ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3585 from facebook/dev

Browse Source 
Preparation for release v1.5.5
This commit is contained in:
Yann Collet
2023-04-01 18:06:35 -07:00
committed by GitHub
parent 945f27758c 9b4833df2d
commit 73e19c4cf1
115 changed files with 3061 additions and 1458 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/images/primary/Dockerfile
+1 -1
View File
@@ -1,4 +1,4 @@
FROM circleci/buildpack-deps:focal
FROM circleci/buildpack-deps@sha256:f6f10c11b7b8ccfd4f4a5b830c3256803604ce61292b60cb22e26b12f62b0e8c
RUN sudo dpkg --add-architecture i386
RUN sudo apt-get -y -qq update
.github/workflows/dev-long-tests.yml
+50 -24
View File
@@ -9,11 +9,13 @@ on:
pull_request:
branches: [ dev, release, actionsTest ]
permissions: read-all
jobs:
make-all:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: make all
run: make all
@@ -24,7 +26,7 @@ jobs:
DEVNULLRIGHTS: 1
READFROMBLOCKDEVICE: 1
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: make test
run: make test
@@ -32,28 +34,42 @@ jobs:
make-test-osx:
runs-on: macos-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: OS-X test
run: make test # make -c lib all doesn't work because of the fact that it's not a tty
# lasts ~24mn
make-test-32bit:
runs-on: ubuntu-latest
env:
DEVNULLRIGHTS: 1
READFROMBLOCKDEVICE: 1
steps:
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: make test
run: |
sudo apt-get -qqq update
make libc6install
CFLAGS="-m32" make test
no-intrinsics-fuzztest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: no intrinsics fuzztest
run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
tsan-zstreamtest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: thread sanitizer zstreamtest
run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
ubsan-zstreamtest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: undefined behavior sanitizer zstreamtest
run: CC=clang make uasan-test-zstream
@@ -61,7 +77,7 @@ jobs:
tsan-fuzztest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: thread sanitizer fuzztest
run: CC=clang make tsan-fuzztest
@@ -69,7 +85,7 @@ jobs:
big-tests-zstreamtest32:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: zstream tests in 32bit mode, with big tests
run: |
sudo apt-get -qqq update
@@ -80,7 +96,7 @@ jobs:
gcc-8-asan-ubsan-testzstd:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: gcc-8 + ASan + UBSan + Test Zstd
# See https://askubuntu.com/a/1428822
run: |
@@ -92,14 +108,14 @@ jobs:
clang-asan-ubsan-testzstd:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: clang + ASan + UBSan + Test Zstd
run: CC=clang make -j uasan-test-zstd </dev/null V=1
gcc-asan-ubsan-testzstd-32bit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: ASan + UBSan + Test Zstd, 32bit mode
run: |
sudo apt-get -qqq update
@@ -113,7 +129,7 @@ jobs:
gcc-8-asan-ubsan-fuzz:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: gcc-8 + ASan + UBSan + Fuzz Test
# See https://askubuntu.com/a/1428822
run: |
@@ -125,14 +141,14 @@ jobs:
clang-asan-ubsan-fuzz:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: clang + ASan + UBSan + Fuzz Test
run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
gcc-asan-ubsan-fuzz32:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: ASan + UBSan + Fuzz Test 32bit
run: |
sudo apt-get -qqq update
@@ -142,7 +158,7 @@ jobs:
clang-asan-ubsan-fuzz32:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: clang + ASan + UBSan + Fuzz Test 32bit
run: |
sudo apt-get -qqq update
@@ -152,28 +168,38 @@ jobs:
asan-ubsan-regression:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: ASan + UBSan + Regression Test
run: make -j uasanregressiontest
clang-ubsan-regression:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: clang + ASan + UBSan + Regression Test
run: CC=clang make -j uasanregressiontest
msan-regression:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: MSan + Regression Test
run: make -j msanregressiontest
clang-msan-fuzz-unoptimized:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: clang + MSan + Fuzz Test
run: |
sudo apt-get -qqq update
sudo apt-get install clang
CC=clang MOREFLAGS="-O0" make clean msan-fuzztest
clang-msan-fuzz:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: clang + MSan + Fuzz Test
run: |
sudo apt-get -qqq update
@@ -184,7 +210,7 @@ jobs:
clang-msan-testzstd:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: clang + MSan + Test Zstd
run: |
sudo apt-get update
@@ -194,7 +220,7 @@ jobs:
armfuzz:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Qemu ARM emulation + Fuzz Test
run: |
sudo apt-get -qqq update
@@ -204,7 +230,7 @@ jobs:
valgrind-fuzz-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: valgrind + fuzz test stack mode # ~ 7mn
shell: 'script -q -e -c "bash {0}"'
run: |
@@ -220,8 +246,8 @@ jobs:
run:
shell: msys2 {0}
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: msys2/setup-msys2@v2
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- uses: msys2/setup-msys2@5beef6d11f48bba68b9eb503e3adc60b23c0cc36 # tag=v2
with:
msystem: MINGW64
install: make
.github/workflows/dev-short-tests.yml
+90 -49
View File
@@ -10,25 +10,27 @@ on:
pull_request:
branches: [ dev, release, actionsTest ]
permissions: read-all
jobs:
linux-kernel:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: linux kernel, library + build + test
run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
benchmarking:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: make benchmarking
run: make benchmarking
check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: make check on 32-bit
run: |
sudo apt update
@@ -38,7 +40,7 @@ jobs:
check-x32:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
env:
CHECK_CONSTRAINED_MEM: true
@@ -50,16 +52,24 @@ jobs:
build-c89:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: ensure zstd can be build with c89/c90 compilers (+ long long support + variadic macros)
run: |
make c89build V=1
build-zstd-dll:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: build zstd bin against a dynamic lib (debuglevel for more dependencies)
run: |
make -C lib lib-mt-release
DEBUGLEVEL=2 make -C programs zstd-dll
gcc-7-libzstd:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: gcc-7 + libzstdmt compilation
# See https://askubuntu.com/a/1428822
run: |
@@ -76,7 +86,7 @@ jobs:
cmake-build-and-test-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: cmake build and test check
run: |
FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild
@@ -87,7 +97,7 @@ jobs:
cpp-gnu90-c99-compatibility:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: C++, gnu90 and c99 compatibility
run: |
make cxxtest
@@ -101,7 +111,7 @@ jobs:
mingw-cross-compilation:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: mingw cross-compilation
run: |
# sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
@@ -112,7 +122,7 @@ jobs:
armbuild:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: ARM Build Test
run: |
sudo apt-get -qqq update
@@ -122,7 +132,7 @@ jobs:
bourne-shell:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Bourne shell compatibility (shellcheck)
run: |
wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
@@ -132,7 +142,7 @@ jobs:
zlib-wrapper:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: zlib wrapper test
run: |
sudo apt-get -qqq update
@@ -143,7 +153,7 @@ jobs:
lz4-threadpool-libs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: LZ4, thread pool, and libs build testslib wrapper test
run: |
make lz4install
@@ -157,7 +167,7 @@ jobs:
gcc-make-tests-32bit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Make all, 32bit mode
run: |
sudo apt-get -qqq update
@@ -167,7 +177,7 @@ jobs:
gcc-8-make:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: gcc-8 build
# See https://askubuntu.com/a/1428822
run: |
@@ -176,10 +186,29 @@ jobs:
make gcc8install
CC=gcc-8 CFLAGS="-Werror" make -j all
make-external-compressors:
strategy:
matrix:
include:
- name: "no external compressors"
flags: "HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0"
- name: "only zlib"
flags: "HAVE_ZLIB=1 HAVE_LZ4=0 HAVE_LZMA=0"
- name: "only lz4"
flags: "HAVE_ZLIB=0 HAVE_LZ4=1 HAVE_LZMA=0"
- name: "only lzma"
flags: "HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=1"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Build with ${{matrix.name}}
run: ${{matrix.flags}} make zstd
implicit-fall-through:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: -Wimplicit-fallthrough build
run: |
make clean
@@ -190,7 +219,7 @@ jobs:
meson-linux:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Install packages
run: |
sudo apt-get update
@@ -214,7 +243,7 @@ jobs:
meson-windows:
runs-on: windows-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Install packages
run: pip install --pre meson
- name: Initialize the MSVC dev command prompt
@@ -241,11 +270,13 @@ jobs:
- generator: "Visual Studio 17 2022"
flags: "-A Win32"
- generator: "MinGW Makefiles"
- generator: "Visual Studio 17 2022"
flags: "-T ClangCL"
runs-on: windows-2022
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.3
uses: microsoft/setup-msbuild@1ff57057b5cfdc39105cd07a01d78e9b0ea0c14c # tag=v1.3
- name: Build
working-directory: ${{env.GITHUB_WORKSPACE}}
run: |
@@ -257,34 +288,45 @@ jobs:
msbuild-visual-studio:
strategy:
fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed.
matrix:
include: [
{ name: "VS 2022 x64 Debug", platform: x64, configuration: Debug, toolset: v143, runner: "windows-2022"},
{ name: "VS 2022 Win32 Debug", platform: Win32, configuration: Debug, toolset: v143, runner: "windows-2022"},
{ name: "VS 2022 x64 Release", platform: x64, configuration: Release, toolset: v143, runner: "windows-2022"},
{ name: "VS 2022 Win32 Release", platform: Win32, configuration: Release, toolset: v143, runner: "windows-2022"},
{ name: "VS 2019 x64 Release", platform: Win32, configuration: Release, toolset: v142, runner: "windows-2019"},
{ name: "VS 2019 Win32 Release", platform: x64, configuration: Release, toolset: v142, runner: "windows-2019"},
{ name: "VS 2022 x64 Debug", platform: x64, configuration: Debug, toolset: v143, runner: "windows-2022", arch: "" },
{ name: "VS 2022 Win32 Debug", platform: Win32, configuration: Debug, toolset: v143, runner: "windows-2022", arch: "" },
{ name: "VS 2022 x64 Release", platform: x64, configuration: Release, toolset: v143, runner: "windows-2022", arch: ""},
{ name: "VS 2022 Win32 Release", platform: Win32, configuration: Release, toolset: v143, runner: "windows-2022", arch: ""},
{ name: "VS 2019 x64 Release", platform: Win32, configuration: Release, toolset: v142, runner: "windows-2019", arch: ""},
{ name: "VS 2019 Win32 Release", platform: x64, configuration: Release, toolset: v142, runner: "windows-2019", arch: ""},
{ name: "VS 2022 x64 Release AVX2", platform: x64, configuration: Release, toolset: v143, runner: "windows-2022", arch: "AdvancedVectorExtensions2" },
]
runs-on: ${{matrix.runner}}
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.3
- name: Build
uses: microsoft/setup-msbuild@1ff57057b5cfdc39105cd07a01d78e9b0ea0c14c # tag=v1.3
- name: Build ${{matrix.name}}
working-directory: ${{env.GITHUB_WORKSPACE}}
# See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
if: ${{ matrix.arch == '' }}
run: >
msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=${{matrix.toolset}}
/t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}}
/t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}} /warnaserror
- name: Build ${{matrix.name}}
working-directory: ${{env.GITHUB_WORKSPACE}}
# See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
if: ${{ matrix.arch != '' }}
run: >
msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=${{matrix.toolset}}
/t:Clean,Build /p:Platform=${{matrix.platform}} /p:Configuration=${{matrix.configuration}} /warnaserror
/p:InstructionSet=${{matrix.arch}}
# This tests that we don't accidently grow the size too much.
# This tests that we don't accidentally grow the size too much.
# If the size grows intentionally, you can raise these numbers.
# But we do need to think about binary size, since it is a concern.
libzstd-size:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: libzstd size test
run: |
make clean && make -j -C lib libzstd && ./tests/check_size.py lib/libzstd.so 1100000
@@ -295,7 +337,7 @@ jobs:
minimal-decompressor-macros:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: minimal decompressor macros
run: |
make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
@@ -310,7 +352,7 @@ jobs:
dynamic-bmi2:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: dynamic bmi2 tests
run: |
make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
@@ -322,13 +364,12 @@ jobs:
test-variants:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: make all variants & validate
run: |
make -j -C programs allVariants MOREFLAGS=-O0
./tests/test-variants.sh
qemu-consistency:
name: QEMU ${{ matrix.name }}
runs-on: ubuntu-20.04
@@ -348,7 +389,7 @@ jobs:
XCC: ${{ matrix.xcc }}
XEMU: ${{ matrix.xemu }}
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: apt update & install
run: |
sudo apt-get update
@@ -404,8 +445,8 @@ jobs:
run:
shell: msys2 {0}
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: msys2/setup-msys2@v2
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- uses: msys2/setup-msys2@5beef6d11f48bba68b9eb503e3adc60b23c0cc36 # tag=v2
with:
msystem: ${{ matrix.msystem }}
install: make diffutils
@@ -440,9 +481,9 @@ jobs:
platform: [x64, Win32]
configuration: [Release]
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@v1.3
uses: microsoft/setup-msbuild@1ff57057b5cfdc39105cd07a01d78e9b0ea0c14c # tag=v1.3
- name: Build and run tests
working-directory: ${{env.GITHUB_WORKSPACE}}
env:
@@ -461,7 +502,7 @@ jobs:
runs-on: windows-latest
steps:
- run: git config --global core.autocrlf input
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- uses: cygwin/cygwin-install-action@f5e0f048310c425e84bc789f493a828c6dc80a25 # tag=master
with:
platform: x86_64
@@ -486,7 +527,7 @@ jobs:
intel-cet-compatibility:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Build Zstd
run: |
make -j zstd V=1
@@ -507,7 +548,7 @@ jobs:
container:
image: debian:testing
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Install dependencies
run: |
apt -y update
@@ -522,7 +563,7 @@ jobs:
versions-compatibility:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Versions Compatibility Test
run: |
make -C tests versionsTest
@@ -530,7 +571,7 @@ jobs:
clangbuild:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: make clangbuild
run: |
make clangbuild
@@ -538,7 +579,7 @@ jobs:
clang-pgo:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Build PGO Zstd with Clang
env:
CC: clang-14
@@ -550,7 +591,7 @@ jobs:
gcc-pgo:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Build PGO Zstd with GCC
env:
CC: gcc
@@ -577,7 +618,7 @@ jobs:
# sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
# sudo apt-get update
# sudo apt-get install -y intel-basekit intel-hpckit
# - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
# - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
# - name: make check
# run: |
# make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
.github/workflows/publish-release-artifacts.yml
+4 -6
View File
@@ -5,21 +5,19 @@ on:
types:
- published
permissions:
contents: read
permissions: read-all
jobs:
publish-release-artifacts:
permissions:
contents: read # to fetch code (actions/checkout)
actions: write # to attach binaries to release artifacts (skx/github-action-publish-binaries)
contents: write # to fetch code and upload artifacts
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/tags/')
steps:
- name: Checkout
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- name: Archive
env:
@@ -68,7 +66,7 @@ jobs:
fi
- name: Publish
uses: skx/github-action-publish-binaries@release-2.0
uses: skx/github-action-publish-binaries@b9ca5643b2f1d7371a6cba7f35333f1461bbc703 # tag=release-2.0
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
.github/workflows/scorecards.yml
+2 -2
View File
@@ -27,7 +27,7 @@ jobs:
steps:
- name: "Checkout code"
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
with:
persist-credentials: false
@@ -59,6 +59,6 @@ jobs:
# Upload the results to GitHub's code scanning dashboard.
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@3ebbd71c74ef574dbc558c82f70e52732c8b44fe # tag=v2.2.1
uses: github/codeql-action/upload-sarif@67a35a08586135a9573f4327e904ecbf517a882d # tag=v2.2.8
with:
sarif_file: results.sarif
.github/workflows/windows-artifacts.yml
+48
View File
@@ -0,0 +1,48 @@
name: windows-artifacts
on:
push:
branches: [ release, test_artifacts, win_artifacts ]
permissions: read-all
jobs:
windows-64-artifacts:
# see https://ariya.io/2020/07/on-github-actions-with-msys2
runs-on: windows-latest
defaults:
run:
shell: msys2 {0}
steps:
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3
- uses: msys2/setup-msys2@5beef6d11f48bba68b9eb503e3adc60b23c0cc36 # tag=v2
with:
msystem: MINGW64
install: make zlib git p7zip mingw-w64-x86_64-gcc
update: true
- name: display versions
run: |
make -v
cc -v
- name: Building zlib to static link
run: |
git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib
make -C zlib -f win32/Makefile.gcc libz.a
- name: Building zstd programs in 64-bit mode
run: |
CPPFLAGS=-I../zlib LDFLAGS=../zlib/libz.a make -j allzstd MOREFLAGS=-static V=1
- name: Create artifacts
run: |
./lib/dll/example/build_package.bat
cd bin/
7z a -tzip -mx9 zstd-win-release-win64.zip *
cd ..
- name: Publish zstd-win-release-win64.zip
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # tag=v3
with:
path: ${{ github.workspace }}/bin/zstd-win-release-win64.zip
name: zstd-win-release-win64.zip
CHANGELOG
+19
View File
@@ -1,3 +1,22 @@
v1.5.5 (Apr 2023)
fix: fix rare corruption bug affecting the high compression mode, reported by @danlark1 (#3517, @terrelln)
perf: improve mid-level compression speed (#3529, #3533, #3543, @yoniko and #3552, @terrelln)
lib: deprecated bufferless block-level API (#3534) by @terrelln
cli: mmap large dictionaries to save memory, by @daniellerozenblit
cli: improve speed of --patch-from mode (~+50%) (#3545) by @daniellerozenblit
cli: improve i/o speed (~+10%) when processing lots of small files (#3479) by @felixhandte
cli: zstd no longer crashes when requested to write into write-protected directory (#3541) by @felixhandte
cli: fix decompression into block device using -o (#3584, @Cyan4973) reported by @georgmu
build: fix zstd CLI compiled with lzma support but not zlib support (#3494) by @Hello71
build: fix cmake does no longer require 3.18 as minimum version (#3510) by @kou
build: fix MSVC+ClangCL linking issue (#3569) by @tru
build: fix zstd-dll, version of zstd CLI that links to the dynamic library (#3496) by @yoniko
build: fix MSVC warnings (#3495) by @embg
doc: updated zstd specification to clarify corner cases, by @Cyan4973
doc: document how to create fat binaries for macos (#3568) by @rickmark
misc: improve seekable format ingestion speed (~+100%) for very small chunk sizes (#3544) by @Cyan4973
misc: tests/fullbench can benchmark multiple files (#3516) by @dloidolt
v1.5.4 (Feb 2023)
perf: +20% faster huffman decompression for targets that can't compile x64 assembly (#3449, @terrelln)
perf: up to +10% faster streaming compression at levels 1-2 (#3114, @embg)
Makefile
+26 -26
View File
@@ -200,15 +200,15 @@ travis-install:
.PHONY: gcc5build gcc6build gcc7build clangbuild m32build armbuild aarch64build ppcbuild ppc64build
gcc5build: clean
gcc-5 -v
CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror"
CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
gcc6build: clean
gcc-6 -v
CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror"
CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
gcc7build: clean
gcc-7 -v
CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror"
CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror $(MOREFLAGS)"
clangbuild: clean
clang -v
@@ -232,17 +232,17 @@ ppc64build: clean
.PHONY: armfuzz aarch64fuzz ppcfuzz ppc64fuzz
armfuzz: clean
CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
aarch64fuzz: clean
ld -v
CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
ppcfuzz: clean
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
ppc64fuzz: clean
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) fuzztest
.PHONY: cxxtest gcc5test gcc6test armtest aarch64test ppctest ppc64test
cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
@@ -251,34 +251,34 @@ cxxtest: clean
gcc5test: clean
gcc-5 -v
$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror"
$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror $(MOREFLAGS)"
gcc6test: clean
gcc-6 -v
$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror $(MOREFLAGS)"
armtest: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
aarch64test:
$(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
ppctest: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" FUZZER_FLAGS=--no-big-tests
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
ppc64test: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
.PHONY: arm-ppc-compilation
arm-ppc-compilation:
$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)"
$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static $(MOREFLAGS)"
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static $(MOREFLAGS)"
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static $(MOREFLAGS)"
regressiontest:
$(MAKE) -C $(FUZZDIR) regressiontest
@@ -308,31 +308,31 @@ update_regressionResults:
# run UBsan with -fsanitize-recover=pointer-overflow
# this only works with recent compilers such as gcc 8+
usan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror"
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=undefined -Werror $(MOREFLAGS)"
asan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror"
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror $(MOREFLAGS)"
asan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror" $(MAKE) -C $(TESTDIR) $*
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
msan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason
msan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror $(MOREFLAGS)" FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)" $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*
asan32: clean
$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address $(MOREFLAGS)"
uasan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror"
$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror $(MOREFLAGS)"
uasan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $*
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=pointer-overflow -fsanitize=address,undefined -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $*
tsan-%: clean
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests
LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror $(MOREFLAGS)" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS="--no-big-tests $(FUZZER_FLAGS)"
.PHONY: apt-install
apt-install:
README.md
+12
View File
@@ -154,6 +154,18 @@ to create `zstd` binary, and `libzstd` dynamic and static libraries.
By default, `CMAKE_BUILD_TYPE` is set to `Release`.
#### Support for Fat (Universal2) Output
`zstd` can be built and installed with support for both Apple Silicon (M1/M2) as well as Intel by using CMake's Universal2 support.
To perform a Fat/Universal2 build and install use the following commands:
```bash
cmake -B build-cmake-debug -S build/cmake -G Ninja -DCMAKE_OSX_ARCHITECTURES="x86_64;x86_64h;arm64"
cd build-cmake-debug
ninja
sudo ninja install
```
### Meson
A Meson project is provided within [`build/meson`](build/meson). Follow
build/VS2010/libzstd-dll/libzstd-dll.vcxproj
-1
View File
@@ -169,7 +169,6 @@
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<MinimalRebuild>true</MinimalRebuild>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<DebugInformationFormat>EditAndContinue</DebugInformationFormat>
build/VS2010/libzstd/libzstd.vcxproj
-1
View File
@@ -162,7 +162,6 @@
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<MinimalRebuild>true</MinimalRebuild>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<DebugInformationFormat>EditAndContinue</DebugInformationFormat>
build/VS2010/zstd/zstd.vcxproj
+5
View File
@@ -114,6 +114,7 @@
<RootNamespace>zstd</RootNamespace>
<OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
<InstructionSet>NotSet</InstructionSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
@@ -187,6 +188,7 @@
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -203,6 +205,7 @@
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -222,6 +225,7 @@
<EnablePREfast>false</EnablePREfast>
<TreatWarningAsError>false</TreatWarningAsError>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@@ -244,6 +248,7 @@
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<AdditionalOptions>/DZSTD_MULTITHREAD %(AdditionalOptions)</AdditionalOptions>
<EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
build/cmake/CMakeModules/AddZstdCompilationFlags.cmake
+24 -2
View File
@@ -1,6 +1,15 @@
include(CheckCXXCompilerFlag)
include(CheckCCompilerFlag)
include(CheckLinkerFlag)
# VERSION_GREATER_EQUAL requires CMake 3.7 or later.
# https://cmake.org/cmake/help/latest/command/if.html#version-greater-equal
if (CMAKE_VERSION VERSION_LESS 3.18)
set(ZSTD_HAVE_CHECK_LINKER_FLAG false)
else ()
set(ZSTD_HAVE_CHECK_LINKER_FLAG true)
endif ()
if (ZSTD_HAVE_CHECK_LINKER_FLAG)
include(CheckLinkerFlag)
endif()
function(EnableCompilerFlag _flag _C _CXX _LD)
string(REGEX REPLACE "\\+" "PLUS" varname "${_flag}")
@@ -20,7 +29,20 @@ function(EnableCompilerFlag _flag _C _CXX _LD)
endif ()
endif ()
if (_LD)
CHECK_LINKER_FLAG(C ${_flag} LD_FLAG_${varname})
# We never add a linker flag with CMake < 3.18. We will
# implement CHECK_LINKER_FLAG() like feature for CMake < 3.18
# or require CMake >= 3.18 when we need to add a required
# linker flag in future.
#
# We also skip linker flags check for MSVC compilers (which includes
# clang-cl) since currently check_linker_flag() doesn't give correct
# results for this configuration,
# see: https://gitlab.kitware.com/cmake/cmake/-/issues/22023
if (ZSTD_HAVE_CHECK_LINKER_FLAG AND NOT MSVC)
CHECK_LINKER_FLAG(C ${_flag} LD_FLAG_${varname})
else ()
set(LD_FLAG_${varname} false)
endif ()
if (LD_FLAG_${varname})
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${_flag}" PARENT_SCOPE)
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${_flag}" PARENT_SCOPE)
build/meson/meson.build
+1 -1
View File
@@ -132,7 +132,7 @@ endif
subdir('lib')
if bin_programs
if bin_programs or bin_tests
subdir('programs')
endif
build/meson/programs/meson.build
+8 -1
View File
@@ -72,7 +72,14 @@ zstd = executable('zstd',
c_args: zstd_c_args,
dependencies: zstd_deps,
export_dynamic: export_dynamic_on_windows, # Since Meson 0.45.0
install: true)
build_by_default: bin_programs,
install: bin_programs)
if not bin_programs
# we generate rules to build the programs, but don't install anything
# so do not continue to installing scripts and manpages
subdir_done()
endif
zstd_frugal_sources = [join_paths(zstd_rootdir, 'programs/zstdcli.c'),
join_paths(zstd_rootdir, 'programs/timefn.c'),
build/meson/tests/meson.build
+1 -1
View File
@@ -162,7 +162,7 @@ if host_machine_os != os_windows
playTests_sh,
args: opt,
env: ['ZSTD_BIN=' + zstd.full_path(), 'DATAGEN_BIN=./datagen'],
depends: [datagen],
depends: [datagen, zstd],
suite: suite,
workdir: meson.current_build_dir(),
timeout: 2800) # Timeout should work on HDD drive
contrib/docker/Dockerfile
+2 -2
View File
@@ -1,13 +1,13 @@
# Dockerfile
# First image to build the binary
FROM alpine as builder
FROM alpine@sha256:69665d02cb32192e52e07644d76bc6f25abeb5410edc1c7a81a10ba3f0efb90a as builder
RUN apk --no-cache add make gcc libc-dev
COPY . /src
RUN mkdir /pkg && cd /src && make && make DESTDIR=/pkg install
# Second minimal image to only keep the built binary
FROM alpine
FROM alpine@sha256:69665d02cb32192e52e07644d76bc6f25abeb5410edc1c7a81a10ba3f0efb90a
# Copy the built files
COPY --from=builder /pkg /
contrib/linux-kernel/zstd_common_module.c
-3
View File
@@ -24,9 +24,6 @@ EXPORT_SYMBOL_GPL(HUF_readStats_wksp);
EXPORT_SYMBOL_GPL(ZSTD_isError);
EXPORT_SYMBOL_GPL(ZSTD_getErrorName);
EXPORT_SYMBOL_GPL(ZSTD_getErrorCode);
EXPORT_SYMBOL_GPL(ZSTD_customMalloc);
EXPORT_SYMBOL_GPL(ZSTD_customCalloc);
EXPORT_SYMBOL_GPL(ZSTD_customFree);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("Zstd Common");
contrib/linux-kernel/zstd_deps.h
+1 -1
View File
@@ -84,7 +84,7 @@ static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
#include <linux/kernel.h>
#define assert(x) WARN_ON((x))
#define assert(x) WARN_ON(!(x))
#endif /* ZSTD_DEPS_ASSERT */
#endif /* ZSTD_DEPS_NEED_ASSERT */
contrib/pzstd/Makefile
+7 -1
View File
@@ -37,10 +37,16 @@ CFLAGS += -Wno-deprecated-declarations
PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
GTEST_INC = -isystem googletest/googletest/include
# If default C++ version is older than C++11, explicitly set C++11, which is the
# minimum required by the code.
ifeq ($(shell echo "\043if __cplusplus < 201103L\n\043error\n\043endif" | $(CXX) -x c++ -Werror -c - -o /dev/null 2>/dev/null && echo 1 || echo 0),0)
PZSTD_CXX_STD := -std=c++11
endif
PZSTD_CPPFLAGS = $(PZSTD_INC)
PZSTD_CCXXFLAGS =
PZSTD_CFLAGS = $(PZSTD_CCXXFLAGS)
PZSTD_CXXFLAGS = $(PZSTD_CCXXFLAGS) -std=c++11
PZSTD_CXXFLAGS = $(PZSTD_CCXXFLAGS) $(PZSTD_CXX_STD)
PZSTD_LDFLAGS =
EXTRA_FLAGS =
ALL_CFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS) $(PZSTD_CFLAGS)
contrib/seekable_format/README.md
+42
View File
@@ -0,0 +1,42 @@
# Zstandard Seekable Format
The seekable format splits compressed data into a series of independent "frames",
each compressed individually,
so that decompression of a section in the middle of an archive
only requires zstd to decompress at most a frame's worth of extra data,
instead of the entire archive.
The frames are appended, so that the decompression of the entire payload
still regenerates the original content, using any compliant zstd decoder.
On top of that, the seekable format generates a jump table,
which makes it possible to jump directly to the position of the relevant frame
when requesting only a segment of the data.
The jump table is simply ignored by zstd decoders unaware of the seekable format.
The format is delivered with an API to create seekable archives
and to retrieve arbitrary segments inside the archive.
### Maximum Frame Size parameter
When creating a seekable archive, the main parameter is the maximum frame size.
At compression time, user can manually select the boundaries between segments,
but they don't have to: long segments will be automatically split
when larger than selected maximum frame size.
Small frame sizes reduce decompression cost when requesting small segments,
because the decoder will nonetheless have to decompress an entire frame
to recover just a single byte from it.
A good rule of thumb is to select a maximum frame size roughly equivalent
to the access pattern when it's known.
For example, if the application tends to request 4KB blocks,
then it's a good idea to set a maximum frame size in the vicinity of 4 KB.
But small frame sizes also reduce compression ratio,
and increase the cost for the jump table,
so there is a balance to find.
In general, try to avoid really tiny frame sizes (<1 KB),
which would have a large negative impact on compression ratio.
contrib/seekable_format/examples/parallel_compression.c
+1 -1
View File
@@ -25,7 +25,7 @@
#include "pool.h" // use zstd thread pool for demo
#include "zstd_seekable.h"
#include "../zstd_seekable.h"
static void* malloc_orDie(size_t size)
{
contrib/seekable_format/examples/parallel_processing.c
+1 -1
View File
@@ -29,7 +29,7 @@
#include "pool.h" // use zstd thread pool for demo
#include "zstd_seekable.h"
#include "../zstd_seekable.h"
#define MIN(a, b) ((a) < (b) ? (a) : (b))
contrib/seekable_format/examples/seekable_compression.c
+10 -7
View File
@@ -13,7 +13,7 @@
#define ZSTD_STATIC_LINKING_ONLY
#include <zstd.h> // presumes zstd library is installed
#include "zstd_seekable.h"
#include "../zstd_seekable.h"
static void* malloc_orDie(size_t size)
{
@@ -112,20 +112,23 @@ static char* createOutFilename_orDie(const char* filename)
return (char*)outSpace;
}
int main(int argc, const char** argv) {
#define CLEVEL_DEFAULT 5
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
if (argc!=3) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE FRAME_SIZE\n", exeName);
if (argc<3 || argc>4) {
printf("wrong arguments \n");
printf("usage: \n");
printf("%s FILE FRAME_SIZE [LEVEL] \n", exeName);
return 1;
}
{ const char* const inFileName = argv[1];
unsigned const frameSize = (unsigned)atoi(argv[2]);
int const cLevel = (argc==4) ? atoi(argv[3]) : CLEVEL_DEFAULT;
char* const outFileName = createOutFilename_orDie(inFileName);
compressFile_orDie(inFileName, outFileName, 5, frameSize);
compressFile_orDie(inFileName, outFileName, cLevel, frameSize);
free(outFileName);
}
contrib/seekable_format/examples/seekable_decompression.c
+1 -1
View File
@@ -16,7 +16,7 @@
#include <zstd.h> // presumes zstd library is installed
#include <zstd_errors.h>
#include "zstd_seekable.h"
#include "../zstd_seekable.h"
#define MIN(a, b) ((a) < (b) ? (a) : (b))
contrib/seekable_format/tests/seekable_tests.c
+134 -1
View File
@@ -3,8 +3,55 @@
#include <stdlib.h> // malloc
#include <stdio.h>
#include <assert.h>
#include <string.h>
#include "zstd_seekable.h"
#include "../zstd_seekable.h"
/* ZSTD_seekable_customFile implementation that reads/seeks a buffer while keeping track of total bytes read */
typedef struct {
const void *ptr;
size_t size;
size_t pos;
size_t totalRead;
} buffWrapperWithTotal_t;
static int readBuffWithTotal(void* opaque, void* buffer, size_t n)
{
buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*)opaque;
assert(buff != NULL);
if (buff->pos + n > buff->size) return -1;
memcpy(buffer, (const char*)buff->ptr + buff->pos, n);
buff->pos += n;
buff->totalRead += n;
return 0;
}
static int seekBuffWithTotal(void* opaque, long long offset, int origin)
{
buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*) opaque;
unsigned long long newOffset;
assert(buff != NULL);
switch (origin) {
case SEEK_SET:
assert(offset >= 0);
newOffset = (unsigned long long)offset;
break;
case SEEK_CUR:
newOffset = (unsigned long long)((long long)buff->pos + offset);
break;
case SEEK_END:
newOffset = (unsigned long long)((long long)buff->size + offset);
break;
default:
assert(0); /* not possible */
}
if (newOffset > buff->size) {
return -1;
}
buff->pos = newOffset;
return 0;
}
/* Basic unit tests for zstd seekable format */
int main(int argc, const char** argv)
@@ -220,6 +267,92 @@ int main(int argc, const char** argv)
}
printf("Success!\n");
printf("Test %u - multiple decompress calls: ", testNb++);
{ char const inBuffer[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt";
size_t const inSize = sizeof(inBuffer);
size_t const seekCapacity = 5000;
void* const seekBuffer = malloc(seekCapacity);
assert(seekBuffer != NULL);
size_t seekSize;
size_t const outCapacity = inSize;
char* const outBuffer = malloc(outCapacity);
assert(outBuffer != NULL);
ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream();
assert(zscs != NULL);
/* compress test data with a small frame size to ensure multiple frames in the output */
unsigned const maxFrameSize = 40;
{ size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, maxFrameSize);
assert(!ZSTD_isError(initStatus));
}
{ ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity };
ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize };
while (inb.pos < inb.size) {
size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb);
assert(!ZSTD_isError(cStatus));
}
size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb);
assert(!ZSTD_isError(endStatus));
seekSize = outb.pos;
}
ZSTD_seekable* const stream = ZSTD_seekable_create();
assert(stream != NULL);
buffWrapperWithTotal_t buffWrapper = {seekBuffer, seekSize, 0, 0};
{ ZSTD_seekable_customFile srcFile = {&buffWrapper, &readBuffWithTotal, &seekBuffWithTotal};
size_t const initStatus = ZSTD_seekable_initAdvanced(stream, srcFile);
assert(!ZSTD_isError(initStatus)); }
/* Perform a series of small reads and seeks (repeatedly read 1 byte and skip 1 byte)
and check that we didn't reread input data unnecessarily */
size_t pos;
for (pos = 0; pos < inSize; pos += 2) {
size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, 1, pos);
if (decStatus != 1 || outBuffer[0] != inBuffer[pos]) {
goto _test_error;
}
}
if (buffWrapper.totalRead > seekSize) {
/* We read more than the compressed size, meaning there were some rereads.
This is unneeded because we only seeked forward. */
printf("Too much data read: %zu read, with compressed size %zu\n", buffWrapper.totalRead, seekSize);
goto _test_error;
}
/* Perform some reads and seeks to ensure correctness */
struct {
size_t offset;
size_t size;
} const tests[] = { /* Assume the frame size is 40 */
{20, 40}, /* read partial data from two frames */
{60, 10}, /* continue reading from the same offset */
{50, 20}, /* seek backward within the same frame */
{10, 10}, /* seek backward to a different frame */
{25, 10}, /* seek forward within the same frame */
{60, 10}, /* seek forward to a different frame */
};
size_t idx;
for (idx = 0; idx < sizeof(tests) / sizeof(tests[0]); idx++) {
size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, tests[idx].size, tests[idx].offset);
if (decStatus != tests[idx].size || memcmp(outBuffer, inBuffer + tests[idx].offset, tests[idx].size) != 0) {
goto _test_error;
}
}
free(seekBuffer);
free(outBuffer);
ZSTD_seekable_freeCStream(zscs);
ZSTD_seekable_free(stream);
}
printf("Success!\n");
/* TODO: Add more tests */
printf("Finished tests\n");
return 0;
contrib/seekable_format/zstd_seekable.h
+15 -6
View File
@@ -15,8 +15,8 @@ extern "C" {
#define ZSTD_SEEKABLE_MAXFRAMES 0x8000000U
/* Limit the maximum size to avoid any potential issues storing the compressed size */
#define ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE 0x80000000U
/* Limit maximum size to avoid potential issues storing the compressed size */
#define ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE 0x40000000U
/*-****************************************************************************
* Seekable Format
@@ -48,10 +48,19 @@ typedef struct ZSTD_seekTable_s ZSTD_seekTable;
*
* Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object
* for a new compression operation.
* `maxFrameSize` indicates the size at which to automatically start a new
* seekable frame. `maxFrameSize == 0` implies the default maximum size.
* `checksumFlag` indicates whether or not the seek table should include frame
* checksums on the uncompressed data for verification.
* - `maxFrameSize` indicates the size at which to automatically start a new
* seekable frame.
* `maxFrameSize == 0` implies the default maximum size.
* Smaller frame sizes allow faster decompression of small segments,
* since retrieving a single byte requires decompression of
* the full frame where the byte belongs.
* In general, size the frames to roughly correspond to
* the access granularity (when it's known).
* But small sizes also reduce compression ratio.
* Avoid really tiny frame sizes (< 1 KB),
* that would hurt compression ratio considerably.
* - `checksumFlag` indicates whether or not the seek table should include frame
* checksums on the uncompressed data for verification.
* @return : a size hint for input to provide for compression, or an error code
* checkable with ZSTD_isError()
*
contrib/seekable_format/zstdseek_compress.c
+2
View File
@@ -230,6 +230,8 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
const BYTE* const inBase = (const BYTE*) input->src + input->pos;
size_t inLen = input->size - input->pos;
assert(zcs->maxFrameSize < INT_MAX);
ZSTD_CCtx_setParameter(zcs->cstream, ZSTD_c_srcSizeHint, (int)zcs->maxFrameSize);
inLen = MIN(inLen, (size_t)(zcs->maxFrameSize - zcs->frameDSize));
/* if we haven't finished flushing the last frame, don't start writing a new one */
contrib/seekable_format/zstdseek_decompress.c
+1 -1
View File
@@ -493,7 +493,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
size_t srcBytesRead = 0;
do {
/* check if we can continue from a previous decompress job */
if (targetFrame != zs->curFrame || offset != zs->decompressedOffset) {
if (targetFrame != zs->curFrame || offset < zs->decompressedOffset) {
zs->decompressedOffset = zs->seekTable.entries[targetFrame].dOffset;
zs->curFrame = targetFrame;
doc/zstd_compression_format.md
+31 -15
View File
@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
### Version
0.3.7 (2020-12-09)
0.3.9 (2023-03-08)
Introduction
@@ -470,6 +470,7 @@ This field uses 2 lowest bits of first byte, describing 4 different block types
repeated `Regenerated_Size` times.
- `Compressed_Literals_Block` - This is a standard Huffman-compressed block,
starting with a Huffman tree description.
In this mode, there are at least 2 different literals represented in the Huffman tree description.
See details below.
- `Treeless_Literals_Block` - This is a Huffman-compressed block,
using Huffman tree _from previous Huffman-compressed literals block_.
@@ -533,15 +534,20 @@ __`Size_Format` for `Compressed_Literals_Block` and `Treeless_Literals_Block`__
Both `Compressed_Size` and `Regenerated_Size` fields follow __little-endian__ convention.
Note: `Compressed_Size` __includes__ the size of the Huffman Tree description
_when_ it is present.
Note 2: `Compressed_Size` can never be `==0`.
Even in single-stream scenario, assuming an empty content, it must be `>=1`,
since it contains at least the final end bit flag.
In 4-streams scenario, a valid `Compressed_Size` is necessarily `>= 10`
(6 bytes for the jump table, + 4x1 bytes for the 4 streams).
4 streams is superior to 1 stream in decompression speed,
4 streams is faster than 1 stream in decompression speed,
by exploiting instruction level parallelism.
But it's also more expensive,
costing on average ~7.3 bytes more than the 1 stream mode, mostly from the jump table.
In general, use the 4 streams mode when there are more literals to decode,
to favor higher decompression speeds.
Beyond 1KB, the 4 streams mode is compulsory anyway.
Note that beyond >1KB of literals, the 4 streams mode is compulsory.
Note that a minimum of 6 bytes is required for the 4 streams mode.
That's a technical minimum, but it's not recommended to employ the 4 streams mode
@@ -566,6 +572,7 @@ or from a dictionary.
### `Huffman_Tree_Description`
This section is only present when `Literals_Block_Type` type is `Compressed_Literals_Block` (`2`).
The tree describes the weights of all literals symbols that can be present in the literals block, at least 2 and up to 256.
The format of the Huffman tree description can be found at [Huffman Tree description](#huffman-tree-description).
The size of `Huffman_Tree_Description` is determined during decoding process,
it must be used to determine where streams begin.
@@ -575,10 +582,10 @@ it must be used to determine where streams begin.
### Jump Table
The Jump Table is only present when there are 4 Huffman-coded streams.
Reminder : Huffman compressed data consists of either 1 or 4 Huffman-coded streams.
Reminder : Huffman compressed data consists of either 1 or 4 streams.
If only one stream is present, it is a single bitstream occupying the entire
remaining portion of the literals block, encoded as described within
remaining portion of the literals block, encoded as described in
[Huffman-Coded Streams](#huffman-coded-streams).
If there are four streams, `Literals_Section_Header` only provided
@@ -589,17 +596,18 @@ except for the last stream which may be up to 3 bytes smaller,
to reach a total decompressed size as specified in `Regenerated_Size`.
The compressed size of each stream is provided explicitly in the Jump Table.
Jump Table is 6 bytes long, and consist of three 2-byte __little-endian__ fields,
Jump Table is 6 bytes long, and consists of three 2-byte __little-endian__ fields,
describing the compressed sizes of the first three streams.
`Stream4_Size` is computed from total `Total_Streams_Size` minus sizes of other streams.
`Stream4_Size` is computed from `Total_Streams_Size` minus sizes of other streams:
`Stream4_Size = Total_Streams_Size - 6 - Stream1_Size - Stream2_Size - Stream3_Size`.
Note: if `Stream1_Size + Stream2_Size + Stream3_Size > Total_Streams_Size`,
`Stream4_Size` is necessarily `>= 1`. Therefore,
if `Total_Streams_Size < Stream1_Size + Stream2_Size + Stream3_Size + 6 + 1`,
data is considered corrupted.
Each of these 4 bitstreams is then decoded independently as a Huffman-Coded stream,
as described at [Huffman-Coded Streams](#huffman-coded-streams)
as described in [Huffman-Coded Streams](#huffman-coded-streams)
Sequences Section
@@ -1197,7 +1205,7 @@ Huffman Coding
--------------
Zstandard Huffman-coded streams are read backwards,
similar to the FSE bitstreams.
Therefore, to find the start of the bitstream, it is therefore to
Therefore, to find the start of the bitstream, it is required to
know the offset of the last byte of the Huffman-coded stream.
After writing the last bit containing information, the compressor
@@ -1239,9 +1247,15 @@ Transformation from `Weight` to `Number_of_Bits` follows this formula :
```
Number_of_Bits = Weight ? (Max_Number_of_Bits + 1 - Weight) : 0
```
The last symbol's `Weight` is deduced from previously decoded ones,
by completing to the nearest power of 2.
This power of 2 gives `Max_Number_of_Bits`, the depth of the current tree.
When a literal value is not present, it receives a `Weight` of 0.
The least frequent symbol receives a `Weight` of 1.
Consequently, the `Weight` 1 is necessarily present.
The most frequent symbol receives a `Weight` anywhere between 1 and 11 (max).
The last symbol's `Weight` is deduced from previously retrieved Weights,
by completing to the nearest power of 2. It's necessarily non 0.
If it's not possible to reach a clean power of 2 with a single `Weight` value,
the Huffman Tree Description is considered invalid.
This final power of 2 gives `Max_Number_of_Bits`, the depth of the current tree.
`Max_Number_of_Bits` must be <= 11,
otherwise the representation is considered corrupted.
@@ -1254,7 +1268,7 @@ Let's presume the following Huffman tree must be described :
The tree depth is 4, since its longest elements uses 4 bits
(longest elements are the one with smallest frequency).
Value `5` will not be listed, as it can be determined from values for 0-4,
Literal value `5` will not be listed, as it can be determined from previous values 0-4,
nor will values above `5` as they are all 0.
Values from `0` to `4` will be listed using `Weight` instead of `Number_of_Bits`.
Weight formula is :
@@ -1274,7 +1288,7 @@ The `Weight` of `5` can be determined by advancing to the next power of 2.
The sum of `2^(Weight-1)` (excluding 0's) is :
`8 + 4 + 2 + 0 + 1 = 15`.
Nearest larger power of 2 value is 16.
Therefore, `Max_Number_of_Bits = 4` and `Weight[5] = 16-15 = 1`.
Therefore, `Max_Number_of_Bits = 4` and `Weight[5] = log_2(16 - 15) + 1 = 1`.
#### Huffman Tree header
@@ -1683,6 +1697,8 @@ or at least provide a meaningful error code explaining for which reason it canno
Version changes
---------------
- 0.3.9 : clarifications for Huffman-compressed literal sizes.
- 0.3.8 : clarifications for Huffman Blocks and Huffman Tree descriptions.
- 0.3.7 : clarifications for Repeat_Offsets, matching RFC8878
- 0.3.6 : clarifications for Dictionary_ID
- 0.3.5 : clarifications for Block_Maximum_Size
doc/zstd_manual.html
+127 -83
View File
@@ -1,10 +1,10 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>zstd 1.5.4 Manual</title>
<title>zstd 1.5.5 Manual</title>
</head>
<body>
<h1>zstd 1.5.4 Manual</h1>
<h1>zstd 1.5.5 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
@@ -22,15 +22,15 @@
<li><a href="#Chapter12">Dictionary helper functions</a></li>
<li><a href="#Chapter13">Advanced dictionary and prefix API (Requires v1.4.0+)</a></li>
<li><a href="#Chapter14">experimental API (static linking only)</a></li>
<li><a href="#Chapter15">Frame size functions</a></li>
<li><a href="#Chapter15">Frame header and size functions</a></li>
<li><a href="#Chapter16">Memory management</a></li>
<li><a href="#Chapter17">Advanced compression functions</a></li>
<li><a href="#Chapter18">Advanced decompression functions</a></li>
<li><a href="#Chapter19">Advanced streaming functions</a></li>
<li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions</a></li>
<li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions (DEPRECATED)</a></li>
<li><a href="#Chapter21">Buffer-less streaming compression (synchronous mode)</a></li>
<li><a href="#Chapter22">Buffer-less streaming decompression (synchronous mode)</a></li>
<li><a href="#Chapter23">Block level API</a></li>
<li><a href="#Chapter23">Block level API (DEPRECATED)</a></li>
</ol>
<hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre>
@@ -80,7 +80,8 @@
const void* src, size_t srcSize,
int compressionLevel);
</b><p> Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
enough space to successfully compress the data.
@return : compressed size written into `dst` (<= `dstCapacity),
or an error code if it fails (which can be tested using ZSTD_isError()).
</p></pre><BR>
@@ -486,7 +487,8 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); </b>/* accept NULL pointer */<b>
Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
- Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
- The function is always blocking, returns when compression is completed.
Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
enough space to successfully compress the data, though it is possible it fails for other reasons.
@return : compressed size written into `dst` (<= `dstCapacity),
or an error code if it fails (which can be tested using ZSTD_isError()).
@@ -866,9 +868,11 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds); </b>/* accept NULL pointer */<b>
<a name="Chapter13"></a><h2>Advanced dictionary and prefix API (Requires v1.4.0+)</h2><pre>
This API allows dictionaries to be used with ZSTD_compress2(),
ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
only reset with the context is reset with ZSTD_reset_parameters or
ZSTD_reset_session_and_parameters. Prefixes are single-use.
ZSTD_compressStream2(), and ZSTD_decompressDCtx().
Dictionaries are sticky, they remain valid when same context is re-used,
they only reset when the context is reset
with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters.
In contrast, Prefixes are single-use.
<BR></pre>
<pre><b>size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
@@ -888,7 +892,11 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds); </b>/* accept NULL pointer */<b>
Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
In such a case, dictionary buffer must outlive its users.
Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
to precisely select how dictionary content must be interpreted.
to precisely select how dictionary content must be interpreted.
Note 5 : This method does not benefit from LDM (long distance mode).
If you want to employ LDM on some large dictionary content,
prefer employing ZSTD_CCtx_refPrefix() described below.
</p></pre><BR>
<pre><b>size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
@@ -912,6 +920,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds); </b>/* accept NULL pointer */<b>
Decompression will need same prefix to properly regenerate data.
Compressing with a prefix is similar in outcome as performing a diff and compressing it,
but performs much faster, especially during decompression (compression speed is tunable with compression level).
This method is compatible with LDM (long distance mode).
@result : 0, or an error code (which can be tested with ZSTD_isError()).
Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
Note 1 : Prefix buffer is referenced. It **must** outlive compression.
@@ -1146,7 +1155,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
ZSTD_ps_disable = 2 </b>/* Do not use the feature */<b>
} ZSTD_paramSwitch_e;
</b></pre><BR>
<a name="Chapter15"></a><h2>Frame size functions</h2><pre></pre>
<a name="Chapter15"></a><h2>Frame header and size functions</h2><pre></pre>
<pre><b>ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
</b><p> `src` should point to the start of a series of ZSTD encoded and/or skippable frames
@@ -1192,6 +1201,31 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
or an error code (if srcSize is too small)
</p></pre><BR>
<pre><b>typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
</b></pre><BR>
<pre><b>typedef struct {
unsigned long long frameContentSize; </b>/* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */<b>
unsigned long long windowSize; </b>/* can be very large, up to <= frameContentSize */<b>
unsigned blockSizeMax;
ZSTD_frameType_e frameType; </b>/* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */<b>
unsigned headerSize;
unsigned dictID;
unsigned checksumFlag;
unsigned _reserved1;
unsigned _reserved2;
} ZSTD_frameHeader;
</b></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input */<b>
</b>/*! ZSTD_getFrameHeader_advanced() :<b>
* same as ZSTD_getFrameHeader(),
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
</b><p> decode Frame Header, or requires larger `srcSize`.
@return : 0, `zfhPtr` is correctly filled,
>0, `srcSize` is too small, value is wanted `srcSize` amount,
or an error code, which can be tested using ZSTD_isError()
</p></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
</b><p> Zstd supports in-place decompression, where the input and output buffers overlap.
In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
@@ -1505,10 +1539,24 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPo
</p></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
</b><p> Set all parameters provided within @cparams into the working @cctx.
</b><p> Set all parameters provided within @p cparams into the working @p cctx.
Note : if modifying parameters during compression (MT mode only),
note that changes to the .windowLog parameter will be ignored.
@return 0 on success, or an error code (can be checked with ZSTD_isError())
@return 0 on success, or an error code (can be checked with ZSTD_isError()).
On failure, no parameters are updated.
</p></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams);
</b><p> Set all parameters provided within @p fparams into the working @p cctx.
@return 0 on success, or an error code (can be checked with ZSTD_isError()).
</p></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params);
</b><p> Set all parameters provided within @p params into the working @p cctx.
@return 0 on success, or an error code (can be checked with ZSTD_isError()).
</p></pre><BR>
<pre><b>ZSTD_DEPRECATED("use ZSTD_compress2")
@@ -1754,12 +1802,9 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize,
ZSTD_parameters params,
unsigned long long pledgedSrcSize);
</b><p> This function is DEPRECATED, and is approximately equivalent to:
</b><p> This function is DEPRECATED, and is equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
// Pseudocode: Set each zstd parameter and leave the rest as-is.
for ((param, value) : params) {
ZSTD_CCtx_setParameter(zcs, param, value);
}
ZSTD_CCtx_setParams(zcs, params);
ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
@@ -1788,12 +1833,9 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
const ZSTD_CDict* cdict,
ZSTD_frameParameters fParams,
unsigned long long pledgedSrcSize);
</b><p> This function is DEPRECATED, and is approximately equivalent to:
</b><p> This function is DEPRECATED, and is equivalent to:
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
// Pseudocode: Set each zstd frame parameter and leave the rest as-is.
for ((fParam, value) : fParams) {
ZSTD_CCtx_setParameter(zcs, fParam, value);
}
ZSTD_CCtx_setFParams(zcs, fParams);
ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
ZSTD_CCtx_refCDict(zcs, cdict);
@@ -1880,10 +1922,42 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
</p></pre><BR>
<a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
But it's also a complex one, with several restrictions, documented below.
Prefer normal streaming API for an easier experience.
<pre><b>ZSTDLIB_STATIC_API void
ZSTD_registerSequenceProducer(
ZSTD_CCtx* cctx,
void* sequenceProducerState,
ZSTD_sequenceProducer_F* sequenceProducer
);
</b><p> Instruct zstd to use a block-level external sequence producer function.
The sequenceProducerState must be initialized by the caller, and the caller is
responsible for managing its lifetime. This parameter is sticky across
compressions. It will remain set until the user explicitly resets compression
parameters.
Sequence producer registration is considered to be an "advanced parameter",
part of the "advanced API". This means it will only have an effect on compression
APIs which respect advanced parameters, such as compress2() and compressStream2().
Older compression APIs such as compressCCtx(), which predate the introduction of
"advanced parameters", will ignore any external sequence producer setting.
The sequence producer can be "cleared" by registering a NULL function pointer. This
removes all limitations described above in the "LIMITATIONS" section of the API docs.
The user is strongly encouraged to read the full API documentation (above) before
calling this function.
</p></pre><BR>
<a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions (DEPRECATED)</h2><pre>
This API is deprecated, and will be removed in a future version.
It allows streaming (de)compression with user allocated buffers.
However, it is hard to use, and not as well tested as the rest of
our API.
Please use the normal streaming API instead: ZSTD_compressStream2,
and ZSTD_decompressStream.
If there is functionality that you need, but it doesn't provide,
please open an issue on our GitHub.
<BR></pre>
@@ -1914,8 +1988,11 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
`ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
<BR></pre>
<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
<h3>Buffer-less streaming compression functions</h3><pre></pre><b><pre>ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); </b>/**< note: fails if cdict==NULL */<b>
</pre></b><BR>
<pre><b>size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
@@ -1993,36 +2070,25 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const Z
For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
<BR></pre>
<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre>typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
typedef struct {
unsigned long long frameContentSize; </b>/* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */<b>
unsigned long long windowSize; </b>/* can be very large, up to <= frameContentSize */<b>
unsigned blockSizeMax;
ZSTD_frameType_e frameType; </b>/* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */<b>
unsigned headerSize;
unsigned dictID;
unsigned checksumFlag;
unsigned _reserved1;
unsigned _reserved2;
} ZSTD_frameHeader;
</pre></b><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input */<b>
</b>/*! ZSTD_getFrameHeader_advanced() :<b>
* same as ZSTD_getFrameHeader(),
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
</b><p> decode Frame Header, or requires larger `srcSize`.
@return : 0, `zfhPtr` is correctly filled,
>0, `srcSize` is too small, value is wanted `srcSize` amount,
or an error code, which can be tested using ZSTD_isError()
</p></pre><BR>
<h3>Buffer-less streaming decompression functions</h3><pre></pre><b><pre></pre></b><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
</b></pre><BR>
<pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
</b></pre><BR>
<a name="Chapter23"></a><h2>Block level API</h2><pre></pre>
<a name="Chapter23"></a><h2>Block level API (DEPRECATED)</h2><pre></pre>
<pre><b></b><p> Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
<pre><b></b><p> You can get the frame header down to 2 bytes by setting:
- ZSTD_c_format = ZSTD_f_zstd1_magicless
- ZSTD_c_contentSizeFlag = 0
- ZSTD_c_checksumFlag = 0
- ZSTD_c_dictIDFlag = 0
This API is not as well tested as our normal API, so we recommend not using it.
We will be removing it in a future version. If the normal API doesn't provide
the functionality you need, please open a GitHub issue.
Block functions produce and decode raw zstd blocks, without frame metadata.
Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
A few rules to respect :
@@ -2046,36 +2112,14 @@ ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowS
Use ZSTD_insertBlock() for such a case.
</p></pre><BR>
<h3>Raw zstd block functions</h3><pre></pre><b><pre>ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
<h3>Raw zstd block functions</h3><pre></pre><b><pre>ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); </b>/**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */<b>
</pre></b><BR>
<pre><b>ZSTDLIB_STATIC_API void
ZSTD_registerSequenceProducer(
ZSTD_CCtx* cctx,
void* sequenceProducerState,
ZSTD_sequenceProducer_F* sequenceProducer
);
</b><p> Instruct zstd to use a block-level external sequence producer function.
The sequenceProducerState must be initialized by the caller, and the caller is
responsible for managing its lifetime. This parameter is sticky across
compressions. It will remain set until the user explicitly resets compression
parameters.
Sequence producer registration is considered to be an "advanced parameter",
part of the "advanced API". This means it will only have an effect on compression
APIs which respect advanced parameters, such as compress2() and compressStream2().
Older compression APIs such as compressCCtx(), which predate the introduction of
"advanced parameters", will ignore any external sequence producer setting.
The sequence producer can be "cleared" by registering a NULL function pointer. This
removes all limitations described above in the "LIMITATIONS" section of the API docs.
The user is strongly encouraged to read the full API documentation (above) before
calling this function.
</p></pre><BR>
</html>
</body>
lib/common/allocations.h
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* This file provides custom allocation primitives
*/
#define ZSTD_DEPS_NEED_MALLOC
#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
#include "mem.h" /* MEM_STATIC */
#define ZSTD_STATIC_LINKING_ONLY
#include "../zstd.h" /* ZSTD_customMem */
#ifndef ZSTD_ALLOCATIONS_H
#define ZSTD_ALLOCATIONS_H
/* custom memory allocation functions */
MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
{
if (customMem.customAlloc)
return customMem.customAlloc(customMem.opaque, size);
return ZSTD_malloc(size);
}
MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
{
if (customMem.customAlloc) {
/* calloc implemented as malloc+memset;
* not as efficient as calloc, but next best guess for custom malloc */
void* const ptr = customMem.customAlloc(customMem.opaque, size);
ZSTD_memset(ptr, 0, size);
return ptr;
}
return ZSTD_calloc(1, size);
}
MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
{
if (ptr!=NULL) {
if (customMem.customFree)
customMem.customFree(customMem.opaque, ptr);
else
ZSTD_free(ptr);
}
}
#endif /* ZSTD_ALLOCATIONS_H */
lib/common/bits.h
+30 -5
View File
@@ -17,7 +17,7 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
{
assert(val != 0);
{
static const int DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3,
static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3,
30, 22, 20, 15, 25, 17, 4, 8,
31, 27, 13, 23, 21, 19, 16, 7,
26, 12, 18, 6, 11, 5, 10, 9};
@@ -30,7 +30,7 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
assert(val != 0);
# if defined(_MSC_VER)
# if STATIC_BMI2 == 1
return _tzcnt_u32(val);
return (unsigned)_tzcnt_u32(val);
# else
if (val != 0) {
unsigned long r;
@@ -69,7 +69,7 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
assert(val != 0);
# if defined(_MSC_VER)
# if STATIC_BMI2 == 1
return _lzcnt_u32(val);
return (unsigned)_lzcnt_u32(val);
# else
if (val != 0) {
unsigned long r;
@@ -92,7 +92,7 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
assert(val != 0);
# if defined(_MSC_VER) && defined(_WIN64)
# if STATIC_BMI2 == 1
return _tzcnt_u64(val);
return (unsigned)_tzcnt_u64(val);
# else
if (val != 0) {
unsigned long r;
@@ -123,7 +123,7 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
assert(val != 0);
# if defined(_MSC_VER) && defined(_WIN64)
# if STATIC_BMI2 == 1
return _lzcnt_u64(val);
return (unsigned)_lzcnt_u64(val);
# else
if (val != 0) {
unsigned long r;
@@ -172,4 +172,29 @@ MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCo
return 31 - ZSTD_countLeadingZeros32(val);
}
/* ZSTD_rotateRight_*():
* Rotates a bitfield to the right by "count" bits.
* https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts
*/
MEM_STATIC
U64 ZSTD_rotateRight_U64(U64 const value, U32 count) {
assert(count < 64);
count &= 0x3F; /* for fickle pattern recognition */
return (value >> count) | (U64)(value << ((0U - count) & 0x3F));
}
MEM_STATIC
U32 ZSTD_rotateRight_U32(U32 const value, U32 count) {
assert(count < 32);
count &= 0x1F; /* for fickle pattern recognition */
return (value >> count) | (U32)(value << ((0U - count) & 0x1F));
}
MEM_STATIC
U16 ZSTD_rotateRight_U16(U16 const value, U32 count) {
assert(count < 16);
count &= 0x0F; /* for fickle pattern recognition */
return (value >> count) | (U16)(value << ((0U - count) & 0x0F));
}
#endif /* ZSTD_BITS_H */
lib/common/bitstream.h
+1 -1
View File
@@ -396,7 +396,7 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
* This function is safe, it guarantees it will not read beyond src buffer.
* @return : status of `BIT_DStream_t` internal register.
* when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
MEM_STATIC FORCE_INLINE_ATTR BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
{
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */
return BIT_DStream_overflow;
lib/common/compiler.h
+4
View File
@@ -311,6 +311,10 @@ void __msan_poison(const volatile void *a, size_t size);
/* Returns the offset of the first (at least partially) poisoned byte in the
memory range, or -1 if the whole range is good. */
intptr_t __msan_test_shadow(const volatile void *x, size_t size);
/* Print shadow and origin for the memory range to stderr in a human-readable
format. */
void __msan_print_shadow(const volatile void *x, size_t size);
#endif
#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE)
lib/common/pool.c
+1 -1
View File
@@ -10,9 +10,9 @@
/* ====== Dependencies ======= */
#include "../common/allocations.h" /* ZSTD_customCalloc, ZSTD_customFree */
#include "zstd_deps.h" /* size_t */
#include "debug.h" /* assert */
#include "zstd_internal.h" /* ZSTD_customCalloc, ZSTD_customFree */
#include "pool.h"
/* ====== Compiler specifics ====== */
lib/common/threading.c
+1 -1
View File
@@ -47,7 +47,7 @@ static unsigned __stdcall worker(void *arg)
void* (*start_routine)(void*);
void* thread_arg;
/* Inialized thread_arg and start_routine and signal main thread that we don't need it
/* Initialized thread_arg and start_routine and signal main thread that we don't need it
* to wait any longer.
*/
{
lib/common/zstd_common.c
-35
View File
@@ -14,7 +14,6 @@
* Dependencies
***************************************/
#define ZSTD_DEPS_NEED_MALLOC
#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
#include "error_private.h"
#include "zstd_internal.h"
@@ -47,37 +46,3 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
/*! ZSTD_getErrorString() :
* provides error code string from enum */
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
/*=**************************************************************
* Custom allocator
****************************************************************/
void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
{
if (customMem.customAlloc)
return customMem.customAlloc(customMem.opaque, size);
return ZSTD_malloc(size);
}
void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
{
if (customMem.customAlloc) {
/* calloc implemented as malloc+memset;
* not as efficient as calloc, but next best guess for custom malloc */
void* const ptr = customMem.customAlloc(customMem.opaque, size);
ZSTD_memset(ptr, 0, size);
return ptr;
}
return ZSTD_calloc(1, size);
}
void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
{
if (ptr!=NULL) {
if (customMem.customFree)
customMem.customFree(customMem.opaque, ptr);
else
ZSTD_free(ptr);
}
}
lib/common/zstd_internal.h
-5
View File
@@ -350,11 +350,6 @@ typedef struct {
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
/* custom memory allocation functions */
void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);
void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem);
void ZSTD_customFree(void* ptr, ZSTD_customMem customMem);
/* ZSTD_invalidateRepCodes() :
* ensures next compression will not use repcodes from previous block.
lib/compress/zstd_compress.c
+196 -95
View File
@@ -11,6 +11,7 @@
/*-*************************************
* Dependencies
***************************************/
#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
#include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */
#include "../common/mem.h"
#include "hist.h" /* HIST_countFast_wksp */
@@ -26,7 +27,7 @@
#include "zstd_opt.h"
#include "zstd_ldm.h"
#include "zstd_compress_superblock.h"
#include "../common/bits.h" /* ZSTD_highbit32 */
#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_rotateRight_U64 */
/* ***************************************************************
* Tuning parameters
@@ -1177,16 +1178,39 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams)
{
ZSTD_STATIC_ASSERT(sizeof(cparams) == 7 * 4 /* all params are listed below */);
DEBUGLOG(4, "ZSTD_CCtx_setCParams");
assert(cctx != NULL);
if (cctx->streamStage != zcss_init) {
/* All parameters in @cparams are allowed to be updated during MT compression.
* This must be signaled, so that MT compression picks up the changes */
cctx->cParamsChanged = 1;
}
/* only update if parameters are valid */
/* only update if all parameters are valid */
FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), "");
cctx->requestedParams.cParams = cparams;
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, cparams.windowLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, cparams.chainLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, cparams.hashLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, cparams.searchLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, cparams.minMatch), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, cparams.targetLength), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, cparams.strategy), "");
return 0;
}
size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams)
{
ZSTD_STATIC_ASSERT(sizeof(fparams) == 3 * 4 /* all params are listed below */);
DEBUGLOG(4, "ZSTD_CCtx_setFParams");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, fparams.contentSizeFlag != 0), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, fparams.checksumFlag != 0), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, fparams.noDictIDFlag == 0), "");
return 0;
}
size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params)
{
DEBUGLOG(4, "ZSTD_CCtx_setParams");
/* First check cParams, because we want to update all or none. */
FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), "");
/* Next set fParams, because this could fail if the cctx isn't in init stage. */
FORWARD_IF_ERROR(ZSTD_CCtx_setFParams(cctx, params.fParams), "");
/* Finally set cParams, which should succeed. */
FORWARD_IF_ERROR(ZSTD_CCtx_setCParams(cctx, params.cParams), "");
return 0;
}
@@ -1592,7 +1616,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
+ ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t))
+ ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)
? ZSTD_cwksp_aligned_alloc_size(hSize*sizeof(U16))
? ZSTD_cwksp_aligned_alloc_size(hSize)
: 0;
size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
? optPotentialSpace
@@ -1883,6 +1907,19 @@ typedef enum {
ZSTD_resetTarget_CCtx
} ZSTD_resetTarget_e;
/* Mixes bits in a 64 bits in a value, based on XXH3_rrmxmx */
static U64 ZSTD_bitmix(U64 val, U64 len) {
val ^= ZSTD_rotateRight_U64(val, 49) ^ ZSTD_rotateRight_U64(val, 24);
val *= 0x9FB21C651E98DF25ULL;
val ^= (val >> 35) + len ;
val *= 0x9FB21C651E98DF25ULL;
return val ^ (val >> 28);
}
/* Mixes in the hashSalt and hashSaltEntropy to create a new hashSalt */
static void ZSTD_advanceHashSalt(ZSTD_matchState_t* ms) {
ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4);
}
static size_t
ZSTD_reset_matchState(ZSTD_matchState_t* ms,
@@ -1910,6 +1947,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
}
ms->hashLog3 = hashLog3;
ms->lazySkipping = 0;
ZSTD_invalidateMatchState(ms);
@@ -1931,6 +1969,27 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
ZSTD_cwksp_clean_tables(ws);
}
if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
/* Row match finder needs an additional table of hashes ("tags") */
size_t const tagTableSize = hSize;
/* We want to generate a new salt in case we reset a Cctx, but we always want to use
* 0 when we reset a Cdict */
if(forWho == ZSTD_resetTarget_CCtx) {
ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize);
ZSTD_advanceHashSalt(ms);
} else {
/* When we are not salting we want to always memset the memory */
ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned(ws, tagTableSize);
ZSTD_memset(ms->tagTable, 0, tagTableSize);
ms->hashSalt = 0;
}
{ /* Switch to 32-entry rows if searchLog is 5 (or more) */
U32 const rowLog = BOUNDED(4, cParams->searchLog, 6);
assert(cParams->hashLog >= rowLog);
ms->rowHashLog = cParams->hashLog - rowLog;
}
}
/* opt parser space */
if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) {
DEBUGLOG(4, "reserving optimal parser space");
@@ -1942,19 +2001,6 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
}
if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
{ /* Row match finder needs an additional table of hashes ("tags") */
size_t const tagTableSize = hSize*sizeof(U16);
ms->tagTable = (U16*)ZSTD_cwksp_reserve_aligned(ws, tagTableSize);
if (ms->tagTable) ZSTD_memset(ms->tagTable, 0, tagTableSize);
}
{ /* Switch to 32-entry rows if searchLog is 5 (or more) */
U32 const rowLog = BOUNDED(4, cParams->searchLog, 6);
assert(cParams->hashLog >= rowLog);
ms->rowHashLog = cParams->hashLog - rowLog;
}
}
ms->cParams = *cParams;
RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
@@ -2101,45 +2147,16 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock);
/* ZSTD_wildcopy() is used to copy into the literals buffer,
* so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes.
*/
zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH);
zc->seqStore.maxNbLit = blockSize;
/* buffers */
zc->bufferedPolicy = zbuff;
zc->inBuffSize = buffInSize;
zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize);
zc->outBuffSize = buffOutSize;
zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize);
/* ldm bucketOffsets table */
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
/* TODO: avoid memset? */
size_t const numBuckets =
((size_t)1) << (params->ldmParams.hashLog -
params->ldmParams.bucketSizeLog);
zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets);
ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets);
}
/* sequences storage */
ZSTD_referenceExternalSequences(zc, NULL, 0);
zc->seqStore.maxNbSeq = maxNbSeq;
zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef));
FORWARD_IF_ERROR(ZSTD_reset_matchState(
&zc->blockState.matchState,
ws,
&params->cParams,
params->useRowMatchFinder,
crp,
needsIndexReset,
ZSTD_resetTarget_CCtx), "");
&zc->blockState.matchState,
ws,
&params->cParams,
params->useRowMatchFinder,
crp,
needsIndexReset,
ZSTD_resetTarget_CCtx), "");
zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef));
/* ldm hash table */
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
@@ -2162,8 +2179,39 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
(ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
}
/* buffers */
/* ZSTD_wildcopy() is used to copy into the literals buffer,
* so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes.
*/
zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH);
zc->seqStore.maxNbLit = blockSize;
zc->bufferedPolicy = zbuff;
zc->inBuffSize = buffInSize;
zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize);
zc->outBuffSize = buffOutSize;
zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize);
/* ldm bucketOffsets table */
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
/* TODO: avoid memset? */
size_t const numBuckets =
((size_t)1) << (params->ldmParams.hashLog -
params->ldmParams.bucketSizeLog);
zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets);
ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets);
}
/* sequences storage */
ZSTD_referenceExternalSequences(zc, NULL, 0);
zc->seqStore.maxNbSeq = maxNbSeq;
zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace));
assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace));
zc->initialized = 1;
@@ -2338,10 +2386,11 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
}
/* copy tag table */
if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) {
size_t const tagTableSize = hSize*sizeof(U16);
size_t const tagTableSize = hSize;
ZSTD_memcpy(cctx->blockState.matchState.tagTable,
cdict->matchState.tagTable,
tagTableSize);
cdict->matchState.tagTable,
tagTableSize);
cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt;
}
}
@@ -3858,9 +3907,10 @@ ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRe
const seqStore_t* const seqStore, U32 const nbSeq)
{
U32 idx = 0;
U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengthPos : nbSeq;
for (; idx < nbSeq; ++idx) {
seqDef* const seq = seqStore->sequencesStart + idx;
U32 const ll0 = (seq->litLength == 0);
U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx);
U32 const offBase = seq->offBase;
assert(offBase > 0);
if (OFFBASE_IS_REPCODE(offBase)) {
@@ -4576,31 +4626,51 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
}
}
size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize);
return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */);
}
/* NOTE: Must just wrap ZSTD_compressContinue_public() */
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
return ZSTD_compressContinue_public(cctx, dst, dstCapacity, src, srcSize);
}
size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
static size_t ZSTD_getBlockSize_deprecated(const ZSTD_CCtx* cctx)
{
ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams;
assert(!ZSTD_checkCParams(cParams));
return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog);
}
size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
/* NOTE: Must just wrap ZSTD_getBlockSize_deprecated() */
size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
{
return ZSTD_getBlockSize_deprecated(cctx);
}
/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */
size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize);
{ size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
{ size_t const blockSizeMax = ZSTD_getBlockSize_deprecated(cctx);
RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); }
return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
}
/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */
size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
return ZSTD_compressBlock_deprecated(cctx, dst, dstCapacity, src, srcSize);
}
/*! ZSTD_loadDictionaryContent() :
* @return : 0, or an error code
*/
@@ -4644,31 +4714,42 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
ip = iend - maxDictSize;
src = ip;
srcSize = maxDictSize;
} }
}
}
if (srcSize > ZSTD_CHUNKSIZE_MAX) {
/* We must have cleared our windows when our source is this large. */
assert(ZSTD_window_isEmpty(ms->window));
if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window));
}
ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0);
DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder);
ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0);
ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
ms->forceNonContiguous = params->deterministicRefPrefix;
if (loadLdmDict) {
if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */
ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0);
ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base);
ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
}
/* If the dict is larger than we can reasonably index in our tables, only load the suffix. */
if (params->cParams.strategy < ZSTD_btultra) {
U32 maxDictSize = 8U << MIN(MAX(params->cParams.hashLog, params->cParams.chainLog), 28);
if (srcSize > maxDictSize) {
ip = iend - maxDictSize;
src = ip;
srcSize = maxDictSize;
}
}
ms->nextToUpdate = (U32)(ip - ms->window.base);
ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
ms->forceNonContiguous = params->deterministicRefPrefix;
if (srcSize <= HASH_READ_SIZE) return 0;
ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend);
if (loadLdmDict)
ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
switch(params->cParams.strategy)
{
case ZSTD_fast:
@@ -4688,7 +4769,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
} else {
assert(params->useRowMatchFinder != ZSTD_ps_auto);
if (params->useRowMatchFinder == ZSTD_ps_enable) {
size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16);
size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog);
ZSTD_memset(ms->tagTable, 0, tagTableSize);
ZSTD_row_update(ms, iend-HASH_READ_SIZE);
DEBUGLOG(4, "Using row-based hash table for lazy dict");
@@ -4991,8 +5072,8 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
&cctxParams, pledgedSrcSize);
}
size_t
ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
static size_t
ZSTD_compressBegin_usingDict_deprecated(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_CCtx_params cctxParams;
{ ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
@@ -5003,9 +5084,15 @@ ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize,
&cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
}
size_t
ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
{
return ZSTD_compressBegin_usingDict_deprecated(cctx, dict, dictSize, compressionLevel);
}
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
{
return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
return ZSTD_compressBegin_usingDict_deprecated(cctx, NULL, 0, compressionLevel);
}
@@ -5075,9 +5162,9 @@ void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize)
#endif
}
size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
size_t endResult;
size_t const cSize = ZSTD_compressContinue_internal(cctx,
@@ -5101,6 +5188,14 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
return cSize + endResult;
}
/* NOTE: Must just wrap ZSTD_compressEnd_public() */
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize);
}
size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@@ -5129,7 +5224,7 @@ size_t ZSTD_compress_advanced_internal(
FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
params, srcSize, ZSTDb_not_buffered) , "");
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize);
}
size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
@@ -5451,6 +5546,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
params.cParams = cParams;
params.useRowMatchFinder = useRowMatchFinder;
cdict->useRowMatchFinder = useRowMatchFinder;
cdict->compressionLevel = ZSTD_NO_CLEVEL;
if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dict, dictSize,
@@ -5530,12 +5626,17 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
/* ZSTD_compressBegin_usingCDict() :
* cdict must be != NULL */
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{
ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN);
}
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{
return ZSTD_compressBegin_usingCDict_deprecated(cctx, cdict);
}
/*! ZSTD_compress_usingCDict_internal():
* Implementation of various ZSTD_compress_usingCDict* functions.
*/
@@ -5545,7 +5646,7 @@ static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx,
const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
{
FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize);
}
/*! ZSTD_compress_usingCDict_advanced():
@@ -5803,7 +5904,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
|| zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */
&& (zcs->inBuffPos == 0) ) {
/* shortcut to compression pass directly into output buffer */
size_t const cSize = ZSTD_compressEnd(zcs,
size_t const cSize = ZSTD_compressEnd_public(zcs,
op, oend-op, ip, iend-ip);
DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize);
FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed");
@@ -5861,9 +5962,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
if (inputBuffered) {
unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
cSize = lastBlock ?
ZSTD_compressEnd(zcs, cDst, oSize,
ZSTD_compressEnd_public(zcs, cDst, oSize,
zcs->inBuff + zcs->inToCompress, iSize) :
ZSTD_compressContinue(zcs, cDst, oSize,
ZSTD_compressContinue_public(zcs, cDst, oSize,
zcs->inBuff + zcs->inToCompress, iSize);
FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
zcs->frameEnded = lastBlock;
@@ -5879,8 +5980,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
} else { /* !inputBuffered, hence ZSTD_bm_stable */
unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend);
cSize = lastBlock ?
ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) :
ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize);
ZSTD_compressEnd_public(zcs, cDst, oSize, ip, iSize) :
ZSTD_compressContinue_public(zcs, cDst, oSize, ip, iSize);
/* Consume the input prior to error checking to mirror buffered mode. */
if (ip) ip += iSize;
FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
lib/compress/zstd_compress_internal.h
+67 -13
View File
@@ -226,8 +226,10 @@ struct ZSTD_matchState_t {
U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */
U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/
U16* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */
BYTE* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */
U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */
U64 hashSalt; /* For row-based matchFinder: salts the hash for re-use of tag table */
U32 hashSaltEntropy; /* For row-based matchFinder: collects entropy for salt generation */
U32* hashTable;
U32* hashTable3;
@@ -247,6 +249,13 @@ struct ZSTD_matchState_t {
* This behavior is controlled from the cctx ms.
* This parameter has no effect in the cdict ms. */
int prefetchCDictTables;
/* When == 0, lazy match finders insert every position.
* When != 0, lazy match finders only insert positions they search.
* This allows them to skip much faster over incompressible data,
* at a small cost to compression ratio.
*/
int lazySkipping;
};
typedef struct {
@@ -787,28 +796,35 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match,
* Hashes
***************************************/
static const U32 prime3bytes = 506832829U;
static U32 ZSTD_hash3(U32 u, U32 h) { assert(h <= 32); return ((u << (32-24)) * prime3bytes) >> (32-h) ; }
MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */
static U32 ZSTD_hash3(U32 u, U32 h, U32 s) { assert(h <= 32); return (((u << (32-24)) * prime3bytes) ^ s) >> (32-h) ; }
MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h, 0); } /* only in zstd_opt.h */
MEM_STATIC size_t ZSTD_hash3PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash3(MEM_readLE32(ptr), h, s); }
static const U32 prime4bytes = 2654435761U;
static U32 ZSTD_hash4(U32 u, U32 h) { assert(h <= 32); return (u * prime4bytes) >> (32-h) ; }
static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_readLE32(ptr), h); }
static U32 ZSTD_hash4(U32 u, U32 h, U32 s) { assert(h <= 32); return ((u * prime4bytes) ^ s) >> (32-h) ; }
static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_readLE32(ptr), h, 0); }
static size_t ZSTD_hash4PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash4(MEM_readLE32(ptr), h, s); }
static const U64 prime5bytes = 889523592379ULL;
static size_t ZSTD_hash5(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; }
static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); }
static size_t ZSTD_hash5(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-40)) * prime5bytes) ^ s) >> (64-h)) ; }
static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h, 0); }
static size_t ZSTD_hash5PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash5(MEM_readLE64(p), h, s); }
static const U64 prime6bytes = 227718039650203ULL;
static size_t ZSTD_hash6(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
static size_t ZSTD_hash6(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-48)) * prime6bytes) ^ s) >> (64-h)) ; }
static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h, 0); }
static size_t ZSTD_hash6PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash6(MEM_readLE64(p), h, s); }
static const U64 prime7bytes = 58295818150454627ULL;
static size_t ZSTD_hash7(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; }
static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); }
static size_t ZSTD_hash7(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-56)) * prime7bytes) ^ s) >> (64-h)) ; }
static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h, 0); }
static size_t ZSTD_hash7PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash7(MEM_readLE64(p), h, s); }
static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
static size_t ZSTD_hash8(U64 u, U32 h) { assert(h <= 64); return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
static size_t ZSTD_hash8(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u) * prime8bytes) ^ s) >> (64-h)) ; }
static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h, 0); }
static size_t ZSTD_hash8PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash8(MEM_readLE64(p), h, s); }
MEM_STATIC FORCE_INLINE_ATTR
size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
@@ -828,6 +844,24 @@ size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
}
}
MEM_STATIC FORCE_INLINE_ATTR
size_t ZSTD_hashPtrSalted(const void* p, U32 hBits, U32 mls, const U64 hashSalt) {
/* Although some of these hashes do support hBits up to 64, some do not.
* To be on the safe side, always avoid hBits > 32. */
assert(hBits <= 32);
switch(mls)
{
default:
case 4: return ZSTD_hash4PtrS(p, hBits, (U32)hashSalt);
case 5: return ZSTD_hash5PtrS(p, hBits, hashSalt);
case 6: return ZSTD_hash6PtrS(p, hBits, hashSalt);
case 7: return ZSTD_hash7PtrS(p, hBits, hashSalt);
case 8: return ZSTD_hash8PtrS(p, hBits, hashSalt);
}
}
/** ZSTD_ipow() :
* Return base^exponent.
*/
@@ -1475,4 +1509,24 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
/* ===============================================================
* Deprecated definitions that are still used internally to avoid
* deprecation warnings. These functions are exactly equivalent to
* their public variants, but avoid the deprecation warnings.
* =============================================================== */
size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
#endif /* ZSTD_COMPRESS_H */
lib/compress/zstd_cwksp.h
+123 -59
View File
@@ -14,7 +14,9 @@
/*-*************************************
* Dependencies
***************************************/
#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */
#include "../common/zstd_internal.h"
#include "../common/portability_macros.h"
#if defined (__cplusplus)
extern "C" {
@@ -44,8 +46,9 @@ extern "C" {
***************************************/
typedef enum {
ZSTD_cwksp_alloc_objects,
ZSTD_cwksp_alloc_buffers,
ZSTD_cwksp_alloc_aligned
ZSTD_cwksp_alloc_aligned_init_once,
ZSTD_cwksp_alloc_aligned,
ZSTD_cwksp_alloc_buffers
} ZSTD_cwksp_alloc_phase_e;
/**
@@ -98,8 +101,8 @@ typedef enum {
*
* Workspace Layout:
*
* [ ... workspace ... ]
* [objects][tables ... ->] free space [<- ... aligned][<- ... buffers]
* [ ... workspace ... ]
* [objects][tables ->] free space [<- buffers][<- aligned][<- init once]
*
* The various objects that live in the workspace are divided into the
* following categories, and are allocated separately:
@@ -123,9 +126,18 @@ typedef enum {
* uint32_t arrays, all of whose values are between 0 and (nextSrc - base).
* Their sizes depend on the cparams. These tables are 64-byte aligned.
*
* - Aligned: these buffers are used for various purposes that require 4 byte
* alignment, but don't require any initialization before they're used. These
* buffers are each aligned to 64 bytes.
* - Init once: these buffers require to be initialized at least once before
* use. They should be used when we want to skip memory initialization
* while not triggering memory checkers (like Valgrind) when reading from
* from this memory without writing to it first.
* These buffers should be used carefully as they might contain data
* from previous compressions.
* Buffers are aligned to 64 bytes.
*
* - Aligned: these buffers don't require any initialization before they're
* used. The user of the buffer should make sure they write into a buffer
* location before reading from it.
* Buffers are aligned to 64 bytes.
*
* - Buffers: these buffers are used for various purposes that don't require
* any alignment or initialization before they're used. This means they can
@@ -137,8 +149,9 @@ typedef enum {
* correctly packed into the workspace buffer. That order is:
*
* 1. Objects
* 2. Buffers
* 3. Aligned/Tables
* 2. Init once / Tables
* 3. Aligned / Tables
* 4. Buffers / Tables
*
* Attempts to reserve objects of different types out of order will fail.
*/
@@ -150,6 +163,7 @@ typedef struct {
void* tableEnd;
void* tableValidEnd;
void* allocStart;
void* initOnceStart;
BYTE allocFailed;
int workspaceOversizedDuration;
@@ -162,6 +176,7 @@ typedef struct {
***************************************/
MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws);
MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws);
MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
(void)ws;
@@ -171,6 +186,20 @@ MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
assert(ws->tableEnd <= ws->allocStart);
assert(ws->tableValidEnd <= ws->allocStart);
assert(ws->allocStart <= ws->workspaceEnd);
assert(ws->initOnceStart <= ZSTD_cwksp_initialAllocStart(ws));
assert(ws->workspace <= ws->initOnceStart);
#if ZSTD_MEMORY_SANITIZER
{
intptr_t const offset = __msan_test_shadow(ws->initOnceStart,
(U8*)ZSTD_cwksp_initialAllocStart(ws) - (U8*)ws->initOnceStart);
#if defined(ZSTD_MSAN_PRINT)
if(offset!=-1) {
__msan_print_shadow((U8*)ws->initOnceStart + offset - 8, 32);
}
#endif
assert(offset==-1);
};
#endif
}
/**
@@ -217,14 +246,10 @@ MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) {
* for internal purposes (currently only alignment).
*/
MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) {
/* For alignment, the wksp will always allocate an additional n_1=[1, 64] bytes
* to align the beginning of tables section, as well as another n_2=[0, 63] bytes
* to align the beginning of the aligned section.
*
* n_1 + n_2 == 64 bytes if the cwksp is freshly allocated, due to tables and
* aligneds being sized in multiples of 64 bytes.
/* For alignment, the wksp will always allocate an additional 2*ZSTD_CWKSP_ALIGNMENT_BYTES
* bytes to align the beginning of tables section and end of buffers;
*/
size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES;
size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES * 2;
return slackSpace;
}
@@ -237,10 +262,18 @@ MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignByt
size_t const alignBytesMask = alignBytes - 1;
size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask;
assert((alignBytes & alignBytesMask) == 0);
assert(bytes != ZSTD_CWKSP_ALIGNMENT_BYTES);
assert(bytes < alignBytes);
return bytes;
}
/**
* Returns the initial value for allocStart which is used to determine the position from
* which we can allocate from the end of the workspace.
*/
MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) {
return (void*)((size_t)ws->workspaceEnd & ~(ZSTD_CWKSP_ALIGNMENT_BYTES-1));
}
/**
* Internal function. Do not use directly.
* Reserves the given number of bytes within the aligned/buffer segment of the wksp,
@@ -281,27 +314,16 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase
{
assert(phase >= ws->phase);
if (phase > ws->phase) {
/* Going from allocating objects to allocating buffers */
if (ws->phase < ZSTD_cwksp_alloc_buffers &&
phase >= ZSTD_cwksp_alloc_buffers) {
/* Going from allocating objects to allocating initOnce / tables */
if (ws->phase < ZSTD_cwksp_alloc_aligned_init_once &&
phase >= ZSTD_cwksp_alloc_aligned_init_once) {
ws->tableValidEnd = ws->objectEnd;
}
ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws);
/* Going from allocating buffers to allocating aligneds/tables */
if (ws->phase < ZSTD_cwksp_alloc_aligned &&
phase >= ZSTD_cwksp_alloc_aligned) {
{ /* Align the start of the "aligned" to 64 bytes. Use [1, 64] bytes. */
size_t const bytesToAlign =
ZSTD_CWKSP_ALIGNMENT_BYTES - ZSTD_cwksp_bytes_to_align_ptr(ws->allocStart, ZSTD_CWKSP_ALIGNMENT_BYTES);
DEBUGLOG(5, "reserving aligned alignment addtl space: %zu", bytesToAlign);
ZSTD_STATIC_ASSERT((ZSTD_CWKSP_ALIGNMENT_BYTES & (ZSTD_CWKSP_ALIGNMENT_BYTES - 1)) == 0); /* power of 2 */
RETURN_ERROR_IF(!ZSTD_cwksp_reserve_internal_buffer_space(ws, bytesToAlign),
memory_allocation, "aligned phase - alignment initial allocation failed!");
}
{ /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */
void* const alloc = ws->objectEnd;
void *const alloc = ws->objectEnd;
size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES);
void* const objectEnd = (BYTE*)alloc + bytesToAlign;
void *const objectEnd = (BYTE *) alloc + bytesToAlign;
DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign);
RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation,
"table phase - alignment initial allocation failed!");
@@ -309,7 +331,9 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase
ws->tableEnd = objectEnd; /* table area starts being empty */
if (ws->tableValidEnd < ws->tableEnd) {
ws->tableValidEnd = ws->tableEnd;
} } }
}
}
}
ws->phase = phase;
ZSTD_cwksp_assert_internal_consistency(ws);
}
@@ -321,7 +345,7 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase
*/
MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr)
{
return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
return (ptr != NULL) && (ws->workspace <= ptr) && (ptr < ws->workspaceEnd);
}
/**
@@ -366,6 +390,36 @@ MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes)
return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers);
}
/**
* Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
* This memory has been initialized at least once in the past.
* This doesn't mean it has been initialized this time, and it might contain data from previous
* operations.
* The main usage is for algorithms that might need read access into uninitialized memory.
* The algorithm must maintain safety under these conditions and must make sure it doesn't
* leak any of the past data (directly or in side channels).
*/
MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t bytes)
{
size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES);
void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once);
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
if(ptr && ptr < ws->initOnceStart) {
/* We assume the memory following the current allocation is either:
* 1. Not usable as initOnce memory (end of workspace)
* 2. Another initOnce buffer that has been allocated before (and so was previously memset)
* 3. An ASAN redzone, in which case we don't want to write on it
* For these reasons it should be fine to not explicitly zero every byte up to ws->initOnceStart.
* Note that we assume here that MSAN and ASAN cannot run in the same time. */
ZSTD_memset(ptr, 0, MIN((size_t)((U8*)ws->initOnceStart - (U8*)ptr), alignedBytes));
ws->initOnceStart = ptr;
}
#if ZSTD_MEMORY_SANITIZER
assert(__msan_test_shadow(ptr, bytes) == -1);
#endif
return ptr;
}
/**
* Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
*/
@@ -384,13 +438,17 @@ MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes)
*/
MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes)
{
const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned;
const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned_init_once;
void* alloc;
void* end;
void* top;
if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) {
return NULL;
/* We can only start allocating tables after we are done reserving space for objects at the
* start of the workspace */
if(ws->phase < phase) {
if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) {
return NULL;
}
}
alloc = ws->tableEnd;
end = (BYTE *)alloc + bytes;
@@ -469,11 +527,19 @@ MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws)
#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
/* To validate that the table re-use logic is sound, and that we don't
* access table space that we haven't cleaned, we re-"poison" the table
* space every time we mark it dirty. */
* space every time we mark it dirty.
* Since tableValidEnd space and initOnce space may overlap we don't poison
* the initOnce portion as it break its promise. This means that this poisoning
* check isn't always applied fully. */
{
size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
assert(__msan_test_shadow(ws->objectEnd, size) == -1);
__msan_poison(ws->objectEnd, size);
if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) {
__msan_poison(ws->objectEnd, size);
} else {
assert(ws->initOnceStart >= ws->objectEnd);
__msan_poison(ws->objectEnd, (BYTE*)ws->initOnceStart - (BYTE*)ws->objectEnd);
}
}
#endif
@@ -538,11 +604,14 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
/* To validate that the context re-use logic is sound, and that we don't
* access stuff that this compression hasn't initialized, we re-"poison"
* the workspace (or at least the non-static, non-table parts of it)
* every time we start a new compression. */
* the workspace except for the areas in which we expect memory re-use
* without initialization (objects, valid tables area and init once
* memory). */
{
size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd;
__msan_poison(ws->tableValidEnd, size);
if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) {
size_t size = (BYTE*)ws->initOnceStart - (BYTE*)ws->tableValidEnd;
__msan_poison(ws->tableValidEnd, size);
}
}
#endif
@@ -558,10 +627,10 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
#endif
ws->tableEnd = ws->objectEnd;
ws->allocStart = ws->workspaceEnd;
ws->allocStart = ZSTD_cwksp_initialAllocStart(ws);
ws->allocFailed = 0;
if (ws->phase > ZSTD_cwksp_alloc_buffers) {
ws->phase = ZSTD_cwksp_alloc_buffers;
if (ws->phase > ZSTD_cwksp_alloc_aligned_init_once) {
ws->phase = ZSTD_cwksp_alloc_aligned_init_once;
}
ZSTD_cwksp_assert_internal_consistency(ws);
}
@@ -578,6 +647,7 @@ MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_c
ws->workspaceEnd = (BYTE*)start + size;
ws->objectEnd = ws->workspace;
ws->tableValidEnd = ws->objectEnd;
ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws);
ws->phase = ZSTD_cwksp_alloc_objects;
ws->isStatic = isStatic;
ZSTD_cwksp_clear(ws);
@@ -630,17 +700,11 @@ MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
* Returns if the estimated space needed for a wksp is within an acceptable limit of the
* actual amount of space used.
*/
MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp* const ws,
size_t const estimatedSpace, int resizedWorkspace) {
if (resizedWorkspace) {
/* Resized/newly allocated wksp should have exact bounds */
return ZSTD_cwksp_used(ws) == estimatedSpace;
} else {
/* Due to alignment, when reusing a workspace, we can actually consume 63 fewer or more bytes
* than estimatedSpace. See the comments in zstd_cwksp.h for details.
*/
return (ZSTD_cwksp_used(ws) >= estimatedSpace - 63) && (ZSTD_cwksp_used(ws) <= estimatedSpace + 63);
}
MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp *const ws, size_t const estimatedSpace) {
/* We have an alignment space between objects and tables between tables and buffers, so we can have up to twice
* the alignment bytes difference between estimation and actual usage */
return (estimatedSpace - ZSTD_cwksp_slack_space_required()) <= ZSTD_cwksp_used(ws) &&
ZSTD_cwksp_used(ws) <= estimatedSpace;
}
lib/compress/zstd_lazy.c
+92 -62
View File
@@ -12,6 +12,8 @@
#include "zstd_lazy.h"
#include "../common/bits.h" /* ZSTD_countTrailingZeros64 */
#define kLazySkippingStep 8
/*-*************************************
* Binary Tree search
@@ -618,7 +620,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal(
ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams,
const BYTE* ip, U32 const mls)
const BYTE* ip, U32 const mls, U32 const lazySkipping)
{
U32* const hashTable = ms->hashTable;
const U32 hashLog = cParams->hashLog;
@@ -633,6 +635,9 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal(
NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
hashTable[h] = idx;
idx++;
/* Stop inserting every position when in the lazy skipping mode. */
if (lazySkipping)
break;
}
ms->nextToUpdate = target;
@@ -641,7 +646,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal(
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
const ZSTD_compressionParameters* const cParams = &ms->cParams;
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch);
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch, /* lazySkipping*/ 0);
}
/* inlining is important to hardwire a hot branch (template emulation) */
@@ -685,7 +690,7 @@ size_t ZSTD_HcFindBestMatch(
}
/* HC4 match finder */
matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls);
matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls, ms->lazySkipping);
for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) {
size_t currentMl=0;
@@ -758,7 +763,6 @@ size_t ZSTD_HcFindBestMatch(
* (SIMD) Row-based matchfinder
***********************************/
/* Constants for row-based hash */
#define ZSTD_ROW_HASH_TAG_OFFSET 16 /* byte offset of hashes in the match state's tagTable from the beginning of a row */
#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1)
#define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */
@@ -774,39 +778,15 @@ MEM_STATIC U32 ZSTD_VecMask_next(ZSTD_VecMask val) {
return ZSTD_countTrailingZeros64(val);
}
/* ZSTD_rotateRight_*():
* Rotates a bitfield to the right by "count" bits.
* https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts
*/
FORCE_INLINE_TEMPLATE
U64 ZSTD_rotateRight_U64(U64 const value, U32 count) {
assert(count < 64);
count &= 0x3F; /* for fickle pattern recognition */
return (value >> count) | (U64)(value << ((0U - count) & 0x3F));
}
FORCE_INLINE_TEMPLATE
U32 ZSTD_rotateRight_U32(U32 const value, U32 count) {
assert(count < 32);
count &= 0x1F; /* for fickle pattern recognition */
return (value >> count) | (U32)(value << ((0U - count) & 0x1F));
}
FORCE_INLINE_TEMPLATE
U16 ZSTD_rotateRight_U16(U16 const value, U32 count) {
assert(count < 16);
count &= 0x0F; /* for fickle pattern recognition */
return (value >> count) | (U16)(value << ((0U - count) & 0x0F));
}
/* ZSTD_row_nextIndex():
* Returns the next index to insert at within a tagTable row, and updates the "head"
* value to reflect the update. Essentially cycles backwards from [0, {entries per row})
* value to reflect the update. Essentially cycles backwards from [1, {entries per row})
*/
FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) {
U32 const next = (*tagRow - 1) & rowMask;
*tagRow = (BYTE)next;
return next;
U32 next = (*tagRow-1) & rowMask;
next += (next == 0) ? rowMask : 0; /* skip first position */
*tagRow = (BYTE)next;
return next;
}
/* ZSTD_isAligned():
@@ -820,7 +800,7 @@ MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) {
/* ZSTD_row_prefetch():
* Performs prefetching for the hashTable and tagTable at a given row.
*/
FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, U16 const* tagTable, U32 const relRow, U32 const rowLog) {
FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, BYTE const* tagTable, U32 const relRow, U32 const rowLog) {
PREFETCH_L1(hashTable + relRow);
if (rowLog >= 5) {
PREFETCH_L1(hashTable + relRow + 16);
@@ -844,13 +824,13 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const B
U32 idx, const BYTE* const iLimit)
{
U32 const* const hashTable = ms->hashTable;
U16 const* const tagTable = ms->tagTable;
BYTE const* const tagTable = ms->tagTable;
U32 const hashLog = ms->rowHashLog;
U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1);
U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch);
for (; idx < lim; ++idx) {
U32 const hash = (U32)ZSTD_hashPtr(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
U32 const hash = (U32)ZSTD_hashPtrSalted(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt);
U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash;
@@ -866,11 +846,12 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const B
* base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable.
*/
FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable,
U16 const* tagTable, BYTE const* base,
BYTE const* tagTable, BYTE const* base,
U32 idx, U32 const hashLog,
U32 const rowLog, U32 const mls)
U32 const rowLog, U32 const mls,
U64 const hashSalt)
{
U32 const newHash = (U32)ZSTD_hashPtr(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
U32 const newHash = (U32)ZSTD_hashPtrSalted(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt);
U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
{ U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK];
@@ -888,22 +869,21 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms,
U32 const rowMask, U32 const useCache)
{
U32* const hashTable = ms->hashTable;
U16* const tagTable = ms->tagTable;
BYTE* const tagTable = ms->tagTable;
U32 const hashLog = ms->rowHashLog;
const BYTE* const base = ms->window.base;
DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx);
for (; updateStartIdx < updateEndIdx; ++updateStartIdx) {
U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls)
: (U32)ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls, ms->hashSalt)
: (U32)ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt);
U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
U32* const row = hashTable + relRow;
BYTE* tagRow = (BYTE*)(tagTable + relRow); /* Though tagTable is laid out as a table of U16, each tag is only 1 byte.
Explicit cast allows us to get exact desired position within each row */
BYTE* tagRow = tagTable + relRow;
U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
assert(hash == ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls));
((BYTE*)tagRow)[pos + ZSTD_ROW_HASH_TAG_OFFSET] = hash & ZSTD_ROW_HASH_TAG_MASK;
assert(hash == ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt));
tagRow[pos] = hash & ZSTD_ROW_HASH_TAG_MASK;
row[pos] = updateStartIdx;
}
}
@@ -1059,7 +1039,7 @@ ZSTD_row_getNEONMask(const U32 rowEntries, const BYTE* const src, const BYTE tag
FORCE_INLINE_TEMPLATE ZSTD_VecMask
ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGrouped, const U32 rowEntries)
{
const BYTE* const src = tagRow + ZSTD_ROW_HASH_TAG_OFFSET;
const BYTE* const src = tagRow;
assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES);
assert(ZSTD_row_matchMaskGroupWidth(rowEntries) * rowEntries <= sizeof(ZSTD_VecMask) * 8);
@@ -1144,7 +1124,7 @@ size_t ZSTD_RowFindBestMatch(
const U32 rowLog)
{
U32* const hashTable = ms->hashTable;
U16* const tagTable = ms->tagTable;
BYTE* const tagTable = ms->tagTable;
U32* const hashCache = ms->hashCache;
const U32 hashLog = ms->rowHashLog;
const ZSTD_compressionParameters* const cParams = &ms->cParams;
@@ -1163,8 +1143,10 @@ size_t ZSTD_RowFindBestMatch(
const U32 rowMask = rowEntries - 1;
const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */
const U32 groupWidth = ZSTD_row_matchMaskGroupWidth(rowEntries);
const U64 hashSalt = ms->hashSalt;
U32 nbAttempts = 1U << cappedSearchLog;
size_t ml=4-1;
U32 hash;
/* DMS/DDS variables that may be referenced laster */
const ZSTD_matchState_t* const dms = ms->dictMatchState;
@@ -1188,7 +1170,7 @@ size_t ZSTD_RowFindBestMatch(
if (dictMode == ZSTD_dictMatchState) {
/* Prefetch DMS rows */
U32* const dmsHashTable = dms->hashTable;
U16* const dmsTagTable = dms->tagTable;
BYTE* const dmsTagTable = dms->tagTable;
U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK;
@@ -1198,9 +1180,19 @@ size_t ZSTD_RowFindBestMatch(
}
/* Update the hashTable and tagTable up to (but not including) ip */
ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */);
if (!ms->lazySkipping) {
ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */);
hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls, hashSalt);
} else {
/* Stop inserting every position when in the lazy skipping mode.
* The hash cache is also not kept up to date in this mode.
*/
hash = (U32)ZSTD_hashPtrSalted(ip, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt);
ms->nextToUpdate = curr;
}
ms->hashSaltEntropy += hash; /* collect salt entropy */
{ /* Get the hash for ip, compute the appropriate row */
U32 const hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls);
U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK;
U32* const row = hashTable + relRow;
@@ -1212,9 +1204,10 @@ size_t ZSTD_RowFindBestMatch(
ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, headGrouped, rowEntries);
/* Cycle through the matches and prefetch */
for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) {
for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) {
U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask;
U32 const matchIndex = row[matchPos];
if(matchPos == 0) continue;
assert(numMatches < rowEntries);
if (matchIndex < lowLimit)
break;
@@ -1224,13 +1217,14 @@ size_t ZSTD_RowFindBestMatch(
PREFETCH_L1(dictBase + matchIndex);
}
matchBuffer[numMatches++] = matchIndex;
--nbAttempts;
}
/* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop
in ZSTD_row_update_internal() at the next search. */
{
U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
tagRow[pos + ZSTD_ROW_HASH_TAG_OFFSET] = (BYTE)tag;
tagRow[pos] = (BYTE)tag;
row[pos] = ms->nextToUpdate++;
}
@@ -1281,13 +1275,15 @@ size_t ZSTD_RowFindBestMatch(
size_t currMatch = 0;
ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, headGrouped, rowEntries);
for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) {
for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) {
U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask;
U32 const matchIndex = dmsRow[matchPos];
if(matchPos == 0) continue;
if (matchIndex < dmsLowestIndex)
break;
PREFETCH_L1(dmsBase + matchIndex);
matchBuffer[numMatches++] = matchIndex;
--nbAttempts;
}
/* Return the longest match */
@@ -1544,10 +1540,11 @@ ZSTD_compressBlock_lazy_generic(
assert(offset_2 <= dictAndPrefixLength);
}
/* Reset the lazy skipping state */
ms->lazySkipping = 0;
if (searchMethod == search_rowHash) {
ZSTD_row_fillHashCache(ms, base, rowLog,
MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */),
ms->nextToUpdate, ilimit);
ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
}
/* Match Loop */
@@ -1591,7 +1588,16 @@ ZSTD_compressBlock_lazy_generic(
}
if (matchLength < 4) {
ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */
size_t const step = ((size_t)(ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */;
ip += step;
/* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time.
* In this mode we stop inserting every position into our tables, and only insert
* positions that we search, which is one in step positions.
* The exact cutoff is flexible, I've just chosen a number that is reasonably high,
* so we minimize the compression ratio loss in "normal" scenarios. This mode gets
* triggered once we've gone 2KB without finding any matches.
*/
ms->lazySkipping = step > kLazySkippingStep;
continue;
}
@@ -1695,6 +1701,13 @@ _storeSequence:
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength);
anchor = ip = start + matchLength;
}
if (ms->lazySkipping) {
/* We've found a match, disable lazy skipping mode, and refill the hash cache. */
if (searchMethod == search_rowHash) {
ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
}
ms->lazySkipping = 0;
}
/* check immediate repcode */
if (isDxS) {
@@ -1912,12 +1925,13 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod);
/* Reset the lazy skipping state */
ms->lazySkipping = 0;
/* init */
ip += (ip == prefixStart);
if (searchMethod == search_rowHash) {
ZSTD_row_fillHashCache(ms, base, rowLog,
MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */),
ms->nextToUpdate, ilimit);
ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
}
/* Match Loop */
@@ -1955,7 +1969,16 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
}
if (matchLength < 4) {
ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */
size_t const step = ((size_t)(ip-anchor) >> kSearchStrength);
ip += step + 1; /* jump faster over incompressible sections */
/* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time.
* In this mode we stop inserting every position into our tables, and only insert
* positions that we search, which is one in step positions.
* The exact cutoff is flexible, I've just chosen a number that is reasonably high,
* so we minimize the compression ratio loss in "normal" scenarios. This mode gets
* triggered once we've gone 2KB without finding any matches.
*/
ms->lazySkipping = step > kLazySkippingStep;
continue;
}
@@ -2041,6 +2064,13 @@ _storeSequence:
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength);
anchor = ip = start + matchLength;
}
if (ms->lazySkipping) {
/* We've found a match, disable lazy skipping mode, and refill the hash cache. */
if (searchMethod == search_rowHash) {
ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
}
ms->lazySkipping = 0;
}
/* check immediate repcode */
while (ip <= ilimit) {
lib/compress/zstd_opt.c
+2
View File
@@ -1086,6 +1086,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
ZSTD_optimal_t lastSequence;
ZSTD_optLdm_t optLdm;
ZSTD_memset(&lastSequence, 0, sizeof(ZSTD_optimal_t));
optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore;
optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0;
ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip));
lib/compress/zstdmt_compress.c
+5 -4
View File
@@ -20,6 +20,7 @@
/* ====== Dependencies ====== */
#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset, INT_MAX, UINT_MAX */
#include "../common/mem.h" /* MEM_STATIC */
#include "../common/pool.h" /* threadpool */
@@ -719,7 +720,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID);
if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */
size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0);
size_t const hSize = ZSTD_compressContinue_public(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0);
if (ZSTD_isError(hSize)) JOB_ERROR(hSize);
DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize);
ZSTD_invalidateRepCodes(cctx);
@@ -737,7 +738,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks);
assert(job->cSize == 0);
for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) {
size_t const cSize = ZSTD_compressContinue(cctx, op, oend-op, ip, chunkSize);
size_t const cSize = ZSTD_compressContinue_public(cctx, op, oend-op, ip, chunkSize);
if (ZSTD_isError(cSize)) JOB_ERROR(cSize);
ip += chunkSize;
op += cSize; assert(op < oend);
@@ -757,8 +758,8 @@ static void ZSTDMT_compressionJob(void* jobDescription)
size_t const lastBlockSize1 = job->src.size & (chunkSize-1);
size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1;
size_t const cSize = (job->lastJob) ?
ZSTD_compressEnd (cctx, op, oend-op, ip, lastBlockSize) :
ZSTD_compressContinue(cctx, op, oend-op, ip, lastBlockSize);
ZSTD_compressEnd_public(cctx, op, oend-op, ip, lastBlockSize) :
ZSTD_compressContinue_public(cctx, op, oend-op, ip, lastBlockSize);
if (ZSTD_isError(cSize)) JOB_ERROR(cSize);
lastCBlockSize = cSize;
} }
lib/decompress/huf_decompress.c
+4 -4
View File
@@ -696,7 +696,7 @@ void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
/* Copy the arguments to local variables */
ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
ZSTD_memcpy(&ip, &args->ip, sizeof(ip));
ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip));
ZSTD_memcpy(&op, &args->op, sizeof(op));
assert(MEM_isLittleEndian());
@@ -779,7 +779,7 @@ _out:
/* Save the final values of each of the state variables back to args. */
ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
ZSTD_memcpy(&args->ip, &ip, sizeof(ip));
ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip));
ZSTD_memcpy(&args->op, &op, sizeof(op));
}
@@ -1476,7 +1476,7 @@ void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
/* Copy the arguments to local registers. */
ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
ZSTD_memcpy(&ip, &args->ip, sizeof(ip));
ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip));
ZSTD_memcpy(&op, &args->op, sizeof(op));
oend[0] = op[1];
@@ -1599,7 +1599,7 @@ _out:
/* Save the final values of each of the state variables back to args. */
ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
ZSTD_memcpy(&args->ip, &ip, sizeof(ip));
ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip));
ZSTD_memcpy(&args->op, &op, sizeof(op));
}
lib/decompress/zstd_ddict.c
+1
View File
@@ -14,6 +14,7 @@
/*-*******************************************************
* Dependencies
*********************************************************/
#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */
#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
#include "../common/cpu.h" /* bmi2 */
#include "../common/mem.h" /* low level memory routines */
lib/decompress/zstd_decompress.c
+1
View File
@@ -55,6 +55,7 @@
/*-*******************************************************
* Dependencies
*********************************************************/
#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
#include "../common/mem.h" /* low level memory routines */
#define FSE_STATIC_LINKING_ONLY
lib/decompress/zstd_decompress_block.c
+16 -17
View File
@@ -1232,11 +1232,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
/* sequence */
{ size_t offset;
#if defined(__clang__)
if (LIKELY(ofBits > 1)) {
#else
if (ofBits > 1) {
#endif
ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32);
@@ -1273,11 +1269,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
seq.offset = offset;
}
#if defined(__clang__)
if (UNLIKELY(mlBits > 0))
#else
if (mlBits > 0)
#endif
seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/);
if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
@@ -1287,11 +1279,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
/* Ensure there are enough bits to read the rest of data in 64-bit mode. */
ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
#if defined(__clang__)
if (UNLIKELY(llBits > 0))
#else
if (llBits > 0)
#endif
seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
if (MEM_32bits())
@@ -1987,7 +1975,7 @@ ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
/**
* @returns The total size of the history referencable by zstd, including
* @returns The total size of the history referenceable by zstd, including
* both the prefix and the extDict. At @p op any offset larger than this
* is invalid.
*/
@@ -2124,7 +2112,9 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
ip += seqHSize;
srcSize -= seqHSize;
RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall,
"invalid dst");
/* If we could potentially have long offsets, or we might want to use the prefetch decoder,
* compute information about the share of long offsets, and the maximum nbAdditionalBits.
@@ -2181,9 +2171,9 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize)
}
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
size_t dSize;
ZSTD_checkContinuity(dctx, dst, dstCapacity);
@@ -2191,3 +2181,12 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
dctx->previousDstEnd = (char*)dst + dSize;
return dSize;
}
/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize);
}
lib/decompress/zstd_decompress_block.h
+5
View File
@@ -64,5 +64,10 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
unsigned tableLog, void* wksp, size_t wkspSize,
int bmi2);
/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */
size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
#endif /* ZSTD_DEC_BLOCK_H */
lib/dictBuilder/zdict.c
+2 -2
View File
@@ -566,11 +566,11 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
size_t cSize;
if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */
{ size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict);
{ size_t const errorCode = ZSTD_compressBegin_usingCDict_deprecated(esr.zc, esr.dict);
if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; }
}
cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
cSize = ZSTD_compressBlock_deprecated(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; }
if (cSize) { /* if == 0; block is not compressible */
lib/zstd.h
+275 -229
View File
@@ -106,7 +106,7 @@ extern "C" {
/*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 5
#define ZSTD_VERSION_RELEASE 4
#define ZSTD_VERSION_RELEASE 5
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
/*! ZSTD_versionNumber() :
@@ -148,7 +148,8 @@ ZSTDLIB_API const char* ZSTD_versionString(void);
***************************************/
/*! ZSTD_compress() :
* Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
* Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
* NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
* enough space to successfully compress the data.
* @return : compressed size written into `dst` (<= `dstCapacity),
* or an error code if it fails (which can be tested using ZSTD_isError()). */
ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
@@ -578,7 +579,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
* Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
* - The function is always blocking, returns when compression is completed.
* Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
* NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
* enough space to successfully compress the data, though it is possible it fails for other reasons.
* @return : compressed size written into `dst` (<= `dstCapacity),
* or an error code if it fails (which can be tested using ZSTD_isError()).
*/
@@ -1018,9 +1020,11 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
* Advanced dictionary and prefix API (Requires v1.4.0+)
*
* This API allows dictionaries to be used with ZSTD_compress2(),
* ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
* only reset with the context is reset with ZSTD_reset_parameters or
* ZSTD_reset_session_and_parameters. Prefixes are single-use.
* ZSTD_compressStream2(), and ZSTD_decompressDCtx().
* Dictionaries are sticky, they remain valid when same context is re-used,
* they only reset when the context is reset
* with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters.
* In contrast, Prefixes are single-use.
******************************************************************************/
@@ -1041,7 +1045,11 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
* Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
* In such a case, dictionary buffer must outlive its users.
* Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
* to precisely select how dictionary content must be interpreted. */
* to precisely select how dictionary content must be interpreted.
* Note 5 : This method does not benefit from LDM (long distance mode).
* If you want to employ LDM on some large dictionary content,
* prefer employing ZSTD_CCtx_refPrefix() described below.
*/
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
@@ -1064,6 +1072,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
* Decompression will need same prefix to properly regenerate data.
* Compressing with a prefix is similar in outcome as performing a diff and compressing it,
* but performs much faster, especially during decompression (compression speed is tunable with compression level).
* This method is compatible with LDM (long distance mode).
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
* Note 1 : Prefix buffer is referenced. It **must** outlive compression.
@@ -1387,7 +1396,7 @@ typedef enum {
} ZSTD_paramSwitch_e;
/***************************************
* Frame size functions
* Frame header and size functions
***************************************/
/*! ZSTD_findDecompressedSize() :
@@ -1434,6 +1443,30 @@ ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size
* or an error code (if srcSize is too small) */
ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
typedef struct {
unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
unsigned blockSizeMax;
ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
unsigned headerSize;
unsigned dictID;
unsigned checksumFlag;
unsigned _reserved1;
unsigned _reserved2;
} ZSTD_frameHeader;
/*! ZSTD_getFrameHeader() :
* decode Frame Header, or requires larger `srcSize`.
* @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
* or an error code, which can be tested using ZSTD_isError() */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
/*! ZSTD_getFrameHeader_advanced() :
* same as ZSTD_getFrameHeader(),
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
/*! ZSTD_decompressionMargin() :
* Zstd supports in-place decompression, where the input and output buffers overlap.
* In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
@@ -1803,12 +1836,26 @@ ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
/*! ZSTD_CCtx_setCParams() :
* Set all parameters provided within @cparams into the working @cctx.
* Set all parameters provided within @p cparams into the working @p cctx.
* Note : if modifying parameters during compression (MT mode only),
* note that changes to the .windowLog parameter will be ignored.
* @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
* @return 0 on success, or an error code (can be checked with ZSTD_isError()).
* On failure, no parameters are updated.
*/
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
/*! ZSTD_CCtx_setFParams() :
* Set all parameters provided within @p fparams into the working @p cctx.
* @return 0 on success, or an error code (can be checked with ZSTD_isError()).
*/
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams);
/*! ZSTD_CCtx_setParams() :
* Set all parameters provided within @p params into the working @p cctx.
* @return 0 on success, or an error code (can be checked with ZSTD_isError()).
*/
ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params);
/*! ZSTD_compress_advanced() :
* Note : this function is now DEPRECATED.
* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
@@ -2134,7 +2181,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* This parameter can be used to set an upper bound on the blocksize
* that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
* bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
* compressBound() innacurate). Only currently meant to be used for testing.
* compressBound() inaccurate). Only currently meant to be used for testing.
*
*/
#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
@@ -2452,12 +2499,9 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
int compressionLevel);
/*! ZSTD_initCStream_advanced() :
* This function is DEPRECATED, and is approximately equivalent to:
* This function is DEPRECATED, and is equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* // Pseudocode: Set each zstd parameter and leave the rest as-is.
* for ((param, value) : params) {
* ZSTD_CCtx_setParameter(zcs, param, value);
* }
* ZSTD_CCtx_setParams(zcs, params);
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
*
@@ -2486,12 +2530,9 @@ ZSTDLIB_STATIC_API
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
/*! ZSTD_initCStream_usingCDict_advanced() :
* This function is DEPRECATED, and is approximately equivalent to:
* This function is DEPRECATED, and is equivalent to:
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
* // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
* for ((fParam, value) : fParams) {
* ZSTD_CCtx_setParameter(zcs, fParam, value);
* }
* ZSTD_CCtx_setFParams(zcs, fParams);
* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
* ZSTD_CCtx_refCDict(zcs, cdict);
*
@@ -2598,214 +2639,6 @@ ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
/*********************************************************************
* Buffer-less and synchronous inner streaming functions
*
* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
* But it's also a complex one, with several restrictions, documented below.
* Prefer normal streaming API for an easier experience.
********************************************************************* */
/**
Buffer-less streaming compression (synchronous mode)
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
Start by initializing a context.
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
Then, consume your input using ZSTD_compressContinue().
There are some important considerations to keep in mind when using this advanced function :
- ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
- Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
- Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
Worst case evaluation is provided by ZSTD_compressBound().
ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
- ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
- ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
In which case, it will "discard" the relevant memory section from its history.
Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
`ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
*/
/*===== Buffer-less streaming compression functions =====*/
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
ZSTDLIB_STATIC_API
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
ZSTD_DEPRECATED("use advanced API to access custom parameters")
ZSTDLIB_STATIC_API
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTD_DEPRECATED("use advanced API to access custom parameters")
ZSTDLIB_STATIC_API
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
/**
Buffer-less streaming decompression (synchronous mode)
A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times.
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
Data fragment must be large enough to ensure successful decoding.
`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
>0 : `srcSize` is too small, please provide at least result bytes on next attempt.
errorCode, which can be tested using ZSTD_isError().
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
As a consequence, check that values remain within valid application range.
For example, do not allocate memory blindly, check that `windowSize` is within expectation.
Each application can set its own limits, depending on local restrictions.
For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
ZSTD_decompressContinue() is very sensitive to contiguity,
if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
There are multiple ways to guarantee this condition.
The most memory efficient way is to use a round buffer of sufficient size.
Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
which can return an error code if required value is too large for current system (in 32-bits mode).
In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
At which point, decoding can resume from the beginning of the buffer.
Note that already decoded data stored in the buffer should be flushed before being overwritten.
There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
Finally, if you control the compression process, you can also ignore all buffer size rules,
as long as the encoder and decoder progress in "lock-step",
aka use exactly the same buffer sizes, break contiguity at the same place, etc.
Once buffers are setup, start decompression, with ZSTD_decompressBegin().
If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
It can also be an error code, which can be tested with ZSTD_isError().
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
Context can then be reset to start a new decompression.
Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
This information is not required to properly decode a frame.
== Special case : skippable frames ==
Skippable frames allow integration of user-defined data into a flow of concatenated frames.
Skippable frames will be ignored (skipped) by decompressor.
The format of skippable frames is as follows :
a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
c) Frame Content - any content (User Data) of length equal to Frame Size
For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
*/
/*===== Buffer-less streaming decompression functions =====*/
typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
typedef struct {
unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
unsigned blockSizeMax;
ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
unsigned headerSize;
unsigned dictID;
unsigned checksumFlag;
unsigned _reserved1;
unsigned _reserved2;
} ZSTD_frameHeader;
/*! ZSTD_getFrameHeader() :
* decode Frame Header, or requires larger `srcSize`.
* @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
* or an error code, which can be tested using ZSTD_isError() */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
/*! ZSTD_getFrameHeader_advanced() :
* same as ZSTD_getFrameHeader(),
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/* misc */
ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
/* ============================ */
/** Block level API */
/* ============================ */
/*!
Block functions produce and decode raw zstd blocks, without frame metadata.
Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
A few rules to respect :
- Compressing and decompressing require a context structure
+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
- It is necessary to init context before starting
+ compression : any ZSTD_compressBegin*() variant, including with dictionary
+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
+ For inputs larger than a single block, consider using regular ZSTD_compress() instead.
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
===> In which case, nothing is produced into `dst` !
+ User __must__ test for such outcome and deal directly with uncompressed data
+ A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
Doing so would mess up with statistics history, leading to potential data corruption.
+ ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
+ In case of multiple successive blocks, should some of them be uncompressed,
decoder must be informed of their existence in order to follow proper history.
Use ZSTD_insertBlock() for such a case.
*/
/*===== Raw zstd block functions =====*/
ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
*
* *** OVERVIEW ***
@@ -2967,6 +2800,219 @@ ZSTD_registerSequenceProducer(
ZSTD_sequenceProducer_F* sequenceProducer
);
/*********************************************************************
* Buffer-less and synchronous inner streaming functions (DEPRECATED)
*
* This API is deprecated, and will be removed in a future version.
* It allows streaming (de)compression with user allocated buffers.
* However, it is hard to use, and not as well tested as the rest of
* our API.
*
* Please use the normal streaming API instead: ZSTD_compressStream2,
* and ZSTD_decompressStream.
* If there is functionality that you need, but it doesn't provide,
* please open an issue on our GitHub.
********************************************************************* */
/**
Buffer-less streaming compression (synchronous mode)
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
Start by initializing a context.
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
Then, consume your input using ZSTD_compressContinue().
There are some important considerations to keep in mind when using this advanced function :
- ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
- Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
- Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
Worst case evaluation is provided by ZSTD_compressBound().
ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
- ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog).
It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
- ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
In which case, it will "discard" the relevant memory section from its history.
Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
`ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
*/
/*===== Buffer-less streaming compression functions =====*/
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */
ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
ZSTDLIB_STATIC_API
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
ZSTD_DEPRECATED("use advanced API to access custom parameters")
ZSTDLIB_STATIC_API
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
ZSTD_DEPRECATED("use advanced API to access custom parameters")
ZSTDLIB_STATIC_API
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
/**
Buffer-less streaming decompression (synchronous mode)
A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times.
First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
Data fragment must be large enough to ensure successful decoding.
`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
>0 : `srcSize` is too small, please provide at least result bytes on next attempt.
errorCode, which can be tested using ZSTD_isError().
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
As a consequence, check that values remain within valid application range.
For example, do not allocate memory blindly, check that `windowSize` is within expectation.
Each application can set its own limits, depending on local restrictions.
For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
ZSTD_decompressContinue() is very sensitive to contiguity,
if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
There are multiple ways to guarantee this condition.
The most memory efficient way is to use a round buffer of sufficient size.
Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
which can return an error code if required value is too large for current system (in 32-bits mode).
In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
At which point, decoding can resume from the beginning of the buffer.
Note that already decoded data stored in the buffer should be flushed before being overwritten.
There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
Finally, if you control the compression process, you can also ignore all buffer size rules,
as long as the encoder and decoder progress in "lock-step",
aka use exactly the same buffer sizes, break contiguity at the same place, etc.
Once buffers are setup, start decompression, with ZSTD_decompressBegin().
If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
It can also be an error code, which can be tested with ZSTD_isError().
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
Context can then be reset to start a new decompression.
Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
This information is not required to properly decode a frame.
== Special case : skippable frames ==
Skippable frames allow integration of user-defined data into a flow of concatenated frames.
Skippable frames will be ignored (skipped) by decompressor.
The format of skippable frames is as follows :
a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
c) Frame Content - any content (User Data) of length equal to Frame Size
For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
*/
/*===== Buffer-less streaming decompression functions =====*/
ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/* misc */
ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
/* ========================================= */
/** Block level API (DEPRECATED) */
/* ========================================= */
/*!
This API is deprecated in favor of the regular compression API.
You can get the frame header down to 2 bytes by setting:
- ZSTD_c_format = ZSTD_f_zstd1_magicless
- ZSTD_c_contentSizeFlag = 0
- ZSTD_c_checksumFlag = 0
- ZSTD_c_dictIDFlag = 0
This API is not as well tested as our normal API, so we recommend not using it.
We will be removing it in a future version. If the normal API doesn't provide
the functionality you need, please open a GitHub issue.
Block functions produce and decode raw zstd blocks, without frame metadata.
Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
A few rules to respect :
- Compressing and decompressing require a context structure
+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
- It is necessary to init context before starting
+ compression : any ZSTD_compressBegin*() variant, including with dictionary
+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary
- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
+ If input is larger than a block size, it's necessary to split input data into multiple blocks
+ For inputs larger than a single block, consider using regular ZSTD_compress() instead.
Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
===> In which case, nothing is produced into `dst` !
+ User __must__ test for such outcome and deal directly with uncompressed data
+ A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
Doing so would mess up with statistics history, leading to potential data corruption.
+ ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
+ In case of multiple successive blocks, should some of them be uncompressed,
decoder must be informed of their existence in order to follow proper history.
Use ZSTD_insertBlock() for such a case.
*/
/*===== Raw zstd block functions =====*/
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
programs/Makefile
+1 -1
View File
@@ -225,7 +225,7 @@ zstd-noxz : zstd
.PHONY: zstd-dll
zstd-dll : LDFLAGS+= -L$(LIBZSTD)
zstd-dll : LDLIBS += -lzstd
zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c
zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c pool.c threading.c
zstd-dll : zstd
programs/benchzstd.c
+31 -18
View File
@@ -697,9 +697,9 @@ static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedS
displayLevel, displayName, adv);
}
BMK_benchOutcome_t BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv)
int BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv)
{
char name[20] = {0};
size_t const benchedSize = 10000000;
@@ -707,12 +707,16 @@ BMK_benchOutcome_t BMK_syntheticTest(int cLevel, double compressibility,
BMK_benchOutcome_t res;
if (cLevel > ZSTD_maxCLevel()) {
RETURN_ERROR(15, BMK_benchOutcome_t, "Invalid Compression Level");
DISPLAYLEVEL(1, "Invalid Compression Level");
return 15;
}
/* Memory allocation */
srcBuffer = malloc(benchedSize);
if (!srcBuffer) RETURN_ERROR(21, BMK_benchOutcome_t, "not enough memory");
if (!srcBuffer) {
DISPLAYLEVEL(1, "allocation error : not enough memory");
return 16;
}
/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
@@ -728,7 +732,7 @@ BMK_benchOutcome_t BMK_syntheticTest(int cLevel, double compressibility,
/* clean up */
free(srcBuffer);
return res;
return !BMK_isSuccessful_benchOutcome(res);
}
@@ -790,7 +794,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
return 0;
}
BMK_benchOutcome_t BMK_benchFilesAdvanced(
int BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel,
const ZSTD_compressionParameters* compressionParams,
@@ -805,19 +809,25 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
if (!nbFiles) {
RETURN_ERROR(14, BMK_benchOutcome_t, "No Files to Benchmark");
DISPLAYLEVEL(1, "No Files to Benchmark");
return 13;
}
if (cLevel > ZSTD_maxCLevel()) {
RETURN_ERROR(15, BMK_benchOutcome_t, "Invalid Compression Level");
DISPLAYLEVEL(1, "Invalid Compression Level");
return 14;
}
if (totalSizeToLoad == UTIL_FILESIZE_UNKNOWN) {
RETURN_ERROR(9, BMK_benchOutcome_t, "Error loading files");
DISPLAYLEVEL(1, "Error loading files");
return 15;
}
fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t));
if (!fileSizes) RETURN_ERROR(12, BMK_benchOutcome_t, "not enough memory for fileSizes");
if (!fileSizes) {
DISPLAYLEVEL(1, "not enough memory for fileSizes");
return 16;
}
/* Load dictionary */
if (dictFileName != NULL) {
@@ -825,18 +835,21 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
if (dictFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYLEVEL(1, "error loading %s : %s \n", dictFileName, strerror(errno));
free(fileSizes);
RETURN_ERROR(9, BMK_benchOutcome_t, "benchmark aborted");
DISPLAYLEVEL(1, "benchmark aborted");
return 17;
}
if (dictFileSize > 64 MB) {
free(fileSizes);
RETURN_ERROR(10, BMK_benchOutcome_t, "dictionary file %s too large", dictFileName);
DISPLAYLEVEL(1, "dictionary file %s too large", dictFileName);
return 18;
}
dictBufferSize = (size_t)dictFileSize;
dictBuffer = malloc(dictBufferSize);
if (dictBuffer==NULL) {
free(fileSizes);
RETURN_ERROR(11, BMK_benchOutcome_t, "not enough memory for dictionary (%u bytes)",
DISPLAYLEVEL(1, "not enough memory for dictionary (%u bytes)",
(unsigned)dictBufferSize);
return 19;
}
{ int const errorCode = BMK_loadFiles(dictBuffer, dictBufferSize,
@@ -858,7 +871,8 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
if (!srcBuffer) {
free(dictBuffer);
free(fileSizes);
RETURN_ERROR(12, BMK_benchOutcome_t, "not enough memory");
DISPLAYLEVEL(1, "not enough memory for srcBuffer");
return 20;
}
/* Load input buffer */
@@ -886,12 +900,11 @@ _cleanUp:
free(srcBuffer);
free(dictBuffer);
free(fileSizes);
return res;
return !BMK_isSuccessful_benchOutcome(res);
}
BMK_benchOutcome_t BMK_benchFiles(
const char* const * fileNamesTable, unsigned nbFiles,
int BMK_benchFiles(const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel)
programs/benchzstd.h
+17 -34
View File
@@ -81,21 +81,13 @@ BMK_benchResult_t BMK_extract_benchResult(BMK_benchOutcome_t outcome);
* 2 : + result + interaction + warnings;
* 3 : + information;
* 4 : + debug
* @return:
* a variant, which expresses either an error, or a valid result.
* Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* If yes, extract the valid result with BMK_extract_benchResult(),
* it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
* @return: 0 on success, !0 on error
*/
BMK_benchOutcome_t BMK_benchFiles(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel);
int BMK_benchFiles(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel);
typedef enum {
@@ -126,11 +118,11 @@ BMK_advancedParams_t BMK_initAdvancedParams(void);
/*! BMK_benchFilesAdvanced():
* Same as BMK_benchFiles(),
* with more controls, provided through advancedParams_t structure */
BMK_benchOutcome_t BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
int BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
/*! BMK_syntheticTest() -- called from zstdcli */
/* Generates a sample with datagen, using compressibility argument */
@@ -139,20 +131,11 @@ BMK_benchOutcome_t BMK_benchFilesAdvanced(
* compressionParams - basic compression Parameters
* displayLevel - see benchFiles
* adv - see advanced_Params_t
* @return:
* a variant, which expresses either an error, or a valid result.
* Use BMK_isSuccessful_benchOutcome() to check if function was successful.
* If yes, extract the valid result with BMK_extract_benchResult(),
* it will contain :
* .cSpeed: compression speed in bytes per second,
* .dSpeed: decompression speed in bytes per second,
* .cSize : compressed size, in bytes
* .cMem : memory budget required for the compression context
* @return: 0 on success, !0 on error
*/
BMK_benchOutcome_t BMK_syntheticTest(
int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
int BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
@@ -190,8 +173,8 @@ BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
int displayLevel, const char* displayName);
/* BMK_benchMemAdvanced() : same as BMK_benchMem()
* with following additional options :
/* BMK_benchMemAdvanced() : used by Paramgrill
* same as BMK_benchMem() with following additional options :
* dstBuffer - destination buffer to write compressed output in, NULL if none provided.
* dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL
* adv = see advancedParams_t
programs/fileio.c
+248 -57
View File
@@ -485,6 +485,11 @@ void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
prefs->passThrough = (value != 0);
}
void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_paramSwitch_e value)
{
prefs->mmapDict = value;
}
/* FIO_ctx_t functions */
void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) {
@@ -576,6 +581,8 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
const char* srcFileName, const char* dstFileName,
const int mode)
{
int isDstRegFile;
if (prefs->testMode) return NULL; /* do not open file in test mode */
assert(dstFileName != NULL);
@@ -595,11 +602,16 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
return NULL;
}
isDstRegFile = UTIL_isRegularFile(dstFileName); /* invoke once */
if (prefs->sparseFileSupport == 1) {
if (!isDstRegFile) {
prefs->sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is disabled when output is not a file \n");
}
prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
}
if (UTIL_isRegularFile(dstFileName)) {
if (isDstRegFile) {
/* Check if destination file already exists */
#if !defined(_WIN32)
/* this test does not work on Windows :
@@ -644,39 +656,34 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
#endif
if (f == NULL) {
DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
} else {
/* An increased buffer size can provide a significant performance
* boost on some platforms. Note that providing a NULL buf with a
* size that's not 0 is not defined in ANSI C, but is defined in an
* extension. There are three possibilities here:
* 1. Libc supports the extended version and everything is good.
* 2. Libc ignores the size when buf is NULL, in which case
* everything will continue as if we didn't call `setvbuf()`.
* 3. We fail the call and execution continues but a warning
* message might be shown.
* In all cases due execution continues. For now, I believe that
* this is a more cost-effective solution than managing the buffers
* allocations ourselves (will require an API change).
*/
if (setvbuf(f, NULL, _IOFBF, 1 MB)) {
DISPLAYLEVEL(2, "Warning: setvbuf failed for %s\n", dstFileName);
}
}
/* An increased buffer size can provide a significant performance boost on some platforms.
* Note that providing a NULL buf with a size that's not 0 is not defined in ANSI C, but is defined
* in an extension. There are three possibilities here -
* 1. Libc supports the extended version and everything is good.
* 2. Libc ignores the size when buf is NULL, in which case everything will continue as if we didn't
* call `setvbuf`.
* 3. We fail the call and execution continues but a warning message might be shown.
* In all cases due execution continues. For now, I believe that this is a more cost-effective
* solution than managing the buffers allocations ourselves (will require an API change). */
if(setvbuf(f, NULL, _IOFBF, 1 MB))
DISPLAYLEVEL(2, "Warning: setvbuf failed for %s\n", dstFileName);
return f;
}
}
/*! FIO_createDictBuffer() :
* creates a buffer, pointed by `*bufferPtr`,
* loads `filename` content into it, up to DICTSIZE_MAX bytes.
* @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer
/* FIO_getDictFileStat() :
*/
static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
FILE* fileHandle;
U64 fileSize;
assert(bufferPtr != NULL);
static void FIO_getDictFileStat(const char* fileName, stat_t* dictFileStat) {
assert(dictFileStat != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
if (fileName == NULL) return;
if (!UTIL_stat(fileName, dictFileStat)) {
EXM_THROW(31, "Stat failed on dictionary file %s: %s", fileName, strerror(errno));
@@ -685,6 +692,26 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p
if (!UTIL_isRegularFileStat(dictFileStat)) {
EXM_THROW(32, "Dictionary %s must be a regular file.", fileName);
}
}
/* FIO_setDictBufferMalloc() :
* allocates a buffer, pointed by `dict->dictBuffer`,
* loads `filename` content into it, up to DICTSIZE_MAX bytes.
* @return : loaded size
* if fileName==NULL, returns 0 and a NULL pointer
*/
static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
FILE* fileHandle;
U64 fileSize;
void** bufferPtr = &dict->dictBuffer;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = fopen(fileName, "rb");
@@ -712,6 +739,130 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p
return (size_t)fileSize;
}
#if (PLATFORM_POSIX_VERSION > 0)
#include <sys/mman.h>
static void FIO_munmap(FIO_Dict_t* dict)
{
munmap(dict->dictBuffer, dict->dictBufferSize);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
}
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
int fileHandle;
U64 fileSize;
void** bufferPtr = &dict->dictBuffer;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = open(fileName, O_RDONLY);
if (fileHandle == -1) {
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
}
fileSize = UTIL_getFileSizeStat(dictFileStat);
{
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
}
*bufferPtr = mmap(NULL, (size_t)fileSize, PROT_READ, MAP_PRIVATE, fileHandle, 0);
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
close(fileHandle);
return (size_t)fileSize;
}
#elif defined(_MSC_VER) || defined(_WIN32)
#include <windows.h>
static void FIO_munmap(FIO_Dict_t* dict)
{
UnmapViewOfFile(dict->dictBuffer);
CloseHandle(dict->dictHandle);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
}
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
HANDLE fileHandle, mapping;
U64 fileSize;
void** bufferPtr = &dict->dictBuffer;
assert(bufferPtr != NULL);
assert(dictFileStat != NULL);
*bufferPtr = NULL;
if (fileName == NULL) return 0;
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
fileHandle = CreateFileA(fileName, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_READONLY, NULL);
if (fileHandle == INVALID_HANDLE_VALUE) {
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
}
fileSize = UTIL_getFileSizeStat(dictFileStat);
{
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
if (fileSize > dictSizeMax) {
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
}
}
mapping = CreateFileMapping(fileHandle, NULL, PAGE_READONLY, 0, 0, NULL);
if (mapping == NULL) {
EXM_THROW(35, "Couldn't map dictionary %s: %s", fileName, strerror(errno));
}
*bufferPtr = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, (DWORD)fileSize); /* we can only cast to DWORD here because dictSize <= 2GB */
if (*bufferPtr==NULL) EXM_THROW(36, "%s", strerror(errno));
dict->dictHandle = fileHandle;
return (size_t)fileSize;
}
#else
static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
{
return FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat);
}
static void FIO_munmap(FIO_Dict_t* dict) {
free(dict->dictBuffer);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
}
#endif
static void FIO_freeDict(FIO_Dict_t* dict) {
if (dict->dictBufferType == FIO_mallocDict) {
free(dict->dictBuffer);
dict->dictBuffer = NULL;
dict->dictBufferSize = 0;
} else if (dict->dictBufferType == FIO_mmapDict) {
FIO_munmap(dict);
} else {
assert(0); /* Should not reach this case */
}
}
static void FIO_initDict(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat, FIO_dictBufferType_t dictBufferType) {
dict->dictBufferType = dictBufferType;
if (dict->dictBufferType == FIO_mallocDict) {
dict->dictBufferSize = FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat);
} else if (dict->dictBufferType == FIO_mmapDict) {
dict->dictBufferSize = FIO_setDictBufferMMap(dict, fileName, prefs, dictFileStat);
} else {
assert(0); /* Should not reach this case */
}
}
/* FIO_checkFilenameCollisions() :
@@ -914,8 +1065,7 @@ static ZSTD_outBuffer setOutBuffer(void* buf, size_t s, size_t pos)
* Compression
************************************************************************/
typedef struct {
void* dictBuffer;
size_t dictBufferSize;
FIO_Dict_t dict;
const char* dictFileName;
stat_t dictFileStat;
ZSTD_CStream* cctx;
@@ -961,6 +1111,9 @@ static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
const char* dictFileName, unsigned long long const maxSrcFileSize,
int cLevel, ZSTD_compressionParameters comprParams) {
int useMMap = prefs->mmapDict == ZSTD_ps_enable;
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
FIO_dictBufferType_t dictBufferType;
cRess_t ress;
memset(&ress, 0, sizeof(ress));
@@ -970,19 +1123,25 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx",
strerror(errno));
FIO_getDictFileStat(dictFileName, &ress.dictFileStat);
/* need to update memLimit before calling createDictBuffer
* because of memLimit check inside it */
if (prefs->patchFromMode) {
U64 const dictSize = UTIL_getFileSizeStat(&ress.dictFileStat);
unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
useMMap |= dictSize > prefs->memLimit;
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
}
ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs, &ress.dictFileStat); /* works with dictFileName==NULL */
dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
FIO_initDict(&ress.dict, dictFileName, prefs, &ress.dictFileStat, dictBufferType); /* works with dictFileName==NULL */
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
/* Advanced parameters, including dictionary */
if (dictFileName && (ress.dictBuffer==NULL))
if (dictFileName && (ress.dict.dictBuffer==NULL))
EXM_THROW(32, "allocation error : can't create dictBuffer");
ress.dictFileName = dictFileName;
@@ -1032,17 +1191,17 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
#endif
/* dictionary */
if (prefs->patchFromMode) {
CHECK( ZSTD_CCtx_refPrefix(ress.cctx, ress.dictBuffer, ress.dictBufferSize) );
CHECK( ZSTD_CCtx_refPrefix(ress.cctx, ress.dict.dictBuffer, ress.dict.dictBufferSize) );
} else {
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, ress.dictBuffer, ress.dictBufferSize) );
CHECK( ZSTD_CCtx_loadDictionary_byReference(ress.cctx, ress.dict.dictBuffer, ress.dict.dictBufferSize) );
}
return ress;
}
static void FIO_freeCResources(const cRess_t* const ress)
static void FIO_freeCResources(cRess_t* const ress)
{
free(ress->dictBuffer);
FIO_freeDict(&(ress->dict));
AIO_WritePool_free(ress->writeCtx);
AIO_ReadPool_free(ress->readCtx);
ZSTD_freeCStream(ress->cctx); /* never fails */
@@ -1173,8 +1332,8 @@ FIO_compressLzmaFrame(cRess_t* ress,
}
writeJob =AIO_WritePool_acquireJob(ress->writeCtx);
strm.next_out = (Bytef*)writeJob->buffer;
strm.avail_out = (uInt)writeJob->bufferSize;
strm.next_out = (BYTE*)writeJob->buffer;
strm.avail_out = writeJob->bufferSize;
strm.next_in = 0;
strm.avail_in = 0;
@@ -1201,7 +1360,7 @@ FIO_compressLzmaFrame(cRess_t* ress,
writeJob->usedBufferSize = compBytes;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
outFileSize += compBytes;
strm.next_out = (Bytef*)writeJob->buffer;
strm.next_out = (BYTE*)writeJob->buffer;
strm.avail_out = writeJob->bufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
@@ -1681,6 +1840,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
int result;
int transferStat = 0;
FILE *dstFile;
int dstFd = -1;
assert(AIO_ReadPool_getFile(ress.readCtx) != NULL);
if (AIO_WritePool_getFile(ress.writeCtx) == NULL) {
@@ -1696,6 +1856,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName);
dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFileInitialPermissions);
if (dstFile==NULL) return 1; /* could not open dstFileName */
dstFd = fileno(dstFile);
AIO_WritePool_setFile(ress.writeCtx, dstFile);
/* Must only be added after FIO_openDstFile() succeeds.
* Otherwise we may delete the destination file if it already exists,
@@ -1709,14 +1870,20 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
if (closeDstFile) {
clearHandler();
if (transferStat) {
UTIL_setFDStat(dstFd, dstFileName, srcFileStat);
}
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName);
if (AIO_WritePool_closeFile(ress.writeCtx)) { /* error closing file */
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result=1;
}
if (transferStat) {
UTIL_setFileStat(dstFileName, srcFileStat);
UTIL_utime(dstFileName, srcFileStat);
}
if ( (result != 0) /* operation failure */
&& strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */
) {
@@ -1873,7 +2040,7 @@ int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const
const char* srcFileName, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams)
{
cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
cRess_t ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
#define DISPLAY_LEVEL_DEFAULT 2
@@ -2043,6 +2210,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
* Decompression
***************************************************************************/
typedef struct {
FIO_Dict_t dict;
ZSTD_DStream* dctx;
WritePoolCtx_t *writeCtx;
ReadPoolCtx_t *readCtx;
@@ -2050,11 +2218,19 @@ typedef struct {
static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
{
int useMMap = prefs->mmapDict == ZSTD_ps_enable;
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
stat_t statbuf;
dRess_t ress;
memset(&ress, 0, sizeof(ress));
if (prefs->patchFromMode)
FIO_adjustMemLimitForPatchFromMode(prefs, UTIL_getFileSize(dictFileName), 0 /* just use the dict size */);
FIO_getDictFileStat(dictFileName, &statbuf);
if (prefs->patchFromMode){
U64 const dictSize = UTIL_getFileSizeStat(&statbuf);
useMMap |= dictSize > prefs->memLimit;
FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
}
/* Allocation */
ress.dctx = ZSTD_createDStream();
@@ -2064,29 +2240,34 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag));
/* dictionary */
{ void* dictBuffer;
stat_t statbuf;
size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs, &statbuf);
CHECK( ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
CHECK( ZSTD_DCtx_loadDictionary(ress.dctx, dictBuffer, dictBufferSize) );
free(dictBuffer);
{
FIO_dictBufferType_t dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
FIO_initDict(&ress.dict, dictFileName, prefs, &statbuf, dictBufferType);
CHECK(ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
if (prefs->patchFromMode){
CHECK(ZSTD_DCtx_refPrefix(ress.dctx, ress.dict.dictBuffer, ress.dict.dictBufferSize));
} else {
CHECK(ZSTD_DCtx_loadDictionary_byReference(ress.dctx, ress.dict.dictBuffer, ress.dict.dictBufferSize));
}
}
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_DStreamOutSize());
ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_DStreamInSize());
return ress;
}
static void FIO_freeDResources(dRess_t ress)
{
FIO_freeDict(&(ress.dict));
CHECK( ZSTD_freeDStream(ress.dctx) );
AIO_WritePool_free(ress.writeCtx);
AIO_ReadPool_free(ress.readCtx);
}
/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
@return : 0 (no error) */
/* FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
* @return : 0 (no error) */
static int FIO_passThrough(dRess_t *ress)
{
size_t const blockSize = MIN(MIN(64 KB, ZSTD_DStreamInSize()), ZSTD_DStreamOutSize());
@@ -2114,7 +2295,8 @@ static int FIO_passThrough(dRess_t *ress)
static void
FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
const dRess_t* ress,
size_t err, const char* srcFileName)
size_t err,
const char* srcFileName)
{
ZSTD_frameHeader header;
@@ -2316,8 +2498,8 @@ FIO_decompressLzmaFrame(dRess_t* ress,
}
writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
strm.next_out = (Bytef*)writeJob->buffer;
strm.avail_out = (uInt)writeJob->bufferSize;
strm.next_out = (BYTE*)writeJob->buffer;
strm.avail_out = writeJob->bufferSize;
strm.next_in = (BYTE const*)ress->readCtx->srcBuffer;
strm.avail_in = ress->readCtx->srcBufferLoaded;
@@ -2345,7 +2527,7 @@ FIO_decompressLzmaFrame(dRess_t* ress,
writeJob->usedBufferSize = decompBytes;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
outFileSize += decompBytes;
strm.next_out = (Bytef*)writeJob->buffer;
strm.next_out = (BYTE*)writeJob->buffer;
strm.avail_out = writeJob->bufferSize;
} }
if (ret == LZMA_STREAM_END) break;
@@ -2540,6 +2722,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
int result;
int releaseDstFile = 0;
int transferStat = 0;
int dstFd = 0;
if ((AIO_WritePool_getFile(ress.writeCtx) == NULL) && (prefs->testMode == 0)) {
FILE *dstFile;
@@ -2555,6 +2738,7 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions);
if (dstFile==NULL) return 1;
dstFd = fileno(dstFile);
AIO_WritePool_setFile(ress.writeCtx, dstFile);
/* Must only be added after FIO_openDstFile() succeeds.
@@ -2568,13 +2752,18 @@ static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
if (releaseDstFile) {
clearHandler();
if (transferStat) {
UTIL_setFDStat(dstFd, dstFileName, srcFileStat);
}
if (AIO_WritePool_closeFile(ress.writeCtx)) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result = 1;
}
if (transferStat) {
UTIL_setFileStat(dstFileName, srcFileStat);
UTIL_utime(dstFileName, srcFileStat);
}
if ( (result != 0) /* operation failure */
@@ -2655,6 +2844,8 @@ int FIO_decompressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs,
int const decodingError = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName);
FIO_freeDResources(ress);
return decodingError;
}
programs/fileio.h
+1
View File
@@ -106,6 +106,7 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value);
void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_paramSwitch_e value);
/* FIO_ctx_t functions */
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);
programs/fileio_types.h
+12
View File
@@ -69,6 +69,18 @@ typedef struct FIO_prefs_s {
int contentSize;
int allowBlockDevices;
int passThrough;
ZSTD_paramSwitch_e mmapDict;
} FIO_prefs_t;
typedef enum {FIO_mallocDict, FIO_mmapDict} FIO_dictBufferType_t;
typedef struct {
void* dictBuffer;
size_t dictBufferSize;
FIO_dictBufferType_t dictBufferType;
#if defined(_MSC_VER) || defined(_WIN32)
HANDLE dictHandle;
#endif
} FIO_Dict_t;
#endif /* FILEIO_TYPES_HEADER */
programs/timefn.c
+1 -1
View File
@@ -88,7 +88,7 @@ UTIL_time_t UTIL_getTime(void)
/* C11 requires support of timespec_get().
* However, FreeBSD 11 claims C11 compliance while lacking timespec_get().
* Double confirm timespec_get() support by checking the definition of TIME_UTC.
* However, some versions of Android manage to simultanously define TIME_UTC
* However, some versions of Android manage to simultaneously define TIME_UTC
* and lack timespec_get() support... */
#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
&& defined(TIME_UTC) && !defined(__ANDROID__)
programs/util.c
+76 -16
View File
@@ -102,6 +102,17 @@ UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size)
#define chmod _chmod
#endif
#ifndef ZSTD_HAVE_FCHMOD
#if PLATFORM_POSIX_VERSION >= 199309L
#define ZSTD_HAVE_FCHMOD
#endif
#endif
#ifndef ZSTD_HAVE_FCHOWN
#if PLATFORM_POSIX_VERSION >= 200809L
#define ZSTD_HAVE_FCHOWN
#endif
#endif
/*-****************************************
* Console log
@@ -147,21 +158,38 @@ void UTIL_traceFileStat(void)
g_traceFileStat = 1;
}
int UTIL_stat(const char* filename, stat_t* statbuf)
int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf)
{
int ret;
UTIL_TRACE_CALL("UTIL_stat(%s)", filename);
UTIL_TRACE_CALL("UTIL_stat(%d, %s)", fd, filename);
#if defined(_MSC_VER)
ret = !_stat64(filename, statbuf);
if (fd >= 0) {
ret = !_fstat64(fd, statbuf);
} else {
ret = !_stat64(filename, statbuf);
}
#elif defined(__MINGW32__) && defined (__MSVCRT__)
ret = !_stati64(filename, statbuf);
if (fd >= 0) {
ret = !_fstati64(fd, statbuf);
} else {
ret = !_stati64(filename, statbuf);
}
#else
ret = !stat(filename, statbuf);
if (fd >= 0) {
ret = !fstat(fd, statbuf);
} else {
ret = !stat(filename, statbuf);
}
#endif
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_stat(const char* filename, stat_t* statbuf)
{
return UTIL_fstat(-1, filename, statbuf);
}
int UTIL_isRegularFile(const char* infilename)
{
stat_t statbuf;
@@ -183,11 +211,16 @@ int UTIL_isRegularFileStat(const stat_t* statbuf)
/* like chmod, but avoid changing permission of /dev/null */
int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
{
return UTIL_fchmod(-1, filename, statbuf, permissions);
}
int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions)
{
stat_t localStatBuf;
UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions);
if (statbuf == NULL) {
if (!UTIL_stat(filename, &localStatBuf)) {
if (!UTIL_fstat(fd, filename, &localStatBuf)) {
UTIL_TRACE_RET(0);
return 0;
}
@@ -197,9 +230,20 @@ int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions)
UTIL_TRACE_RET(0);
return 0; /* pretend success, but don't change anything */
}
UTIL_TRACE_CALL("chmod");
#ifdef ZSTD_HAVE_FCHMOD
if (fd >= 0) {
int ret;
UTIL_TRACE_CALL("fchmod");
ret = fchmod(fd, permissions);
UTIL_TRACE_RET(ret);
UTIL_TRACE_RET(ret);
return ret;
} else
#endif
{
int const ret = chmod(filename, permissions);
int ret;
UTIL_TRACE_CALL("chmod");
ret = chmod(filename, permissions);
UTIL_TRACE_RET(ret);
UTIL_TRACE_RET(ret);
return ret;
@@ -236,19 +280,21 @@ int UTIL_utime(const char* filename, const stat_t *statbuf)
}
int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
{
return UTIL_setFDStat(-1, filename, statbuf);
}
int UTIL_setFDStat(const int fd, const char *filename, const stat_t *statbuf)
{
int res = 0;
stat_t curStatBuf;
UTIL_TRACE_CALL("UTIL_setFileStat(%s)", filename);
UTIL_TRACE_CALL("UTIL_setFileStat(%d, %s)", fd, filename);
if (!UTIL_stat(filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) {
if (!UTIL_fstat(fd, filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) {
UTIL_TRACE_RET(-1);
return -1;
}
/* set access and modification times */
res += UTIL_utime(filename, statbuf);
/* Mimic gzip's behavior:
*
* "Change the group first, then the permissions, then the owner.
@@ -258,13 +304,27 @@ int UTIL_setFileStat(const char *filename, const stat_t *statbuf)
* setgid bits." */
#if !defined(_WIN32)
res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */
#ifdef ZSTD_HAVE_FCHOWN
if (fd >= 0) {
res += fchown(fd, -1, statbuf->st_gid); /* Apply group ownership */
} else
#endif
{
res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */
}
#endif
res += UTIL_chmod(filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */
res += UTIL_fchmod(fd, filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */
#ifdef ZSTD_HAVE_FCHOWN
if (fd >= 0) {
res += fchown(fd, statbuf->st_uid, -1); /* Apply user ownership */
} else
#endif
{
res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */
}
#endif
errno = 0;
programs/util.h
+13
View File
@@ -126,15 +126,25 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const
/**
* Calls platform's equivalent of stat() on filename and writes info to statbuf.
* Returns success (1) or failure (0).
*
* UTIL_fstat() is like UTIL_stat() but takes an optional fd that refers to the
* file in question. It turns out that this can be meaningfully faster. If fd is
* -1, behaves just like UTIL_stat() (i.e., falls back to using the filename).
*/
int UTIL_stat(const char* filename, stat_t* statbuf);
int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf);
/**
* Instead of getting a file's stats, this updates them with the info in the
* provided stat_t. Currently sets owner, group, atime, and mtime. Will only
* update this info for regular files.
*
* UTIL_setFDStat() also takes an fd, and will preferentially use that to
* indicate which file to modify, If fd is -1, it will fall back to using the
* filename.
*/
int UTIL_setFileStat(const char* filename, const stat_t* statbuf);
int UTIL_setFDStat(const int fd, const char* filename, const stat_t* statbuf);
/**
* Set atime to now and mtime to the st_mtim in statbuf.
@@ -159,8 +169,11 @@ U64 UTIL_getFileSizeStat(const stat_t* statbuf);
* Like chmod(), but only modifies regular files. Provided statbuf may be NULL,
* in which case this function will stat() the file internally, in order to
* check whether it should be modified.
*
* If fd is -1, fd is ignored and the filename is used.
*/
int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions);
int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions);
/*
* In the absence of a pre-existing stat result on the file in question, these
programs/zstd.1
+2 -2
View File
@@ -1,4 +1,4 @@
.TH "ZSTD" "1" "February 2023" "zstd 1.5.4" "User Commands"
.TH "ZSTD" "1" "March 2023" "zstd 1.5.5" "User Commands"
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
.SH "SYNOPSIS"
@@ -105,7 +105,7 @@ Note: for all levels, you can use \fB\-\-patch\-from\fR in \fB\-\-single\-thread
.IP
Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e\. 4096), and by setting a large \fB\-\-zstd=chainLog=\fR\.
.IP "\[ci]" 4
\fB\-\-rsyncable\fR: \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and the faster compression levels will see a small compression speed hit\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your mileage may vary\.
\fB\-\-rsyncable\fR: \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and a potential impact to compression speed, perceptible at higher speeds, for example when combining \fB\-\-rsyncable\fR with many parallel worker threads\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your mileage may vary\.
.IP "\[ci]" 4
\fB\-C\fR, \fB\-\-[no\-]check\fR: add integrity check computed from uncompressed data (default: enabled)
.IP "\[ci]" 4
programs/zstd.1.md
+4 -3
View File
@@ -177,9 +177,10 @@ the last one takes effect.
(i.e. 4096), and by setting a large `--zstd=chainLog=`.
* `--rsyncable`:
`zstd` will periodically synchronize the compression state to make the
compressed file more rsync-friendly. There is a negligible impact to
compression ratio, and the faster compression levels will see a small
compression speed hit.
compressed file more rsync-friendly.
There is a negligible impact to compression ratio,
and a potential impact to compression speed, perceptible at higher speeds,
for example when combining `--rsyncable` with many parallel worker threads.
This feature does not work with `--single-thread`. You probably don't want
to use it with long range mode, since it will decrease the effectiveness of
the synchronization points, but your mileage may vary.
programs/zstdcli.c
+10 -8
View File
@@ -37,7 +37,7 @@
#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
#ifndef ZSTD_NOBENCH
# include "benchzstd.h" /* BMK_benchFiles */
# include "benchzstd.h" /* BMK_benchFilesAdvanced */
#endif
#ifndef ZSTD_NODICT
# include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
@@ -165,7 +165,7 @@ static void usage(FILE* f, const char* programName)
#endif
DISPLAY_F(f, " -D DICT Use DICT as the dictionary for compression or decompression.\n\n");
DISPLAY_F(f, " -f, --force Disable input and output checks. Allows overwriting existing files,\n");
DISPLAY_F(f, " receiving input from the console, printing ouput to STDOUT, and\n");
DISPLAY_F(f, " receiving input from the console, printing output to STDOUT, and\n");
DISPLAY_F(f, " operating on links, block devices, etc. Unrecognized formats will be\n");
DISPLAY_F(f, " passed-through through as-is.\n\n");
@@ -254,6 +254,7 @@ static void usage_advanced(const char* programName)
DISPLAYOUT("\n");
DISPLAYOUT(" --format=zstd Compress files to the `.zst` format. [Default]\n");
DISPLAYOUT(" --mmap-dict Memory-map dictionary file rather than mallocing and loading all at once");
#ifdef ZSTD_GZCOMPRESS
DISPLAYOUT(" --format=gzip Compress files to the `.gz` format.\n");
#endif
@@ -851,6 +852,7 @@ int main(int argCount, const char* argv[])
ultra=0,
contentSize=1,
removeSrcFile=0;
ZSTD_paramSwitch_e mmapDict=ZSTD_ps_auto;
ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
FIO_compressionType_t cType = FIO_zstdCompression;
unsigned nbWorkers = 0;
@@ -984,6 +986,8 @@ int main(int argCount, const char* argv[])
if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; }
if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; cType = FIO_zstdCompression; continue; }
if (!strcmp(argument, "--mmap-dict")) { mmapDict = ZSTD_ps_enable; continue; }
if (!strcmp(argument, "--no-mmap-dict")) { mmapDict = ZSTD_ps_disable; continue; }
#ifdef ZSTD_GZCOMPRESS
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; cType = FIO_gzipCompression; continue; }
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
@@ -1391,18 +1395,15 @@ int main(int argCount, const char* argv[])
int c;
DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
for(c = cLevel; c <= cLevelLast; c++) {
BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
operationResult = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
} }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_benchOutcome_t const bo = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
operationResult = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
} }
} else {
for(; cLevel <= cLevelLast; cLevel++) {
BMK_benchOutcome_t const bo = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
if (!BMK_isSuccessful_benchOutcome(bo)) return 1;
operationResult = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
} }
#else
@@ -1526,6 +1527,7 @@ int main(int argCount, const char* argv[])
FIO_setNotificationLevel(g_displayLevel);
FIO_setAllowBlockDevices(prefs, allowBlockDevices);
FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL);
FIO_setMMapDict(prefs, mmapDict);
if (memLimit == 0) {
if (compressionParams.windowLog == 0) {
memLimit = (U32)1 << g_defaultMaxWindowLog;
programs/zstdgrep.1
+1 -1
View File
@@ -1,4 +1,4 @@
.TH "ZSTDGREP" "1" "February 2023" "zstd 1.5.4" "User Commands"
.TH "ZSTDGREP" "1" "March 2023" "zstd 1.5.5" "User Commands"
.SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
.SH "SYNOPSIS"
programs/zstdless.1
+1 -1
View File
@@ -1,4 +1,4 @@
.TH "ZSTDLESS" "1" "February 2023" "zstd 1.5.4" "User Commands"
.TH "ZSTDLESS" "1" "March 2023" "zstd 1.5.5" "User Commands"
.SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files
.SH "SYNOPSIS"
tests/Makefile
+6 -2
View File
@@ -95,7 +95,7 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus
dll: fuzzer-dll zstreamtest-dll
.PHONY: zstd zstd32 zstd-nolegacy # only external makefile knows how to build or update them
zstd zstd32 zstd-nolegacy:
zstd zstd32 zstd-nolegacy zstd-dll:
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
.PHONY: libzstd
@@ -328,13 +328,17 @@ test-all: test test32 test-decodecorpus-cli
test-zstd: ZSTD = $(PRGDIR)/zstd
test-zstd: zstd
.PHONY: test-zstd-dll
test-zstd-dll: ZSTD = $(PRGDIR)/zstd
test-zstd-dll: zstd-dll
test-zstd32: ZSTD = $(PRGDIR)/zstd32
test-zstd32: zstd32
test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy
test-zstd-nolegacy: zstd-nolegacy
test-zstd test-zstd32 test-zstd-nolegacy: datagen
test-zstd test-zstd32 test-zstd-nolegacy test-zstd-dll: datagen
file $(ZSTD)
EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)
tests/cli-tests/basic/help.sh.stdout.glob
+1 -1
View File
@@ -16,7 +16,7 @@ Options:
-D DICT Use DICT as the dictionary for compression or decompression.
-f, --force Disable input and output checks. Allows overwriting existing files,
receiving input from the console, printing ouput to STDOUT, and
receiving input from the console, printing output to STDOUT, and
operating on links, block devices, etc. Unrecognized formats will be
passed-through through as-is.
tests/cli-tests/compression/basic.sh
+4 -2
View File
@@ -25,8 +25,10 @@ zstd --stdout file | zstd -t
println bob | zstd | zstd -t
# Test keeping input file when compressing to stdout in gzip mode
$ZSTD_SYMLINK_DIR/gzip -c file | zstd -t ; test -f file
$ZSTD_SYMLINK_DIR/gzip --stdout file | zstd -t ; test -f file
if $(command -v $ZSTD_SYMLINK_DIR/gzip); then
$ZSTD_SYMLINK_DIR/gzip -c file | zstd -t ; test -f file
$ZSTD_SYMLINK_DIR/gzip --stdout file | zstd -t ; test -f file
fi
# Test --rm
cp file file-rm
tests/cli-tests/compression/golden.sh
+4
View File
@@ -10,3 +10,7 @@ zstd -r -t golden-compressed/
zstd --target-compressed-block-size=1024 -rf golden/ --output-dir-mirror golden-compressed/
zstd -r -t golden-compressed/
# PR #3517 block splitter corruption test
zstd -rf -19 --zstd=mml=7 golden/ --output-dir-mirror golden-compressed/
zstd -r -t golden-compressed/
tests/cli-tests/compression/gzip-compat.sh
+8 -6
View File
@@ -6,10 +6,12 @@ set -e
# set -v
# Test gzip specific compression option
$ZSTD_SYMLINK_DIR/gzip --fast file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
$ZSTD_SYMLINK_DIR/gzip --best file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
if $(command -v $ZSTD_SYMLINK_DIR/gzip); then
$ZSTD_SYMLINK_DIR/gzip --fast file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
$ZSTD_SYMLINK_DIR/gzip --best file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
# Test -n / --no-name: do not embed original filename in archive
$ZSTD_SYMLINK_DIR/gzip -n file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
$ZSTD_SYMLINK_DIR/gzip --no-name file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
$ZSTD_SYMLINK_DIR/gzip -c --no-name file | grep -qv file
# Test -n / --no-name: do not embed original filename in archive
$ZSTD_SYMLINK_DIR/gzip -n file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
$ZSTD_SYMLINK_DIR/gzip --no-name file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
$ZSTD_SYMLINK_DIR/gzip -c --no-name file | grep -qv file
fi
tests/cli-tests/compression/window-resize.sh
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/sh
datagen -g1G > file
zstd --long=31 -1 --single-thread --no-content-size -f file
zstd --long=30 -1 --single-thread --no-content-size -f file
zstd -l -v file.zst
# We want to ignore stderr (its outputting "*** zstd command line interface
tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh
Executable
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
# motivated by issue #3523
datagen > file
mkdir out
chmod 000 out
zstd file -q --trace-file-stat -o out/file.zst
zstd -tq out/file.zst
chmod 777 out
tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact
+26
View File
@@ -0,0 +1,26 @@
Trace:FileStat: > UTIL_isLink(file)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isSameFile(file, out/file.zst)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_stat(-1, out/file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(out/file.zst)
Trace:FileStat: > UTIL_stat(-1, out/file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
zstd: out/file.zst: Permission denied
zstd: can't stat out/file.zst : Permission denied -- ignored
tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
+13 -13
View File
@@ -3,40 +3,40 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isSameFile(file, file.zst)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_setFileStat(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_setFileStat(4, file.zst)
Trace:FileStat: > UTIL_stat(4, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_utime(file.zst)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_chmod(file.zst, 0642)
Trace:FileStat: > chmod
Trace:FileStat: > fchmod
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_utime(file.zst)
Trace:FileStat: < 0
tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
+5 -5
View File
@@ -5,20 +5,20 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
+5 -5
View File
@@ -3,22 +3,22 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
+3 -3
View File
@@ -5,14 +5,14 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
+11 -11
View File
@@ -5,34 +5,34 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isSameFile(file.zst, file)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_setFileStat(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_setFileStat(4, file)
Trace:FileStat: > UTIL_stat(4, file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_utime(file)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_chmod(file, 0642)
Trace:FileStat: > chmod
Trace:FileStat: > fchmod
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_utime(file)
Trace:FileStat: < 0
tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
+3 -3
View File
@@ -5,14 +5,14 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isDirectoryStat()
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
+4 -4
View File
@@ -3,18 +3,18 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
Trace:FileStat: < 1
tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
+2 -2
View File
@@ -5,10 +5,10 @@ Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
tests/cli-tests/run.py
+9 -16
View File
@@ -109,25 +109,17 @@ def pop_line(data: bytes) -> typing.Tuple[typing.Optional[bytes], bytes]:
the first line always ends in a :\n:, even if it is the last line and :data:
doesn't end in :\n:.
"""
NEWLINE = b"\n"[0]
NEWLINE = b"\n"
if data == b'':
return (None, data)
newline_idx = data.find(b"\n")
if newline_idx == -1:
end_idx = len(data)
else:
end_idx = newline_idx + 1
parts = data.split(NEWLINE, maxsplit=1)
line = parts[0] + NEWLINE
if len(parts) == 1:
return line, b''
line = data[:end_idx]
data = data[end_idx:]
assert len(line) != 0
if line[-1] != NEWLINE:
line += NEWLINE
return (line, data)
return line, parts[1]
def glob_line_matches(actual: bytes, expect: bytes) -> bool:
@@ -535,7 +527,8 @@ class TestSuite:
subprocess.run(
args=[script],
stdin=subprocess.DEVNULL,
capture_output=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd,
env=env,
check=True,
@@ -647,7 +640,7 @@ if __name__ == "__main__":
help="Preserve the scratch directory TEST_DIR/scratch/ for debugging purposes."
)
parser.add_argument("--verbose", action="store_true", help="Verbose test output.")
parser.add_argument("--timeout", default=60, type=int, help="Test case timeout in seconds. Set to 0 to disable timeouts.")
parser.add_argument("--timeout", default=200, type=int, help="Test case timeout in seconds. Set to 0 to disable timeouts.")
parser.add_argument(
"--exec-prefix",
default=None,
tests/decodecorpus.c
+3 -2
View File
@@ -26,6 +26,7 @@
/* Direct access to internal compression functions is required */
#include "compress/zstd_compress.c" /* ZSTD_resetSeqStore, ZSTD_storeSeq, *_TO_OFFBASE, HIST_countFast_wksp, HIST_isError */
#include "decompress/zstd_decompress_block.h" /* ZSTD_decompressBlock_deprecated */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */
@@ -1434,7 +1435,7 @@ static size_t testDecodeWithDict(U32 seed, genType_e genType)
ZSTD_freeDCtx(dctx);
goto dictTestCleanup;
}
ret = ZSTD_decompressBlock(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE,
ret = ZSTD_decompressBlock_deprecated(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE,
fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart);
}
ZSTD_freeDCtx(dctx);
@@ -1461,7 +1462,7 @@ static size_t testDecodeRawBlock(frame_t* fr)
size_t ret = ZSTD_decompressBegin(dctx);
if (ZSTD_isError(ret)) return ret;
ret = ZSTD_decompressBlock(
ret = ZSTD_decompressBlock_deprecated(
dctx,
DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE,
fr->dataStart, (BYTE*)fr->data - (BYTE*)fr->dataStart);
tests/fullbench.c
+3 -1
View File
@@ -774,7 +774,9 @@ static int benchFiles(U32 benchNb,
} else {
for (benchNb=0; benchNb<100; benchNb++) {
benchMem(benchNb, origBuff, benchedSize, cLevel, cparams);
} }
}
benchNb = 0;
}
free(origBuff);
} }
tests/fuzz/Makefile
+13 -11
View File
@@ -34,13 +34,13 @@ ZSTDDIR = ../../lib
PRGDIR = ../../programs
CONTRIBDIR = ../../contrib
# TODO(embg) make it possible to plug in an arbitrary matchfinder as a .o file
MATCHFINDER_DIR = $(CONTRIBDIR)/externalSequenceProducer
MATCHFINDER_SRC = $(MATCHFINDER_DIR)/sequence_producer.c
DEFAULT_SEQ_PROD_DIR = $(CONTRIBDIR)/externalSequenceProducer
DEFAULT_SEQ_PROD_SRC = $(DEFAULT_SEQ_PROD_DIR)/sequence_producer.c
THIRD_PARTY_SEQ_PROD_OBJ ?=
FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \
-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -I$(MATCHFINDER_DIR) \
-I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -I$(DEFAULT_SEQ_PROD_DIR) \
-DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS)
FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
@@ -75,7 +75,7 @@ FUZZ_SRC := \
$(ZSTDCOMP_SRC) \
$(ZSTDDICT_SRC) \
$(ZSTDLEGACY_SRC) \
$(MATCHFINDER_SRC)
$(DEFAULT_SEQ_PROD_SRC)
FUZZ_SRC := $(sort $(wildcard $(FUZZ_SRC)))
FUZZ_D_OBJ1 := $(subst $(ZSTDDIR)/common/,d_lib_common_,$(FUZZ_SRC))
@@ -84,10 +84,11 @@ FUZZ_D_OBJ3 := $(subst $(ZSTDDIR)/decompress/,d_lib_decompress_,$(FUZZ_D_OBJ2))
FUZZ_D_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,d_lib_dictBuilder_,$(FUZZ_D_OBJ3))
FUZZ_D_OBJ5 := $(subst $(ZSTDDIR)/legacy/,d_lib_legacy_,$(FUZZ_D_OBJ4))
FUZZ_D_OBJ6 := $(subst $(PRGDIR)/,d_prg_,$(FUZZ_D_OBJ5))
FUZZ_D_OBJ7 := $(subst $(MATCHFINDER_DIR)/,d_matchfinder_,$(FUZZ_D_OBJ6))
FUZZ_D_OBJ7 := $(subst $(DEFAULT_SEQ_PROD_DIR)/,d_default_seq_prod_,$(FUZZ_D_OBJ6))
FUZZ_D_OBJ8 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ7))
FUZZ_D_OBJ9 := $(FUZZ_D_OBJ8:.c=.o)
FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ9:.S=.o)
FUZZ_D_OBJ10 := $(THIRD_PARTY_SEQ_PROD_OBJ) $(FUZZ_D_OBJ9)
FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ10:.S=.o)
FUZZ_RT_OBJ1 := $(subst $(ZSTDDIR)/common/,rt_lib_common_,$(FUZZ_SRC))
FUZZ_RT_OBJ2 := $(subst $(ZSTDDIR)/compress/,rt_lib_compress_,$(FUZZ_RT_OBJ1))
@@ -95,10 +96,11 @@ FUZZ_RT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,rt_lib_decompress_,$(FUZZ_RT_OBJ2
FUZZ_RT_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,rt_lib_dictBuilder_,$(FUZZ_RT_OBJ3))
FUZZ_RT_OBJ5 := $(subst $(ZSTDDIR)/legacy/,rt_lib_legacy_,$(FUZZ_RT_OBJ4))
FUZZ_RT_OBJ6 := $(subst $(PRGDIR)/,rt_prg_,$(FUZZ_RT_OBJ5))
FUZZ_RT_OBJ7 := $(subst $(MATCHFINDER_DIR)/,rt_matchfinder_,$(FUZZ_RT_OBJ6))
FUZZ_RT_OBJ7 := $(subst $(DEFAULT_SEQ_PROD_DIR)/,rt_default_seq_prod_,$(FUZZ_RT_OBJ6))
FUZZ_RT_OBJ8 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ7))
FUZZ_RT_OBJ9 := $(FUZZ_RT_OBJ8:.c=.o)
FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ9:.S=.o)
FUZZ_RT_OBJ10 := $(THIRD_PARTY_SEQ_PROD_OBJ) $(FUZZ_RT_OBJ9)
FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ10:.S=.o)
.PHONY: default all clean cleanall
@@ -151,7 +153,7 @@ rt_prg_%.o: $(PRGDIR)/%.c
rt_fuzz_%.o: %.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
rt_matchfinder_%.o: $(MATCHFINDER_DIR)/%.c
rt_default_seq_prod_%.o: $(DEFAULT_SEQ_PROD_DIR)/%.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@
d_lib_common_%.o: $(ZSTDDIR)/common/%.c
@@ -178,7 +180,7 @@ d_prg_%.o: $(PRGDIR)/%.c
d_fuzz_%.o: %.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
d_matchfinder_%.o: $(MATCHFINDER_DIR)/%.c
d_default_seq_prod_%.o: $(DEFAULT_SEQ_PROD_DIR)/%.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@
simple_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o
tests/fuzz/README.md
+5 -1
View File
@@ -61,7 +61,7 @@ Alternatively, you can fuzz all targets in parallel, using one core per target:
```
python3 ./fuzz.py list | xargs -P$(python3 ./fuzz.py list | wc -l) -I__ sh -c "python3 ./fuzz.py libfuzzer __ 2>&1 | tee __.log"
```
Either way, to double-check that no crashes were found, run `ls corpora/*crash`.
Either way, to double-check that no crashes were found, run `ls corpora/*crash`.
If any crashes were found, you can use the hashes to reproduce them.
## LibFuzzer
@@ -113,3 +113,7 @@ CC=clang CXX=clang++ ./fuzz.py build all --enable-asan --enable-ubsan
CC=clang CXX=clang++ ./fuzz.py build all --enable-msan
./fuzz.py regression all
```
## Fuzzing a custom sequence producer plugin
Sequence producer plugin authors can use the zstd fuzzers to stress-test their code.
See the documentation in `fuzz_third_party_seq_prod.h` for details.
tests/fuzz/block_decompress.c
+5 -1
View File
@@ -13,6 +13,7 @@
* decompression function to ensure the decompressor never crashes.
*/
#include "fuzz_data_producer.h"
#define ZSTD_STATIC_LINKING_ONLY
#include <stddef.h>
@@ -28,11 +29,12 @@ static size_t bufSize = 0;
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
size_t const neededBufSize = ZSTD_BLOCKSIZE_MAX;
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
/* Allocate all buffers and contexts if not already allocated */
if (neededBufSize > bufSize) {
free(rBuf);
rBuf = FUZZ_malloc(neededBufSize);
rBuf = FUZZ_malloc_rand(neededBufSize, producer);
bufSize = neededBufSize;
}
if (!dctx) {
@@ -42,6 +44,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
ZSTD_decompressBegin(dctx);
ZSTD_decompressBlock(dctx, rBuf, neededBufSize, src, size);
FUZZ_dataProducer_free(producer);
#ifndef STATEFUL_FUZZING
ZSTD_freeDCtx(dctx); dctx = NULL;
#endif
tests/fuzz/block_round_trip.c
+4
View File
@@ -23,6 +23,7 @@
#include "zstd.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
#include "fuzz_third_party_seq_prod.h"
static ZSTD_CCtx *cctx = NULL;
static ZSTD_DCtx *dctx = NULL;
@@ -54,6 +55,8 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
FUZZ_SEQ_PROD_SETUP();
/* Give a random portion of src data to the producer, to use for
parameter generation. The rest will be used for (de)compression */
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
@@ -95,5 +98,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
ZSTD_freeCCtx(cctx); cctx = NULL;
ZSTD_freeDCtx(dctx); dctx = NULL;
#endif
FUZZ_SEQ_PROD_TEARDOWN();
return 0;
}
tests/fuzz/decompress_dstSize_tooSmall.c
+4
View File
@@ -22,12 +22,15 @@
#include "zstd_errors.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
#include "fuzz_third_party_seq_prod.h"
static ZSTD_CCtx *cctx = NULL;
static ZSTD_DCtx *dctx = NULL;
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
FUZZ_SEQ_PROD_SETUP();
/* Give a random portion of src data to the producer, to use for
parameter generation. The rest will be used for (de)compression */
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
@@ -66,5 +69,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
ZSTD_freeCCtx(cctx); cctx = NULL;
ZSTD_freeDCtx(dctx); dctx = NULL;
#endif
FUZZ_SEQ_PROD_TEARDOWN();
return 0;
}
tests/fuzz/dictionary_decompress.c
+4
View File
@@ -20,11 +20,14 @@
#include "fuzz_helpers.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
#include "fuzz_third_party_seq_prod.h"
static ZSTD_DCtx *dctx = NULL;
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
FUZZ_SEQ_PROD_SETUP();
/* Give a random portion of src data to the producer, to use for
parameter generation. The rest will be used for (de)compression */
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
@@ -69,5 +72,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
#ifndef STATEFUL_FUZZING
ZSTD_freeDCtx(dctx); dctx = NULL;
#endif
FUZZ_SEQ_PROD_TEARDOWN();
return 0;
}
tests/fuzz/dictionary_loader.c
+4 -1
View File
@@ -20,6 +20,7 @@
#include "fuzz_helpers.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
#include "fuzz_third_party_seq_prod.h"
/**
* Compresses the data and returns the compressed size or an error.
@@ -35,7 +36,7 @@ static size_t compress(void* compressed, size_t compressedCapacity,
if (refPrefix)
FUZZ_ZASSERT(ZSTD_CCtx_refPrefix_advanced(
cctx, dict, dictSize, dictContentType));
else
else
FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced(
cctx, dict, dictSize, dictLoadMethod, dictContentType));
size_t const compressedSize = ZSTD_compress2(
@@ -67,6 +68,7 @@ static size_t decompress(void* result, size_t resultCapacity,
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
FUZZ_SEQ_PROD_SETUP();
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
int const refPrefix = FUZZ_dataProducer_uint32Range(producer, 0, 1) != 0;
ZSTD_dictLoadMethod_e const dlm =
@@ -99,5 +101,6 @@ out:
free(cBuf);
free(rBuf);
FUZZ_dataProducer_free(producer);
FUZZ_SEQ_PROD_TEARDOWN();
return 0;
}
tests/fuzz/dictionary_round_trip.c
+4
View File
@@ -21,6 +21,7 @@
#include "fuzz_helpers.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
#include "fuzz_third_party_seq_prod.h"
static ZSTD_CCtx *cctx = NULL;
static ZSTD_DCtx *dctx = NULL;
@@ -108,6 +109,8 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
FUZZ_SEQ_PROD_SETUP();
/* Give a random portion of src data to the producer, to use for
parameter generation. The rest will be used for (de)compression */
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
@@ -147,5 +150,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
ZSTD_freeCCtx(cctx); cctx = NULL;
ZSTD_freeDCtx(dctx); dctx = NULL;
#endif
FUZZ_SEQ_PROD_TEARDOWN();
return 0;
}
tests/fuzz/dictionary_stream_round_trip.c
+3
View File
@@ -22,6 +22,7 @@
#include "fuzz_helpers.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
#include "fuzz_third_party_seq_prod.h"
ZSTD_CCtx *cctx = NULL;
static ZSTD_DCtx *dctx = NULL;
@@ -147,6 +148,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
FUZZ_SEQ_PROD_SETUP();
size_t neededBufSize;
/* Give a random portion of src data to the producer, to use for
@@ -202,5 +204,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
ZSTD_freeCCtx(cctx); cctx = NULL;
ZSTD_freeDCtx(dctx); dctx = NULL;
#endif
FUZZ_SEQ_PROD_TEARDOWN();
return 0;
}
tests/fuzz/fuzz.h
+5
View File
@@ -31,6 +31,11 @@
* This is the canonical flag to enable deterministic builds for fuzzing.
* Changes to zstd for fuzzing are gated behind this define.
* It is recommended to define this when building zstd for fuzzing.
* @param FUZZ_THIRD_PARTY_SEQ_PROD
* This flag allows sequence producer plugin authors to replace the built-in
* default sequence producer with their own code. If you are not a plugin
* author, you should not define this flag. See the docs at
* fuzz_third_party_seq_prod.h for more information.
*/
#ifndef FUZZ_H
tests/fuzz/fuzz.py
+11
View File
@@ -78,6 +78,7 @@ CFLAGS = os.environ.get('CFLAGS', '-O3')
CXXFLAGS = os.environ.get('CXXFLAGS', CFLAGS)
LDFLAGS = os.environ.get('LDFLAGS', '')
MFLAGS = os.environ.get('MFLAGS', '-j')
THIRD_PARTY_SEQ_PROD_OBJ = os.environ.get('THIRD_PARTY_SEQ_PROD_OBJ', '')
# Fuzzing environment variables
LIB_FUZZING_ENGINE = os.environ.get('LIB_FUZZING_ENGINE', 'libregression.a')
@@ -319,6 +320,12 @@ def build_parser(args):
dest='stateful_fuzzing',
action='store_true',
help='Reuse contexts between runs (makes reproduction impossible)')
parser.add_argument(
'--custom-seq-prod',
dest='third_party_seq_prod_obj',
type=str,
default=THIRD_PARTY_SEQ_PROD_OBJ,
help='Path to an object file with symbols for fuzzing your sequence producer plugin.')
parser.add_argument(
'--cc',
dest='cc',
@@ -450,6 +457,10 @@ def build(args):
if args.stateful_fuzzing:
cppflags += ['-DSTATEFUL_FUZZING']
if args.third_party_seq_prod_obj:
cppflags += ['-DFUZZ_THIRD_PARTY_SEQ_PROD']
mflags += ['THIRD_PARTY_SEQ_PROD_OBJ={}'.format(args.third_party_seq_prod_obj)]
if args.fuzzing_mode:
cppflags += ['-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION']
tests/fuzz/fuzz_data_producer.c
+1
View File
@@ -8,6 +8,7 @@
* You may select, at your option, one of the above-listed licenses.
*/
#include "fuzz_helpers.h"
#include "fuzz_data_producer.h"
struct FUZZ_dataProducer_s{
tests/fuzz/fuzz_data_producer.h
-1
View File
@@ -24,7 +24,6 @@
#include <stdio.h>
#include <stdlib.h>
#include "fuzz_helpers.h"
/* Struct used for maintaining the state of the data */
typedef struct FUZZ_dataProducer_s FUZZ_dataProducer_t;
tests/fuzz/fuzz_helpers.c
+16
View File
@@ -23,6 +23,22 @@ void* FUZZ_malloc(size_t size)
return NULL;
}
void* FUZZ_malloc_rand(size_t size, FUZZ_dataProducer_t *producer)
{
if (size > 0) {
void* const mem = malloc(size);
FUZZ_ASSERT(mem);
return mem;
} else {
uintptr_t ptr = 0;
/* Add +- 1M 50% of the time */
if (FUZZ_dataProducer_uint32Range(producer, 0, 1))
FUZZ_dataProducer_int32Range(producer, -1000000, 1000000);
return (void*)ptr;
}
}
int FUZZ_memcmp(void const* lhs, void const* rhs, size_t size)
{
if (size == 0) {
tests/fuzz/fuzz_helpers.h
+7
View File
@@ -19,6 +19,7 @@
#include "fuzz.h"
#include "xxhash.h"
#include "zstd.h"
#include "fuzz_data_producer.h"
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
@@ -62,6 +63,12 @@ extern "C" {
*/
void* FUZZ_malloc(size_t size);
/**
* malloc except returns random pointer for zero sized data and FUZZ_ASSERT
* that malloc doesn't fail.
*/
void* FUZZ_malloc_rand(size_t size, FUZZ_dataProducer_t *producer);
/**
* memcmp but accepts NULL.
*/
tests/fuzz/fuzz_third_party_seq_prod.h
+116
View File
@@ -0,0 +1,116 @@
/*
* Copyright (c) Yann Collet, Meta Platforms, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef EXAMPLE_SEQ_PROD_H
#define EXAMPLE_SEQ_PROD_H
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
/* *** INTERFACE FOR FUZZING THIRD-PARTY SEQUENCE PRODUCER PLUGINS ***
* Fuzz-testing for the external sequence producer API was introduced in PR #3437.
* However, the setup in #3437 only allows fuzzers to exercise the implementation of the
* API itself (the code in the core zstd library which interacts with your plugin).
*
* This header defines an interface for plugin authors to link their code into the fuzzer
* build. Plugin authors can provide an object file implementing the symbols below,
* and those symbols will replace the default ones provided by #3437.
*
* To fuzz your plugin, follow these steps:
* - Build your object file with a recent version of clang. Building with gcc is not supported.
* - Build your object file using appropriate flags for fuzzing. For example:
* `-g -fno-omit-frame-pointer -fsanitize=undefined,address,fuzzer`
* - Build the fuzzer binaries with options corresponding to the flags you chose. Use --custom-seq-prod= to pass in your object file:
* `./fuzz.py build all --enable-fuzzer --enable-asan --enable-ubsan --cc clang --cxx clang++ --custom-seq-prod=your_object.o`
*
* An example implementation of this header is provided at tests/fuzz/seq_prod_fuzz_example/.
* Use these commands to fuzz with the example code:
* $ make corpora
* $ make -C seq_prod_fuzz_example/
* $ python3 ./fuzz.py build all --enable-fuzzer --enable-asan --enable-ubsan --cc clang --cxx clang++ --custom-seq-prod=seq_prod_fuzz_example/example_seq_prod.o
* $ python3 ./fuzz.py libfuzzer simple_round_trip
*/
/* The fuzzer will call this function before each test-case. It should run any
* setup actions (such as starting a hardware device) needed for fuzzing.
*
* The fuzzer will assert() that the return value is zero. To signal an error,
* please return a non-zero value. */
size_t FUZZ_seqProdSetup(void);
/* The fuzzer will call this function after each test-case. It should free
* resources aquired by FUZZ_seqProdSetup() to prevent leaks across test-cases.
*
* The fuzzer will assert() that the return value is zero. To signal an error,
* please return a non-zero value. */
size_t FUZZ_seqProdTearDown(void);
/* The fuzzer will call this function before each test-case, only after calling
* FUZZ_seqProdSetup(), to obtain a sequence producer state which can be passed
* into ZSTD_registerSequenceProducer().
*
* All compressions which are part of a test-case will share a single sequence
* producer state. Sharing the state object is safe because the fuzzers currently
* don't exercise the sequence producer API in multi-threaded scenarios. We may
* need a new approach in the future to support multi-threaded fuzzing.
*
* The fuzzer will assert() that the return value is not NULL. To signal an error,
* please return NULL. */
void* FUZZ_createSeqProdState(void);
/* The fuzzer will call this function after each test-case. It should free any
* resources aquired by FUZZ_createSeqProdState().
*
* The fuzzer will assert() that the return value is zero. To signal an error,
* please return a non-zero value. */
size_t FUZZ_freeSeqProdState(void* sequenceProducerState);
/* This is the sequence producer function you would like to fuzz! It will receive
* the void* returned by FUZZ_createSeqProdState() on each invocation. */
size_t FUZZ_thirdPartySeqProd(void* sequenceProducerState,
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
const void* src, size_t srcSize,
const void* dict, size_t dictSize,
int compressionLevel,
size_t windowSize);
/* These macros are internal helpers. You do not need to worry about them. */
#ifdef FUZZ_THIRD_PARTY_SEQ_PROD
#define FUZZ_SEQ_PROD_SETUP() \
do { \
FUZZ_ASSERT(FUZZ_seqProdSetup() == 0); \
FUZZ_seqProdState = FUZZ_createSeqProdState(); \
FUZZ_ASSERT(FUZZ_seqProdState != NULL); \
} while (0)
#else
#define FUZZ_SEQ_PROD_SETUP()
#endif
#ifdef FUZZ_THIRD_PARTY_SEQ_PROD
#define FUZZ_SEQ_PROD_TEARDOWN() \
do { \
FUZZ_ASSERT(FUZZ_freeSeqProdState(FUZZ_seqProdState) == 0); \
FUZZ_ASSERT(FUZZ_seqProdTearDown() == 0); \
} while (0)
#else
#define FUZZ_SEQ_PROD_TEARDOWN()
#endif
#ifdef __cplusplus
}
#endif
#endif /* EXAMPLE_SEQ_PROD_H */
tests/fuzz/raw_dictionary_round_trip.c
+5 -1
View File
@@ -21,6 +21,7 @@
#include "fuzz_helpers.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
#include "fuzz_third_party_seq_prod.h"
static ZSTD_CCtx *cctx = NULL;
static ZSTD_DCtx *dctx = NULL;
@@ -42,7 +43,7 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
FUZZ_ZASSERT(ZSTD_CCtx_refPrefix_advanced(
cctx, dict, dictSize,
ZSTD_dct_rawContent));
else
else
FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced(
cctx, dict, dictSize,
(ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1),
@@ -68,6 +69,8 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
FUZZ_SEQ_PROD_SETUP();
/* Give a random portion of src data to the producer, to use for
parameter generation. The rest will be used for (de)compression */
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
@@ -111,5 +114,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
ZSTD_freeCCtx(cctx); cctx = NULL;
ZSTD_freeDCtx(dctx); dctx = NULL;
#endif
FUZZ_SEQ_PROD_TEARDOWN();
return 0;
}
tests/fuzz/seq_prod_fuzz_example/Makefile
+16
View File
@@ -0,0 +1,16 @@
# Copyright (c) Yann Collet, Meta Platforms, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
CC = clang
CFLAGS = -g -fno-omit-frame-pointer -fsanitize=undefined,address,fuzzer -I../ -I../../../lib/
.PHONY: default
default: example_seq_prod.o
example_seq_prod.o: example_seq_prod.c
$(CC) -c $(CFLAGS) $^ -o $@
tests/fuzz/seq_prod_fuzz_example/README.md
+12
View File
@@ -0,0 +1,12 @@
# Fuzzing a Custom Sequence Producer Plugin
This directory contains example code for using a custom sequence producer in the zstd fuzzers.
You can build and run the code in this directory using these commands:
```
$ make corpora
$ make -C seq_prod_fuzz_example/
$ python3 ./fuzz.py build all --enable-fuzzer --enable-asan --enable-ubsan --cc clang --cxx clang++ --custom-seq-prod=seq_prod_fuzz_example/example_seq_prod.o
$ python3 ./fuzz.py libfuzzer simple_round_trip
```
See `../fuzz_third_party_seq_prod.h` and `../README.md` for more information on zstd fuzzing.
tests/fuzz/seq_prod_fuzz_example/example_seq_prod.c
+52
View File
@@ -0,0 +1,52 @@
/*
* Copyright (c) Yann Collet, Meta Platforms, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "fuzz_third_party_seq_prod.h"
#include <assert.h>
#include <stdlib.h>
#include <string.h>
_Thread_local size_t threadLocalState;
size_t FUZZ_seqProdSetup(void) {
threadLocalState = 0;
return 0;
}
size_t FUZZ_seqProdTearDown(void) {
return 0;
}
void* FUZZ_createSeqProdState(void) {
return calloc(1, sizeof(size_t));
}
size_t FUZZ_freeSeqProdState(void* state) {
free(state);
return 0;
}
size_t FUZZ_thirdPartySeqProd(
void* sequenceProducerState,
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
const void* src, size_t srcSize,
const void* dict, size_t dictSize,
int compressionLevel,
size_t windowSize
) {
/* Try to catch unsafe use of the shared state */
size_t* const sharedStatePtr = (size_t*)sequenceProducerState;
assert(*sharedStatePtr == threadLocalState);
(*sharedStatePtr)++; threadLocalState++;
/* Check that fallback is enabled when FUZZ_THIRD_PARTY_SEQ_PROD is defined */
return ZSTD_SEQUENCE_PRODUCER_ERROR;
}
tests/fuzz/sequence_compression_api.c
+4
View File
@@ -25,6 +25,7 @@
#include "fuzz_helpers.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
#include "fuzz_third_party_seq_prod.h"
static ZSTD_CCtx* cctx = NULL;
static ZSTD_DCtx* dctx = NULL;
@@ -263,6 +264,8 @@ static size_t roundTripTest(void* result, size_t resultCapacity,
int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
{
FUZZ_SEQ_PROD_SETUP();
void* rBuf;
size_t rBufSize;
void* cBuf;
@@ -373,5 +376,6 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
free(generatedSrc); generatedSrc = NULL;
free(literalsBuffer); literalsBuffer = NULL;
#endif
FUZZ_SEQ_PROD_TEARDOWN();
return 0;
}
tests/fuzz/simple_compress.c
+4
View File
@@ -22,11 +22,14 @@
#include "zstd_errors.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
#include "fuzz_third_party_seq_prod.h"
static ZSTD_CCtx *cctx = NULL;
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
FUZZ_SEQ_PROD_SETUP();
/* Give a random portion of src data to the producer, to use for
parameter generation. The rest will be used for (de)compression */
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
@@ -52,5 +55,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
#ifndef STATEFUL_FUZZING
ZSTD_freeCCtx(cctx); cctx = NULL;
#endif
FUZZ_SEQ_PROD_TEARDOWN();
return 0;
}
tests/fuzz/simple_round_trip.c
+4
View File
@@ -22,6 +22,7 @@
#include "fuzz_helpers.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
#include "fuzz_third_party_seq_prod.h"
static ZSTD_CCtx *cctx = NULL;
static ZSTD_DCtx *dctx = NULL;
@@ -129,6 +130,8 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
FUZZ_SEQ_PROD_SETUP();
size_t const rBufSize = size;
void* rBuf = FUZZ_malloc(rBufSize);
size_t cBufSize = ZSTD_compressBound(size);
@@ -164,5 +167,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
ZSTD_freeCCtx(cctx); cctx = NULL;
ZSTD_freeDCtx(dctx); dctx = NULL;
#endif
FUZZ_SEQ_PROD_TEARDOWN();
return 0;
}
tests/fuzz/stream_round_trip.c
+3
View File
@@ -22,6 +22,7 @@
#include "fuzz_helpers.h"
#include "zstd_helpers.h"
#include "fuzz_data_producer.h"
#include "fuzz_third_party_seq_prod.h"
ZSTD_CCtx *cctx = NULL;
static ZSTD_DCtx *dctx = NULL;
@@ -133,6 +134,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
FUZZ_SEQ_PROD_SETUP();
size_t neededBufSize;
/* Give a random portion of src data to the producer, to use for
@@ -191,5 +193,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
ZSTD_freeCCtx(cctx); cctx = NULL;
ZSTD_freeDCtx(dctx); dctx = NULL;
#endif
FUZZ_SEQ_PROD_TEARDOWN();
return 0;
}
tests/fuzz/zstd_helpers.c
+20
View File
@@ -18,10 +18,13 @@
#include "zstd.h"
#include "zdict.h"
#include "sequence_producer.h"
#include "fuzz_third_party_seq_prod.h"
const int kMinClevel = -3;
const int kMaxClevel = 19;
void* FUZZ_seqProdState = NULL;
static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value)
{
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
@@ -72,12 +75,25 @@ ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer)
}
static void setSequenceProducerParams(ZSTD_CCtx *cctx, FUZZ_dataProducer_t *producer) {
#ifdef FUZZ_THIRD_PARTY_SEQ_PROD
ZSTD_registerSequenceProducer(
cctx,
FUZZ_seqProdState,
FUZZ_thirdPartySeqProd
);
#else
ZSTD_registerSequenceProducer(
cctx,
NULL,
simpleSequenceProducer
);
#endif
#ifdef FUZZ_THIRD_PARTY_SEQ_PROD
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableSeqProducerFallback, 1));
#else
setRand(cctx, ZSTD_c_enableSeqProducerFallback, 0, 1, producer);
#endif
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0));
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
}
@@ -137,11 +153,15 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
setRand(cctx, ZSTD_c_targetCBlockSize, ZSTD_TARGETCBLOCKSIZE_MIN, ZSTD_TARGETCBLOCKSIZE_MAX, producer);
}
#ifdef FUZZ_THIRD_PARTY_SEQ_PROD
setSequenceProducerParams(cctx, producer);
#else
if (FUZZ_dataProducer_uint32Range(producer, 0, 10) == 1) {
setSequenceProducerParams(cctx, producer);
} else {
ZSTD_registerSequenceProducer(cctx, NULL, NULL);
}
#endif
}
FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer)
tests/fuzz/zstd_helpers.h
+4
View File
@@ -45,6 +45,10 @@ typedef struct {
*/
FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer);
#ifdef FUZZ_THIRD_PARTY_SEQ_PROD
extern void* FUZZ_seqProdState;
#endif
#ifdef __cplusplus
}
#endif
tests/fuzzer.c
+195 -1
View File
@@ -1286,6 +1286,73 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Check block splitter with 64K literal length : ", testNb++);
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
size_t const srcSize = 256 * 1024;
U32 const compressibleLenU32 = 32 * 1024 / 4;
U32 const blockSizeU32 = 128 * 1024 / 4;
U32 const litLenU32 = 64 * 1024 / 4;
U32* data = (U32*)malloc(srcSize);
size_t dSize;
if (data == NULL || cctx == NULL) goto _output_error;
/* Generate data without any matches */
RDG_genBuffer(data, srcSize, 0.0, 0.01, 2654435761U);
/* Generate 32K of compressible data */
RDG_genBuffer(data, compressibleLenU32 * 4, 0.5, 0.5, 0xcafebabe);
/* Add a match of offset=12, length=8 at idx=16, 32, 48, 64 */
data[compressibleLenU32 + 0] = 0xFFFFFFFF;
data[compressibleLenU32 + 1] = 0xEEEEEEEE;
data[compressibleLenU32 + 4] = 0xFFFFFFFF;
data[compressibleLenU32 + 5] = 0xEEEEEEEE;
/* Add a match of offset=16, length=8 at idx=64K + 64.
* This generates a sequence with llen=64K, and repeat code 1.
* The block splitter thought this was ll0, and corrupted the
* repeat offset history.
*/
data[compressibleLenU32 + litLenU32 + 2 + 0] = 0xDDDDDDDD;
data[compressibleLenU32 + litLenU32 + 2 + 1] = 0xCCCCCCCC;
data[compressibleLenU32 + litLenU32 + 2 + 4] = 0xDDDDDDDD;
data[compressibleLenU32 + litLenU32 + 2 + 5] = 0xCCCCCCCC;
/* Add a match of offset=16, length=8 at idx=128K + 16.
* This should generate a sequence with repeat code = 1.
* But the block splitters mistake caused zstd to generate
* repeat code = 2, corrupting the data.
*/
data[blockSizeU32] = 0xBBBBBBBB;
data[blockSizeU32 + 1] = 0xAAAAAAAA;
data[blockSizeU32 + 4] = 0xBBBBBBBB;
data[blockSizeU32 + 5] = 0xAAAAAAAA;
/* Generate a golden file from this data in case datagen changes and
* doesn't generate the exact same data. We will also test this golden file.
*/
if (0) {
FILE* f = fopen("golden-compression/PR-3517-block-splitter-corruption-test", "wb");
fwrite(data, 1, srcSize, f);
fclose(f);
}
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize);
CHECK_Z(cSize);
dSize = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
CHECK_Z(dSize);
CHECK_EQ(dSize, srcSize);
CHECK(!memcmp(decodedBuffer, data, srcSize));
free(data);
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++);
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
@@ -1583,6 +1650,133 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCParams() : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
int value;
ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0);
cparams.strategy = -1;
/* Set invalid cParams == no change. */
CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
CHECK_EQ(value, 0);
cparams = ZSTD_getCParams(12, 0, 0);
CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
CHECK_EQ(value, (int)cparams.windowLog);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
CHECK_EQ(value, (int)cparams.chainLog);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
CHECK_EQ(value, (int)cparams.hashLog);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
CHECK_EQ(value, (int)cparams.searchLog);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
CHECK_EQ(value, (int)cparams.minMatch);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
CHECK_EQ(value, (int)cparams.targetLength);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
CHECK_EQ(value, (int)cparams.strategy);
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setFParams() : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
int value;
ZSTD_frameParameters fparams = {0, 1, 1};
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
CHECK_EQ(value, 1);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
CHECK_EQ(value, 1);
CHECK_Z(ZSTD_CCtx_setFParams(cctx, fparams));
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
CHECK_EQ(value, fparams.contentSizeFlag);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
CHECK_EQ(value, fparams.checksumFlag);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
CHECK_EQ(value, !fparams.noDictIDFlag);
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCarams() : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
int value;
ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
params.cParams.strategy = -1;
/* Set invalid params == no change. */
CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params)));
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
CHECK_EQ(value, 1);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
CHECK_EQ(value, 0);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
CHECK_EQ(value, 1);
params = ZSTD_getParams(12, 0, 0);
params.fParams.contentSizeFlag = 0;
params.fParams.checksumFlag = 1;
params.fParams.noDictIDFlag = 1;
CHECK_Z(ZSTD_CCtx_setParams(cctx, params));
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
CHECK_EQ(value, (int)params.cParams.windowLog);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
CHECK_EQ(value, (int)params.cParams.chainLog);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
CHECK_EQ(value, (int)params.cParams.hashLog);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
CHECK_EQ(value, (int)params.cParams.searchLog);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
CHECK_EQ(value, (int)params.cParams.minMatch);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
CHECK_EQ(value, (int)params.cParams.targetLength);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
CHECK_EQ(value, (int)params.cParams.strategy);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
CHECK_EQ(value, params.fParams.contentSizeFlag);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
CHECK_EQ(value, params.fParams.checksumFlag);
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
CHECK_EQ(value, !params.fParams.noDictIDFlag);
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_outBuffer out = {NULL, 0, 0};
@@ -2228,7 +2422,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
3663, 3662, 3661, 3660, 3660,
3660, 3660, 3660 };
size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2820, 2940,
2950, 2950, 2925, 2900, 2891,
2950, 2950, 2925, 2900, 2892,
2910, 2910, 2910, 2780, 2775,
2765, 2760, 2755, 2754, 2753,
2753, 2753, 2753 };
tests/golden-compression/PR-3517-block-splitter-corruption-test
+1
View File
File diff suppressed because one or more lines are too long
tests/regression/results.csv
+445 -445
View File
@@ -6,10 +6,10 @@ silesia.tar, level 0, compress
silesia.tar, level 1, compress simple, 5327717
silesia.tar, level 3, compress simple, 4854086
silesia.tar, level 4, compress simple, 4791503
silesia.tar, level 5, compress simple, 4677740
silesia.tar, level 6, compress simple, 4613242
silesia.tar, level 7, compress simple, 4576661
silesia.tar, level 9, compress simple, 4552899
silesia.tar, level 5, compress simple, 4679004
silesia.tar, level 6, compress simple, 4614561
silesia.tar, level 7, compress simple, 4579828
silesia.tar, level 9, compress simple, 4555448
silesia.tar, level 13, compress simple, 4502956
silesia.tar, level 16, compress simple, 4360546
silesia.tar, level 19, compress simple, 4265911
@@ -25,8 +25,8 @@ github.tar, level 3, compress
github.tar, level 4, compress simple, 38893
github.tar, level 5, compress simple, 39651
github.tar, level 6, compress simple, 39282
github.tar, level 7, compress simple, 38110
github.tar, level 9, compress simple, 36760
github.tar, level 7, compress simple, 38005
github.tar, level 9, compress simple, 36723
github.tar, level 13, compress simple, 35501
github.tar, level 16, compress simple, 40466
github.tar, level 19, compress simple, 32276
@@ -40,10 +40,10 @@ silesia, level 0, compress
silesia, level 1, compress cctx, 5306632
silesia, level 3, compress cctx, 4842075
silesia, level 4, compress cctx, 4779186
silesia, level 5, compress cctx, 4666323
silesia, level 6, compress cctx, 4603066
silesia, level 7, compress cctx, 4566984
silesia, level 9, compress cctx, 4543018
silesia, level 5, compress cctx, 4667668
silesia, level 6, compress cctx, 4604351
silesia, level 7, compress cctx, 4570271
silesia, level 9, compress cctx, 4545850
silesia, level 13, compress cctx, 4493990
silesia, level 16, compress cctx, 4360041
silesia, level 19, compress cctx, 4296055
@@ -53,33 +53,33 @@ silesia, multithreaded long distance mode, compress
silesia, small window log, compress cctx, 7082951
silesia, small hash log, compress cctx, 6526141
silesia, small chain log, compress cctx, 4912197
silesia, explicit params, compress cctx, 4794052
silesia, explicit params, compress cctx, 4794318
silesia, uncompressed literals, compress cctx, 4842075
silesia, uncompressed literals optimal, compress cctx, 4296055
silesia, huffman literals, compress cctx, 6172202
silesia, multithreaded with advanced params, compress cctx, 4842075
github, level -5, compress cctx, 204407
github, level -5 with dict, compress cctx, 52059
github, level -5 with dict, compress cctx, 47581
github, level -3, compress cctx, 193253
github, level -3 with dict, compress cctx, 46787
github, level -3 with dict, compress cctx, 43043
github, level -1, compress cctx, 175468
github, level -1 with dict, compress cctx, 43585
github, level -1 with dict, compress cctx, 42044
github, level 0, compress cctx, 136332
github, level 0 with dict, compress cctx, 41534
github, level 1, compress cctx, 142365
github, level 1 with dict, compress cctx, 42259
github, level 1 with dict, compress cctx, 41715
github, level 3, compress cctx, 136332
github, level 3 with dict, compress cctx, 41534
github, level 4, compress cctx, 136199
github, level 4 with dict, compress cctx, 41725
github, level 5, compress cctx, 135121
github, level 5 with dict, compress cctx, 38759
github, level 5 with dict, compress cctx, 38755
github, level 6, compress cctx, 135122
github, level 6 with dict, compress cctx, 38669
github, level 6 with dict, compress cctx, 38665
github, level 7, compress cctx, 135122
github, level 7 with dict, compress cctx, 38755
github, level 7 with dict, compress cctx, 38759
github, level 9, compress cctx, 135122
github, level 9 with dict, compress cctx, 39398
github, level 9 with dict, compress cctx, 39362
github, level 13, compress cctx, 132878
github, level 13 with dict, compress cctx, 39948
github, level 16, compress cctx, 133209
@@ -104,10 +104,10 @@ silesia, level 0, zstdcli,
silesia, level 1, zstdcli, 5306680
silesia, level 3, zstdcli, 4842123
silesia, level 4, zstdcli, 4779234
silesia, level 5, zstdcli, 4666371
silesia, level 6, zstdcli, 4603114
silesia, level 7, zstdcli, 4567032
silesia, level 9, zstdcli, 4543066
silesia, level 5, zstdcli, 4667716
silesia, level 6, zstdcli, 4604399
silesia, level 7, zstdcli, 4570319
silesia, level 9, zstdcli, 4545898
silesia, level 13, zstdcli, 4494038
silesia, level 16, zstdcli, 4360089
silesia, level 19, zstdcli, 4296103
@@ -117,7 +117,7 @@ silesia, multithreaded long distance mode, zstdcli,
silesia, small window log, zstdcli, 7095048
silesia, small hash log, zstdcli, 6526189
silesia, small chain log, zstdcli, 4912245
silesia, explicit params, zstdcli, 4795432
silesia, explicit params, zstdcli, 4795840
silesia, uncompressed literals, zstdcli, 5120614
silesia, uncompressed literals optimal, zstdcli, 4319566
silesia, huffman literals, zstdcli, 5321417
@@ -129,10 +129,10 @@ silesia.tar, level 0, zstdcli,
silesia.tar, level 1, zstdcli, 5329010
silesia.tar, level 3, zstdcli, 4854164
silesia.tar, level 4, zstdcli, 4792352
silesia.tar, level 5, zstdcli, 4678682
silesia.tar, level 6, zstdcli, 4614125
silesia.tar, level 7, zstdcli, 4578719
silesia.tar, level 9, zstdcli, 4552903
silesia.tar, level 5, zstdcli, 4679860
silesia.tar, level 6, zstdcli, 4615355
silesia.tar, level 7, zstdcli, 4581791
silesia.tar, level 9, zstdcli, 4555452
silesia.tar, level 13, zstdcli, 4502960
silesia.tar, level 16, zstdcli, 4360550
silesia.tar, level 19, zstdcli, 4265915
@@ -143,21 +143,21 @@ silesia.tar, multithreaded long distance mode, zstdcli,
silesia.tar, small window log, zstdcli, 7100701
silesia.tar, small hash log, zstdcli, 6529264
silesia.tar, small chain log, zstdcli, 4917022
silesia.tar, explicit params, zstdcli, 4820713
silesia.tar, explicit params, zstdcli, 4821112
silesia.tar, uncompressed literals, zstdcli, 5122571
silesia.tar, uncompressed literals optimal, zstdcli, 4310145
silesia.tar, huffman literals, zstdcli, 5342074
silesia.tar, multithreaded with advanced params, zstdcli, 5122571
github, level -5, zstdcli, 206407
github, level -5 with dict, zstdcli, 48718
github, level -5 with dict, zstdcli, 47832
github, level -3, zstdcli, 195253
github, level -3 with dict, zstdcli, 47395
github, level -3 with dict, zstdcli, 46671
github, level -1, zstdcli, 177468
github, level -1 with dict, zstdcli, 45170
github, level -1 with dict, zstdcli, 43825
github, level 0, zstdcli, 138332
github, level 0 with dict, zstdcli, 43148
github, level 1, zstdcli, 144365
github, level 1 with dict, zstdcli, 43682
github, level 1 with dict, zstdcli, 43266
github, level 3, zstdcli, 138332
github, level 3 with dict, zstdcli, 43148
github, level 4, zstdcli, 138199
@@ -165,9 +165,9 @@ github, level 4 with dict, zstdcli,
github, level 5, zstdcli, 137121
github, level 5 with dict, zstdcli, 40728
github, level 6, zstdcli, 137122
github, level 6 with dict, zstdcli, 40636
github, level 6 with dict, zstdcli, 40638
github, level 7, zstdcli, 137122
github, level 7 with dict, zstdcli, 40745
github, level 7 with dict, zstdcli, 40749
github, level 9, zstdcli, 137122
github, level 9 with dict, zstdcli, 41393
github, level 13, zstdcli, 134878
@@ -188,27 +188,27 @@ github, uncompressed literals optimal, zstdcli,
github, huffman literals, zstdcli, 144365
github, multithreaded with advanced params, zstdcli, 167911
github.tar, level -5, zstdcli, 52119
github.tar, level -5 with dict, zstdcli, 50978
github.tar, level -5 with dict, zstdcli, 51101
github.tar, level -3, zstdcli, 45682
github.tar, level -3 with dict, zstdcli, 44660
github.tar, level -3 with dict, zstdcli, 44738
github.tar, level -1, zstdcli, 42564
github.tar, level -1 with dict, zstdcli, 41155
github.tar, level -1 with dict, zstdcli, 41357
github.tar, level 0, zstdcli, 38835
github.tar, level 0 with dict, zstdcli, 37999
github.tar, level 1, zstdcli, 39204
github.tar, level 1 with dict, zstdcli, 38093
github.tar, level 1 with dict, zstdcli, 38123
github.tar, level 3, zstdcli, 38835
github.tar, level 3 with dict, zstdcli, 37999
github.tar, level 4, zstdcli, 38897
github.tar, level 4 with dict, zstdcli, 37952
github.tar, level 5, zstdcli, 39655
github.tar, level 5 with dict, zstdcli, 39071
github.tar, level 5 with dict, zstdcli, 39073
github.tar, level 6, zstdcli, 39286
github.tar, level 6 with dict, zstdcli, 38638
github.tar, level 7, zstdcli, 38114
github.tar, level 7 with dict, zstdcli, 37886
github.tar, level 9, zstdcli, 36764
github.tar, level 9 with dict, zstdcli, 36632
github.tar, level 6 with dict, zstdcli, 38647
github.tar, level 7, zstdcli, 38009
github.tar, level 7 with dict, zstdcli, 37861
github.tar, level 9, zstdcli, 36727
github.tar, level 9 with dict, zstdcli, 36686
github.tar, level 13, zstdcli, 35505
github.tar, level 13 with dict, zstdcli, 37134
github.tar, level 16, zstdcli, 40470
@@ -235,18 +235,18 @@ silesia, level 0, advanced
silesia, level 1, advanced one pass, 5306632
silesia, level 3, advanced one pass, 4842075
silesia, level 4, advanced one pass, 4779186
silesia, level 5 row 1, advanced one pass, 4666323
silesia, level 5 row 2, advanced one pass, 4670136
silesia, level 5, advanced one pass, 4666323
silesia, level 6, advanced one pass, 4603066
silesia, level 7 row 1, advanced one pass, 4566984
silesia, level 7 row 2, advanced one pass, 4564868
silesia, level 7, advanced one pass, 4566984
silesia, level 9, advanced one pass, 4543018
silesia, level 11 row 1, advanced one pass, 4505046
silesia, level 11 row 2, advanced one pass, 4503116
silesia, level 12 row 1, advanced one pass, 4505046
silesia, level 12 row 2, advanced one pass, 4503116
silesia, level 5 row 1, advanced one pass, 4667668
silesia, level 5 row 2, advanced one pass, 4670326
silesia, level 5, advanced one pass, 4667668
silesia, level 6, advanced one pass, 4604351
silesia, level 7 row 1, advanced one pass, 4570271
silesia, level 7 row 2, advanced one pass, 4565169
silesia, level 7, advanced one pass, 4570271
silesia, level 9, advanced one pass, 4545850
silesia, level 11 row 1, advanced one pass, 4505658
silesia, level 11 row 2, advanced one pass, 4503429
silesia, level 12 row 1, advanced one pass, 4505658
silesia, level 12 row 2, advanced one pass, 4503429
silesia, level 13, advanced one pass, 4493990
silesia, level 16, advanced one pass, 4360041
silesia, level 19, advanced one pass, 4296055
@@ -257,7 +257,7 @@ silesia, multithreaded long distance mode, advanced
silesia, small window log, advanced one pass, 7095000
silesia, small hash log, advanced one pass, 6526141
silesia, small chain log, advanced one pass, 4912197
silesia, explicit params, advanced one pass, 4795432
silesia, explicit params, advanced one pass, 4795840
silesia, uncompressed literals, advanced one pass, 5120566
silesia, uncompressed literals optimal, advanced one pass, 4319518
silesia, huffman literals, advanced one pass, 5321369
@@ -269,18 +269,18 @@ silesia.tar, level 0, advanced
silesia.tar, level 1, advanced one pass, 5327717
silesia.tar, level 3, advanced one pass, 4854086
silesia.tar, level 4, advanced one pass, 4791503
silesia.tar, level 5 row 1, advanced one pass, 4677740
silesia.tar, level 5 row 2, advanced one pass, 4682161
silesia.tar, level 5, advanced one pass, 4677740
silesia.tar, level 6, advanced one pass, 4613242
silesia.tar, level 7 row 1, advanced one pass, 4576661
silesia.tar, level 7 row 2, advanced one pass, 4575393
silesia.tar, level 7, advanced one pass, 4576661
silesia.tar, level 9, advanced one pass, 4552899
silesia.tar, level 11 row 1, advanced one pass, 4514432
silesia.tar, level 11 row 2, advanced one pass, 4513604
silesia.tar, level 12 row 1, advanced one pass, 4514049
silesia.tar, level 12 row 2, advanced one pass, 4513797
silesia.tar, level 5 row 1, advanced one pass, 4679004
silesia.tar, level 5 row 2, advanced one pass, 4682334
silesia.tar, level 5, advanced one pass, 4679004
silesia.tar, level 6, advanced one pass, 4614561
silesia.tar, level 7 row 1, advanced one pass, 4579828
silesia.tar, level 7 row 2, advanced one pass, 4575602
silesia.tar, level 7, advanced one pass, 4579828
silesia.tar, level 9, advanced one pass, 4555448
silesia.tar, level 11 row 1, advanced one pass, 4514962
silesia.tar, level 11 row 2, advanced one pass, 4513816
silesia.tar, level 12 row 1, advanced one pass, 4514517
silesia.tar, level 12 row 2, advanced one pass, 4514007
silesia.tar, level 13, advanced one pass, 4502956
silesia.tar, level 16, advanced one pass, 4360546
silesia.tar, level 19, advanced one pass, 4265911
@@ -291,105 +291,105 @@ silesia.tar, multithreaded long distance mode, advanced
silesia.tar, small window log, advanced one pass, 7100655
silesia.tar, small hash log, advanced one pass, 6529206
silesia.tar, small chain log, advanced one pass, 4917041
silesia.tar, explicit params, advanced one pass, 4806855
silesia.tar, explicit params, advanced one pass, 4807274
silesia.tar, uncompressed literals, advanced one pass, 5122473
silesia.tar, uncompressed literals optimal, advanced one pass, 4310141
silesia.tar, huffman literals, advanced one pass, 5341705
silesia.tar, multithreaded with advanced params, advanced one pass, 5122567
github, level -5, advanced one pass, 204407
github, level -5 with dict, advanced one pass, 46718
github, level -5 with dict, advanced one pass, 45832
github, level -3, advanced one pass, 193253
github, level -3 with dict, advanced one pass, 45395
github, level -3 with dict, advanced one pass, 44671
github, level -1, advanced one pass, 175468
github, level -1 with dict, advanced one pass, 43170
github, level -1 with dict, advanced one pass, 41825
github, level 0, advanced one pass, 136332
github, level 0 with dict, advanced one pass, 41148
github, level 0 with dict dms, advanced one pass, 41148
github, level 0 with dict dds, advanced one pass, 41148
github, level 0 with dict copy, advanced one pass, 41124
github, level 0 with dict load, advanced one pass, 42252
github, level 0 with dict load, advanced one pass, 41847
github, level 1, advanced one pass, 142365
github, level 1 with dict, advanced one pass, 41682
github, level 1 with dict dms, advanced one pass, 41682
github, level 1 with dict dds, advanced one pass, 41682
github, level 1 with dict copy, advanced one pass, 41698
github, level 1 with dict load, advanced one pass, 43814
github, level 1 with dict, advanced one pass, 41266
github, level 1 with dict dms, advanced one pass, 41266
github, level 1 with dict dds, advanced one pass, 41266
github, level 1 with dict copy, advanced one pass, 41279
github, level 1 with dict load, advanced one pass, 43331
github, level 3, advanced one pass, 136332
github, level 3 with dict, advanced one pass, 41148
github, level 3 with dict dms, advanced one pass, 41148
github, level 3 with dict dds, advanced one pass, 41148
github, level 3 with dict copy, advanced one pass, 41124
github, level 3 with dict load, advanced one pass, 42252
github, level 3 with dict load, advanced one pass, 41847
github, level 4, advanced one pass, 136199
github, level 4 with dict, advanced one pass, 41251
github, level 4 with dict dms, advanced one pass, 41251
github, level 4 with dict dds, advanced one pass, 41251
github, level 4 with dict copy, advanced one pass, 41216
github, level 4 with dict load, advanced one pass, 41159
github, level 4 with dict load, advanced one pass, 41548
github, level 5 row 1, advanced one pass, 134584
github, level 5 row 1 with dict dms, advanced one pass, 38758
github, level 5 row 1 with dict dms, advanced one pass, 38754
github, level 5 row 1 with dict dds, advanced one pass, 38728
github, level 5 row 1 with dict copy, advanced one pass, 38759
github, level 5 row 1 with dict load, advanced one pass, 41518
github, level 5 row 1 with dict copy, advanced one pass, 38755
github, level 5 row 1 with dict load, advanced one pass, 41899
github, level 5 row 2, advanced one pass, 135121
github, level 5 row 2 with dict dms, advanced one pass, 38938
github, level 5 row 2 with dict dds, advanced one pass, 38732
github, level 5 row 2 with dict copy, advanced one pass, 38934
github, level 5 row 2 with dict load, advanced one pass, 40725
github, level 5 row 2 with dict load, advanced one pass, 41248
github, level 5, advanced one pass, 135121
github, level 5 with dict, advanced one pass, 38758
github, level 5 with dict dms, advanced one pass, 38758
github, level 5 with dict, advanced one pass, 38754
github, level 5 with dict dms, advanced one pass, 38754
github, level 5 with dict dds, advanced one pass, 38728
github, level 5 with dict copy, advanced one pass, 38759
github, level 5 with dict load, advanced one pass, 40725
github, level 5 with dict copy, advanced one pass, 38755
github, level 5 with dict load, advanced one pass, 41248
github, level 6, advanced one pass, 135122
github, level 6 with dict, advanced one pass, 38671
github, level 6 with dict dms, advanced one pass, 38671
github, level 6 with dict dds, advanced one pass, 38636
github, level 6 with dict copy, advanced one pass, 38669
github, level 6 with dict load, advanced one pass, 40695
github, level 6 with dict, advanced one pass, 38669
github, level 6 with dict dms, advanced one pass, 38669
github, level 6 with dict dds, advanced one pass, 38638
github, level 6 with dict copy, advanced one pass, 38665
github, level 6 with dict load, advanced one pass, 41153
github, level 7 row 1, advanced one pass, 134584
github, level 7 row 1 with dict dms, advanced one pass, 38758
github, level 7 row 1 with dict dds, advanced one pass, 38745
github, level 7 row 1 with dict copy, advanced one pass, 38755
github, level 7 row 1 with dict load, advanced one pass, 43154
github, level 7 row 1 with dict dms, advanced one pass, 38765
github, level 7 row 1 with dict dds, advanced one pass, 38749
github, level 7 row 1 with dict copy, advanced one pass, 38759
github, level 7 row 1 with dict load, advanced one pass, 43227
github, level 7 row 2, advanced one pass, 135122
github, level 7 row 2 with dict dms, advanced one pass, 38860
github, level 7 row 2 with dict dds, advanced one pass, 38766
github, level 7 row 2 with dict copy, advanced one pass, 38834
github, level 7 row 2 with dict load, advanced one pass, 40695
github, level 7 row 2 with dict load, advanced one pass, 41153
github, level 7, advanced one pass, 135122
github, level 7 with dict, advanced one pass, 38758
github, level 7 with dict dms, advanced one pass, 38758
github, level 7 with dict dds, advanced one pass, 38745
github, level 7 with dict copy, advanced one pass, 38755
github, level 7 with dict load, advanced one pass, 40695
github, level 7 with dict, advanced one pass, 38765
github, level 7 with dict dms, advanced one pass, 38765
github, level 7 with dict dds, advanced one pass, 38749
github, level 7 with dict copy, advanced one pass, 38759
github, level 7 with dict load, advanced one pass, 41153
github, level 9, advanced one pass, 135122
github, level 9 with dict, advanced one pass, 39437
github, level 9 with dict dms, advanced one pass, 39437
github, level 9 with dict, advanced one pass, 39439
github, level 9 with dict dms, advanced one pass, 39439
github, level 9 with dict dds, advanced one pass, 39393
github, level 9 with dict copy, advanced one pass, 39398
github, level 9 with dict load, advanced one pass, 41710
github, level 9 with dict copy, advanced one pass, 39362
github, level 9 with dict load, advanced one pass, 42148
github, level 11 row 1, advanced one pass, 135367
github, level 11 row 1 with dict dms, advanced one pass, 39671
github, level 11 row 1 with dict dds, advanced one pass, 39671
github, level 11 row 1 with dict copy, advanced one pass, 39651
github, level 11 row 1 with dict load, advanced one pass, 41360
github, level 11 row 1 with dict load, advanced one pass, 41744
github, level 11 row 2, advanced one pass, 135367
github, level 11 row 2 with dict dms, advanced one pass, 39671
github, level 11 row 2 with dict dds, advanced one pass, 39671
github, level 11 row 2 with dict copy, advanced one pass, 39651
github, level 11 row 2 with dict load, advanced one pass, 41360
github, level 11 row 2 with dict load, advanced one pass, 41744
github, level 12 row 1, advanced one pass, 134402
github, level 12 row 1 with dict dms, advanced one pass, 39677
github, level 12 row 1 with dict dds, advanced one pass, 39677
github, level 12 row 1 with dict copy, advanced one pass, 39677
github, level 12 row 1 with dict load, advanced one pass, 41166
github, level 12 row 1 with dict load, advanced one pass, 41553
github, level 12 row 2, advanced one pass, 134402
github, level 12 row 2 with dict dms, advanced one pass, 39677
github, level 12 row 2 with dict dds, advanced one pass, 39677
github, level 12 row 2 with dict copy, advanced one pass, 39677
github, level 12 row 2 with dict load, advanced one pass, 41166
github, level 12 row 2 with dict load, advanced one pass, 41553
github, level 13, advanced one pass, 132878
github, level 13 with dict, advanced one pass, 39900
github, level 13 with dict dms, advanced one pass, 39900
@@ -422,11 +422,11 @@ github, uncompressed literals optimal, advanced
github, huffman literals, advanced one pass, 142365
github, multithreaded with advanced params, advanced one pass, 165911
github.tar, level -5, advanced one pass, 52115
github.tar, level -5 with dict, advanced one pass, 50974
github.tar, level -5 with dict, advanced one pass, 51097
github.tar, level -3, advanced one pass, 45678
github.tar, level -3 with dict, advanced one pass, 44656
github.tar, level -3 with dict, advanced one pass, 44734
github.tar, level -1, advanced one pass, 42560
github.tar, level -1 with dict, advanced one pass, 41151
github.tar, level -1 with dict, advanced one pass, 41353
github.tar, level 0, advanced one pass, 38831
github.tar, level 0 with dict, advanced one pass, 37995
github.tar, level 0 with dict dms, advanced one pass, 38003
@@ -434,10 +434,10 @@ github.tar, level 0 with dict dds, advanced
github.tar, level 0 with dict copy, advanced one pass, 37995
github.tar, level 0 with dict load, advanced one pass, 37956
github.tar, level 1, advanced one pass, 39200
github.tar, level 1 with dict, advanced one pass, 38089
github.tar, level 1 with dict dms, advanced one pass, 38294
github.tar, level 1 with dict dds, advanced one pass, 38294
github.tar, level 1 with dict copy, advanced one pass, 38089
github.tar, level 1 with dict, advanced one pass, 38119
github.tar, level 1 with dict dms, advanced one pass, 38406
github.tar, level 1 with dict dds, advanced one pass, 38406
github.tar, level 1 with dict copy, advanced one pass, 38119
github.tar, level 1 with dict load, advanced one pass, 38364
github.tar, level 3, advanced one pass, 38831
github.tar, level 3 with dict, advanced one pass, 37995
@@ -452,64 +452,64 @@ github.tar, level 4 with dict dds, advanced
github.tar, level 4 with dict copy, advanced one pass, 37948
github.tar, level 4 with dict load, advanced one pass, 37927
github.tar, level 5 row 1, advanced one pass, 39651
github.tar, level 5 row 1 with dict dms, advanced one pass, 39059
github.tar, level 5 row 1 with dict dds, advanced one pass, 39067
github.tar, level 5 row 1 with dict copy, advanced one pass, 39082
github.tar, level 5 row 1 with dict load, advanced one pass, 38999
github.tar, level 5 row 1 with dict dms, advanced one pass, 39043
github.tar, level 5 row 1 with dict dds, advanced one pass, 39069
github.tar, level 5 row 1 with dict copy, advanced one pass, 39145
github.tar, level 5 row 1 with dict load, advanced one pass, 39000
github.tar, level 5 row 2, advanced one pass, 39701
github.tar, level 5 row 2 with dict dms, advanced one pass, 39365
github.tar, level 5 row 2 with dict dds, advanced one pass, 39233
github.tar, level 5 row 2 with dict copy, advanced one pass, 39715
github.tar, level 5 row 2 with dict load, advanced one pass, 39158
github.tar, level 5, advanced one pass, 39651
github.tar, level 5 with dict, advanced one pass, 39082
github.tar, level 5 with dict dms, advanced one pass, 39059
github.tar, level 5 with dict dds, advanced one pass, 39067
github.tar, level 5 with dict copy, advanced one pass, 39082
github.tar, level 5 with dict load, advanced one pass, 38999
github.tar, level 5 with dict, advanced one pass, 39145
github.tar, level 5 with dict dms, advanced one pass, 39043
github.tar, level 5 with dict dds, advanced one pass, 39069
github.tar, level 5 with dict copy, advanced one pass, 39145
github.tar, level 5 with dict load, advanced one pass, 39000
github.tar, level 6, advanced one pass, 39282
github.tar, level 6 with dict, advanced one pass, 38656
github.tar, level 6 with dict dms, advanced one pass, 38636
github.tar, level 6 with dict dds, advanced one pass, 38634
github.tar, level 6 with dict dms, advanced one pass, 38640
github.tar, level 6 with dict dds, advanced one pass, 38643
github.tar, level 6 with dict copy, advanced one pass, 38656
github.tar, level 6 with dict load, advanced one pass, 38648
github.tar, level 7 row 1, advanced one pass, 38110
github.tar, level 7 row 1 with dict dms, advanced one pass, 37858
github.tar, level 7 row 1 with dict dds, advanced one pass, 37882
github.tar, level 7 row 1 with dict copy, advanced one pass, 37865
github.tar, level 7 row 1 with dict load, advanced one pass, 37436
github.tar, level 6 with dict load, advanced one pass, 38647
github.tar, level 7 row 1, advanced one pass, 38005
github.tar, level 7 row 1 with dict dms, advanced one pass, 37832
github.tar, level 7 row 1 with dict dds, advanced one pass, 37857
github.tar, level 7 row 1 with dict copy, advanced one pass, 37839
github.tar, level 7 row 1 with dict load, advanced one pass, 37286
github.tar, level 7 row 2, advanced one pass, 38077
github.tar, level 7 row 2 with dict dms, advanced one pass, 38012
github.tar, level 7 row 2 with dict dds, advanced one pass, 38014
github.tar, level 7 row 2 with dict copy, advanced one pass, 38101
github.tar, level 7 row 2 with dict load, advanced one pass, 37402
github.tar, level 7, advanced one pass, 38110
github.tar, level 7 with dict, advanced one pass, 37865
github.tar, level 7 with dict dms, advanced one pass, 37858
github.tar, level 7 with dict dds, advanced one pass, 37882
github.tar, level 7 with dict copy, advanced one pass, 37865
github.tar, level 7 with dict load, advanced one pass, 37436
github.tar, level 9, advanced one pass, 36760
github.tar, level 9 with dict, advanced one pass, 36484
github.tar, level 9 with dict dms, advanced one pass, 36567
github.tar, level 9 with dict dds, advanced one pass, 36628
github.tar, level 9 with dict copy, advanced one pass, 36484
github.tar, level 9 with dict load, advanced one pass, 36401
github.tar, level 11 row 1, advanced one pass, 36081
github.tar, level 7, advanced one pass, 38005
github.tar, level 7 with dict, advanced one pass, 37839
github.tar, level 7 with dict dms, advanced one pass, 37832
github.tar, level 7 with dict dds, advanced one pass, 37857
github.tar, level 7 with dict copy, advanced one pass, 37839
github.tar, level 7 with dict load, advanced one pass, 37286
github.tar, level 9, advanced one pass, 36723
github.tar, level 9 with dict, advanced one pass, 36531
github.tar, level 9 with dict dms, advanced one pass, 36615
github.tar, level 9 with dict dds, advanced one pass, 36682
github.tar, level 9 with dict copy, advanced one pass, 36531
github.tar, level 9 with dict load, advanced one pass, 36322
github.tar, level 11 row 1, advanced one pass, 36085
github.tar, level 11 row 1 with dict dms, advanced one pass, 36963
github.tar, level 11 row 1 with dict dds, advanced one pass, 36963
github.tar, level 11 row 1 with dict copy, advanced one pass, 36557
github.tar, level 11 row 1 with dict load, advanced one pass, 36434
github.tar, level 11 row 1 with dict load, advanced one pass, 36423
github.tar, level 11 row 2, advanced one pass, 36110
github.tar, level 11 row 2 with dict dms, advanced one pass, 36963
github.tar, level 11 row 2 with dict dds, advanced one pass, 36963
github.tar, level 11 row 2 with dict copy, advanced one pass, 36557
github.tar, level 11 row 2 with dict load, advanced one pass, 36459
github.tar, level 12 row 1, advanced one pass, 36081
github.tar, level 12 row 1, advanced one pass, 36085
github.tar, level 12 row 1 with dict dms, advanced one pass, 36986
github.tar, level 12 row 1 with dict dds, advanced one pass, 36986
github.tar, level 12 row 1 with dict copy, advanced one pass, 36609
github.tar, level 12 row 1 with dict load, advanced one pass, 36434
github.tar, level 12 row 1 with dict load, advanced one pass, 36423
github.tar, level 12 row 2, advanced one pass, 36110
github.tar, level 12 row 2 with dict dms, advanced one pass, 36986
github.tar, level 12 row 2 with dict dds, advanced one pass, 36986
@@ -553,18 +553,18 @@ silesia, level 0, advanced
silesia, level 1, advanced one pass small out, 5306632
silesia, level 3, advanced one pass small out, 4842075
silesia, level 4, advanced one pass small out, 4779186
silesia, level 5 row 1, advanced one pass small out, 4666323
silesia, level 5 row 2, advanced one pass small out, 4670136
silesia, level 5, advanced one pass small out, 4666323
silesia, level 6, advanced one pass small out, 4603066
silesia, level 7 row 1, advanced one pass small out, 4566984
silesia, level 7 row 2, advanced one pass small out, 4564868
silesia, level 7, advanced one pass small out, 4566984
silesia, level 9, advanced one pass small out, 4543018
silesia, level 11 row 1, advanced one pass small out, 4505046
silesia, level 11 row 2, advanced one pass small out, 4503116
silesia, level 12 row 1, advanced one pass small out, 4505046
silesia, level 12 row 2, advanced one pass small out, 4503116
silesia, level 5 row 1, advanced one pass small out, 4667668
silesia, level 5 row 2, advanced one pass small out, 4670326
silesia, level 5, advanced one pass small out, 4667668
silesia, level 6, advanced one pass small out, 4604351
silesia, level 7 row 1, advanced one pass small out, 4570271
silesia, level 7 row 2, advanced one pass small out, 4565169
silesia, level 7, advanced one pass small out, 4570271
silesia, level 9, advanced one pass small out, 4545850
silesia, level 11 row 1, advanced one pass small out, 4505658
silesia, level 11 row 2, advanced one pass small out, 4503429
silesia, level 12 row 1, advanced one pass small out, 4505658
silesia, level 12 row 2, advanced one pass small out, 4503429
silesia, level 13, advanced one pass small out, 4493990
silesia, level 16, advanced one pass small out, 4360041
silesia, level 19, advanced one pass small out, 4296055
@@ -575,7 +575,7 @@ silesia, multithreaded long distance mode, advanced
silesia, small window log, advanced one pass small out, 7095000
silesia, small hash log, advanced one pass small out, 6526141
silesia, small chain log, advanced one pass small out, 4912197
silesia, explicit params, advanced one pass small out, 4795432
silesia, explicit params, advanced one pass small out, 4795840
silesia, uncompressed literals, advanced one pass small out, 5120566
silesia, uncompressed literals optimal, advanced one pass small out, 4319518
silesia, huffman literals, advanced one pass small out, 5321369
@@ -587,18 +587,18 @@ silesia.tar, level 0, advanced
silesia.tar, level 1, advanced one pass small out, 5327717
silesia.tar, level 3, advanced one pass small out, 4854086
silesia.tar, level 4, advanced one pass small out, 4791503
silesia.tar, level 5 row 1, advanced one pass small out, 4677740
silesia.tar, level 5 row 2, advanced one pass small out, 4682161
silesia.tar, level 5, advanced one pass small out, 4677740
silesia.tar, level 6, advanced one pass small out, 4613242
silesia.tar, level 7 row 1, advanced one pass small out, 4576661
silesia.tar, level 7 row 2, advanced one pass small out, 4575393
silesia.tar, level 7, advanced one pass small out, 4576661
silesia.tar, level 9, advanced one pass small out, 4552899
silesia.tar, level 11 row 1, advanced one pass small out, 4514432
silesia.tar, level 11 row 2, advanced one pass small out, 4513604
silesia.tar, level 12 row 1, advanced one pass small out, 4514049
silesia.tar, level 12 row 2, advanced one pass small out, 4513797
silesia.tar, level 5 row 1, advanced one pass small out, 4679004
silesia.tar, level 5 row 2, advanced one pass small out, 4682334
silesia.tar, level 5, advanced one pass small out, 4679004
silesia.tar, level 6, advanced one pass small out, 4614561
silesia.tar, level 7 row 1, advanced one pass small out, 4579828
silesia.tar, level 7 row 2, advanced one pass small out, 4575602
silesia.tar, level 7, advanced one pass small out, 4579828
silesia.tar, level 9, advanced one pass small out, 4555448
silesia.tar, level 11 row 1, advanced one pass small out, 4514962
silesia.tar, level 11 row 2, advanced one pass small out, 4513816
silesia.tar, level 12 row 1, advanced one pass small out, 4514517
silesia.tar, level 12 row 2, advanced one pass small out, 4514007
silesia.tar, level 13, advanced one pass small out, 4502956
silesia.tar, level 16, advanced one pass small out, 4360546
silesia.tar, level 19, advanced one pass small out, 4265911
@@ -609,105 +609,105 @@ silesia.tar, multithreaded long distance mode, advanced
silesia.tar, small window log, advanced one pass small out, 7100655
silesia.tar, small hash log, advanced one pass small out, 6529206
silesia.tar, small chain log, advanced one pass small out, 4917041
silesia.tar, explicit params, advanced one pass small out, 4806855
silesia.tar, explicit params, advanced one pass small out, 4807274
silesia.tar, uncompressed literals, advanced one pass small out, 5122473
silesia.tar, uncompressed literals optimal, advanced one pass small out, 4310141
silesia.tar, huffman literals, advanced one pass small out, 5341705
silesia.tar, multithreaded with advanced params, advanced one pass small out, 5122567
github, level -5, advanced one pass small out, 204407
github, level -5 with dict, advanced one pass small out, 46718
github, level -5 with dict, advanced one pass small out, 45832
github, level -3, advanced one pass small out, 193253
github, level -3 with dict, advanced one pass small out, 45395
github, level -3 with dict, advanced one pass small out, 44671
github, level -1, advanced one pass small out, 175468
github, level -1 with dict, advanced one pass small out, 43170
github, level -1 with dict, advanced one pass small out, 41825
github, level 0, advanced one pass small out, 136332
github, level 0 with dict, advanced one pass small out, 41148
github, level 0 with dict dms, advanced one pass small out, 41148
github, level 0 with dict dds, advanced one pass small out, 41148
github, level 0 with dict copy, advanced one pass small out, 41124
github, level 0 with dict load, advanced one pass small out, 42252
github, level 0 with dict load, advanced one pass small out, 41847
github, level 1, advanced one pass small out, 142365
github, level 1 with dict, advanced one pass small out, 41682
github, level 1 with dict dms, advanced one pass small out, 41682
github, level 1 with dict dds, advanced one pass small out, 41682
github, level 1 with dict copy, advanced one pass small out, 41698
github, level 1 with dict load, advanced one pass small out, 43814
github, level 1 with dict, advanced one pass small out, 41266
github, level 1 with dict dms, advanced one pass small out, 41266
github, level 1 with dict dds, advanced one pass small out, 41266
github, level 1 with dict copy, advanced one pass small out, 41279
github, level 1 with dict load, advanced one pass small out, 43331
github, level 3, advanced one pass small out, 136332
github, level 3 with dict, advanced one pass small out, 41148
github, level 3 with dict dms, advanced one pass small out, 41148
github, level 3 with dict dds, advanced one pass small out, 41148
github, level 3 with dict copy, advanced one pass small out, 41124
github, level 3 with dict load, advanced one pass small out, 42252
github, level 3 with dict load, advanced one pass small out, 41847
github, level 4, advanced one pass small out, 136199
github, level 4 with dict, advanced one pass small out, 41251
github, level 4 with dict dms, advanced one pass small out, 41251
github, level 4 with dict dds, advanced one pass small out, 41251
github, level 4 with dict copy, advanced one pass small out, 41216
github, level 4 with dict load, advanced one pass small out, 41159
github, level 4 with dict load, advanced one pass small out, 41548
github, level 5 row 1, advanced one pass small out, 134584
github, level 5 row 1 with dict dms, advanced one pass small out, 38758
github, level 5 row 1 with dict dms, advanced one pass small out, 38754
github, level 5 row 1 with dict dds, advanced one pass small out, 38728
github, level 5 row 1 with dict copy, advanced one pass small out, 38759
github, level 5 row 1 with dict load, advanced one pass small out, 41518
github, level 5 row 1 with dict copy, advanced one pass small out, 38755
github, level 5 row 1 with dict load, advanced one pass small out, 41899
github, level 5 row 2, advanced one pass small out, 135121
github, level 5 row 2 with dict dms, advanced one pass small out, 38938
github, level 5 row 2 with dict dds, advanced one pass small out, 38732
github, level 5 row 2 with dict copy, advanced one pass small out, 38934
github, level 5 row 2 with dict load, advanced one pass small out, 40725
github, level 5 row 2 with dict load, advanced one pass small out, 41248
github, level 5, advanced one pass small out, 135121
github, level 5 with dict, advanced one pass small out, 38758
github, level 5 with dict dms, advanced one pass small out, 38758
github, level 5 with dict, advanced one pass small out, 38754
github, level 5 with dict dms, advanced one pass small out, 38754
github, level 5 with dict dds, advanced one pass small out, 38728
github, level 5 with dict copy, advanced one pass small out, 38759
github, level 5 with dict load, advanced one pass small out, 40725
github, level 5 with dict copy, advanced one pass small out, 38755
github, level 5 with dict load, advanced one pass small out, 41248
github, level 6, advanced one pass small out, 135122
github, level 6 with dict, advanced one pass small out, 38671
github, level 6 with dict dms, advanced one pass small out, 38671
github, level 6 with dict dds, advanced one pass small out, 38636
github, level 6 with dict copy, advanced one pass small out, 38669
github, level 6 with dict load, advanced one pass small out, 40695
github, level 6 with dict, advanced one pass small out, 38669
github, level 6 with dict dms, advanced one pass small out, 38669
github, level 6 with dict dds, advanced one pass small out, 38638
github, level 6 with dict copy, advanced one pass small out, 38665
github, level 6 with dict load, advanced one pass small out, 41153
github, level 7 row 1, advanced one pass small out, 134584
github, level 7 row 1 with dict dms, advanced one pass small out, 38758
github, level 7 row 1 with dict dds, advanced one pass small out, 38745
github, level 7 row 1 with dict copy, advanced one pass small out, 38755
github, level 7 row 1 with dict load, advanced one pass small out, 43154
github, level 7 row 1 with dict dms, advanced one pass small out, 38765
github, level 7 row 1 with dict dds, advanced one pass small out, 38749
github, level 7 row 1 with dict copy, advanced one pass small out, 38759
github, level 7 row 1 with dict load, advanced one pass small out, 43227
github, level 7 row 2, advanced one pass small out, 135122
github, level 7 row 2 with dict dms, advanced one pass small out, 38860
github, level 7 row 2 with dict dds, advanced one pass small out, 38766
github, level 7 row 2 with dict copy, advanced one pass small out, 38834
github, level 7 row 2 with dict load, advanced one pass small out, 40695
github, level 7 row 2 with dict load, advanced one pass small out, 41153
github, level 7, advanced one pass small out, 135122
github, level 7 with dict, advanced one pass small out, 38758
github, level 7 with dict dms, advanced one pass small out, 38758
github, level 7 with dict dds, advanced one pass small out, 38745
github, level 7 with dict copy, advanced one pass small out, 38755
github, level 7 with dict load, advanced one pass small out, 40695
github, level 7 with dict, advanced one pass small out, 38765
github, level 7 with dict dms, advanced one pass small out, 38765
github, level 7 with dict dds, advanced one pass small out, 38749
github, level 7 with dict copy, advanced one pass small out, 38759
github, level 7 with dict load, advanced one pass small out, 41153
github, level 9, advanced one pass small out, 135122
github, level 9 with dict, advanced one pass small out, 39437
github, level 9 with dict dms, advanced one pass small out, 39437
github, level 9 with dict, advanced one pass small out, 39439
github, level 9 with dict dms, advanced one pass small out, 39439
github, level 9 with dict dds, advanced one pass small out, 39393
github, level 9 with dict copy, advanced one pass small out, 39398
github, level 9 with dict load, advanced one pass small out, 41710
github, level 9 with dict copy, advanced one pass small out, 39362
github, level 9 with dict load, advanced one pass small out, 42148
github, level 11 row 1, advanced one pass small out, 135367
github, level 11 row 1 with dict dms, advanced one pass small out, 39671
github, level 11 row 1 with dict dds, advanced one pass small out, 39671
github, level 11 row 1 with dict copy, advanced one pass small out, 39651
github, level 11 row 1 with dict load, advanced one pass small out, 41360
github, level 11 row 1 with dict load, advanced one pass small out, 41744
github, level 11 row 2, advanced one pass small out, 135367
github, level 11 row 2 with dict dms, advanced one pass small out, 39671
github, level 11 row 2 with dict dds, advanced one pass small out, 39671
github, level 11 row 2 with dict copy, advanced one pass small out, 39651
github, level 11 row 2 with dict load, advanced one pass small out, 41360
github, level 11 row 2 with dict load, advanced one pass small out, 41744
github, level 12 row 1, advanced one pass small out, 134402
github, level 12 row 1 with dict dms, advanced one pass small out, 39677
github, level 12 row 1 with dict dds, advanced one pass small out, 39677
github, level 12 row 1 with dict copy, advanced one pass small out, 39677
github, level 12 row 1 with dict load, advanced one pass small out, 41166
github, level 12 row 1 with dict load, advanced one pass small out, 41553
github, level 12 row 2, advanced one pass small out, 134402
github, level 12 row 2 with dict dms, advanced one pass small out, 39677
github, level 12 row 2 with dict dds, advanced one pass small out, 39677
github, level 12 row 2 with dict copy, advanced one pass small out, 39677
github, level 12 row 2 with dict load, advanced one pass small out, 41166
github, level 12 row 2 with dict load, advanced one pass small out, 41553
github, level 13, advanced one pass small out, 132878
github, level 13 with dict, advanced one pass small out, 39900
github, level 13 with dict dms, advanced one pass small out, 39900
@@ -740,11 +740,11 @@ github, uncompressed literals optimal, advanced
github, huffman literals, advanced one pass small out, 142365
github, multithreaded with advanced params, advanced one pass small out, 165911
github.tar, level -5, advanced one pass small out, 52115
github.tar, level -5 with dict, advanced one pass small out, 50974
github.tar, level -5 with dict, advanced one pass small out, 51097
github.tar, level -3, advanced one pass small out, 45678
github.tar, level -3 with dict, advanced one pass small out, 44656
github.tar, level -3 with dict, advanced one pass small out, 44734
github.tar, level -1, advanced one pass small out, 42560
github.tar, level -1 with dict, advanced one pass small out, 41151
github.tar, level -1 with dict, advanced one pass small out, 41353
github.tar, level 0, advanced one pass small out, 38831
github.tar, level 0 with dict, advanced one pass small out, 37995
github.tar, level 0 with dict dms, advanced one pass small out, 38003
@@ -752,10 +752,10 @@ github.tar, level 0 with dict dds, advanced
github.tar, level 0 with dict copy, advanced one pass small out, 37995
github.tar, level 0 with dict load, advanced one pass small out, 37956
github.tar, level 1, advanced one pass small out, 39200
github.tar, level 1 with dict, advanced one pass small out, 38089
github.tar, level 1 with dict dms, advanced one pass small out, 38294
github.tar, level 1 with dict dds, advanced one pass small out, 38294
github.tar, level 1 with dict copy, advanced one pass small out, 38089
github.tar, level 1 with dict, advanced one pass small out, 38119
github.tar, level 1 with dict dms, advanced one pass small out, 38406
github.tar, level 1 with dict dds, advanced one pass small out, 38406
github.tar, level 1 with dict copy, advanced one pass small out, 38119
github.tar, level 1 with dict load, advanced one pass small out, 38364
github.tar, level 3, advanced one pass small out, 38831
github.tar, level 3 with dict, advanced one pass small out, 37995
@@ -770,64 +770,64 @@ github.tar, level 4 with dict dds, advanced
github.tar, level 4 with dict copy, advanced one pass small out, 37948
github.tar, level 4 with dict load, advanced one pass small out, 37927
github.tar, level 5 row 1, advanced one pass small out, 39651
github.tar, level 5 row 1 with dict dms, advanced one pass small out, 39059
github.tar, level 5 row 1 with dict dds, advanced one pass small out, 39067
github.tar, level 5 row 1 with dict copy, advanced one pass small out, 39082
github.tar, level 5 row 1 with dict load, advanced one pass small out, 38999
github.tar, level 5 row 1 with dict dms, advanced one pass small out, 39043
github.tar, level 5 row 1 with dict dds, advanced one pass small out, 39069
github.tar, level 5 row 1 with dict copy, advanced one pass small out, 39145
github.tar, level 5 row 1 with dict load, advanced one pass small out, 39000
github.tar, level 5 row 2, advanced one pass small out, 39701
github.tar, level 5 row 2 with dict dms, advanced one pass small out, 39365
github.tar, level 5 row 2 with dict dds, advanced one pass small out, 39233
github.tar, level 5 row 2 with dict copy, advanced one pass small out, 39715
github.tar, level 5 row 2 with dict load, advanced one pass small out, 39158
github.tar, level 5, advanced one pass small out, 39651
github.tar, level 5 with dict, advanced one pass small out, 39082
github.tar, level 5 with dict dms, advanced one pass small out, 39059
github.tar, level 5 with dict dds, advanced one pass small out, 39067
github.tar, level 5 with dict copy, advanced one pass small out, 39082
github.tar, level 5 with dict load, advanced one pass small out, 38999
github.tar, level 5 with dict, advanced one pass small out, 39145
github.tar, level 5 with dict dms, advanced one pass small out, 39043
github.tar, level 5 with dict dds, advanced one pass small out, 39069
github.tar, level 5 with dict copy, advanced one pass small out, 39145
github.tar, level 5 with dict load, advanced one pass small out, 39000
github.tar, level 6, advanced one pass small out, 39282
github.tar, level 6 with dict, advanced one pass small out, 38656
github.tar, level 6 with dict dms, advanced one pass small out, 38636
github.tar, level 6 with dict dds, advanced one pass small out, 38634
github.tar, level 6 with dict dms, advanced one pass small out, 38640
github.tar, level 6 with dict dds, advanced one pass small out, 38643
github.tar, level 6 with dict copy, advanced one pass small out, 38656
github.tar, level 6 with dict load, advanced one pass small out, 38648
github.tar, level 7 row 1, advanced one pass small out, 38110
github.tar, level 7 row 1 with dict dms, advanced one pass small out, 37858
github.tar, level 7 row 1 with dict dds, advanced one pass small out, 37882
github.tar, level 7 row 1 with dict copy, advanced one pass small out, 37865
github.tar, level 7 row 1 with dict load, advanced one pass small out, 37436
github.tar, level 6 with dict load, advanced one pass small out, 38647
github.tar, level 7 row 1, advanced one pass small out, 38005
github.tar, level 7 row 1 with dict dms, advanced one pass small out, 37832
github.tar, level 7 row 1 with dict dds, advanced one pass small out, 37857
github.tar, level 7 row 1 with dict copy, advanced one pass small out, 37839
github.tar, level 7 row 1 with dict load, advanced one pass small out, 37286
github.tar, level 7 row 2, advanced one pass small out, 38077
github.tar, level 7 row 2 with dict dms, advanced one pass small out, 38012
github.tar, level 7 row 2 with dict dds, advanced one pass small out, 38014
github.tar, level 7 row 2 with dict copy, advanced one pass small out, 38101
github.tar, level 7 row 2 with dict load, advanced one pass small out, 37402
github.tar, level 7, advanced one pass small out, 38110
github.tar, level 7 with dict, advanced one pass small out, 37865
github.tar, level 7 with dict dms, advanced one pass small out, 37858
github.tar, level 7 with dict dds, advanced one pass small out, 37882
github.tar, level 7 with dict copy, advanced one pass small out, 37865
github.tar, level 7 with dict load, advanced one pass small out, 37436
github.tar, level 9, advanced one pass small out, 36760
github.tar, level 9 with dict, advanced one pass small out, 36484
github.tar, level 9 with dict dms, advanced one pass small out, 36567
github.tar, level 9 with dict dds, advanced one pass small out, 36628
github.tar, level 9 with dict copy, advanced one pass small out, 36484
github.tar, level 9 with dict load, advanced one pass small out, 36401
github.tar, level 11 row 1, advanced one pass small out, 36081
github.tar, level 7, advanced one pass small out, 38005
github.tar, level 7 with dict, advanced one pass small out, 37839
github.tar, level 7 with dict dms, advanced one pass small out, 37832
github.tar, level 7 with dict dds, advanced one pass small out, 37857
github.tar, level 7 with dict copy, advanced one pass small out, 37839
github.tar, level 7 with dict load, advanced one pass small out, 37286
github.tar, level 9, advanced one pass small out, 36723
github.tar, level 9 with dict, advanced one pass small out, 36531
github.tar, level 9 with dict dms, advanced one pass small out, 36615
github.tar, level 9 with dict dds, advanced one pass small out, 36682
github.tar, level 9 with dict copy, advanced one pass small out, 36531
github.tar, level 9 with dict load, advanced one pass small out, 36322
github.tar, level 11 row 1, advanced one pass small out, 36085
github.tar, level 11 row 1 with dict dms, advanced one pass small out, 36963
github.tar, level 11 row 1 with dict dds, advanced one pass small out, 36963
github.tar, level 11 row 1 with dict copy, advanced one pass small out, 36557
github.tar, level 11 row 1 with dict load, advanced one pass small out, 36434
github.tar, level 11 row 1 with dict load, advanced one pass small out, 36423
github.tar, level 11 row 2, advanced one pass small out, 36110
github.tar, level 11 row 2 with dict dms, advanced one pass small out, 36963
github.tar, level 11 row 2 with dict dds, advanced one pass small out, 36963
github.tar, level 11 row 2 with dict copy, advanced one pass small out, 36557
github.tar, level 11 row 2 with dict load, advanced one pass small out, 36459
github.tar, level 12 row 1, advanced one pass small out, 36081
github.tar, level 12 row 1, advanced one pass small out, 36085
github.tar, level 12 row 1 with dict dms, advanced one pass small out, 36986
github.tar, level 12 row 1 with dict dds, advanced one pass small out, 36986
github.tar, level 12 row 1 with dict copy, advanced one pass small out, 36609
github.tar, level 12 row 1 with dict load, advanced one pass small out, 36434
github.tar, level 12 row 1 with dict load, advanced one pass small out, 36423
github.tar, level 12 row 2, advanced one pass small out, 36110
github.tar, level 12 row 2 with dict dms, advanced one pass small out, 36986
github.tar, level 12 row 2 with dict dds, advanced one pass small out, 36986
@@ -871,18 +871,18 @@ silesia, level 0, advanced
silesia, level 1, advanced streaming, 5306388
silesia, level 3, advanced streaming, 4842075
silesia, level 4, advanced streaming, 4779186
silesia, level 5 row 1, advanced streaming, 4666323
silesia, level 5 row 2, advanced streaming, 4670136
silesia, level 5, advanced streaming, 4666323
silesia, level 6, advanced streaming, 4603066
silesia, level 7 row 1, advanced streaming, 4566984
silesia, level 7 row 2, advanced streaming, 4564868
silesia, level 7, advanced streaming, 4566984
silesia, level 9, advanced streaming, 4543018
silesia, level 11 row 1, advanced streaming, 4505046
silesia, level 11 row 2, advanced streaming, 4503116
silesia, level 12 row 1, advanced streaming, 4505046
silesia, level 12 row 2, advanced streaming, 4503116
silesia, level 5 row 1, advanced streaming, 4667668
silesia, level 5 row 2, advanced streaming, 4670326
silesia, level 5, advanced streaming, 4667668
silesia, level 6, advanced streaming, 4604351
silesia, level 7 row 1, advanced streaming, 4570271
silesia, level 7 row 2, advanced streaming, 4565169
silesia, level 7, advanced streaming, 4570271
silesia, level 9, advanced streaming, 4545850
silesia, level 11 row 1, advanced streaming, 4505658
silesia, level 11 row 2, advanced streaming, 4503429
silesia, level 12 row 1, advanced streaming, 4505658
silesia, level 12 row 2, advanced streaming, 4503429
silesia, level 13, advanced streaming, 4493990
silesia, level 16, advanced streaming, 4360041
silesia, level 19, advanced streaming, 4296055
@@ -893,7 +893,7 @@ silesia, multithreaded long distance mode, advanced
silesia, small window log, advanced streaming, 7111103
silesia, small hash log, advanced streaming, 6526141
silesia, small chain log, advanced streaming, 4912197
silesia, explicit params, advanced streaming, 4795452
silesia, explicit params, advanced streaming, 4795857
silesia, uncompressed literals, advanced streaming, 5120566
silesia, uncompressed literals optimal, advanced streaming, 4319518
silesia, huffman literals, advanced streaming, 5321370
@@ -905,18 +905,18 @@ silesia.tar, level 0, advanced
silesia.tar, level 1, advanced streaming, 5327708
silesia.tar, level 3, advanced streaming, 4859271
silesia.tar, level 4, advanced streaming, 4797470
silesia.tar, level 5 row 1, advanced streaming, 4677748
silesia.tar, level 5 row 2, advanced streaming, 4682169
silesia.tar, level 5, advanced streaming, 4677748
silesia.tar, level 6, advanced streaming, 4613246
silesia.tar, level 7 row 1, advanced streaming, 4576664
silesia.tar, level 7 row 2, advanced streaming, 4575394
silesia.tar, level 7, advanced streaming, 4576664
silesia.tar, level 9, advanced streaming, 4552900
silesia.tar, level 11 row 1, advanced streaming, 4514433
silesia.tar, level 11 row 2, advanced streaming, 4513604
silesia.tar, level 12 row 1, advanced streaming, 4514049
silesia.tar, level 12 row 2, advanced streaming, 4513797
silesia.tar, level 5 row 1, advanced streaming, 4679020
silesia.tar, level 5 row 2, advanced streaming, 4682355
silesia.tar, level 5, advanced streaming, 4679020
silesia.tar, level 6, advanced streaming, 4614558
silesia.tar, level 7 row 1, advanced streaming, 4579823
silesia.tar, level 7 row 2, advanced streaming, 4575601
silesia.tar, level 7, advanced streaming, 4579823
silesia.tar, level 9, advanced streaming, 4555445
silesia.tar, level 11 row 1, advanced streaming, 4514959
silesia.tar, level 11 row 2, advanced streaming, 4513810
silesia.tar, level 12 row 1, advanced streaming, 4514514
silesia.tar, level 12 row 2, advanced streaming, 4514003
silesia.tar, level 13, advanced streaming, 4502956
silesia.tar, level 16, advanced streaming, 4360546
silesia.tar, level 19, advanced streaming, 4265911
@@ -927,105 +927,105 @@ silesia.tar, multithreaded long distance mode, advanced
silesia.tar, small window log, advanced streaming, 7117559
silesia.tar, small hash log, advanced streaming, 6529209
silesia.tar, small chain log, advanced streaming, 4917021
silesia.tar, explicit params, advanced streaming, 4806873
silesia.tar, explicit params, advanced streaming, 4807288
silesia.tar, uncompressed literals, advanced streaming, 5127423
silesia.tar, uncompressed literals optimal, advanced streaming, 4310141
silesia.tar, huffman literals, advanced streaming, 5341712
silesia.tar, multithreaded with advanced params, advanced streaming, 5122567
github, level -5, advanced streaming, 204407
github, level -5 with dict, advanced streaming, 46718
github, level -5 with dict, advanced streaming, 45832
github, level -3, advanced streaming, 193253
github, level -3 with dict, advanced streaming, 45395
github, level -3 with dict, advanced streaming, 44671
github, level -1, advanced streaming, 175468
github, level -1 with dict, advanced streaming, 43170
github, level -1 with dict, advanced streaming, 41825
github, level 0, advanced streaming, 136332
github, level 0 with dict, advanced streaming, 41148
github, level 0 with dict dms, advanced streaming, 41148
github, level 0 with dict dds, advanced streaming, 41148
github, level 0 with dict copy, advanced streaming, 41124
github, level 0 with dict load, advanced streaming, 42252
github, level 0 with dict load, advanced streaming, 41847
github, level 1, advanced streaming, 142365
github, level 1 with dict, advanced streaming, 41682
github, level 1 with dict dms, advanced streaming, 41682
github, level 1 with dict dds, advanced streaming, 41682
github, level 1 with dict copy, advanced streaming, 41698
github, level 1 with dict load, advanced streaming, 43814
github, level 1 with dict, advanced streaming, 41266
github, level 1 with dict dms, advanced streaming, 41266
github, level 1 with dict dds, advanced streaming, 41266
github, level 1 with dict copy, advanced streaming, 41279
github, level 1 with dict load, advanced streaming, 43331
github, level 3, advanced streaming, 136332
github, level 3 with dict, advanced streaming, 41148
github, level 3 with dict dms, advanced streaming, 41148
github, level 3 with dict dds, advanced streaming, 41148
github, level 3 with dict copy, advanced streaming, 41124
github, level 3 with dict load, advanced streaming, 42252
github, level 3 with dict load, advanced streaming, 41847
github, level 4, advanced streaming, 136199
github, level 4 with dict, advanced streaming, 41251
github, level 4 with dict dms, advanced streaming, 41251
github, level 4 with dict dds, advanced streaming, 41251
github, level 4 with dict copy, advanced streaming, 41216
github, level 4 with dict load, advanced streaming, 41159
github, level 4 with dict load, advanced streaming, 41548
github, level 5 row 1, advanced streaming, 134584
github, level 5 row 1 with dict dms, advanced streaming, 38758
github, level 5 row 1 with dict dms, advanced streaming, 38754
github, level 5 row 1 with dict dds, advanced streaming, 38728
github, level 5 row 1 with dict copy, advanced streaming, 38759
github, level 5 row 1 with dict load, advanced streaming, 41518
github, level 5 row 1 with dict copy, advanced streaming, 38755
github, level 5 row 1 with dict load, advanced streaming, 41899
github, level 5 row 2, advanced streaming, 135121
github, level 5 row 2 with dict dms, advanced streaming, 38938
github, level 5 row 2 with dict dds, advanced streaming, 38732
github, level 5 row 2 with dict copy, advanced streaming, 38934
github, level 5 row 2 with dict load, advanced streaming, 40725
github, level 5 row 2 with dict load, advanced streaming, 41248
github, level 5, advanced streaming, 135121
github, level 5 with dict, advanced streaming, 38758
github, level 5 with dict dms, advanced streaming, 38758
github, level 5 with dict, advanced streaming, 38754
github, level 5 with dict dms, advanced streaming, 38754
github, level 5 with dict dds, advanced streaming, 38728
github, level 5 with dict copy, advanced streaming, 38759
github, level 5 with dict load, advanced streaming, 40725
github, level 5 with dict copy, advanced streaming, 38755
github, level 5 with dict load, advanced streaming, 41248
github, level 6, advanced streaming, 135122
github, level 6 with dict, advanced streaming, 38671
github, level 6 with dict dms, advanced streaming, 38671
github, level 6 with dict dds, advanced streaming, 38636
github, level 6 with dict copy, advanced streaming, 38669
github, level 6 with dict load, advanced streaming, 40695
github, level 6 with dict, advanced streaming, 38669
github, level 6 with dict dms, advanced streaming, 38669
github, level 6 with dict dds, advanced streaming, 38638
github, level 6 with dict copy, advanced streaming, 38665
github, level 6 with dict load, advanced streaming, 41153
github, level 7 row 1, advanced streaming, 134584
github, level 7 row 1 with dict dms, advanced streaming, 38758
github, level 7 row 1 with dict dds, advanced streaming, 38745
github, level 7 row 1 with dict copy, advanced streaming, 38755
github, level 7 row 1 with dict load, advanced streaming, 43154
github, level 7 row 1 with dict dms, advanced streaming, 38765
github, level 7 row 1 with dict dds, advanced streaming, 38749
github, level 7 row 1 with dict copy, advanced streaming, 38759
github, level 7 row 1 with dict load, advanced streaming, 43227
github, level 7 row 2, advanced streaming, 135122
github, level 7 row 2 with dict dms, advanced streaming, 38860
github, level 7 row 2 with dict dds, advanced streaming, 38766
github, level 7 row 2 with dict copy, advanced streaming, 38834
github, level 7 row 2 with dict load, advanced streaming, 40695
github, level 7 row 2 with dict load, advanced streaming, 41153
github, level 7, advanced streaming, 135122
github, level 7 with dict, advanced streaming, 38758
github, level 7 with dict dms, advanced streaming, 38758
github, level 7 with dict dds, advanced streaming, 38745
github, level 7 with dict copy, advanced streaming, 38755
github, level 7 with dict load, advanced streaming, 40695
github, level 7 with dict, advanced streaming, 38765
github, level 7 with dict dms, advanced streaming, 38765
github, level 7 with dict dds, advanced streaming, 38749
github, level 7 with dict copy, advanced streaming, 38759
github, level 7 with dict load, advanced streaming, 41153
github, level 9, advanced streaming, 135122
github, level 9 with dict, advanced streaming, 39437
github, level 9 with dict dms, advanced streaming, 39437
github, level 9 with dict, advanced streaming, 39439
github, level 9 with dict dms, advanced streaming, 39439
github, level 9 with dict dds, advanced streaming, 39393
github, level 9 with dict copy, advanced streaming, 39398
github, level 9 with dict load, advanced streaming, 41710
github, level 9 with dict copy, advanced streaming, 39362
github, level 9 with dict load, advanced streaming, 42148
github, level 11 row 1, advanced streaming, 135367
github, level 11 row 1 with dict dms, advanced streaming, 39671
github, level 11 row 1 with dict dds, advanced streaming, 39671
github, level 11 row 1 with dict copy, advanced streaming, 39651
github, level 11 row 1 with dict load, advanced streaming, 41360
github, level 11 row 1 with dict load, advanced streaming, 41744
github, level 11 row 2, advanced streaming, 135367
github, level 11 row 2 with dict dms, advanced streaming, 39671
github, level 11 row 2 with dict dds, advanced streaming, 39671
github, level 11 row 2 with dict copy, advanced streaming, 39651
github, level 11 row 2 with dict load, advanced streaming, 41360
github, level 11 row 2 with dict load, advanced streaming, 41744
github, level 12 row 1, advanced streaming, 134402
github, level 12 row 1 with dict dms, advanced streaming, 39677
github, level 12 row 1 with dict dds, advanced streaming, 39677
github, level 12 row 1 with dict copy, advanced streaming, 39677
github, level 12 row 1 with dict load, advanced streaming, 41166
github, level 12 row 1 with dict load, advanced streaming, 41553
github, level 12 row 2, advanced streaming, 134402
github, level 12 row 2 with dict dms, advanced streaming, 39677
github, level 12 row 2 with dict dds, advanced streaming, 39677
github, level 12 row 2 with dict copy, advanced streaming, 39677
github, level 12 row 2 with dict load, advanced streaming, 41166
github, level 12 row 2 with dict load, advanced streaming, 41553
github, level 13, advanced streaming, 132878
github, level 13 with dict, advanced streaming, 39900
github, level 13 with dict dms, advanced streaming, 39900
@@ -1058,11 +1058,11 @@ github, uncompressed literals optimal, advanced
github, huffman literals, advanced streaming, 142365
github, multithreaded with advanced params, advanced streaming, 165911
github.tar, level -5, advanced streaming, 52152
github.tar, level -5 with dict, advanced streaming, 51045
github.tar, level -5 with dict, advanced streaming, 51181
github.tar, level -3, advanced streaming, 45678
github.tar, level -3 with dict, advanced streaming, 44656
github.tar, level -3 with dict, advanced streaming, 44734
github.tar, level -1, advanced streaming, 42560
github.tar, level -1 with dict, advanced streaming, 41151
github.tar, level -1 with dict, advanced streaming, 41353
github.tar, level 0, advanced streaming, 38831
github.tar, level 0 with dict, advanced streaming, 37995
github.tar, level 0 with dict dms, advanced streaming, 38003
@@ -1070,10 +1070,10 @@ github.tar, level 0 with dict dds, advanced
github.tar, level 0 with dict copy, advanced streaming, 37995
github.tar, level 0 with dict load, advanced streaming, 37956
github.tar, level 1, advanced streaming, 39200
github.tar, level 1 with dict, advanced streaming, 38089
github.tar, level 1 with dict dms, advanced streaming, 38294
github.tar, level 1 with dict dds, advanced streaming, 38294
github.tar, level 1 with dict copy, advanced streaming, 38089
github.tar, level 1 with dict, advanced streaming, 38119
github.tar, level 1 with dict dms, advanced streaming, 38406
github.tar, level 1 with dict dds, advanced streaming, 38406
github.tar, level 1 with dict copy, advanced streaming, 38119
github.tar, level 1 with dict load, advanced streaming, 38364
github.tar, level 3, advanced streaming, 38831
github.tar, level 3 with dict, advanced streaming, 37995
@@ -1088,64 +1088,64 @@ github.tar, level 4 with dict dds, advanced
github.tar, level 4 with dict copy, advanced streaming, 37948
github.tar, level 4 with dict load, advanced streaming, 37927
github.tar, level 5 row 1, advanced streaming, 39651
github.tar, level 5 row 1 with dict dms, advanced streaming, 39059
github.tar, level 5 row 1 with dict dds, advanced streaming, 39067
github.tar, level 5 row 1 with dict copy, advanced streaming, 39082
github.tar, level 5 row 1 with dict load, advanced streaming, 38999
github.tar, level 5 row 1 with dict dms, advanced streaming, 39043
github.tar, level 5 row 1 with dict dds, advanced streaming, 39069
github.tar, level 5 row 1 with dict copy, advanced streaming, 39145
github.tar, level 5 row 1 with dict load, advanced streaming, 39000
github.tar, level 5 row 2, advanced streaming, 39701
github.tar, level 5 row 2 with dict dms, advanced streaming, 39365
github.tar, level 5 row 2 with dict dds, advanced streaming, 39233
github.tar, level 5 row 2 with dict copy, advanced streaming, 39715
github.tar, level 5 row 2 with dict load, advanced streaming, 39158
github.tar, level 5, advanced streaming, 39651
github.tar, level 5 with dict, advanced streaming, 39082
github.tar, level 5 with dict dms, advanced streaming, 39059
github.tar, level 5 with dict dds, advanced streaming, 39067
github.tar, level 5 with dict copy, advanced streaming, 39082
github.tar, level 5 with dict load, advanced streaming, 38999
github.tar, level 5 with dict, advanced streaming, 39145
github.tar, level 5 with dict dms, advanced streaming, 39043
github.tar, level 5 with dict dds, advanced streaming, 39069
github.tar, level 5 with dict copy, advanced streaming, 39145
github.tar, level 5 with dict load, advanced streaming, 39000
github.tar, level 6, advanced streaming, 39282
github.tar, level 6 with dict, advanced streaming, 38656
github.tar, level 6 with dict dms, advanced streaming, 38636
github.tar, level 6 with dict dds, advanced streaming, 38634
github.tar, level 6 with dict dms, advanced streaming, 38640
github.tar, level 6 with dict dds, advanced streaming, 38643
github.tar, level 6 with dict copy, advanced streaming, 38656
github.tar, level 6 with dict load, advanced streaming, 38648
github.tar, level 7 row 1, advanced streaming, 38110
github.tar, level 7 row 1 with dict dms, advanced streaming, 37858
github.tar, level 7 row 1 with dict dds, advanced streaming, 37882
github.tar, level 7 row 1 with dict copy, advanced streaming, 37865
github.tar, level 7 row 1 with dict load, advanced streaming, 37436
github.tar, level 6 with dict load, advanced streaming, 38647
github.tar, level 7 row 1, advanced streaming, 38005
github.tar, level 7 row 1 with dict dms, advanced streaming, 37832
github.tar, level 7 row 1 with dict dds, advanced streaming, 37857
github.tar, level 7 row 1 with dict copy, advanced streaming, 37839
github.tar, level 7 row 1 with dict load, advanced streaming, 37286
github.tar, level 7 row 2, advanced streaming, 38077
github.tar, level 7 row 2 with dict dms, advanced streaming, 38012
github.tar, level 7 row 2 with dict dds, advanced streaming, 38014
github.tar, level 7 row 2 with dict copy, advanced streaming, 38101
github.tar, level 7 row 2 with dict load, advanced streaming, 37402
github.tar, level 7, advanced streaming, 38110
github.tar, level 7 with dict, advanced streaming, 37865
github.tar, level 7 with dict dms, advanced streaming, 37858
github.tar, level 7 with dict dds, advanced streaming, 37882
github.tar, level 7 with dict copy, advanced streaming, 37865
github.tar, level 7 with dict load, advanced streaming, 37436
github.tar, level 9, advanced streaming, 36760
github.tar, level 9 with dict, advanced streaming, 36484
github.tar, level 9 with dict dms, advanced streaming, 36567
github.tar, level 9 with dict dds, advanced streaming, 36628
github.tar, level 9 with dict copy, advanced streaming, 36484
github.tar, level 9 with dict load, advanced streaming, 36401
github.tar, level 11 row 1, advanced streaming, 36081
github.tar, level 7, advanced streaming, 38005
github.tar, level 7 with dict, advanced streaming, 37839
github.tar, level 7 with dict dms, advanced streaming, 37832
github.tar, level 7 with dict dds, advanced streaming, 37857
github.tar, level 7 with dict copy, advanced streaming, 37839
github.tar, level 7 with dict load, advanced streaming, 37286
github.tar, level 9, advanced streaming, 36723
github.tar, level 9 with dict, advanced streaming, 36531
github.tar, level 9 with dict dms, advanced streaming, 36615
github.tar, level 9 with dict dds, advanced streaming, 36682
github.tar, level 9 with dict copy, advanced streaming, 36531
github.tar, level 9 with dict load, advanced streaming, 36322
github.tar, level 11 row 1, advanced streaming, 36085
github.tar, level 11 row 1 with dict dms, advanced streaming, 36963
github.tar, level 11 row 1 with dict dds, advanced streaming, 36963
github.tar, level 11 row 1 with dict copy, advanced streaming, 36557
github.tar, level 11 row 1 with dict load, advanced streaming, 36434
github.tar, level 11 row 1 with dict load, advanced streaming, 36423
github.tar, level 11 row 2, advanced streaming, 36110
github.tar, level 11 row 2 with dict dms, advanced streaming, 36963
github.tar, level 11 row 2 with dict dds, advanced streaming, 36963
github.tar, level 11 row 2 with dict copy, advanced streaming, 36557
github.tar, level 11 row 2 with dict load, advanced streaming, 36459
github.tar, level 12 row 1, advanced streaming, 36081
github.tar, level 12 row 1, advanced streaming, 36085
github.tar, level 12 row 1 with dict dms, advanced streaming, 36986
github.tar, level 12 row 1 with dict dds, advanced streaming, 36986
github.tar, level 12 row 1 with dict copy, advanced streaming, 36609
github.tar, level 12 row 1 with dict load, advanced streaming, 36434
github.tar, level 12 row 1 with dict load, advanced streaming, 36423
github.tar, level 12 row 2, advanced streaming, 36110
github.tar, level 12 row 2 with dict dms, advanced streaming, 36986
github.tar, level 12 row 2 with dict dds, advanced streaming, 36986
@@ -1189,10 +1189,10 @@ silesia, level 0, old stre
silesia, level 1, old streaming, 5306388
silesia, level 3, old streaming, 4842075
silesia, level 4, old streaming, 4779186
silesia, level 5, old streaming, 4666323
silesia, level 6, old streaming, 4603066
silesia, level 7, old streaming, 4566984
silesia, level 9, old streaming, 4543018
silesia, level 5, old streaming, 4667668
silesia, level 6, old streaming, 4604351
silesia, level 7, old streaming, 4570271
silesia, level 9, old streaming, 4545850
silesia, level 13, old streaming, 4493990
silesia, level 16, old streaming, 4360041
silesia, level 19, old streaming, 4296055
@@ -1207,10 +1207,10 @@ silesia.tar, level 0, old stre
silesia.tar, level 1, old streaming, 5327708
silesia.tar, level 3, old streaming, 4859271
silesia.tar, level 4, old streaming, 4797470
silesia.tar, level 5, old streaming, 4677748
silesia.tar, level 6, old streaming, 4613246
silesia.tar, level 7, old streaming, 4576664
silesia.tar, level 9, old streaming, 4552900
silesia.tar, level 5, old streaming, 4679020
silesia.tar, level 6, old streaming, 4614558
silesia.tar, level 7, old streaming, 4579823
silesia.tar, level 9, old streaming, 4555445
silesia.tar, level 13, old streaming, 4502956
silesia.tar, level 16, old streaming, 4360546
silesia.tar, level 19, old streaming, 4265911
@@ -1219,27 +1219,27 @@ silesia.tar, uncompressed literals, old stre
silesia.tar, uncompressed literals optimal, old streaming, 4265911
silesia.tar, huffman literals, old streaming, 6179056
github, level -5, old streaming, 204407
github, level -5 with dict, old streaming, 46718
github, level -5 with dict, old streaming, 45832
github, level -3, old streaming, 193253
github, level -3 with dict, old streaming, 45395
github, level -3 with dict, old streaming, 44671
github, level -1, old streaming, 175468
github, level -1 with dict, old streaming, 43170
github, level -1 with dict, old streaming, 41825
github, level 0, old streaming, 136332
github, level 0 with dict, old streaming, 41148
github, level 1, old streaming, 142365
github, level 1 with dict, old streaming, 41682
github, level 1 with dict, old streaming, 41266
github, level 3, old streaming, 136332
github, level 3 with dict, old streaming, 41148
github, level 4, old streaming, 136199
github, level 4 with dict, old streaming, 41251
github, level 5, old streaming, 135121
github, level 5 with dict, old streaming, 38758
github, level 5 with dict, old streaming, 38754
github, level 6, old streaming, 135122
github, level 6 with dict, old streaming, 38671
github, level 6 with dict, old streaming, 38669
github, level 7, old streaming, 135122
github, level 7 with dict, old streaming, 38758
github, level 7 with dict, old streaming, 38765
github, level 9, old streaming, 135122
github, level 9 with dict, old streaming, 39437
github, level 9 with dict, old streaming, 39439
github, level 13, old streaming, 132878
github, level 13 with dict, old streaming, 39900
github, level 16, old streaming, 133209
@@ -1252,27 +1252,27 @@ github, uncompressed literals, old stre
github, uncompressed literals optimal, old streaming, 132879
github, huffman literals, old streaming, 175468
github.tar, level -5, old streaming, 52152
github.tar, level -5 with dict, old streaming, 51045
github.tar, level -5 with dict, old streaming, 51181
github.tar, level -3, old streaming, 45678
github.tar, level -3 with dict, old streaming, 44656
github.tar, level -3 with dict, old streaming, 44734
github.tar, level -1, old streaming, 42560
github.tar, level -1 with dict, old streaming, 41151
github.tar, level -1 with dict, old streaming, 41353
github.tar, level 0, old streaming, 38831
github.tar, level 0 with dict, old streaming, 37995
github.tar, level 1, old streaming, 39200
github.tar, level 1 with dict, old streaming, 38089
github.tar, level 1 with dict, old streaming, 38119
github.tar, level 3, old streaming, 38831
github.tar, level 3 with dict, old streaming, 37995
github.tar, level 4, old streaming, 38893
github.tar, level 4 with dict, old streaming, 37948
github.tar, level 5, old streaming, 39651
github.tar, level 5 with dict, old streaming, 39082
github.tar, level 5 with dict, old streaming, 39145
github.tar, level 6, old streaming, 39282
github.tar, level 6 with dict, old streaming, 38656
github.tar, level 7, old streaming, 38110
github.tar, level 7 with dict, old streaming, 37865
github.tar, level 9, old streaming, 36760
github.tar, level 9 with dict, old streaming, 36484
github.tar, level 7, old streaming, 38005
github.tar, level 7 with dict, old streaming, 37839
github.tar, level 9, old streaming, 36723
github.tar, level 9 with dict, old streaming, 36531
github.tar, level 13, old streaming, 35501
github.tar, level 13 with dict, old streaming, 37130
github.tar, level 16, old streaming, 40466
@@ -1291,10 +1291,10 @@ silesia, level 0, old stre
silesia, level 1, old streaming advanced, 5306388
silesia, level 3, old streaming advanced, 4842075
silesia, level 4, old streaming advanced, 4779186
silesia, level 5, old streaming advanced, 4666323
silesia, level 6, old streaming advanced, 4603066
silesia, level 7, old streaming advanced, 4566984
silesia, level 9, old streaming advanced, 4543018
silesia, level 5, old streaming advanced, 4667668
silesia, level 6, old streaming advanced, 4604351
silesia, level 7, old streaming advanced, 4570271
silesia, level 9, old streaming advanced, 4545850
silesia, level 13, old streaming advanced, 4493990
silesia, level 16, old streaming advanced, 4360041
silesia, level 19, old streaming advanced, 4296055
@@ -1305,7 +1305,7 @@ silesia, multithreaded long distance mode, old stre
silesia, small window log, old streaming advanced, 7111103
silesia, small hash log, old streaming advanced, 6526141
silesia, small chain log, old streaming advanced, 4912197
silesia, explicit params, old streaming advanced, 4795452
silesia, explicit params, old streaming advanced, 4795857
silesia, uncompressed literals, old streaming advanced, 4842075
silesia, uncompressed literals optimal, old streaming advanced, 4296055
silesia, huffman literals, old streaming advanced, 6172207
@@ -1317,10 +1317,10 @@ silesia.tar, level 0, old stre
silesia.tar, level 1, old streaming advanced, 5327708
silesia.tar, level 3, old streaming advanced, 4859271
silesia.tar, level 4, old streaming advanced, 4797470
silesia.tar, level 5, old streaming advanced, 4677748
silesia.tar, level 6, old streaming advanced, 4613246
silesia.tar, level 7, old streaming advanced, 4576664
silesia.tar, level 9, old streaming advanced, 4552900
silesia.tar, level 5, old streaming advanced, 4679020
silesia.tar, level 6, old streaming advanced, 4614558
silesia.tar, level 7, old streaming advanced, 4579823
silesia.tar, level 9, old streaming advanced, 4555445
silesia.tar, level 13, old streaming advanced, 4502956
silesia.tar, level 16, old streaming advanced, 4360546
silesia.tar, level 19, old streaming advanced, 4265911
@@ -1331,17 +1331,17 @@ silesia.tar, multithreaded long distance mode, old stre
silesia.tar, small window log, old streaming advanced, 7117562
silesia.tar, small hash log, old streaming advanced, 6529209
silesia.tar, small chain log, old streaming advanced, 4917021
silesia.tar, explicit params, old streaming advanced, 4806873
silesia.tar, explicit params, old streaming advanced, 4807288
silesia.tar, uncompressed literals, old streaming advanced, 4859271
silesia.tar, uncompressed literals optimal, old streaming advanced, 4265911
silesia.tar, huffman literals, old streaming advanced, 6179056
silesia.tar, multithreaded with advanced params, old streaming advanced, 4859271
github, level -5, old streaming advanced, 213265
github, level -5 with dict, old streaming advanced, 49562
github, level -5 with dict, old streaming advanced, 46708
github, level -3, old streaming advanced, 196126
github, level -3 with dict, old streaming advanced, 44956
github, level -3 with dict, old streaming advanced, 45476
github, level -1, old streaming advanced, 181107
github, level -1 with dict, old streaming advanced, 42383
github, level -1 with dict, old streaming advanced, 42060
github, level 0, old streaming advanced, 141104
github, level 0 with dict, old streaming advanced, 41113
github, level 1, old streaming advanced, 143693
@@ -1355,9 +1355,9 @@ github, level 5 with dict, old stre
github, level 6, old streaming advanced, 138676
github, level 6 with dict, old streaming advanced, 38744
github, level 7, old streaming advanced, 138676
github, level 7 with dict, old streaming advanced, 38924
github, level 7 with dict, old streaming advanced, 38875
github, level 9, old streaming advanced, 138676
github, level 9 with dict, old streaming advanced, 38981
github, level 9 with dict, old streaming advanced, 38941
github, level 13, old streaming advanced, 138676
github, level 13 with dict, old streaming advanced, 39725
github, level 16, old streaming advanced, 138575
@@ -1378,11 +1378,11 @@ github, uncompressed literals optimal, old stre
github, huffman literals, old streaming advanced, 181107
github, multithreaded with advanced params, old streaming advanced, 141104
github.tar, level -5, old streaming advanced, 52152
github.tar, level -5 with dict, old streaming advanced, 50988
github.tar, level -5 with dict, old streaming advanced, 51129
github.tar, level -3, old streaming advanced, 45678
github.tar, level -3 with dict, old streaming advanced, 44729
github.tar, level -3 with dict, old streaming advanced, 44986
github.tar, level -1, old streaming advanced, 42560
github.tar, level -1 with dict, old streaming advanced, 41589
github.tar, level -1 with dict, old streaming advanced, 41650
github.tar, level 0, old streaming advanced, 38831
github.tar, level 0 with dict, old streaming advanced, 38013
github.tar, level 1, old streaming advanced, 39200
@@ -1392,13 +1392,13 @@ github.tar, level 3 with dict, old stre
github.tar, level 4, old streaming advanced, 38893
github.tar, level 4 with dict, old streaming advanced, 38063
github.tar, level 5, old streaming advanced, 39651
github.tar, level 5 with dict, old streaming advanced, 38997
github.tar, level 5 with dict, old streaming advanced, 39018
github.tar, level 6, old streaming advanced, 39282
github.tar, level 6 with dict, old streaming advanced, 38640
github.tar, level 7, old streaming advanced, 38110
github.tar, level 7 with dict, old streaming advanced, 37387
github.tar, level 9, old streaming advanced, 36760
github.tar, level 9 with dict, old streaming advanced, 36312
github.tar, level 6 with dict, old streaming advanced, 38635
github.tar, level 7, old streaming advanced, 38005
github.tar, level 7 with dict, old streaming advanced, 37264
github.tar, level 9, old streaming advanced, 36723
github.tar, level 9 with dict, old streaming advanced, 36241
github.tar, level 13, old streaming advanced, 35501
github.tar, level 13 with dict, old streaming advanced, 35807
github.tar, level 16, old streaming advanced, 40466
@@ -1418,62 +1418,62 @@ github.tar, uncompressed literals, old stre
github.tar, uncompressed literals optimal, old streaming advanced, 32276
github.tar, huffman literals, old streaming advanced, 42560
github.tar, multithreaded with advanced params, old streaming advanced, 38831
github, level -5 with dict, old streaming cdict, 46718
github, level -3 with dict, old streaming cdict, 45395
github, level -1 with dict, old streaming cdict, 43170
github, level -5 with dict, old streaming cdict, 45832
github, level -3 with dict, old streaming cdict, 44671
github, level -1 with dict, old streaming cdict, 41825
github, level 0 with dict, old streaming cdict, 41148
github, level 1 with dict, old streaming cdict, 41682
github, level 1 with dict, old streaming cdict, 41266
github, level 3 with dict, old streaming cdict, 41148
github, level 4 with dict, old streaming cdict, 41251
github, level 5 with dict, old streaming cdict, 38758
github, level 6 with dict, old streaming cdict, 38671
github, level 7 with dict, old streaming cdict, 38758
github, level 9 with dict, old streaming cdict, 39437
github, level 5 with dict, old streaming cdict, 38754
github, level 6 with dict, old streaming cdict, 38669
github, level 7 with dict, old streaming cdict, 38765
github, level 9 with dict, old streaming cdict, 39439
github, level 13 with dict, old streaming cdict, 39900
github, level 16 with dict, old streaming cdict, 37577
github, level 19 with dict, old streaming cdict, 37576
github, no source size with dict, old streaming cdict, 40654
github.tar, level -5 with dict, old streaming cdict, 51191
github.tar, level -3 with dict, old streaming cdict, 44821
github.tar, level -1 with dict, old streaming cdict, 41775
github.tar, level -5 with dict, old streaming cdict, 51286
github.tar, level -3 with dict, old streaming cdict, 45147
github.tar, level -1 with dict, old streaming cdict, 41865
github.tar, level 0 with dict, old streaming cdict, 37956
github.tar, level 1 with dict, old streaming cdict, 38364
github.tar, level 3 with dict, old streaming cdict, 37956
github.tar, level 4 with dict, old streaming cdict, 37927
github.tar, level 5 with dict, old streaming cdict, 38999
github.tar, level 6 with dict, old streaming cdict, 38648
github.tar, level 7 with dict, old streaming cdict, 37436
github.tar, level 9 with dict, old streaming cdict, 36401
github.tar, level 5 with dict, old streaming cdict, 39000
github.tar, level 6 with dict, old streaming cdict, 38647
github.tar, level 7 with dict, old streaming cdict, 37286
github.tar, level 9 with dict, old streaming cdict, 36322
github.tar, level 13 with dict, old streaming cdict, 36010
github.tar, level 16 with dict, old streaming cdict, 39081
github.tar, level 19 with dict, old streaming cdict, 32479
github.tar, no source size with dict, old streaming cdict, 38000
github, level -5 with dict, old streaming advanced cdict, 49562
github, level -3 with dict, old streaming advanced cdict, 44956
github, level -1 with dict, old streaming advanced cdict, 42383
github, level -5 with dict, old streaming advanced cdict, 46708
github, level -3 with dict, old streaming advanced cdict, 45476
github, level -1 with dict, old streaming advanced cdict, 42060
github, level 0 with dict, old streaming advanced cdict, 41113
github, level 1 with dict, old streaming advanced cdict, 42430
github, level 3 with dict, old streaming advanced cdict, 41113
github, level 4 with dict, old streaming advanced cdict, 41084
github, level 5 with dict, old streaming advanced cdict, 38723
github, level 6 with dict, old streaming advanced cdict, 38744
github, level 7 with dict, old streaming advanced cdict, 38924
github, level 9 with dict, old streaming advanced cdict, 38981
github, level 7 with dict, old streaming advanced cdict, 38875
github, level 9 with dict, old streaming advanced cdict, 38941
github, level 13 with dict, old streaming advanced cdict, 39725
github, level 16 with dict, old streaming advanced cdict, 40789
github, level 19 with dict, old streaming advanced cdict, 37576
github, no source size with dict, old streaming advanced cdict, 40608
github.tar, level -5 with dict, old streaming advanced cdict, 50854
github.tar, level -3 with dict, old streaming advanced cdict, 44571
github.tar, level -1 with dict, old streaming advanced cdict, 41477
github.tar, level -5 with dict, old streaming advanced cdict, 50791
github.tar, level -3 with dict, old streaming advanced cdict, 44926
github.tar, level -1 with dict, old streaming advanced cdict, 41482
github.tar, level 0 with dict, old streaming advanced cdict, 38013
github.tar, level 1 with dict, old streaming advanced cdict, 38168
github.tar, level 3 with dict, old streaming advanced cdict, 38013
github.tar, level 4 with dict, old streaming advanced cdict, 38063
github.tar, level 5 with dict, old streaming advanced cdict, 38997
github.tar, level 6 with dict, old streaming advanced cdict, 38640
github.tar, level 7 with dict, old streaming advanced cdict, 37387
github.tar, level 9 with dict, old streaming advanced cdict, 36312
github.tar, level 5 with dict, old streaming advanced cdict, 39018
github.tar, level 6 with dict, old streaming advanced cdict, 38635
github.tar, level 7 with dict, old streaming advanced cdict, 37264
github.tar, level 9 with dict, old streaming advanced cdict, 36241
github.tar, level 13 with dict, old streaming advanced cdict, 35807
github.tar, level 16 with dict, old streaming advanced cdict, 38578
github.tar, level 19 with dict, old streaming advanced cdict, 32704
1 Data Config Method Total compressed size
6 silesia.tar level 1 compress simple 5327717
7 silesia.tar level 3 compress simple 4854086
8 silesia.tar level 4 compress simple 4791503
9 silesia.tar level 5 compress simple 4677740 4679004
10 silesia.tar level 6 compress simple 4613242 4614561
11 silesia.tar level 7 compress simple 4576661 4579828
12 silesia.tar level 9 compress simple 4552899 4555448
13 silesia.tar level 13 compress simple 4502956
14 silesia.tar level 16 compress simple 4360546
15 silesia.tar level 19 compress simple 4265911
25 github.tar level 4 compress simple 38893
26 github.tar level 5 compress simple 39651
27 github.tar level 6 compress simple 39282
28 github.tar level 7 compress simple 38110 38005
29 github.tar level 9 compress simple 36760 36723
30 github.tar level 13 compress simple 35501
31 github.tar level 16 compress simple 40466
32 github.tar level 19 compress simple 32276
40 silesia level 1 compress cctx 5306632
41 silesia level 3 compress cctx 4842075
42 silesia level 4 compress cctx 4779186
43 silesia level 5 compress cctx 4666323 4667668
44 silesia level 6 compress cctx 4603066 4604351
45 silesia level 7 compress cctx 4566984 4570271
46 silesia level 9 compress cctx 4543018 4545850
47 silesia level 13 compress cctx 4493990
48 silesia level 16 compress cctx 4360041
49 silesia level 19 compress cctx 4296055
53 silesia small window log compress cctx 7082951
54 silesia small hash log compress cctx 6526141
55 silesia small chain log compress cctx 4912197
56 silesia explicit params compress cctx 4794052 4794318
57 silesia uncompressed literals compress cctx 4842075
58 silesia uncompressed literals optimal compress cctx 4296055
59 silesia huffman literals compress cctx 6172202
60 silesia multithreaded with advanced params compress cctx 4842075
61 github level -5 compress cctx 204407
62 github level -5 with dict compress cctx 52059 47581
63 github level -3 compress cctx 193253
64 github level -3 with dict compress cctx 46787 43043
65 github level -1 compress cctx 175468
66 github level -1 with dict compress cctx 43585 42044
67 github level 0 compress cctx 136332
68 github level 0 with dict compress cctx 41534
69 github level 1 compress cctx 142365
70 github level 1 with dict compress cctx 42259 41715
71 github level 3 compress cctx 136332
72 github level 3 with dict compress cctx 41534
73 github level 4 compress cctx 136199
74 github level 4 with dict compress cctx 41725
75 github level 5 compress cctx 135121
76 github level 5 with dict compress cctx 38759 38755
77 github level 6 compress cctx 135122
78 github level 6 with dict compress cctx 38669 38665
79 github level 7 compress cctx 135122
80 github level 7 with dict compress cctx 38755 38759
81 github level 9 compress cctx 135122
82 github level 9 with dict compress cctx 39398 39362
83 github level 13 compress cctx 132878
84 github level 13 with dict compress cctx 39948
85 github level 16 compress cctx 133209
104 silesia level 1 zstdcli 5306680
105 silesia level 3 zstdcli 4842123
106 silesia level 4 zstdcli 4779234
107 silesia level 5 zstdcli 4666371 4667716
108 silesia level 6 zstdcli 4603114 4604399
109 silesia level 7 zstdcli 4567032 4570319
110 silesia level 9 zstdcli 4543066 4545898
111 silesia level 13 zstdcli 4494038
112 silesia level 16 zstdcli 4360089
113 silesia level 19 zstdcli 4296103
117 silesia small window log zstdcli 7095048
118 silesia small hash log zstdcli 6526189
119 silesia small chain log zstdcli 4912245
120 silesia explicit params zstdcli 4795432 4795840
121 silesia uncompressed literals zstdcli 5120614
122 silesia uncompressed literals optimal zstdcli 4319566
123 silesia huffman literals zstdcli 5321417
129 silesia.tar level 1 zstdcli 5329010
130 silesia.tar level 3 zstdcli 4854164
131 silesia.tar level 4 zstdcli 4792352
132 silesia.tar level 5 zstdcli 4678682 4679860
133 silesia.tar level 6 zstdcli 4614125 4615355
134 silesia.tar level 7 zstdcli 4578719 4581791
135 silesia.tar level 9 zstdcli 4552903 4555452
136 silesia.tar level 13 zstdcli 4502960
137 silesia.tar level 16 zstdcli 4360550
138 silesia.tar level 19 zstdcli 4265915
143 silesia.tar small window log zstdcli 7100701
144 silesia.tar small hash log zstdcli 6529264
145 silesia.tar small chain log zstdcli 4917022
146 silesia.tar explicit params zstdcli 4820713 4821112
147 silesia.tar uncompressed literals zstdcli 5122571
148 silesia.tar uncompressed literals optimal zstdcli 4310145
149 silesia.tar huffman literals zstdcli 5342074
150 silesia.tar multithreaded with advanced params zstdcli 5122571
151 github level -5 zstdcli 206407
152 github level -5 with dict zstdcli 48718 47832
153 github level -3 zstdcli 195253
154 github level -3 with dict zstdcli 47395 46671
155 github level -1 zstdcli 177468
156 github level -1 with dict zstdcli 45170 43825
157 github level 0 zstdcli 138332
158 github level 0 with dict zstdcli 43148
159 github level 1 zstdcli 144365
160 github level 1 with dict zstdcli 43682 43266
161 github level 3 zstdcli 138332
162 github level 3 with dict zstdcli 43148
163 github level 4 zstdcli 138199
165 github level 5 zstdcli 137121
166 github level 5 with dict zstdcli 40728
167 github level 6 zstdcli 137122
168 github level 6 with dict zstdcli 40636 40638
169 github level 7 zstdcli 137122
170 github level 7 with dict zstdcli 40745 40749
171 github level 9 zstdcli 137122
172 github level 9 with dict zstdcli 41393
173 github level 13 zstdcli 134878
188 github huffman literals zstdcli 144365
189 github multithreaded with advanced params zstdcli 167911
190 github.tar level -5 zstdcli 52119
191 github.tar level -5 with dict zstdcli 50978 51101
192 github.tar level -3 zstdcli 45682
193 github.tar level -3 with dict zstdcli 44660 44738
194 github.tar level -1 zstdcli 42564
195 github.tar level -1 with dict zstdcli 41155 41357
196 github.tar level 0 zstdcli 38835
197 github.tar level 0 with dict zstdcli 37999
198 github.tar level 1 zstdcli 39204
199 github.tar level 1 with dict zstdcli 38093 38123
200 github.tar level 3 zstdcli 38835
201 github.tar level 3 with dict zstdcli 37999
202 github.tar level 4 zstdcli 38897
203 github.tar level 4 with dict zstdcli 37952
204 github.tar level 5 zstdcli 39655
205 github.tar level 5 with dict zstdcli 39071 39073
206 github.tar level 6 zstdcli 39286
207 github.tar level 6 with dict zstdcli 38638 38647
208 github.tar level 7 zstdcli 38114 38009
209 github.tar level 7 with dict zstdcli 37886 37861
210 github.tar level 9 zstdcli 36764 36727
211 github.tar level 9 with dict zstdcli 36632 36686
212 github.tar level 13 zstdcli 35505
213 github.tar level 13 with dict zstdcli 37134
214 github.tar level 16 zstdcli 40470
235 silesia level 1 advanced one pass 5306632
236 silesia level 3 advanced one pass 4842075
237 silesia level 4 advanced one pass 4779186
238 silesia level 5 row 1 advanced one pass 4666323 4667668
239 silesia level 5 row 2 advanced one pass 4670136 4670326
240 silesia level 5 advanced one pass 4666323 4667668
241 silesia level 6 advanced one pass 4603066 4604351
242 silesia level 7 row 1 advanced one pass 4566984 4570271
243 silesia level 7 row 2 advanced one pass 4564868 4565169
244 silesia level 7 advanced one pass 4566984 4570271
245 silesia level 9 advanced one pass 4543018 4545850
246 silesia level 11 row 1 advanced one pass 4505046 4505658
247 silesia level 11 row 2 advanced one pass 4503116 4503429
248 silesia level 12 row 1 advanced one pass 4505046 4505658
249 silesia level 12 row 2 advanced one pass 4503116 4503429
250 silesia level 13 advanced one pass 4493990
251 silesia level 16 advanced one pass 4360041
252 silesia level 19 advanced one pass 4296055
257 silesia small window log advanced one pass 7095000
258 silesia small hash log advanced one pass 6526141
259 silesia small chain log advanced one pass 4912197
260 silesia explicit params advanced one pass 4795432 4795840
261 silesia uncompressed literals advanced one pass 5120566
262 silesia uncompressed literals optimal advanced one pass 4319518
263 silesia huffman literals advanced one pass 5321369
269 silesia.tar level 1 advanced one pass 5327717
270 silesia.tar level 3 advanced one pass 4854086
271 silesia.tar level 4 advanced one pass 4791503
272 silesia.tar level 5 row 1 advanced one pass 4677740 4679004
273 silesia.tar level 5 row 2 advanced one pass 4682161 4682334
274 silesia.tar level 5 advanced one pass 4677740 4679004
275 silesia.tar level 6 advanced one pass 4613242 4614561
276 silesia.tar level 7 row 1 advanced one pass 4576661 4579828
277 silesia.tar level 7 row 2 advanced one pass 4575393 4575602
278 silesia.tar level 7 advanced one pass 4576661 4579828
279 silesia.tar level 9 advanced one pass 4552899 4555448
280 silesia.tar level 11 row 1 advanced one pass 4514432 4514962
281 silesia.tar level 11 row 2 advanced one pass 4513604 4513816
282 silesia.tar level 12 row 1 advanced one pass 4514049 4514517
283 silesia.tar level 12 row 2 advanced one pass 4513797 4514007
284 silesia.tar level 13 advanced one pass 4502956
285 silesia.tar level 16 advanced one pass 4360546
286 silesia.tar level 19 advanced one pass 4265911
291 silesia.tar small window log advanced one pass 7100655
292 silesia.tar small hash log advanced one pass 6529206
293 silesia.tar small chain log advanced one pass 4917041
294 silesia.tar explicit params advanced one pass 4806855 4807274
295 silesia.tar uncompressed literals advanced one pass 5122473
296 silesia.tar uncompressed literals optimal advanced one pass 4310141
297 silesia.tar huffman literals advanced one pass 5341705
298 silesia.tar multithreaded with advanced params advanced one pass 5122567
299 github level -5 advanced one pass 204407
300 github level -5 with dict advanced one pass 46718 45832
301 github level -3 advanced one pass 193253
302 github level -3 with dict advanced one pass 45395 44671
303 github level -1 advanced one pass 175468
304 github level -1 with dict advanced one pass 43170 41825
305 github level 0 advanced one pass 136332
306 github level 0 with dict advanced one pass 41148
307 github level 0 with dict dms advanced one pass 41148
308 github level 0 with dict dds advanced one pass 41148
309 github level 0 with dict copy advanced one pass 41124
310 github level 0 with dict load advanced one pass 42252 41847
311 github level 1 advanced one pass 142365
312 github level 1 with dict advanced one pass 41682 41266
313 github level 1 with dict dms advanced one pass 41682 41266
314 github level 1 with dict dds advanced one pass 41682 41266
315 github level 1 with dict copy advanced one pass 41698 41279
316 github level 1 with dict load advanced one pass 43814 43331
317 github level 3 advanced one pass 136332
318 github level 3 with dict advanced one pass 41148
319 github level 3 with dict dms advanced one pass 41148
320 github level 3 with dict dds advanced one pass 41148
321 github level 3 with dict copy advanced one pass 41124
322 github level 3 with dict load advanced one pass 42252 41847
323 github level 4 advanced one pass 136199
324 github level 4 with dict advanced one pass 41251
325 github level 4 with dict dms advanced one pass 41251
326 github level 4 with dict dds advanced one pass 41251
327 github level 4 with dict copy advanced one pass 41216
328 github level 4 with dict load advanced one pass 41159 41548
329 github level 5 row 1 advanced one pass 134584
330 github level 5 row 1 with dict dms advanced one pass 38758 38754
331 github level 5 row 1 with dict dds advanced one pass 38728
332 github level 5 row 1 with dict copy advanced one pass 38759 38755
333 github level 5 row 1 with dict load advanced one pass 41518 41899
334 github level 5 row 2 advanced one pass 135121
335 github level 5 row 2 with dict dms advanced one pass 38938
336 github level 5 row 2 with dict dds advanced one pass 38732
337 github level 5 row 2 with dict copy advanced one pass 38934
338 github level 5 row 2 with dict load advanced one pass 40725 41248
339 github level 5 advanced one pass 135121
340 github level 5 with dict advanced one pass 38758 38754
341 github level 5 with dict dms advanced one pass 38758 38754
342 github level 5 with dict dds advanced one pass 38728
343 github level 5 with dict copy advanced one pass 38759 38755
344 github level 5 with dict load advanced one pass 40725 41248
345 github level 6 advanced one pass 135122
346 github level 6 with dict advanced one pass 38671 38669
347 github level 6 with dict dms advanced one pass 38671 38669
348 github level 6 with dict dds advanced one pass 38636 38638
349 github level 6 with dict copy advanced one pass 38669 38665
350 github level 6 with dict load advanced one pass 40695 41153
351 github level 7 row 1 advanced one pass 134584
352 github level 7 row 1 with dict dms advanced one pass 38758 38765
353 github level 7 row 1 with dict dds advanced one pass 38745 38749
354 github level 7 row 1 with dict copy advanced one pass 38755 38759
355 github level 7 row 1 with dict load advanced one pass 43154 43227
356 github level 7 row 2 advanced one pass 135122
357 github level 7 row 2 with dict dms advanced one pass 38860
358 github level 7 row 2 with dict dds advanced one pass 38766
359 github level 7 row 2 with dict copy advanced one pass 38834
360 github level 7 row 2 with dict load advanced one pass 40695 41153
361 github level 7 advanced one pass 135122
362 github level 7 with dict advanced one pass 38758 38765
363 github level 7 with dict dms advanced one pass 38758 38765
364 github level 7 with dict dds advanced one pass 38745 38749
365 github level 7 with dict copy advanced one pass 38755 38759
366 github level 7 with dict load advanced one pass 40695 41153
367 github level 9 advanced one pass 135122
368 github level 9 with dict advanced one pass 39437 39439
369 github level 9 with dict dms advanced one pass 39437 39439
370 github level 9 with dict dds advanced one pass 39393
371 github level 9 with dict copy advanced one pass 39398 39362
372 github level 9 with dict load advanced one pass 41710 42148
373 github level 11 row 1 advanced one pass 135367
374 github level 11 row 1 with dict dms advanced one pass 39671
375 github level 11 row 1 with dict dds advanced one pass 39671
376 github level 11 row 1 with dict copy advanced one pass 39651
377 github level 11 row 1 with dict load advanced one pass 41360 41744
378 github level 11 row 2 advanced one pass 135367
379 github level 11 row 2 with dict dms advanced one pass 39671
380 github level 11 row 2 with dict dds advanced one pass 39671
381 github level 11 row 2 with dict copy advanced one pass 39651
382 github level 11 row 2 with dict load advanced one pass 41360 41744
383 github level 12 row 1 advanced one pass 134402
384 github level 12 row 1 with dict dms advanced one pass 39677
385 github level 12 row 1 with dict dds advanced one pass 39677
386 github level 12 row 1 with dict copy advanced one pass 39677
387 github level 12 row 1 with dict load advanced one pass 41166 41553
388 github level 12 row 2 advanced one pass 134402
389 github level 12 row 2 with dict dms advanced one pass 39677
390 github level 12 row 2 with dict dds advanced one pass 39677
391 github level 12 row 2 with dict copy advanced one pass 39677
392 github level 12 row 2 with dict load advanced one pass 41166 41553
393 github level 13 advanced one pass 132878
394 github level 13 with dict advanced one pass 39900
395 github level 13 with dict dms advanced one pass 39900
422 github huffman literals advanced one pass 142365
423 github multithreaded with advanced params advanced one pass 165911
424 github.tar level -5 advanced one pass 52115
425 github.tar level -5 with dict advanced one pass 50974 51097
426 github.tar level -3 advanced one pass 45678
427 github.tar level -3 with dict advanced one pass 44656 44734
428 github.tar level -1 advanced one pass 42560
429 github.tar level -1 with dict advanced one pass 41151 41353
430 github.tar level 0 advanced one pass 38831
431 github.tar level 0 with dict advanced one pass 37995
432 github.tar level 0 with dict dms advanced one pass 38003
434 github.tar level 0 with dict copy advanced one pass 37995
435 github.tar level 0 with dict load advanced one pass 37956
436 github.tar level 1 advanced one pass 39200
437 github.tar level 1 with dict advanced one pass 38089 38119
438 github.tar level 1 with dict dms advanced one pass 38294 38406
439 github.tar level 1 with dict dds advanced one pass 38294 38406
440 github.tar level 1 with dict copy advanced one pass 38089 38119
441 github.tar level 1 with dict load advanced one pass 38364
442 github.tar level 3 advanced one pass 38831
443 github.tar level 3 with dict advanced one pass 37995
452 github.tar level 4 with dict copy advanced one pass 37948
453 github.tar level 4 with dict load advanced one pass 37927
454 github.tar level 5 row 1 advanced one pass 39651
455 github.tar level 5 row 1 with dict dms advanced one pass 39059 39043
456 github.tar level 5 row 1 with dict dds advanced one pass 39067 39069
457 github.tar level 5 row 1 with dict copy advanced one pass 39082 39145
458 github.tar level 5 row 1 with dict load advanced one pass 38999 39000
459 github.tar level 5 row 2 advanced one pass 39701
460 github.tar level 5 row 2 with dict dms advanced one pass 39365
461 github.tar level 5 row 2 with dict dds advanced one pass 39233
462 github.tar level 5 row 2 with dict copy advanced one pass 39715
463 github.tar level 5 row 2 with dict load advanced one pass 39158
464 github.tar level 5 advanced one pass 39651
465 github.tar level 5 with dict advanced one pass 39082 39145
466 github.tar level 5 with dict dms advanced one pass 39059 39043
467 github.tar level 5 with dict dds advanced one pass 39067 39069
468 github.tar level 5 with dict copy advanced one pass 39082 39145
469 github.tar level 5 with dict load advanced one pass 38999 39000
470 github.tar level 6 advanced one pass 39282
471 github.tar level 6 with dict advanced one pass 38656
472 github.tar level 6 with dict dms advanced one pass 38636 38640
473 github.tar level 6 with dict dds advanced one pass 38634 38643
474 github.tar level 6 with dict copy advanced one pass 38656
475 github.tar level 6 with dict load advanced one pass 38648 38647
476 github.tar level 7 row 1 advanced one pass 38110 38005
477 github.tar level 7 row 1 with dict dms advanced one pass 37858 37832
478 github.tar level 7 row 1 with dict dds advanced one pass 37882 37857
479 github.tar level 7 row 1 with dict copy advanced one pass 37865 37839
480 github.tar level 7 row 1 with dict load advanced one pass 37436 37286
481 github.tar level 7 row 2 advanced one pass 38077
482 github.tar level 7 row 2 with dict dms advanced one pass 38012
483 github.tar level 7 row 2 with dict dds advanced one pass 38014
484 github.tar level 7 row 2 with dict copy advanced one pass 38101
485 github.tar level 7 row 2 with dict load advanced one pass 37402
486 github.tar level 7 advanced one pass 38110 38005
487 github.tar level 7 with dict advanced one pass 37865 37839
488 github.tar level 7 with dict dms advanced one pass 37858 37832
489 github.tar level 7 with dict dds advanced one pass 37882 37857
490 github.tar level 7 with dict copy advanced one pass 37865 37839
491 github.tar level 7 with dict load advanced one pass 37436 37286
492 github.tar level 9 advanced one pass 36760 36723
493 github.tar level 9 with dict advanced one pass 36484 36531
494 github.tar level 9 with dict dms advanced one pass 36567 36615
495 github.tar level 9 with dict dds advanced one pass 36628 36682
496 github.tar level 9 with dict copy advanced one pass 36484 36531
497 github.tar level 9 with dict load advanced one pass 36401 36322
498 github.tar level 11 row 1 advanced one pass 36081 36085
499 github.tar level 11 row 1 with dict dms advanced one pass 36963
500 github.tar level 11 row 1 with dict dds advanced one pass 36963
501 github.tar level 11 row 1 with dict copy advanced one pass 36557
502 github.tar level 11 row 1 with dict load advanced one pass 36434 36423
503 github.tar level 11 row 2 advanced one pass 36110
504 github.tar level 11 row 2 with dict dms advanced one pass 36963
505 github.tar level 11 row 2 with dict dds advanced one pass 36963
506 github.tar level 11 row 2 with dict copy advanced one pass 36557
507 github.tar level 11 row 2 with dict load advanced one pass 36459
508 github.tar level 12 row 1 advanced one pass 36081 36085
509 github.tar level 12 row 1 with dict dms advanced one pass 36986
510 github.tar level 12 row 1 with dict dds advanced one pass 36986
511 github.tar level 12 row 1 with dict copy advanced one pass 36609
512 github.tar level 12 row 1 with dict load advanced one pass 36434 36423
513 github.tar level 12 row 2 advanced one pass 36110
514 github.tar level 12 row 2 with dict dms advanced one pass 36986
515 github.tar level 12 row 2 with dict dds advanced one pass 36986
553 silesia level 1 advanced one pass small out 5306632
554 silesia level 3 advanced one pass small out 4842075
555 silesia level 4 advanced one pass small out 4779186
556 silesia level 5 row 1 advanced one pass small out 4666323 4667668
557 silesia level 5 row 2 advanced one pass small out 4670136 4670326
558 silesia level 5 advanced one pass small out 4666323 4667668
559 silesia level 6 advanced one pass small out 4603066 4604351
560 silesia level 7 row 1 advanced one pass small out 4566984 4570271
561 silesia level 7 row 2 advanced one pass small out 4564868 4565169
562 silesia level 7 advanced one pass small out 4566984 4570271
563 silesia level 9 advanced one pass small out 4543018 4545850
564 silesia level 11 row 1 advanced one pass small out 4505046 4505658
565 silesia level 11 row 2 advanced one pass small out 4503116 4503429
566 silesia level 12 row 1 advanced one pass small out 4505046 4505658
567 silesia level 12 row 2 advanced one pass small out 4503116 4503429
568 silesia level 13 advanced one pass small out 4493990
569 silesia level 16 advanced one pass small out 4360041
570 silesia level 19 advanced one pass small out 4296055
575 silesia small window log advanced one pass small out 7095000
576 silesia small hash log advanced one pass small out 6526141
577 silesia small chain log advanced one pass small out 4912197
578 silesia explicit params advanced one pass small out 4795432 4795840
579 silesia uncompressed literals advanced one pass small out 5120566
580 silesia uncompressed literals optimal advanced one pass small out 4319518
581 silesia huffman literals advanced one pass small out 5321369
587 silesia.tar level 1 advanced one pass small out 5327717
588 silesia.tar level 3 advanced one pass small out 4854086
589 silesia.tar level 4 advanced one pass small out 4791503
590 silesia.tar level 5 row 1 advanced one pass small out 4677740 4679004
591 silesia.tar level 5 row 2 advanced one pass small out 4682161 4682334
592 silesia.tar level 5 advanced one pass small out 4677740 4679004
593 silesia.tar level 6 advanced one pass small out 4613242 4614561
594 silesia.tar level 7 row 1 advanced one pass small out 4576661 4579828
595 silesia.tar level 7 row 2 advanced one pass small out 4575393 4575602
596 silesia.tar level 7 advanced one pass small out 4576661 4579828
597 silesia.tar level 9 advanced one pass small out 4552899 4555448
598 silesia.tar level 11 row 1 advanced one pass small out 4514432 4514962
599 silesia.tar level 11 row 2 advanced one pass small out 4513604 4513816
600 silesia.tar level 12 row 1 advanced one pass small out 4514049 4514517
601 silesia.tar level 12 row 2 advanced one pass small out 4513797 4514007
602 silesia.tar level 13 advanced one pass small out 4502956
603 silesia.tar level 16 advanced one pass small out 4360546
604 silesia.tar level 19 advanced one pass small out 4265911
609 silesia.tar small window log advanced one pass small out 7100655
610 silesia.tar small hash log advanced one pass small out 6529206
611 silesia.tar small chain log advanced one pass small out 4917041
612 silesia.tar explicit params advanced one pass small out 4806855 4807274
613 silesia.tar uncompressed literals advanced one pass small out 5122473
614 silesia.tar uncompressed literals optimal advanced one pass small out 4310141
615 silesia.tar huffman literals advanced one pass small out 5341705
616 silesia.tar multithreaded with advanced params advanced one pass small out 5122567
617 github level -5 advanced one pass small out 204407
618 github level -5 with dict advanced one pass small out 46718 45832
619 github level -3 advanced one pass small out 193253
620 github level -3 with dict advanced one pass small out 45395 44671
621 github level -1 advanced one pass small out 175468
622 github level -1 with dict advanced one pass small out 43170 41825
623 github level 0 advanced one pass small out 136332
624 github level 0 with dict advanced one pass small out 41148
625 github level 0 with dict dms advanced one pass small out 41148
626 github level 0 with dict dds advanced one pass small out 41148
627 github level 0 with dict copy advanced one pass small out 41124
628 github level 0 with dict load advanced one pass small out 42252 41847
629 github level 1 advanced one pass small out 142365
630 github level 1 with dict advanced one pass small out 41682 41266
631 github level 1 with dict dms advanced one pass small out 41682 41266
632 github level 1 with dict dds advanced one pass small out 41682 41266
633 github level 1 with dict copy advanced one pass small out 41698 41279
634 github level 1 with dict load advanced one pass small out 43814 43331
635 github level 3 advanced one pass small out 136332
636 github level 3 with dict advanced one pass small out 41148
637 github level 3 with dict dms advanced one pass small out 41148
638 github level 3 with dict dds advanced one pass small out 41148
639 github level 3 with dict copy advanced one pass small out 41124
640 github level 3 with dict load advanced one pass small out 42252 41847
641 github level 4 advanced one pass small out 136199
642 github level 4 with dict advanced one pass small out 41251
643 github level 4 with dict dms advanced one pass small out 41251
644 github level 4 with dict dds advanced one pass small out 41251
645 github level 4 with dict copy advanced one pass small out 41216
646 github level 4 with dict load advanced one pass small out 41159 41548
647 github level 5 row 1 advanced one pass small out 134584
648 github level 5 row 1 with dict dms advanced one pass small out 38758 38754
649 github level 5 row 1 with dict dds advanced one pass small out 38728
650 github level 5 row 1 with dict copy advanced one pass small out 38759 38755
651 github level 5 row 1 with dict load advanced one pass small out 41518 41899
652 github level 5 row 2 advanced one pass small out 135121
653 github level 5 row 2 with dict dms advanced one pass small out 38938
654 github level 5 row 2 with dict dds advanced one pass small out 38732
655 github level 5 row 2 with dict copy advanced one pass small out 38934
656 github level 5 row 2 with dict load advanced one pass small out 40725 41248
657 github level 5 advanced one pass small out 135121
658 github level 5 with dict advanced one pass small out 38758 38754
659 github level 5 with dict dms advanced one pass small out 38758 38754
660 github level 5 with dict dds advanced one pass small out 38728
661 github level 5 with dict copy advanced one pass small out 38759 38755
662 github level 5 with dict load advanced one pass small out 40725 41248
663 github level 6 advanced one pass small out 135122
664 github level 6 with dict advanced one pass small out 38671 38669
665 github level 6 with dict dms advanced one pass small out 38671 38669
666 github level 6 with dict dds advanced one pass small out 38636 38638
667 github level 6 with dict copy advanced one pass small out 38669 38665
668 github level 6 with dict load advanced one pass small out 40695 41153
669 github level 7 row 1 advanced one pass small out 134584
670 github level 7 row 1 with dict dms advanced one pass small out 38758 38765
671 github level 7 row 1 with dict dds advanced one pass small out 38745 38749
672 github level 7 row 1 with dict copy advanced one pass small out 38755 38759
673 github level 7 row 1 with dict load advanced one pass small out 43154 43227
674 github level 7 row 2 advanced one pass small out 135122
675 github level 7 row 2 with dict dms advanced one pass small out 38860
676 github level 7 row 2 with dict dds advanced one pass small out 38766
677 github level 7 row 2 with dict copy advanced one pass small out 38834
678 github level 7 row 2 with dict load advanced one pass small out 40695 41153
679 github level 7 advanced one pass small out 135122
680 github level 7 with dict advanced one pass small out 38758 38765
681 github level 7 with dict dms advanced one pass small out 38758 38765
682 github level 7 with dict dds advanced one pass small out 38745 38749
683 github level 7 with dict copy advanced one pass small out 38755 38759
684 github level 7 with dict load advanced one pass small out 40695 41153
685 github level 9 advanced one pass small out 135122
686 github level 9 with dict advanced one pass small out 39437 39439
687 github level 9 with dict dms advanced one pass small out 39437 39439
688 github level 9 with dict dds advanced one pass small out 39393
689 github level 9 with dict copy advanced one pass small out 39398 39362
690 github level 9 with dict load advanced one pass small out 41710 42148
691 github level 11 row 1 advanced one pass small out 135367
692 github level 11 row 1 with dict dms advanced one pass small out 39671
693 github level 11 row 1 with dict dds advanced one pass small out 39671
694 github level 11 row 1 with dict copy advanced one pass small out 39651
695 github level 11 row 1 with dict load advanced one pass small out 41360 41744
696 github level 11 row 2 advanced one pass small out 135367
697 github level 11 row 2 with dict dms advanced one pass small out 39671
698 github level 11 row 2 with dict dds advanced one pass small out 39671
699 github level 11 row 2 with dict copy advanced one pass small out 39651
700 github level 11 row 2 with dict load advanced one pass small out 41360 41744
701 github level 12 row 1 advanced one pass small out 134402
702 github level 12 row 1 with dict dms advanced one pass small out 39677
703 github level 12 row 1 with dict dds advanced one pass small out 39677
704 github level 12 row 1 with dict copy advanced one pass small out 39677
705 github level 12 row 1 with dict load advanced one pass small out 41166 41553
706 github level 12 row 2 advanced one pass small out 134402
707 github level 12 row 2 with dict dms advanced one pass small out 39677
708 github level 12 row 2 with dict dds advanced one pass small out 39677
709 github level 12 row 2 with dict copy advanced one pass small out 39677
710 github level 12 row 2 with dict load advanced one pass small out 41166 41553
711 github level 13 advanced one pass small out 132878
712 github level 13 with dict advanced one pass small out 39900
713 github level 13 with dict dms advanced one pass small out 39900
740 github huffman literals advanced one pass small out 142365
741 github multithreaded with advanced params advanced one pass small out 165911
742 github.tar level -5 advanced one pass small out 52115
743 github.tar level -5 with dict advanced one pass small out 50974 51097
744 github.tar level -3 advanced one pass small out 45678
745 github.tar level -3 with dict advanced one pass small out 44656 44734
746 github.tar level -1 advanced one pass small out 42560
747 github.tar level -1 with dict advanced one pass small out 41151 41353
748 github.tar level 0 advanced one pass small out 38831
749 github.tar level 0 with dict advanced one pass small out 37995
750 github.tar level 0 with dict dms advanced one pass small out 38003
752 github.tar level 0 with dict copy advanced one pass small out 37995
753 github.tar level 0 with dict load advanced one pass small out 37956
754 github.tar level 1 advanced one pass small out 39200
755 github.tar level 1 with dict advanced one pass small out 38089 38119
756 github.tar level 1 with dict dms advanced one pass small out 38294 38406
757 github.tar level 1 with dict dds advanced one pass small out 38294 38406
758 github.tar level 1 with dict copy advanced one pass small out 38089 38119
759 github.tar level 1 with dict load advanced one pass small out 38364
760 github.tar level 3 advanced one pass small out 38831
761 github.tar level 3 with dict advanced one pass small out 37995
770 github.tar level 4 with dict copy advanced one pass small out 37948
771 github.tar level 4 with dict load advanced one pass small out 37927
772 github.tar level 5 row 1 advanced one pass small out 39651
773 github.tar level 5 row 1 with dict dms advanced one pass small out 39059 39043
774 github.tar level 5 row 1 with dict dds advanced one pass small out 39067 39069
775 github.tar level 5 row 1 with dict copy advanced one pass small out 39082 39145
776 github.tar level 5 row 1 with dict load advanced one pass small out 38999 39000
777 github.tar level 5 row 2 advanced one pass small out 39701
778 github.tar level 5 row 2 with dict dms advanced one pass small out 39365
779 github.tar level 5 row 2 with dict dds advanced one pass small out 39233
780 github.tar level 5 row 2 with dict copy advanced one pass small out 39715
781 github.tar level 5 row 2 with dict load advanced one pass small out 39158
782 github.tar level 5 advanced one pass small out 39651
783 github.tar level 5 with dict advanced one pass small out 39082 39145
784 github.tar level 5 with dict dms advanced one pass small out 39059 39043
785 github.tar level 5 with dict dds advanced one pass small out 39067 39069
786 github.tar level 5 with dict copy advanced one pass small out 39082 39145
787 github.tar level 5 with dict load advanced one pass small out 38999 39000
788 github.tar level 6 advanced one pass small out 39282
789 github.tar level 6 with dict advanced one pass small out 38656
790 github.tar level 6 with dict dms advanced one pass small out 38636 38640
791 github.tar level 6 with dict dds advanced one pass small out 38634 38643
792 github.tar level 6 with dict copy advanced one pass small out 38656
793 github.tar level 6 with dict load advanced one pass small out 38648 38647
794 github.tar level 7 row 1 advanced one pass small out 38110 38005
795 github.tar level 7 row 1 with dict dms advanced one pass small out 37858 37832
796 github.tar level 7 row 1 with dict dds advanced one pass small out 37882 37857
797 github.tar level 7 row 1 with dict copy advanced one pass small out 37865 37839
798 github.tar level 7 row 1 with dict load advanced one pass small out 37436 37286
799 github.tar level 7 row 2 advanced one pass small out 38077
800 github.tar level 7 row 2 with dict dms advanced one pass small out 38012
801 github.tar level 7 row 2 with dict dds advanced one pass small out 38014
802 github.tar level 7 row 2 with dict copy advanced one pass small out 38101
803 github.tar level 7 row 2 with dict load advanced one pass small out 37402
804 github.tar level 7 advanced one pass small out 38110 38005
805 github.tar level 7 with dict advanced one pass small out 37865 37839
806 github.tar level 7 with dict dms advanced one pass small out 37858 37832
807 github.tar level 7 with dict dds advanced one pass small out 37882 37857
808 github.tar level 7 with dict copy advanced one pass small out 37865 37839
809 github.tar level 7 with dict load advanced one pass small out 37436 37286
810 github.tar level 9 advanced one pass small out 36760 36723
811 github.tar level 9 with dict advanced one pass small out 36484 36531
812 github.tar level 9 with dict dms advanced one pass small out 36567 36615
813 github.tar level 9 with dict dds advanced one pass small out 36628 36682
814 github.tar level 9 with dict copy advanced one pass small out 36484 36531
815 github.tar level 9 with dict load advanced one pass small out 36401 36322
816 github.tar level 11 row 1 advanced one pass small out 36081 36085
817 github.tar level 11 row 1 with dict dms advanced one pass small out 36963
818 github.tar level 11 row 1 with dict dds advanced one pass small out 36963
819 github.tar level 11 row 1 with dict copy advanced one pass small out 36557
820 github.tar level 11 row 1 with dict load advanced one pass small out 36434 36423
821 github.tar level 11 row 2 advanced one pass small out 36110
822 github.tar level 11 row 2 with dict dms advanced one pass small out 36963
823 github.tar level 11 row 2 with dict dds advanced one pass small out 36963
824 github.tar level 11 row 2 with dict copy advanced one pass small out 36557
825 github.tar level 11 row 2 with dict load advanced one pass small out 36459
826 github.tar level 12 row 1 advanced one pass small out 36081 36085
827 github.tar level 12 row 1 with dict dms advanced one pass small out 36986
828 github.tar level 12 row 1 with dict dds advanced one pass small out 36986
829 github.tar level 12 row 1 with dict copy advanced one pass small out 36609
830 github.tar level 12 row 1 with dict load advanced one pass small out 36434 36423
831 github.tar level 12 row 2 advanced one pass small out 36110
832 github.tar level 12 row 2 with dict dms advanced one pass small out 36986
833 github.tar level 12 row 2 with dict dds advanced one pass small out 36986
871 silesia level 1 advanced streaming 5306388
872 silesia level 3 advanced streaming 4842075
873 silesia level 4 advanced streaming 4779186
874 silesia level 5 row 1 advanced streaming 4666323 4667668
875 silesia level 5 row 2 advanced streaming 4670136 4670326
876 silesia level 5 advanced streaming 4666323 4667668
877 silesia level 6 advanced streaming 4603066 4604351
878 silesia level 7 row 1 advanced streaming 4566984 4570271
879 silesia level 7 row 2 advanced streaming 4564868 4565169
880 silesia level 7 advanced streaming 4566984 4570271
881 silesia level 9 advanced streaming 4543018 4545850
882 silesia level 11 row 1 advanced streaming 4505046 4505658
883 silesia level 11 row 2 advanced streaming 4503116 4503429
884 silesia level 12 row 1 advanced streaming 4505046 4505658
885 silesia level 12 row 2 advanced streaming 4503116 4503429
886 silesia level 13 advanced streaming 4493990
887 silesia level 16 advanced streaming 4360041
888 silesia level 19 advanced streaming 4296055
893 silesia small window log advanced streaming 7111103
894 silesia small hash log advanced streaming 6526141
895 silesia small chain log advanced streaming 4912197
896 silesia explicit params advanced streaming 4795452 4795857
897 silesia uncompressed literals advanced streaming 5120566
898 silesia uncompressed literals optimal advanced streaming 4319518
899 silesia huffman literals advanced streaming 5321370
905 silesia.tar level 1 advanced streaming 5327708
906 silesia.tar level 3 advanced streaming 4859271
907 silesia.tar level 4 advanced streaming 4797470
908 silesia.tar level 5 row 1 advanced streaming 4677748 4679020
909 silesia.tar level 5 row 2 advanced streaming 4682169 4682355
910 silesia.tar level 5 advanced streaming 4677748 4679020
911 silesia.tar level 6 advanced streaming 4613246 4614558
912 silesia.tar level 7 row 1 advanced streaming 4576664 4579823
913 silesia.tar level 7 row 2 advanced streaming 4575394 4575601
914 silesia.tar level 7 advanced streaming 4576664 4579823
915 silesia.tar level 9 advanced streaming 4552900 4555445
916 silesia.tar level 11 row 1 advanced streaming 4514433 4514959
917 silesia.tar level 11 row 2 advanced streaming 4513604 4513810
918 silesia.tar level 12 row 1 advanced streaming 4514049 4514514
919 silesia.tar level 12 row 2 advanced streaming 4513797 4514003
920 silesia.tar level 13 advanced streaming 4502956
921 silesia.tar level 16 advanced streaming 4360546
922 silesia.tar level 19 advanced streaming 4265911
927 silesia.tar small window log advanced streaming 7117559
928 silesia.tar small hash log advanced streaming 6529209
929 silesia.tar small chain log advanced streaming 4917021
930 silesia.tar explicit params advanced streaming 4806873 4807288
931 silesia.tar uncompressed literals advanced streaming 5127423
932 silesia.tar uncompressed literals optimal advanced streaming 4310141
933 silesia.tar huffman literals advanced streaming 5341712
934 silesia.tar multithreaded with advanced params advanced streaming 5122567
935 github level -5 advanced streaming 204407
936 github level -5 with dict advanced streaming 46718 45832
937 github level -3 advanced streaming 193253
938 github level -3 with dict advanced streaming 45395 44671
939 github level -1 advanced streaming 175468
940 github level -1 with dict advanced streaming 43170 41825
941 github level 0 advanced streaming 136332
942 github level 0 with dict advanced streaming 41148
943 github level 0 with dict dms advanced streaming 41148
944 github level 0 with dict dds advanced streaming 41148
945 github level 0 with dict copy advanced streaming 41124
946 github level 0 with dict load advanced streaming 42252 41847
947 github level 1 advanced streaming 142365
948 github level 1 with dict advanced streaming 41682 41266
949 github level 1 with dict dms advanced streaming 41682 41266
950 github level 1 with dict dds advanced streaming 41682 41266
951 github level 1 with dict copy advanced streaming 41698 41279
952 github level 1 with dict load advanced streaming 43814 43331
953 github level 3 advanced streaming 136332
954 github level 3 with dict advanced streaming 41148
955 github level 3 with dict dms advanced streaming 41148
956 github level 3 with dict dds advanced streaming 41148
957 github level 3 with dict copy advanced streaming 41124
958 github level 3 with dict load advanced streaming 42252 41847
959 github level 4 advanced streaming 136199
960 github level 4 with dict advanced streaming 41251
961 github level 4 with dict dms advanced streaming 41251
962 github level 4 with dict dds advanced streaming 41251
963 github level 4 with dict copy advanced streaming 41216
964 github level 4 with dict load advanced streaming 41159 41548
965 github level 5 row 1 advanced streaming 134584
966 github level 5 row 1 with dict dms advanced streaming 38758 38754
967 github level 5 row 1 with dict dds advanced streaming 38728
968 github level 5 row 1 with dict copy advanced streaming 38759 38755
969 github level 5 row 1 with dict load advanced streaming 41518 41899
970 github level 5 row 2 advanced streaming 135121
971 github level 5 row 2 with dict dms advanced streaming 38938
972 github level 5 row 2 with dict dds advanced streaming 38732
973 github level 5 row 2 with dict copy advanced streaming 38934
974 github level 5 row 2 with dict load advanced streaming 40725 41248
975 github level 5 advanced streaming 135121
976 github level 5 with dict advanced streaming 38758 38754
977 github level 5 with dict dms advanced streaming 38758 38754
978 github level 5 with dict dds advanced streaming 38728
979 github level 5 with dict copy advanced streaming 38759 38755
980 github level 5 with dict load advanced streaming 40725 41248
981 github level 6 advanced streaming 135122
982 github level 6 with dict advanced streaming 38671 38669
983 github level 6 with dict dms advanced streaming 38671 38669
984 github level 6 with dict dds advanced streaming 38636 38638
985 github level 6 with dict copy advanced streaming 38669 38665
986 github level 6 with dict load advanced streaming 40695 41153
987 github level 7 row 1 advanced streaming 134584
988 github level 7 row 1 with dict dms advanced streaming 38758 38765
989 github level 7 row 1 with dict dds advanced streaming 38745 38749
990 github level 7 row 1 with dict copy advanced streaming 38755 38759
991 github level 7 row 1 with dict load advanced streaming 43154 43227
992 github level 7 row 2 advanced streaming 135122
993 github level 7 row 2 with dict dms advanced streaming 38860
994 github level 7 row 2 with dict dds advanced streaming 38766
995 github level 7 row 2 with dict copy advanced streaming 38834
996 github level 7 row 2 with dict load advanced streaming 40695 41153
997 github level 7 advanced streaming 135122
998 github level 7 with dict advanced streaming 38758 38765
999 github level 7 with dict dms advanced streaming 38758 38765
1000 github level 7 with dict dds advanced streaming 38745 38749
1001 github level 7 with dict copy advanced streaming 38755 38759
1002 github level 7 with dict load advanced streaming 40695 41153
1003 github level 9 advanced streaming 135122
1004 github level 9 with dict advanced streaming 39437 39439
1005 github level 9 with dict dms advanced streaming 39437 39439
1006 github level 9 with dict dds advanced streaming 39393
1007 github level 9 with dict copy advanced streaming 39398 39362
1008 github level 9 with dict load advanced streaming 41710 42148
1009 github level 11 row 1 advanced streaming 135367
1010 github level 11 row 1 with dict dms advanced streaming 39671
1011 github level 11 row 1 with dict dds advanced streaming 39671
1012 github level 11 row 1 with dict copy advanced streaming 39651
1013 github level 11 row 1 with dict load advanced streaming 41360 41744
1014 github level 11 row 2 advanced streaming 135367
1015 github level 11 row 2 with dict dms advanced streaming 39671
1016 github level 11 row 2 with dict dds advanced streaming 39671
1017 github level 11 row 2 with dict copy advanced streaming 39651
1018 github level 11 row 2 with dict load advanced streaming 41360 41744
1019 github level 12 row 1 advanced streaming 134402
1020 github level 12 row 1 with dict dms advanced streaming 39677
1021 github level 12 row 1 with dict dds advanced streaming 39677
1022 github level 12 row 1 with dict copy advanced streaming 39677
1023 github level 12 row 1 with dict load advanced streaming 41166 41553
1024 github level 12 row 2 advanced streaming 134402
1025 github level 12 row 2 with dict dms advanced streaming 39677
1026 github level 12 row 2 with dict dds advanced streaming 39677
1027 github level 12 row 2 with dict copy advanced streaming 39677
1028 github level 12 row 2 with dict load advanced streaming 41166 41553
1029 github level 13 advanced streaming 132878
1030 github level 13 with dict advanced streaming 39900
1031 github level 13 with dict dms advanced streaming 39900
1058 github huffman literals advanced streaming 142365
1059 github multithreaded with advanced params advanced streaming 165911
1060 github.tar level -5 advanced streaming 52152
1061 github.tar level -5 with dict advanced streaming 51045 51181
1062 github.tar level -3 advanced streaming 45678
1063 github.tar level -3 with dict advanced streaming 44656 44734
1064 github.tar level -1 advanced streaming 42560
1065 github.tar level -1 with dict advanced streaming 41151 41353
1066 github.tar level 0 advanced streaming 38831
1067 github.tar level 0 with dict advanced streaming 37995
1068 github.tar level 0 with dict dms advanced streaming 38003
1070 github.tar level 0 with dict copy advanced streaming 37995
1071 github.tar level 0 with dict load advanced streaming 37956
1072 github.tar level 1 advanced streaming 39200
1073 github.tar level 1 with dict advanced streaming 38089 38119
1074 github.tar level 1 with dict dms advanced streaming 38294 38406
1075 github.tar level 1 with dict dds advanced streaming 38294 38406
1076 github.tar level 1 with dict copy advanced streaming 38089 38119
1077 github.tar level 1 with dict load advanced streaming 38364
1078 github.tar level 3 advanced streaming 38831
1079 github.tar level 3 with dict advanced streaming 37995
1088 github.tar level 4 with dict copy advanced streaming 37948
1089 github.tar level 4 with dict load advanced streaming 37927
1090 github.tar level 5 row 1 advanced streaming 39651
1091 github.tar level 5 row 1 with dict dms advanced streaming 39059 39043
1092 github.tar level 5 row 1 with dict dds advanced streaming 39067 39069
1093 github.tar level 5 row 1 with dict copy advanced streaming 39082 39145
1094 github.tar level 5 row 1 with dict load advanced streaming 38999 39000
1095 github.tar level 5 row 2 advanced streaming 39701
1096 github.tar level 5 row 2 with dict dms advanced streaming 39365
1097 github.tar level 5 row 2 with dict dds advanced streaming 39233
1098 github.tar level 5 row 2 with dict copy advanced streaming 39715
1099 github.tar level 5 row 2 with dict load advanced streaming 39158
1100 github.tar level 5 advanced streaming 39651
1101 github.tar level 5 with dict advanced streaming 39082 39145
1102 github.tar level 5 with dict dms advanced streaming 39059 39043
1103 github.tar level 5 with dict dds advanced streaming 39067 39069
1104 github.tar level 5 with dict copy advanced streaming 39082 39145
1105 github.tar level 5 with dict load advanced streaming 38999 39000
1106 github.tar level 6 advanced streaming 39282
1107 github.tar level 6 with dict advanced streaming 38656
1108 github.tar level 6 with dict dms advanced streaming 38636 38640
1109 github.tar level 6 with dict dds advanced streaming 38634 38643
1110 github.tar level 6 with dict copy advanced streaming 38656
1111 github.tar level 6 with dict load advanced streaming 38648 38647
1112 github.tar level 7 row 1 advanced streaming 38110 38005
1113 github.tar level 7 row 1 with dict dms advanced streaming 37858 37832
1114 github.tar level 7 row 1 with dict dds advanced streaming 37882 37857
1115 github.tar level 7 row 1 with dict copy advanced streaming 37865 37839
1116 github.tar level 7 row 1 with dict load advanced streaming 37436 37286
1117 github.tar level 7 row 2 advanced streaming 38077
1118 github.tar level 7 row 2 with dict dms advanced streaming 38012
1119 github.tar level 7 row 2 with dict dds advanced streaming 38014
1120 github.tar level 7 row 2 with dict copy advanced streaming 38101
1121 github.tar level 7 row 2 with dict load advanced streaming 37402
1122 github.tar level 7 advanced streaming 38110 38005
1123 github.tar level 7 with dict advanced streaming 37865 37839
1124 github.tar level 7 with dict dms advanced streaming 37858 37832
1125 github.tar level 7 with dict dds advanced streaming 37882 37857
1126 github.tar level 7 with dict copy advanced streaming 37865 37839
1127 github.tar level 7 with dict load advanced streaming 37436 37286
1128 github.tar level 9 advanced streaming 36760 36723
1129 github.tar level 9 with dict advanced streaming 36484 36531
1130 github.tar level 9 with dict dms advanced streaming 36567 36615
1131 github.tar level 9 with dict dds advanced streaming 36628 36682
1132 github.tar level 9 with dict copy advanced streaming 36484 36531
1133 github.tar level 9 with dict load advanced streaming 36401 36322
1134 github.tar level 11 row 1 advanced streaming 36081 36085
1135 github.tar level 11 row 1 with dict dms advanced streaming 36963
1136 github.tar level 11 row 1 with dict dds advanced streaming 36963
1137 github.tar level 11 row 1 with dict copy advanced streaming 36557
1138 github.tar level 11 row 1 with dict load advanced streaming 36434 36423
1139 github.tar level 11 row 2 advanced streaming 36110
1140 github.tar level 11 row 2 with dict dms advanced streaming 36963
1141 github.tar level 11 row 2 with dict dds advanced streaming 36963
1142 github.tar level 11 row 2 with dict copy advanced streaming 36557
1143 github.tar level 11 row 2 with dict load advanced streaming 36459
1144 github.tar level 12 row 1 advanced streaming 36081 36085
1145 github.tar level 12 row 1 with dict dms advanced streaming 36986
1146 github.tar level 12 row 1 with dict dds advanced streaming 36986
1147 github.tar level 12 row 1 with dict copy advanced streaming 36609
1148 github.tar level 12 row 1 with dict load advanced streaming 36434 36423
1149 github.tar level 12 row 2 advanced streaming 36110
1150 github.tar level 12 row 2 with dict dms advanced streaming 36986
1151 github.tar level 12 row 2 with dict dds advanced streaming 36986
1189 silesia level 1 old streaming 5306388
1190 silesia level 3 old streaming 4842075
1191 silesia level 4 old streaming 4779186
1192 silesia level 5 old streaming 4666323 4667668
1193 silesia level 6 old streaming 4603066 4604351
1194 silesia level 7 old streaming 4566984 4570271
1195 silesia level 9 old streaming 4543018 4545850
1196 silesia level 13 old streaming 4493990
1197 silesia level 16 old streaming 4360041
1198 silesia level 19 old streaming 4296055
1207 silesia.tar level 1 old streaming 5327708
1208 silesia.tar level 3 old streaming 4859271
1209 silesia.tar level 4 old streaming 4797470
1210 silesia.tar level 5 old streaming 4677748 4679020
1211 silesia.tar level 6 old streaming 4613246 4614558
1212 silesia.tar level 7 old streaming 4576664 4579823
1213 silesia.tar level 9 old streaming 4552900 4555445
1214 silesia.tar level 13 old streaming 4502956
1215 silesia.tar level 16 old streaming 4360546
1216 silesia.tar level 19 old streaming 4265911
1219 silesia.tar uncompressed literals optimal old streaming 4265911
1220 silesia.tar huffman literals old streaming 6179056
1221 github level -5 old streaming 204407
1222 github level -5 with dict old streaming 46718 45832
1223 github level -3 old streaming 193253
1224 github level -3 with dict old streaming 45395 44671
1225 github level -1 old streaming 175468
1226 github level -1 with dict old streaming 43170 41825
1227 github level 0 old streaming 136332
1228 github level 0 with dict old streaming 41148
1229 github level 1 old streaming 142365
1230 github level 1 with dict old streaming 41682 41266
1231 github level 3 old streaming 136332
1232 github level 3 with dict old streaming 41148
1233 github level 4 old streaming 136199
1234 github level 4 with dict old streaming 41251
1235 github level 5 old streaming 135121
1236 github level 5 with dict old streaming 38758 38754
1237 github level 6 old streaming 135122
1238 github level 6 with dict old streaming 38671 38669
1239 github level 7 old streaming 135122
1240 github level 7 with dict old streaming 38758 38765
1241 github level 9 old streaming 135122
1242 github level 9 with dict old streaming 39437 39439
1243 github level 13 old streaming 132878
1244 github level 13 with dict old streaming 39900
1245 github level 16 old streaming 133209
1252 github uncompressed literals optimal old streaming 132879
1253 github huffman literals old streaming 175468
1254 github.tar level -5 old streaming 52152
1255 github.tar level -5 with dict old streaming 51045 51181
1256 github.tar level -3 old streaming 45678
1257 github.tar level -3 with dict old streaming 44656 44734
1258 github.tar level -1 old streaming 42560
1259 github.tar level -1 with dict old streaming 41151 41353
1260 github.tar level 0 old streaming 38831
1261 github.tar level 0 with dict old streaming 37995
1262 github.tar level 1 old streaming 39200
1263 github.tar level 1 with dict old streaming 38089 38119
1264 github.tar level 3 old streaming 38831
1265 github.tar level 3 with dict old streaming 37995
1266 github.tar level 4 old streaming 38893
1267 github.tar level 4 with dict old streaming 37948
1268 github.tar level 5 old streaming 39651
1269 github.tar level 5 with dict old streaming 39082 39145
1270 github.tar level 6 old streaming 39282
1271 github.tar level 6 with dict old streaming 38656
1272 github.tar level 7 old streaming 38110 38005
1273 github.tar level 7 with dict old streaming 37865 37839
1274 github.tar level 9 old streaming 36760 36723
1275 github.tar level 9 with dict old streaming 36484 36531
1276 github.tar level 13 old streaming 35501
1277 github.tar level 13 with dict old streaming 37130
1278 github.tar level 16 old streaming 40466
1291 silesia level 1 old streaming advanced 5306388
1292 silesia level 3 old streaming advanced 4842075
1293 silesia level 4 old streaming advanced 4779186
1294 silesia level 5 old streaming advanced 4666323 4667668
1295 silesia level 6 old streaming advanced 4603066 4604351
1296 silesia level 7 old streaming advanced 4566984 4570271
1297 silesia level 9 old streaming advanced 4543018 4545850
1298 silesia level 13 old streaming advanced 4493990
1299 silesia level 16 old streaming advanced 4360041
1300 silesia level 19 old streaming advanced 4296055
1305 silesia small window log old streaming advanced 7111103
1306 silesia small hash log old streaming advanced 6526141
1307 silesia small chain log old streaming advanced 4912197
1308 silesia explicit params old streaming advanced 4795452 4795857
1309 silesia uncompressed literals old streaming advanced 4842075
1310 silesia uncompressed literals optimal old streaming advanced 4296055
1311 silesia huffman literals old streaming advanced 6172207
1317 silesia.tar level 1 old streaming advanced 5327708
1318 silesia.tar level 3 old streaming advanced 4859271
1319 silesia.tar level 4 old streaming advanced 4797470
1320 silesia.tar level 5 old streaming advanced 4677748 4679020
1321 silesia.tar level 6 old streaming advanced 4613246 4614558
1322 silesia.tar level 7 old streaming advanced 4576664 4579823
1323 silesia.tar level 9 old streaming advanced 4552900 4555445
1324 silesia.tar level 13 old streaming advanced 4502956
1325 silesia.tar level 16 old streaming advanced 4360546
1326 silesia.tar level 19 old streaming advanced 4265911
1331 silesia.tar small window log old streaming advanced 7117562
1332 silesia.tar small hash log old streaming advanced 6529209
1333 silesia.tar small chain log old streaming advanced 4917021
1334 silesia.tar explicit params old streaming advanced 4806873 4807288
1335 silesia.tar uncompressed literals old streaming advanced 4859271
1336 silesia.tar uncompressed literals optimal old streaming advanced 4265911
1337 silesia.tar huffman literals old streaming advanced 6179056
1338 silesia.tar multithreaded with advanced params old streaming advanced 4859271
1339 github level -5 old streaming advanced 213265
1340 github level -5 with dict old streaming advanced 49562 46708
1341 github level -3 old streaming advanced 196126
1342 github level -3 with dict old streaming advanced 44956 45476
1343 github level -1 old streaming advanced 181107
1344 github level -1 with dict old streaming advanced 42383 42060
1345 github level 0 old streaming advanced 141104
1346 github level 0 with dict old streaming advanced 41113
1347 github level 1 old streaming advanced 143693
1355 github level 6 old streaming advanced 138676
1356 github level 6 with dict old streaming advanced 38744
1357 github level 7 old streaming advanced 138676
1358 github level 7 with dict old streaming advanced 38924 38875
1359 github level 9 old streaming advanced 138676
1360 github level 9 with dict old streaming advanced 38981 38941
1361 github level 13 old streaming advanced 138676
1362 github level 13 with dict old streaming advanced 39725
1363 github level 16 old streaming advanced 138575
1378 github huffman literals old streaming advanced 181107
1379 github multithreaded with advanced params old streaming advanced 141104
1380 github.tar level -5 old streaming advanced 52152
1381 github.tar level -5 with dict old streaming advanced 50988 51129
1382 github.tar level -3 old streaming advanced 45678
1383 github.tar level -3 with dict old streaming advanced 44729 44986
1384 github.tar level -1 old streaming advanced 42560
1385 github.tar level -1 with dict old streaming advanced 41589 41650
1386 github.tar level 0 old streaming advanced 38831
1387 github.tar level 0 with dict old streaming advanced 38013
1388 github.tar level 1 old streaming advanced 39200
1392 github.tar level 4 old streaming advanced 38893
1393 github.tar level 4 with dict old streaming advanced 38063
1394 github.tar level 5 old streaming advanced 39651
1395 github.tar level 5 with dict old streaming advanced 38997 39018
1396 github.tar level 6 old streaming advanced 39282
1397 github.tar level 6 with dict old streaming advanced 38640 38635
1398 github.tar level 7 old streaming advanced 38110 38005
1399 github.tar level 7 with dict old streaming advanced 37387 37264
1400 github.tar level 9 old streaming advanced 36760 36723
1401 github.tar level 9 with dict old streaming advanced 36312 36241
1402 github.tar level 13 old streaming advanced 35501
1403 github.tar level 13 with dict old streaming advanced 35807
1404 github.tar level 16 old streaming advanced 40466
1418 github.tar uncompressed literals optimal old streaming advanced 32276
1419 github.tar huffman literals old streaming advanced 42560
1420 github.tar multithreaded with advanced params old streaming advanced 38831
1421 github level -5 with dict old streaming cdict 46718 45832
1422 github level -3 with dict old streaming cdict 45395 44671
1423 github level -1 with dict old streaming cdict 43170 41825
1424 github level 0 with dict old streaming cdict 41148
1425 github level 1 with dict old streaming cdict 41682 41266
1426 github level 3 with dict old streaming cdict 41148
1427 github level 4 with dict old streaming cdict 41251
1428 github level 5 with dict old streaming cdict 38758 38754
1429 github level 6 with dict old streaming cdict 38671 38669
1430 github level 7 with dict old streaming cdict 38758 38765
1431 github level 9 with dict old streaming cdict 39437 39439
1432 github level 13 with dict old streaming cdict 39900
1433 github level 16 with dict old streaming cdict 37577
1434 github level 19 with dict old streaming cdict 37576
1435 github no source size with dict old streaming cdict 40654
1436 github.tar level -5 with dict old streaming cdict 51191 51286
1437 github.tar level -3 with dict old streaming cdict 44821 45147
1438 github.tar level -1 with dict old streaming cdict 41775 41865
1439 github.tar level 0 with dict old streaming cdict 37956
1440 github.tar level 1 with dict old streaming cdict 38364
1441 github.tar level 3 with dict old streaming cdict 37956
1442 github.tar level 4 with dict old streaming cdict 37927
1443 github.tar level 5 with dict old streaming cdict 38999 39000
1444 github.tar level 6 with dict old streaming cdict 38648 38647
1445 github.tar level 7 with dict old streaming cdict 37436 37286
1446 github.tar level 9 with dict old streaming cdict 36401 36322
1447 github.tar level 13 with dict old streaming cdict 36010
1448 github.tar level 16 with dict old streaming cdict 39081
1449 github.tar level 19 with dict old streaming cdict 32479
1450 github.tar no source size with dict old streaming cdict 38000
1451 github level -5 with dict old streaming advanced cdict 49562 46708
1452 github level -3 with dict old streaming advanced cdict 44956 45476
1453 github level -1 with dict old streaming advanced cdict 42383 42060
1454 github level 0 with dict old streaming advanced cdict 41113
1455 github level 1 with dict old streaming advanced cdict 42430
1456 github level 3 with dict old streaming advanced cdict 41113
1457 github level 4 with dict old streaming advanced cdict 41084
1458 github level 5 with dict old streaming advanced cdict 38723
1459 github level 6 with dict old streaming advanced cdict 38744
1460 github level 7 with dict old streaming advanced cdict 38924 38875
1461 github level 9 with dict old streaming advanced cdict 38981 38941
1462 github level 13 with dict old streaming advanced cdict 39725
1463 github level 16 with dict old streaming advanced cdict 40789
1464 github level 19 with dict old streaming advanced cdict 37576
1465 github no source size with dict old streaming advanced cdict 40608
1466 github.tar level -5 with dict old streaming advanced cdict 50854 50791
1467 github.tar level -3 with dict old streaming advanced cdict 44571 44926
1468 github.tar level -1 with dict old streaming advanced cdict 41477 41482
1469 github.tar level 0 with dict old streaming advanced cdict 38013
1470 github.tar level 1 with dict old streaming advanced cdict 38168
1471 github.tar level 3 with dict old streaming advanced cdict 38013
1472 github.tar level 4 with dict old streaming advanced cdict 38063
1473 github.tar level 5 with dict old streaming advanced cdict 38997 39018
1474 github.tar level 6 with dict old streaming advanced cdict 38640 38635
1475 github.tar level 7 with dict old streaming advanced cdict 37387 37264
1476 github.tar level 9 with dict old streaming advanced cdict 36312 36241
1477 github.tar level 13 with dict old streaming advanced cdict 35807
1478 github.tar level 16 with dict old streaming advanced cdict 38578
1479 github.tar level 19 with dict old streaming advanced cdict 32704
zlibWrapper/examples/zwrapbench.c
+6 -6
View File
@@ -26,7 +26,7 @@
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"
#include "zstd_zlibwrapper.h"
#include "../zstd_zlibwrapper.h"
@@ -109,17 +109,17 @@ static unsigned g_nbIterations = NBLOOPS;
static size_t g_blockSize = 0;
int g_additionalParam = 0;
void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
static void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
void BMK_SetNbIterations(unsigned nbLoops)
static void BMK_SetNbIterations(unsigned nbLoops)
{
g_nbIterations = nbLoops;
DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations);
}
void BMK_SetBlockSize(size_t blockSize)
static void BMK_SetBlockSize(size_t blockSize)
{
g_blockSize = blockSize;
DISPLAYLEVEL(2, "using blocks of size %u KB \n", (unsigned)(blockSize>>10));
@@ -798,7 +798,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
}
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
static int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast)
{
double const compressibility = (double)g_compressibilityDefault / 100;