Compare commits

..
Author SHA1 Message Date
Yann ColletandGitHub 794ea1b0af Merge pull request #3984 from facebook/dev
v1.5.6
2024-03-21 15:05:51 -07:00
Yann ColletandGitHub 63779c7982 Merge pull request #3595 from facebook/dev
v1.5.5 last changes
2023-04-04 13:13:52 -07:00
Yann ColletandGitHub d507e02940 Merge pull request #3590 from facebook/dev
v1.5.5
2023-04-03 14:38:22 -07:00
Yann ColletandGitHub 73e19c4cf1 Merge pull request #3585 from facebook/dev
Preparation for release v1.5.5
2023-04-01 18:06:35 -07:00
Yann ColletandGitHub 945f27758c Merge pull request #3487 from facebook/dev
release v1.5.4
2023-02-09 16:41:50 -08:00
Felix HandteandGitHub e47e674cd0 Merge pull request #2995 from facebook/v1.5.2-rc
Zstandard v1.5.2
2022-01-20 16:17:18 -05:00
Yann ColletandGitHub 791626dfb9 Merge pull request #2942 from facebook/dev
update man pages for v1.5.1
2021-12-20 14:49:18 -08:00
Yann ColletandGitHub f4a541b021 Merge pull request #2941 from facebook/dev
v1.5.1
2021-12-20 13:49:33 -08:00
senandGitHub a488ba114e Zstd 1.5.0 Release
Zstd 1.5.0 Release
2021-05-14 10:59:34 -04:00
Felix HandteandGitHub e4558ffd1d Merge pull request #2515 from facebook/dev
ZStandard v1.4.9
2021-03-02 17:20:57 -05:00
Yann ColletandGitHub 97a3da1df0 Merge pull request #2435 from facebook/dev
v1.4.8 hotfix
2020-12-18 16:39:42 -08:00
240 changed files with 5310 additions and 11920 deletions
+123
View File
@@ -0,0 +1,123 @@
version: 2
jobs:
# the first half of the jobs are in this test
short-tests-0:
# TODO: Create a small custom docker image with all the dependencies we need
# preinstalled to reduce installation time.
docker:
- image: fbopensource/zstd-circleci-primary:0.0.1
steps:
- checkout
- run:
name: Test
command: |
./tests/test-license.py
cc -v
CFLAGS="-O0 -Werror -pedantic" make allmost; make clean
make c99build; make clean
make c11build; make clean
make -j regressiontest; make clean
make shortest; make clean
make cxxtest; make clean
# the second half of the jobs are in this test
short-tests-1:
docker:
- image: fbopensource/zstd-circleci-primary:0.0.1
steps:
- checkout
- run:
name: Test
command: |
make gnu90build; make clean
make gnu99build; make clean
make ppc64build V=1; make clean
make ppcbuild V=1; make clean
make armbuild V=1; make clean
make aarch64build V=1; make clean
make -C tests test-legacy test-longmatch; make clean
make -C lib libzstd-nomt; make clean
# This step should only be run in a cron job
regression-test:
docker:
- image: fbopensource/zstd-circleci-primary:0.0.1
environment:
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
steps:
- checkout
# Restore the cached resources.
- restore_cache:
# We try our best to bust the cache when the data changes by hashing
# data.c. If that doesn't work, simply update the version number here
# and below. If we fail to bust the cache, the regression testing will
# still work, since it has its own stamp, but will need to redownload
# everything.
keys:
- regression-cache-{{ checksum "tests/regression/data.c" }}-v0
- run:
name: Regression Test
command: |
make -C programs zstd
make -C tests/regression test
mkdir -p $CIRCLE_ARTIFACTS
./tests/regression/test \
--cache tests/regression/cache \
--output $CIRCLE_ARTIFACTS/results.csv \
--zstd programs/zstd
echo "NOTE: The new results.csv is uploaded as an artifact to this job"
echo " If this fails, go to the Artifacts pane in CircleCI, "
echo " download /tmp/circleci-artifacts/results.csv, and if they "
echo " are still good, copy it into the repo and commit it."
echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
# Only save the cache on success (default), since if the failure happened
# before we stamp the data cache, we will have a bad cache for this key.
- save_cache:
key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
paths:
- tests/regression/cache
- store_artifacts:
path: /tmp/circleci-artifacts
workflows:
version: 2
commit:
jobs:
# Run the tests in parallel
- short-tests-0
- short-tests-1
- regression-test
nightly:
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- release
- dev
- master
jobs:
# Run daily regression tests
- regression-test
# Longer tests
#- make -C tests test-zstd-nolegacy && make clean
#- pyenv global 3.4.4; make -C tests versionsTest && make clean
#- make zlibwrapper && make clean
#- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
#- make uasan && make clean
#- make asan32 && make clean
#- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
# Valgrind tests
#- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
#- make -C tests valgrindTest && make clean
# ARM, AArch64, PowerPC, PowerPC64 tests
#- make ppctest && make clean
#- make ppc64test && make clean
#- make armtest && make clean
#- make aarch64test && make clean
+9
View File
@@ -0,0 +1,9 @@
FROM circleci/buildpack-deps@sha256:f6f10c11b7b8ccfd4f4a5b830c3256803604ce61292b60cb22e26b12f62b0e8c
RUN sudo dpkg --add-architecture i386
RUN sudo apt-get -y -qq update
RUN sudo apt-get -y install \
gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
libcurl4-openssl-dev
+4 -3
View File
@@ -1,9 +1,10 @@
task:
name: FreeBSD (make check)
name: FreeBSD (shortest)
freebsd_instance:
matrix:
image_family: freebsd-15-0-amd64-zfs
image_family: freebsd-14-0
image_family: freebsd-13-2
install_script: pkg install -y gmake coreutils
script: |
MOREFLAGS="-Werror" gmake -j all
gmake check
gmake shortest
+1 -1
View File
@@ -3,4 +3,4 @@ updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
interval: "weekly"
-50
View File
@@ -1,50 +0,0 @@
name: Android NDK Build
on:
pull_request:
branches: [ dev, release, actionsTest ]
push:
branches: [ actionsTest, '*ndk*' ]
permissions: read-all
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: Set up JDK 17
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
with:
java-version: '17'
distribution: 'temurin'
- name: Setup Android SDK
uses: android-actions/setup-android@651bceb6f9ca583f16b8d75b62c36ded2ae6fc9c # v4.0.0
- name: Install Android NDK
run: |
sdkmanager --install "ndk;27.0.12077973"
echo "ANDROID_NDK_HOME=$ANDROID_SDK_ROOT/ndk/27.0.12077973" >> $GITHUB_ENV
- name: Build with NDK
run: |
export PATH=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH
make CC=aarch64-linux-android21-clang \
AR=llvm-ar \
RANLIB=llvm-ranlib \
STRIP=llvm-strip
- name: Build with CMake and NDK
run: |
mkdir -p build-android
cd build-android
cmake --version
cmake ../build/cmake \
-DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake \
-DANDROID_ABI=arm64-v8a \
-DANDROID_PLATFORM=android-21 \
-DCMAKE_BUILD_TYPE=Release
cmake --build . --parallel
-168
View File
@@ -1,168 +0,0 @@
name: cmake-tests
# CMake-specific build and test workflows
# This workflow validates zstd builds across different CMake configurations,
# platforms, and edge cases to ensure broad compatibility.
concurrency:
group: cmake-${{ github.ref }}
cancel-in-progress: true
on:
pull_request:
branches: [ dev, release, actionsTest ]
permissions: read-all
env:
# Centralized test timeouts for consistency
QUICK_TEST_TIME: "30s"
STANDARD_TEST_TIME: "1mn"
# Common CMake flags
COMMON_CMAKE_FLAGS: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
jobs:
# Basic cmake build using the root CMakeLists.txt
# Provides a lightweight sanity check that the top-level project config builds
# with the default Unix Makefiles generator driven purely through cmake commands
cmake-root-basic:
name: "CMake Root Build"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: Configure (Root)
run: |
cmake -S . -B cmake-build -DCMAKE_BUILD_TYPE=Release ${{ env.COMMON_CMAKE_FLAGS }}
- name: Build (Root)
run: |
cmake --build cmake-build --config Release
# Ubuntu-based cmake build using make wrapper
# This test uses the make-driven cmake build to ensure compatibility
# with the existing build system integration
cmake-ubuntu-basic:
name: "CMake build using make wrapper"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: Install dependencies
run: |
sudo apt install liblzma-dev # Required for compression algorithms
- name: CMake build and test via make
run: |
# Use make wrapper for cmake build with quick test timeouts
FUZZERTEST=-T${{ env.STANDARD_TEST_TIME }} ZSTREAM_TESTTIME=-T${{ env.STANDARD_TEST_TIME }} make cmakebuild V=1
# Cross-platform cmake build with edge case: source paths containing spaces
# This test ensures cmake handles filesystem paths with spaces correctly
# across different operating systems and build generators
cmake-cross-platform-spaces:
name: "CMake Cross-Platform (Spaces in Path)"
runs-on: ${{ matrix.os }}
strategy:
matrix:
include:
- os: ubuntu-latest
generator: "Unix Makefiles"
name: "Linux"
- os: windows-latest
generator: "NMake Makefiles"
name: "Windows NMake"
- os: macos-latest
generator: "Unix Makefiles"
name: "macOS"
env:
SRC_DIR: "source directory with spaces"
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
with:
path: "${{ env.SRC_DIR }}"
- uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
if: ${{ matrix.generator == 'NMake Makefiles' }}
- name: "CMake build and install (${{ matrix.name }})"
run: |
# Test Release build with installation to verify packaging
cmake -S "${{ env.SRC_DIR }}/build/cmake" -B build -DBUILD_TESTING=ON -G "${{ matrix.generator }}" -DCMAKE_BUILD_TYPE=Release --install-prefix "${{ runner.temp }}/install"
cmake --build build --config Release
cmake --install build --config Release
# Windows-specific cmake testing with Visual Studio 2022
# Tests multiple generators and toolchains to ensure broad Windows compatibility
# including MSVC (x64, Win32, ARM64), MinGW, and Clang-CL with various architectures and optimizations
cmake-windows-comprehensive:
name: "CMake Windows VS2022 (${{ matrix.name }})"
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
include:
- generator: "Visual Studio 17 2022"
flags: "-A x64"
name: "MSVC x64"
runner: "windows-2022"
cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
- generator: "Visual Studio 17 2022"
flags: "-A Win32"
name: "MSVC Win32"
runner: "windows-2022"
cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
- generator: "Visual Studio 17 2022"
flags: "-A x64"
name: "MSVC x64 (No ZSTD_BUILD_TESTS)"
runner: "windows-2022"
# Intentionally omit ZSTD_BUILD_TESTS to reproduce the CXX language configuration bug
cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON"
- generator: "Visual Studio 17 2022"
flags: "-A ARM64"
name: "MSVC ARM64"
runner: "windows-11-arm" # githuh runner for WOA instance
- generator: "MinGW Makefiles"
flags: ""
name: "MinGW"
runner: "windows-2022"
cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
- generator: "Visual Studio 17 2022"
flags: "-T ClangCL"
name: "Clang-CL"
runner: "windows-2022"
cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
- generator: "Visual Studio 17 2022"
flags: "-T ClangCL -A x64 -DCMAKE_C_FLAGS=/arch:AVX2"
name: "Clang-CL AVX2"
runner: "windows-2022"
cmake_extra_flags: "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DZSTD_BUILD_TESTS=ON"
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@30375c66a4eea26614e0d39710365f22f8b0af57 # tag=v3.0.0
- name: "Configure CMake (${{ matrix.name }})"
run: |
cd build\cmake
mkdir build
cd build
cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} -DCMAKE_BUILD_TYPE=Debug ${{ matrix.cmake_extra_flags }} -DZSTD_ZSTREAM_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FUZZER_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FULLBENCH_FLAGS=-i0 ..
- name: "Build (${{ matrix.name }})"
run: |
cd build\cmake\build
cmake.exe --build .
- name: "Test (${{ matrix.name }})"
run: |
cd build\cmake\build
ctest.exe -V -C Debug
# macOS ARM64 (Apple Silicon) specific cmake testing
# Validates zstd builds and runs correctly on Apple Silicon architecture
# Uses native ARM64 hardware for optimal performance and compatibility testing
cmake-macos-arm64:
name: "CMake macOS ARM64 (Apple Silicon)"
runs-on: macos-14 # ARM64 runner
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: "CMake build and test (ARM64)"
run: |
# Configure and build with ARM64-specific optimizations
cd build/cmake
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release ${{ env.COMMON_CMAKE_FLAGS }} -DZSTD_ZSTREAM_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FUZZER_FLAGS=-T${{ env.QUICK_TEST_TIME }} -DZSTD_FULLBENCH_FLAGS=-i1 ..
make -j$(sysctl -n hw.ncpu)
ctest -V
+17 -32
View File
@@ -3,12 +3,7 @@ on:
push:
branches:
- dev
pull_request:
branches:
- dev
permissions: read-all
jobs:
short-tests-0:
runs-on: ubuntu-latest
@@ -17,7 +12,7 @@ jobs:
image: fbopensource/zstd-circleci-primary:0.0.1
options: --entrypoint /bin/bash
steps:
- uses: actions/checkout@v6.0.2
- uses: actions/checkout@v4
- name: Install Dependencies
run: |
sudo apt-get update
@@ -30,9 +25,8 @@ jobs:
make c99build; make clean
make c11build; make clean
make -j regressiontest; make clean
make check; make clean
make shortest; make clean
make cxxtest; make clean
short-tests-1:
runs-on: ubuntu-latest
services:
@@ -40,30 +34,21 @@ jobs:
image: fbopensource/zstd-circleci-primary:0.0.1
options: --entrypoint /bin/bash
steps:
- uses: actions/checkout@v6.0.2
- uses: actions/checkout@v4
- name: Install Dependencies
run: |
sudo apt-get update
sudo apt-get install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu libc6-dev-ppc64-powerpc-cross libcurl4-gnutls-dev lib64gcc-13-dev-powerpc-cross
- name: gnu90 build
run: make gnu90build && make clean
- name: gnu99 build
run: make gnu99build && make clean
- name: ppc64 build
run: make ppc64build V=1 && make clean
- name: ppc build
run: make ppcbuild V=1 && make clean
- name: arm build
run: make armbuild V=1 && make clean
- name: aarch64 build
run: make aarch64build V=1 && make clean
- name: test-legacy
run: make -C tests test-legacy V=1 && make clean
- name: test-longmatch
run: make -C tests test-longmatch V=1 && make clean
- name: libzstd-nomt build
run: make -C lib libzstd-nomt V=1 && make clean
sudo apt-get install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu libc6-dev-ppc64-powerpc-cross libcurl4-gnutls-dev lib64gcc-11-dev-powerpc-cross
- name: Test
run: |-
make gnu90build; make clean
make gnu99build; make clean
make ppc64build V=1; make clean
make ppcbuild V=1; make clean
make armbuild V=1; make clean
make aarch64build V=1; make clean
make -C tests test-legacy test-longmatch; make clean
make -C lib libzstd-nomt; make clean
regression-test:
runs-on: ubuntu-latest
services:
@@ -73,9 +58,9 @@ jobs:
env:
CIRCLE_ARTIFACTS: "/tmp/circleci-artifacts"
steps:
- uses: actions/checkout@v6.0.2
- uses: actions/checkout@v4
- name: restore_cache
uses: actions/cache@v5
uses: actions/cache@v4
with:
key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
path: tests/regression/cache
@@ -99,6 +84,6 @@ jobs:
echo " are still good, copy it into the repo and commit it."
echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
- uses: actions/upload-artifact@v7.0.1
- uses: actions/upload-artifact@v4
with:
path: "/tmp/circleci-artifacts"
+69 -96
View File
@@ -1,5 +1,5 @@
name: dev-long-tests
# Tests generally longer than 10mn
# Tests longer than 10mn
concurrency:
group: long-${{ github.ref }}
@@ -12,106 +12,91 @@ on:
permissions: read-all
jobs:
# lasts ~7mn
make-all:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: make all
run: make all
# lasts ~19mn
# lasts ~24mn
make-test:
runs-on: ubuntu-latest
env:
DEVNULLRIGHTS: 1
READFROMBLOCKDEVICE: 1
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: make test
run: |
make test
make -j zstd
./tests/test_process_substitution.bash ./zstd
# lasts ~16mn
make-test-macos:
runs-on: macos-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: make test on macos
run: make test
# lasts ~10mn
# lasts ~26mn
make-test-osx:
runs-on: macos-latest
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: OS-X test
run: make test # make -c lib all doesn't work because of the fact that it's not a tty
# lasts ~24mn
make-test-32bit:
runs-on: ubuntu-latest
env:
DEVNULLRIGHTS: 1
READFROMBLOCKDEVICE: 1
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: make test # note: `make -j test success` seems to require a clean state
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: make test
run: |
sudo apt-get -qqq update
make libc6install
make clean
CFLAGS="-m32 -O2" make -j test V=1
CFLAGS="-m32" make test
# lasts ~7mn
test-largeDictionary:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: largeDictionary
run: |
CFLAGS="-Werror -O3" make -j -C tests test-largeDictionary
# lasts ~9mn
no-intrinsics-fuzztest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: no intrinsics fuzztest
run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
# lasts ~8mn
tsan-zstreamtest:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: thread sanitizer zstreamtest
run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
uasan-zstreamtest:
runs-on: ubuntu-latest
ubsan-zstreamtest:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: ub + address sanitizer on zstreamtest
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: undefined behavior sanitizer zstreamtest
run: CC=clang make uasan-test-zstream
# lasts ~11mn
# lasts ~15mn
tsan-fuzztest:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: thread sanitizer fuzztest
run: CC=clang make tsan-fuzztest
big-tests-zstreamtest32:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: zstream tests in 32bit mode, with big tests
run: |
sudo apt-get -qqq update
make libc6install
CC=clang make -C tests test-zstream32 FUZZER_FLAGS="--big-tests"
# lasts ~13mn
# lasts ~23mn
gcc-8-asan-ubsan-testzstd:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: gcc-8 + ASan + UBSan + Test Zstd
# See https://askubuntu.com/a/1428822
run: |
@@ -121,16 +106,16 @@ jobs:
CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
clang-asan-ubsan-testzstd:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: clang + ASan + UBSan + Test Zstd
run: CC=clang make -j uasan-test-zstd </dev/null V=1
gcc-asan-ubsan-testzstd-32bit:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: ASan + UBSan + Test Zstd, 32bit mode
run: |
sudo apt-get -qqq update
@@ -142,9 +127,9 @@ jobs:
# so any data coming from these libraries is always considered "uninitialized"
gcc-8-asan-ubsan-fuzz:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: gcc-8 + ASan + UBSan + Fuzz Test
# See https://askubuntu.com/a/1428822
run: |
@@ -154,72 +139,57 @@ jobs:
CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
clang-asan-ubsan-fuzz:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: clang + ASan + UBSan + Fuzz Test
run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
gcc-asan-ubsan-fuzz32:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: ASan + UBSan + Fuzz Test 32bit
run: |
sudo apt-get -qqq update
make libc6install
CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
clang-asan-fuzz32:
runs-on: ubuntu-latest
clang-asan-ubsan-fuzz32:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: clang + ASan + Fuzz Test 32bit
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: clang + ASan + UBSan + Fuzz Test 32bit
run: |
sudo apt-get -qqq update
make libc6install
CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make asan-fuzztest
# The following test seems to have issues on github CI specifically,
# it does not provide the `__mulodi4` instruction emulation
# required for signed 64-bit multiplication.
# Replaced by asan-only test (above)
#
# clang-asan-ubsan-fuzz32:
# runs-on: ubuntu-20.04
# steps:
# - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
# - name: clang + ASan + UBSan + Fuzz Test 32bit
# run: |
# sudo apt-get -qqq update
# make libc6install
# CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
asan-ubsan-regression:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: ASan + UBSan + Regression Test
run: make -j uasanregressiontest
clang-asan-ubsan-regression:
runs-on: ubuntu-latest
clang-ubsan-regression:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: clang + ASan + UBSan + Regression Test
run: CC=clang make -j uasanregressiontest
msan-regression:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: MSan + Regression Test
run: make -j msanregressiontest
clang-msan-fuzz-unoptimized:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: clang + MSan + Fuzz Test
run: |
sudo apt-get -qqq update
@@ -227,27 +197,30 @@ jobs:
CC=clang MOREFLAGS="-O0" make clean msan-fuzztest
clang-msan-fuzz:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: clang + MSan + Fuzz Test
run: |
sudo apt-get -qqq update
sudo apt-get install clang
CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest
# lasts ~24mn
clang-msan-testzstd:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: clang + MSan + Test Zstd
run: |
CC=clang make -j msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
sudo apt-get update
sudo apt-get install clang
CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
armfuzz:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Qemu ARM emulation + Fuzz Test
run: |
sudo apt-get -qqq update
@@ -257,7 +230,7 @@ jobs:
valgrind-fuzz-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: valgrind + fuzz test stack mode # ~ 7mn
shell: 'script -q -e -c "bash {0}"'
run: |
@@ -273,8 +246,8 @@ jobs:
run:
shell: msys2 {0}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c # tag=v2.31.1
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- uses: msys2/setup-msys2@cc11e9188b693c2b100158c3322424c4cc1dadea # tag=v2.22.0
with:
msystem: MINGW64
install: make
@@ -296,7 +269,7 @@ jobs:
# lasts ~20mn
oss-fuzz:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
@@ -317,7 +290,7 @@ jobs:
dry-run: false
sanitizer: ${{ matrix.sanitizer }}
- name: Upload Crash
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # tag=v7.0.1
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # tag=v4.3.1
if: failure() && steps.build.outcome == 'success'
with:
name: ${{ matrix.sanitizer }}-artifacts
+134 -181
View File
@@ -16,40 +16,51 @@ jobs:
linux-kernel:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: linux kernel, library + build + test
run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable"
benchmarking:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: make benchmarking
run: make benchmarking
check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: make check on 32-bit
run: |
sudo apt update
APT_PACKAGES="gcc-multilib" make apt-install
CFLAGS="-m32 -O1 -fstack-protector" make check V=1
CFLAGS="-m32 -O1 -fstack-protector" make V=1 -C tests test-cli-tests
check-x32:
runs-on: ubuntu-20.04 # ubuntu-latest == ubuntu-22.04 have issues currently with x32
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: make check on x32 ABI # https://en.wikipedia.org/wiki/X32_ABI
env:
CHECK_CONSTRAINED_MEM: true
run: |
sudo apt update
APT_PACKAGES="gcc-multilib" make apt-install
CFLAGS="-mx32 -O1 -fstack-protector" make check V=1
build-c89:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: ensure zstd can be built with c89/c90 compilers (+ long long support + variadic macros)
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: ensure zstd can be build with c89/c90 compilers (+ long long support + variadic macros)
run: |
make c89build V=1
build-zstd-dll:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: build zstd bin against a dynamic lib (debuglevel for more dependencies)
run: |
make -C lib lib-mt-release
@@ -58,7 +69,7 @@ jobs:
gcc-7-libzstd:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: gcc-7 + libzstdmt compilation
# See https://askubuntu.com/a/1428822
run: |
@@ -72,10 +83,18 @@ jobs:
# candidate test (for discussion) : underlink test
# LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
cmake-build-and-test-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: cmake build and test
run: |
FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild V=1
cpp-gnu90-c99-compatibility:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: C++, gnu90 and c99 compatibility
run: |
make cxxtest
@@ -89,7 +108,7 @@ jobs:
mingw-cross-compilation:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: mingw cross-compilation
run: |
# sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work)
@@ -100,7 +119,7 @@ jobs:
armbuild:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: ARM Build Test
run: |
sudo apt-get -qqq update
@@ -110,7 +129,7 @@ jobs:
bourne-shell:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Bourne shell compatibility (shellcheck)
run: |
wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz
@@ -120,22 +139,21 @@ jobs:
zlib-wrapper:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: install valgrind
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: zlib wrapper test
run: |
sudo apt-get -qqq update
make valgrindinstall V=1
- name: zlib wrapper test
run: make -C zlibWrapper test V=1
- name: zlib wrapper test under valgrind
run: make -C zlibWrapper test-valgrind V=1
make valgrindinstall
make -C zlibWrapper test
make -C zlibWrapper test-valgrind
lz4-threadpool-libs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: LZ4, thread pool, and libs build testslib wrapper test
run: |
make lz4install
make -C tests test-lz4
make check < /dev/null | tee # mess with lz4 console detection
make clean
@@ -143,41 +161,20 @@ jobs:
make clean
bash tests/libzstd_builds.sh
gcc-make-all-avx2:
gcc-make-tests-32bit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: Make all, with AVX2
run: |
sudo apt-get -qqq update
make libc6install
CFLAGS="-Werror -mavx2" make -j all
gcc-make-all-32bit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Make all, 32bit mode
run: |
sudo apt-get -qqq update
make libc6install
CFLAGS="-Werror -m32" make -j all32
gcc-make-all-32bit-avx2:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: Make all, 32bit + AVX2 mode
run: |
sudo apt-get -qqq update
make libc6install
CPPFLAGS="-DSTATIC_BMI2=1" CFLAGS="-Werror -m32 -mavx2 -mbmi2" make -j all32
gcc-8-make:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: gcc-8 build
# See https://askubuntu.com/a/1428822
run: |
@@ -200,17 +197,15 @@ jobs:
flags: "HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=1"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Build with ${{matrix.name}}
run: |
sudo apt install liblzma-dev
${{matrix.flags}} make zstd
run: ${{matrix.flags}} make zstd
implicit-fall-through:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: -Wimplicit-fallthrough build
run: |
make clean
@@ -221,11 +216,11 @@ jobs:
meson-linux:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Install packages
run: |
sudo apt-get update
sudo apt-get -y install build-essential python3-pip ninja-build liblz4-dev liblzma-dev
sudo apt-get -y install build-essential python3-pip ninja-build liblz4-dev
pip install --pre meson
- name: Build with Meson
run: |
@@ -237,59 +232,15 @@ jobs:
-Dbin_tests=true \
-Dbin_contrib=true \
-Ddefault_library=both \
build/meson mesonBuild
ninja -C mesonBuild/
meson test -C mesonBuild/ --print-errorlogs
meson install -C mesonBuild --destdir staging/
meson-mingw-cross-compilation:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: Install packages
run: |
sudo apt-get -qqq update
sudo apt-get -y install build-essential python3-pip ninja-build {gcc,g++}-mingw-w64-x86-64
pip install --pre meson
- name: Build with Meson
run: |
cat > cross.ini <<EOF
[binaries]
ar = 'x86_64-w64-mingw32-ar'
c = 'x86_64-w64-mingw32-gcc'
cpp = 'x86_64-w64-mingw32-g++'
ld = 'x86_64-w64-mingw32-ld'
objcopy = 'x86_64-w64-mingw32-objcopy'
objdump = 'x86_64-w64-mingw32-objdump'
strip = 'x86_64-w64-mingw32-strip'
windres = 'x86_64-w64-mingw32-windres'
[host_machine]
system = 'windows'
endian = 'little'
cpu_family = 'x86_64'
cpu = 'x86_64'
EOF
# pzstd doesn't build; skip -Dbin_contrib=true
meson setup \
--buildtype=debugoptimized \
--cross-file=cross.ini \
-Db_lundef=false \
-Dbin_programs=true \
-Dbin_tests=true \
-Ddefault_library=both \
build/meson builddir
ninja -C builddir/
if grep -- -pthread builddir/meson-private/libzstd.pc; then
echo "Error: found stray pthread dependency"
exit 1
fi
meson test -C builddir/ --print-errorlogs
meson install -C builddir --destdir staging/
meson-windows:
runs-on: windows-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Install packages
run: pip install --pre meson
- name: Configure with Meson
@@ -305,6 +256,32 @@ jobs:
run: |
meson install -C builddir --destdir staging/
cmake-visual-2022:
strategy:
matrix:
include:
- generator: "Visual Studio 17 2022"
flags: "-A x64"
- generator: "Visual Studio 17 2022"
flags: "-A Win32"
- generator: "MinGW Makefiles"
- generator: "Visual Studio 17 2022"
flags: "-T ClangCL"
runs-on: windows-2022
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # tag=v2.0.0
- name: Build & Test
working-directory: ${{env.GITHUB_WORKSPACE}}
run: |
cd build\cmake
mkdir build
cd build
cmake.exe -G "${{matrix.generator}}" ${{matrix.flags}} -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZSTREAM_FLAGS=-T30s -DZSTD_FUZZER_FLAGS=-T30s -DZSTD_FULLBENCH_FLAGS=-i0 ..
cmake.exe --build .
ctest.exe -V -C Debug
msbuild-visual-studio:
strategy:
fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed.
@@ -314,14 +291,15 @@ jobs:
{ name: "VS 2022 Win32 Debug", platform: Win32, configuration: Debug, toolset: v143, runner: "windows-2022", arch: "" },
{ name: "VS 2022 x64 Release", platform: x64, configuration: Release, toolset: v143, runner: "windows-2022", arch: ""},
{ name: "VS 2022 Win32 Release", platform: Win32, configuration: Release, toolset: v143, runner: "windows-2022", arch: ""},
{ name: "VS 2025 x64 Debug", platform: x64, configuration: Debug, toolset: v143, runner: "windows-2025", arch: ""},
{ name: "VS 2019 x64 Release", platform: Win32, configuration: Release, toolset: v142, runner: "windows-2019", arch: ""},
{ name: "VS 2019 Win32 Release", platform: x64, configuration: Release, toolset: v142, runner: "windows-2019", arch: ""},
{ name: "VS 2022 x64 Release AVX2", platform: x64, configuration: Release, toolset: v143, runner: "windows-2022", arch: "AdvancedVectorExtensions2" },
]
runs-on: ${{matrix.runner}}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@30375c66a4eea26614e0d39710365f22f8b0af57 # tag=v3.0.0
uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # tag=v2.0.0
- name: Build ${{matrix.name}}
working-directory: ${{env.GITHUB_WORKSPACE}}
# See https://docs.microsoft.com/visualstudio/msbuild/msbuild-command-line-reference
@@ -344,7 +322,7 @@ jobs:
libzstd-size:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: libzstd size test
run: |
make clean && make -j -C lib libzstd && ./tests/check_size.py lib/libzstd.so 1100000
@@ -355,7 +333,7 @@ jobs:
minimal-decompressor-macros:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: minimal decompressor macros
run: |
make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
@@ -372,7 +350,7 @@ jobs:
dynamic-bmi2:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: dynamic bmi2 tests
run: |
make clean && make -j check MOREFLAGS="-O0 -Werror -mbmi2"
@@ -384,7 +362,7 @@ jobs:
test-variants:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: make all variants & validate
run: |
make -j -C programs allVariants MOREFLAGS=-O0
@@ -392,18 +370,18 @@ jobs:
qemu-consistency:
name: QEMU ${{ matrix.name }}
runs-on: ubuntu-24.04
runs-on: ubuntu-20.04
strategy:
fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed.
matrix:
include: [
{ name: ARM, xcc_pkg: gcc-arm-linux-gnueabi, xcc: arm-linux-gnueabi-gcc, xemu_pkg: qemu-system-arm, xemu: qemu-arm-static },
{ name: ARM64, xcc_pkg: gcc-aarch64-linux-gnu, xcc: aarch64-linux-gnu-gcc, xemu_pkg: qemu-system-aarch64,xemu: qemu-aarch64-static },
{ name: ARM64, xcc_pkg: gcc-aarch64-linux-gnu, xcc: aarch64-linux-gnu-gcc, xemu_pkg: qemu-system-arm, xemu: qemu-aarch64-static },
{ name: PPC, xcc_pkg: gcc-powerpc-linux-gnu, xcc: powerpc-linux-gnu-gcc, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc-static },
{ name: PPC64LE, xcc_pkg: gcc-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64le-static },
{ name: S390X, xcc_pkg: gcc-s390x-linux-gnu, xcc: s390x-linux-gnu-gcc, xemu_pkg: qemu-system-s390x, xemu: qemu-s390x-static },
{ name: MIPS, xcc_pkg: gcc-mips-linux-gnu, xcc: mips-linux-gnu-gcc, xemu_pkg: qemu-system-mips, xemu: qemu-mips-static },
{ name: RISC-V, xcc_pkg: gcc-14-riscv64-linux-gnu, xcc: riscv64-linux-gnu-gcc-14, xemu_pkg: qemu-system-riscv64,xemu: qemu-riscv64-static },
{ name: RISC-V, xcc_pkg: gcc-riscv64-linux-gnu, xcc: riscv64-linux-gnu-gcc, xemu_pkg: qemu-system-riscv64,xemu: qemu-riscv64-static },
{ name: M68K, xcc_pkg: gcc-m68k-linux-gnu, xcc: m68k-linux-gnu-gcc, xemu_pkg: qemu-system-m68k, xemu: qemu-m68k-static },
{ name: SPARC, xcc_pkg: gcc-sparc64-linux-gnu, xcc: sparc64-linux-gnu-gcc, xemu_pkg: qemu-system-sparc, xemu: qemu-sparc64-static },
]
@@ -411,7 +389,7 @@ jobs:
XCC: ${{ matrix.xcc }}
XEMU: ${{ matrix.xemu }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: apt update & install
run: |
sudo apt-get update
@@ -431,15 +409,8 @@ jobs:
- name: ARM64
if: ${{ matrix.name == 'ARM64' }}
run: |
make clean
LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j check
LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j -C tests test-cli-tests
CFLAGS="-O3 -march=armv8.2-a+sve2" LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j check
CFLAGS="-O3 -march=armv8.2-a+sve2" LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make -j -C tests test-cli-tests
# This test is only compatible with standard libraries that support BTI (Branch Target Identification).
# Unfortunately, the standard library provided on Ubuntu 24.04 does not have this feature enabled.
# make clean
# LDFLAGS="-static -z force-bti" MOREFLAGS="-mbranch-protection=standard" CC=$XCC QEMU_SYS=$XEMU make check V=1
LDFLAGS="-static -z force-bti" MOREFLAGS="-mbranch-protection=standard" CC=$XCC QEMU_SYS=$XEMU make clean check
LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
- name: PPC
if: ${{ matrix.name == 'PPC' }}
run: |
@@ -460,9 +431,6 @@ jobs:
if: ${{ matrix.name == 'RISC-V' }}
run: |
LDFLAGS="-static" CC=$XCC QEMU_SYS=$XEMU make clean check
CFLAGS="-march=rv64gcv -O3" LDFLAGS="-static -DMEM_FORCE_MEMORY_ACCESS=0" CC=$XCC QEMU_SYS="$XEMU -cpu rv64,v=true,vlen=128" make clean check
CFLAGS="-march=rv64gcv -O3" LDFLAGS="-static -DMEM_FORCE_MEMORY_ACCESS=0" CC=$XCC QEMU_SYS="$XEMU -cpu rv64,v=true,vlen=256" make clean check
CFLAGS="-march=rv64gcv -O3" LDFLAGS="-static -DMEM_FORCE_MEMORY_ACCESS=0" CC=$XCC QEMU_SYS="$XEMU -cpu rv64,v=true,vlen=512" make clean check
- name: M68K
if: ${{ matrix.name == 'M68K' }}
run: |
@@ -486,8 +454,8 @@ jobs:
run:
shell: msys2 {0}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c # tag=v2.31.1
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- uses: msys2/setup-msys2@cc11e9188b693c2b100158c3322424c4cc1dadea # tag=v2.22.0
with:
msystem: ${{ matrix.msystem }}
install: make diffutils
@@ -522,9 +490,9 @@ jobs:
platform: [x64, Win32]
configuration: [Release]
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Add MSBuild to PATH
uses: microsoft/setup-msbuild@30375c66a4eea26614e0d39710365f22f8b0af57 # tag=v3.0.0
uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # tag=v2.0.0
- name: Build and run tests
working-directory: ${{env.GITHUB_WORKSPACE}}
env:
@@ -543,8 +511,8 @@ jobs:
runs-on: windows-latest
steps:
- run: git config --global core.autocrlf input
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: cygwin/cygwin-install-action@3f0a3f9f988f7e96b8c18098ae05eaec175f5b52 # tag=v6
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- uses: cygwin/cygwin-install-action@006ad0b0946ca6d0a3ea2d4437677fa767392401 # tag=master
with:
platform: x86_64
packages: >-
@@ -564,18 +532,13 @@ jobs:
make -j allzstd &&
make -C tests fuzzer &&
./tests/fuzzer.exe -v -T1m
- name: cygwin install test
shell: C:\cygwin\bin\bash.exe --noprofile --norc -eo pipefail '{0}'
run: >-
make -j &&
make install
pkg-config:
runs-on: ubuntu-latest
container:
image: debian:testing
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Install dependencies
run: |
apt -y update
@@ -590,7 +553,7 @@ jobs:
versions-compatibility:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Versions Compatibility Test
run: |
make -C tests versionsTest
@@ -598,15 +561,27 @@ jobs:
clangbuild:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: make clangbuild
run: |
make clangbuild
clang-pgo:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Build PGO Zstd with Clang
env:
CC: clang-14
LLVM_PROFDATA: llvm-profdata-14
run: |
make -C programs zstd-pgo
./programs/zstd -b
gcc-pgo:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Build PGO Zstd with GCC
env:
CC: gcc
@@ -614,34 +589,10 @@ jobs:
make -C programs zstd-pgo
./programs/zstd -b
clang-pgo:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: Build PGO Zstd with Clang
env:
CC: clang
run: |
sudo apt install -y llvm
llvm-profdata --version
make -C programs zstd-pgo
./programs/zstd -b
musl-build:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: Install musl-tools
run: |
sudo apt install -y musl-tools
- name: Compile with musl-gcc and test-zstd
run: |
CC=musl-gcc CFLAGS="-Werror -O3" CPPFLAGS=-DZDICT_QSORT=ZDICT_QSORT_C90 make -j -C tests test-zstd V=1
intel-cet-compatibility:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
- name: Build Zstd
run: |
make -j zstd V=1
@@ -657,24 +608,26 @@ jobs:
run: |
sde-external-9.33.0-2024-01-07-lin/sde -cet -cet-raise 0 -cet-endbr-exe -cet-stderr -cet-abort -- ./zstd -b3
icx:
# install instructions: https://www.intel.com/content/www/us/en/docs/oneapi/installation-guide-linux/2025-0/apt-005.html
name: icx-check
runs-on: ubuntu-latest
steps:
- name: install icx
run: |
# download the key to system keyring
wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
| gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
# add signed entry to apt sources and configure the APT client to use Intel repository:
echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
sudo apt-get update
sudo apt-get install -y intel-basekit intel-hpckit
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
- name: make check
run: |
source /opt/intel/oneapi/setvars.sh
make CC=icx check
make CC=icx -C tests test-cli-tests
# Failing tests, for reference
# icc tests are currently failing on Github Actions, likely to issues during installation stage
#
# icc:
# name: icc-check
# runs-on: ubuntu-latest
# steps:
# - name: install icc
# run: |
# export DEBIAN_FRONTEND=noninteractive
# sudo apt-get -qqq update
# sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
# sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
# sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
# sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
# sudo apt-get update
# sudo apt-get install -y intel-basekit intel-hpckit
# - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v4.1.1
# - name: make check
# run: |
# make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check
+32 -5
View File
@@ -1,24 +1,51 @@
name: facebook/zstd/nightly
on:
schedule:
- cron: '0 0 * * *'
- cron: 0 0 * * *
push:
branches:
- release
- dev
- '*nightly*'
- master
permissions: read-all
jobs:
regression-test:
runs-on: ubuntu-latest
services:
docker:
image: fbopensource/zstd-circleci-primary:0.0.1
options: --entrypoint /bin/bash
env:
CIRCLE_ARTIFACTS: "/tmp/circleci-artifacts"
steps:
- uses: actions/checkout@v6.0.2
- name: Install dependencies
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
key: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
path: tests/regression/cache
restore-keys: regression-cache-{{ checksum "tests/regression/data.c" }}-v0
- uses: actions/upload-artifact@v4
with:
path: "/tmp/circleci-artifacts"
- name: Install Dependencies
run: |
sudo apt-get update
sudo apt-get install libcurl4-gnutls-dev
- name: Regression Test
run: |
make -C programs zstd
make -C tests/regression test
mkdir -p $CIRCLE_ARTIFACTS
./tests/regression/test \
--cache tests/regression/cache \
--output $CIRCLE_ARTIFACTS/results.csv \
--zstd programs/zstd
echo "NOTE: The new results.csv is uploaded as an artifact to this job"
echo " If this fails, go to the Artifacts pane in CircleCI, "
echo " download /tmp/circleci-artifacts/results.csv, and if they "
echo " are still good, copy it into the repo and commit it."
echo "> diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv"
diff tests/regression/results.csv $CIRCLE_ARTIFACTS/results.csv
# Longer tests
#- make -C tests test-zstd-nolegacy && make clean
@@ -17,7 +17,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v3
- name: Archive
env:
-64
View File
@@ -1,64 +0,0 @@
name: release_checks
on:
push:
branches:
- release
pull_request:
branches:
- release
permissions: read-all
jobs:
verify-manual:
runs-on: ubuntu-latest
steps:
- name: Check out repository
uses: actions/checkout@v6.0.2
- name: Save current manual
run: mv doc/zstd_manual.html doc/zstd_manual_saved.html
- name: Generate new manual
run: make manual
- name: Compare manuals
run: |
if ! cmp -s doc/zstd_manual.html doc/zstd_manual_saved.html; then
echo "The API manual was not updated before release !"
exit 1
fi
verify-man-pages:
runs-on: ubuntu-latest
steps:
- name: Check out repository
uses: actions/checkout@v6.0.2
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y ruby ruby-dev
sudo gem install ronn
- name: Display ronn version
run: ronn --version
- name: Save current man pages
run: |
mv programs/zstd.1 programs/zstd.1.saved
mv programs/zstdgrep.1 programs/zstdgrep.1.saved
mv programs/zstdless.1 programs/zstdless.1.saved
- name: Generate new manual pages
run: make -C programs man
- name: Compare man pages
run: |
for file in zstd.1 zstdgrep.1 zstdless.1; do
if ! cmp -s programs/$file programs/$file.saved; then
echo "The man page $file should have been updated."
exit 1
fi
done
+4 -4
View File
@@ -27,12 +27,12 @@ jobs:
steps:
- name: "Checkout code"
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v3
with:
persist-credentials: false
- name: "Run analysis"
uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # tag=v2.4.1
uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # tag=v2.3.1
with:
results_file: results.sarif
results_format: sarif
@@ -51,7 +51,7 @@ jobs:
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
# format to the repository Actions tab.
- name: "Upload artifact"
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # tag=v7.0.1
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # tag=v4.3.1
with:
name: SARIF file
path: results.sarif
@@ -59,6 +59,6 @@ jobs:
# Upload the results to GitHub's code scanning dashboard.
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # tag=v4.36.2
uses: github/codeql-action/upload-sarif@3ab4101902695724f9365a384f86c1074d94e18c # tag=v3.24.7
with:
sarif_file: results.sarif
+21 -82
View File
@@ -2,7 +2,7 @@ name: windows-artifacts
on:
push:
branches: [ test_artifacts, win_artifacts, release ]
branches: [ test_artifacts, win_artifacts ]
release:
types:
- published
@@ -11,109 +11,48 @@ permissions: read-all
jobs:
windows-artifacts:
permissions:
contents: write # to fetch code and upload artifacts
# For msys2, see https://ariya.io/2020/07/on-github-actions-with-msys2
runs-on: ${{ matrix.shell == 'cmake' && 'windows-11-arm' || 'windows-latest' }}
# see https://ariya.io/2020/07/on-github-actions-with-msys2
runs-on: windows-latest
# see https://github.com/msys2/setup-msys2
strategy:
# For msys2, see https://github.com/msys2/setup-msys2
matrix:
include:
- { msystem: mingw64, env: x86_64, ziparch: win64, shell: msys2 }
- { msystem: mingw32, env: i686, ziparch: win32, shell: msys2 }
- { msystem: null, env: arm64, ziparch: win-arm64, shell: cmake }
- { msystem: mingw64, env: x86_64, ziparch: win64 }
- { msystem: mingw32, env: i686, ziparch: win32 }
defaults:
run:
shell: ${{ matrix.shell == 'cmake' && 'pwsh' || 'msys2 {0}' }}
shell: msys2 {0}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # tag=v6.0.2
# MSYS2 setup
- uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c # tag=v2.31.1
if: matrix.shell == 'msys2'
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # tag=v3
- uses: msys2/setup-msys2@5beef6d11f48bba68b9eb503e3adc60b23c0cc36 # tag=v2
with:
msystem: ${{ matrix.msystem }}
install: make p7zip git mingw-w64-${{matrix.env}}-gcc
install: make zlib git p7zip mingw-w64-${{matrix.env}}-gcc
update: true
- name: display versions (MSYS2)
if: matrix.shell == 'msys2'
- name: display versions
run: |
make -v
cc -v
- name: display versions (CMake)
if: matrix.shell == 'cmake'
run: |
cmake --version
# Build dependencies (MSYS2 only)
- name: Building zlib to static link
if: matrix.shell == 'msys2'
run: |
git clone --depth 1 --branch v1.3.1 https://github.com/madler/zlib
git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib
make -C zlib -f win32/Makefile.gcc libz.a
- name: Building lz4 to static link
if: matrix.shell == 'msys2'
run: |
git clone --depth 1 --branch v1.10.0 https://github.com/lz4/lz4
# ensure both libraries use the same version of libxxhash
cp lib/common/xxhash.* lz4/lib
CPPFLAGS=-DXXH_NAMESPACE=LZ4_ make -C lz4/lib liblz4.a V=1
# Build zstd
- name: Building zstd programs
if: matrix.shell == 'msys2'
run: |
CPPFLAGS="-I../zlib -I../lz4/lib" LDFLAGS=-static make -j allzstd V=1 HAVE_ZLIB=1 HAVE_LZ4=1 HAVE_LZMA=0 LDLIBS="../zlib/libz.a ../lz4/lib/liblz4.a"
CPPFLAGS=-I../zlib LDFLAGS=../zlib/libz.a make -j allzstd MOREFLAGS=-static V=1
- name: Build zstd (CMake ARM64)
if: matrix.shell == 'cmake'
- name: Create artifacts
run: |
cd build\cmake
mkdir build
cd build
cmake.exe -G "Visual Studio 17 2022" -A ARM64 -DCMAKE_BUILD_TYPE=Release -DZSTD_BUILD_PROGRAMS=ON -DZSTD_BUILD_SHARED=ON -DZSTD_BUILD_STATIC=ON ..
cmake.exe --build . --config Release --parallel
- name: Create artifacts (MSYS2)
if: matrix.shell == 'msys2'
run: |
./lib/dll/example/build_package.bat || exit 1
./lib/dll/example/build_package.bat
mv bin/ zstd-${{ github.ref_name }}-${{matrix.ziparch}}/
7z a -tzip -mx9 zstd-${{ github.ref_name }}-${{matrix.ziparch}}.zip zstd-${{ github.ref_name }}-${{matrix.ziparch}}/
cd ..
- name: Create artifacts (CMake)
if: matrix.shell == 'cmake'
run: |
.\lib\dll\example\build_package.bat
if ($LASTEXITCODE -ne 0) { exit 1 }
mv bin/ zstd-${{ github.ref_name }}-${{matrix.ziparch}}/
- name: Publish zstd-$VERSION-${{matrix.ziparch}}.zip for manual inspection
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # tag=v7.0.1
- name: Publish zstd-$VERSION-${{matrix.ziparch}}.zip
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # tag=v4.3.1
with:
compression-level: 9 # maximum compression
if-no-files-found: error # defaults to `warn`
path: ${{ github.workspace }}/zstd-${{ github.ref_name }}-${{matrix.ziparch}}/
name: zstd-${{ github.ref_name }}-${{matrix.ziparch}}
- name: Package artifact for upload (MSYS2)
if: matrix.shell == 'msys2'
run: |
7z a -tzip -mx9 "$(cygpath -u '${{ github.workspace }}/zstd-${{ github.ref_name }}-${{ matrix.ziparch }}.zip')" "$(cygpath -u '${{ github.workspace }}/zstd-${{ github.ref_name }}-${{ matrix.ziparch }}')"
- name: Package artifact for upload (CMake)
if: matrix.shell == 'cmake'
run: |
Compress-Archive -Path "zstd-${{ github.ref_name }}-${{ matrix.ziparch }}" -DestinationPath "zstd-${{ github.ref_name }}-${{ matrix.ziparch }}.zip" -CompressionLevel Optimal
- name: Upload release asset
if: github.event_name == 'release'
shell: pwsh
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release upload "${{ github.ref_name }}" "$env:GITHUB_WORKSPACE/zstd-${{ github.ref_name }}-${{ matrix.ziparch }}.zip" --clobber
path: ${{ github.workspace }}/zstd-${{ github.ref_name }}-${{matrix.ziparch}}.zip
name: zstd-${{ github.ref_name }}-${{matrix.ziparch}}.zip
+10 -13
View File
@@ -12,8 +12,6 @@
*.so
*.so.*
*.dylib
*.framework
*.xcframework
# Executables
/zstd
@@ -22,6 +20,16 @@ zstdmt
*.out
*.app
# Test artefacts
tmp*
*.zst
*.zstd
dictionary.
dictionary
NUL
cmakebuild/
install/
# Build artefacts
contrib/linux-kernel/linux/
projects/
@@ -30,17 +38,6 @@ bin/
buck-out/
build-*
*.gcda
cmakebuild/
cmake-build/
# Test artefacts
tmp*
*.zst
*.zstd
dictionary.
dictionary
NUL
install/
# IDE
.clang_complete
+2 -32
View File
@@ -1,39 +1,9 @@
v1.6.0 (Dec 2025)
api: legacy format support is now disabled by default
build: `ZSTD_LEGACY_SUPPORT` defaults to `0` in Makefile and CMake
V1.5.7 (Feb 2025)
fix: compression bug in 32-bit mode associated with long-lasting sessions
api: new method `ZSTD_compressSequencesAndLiterals()` (#4217, #4232)
api: `ZSTD_getFrameHeader()` works on skippable frames (#4228)
perf: substantial compression speed improvements (up to +30%) on small data, by @TocarIP (#4144) and @cyan4973 (#4165)
perf: improved compression speed (~+5%) for dictionary compression at low levels (#4170)
perf: much faster speed for `--patch-from` at high compression levels (#4276)
perf: higher `--patch-from` compression ratios, notably at high levels (#4288)
perf: better speed for binaries on Windows (@pps83) and when compiled with Visual Studio (@MessyHack)
perf: slight compression ratio improvement thanks to better block boundaries (#4136, #4176, #4178)
perf: slight compression ratio improvement for `dfast`, aka levels 3 and 4 (#4171)
perf: runtime bmi2 detection enabled on x86 32-bit mode (#4251)
cli: multi-threading as default CLI setting, by @daniellerozenblit
cli: new `--max` command (#4290)
build: improve `msbuild` version autodetection, support VS2022, by @ManuelBlanc
build: fix `meson` build by @artem and @Victor-C-Zhang, and on Windows by @bgilbert
build: compatibility with Apple Framework, by @Treata11
build: improve icc/icx compatibility, by @josepho0918 and @luau-project
build: improve compatibility with Android NDK, by Adenilson Cavalcanti
portability: linux kernel branch, with improved support for Sequence producers (@embg, @gcabiddu, @cyan4973)
portability: improved qnx compatibility, suggested by @rainbowball
portability: improved install script for FreeBSD, by @sunpoet
portability: fixed test suite compatibility with gnu hurd, by @diegonc
doc: clarify specification, by @elasota
misc: improved tests/decodecorpus validation tool (#4102), by antmicro
V1.5.6 (Mar 2024)
api: Promote `ZSTD_c_targetCBlockSize` to Stable API by @felixhandte
api: new `ZSTD_d_maxBlockSize` experimental parameter, to reduce streaming decompression memory, by @terrelln
perf: improve performance of param `ZSTD_c_targetCBlockSize`, by @Cyan4973
perf: improved compression of arrays of integers at high compression, by @Cyan4973
lib: reduce binary size with selective build-time exclusion, by @felixhandte
lib: reduce binary size with selective built-time exclusion, by @felixhandte
lib: improved huffman speed on small data and linux kernel, by @terrelln
lib: accept dictionaries with partial literal tables, by @terrelln
lib: fix CCtx size estimation with external sequence producer, by @embg
@@ -519,7 +489,7 @@ misc: added /contrib/docker script by @gyscos
v1.3.3 (Dec 21, 2017)
perf: faster zstd_opt strategy (levels 16-19)
fix : bug #944 : multithreading with shared dictionary and large data, reported by @gsliepen
fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
cli : fix : content size written in header by default
cli : fix : improved LZ4 format support, by @felixhandte
cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
-11
View File
@@ -1,11 +0,0 @@
cmake_minimum_required(VERSION 3.10)
# Thin wrapper so `cmake -S .` behaves like `cmake -S build/cmake`.
# Policy lives in build/cmake; keep parent project language-less.
project(zstd-superbuild LANGUAGES NONE)
if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR)
message(FATAL_ERROR "In-source builds are not supported. Specify -B <build-dir>.")
endif()
add_subdirectory(build/cmake)
+2 -7
View File
@@ -60,7 +60,7 @@ Our contribution process works in three main stages:
* Note: run local tests to ensure that your changes didn't break existing functionality
* Quick check
```
make check
make shortest
```
* Longer check
```
@@ -373,12 +373,7 @@ counter `L1-dcache-load-misses`
#### Visual Studio
Build Zstd with symbols first (for example `cmake -B build -S build/cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo && ninja -C build zstd`) so the profiler resolves call stacks.
* Launch Visual Studios Performance Profiler (`Alt+F2`), enable CPU Usage (optionally Instrumentation), and point it at the `programs/zstd` benchmark you want to run.
* If you prefer to start the benchmark from a terminal, use “Attach to running process” to latch onto it mid-run; keep frame pointers (`-fno-omit-frame-pointer`) for clean stacks.
* When you stop the capture, review the call tree, hot path, and annotated source panes
* Microsofts [Performance Profiling docs](https://learn.microsoft.com/en-us/visualstudio/profiling/?view=vs-2022) cover deeper sampling, ETW, and collection options if required.
TODO
## Issues
We use GitHub issues to track public bugs. Please ensure your description is
+23 -36
View File
@@ -85,10 +85,14 @@ test:
$(MAKE) -C $(TESTDIR) $@
ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@
## shortest: same as `make check`
.PHONY: shortest
shortest:
$(Q)$(MAKE) -C $(TESTDIR) $@
## check: run basic tests for `zstd` cli
.PHONY: check
check:
$(Q)$(MAKE) -C $(TESTDIR) $@
check: shortest
.PHONY: automated_benchmarking
automated_benchmarking:
@@ -141,16 +145,13 @@ clean:
$(Q)$(MAKE) -C contrib/largeNbDicts $@ > $(VOID)
$(Q)$(MAKE) -C contrib/externalSequenceProducer $@ > $(VOID)
$(Q)$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
$(Q)$(RM) -r lz4 cmakebuild mesonbuild install
$(Q)$(RM) -r lz4 cmakebuild install
@echo Cleaning completed
LIBZSTD_MK_DIR = $(ZSTDDIR)
include $(LIBZSTD_MK_DIR)/install_oses.mk # UNAME, INSTALL_OS_LIST
#------------------------------------------------------------------------------
# make install is validated only for Linux, macOS, Hurd and some BSD targets
#------------------------------------------------------------------------------
ifneq (,$(filter $(INSTALL_OS_LIST),$(UNAME)))
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT CYGWIN_NT Haiku AIX))
HOST_OS = POSIX
@@ -199,9 +200,9 @@ travis-install:
.PHONY: clangbuild-darwin-fat
clangbuild-darwin-fat: clean
clang -v
CXX=clang++ CC=clang CFLAGS+="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch arm64" $(MAKE) zstd-release
CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch arm64" $(MAKE) zstd-release
mv programs/zstd programs/zstd_arm64
CXX=clang++ CC=clang CFLAGS+="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch x86_64" $(MAKE) zstd-release
CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation -arch x86_64" $(MAKE) zstd-release
mv programs/zstd programs/zstd_x64
lipo -create programs/zstd_x64 programs/zstd_arm64 -output programs/zstd
@@ -292,16 +293,16 @@ regressiontest:
$(MAKE) -C $(FUZZDIR) regressiontest
uasanregressiontest:
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined -Werror" CXXFLAGS="-O3 -fsanitize=address,undefined -Werror"
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=address,undefined" CXXFLAGS="-O3 -fsanitize=address,undefined"
msanregressiontest:
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory -Werror" CXXFLAGS="-O3 -fsanitize=memory -Werror"
$(MAKE) -C $(FUZZDIR) regressiontest CC=clang CXX=clang++ CFLAGS="-O3 -fsanitize=memory" CXXFLAGS="-O3 -fsanitize=memory"
update_regressionResults : REGRESS_RESULTS_DIR := /tmp/regress_results_dir/
update_regressionResults:
$(MAKE) -j -C programs zstd
$(MAKE) -j -C tests/regression test
$(RM) -r $(REGRESS_RESULTS_DIR)
$(MAKE) -C programs zstd
$(MAKE) -C tests/regression test
$(RM) -rf $(REGRESS_RESULTS_DIR)
$(MKDIR) $(REGRESS_RESULTS_DIR)
./tests/regression/test \
--cache tests/regression/cache \
@@ -354,7 +355,7 @@ apt-add-repo:
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
sudo apt-get update -y -qq
.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install
.PHONY: ppcinstall arminstall valgrindinstall libc6install gcc6install gcc7install gcc8install gpp6install clang38install lz4install
ppcinstall:
APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
@@ -382,22 +383,26 @@ gpp6install: apt-add-repo
clang38install:
APT_PACKAGES="clang-3.8" $(MAKE) apt-install
# Ubuntu 14.04 ships a too-old lz4
lz4install:
[ -e lz4 ] || git clone https://github.com/lz4/lz4 && sudo $(MAKE) -C lz4 install
endif
ifneq (,$(filter MSYS%,$(shell sh -c 'MSYSTEM="MSYS" uname') ))
ifneq (,$(filter MSYS%,$(shell uname)))
HOST_OS = MSYS
endif
#------------------------------------------------------------------------
# target specific tests
#------------------------------------------------------------------------
ifneq (,$(filter MSYS POSIX,$(HOST_OS)))
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
CMAKE ?= cmake
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
ifneq (,$(filter MSYS%,$(shell sh -c 'MSYSTEM="MSYS" uname')))
ifneq (,$(filter MSYS%,$(shell uname)))
CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
endif
@@ -410,24 +415,6 @@ cmakebuild:
$(CMAKE) --build cmakebuild --target install -- -j V=1
cd cmakebuild; ctest -V -L Medium
MESON ?= meson
NINJA ?= ninja
.PHONY: mesonbuild
mesonbuild:
$(MESON) setup \
--buildtype=debugoptimized \
-Db_lundef=false \
-Dauto_features=enabled \
-Dbin_programs=true \
-Dbin_tests=true \
-Dbin_contrib=true \
-Ddefault_library=both \
build/meson mesonbuild
$(NINJA) -C mesonbuild/
$(MESON) test -C mesonbuild/ --print-errorlogs
$(MESON) install -C mesonbuild --destdir staging/
.PHONY: c89build gnu90build c99build gnu99build c11build bmix64build bmix32build bmi32build staticAnalyze
c89build: clean
$(CC) -v
+38 -55
View File
@@ -29,10 +29,10 @@ a list of known ports and bindings is provided on [Zstandard homepage](https://f
## Benchmarks
For reference, several fast compression algorithms were tested and compared
on a desktop featuring a Core i7-9700K CPU @ 4.9GHz
and running Ubuntu 24.04 (`Linux 6.8.0-53-generic`),
on a desktop running Ubuntu 20.04 (`Linux 5.11.0-41-generic`),
with a Core i7-9700K CPU @ 4.9GHz,
using [lzbench], an open-source in-memory benchmark by @inikep
compiled with [gcc] 14.2.0,
compiled with [gcc] 9.3.0,
on the [Silesia compression corpus].
[lzbench]: https://github.com/inikep/lzbench
@@ -41,23 +41,24 @@ on the [Silesia compression corpus].
| Compressor name | Ratio | Compression| Decompress.|
| --------------- | ------| -----------| ---------- |
| **zstd 1.5.7 -1** | 2.896 | 510 MB/s | 1550 MB/s |
| brotli 1.1.0 -1 | 2.883 | 290 MB/s | 425 MB/s |
| [zlib] 1.3.1 -1 | 2.743 | 105 MB/s | 390 MB/s |
| **zstd 1.5.7 --fast=1** | 2.439 | 545 MB/s | 1850 MB/s |
| quicklz 1.5.0 -1 | 2.238 | 520 MB/s | 750 MB/s |
| **zstd 1.5.7 --fast=4** | 2.146 | 665 MB/s | 2050 MB/s |
| lzo1x 2.10 -1 | 2.106 | 650 MB/s | 780 MB/s |
| [lz4] 1.10.0 | 2.101 | 675 MB/s | 3850 MB/s |
| snappy 1.2.1 | 2.089 | 520 MB/s | 1500 MB/s |
| lzf 3.6 -1 | 2.077 | 410 MB/s | 820 MB/s |
| **zstd 1.5.1 -1** | 2.887 | 530 MB/s | 1700 MB/s |
| [zlib] 1.2.11 -1 | 2.743 | 95 MB/s | 400 MB/s |
| brotli 1.0.9 -0 | 2.702 | 395 MB/s | 450 MB/s |
| **zstd 1.5.1 --fast=1** | 2.437 | 600 MB/s | 2150 MB/s |
| **zstd 1.5.1 --fast=3** | 2.239 | 670 MB/s | 2250 MB/s |
| quicklz 1.5.0 -1 | 2.238 | 540 MB/s | 760 MB/s |
| **zstd 1.5.1 --fast=4** | 2.148 | 710 MB/s | 2300 MB/s |
| lzo1x 2.10 -1 | 2.106 | 660 MB/s | 845 MB/s |
| [lz4] 1.9.3 | 2.101 | 740 MB/s | 4500 MB/s |
| lzf 3.6 -1 | 2.077 | 410 MB/s | 830 MB/s |
| snappy 1.1.9 | 2.073 | 550 MB/s | 1750 MB/s |
[zlib]: https://www.zlib.net/
[lz4]: https://lz4.github.io/lz4/
The negative compression levels, specified with `--fast=#`,
offer faster compression and decompression speed
at the cost of compression ratio.
at the cost of compression ratio (compared to level 1).
Zstd can also offer stronger compression ratios at the cost of compression speed.
Speed vs Compression trade-off is configurable by small increments.
@@ -120,40 +121,33 @@ Dictionary gains are mostly effective in the first few KB. Then, the compression
## Build instructions
`make` is the main build system of this project.
It is the reference, and other build systems are periodically updated to stay compatible.
However, small drifts and feature differences can be present, since perfect synchronization is difficult.
For this reason, when your build system allows it, prefer employing `make`.
`make` is the officially maintained build system of this project.
All other build systems are "compatible" and 3rd-party maintained,
they may feature small differences in advanced options.
When your system allows it, prefer using `make` to build `zstd` and `libzstd`.
### Makefile
Assuming your system supports standard `make` (or `gmake`),
just invoking `make` in root directory generates `zstd` cli at root,
and also generates `libzstd` into `lib/`.
If your system is compatible with standard `make` (or `gmake`),
invoking `make` in root directory will generate `zstd` cli in root directory.
It will also create `libzstd` into `lib/`.
Other standard targets include:
- `make install` : install zstd cli, library and man pages
- `make check` : run `zstd`, test its essential behavior on local platform
Other available options include:
- `make install` : create and install zstd cli, library and man pages
- `make check` : create and run `zstd`, test its behavior on local platform
The `Makefile` follows the [GNU Standard Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html),
allowing staged install, standard compilation flags, directory variables and command variables.
allowing staged install, standard flags, directory variables and command variables.
For advanced use cases, specialized flags which control binary generation and installation paths are documented
in [`lib/README.md`](lib/README.md#modular-build) for the `libzstd` library
For advanced use cases, specialized compilation flags which control binary generation
are documented in [`lib/README.md`](lib/README.md#modular-build) for the `libzstd` library
and in [`programs/README.md`](programs/README.md#compilation-variables) for the `zstd` CLI.
### cmake
A `cmake` project generator is available for generating Makefiles or other build scripts
to create the `zstd` binary as well as `libzstd` dynamic and static libraries.
The repository root now contains a minimal `CMakeLists.txt` that forwards to `build/cmake`,
so you can configure the project with a standard `cmake -S .` invocation,
while the historical `cmake -S build/cmake` entry point remains fully supported.
```bash
cmake -S . -B build-cmake
cmake --build build-cmake
```
A `cmake` project generator is provided within `build/cmake`.
It can generate Makefiles or other build scripts
to create `zstd` binary, and `libzstd` dynamic and static libraries.
By default, `CMAKE_BUILD_TYPE` is set to `Release`.
@@ -163,7 +157,7 @@ By default, `CMAKE_BUILD_TYPE` is set to `Release`.
To perform a Fat/Universal2 build and install use the following commands:
```bash
cmake -S . -B build-cmake-debug -G Ninja -DCMAKE_OSX_ARCHITECTURES="x86_64;x86_64h;arm64"
cmake -B build-cmake-debug -S build/cmake -G Ninja -DCMAKE_OSX_ARCHITECTURES="x86_64;x86_64h;arm64"
cd build-cmake-debug
ninja
sudo ninja install
@@ -191,25 +185,13 @@ You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) depe
The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors.
If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
### Conan
You can install pre-built binaries for zstd or build it from source using [Conan](https://conan.io/). Use the following command:
```bash
conan install --requires="zstd/[*]" --build=missing
```
The zstd Conan recipe is kept up to date by Conan maintainers and community contributors.
If the version is out of date, please [create an issue or pull request](https://github.com/conan-io/conan-center-index) on the ConanCenterIndex repository.
### Visual Studio (Windows)
Going into `build` directory, you will find additional possibilities:
- Projects for Visual Studio 2008 and 2010.
- Projects for Visual Studio 2005, 2008 and 2010.
+ VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017.
- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`,
which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
- It is now recommended to generate Visual Studio solutions from `cmake`
### Buck
@@ -218,7 +200,7 @@ The output binary will be in `buck-out/gen/programs/`.
### Bazel
You can integrate zstd into your Bazel project by using the module hosted on the [Bazel Central Repository](https://registry.bazel.build/modules/zstd).
You easily can integrate zstd into your Bazel project by using the module hosted on the [Bazel Central Repository](https://registry.bazel.build/modules/zstd).
## Testing
@@ -229,9 +211,9 @@ For information on CI testing, please refer to `TESTING.md`.
## Status
Zstandard is deployed within Meta and many other large cloud infrastructures,
to compress humongous amounts of data in various formats and use cases.
It is also continuously fuzzed for security issues by Google's [oss-fuzz](https://github.com/google/oss-fuzz/tree/master/projects/zstd) program.
Zstandard is currently deployed within Facebook and many other large cloud infrastructures.
It is run continuously to compress large amounts of data in multiple formats and use cases.
Zstandard is considered safe for production environments.
## License
@@ -240,5 +222,6 @@ Zstandard is dual-licensed under [BSD](LICENSE) OR [GPLv2](COPYING).
## Contributing
The `dev` branch is the one where all contributions are merged before reaching `release`.
If you plan to propose a patch, please commit into the `dev` branch, or its own feature branch.
Direct commit to `release` are not permitted.
For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
+4 -4
View File
@@ -45,7 +45,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -122,7 +122,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
@@ -197,7 +197,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -275,7 +275,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
+4 -4
View File
@@ -45,7 +45,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -121,7 +121,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
@@ -195,7 +195,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -272,7 +272,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
@@ -0,0 +1,189 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{00000000-1CC8-4FD7-9281-6B8DBB9D3DF8}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>fullbench-dll</RootNamespace>
<OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
<ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast>
<TreatWarningAsError>false</TreatWarningAsError>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
<ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalLibraryDirectories>$(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\programs\util.c" />
<ClCompile Include="..\..\..\programs\timefn.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\benchfn.c" />
<ClCompile Include="..\..\..\tests\fullbench.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\programs\datagen.h" />
<ClInclude Include="..\..\..\programs\benchfn.h" />
<ClInclude Include="..\..\..\programs\util.h" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\libzstd-dll\libzstd-dll.vcxproj">
<Project>{00000000-94d5-4bf9-8a50-7bd9929a0850}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
-2
View File
@@ -170,7 +170,6 @@
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
<ClCompile Include="..\..\..\lib\compress\zstdmt_compress.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
@@ -184,7 +183,6 @@
<ClCompile Include="..\..\..\programs\util.c" />
<ClCompile Include="..\..\..\programs\timefn.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\lorem.c" />
<ClCompile Include="..\..\..\programs\benchfn.c" />
<ClCompile Include="..\..\..\tests\fullbench.c" />
</ItemGroup>
-1
View File
@@ -170,7 +170,6 @@
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
+4 -5
View File
@@ -34,7 +34,6 @@
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -169,7 +168,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<DebugInformationFormat>EditAndContinue</DebugInformationFormat>
@@ -188,7 +187,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -208,7 +207,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
@@ -229,7 +228,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+4 -5
View File
@@ -34,7 +34,6 @@
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -162,7 +161,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<DebugInformationFormat>EditAndContinue</DebugInformationFormat>
@@ -181,7 +180,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -201,7 +200,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
@@ -222,7 +221,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+5
View File
@@ -7,6 +7,11 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcx
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcxproj", "{61ABD629-1CC8-4FD7-9281-6B8DBB9D3DF8}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll\fullbench-dll.vcxproj", "{00000000-1CC8-4FD7-9281-6B8DBB9D3DF8}"
ProjectSection(ProjectDependencies) = postProject
{00000000-94D5-4BF9-8A50-7BD9929A0850} = {00000000-94D5-4BF9-8A50-7BD9929A0850}
EndProjectSection
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{037E781E-81A6-494B-B1B3-438AB1200523}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libzstd", "libzstd\libzstd.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}"
+4 -5
View File
@@ -35,7 +35,6 @@
<ClCompile Include="..\..\..\lib\compress\zstd_compress_literals.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress_sequences.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress_superblock.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_preSplit.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_fast.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_double_fast.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_lazy.c" />
@@ -187,7 +186,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
@@ -204,7 +203,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<EnableEnhancedInstructionSet>$(InstructionSet)</EnableEnhancedInstructionSet>
@@ -223,7 +222,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast>
<TreatWarningAsError>false</TreatWarningAsError>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@@ -245,7 +244,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
-7
View File
@@ -1,7 +0,0 @@
@echo off
rem build 32-bit
call "%~p0%build.generic.cmd" preview Win32 Release v143
rem build 64-bit
call "%~p0%build.generic.cmd" preview x64 Release v143
+22 -17
View File
@@ -2,7 +2,7 @@
IF "%1%" == "" GOTO display_help
SETLOCAL ENABLEDELAYEDEXPANSION
SETLOCAL
SET msbuild_version=%1
@@ -19,34 +19,39 @@ GOTO build
:display_help
echo Syntax: build.generic.cmd msbuild_version msbuild_platform msbuild_configuration msbuild_toolset
echo msbuild_version: VS installed version (latest, VS2012, VS2013, VS2015, VS2017, VS2019, VS2022, ...)
echo msbuild_version: VS installed version (VS2012, VS2013, VS2015, VS2017, VS2019, ...)
echo msbuild_platform: Platform (x64 or Win32)
echo msbuild_configuration: VS configuration (Release or Debug)
echo msbuild_toolset: Platform Toolset (v100, v110, v120, v140, v141, v142, v143, ...)
echo msbuild_toolset: Platform Toolset (v100, v110, v120, v140, v141, v142, ...)
EXIT /B 1
:build
SET msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe"
SET msbuild_vs2017community="%programfiles(x86)%\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe"
SET msbuild_vs2017professional="%programfiles(x86)%\Microsoft Visual Studio\2017\Professional\MSBuild\15.0\Bin\MSBuild.exe"
SET msbuild_vs2017enterprise="%programfiles(x86)%\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\MSBuild.exe"
IF %msbuild_version% == VS2013 SET msbuild="%programfiles(x86)%\MSBuild\12.0\Bin\MSBuild.exe"
IF %msbuild_version% == VS2015 SET msbuild="%programfiles(x86)%\MSBuild\14.0\Bin\MSBuild.exe"
IF %msbuild_version% == VS2017 SET vswhere_params=-version [15,16) -products *
IF %msbuild_version% == VS2017Community SET vswhere_params=-version [15,16) -products Community
IF %msbuild_version% == VS2017Enterprise SET vswhere_params=-version [15,16) -products Enterprise
IF %msbuild_version% == VS2017Professional SET vswhere_params=-version [15,16) -products Professional
IF %msbuild_version% == VS2019 SET vswhere_params=-version [16,17) -products *
IF %msbuild_version% == VS2022 SET vswhere_params=-version [17,18) -products *
REM Add the next Visual Studio version here.
IF %msbuild_version% == latest SET vswhere_params=-latest -products *
IF %msbuild_version% == preview SET vswhere_params=-prerelease -products *
IF %msbuild_version% == VS2017Community SET msbuild=%msbuild_vs2017community%
IF %msbuild_version% == VS2017Professional SET msbuild=%msbuild_vs2017professional%
IF %msbuild_version% == VS2017Enterprise SET msbuild=%msbuild_vs2017enterprise%
IF %msbuild_version% == VS2017 (
IF EXIST %msbuild_vs2017community% SET msbuild=%msbuild_vs2017community%
IF EXIST %msbuild_vs2017professional% SET msbuild=%msbuild_vs2017professional%
IF EXIST %msbuild_vs2017enterprise% SET msbuild=%msbuild_vs2017enterprise%
)
IF NOT DEFINED vswhere_params GOTO skip_vswhere
SET vswhere="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
FOR /F "USEBACKQ TOKENS=*" %%F IN (`%vswhere% !vswhere_params! -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe`) DO (
SET msbuild="%%F"
:: VS2019
SET msbuild_vs2019community="%programfiles(x86)%\Microsoft Visual Studio\2019\Community\MSBuild\Current\Bin\MSBuild.exe"
SET msbuild_vs2019professional="%programfiles(x86)%\Microsoft Visual Studio\2019\Professional\MSBuild\Current\Bin\MSBuild.exe"
SET msbuild_vs2019enterprise="%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe"
IF %msbuild_version% == VS2019 (
IF EXIST %msbuild_vs2019community% SET msbuild=%msbuild_vs2019community%
IF EXIST %msbuild_vs2019professional% SET msbuild=%msbuild_vs2019professional%
IF EXIST %msbuild_vs2019enterprise% SET msbuild=%msbuild_vs2019enterprise%
)
:skip_vswhere
SET project="%~p0\..\VS2010\zstd.sln"
+173 -38
View File
@@ -7,75 +7,210 @@
# in the COPYING file in the root directory of this source tree).
# ################################################################
cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
# As of 2018-12-26 ZSTD has been validated to build with cmake version 3.13.2 new policies.
# Set and use the newest cmake policies that are validated to work
set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3")
set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") #Policies never changed at PATCH level
if("${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND
"${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
else()
set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0")
endif()
cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
set(CMAKE_BUILD_WITH_INSTALL_RPATH on)
#-----------------------------------------------------------------------------
# Setup CMake environment
#-----------------------------------------------------------------------------
set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
# Define project paths
set(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib)
# Parse version
include(GetZstdLibraryVersion)
GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH)
#-----------------------------------------------------------------------------
# Configure CMake policies and version
#-----------------------------------------------------------------------------
include(ZstdVersion)
#-----------------------------------------------------------------------------
# Project declaration
#-----------------------------------------------------------------------------
project(zstd
VERSION "${ZSTD_FULL_VERSION}"
LANGUAGES C # Main library is in C and ASM, ASM is enabled conditionally
HOMEPAGE_URL "${zstd_HOMEPAGE_URL}"
DESCRIPTION "${zstd_DESCRIPTION}"
)
VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}"
LANGUAGES C # Main library is in C
ASM # And ASM
CXX # Testing contributed code also utilizes CXX
)
#-----------------------------------------------------------------------------
# Build type configuration
#-----------------------------------------------------------------------------
message(STATUS "ZSTD VERSION: ${zstd_VERSION}")
set(zstd_HOMEPAGE_URL "https://facebook.github.io/zstd")
set(zstd_DESCRIPTION "Zstandard is a real-time compression algorithm, providing high compression ratios.")
# Set a default build type if none was specified
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
message(STATUS "Setting build type to 'Release' as none was specified.")
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
# Set the possible values of build type for cmake-gui
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
endif()
#-----------------------------------------------------------------------------
# Include standard modules
#-----------------------------------------------------------------------------
include(GNUInstallDirs)
#-----------------------------------------------------------------------------
# Display installation information
# Add extra compilation flags
#-----------------------------------------------------------------------------
include(AddZstdCompilationFlags)
ADD_ZSTD_COMPILATION_FLAGS()
# Always hide XXHash symbols
add_definitions(-DXXH_NAMESPACE=ZSTD_)
#-----------------------------------------------------------------------------
# Installation variables
#-----------------------------------------------------------------------------
message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}")
message(STATUS "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
#-----------------------------------------------------------------------------
# Configure build options
# Options
#-----------------------------------------------------------------------------
include(ZstdOptions)
# Legacy support
option(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" ON)
if (ZSTD_LEGACY_SUPPORT)
message(STATUS "ZSTD_LEGACY_SUPPORT defined!")
set(ZSTD_LEGACY_LEVEL 5 CACHE STRING "")
add_definitions(-DZSTD_LEGACY_SUPPORT=${ZSTD_LEGACY_LEVEL})
else ()
message(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
add_definitions(-DZSTD_LEGACY_SUPPORT=0)
endif ()
if (ANDROID)
set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT OFF)
else()
set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT ON)
endif()
# Multi-threading support
option(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ${ZSTD_MULTITHREAD_SUPPORT_DEFAULT})
if (ZSTD_MULTITHREAD_SUPPORT)
message(STATUS "ZSTD_MULTITHREAD_SUPPORT is enabled")
else ()
message(STATUS "ZSTD_MULTITHREAD_SUPPORT is disabled")
endif ()
option(ZSTD_BUILD_PROGRAMS "BUILD PROGRAMS" ON)
option(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
# Respect the conventional CMake option for enabling tests if it was specified on the first configure
if (BUILD_TESTING)
set(ZSTD_BUILD_TESTS_default ON)
else()
set(ZSTD_BUILD_TESTS_default OFF)
endif()
option(ZSTD_BUILD_TESTS "BUILD TESTS" ${ZSTD_BUILD_TESTS_default})
if (MSVC)
option(ZSTD_USE_STATIC_RUNTIME "LINK TO STATIC RUN-TIME LIBRARIES" OFF)
endif ()
#-----------------------------------------------------------------------------
# Configure compilation flags
# External dependencies
#-----------------------------------------------------------------------------
include(AddZstdCompilationFlags)
ADD_ZSTD_COMPILATION_FLAGS(ON ZSTD_ENABLE_CXX ON)
# Define a function to handle special thread settings for HP-UX
# See https://github.com/facebook/zstd/pull/3862 for details.
function(setup_hpux_threads)
find_package(Threads)
if (NOT Threads_FOUND)
set(CMAKE_USE_PTHREADS_INIT 1 PARENT_SCOPE)
set(CMAKE_THREAD_LIBS_INIT -lpthread PARENT_SCOPE)
set(CMAKE_HAVE_THREADS_LIBRARY 1 PARENT_SCOPE)
set(Threads_FOUND TRUE PARENT_SCOPE)
endif()
endfunction()
if (ZSTD_MULTITHREAD_SUPPORT AND UNIX)
if (CMAKE_SYSTEM_NAME MATCHES "HP-UX")
setup_hpux_threads()
else()
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
endif()
if (CMAKE_USE_PTHREADS_INIT)
set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
else()
message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
endif()
endif ()
#-----------------------------------------------------------------------------
# Configure dependencies
# Add source directories
#-----------------------------------------------------------------------------
include(ZstdDependencies)
add_subdirectory(lib)
option(ZSTD_PROGRAMS_LINK_SHARED "PROGRAMS LINK SHARED" OFF)
if (ZSTD_BUILD_PROGRAMS)
if (NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
message(SEND_ERROR "You need to build static library to build zstd CLI")
elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
message(SEND_ERROR "You need to build shared library to build zstd CLI")
endif ()
add_subdirectory(programs)
endif ()
if (ZSTD_BUILD_TESTS)
enable_testing()
if (NOT ZSTD_BUILD_STATIC)
message(SEND_ERROR "You need to build static library to build tests")
endif ()
add_subdirectory(tests)
endif ()
if (ZSTD_BUILD_CONTRIB)
add_subdirectory(contrib)
endif ()
#-----------------------------------------------------------------------------
# Configure build targets
# Add clean-all target
#-----------------------------------------------------------------------------
include(ZstdBuild)
add_custom_target(clean-all
COMMAND ${CMAKE_BUILD_TOOL} clean
COMMAND rm -rf ${CMAKE_BINARY_DIR}/
)
#-----------------------------------------------------------------------------
# Configure package generation
# Generate Package Config files
#
# This section is based on the boiler plate code from:
# https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#creating-packages
#-----------------------------------------------------------------------------
include(ZstdPackage)
include(CMakePackageConfigHelpers)
write_basic_package_version_file(
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
VERSION ${zstd_VERSION}
COMPATIBILITY SameMajorVersion
)
# A Package Config file that works from the build directory
export(EXPORT zstdExports
FILE "${CMAKE_CURRENT_BINARY_DIR}/zstdTargets.cmake"
NAMESPACE zstd::
)
# A Package Config file that works from the installation directory
set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/zstd)
install(EXPORT zstdExports
FILE zstdTargets.cmake
NAMESPACE zstd::
DESTINATION ${ConfigPackageLocation}
)
configure_package_config_file(
zstdConfig.cmake.in
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
INSTALL_DESTINATION ${ConfigPackageLocation}
)
install(FILES
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
DESTINATION ${ConfigPackageLocation}
)
@@ -1,12 +1,11 @@
include(CheckCXXCompilerFlag)
include(CheckCCompilerFlag)
if(CMAKE_CXX_COMPILER)
include(CheckCXXCompilerFlag)
endif()
if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
set(ZSTD_HAVE_CHECK_LINKER_FLAG true)
else ()
# VERSION_GREATER_EQUAL requires CMake 3.7 or later.
# https://cmake.org/cmake/help/latest/command/if.html#version-greater-equal
if (CMAKE_VERSION VERSION_LESS 3.18)
set(ZSTD_HAVE_CHECK_LINKER_FLAG false)
else ()
set(ZSTD_HAVE_CHECK_LINKER_FLAG true)
endif ()
if (ZSTD_HAVE_CHECK_LINKER_FLAG)
include(CheckLinkerFlag)
@@ -23,7 +22,7 @@ function(EnableCompilerFlag _flag _C _CXX _LD)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_flag}" PARENT_SCOPE)
endif ()
endif ()
if (_CXX AND CMAKE_CXX_COMPILER)
if (_CXX)
CHECK_CXX_COMPILER_FLAG(${_flag} CXX_FLAG_${varname})
if (CXX_FLAG_${varname})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_flag}" PARENT_SCOPE)
@@ -51,65 +50,49 @@ function(EnableCompilerFlag _flag _C _CXX _LD)
endif ()
endfunction()
macro(ADD_ZSTD_COMPILATION_FLAGS _C _CXX _LD)
# We set ZSTD_HAS_NOEXECSTACK if we are certain we've set all the required
# compiler flags to mark the stack as non-executable.
set(ZSTD_HAS_NOEXECSTACK false)
macro(ADD_ZSTD_COMPILATION_FLAGS)
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" OR MINGW) #Not only UNIX but also WIN32 for MinGW
# It's possible to select the exact standard used for compilation.
# It's not necessary, but can be employed for specific purposes.
# Note that zstd source code is compatible with both C++98 and above
# and C-gnu90 (c90 + long long + variadic macros ) and above
# EnableCompilerFlag("-std=c++11" false true) # Set C++ compilation to c++11 standard
# EnableCompilerFlag("-std=c99" true false) # Set C compilation to c99 standard
# EnableCompilerFlag("-std=c99" true false) # Set C compiation to c99 standard
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND MSVC)
# clang-cl normally maps -Wall to -Weverything.
EnableCompilerFlag("/clang:-Wall" _C _CXX false)
EnableCompilerFlag("/clang:-Wall" true true false)
else ()
EnableCompilerFlag("-Wall" _C _CXX false)
EnableCompilerFlag("-Wall" true true false)
endif ()
EnableCompilerFlag("-Wextra" _C _CXX false)
EnableCompilerFlag("-Wundef" _C _CXX false)
EnableCompilerFlag("-Wshadow" _C _CXX false)
EnableCompilerFlag("-Wcast-align" _C _CXX false)
EnableCompilerFlag("-Wcast-qual" _C _CXX false)
EnableCompilerFlag("-Wstrict-prototypes" _C false false)
EnableCompilerFlag("-Wextra" true true false)
EnableCompilerFlag("-Wundef" true true false)
EnableCompilerFlag("-Wshadow" true true false)
EnableCompilerFlag("-Wcast-align" true true false)
EnableCompilerFlag("-Wcast-qual" true true false)
EnableCompilerFlag("-Wstrict-prototypes" true false false)
# Enable asserts in Debug mode
if (CMAKE_BUILD_TYPE MATCHES "Debug")
EnableCompilerFlag("-DDEBUGLEVEL=1" _C _CXX false)
EnableCompilerFlag("-DDEBUGLEVEL=1" true true false)
endif ()
# Add noexecstack flags
# LDFLAGS
EnableCompilerFlag("-Wl,-z,noexecstack" false false _LD)
EnableCompilerFlag("-z noexecstack" false false true)
# CFLAGS & CXXFLAGS
EnableCompilerFlag("-Qunused-arguments" _C _CXX false)
EnableCompilerFlag("-Wa,--noexecstack" _C _CXX false)
# NOTE: Using 3 nested ifs because the variables are sometimes
# empty if the condition is false, and sometimes equal to false.
# This implicitly converts them to truthy values. There may be
# a better way to do this, but this reliably works.
if (${LD_FLAG_WL_Z_NOEXECSTACK})
if (${C_FLAG_WA_NOEXECSTACK})
if (${CXX_FLAG_WA_NOEXECSTACK})
# We've succeeded in marking the stack as non-executable
set(ZSTD_HAS_NOEXECSTACK true)
endif()
endif()
endif()
EnableCompilerFlag("-Qunused-arguments" true true false)
EnableCompilerFlag("-Wa,--noexecstack" true true false)
elseif (MSVC) # Add specific compilation flags for Windows Visual
set(ACTIVATE_MULTITHREADED_COMPILATION "ON" CACHE BOOL "activate multi-threaded compilation (/MP flag)")
if (CMAKE_GENERATOR MATCHES "Visual Studio" AND ACTIVATE_MULTITHREADED_COMPILATION)
EnableCompilerFlag("/MP" _C _CXX false)
EnableCompilerFlag("/MP" true true false)
endif ()
# UNICODE SUPPORT
EnableCompilerFlag("/D_UNICODE" _C _CXX false)
EnableCompilerFlag("/DUNICODE" _C _CXX false)
EnableCompilerFlag("/D_UNICODE" true true false)
EnableCompilerFlag("/DUNICODE" true true false)
# Enable asserts in Debug mode
if (CMAKE_BUILD_TYPE MATCHES "Debug")
EnableCompilerFlag("/DDEBUGLEVEL=1" _C _CXX false)
EnableCompilerFlag("/DDEBUGLEVEL=1" true true false)
endif ()
endif ()
-42
View File
@@ -1,42 +0,0 @@
# ################################################################
# ZSTD Build Targets Configuration
# ################################################################
# Always build the library first (this defines ZSTD_BUILD_STATIC/SHARED options)
add_subdirectory(lib)
# Validate build configuration after lib options are defined
if(ZSTD_BUILD_PROGRAMS)
if(NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED)
message(SEND_ERROR "Static library required to build zstd CLI programs")
elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED)
message(SEND_ERROR "Shared library required to build zstd CLI programs")
endif()
endif()
if(ZSTD_BUILD_TESTS AND NOT ZSTD_BUILD_STATIC)
message(SEND_ERROR "Static library required to build test suite")
endif()
# Add programs if requested
if(ZSTD_BUILD_PROGRAMS)
add_subdirectory(programs)
endif()
# Add tests if requested
if(ZSTD_BUILD_TESTS)
enable_testing()
add_subdirectory(tests)
endif()
# Add contrib utilities if requested
if(ZSTD_BUILD_CONTRIB)
add_subdirectory(contrib)
endif()
# Clean-all target for thorough cleanup
add_custom_target(clean-all
COMMAND ${CMAKE_BUILD_TOOL} clean
COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/
COMMENT "Performing complete clean including build directory"
)
@@ -1,30 +0,0 @@
# ################################################################
# ZSTD Dependencies Configuration
# ################################################################
# Function to handle HP-UX thread configuration
function(setup_hpux_threads)
find_package(Threads)
if(NOT Threads_FOUND)
set(CMAKE_USE_PTHREADS_INIT 1 PARENT_SCOPE)
set(CMAKE_THREAD_LIBS_INIT -lpthread PARENT_SCOPE)
set(CMAKE_HAVE_THREADS_LIBRARY 1 PARENT_SCOPE)
set(Threads_FOUND TRUE PARENT_SCOPE)
endif()
endfunction()
# Configure threading support
if(ZSTD_MULTITHREAD_SUPPORT AND UNIX)
if(CMAKE_SYSTEM_NAME MATCHES "HP-UX")
setup_hpux_threads()
else()
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
endif()
if(CMAKE_USE_PTHREADS_INIT)
set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}")
else()
message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads")
endif()
endif()
@@ -1,68 +0,0 @@
# ################################################################
# ZSTD Build Options Configuration
# ################################################################
# Legacy support configuration (disabled by default)
option(ZSTD_LEGACY_SUPPORT "Enable legacy format support" OFF)
if(ZSTD_LEGACY_SUPPORT)
message(STATUS "ZSTD_LEGACY_SUPPORT enabled")
set(ZSTD_LEGACY_LEVEL 5 CACHE STRING "Legacy support level")
add_definitions(-DZSTD_LEGACY_SUPPORT=${ZSTD_LEGACY_LEVEL})
else()
message(STATUS "ZSTD_LEGACY_SUPPORT disabled")
add_definitions(-DZSTD_LEGACY_SUPPORT=0)
endif()
# Platform-specific options
if(APPLE)
option(ZSTD_FRAMEWORK "Build as Apple Framework" OFF)
endif()
# Android-specific configuration
if(ANDROID)
set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT OFF)
# Handle old Android API levels
if((NOT ANDROID_PLATFORM_LEVEL) OR (ANDROID_PLATFORM_LEVEL VERSION_LESS 24))
message(STATUS "Configuring for old Android API - disabling fseeko/ftello")
add_compile_definitions(LIBC_NO_FSEEKO)
endif()
else()
set(ZSTD_MULTITHREAD_SUPPORT_DEFAULT ON)
endif()
# Multi-threading support
option(ZSTD_MULTITHREAD_SUPPORT "Enable multi-threading support" ${ZSTD_MULTITHREAD_SUPPORT_DEFAULT})
if(ZSTD_MULTITHREAD_SUPPORT)
message(STATUS "Multi-threading support enabled")
else()
message(STATUS "Multi-threading support disabled")
endif()
# Build component options
option(ZSTD_BUILD_PROGRAMS "Build command-line programs" ON)
option(ZSTD_BUILD_CONTRIB "Build contrib utilities" OFF)
option(ZSTD_PROGRAMS_LINK_SHARED "Link programs against shared library" OFF)
# Test configuration
if(BUILD_TESTING)
set(ZSTD_BUILD_TESTS_default ON)
else()
set(ZSTD_BUILD_TESTS_default OFF)
endif()
option(ZSTD_BUILD_TESTS "Build test suite" ${ZSTD_BUILD_TESTS_default})
# MSVC-specific options
if(MSVC)
option(ZSTD_USE_STATIC_RUNTIME "Link to static runtime libraries" OFF)
endif()
# C++ support (needed for tests)
set(ZSTD_ENABLE_CXX ${ZSTD_BUILD_TESTS})
if(ZSTD_ENABLE_CXX)
enable_language(CXX)
endif()
# Set global definitions
add_definitions(-DXXH_NAMESPACE=ZSTD_)
@@ -1,45 +0,0 @@
# ################################################################
# ZSTD Package Configuration
# ################################################################
include(CMakePackageConfigHelpers)
# Generate version file
write_basic_package_version_file(
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
VERSION ${zstd_VERSION}
COMPATIBILITY SameMajorVersion
)
# Configure package for installation
set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/zstd)
foreach(target_suffix IN ITEMS "_shared" "_static" "")
if(TARGET "libzstd${target_suffix}")
# Export targets for build directory
export(EXPORT "zstdExports${target_suffix}"
FILE "${CMAKE_CURRENT_BINARY_DIR}/zstdTargets${target_suffix}.cmake"
NAMESPACE zstd::
)
# Install exported targets
install(EXPORT "zstdExports${target_suffix}"
FILE "zstdTargets${target_suffix}.cmake"
NAMESPACE zstd::
DESTINATION ${ConfigPackageLocation}
)
endif()
endforeach()
# Configure and install package config file
configure_package_config_file(
zstdConfig.cmake.in
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
INSTALL_DESTINATION ${ConfigPackageLocation}
)
# Install config files
install(FILES
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake"
"${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake"
DESTINATION ${ConfigPackageLocation}
)
@@ -1,31 +0,0 @@
# ################################################################
# ZSTD Version Configuration
# ################################################################
# Setup CMake policy version
set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3")
set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13")
# Determine appropriate policy version
if("${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND
"${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}")
set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}")
else()
set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0")
endif()
cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION})
# Parse version from header file
include(GetZstdLibraryVersion)
GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH)
# Set version variables
set(ZSTD_SHORT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}")
set(ZSTD_FULL_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
# Project metadata
set(zstd_HOMEPAGE_URL "https://facebook.github.io/zstd")
set(zstd_DESCRIPTION "Zstandard is a real-time compression algorithm, providing high compression ratios.")
message(STATUS "ZSTD VERSION: ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}")
-19
View File
@@ -7,15 +7,6 @@ variables.
## How to build
You can configure the project from the repository root thanks to the forwarding
`CMakeLists.txt`:
```sh
cmake -S . -B build-cmake
cmake --build build-cmake
```
The historical workflow that starts configuration from `build/cmake` continues
to work as described below.
As cmake doesn't support command like `cmake clean`, it's recommended to perform an "out of source build".
To do this, you can create a new directory and build in it:
```sh
@@ -50,16 +41,6 @@ cmake -DZSTD_BUILD_TESTS=ON -DZSTD_LEGACY_SUPPORT=OFF ..
make
```
**Apple Frameworks**
It's generally recommended to have CMake with versions higher than 3.14 for [iOS-derived platforms](https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html#id27).
```sh
cmake -S. -B build-cmake -DZSTD_FRAMEWORK=ON -DCMAKE_SYSTEM_NAME=iOS
```
Or you can utilize [iOS-CMake](https://github.com/leetal/ios-cmake) toolchain for CMake versions lower than 3.14
```sh
cmake -B build -G Xcode -DCMAKE_TOOLCHAIN_FILE=<Path To ios.toolchain.cmake> -DPLATFORM=OS64 -DZSTD_FRAMEWORK=ON
```
### how to use it with CMake FetchContent
For all options available, you can see it on <https://github.com/facebook/zstd/blob/dev/build/cmake/lib/CMakeLists.txt>
+17 -59
View File
@@ -7,7 +7,7 @@
# in the COPYING file in the root directory of this source tree).
# ################################################################
project(libzstd C) # ASM language is conditionally enabled where supported
project(libzstd C ASM)
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
option(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
@@ -39,8 +39,7 @@ file(GLOB DecompressSources ${LIBRARY_DIR}/decompress/*.c)
if (MSVC)
add_compile_options(-DZSTD_DISABLE_ASM)
else ()
if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|AMD64.*|x86_64.*|X86_64.*" AND ${ZSTD_HAS_NOEXECSTACK})
enable_language(ASM)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|AMD64.*|x86_64.*|X86_64.*")
set(DecompressSources ${DecompressSources} ${LIBRARY_DIR}/decompress/huf_decompress_amd64.S)
else()
add_compile_options(-DZSTD_DISABLE_ASM)
@@ -98,11 +97,9 @@ if (ZSTD_LEGACY_SUPPORT)
${LIBRARY_LEGACY_DIR}/zstd_v07.h)
endif ()
if (MSVC AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang"))
if (MSVC)
set(MSVC_RESOURCE_DIR ${ZSTD_SOURCE_DIR}/build/VS2010/libzstd-dll)
set(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc)
else()
set(PlatformDependResources)
endif ()
# Explicitly set the language to C for all files, including ASM files.
@@ -115,19 +112,21 @@ endif()
macro (add_definition target var)
if (NOT ("${${var}}" STREQUAL ""))
target_compile_definitions(${target} PUBLIC "${var}=__attribute__((visibility(\"${${var}}\")))")
set_property(TARGET ${target} APPEND PROPERTY COMPILE_DEFINITIONS "${var}=__attribute__((visibility(\"${${var}}\")))")
endif ()
endmacro ()
# Define directories containing the library's public headers
set(PUBLIC_INCLUDE_DIRS ${LIBRARY_DIR})
set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} /I \"${LIBRARY_DIR}\"")
# Split project to static and shared libraries build
set(library_targets)
if (ZSTD_BUILD_SHARED)
add_library(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources})
target_include_directories(libzstd_shared INTERFACE $<BUILD_INTERFACE:${PUBLIC_INCLUDE_DIRS}>)
list(APPEND library_targets libzstd_shared)
if (ZSTD_MULTITHREAD_SUPPORT)
target_compile_definitions(libzstd_shared PUBLIC ZSTD_MULTITHREAD)
set_property(TARGET libzstd_shared APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
if (UNIX)
target_link_libraries(libzstd_shared ${THREADS_LIBS})
endif ()
@@ -139,8 +138,9 @@ endif ()
if (ZSTD_BUILD_STATIC)
add_library(libzstd_static STATIC ${Sources} ${Headers})
target_include_directories(libzstd_static INTERFACE $<BUILD_INTERFACE:${PUBLIC_INCLUDE_DIRS}>)
list(APPEND library_targets libzstd_static)
if (ZSTD_MULTITHREAD_SUPPORT)
target_compile_definitions(libzstd_static PUBLIC ZSTD_MULTITHREAD)
set_property(TARGET libzstd_static APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
if (UNIX)
target_link_libraries(libzstd_static ${THREADS_LIBS})
endif ()
@@ -157,6 +157,7 @@ if (ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
endif ()
add_library(libzstd INTERFACE)
target_link_libraries(libzstd INTERFACE libzstd_shared)
list(APPEND library_targets libzstd)
endif ()
if (ZSTD_BUILD_STATIC AND NOT ZSTD_BUILD_SHARED)
if (BUILD_SHARED_LIBS)
@@ -164,6 +165,7 @@ if (ZSTD_BUILD_STATIC AND NOT ZSTD_BUILD_SHARED)
endif ()
add_library(libzstd INTERFACE)
target_link_libraries(libzstd INTERFACE libzstd_static)
list(APPEND library_targets libzstd)
endif ()
if (ZSTD_BUILD_SHARED AND ZSTD_BUILD_STATIC)
# If both ZSTD_BUILD_SHARED and ZSTD_BUILD_STATIC are set, which is the
@@ -172,9 +174,11 @@ if (ZSTD_BUILD_SHARED AND ZSTD_BUILD_STATIC)
if (BUILD_SHARED_LIBS)
add_library(libzstd INTERFACE)
target_link_libraries(libzstd INTERFACE libzstd_shared)
list(APPEND library_targets libzstd)
else ()
add_library(libzstd INTERFACE)
target_link_libraries(libzstd INTERFACE libzstd_static)
list(APPEND library_targets libzstd)
endif ()
endif ()
@@ -201,28 +205,8 @@ if (ZSTD_BUILD_SHARED)
libzstd_shared
PROPERTIES
OUTPUT_NAME zstd
VERSION ${ZSTD_FULL_VERSION}
VERSION ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}
SOVERSION ${zstd_VERSION_MAJOR})
if (ZSTD_FRAMEWORK)
set_target_properties(
libzstd_shared
PROPERTIES
FRAMEWORK TRUE
FRAMEWORK_VERSION "${ZSTD_FULL_VERSION}"
PRODUCT_BUNDLE_IDENTIFIER "github.com/facebook/zstd"
XCODE_ATTRIBUTE_INSTALL_PATH "@rpath"
PUBLIC_HEADER "${PublicHeaders}"
OUTPUT_NAME "zstd"
XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY ""
XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO"
XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED "NO"
MACOSX_FRAMEWORK_IDENTIFIER "github.com/facebook/zstd"
MACOSX_FRAMEWORK_BUNDLE_VERSION "${ZSTD_FULL_VERSION}"
MACOSX_FRAMEWORK_SHORT_VERSION_STRING "${ZSTD_SHORT_VERSION}"
MACOSX_RPATH TRUE
RESOURCE ${PublicHeaders})
endif ()
endif ()
if (ZSTD_BUILD_STATIC)
@@ -231,26 +215,6 @@ if (ZSTD_BUILD_STATIC)
PROPERTIES
POSITION_INDEPENDENT_CODE On
OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME})
if (ZSTD_FRAMEWORK)
set_target_properties(
libzstd_static
PROPERTIES
FRAMEWORK TRUE
FRAMEWORK_VERSION "${ZSTD_FULL_VERSION}"
PRODUCT_BUNDLE_IDENTIFIER "github.com/facebook/zstd/${STATIC_LIBRARY_BASE_NAME}"
XCODE_ATTRIBUTE_INSTALL_PATH "@rpath"
PUBLIC_HEADER "${PublicHeaders}"
OUTPUT_NAME "${STATIC_LIBRARY_BASE_NAME}"
XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY ""
XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO"
XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED "NO"
MACOSX_FRAMEWORK_IDENTIFIER "github.com/facebook/zstd/${STATIC_LIBRARY_BASE_NAME}"
MACOSX_FRAMEWORK_BUNDLE_VERSION "${ZSTD_FULL_VERSION}"
MACOSX_FRAMEWORK_SHORT_VERSION_STRING "${ZSTD_SHORT_VERSION}"
MACOSX_RPATH TRUE
RESOURCE ${PublicHeaders})
endif ()
endif ()
# pkg-config
@@ -268,20 +232,14 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libzstd.pc" DESTINATION "${CMAKE_INST
# install target
install(FILES ${PublicHeaders} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
foreach(target_suffix IN ITEMS "_shared" "_static" "")
if(TARGET "libzstd${target_suffix}")
install(TARGETS "libzstd${target_suffix}"
EXPORT "zstdExports${target_suffix}"
install(TARGETS ${library_targets}
EXPORT zstdExports
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}"
FRAMEWORK DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT runtime OPTIONAL
PUBLIC_HEADER DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
)
endif()
endforeach()
# uninstall target
if (NOT TARGET uninstall)
-3
View File
@@ -5,9 +5,6 @@ if(@ZSTD_MULTITHREAD_SUPPORT@ AND "@UNIX@")
find_dependency(Threads)
endif()
foreach(lib_suffix IN ITEMS "_shared" "_static")
include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets${lib_suffix}.cmake" OPTIONAL)
endforeach()
include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets.cmake")
check_required_components("zstd")
+1 -1
View File
@@ -20,6 +20,6 @@ pzstd = executable('pzstd',
pzstd_sources,
cpp_args: pzstd_warning_flags,
include_directories: pzstd_includes,
dependencies: [ libzstd_internal_dep, thread_dep ],
dependencies: [ libzstd_dep, thread_dep ],
override_options: ['b_ndebug=true'],
install: true)
+4 -12
View File
@@ -30,7 +30,6 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_literals.c'),
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_sequences.c'),
join_paths(zstd_rootdir, 'lib/compress/zstd_compress_superblock.c'),
join_paths(zstd_rootdir, 'lib/compress/zstd_preSplit.c'),
join_paths(zstd_rootdir, 'lib/compress/zstdmt_compress.c'),
join_paths(zstd_rootdir, 'lib/compress/zstd_fast.c'),
join_paths(zstd_rootdir, 'lib/compress/zstd_double_fast.c'),
@@ -125,11 +124,7 @@ libzstd = library('zstd',
version: zstd_libversion)
libzstd_dep = declare_dependency(link_with: libzstd,
include_directories: join_paths(zstd_rootdir,'lib')) # Do not expose private headers
if meson.version().version_compare('>=0.54.0')
meson.override_dependency('libzstd', libzstd_dep)
endif
include_directories: libzstd_includes)
# we link to both:
# - the shared library (for public symbols)
@@ -139,8 +134,7 @@ endif
# -fvisibility=hidden means those cannot be found
if get_option('default_library') == 'static'
libzstd_static = libzstd
libzstd_internal_dep = declare_dependency(link_with: libzstd,
include_directories: libzstd_includes)
libzstd_internal_dep = libzstd_dep
else
if get_option('default_library') == 'shared'
libzstd_static = static_library('zstd_objlib',
@@ -153,13 +147,11 @@ else
if cc_id == compiler_msvc
# msvc does not actually support linking to both, but errors out with:
# error LNK2005: ZSTD_<foo> already defined in zstd.lib(zstd-1.dll)
libzstd_internal_dep = declare_dependency(link_with: libzstd_static,
include_directories: libzstd_includes)
libzstd_internal_dep = declare_dependency(link_with: libzstd_static)
else
libzstd_internal_dep = declare_dependency(link_with: libzstd,
# the static library must be linked after the shared one
dependencies: declare_dependency(link_with: libzstd_static),
include_directories: libzstd_includes)
dependencies: declare_dependency(link_with: libzstd_static))
endif
endif
+6 -16
View File
@@ -10,7 +10,7 @@
project('zstd',
['c', 'cpp'],
license: 'BSD-3-Clause OR GPL-2.0-only',
license: ['BSD', 'GPLv2'],
default_options : [
# There shouldn't be any need to force a C standard convention for zstd
# but in case one would want that anyway, this can be done here.
@@ -65,6 +65,7 @@ zstd_docdir = join_paths(zstd_datadir, 'doc', meson.project_name())
# Built-in options
use_debug = get_option('debug')
buildtype = get_option('buildtype')
default_library_type = get_option('default_library')
# Custom options
@@ -87,13 +88,8 @@ feature_lz4 = get_option('lz4')
# =============================================================================
libm_dep = cc.find_library('m', required: false)
if host_machine_os == os_windows
thread_dep = dependency('', required: false)
use_multi_thread = not feature_multi_thread.disabled()
else
thread_dep = dependency('threads', required: feature_multi_thread)
use_multi_thread = thread_dep.found()
endif
thread_dep = dependency('threads', required: feature_multi_thread)
use_multi_thread = thread_dep.found()
# Arguments in dependency should be equivalent to those passed to pkg-config
zlib_dep = dependency('zlib', required: feature_zlib)
use_zlib = zlib_dep.found()
@@ -115,16 +111,10 @@ if [compiler_gcc, compiler_clang].contains(cc_id)
if cc_id == compiler_clang
common_warning_flags += ['-Wconversion', '-Wno-sign-conversion', '-Wdocumentation']
endif
noexecstack_flags = ['-Wa,--noexecstack' ]
noexecstack_link_flags = ['-Wl,-z,noexecstack']
cc_compile_flags = cc.get_supported_arguments(common_warning_flags + noexecstack_flags + ['-Wstrict-prototypes'])
cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags + noexecstack_flags)
cc_compile_flags = cc.get_supported_arguments(common_warning_flags + ['-Wstrict-prototypes'])
cxx_compile_flags = cxx.get_supported_arguments(common_warning_flags)
add_project_arguments(cc_compile_flags, language : 'c')
add_project_arguments(cxx_compile_flags, language : 'cpp')
cc_link_flags = cc.get_supported_link_arguments(noexecstack_link_flags)
cxx_link_flags = cxx.get_supported_link_arguments(noexecstack_link_flags)
add_project_link_arguments(cc_link_flags, language: 'c')
add_project_link_arguments(cxx_link_flags, language: 'cpp')
elif cc_id == compiler_msvc
msvc_compile_flags = [ '/D_UNICODE', '/DUNICODE' ]
if use_multi_thread
+3 -3
View File
@@ -10,8 +10,8 @@
# Read guidelines from https://wiki.gnome.org/Initiatives/GnomeGoals/MesonPorting
option('legacy_level', type: 'integer', min: 0, max: 7, value: 0,
description: 'Support legacy format: 0=disabled, 1-7=support v0.1+ to v0.7+')
option('legacy_level', type: 'integer', min: 0, max: 7, value: 5,
description: 'Support any legacy format: 7 to 1 for v0.7+ to v0.1+')
option('debug_level', type: 'integer', min: 0, max: 9, value: 1,
description: 'Enable run-time debug. See lib/common/debug.h')
option('backtrace', type: 'feature', value: 'disabled',
@@ -27,7 +27,7 @@ option('bin_contrib', type: 'boolean', value: false,
description: 'Enable contrib build')
option('multi_thread', type: 'feature', value: 'enabled',
description: 'Enable multi-threading when pthread or Windows is detected')
description: 'Enable multi-threading when pthread is detected')
option('zlib', type: 'feature', value: 'auto',
description: 'Enable zlib support')
option('lzma', type: 'feature', value: 'auto',
+1 -1
View File
@@ -93,7 +93,7 @@ roundTripCrash = executable('roundTripCrash',
longmatch_sources = [join_paths(zstd_rootdir, 'tests/longmatch.c')]
longmatch = executable('longmatch',
longmatch_sources,
dependencies: [ libzstd_internal_dep ],
dependencies: [ libzstd_dep ],
install: false)
invalidDictionaries_sources = [join_paths(zstd_rootdir, 'tests/invalidDictionaries.c')]
+1 -1
View File
@@ -19,7 +19,7 @@ def valgrindTest(valgrind, datagen, fuzzer, zstd, fullbench):
subprocess.check_call([*VALGRIND_ARGS, datagen, '-g50M'], stdout=subprocess.DEVNULL)
if subprocess.call([*VALGRIND_ARGS, zstd, '--fake-stdin-is-console'],
if subprocess.call([*VALGRIND_ARGS, zstd],
stdout=subprocess.DEVNULL) == 0:
raise subprocess.SubprocessError('zstd without argument should have failed')
-1
View File
@@ -4,7 +4,6 @@ zstddeclib.c
zstdenclib.c
zstd.c
zstd.h
zstd_errors.h
# test artifacts
temp*
@@ -70,7 +70,6 @@ echo "Single file library creation script: PASSED"
# Copy the header to here (for the tests)
cp "$ZSTD_SRC_ROOT/zstd.h" examples/zstd.h
cp "$ZSTD_SRC_ROOT/zstd_errors.h" examples/zstd_errors.h
# Compile the generated output
cc -Wall -Wextra -Werror -Wshadow -pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c
-1
View File
@@ -69,7 +69,6 @@
#include "compress/zstd_compress_literals.c"
#include "compress/zstd_compress_sequences.c"
#include "compress/zstd_compress_superblock.c"
#include "compress/zstd_preSplit.c"
#include "compress/zstd_compress.c"
#include "compress/zstd_double_fast.c"
#include "compress/zstd_fast.c"
+2 -3
View File
@@ -27,7 +27,6 @@ do { \
} while (0) \
int main(int argc, char *argv[]) {
int retn = 0;
if (argc != 2) {
printf("Usage: externalSequenceProducer <file>\n");
return 1;
@@ -97,12 +96,12 @@ int main(int argc, char *argv[]) {
break;
}
}
retn = 1;
return 1;
}
ZSTD_freeCCtx(zc);
free(src);
free(dst);
free(val);
return retn;
return 0;
}
+1 -1
View File
@@ -40,7 +40,7 @@ gen_html: gen_html.cpp
$(ZSTDMANUAL): gen_html $(ZSTDAPI)
echo "Update zstd manual in /doc"
./gen_html$(EXT) $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL)
./gen_html $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL)
.PHONY: manual
manual: gen_html $(ZSTDMANUAL)
-1
View File
@@ -211,7 +211,6 @@ int main(int argc, char *argv[]) {
ostream << "<html>\n<head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=ISO-8859-1\">\n<title>" << version << "</title>\n</head>\n<body>" << endl;
ostream << "<h1>" << version << "</h1>\n";
ostream << "Note: the content of this file has been automatically generated by parsing \"zstd.h\" \n";
ostream << "<hr>\n<a name=\"Contents\"></a><h2>Contents</h2>\n<ol>\n";
for (size_t i=0; i<chapters.size(); i++)
+3 -7
View File
@@ -45,9 +45,7 @@
#define RUN_TIME_DEFAULT_MS 1000
#define BENCH_TIME_DEFAULT_MS (BENCH_TIME_DEFAULT_S * RUN_TIME_DEFAULT_MS)
#ifndef ZSTD_DISPLAY_LEVEL_DEFAULT
# define ZSTD_DISPLAY_LEVEL_DEFAULT 3
#endif
#define DISPLAY_LEVEL_DEFAULT 3
#define BENCH_SIZE_MAX (1200 MB)
@@ -63,7 +61,7 @@
#define DISPLAY(...) fprintf(stdout, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
static int g_displayLevel = ZSTD_DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
/*--- buffer_t ---*/
@@ -741,8 +739,6 @@ static int benchMem(slice_collection_t dstBlocks, slice_collection_t srcBlocks,
/* BMK_benchTimedFn may not run exactly nbRounds iterations */
double speedAggregated =
aggregateData(speedPerRound, roundNb + 1, metricAggregatePref);
free(speedPerRound);
if (metricAggregatePref == fastest)
DISPLAY("Fastest Speed : %.1f MB/s \n", speedAggregated);
else
@@ -1029,7 +1025,7 @@ int main (int argc, const char** argv)
unsigned nbBlocks = 0; /* determine nbBlocks automatically, from source and blockSize */
ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto;
ZSTD_dictAttachPref_e dictAttachPref = ZSTD_dictDefaultAttach;
ZSTD_ParamSwitch_e prefetchCDictTables = ZSTD_ps_auto;
ZSTD_paramSwitch_e prefetchCDictTables = ZSTD_ps_auto;
metricAggregatePref_e metricAggregatePref = fastest;
for (int argNb = 1; argNb < argc ; argNb++) {
-1
View File
@@ -26,7 +26,6 @@ zstd_compress-y := \
compress/zstd_lazy.o \
compress/zstd_ldm.o \
compress/zstd_opt.o \
compress/zstd_preSplit.o \
zstd_decompress-y := \
zstd_decompress_module.o \
+2 -246
View File
@@ -77,30 +77,6 @@ int zstd_min_clevel(void);
*/
int zstd_max_clevel(void);
/**
* zstd_default_clevel() - default compression level
*
* Return: Default compression level.
*/
int zstd_default_clevel(void);
/**
* struct zstd_custom_mem - custom memory allocation
*/
typedef ZSTD_customMem zstd_custom_mem;
/**
* struct zstd_dict_load_method - Dictionary load method.
* See zstd_lib.h.
*/
typedef ZSTD_dictLoadMethod_e zstd_dict_load_method;
/**
* struct zstd_dict_content_type - Dictionary context type.
* See zstd_lib.h.
*/
typedef ZSTD_dictContentType_e zstd_dict_content_type;
/* ====== Parameter Selection ====== */
/**
@@ -160,32 +136,9 @@ typedef ZSTD_parameters zstd_parameters;
zstd_parameters zstd_get_params(int level,
unsigned long long estimated_src_size);
/**
* zstd_get_cparams() - returns zstd_compression_parameters for selected level
* @level: The compression level
* @estimated_src_size: The estimated source size to compress or 0
* if unknown.
* @dict_size: Dictionary size.
*
* Return: The selected zstd_compression_parameters.
*/
zstd_compression_parameters zstd_get_cparams(int level,
unsigned long long estimated_src_size, size_t dict_size);
/* ====== Single-pass Compression ====== */
typedef ZSTD_CCtx zstd_cctx;
typedef ZSTD_cParameter zstd_cparameter;
/**
* zstd_cctx_set_param() - sets a compression parameter
* @cctx: The context. Must have been initialized with zstd_init_cctx().
* @param: The parameter to set.
* @value: The value to set the parameter to.
*
* Return: Zero or an error, which can be checked using zstd_is_error().
*/
size_t zstd_cctx_set_param(zstd_cctx *cctx, zstd_cparameter param, int value);
/* ====== Single-pass Compression ====== */
/**
* zstd_cctx_workspace_bound() - max memory needed to initialize a zstd_cctx
@@ -200,20 +153,6 @@ size_t zstd_cctx_set_param(zstd_cctx *cctx, zstd_cparameter param, int value);
*/
size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *parameters);
/**
* zstd_cctx_workspace_bound_with_ext_seq_prod() - max memory needed to
* initialize a zstd_cctx when using the block-level external sequence
* producer API.
* @parameters: The compression parameters to be used.
*
* If multiple compression parameters might be used, the caller must call
* this function for each set of parameters and use the maximum size.
*
* Return: A lower bound on the size of the workspace that is passed to
* zstd_init_cctx().
*/
size_t zstd_cctx_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *parameters);
/**
* zstd_init_cctx() - initialize a zstd compression context
* @workspace: The workspace to emplace the context into. It must outlive
@@ -241,71 +180,6 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size);
size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
const void *src, size_t src_size, const zstd_parameters *parameters);
/**
* zstd_create_cctx_advanced() - Create compression context
* @custom_mem: Custom allocator.
*
* Return: NULL on error, pointer to compression context otherwise.
*/
zstd_cctx *zstd_create_cctx_advanced(zstd_custom_mem custom_mem);
/**
* zstd_free_cctx() - Free compression context
* @cdict: Pointer to compression context.
*
* Return: Always 0.
*/
size_t zstd_free_cctx(zstd_cctx* cctx);
/**
* struct zstd_cdict - Compression dictionary.
* See zstd_lib.h.
*/
typedef ZSTD_CDict zstd_cdict;
/**
* zstd_create_cdict_byreference() - Create compression dictionary
* @dict: Pointer to dictionary buffer.
* @dict_size: Size of the dictionary buffer.
* @dict_load_method: Dictionary load method.
* @dict_content_type: Dictionary content type.
* @custom_mem: Memory allocator.
*
* Note, this uses @dict by reference (ZSTD_dlm_byRef), so it should be
* free before zstd_cdict is destroyed.
*
* Return: NULL on error, pointer to compression dictionary
* otherwise.
*/
zstd_cdict *zstd_create_cdict_byreference(const void *dict, size_t dict_size,
zstd_compression_parameters cparams,
zstd_custom_mem custom_mem);
/**
* zstd_free_cdict() - Free compression dictionary
* @cdict: Pointer to compression dictionary.
*
* Return: Always 0.
*/
size_t zstd_free_cdict(zstd_cdict* cdict);
/**
* zstd_compress_using_cdict() - compress src into dst using a dictionary
* @cctx: The context. Must have been initialized with zstd_init_cctx().
* @dst: The buffer to compress src into.
* @dst_capacity: The size of the destination buffer. May be any size, but
* ZSTD_compressBound(srcSize) is guaranteed to be large enough.
* @src: The data to compress.
* @src_size: The size of the data to compress.
* @cdict: The dictionary to be used.
*
* Return: The compressed size or an error, which can be checked using
* zstd_is_error().
*/
size_t zstd_compress_using_cdict(zstd_cctx *cctx, void *dst,
size_t dst_capacity, const void *src, size_t src_size,
const zstd_cdict *cdict);
/* ====== Single-pass Decompression ====== */
typedef ZSTD_DCtx zstd_dctx;
@@ -346,71 +220,6 @@ zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size);
size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
const void *src, size_t src_size);
/**
* struct zstd_ddict - Decompression dictionary.
* See zstd_lib.h.
*/
typedef ZSTD_DDict zstd_ddict;
/**
* zstd_create_ddict_byreference() - Create decompression dictionary
* @dict: Pointer to dictionary buffer.
* @dict_size: Size of the dictionary buffer.
* @dict_load_method: Dictionary load method.
* @dict_content_type: Dictionary content type.
* @custom_mem: Memory allocator.
*
* Note, this uses @dict by reference (ZSTD_dlm_byRef), so it should be
* free before zstd_ddict is destroyed.
*
* Return: NULL on error, pointer to decompression dictionary
* otherwise.
*/
zstd_ddict *zstd_create_ddict_byreference(const void *dict, size_t dict_size,
zstd_custom_mem custom_mem);
/**
* zstd_free_ddict() - Free decompression dictionary
* @dict: Pointer to the dictionary.
*
* Return: Always 0.
*/
size_t zstd_free_ddict(zstd_ddict *ddict);
/**
* zstd_create_dctx_advanced() - Create decompression context
* @custom_mem: Custom allocator.
*
* Return: NULL on error, pointer to decompression context otherwise.
*/
zstd_dctx *zstd_create_dctx_advanced(zstd_custom_mem custom_mem);
/**
* zstd_free_dctx() -- Free decompression context
* @dctx: Pointer to decompression context.
* Return: Always 0.
*/
size_t zstd_free_dctx(zstd_dctx *dctx);
/**
* zstd_decompress_using_ddict() - decompress src into dst using a dictionary
* @dctx: The decompression context.
* @dst: The buffer to decompress src into.
* @dst_capacity: The size of the destination buffer. Must be at least as large
* as the decompressed size. If the caller cannot upper bound the
* decompressed size, then it's better to use the streaming API.
* @src: The zstd compressed data to decompress. Multiple concatenated
* frames and skippable frames are allowed.
* @src_size: The exact size of the data to decompress.
* @ddict: The dictionary to be used.
*
* Return: The decompressed size or an error, which can be checked using
* zstd_is_error().
*/
size_t zstd_decompress_using_ddict(zstd_dctx *dctx,
void *dst, size_t dst_capacity, const void *src, size_t src_size,
const zstd_ddict *ddict);
/* ====== Streaming Buffers ====== */
/**
@@ -448,16 +257,6 @@ typedef ZSTD_CStream zstd_cstream;
*/
size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams);
/**
* zstd_cstream_workspace_bound_with_ext_seq_prod() - memory needed to initialize
* a zstd_cstream when using the block-level external sequence producer API.
* @cparams: The compression parameters to be used for compression.
*
* Return: A lower bound on the size of the workspace that is passed to
* zstd_init_cstream().
*/
size_t zstd_cstream_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *cparams);
/**
* zstd_init_cstream() - initialize a zstd streaming compression context
* @parameters The zstd parameters to use for compression.
@@ -617,18 +416,6 @@ size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output,
*/
size_t zstd_find_frame_compressed_size(const void *src, size_t src_size);
/**
* zstd_register_sequence_producer() - exposes the zstd library function
* ZSTD_registerSequenceProducer(). This is used for the block-level external
* sequence producer API. See upstream zstd.h for detailed documentation.
*/
typedef ZSTD_sequenceProducer_F zstd_sequence_producer_f;
void zstd_register_sequence_producer(
zstd_cctx *cctx,
void* sequence_producer_state,
zstd_sequence_producer_f sequence_producer
);
/**
* struct zstd_frame_params - zstd frame parameters stored in the frame header
* @frameContentSize: The frame content size, or ZSTD_CONTENTSIZE_UNKNOWN if not
@@ -642,7 +429,7 @@ void zstd_register_sequence_producer(
*
* See zstd_lib.h.
*/
typedef ZSTD_FrameHeader zstd_frame_header;
typedef ZSTD_frameHeader zstd_frame_header;
/**
* zstd_get_frame_header() - extracts parameters from a zstd or skippable frame
@@ -657,35 +444,4 @@ typedef ZSTD_FrameHeader zstd_frame_header;
size_t zstd_get_frame_header(zstd_frame_header *params, const void *src,
size_t src_size);
/**
* struct zstd_sequence - a sequence of literals or a match
*
* @offset: The offset of the match
* @litLength: The literal length of the sequence
* @matchLength: The match length of the sequence
* @rep: Represents which repeat offset is used
*/
typedef ZSTD_Sequence zstd_sequence;
/**
* zstd_compress_sequences_and_literals() - compress an array of zstd_sequence and literals
*
* @cctx: The zstd compression context.
* @dst: The buffer to compress the data into.
* @dst_capacity: The size of the destination buffer.
* @in_seqs: The array of zstd_sequence to compress.
* @in_seqs_size: The number of sequences in in_seqs.
* @literals: The literals associated to the sequences to be compressed.
* @lit_size: The size of the literals in the literals buffer.
* @lit_capacity: The size of the literals buffer.
* @decompressed_size: The size of the input data
*
* Return: The compressed size or an error, which can be checked using
* zstd_is_error().
*/
size_t zstd_compress_sequences_and_literals(zstd_cctx *cctx, void* dst, size_t dst_capacity,
const zstd_sequence *in_seqs, size_t in_seqs_size,
const void* literals, size_t lit_size, size_t lit_capacity,
size_t decompressed_size);
#endif /* LINUX_ZSTD_H */
+1 -1
View File
@@ -15,7 +15,7 @@
/*-****************************************
* Dependencies
******************************************/
#include <linux/unaligned.h> /* get_unaligned, put_unaligned* */
#include <asm/unaligned.h> /* get_unaligned, put_unaligned* */
#include <linux/compiler.h> /* inline */
#include <linux/swab.h> /* swab32, swab64 */
#include <linux/types.h> /* size_t, ptrdiff_t */
@@ -296,7 +296,7 @@ XXH_API void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
#include <linux/unaligned.h>
#include <asm/unaligned.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/module.h>
-122
View File
@@ -16,7 +16,6 @@
#include "common/zstd_deps.h"
#include "common/zstd_internal.h"
#include "compress/zstd_compress_internal.h"
#define ZSTD_FORWARD_IF_ERR(ret) \
do { \
@@ -67,12 +66,6 @@ int zstd_max_clevel(void)
}
EXPORT_SYMBOL(zstd_max_clevel);
int zstd_default_clevel(void)
{
return ZSTD_defaultCLevel();
}
EXPORT_SYMBOL(zstd_default_clevel);
size_t zstd_compress_bound(size_t src_size)
{
return ZSTD_compressBound(src_size);
@@ -86,71 +79,12 @@ zstd_parameters zstd_get_params(int level,
}
EXPORT_SYMBOL(zstd_get_params);
zstd_compression_parameters zstd_get_cparams(int level,
unsigned long long estimated_src_size, size_t dict_size)
{
return ZSTD_getCParams(level, estimated_src_size, dict_size);
}
EXPORT_SYMBOL(zstd_get_cparams);
size_t zstd_cctx_set_param(zstd_cctx *cctx, ZSTD_cParameter param, int value)
{
return ZSTD_CCtx_setParameter(cctx, param, value);
}
EXPORT_SYMBOL(zstd_cctx_set_param);
size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *cparams)
{
return ZSTD_estimateCCtxSize_usingCParams(*cparams);
}
EXPORT_SYMBOL(zstd_cctx_workspace_bound);
// Used by zstd_cctx_workspace_bound_with_ext_seq_prod()
static size_t dummy_external_sequence_producer(
void *sequenceProducerState,
ZSTD_Sequence *outSeqs, size_t outSeqsCapacity,
const void *src, size_t srcSize,
const void *dict, size_t dictSize,
int compressionLevel,
size_t windowSize)
{
(void)sequenceProducerState;
(void)outSeqs; (void)outSeqsCapacity;
(void)src; (void)srcSize;
(void)dict; (void)dictSize;
(void)compressionLevel;
(void)windowSize;
return ZSTD_SEQUENCE_PRODUCER_ERROR;
}
static void init_cctx_params_from_compress_params(
ZSTD_CCtx_params *cctx_params,
const zstd_compression_parameters *compress_params)
{
ZSTD_parameters zstd_params;
memset(&zstd_params, 0, sizeof(zstd_params));
zstd_params.cParams = *compress_params;
ZSTD_CCtxParams_init_advanced(cctx_params, zstd_params);
}
size_t zstd_cctx_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *compress_params)
{
ZSTD_CCtx_params cctx_params;
init_cctx_params_from_compress_params(&cctx_params, compress_params);
ZSTD_CCtxParams_registerSequenceProducer(&cctx_params, NULL, dummy_external_sequence_producer);
return ZSTD_estimateCCtxSize_usingCCtxParams(&cctx_params);
}
EXPORT_SYMBOL(zstd_cctx_workspace_bound_with_ext_seq_prod);
size_t zstd_cstream_workspace_bound_with_ext_seq_prod(const zstd_compression_parameters *compress_params)
{
ZSTD_CCtx_params cctx_params;
init_cctx_params_from_compress_params(&cctx_params, compress_params);
ZSTD_CCtxParams_registerSequenceProducer(&cctx_params, NULL, dummy_external_sequence_producer);
return ZSTD_estimateCStreamSize_usingCCtxParams(&cctx_params);
}
EXPORT_SYMBOL(zstd_cstream_workspace_bound_with_ext_seq_prod);
zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size)
{
if (workspace == NULL)
@@ -159,33 +93,6 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size)
}
EXPORT_SYMBOL(zstd_init_cctx);
zstd_cctx *zstd_create_cctx_advanced(zstd_custom_mem custom_mem)
{
return ZSTD_createCCtx_advanced(custom_mem);
}
EXPORT_SYMBOL(zstd_create_cctx_advanced);
size_t zstd_free_cctx(zstd_cctx *cctx)
{
return ZSTD_freeCCtx(cctx);
}
EXPORT_SYMBOL(zstd_free_cctx);
zstd_cdict *zstd_create_cdict_byreference(const void *dict, size_t dict_size,
zstd_compression_parameters cparams,
zstd_custom_mem custom_mem)
{
return ZSTD_createCDict_advanced(dict, dict_size, ZSTD_dlm_byRef,
ZSTD_dct_auto, cparams, custom_mem);
}
EXPORT_SYMBOL(zstd_create_cdict_byreference);
size_t zstd_free_cdict(zstd_cdict *cdict)
{
return ZSTD_freeCDict(cdict);
}
EXPORT_SYMBOL(zstd_free_cdict);
size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
const void *src, size_t src_size, const zstd_parameters *parameters)
{
@@ -194,15 +101,6 @@ size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
}
EXPORT_SYMBOL(zstd_compress_cctx);
size_t zstd_compress_using_cdict(zstd_cctx *cctx, void *dst,
size_t dst_capacity, const void *src, size_t src_size,
const ZSTD_CDict *cdict)
{
return ZSTD_compress_usingCDict(cctx, dst, dst_capacity,
src, src_size, cdict);
}
EXPORT_SYMBOL(zstd_compress_using_cdict);
size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams)
{
return ZSTD_estimateCStreamSize_usingCParams(*cparams);
@@ -262,25 +160,5 @@ size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output)
}
EXPORT_SYMBOL(zstd_end_stream);
void zstd_register_sequence_producer(
zstd_cctx *cctx,
void* sequence_producer_state,
zstd_sequence_producer_f sequence_producer
) {
ZSTD_registerSequenceProducer(cctx, sequence_producer_state, sequence_producer);
}
EXPORT_SYMBOL(zstd_register_sequence_producer);
size_t zstd_compress_sequences_and_literals(zstd_cctx *cctx, void* dst, size_t dst_capacity,
const zstd_sequence *in_seqs, size_t in_seqs_size,
const void* literals, size_t lit_size, size_t lit_capacity,
size_t decompressed_size)
{
return ZSTD_compressSequencesAndLiterals(cctx, dst, dst_capacity, in_seqs,
in_seqs_size, literals, lit_size,
lit_capacity, decompressed_size);
}
EXPORT_SYMBOL(zstd_compress_sequences_and_literals);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("Zstd Compressor");
@@ -44,33 +44,6 @@ size_t zstd_dctx_workspace_bound(void)
}
EXPORT_SYMBOL(zstd_dctx_workspace_bound);
zstd_dctx *zstd_create_dctx_advanced(zstd_custom_mem custom_mem)
{
return ZSTD_createDCtx_advanced(custom_mem);
}
EXPORT_SYMBOL(zstd_create_dctx_advanced);
size_t zstd_free_dctx(zstd_dctx *dctx)
{
return ZSTD_freeDCtx(dctx);
}
EXPORT_SYMBOL(zstd_free_dctx);
zstd_ddict *zstd_create_ddict_byreference(const void *dict, size_t dict_size,
zstd_custom_mem custom_mem)
{
return ZSTD_createDDict_advanced(dict, dict_size, ZSTD_dlm_byRef,
ZSTD_dct_auto, custom_mem);
}
EXPORT_SYMBOL(zstd_create_ddict_byreference);
size_t zstd_free_ddict(zstd_ddict *ddict)
{
return ZSTD_freeDDict(ddict);
}
EXPORT_SYMBOL(zstd_free_ddict);
zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size)
{
if (workspace == NULL)
@@ -86,15 +59,6 @@ size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
}
EXPORT_SYMBOL(zstd_decompress_dctx);
size_t zstd_decompress_using_ddict(zstd_dctx *dctx,
void *dst, size_t dst_capacity, const void* src, size_t src_size,
const zstd_ddict* ddict)
{
return ZSTD_decompress_usingDDict(dctx, dst, dst_capacity, src,
src_size, ddict);
}
EXPORT_SYMBOL(zstd_decompress_using_ddict);
size_t zstd_dstream_workspace_bound(size_t max_window_size)
{
return ZSTD_estimateDStreamSize(max_window_size);
-1
View File
@@ -2,7 +2,6 @@
-- Basic usage: project_zstd(ZSTD_DIR)
function project_zstd(dir, compression, decompression, deprecated, dictbuilder, legacy)
if string.sub(dir, -1, 1) ~= '/' then dir = dir .. '/' end
if compression == nil then compression = true end
if decompression == nil then decompression = true end
if deprecated == nil then deprecated = false end
-1
View File
@@ -10,7 +10,6 @@
#include <cstdio>
#include <mutex>
#include <chrono>
namespace pzstd {
+2 -2
View File
@@ -322,7 +322,7 @@ Options::Status Options::parse(int argc, const char **argv) {
g_utilDisplayLevel = verbosity;
// Remove local input files that are symbolic links
if (!followLinks) {
localInputFiles.erase(std::remove_if(localInputFiles.begin(), localInputFiles.end(),
std::remove_if(localInputFiles.begin(), localInputFiles.end(),
[&](const char *path) {
bool isLink = UTIL_isLink(path);
if (isLink && verbosity >= 2) {
@@ -332,7 +332,7 @@ Options::Status Options::parse(int argc, const char **argv) {
path);
}
return isLink;
}), localInputFiles.end());
});
}
// Translate input files/directories into files to (de)compress
+2 -10
View File
@@ -269,10 +269,7 @@ static void compress(
std::shared_ptr<BufferWorkQueue> out,
size_t maxInputSize) {
auto& errorHolder = state.errorHolder;
auto guard = makeScopeGuard([&] {
in->finish();
out->finish();
});
auto guard = makeScopeGuard([&] { out->finish(); });
// Initialize the CCtx
auto ctx = state.cStreamPool->get();
if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_CStream")) {
@@ -434,10 +431,7 @@ static void decompress(
std::shared_ptr<BufferWorkQueue> in,
std::shared_ptr<BufferWorkQueue> out) {
auto& errorHolder = state.errorHolder;
auto guard = makeScopeGuard([&] {
in->finish();
out->finish();
});
auto guard = makeScopeGuard([&] { out->finish(); });
// Initialize the DCtx
auto ctx = state.dStreamPool->get();
if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_DStream")) {
@@ -584,7 +578,6 @@ std::uint64_t writeFile(
FILE* outputFd,
bool decompress) {
auto& errorHolder = state.errorHolder;
auto outsFinishGuard = makeScopeGuard([&outs] { outs.finish(); });
auto lineClearGuard = makeScopeGuard([&state] {
state.log.clear(kLogInfo);
});
@@ -592,7 +585,6 @@ std::uint64_t writeFile(
std::shared_ptr<BufferWorkQueue> out;
// Grab the output queue for each decompression job (in order).
while (outs.pop(out)) {
auto outFinishGuard = makeScopeGuard([&out] { out->finish(); });
if (errorHolder.hasError()) {
continue;
}
+2
View File
@@ -7,7 +7,9 @@
* in the COPYING file in the root directory of this source tree).
*/
#include "Pzstd.h"
extern "C" {
#include "datagen.h"
}
#include "test/RoundTrip.h"
#include "utils/ScopeGuard.h"
+2
View File
@@ -6,7 +6,9 @@
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
extern "C" {
#include "datagen.h"
}
#include "Options.h"
#include "test/RoundTrip.h"
#include "utils/ScopeGuard.h"
+3 -4
View File
@@ -115,14 +115,13 @@ class WorkQueue {
}
/**
* Promise that either the reader side or the writer side is done.
* If the writer is done, `push()` won't be called again, so once the queue
* is empty there will never be any more work. If the reader is done, `pop()`
* won't be called again, so further items pushed will just be ignored.
* Promise that `push()` won't be called again, so once the queue is empty
* there will never any more work.
*/
void finish() {
{
std::lock_guard<std::mutex> lock(mutex_);
assert(!done_);
done_ = true;
}
readerCv_.notify_all();
+1 -3
View File
@@ -11,8 +11,6 @@
ZSTDLIB_PATH = ../../../lib
ZSTDLIB_NAME = libzstd.a
# Parallel tools only make sense against multi-threaded libzstd
ZSTDLIB_TARGET = $(ZSTDLIB_NAME)-mt
ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I../../../lib -I../../../lib/common
@@ -30,7 +28,7 @@ all: seekable_compression seekable_decompression seekable_decompression_mem \
parallel_processing
$(ZSTDLIB):
$(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_TARGET)
make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
seekable_compression : seekable_compression.c $(SEEKABLE_OBJS)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -23,8 +23,6 @@
#include "xxhash.h"
#define ZSTD_MULTITHREAD 1
#include "threading.h"
#include "pool.h" // use zstd thread pool for demo
#include "../zstd_seekable.h"
@@ -74,87 +72,114 @@ static size_t fclose_orDie(FILE* file)
exit(6);
}
struct state {
FILE* fout;
ZSTD_pthread_mutex_t mutex;
size_t nextID;
struct job* pending;
ZSTD_frameLog* frameLog;
const int compressionLevel;
};
static void fseek_orDie(FILE* file, long int offset, int origin)
{
if (!fseek(file, offset, origin)) {
if (!fflush(file)) return;
}
/* error */
perror("fseek");
exit(7);
}
static long int ftell_orDie(FILE* file)
{
long int off = ftell(file);
if (off != -1) return off;
/* error */
perror("ftell");
exit(8);
}
struct job {
size_t id;
struct job* next;
struct state* state;
void* src;
const void* src;
size_t srcSize;
void* dst;
size_t dstSize;
unsigned checksum;
int compressionLevel;
int done;
};
static void addPending_inmutex(struct state* state, struct job* job)
{
struct job** p = &state->pending;
while (*p && (*p)->id < job->id)
p = &(*p)->next;
job->next = *p;
*p = job;
}
static void flushFrame(struct state* state, struct job* job)
{
fwrite_orDie(job->dst, job->dstSize, state->fout);
free(job->dst);
size_t ret = ZSTD_seekable_logFrame(state->frameLog, (unsigned)job->dstSize, (unsigned)job->srcSize, job->checksum);
if (ZSTD_isError(ret)) {
fprintf(stderr, "ZSTD_seekable_logFrame() error : %s \n", ZSTD_getErrorName(ret));
exit(12);
}
}
static void flushPending_inmutex(struct state* state)
{
while (state->pending && state->pending->id == state->nextID) {
struct job* p = state->pending;
state->pending = p->next;
flushFrame(state, p);
free(p);
state->nextID++;
}
}
static void finishFrame(struct job* job)
{
struct state *state = job->state;
ZSTD_pthread_mutex_lock(&state->mutex);
addPending_inmutex(state, job);
flushPending_inmutex(state);
ZSTD_pthread_mutex_unlock(&state->mutex);
}
static void compressFrame(void* opaque)
{
struct job* job = opaque;
job->checksum = (unsigned)XXH64(job->src, job->srcSize, 0);
job->checksum = XXH64(job->src, job->srcSize, 0);
size_t ret = ZSTD_compress(job->dst, job->dstSize, job->src, job->srcSize, job->state->compressionLevel);
size_t ret = ZSTD_compress(job->dst, job->dstSize, job->src, job->srcSize, job->compressionLevel);
if (ZSTD_isError(ret)) {
fprintf(stderr, "ZSTD_compress() error : %s \n", ZSTD_getErrorName(ret));
exit(20);
}
job->dstSize = ret;
job->done = 1;
}
// No longer need
free(job->src);
job->src = NULL;
static void compressFile_orDie(const char* fname, const char* outName, int cLevel, unsigned frameSize, int nbThreads)
{
POOL_ctx* pool = POOL_create(nbThreads, nbThreads);
if (pool == NULL) { fprintf(stderr, "POOL_create() error \n"); exit(9); }
finishFrame(job);
FILE* const fin = fopen_orDie(fname, "rb");
FILE* const fout = fopen_orDie(outName, "wb");
if (ZSTD_compressBound(frameSize) > 0xFFFFFFFFU) { fprintf(stderr, "Frame size too large \n"); exit(10); }
unsigned dstSize = ZSTD_compressBound(frameSize);
fseek_orDie(fin, 0, SEEK_END);
long int length = ftell_orDie(fin);
fseek_orDie(fin, 0, SEEK_SET);
size_t numFrames = (length + frameSize - 1) / frameSize;
struct job* jobs = malloc_orDie(sizeof(struct job) * numFrames);
size_t i;
for(i = 0; i < numFrames; i++) {
void* in = malloc_orDie(frameSize);
void* out = malloc_orDie(dstSize);
size_t inSize = fread_orDie(in, frameSize, fin);
jobs[i].src = in;
jobs[i].srcSize = inSize;
jobs[i].dst = out;
jobs[i].dstSize = dstSize;
jobs[i].compressionLevel = cLevel;
jobs[i].done = 0;
POOL_add(pool, compressFrame, &jobs[i]);
}
ZSTD_frameLog* fl = ZSTD_seekable_createFrameLog(1);
if (fl == NULL) { fprintf(stderr, "ZSTD_seekable_createFrameLog() failed \n"); exit(11); }
for (i = 0; i < numFrames; i++) {
while (!jobs[i].done) SLEEP(5); /* wake up every 5 milliseconds to check */
fwrite_orDie(jobs[i].dst, jobs[i].dstSize, fout);
free((void*)jobs[i].src);
free(jobs[i].dst);
size_t ret = ZSTD_seekable_logFrame(fl, jobs[i].dstSize, jobs[i].srcSize, jobs[i].checksum);
if (ZSTD_isError(ret)) { fprintf(stderr, "ZSTD_seekable_logFrame() error : %s \n", ZSTD_getErrorName(ret)); }
}
{ unsigned char seekTableBuff[1024];
ZSTD_outBuffer out = {seekTableBuff, 1024, 0};
while (ZSTD_seekable_writeSeekTable(fl, &out) != 0) {
fwrite_orDie(seekTableBuff, out.pos, fout);
out.pos = 0;
}
fwrite_orDie(seekTableBuff, out.pos, fout);
}
ZSTD_seekable_freeFrameLog(fl);
free(jobs);
fclose_orDie(fout);
fclose_orDie(fin);
}
static const char* createOutFilename_orDie(const char* filename)
@@ -168,72 +193,6 @@ static const char* createOutFilename_orDie(const char* filename)
return (const char*)outSpace;
}
static void openInOut_orDie(const char* fname, FILE** fin, FILE** fout) {
if (strcmp(fname, "-") == 0) {
*fin = stdin;
*fout = stdout;
} else {
*fin = fopen_orDie(fname, "rb");
const char* outName = createOutFilename_orDie(fname);
*fout = fopen_orDie(outName, "wb");
}
}
static void compressFile_orDie(const char* fname, int cLevel, unsigned frameSize, size_t nbThreads)
{
struct state state = {
.nextID = 0,
.pending = NULL,
.compressionLevel = cLevel,
};
ZSTD_pthread_mutex_init(&state.mutex, NULL);
state.frameLog = ZSTD_seekable_createFrameLog(1);
if (state.frameLog == NULL) { fprintf(stderr, "ZSTD_seekable_createFrameLog() failed \n"); exit(11); }
POOL_ctx* pool = POOL_create(nbThreads, nbThreads);
if (pool == NULL) { fprintf(stderr, "POOL_create() error \n"); exit(9); }
FILE* fin;
openInOut_orDie(fname, &fin, &state.fout);
if (ZSTD_compressBound(frameSize) > 0xFFFFFFFFU) { fprintf(stderr, "Frame size too large \n"); exit(10); }
size_t dstSize = ZSTD_compressBound(frameSize);
for (size_t id = 0; 1; id++) {
struct job* job = malloc_orDie(sizeof(struct job));
job->id = id;
job->next = NULL;
job->state = &state;
job->src = malloc_orDie(frameSize);
job->dst = malloc_orDie(dstSize);
job->srcSize = fread_orDie(job->src, frameSize, fin);
job->dstSize = dstSize;
POOL_add(pool, compressFrame, job);
if (feof(fin))
break;
}
POOL_joinJobs(pool);
POOL_free(pool);
if (state.pending) {
fprintf(stderr, "Unexpected leftover output blocks!\n");
exit(13);
}
{ unsigned char seekTableBuff[1024];
ZSTD_outBuffer out = {seekTableBuff, 1024, 0};
while (ZSTD_seekable_writeSeekTable(state.frameLog, &out) != 0) {
fwrite_orDie(seekTableBuff, out.pos, state.fout);
out.pos = 0;
}
fwrite_orDie(seekTableBuff, out.pos, state.fout);
}
ZSTD_seekable_freeFrameLog(state.frameLog);
fclose_orDie(state.fout);
fclose_orDie(fin);
}
int main(int argc, const char** argv) {
const char* const exeName = argv[0];
if (argc!=4) {
@@ -245,9 +204,10 @@ int main(int argc, const char** argv) {
{ const char* const inFileName = argv[1];
unsigned const frameSize = (unsigned)atoi(argv[2]);
size_t const nbThreads = (size_t)atoi(argv[3]);
int const nbThreads = atoi(argv[3]);
compressFile_orDie(inFileName, 5, frameSize, nbThreads);
const char* const outFileName = createOutFilename_orDie(inFileName);
compressFile_orDie(inFileName, outFileName, 5, frameSize, nbThreads);
}
return 0;
@@ -100,11 +100,13 @@ struct sum_job {
const char* fname;
unsigned long long sum;
unsigned frameNb;
int done;
};
static void sumFrame(void* opaque)
{
struct sum_job* job = (struct sum_job*)opaque;
job->done = 0;
FILE* const fin = fopen_orDie(job->fname, "rb");
@@ -126,6 +128,7 @@ static void sumFrame(void* opaque)
sum += data[i];
}
job->sum = sum;
job->done = 1;
fclose(fin);
ZSTD_seekable_free(seekable);
@@ -150,14 +153,14 @@ static void sumFile_orDie(const char* fname, int nbThreads)
unsigned fnb;
for (fnb = 0; fnb < numFrames; fnb++) {
jobs[fnb] = (struct sum_job){ fname, 0, fnb };
jobs[fnb] = (struct sum_job){ fname, 0, fnb, 0 };
POOL_add(pool, sumFrame, &jobs[fnb]);
}
POOL_joinJobs(pool);
unsigned long long total = 0;
for (fnb = 0; fnb < numFrames; fnb++) {
while (!jobs[fnb].done) SLEEP(5); /* wake up every 5 milliseconds to check */
total += jobs[fnb].sum;
}
-1
View File
@@ -1,2 +1 @@
seekable_tests
data.txt
+2 -22
View File
@@ -24,7 +24,7 @@ SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
.PHONY: default clean test
default: test
test: seekable_tests parallel_compression_test
test: seekable_tests
./seekable_tests
$(ZSTDLIB):
@@ -32,27 +32,7 @@ $(ZSTDLIB):
seekable_tests : $(SEEKABLE_OBJS)
EXAMPLES_PATH = ../examples
PARALLEL_COMPRESSION = $(EXAMPLES_PATH)/parallel_compression
DATAGEN_PATH = ../../../tests
DATAGEN = $(DATAGEN_PATH)/datagen
$(PARALLEL_COMPRESSION):
$(MAKE) -C $(EXAMPLES_PATH) parallel_compression
$(DATAGEN):
$(MAKE) -C $(DATAGEN_PATH) datagen
data.txt: $(DATAGEN)
$(DATAGEN) -g100M > $@
parallel_compression_test: $(PARALLEL_COMPRESSION) data.txt
ulimit -Sv 102400; $(PARALLEL_COMPRESSION) data.txt 1048576 2
.PHONY: parallel_compression_test parallel_comp
clean:
@$(RM) core *.o tmp* result* *.zst \
seekable_tests data.txt
seekable_tests
@echo Cleaning completed
@@ -316,10 +316,6 @@ int main(int argc, const char** argv)
for (pos = 0; pos < inSize; pos += 2) {
size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, 1, pos);
if (decStatus != 1 || outBuffer[0] != inBuffer[pos]) {
free(seekBuffer);
free(outBuffer);
ZSTD_seekable_freeCStream(zscs);
ZSTD_seekable_free(stream);
goto _test_error;
}
}
@@ -327,10 +323,6 @@ int main(int argc, const char** argv)
/* We read more than the compressed size, meaning there were some rereads.
This is unneeded because we only seeked forward. */
printf("Too much data read: %zu read, with compressed size %zu\n", buffWrapper.totalRead, seekSize);
free(seekBuffer);
free(outBuffer);
ZSTD_seekable_freeCStream(zscs);
ZSTD_seekable_free(stream);
goto _test_error;
}
@@ -350,10 +342,6 @@ int main(int argc, const char** argv)
for (idx = 0; idx < sizeof(tests) / sizeof(tests[0]); idx++) {
size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, tests[idx].size, tests[idx].offset);
if (decStatus != tests[idx].size || memcmp(outBuffer, inBuffer + tests[idx].offset, tests[idx].size) != 0) {
free(seekBuffer);
free(outBuffer);
ZSTD_seekable_freeCStream(zscs);
ZSTD_seekable_free(stream);
goto _test_error;
}
}
+3 -3
View File
@@ -1,13 +1,13 @@
#ifndef SEEKABLE_H
#define SEEKABLE_H
#include <stdio.h>
#include "zstd.h" /* ZSTDLIB_API */
#if defined (__cplusplus)
extern "C" {
#endif
#include <stdio.h>
#include "zstd.h" /* ZSTDLIB_API */
#define ZSTD_seekTableFooterSize 9
@@ -77,10 +77,7 @@
/* ************************************************************
* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
***************************************************************/
#if defined(LIBC_NO_FSEEKO)
/* Some older libc implementations don't include these functions (e.g. Bionic < 24) */
# define LONG_SEEK fseek
#elif defined(_MSC_VER) && _MSC_VER >= 1400
#if defined(_MSC_VER) && _MSC_VER >= 1400
# define LONG_SEEK _fseeki64
#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
# define LONG_SEEK fseeko
@@ -255,8 +252,6 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs)
{
assert(zs != NULL);
if (zs->seekTable.entries == NULL) return NULL;
ZSTD_seekTable* const st = (ZSTD_seekTable*)malloc(sizeof(ZSTD_seekTable));
if (st==NULL) return NULL;
-20
View File
@@ -18,26 +18,6 @@ This document lists a few known cases where invalid data was formerly accepted
by the decoder, and what has changed since.
Truncated Huffman states
------------------------
**Last affected version**: v1.5.6
**Produced by the reference compressor**: No
**Example Frame**: `28b5 2ffd 0000 5500 0072 8001 0420 7e1f 02aa 00`
When using FSE-compressed Huffman weights, the compressed weight bitstream
could contain fewer bits than necessary to decode the initial states.
The reference decompressor up to v1.5.6 will decode truncated or missing
initial states as zero, which can result in a valid Huffman tree if only
the second state is truncated.
In newer versions, truncated initial states are reported as a corruption
error by the decoder.
Offset == 0
-----------
+1 -1
View File
@@ -10,7 +10,7 @@
ZSTD ?= zstd # note: requires zstd installation on local system
UNAME?= $(shell sh -c 'MSYSTEM="MSYS" uname')
UNAME?= $(shell uname)
ifeq ($(UNAME), SunOS)
DIFF ?= gdiff
else
+107 -147
View File
@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
### Version
0.4.5 (2026-05-14)
0.4.0 (2023-06-05)
Introduction
@@ -387,24 +387,20 @@ When `Block_Type` is `RLE_Block`, since `Block_Content`s size is always 1,
__`Block_Content`__ and __`Block_Maximum_Size`__
The size of `Block_Content` is limited by `Block_Maximum_Size`, which is
determined once for a given frame and is the smallest of:
The size of `Block_Content` is limited by `Block_Maximum_Size`,
which is the smallest of:
- `Window_Size`
- 128 KiB (131.072 bytes)
- 128 KB
Both the `Block_Content` and the decompressed size of any block in the frame must
be no larger than `Block_Maximum_Size`.
`Block_Maximum_Size` is constant for a given frame.
This maximum is applicable to both the decompressed size
and the compressed size of any block in the frame.
The reasoning for this limit is that a decoder can read this information
at the beginning of a frame and use it to allocate buffers.
The guarantees on the size of blocks ensure that
the buffers will be large enough for any following block of the valid frame.
If a compressed block is larger than its uncompressed content, it is recommended
to send it uncompressed (i.e., a `Raw_Block`). However, as long as
`Block_Content` is no larger than `Block_Maximum_Size`, it is legal to send such
a compressed block, even if it's larger than its uncompressed content.
Compressed Blocks
-----------------
@@ -1042,54 +1038,53 @@ and to compress Huffman headers.
FSE
---
FSE, short for Finite State Entropy, is an entropy codec based on [ANS].
FSE encoding/decoding involves a state that is carried over between symbols.
Decoding must be done in the opposite direction as encoding.
FSE encoding/decoding involves a state that is carried over between symbols,
so decoding must be done in the opposite direction as encoding.
Therefore, all FSE bitstreams are read from end to beginning.
Note that the order of the bits in the stream is not reversed,
we just read each multi-bits element in the reverse order they are encoded.
we just read the elements in the reverse order they are written.
For additional details on FSE, see [Finite State Entropy].
[Finite State Entropy]:https://github.com/Cyan4973/FiniteStateEntropy/
FSE decoding is directed by a decoding table with a power of 2 size, each row containing three elements:
FSE decoding involves a decoding table which has a power of 2 size, and contain three elements:
`Symbol`, `Num_Bits`, and `Baseline`.
The `log2` of the table size is its `Accuracy_Log`.
An FSE state value represents an index in this table.
To obtain the initial state value, consume `Accuracy_Log` bits from the stream as a __little-endian__ value.
The first symbol in the stream is the `Symbol` indicated in the table for that state.
The next symbol in the stream is the `Symbol` indicated in the table for that state.
To obtain the next state value,
the decoder should consume `Num_Bits` bits from the stream as a __little-endian__ value and add it to `Baseline`.
[ANS]: https://en.wikipedia.org/wiki/Asymmetric_Numeral_Systems
### FSE Table Description
To decode an FSE bitstream, it is necessary to build its FSE decoding table.
The decoding table is derived from a distribution of Probabilities.
The Zstandard format encodes distributions of Probabilities as follows:
To decode FSE streams, it is necessary to construct the decoding table.
The Zstandard format encodes FSE table descriptions as follows:
The distribution of probabilities is described in a bitstream which is read forward,
in __little-endian__ fashion.
The amount of bytes consumed from the bitstream to describe the distribution
is discovered at the end of the decoding process.
An FSE distribution table describes the probabilities of all symbols
from `0` to the last present one (included)
on a normalized scale of `1 << Accuracy_Log` .
Note that there must be two or more symbols with nonzero probability.
The bitstream starts by reporting on which scale the distribution operates.
It's a bitstream which is read forward, in __little-endian__ fashion.
It's not necessary to know bitstream exact size,
it will be discovered and reported by the decoding process.
The bitstream starts by reporting on which scale it operates.
Let's `low4Bits` designate the lowest 4 bits of the first byte :
`Accuracy_Log = low4bits + 5`.
An FSE distribution table describes the probabilities of all symbols
from `0` to the last present one (included) in natural order.
The sum of probabilities is normalized to reach a power of 2 total of `1 << Accuracy_Log` .
There must be two or more symbols with non-zero probabilities.
The number of bits used to decode each probability is variable.
Then follows each symbol value, from `0` to last present one.
The number of bits used by each field is variable.
It depends on :
- Remaining probabilities + 1 :
__example__ :
Presuming an `Accuracy_Log` of 8,
and presuming 100 probability points have already been distributed,
and presuming 100 probabilities points have already been distributed,
the decoder may read any value from `0` to `256 - 100 + 1 == 157` (inclusive).
Therefore, it may read up to `log2sup(157) == 8` bits, where `log2sup(N)`
is the smallest integer `T` that satisfies `(1 << T) > N`.
@@ -1103,133 +1098,114 @@ It depends on :
values from 98 to 157 use 8 bits.
This is achieved through this scheme :
| 8-bit field read | Value decoded | Nb of bits consumed |
| ---------------- | ------------- | ------------------- |
| Value read | Value decoded | Number of bits used |
| ---------- | ------------- | ------------------- |
| 0 - 97 | 0 - 97 | 7 |
| 98 - 127 | 98 - 127 | 8 |
| 128 - 225 | 0 - 97 | 7 |
| 226 - 255 | 128 - 157 | 8 |
Probability is derived from Value decoded using the following formula:
`Probality = Value - 1`
Symbols probabilities are read one by one, in order.
Consequently, a Probability of `0` is described by a Value `1`.
Probability is obtained from Value decoded by following formula :
`Proba = value - 1`
A Value `0` is used to signal a special case, named "Probability `-1`".
It describes a probability which should have been "less than 1".
Its effect on the decoding table building process is described in the [next section].
For the purpose of counting total allocated probability points, it counts as one.
It means value `0` becomes negative probability `-1`.
`-1` is a special probability, which means "less than 1".
Its effect on distribution table is described in the [next section].
For the purpose of calculating total allocated probability points, it counts as one.
[next section]:#from-normalized-distribution-to-decoding-tables
Symbols probabilities are read one by one, in order.
After each probability is decoded, the total nb of probability points is updated.
This is used to determine how many bits must be read to decode the probability of next symbol.
When a symbol has a __probability__ of `zero` (decoded from reading a Value `1`),
When a symbol has a __probability__ of `zero`,
it is followed by a 2-bits repeat flag.
This repeat flag tells how many probabilities of zeroes follow the current one.
It provides a number ranging from 0 to 3.
If it is a 3, another 2-bits repeat flag follows, and so on.
When the Probability for a symbol makes cumulated total reach `1 << Accuracy_Log`,
then it's the last symbol, and decoding is complete.
When last symbol reaches cumulated total of `1 << Accuracy_Log`,
decoding is complete.
If the last symbol makes cumulated total go above `1 << Accuracy_Log`,
distribution is considered corrupted.
If this process results in a non-zero probability for a value outside of the
valid range of values that the FSE table is defined for, even if that value is
not used, then the data is considered corrupted.
Then the decoder can tell how many bytes were used in this process,
and how many symbols are present.
The bitstream consumes a round number of bytes.
Any remaining bit within the last byte is just unused.
If this process results in a non-zero probability for a symbol outside of the
valid range of symbols that the FSE table is defined for, even if that symbol is
not used, then the data is considered corrupted.
For the specific case of offset codes,
a decoder implementation may reject a frame containing a non-zero probability
for an offset code larger than the largest offset code supported by the decoder
implementation.
#### From normalized distribution to decoding tables
The normalized distribution of probabilities is enough
The distribution of normalized probabilities is enough
to create a unique decoding table.
It is generated using the following build rule :
It follows the following build rule :
The table has a size of `Table_Size = 1 << Accuracy_Log`.
Each row specifies the decoded symbol,
and instructions to reach the next state (`Number_of_Bits` and `Baseline`).
Each cell describes the symbol decoded,
and instructions to get the next state (`Number_of_Bits` and `Baseline`).
Symbols are first scanned in their natural order for "less than 1" probabilities
(previously decoded from a Value of `0`).
Symbols with this special probability are being attributed a single row,
Symbols are scanned in their natural order for "less than 1" probabilities.
Symbols with this probability are being attributed a single cell,
starting from the end of the table and retreating.
These symbols define a full state reset, reading `Accuracy_Log` bits.
Then, all remaining symbols, sorted in natural order, are allocated rows.
Starting from smallest present symbol, and table position `0`,
each symbol gets allocated as many rows as its probability.
Then, all remaining symbols, sorted in natural order, are allocated cells.
Starting from symbol `0` (if it exists), and table position `0`,
each symbol gets allocated as many cells as its probability.
Cell allocation is spread, not linear :
each successor position follows this rule :
Row allocation is not linear, it follows this order, in modular arithmetic:
```
position += (tableSize>>1) + (tableSize>>3) + 3;
position &= tableSize-1;
```
Using above ordering rule, each symbol gets allocated as many rows as its probability.
If a position is already occupied by a "less than 1" probability symbol,
it is simply skipped, and the next position is allocated instead.
Once enough rows have been allocated for the current symbol,
the allocation process continues, using the next symbol, in natural order.
This process guarantees that the table is entirely and exactly filled.
A position is skipped if already occupied by a "less than 1" probability symbol.
`position` does not reset between symbols, it simply iterates through
each position in the table, switching to the next symbol when enough
states have been allocated to the current one.
Each row specifies a decoded symbol, and is accessed by current state value.
It also specifies `Number_of_Bits` and `Baseline`, which are required to determine next state value.
The process guarantees that the table is entirely filled.
Each cell corresponds to a state value, which contains the symbol being decoded.
To correctly set these fields, it's necessary to sort all occurrences of each symbol in state value order,
and then attribute N+1 bits to lower rows, and N bits to higher rows,
following the process described below (using an example):
To add the `Number_of_Bits` and `Baseline` required to retrieve next state,
it's first necessary to sort all occurrences of each symbol in state order.
Lower states will need 1 more bit than higher ones.
The process is repeated for each symbol.
__Example__ :
Presuming an `Accuracy_Log` of 7,
let's imagine a symbol with a Probability of 5:
it receives 5 rows, corresponding to 5 state values between `0` and `127`.
Presuming a symbol has a probability of 5,
it receives 5 cells, corresponding to 5 state values.
These state values are then sorted in natural order.
In this example, the first state value happens to be `1` (after unspecified previous symbols).
The next 4 states are then determined using above modular arithmetic rule,
which specifies to add `64+16+3 = 83` modulo `128` to jump to next position,
producing the following series: `1`, `84`, `39`, `122`, `77` (modular arithmetic).
(note: the next symbol will then start at `32`).
Next power of 2 after 5 is 8.
Space of probabilities must be divided into 8 equal parts.
Presuming the `Accuracy_Log` is 7, it defines a space of 128 states.
Divided by 8, each share is 16 large.
These state values are then sorted in natural order,
resulting in the following series: `1`, `39`, `77`, `84`, `122`.
The next power of 2 after 5 is 8.
Therefore, the probability space will be divided into 8 equal parts.
Since the probability space is `1<<7 = 128` large, each share is `128/8 = 16` large.
In order to reach 8 shares, the `8-5 = 3` lowest states will count "double",
In order to reach 8 shares, 8-5=3 lowest states will count "double",
doubling their shares (32 in width), hence requiring one more bit.
Baseline is assigned starting from the lowest state using fewer bits,
continuing in natural state order, looping back at the beginning.
Each state takes its allocated range from Baseline, sized by its `Number_of_Bits`.
Baseline is assigned starting from the higher states using fewer bits,
increasing at each state, then resuming at the first state,
each state takes its allocated width from Baseline.
| state order | 0 | 1 | 2 | 3 | 4 |
| ---------------- | ----- | ----- | ------ | ---- | ------ |
| state value | 1 | 39 | 77 | 84 | 122 |
| width | 32 | 32 | 32 | 16 | 16 |
| `Number_of_Bits` | 5 | 5 | 5 | 4 | 4 |
| allocation order | 3 | 4 | 5 | 1 | 2 |
| range number | 2 | 4 | 6 | 0 | 1 |
| `Baseline` | 32 | 64 | 96 | 0 | 16 |
| range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
During decoding, the next state value is determined by using current state value as row number,
then reading the required `Number_of_Bits` from the bitstream, and adding the specified `Baseline`.
During decoding, the next state value is determined from current state value,
by reading the required `Number_of_Bits`, and adding the specified `Baseline`.
Note:
as a trivial example, it follows that, for a symbol with a Probability of `1`,
`Baseline` is necessarily `0`, and `Number_of_Bits` is necessarily `Accuracy_Log`.
See [Appendix A] to see the outcome of this process applied to the default distributions.
See [Appendix A] for the results of this process applied to the default distributions.
[Appendix A]: #appendix-a---decoding-tables-for-predefined-codes
@@ -1274,13 +1250,13 @@ This specification limits maximum code length to 11 bits.
#### Representation
All literal symbols from zero (included) to last present one (excluded)
All literal values from zero (included) to last present one (excluded)
are represented by `Weight` with values from `0` to `Max_Number_of_Bits`.
Transformation from `Weight` to `Number_of_Bits` follows this formula :
```
Number_of_Bits = Weight ? (Max_Number_of_Bits + 1 - Weight) : 0
```
When a literal symbol is not present, it receives a `Weight` of 0.
When a literal value is not present, it receives a `Weight` of 0.
The least frequent symbol receives a `Weight` of 1.
If no literal has a `Weight` of 1, then the data is considered corrupted.
If there are not at least two literals with non-zero `Weight`, then the data
@@ -1297,38 +1273,33 @@ otherwise the representation is considered corrupted.
__Example__ :
Let's presume the following Huffman tree must be described :
| literal symbol | A | B | C | D | E | F |
| literal value | 0 | 1 | 2 | 3 | 4 | 5 |
| ---------------- | --- | --- | --- | --- | --- | --- |
| `Number_of_Bits` | 1 | 2 | 3 | 0 | 4 | 4 |
The tree depth is 4, since its longest elements uses 4 bits
(longest elements are the ones with smallest frequency).
All symbols will now receive a `Weight` instead of `Number_of_Bits`.
(longest elements are the one with smallest frequency).
Literal value `5` will not be listed, as it can be determined from previous values 0-4,
nor will values above `5` as they are all 0.
Values from `0` to `4` will be listed using `Weight` instead of `Number_of_Bits`.
Weight formula is :
```
Weight = Number_of_Bits ? (Max_Number_of_Bits + 1 - Number_of_Bits) : 0
```
It gives the following series of Weights :
It gives the following series of weights :
| literal symbol | A | B | C | D | E | F |
| -------------- | --- | --- | --- | --- | --- | --- |
| `Weight` | 4 | 3 | 2 | 0 | 1 | 1 |
This list will be sent to the decoder, with the following modifications:
- `F` will not be listed, because it can be determined from previous symbols
- nor will symbols above `F` as they are all 0
- on the other hand, all symbols before `A`, starting with `\0`, will be listed, with a Weight of 0.
| literal value | 0 | 1 | 2 | 3 | 4 |
| ------------- | --- | --- | --- | --- | --- |
| `Weight` | 4 | 3 | 2 | 0 | 1 |
The decoder will do the inverse operation :
having collected weights of literal symbols from `A` to `E`,
it knows the last literal, `F`, is present with a non-zero `Weight`.
The `Weight` of `F` can be determined by advancing to the next power of 2.
having collected weights of literal symbols from `0` to `4`,
it knows the last literal, `5`, is present with a non-zero `Weight`.
The `Weight` of `5` can be determined by advancing to the next power of 2.
The sum of `2^(Weight-1)` (excluding 0's) is :
`8 + 4 + 2 + 0 + 1 = 15`.
Nearest larger power of 2 value is 16.
Therefore, `Max_Number_of_Bits = log2(16) = 4` and `Weight[F] = log_2(16 - 15) + 1 = 1`.
Therefore, `Max_Number_of_Bits = 4` and `Weight[5] = log_2(16 - 15) + 1 = 1`.
#### Huffman Tree header
@@ -1368,7 +1339,7 @@ sharing a single distribution table.
To decode an FSE bitstream, it is necessary to know its compressed size.
Compressed size is provided by `headerByte`.
It's also necessary to know its _maximum possible_ decompressed size,
which is `255`, since literal symbols span from `0` to `255`,
which is `255`, since literal values span from `0` to `255`,
and last symbol's `Weight` is not represented.
An FSE bitstream starts by a header, describing probabilities distribution.
@@ -1393,10 +1364,6 @@ symbols for each of the final states are decoded and the process is complete.
If this process would produce more weights than the maximum number of decoded
weights (255), then the data is considered corrupted.
If either of the 2 initial states are absent or truncated, then the data is
considered corrupted. Consequently, it is not possible to encode fewer than
2 weights using this mode.
#### Conversion from weights to Huffman prefix codes
All present symbols shall now have a `Weight` value.
@@ -1404,28 +1371,26 @@ It is possible to transform weights into `Number_of_Bits`, using this formula:
```
Number_of_Bits = (Weight>0) ? Max_Number_of_Bits + 1 - Weight : 0
```
In order to determine which prefix code is assigned to each Symbol,
Symbols are first sorted by `Weight`, then by natural sequential order.
Symbols are sorted by `Weight`.
Within same `Weight`, symbols keep natural sequential order.
Symbols with a `Weight` of zero are removed.
Then, starting from lowest `Weight` (hence highest `Number_of_Bits`),
prefix codes are assigned in ascending order.
Then, starting from lowest `Weight`, prefix codes are distributed in sequential order.
__Example__ :
Let's assume the following list of weights has been decoded:
Let's presume the following list of weights has been decoded :
| Literal | A | B | C | D | E | F |
| Literal | 0 | 1 | 2 | 3 | 4 | 5 |
| -------- | --- | --- | --- | --- | --- | --- |
| `Weight` | 4 | 3 | 2 | 0 | 1 | 1 |
Sorted by weight and then natural sequential order,
it gives the following prefix codes distribution:
it gives the following distribution :
| Literal | D | E | F | C | B | A |
| ---------------- | --- | ---- | ---- | ---- | ---- | ---- |
| Literal | 3 | 4 | 5 | 2 | 1 | 0 |
| ---------------- | --- | --- | --- | --- | --- | ---- |
| `Weight` | 0 | 1 | 1 | 2 | 3 | 4 |
| `Number_of_Bits` | 0 | 4 | 4 | 3 | 2 | 1 |
| prefix code | N/A | 0000 | 0001 | 001 | 01 | 1 |
| ascending order | N/A | 0000 | 0001 | 001x | 01xx | 1xxx |
| prefix codes | N/A | 0000| 0001| 001 | 01 | 1 |
### Huffman-coded Streams
@@ -1448,10 +1413,10 @@ it's possible to read the bitstream in a __little-endian__ fashion,
keeping track of already used bits. Since the bitstream is encoded in reverse
order, starting from the end read symbols in forward order.
For example, if the literal sequence `ABEF` was encoded using above prefix code,
For example, if the literal sequence "0145" was encoded using above prefix code,
it would be encoded (in reverse order) as:
|Symbol | F | E | B | A | Padding |
|Symbol | 5 | 4 | 1 | 0 | Padding |
|--------|------|------|----|---|---------|
|Encoding|`0000`|`0001`|`01`|`1`| `00001` |
@@ -1746,11 +1711,6 @@ or at least provide a meaningful error code explaining for which reason it canno
Version changes
---------------
- 0.4.5 : minor clarification regarding Block_Maximum_Size
- 0.4.4 : minor clarification for block size
- 0.4.3 : clarifications for Huffman prefix code assignment example
- 0.4.2 : refactor FSE table construction process, inspired by Donald Pian
- 0.4.1 : clarifications on a few error scenarios, by Eric Lasota
- 0.4.0 : fixed imprecise behavior for nbSeq==0, detected by Igor Pavlov
- 0.3.9 : clarifications for Huffman-compressed literal sizes.
- 0.3.8 : clarifications for Huffman Blocks and Huffman Tree descriptions.
+88 -162
View File
@@ -1,17 +1,16 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>zstd 1.5.7 Manual</title>
<title>zstd 1.5.6 Manual</title>
</head>
<body>
<h1>zstd 1.5.7 Manual</h1>
Note: the content of this file has been automatically generated by parsing "zstd.h"
<h1>zstd 1.5.6 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
<li><a href="#Chapter1">Introduction</a></li>
<li><a href="#Chapter2">Version</a></li>
<li><a href="#Chapter3">Simple Core API</a></li>
<li><a href="#Chapter3">Simple API</a></li>
<li><a href="#Chapter4">Explicit context</a></li>
<li><a href="#Chapter5">Advanced compression API (Requires v1.4.0+)</a></li>
<li><a href="#Chapter6">Advanced decompression API (Requires v1.4.0+)</a></li>
@@ -75,7 +74,7 @@ Note: the content of this file has been automatically generated by parsing "zstd
</b><p> Return runtime library version, like "1.4.5". Requires v1.3.0+.
</p></pre><BR>
<a name="Chapter3"></a><h2>Simple Core API</h2><pre></pre>
<a name="Chapter3"></a><h2>Simple API</h2><pre></pre>
<pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
@@ -90,16 +89,12 @@ Note: the content of this file has been automatically generated by parsing "zstd
<pre><b>size_t ZSTD_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
</b><p> `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
Multiple compressed frames can be decompressed at once with this method.
The result will be the concatenation of all decompressed frames, back to back.
`dstCapacity` is an upper bound of originalSize to regenerate.
First frame's decompressed size can be extracted using ZSTD_getFrameContentSize().
If maximum upper bound isn't known, prefer using streaming mode to decompress data.
If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTD_isError()).
</p></pre><BR>
<h3>Decompression helper functions</h3><pre></pre><b><pre></pre></b><BR>
<pre><b>#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
@@ -110,8 +105,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
- ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
- ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
note 1 : a 0 return value means the frame is valid but "empty".
When invoking this method on a skippable frame, it will return 0.
note 2 : decompressed size is an optional field, it may not be present (typically in streaming mode).
note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
In which case, it's necessary to use streaming mode to decompress data.
Optionally, application can rely on some implicit limit,
@@ -129,8 +123,9 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
</p></pre><BR>
<pre><b>ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
ZSTDLIB_API
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
</b><p> This function is now obsolete, in favor of ZSTD_getFrameContentSize().
</b><p> NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
Both functions work the same way, but ZSTD_getDecompressedSize() blends
"empty", "unknown" and "error" results to the same return value (0),
while ZSTD_getFrameContentSize() gives them separate return values.
@@ -143,42 +138,34 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
@return : the compressed size of the first frame starting at `src`,
suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
or an error code if input is invalid
Note 1: this method is called _find*() because it's not enough to read the header,
it may have to scan through the frame's content, to reach its end.
Note 2: this method also works with Skippable Frames. In which case,
it returns the size of the complete skippable frame,
which is always equal to its content size + 8 bytes for headers.
</p></pre><BR>
<h3>Compression helper functions</h3><pre></pre><b><pre></pre></b><BR>
<pre><b>#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U)
<h3>Helper functions</h3><pre></pre><b><pre></b>/* ZSTD_compressBound() :<b>
* maximum compressed size in worst case single-pass scenario.
* When invoking `ZSTD_compress()` or any other one-pass compression function,
* it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
* as it eliminates one potential failure scenario,
* aka not enough room in dst buffer to write the compressed frame.
* Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
* In which case, ZSTD_compressBound() will return an error code
* which can be tested using ZSTD_isError().
*
* ZSTD_COMPRESSBOUND() :
* same as ZSTD_compressBound(), but as a macro.
* It can be used to produce constants, which can be useful for static allocation,
* for example to size a static array on stack.
* Will produce constant value 0 if srcSize too large.
*/
#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U)
#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b>
size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b>
</b><p> maximum compressed size in worst case single-pass scenario.
When invoking `ZSTD_compress()`, or any other one-pass compression function,
it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
as it eliminates one potential failure scenario,
aka not enough room in dst buffer to write the compressed frame.
Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE .
In which case, ZSTD_compressBound() will return an error code
which can be tested using ZSTD_isError().
ZSTD_COMPRESSBOUND() :
same as ZSTD_compressBound(), but as a macro.
It can be used to produce constants, which can be useful for static allocation,
for example to size a static array on stack.
Will produce constant value 0 if srcSize is too large.
</p></pre><BR>
<h3>Error helper functions</h3><pre></pre><b><pre></b>/* ZSTD_isError() :<b>
</b>/* ZSTD_isError() :<b>
* Most ZSTD_* functions returning a size_t value can be tested for error,
* using ZSTD_isError().
* @return 1 if error, 0 otherwise
*/
unsigned ZSTD_isError(size_t result); </b>/*!< tells if a `size_t` function result is an error code */<b>
ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); </b>/* convert a result into an error code, which can be compared to error enum list */<b>
const char* ZSTD_getErrorName(size_t result); </b>/*!< provides readable string from a function result */<b>
unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` function result is an error code */<b>
const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b>
int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed, requires v1.4.0+ */<b>
int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
int ZSTD_defaultCLevel(void); </b>/*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */<b>
@@ -186,17 +173,17 @@ int ZSTD_defaultCLevel(void); </b>/*!< default compression leve
<a name="Chapter4"></a><h2>Explicit context</h2><pre></pre>
<h3>Compression context</h3><pre> When compressing many times,
it is recommended to allocate a compression context just once,
it is recommended to allocate a context just once,
and reuse it for each successive compression operation.
This will make the workload easier for system's memory.
This will make workload friendlier for system's memory.
Note : re-using context is just a speed / resource optimization.
It doesn't change the compression ratio, which remains identical.
Note 2: For parallel execution in multi-threaded environments,
use one different context per thread .
Note 2 : In multi-threaded environments,
use one different context per thread for parallel execution.
</pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
ZSTD_CCtx* ZSTD_createCCtx(void);
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); </b>/* compatible with NULL pointer */<b>
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); </b>/* accept NULL pointer */<b>
</pre></b><BR>
<pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
@@ -207,7 +194,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); </b>/* compatible with NULL pointer
this function compresses at the requested compression level,
__ignoring any other advanced parameter__ .
If any advanced parameter was set using the advanced API,
they will all be reset. Only @compressionLevel remains.
they will all be reset. Only `compressionLevel` remains.
</p></pre><BR>
@@ -311,7 +298,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); </b>/* accept NULL pointer */<b>
* Special: value 0 means "use default strategy". */
ZSTD_c_targetCBlockSize=130, </b>/* v1.5.6+<b>
* Attempts to fit compressed block size into approximately targetCBlockSize.
* Attempts to fit compressed block size into approximatively targetCBlockSize.
* Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
* Note that it's not a guarantee, just a convergence target (default:0).
* No target when targetCBlockSize == 0.
@@ -407,8 +394,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); </b>/* accept NULL pointer */<b>
* ZSTD_c_stableOutBuffer
* ZSTD_c_blockDelimiters
* ZSTD_c_validateSequences
* ZSTD_c_blockSplitterLevel
* ZSTD_c_splitAfterSequences
* ZSTD_c_useBlockSplitter
* ZSTD_c_useRowMatchFinder
* ZSTD_c_prefetchCDictTables
* ZSTD_c_enableSeqProducerFallback
@@ -435,8 +421,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); </b>/* accept NULL pointer */<b>
ZSTD_c_experimentalParam16=1013,
ZSTD_c_experimentalParam17=1014,
ZSTD_c_experimentalParam18=1015,
ZSTD_c_experimentalParam19=1016,
ZSTD_c_experimentalParam20=1017
ZSTD_c_experimentalParam19=1016
} ZSTD_cParameter;
</b></pre><BR>
<pre><b>typedef struct {
@@ -733,7 +718,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
<a name="Chapter9"></a><h2>Streaming decompression - HowTo</h2><pre>
A ZSTD_DStream object is required to track streaming operations.
Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
ZSTD_DStream objects can be re-employed multiple times.
ZSTD_DStream objects can be reused multiple times.
Use ZSTD_initDStream() to start a new decompression operation.
@return : recommended first input size
@@ -743,21 +728,16 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
The function will update both `pos` fields.
If `input.pos < input.size`, some input has not been consumed.
It's up to the caller to present again remaining data.
The function tries to flush all data decoded immediately, respecting output buffer size.
If `output.pos < output.size`, decoder has flushed everything it could.
However, when `output.pos == output.size`, it's more difficult to know.
If @return > 0, the frame is not complete, meaning
either there is still some data left to flush within internal buffers,
or there is more input to read to complete the frame (or both).
But if `output.pos == output.size`, there might be some data left within internal buffers.,
In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
@return : 0 when a frame is completely decoded and fully flushed,
or an error code, which can be tested using ZSTD_isError(),
or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
the return value is a suggested next input size (just a hint for better latency)
that will never request more than the remaining content of the compressed frame.
that will never request more than the remaining frame size.
<BR></pre>
@@ -783,10 +763,9 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds); </b>/* accept NULL pointer */<b>
Function will update both input and output `pos` fields exposing current state via these fields:
- `input.pos < input.size`, some input remaining and caller should provide remaining input
on the next call.
- `output.pos < output.size`, decoder flushed internal output buffer.
- `output.pos == output.size`, unflushed data potentially present in the internal buffers,
check ZSTD_decompressStream() @return value,
if > 0, invoke it again to flush remaining data to output.
- `output.pos < output.size`, decoder finished and flushed all remaining buffers.
- `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
call ZSTD_decompressStream() again to flush remaining data to output.
Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
@return : 0 when a frame is completely decoded and fully flushed,
@@ -1088,7 +1067,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
*
* Note: This field is optional. ZSTD_generateSequences() will calculate the value of
* 'rep', but repeat offsets do not necessarily need to be calculated from an external
* sequence provider perspective. For example, ZSTD_compressSequences() does not
* sequence provider's perspective. For example, ZSTD_compressSequences() does not
* use this 'rep' field at all (as of now).
*/
} ZSTD_Sequence;
@@ -1193,14 +1172,14 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
} ZSTD_literalCompressionMode_e;
</b></pre><BR>
<pre><b>typedef enum {
</b>/* Note: This enum controls features which are conditionally beneficial.<b>
* Zstd can take a decision on whether or not to enable the feature (ZSTD_ps_auto),
* but setting the switch to ZSTD_ps_enable or ZSTD_ps_disable force enable/disable the feature.
</b>/* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final<b>
* decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
* or ZSTD_ps_disable allow for a force enable/disable the feature.
*/
ZSTD_ps_auto = 0, </b>/* Let the library automatically determine whether the feature shall be enabled */<b>
ZSTD_ps_enable = 1, </b>/* Force-enable the feature */<b>
ZSTD_ps_disable = 2 </b>/* Do not use the feature */<b>
} ZSTD_ParamSwitch_e;
} ZSTD_paramSwitch_e;
</b></pre><BR>
<a name="Chapter15"></a><h2>Frame header and size functions</h2><pre></pre>
@@ -1243,33 +1222,33 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
</p></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
</b><p> srcSize must be large enough, aka >= ZSTD_FRAMEHEADERSIZE_PREFIX.
</b><p> srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
@return : size of the Frame Header,
or an error code (if srcSize is too small)
</p></pre><BR>
<pre><b>typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_FrameType_e;
<pre><b>typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
</b></pre><BR>
<pre><b>typedef struct {
unsigned long long frameContentSize; </b>/* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */<b>
unsigned long long windowSize; </b>/* can be very large, up to <= frameContentSize */<b>
unsigned blockSizeMax;
ZSTD_FrameType_e frameType; </b>/* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */<b>
ZSTD_frameType_e frameType; </b>/* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */<b>
unsigned headerSize;
unsigned dictID; </b>/* for ZSTD_skippableFrame, contains the skippable magic variant [0-15] */<b>
unsigned dictID;
unsigned checksumFlag;
unsigned _reserved1;
unsigned _reserved2;
} ZSTD_FrameHeader;
} ZSTD_frameHeader;
</b></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize);
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input */<b>
</b>/*! ZSTD_getFrameHeader_advanced() :<b>
* same as ZSTD_getFrameHeader(),
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
</b><p> decode Frame Header into `zfhPtr`, or requires larger `srcSize`.
@return : 0 => header is complete, `zfhPtr` is correctly filled,
>0 => `srcSize` is too small, @return value is the wanted `srcSize` amount, `zfhPtr` is not filled,
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
</b><p> decode Frame Header, or requires larger `srcSize`.
@return : 0, `zfhPtr` is correctly filled,
>0, `srcSize` is too small, value is wanted `srcSize` amount,
or an error code, which can be tested using ZSTD_isError()
</p></pre><BR>
@@ -1319,9 +1298,9 @@ ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr,
</p></pre><BR>
<pre><b>typedef enum {
ZSTD_sf_noBlockDelimiters = 0, </b>/* ZSTD_Sequence[] has no block delimiters, just sequences */<b>
ZSTD_sf_explicitBlockDelimiters = 1 </b>/* ZSTD_Sequence[] contains explicit block delimiters */<b>
} ZSTD_SequenceFormat_e;
ZSTD_sf_noBlockDelimiters = 0, </b>/* Representation of ZSTD_Sequence has no block delimiters, sequences only */<b>
ZSTD_sf_explicitBlockDelimiters = 1 </b>/* Representation of ZSTD_Sequence contains explicit block delimiters */<b>
} ZSTD_sequenceFormat_e;
</b></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
</b><p> `srcSize` : size of the input buffer
@@ -1332,37 +1311,19 @@ ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr,
</p></pre><BR>
<pre><b>ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()")
ZSTDLIB_STATIC_API size_t
ZSTD_generateSequences(ZSTD_CCtx* zc,
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
const void* src, size_t srcSize);
</b><p> WARNING: This function is meant for debugging and informational purposes ONLY!
Its implementation is flawed, and it will be deleted in a future version.
It is not guaranteed to succeed, as there are several cases where it will give
up and fail. You should NOT use this function in production code.
This function is deprecated, and will be removed in a future version.
Generate sequences using ZSTD_compress2(), given a source buffer.
@param zc The compression context to be used for ZSTD_compress2(). Set any
compression parameters you need on this context.
@param outSeqs The output sequences buffer of size @p outSeqsSize
@param outSeqsCapacity The size of the output sequences buffer.
ZSTD_sequenceBound(srcSize) is an upper bound on the number
of sequences that can be generated.
@param src The source buffer to generate sequences from of size @p srcSize.
@param srcSize The size of the source buffer.
<pre><b></b><p> Generate sequences using ZSTD_compress2(), given a source buffer.
Each block will end with a dummy sequence
with offset == 0, matchLength == 0, and litLength == length of last literals.
litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
simply acts as a block delimiter.
@returns The number of sequences generated, necessarily less than
ZSTD_sequenceBound(srcSize), or an error code that can be checked
with ZSTD_isError().
@zc can be used to insert custom compression params.
This function invokes ZSTD_compress2().
The output of this function can be fed into ZSTD_compressSequences() with CCtx
setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
@return : number of sequences generated
</p></pre><BR>
@@ -1380,14 +1341,13 @@ ZSTD_generateSequences(ZSTD_CCtx* zc,
</p></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t
ZSTD_compressSequences(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* src, size_t srcSize);
</b><p> Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
@src contains the entire input (not just the literals).
If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
If a dictionary is included, then the cctx should reference the dict (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.).
If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
The entire source is compressed into a single frame.
The compression behavior changes based on cctx params. In particular:
@@ -1396,17 +1356,11 @@ ZSTD_compressSequences(ZSTD_CCtx* cctx,
the block size derived from the cctx, and sequences may be split. This is the default setting.
If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
valid block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
When ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, it's possible to decide generating repcodes
using the advanced parameter ZSTD_c_repcodeResolution. Repcodes will improve compression ratio, though the benefit
can vary greatly depending on Sequences. On the other hand, repcode resolution is an expensive operation.
By default, it's disabled at low (<10) compression levels, and enabled above the threshold (>=10).
ZSTD_c_repcodeResolution makes it possible to directly manage this processing in either direction.
If ZSTD_c_validateSequences == 0, this function blindly accepts the Sequences provided. Invalid Sequences cause undefined
behavior. If ZSTD_c_validateSequences == 1, then the function will detect invalid Sequences (see doc/zstd_compression_format.md for
specifics regarding offset/matchlength requirements) and then bail out and return an error.
If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
In addition to the two adjustable experimental params, there are other important cctx params.
- ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
@@ -1414,47 +1368,21 @@ ZSTD_compressSequences(ZSTD_CCtx* cctx,
- ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
Note: Repcodes are, as of now, always re-calculated within this function, ZSTD_Sequence.rep is effectively unused.
Dev Note: Once ability to ingest repcodes become available, the explicit block delims mode must respect those repcodes exactly,
and cannot emit an RLE block that disagrees with the repcode history.
@return : final compressed size, or a ZSTD error code.
</p></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t
ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t nbSequences,
const void* literals, size_t litSize, size_t litBufCapacity,
size_t decompressedSize);
</b><p> This is a variant of ZSTD_compressSequences() which,
instead of receiving (src,srcSize) as input parameter, receives (literals,litSize),
aka all the literals, already extracted and laid out into a single continuous buffer.
This can be useful if the process generating the sequences also happens to generate the buffer of literals,
thus skipping an extraction + caching stage.
It's a speed optimization, useful when the right conditions are met,
but it also features the following limitations:
- Only supports explicit delimiter mode
- Currently does not support Sequences validation (so input Sequences are trusted)
- Not compatible with frame checksum, which must be disabled
- If any block is incompressible, will fail and return an error
- @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error.
- @litBufCapacity is the size of the underlying buffer into which literals are written, starting at address @literals.
@litBufCapacity must be at least 8 bytes larger than @litSize.
- @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error.
Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
and cannot emit an RLE block that disagrees with the repcode history
@return : final compressed size, or a ZSTD error code.
</p></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
unsigned magicVariant);
const void* src, size_t srcSize, unsigned magicVariant);
</b><p> Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
As such, the parameter magicVariant controls the exact skippable frame magic number variant used,
so the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
Returns an error if destination buffer is not large enough, if the source size is not representable
with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
@@ -1463,14 +1391,13 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
</p></pre><BR>
<pre><b>ZSTDLIB_STATIC_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity,
unsigned* magicVariant,
<pre><b>size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
const void* src, size_t srcSize);
</b><p> Retrieves the content of a zstd skippable frame starting at @src, and writes it to @dst buffer.
</b><p> Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
The parameter @magicVariant will receive the magicVariant that was supplied when the frame was written,
i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.
This can be NULL if the caller is not interested in the magicVariant.
The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
in the magicVariant.
Returns an error if destination buffer is not large enough, or if the frame is not skippable.
@@ -1478,7 +1405,7 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
</p></pre><BR>
<pre><b>ZSTDLIB_STATIC_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
<pre><b>unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
</b><p> Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
</p></pre><BR>
@@ -1585,14 +1512,13 @@ static
#ifdef __GNUC__
__attribute__((__unused__))
#endif
ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< this constant defers to stdlib's functions */<b>
</b><p> These prototypes make it possible to pass your own allocation/free functions.
ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
</p></pre><BR>
<pre><b>ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< this constant defers to stdlib's functions */<b>
</b></pre><BR>
<pre><b>typedef struct POOL_ctx_s ZSTD_threadPool;
ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); </b>/* accept NULL pointer */<b>
@@ -2131,7 +2057,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const Z
>0 : `srcSize` is too small, please provide at least result bytes on next attempt.
errorCode, which can be tested using ZSTD_isError().
It fills a ZSTD_FrameHeader structure with important information to correctly decode the frame,
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
As a consequence, check that values remain within valid application range.
+1 -1
View File
@@ -57,7 +57,7 @@ cxx_library(
srcs=glob(['legacy/*.c']),
deps=[':common'],
exported_preprocessor_flags=[
'-DZSTD_LEGACY_SUPPORT=0',
'-DZSTD_LEGACY_SUPPORT=4',
],
)
+14 -34
View File
@@ -63,8 +63,6 @@ CPPFLAGS_DYNLIB += -DZSTD_MULTITHREAD # dynamic library build defaults to multi
LDFLAGS_DYNLIB += -pthread
CPPFLAGS_STATICLIB += # static library build defaults to single-threaded
# pkg-config Libs.private points to LDFLAGS_DYNLIB
PCLIB := $(LDFLAGS_DYNLIB)
ifeq ($(findstring GCC,$(CCVER)),GCC)
decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
@@ -73,15 +71,13 @@ endif
# macOS linker doesn't support -soname, and use different extension
# see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
UNAME_TARGET_SYSTEM ?= $(UNAME)
ifeq ($(UNAME_TARGET_SYSTEM), Darwin)
ifeq ($(UNAME), Darwin)
SHARED_EXT = dylib
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
else
ifeq ($(UNAME_TARGET_SYSTEM), AIX)
ifeq ($(UNAME), AIX)
SONAME_FLAGS =
else
SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
@@ -128,7 +124,7 @@ $(ZSTD_STATICLIB): $(ZSTD_STATICLIB_OBJ)
$(AR) $(ARFLAGS) $@ $^
libzstd.a: $(ZSTD_STATICLIB)
$(CP) $< $@
cp -f $< $@
endif
@@ -166,13 +162,13 @@ $(ZSTD_DYNLIB): $(ZSTD_DYNLIB_OBJ)
$(if $(filter -DZSTD_MULTITHREAD,$(CPPFLAGS)),\
@echo compiling multi-threaded dynamic library $(LIBVER),\
@echo compiling single-threaded dynamic library $(LIBVER))
$(CC) $(FLAGS) $^ $(SONAME_FLAGS) -o $@
$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
@echo creating versioned links
$(LN) -sf $@ libzstd.$(SHARED_EXT_MAJOR)
$(LN) -sf $@ libzstd.$(SHARED_EXT)
ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
ln -sf $@ libzstd.$(SHARED_EXT)
$(LIBZSTD): $(ZSTD_DYNLIB)
$(CP) $< $@
cp -f $< $@
endif # ifndef BUILD_DIR
endif # if windows
@@ -190,15 +186,12 @@ lib : libzstd.a libzstd
%-mt : CPPFLAGS_DYNLIB := -DZSTD_MULTITHREAD
%-mt : CPPFLAGS_STATICLIB := -DZSTD_MULTITHREAD
%-mt : LDFLAGS_DYNLIB := -pthread
%-mt : PCLIB :=
%-mt : PCMTLIB := $(LDFLAGS_DYNLIB)
%-mt : %
@echo multi-threaded build completed
%-nomt : CPPFLAGS_DYNLIB :=
%-nomt : LDFLAGS_DYNLIB :=
%-nomt : CPPFLAGS_STATICLIB :=
%-nomt : PCLIB :=
%-nomt : %
@echo single-threaded build completed
@@ -255,7 +248,7 @@ libzstd-nomt: $(ZSTD_NOMT_FILES)
echo "Error: Found zstdmt in list."; \
exit 1; \
fi
$(CC) $(FLAGS) $^ $(SONAME_FLAGS) -o $@
$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
.PHONY: clean
clean:
@@ -268,7 +261,7 @@ clean:
#-----------------------------------------------------------------------------
# make install is validated only for below listed environments
#-----------------------------------------------------------------------------
ifneq (,$(filter $(INSTALL_OS_LIST),$(UNAME)))
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT CYGWIN_NT))
lib: libzstd.pc
@@ -299,21 +292,13 @@ PCLIBPREFIX := $(if $(findstring $(LIBDIR),$(PCLIBDIR)),,$${exec_prefix})
# to PREFIX, rather than as a resolved value.
PCEXEC_PREFIX := $(if $(HAS_EXPLICIT_EXEC_PREFIX),$(EXEC_PREFIX),$${prefix})
ifneq ($(MT),)
PCLIB :=
PCMTLIB := $(LDFLAGS_DYNLIB)
else
PCLIB := $(LDFLAGS_DYNLIB)
endif
ifneq (,$(filter FreeBSD NetBSD DragonFly,$(UNAME)))
ifneq (,$(filter $(UNAME),FreeBSD NetBSD DragonFly))
PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
else
PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
endif
ifneq (,$(filter SunOS,$(UNAME)))
ifneq (,$(filter $(UNAME),SunOS))
INSTALL ?= ginstall
else
INSTALL ?= install
@@ -323,10 +308,6 @@ INSTALL_PROGRAM ?= $(INSTALL)
INSTALL_DATA ?= $(INSTALL) -m 644
# pkg-config library define.
# For static single-threaded library declare -pthread in Libs.private
# For static multi-threaded library declare -pthread in Libs and Cflags
.PHONY: libzstd.pc
libzstd.pc: libzstd.pc.in
@echo creating pkgconfig
@sed \
@@ -335,8 +316,7 @@ libzstd.pc: libzstd.pc.in
-e 's|@INCLUDEDIR@|$(PCINCPREFIX)$(PCINCDIR)|' \
-e 's|@LIBDIR@|$(PCLIBPREFIX)$(PCLIBDIR)|' \
-e 's|@VERSION@|$(VERSION)|' \
-e 's|@LIBS_MT@|$(PCMTLIB)|' \
-e 's|@LIBS_PRIVATE@|$(PCLIB)|' \
-e 's|@LIBS_PRIVATE@|$(LDFLAGS_DYNLIB)|' \
$< >$@
.PHONY: install
@@ -363,8 +343,8 @@ install-shared:
[ -e $(DESTDIR)$(LIBDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
@echo Installing shared library
$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
$(LN) -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
$(LN) -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
.PHONY: install-includes
install-includes:
+5 -36
View File
@@ -7,30 +7,15 @@ in order to make it easier to select or exclude features.
#### Building
A `Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
including commands variables, staged install, directory variables and standard targets.
- `make` : generates both static and dynamic libraries
- `make install` : install libraries, headers and pkg-config in local system directories
- `make install` : install libraries and headers in target system directories
`libzstd` default scope includes compression, decompression, and dictionary builder.
Note: starting v1.6.0, support for decoding legacy formats is disabled by default.
See _modular build_ below to learn how to enable it.
`libzstd` default scope is pretty large, including compression, decompression, dictionary builder,
and support for decoding legacy formats >= v0.5.0.
The scope can be reduced on demand (see paragraph _modular build_).
#### Multiarch Support
For multiarch systems (like Debian/Ubuntu), libraries should be installed to architecture-specific directories.
When creating packages for such systems, use the `LIBDIR` variable to specify the correct multiarch path:
```bash
# For x86_64 systems on Ubuntu/Debian:
make install PREFIX=/usr LIBDIR=/usr/lib/x86_64-linux-gnu
# For ARM64 systems on Ubuntu/Debian:
make install PREFIX=/usr LIBDIR=/usr/lib/aarch64-linux-gnu
```
This will not only install the files in the correct directories, but also generate the correct paths for `pkg-config`.
#### Multithreading support
@@ -42,16 +27,12 @@ Enabling multithreading requires 2 conditions :
For convenience, we provide a build target to generate multi and single threaded libraries:
- Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`.
Note that the `.pc` generated on calling `make lib-mt` will already include the require Libs and Cflags.
- Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`.
- By default, as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`.
When linking a POSIX program with a multithreaded version of `libzstd`,
note that it's necessary to invoke the `-pthread` flag during link stage.
The `.pc` generated from `make install` or `make install-pc` always assume a single-threaded static library
is compiled. To correctly generate a `.pc` for the multi-threaded static library, set `MT=1` as ENV variable.
Multithreading capabilities are exposed
via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351).
@@ -100,7 +81,7 @@ The file structure is designed to make this selection manually achievable for an
Specifying a number limits versions supported to that version onward.
For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=0` (disabled).
By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`.
Decoding supported legacy format is a transparent capability triggered within decompression functions.
It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`.
Each version does also provide its own set of advanced API.
@@ -164,13 +145,6 @@ The file structure is designed to make this selection manually achievable for an
will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in
the shared library, which is now hidden by default.
- The build macro `STATIC_BMI2` can be set to 1 to force usage of `bmi2` instructions.
It is generally not necessary to set this build macro,
because `STATIC_BMI2` will be automatically set to 1
on detecting the presence of the corresponding instruction set in the compilation target.
It's nonetheless available as an optional manual toggle for better control,
and can also be used to forcefully disable `bmi2` instructions by setting it to 0.
- The build macro `DYNAMIC_BMI2` can be set to 1 or 0 in order to generate binaries
which can detect at runtime the presence of BMI2 instructions, and use them only if present.
These instructions contribute to better performance, notably on the decoder side.
@@ -208,11 +182,6 @@ The file structure is designed to make this selection manually achievable for an
and assembly decoding loops. You may want to use this macro if these loops are
slower on your platform.
- The macro `ZDICT_QSORT` can enforce selection of a specific sorting variant,
which is useful when autodetection fails, for example with older versions of `musl`.
For this scenario, it can be set as `ZDICT_QSORT=ZDICT_QSORT_C90`.
Other selectable suffixes are `_GNU`, `_APPLE`, `_MSVC` and `_C11`.
#### Windows : using MinGW+MSYS to create DLL
DLL can be created using MinGW+MSYS with the `make libzstd` command.
+2 -6
View File
@@ -33,13 +33,9 @@ MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
{
if (customMem.customAlloc) {
/* calloc implemented as malloc+memset */
/* calloc implemented as malloc+memset;
* not as efficient as calloc, but next best guess for custom malloc */
void* const ptr = customMem.customAlloc(customMem.opaque, size);
if (ptr == NULL) {
return NULL;
}
ZSTD_memset(ptr, 0, size);
return ptr;
}
+30 -35
View File
@@ -21,15 +21,15 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
30, 22, 20, 15, 25, 17, 4, 8,
31, 27, 13, 23, 21, 19, 16, 7,
26, 12, 18, 6, 11, 5, 10, 9};
return DeBruijnBytePos[((U32) ((val & (0-val)) * 0x077CB531U)) >> 27];
return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27];
}
}
MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
{
assert(val != 0);
#if defined(_MSC_VER)
# if STATIC_BMI2
# if defined(_MSC_VER)
# if STATIC_BMI2 == 1
return (unsigned)_tzcnt_u32(val);
# else
if (val != 0) {
@@ -37,20 +37,18 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
_BitScanForward(&r, val);
return (unsigned)r;
} else {
__assume(0); /* Should not reach this code path */
/* Should not reach this code path */
__assume(0);
}
# endif
#elif defined(__GNUC__) && (__GNUC__ >= 4)
# elif defined(__GNUC__) && (__GNUC__ >= 4)
return (unsigned)__builtin_ctz(val);
#elif defined(__ICCARM__)
return (unsigned)__builtin_ctz(val);
#else
# else
return ZSTD_countTrailingZeros32_fallback(val);
#endif
# endif
}
MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val)
{
MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
assert(val != 0);
{
static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29,
@@ -69,8 +67,8 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val)
MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
{
assert(val != 0);
#if defined(_MSC_VER)
# if STATIC_BMI2
# if defined(_MSC_VER)
# if STATIC_BMI2 == 1
return (unsigned)_lzcnt_u32(val);
# else
if (val != 0) {
@@ -78,23 +76,22 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
_BitScanReverse(&r, val);
return (unsigned)(31 - r);
} else {
__assume(0); /* Should not reach this code path */
/* Should not reach this code path */
__assume(0);
}
# endif
#elif defined(__GNUC__) && (__GNUC__ >= 4)
# elif defined(__GNUC__) && (__GNUC__ >= 4)
return (unsigned)__builtin_clz(val);
#elif defined(__ICCARM__)
return (unsigned)__builtin_clz(val);
#else
# else
return ZSTD_countLeadingZeros32_fallback(val);
#endif
# endif
}
MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
{
assert(val != 0);
#if defined(_MSC_VER) && defined(_WIN64)
# if STATIC_BMI2
# if defined(_MSC_VER) && defined(_WIN64)
# if STATIC_BMI2 == 1
return (unsigned)_tzcnt_u64(val);
# else
if (val != 0) {
@@ -102,14 +99,13 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
_BitScanForward64(&r, val);
return (unsigned)r;
} else {
__assume(0); /* Should not reach this code path */
/* Should not reach this code path */
__assume(0);
}
# endif
#elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
return (unsigned)__builtin_ctzll(val);
#elif defined(__ICCARM__)
return (unsigned)__builtin_ctzll(val);
#else
# else
{
U32 mostSignificantWord = (U32)(val >> 32);
U32 leastSignificantWord = (U32)val;
@@ -119,14 +115,14 @@ MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
return ZSTD_countTrailingZeros32(leastSignificantWord);
}
}
#endif
# endif
}
MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
{
assert(val != 0);
#if defined(_MSC_VER) && defined(_WIN64)
# if STATIC_BMI2
# if defined(_MSC_VER) && defined(_WIN64)
# if STATIC_BMI2 == 1
return (unsigned)_lzcnt_u64(val);
# else
if (val != 0) {
@@ -134,14 +130,13 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
_BitScanReverse64(&r, val);
return (unsigned)(63 - r);
} else {
__assume(0); /* Should not reach this code path */
/* Should not reach this code path */
__assume(0);
}
# endif
#elif defined(__GNUC__) && (__GNUC__ >= 4)
# elif defined(__GNUC__) && (__GNUC__ >= 4)
return (unsigned)(__builtin_clzll(val));
#elif defined(__ICCARM__)
return (unsigned)(__builtin_clzll(val));
#else
# else
{
U32 mostSignificantWord = (U32)(val >> 32);
U32 leastSignificantWord = (U32)val;
@@ -151,7 +146,7 @@ MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
return ZSTD_countLeadingZeros32(mostSignificantWord);
}
}
#endif
# endif
}
MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val)
+29 -26
View File
@@ -14,6 +14,9 @@
#ifndef BITSTREAM_H_MODULE
#define BITSTREAM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*
* This API consists of small unitary functions, which must be inlined for best performance.
* Since link-time-optimization is not available for all compilers,
@@ -29,6 +32,7 @@
#include "error_private.h" /* error codes and messages */
#include "bits.h" /* ZSTD_highbit32 */
/*=========================================
* Target specific
=========================================*/
@@ -48,13 +52,12 @@
/*-******************************************
* bitStream encoding API (write forward)
********************************************/
typedef size_t BitContainerType;
/* bitStream can mix input from multiple sources.
* A critical property of these streams is that they encode and decode in **reverse** direction.
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
*/
typedef struct {
BitContainerType bitContainer;
size_t bitContainer;
unsigned bitPos;
char* startPtr;
char* ptr;
@@ -62,7 +65,7 @@ typedef struct {
} BIT_CStream_t;
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
@@ -71,7 +74,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
*
* bits are first added to a local register.
* Local register is BitContainerType, 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
* Writing data into memory is an explicit operation, performed by the flushBits function.
* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
* After a flushBits, a maximum of 7 bits might still be stored into local register.
@@ -87,6 +90,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
/*-********************************************
* bitStream decoding API (read backward)
**********************************************/
typedef size_t BitContainerType;
typedef struct {
BitContainerType bitContainer;
unsigned bitsConsumed;
@@ -102,8 +106,8 @@ typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */
} BIT_DStream_status; /* result of BIT_reloadDStream() */
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
@@ -121,7 +125,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
/*-****************************************
* unsafe API
******************************************/
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, BitContainerType value, unsigned nbBits);
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
@@ -159,15 +163,10 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
return 0;
}
FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitContainer, U32 const nbBits)
FORCE_INLINE_TEMPLATE size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
{
#if STATIC_BMI2 && !defined(ZSTD_NO_INTRINSICS)
# if (defined(__x86_64__) || defined(_M_X64)) && !defined(__ILP32__)
#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS)
return _bzhi_u64(bitContainer, nbBits);
# else
DEBUG_STATIC_ASSERT(sizeof(bitContainer) == sizeof(U32));
return _bzhi_u32(bitContainer, nbBits);
# endif
#else
assert(nbBits < BIT_MASK_SIZE);
return bitContainer & BIT_mask[nbBits];
@@ -178,7 +177,7 @@ FORCE_INLINE_TEMPLATE BitContainerType BIT_getLowerBits(BitContainerType bitCont
* can add up to 31 bits into `bitC`.
* Note : does not check for register overflow ! */
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
BitContainerType value, unsigned nbBits)
size_t value, unsigned nbBits)
{
DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
assert(nbBits < BIT_MASK_SIZE);
@@ -191,7 +190,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
* works only if `value` is _clean_,
* meaning all high bits above nbBits are 0 */
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
BitContainerType value, unsigned nbBits)
size_t value, unsigned nbBits)
{
assert((value>>nbBits) == 0);
assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
@@ -238,7 +237,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
BIT_addBitsFast(bitC, 1, 1); /* endMark */
BIT_flushBits(bitC);
if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
return (size_t)(bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
}
@@ -299,12 +298,12 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
return srcSize;
}
FORCE_INLINE_TEMPLATE BitContainerType BIT_getUpperBits(BitContainerType bitContainer, U32 const start)
FORCE_INLINE_TEMPLATE size_t BIT_getUpperBits(BitContainerType bitContainer, U32 const start)
{
return bitContainer >> start;
}
FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits)
FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits)
{
U32 const regMask = sizeof(bitContainer)*8 - 1;
/* if start > regMask, bitstream is corrupted, and result is undefined */
@@ -314,7 +313,7 @@ FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitCon
* such cpus old (pre-Haswell, 2013) and their performance is not of that
* importance.
*/
#if defined(__x86_64__) || defined(_M_X64)
#if defined(__x86_64__) || defined(_M_X86)
return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
#else
return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
@@ -327,7 +326,7 @@ FORCE_INLINE_TEMPLATE BitContainerType BIT_getMiddleBits(BitContainerType bitCon
* On 32-bits, maxNbBits==24.
* On 64-bits, maxNbBits==56.
* @return : value extracted */
FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
FORCE_INLINE_TEMPLATE size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
{
/* arbitrate between double-shift and shift+mask */
#if 1
@@ -343,7 +342,7 @@ FORCE_INLINE_TEMPLATE BitContainerType BIT_lookBits(const BIT_DStream_t* bitD,
/*! BIT_lookBitsFast() :
* unsafe version; only works if nbBits >= 1 */
MEM_STATIC BitContainerType BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
{
U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
assert(nbBits >= 1);
@@ -359,18 +358,18 @@ FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
* Read (consume) next n bits from local register and update.
* Pay attention to not read more than nbBits contained into local register.
* @return : extracted value. */
FORCE_INLINE_TEMPLATE BitContainerType BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
FORCE_INLINE_TEMPLATE size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
{
BitContainerType const value = BIT_lookBits(bitD, nbBits);
size_t const value = BIT_lookBits(bitD, nbBits);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_readBitsFast() :
* unsafe version; only works if nbBits >= 1 */
MEM_STATIC BitContainerType BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
{
BitContainerType const value = BIT_lookBitsFast(bitD, nbBits);
size_t const value = BIT_lookBitsFast(bitD, nbBits);
assert(nbBits >= 1);
BIT_skipBits(bitD, nbBits);
return value;
@@ -451,4 +450,8 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
}
#if defined (__cplusplus)
}
#endif
#endif /* BITSTREAM_H_MODULE */
+28 -60
View File
@@ -27,7 +27,7 @@
# define INLINE_KEYWORD
#endif
#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
#if defined(__GNUC__) || defined(__ICCARM__)
# define FORCE_INLINE_ATTR __attribute__((always_inline))
#elif defined(_MSC_VER)
# define FORCE_INLINE_ATTR __forceinline
@@ -54,7 +54,7 @@
#endif
/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
#if defined(__GNUC__)
# define UNUSED_ATTR __attribute__((unused))
#else
# define UNUSED_ATTR
@@ -95,8 +95,6 @@
#ifndef MEM_STATIC /* already defined in Linux Kernel mem.h */
#if defined(__GNUC__)
# define MEM_STATIC static __inline UNUSED_ATTR
#elif defined(__IAR_SYSTEMS_ICC__)
# define MEM_STATIC static inline UNUSED_ATTR
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define MEM_STATIC static inline
#elif defined(_MSC_VER)
@@ -110,7 +108,7 @@
#ifdef _MSC_VER
# define FORCE_NOINLINE static __declspec(noinline)
#else
# if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
# if defined(__GNUC__) || defined(__ICCARM__)
# define FORCE_NOINLINE static __attribute__((__noinline__))
# else
# define FORCE_NOINLINE static
@@ -119,7 +117,7 @@
/* target attribute */
#if defined(__GNUC__) || defined(__IAR_SYSTEMS_ICC__)
#if defined(__GNUC__) || defined(__ICCARM__)
# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
#else
# define TARGET_ATTRIBUTE(target)
@@ -207,44 +205,35 @@
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
#endif
/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/
#ifndef STATIC_BMI2
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))
# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2
# define STATIC_BMI2 1
# endif
# elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__)
# define STATIC_BMI2 1
# endif
#endif
#ifndef STATIC_BMI2
#define STATIC_BMI2 0
#endif
/* compile time determination of SIMD support */
#if !defined(ZSTD_NO_INTRINSICS)
# if defined(__AVX2__)
# define ZSTD_ARCH_X86_AVX2
# endif
# if defined(__SSE2__) || defined(_M_X64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
# define ZSTD_ARCH_X86_SSE2
# endif
# if defined(__ARM_NEON) || defined(_M_ARM64)
# define ZSTD_ARCH_ARM_NEON
# endif
# if defined(__ARM_FEATURE_SVE)
# define ZSTD_ARCH_ARM_SVE
# endif
# if defined(__ARM_FEATURE_SVE2)
# define ZSTD_ARCH_ARM_SVE2
# endif
# if defined(__riscv) && defined(__riscv_vector)
# if ((defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 14) || \
(defined(__clang__) && __clang_major__ >= 19))
#define ZSTD_ARCH_RISCV_RVV
# endif
#endif
#
# if defined(ZSTD_ARCH_X86_AVX2)
# include <immintrin.h>
# endif
# if defined(ZSTD_ARCH_X86_SSE2)
# include <emmintrin.h>
# elif defined(ZSTD_ARCH_ARM_NEON)
# include <arm_neon.h>
# endif
# if defined(ZSTD_ARCH_ARM_SVE) || defined(ZSTD_ARCH_ARM_SVE2)
# include <arm_sve.h>
# endif
# if defined(ZSTD_ARCH_RISCV_RVV)
# include <riscv_vector.h>
# endif
#endif
/* C-language Attributes are added in C23. */
@@ -284,15 +273,9 @@
#endif
/*-**************************************************************
* Alignment
* Alignment check
*****************************************************************/
/* @return 1 if @u is a 2^n value, 0 otherwise
* useful to check a value is valid for alignment restrictions */
MEM_STATIC int ZSTD_isPower2(size_t u) {
return (u & (u-1)) == 0;
}
/* this test was initially positioned in mem.h,
* but this file is removed (or replaced) for linux kernel
* so it's now hosted in compiler.h,
@@ -318,21 +301,6 @@ MEM_STATIC int ZSTD_isPower2(size_t u) {
# endif
#endif /* ZSTD_ALIGNOF */
#ifndef ZSTD_ALIGNED
/* C90-compatible alignment macro (GCC/Clang). Adjust for other compilers if needed. */
# if defined(__GNUC__) || defined(__clang__)
# define ZSTD_ALIGNED(a) __attribute__((aligned(a)))
# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
# define ZSTD_ALIGNED(a) _Alignas(a)
#elif defined(_MSC_VER)
# define ZSTD_ALIGNED(n) __declspec(align(n))
# else
/* this compiler will require its own alignment instruction */
# define ZSTD_ALIGNED(...)
# endif
#endif /* ZSTD_ALIGNED */
/*-**************************************************************
* Sanitizer
*****************************************************************/
@@ -356,7 +324,7 @@ MEM_STATIC int ZSTD_isPower2(size_t u) {
#endif
/**
* Helper function to perform a wrapped pointer difference without triggering
* Helper function to perform a wrapped pointer difference without trigging
* UBSAN.
*
* @returns lhs - rhs with wrapping
@@ -375,9 +343,9 @@ ptrdiff_t ZSTD_wrappedPtrDiff(unsigned char const* lhs, unsigned char const* rhs
*/
MEM_STATIC
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
const void* ZSTD_wrappedPtrAdd(const void* ptr, ptrdiff_t add)
unsigned char const* ZSTD_wrappedPtrAdd(unsigned char const* ptr, ptrdiff_t add)
{
return (const char*)ptr + add;
return ptr + add;
}
/**
@@ -388,9 +356,9 @@ const void* ZSTD_wrappedPtrAdd(const void* ptr, ptrdiff_t add)
*/
MEM_STATIC
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
const void* ZSTD_wrappedPtrSub(const void* ptr, ptrdiff_t sub)
unsigned char const* ZSTD_wrappedPtrSub(unsigned char const* ptr, ptrdiff_t sub)
{
return (const char*)ptr - sub;
return ptr - sub;
}
/**
@@ -400,9 +368,9 @@ const void* ZSTD_wrappedPtrSub(const void* ptr, ptrdiff_t sub)
* @returns `ptr + add` except it defines `NULL + 0 == NULL`.
*/
MEM_STATIC
void* ZSTD_maybeNullPtrAdd(void* ptr, ptrdiff_t add)
unsigned char* ZSTD_maybeNullPtrAdd(unsigned char* ptr, ptrdiff_t add)
{
return add > 0 ? (char*)ptr + add : ptr;
return add > 0 ? ptr + add : ptr;
}
/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an
+1 -1
View File
@@ -35,7 +35,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
U32 f7b = 0;
U32 f7c = 0;
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
#if !defined(_M_X64) || !defined(__clang__) || __clang_major__ >= 16
#if !defined(__clang__)
int reg[4];
__cpuid((int*)reg, 0);
{
+9
View File
@@ -32,6 +32,10 @@
#ifndef DEBUG_H_12987983217
#define DEBUG_H_12987983217
#if defined (__cplusplus)
extern "C" {
#endif
/* static assert is triggered at compile time, leaving no runtime artefact.
* static assert only works with compile-time constants.
@@ -104,4 +108,9 @@ extern int g_debuglevel; /* the variable is only declared,
# define DEBUGLOG(l, ...) do { } while (0) /* disabled */
#endif
#if defined (__cplusplus)
}
#endif
#endif /* DEBUG_H_12987983217 */
-1
View File
@@ -40,7 +40,6 @@ const char* ERR_getErrorString(ERR_enum code)
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(cannotProduce_uncompressedBlock): return "This mode cannot generate an uncompressed block";
case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
+10
View File
@@ -13,6 +13,11 @@
#ifndef ERROR_H_MODULE
#define ERROR_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* Dependencies
******************************************/
@@ -21,6 +26,7 @@
#include "debug.h"
#include "zstd_deps.h" /* size_t */
/* ****************************************
* Compiler-specific
******************************************/
@@ -155,4 +161,8 @@ void _force_has_format_string(const char *format, ...) {
} \
} while(0)
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_H_MODULE */
+17 -2
View File
@@ -11,6 +11,11 @@
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
****************************************************************** */
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef FSE_H
#define FSE_H
@@ -20,6 +25,7 @@
******************************************/
#include "zstd_deps.h" /* size_t, ptrdiff_t */
/*-*****************************************
* FSE_PUBLIC_API : control library symbols visibility
******************************************/
@@ -226,8 +232,11 @@ If there is an error, the function will return an error code, which can be teste
#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
#define FSE_H_FSE_STATIC_LINKING_ONLY
/* *** Dependency *** */
#include "bitstream.h"
/* *****************************************
* Static allocation
*******************************************/
@@ -456,13 +465,13 @@ MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, un
FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* const stateTable = (const U16*)(statePtr->stateTable);
U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
BIT_addBits(bitC, (BitContainerType)statePtr->value, nbBitsOut);
BIT_addBits(bitC, (size_t)statePtr->value, nbBitsOut);
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
{
BIT_addBits(bitC, (BitContainerType)statePtr->value, statePtr->stateLog);
BIT_addBits(bitC, (size_t)statePtr->value, statePtr->stateLog);
BIT_flushBits(bitC);
}
@@ -622,4 +631,10 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3)
#endif /* FSE_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
-2
View File
@@ -190,8 +190,6 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
FSE_initDState(&state1, &bitD, dt);
FSE_initDState(&state2, &bitD, dt);
RETURN_ERROR_IF(BIT_reloadDStream(&bitD)==BIT_DStream_overflow, corruption_detected, "");
#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
/* 4 symbols per loop */
+9
View File
@@ -12,6 +12,10 @@
* You may select, at your option, one of the above-listed licenses.
****************************************************************** */
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef HUF_H_298734234
#define HUF_H_298734234
@@ -21,6 +25,7 @@
#define FSE_STATIC_LINKING_ONLY
#include "fse.h"
/* *** Tool functions *** */
#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */
size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
@@ -275,3 +280,7 @@ size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize
#endif
#endif /* HUF_H_298734234 */
#if defined (__cplusplus)
}
#endif
+11 -7
View File
@@ -11,6 +11,10 @@
#ifndef MEM_H_MODULE
#define MEM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
@@ -26,8 +30,6 @@
#if defined(_MSC_VER) /* Visual Studio */
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#elif defined(__ICCARM__)
# include <intrinsics.h>
#endif
/*-**************************************************************
@@ -72,6 +74,7 @@
typedef signed long long S64;
#endif
/*-**************************************************************
* Memory I/O API
*****************************************************************/
@@ -147,12 +150,10 @@ MEM_STATIC unsigned MEM_isLittleEndian(void)
return 1;
#elif defined(__clang__) && __BIG_ENDIAN__
return 0;
#elif defined(_MSC_VER) && (_M_X64 || _M_IX86)
#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86)
return 1;
#elif defined(__DMC__) && defined(_M_IX86)
return 1;
#elif defined(__IAR_SYSTEMS_ICC__) && __LITTLE_ENDIAN__
return 1;
#else
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
return one.c[0];
@@ -245,8 +246,6 @@ MEM_STATIC U32 MEM_swap32(U32 in)
#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
|| (defined(__clang__) && __has_builtin(__builtin_bswap32))
return __builtin_bswap32(in);
#elif defined(__ICCARM__)
return __REV(in);
#else
return MEM_swap32_fallback(in);
#endif
@@ -419,4 +418,9 @@ MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
/* code only tested on 32 and 64 bits systems */
MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
#if defined (__cplusplus)
}
#endif
#endif /* MEM_H_MODULE */
+16 -13
View File
@@ -142,17 +142,19 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
/* Allocate space for the thread handles */
ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
ctx->threadCapacity = 0;
ctx->threadLimit = numThreads;
ctx->customMem = customMem;
/* Check for errors */
if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
/* Initialize the threads */
while (ctx->threadCapacity < numThreads) {
if (ZSTD_pthread_create(&ctx->threads[ctx->threadCapacity++], NULL, &POOL_thread, ctx)) {
--ctx->threadCapacity;
{ size_t i;
for (i = 0; i < numThreads; ++i) {
if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
ctx->threadCapacity = i;
POOL_free(ctx);
return NULL;
}
} }
ctx->threadCapacity = numThreads;
ctx->threadLimit = numThreads;
}
return ctx;
}
@@ -218,22 +220,23 @@ static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
return 0;
}
/* numThreads > threadCapacity */
ctx->threadLimit = numThreads;
{ ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem);
if (!threadPool) return 1;
/* extend existing thread pool */
/* replace existing thread pool */
ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(ZSTD_pthread_t));
ZSTD_customFree(ctx->threads, ctx->customMem);
ctx->threads = threadPool;
/* Initialize additional threads */
while (ctx->threadCapacity < numThreads) {
if (ZSTD_pthread_create(&threadPool[ctx->threadCapacity++], NULL, &POOL_thread, ctx)) {
--ctx->threadCapacity;
{ size_t threadId;
for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
ctx->threadCapacity = threadId;
return 1;
}
}
}
} }
} }
/* successfully expanded */
ctx->threadCapacity = numThreads;
ctx->threadLimit = numThreads;
return 0;
}
+9
View File
@@ -11,6 +11,10 @@
#ifndef POOL_H
#define POOL_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "zstd_deps.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */
@@ -78,4 +82,9 @@ void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
*/
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
#if defined (__cplusplus)
}
#endif
#endif
+9 -41
View File
@@ -74,39 +74,26 @@
# define ZSTD_HIDE_ASM_FUNCTION(func)
#endif
/* Compile time determination of BMI2 support */
#ifndef STATIC_BMI2
# if defined(__BMI2__)
# define STATIC_BMI2 1
# elif defined(_MSC_VER) && defined(__AVX2__)
# define STATIC_BMI2 1 /* MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 */
# endif
#endif
#ifndef STATIC_BMI2
# define STATIC_BMI2 0
#endif
/* Enable runtime BMI2 dispatch based on the CPU.
* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
*/
#ifndef DYNAMIC_BMI2
# if ((defined(__clang__) && __has_attribute(__target__)) \
#if ((defined(__clang__) && __has_attribute(__target__)) \
|| (defined(__GNUC__) \
&& (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
&& (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)) \
&& (defined(__x86_64__) || defined(_M_X64)) \
&& !defined(__BMI2__)
# define DYNAMIC_BMI2 1
# else
# define DYNAMIC_BMI2 0
# endif
# define DYNAMIC_BMI2 1
#else
# define DYNAMIC_BMI2 0
#endif
#endif
/**
* Only enable assembly for GNU C compatible compilers,
* Only enable assembly for GNUC compatible compilers,
* because other platforms may not support GAS assembly syntax.
*
* Only enable assembly for Linux / MacOS / Win32, other platforms may
* Only enable assembly for Linux / MacOS, other platforms may
* work, but they haven't been tested. This could likely be
* extended to BSD systems.
*
@@ -114,7 +101,7 @@
* 100% of code to be instrumented to work.
*/
#if defined(__GNUC__)
# if defined(__linux__) || defined(__linux) || defined(__APPLE__) || defined(_WIN32)
# if defined(__linux__) || defined(__linux) || defined(__APPLE__)
# if ZSTD_MEMORY_SANITIZER
# define ZSTD_ASM_SUPPORTED 0
# elif ZSTD_DATAFLOW_SANITIZER
@@ -168,23 +155,4 @@
# define ZSTD_CET_ENDBRANCH
#endif
/**
* ZSTD_IS_DETERMINISTIC_BUILD must be set to 0 if any compilation macro is
* active that impacts the compressed output.
*
* NOTE: ZSTD_MULTITHREAD is allowed to be set or unset.
*/
#if defined(ZSTD_CLEVEL_DEFAULT) \
|| defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \
|| defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \
|| defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
|| defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \
|| defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \
|| defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \
|| defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR)
# define ZSTD_IS_DETERMINISTIC_BUILD 0
#else
# define ZSTD_IS_DETERMINISTIC_BUILD 1
#endif
#endif /* ZSTD_PORTABILITY_MACROS_H */
+2 -16
View File
@@ -143,14 +143,7 @@ int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t con
*mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t));
if (!*mutex)
return 1;
{
int const ret = pthread_mutex_init(*mutex, attr);
if (ret != 0) {
ZSTD_free(*mutex);
*mutex = NULL;
}
return ret;
}
return pthread_mutex_init(*mutex, attr);
}
int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
@@ -171,14 +164,7 @@ int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const*
*cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t));
if (!*cond)
return 1;
{
int const ret = pthread_cond_init(*cond, attr);
if (ret != 0) {
ZSTD_free(*cond);
*cond = NULL;
}
return ret;
}
return pthread_cond_init(*cond, attr);
}
int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
+8
View File
@@ -16,6 +16,10 @@
#include "debug.h"
#if defined (__cplusplus)
extern "C" {
#endif
#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
/**
@@ -68,6 +72,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread);
* add here more wrappers as required
*/
#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */
/* === POSIX Systems === */
# include <pthread.h>
@@ -138,5 +143,8 @@ typedef int ZSTD_pthread_cond_t;
#endif /* ZSTD_MULTITHREAD */
#if defined (__cplusplus)
}
#endif
#endif /* THREADING_H_938743 */
+19 -93
View File
@@ -227,6 +227,10 @@
* xxHash prototypes and implementation
*/
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************
* INLINE mode
******************************/
@@ -533,9 +537,6 @@
/*! @brief Version number, encoded as two digits each */
#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @brief Obtains the xxHash version.
*
@@ -546,9 +547,6 @@ extern "C" {
*/
XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void);
#if defined (__cplusplus)
}
#endif
/* ****************************
* Common basic types
@@ -595,10 +593,6 @@ typedef uint32_t XXH32_hash_t;
# endif
#endif
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @}
*
@@ -827,9 +821,6 @@ XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canoni
#endif
/*! @endcond */
#if defined (__cplusplus)
} /* end of extern "C" */
#endif
/*!
* @}
@@ -868,9 +859,6 @@ typedef uint64_t XXH64_hash_t;
# endif
#endif
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @}
*
@@ -1574,11 +1562,6 @@ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE con
#endif /* !XXH_NO_XXH3 */
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif /* XXH_NO_LONG_LONG */
/*!
@@ -1765,10 +1748,6 @@ struct XXH3_state_s {
} while(0)
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @brief Calculates the 128-bit hash of @p data using XXH3.
*
@@ -1984,13 +1963,8 @@ XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
XXH64_hash_t seed64);
#endif /* !XXH_NO_STREAM */
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif /* !XXH_NO_XXH3 */
#endif /* XXH_NO_LONG_LONG */
#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
# define XXH_IMPLEMENTATION
#endif
@@ -2289,12 +2263,10 @@ XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
* @{
*/
/* *************************************
* Includes & Memory related functions
***************************************/
#include <string.h> /* memcmp, memcpy */
#include <limits.h> /* ULLONG_MAX */
#if defined(XXH_NO_STREAM)
/* nothing */
#elif defined(XXH_NO_STDLIB)
@@ -2308,17 +2280,9 @@ XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
* without access to dynamic allocation.
*/
#if defined (__cplusplus)
extern "C" {
#endif
static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; }
static void XXH_free(void* p) { (void)p; }
#if defined (__cplusplus)
} /* extern "C" */
#endif
#else
/*
@@ -2327,9 +2291,6 @@ static void XXH_free(void* p) { (void)p; }
*/
#include <stdlib.h>
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @internal
* @brief Modify this function to use a different routine than malloc().
@@ -2342,15 +2303,10 @@ static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); }
*/
static void XXH_free(void* p) { free(p); }
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif /* XXH_NO_STDLIB */
#if defined (__cplusplus)
extern "C" {
#endif
#include <string.h>
/*!
* @internal
* @brief Modify this function to use a different routine than memcpy().
@@ -2360,9 +2316,8 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size)
return memcpy(dest,src,size);
}
#if defined (__cplusplus)
} /* extern "C" */
#endif
#include <limits.h> /* ULLONG_MAX */
/* *************************************
* Compiler Specific Options
@@ -2497,10 +2452,6 @@ typedef XXH32_hash_t xxh_u32;
# define U32 xxh_u32
#endif
#if defined (__cplusplus)
extern "C" {
#endif
/* *** Memory access *** */
/*!
@@ -3657,10 +3608,6 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_can
return XXH_readBE64(src);
}
#if defined (__cplusplus)
}
#endif
#ifndef XXH_NO_XXH3
/* *********************************************************************
@@ -3892,7 +3839,7 @@ enum XXH_VECTOR_TYPE /* fake enum */ {
# define XXH_VECTOR XXH_AVX512
# elif defined(__AVX2__)
# define XXH_VECTOR XXH_AVX2
# elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
# define XXH_VECTOR XXH_SSE2
# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
|| (defined(__s390x__) && defined(__VEC__)) \
@@ -3981,10 +3928,6 @@ enum XXH_VECTOR_TYPE /* fake enum */ {
# pragma GCC optimize("-O2")
#endif
#if defined (__cplusplus)
extern "C" {
#endif
#if XXH_VECTOR == XXH_NEON
/*
@@ -4107,10 +4050,6 @@ XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
# endif
#endif /* XXH_VECTOR == XXH_NEON */
#if defined (__cplusplus)
} /* extern "C" */
#endif
/*
* VSX and Z Vector helpers.
*
@@ -4172,9 +4111,6 @@ typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING;
# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
# define XXH_vec_revb vec_revb
# else
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* A polyfill for POWER9's vec_revb().
*/
@@ -4184,15 +4120,9 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
return vec_perm(val, val, vByteSwap);
}
#if defined (__cplusplus)
} /* extern "C" */
#endif
# endif
# endif /* XXH_VSX_BE */
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* Performs an unaligned vector load and byte swaps it on big endian.
*/
@@ -4237,11 +4167,6 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
return result;
}
# endif /* XXH_vec_mulo, XXH_vec_mule */
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif /* XXH_VECTOR == XXH_VSX */
#if XXH_VECTOR == XXH_SVE
@@ -4275,9 +4200,7 @@ do { \
# endif
#endif /* XXH_NO_PREFETCH */
#if defined (__cplusplus)
extern "C" {
#endif
/* ==========================================
* XXH3 default settings
* ========================================== */
@@ -6954,6 +6877,8 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_
#endif /* !XXH_NO_STREAM */
/* 128-bit utility functions */
#include <string.h> /* memcmp, memcpy */
/* return : 1 is equal, 0 if different */
/*! @ingroup XXH3_family */
XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
@@ -7080,15 +7005,16 @@ XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed)
# pragma GCC pop_options
#endif
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif /* XXH_NO_LONG_LONG */
#endif /* XXH_NO_XXH3 */
/*!
* @}
*/
#endif /* XXH_IMPLEMENTATION */
#if defined (__cplusplus)
} /* extern "C" */
#endif
-9
View File
@@ -46,12 +46,3 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
/*! ZSTD_getErrorString() :
* provides error code string from enum */
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
int ZSTD_isDeterministicBuild(void)
{
#if ZSTD_IS_DETERMINISTIC_BUILD
return 1;
#else
return 0;
#endif
}
-12
View File
@@ -24,18 +24,6 @@
#ifndef ZSTD_DEPS_COMMON
#define ZSTD_DEPS_COMMON
/* Even though we use qsort_r only for the dictionary builder, the macro
* _GNU_SOURCE has to be declared *before* the inclusion of any standard
* header and the script 'combine.sh' combines the whole zstd source code
* in a single file.
*/
#if defined(__linux) || defined(__linux__) || defined(linux) || defined(__gnu_linux__) || \
defined(__CYGWIN__) || defined(__MSYS__)
#if !defined(_GNU_SOURCE) && !defined(__ANDROID__) /* NDK doesn't ship qsort_r(). */
#define _GNU_SOURCE
#endif
#endif
#include <limits.h>
#include <stddef.h>
#include <string.h>
+71 -5
View File
@@ -39,6 +39,10 @@
# define ZSTD_TRACE 0
#endif
#if defined (__cplusplus)
extern "C" {
#endif
/* ---- static assert (debug) --- */
#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
#define ZSTD_isError ERR_isError /* for inlining */
@@ -91,7 +95,7 @@ typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */
#define MIN_LITERALS_FOR_4_STREAMS 6
typedef enum { set_basic, set_rle, set_compressed, set_repeat } SymbolEncodingType_e;
typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
#define LONGNBSEQ 0x7F00
@@ -168,7 +172,7 @@ static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
* Shared functions to include for inlining
*********************************************/
static void ZSTD_copy8(void* dst, const void* src) {
#if defined(ZSTD_ARCH_ARM_NEON) && !defined(__aarch64__)
#if defined(ZSTD_ARCH_ARM_NEON)
vst1_u8((uint8_t*)dst, vld1_u8((const uint8_t*)src));
#else
ZSTD_memcpy(dst, src, 8);
@@ -185,8 +189,6 @@ static void ZSTD_copy16(void* dst, const void* src) {
vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src));
#elif defined(ZSTD_ARCH_X86_SSE2)
_mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src));
#elif defined(ZSTD_ARCH_RISCV_RVV)
__riscv_vse8_v_u8m1((uint8_t*)dst, __riscv_vle8_v_u8m1((const uint8_t*)src, 16), 16);
#elif defined(__clang__)
ZSTD_memmove(dst, src, 16);
#else
@@ -215,7 +217,7 @@ typedef enum {
* The src buffer must be before the dst buffer.
*/
MEM_STATIC FORCE_INLINE_ATTR
void ZSTD_wildcopy(void* dst, const void* src, size_t length, ZSTD_overlap_e const ovtype)
void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype)
{
ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
const BYTE* ip = (const BYTE*)src;
@@ -276,6 +278,62 @@ typedef enum {
/*-*******************************************
* Private declarations
*********************************************/
typedef struct seqDef_s {
U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
U16 litLength;
U16 mlBase; /* mlBase == matchLength - MINMATCH */
} seqDef;
/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
typedef enum {
ZSTD_llt_none = 0, /* no longLengthType */
ZSTD_llt_literalLength = 1, /* represents a long literal */
ZSTD_llt_matchLength = 2 /* represents a long match */
} ZSTD_longLengthType_e;
typedef struct {
seqDef* sequencesStart;
seqDef* sequences; /* ptr to end of sequences */
BYTE* litStart;
BYTE* lit; /* ptr to end of literals */
BYTE* llCode;
BYTE* mlCode;
BYTE* ofCode;
size_t maxNbSeq;
size_t maxNbLit;
/* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
* in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
* the existing value of the litLength or matchLength by 0x10000.
*/
ZSTD_longLengthType_e longLengthType;
U32 longLengthPos; /* Index of the sequence to apply long length modification to */
} seqStore_t;
typedef struct {
U32 litLength;
U32 matchLength;
} ZSTD_sequenceLength;
/**
* Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences
* indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
*/
MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq)
{
ZSTD_sequenceLength seqLen;
seqLen.litLength = seq->litLength;
seqLen.matchLength = seq->mlBase + MINMATCH;
if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
if (seqStore->longLengthType == ZSTD_llt_literalLength) {
seqLen.litLength += 0x10000;
}
if (seqStore->longLengthType == ZSTD_llt_matchLength) {
seqLen.matchLength += 0x10000;
}
}
return seqLen;
}
/**
* Contains the compressed frame size and an upper-bound for the decompressed frame size.
@@ -289,6 +347,10 @@ typedef struct {
unsigned long long decompressedBound;
} ZSTD_frameSizeInfo; /* decompress & legacy */
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
/* ZSTD_invalidateRepCodes() :
* ensures next compression will not use repcodes from previous block.
* Note : only works with regular variant;
@@ -323,4 +385,8 @@ MEM_STATIC int ZSTD_cpuSupportsBmi2(void)
return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid);
}
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_CCOMMON_H_MODULE */
+12 -5
View File
@@ -11,20 +11,23 @@
#ifndef ZSTD_TRACE_H
#define ZSTD_TRACE_H
#if defined (__cplusplus)
extern "C" {
#endif
#include <stddef.h>
/* weak symbol support
* For now, enable conservatively:
* - Only GNUC
* - Only ELF
* - Only x86-64, i386, aarch64 and risc-v.
* - Only x86-64, i386 and aarch64
* Also, explicitly disable on platforms known not to work so they aren't
* forgotten in the future.
*/
#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \
defined(__GNUC__) && defined(__ELF__) && \
(defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \
defined(_M_IX86) || defined(__aarch64__) || defined(__riscv)) && \
(defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(__aarch64__)) && \
!defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
!defined(__CYGWIN__) && !defined(_AIX)
# define ZSTD_HAVE_WEAK_SYMBOLS 1
@@ -61,7 +64,7 @@ typedef struct {
/**
* Non-zero if streaming (de)compression is used.
*/
int streaming;
unsigned streaming;
/**
* The dictionary ID.
*/
@@ -70,7 +73,7 @@ typedef struct {
* Is the dictionary cold?
* Only set on decompression.
*/
int dictionaryIsCold;
unsigned dictionaryIsCold;
/**
* The dictionary size or zero if no dictionary.
*/
@@ -153,4 +156,8 @@ ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
#endif /* ZSTD_TRACE */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_TRACE_H */
+3 -268
View File
@@ -19,12 +19,6 @@
#include "../common/error_private.h" /* ERROR */
#include "hist.h"
#if defined(ZSTD_ARCH_ARM_SVE2)
#define HIST_FAST_THRESHOLD 500
#else
#define HIST_FAST_THRESHOLD 1500
#endif
/* --- Error management --- */
unsigned HIST_isError(size_t code) { return ERR_isError(code); }
@@ -32,16 +26,6 @@ unsigned HIST_isError(size_t code) { return ERR_isError(code); }
/*-**************************************************************
* Histogram functions
****************************************************************/
void HIST_add(unsigned* count, const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
const BYTE* const end = ip + srcSize;
while (ip<end) {
count[*ip++]++;
}
}
unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize)
{
@@ -71,244 +55,6 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e;
#if defined(ZSTD_ARCH_ARM_SVE2)
FORCE_INLINE_TEMPLATE size_t min_size(size_t a, size_t b) { return a < b ? a : b; }
static
svuint16_t HIST_count_6_sve2(const BYTE* const src, size_t size, U32* const dst,
const svuint8_t c0, const svuint8_t c1,
const svuint8_t c2, const svuint8_t c3,
const svuint8_t c4, const svuint8_t c5,
const svuint16_t histmax, size_t maxCount)
{
const svbool_t vl128 = svptrue_pat_b8(SV_VL16);
svuint16_t hh0 = svdup_n_u16(0);
svuint16_t hh1 = svdup_n_u16(0);
svuint16_t hh2 = svdup_n_u16(0);
svuint16_t hh3 = svdup_n_u16(0);
svuint16_t hh4 = svdup_n_u16(0);
svuint16_t hh5 = svdup_n_u16(0);
svuint16_t hh6 = svdup_n_u16(0);
svuint16_t hh7 = svdup_n_u16(0);
svuint16_t hh8 = svdup_n_u16(0);
svuint16_t hh9 = svdup_n_u16(0);
svuint16_t hha = svdup_n_u16(0);
svuint16_t hhb = svdup_n_u16(0);
size_t i = 0;
while (i < size) {
/* We can only accumulate 15 (15 * 16 <= 255) iterations of histogram
* in 8-bit accumulators! */
const size_t size240 = min_size(i + 240, size);
svbool_t pred = svwhilelt_b8_u64(i, size);
svuint8_t c = svld1rq_u8(pred, src + i);
svuint8_t h0 = svhistseg_u8(c0, c);
svuint8_t h1 = svhistseg_u8(c1, c);
svuint8_t h2 = svhistseg_u8(c2, c);
svuint8_t h3 = svhistseg_u8(c3, c);
svuint8_t h4 = svhistseg_u8(c4, c);
svuint8_t h5 = svhistseg_u8(c5, c);
for (i += 16; i < size240; i += 16) {
pred = svwhilelt_b8_u64(i, size);
c = svld1rq_u8(pred, src + i);
h0 = svadd_u8_x(vl128, h0, svhistseg_u8(c0, c));
h1 = svadd_u8_x(vl128, h1, svhistseg_u8(c1, c));
h2 = svadd_u8_x(vl128, h2, svhistseg_u8(c2, c));
h3 = svadd_u8_x(vl128, h3, svhistseg_u8(c3, c));
h4 = svadd_u8_x(vl128, h4, svhistseg_u8(c4, c));
h5 = svadd_u8_x(vl128, h5, svhistseg_u8(c5, c));
}
hh0 = svaddwb_u16(hh0, h0);
hh1 = svaddwt_u16(hh1, h0);
hh2 = svaddwb_u16(hh2, h1);
hh3 = svaddwt_u16(hh3, h1);
hh4 = svaddwb_u16(hh4, h2);
hh5 = svaddwt_u16(hh5, h2);
hh6 = svaddwb_u16(hh6, h3);
hh7 = svaddwt_u16(hh7, h3);
hh8 = svaddwb_u16(hh8, h4);
hh9 = svaddwt_u16(hh9, h4);
hha = svaddwb_u16(hha, h5);
hhb = svaddwt_u16(hhb, h5);
}
svst1_u32(svwhilelt_b32_u64( 0, maxCount), dst + 0, svshllb_n_u32(hh0, 0));
svst1_u32(svwhilelt_b32_u64( 4, maxCount), dst + 4, svshllt_n_u32(hh0, 0));
svst1_u32(svwhilelt_b32_u64( 8, maxCount), dst + 8, svshllb_n_u32(hh1, 0));
svst1_u32(svwhilelt_b32_u64(12, maxCount), dst + 12, svshllt_n_u32(hh1, 0));
svst1_u32(svwhilelt_b32_u64(16, maxCount), dst + 16, svshllb_n_u32(hh2, 0));
svst1_u32(svwhilelt_b32_u64(20, maxCount), dst + 20, svshllt_n_u32(hh2, 0));
svst1_u32(svwhilelt_b32_u64(24, maxCount), dst + 24, svshllb_n_u32(hh3, 0));
svst1_u32(svwhilelt_b32_u64(28, maxCount), dst + 28, svshllt_n_u32(hh3, 0));
svst1_u32(svwhilelt_b32_u64(32, maxCount), dst + 32, svshllb_n_u32(hh4, 0));
svst1_u32(svwhilelt_b32_u64(36, maxCount), dst + 36, svshllt_n_u32(hh4, 0));
svst1_u32(svwhilelt_b32_u64(40, maxCount), dst + 40, svshllb_n_u32(hh5, 0));
svst1_u32(svwhilelt_b32_u64(44, maxCount), dst + 44, svshllt_n_u32(hh5, 0));
svst1_u32(svwhilelt_b32_u64(48, maxCount), dst + 48, svshllb_n_u32(hh6, 0));
svst1_u32(svwhilelt_b32_u64(52, maxCount), dst + 52, svshllt_n_u32(hh6, 0));
svst1_u32(svwhilelt_b32_u64(56, maxCount), dst + 56, svshllb_n_u32(hh7, 0));
svst1_u32(svwhilelt_b32_u64(60, maxCount), dst + 60, svshllt_n_u32(hh7, 0));
svst1_u32(svwhilelt_b32_u64(64, maxCount), dst + 64, svshllb_n_u32(hh8, 0));
svst1_u32(svwhilelt_b32_u64(68, maxCount), dst + 68, svshllt_n_u32(hh8, 0));
svst1_u32(svwhilelt_b32_u64(72, maxCount), dst + 72, svshllb_n_u32(hh9, 0));
svst1_u32(svwhilelt_b32_u64(76, maxCount), dst + 76, svshllt_n_u32(hh9, 0));
svst1_u32(svwhilelt_b32_u64(80, maxCount), dst + 80, svshllb_n_u32(hha, 0));
svst1_u32(svwhilelt_b32_u64(84, maxCount), dst + 84, svshllt_n_u32(hha, 0));
svst1_u32(svwhilelt_b32_u64(88, maxCount), dst + 88, svshllb_n_u32(hhb, 0));
svst1_u32(svwhilelt_b32_u64(92, maxCount), dst + 92, svshllt_n_u32(hhb, 0));
hh0 = svmax_u16_x(vl128, hh0, hh1);
hh2 = svmax_u16_x(vl128, hh2, hh3);
hh4 = svmax_u16_x(vl128, hh4, hh5);
hh6 = svmax_u16_x(vl128, hh6, hh7);
hh8 = svmax_u16_x(vl128, hh8, hh9);
hha = svmax_u16_x(vl128, hha, hhb);
hh0 = svmax_u16_x(vl128, hh0, hh2);
hh4 = svmax_u16_x(vl128, hh4, hh6);
hh8 = svmax_u16_x(vl128, hh8, hha);
hh0 = svmax_u16_x(vl128, hh0, hh4);
hh8 = svmax_u16_x(vl128, hh8, histmax);
return svmax_u16_x(vl128, hh0, hh8);
}
static size_t HIST_count_sve2(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize,
HIST_checkInput_e check)
{
const BYTE* ip = (const BYTE*)source;
const size_t maxCount = *maxSymbolValuePtr + 1;
assert(*maxSymbolValuePtr <= 255);
if (!sourceSize) {
ZSTD_memset(count, 0, maxCount * sizeof(*count));
*maxSymbolValuePtr = 0;
return 0;
}
{ const svbool_t vl128 = svptrue_pat_b8(SV_VL16);
const svuint8_t c0 = svreinterpret_u8(svindex_u32(0x0C040800, 0x01010101));
const svuint8_t c1 = svadd_n_u8_x(vl128, c0, 16);
const svuint8_t c2 = svadd_n_u8_x(vl128, c0, 32);
const svuint8_t c3 = svadd_n_u8_x(vl128, c1, 32);
svuint8_t symbolMax = svdup_n_u8(0);
svuint16_t hh0 = svdup_n_u16(0);
svuint16_t hh1 = svdup_n_u16(0);
svuint16_t hh2 = svdup_n_u16(0);
svuint16_t hh3 = svdup_n_u16(0);
svuint16_t hh4 = svdup_n_u16(0);
svuint16_t hh5 = svdup_n_u16(0);
svuint16_t hh6 = svdup_n_u16(0);
svuint16_t hh7 = svdup_n_u16(0);
svuint16_t max;
size_t maxSymbolValue;
size_t i = 0;
while (i < sourceSize) {
/* We can only accumulate 15 (15 * 16 <= 255) iterations of
* histogram in 8-bit accumulators! */
const size_t size240 = min_size(i + 240, sourceSize);
svbool_t pred = svwhilelt_b8_u64(i, sourceSize);
svuint8_t c = svld1rq_u8(pred, ip + i);
svuint8_t h0 = svhistseg_u8(c0, c);
svuint8_t h1 = svhistseg_u8(c1, c);
svuint8_t h2 = svhistseg_u8(c2, c);
svuint8_t h3 = svhistseg_u8(c3, c);
symbolMax = svmax_u8_x(vl128, symbolMax, c);
for (i += 16; i < size240; i += 16) {
pred = svwhilelt_b8_u64(i, sourceSize);
c = svld1rq_u8(pred, ip + i);
h0 = svadd_u8_x(vl128, h0, svhistseg_u8(c0, c));
h1 = svadd_u8_x(vl128, h1, svhistseg_u8(c1, c));
h2 = svadd_u8_x(vl128, h2, svhistseg_u8(c2, c));
h3 = svadd_u8_x(vl128, h3, svhistseg_u8(c3, c));
symbolMax = svmax_u8_x(vl128, symbolMax, c);
}
hh0 = svaddwb_u16(hh0, h0);
hh1 = svaddwt_u16(hh1, h0);
hh2 = svaddwb_u16(hh2, h1);
hh3 = svaddwt_u16(hh3, h1);
hh4 = svaddwb_u16(hh4, h2);
hh5 = svaddwt_u16(hh5, h2);
hh6 = svaddwb_u16(hh6, h3);
hh7 = svaddwt_u16(hh7, h3);
}
maxSymbolValue = svmaxv_u8(vl128, symbolMax);
if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall);
*maxSymbolValuePtr = maxSymbolValue;
/* If the buffer size is not divisible by 16, the last elements of the final
* vector register read will be zeros, and these elements must be subtracted
* from the histogram.
*/
hh0 = svsub_n_u16_m(svptrue_pat_b32(SV_VL1), hh0, -sourceSize & 15);
svst1_u32(svwhilelt_b32_u64( 0, maxCount), count + 0, svshllb_n_u32(hh0, 0));
svst1_u32(svwhilelt_b32_u64( 4, maxCount), count + 4, svshllt_n_u32(hh0, 0));
svst1_u32(svwhilelt_b32_u64( 8, maxCount), count + 8, svshllb_n_u32(hh1, 0));
svst1_u32(svwhilelt_b32_u64(12, maxCount), count + 12, svshllt_n_u32(hh1, 0));
svst1_u32(svwhilelt_b32_u64(16, maxCount), count + 16, svshllb_n_u32(hh2, 0));
svst1_u32(svwhilelt_b32_u64(20, maxCount), count + 20, svshllt_n_u32(hh2, 0));
svst1_u32(svwhilelt_b32_u64(24, maxCount), count + 24, svshllb_n_u32(hh3, 0));
svst1_u32(svwhilelt_b32_u64(28, maxCount), count + 28, svshllt_n_u32(hh3, 0));
svst1_u32(svwhilelt_b32_u64(32, maxCount), count + 32, svshllb_n_u32(hh4, 0));
svst1_u32(svwhilelt_b32_u64(36, maxCount), count + 36, svshllt_n_u32(hh4, 0));
svst1_u32(svwhilelt_b32_u64(40, maxCount), count + 40, svshllb_n_u32(hh5, 0));
svst1_u32(svwhilelt_b32_u64(44, maxCount), count + 44, svshllt_n_u32(hh5, 0));
svst1_u32(svwhilelt_b32_u64(48, maxCount), count + 48, svshllb_n_u32(hh6, 0));
svst1_u32(svwhilelt_b32_u64(52, maxCount), count + 52, svshllt_n_u32(hh6, 0));
svst1_u32(svwhilelt_b32_u64(56, maxCount), count + 56, svshllb_n_u32(hh7, 0));
svst1_u32(svwhilelt_b32_u64(60, maxCount), count + 60, svshllt_n_u32(hh7, 0));
hh0 = svmax_u16_x(vl128, hh0, hh1);
hh2 = svmax_u16_x(vl128, hh2, hh3);
hh4 = svmax_u16_x(vl128, hh4, hh5);
hh6 = svmax_u16_x(vl128, hh6, hh7);
hh0 = svmax_u16_x(vl128, hh0, hh2);
hh4 = svmax_u16_x(vl128, hh4, hh6);
max = svmax_u16_x(vl128, hh0, hh4);
maxSymbolValue = min_size(maxSymbolValue, maxCount);
if (maxSymbolValue >= 64) {
const svuint8_t c4 = svadd_n_u8_x(vl128, c0, 64);
const svuint8_t c5 = svadd_n_u8_x(vl128, c1, 64);
const svuint8_t c6 = svadd_n_u8_x(vl128, c2, 64);
const svuint8_t c7 = svadd_n_u8_x(vl128, c3, 64);
const svuint8_t c8 = svadd_n_u8_x(vl128, c0, 128);
const svuint8_t c9 = svadd_n_u8_x(vl128, c1, 128);
max = HIST_count_6_sve2(ip, sourceSize, count + 64, c4, c5, c6, c7,
c8, c9, max, maxCount - 64);
if (maxSymbolValue >= 160) {
const svuint8_t ca = svadd_n_u8_x(vl128, c2, 128);
const svuint8_t cb = svadd_n_u8_x(vl128, c3, 128);
const svuint8_t cc = svadd_n_u8_x(vl128, c4, 128);
const svuint8_t cd = svadd_n_u8_x(vl128, c5, 128);
const svuint8_t ce = svadd_n_u8_x(vl128, c6, 128);
const svuint8_t cf = svadd_n_u8_x(vl128, c7, 128);
max = HIST_count_6_sve2(ip, sourceSize, count + 160, ca, cb, cc,
cd, ce, cf, max, maxCount - 160);
} else if (maxCount > 160) {
ZSTD_memset(count + 160, 0, (maxCount - 160) * sizeof(*count));
}
} else if (maxCount > 64) {
ZSTD_memset(count + 64, 0, (maxCount - 64) * sizeof(*count));
}
return svmaxv_u16(vl128, max);
}
}
#endif
/* HIST_count_parallel_wksp() :
* store histogram into 4 intermediate tables, recombined at the end.
* this design makes better use of OoO cpus,
@@ -317,8 +63,8 @@ static size_t HIST_count_sve2(unsigned* count, unsigned* maxSymbolValuePtr,
* `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32.
* @return : largest histogram frequency,
* or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */
static UNUSED_ATTR
size_t HIST_count_parallel_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
static size_t HIST_count_parallel_wksp(
unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize,
HIST_checkInput_e check,
U32* const workSpace)
@@ -395,17 +141,11 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize,
void* workSpace, size_t workSpaceSize)
{
if (sourceSize < HIST_FAST_THRESHOLD) /* heuristic threshold */
if (sourceSize < 1500) /* heuristic threshold */
return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize);
#if defined(ZSTD_ARCH_ARM_SVE2)
(void)workSpace;
(void)workSpaceSize;
return HIST_count_sve2(count, maxSymbolValuePtr, source, sourceSize, trustInput);
#else
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace);
#endif
}
/* HIST_count_wksp() :
@@ -415,15 +155,10 @@ size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize,
void* workSpace, size_t workSpaceSize)
{
#if defined(ZSTD_ARCH_ARM_SVE2)
if (*maxSymbolValuePtr < 255)
return HIST_count_sve2(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue);
#else
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
if (*maxSymbolValuePtr < 255)
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace);
#endif
*maxSymbolValuePtr = 255;
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize);
}
-11
View File
@@ -35,11 +35,7 @@ unsigned HIST_isError(size_t code); /**< tells if a return value is an error co
/* --- advanced histogram functions --- */
#if defined(__ARM_FEATURE_SVE2)
#define HIST_WKSP_SIZE_U32 0
#else
#define HIST_WKSP_SIZE_U32 1024
#endif
#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned))
/** HIST_count_wksp() :
* Same as HIST_count(), but using an externally provided scratch buffer.
@@ -77,10 +73,3 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
*/
unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize);
/*! HIST_add() :
* Lowest level: just add nb of occurrences of characters from @src into @count.
* @count is not reset. @count array is presumed large enough (i.e. 1 KB).
@ This function does not need any additional stack memory.
*/
void HIST_add(unsigned* count, const void* src, size_t srcSize);
-1
View File
@@ -425,7 +425,6 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits
* gain back half the rank.
*/
U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1;
assert(nBitsToDecrease <= HUF_TABLELOG_MAX+1);
for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
U32 const highPos = rankLast[nBitsToDecrease];
U32 const lowPos = rankLast[nBitsToDecrease-1];
+350 -1567
View File
@@ -14,7 +14,6 @@
#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
#include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */
#include "../common/mem.h"
#include "../common/error_private.h"
#include "hist.h" /* HIST_countFast_wksp */
#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */
#include "../common/fse.h"
@@ -49,21 +48,13 @@
* in log format, aka 17 => 1 << 17 == 128Ki positions.
* This structure is only used in zstd_opt.
* Since allocation is centralized for all strategies, it has to be known here.
* The actual (selected) size of the hash table is then stored in ZSTD_MatchState_t.hashLog3,
* The actual (selected) size of the hash table is then stored in ZSTD_matchState_t.hashLog3,
* so that zstd_opt.c doesn't need to know about this constant.
*/
#ifndef ZSTD_HASHLOG3_MAX
# define ZSTD_HASHLOG3_MAX 17
#endif
/*-*************************************
* Forward declarations
***************************************/
size_t convertSequences_noRepcodes(SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs,
size_t nbSequences);
/*-*************************************
* Helper functions
***************************************/
@@ -91,12 +82,12 @@ struct ZSTD_CDict_s {
ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */
U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */
ZSTD_cwksp workspace;
ZSTD_MatchState_t matchState;
ZSTD_matchState_t matchState;
ZSTD_compressedBlockState_t cBlockState;
ZSTD_customMem customMem;
U32 dictID;
int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */
ZSTD_ParamSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use
ZSTD_paramSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use
* row-based matchfinder. Unless the cdict is reloaded, we will use
* the same greedy/lazy matchfinder at compression time.
*/
@@ -146,12 +137,11 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize)
ZSTD_cwksp_move(&cctx->workspace, &ws);
cctx->staticSize = workspaceSize;
/* statically sized space. tmpWorkspace never moves (but prev/next block swap places) */
if (!ZSTD_cwksp_check_available(&cctx->workspace, TMP_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL;
/* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */
if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL;
cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t));
cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t));
cctx->tmpWorkspace = ZSTD_cwksp_reserve_object(&cctx->workspace, TMP_WORKSPACE_SIZE);
cctx->tmpWkspSize = TMP_WORKSPACE_SIZE;
cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE);
cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid());
return cctx;
}
@@ -227,7 +217,7 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
}
/* private API call, for dictBuilder only */
const SeqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
/* Returns true if the strategy supports using a row based matchfinder */
static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) {
@@ -237,31 +227,32 @@ static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) {
/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder
* for this compression.
*/
static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_ParamSwitch_e mode) {
static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_paramSwitch_e mode) {
assert(mode != ZSTD_ps_auto);
return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable);
}
/* Returns row matchfinder usage given an initial mode and cParams */
static ZSTD_ParamSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_ParamSwitch_e mode,
static ZSTD_paramSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_paramSwitch_e mode,
const ZSTD_compressionParameters* const cParams) {
#ifdef ZSTD_LINUX_KERNEL
/* The Linux Kernel does not use SIMD, and 128KB is a very common size, e.g. in BtrFS.
* The row match finder is slower for this size without SIMD, so disable it.
*/
const unsigned kWindowLogLowerBound = 17;
#if defined(ZSTD_ARCH_X86_SSE2) || defined(ZSTD_ARCH_ARM_NEON)
int const kHasSIMD128 = 1;
#else
const unsigned kWindowLogLowerBound = 14;
int const kHasSIMD128 = 0;
#endif
if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */
mode = ZSTD_ps_disable;
if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode;
if (cParams->windowLog > kWindowLogLowerBound) mode = ZSTD_ps_enable;
if (kHasSIMD128) {
if (cParams->windowLog > 14) mode = ZSTD_ps_enable;
} else {
if (cParams->windowLog > 17) mode = ZSTD_ps_enable;
}
return mode;
}
/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */
static ZSTD_ParamSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_ParamSwitch_e mode,
static ZSTD_paramSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_paramSwitch_e mode,
const ZSTD_compressionParameters* const cParams) {
if (mode != ZSTD_ps_auto) return mode;
return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable;
@@ -269,7 +260,7 @@ static ZSTD_ParamSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_ParamSwitch_e mode,
/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */
static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
const ZSTD_ParamSwitch_e useRowMatchFinder,
const ZSTD_paramSwitch_e useRowMatchFinder,
const U32 forDDSDict) {
assert(useRowMatchFinder != ZSTD_ps_auto);
/* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate.
@@ -282,7 +273,7 @@ static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
* enable long distance matching (wlog >= 27, strategy >= btopt).
* Returns ZSTD_ps_disable otherwise.
*/
static ZSTD_ParamSwitch_e ZSTD_resolveEnableLdm(ZSTD_ParamSwitch_e mode,
static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
const ZSTD_compressionParameters* const cParams) {
if (mode != ZSTD_ps_auto) return mode;
return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable;
@@ -301,7 +292,7 @@ static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
}
}
static ZSTD_ParamSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_ParamSwitch_e value, int cLevel) {
static ZSTD_paramSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_paramSwitch_e value, int cLevel) {
if (value != ZSTD_ps_auto) return value;
if (cLevel < 10) {
return ZSTD_ps_disable;
@@ -331,7 +322,7 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog);
assert(cctxParams.ldmParams.hashRateLog < 32);
}
cctxParams.postBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.postBlockSplitter, &cParams);
cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize);
@@ -399,13 +390,13 @@ ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams,
*/
cctxParams->compressionLevel = compressionLevel;
cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, &params->cParams);
cctxParams->postBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->postBlockSplitter, &params->cParams);
cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, &params->cParams);
cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, &params->cParams);
cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize);
cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel);
DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
cctxParams->useRowMatchFinder, cctxParams->postBlockSplitter, cctxParams->ldmParams.enableLdm);
cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
}
size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
@@ -606,16 +597,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
bounds.upperBound = 1;
return bounds;
case ZSTD_c_splitAfterSequences:
case ZSTD_c_useBlockSplitter:
bounds.lowerBound = (int)ZSTD_ps_auto;
bounds.upperBound = (int)ZSTD_ps_disable;
return bounds;
case ZSTD_c_blockSplitterLevel:
bounds.lowerBound = 0;
bounds.upperBound = ZSTD_BLOCKSPLITTER_LEVEL_MAX;
return bounds;
case ZSTD_c_useRowMatchFinder:
bounds.lowerBound = (int)ZSTD_ps_auto;
bounds.upperBound = (int)ZSTD_ps_disable;
@@ -641,7 +627,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
return bounds;
case ZSTD_c_repcodeResolution:
case ZSTD_c_searchForExternalRepcodes:
bounds.lowerBound = (int)ZSTD_ps_auto;
bounds.upperBound = (int)ZSTD_ps_disable;
return bounds;
@@ -682,7 +668,6 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
case ZSTD_c_minMatch:
case ZSTD_c_targetLength:
case ZSTD_c_strategy:
case ZSTD_c_blockSplitterLevel:
return 1;
case ZSTD_c_format:
@@ -709,13 +694,13 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
case ZSTD_c_stableOutBuffer:
case ZSTD_c_blockDelimiters:
case ZSTD_c_validateSequences:
case ZSTD_c_splitAfterSequences:
case ZSTD_c_useBlockSplitter:
case ZSTD_c_useRowMatchFinder:
case ZSTD_c_deterministicRefPrefix:
case ZSTD_c_prefetchCDictTables:
case ZSTD_c_enableSeqProducerFallback:
case ZSTD_c_maxBlockSize:
case ZSTD_c_repcodeResolution:
case ZSTD_c_searchForExternalRepcodes:
default:
return 0;
}
@@ -768,14 +753,13 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
case ZSTD_c_stableOutBuffer:
case ZSTD_c_blockDelimiters:
case ZSTD_c_validateSequences:
case ZSTD_c_splitAfterSequences:
case ZSTD_c_blockSplitterLevel:
case ZSTD_c_useBlockSplitter:
case ZSTD_c_useRowMatchFinder:
case ZSTD_c_deterministicRefPrefix:
case ZSTD_c_prefetchCDictTables:
case ZSTD_c_enableSeqProducerFallback:
case ZSTD_c_maxBlockSize:
case ZSTD_c_repcodeResolution:
case ZSTD_c_searchForExternalRepcodes:
break;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@@ -873,7 +857,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
}
case ZSTD_c_literalCompressionMode : {
const ZSTD_ParamSwitch_e lcm = (ZSTD_ParamSwitch_e)value;
const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value;
BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm);
CCtxParams->literalCompressionMode = lcm;
return CCtxParams->literalCompressionMode;
@@ -899,7 +883,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
value = ZSTDMT_JOBSIZE_MIN;
FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), "");
assert(value >= 0);
CCtxParams->jobSize = (size_t)value;
CCtxParams->jobSize = value;
return CCtxParams->jobSize;
#endif
@@ -929,7 +913,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_enableLongDistanceMatching :
BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value);
CCtxParams->ldmParams.enableLdm = (ZSTD_ParamSwitch_e)value;
CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value;
return CCtxParams->ldmParams.enableLdm;
case ZSTD_c_ldmHashLog :
@@ -982,7 +966,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_blockDelimiters:
BOUNDCHECK(ZSTD_c_blockDelimiters, value);
CCtxParams->blockDelimiters = (ZSTD_SequenceFormat_e)value;
CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value;
return CCtxParams->blockDelimiters;
case ZSTD_c_validateSequences:
@@ -990,19 +974,14 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
CCtxParams->validateSequences = value;
return (size_t)CCtxParams->validateSequences;
case ZSTD_c_splitAfterSequences:
BOUNDCHECK(ZSTD_c_splitAfterSequences, value);
CCtxParams->postBlockSplitter = (ZSTD_ParamSwitch_e)value;
return CCtxParams->postBlockSplitter;
case ZSTD_c_blockSplitterLevel:
BOUNDCHECK(ZSTD_c_blockSplitterLevel, value);
CCtxParams->preBlockSplitter_level = value;
return (size_t)CCtxParams->preBlockSplitter_level;
case ZSTD_c_useBlockSplitter:
BOUNDCHECK(ZSTD_c_useBlockSplitter, value);
CCtxParams->useBlockSplitter = (ZSTD_paramSwitch_e)value;
return CCtxParams->useBlockSplitter;
case ZSTD_c_useRowMatchFinder:
BOUNDCHECK(ZSTD_c_useRowMatchFinder, value);
CCtxParams->useRowMatchFinder = (ZSTD_ParamSwitch_e)value;
CCtxParams->useRowMatchFinder = (ZSTD_paramSwitch_e)value;
return CCtxParams->useRowMatchFinder;
case ZSTD_c_deterministicRefPrefix:
@@ -1012,7 +991,7 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_prefetchCDictTables:
BOUNDCHECK(ZSTD_c_prefetchCDictTables, value);
CCtxParams->prefetchCDictTables = (ZSTD_ParamSwitch_e)value;
CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value;
return CCtxParams->prefetchCDictTables;
case ZSTD_c_enableSeqProducerFallback:
@@ -1023,13 +1002,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
case ZSTD_c_maxBlockSize:
if (value!=0) /* 0 ==> default */
BOUNDCHECK(ZSTD_c_maxBlockSize, value);
assert(value>=0);
CCtxParams->maxBlockSize = (size_t)value;
CCtxParams->maxBlockSize = value;
return CCtxParams->maxBlockSize;
case ZSTD_c_repcodeResolution:
BOUNDCHECK(ZSTD_c_repcodeResolution, value);
CCtxParams->searchForExternalRepcodes = (ZSTD_ParamSwitch_e)value;
case ZSTD_c_searchForExternalRepcodes:
BOUNDCHECK(ZSTD_c_searchForExternalRepcodes, value);
CCtxParams->searchForExternalRepcodes = (ZSTD_paramSwitch_e)value;
return CCtxParams->searchForExternalRepcodes;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@@ -1047,7 +1025,7 @@ size_t ZSTD_CCtxParams_getParameter(
switch(param)
{
case ZSTD_c_format :
*value = (int)CCtxParams->format;
*value = CCtxParams->format;
break;
case ZSTD_c_compressionLevel :
*value = CCtxParams->compressionLevel;
@@ -1062,16 +1040,16 @@ size_t ZSTD_CCtxParams_getParameter(
*value = (int)CCtxParams->cParams.chainLog;
break;
case ZSTD_c_searchLog :
*value = (int)CCtxParams->cParams.searchLog;
*value = CCtxParams->cParams.searchLog;
break;
case ZSTD_c_minMatch :
*value = (int)CCtxParams->cParams.minMatch;
*value = CCtxParams->cParams.minMatch;
break;
case ZSTD_c_targetLength :
*value = (int)CCtxParams->cParams.targetLength;
*value = CCtxParams->cParams.targetLength;
break;
case ZSTD_c_strategy :
*value = (int)CCtxParams->cParams.strategy;
*value = (unsigned)CCtxParams->cParams.strategy;
break;
case ZSTD_c_contentSizeFlag :
*value = CCtxParams->fParams.contentSizeFlag;
@@ -1086,10 +1064,10 @@ size_t ZSTD_CCtxParams_getParameter(
*value = CCtxParams->forceWindow;
break;
case ZSTD_c_forceAttachDict :
*value = (int)CCtxParams->attachDictPref;
*value = CCtxParams->attachDictPref;
break;
case ZSTD_c_literalCompressionMode :
*value = (int)CCtxParams->literalCompressionMode;
*value = CCtxParams->literalCompressionMode;
break;
case ZSTD_c_nbWorkers :
#ifndef ZSTD_MULTITHREAD
@@ -1123,19 +1101,19 @@ size_t ZSTD_CCtxParams_getParameter(
*value = CCtxParams->enableDedicatedDictSearch;
break;
case ZSTD_c_enableLongDistanceMatching :
*value = (int)CCtxParams->ldmParams.enableLdm;
*value = CCtxParams->ldmParams.enableLdm;
break;
case ZSTD_c_ldmHashLog :
*value = (int)CCtxParams->ldmParams.hashLog;
*value = CCtxParams->ldmParams.hashLog;
break;
case ZSTD_c_ldmMinMatch :
*value = (int)CCtxParams->ldmParams.minMatchLength;
*value = CCtxParams->ldmParams.minMatchLength;
break;
case ZSTD_c_ldmBucketSizeLog :
*value = (int)CCtxParams->ldmParams.bucketSizeLog;
*value = CCtxParams->ldmParams.bucketSizeLog;
break;
case ZSTD_c_ldmHashRateLog :
*value = (int)CCtxParams->ldmParams.hashRateLog;
*value = CCtxParams->ldmParams.hashRateLog;
break;
case ZSTD_c_targetCBlockSize :
*value = (int)CCtxParams->targetCBlockSize;
@@ -1155,11 +1133,8 @@ size_t ZSTD_CCtxParams_getParameter(
case ZSTD_c_validateSequences :
*value = (int)CCtxParams->validateSequences;
break;
case ZSTD_c_splitAfterSequences :
*value = (int)CCtxParams->postBlockSplitter;
break;
case ZSTD_c_blockSplitterLevel :
*value = CCtxParams->preBlockSplitter_level;
case ZSTD_c_useBlockSplitter :
*value = (int)CCtxParams->useBlockSplitter;
break;
case ZSTD_c_useRowMatchFinder :
*value = (int)CCtxParams->useRowMatchFinder;
@@ -1176,7 +1151,7 @@ size_t ZSTD_CCtxParams_getParameter(
case ZSTD_c_maxBlockSize:
*value = (int)CCtxParams->maxBlockSize;
break;
case ZSTD_c_repcodeResolution:
case ZSTD_c_searchForExternalRepcodes:
*value = (int)CCtxParams->searchForExternalRepcodes;
break;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@@ -1211,13 +1186,13 @@ size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams)
DEBUGLOG(4, "ZSTD_CCtx_setCParams");
/* only update if all parameters are valid */
FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)cparams.windowLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, (int)cparams.chainLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, (int)cparams.hashLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, (int)cparams.searchLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, (int)cparams.minMatch), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, (int)cparams.targetLength), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, (int)cparams.strategy), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, cparams.windowLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, cparams.chainLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, cparams.hashLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, cparams.searchLog), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, cparams.minMatch), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, cparams.targetLength), "");
FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, cparams.strategy), "");
return 0;
}
@@ -1409,7 +1384,7 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog);
BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch);
BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength);
BOUNDCHECK(ZSTD_c_strategy, (int)cParams.strategy);
BOUNDCHECK(ZSTD_c_strategy, cParams.strategy);
return 0;
}
@@ -1482,15 +1457,15 @@ static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize)
* optimize `cPar` for a specified input (`srcSize` and `dictSize`).
* mostly downsize to reduce memory consumption and initialization latency.
* `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known.
* `mode` is the mode for parameter adjustment. See docs for `ZSTD_CParamMode_e`.
* `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`.
* note : `srcSize==0` means 0!
* condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */
static ZSTD_compressionParameters
ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
unsigned long long srcSize,
size_t dictSize,
ZSTD_CParamMode_e mode,
ZSTD_ParamSwitch_e useRowMatchFinder)
ZSTD_cParamMode_e mode,
ZSTD_paramSwitch_e useRowMatchFinder)
{
const U64 minSrcSize = 513; /* (1<<9) + 1 */
const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
@@ -1634,8 +1609,8 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar,
return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto);
}
static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode);
static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode);
static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
static void ZSTD_overrideCParams(
ZSTD_compressionParameters* cParams,
@@ -1651,12 +1626,11 @@ static void ZSTD_overrideCParams(
}
ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode)
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
{
ZSTD_compressionParameters cParams;
if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) {
assert(CCtxParams->srcSizeHint>=0);
srcSizeHint = (U64)CCtxParams->srcSizeHint;
srcSizeHint = CCtxParams->srcSizeHint;
}
cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode);
if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
@@ -1668,8 +1642,8 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
static size_t
ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
const ZSTD_ParamSwitch_e useRowMatchFinder,
const int enableDedicatedDictSearch,
const ZSTD_paramSwitch_e useRowMatchFinder,
const U32 enableDedicatedDictSearch,
const U32 forCCtx)
{
/* chain table size should be 0 for fast or row-hash strategies */
@@ -1685,14 +1659,14 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
+ hSize * sizeof(U32)
+ h3Size * sizeof(U32);
size_t const optPotentialSpace =
ZSTD_cwksp_aligned64_alloc_size((MaxML+1) * sizeof(U32))
+ ZSTD_cwksp_aligned64_alloc_size((MaxLL+1) * sizeof(U32))
+ ZSTD_cwksp_aligned64_alloc_size((MaxOff+1) * sizeof(U32))
+ ZSTD_cwksp_aligned64_alloc_size((1<<Litbits) * sizeof(U32))
+ ZSTD_cwksp_aligned64_alloc_size(ZSTD_OPT_SIZE * sizeof(ZSTD_match_t))
+ ZSTD_cwksp_aligned64_alloc_size(ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t));
ZSTD_cwksp_aligned_alloc_size((MaxML+1) * sizeof(U32))
+ ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32))
+ ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32))
+ ZSTD_cwksp_aligned_alloc_size((1<<Litbits) * sizeof(U32))
+ ZSTD_cwksp_aligned_alloc_size(ZSTD_OPT_SIZE * sizeof(ZSTD_match_t))
+ ZSTD_cwksp_aligned_alloc_size(ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t));
size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)
? ZSTD_cwksp_aligned64_alloc_size(hSize)
? ZSTD_cwksp_aligned_alloc_size(hSize)
: 0;
size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
? optPotentialSpace
@@ -1719,7 +1693,7 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
const ZSTD_compressionParameters* cParams,
const ldmParams_t* ldmParams,
const int isStatic,
const ZSTD_ParamSwitch_e useRowMatchFinder,
const ZSTD_paramSwitch_e useRowMatchFinder,
const size_t buffInSize,
const size_t buffOutSize,
const U64 pledgedSrcSize,
@@ -1730,16 +1704,16 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer);
size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
+ ZSTD_cwksp_aligned64_alloc_size(maxNbSeq * sizeof(SeqDef))
+ ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
+ 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
size_t const tmpWorkSpace = ZSTD_cwksp_alloc_size(TMP_WORKSPACE_SIZE);
size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE);
size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1);
size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams);
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize);
size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ?
ZSTD_cwksp_aligned64_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0;
ZSTD_cwksp_aligned_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0;
size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize)
@@ -1749,12 +1723,12 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
size_t const externalSeqSpace = useSequenceProducer
? ZSTD_cwksp_aligned64_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
: 0;
size_t const neededSpace =
cctxSpace +
tmpWorkSpace +
entropySpace +
blockStateSpace +
ldmSpace +
ldmSeqSpace +
@@ -1771,19 +1745,15 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
{
ZSTD_compressionParameters const cParams =
ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
ldmParams_t ldmParams = params->ldmParams;
ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder,
ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder,
&cParams);
RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
if (ldmParams.enableLdm == ZSTD_ps_enable) {
ZSTD_ldm_adjustParameters(&ldmParams, &cParams);
}
/* estimateCCtxSize is for one-shot compression. So no buffers should
* be needed. However, we still allocate two 0-sized buffers, which can
* take space under ASAN. */
return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
&cParams, &ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize);
&cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize);
}
size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
@@ -1833,7 +1803,6 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
{ ZSTD_compressionParameters const cParams =
ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
ldmParams_t ldmParams = params->ldmParams;
size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog);
size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered)
? ((size_t)1 << cParams.windowLog) + blockSize
@@ -1841,13 +1810,10 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered)
? ZSTD_compressBound(blockSize) + 1
: 0;
ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, &params->cParams);
ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, &params->cParams);
if (ldmParams.enableLdm == ZSTD_ps_enable) {
ZSTD_ldm_adjustParameters(&ldmParams, &cParams);
}
return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
&cParams, &ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
&cParams, &params->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize);
}
}
@@ -1954,7 +1920,7 @@ void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs)
* Invalidate all the matches in the match finder tables.
* Requires nextSrc and base to be set (can be NULL).
*/
static void ZSTD_invalidateMatchState(ZSTD_MatchState_t* ms)
static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms)
{
ZSTD_window_clear(&ms->window);
@@ -2001,15 +1967,15 @@ static U64 ZSTD_bitmix(U64 val, U64 len) {
}
/* Mixes in the hashSalt and hashSaltEntropy to create a new hashSalt */
static void ZSTD_advanceHashSalt(ZSTD_MatchState_t* ms) {
static void ZSTD_advanceHashSalt(ZSTD_matchState_t* ms) {
ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4);
}
static size_t
ZSTD_reset_matchState(ZSTD_MatchState_t* ms,
ZSTD_reset_matchState(ZSTD_matchState_t* ms,
ZSTD_cwksp* ws,
const ZSTD_compressionParameters* cParams,
const ZSTD_ParamSwitch_e useRowMatchFinder,
const ZSTD_paramSwitch_e useRowMatchFinder,
const ZSTD_compResetPolicy_e crp,
const ZSTD_indexResetPolicy_e forceResetIndex,
const ZSTD_resetTarget_e forWho)
@@ -2063,7 +2029,7 @@ ZSTD_reset_matchState(ZSTD_MatchState_t* ms,
ZSTD_advanceHashSalt(ms);
} else {
/* When we are not salting we want to always memset the memory */
ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned64(ws, tagTableSize);
ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned(ws, tagTableSize);
ZSTD_memset(ms->tagTable, 0, tagTableSize);
ms->hashSalt = 0;
}
@@ -2077,12 +2043,12 @@ ZSTD_reset_matchState(ZSTD_MatchState_t* ms,
/* opt parser space */
if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) {
DEBUGLOG(4, "reserving optimal parser space");
ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (1<<Litbits) * sizeof(unsigned));
ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxLL+1) * sizeof(unsigned));
ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxML+1) * sizeof(unsigned));
ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxOff+1) * sizeof(unsigned));
ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t));
ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t));
ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<<Litbits) * sizeof(unsigned));
ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned));
ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned));
ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned));
ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t));
ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t));
}
ms->cParams = *cParams;
@@ -2130,7 +2096,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
{
ZSTD_cwksp* const ws = &zc->workspace;
DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d",
(U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->postBlockSplitter);
(U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->useBlockSplitter);
assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams)));
zc->isFirstBlock = 1;
@@ -2142,7 +2108,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
params = &zc->appliedParams;
assert(params->useRowMatchFinder != ZSTD_ps_auto);
assert(params->postBlockSplitter != ZSTD_ps_auto);
assert(params->useBlockSplitter != ZSTD_ps_auto);
assert(params->ldmParams.enableLdm != ZSTD_ps_auto);
assert(params->maxBlockSize != 0);
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
@@ -2198,16 +2164,15 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
DEBUGLOG(5, "reserving object space");
/* Statically sized space.
* tmpWorkspace never moves,
* entropyWorkspace never moves,
* though prev/next block swap places */
assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t)));
zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t));
RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock");
zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t));
RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock");
zc->tmpWorkspace = ZSTD_cwksp_reserve_object(ws, TMP_WORKSPACE_SIZE);
RETURN_ERROR_IF(zc->tmpWorkspace == NULL, memory_allocation, "couldn't allocate tmpWorkspace");
zc->tmpWkspSize = TMP_WORKSPACE_SIZE;
zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE);
RETURN_ERROR_IF(zc->entropyWorkspace == NULL, memory_allocation, "couldn't allocate entropyWorkspace");
} }
ZSTD_cwksp_clear(ws);
@@ -2222,7 +2187,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zc->appliedParams.fParams.contentSizeFlag = 0;
DEBUGLOG(4, "pledged content size : %u ; flag : %u",
(unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag);
zc->blockSizeMax = blockSize;
zc->blockSize = blockSize;
XXH64_reset(&zc->xxhState, 0);
zc->stage = ZSTDcs_init;
@@ -2240,15 +2205,15 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
needsIndexReset,
ZSTD_resetTarget_CCtx), "");
zc->seqStore.sequencesStart = (SeqDef*)ZSTD_cwksp_reserve_aligned64(ws, maxNbSeq * sizeof(SeqDef));
zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef));
/* ldm hash table */
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
/* TODO: avoid memset? */
size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog;
zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned64(ws, ldmHSize * sizeof(ldmEntry_t));
zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t));
ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t));
zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned64(ws, maxNbLdmSeq * sizeof(rawSeq));
zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
zc->maxNbLdmSequences = maxNbLdmSeq;
ZSTD_window_init(&zc->ldmState.window);
@@ -2260,7 +2225,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
zc->extSeqBufCapacity = maxNbExternalSeq;
zc->extSeqBuf =
(ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned64(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
(ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
}
/* buffers */
@@ -2479,8 +2444,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
}
/* Zero the hashTable3, since the cdict never fills it */
assert(cctx->blockState.matchState.hashLog3 <= 31);
{ U32 const h3log = cctx->blockState.matchState.hashLog3;
{ int const h3log = cctx->blockState.matchState.hashLog3;
size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
assert(cdict->matchState.hashLog3 == 0);
ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32));
@@ -2489,8 +2453,8 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
ZSTD_cwksp_mark_tables_clean(&cctx->workspace);
/* copy dictionary offsets */
{ ZSTD_MatchState_t const* srcMatchState = &cdict->matchState;
ZSTD_MatchState_t* dstMatchState = &cctx->blockState.matchState;
{ ZSTD_matchState_t const* srcMatchState = &cdict->matchState;
ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState;
dstMatchState->window = srcMatchState->window;
dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
@@ -2548,10 +2512,10 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
/* Copy only compression parameters related to tables. */
params.cParams = srcCCtx->appliedParams.cParams;
assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto);
assert(srcCCtx->appliedParams.postBlockSplitter != ZSTD_ps_auto);
assert(srcCCtx->appliedParams.useBlockSplitter != ZSTD_ps_auto);
assert(srcCCtx->appliedParams.ldmParams.enableLdm != ZSTD_ps_auto);
params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder;
params.postBlockSplitter = srcCCtx->appliedParams.postBlockSplitter;
params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter;
params.ldmParams = srcCCtx->appliedParams.ldmParams;
params.fParams = fParams;
params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize;
@@ -2574,7 +2538,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog)
: 0;
size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
U32 const h3log = srcCCtx->blockState.matchState.hashLog3;
int const h3log = srcCCtx->blockState.matchState.hashLog3;
size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable,
@@ -2592,8 +2556,8 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
/* copy dictionary offsets */
{
const ZSTD_MatchState_t* srcMatchState = &srcCCtx->blockState.matchState;
ZSTD_MatchState_t* dstMatchState = &dstCCtx->blockState.matchState;
const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState;
ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState;
dstMatchState->window = srcMatchState->window;
dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
@@ -2642,7 +2606,7 @@ ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerVa
/* Protect special index values < ZSTD_WINDOW_START_INDEX. */
U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX;
assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */
assert(size < (1U<<31)); /* can be cast to int */
assert(size < (1U<<31)); /* can be casted to int */
#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
/* To validate that the table reuse logic is sound, and that we don't
@@ -2687,7 +2651,7 @@ static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const
/*! ZSTD_reduceIndex() :
* rescale all indexes to avoid future overflow (indexes are U32) */
static void ZSTD_reduceIndex (ZSTD_MatchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue)
static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue)
{
{ U32 const hSize = (U32)1 << params->cParams.hashLog;
ZSTD_reduceTable(ms->hashTable, hSize, reducerValue);
@@ -2714,9 +2678,9 @@ static void ZSTD_reduceIndex (ZSTD_MatchState_t* ms, ZSTD_CCtx_params const* par
/* See doc/zstd_compression_format.md for detailed format description */
int ZSTD_seqToCodes(const SeqStore_t* seqStorePtr)
int ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
{
const SeqDef* const sequences = seqStorePtr->sequencesStart;
const seqDef* const sequences = seqStorePtr->sequencesStart;
BYTE* const llCodeTable = seqStorePtr->llCode;
BYTE* const ofCodeTable = seqStorePtr->ofCode;
BYTE* const mlCodeTable = seqStorePtr->mlCode;
@@ -2759,9 +2723,9 @@ static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams)
* Returns 1 if true, 0 otherwise. */
static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams)
{
DEBUGLOG(5, "ZSTD_blockSplitterEnabled (postBlockSplitter=%d)", cctxParams->postBlockSplitter);
assert(cctxParams->postBlockSplitter != ZSTD_ps_auto);
return (cctxParams->postBlockSplitter == ZSTD_ps_enable);
DEBUGLOG(5, "ZSTD_blockSplitterEnabled (useBlockSplitter=%d)", cctxParams->useBlockSplitter);
assert(cctxParams->useBlockSplitter != ZSTD_ps_auto);
return (cctxParams->useBlockSplitter == ZSTD_ps_enable);
}
/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types
@@ -2785,7 +2749,7 @@ typedef struct {
*/
static ZSTD_symbolEncodingTypeStats_t
ZSTD_buildSequencesStatistics(
const SeqStore_t* seqStorePtr, size_t nbSeq,
const seqStore_t* seqStorePtr, size_t nbSeq,
const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
BYTE* dst, const BYTE* const dstEnd,
ZSTD_strategy strategy, unsigned* countWorkspace,
@@ -2821,7 +2785,7 @@ ZSTD_buildSequencesStatistics(
assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_LitLength, LLFSELog, (SymbolEncodingType_e)stats.LLtype,
CTable_LitLength, LLFSELog, (symbolEncodingType_e)stats.LLtype,
countWorkspace, max, llCodeTable, nbSeq,
LL_defaultNorm, LL_defaultNormLog, MaxLL,
prevEntropy->litlengthCTable,
@@ -2842,7 +2806,7 @@ ZSTD_buildSequencesStatistics(
size_t const mostFrequent = HIST_countFast_wksp(
countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
ZSTD_DefaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
DEBUGLOG(5, "Building OF table");
nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode,
@@ -2853,7 +2817,7 @@ ZSTD_buildSequencesStatistics(
assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_OffsetBits, OffFSELog, (SymbolEncodingType_e)stats.Offtype,
CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)stats.Offtype,
countWorkspace, max, ofCodeTable, nbSeq,
OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
prevEntropy->offcodeCTable,
@@ -2883,7 +2847,7 @@ ZSTD_buildSequencesStatistics(
assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_MatchLength, MLFSELog, (SymbolEncodingType_e)stats.MLtype,
CTable_MatchLength, MLFSELog, (symbolEncodingType_e)stats.MLtype,
countWorkspace, max, mlCodeTable, nbSeq,
ML_defaultNorm, ML_defaultNormLog, MaxML,
prevEntropy->matchlengthCTable,
@@ -2910,12 +2874,11 @@ ZSTD_buildSequencesStatistics(
#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20
MEM_STATIC size_t
ZSTD_entropyCompressSeqStore_internal(
void* dst, size_t dstCapacity,
const void* literals, size_t litSize,
const SeqStore_t* seqStorePtr,
const seqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
void* dst, size_t dstCapacity,
void* entropyWorkspace, size_t entropyWkspSize,
const int bmi2)
{
@@ -2924,7 +2887,7 @@ ZSTD_entropyCompressSeqStore_internal(
FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
const SeqDef* const sequences = seqStorePtr->sequencesStart;
const seqDef* const sequences = seqStorePtr->sequencesStart;
const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
const BYTE* const ofCodeTable = seqStorePtr->ofCode;
const BYTE* const llCodeTable = seqStorePtr->llCode;
@@ -2943,9 +2906,12 @@ ZSTD_entropyCompressSeqStore_internal(
assert(entropyWkspSize >= HUF_WORKSPACE_SIZE);
/* Compress literals */
{ size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
{ const BYTE* const literals = seqStorePtr->litStart;
size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
/* Base suspicion of uncompressibility on ratio of literals to sequences */
int const suspectUncompressible = (numSequences == 0) || (litSize / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
size_t const litSize = (size_t)(seqStorePtr->lit - literals);
size_t const cSize = ZSTD_compressLiterals(
op, dstCapacity,
@@ -3026,35 +2992,33 @@ ZSTD_entropyCompressSeqStore_internal(
return (size_t)(op - ostart);
}
static size_t
ZSTD_entropyCompressSeqStore_wExtLitBuffer(
void* dst, size_t dstCapacity,
const void* literals, size_t litSize,
size_t blockSize,
const SeqStore_t* seqStorePtr,
MEM_STATIC size_t
ZSTD_entropyCompressSeqStore(
const seqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
void* dst, size_t dstCapacity,
size_t srcSize,
void* entropyWorkspace, size_t entropyWkspSize,
int bmi2)
{
size_t const cSize = ZSTD_entropyCompressSeqStore_internal(
dst, dstCapacity,
literals, litSize,
seqStorePtr, prevEntropy, nextEntropy, cctxParams,
dst, dstCapacity,
entropyWorkspace, entropyWkspSize, bmi2);
if (cSize == 0) return 0;
/* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
* Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
*/
if ((cSize == ERROR(dstSize_tooSmall)) & (blockSize <= dstCapacity)) {
if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) {
DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity);
return 0; /* block not compressed */
}
FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed");
/* Check compressibility */
{ size_t const maxCSize = blockSize - ZSTD_minGain(blockSize, cctxParams->cParams.strategy);
{ size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
if (cSize >= maxCSize) return 0; /* block not compressed */
}
DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
@@ -3065,34 +3029,12 @@ ZSTD_entropyCompressSeqStore_wExtLitBuffer(
return cSize;
}
static size_t
ZSTD_entropyCompressSeqStore(
const SeqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
void* dst, size_t dstCapacity,
size_t srcSize,
void* entropyWorkspace, size_t entropyWkspSize,
int bmi2)
{
return ZSTD_entropyCompressSeqStore_wExtLitBuffer(
dst, dstCapacity,
seqStorePtr->litStart, (size_t)(seqStorePtr->lit - seqStorePtr->litStart),
srcSize,
seqStorePtr,
prevEntropy, nextEntropy,
cctxParams,
entropyWorkspace, entropyWkspSize,
bmi2);
}
/* ZSTD_selectBlockCompressor() :
* Not static, but internal use only (used by long distance matcher)
* assumption : strat is a valid strategy */
ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode)
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode)
{
static const ZSTD_BlockCompressor_f blockCompressor[4][ZSTD_STRATEGY_MAX+1] = {
static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = {
{ ZSTD_compressBlock_fast /* default for 0 */,
ZSTD_compressBlock_fast,
ZSTD_COMPRESSBLOCK_DOUBLEFAST,
@@ -3137,13 +3079,13 @@ ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_Para
NULL,
NULL }
};
ZSTD_BlockCompressor_f selectedCompressor;
ZSTD_blockCompressor selectedCompressor;
ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1);
assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat));
DEBUGLOG(5, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder);
assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat));
DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder);
if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) {
static const ZSTD_BlockCompressor_f rowBasedBlockCompressors[4][3] = {
static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = {
{
ZSTD_COMPRESSBLOCK_GREEDY_ROW,
ZSTD_COMPRESSBLOCK_LAZY_ROW,
@@ -3165,7 +3107,7 @@ ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_Para
ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW
}
};
DEBUGLOG(5, "Selecting a row-based matchfinder");
DEBUGLOG(4, "Selecting a row-based matchfinder");
assert(useRowMatchFinder != ZSTD_ps_auto);
selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy];
} else {
@@ -3175,14 +3117,14 @@ ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_Para
return selectedCompressor;
}
static void ZSTD_storeLastLiterals(SeqStore_t* seqStorePtr,
static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr,
const BYTE* anchor, size_t lastLLSize)
{
ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize;
}
void ZSTD_resetSeqStore(SeqStore_t* ssPtr)
void ZSTD_resetSeqStore(seqStore_t* ssPtr)
{
ssPtr->lit = ssPtr->litStart;
ssPtr->sequences = ssPtr->sequencesStart;
@@ -3255,39 +3197,11 @@ static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seq
return litLenSum + matchLenSum;
}
/**
* Function to validate sequences produced by a block compressor.
*/
static void ZSTD_validateSeqStore(const SeqStore_t* seqStore, const ZSTD_compressionParameters* cParams)
{
#if DEBUGLEVEL >= 1
const SeqDef* seq = seqStore->sequencesStart;
const SeqDef* const seqEnd = seqStore->sequences;
size_t const matchLenLowerBound = cParams->minMatch == 3 ? 3 : 4;
for (; seq < seqEnd; ++seq) {
const ZSTD_SequenceLength seqLength = ZSTD_getSequenceLength(seqStore, seq);
assert(seqLength.matchLength >= matchLenLowerBound);
(void)seqLength;
(void)matchLenLowerBound;
}
#else
(void)seqStore;
(void)cParams;
#endif
}
static size_t
ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize,
ZSTD_ParamSwitch_e externalRepSearch);
typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_BuildSeqStore_e;
typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;
static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
{
ZSTD_MatchState_t* const ms = &zc->blockState.matchState;
ZSTD_matchState_t* const ms = &zc->blockState.matchState;
DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize);
assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
/* Assert that we have correctly flushed the ctx params into the ms's copy */
@@ -3348,7 +3262,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
src, srcSize);
assert(zc->externSeqStore.pos <= zc->externSeqStore.size);
} else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) {
RawSeqStore_t ldmSeqStore = kNullRawSeqStore;
rawSeqStore_t ldmSeqStore = kNullRawSeqStore;
/* External matchfinder + LDM is technically possible, just not implemented yet.
* We need to revisit soon and implement it. */
@@ -3399,11 +3313,11 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
/* Return early if there is no error, since we don't need to worry about last literals */
if (!ZSTD_isError(nbPostProcessedSeqs)) {
ZSTD_SequencePosition seqPos = {0,0,0};
ZSTD_sequencePosition seqPos = {0,0,0};
size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->extSeqBuf, nbPostProcessedSeqs);
RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!");
FORWARD_IF_ERROR(
ZSTD_transferSequences_wBlockDelim(
ZSTD_copySequencesToSeqStoreExplicitBlockDelim(
zc, &seqPos,
zc->extSeqBuf, nbPostProcessedSeqs,
src, srcSize,
@@ -3422,7 +3336,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
}
/* Fallback to software matchfinder */
{ ZSTD_BlockCompressor_f const blockCompressor =
{ ZSTD_blockCompressor const blockCompressor =
ZSTD_selectBlockCompressor(
zc->appliedParams.cParams.strategy,
zc->appliedParams.useRowMatchFinder,
@@ -3436,7 +3350,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
} }
} else { /* not long range mode and no external matchfinder */
ZSTD_BlockCompressor_f const blockCompressor = ZSTD_selectBlockCompressor(
ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(
zc->appliedParams.cParams.strategy,
zc->appliedParams.useRowMatchFinder,
dictMode);
@@ -3446,20 +3360,19 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
{ const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize;
ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize);
} }
ZSTD_validateSeqStore(&zc->seqStore, &zc->appliedParams.cParams);
return ZSTDbss_compress;
}
static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const SeqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM])
static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const seqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM])
{
const SeqDef* inSeqs = seqStore->sequencesStart;
const size_t nbInSequences = (size_t)(seqStore->sequences - inSeqs);
const seqDef* inSeqs = seqStore->sequencesStart;
const size_t nbInSequences = seqStore->sequences - inSeqs;
const size_t nbInLiterals = (size_t)(seqStore->lit - seqStore->litStart);
ZSTD_Sequence* outSeqs = seqCollector->seqIndex == 0 ? seqCollector->seqStart : seqCollector->seqStart + seqCollector->seqIndex;
const size_t nbOutSequences = nbInSequences + 1;
size_t nbOutLiterals = 0;
Repcodes_t repcodes;
repcodes_t repcodes;
size_t i;
/* Bounds check that we have enough space for every input sequence
@@ -3545,7 +3458,7 @@ size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
size_t outSeqsSize, const void* src, size_t srcSize)
{
const size_t dstCapacity = ZSTD_compressBound(srcSize);
void* dst; /* Make C90 happy. */
void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem);
SeqCollector seqCollector;
{
int targetCBlockSize;
@@ -3558,7 +3471,6 @@ size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
RETURN_ERROR_IF(nbWorkers != 0, parameter_unsupported, "nbWorkers != 0");
}
dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem);
RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!");
seqCollector.collectSequences = 1;
@@ -3619,7 +3531,7 @@ static int ZSTD_isRLE(const BYTE* src, size_t length) {
* This is just a heuristic based on the compressibility.
* It may return both false positives and false negatives.
*/
static int ZSTD_maybeRLE(SeqStore_t const* seqStore)
static int ZSTD_maybeRLE(seqStore_t const* seqStore)
{
size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart);
@@ -3643,7 +3555,7 @@ writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock)
lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
MEM_writeLE24(op, cBlockHeader);
DEBUGLOG(5, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock);
DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock);
}
/** ZSTD_buildBlockEntropyStats_literals() :
@@ -3781,7 +3693,7 @@ ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy)
* @return : size of fse tables or error code */
static size_t
ZSTD_buildBlockEntropyStats_sequences(
const SeqStore_t* seqStorePtr,
const seqStore_t* seqStorePtr,
const ZSTD_fseCTables_t* prevEntropy,
ZSTD_fseCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
@@ -3805,9 +3717,9 @@ ZSTD_buildBlockEntropyStats_sequences(
entropyWorkspace, entropyWorkspaceSize)
: ZSTD_buildDummySequencesStatistics(nextEntropy);
FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!");
fseMetadata->llType = (SymbolEncodingType_e) stats.LLtype;
fseMetadata->ofType = (SymbolEncodingType_e) stats.Offtype;
fseMetadata->mlType = (SymbolEncodingType_e) stats.MLtype;
fseMetadata->llType = (symbolEncodingType_e) stats.LLtype;
fseMetadata->ofType = (symbolEncodingType_e) stats.Offtype;
fseMetadata->mlType = (symbolEncodingType_e) stats.MLtype;
fseMetadata->lastCountSize = stats.lastCountSize;
return stats.size;
}
@@ -3820,7 +3732,7 @@ ZSTD_buildBlockEntropyStats_sequences(
* Note : also employed in superblock
*/
size_t ZSTD_buildBlockEntropyStats(
const SeqStore_t* seqStorePtr,
const seqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
@@ -3878,7 +3790,7 @@ ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
/* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */
static size_t
ZSTD_estimateBlockSize_symbolType(SymbolEncodingType_e type,
ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
const FSE_CTable* fseCTable,
const U8* additionalBits,
@@ -3969,7 +3881,7 @@ ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
* @return: estimated compressed size of the seqStore, or a zstd error.
*/
static size_t
ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(SeqStore_t* seqStore, ZSTD_CCtx* zc)
ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc)
{
ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()");
@@ -3978,25 +3890,25 @@ ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(SeqStore_t* seqStore, ZSTD_CC
&zc->blockState.nextCBlock->entropy,
&zc->appliedParams,
entropyMetadata,
zc->tmpWorkspace, zc->tmpWkspSize), "");
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), "");
return ZSTD_estimateBlockSize(
seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
seqStore->ofCode, seqStore->llCode, seqStore->mlCode,
(size_t)(seqStore->sequences - seqStore->sequencesStart),
&zc->blockState.nextCBlock->entropy,
entropyMetadata,
zc->tmpWorkspace, zc->tmpWkspSize,
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
(int)(entropyMetadata->hufMetadata.hType == set_compressed), 1);
}
/* Returns literals bytes represented in a seqStore */
static size_t ZSTD_countSeqStoreLiteralsBytes(const SeqStore_t* const seqStore)
static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore)
{
size_t literalsBytes = 0;
size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
size_t i;
for (i = 0; i < nbSeqs; ++i) {
SeqDef const seq = seqStore->sequencesStart[i];
seqDef const seq = seqStore->sequencesStart[i];
literalsBytes += seq.litLength;
if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) {
literalsBytes += 0x10000;
@@ -4005,13 +3917,13 @@ static size_t ZSTD_countSeqStoreLiteralsBytes(const SeqStore_t* const seqStore)
}
/* Returns match bytes represented in a seqStore */
static size_t ZSTD_countSeqStoreMatchBytes(const SeqStore_t* const seqStore)
static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore)
{
size_t matchBytes = 0;
size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
size_t i;
for (i = 0; i < nbSeqs; ++i) {
SeqDef seq = seqStore->sequencesStart[i];
seqDef seq = seqStore->sequencesStart[i];
matchBytes += seq.mlBase + MINMATCH;
if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) {
matchBytes += 0x10000;
@@ -4022,8 +3934,8 @@ static size_t ZSTD_countSeqStoreMatchBytes(const SeqStore_t* const seqStore)
/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx).
* Stores the result in resultSeqStore.
*/
static void ZSTD_deriveSeqStoreChunk(SeqStore_t* resultSeqStore,
const SeqStore_t* originalSeqStore,
static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore,
const seqStore_t* originalSeqStore,
size_t startIdx, size_t endIdx)
{
*resultSeqStore = *originalSeqStore;
@@ -4091,13 +4003,13 @@ ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, c
* 4+ : real_offset+3
*/
static void
ZSTD_seqStore_resolveOffCodes(Repcodes_t* const dRepcodes, Repcodes_t* const cRepcodes,
const SeqStore_t* const seqStore, U32 const nbSeq)
ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
const seqStore_t* const seqStore, U32 const nbSeq)
{
U32 idx = 0;
U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengthPos : nbSeq;
for (; idx < nbSeq; ++idx) {
SeqDef* const seq = seqStore->sequencesStart + idx;
seqDef* const seq = seqStore->sequencesStart + idx;
U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx);
U32 const offBase = seq->offBase;
assert(offBase > 0);
@@ -4127,8 +4039,8 @@ ZSTD_seqStore_resolveOffCodes(Repcodes_t* const dRepcodes, Repcodes_t* const cRe
*/
static size_t
ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc,
const SeqStore_t* const seqStore,
Repcodes_t* const dRep, Repcodes_t* const cRep,
const seqStore_t* const seqStore,
repcodes_t* const dRep, repcodes_t* const cRep,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
U32 lastBlock, U32 isPartition)
@@ -4140,7 +4052,7 @@ ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc,
size_t cSeqsSize;
/* In case of an RLE or raw block, the simulated decompression repcode history must be reset */
Repcodes_t const dRepOriginal = *dRep;
repcodes_t const dRepOriginal = *dRep;
DEBUGLOG(5, "ZSTD_compressSeqStore_singleBlock");
if (isPartition)
ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart));
@@ -4151,7 +4063,7 @@ ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc,
&zc->appliedParams,
op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize,
srcSize,
zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */,
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
zc->bmi2);
FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!");
@@ -4175,18 +4087,18 @@ ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc,
if (cSeqsSize == 0) {
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock);
FORWARD_IF_ERROR(cSize, "Nocompress block failed");
DEBUGLOG(5, "Writing out nocompress block, size: %zu", cSize);
DEBUGLOG(4, "Writing out nocompress block, size: %zu", cSize);
*dRep = dRepOriginal; /* reset simulated decompression repcode history */
} else if (cSeqsSize == 1) {
cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock);
FORWARD_IF_ERROR(cSize, "RLE compress block failed");
DEBUGLOG(5, "Writing out RLE block, size: %zu", cSize);
DEBUGLOG(4, "Writing out RLE block, size: %zu", cSize);
*dRep = dRepOriginal; /* reset simulated decompression repcode history */
} else {
ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
writeBlockHeader(op, cSeqsSize, srcSize, lastBlock);
cSize = ZSTD_blockHeaderSize + cSeqsSize;
DEBUGLOG(5, "Writing out compressed block, size: %zu", cSize);
DEBUGLOG(4, "Writing out compressed block, size: %zu", cSize);
}
if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
@@ -4219,11 +4131,11 @@ typedef struct {
*/
static void
ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx,
ZSTD_CCtx* zc, const SeqStore_t* origSeqStore)
ZSTD_CCtx* zc, const seqStore_t* origSeqStore)
{
SeqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
SeqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
SeqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
size_t estimatedOriginalSize;
size_t estimatedFirstHalfSize;
size_t estimatedSecondHalfSize;
@@ -4293,8 +4205,8 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc,
size_t i = 0;
size_t srcBytesTotal = 0;
U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */
SeqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
SeqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore;
seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore;
size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
/* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history
@@ -4311,11 +4223,11 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc,
*
* See ZSTD_seqStore_resolveOffCodes() for more details.
*/
Repcodes_t dRep;
Repcodes_t cRep;
ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t));
ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t));
ZSTD_memset(nextSeqStore, 0, sizeof(SeqStore_t));
repcodes_t dRep;
repcodes_t cRep;
ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t));
DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
(unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
@@ -4330,8 +4242,8 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc,
lastBlock, 0 /* isPartition */);
FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!");
DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits");
assert(zc->blockSizeMax <= ZSTD_BLOCKSIZE_MAX);
assert(cSizeSingleBlock <= zc->blockSizeMax + ZSTD_blockHeaderSize);
assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX);
assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize);
return cSizeSingleBlock;
}
@@ -4365,12 +4277,12 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc,
dstCapacity -= cSizeChunk;
cSize += cSizeChunk;
*currSeqStore = *nextSeqStore;
assert(cSizeChunk <= zc->blockSizeMax + ZSTD_blockHeaderSize);
assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize);
}
/* cRep and dRep may have diverged during the compression.
* If so, we use the dRep repcodes for the next block.
*/
ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(Repcodes_t));
ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t));
return cSize;
}
@@ -4381,8 +4293,8 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
{
U32 nbSeq;
size_t cSize;
DEBUGLOG(5, "ZSTD_compressBlock_splitBlock");
assert(zc->appliedParams.postBlockSplitter == ZSTD_ps_enable);
DEBUGLOG(4, "ZSTD_compressBlock_splitBlock");
assert(zc->appliedParams.useBlockSplitter == ZSTD_ps_enable);
{ const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
@@ -4392,7 +4304,7 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block");
cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock);
FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
DEBUGLOG(5, "ZSTD_compressBlock_splitBlock: Nocompress block");
DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block");
return cSize;
}
nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart);
@@ -4441,7 +4353,7 @@ ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
&zc->appliedParams,
dst, dstCapacity,
srcSize,
zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */,
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
zc->bmi2);
if (frame &&
@@ -4547,7 +4459,7 @@ static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc,
return cSize;
}
static void ZSTD_overflowCorrectIfNeeded(ZSTD_MatchState_t* ms,
static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
ZSTD_cwksp* ws,
ZSTD_CCtx_params const* params,
void const* ip,
@@ -4571,40 +4483,6 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_MatchState_t* ms,
}
}
#include "zstd_preSplit.h"
static size_t ZSTD_optimalBlockSize(ZSTD_CCtx* cctx, const void* src, size_t srcSize, size_t blockSizeMax, int splitLevel, ZSTD_strategy strat, S64 savings)
{
/* split level based on compression strategy, from `fast` to `btultra2` */
static const int splitLevels[] = { 0, 0, 1, 2, 2, 3, 3, 4, 4, 4 };
/* note: conservatively only split full blocks (128 KB) currently.
* While it's possible to go lower, let's keep it simple for a first implementation.
* Besides, benefits of splitting are reduced when blocks are already small.
*/
if (srcSize < 128 KB || blockSizeMax < 128 KB)
return MIN(srcSize, blockSizeMax);
/* do not split incompressible data though:
* require verified savings to allow pre-splitting.
* Note: as a consequence, the first full block is not split.
*/
if (savings < 3) {
DEBUGLOG(6, "don't attempt splitting: savings (%i) too low", (int)savings);
return 128 KB;
}
/* apply @splitLevel, or use default value (which depends on @strat).
* note that splitting heuristic is still conditioned by @savings >= 3,
* so the first block will not reach this code path */
if (splitLevel == 1) return 128 KB;
if (splitLevel == 0) {
assert(ZSTD_fast <= strat && strat <= ZSTD_btultra2);
splitLevel = splitLevels[strat];
} else {
assert(2 <= splitLevel && splitLevel <= 6);
splitLevel -= 2;
}
return ZSTD_splitBlock(src, blockSizeMax, splitLevel, cctx->tmpWorkspace, cctx->tmpWkspSize);
}
/*! ZSTD_compress_frameChunk() :
* Compress a chunk of data into one or multiple blocks.
* All blocks will be terminated, all input will be consumed.
@@ -4617,36 +4495,29 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
const void* src, size_t srcSize,
U32 lastFrameChunk)
{
size_t blockSizeMax = cctx->blockSizeMax;
size_t blockSize = cctx->blockSize;
size_t remaining = srcSize;
const BYTE* ip = (const BYTE*)src;
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
S64 savings = (S64)cctx->consumedSrcSize - (S64)cctx->producedCSize;
assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX);
DEBUGLOG(5, "ZSTD_compress_frameChunk (srcSize=%u, blockSizeMax=%u)", (unsigned)srcSize, (unsigned)blockSizeMax);
DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize);
if (cctx->appliedParams.fParams.checksumFlag && srcSize)
XXH64_update(&cctx->xxhState, src, srcSize);
while (remaining) {
ZSTD_MatchState_t* const ms = &cctx->blockState.matchState;
size_t const blockSize = ZSTD_optimalBlockSize(cctx,
ip, remaining,
blockSizeMax,
cctx->appliedParams.preBlockSplitter_level,
cctx->appliedParams.cParams.strategy,
savings);
U32 const lastBlock = lastFrameChunk & (blockSize == remaining);
assert(blockSize <= remaining);
ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
/* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
* additional 1. We need to revisit and change this logic to be more consistent */
RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1,
dstSize_tooSmall,
"not enough space to store compressed block");
if (remaining < blockSize) blockSize = remaining;
ZSTD_overflowCorrectIfNeeded(
ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize);
@@ -4684,21 +4555,6 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
}
} /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/
/* @savings is employed to ensure that splitting doesn't worsen expansion of incompressible data.
* Without splitting, the maximum expansion is 3 bytes per full block.
* An adversarial input could attempt to fudge the split detector,
* and make it split incompressible data, resulting in more block headers.
* Note that, since ZSTD_COMPRESSBOUND() assumes a worst case scenario of 1KB per block,
* and the splitter never creates blocks that small (current lower limit is 8 KB),
* there is already no risk to expand beyond ZSTD_COMPRESSBOUND() limit.
* But if the goal is to not expand by more than 3-bytes per 128 KB full block,
* then yes, it becomes possible to make the block splitter oversplit incompressible data.
* Using @savings, we enforce an even more conservative condition,
* requiring the presence of enough savings (at least 3 bytes) to authorize splitting,
* otherwise only full blocks are used.
* But being conservative is fine,
* since splitting barely compressible blocks is not fruitful anyway */
savings += (S64)blockSize - (S64)cSize;
ip += blockSize;
assert(remaining >= blockSize);
@@ -4717,10 +4573,8 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
const ZSTD_CCtx_params* params,
U64 pledgedSrcSize, U32 dictID)
{
BYTE* const op = (BYTE*)dst;
const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID)
{ BYTE* const op = (BYTE*)dst;
U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */
U32 const checksumFlag = params->fParams.checksumFlag>0;
@@ -4818,7 +4672,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
const void* src, size_t srcSize,
U32 frame, U32 lastFrameChunk)
{
ZSTD_MatchState_t* const ms = &cctx->blockState.matchState;
ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
size_t fhSize = 0;
DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u",
@@ -4853,7 +4707,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
src, (BYTE const*)src + srcSize);
}
DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSizeMax);
DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize);
{ size_t const cSize = frame ?
ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */);
@@ -4922,8 +4776,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const
/*! ZSTD_loadDictionaryContent() :
* @return : 0, or an error code
*/
static size_t
ZSTD_loadDictionaryContent(ZSTD_MatchState_t* ms,
static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
ldmState_t* ls,
ZSTD_cwksp* ws,
ZSTD_CCtx_params const* params,
@@ -4973,18 +4826,17 @@ ZSTD_loadDictionaryContent(ZSTD_MatchState_t* ms,
}
ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0);
DEBUGLOG(4, "ZSTD_loadDictionaryContent: useRowMatchFinder=%d", (int)params->useRowMatchFinder);
DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder);
if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */
DEBUGLOG(4, "ZSTD_loadDictionaryContent: Trigger loadLdmDict");
ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0);
ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base);
ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
DEBUGLOG(4, "ZSTD_loadDictionaryContent: ZSTD_ldm_fillHashTable completes");
}
/* If the dict is larger than we can reasonably index in our tables, only load the suffix. */
{ U32 maxDictSize = 1U << MIN(MAX(params->cParams.hashLog + 3, params->cParams.chainLog + 1), 31);
if (params->cParams.strategy < ZSTD_btultra) {
U32 maxDictSize = 8U << MIN(MAX(params->cParams.hashLog, params->cParams.chainLog), 28);
if (srcSize > maxDictSize) {
ip = iend - maxDictSize;
src = ip;
@@ -5048,7 +4900,6 @@ ZSTD_loadDictionaryContent(ZSTD_MatchState_t* ms,
|| !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR)
assert(srcSize >= HASH_READ_SIZE);
DEBUGLOG(4, "Fill %u bytes into the Binary Tree", (unsigned)srcSize);
ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
#else
assert(0); /* shouldn't be called: cparams should've been adjusted. */
@@ -5095,7 +4946,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
{ unsigned maxSymbolValue = 255;
unsigned hasZeroWeights = 1;
size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr,
(size_t)(dictEnd-dictPtr), &hasZeroWeights);
dictEnd-dictPtr, &hasZeroWeights);
/* We only set the loaded table as valid if it contains all non-zero
* weights. Otherwise, we set it to check */
@@ -5107,7 +4958,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
}
{ unsigned offcodeLog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr));
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
/* fill all offset symbols to avoid garbage at end of table */
@@ -5122,7 +4973,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
{ short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
@@ -5136,7 +4987,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
{ short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp(
@@ -5170,7 +5021,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, "");
} } }
return (size_t)(dictPtr - (const BYTE*)dict);
return dictPtr - (const BYTE*)dict;
}
/* Dictionary format :
@@ -5183,7 +5034,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
* dictSize supposed >= 8
*/
static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
ZSTD_MatchState_t* ms,
ZSTD_matchState_t* ms,
ZSTD_cwksp* ws,
ZSTD_CCtx_params const* params,
const void* dict, size_t dictSize,
@@ -5216,7 +5067,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
* @return : dictID, or an error code */
static size_t
ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
ZSTD_MatchState_t* ms,
ZSTD_matchState_t* ms,
ldmState_t* ls,
ZSTD_cwksp* ws,
const ZSTD_CCtx_params* params,
@@ -5293,11 +5144,11 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
cctx->blockState.prevCBlock, &cctx->blockState.matchState,
&cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent,
cdict->dictContentSize, cdict->dictContentType, dtlm,
ZSTD_tfp_forCCtx, cctx->tmpWorkspace)
ZSTD_tfp_forCCtx, cctx->entropyWorkspace)
: ZSTD_compress_insertDictionary(
cctx->blockState.prevCBlock, &cctx->blockState.matchState,
&cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize,
dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->tmpWorkspace);
dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->entropyWorkspace);
FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
assert(dictID <= UINT_MAX);
cctx->dictID = (U32)dictID;
@@ -5401,7 +5252,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
}
cctx->stage = ZSTDcs_created; /* return to "created but no init" status */
return (size_t)(op-ostart);
return op-ostart;
}
void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize)
@@ -5625,16 +5476,14 @@ static size_t ZSTD_initCDict_internal(
return 0;
}
static ZSTD_CDict*
ZSTD_createCDict_advanced_internal(size_t dictSize,
static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_compressionParameters cParams,
ZSTD_ParamSwitch_e useRowMatchFinder,
int enableDedicatedDictSearch,
ZSTD_paramSwitch_e useRowMatchFinder,
U32 enableDedicatedDictSearch,
ZSTD_customMem customMem)
{
if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
DEBUGLOG(3, "ZSTD_createCDict_advanced_internal (dictSize=%u)", (unsigned)dictSize);
{ size_t const workspaceSize =
ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) +
@@ -5671,7 +5520,6 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
{
ZSTD_CCtx_params cctxParams;
ZSTD_memset(&cctxParams, 0, sizeof(cctxParams));
DEBUGLOG(3, "ZSTD_createCDict_advanced, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType);
ZSTD_CCtxParams_init(&cctxParams, 0);
cctxParams.cParams = cParams;
cctxParams.customMem = customMem;
@@ -5692,7 +5540,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced2(
ZSTD_compressionParameters cParams;
ZSTD_CDict* cdict;
DEBUGLOG(3, "ZSTD_createCDict_advanced2, dictSize=%u, mode=%u", (unsigned)dictSize, (unsigned)dictContentType);
DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
if (cctxParams.enableDedicatedDictSearch) {
@@ -5711,7 +5559,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced2(
&cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
}
DEBUGLOG(3, "ZSTD_createCDict_advanced2: DedicatedDictSearch=%u", cctxParams.enableDedicatedDictSearch);
DEBUGLOG(3, "ZSTD_createCDict_advanced2: DDS: %u", cctxParams.enableDedicatedDictSearch);
cctxParams.cParams = cParams;
cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
@@ -5774,7 +5622,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
* workspaceSize: Use ZSTD_estimateCDictSize()
* to determine how large workspace must be.
* cParams : use ZSTD_getCParams() to transform a compression level
* into its relevant cParams.
* into its relevants cParams.
* @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
* Note : there is no corresponding "free" function.
* Since workspace was allocated externally, it must be freed externally.
@@ -5786,7 +5634,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams)
{
ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams);
ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams);
/* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0);
size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
@@ -5797,7 +5645,6 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
ZSTD_CDict* cdict;
ZSTD_CCtx_params params;
DEBUGLOG(4, "ZSTD_initStaticCDict (dictSize==%u)", (unsigned)dictSize);
if ((size_t)workspace & 7) return NULL; /* 8-aligned */
{
@@ -5808,6 +5655,8 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
ZSTD_cwksp_move(&cdict->workspace, &ws);
}
DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u",
(unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize));
if (workspaceSize < neededSize) return NULL;
ZSTD_CCtxParams_init(&params, 0);
@@ -5980,7 +5829,7 @@ size_t ZSTD_CStreamOutSize(void)
return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ;
}
static ZSTD_CParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize)
static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize)
{
if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize))
return ZSTD_cpm_attachDict;
@@ -6112,11 +5961,11 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx)
{
if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
return cctx->blockSizeMax - cctx->stableIn_notConsumed;
return cctx->blockSize - cctx->stableIn_notConsumed;
}
assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered);
{ size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos;
if (hintInSize==0) hintInSize = cctx->blockSizeMax;
if (hintInSize==0) hintInSize = cctx->blockSize;
return hintInSize;
}
}
@@ -6168,13 +6017,12 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
case zcss_load:
if ( (flushMode == ZSTD_e_end)
&& ( (size_t)(oend-op) >= ZSTD_compressBound((size_t)(iend-ip)) /* Enough output space */
&& ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */
|| zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */
&& (zcs->inBuffPos == 0) ) {
/* shortcut to compression pass directly into output buffer */
size_t const cSize = ZSTD_compressEnd_public(zcs,
op, (size_t)(oend-op),
ip, (size_t)(iend-ip));
op, oend-op, ip, iend-ip);
DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize);
FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed");
ip = iend;
@@ -6188,7 +6036,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
size_t const loaded = ZSTD_limitCopy(
zcs->inBuff + zcs->inBuffPos, toLoad,
ip, (size_t)(iend-ip));
ip, iend-ip);
zcs->inBuffPos += loaded;
if (ip) ip += loaded;
if ( (flushMode == ZSTD_e_continue)
@@ -6204,7 +6052,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
} else {
assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable);
if ( (flushMode == ZSTD_e_continue)
&& ( (size_t)(iend - ip) < zcs->blockSizeMax) ) {
&& ( (size_t)(iend - ip) < zcs->blockSize) ) {
/* can't compress a full block : stop here */
zcs->stableIn_notConsumed = (size_t)(iend - ip);
ip = iend; /* pretend to have consumed input */
@@ -6221,9 +6069,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
{ int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered);
void* cDst;
size_t cSize;
size_t oSize = (size_t)(oend-op);
size_t oSize = oend-op;
size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress
: MIN((size_t)(iend - ip), zcs->blockSizeMax);
: MIN((size_t)(iend - ip), zcs->blockSize);
if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable)
cDst = op; /* compress into output buffer, to skip flush stage */
else
@@ -6238,9 +6086,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
zcs->frameEnded = lastBlock;
/* prepare next block */
zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSizeMax;
zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
if (zcs->inBuffTarget > zcs->inBuffSize)
zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSizeMax;
zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize;
DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u",
(unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize);
if (!lastBlock)
@@ -6304,8 +6152,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
}
}
input->pos = (size_t)(ip - istart);
output->pos = (size_t)(op - ostart);
input->pos = ip - istart;
output->pos = op - ostart;
if (zcs->frameEnded) return 0;
return ZSTD_nextInputSizeHint(zcs);
}
@@ -6365,11 +6213,6 @@ static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx,
return 0;
}
/*
* If @endOp == ZSTD_e_end, @inSize becomes pledgedSrcSize.
* Otherwise, it's ignored.
* @return: 0 on success, or a ZSTD_error code otherwise.
*/
static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
ZSTD_EndDirective endOp,
size_t inSize)
@@ -6386,19 +6229,19 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
*/
params.compressionLevel = cctx->cdict->compressionLevel;
}
DEBUGLOG(4, "ZSTD_CCtx_init_compressStream2 : transparent init stage");
DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSize */
{ size_t const dictSize = prefixDict.dict
? prefixDict.dictSize
: (cctx->cdict ? cctx->cdict->dictContentSize : 0);
ZSTD_CParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, &params, cctx->pledgedSrcSizePlusOne - 1);
ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, &params, cctx->pledgedSrcSizePlusOne - 1);
params.cParams = ZSTD_getCParamsFromCCtxParams(
&params, cctx->pledgedSrcSizePlusOne-1,
dictSize, mode);
}
params.postBlockSplitter = ZSTD_resolveBlockSplitterMode(params.postBlockSplitter, &params.cParams);
params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, &params.cParams);
params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, &params.cParams);
params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
@@ -6417,9 +6260,9 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
}
if (params.nbWorkers > 0) {
# if ZSTD_TRACE
#if ZSTD_TRACE
cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0;
# endif
#endif
/* mt context creation */
if (cctx->mtctx == NULL) {
DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u",
@@ -6455,7 +6298,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
/* for small input: avoid automatic flush on reaching end of block, since
* it would require to add a 3-bytes null block to end frame
*/
cctx->inBuffTarget = cctx->blockSizeMax + (cctx->blockSizeMax == pledgedSrcSize);
cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize);
} else {
cctx->inBuffTarget = 0;
}
@@ -6621,11 +6464,11 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
}
/* ZSTD_validateSequence() :
* @offBase : must use the format required by ZSTD_storeSeq()
* @offCode : is presumed to follow format required by ZSTD_storeSeq()
* @returns a ZSTD error code if sequence is not valid
*/
static size_t
ZSTD_validateSequence(U32 offBase, U32 matchLength, U32 minMatch,
ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch,
size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer)
{
U32 const windowSize = 1u << windowLog;
@@ -6636,7 +6479,7 @@ ZSTD_validateSequence(U32 offBase, U32 matchLength, U32 minMatch,
*/
size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize;
size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4;
RETURN_ERROR_IF(offBase > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!");
RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!");
/* Validate maxNbSeq is large enough for the given matchLength and minMatch */
RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch");
return 0;
@@ -6659,27 +6502,21 @@ static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32
return offBase;
}
/* This function scans through an array of ZSTD_Sequence,
* storing the sequences it reads, until it reaches a block delimiter.
* Note that the block delimiter includes the last literals of the block.
* @blockSize must be == sum(sequence_lengths).
* @returns @blockSize on success, and a ZSTD_error otherwise.
*/
static size_t
ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
ZSTD_SequencePosition* seqPos,
size_t
ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize,
ZSTD_ParamSwitch_e externalRepSearch)
ZSTD_paramSwitch_e externalRepSearch)
{
U32 idx = seqPos->idx;
U32 const startIdx = idx;
BYTE const* ip = (BYTE const*)(src);
const BYTE* const iend = ip + blockSize;
Repcodes_t updatedRepcodes;
repcodes_t updatedRepcodes;
U32 dictSize;
DEBUGLOG(5, "ZSTD_transferSequences_wBlockDelim (blockSize = %zu)", blockSize);
DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreExplicitBlockDelim (blockSize = %zu)", blockSize);
if (cctx->cdict) {
dictSize = (U32)cctx->cdict->dictContentSize;
@@ -6688,7 +6525,7 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
} else {
dictSize = 0;
}
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t));
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) {
U32 const litLength = inSeqs[idx].litLength;
U32 const matchLength = inSeqs[idx].matchLength;
@@ -6705,10 +6542,8 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
if (cctx->appliedParams.validateSequences) {
seqPos->posInSrc += litLength + matchLength;
FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch,
seqPos->posInSrc,
cctx->appliedParams.cParams.windowLog, dictSize,
ZSTD_hasExtSeqProd(&cctx->appliedParams)),
FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
cctx->appliedParams.cParams.windowLog, dictSize, ZSTD_hasExtSeqProd(&cctx->appliedParams)),
"Sequence validation failed");
}
RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
@@ -6716,7 +6551,6 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
ip += matchLength + litLength;
}
RETURN_ERROR_IF(idx == inSeqsSize, externalSequences_invalid, "Block delimiter not found.");
/* If we skipped repcode search while parsing, we need to update repcodes now */
assert(externalRepSearch != ZSTD_ps_auto);
@@ -6741,7 +6575,7 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
}
}
ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t));
ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t));
if (inSeqs[idx].litLength) {
DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength);
@@ -6751,35 +6585,21 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
}
RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!");
seqPos->idx = idx+1;
return blockSize;
return 0;
}
/*
* This function attempts to scan through @blockSize bytes in @src
* represented by the sequences in @inSeqs,
* storing any (partial) sequences.
*
* Occasionally, we may want to reduce the actual number of bytes consumed from @src
* to avoid splitting a match, notably if it would produce a match smaller than MINMATCH.
*
* @returns the number of bytes consumed from @src, necessarily <= @blockSize.
* Otherwise, it may return a ZSTD error if something went wrong.
*/
static size_t
ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx,
ZSTD_SequencePosition* seqPos,
size_t
ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize,
ZSTD_ParamSwitch_e externalRepSearch)
const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch)
{
U32 idx = seqPos->idx;
U32 startPosInSequence = seqPos->posInSequence;
U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize;
size_t dictSize;
const BYTE* const istart = (const BYTE*)(src);
const BYTE* ip = istart;
const BYTE* iend = istart + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */
Repcodes_t updatedRepcodes;
BYTE const* ip = (BYTE const*)(src);
BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */
repcodes_t updatedRepcodes;
U32 bytesAdjustment = 0;
U32 finalMatchSplit = 0;
@@ -6793,9 +6613,9 @@ ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx,
} else {
dictSize = 0;
}
DEBUGLOG(5, "ZSTD_transferSequences_noDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize);
DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreNoBlockDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize);
DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t));
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) {
const ZSTD_Sequence currSeq = inSeqs[idx];
U32 litLength = currSeq.litLength;
@@ -6876,40 +6696,35 @@ ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx,
assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength);
seqPos->idx = idx;
seqPos->posInSequence = endPosInSequence;
ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t));
ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t));
iend -= bytesAdjustment;
if (ip != iend) {
/* Store any last literals */
U32 const lastLLSize = (U32)(iend - ip);
U32 lastLLSize = (U32)(iend - ip);
assert(ip <= iend);
DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize);
ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize);
seqPos->posInSrc += lastLLSize;
}
return (size_t)(iend-istart);
return bytesAdjustment;
}
/* @seqPos represents a position within @inSeqs,
* it is read and updated by this function,
* once the goal to produce a block of size @blockSize is reached.
* @return: nb of bytes consumed from @src, necessarily <= @blockSize.
*/
typedef size_t (*ZSTD_SequenceCopier_f)(ZSTD_CCtx* cctx,
ZSTD_SequencePosition* seqPos,
typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize,
ZSTD_ParamSwitch_e externalRepSearch);
static ZSTD_SequenceCopier_f ZSTD_selectSequenceCopier(ZSTD_SequenceFormat_e mode)
const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode)
{
assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, (int)mode));
ZSTD_sequenceCopier sequenceCopier = NULL;
assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode));
if (mode == ZSTD_sf_explicitBlockDelimiters) {
return ZSTD_transferSequences_wBlockDelim;
return ZSTD_copySequencesToSeqStoreExplicitBlockDelim;
} else if (mode == ZSTD_sf_noBlockDelimiters) {
return ZSTD_copySequencesToSeqStoreNoBlockDelim;
}
assert(mode == ZSTD_sf_noBlockDelimiters);
return ZSTD_transferSequences_noDelim;
assert(sequenceCopier != NULL);
return sequenceCopier;
}
/* Discover the size of next block by searching for the delimiter.
@@ -6917,7 +6732,7 @@ static ZSTD_SequenceCopier_f ZSTD_selectSequenceCopier(ZSTD_SequenceFormat_e mod
* otherwise it's an input error.
* The block size retrieved will be later compared to ensure it remains within bounds */
static size_t
blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_SequencePosition seqPos)
blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos)
{
int end = 0;
size_t blockSize = 0;
@@ -6939,17 +6754,20 @@ blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD
return blockSize;
}
static size_t determine_blockSize(ZSTD_SequenceFormat_e mode,
/* More a "target" block size */
static size_t blockSize_noDelimiter(size_t blockSize, size_t remaining)
{
int const lastBlock = (remaining <= blockSize);
return lastBlock ? remaining : blockSize;
}
static size_t determine_blockSize(ZSTD_sequenceFormat_e mode,
size_t blockSize, size_t remaining,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
ZSTD_SequencePosition seqPos)
const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos)
{
DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining);
if (mode == ZSTD_sf_noBlockDelimiters) {
/* Note: more a "target" block size */
return MIN(remaining, blockSize);
}
assert(mode == ZSTD_sf_explicitBlockDelimiters);
if (mode == ZSTD_sf_noBlockDelimiters)
return blockSize_noDelimiter(blockSize, remaining);
{ size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos);
FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters");
if (explicitBlockSize > blockSize)
@@ -6960,7 +6778,7 @@ static size_t determine_blockSize(ZSTD_SequenceFormat_e mode,
}
}
/* Compress all provided sequences, block-by-block.
/* Compress, block-by-block, all of the sequences given.
*
* Returns the cumulative size of all compressed blocks (including their headers),
* otherwise a ZSTD error.
@@ -6973,11 +6791,11 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
{
size_t cSize = 0;
size_t remaining = srcSize;
ZSTD_SequencePosition seqPos = {0, 0, 0};
ZSTD_sequencePosition seqPos = {0, 0, 0};
const BYTE* ip = (BYTE const*)src;
BYTE const* ip = (BYTE const*)src;
BYTE* op = (BYTE*)dst;
ZSTD_SequenceCopier_f const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters);
ZSTD_sequenceCopier const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters);
DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize);
/* Special case: empty frame */
@@ -6993,19 +6811,19 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
while (remaining) {
size_t compressedSeqsSize;
size_t cBlockSize;
size_t additionalByteAdjustment;
size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters,
cctx->blockSizeMax, remaining,
cctx->blockSize, remaining,
inSeqs, inSeqsSize, seqPos);
U32 const lastBlock = (blockSize == remaining);
FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size");
assert(blockSize <= remaining);
ZSTD_resetSeqStore(&cctx->seqStore);
DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize);
blockSize = sequenceCopier(cctx,
&seqPos, inSeqs, inSeqsSize,
ip, blockSize,
cctx->appliedParams.searchForExternalRepcodes);
FORWARD_IF_ERROR(blockSize, "Bad sequence copy");
additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes);
FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy");
blockSize -= additionalByteAdjustment;
/* If blocks are too small, emit as a nocompress block */
/* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
@@ -7013,7 +6831,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) {
cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed");
DEBUGLOG(5, "Block too small (%zu): data remains uncompressed: cSize=%zu", blockSize, cBlockSize);
DEBUGLOG(5, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize);
cSize += cBlockSize;
ip += blockSize;
op += cBlockSize;
@@ -7028,7 +6846,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
&cctx->appliedParams,
op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize,
blockSize,
cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */,
cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
cctx->bmi2);
FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed");
DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize);
@@ -7036,9 +6854,9 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
if (!cctx->isFirstBlock &&
ZSTD_maybeRLE(&cctx->seqStore) &&
ZSTD_isRLE(ip, blockSize)) {
/* Note: don't emit the first block as RLE even if it qualifies because
* doing so will cause the decoder (cli <= v1.4.3 only) to throw an (invalid) error
* "should consume all input error."
/* We don't want to emit our first block as a RLE even if it qualifies because
* doing so will cause the decoder (cli only) to throw a "should consume all input error."
* This is only an issue for zstd <= v1.4.3
*/
compressedSeqsSize = 1;
}
@@ -7091,36 +6909,30 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
{
BYTE* op = (BYTE*)dst;
size_t cSize = 0;
size_t compressedBlocksSize = 0;
size_t frameHeaderSize = 0;
/* Transparent initialization stage, same as compressStream2() */
DEBUGLOG(4, "ZSTD_compressSequences (nbSeqs=%zu,dstCapacity=%zu)", inSeqsSize, dstCapacity);
DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity);
assert(cctx != NULL);
FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed");
/* Begin writing output, starting with frame header */
{ size_t const frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity,
&cctx->appliedParams, srcSize, cctx->dictID);
frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID);
op += frameHeaderSize;
assert(frameHeaderSize <= dstCapacity);
dstCapacity -= frameHeaderSize;
cSize += frameHeaderSize;
}
if (cctx->appliedParams.fParams.checksumFlag && srcSize) {
XXH64_update(&cctx->xxhState, src, srcSize);
}
/* Now generate compressed blocks */
{ size_t const cBlocksSize = ZSTD_compressSequences_internal(cctx,
/* cSize includes block header size and compressed sequences size */
compressedBlocksSize = ZSTD_compressSequences_internal(cctx,
op, dstCapacity,
inSeqs, inSeqsSize,
src, srcSize);
FORWARD_IF_ERROR(cBlocksSize, "Compressing blocks failed!");
cSize += cBlocksSize;
assert(cBlocksSize <= dstCapacity);
dstCapacity -= cBlocksSize;
}
FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!");
cSize += compressedBlocksSize;
dstCapacity -= compressedBlocksSize;
/* Complete with frame checksum, if needed */
if (cctx->appliedParams.fParams.checksumFlag) {
U32 const checksum = (U32) XXH64_digest(&cctx->xxhState);
RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum");
@@ -7133,1033 +6945,6 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
return cSize;
}
#if defined(ZSTD_ARCH_X86_AVX2)
#include <immintrin.h> /* AVX2 intrinsics */
/*
* Convert 2 sequences per iteration, using AVX2 intrinsics:
* - offset -> offBase = offset + 2
* - litLength -> (U16) litLength
* - matchLength -> (U16)(matchLength - 3)
* - rep is ignored
* Store only 8 bytes per SeqDef (offBase[4], litLength[2], mlBase[2]).
*
* At the end, instead of extracting two __m128i,
* we use _mm256_permute4x64_epi64(..., 0xE8) to move lane2 into lane1,
* then store the lower 16 bytes in one go.
*
* @returns 0 on succes, with no long length detected
* @returns > 0 if there is one long length (> 65535),
* indicating the position, and type.
*/
size_t convertSequences_noRepcodes(
SeqDef* dstSeqs,
const ZSTD_Sequence* inSeqs,
size_t nbSequences)
{
/*
* addition:
* For each 128-bit half: (offset+2, litLength+0, matchLength-3, rep+0)
*/
const __m256i addition = _mm256_setr_epi32(
ZSTD_REP_NUM, 0, -MINMATCH, 0, /* for sequence i */
ZSTD_REP_NUM, 0, -MINMATCH, 0 /* for sequence i+1 */
);
/* limit: check if there is a long length */
const __m256i limit = _mm256_set1_epi32(65535);
/*
* shuffle mask for byte-level rearrangement in each 128-bit half:
*
* Input layout (after addition) per 128-bit half:
* [ offset+2 (4 bytes) | litLength (4 bytes) | matchLength (4 bytes) | rep (4 bytes) ]
* We only need:
* offBase (4 bytes) = offset+2
* litLength (2 bytes) = low 2 bytes of litLength
* mlBase (2 bytes) = low 2 bytes of (matchLength)
* => Bytes [0..3, 4..5, 8..9], zero the rest.
*/
const __m256i mask = _mm256_setr_epi8(
/* For the lower 128 bits => sequence i */
0, 1, 2, 3, /* offset+2 */
4, 5, /* litLength (16 bits) */
8, 9, /* matchLength (16 bits) */
(BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80,
(BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80,
/* For the upper 128 bits => sequence i+1 */
16,17,18,19, /* offset+2 */
20,21, /* litLength */
24,25, /* matchLength */
(BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80,
(BYTE)0x80, (BYTE)0x80, (BYTE)0x80, (BYTE)0x80
);
/*
* Next, we'll use _mm256_permute4x64_epi64(vshf, 0xE8).
* Explanation of 0xE8 = 11101000b => [lane0, lane2, lane2, lane3].
* So the lower 128 bits become [lane0, lane2] => combining seq0 and seq1.
*/
#define PERM_LANE_0X_E8 0xE8 /* [0,2,2,3] in lane indices */
size_t longLen = 0, i = 0;
/* AVX permutation depends on the specific definition of target structures */
ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, offset) == 0);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, litLength) == 4);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8);
ZSTD_STATIC_ASSERT(sizeof(SeqDef) == 8);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, offBase) == 0);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, litLength) == 4);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, mlBase) == 6);
/* Process 2 sequences per loop iteration */
for (; i + 1 < nbSequences; i += 2) {
/* Load 2 ZSTD_Sequence (32 bytes) */
__m256i vin = _mm256_loadu_si256((const __m256i*)(const void*)&inSeqs[i]);
/* Add {2, 0, -3, 0} in each 128-bit half */
__m256i vadd = _mm256_add_epi32(vin, addition);
/* Check for long length */
__m256i ll_cmp = _mm256_cmpgt_epi32(vadd, limit); /* 0xFFFFFFFF for element > 65535 */
int ll_res = _mm256_movemask_epi8(ll_cmp);
/* Shuffle bytes so each half gives us the 8 bytes we need */
__m256i vshf = _mm256_shuffle_epi8(vadd, mask);
/*
* Now:
* Lane0 = seq0's 8 bytes
* Lane1 = 0
* Lane2 = seq1's 8 bytes
* Lane3 = 0
*/
/* Permute 64-bit lanes => move Lane2 down into Lane1. */
__m256i vperm = _mm256_permute4x64_epi64(vshf, PERM_LANE_0X_E8);
/*
* Now the lower 16 bytes (Lane0+Lane1) = [seq0, seq1].
* The upper 16 bytes are [Lane2, Lane3] = [seq1, 0], but we won't use them.
*/
/* Store only the lower 16 bytes => 2 SeqDef (8 bytes each) */
_mm_storeu_si128((__m128i *)(void*)&dstSeqs[i], _mm256_castsi256_si128(vperm));
/*
* This writes out 16 bytes total:
* - offset 0..7 => seq0 (offBase, litLength, mlBase)
* - offset 8..15 => seq1 (offBase, litLength, mlBase)
*/
/* check (unlikely) long lengths > 65535
* indices for lengths correspond to bits [4..7], [8..11], [20..23], [24..27]
* => combined mask = 0x0FF00FF0
*/
if (UNLIKELY((ll_res & 0x0FF00FF0) != 0)) {
/* long length detected: let's figure out which one*/
if (inSeqs[i].matchLength > 65535+MINMATCH) {
assert(longLen == 0);
longLen = i + 1;
}
if (inSeqs[i].litLength > 65535) {
assert(longLen == 0);
longLen = i + nbSequences + 1;
}
if (inSeqs[i+1].matchLength > 65535+MINMATCH) {
assert(longLen == 0);
longLen = i + 1 + 1;
}
if (inSeqs[i+1].litLength > 65535) {
assert(longLen == 0);
longLen = i + 1 + nbSequences + 1;
}
}
}
/* Handle leftover if @nbSequences is odd */
if (i < nbSequences) {
/* process last sequence */
assert(i == nbSequences - 1);
dstSeqs[i].offBase = OFFSET_TO_OFFBASE(inSeqs[i].offset);
dstSeqs[i].litLength = (U16)inSeqs[i].litLength;
dstSeqs[i].mlBase = (U16)(inSeqs[i].matchLength - MINMATCH);
/* check (unlikely) long lengths > 65535 */
if (UNLIKELY(inSeqs[i].matchLength > 65535+MINMATCH)) {
assert(longLen == 0);
longLen = i + 1;
}
if (UNLIKELY(inSeqs[i].litLength > 65535)) {
assert(longLen == 0);
longLen = i + nbSequences + 1;
}
}
return longLen;
}
#elif defined (ZSTD_ARCH_RISCV_RVV)
#include <riscv_vector.h>
/*
* Convert `vl` sequences per iteration, using RVV intrinsics:
* - offset -> offBase = offset + 2
* - litLength -> (U16) litLength
* - matchLength -> (U16)(matchLength - 3)
* - rep is ignored
* Store only 8 bytes per SeqDef (offBase[4], litLength[2], mlBase[2]).
*
* @returns 0 on succes, with no long length detected
* @returns > 0 if there is one long length (> 65535),
* indicating the position, and type.
*/
size_t convertSequences_noRepcodes(SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs, size_t nbSequences) {
size_t longLen = 0;
size_t vl = 0;
typedef uint32_t __attribute__((may_alias)) aliased_u32;
/* RVV depends on the specific definition of target structures */
ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, offset) == 0);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, litLength) == 4);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8);
ZSTD_STATIC_ASSERT(sizeof(SeqDef) == 8);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, offBase) == 0);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, litLength) == 4);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, mlBase) == 6);
for (size_t i = 0; i < nbSequences; i += vl) {
vl = __riscv_vsetvl_e32m2(nbSequences-i);
{
// Loading structure member variables
vuint32m2x4_t v_tuple = __riscv_vlseg4e32_v_u32m2x4(
(const aliased_u32*)((const void*)&inSeqs[i]),
vl
);
vuint32m2_t v_offset = __riscv_vget_v_u32m2x4_u32m2(v_tuple, 0);
vuint32m2_t v_lit = __riscv_vget_v_u32m2x4_u32m2(v_tuple, 1);
vuint32m2_t v_match = __riscv_vget_v_u32m2x4_u32m2(v_tuple, 2);
// offset + ZSTD_REP_NUM
vuint32m2_t v_offBase = __riscv_vadd_vx_u32m2(v_offset, ZSTD_REP_NUM, vl);
// Check for integer overflow
// Cast to a 16-bit variable
vbool16_t lit_overflow = __riscv_vmsgtu_vx_u32m2_b16(v_lit, 65535, vl);
vuint16m1_t v_lit_clamped = __riscv_vncvt_x_x_w_u16m1(v_lit, vl);
vbool16_t ml_overflow = __riscv_vmsgtu_vx_u32m2_b16(v_match, 65535+MINMATCH, vl);
vuint16m1_t v_ml_clamped = __riscv_vncvt_x_x_w_u16m1(__riscv_vsub_vx_u32m2(v_match, MINMATCH, vl), vl);
// Pack two 16-bit fields into a 32-bit value (little-endian)
// The lower 16 bits contain litLength, and the upper 16 bits contain mlBase
vuint32m2_t v_lit_ml_combined = __riscv_vsll_vx_u32m2(
__riscv_vwcvtu_x_x_v_u32m2(v_ml_clamped, vl), // Convert matchLength to 32-bit
16,
vl
);
v_lit_ml_combined = __riscv_vor_vv_u32m2(
v_lit_ml_combined,
__riscv_vwcvtu_x_x_v_u32m2(v_lit_clamped, vl),
vl
);
{
// Create a vector of SeqDef structures
// Store the offBase, litLength, and mlBase in a vector of SeqDef
vuint32m2x2_t store_data = __riscv_vcreate_v_u32m2x2(
v_offBase,
v_lit_ml_combined
);
__riscv_vsseg2e32_v_u32m2x2(
(aliased_u32*)((void*)&dstSeqs[i]),
store_data,
vl
);
}
{
// Find the first index where an overflow occurs
int first_ml = __riscv_vfirst_m_b16(ml_overflow, vl);
int first_lit = __riscv_vfirst_m_b16(lit_overflow, vl);
if (UNLIKELY(first_ml != -1)) {
assert(longLen == 0);
longLen = i + first_ml + 1;
}
if (UNLIKELY(first_lit != -1)) {
assert(longLen == 0);
longLen = i + first_lit + 1 + nbSequences;
}
}
}
}
return longLen;
}
/* the vector implementation could also be ported to SSSE3,
* but since this implementation is targeting modern systems (>= Sapphire Rapid),
* it's not useful to develop and maintain code for older pre-AVX2 platforms */
#elif defined(ZSTD_ARCH_ARM_SVE2)
/*
* Checks if any active element in a signed 8-bit integer vector is greater
* than zero.
*
* @param g Governing predicate selecting active lanes.
* @param a Input vector of signed 8-bit integers.
*
* @return True if any active element in `a` is > 0, false otherwise.
*/
FORCE_INLINE_TEMPLATE int cmpgtz_any_s8(svbool_t g, svint8_t a)
{
svbool_t ptest = svcmpgt_n_s8(g, a, 0);
return svptest_any(ptest, ptest);
}
size_t convertSequences_noRepcodes(
SeqDef* dstSeqs,
const ZSTD_Sequence* inSeqs,
size_t nbSequences)
{
/* Process the input with `8 * VL / element` lanes. */
const size_t lanes = 8 * svcntb() / sizeof(ZSTD_Sequence);
size_t longLen = 0;
size_t n = 0;
/* SVE permutation depends on the specific definition of target structures. */
ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, offset) == 0);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, litLength) == 4);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8);
ZSTD_STATIC_ASSERT(sizeof(SeqDef) == 8);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, offBase) == 0);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, litLength) == 4);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, mlBase) == 6);
if (nbSequences >= lanes) {
const svbool_t ptrue = svptrue_b8();
/* 16-bit of {ZSTD_REP_NUM, 0, -MINMATCH, 0} extended to 32-bit lanes. */
const svuint32_t vaddition = svreinterpret_u32(
svunpklo_s32(svreinterpret_s16(svdup_n_u64(ZSTD_REP_NUM | (((U64)(U16)-MINMATCH) << 32)))));
/* For permutation of 16-bit units: 0, 1, 2, 4, 8, 9, 10, 12, ... */
const svuint16_t vmask = svreinterpret_u16(
svindex_u64(0x0004000200010000, 0x0008000800080008));
/* Upper bytes of `litLength` and `matchLength` will be packed into the
* middle of overflow check vector. */
const svbool_t pmid = svcmpne_n_u8(
ptrue, svreinterpret_u8(svdup_n_u64(0x0000FFFFFFFF0000)), 0);
do {
/* Load `lanes` number of `ZSTD_Sequence` into 8 vectors. */
const svuint32_t vin0 = svld1_vnum_u32(ptrue, &inSeqs[n].offset, 0);
const svuint32_t vin1 = svld1_vnum_u32(ptrue, &inSeqs[n].offset, 1);
const svuint32_t vin2 = svld1_vnum_u32(ptrue, &inSeqs[n].offset, 2);
const svuint32_t vin3 = svld1_vnum_u32(ptrue, &inSeqs[n].offset, 3);
const svuint32_t vin4 = svld1_vnum_u32(ptrue, &inSeqs[n].offset, 4);
const svuint32_t vin5 = svld1_vnum_u32(ptrue, &inSeqs[n].offset, 5);
const svuint32_t vin6 = svld1_vnum_u32(ptrue, &inSeqs[n].offset, 6);
const svuint32_t vin7 = svld1_vnum_u32(ptrue, &inSeqs[n].offset, 7);
/* Add {ZSTD_REP_NUM, 0, -MINMATCH, 0} to each structures. */
const svuint16x2_t vadd01 = svcreate2_u16(
svreinterpret_u16(svadd_u32_x(ptrue, vin0, vaddition)),
svreinterpret_u16(svadd_u32_x(ptrue, vin1, vaddition)));
const svuint16x2_t vadd23 = svcreate2_u16(
svreinterpret_u16(svadd_u32_x(ptrue, vin2, vaddition)),
svreinterpret_u16(svadd_u32_x(ptrue, vin3, vaddition)));
const svuint16x2_t vadd45 = svcreate2_u16(
svreinterpret_u16(svadd_u32_x(ptrue, vin4, vaddition)),
svreinterpret_u16(svadd_u32_x(ptrue, vin5, vaddition)));
const svuint16x2_t vadd67 = svcreate2_u16(
svreinterpret_u16(svadd_u32_x(ptrue, vin6, vaddition)),
svreinterpret_u16(svadd_u32_x(ptrue, vin7, vaddition)));
/* Shuffle and pack bytes so each vector contains SeqDef structures. */
const svuint16_t vout01 = svtbl2_u16(vadd01, vmask);
const svuint16_t vout23 = svtbl2_u16(vadd23, vmask);
const svuint16_t vout45 = svtbl2_u16(vadd45, vmask);
const svuint16_t vout67 = svtbl2_u16(vadd67, vmask);
/* Pack the upper 16-bits of 32-bit lanes for overflow check. */
const svuint16_t voverflow01 = svuzp2_u16(svget2_u16(vadd01, 0),
svget2_u16(vadd01, 1));
const svuint16_t voverflow23 = svuzp2_u16(svget2_u16(vadd23, 0),
svget2_u16(vadd23, 1));
const svuint16_t voverflow45 = svuzp2_u16(svget2_u16(vadd45, 0),
svget2_u16(vadd45, 1));
const svuint16_t voverflow67 = svuzp2_u16(svget2_u16(vadd67, 0),
svget2_u16(vadd67, 1));
/* We don't need the whole 16 bits of the overflow part. Only 1 bit
* is needed, so we pack tightly and merge multiple vectors to be
* able to use a single comparison to handle the overflow case.
* However, we also need to handle the possible negative values of
* matchLength parts, so we use signed comparison later. */
const svint8_t voverflow =
svmax_s8_x(pmid,
svtrn1_s8(svreinterpret_s8(voverflow01),
svreinterpret_s8(voverflow23)),
svtrn1_s8(svreinterpret_s8(voverflow45),
svreinterpret_s8(voverflow67)));
/* Store `lanes` number of `SeqDef` structures from 4 vectors. */
svst1_vnum_u32(ptrue, &dstSeqs[n].offBase, 0, svreinterpret_u32(vout01));
svst1_vnum_u32(ptrue, &dstSeqs[n].offBase, 1, svreinterpret_u32(vout23));
svst1_vnum_u32(ptrue, &dstSeqs[n].offBase, 2, svreinterpret_u32(vout45));
svst1_vnum_u32(ptrue, &dstSeqs[n].offBase, 3, svreinterpret_u32(vout67));
/* Check if any enabled lanes of the overflow vector is larger than
* zero, only one such may happen. */
if (UNLIKELY(cmpgtz_any_s8(pmid, voverflow))) {
/* Scalar search for long match is needed because we merged
* multiple overflow bytes with `max`. */
size_t i;
for (i = n; i < n + lanes; i++) {
if (inSeqs[i].matchLength > 65535 + MINMATCH) {
assert(longLen == 0);
longLen = i + 1;
}
if (inSeqs[i].litLength > 65535) {
assert(longLen == 0);
longLen = i + nbSequences + 1;
}
}
}
n += lanes;
} while(n <= nbSequences - lanes);
}
/* Handle remaining elements. */
for (; n < nbSequences; n++) {
dstSeqs[n].offBase = OFFSET_TO_OFFBASE(inSeqs[n].offset);
dstSeqs[n].litLength = (U16)inSeqs[n].litLength;
dstSeqs[n].mlBase = (U16)(inSeqs[n].matchLength - MINMATCH);
/* Check for long length > 65535. */
if (UNLIKELY(inSeqs[n].matchLength > 65535 + MINMATCH)) {
assert(longLen == 0);
longLen = n + 1;
}
if (UNLIKELY(inSeqs[n].litLength > 65535)) {
assert(longLen == 0);
longLen = n + nbSequences + 1;
}
}
return longLen;
}
#elif defined(ZSTD_ARCH_ARM_NEON) && (defined(__aarch64__) || defined(_M_ARM64))
size_t convertSequences_noRepcodes(
SeqDef* dstSeqs,
const ZSTD_Sequence* inSeqs,
size_t nbSequences)
{
size_t longLen = 0;
size_t n = 0;
/* Neon permutation depends on the specific definition of target structures. */
ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, offset) == 0);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, litLength) == 4);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8);
ZSTD_STATIC_ASSERT(sizeof(SeqDef) == 8);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, offBase) == 0);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, litLength) == 4);
ZSTD_STATIC_ASSERT(offsetof(SeqDef, mlBase) == 6);
if (nbSequences > 3) {
static const ZSTD_ALIGNED(16) U32 constAddition[4] = {
ZSTD_REP_NUM, 0, -MINMATCH, 0
};
static const ZSTD_ALIGNED(16) U8 constMask[16] = {
0, 1, 2, 3, 4, 5, 8, 9, 16, 17, 18, 19, 20, 21, 24, 25
};
static const ZSTD_ALIGNED(16) U16 constCounter[8] = {
1, 1, 1, 1, 2, 2, 2, 2
};
const uint32x4_t vaddition = vld1q_u32(constAddition);
const uint8x16_t vmask = vld1q_u8(constMask);
uint16x8_t vcounter = vld1q_u16(constCounter);
uint16x8_t vindex01 = vdupq_n_u16(0);
uint16x8_t vindex23 = vdupq_n_u16(0);
do {
/* Load 4 ZSTD_Sequence (64 bytes). */
const uint32x4_t vin0 = vld1q_u32(&inSeqs[n + 0].offset);
const uint32x4_t vin1 = vld1q_u32(&inSeqs[n + 1].offset);
const uint32x4_t vin2 = vld1q_u32(&inSeqs[n + 2].offset);
const uint32x4_t vin3 = vld1q_u32(&inSeqs[n + 3].offset);
/* Add {ZSTD_REP_NUM, 0, -MINMATCH, 0} to each vector. */
const uint8x16x2_t vadd01 = { {
vreinterpretq_u8_u32(vaddq_u32(vin0, vaddition)),
vreinterpretq_u8_u32(vaddq_u32(vin1, vaddition)),
} };
const uint8x16x2_t vadd23 = { {
vreinterpretq_u8_u32(vaddq_u32(vin2, vaddition)),
vreinterpretq_u8_u32(vaddq_u32(vin3, vaddition)),
} };
/* Shuffle and pack bytes so each vector contains 2 SeqDef structures. */
const uint8x16_t vout01 = vqtbl2q_u8(vadd01, vmask);
const uint8x16_t vout23 = vqtbl2q_u8(vadd23, vmask);
/* Pack the upper 16-bits of 32-bit lanes for overflow check. */
uint16x8_t voverflow01 = vuzp2q_u16(vreinterpretq_u16_u8(vadd01.val[0]),
vreinterpretq_u16_u8(vadd01.val[1]));
uint16x8_t voverflow23 = vuzp2q_u16(vreinterpretq_u16_u8(vadd23.val[0]),
vreinterpretq_u16_u8(vadd23.val[1]));
/* Store 4 SeqDef structures. */
vst1q_u32(&dstSeqs[n + 0].offBase, vreinterpretq_u32_u8(vout01));
vst1q_u32(&dstSeqs[n + 2].offBase, vreinterpretq_u32_u8(vout23));
/* Create masks in case of overflow. */
voverflow01 = vcgtzq_s16(vreinterpretq_s16_u16(voverflow01));
voverflow23 = vcgtzq_s16(vreinterpretq_s16_u16(voverflow23));
/* Update overflow indices. */
vindex01 = vbslq_u16(voverflow01, vcounter, vindex01);
vindex23 = vbslq_u16(voverflow23, vcounter, vindex23);
/* Update counter for overflow check. */
vcounter = vaddq_u16(vcounter, vdupq_n_u16(4));
n += 4;
} while(n < nbSequences - 3);
/* Fixup indices in the second vector, we saved an additional counter
in the loop to update the second overflow index, we need to add 2
here when the indices are not 0. */
{ uint16x8_t nonzero = vtstq_u16(vindex23, vindex23);
vindex23 = vsubq_u16(vindex23, nonzero);
vindex23 = vsubq_u16(vindex23, nonzero);
}
/* Merge indices in the vectors, maximums are needed. */
vindex01 = vmaxq_u16(vindex01, vindex23);
vindex01 = vmaxq_u16(vindex01, vextq_u16(vindex01, vindex01, 4));
/* Compute `longLen`, maximums of matchLength and litLength
with a preference on litLength. */
{ U64 maxLitMatchIndices = vgetq_lane_u64(vreinterpretq_u64_u16(vindex01), 0);
size_t maxLitIndex = (maxLitMatchIndices >> 16) & 0xFFFF;
size_t maxMatchIndex = (maxLitMatchIndices >> 32) & 0xFFFF;
longLen = maxLitIndex > maxMatchIndex ? maxLitIndex + nbSequences
: maxMatchIndex;
}
}
/* Handle remaining elements. */
for (; n < nbSequences; n++) {
dstSeqs[n].offBase = OFFSET_TO_OFFBASE(inSeqs[n].offset);
dstSeqs[n].litLength = (U16)inSeqs[n].litLength;
dstSeqs[n].mlBase = (U16)(inSeqs[n].matchLength - MINMATCH);
/* Check for long length > 65535. */
if (UNLIKELY(inSeqs[n].matchLength > 65535 + MINMATCH)) {
assert(longLen == 0);
longLen = n + 1;
}
if (UNLIKELY(inSeqs[n].litLength > 65535)) {
assert(longLen == 0);
longLen = n + nbSequences + 1;
}
}
return longLen;
}
#else /* No vectorization. */
size_t convertSequences_noRepcodes(
SeqDef* dstSeqs,
const ZSTD_Sequence* inSeqs,
size_t nbSequences)
{
size_t longLen = 0;
size_t n;
for (n=0; n<nbSequences; n++) {
dstSeqs[n].offBase = OFFSET_TO_OFFBASE(inSeqs[n].offset);
dstSeqs[n].litLength = (U16)inSeqs[n].litLength;
dstSeqs[n].mlBase = (U16)(inSeqs[n].matchLength - MINMATCH);
/* Check for long length > 65535. */
if (UNLIKELY(inSeqs[n].matchLength > 65535+MINMATCH)) {
assert(longLen == 0);
longLen = n + 1;
}
if (UNLIKELY(inSeqs[n].litLength > 65535)) {
assert(longLen == 0);
longLen = n + nbSequences + 1;
}
}
return longLen;
}
#endif
/*
* Precondition: Sequences must end on an explicit Block Delimiter
* @return: 0 on success, or an error code.
* Note: Sequence validation functionality has been disabled (removed).
* This is helpful to generate a lean main pipeline, improving performance.
* It may be re-inserted later.
*/
size_t ZSTD_convertBlockSequences(ZSTD_CCtx* cctx,
const ZSTD_Sequence* const inSeqs, size_t nbSequences,
int repcodeResolution)
{
Repcodes_t updatedRepcodes;
size_t seqNb = 0;
DEBUGLOG(5, "ZSTD_convertBlockSequences (nbSequences = %zu)", nbSequences);
RETURN_ERROR_IF(nbSequences >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
"Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t));
/* check end condition */
assert(nbSequences >= 1);
assert(inSeqs[nbSequences-1].matchLength == 0);
assert(inSeqs[nbSequences-1].offset == 0);
/* Convert Sequences from public format to internal format */
if (!repcodeResolution) {
size_t const longl = convertSequences_noRepcodes(cctx->seqStore.sequencesStart, inSeqs, nbSequences-1);
cctx->seqStore.sequences = cctx->seqStore.sequencesStart + nbSequences-1;
if (longl) {
DEBUGLOG(5, "long length");
assert(cctx->seqStore.longLengthType == ZSTD_llt_none);
if (longl <= nbSequences-1) {
DEBUGLOG(5, "long match length detected at pos %zu", longl-1);
cctx->seqStore.longLengthType = ZSTD_llt_matchLength;
cctx->seqStore.longLengthPos = (U32)(longl-1);
} else {
DEBUGLOG(5, "long literals length detected at pos %zu", longl-nbSequences);
assert(longl <= 2* (nbSequences-1));
cctx->seqStore.longLengthType = ZSTD_llt_literalLength;
cctx->seqStore.longLengthPos = (U32)(longl-(nbSequences-1)-1);
}
}
} else {
for (seqNb = 0; seqNb < nbSequences - 1 ; seqNb++) {
U32 const litLength = inSeqs[seqNb].litLength;
U32 const matchLength = inSeqs[seqNb].matchLength;
U32 const ll0 = (litLength == 0);
U32 const offBase = ZSTD_finalizeOffBase(inSeqs[seqNb].offset, updatedRepcodes.rep, ll0);
DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
ZSTD_storeSeqOnly(&cctx->seqStore, litLength, offBase, matchLength);
ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
}
}
/* If we skipped repcode search while parsing, we need to update repcodes now */
if (!repcodeResolution && nbSequences > 1) {
U32* const rep = updatedRepcodes.rep;
if (nbSequences >= 4) {
U32 lastSeqIdx = (U32)nbSequences - 2; /* index of last full sequence */
rep[2] = inSeqs[lastSeqIdx - 2].offset;
rep[1] = inSeqs[lastSeqIdx - 1].offset;
rep[0] = inSeqs[lastSeqIdx].offset;
} else if (nbSequences == 3) {
rep[2] = rep[0];
rep[1] = inSeqs[0].offset;
rep[0] = inSeqs[1].offset;
} else {
assert(nbSequences == 2);
rep[2] = rep[1];
rep[1] = rep[0];
rep[0] = inSeqs[0].offset;
}
}
ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t));
return 0;
}
#if defined(ZSTD_ARCH_X86_AVX2)
BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs)
{
size_t i;
__m256i const zeroVec = _mm256_setzero_si256();
__m256i sumVec = zeroVec; /* accumulates match+lit in 32-bit lanes */
ZSTD_ALIGNED(32) U32 tmp[8]; /* temporary buffer for reduction */
size_t mSum = 0, lSum = 0;
ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16);
/* Process 2 structs (32 bytes) at a time */
for (i = 0; i + 2 <= nbSeqs; i += 2) {
/* Load two consecutive ZSTD_Sequence (8×4 = 32 bytes) */
__m256i data = _mm256_loadu_si256((const __m256i*)(const void*)&seqs[i]);
/* check end of block signal */
__m256i cmp = _mm256_cmpeq_epi32(data, zeroVec);
int cmp_res = _mm256_movemask_epi8(cmp);
/* indices for match lengths correspond to bits [8..11], [24..27]
* => combined mask = 0x0F000F00 */
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8);
if (cmp_res & 0x0F000F00) break;
/* Accumulate in sumVec */
sumVec = _mm256_add_epi32(sumVec, data);
}
/* Horizontal reduction */
_mm256_store_si256((__m256i*)tmp, sumVec);
lSum = tmp[1] + tmp[5];
mSum = tmp[2] + tmp[6];
/* Handle the leftover */
for (; i < nbSeqs; i++) {
lSum += seqs[i].litLength;
mSum += seqs[i].matchLength;
if (seqs[i].matchLength == 0) break; /* end of block */
}
if (i==nbSeqs) {
/* reaching end of sequences: end of block signal was not present */
BlockSummary bs;
bs.nbSequences = ERROR(externalSequences_invalid);
return bs;
}
{ BlockSummary bs;
bs.nbSequences = i+1;
bs.blockSize = lSum + mSum;
bs.litSize = lSum;
return bs;
}
}
#elif defined (ZSTD_ARCH_RISCV_RVV)
BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs)
{
size_t totalMatchSize = 0;
size_t litSize = 0;
size_t i = 0;
int found_terminator = 0;
size_t vl_max = __riscv_vsetvlmax_e32m1();
typedef uint32_t __attribute__((may_alias)) aliased_u32;
vuint32m1_t v_lit_sum = __riscv_vmv_v_x_u32m1(0, vl_max);
vuint32m1_t v_match_sum = __riscv_vmv_v_x_u32m1(0, vl_max);
for (; i < nbSeqs; ) {
size_t vl = __riscv_vsetvl_e32m2(nbSeqs - i);
vuint32m2x4_t v_tuple = __riscv_vlseg4e32_v_u32m2x4(
(const aliased_u32*)((const void*)&seqs[i]),
vl
);
vuint32m2_t v_lit = __riscv_vget_v_u32m2x4_u32m2(v_tuple, 1);
vuint32m2_t v_match = __riscv_vget_v_u32m2x4_u32m2(v_tuple, 2);
// Check if any element has a matchLength of 0
vbool16_t mask = __riscv_vmseq_vx_u32m2_b16(v_match, 0, vl);
int first_zero = __riscv_vfirst_m_b16(mask, vl);
if (first_zero >= 0) {
// Find the first zero byte and set the effective length to that index + 1 to
// recompute the cumulative vector length of literals and matches
vl = first_zero + 1;
// recompute the cumulative vector length of literals and matches
v_lit_sum = __riscv_vredsum_vs_u32m2_u32m1(__riscv_vslidedown_vx_u32m2(v_lit, 0, vl), v_lit_sum, vl);
v_match_sum = __riscv_vredsum_vs_u32m2_u32m1(__riscv_vslidedown_vx_u32m2(v_match, 0, vl), v_match_sum, vl);
i += vl;
found_terminator = 1;
assert(seqs[i - 1].offset == 0);
break;
} else {
v_lit_sum = __riscv_vredsum_vs_u32m2_u32m1(v_lit, v_lit_sum, vl);
v_match_sum = __riscv_vredsum_vs_u32m2_u32m1(v_match, v_match_sum, vl);
i += vl;
}
}
litSize = __riscv_vmv_x_s_u32m1_u32(v_lit_sum);
totalMatchSize = __riscv_vmv_x_s_u32m1_u32(v_match_sum);
if (!found_terminator && i==nbSeqs) {
BlockSummary bs;
bs.nbSequences = ERROR(externalSequences_invalid);
return bs;
}
{ BlockSummary bs;
bs.nbSequences = i;
bs.blockSize = litSize + totalMatchSize;
bs.litSize = litSize;
return bs;
}
}
#else
/*
* The function assumes `litMatchLength` is a packed 64-bit value where the
* lower 32 bits represent the match length. The check varies based on the
* system's endianness:
* - On little-endian systems, it verifies if the entire 64-bit value is at most
* 0xFFFFFFFF, indicating the match length (lower 32 bits) is zero.
* - On big-endian systems, it directly checks if the lower 32 bits are zero.
*
* @returns 1 if the match length is zero, 0 otherwise.
*/
FORCE_INLINE_TEMPLATE int matchLengthHalfIsZero(U64 litMatchLength)
{
if (MEM_isLittleEndian()) {
return litMatchLength <= 0xFFFFFFFFULL;
} else {
return (U32)litMatchLength == 0;
}
}
BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs)
{
/* Use multiple accumulators for efficient use of wide out-of-order machines. */
U64 litMatchSize0 = 0;
U64 litMatchSize1 = 0;
U64 litMatchSize2 = 0;
U64 litMatchSize3 = 0;
size_t n = 0;
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, litLength) + 4 == offsetof(ZSTD_Sequence, matchLength));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) + 4 == offsetof(ZSTD_Sequence, rep));
assert(seqs);
if (nbSeqs > 3) {
/* Process the input in 4 independent streams to reach high throughput. */
do {
/* Load `litLength` and `matchLength` as a packed `U64`. It is safe
* to use 64-bit unsigned arithmetic here because the sum of `litLength`
* and `matchLength` cannot exceed the block size, so the 32-bit
* subparts will never overflow. */
U64 litMatchLength = MEM_read64(&seqs[n].litLength);
litMatchSize0 += litMatchLength;
if (matchLengthHalfIsZero(litMatchLength)) {
assert(seqs[n].offset == 0);
goto _out;
}
litMatchLength = MEM_read64(&seqs[n + 1].litLength);
litMatchSize1 += litMatchLength;
if (matchLengthHalfIsZero(litMatchLength)) {
n += 1;
assert(seqs[n].offset == 0);
goto _out;
}
litMatchLength = MEM_read64(&seqs[n + 2].litLength);
litMatchSize2 += litMatchLength;
if (matchLengthHalfIsZero(litMatchLength)) {
n += 2;
assert(seqs[n].offset == 0);
goto _out;
}
litMatchLength = MEM_read64(&seqs[n + 3].litLength);
litMatchSize3 += litMatchLength;
if (matchLengthHalfIsZero(litMatchLength)) {
n += 3;
assert(seqs[n].offset == 0);
goto _out;
}
n += 4;
} while(n < nbSeqs - 3);
}
for (; n < nbSeqs; n++) {
U64 litMatchLength = MEM_read64(&seqs[n].litLength);
litMatchSize0 += litMatchLength;
if (matchLengthHalfIsZero(litMatchLength)) {
assert(seqs[n].offset == 0);
goto _out;
}
}
/* At this point n == nbSeqs, so no end terminator. */
{ BlockSummary bs;
bs.nbSequences = ERROR(externalSequences_invalid);
return bs;
}
_out:
litMatchSize0 += litMatchSize1 + litMatchSize2 + litMatchSize3;
{ BlockSummary bs;
bs.nbSequences = n + 1;
if (MEM_isLittleEndian()) {
bs.litSize = (U32)litMatchSize0;
bs.blockSize = bs.litSize + (litMatchSize0 >> 32);
} else {
bs.litSize = litMatchSize0 >> 32;
bs.blockSize = bs.litSize + (U32)litMatchSize0;
}
return bs;
}
}
#endif
static size_t
ZSTD_compressSequencesAndLiterals_internal(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t nbSequences,
const void* literals, size_t litSize, size_t srcSize)
{
size_t remaining = srcSize;
size_t cSize = 0;
BYTE* op = (BYTE*)dst;
int const repcodeResolution = (cctx->appliedParams.searchForExternalRepcodes == ZSTD_ps_enable);
assert(cctx->appliedParams.searchForExternalRepcodes != ZSTD_ps_auto);
DEBUGLOG(4, "ZSTD_compressSequencesAndLiterals_internal: nbSeqs=%zu, litSize=%zu", nbSequences, litSize);
RETURN_ERROR_IF(nbSequences == 0, externalSequences_invalid, "Requires at least 1 end-of-block");
/* Special case: empty frame */
if ((nbSequences == 1) && (inSeqs[0].litLength == 0)) {
U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1);
RETURN_ERROR_IF(dstCapacity<3, dstSize_tooSmall, "No room for empty frame block header");
MEM_writeLE24(op, cBlockHeader24);
op += ZSTD_blockHeaderSize;
dstCapacity -= ZSTD_blockHeaderSize;
cSize += ZSTD_blockHeaderSize;
}
while (nbSequences) {
size_t compressedSeqsSize, cBlockSize, conversionStatus;
BlockSummary const block = ZSTD_get1BlockSummary(inSeqs, nbSequences);
U32 const lastBlock = (block.nbSequences == nbSequences);
FORWARD_IF_ERROR(block.nbSequences, "Error while trying to determine nb of sequences for a block");
assert(block.nbSequences <= nbSequences);
RETURN_ERROR_IF(block.litSize > litSize, externalSequences_invalid, "discrepancy: Sequences require more literals than present in buffer");
ZSTD_resetSeqStore(&cctx->seqStore);
conversionStatus = ZSTD_convertBlockSequences(cctx,
inSeqs, block.nbSequences,
repcodeResolution);
FORWARD_IF_ERROR(conversionStatus, "Bad sequence conversion");
inSeqs += block.nbSequences;
nbSequences -= block.nbSequences;
remaining -= block.blockSize;
/* Note: when blockSize is very small, other variant send it uncompressed.
* Here, we still send the sequences, because we don't have the original source to send it uncompressed.
* One could imagine in theory reproducing the source from the sequences,
* but that's complex and costly memory intensive, and goes against the objectives of this variant. */
RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block");
compressedSeqsSize = ZSTD_entropyCompressSeqStore_internal(
op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize,
literals, block.litSize,
&cctx->seqStore,
&cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy,
&cctx->appliedParams,
cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */,
cctx->bmi2);
FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed");
/* note: the spec forbids for any compressed block to be larger than maximum block size */
if (compressedSeqsSize > cctx->blockSizeMax) compressedSeqsSize = 0;
DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize);
litSize -= block.litSize;
literals = (const char*)literals + block.litSize;
/* Note: difficult to check source for RLE block when only Literals are provided,
* but it could be considered from analyzing the sequence directly */
if (compressedSeqsSize == 0) {
/* Sending uncompressed blocks is out of reach, because the source is not provided.
* In theory, one could use the sequences to regenerate the source, like a decompressor,
* but it's complex, and memory hungry, killing the purpose of this variant.
* Current outcome: generate an error code.
*/
RETURN_ERROR(cannotProduce_uncompressedBlock, "ZSTD_compressSequencesAndLiterals cannot generate an uncompressed block");
} else {
U32 cBlockHeader;
assert(compressedSeqsSize > 1); /* no RLE */
/* Error checking and repcodes update */
ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState);
if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
/* Write block header into beginning of block*/
cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3);
MEM_writeLE24(op, cBlockHeader);
cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize;
DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize);
}
cSize += cBlockSize;
op += cBlockSize;
dstCapacity -= cBlockSize;
cctx->isFirstBlock = 0;
DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity);
if (lastBlock) {
assert(nbSequences == 0);
break;
}
}
RETURN_ERROR_IF(litSize != 0, externalSequences_invalid, "literals must be entirely and exactly consumed");
RETURN_ERROR_IF(remaining != 0, externalSequences_invalid, "Sequences must represent a total of exactly srcSize=%zu", srcSize);
DEBUGLOG(4, "cSize final total: %zu", cSize);
return cSize;
}
size_t
ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* literals, size_t litSize, size_t litCapacity,
size_t decompressedSize)
{
BYTE* op = (BYTE*)dst;
size_t cSize = 0;
/* Transparent initialization stage, same as compressStream2() */
DEBUGLOG(4, "ZSTD_compressSequencesAndLiterals (dstCapacity=%zu)", dstCapacity);
assert(cctx != NULL);
if (litCapacity < litSize) {
RETURN_ERROR(workSpace_tooSmall, "literals buffer is not large enough: must be at least 8 bytes larger than litSize (risk of read out-of-bound)");
}
FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, decompressedSize), "CCtx initialization failed");
if (cctx->appliedParams.blockDelimiters == ZSTD_sf_noBlockDelimiters) {
RETURN_ERROR(frameParameter_unsupported, "This mode is only compatible with explicit delimiters");
}
if (cctx->appliedParams.validateSequences) {
RETURN_ERROR(parameter_unsupported, "This mode is not compatible with Sequence validation");
}
if (cctx->appliedParams.fParams.checksumFlag) {
RETURN_ERROR(frameParameter_unsupported, "this mode is not compatible with frame checksum");
}
/* Begin writing output, starting with frame header */
{ size_t const frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity,
&cctx->appliedParams, decompressedSize, cctx->dictID);
op += frameHeaderSize;
assert(frameHeaderSize <= dstCapacity);
dstCapacity -= frameHeaderSize;
cSize += frameHeaderSize;
}
/* Now generate compressed blocks */
{ size_t const cBlocksSize = ZSTD_compressSequencesAndLiterals_internal(cctx,
op, dstCapacity,
inSeqs, inSeqsSize,
literals, litSize, decompressedSize);
FORWARD_IF_ERROR(cBlocksSize, "Compressing blocks failed!");
cSize += cBlocksSize;
assert(cBlocksSize <= dstCapacity);
dstCapacity -= cBlocksSize;
}
DEBUGLOG(4, "Final compressed size: %zu", cSize);
return cSize;
}
/*====== Finalize ======*/
static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs)
@@ -8178,6 +6963,7 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush);
}
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{
ZSTD_inBuffer input = inBuffer_forEndFlush(zcs);
@@ -8258,7 +7044,7 @@ static void ZSTD_dedicatedDictSearch_revertCParams(
}
}
static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode)
static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
{
switch (mode) {
case ZSTD_cpm_unknown:
@@ -8282,8 +7068,8 @@ static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_CParamMo
* @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize.
* Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown.
* Use dictSize == 0 for unknown or unused.
* Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_CParamMode_e`. */
static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode)
* Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */
static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode)
{
U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode);
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB);
@@ -8321,9 +7107,7 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
* same idea as ZSTD_getCParams()
* @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`).
* Fields of `ZSTD_frameParameters` are set to default values */
static ZSTD_parameters
ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode)
{
static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) {
ZSTD_parameters params;
ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode);
DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel);
@@ -8337,8 +7121,7 @@ ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, si
* same idea as ZSTD_getCParams()
* @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`).
* Fields of `ZSTD_frameParameters` are set to default values */
ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize)
{
ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) {
if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN;
return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
}
@@ -8346,8 +7129,8 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH
void ZSTD_registerSequenceProducer(
ZSTD_CCtx* zc,
void* extSeqProdState,
ZSTD_sequenceProducer_F extSeqProdFunc)
{
ZSTD_sequenceProducer_F extSeqProdFunc
) {
assert(zc != NULL);
ZSTD_CCtxParams_registerSequenceProducer(
&zc->requestedParams, extSeqProdState, extSeqProdFunc
@@ -8357,8 +7140,8 @@ void ZSTD_registerSequenceProducer(
void ZSTD_CCtxParams_registerSequenceProducer(
ZSTD_CCtx_params* params,
void* extSeqProdState,
ZSTD_sequenceProducer_F extSeqProdFunc)
{
ZSTD_sequenceProducer_F extSeqProdFunc
) {
assert(params != NULL);
if (extSeqProdFunc != NULL) {
params->extSeqProdFunc = extSeqProdFunc;
+126 -228
View File
@@ -24,7 +24,10 @@
# include "zstdmt_compress.h"
#endif
#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */
#include "zstd_preSplit.h" /* ZSTD_SLIPBLOCK_WORKSPACESIZE */
#if defined (__cplusplus)
extern "C" {
#endif
/*-*************************************
* Constants
@@ -79,70 +82,6 @@ typedef struct {
ZSTD_fseCTables_t fse;
} ZSTD_entropyCTables_t;
/***********************************************
* Sequences *
***********************************************/
typedef struct SeqDef_s {
U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
U16 litLength;
U16 mlBase; /* mlBase == matchLength - MINMATCH */
} SeqDef;
/* Controls whether seqStore has a single "long" litLength or matchLength. See SeqStore_t. */
typedef enum {
ZSTD_llt_none = 0, /* no longLengthType */
ZSTD_llt_literalLength = 1, /* represents a long literal */
ZSTD_llt_matchLength = 2 /* represents a long match */
} ZSTD_longLengthType_e;
typedef struct {
SeqDef* sequencesStart;
SeqDef* sequences; /* ptr to end of sequences */
BYTE* litStart;
BYTE* lit; /* ptr to end of literals */
BYTE* llCode;
BYTE* mlCode;
BYTE* ofCode;
size_t maxNbSeq;
size_t maxNbLit;
/* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
* in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
* the existing value of the litLength or matchLength by 0x10000.
*/
ZSTD_longLengthType_e longLengthType;
U32 longLengthPos; /* Index of the sequence to apply long length modification to */
} SeqStore_t;
typedef struct {
U32 litLength;
U32 matchLength;
} ZSTD_SequenceLength;
/**
* Returns the ZSTD_SequenceLength for the given sequences. It handles the decoding of long sequences
* indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
*/
MEM_STATIC ZSTD_SequenceLength ZSTD_getSequenceLength(SeqStore_t const* seqStore, SeqDef const* seq)
{
ZSTD_SequenceLength seqLen;
seqLen.litLength = seq->litLength;
seqLen.matchLength = seq->mlBase + MINMATCH;
if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
if (seqStore->longLengthType == ZSTD_llt_literalLength) {
seqLen.litLength += 0x10000;
}
if (seqStore->longLengthType == ZSTD_llt_matchLength) {
seqLen.matchLength += 0x10000;
}
}
return seqLen;
}
const SeqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
int ZSTD_seqToCodes(const SeqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
/***********************************************
* Entropy buffer statistics structs and funcs *
***********************************************/
@@ -152,7 +91,7 @@ int ZSTD_seqToCodes(const SeqStore_t* seqStorePtr); /* compress, dictBuilder,
* hufDesSize refers to the size of huffman tree description in bytes.
* This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */
typedef struct {
SymbolEncodingType_e hType;
symbolEncodingType_e hType;
BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE];
size_t hufDesSize;
} ZSTD_hufCTablesMetadata_t;
@@ -163,9 +102,9 @@ typedef struct {
* fseTablesSize refers to the size of fse tables in bytes.
* This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */
typedef struct {
SymbolEncodingType_e llType;
SymbolEncodingType_e ofType;
SymbolEncodingType_e mlType;
symbolEncodingType_e llType;
symbolEncodingType_e ofType;
symbolEncodingType_e mlType;
BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE];
size_t fseTablesSize;
size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
@@ -180,7 +119,7 @@ typedef struct {
* Builds entropy for the block.
* @return : 0 on success or error code */
size_t ZSTD_buildBlockEntropyStats(
const SeqStore_t* seqStorePtr,
const seqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
@@ -209,9 +148,15 @@ typedef struct {
stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */
size_t size; /* The number of sequences. <= capacity. */
size_t capacity; /* The capacity starting from `seq` pointer */
} RawSeqStore_t;
} rawSeqStore_t;
UNUSED_ATTR static const RawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
typedef struct {
U32 idx; /* Index in array of ZSTD_Sequence */
U32 posInSequence; /* Position within sequence at idx */
size_t posInSrc; /* Number of bytes given by sequences provided so far */
} ZSTD_sequencePosition;
UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
typedef struct {
int price; /* price from beginning of segment to this position */
@@ -243,7 +188,7 @@ typedef struct {
U32 offCodeSumBasePrice; /* to compare to log2(offreq) */
ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */
const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */
ZSTD_ParamSwitch_e literalCompressionMode;
ZSTD_paramSwitch_e literalCompressionMode;
} optState_t;
typedef struct {
@@ -265,11 +210,11 @@ typedef struct {
#define ZSTD_WINDOW_START_INDEX 2
typedef struct ZSTD_MatchState_t ZSTD_MatchState_t;
typedef struct ZSTD_matchState_t ZSTD_matchState_t;
#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */
struct ZSTD_MatchState_t {
struct ZSTD_matchState_t {
ZSTD_window_t window; /* State for window round buffer management */
U32 loadedDictEnd; /* index of end of dictionary, within context's referential.
* When loadedDictEnd != 0, a dictionary is in use, and still valid.
@@ -291,15 +236,15 @@ struct ZSTD_MatchState_t {
U32* hashTable3;
U32* chainTable;
int forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */
U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */
int dedicatedDictSearch; /* Indicates whether this matchState is using the
* dedicated dictionary search structure.
*/
optState_t opt; /* optimal parser state */
const ZSTD_MatchState_t* dictMatchState;
const ZSTD_matchState_t* dictMatchState;
ZSTD_compressionParameters cParams;
const RawSeqStore_t* ldmSeqStore;
const rawSeqStore_t* ldmSeqStore;
/* Controls prefetching in some dictMatchState matchfinders.
* This behavior is controlled from the cctx ms.
@@ -317,7 +262,7 @@ struct ZSTD_MatchState_t {
typedef struct {
ZSTD_compressedBlockState_t* prevCBlock;
ZSTD_compressedBlockState_t* nextCBlock;
ZSTD_MatchState_t matchState;
ZSTD_matchState_t matchState;
} ZSTD_blockState_t;
typedef struct {
@@ -344,7 +289,7 @@ typedef struct {
} ldmState_t;
typedef struct {
ZSTD_ParamSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */
ZSTD_paramSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */
U32 hashLog; /* Log size of hashTable */
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
U32 minMatchLength; /* Minimum match length */
@@ -375,7 +320,7 @@ struct ZSTD_CCtx_params_s {
* There is no guarantee that hint is close to actual source size */
ZSTD_dictAttachPref_e attachDictPref;
ZSTD_ParamSwitch_e literalCompressionMode;
ZSTD_paramSwitch_e literalCompressionMode;
/* Multithreading: used to pass parameters to mtctx */
int nbWorkers;
@@ -394,27 +339,14 @@ struct ZSTD_CCtx_params_s {
ZSTD_bufferMode_e outBufferMode;
/* Sequence compression API */
ZSTD_SequenceFormat_e blockDelimiters;
ZSTD_sequenceFormat_e blockDelimiters;
int validateSequences;
/* Block splitting
* @postBlockSplitter executes split analysis after sequences are produced,
* it's more accurate but consumes more resources.
* @preBlockSplitter_level splits before knowing sequences,
* it's more approximative but also cheaper.
* Valid @preBlockSplitter_level values range from 0 to 6 (included).
* 0 means auto, 1 means do not split,
* then levels are sorted in increasing cpu budget, from 2 (fastest) to 6 (slowest).
* Highest @preBlockSplitter_level combines well with @postBlockSplitter.
*/
ZSTD_ParamSwitch_e postBlockSplitter;
int preBlockSplitter_level;
/* Adjust the max block size*/
size_t maxBlockSize;
/* Block splitting */
ZSTD_paramSwitch_e useBlockSplitter;
/* Param for deciding whether to use row-based matchfinder */
ZSTD_ParamSwitch_e useRowMatchFinder;
ZSTD_paramSwitch_e useRowMatchFinder;
/* Always load a dictionary in ext-dict mode (not prefix mode)? */
int deterministicRefPrefix;
@@ -423,7 +355,7 @@ struct ZSTD_CCtx_params_s {
ZSTD_customMem customMem;
/* Controls prefetching in some dictMatchState matchfinders */
ZSTD_ParamSwitch_e prefetchCDictTables;
ZSTD_paramSwitch_e prefetchCDictTables;
/* Controls whether zstd will fall back to an internal matchfinder
* if the external matchfinder returns an error code. */
@@ -435,13 +367,15 @@ struct ZSTD_CCtx_params_s {
void* extSeqProdState;
ZSTD_sequenceProducer_F extSeqProdFunc;
/* Adjust the max block size*/
size_t maxBlockSize;
/* Controls repcode search in external sequence parsing */
ZSTD_ParamSwitch_e searchForExternalRepcodes;
ZSTD_paramSwitch_e searchForExternalRepcodes;
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
#define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
#define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE)
#define TMP_WORKSPACE_SIZE (MAX(ENTROPY_WORKSPACE_SIZE, ZSTD_SLIPBLOCK_WORKSPACESIZE))
/**
* Indicates whether this compression proceeds directly from user-provided
@@ -459,11 +393,11 @@ typedef enum {
*/
#define ZSTD_MAX_NB_BLOCK_SPLITS 196
typedef struct {
SeqStore_t fullSeqStoreChunk;
SeqStore_t firstHalfSeqStore;
SeqStore_t secondHalfSeqStore;
SeqStore_t currSeqStore;
SeqStore_t nextSeqStore;
seqStore_t fullSeqStoreChunk;
seqStore_t firstHalfSeqStore;
seqStore_t secondHalfSeqStore;
seqStore_t currSeqStore;
seqStore_t nextSeqStore;
U32 partitions[ZSTD_MAX_NB_BLOCK_SPLITS];
ZSTD_entropyCTablesMetadata_t entropyMetadata;
@@ -480,7 +414,7 @@ struct ZSTD_CCtx_s {
size_t dictContentSize;
ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */
size_t blockSizeMax;
size_t blockSize;
unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */
unsigned long long consumedSrcSize;
unsigned long long producedCSize;
@@ -492,14 +426,13 @@ struct ZSTD_CCtx_s {
int isFirstBlock;
int initialized;
SeqStore_t seqStore; /* sequences storage ptrs */
seqStore_t seqStore; /* sequences storage ptrs */
ldmState_t ldmState; /* long distance matching state */
rawSeq* ldmSequences; /* Storage for the ldm output sequences */
size_t maxNbLdmSequences;
RawSeqStore_t externSeqStore; /* Mutable reference to external sequences */
rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */
ZSTD_blockState_t blockState;
void* tmpWorkspace; /* used as substitute of stack space - must be aligned for S64 type */
size_t tmpWkspSize;
U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */
/* Whether we are streaming or not */
ZSTD_buffered_policy_e bufferedPolicy;
@@ -573,12 +506,12 @@ typedef enum {
* behavior of taking both the source size and the dict size into account
* when selecting and adjusting parameters.
*/
} ZSTD_CParamMode_e;
} ZSTD_cParamMode_e;
typedef size_t (*ZSTD_BlockCompressor_f) (
ZSTD_MatchState_t* bs, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
typedef size_t (*ZSTD_blockCompressor) (
ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_ParamSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
@@ -624,25 +557,6 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
return 1;
}
/* ZSTD_selectAddr:
* @return index >= lowLimit ? candidate : backup,
* tries to force branchless codegen. */
MEM_STATIC const BYTE*
ZSTD_selectAddr(U32 index, U32 lowLimit, const BYTE* candidate, const BYTE* backup)
{
#if defined(__GNUC__) && defined(__x86_64__)
__asm__ (
"cmp %1, %2\n"
"cmova %3, %0\n"
: "+r"(candidate)
: "r"(index), "r"(lowLimit), "r"(backup)
);
return candidate;
#else
return index >= lowLimit ? candidate : backup;
#endif
}
/* ZSTD_noCompressBlock() :
* Writes uncompressed block to dst buffer from given src.
* Returns the size of the block */
@@ -707,7 +621,7 @@ ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE con
{
assert(iend > ilimit_w);
if (ip <= ilimit_w) {
ZSTD_wildcopy(op, ip, (size_t)(ilimit_w - ip), ZSTD_no_overlap);
ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap);
op += ilimit_w - ip;
ip = ilimit_w;
}
@@ -725,55 +639,14 @@ ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE con
#define OFFBASE_TO_OFFSET(o) (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM)
#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o)) /* returns ID 1,2,3 */
/*! ZSTD_storeSeqOnly() :
* Store a sequence (litlen, litPtr, offBase and matchLength) into SeqStore_t.
* Literals themselves are not copied, but @litPtr is updated.
* @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE().
* @matchLength : must be >= MINMATCH
*/
HINT_INLINE UNUSED_ATTR void
ZSTD_storeSeqOnly(SeqStore_t* seqStorePtr,
size_t litLength,
U32 offBase,
size_t matchLength)
{
assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq);
/* literal Length */
assert(litLength <= ZSTD_BLOCKSIZE_MAX);
if (UNLIKELY(litLength>0xFFFF)) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_literalLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].litLength = (U16)litLength;
/* match offset */
seqStorePtr->sequences[0].offBase = offBase;
/* match Length */
assert(matchLength <= ZSTD_BLOCKSIZE_MAX);
assert(matchLength >= MINMATCH);
{ size_t const mlBase = matchLength - MINMATCH;
if (UNLIKELY(mlBase>0xFFFF)) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_matchLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].mlBase = (U16)mlBase;
}
seqStorePtr->sequences++;
}
/*! ZSTD_storeSeq() :
* Store a sequence (litlen, litPtr, offBase and matchLength) into SeqStore_t.
* Store a sequence (litlen, litPtr, offBase and matchLength) into seqStore_t.
* @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE().
* @matchLength : must be >= MINMATCH
* Allowed to over-read literals up to litLimit.
*/
HINT_INLINE UNUSED_ATTR void
ZSTD_storeSeq(SeqStore_t* seqStorePtr,
ZSTD_storeSeq(seqStore_t* seqStorePtr,
size_t litLength, const BYTE* literals, const BYTE* litLimit,
U32 offBase,
size_t matchLength)
@@ -800,14 +673,36 @@ ZSTD_storeSeq(SeqStore_t* seqStorePtr,
ZSTD_STATIC_ASSERT(WILDCOPY_OVERLENGTH >= 16);
ZSTD_copy16(seqStorePtr->lit, literals);
if (litLength > 16) {
ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, litLength-16, ZSTD_no_overlap);
ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap);
}
} else {
ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w);
}
seqStorePtr->lit += litLength;
ZSTD_storeSeqOnly(seqStorePtr, litLength, offBase, matchLength);
/* literal Length */
if (litLength>0xFFFF) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_literalLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].litLength = (U16)litLength;
/* match offset */
seqStorePtr->sequences[0].offBase = offBase;
/* match Length */
assert(matchLength >= MINMATCH);
{ size_t const mlBase = matchLength - MINMATCH;
if (mlBase>0xFFFF) {
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_matchLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].mlBase = (U16)mlBase;
}
seqStorePtr->sequences++;
}
/* ZSTD_updateRep() :
@@ -836,12 +731,12 @@ ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0)
typedef struct repcodes_s {
U32 rep[3];
} Repcodes_t;
} repcodes_t;
MEM_STATIC Repcodes_t
MEM_STATIC repcodes_t
ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0)
{
Repcodes_t newReps;
repcodes_t newReps;
ZSTD_memcpy(&newReps, rep, sizeof(newReps));
ZSTD_updateRep(newReps.rep, offBase, ll0);
return newReps;
@@ -884,8 +779,8 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match,
size_t const matchLength = ZSTD_count(ip, match, vEnd);
if (match + matchLength != mEnd) return matchLength;
DEBUGLOG(7, "ZSTD_count_2segments: found a 2-parts match (current length==%zu)", matchLength);
DEBUGLOG(7, "distance from match beginning to end dictionary = %i", (int)(mEnd - match));
DEBUGLOG(7, "distance from current pos to end buffer = %i", (int)(iEnd - ip));
DEBUGLOG(7, "distance from match beginning to end dictionary = %zi", mEnd - match);
DEBUGLOG(7, "distance from current pos to end buffer = %zi", iEnd - ip);
DEBUGLOG(7, "next byte : ip==%02X, istart==%02X", ip[matchLength], *iStart);
DEBUGLOG(7, "final match length = %zu", matchLength + ZSTD_count(ip+matchLength, iStart, iEnd));
return matchLength + ZSTD_count(ip+matchLength, iStart, iEnd);
@@ -1023,12 +918,11 @@ MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64
/*-*************************************
* Round buffer management
***************************************/
/* Max @current value allowed:
* In 32-bit mode: we want to avoid crossing the 2 GB limit,
* reducing risks of side effects in case of signed operations on indexes.
* In 64-bit mode: we want to ensure that adding the maximum job size (512 MB)
* doesn't overflow U32 index capacity (4 GB) */
#define ZSTD_CURRENT_MAX (MEM_64bits() ? 3500U MB : 2000U MB)
#if (ZSTD_WINDOWLOG_MAX_64 > 31)
# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX"
#endif
/* Max current allowed */
#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX))
/* Maximum chunk size before overflow correction needs to be called again */
#define ZSTD_CHUNKSIZE_MAX \
( ((U32)-1) /* Maximum ending current index */ \
@@ -1068,7 +962,7 @@ MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window)
* Inspects the provided matchState and figures out what dictMode should be
* passed to the compressor.
*/
MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_MatchState_t *ms)
MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms)
{
return ZSTD_window_hasExtDict(ms->window) ?
ZSTD_extDict :
@@ -1257,7 +1151,7 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
const void* blockEnd,
U32 maxDist,
U32* loadedDictEndPtr,
const ZSTD_MatchState_t** dictMatchStatePtr)
const ZSTD_matchState_t** dictMatchStatePtr)
{
U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
@@ -1302,7 +1196,7 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
const void* blockEnd,
U32 maxDist,
U32* loadedDictEndPtr,
const ZSTD_MatchState_t** dictMatchStatePtr)
const ZSTD_matchState_t** dictMatchStatePtr)
{
assert(loadedDictEndPtr != NULL);
assert(dictMatchStatePtr != NULL);
@@ -1352,7 +1246,7 @@ MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) {
MEM_STATIC
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_window_update(ZSTD_window_t* window,
const void* src, size_t srcSize,
void const* src, size_t srcSize,
int forceNonContiguous)
{
BYTE const* const ip = (BYTE const*)src;
@@ -1380,9 +1274,8 @@ U32 ZSTD_window_update(ZSTD_window_t* window,
/* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
if ( (ip+srcSize > window->dictBase + window->lowLimit)
& (ip < window->dictBase + window->dictLimit)) {
size_t const highInputIdx = (size_t)((ip + srcSize) - window->dictBase);
U32 const lowLimitMax = (highInputIdx > (size_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx;
assert(highInputIdx < UINT_MAX);
ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase;
U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx;
window->lowLimit = lowLimitMax;
DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit);
}
@@ -1392,7 +1285,7 @@ U32 ZSTD_window_update(ZSTD_window_t* window,
/**
* Returns the lowest allowed match index. It may either be in the ext-dict or the prefix.
*/
MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_MatchState_t* ms, U32 curr, unsigned windowLog)
MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
{
U32 const maxDistance = 1U << windowLog;
U32 const lowestValid = ms->window.lowLimit;
@@ -1409,7 +1302,7 @@ MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_MatchState_t* ms, U32 curr, u
/**
* Returns the lowest allowed match index in the prefix.
*/
MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_MatchState_t* ms, U32 curr, unsigned windowLog)
MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog)
{
U32 const maxDistance = 1U << windowLog;
U32 const lowestValid = ms->window.dictLimit;
@@ -1422,13 +1315,6 @@ MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_MatchState_t* ms, U32 curr,
return matchLowest;
}
/* index_safety_check:
* intentional underflow : ensure repIndex isn't overlapping dict + prefix
* @return 1 if values are not overlapping,
* 0 otherwise */
MEM_STATIC int ZSTD_index_overlap_check(const U32 prefixLowestIndex, const U32 repIndex) {
return ((U32)((prefixLowestIndex-1) - repIndex) >= 3);
}
/* debug functions */
@@ -1499,6 +1385,10 @@ MEM_STATIC int ZSTD_comparePackedTags(size_t packedTag1, size_t packedTag2) {
return tag1 == tag2;
}
#if defined (__cplusplus)
}
#endif
/* ===============================================================
* Shared internal declarations
* These prototypes may be called from sources not in lib/compress
@@ -1514,25 +1404,6 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs);
typedef struct {
U32 idx; /* Index in array of ZSTD_Sequence */
U32 posInSequence; /* Position within sequence at idx */
size_t posInSrc; /* Number of bytes given by sequences provided so far */
} ZSTD_SequencePosition;
/* for benchmark */
size_t ZSTD_convertBlockSequences(ZSTD_CCtx* cctx,
const ZSTD_Sequence* const inSeqs, size_t nbSequences,
int repcodeResolution);
typedef struct {
size_t nbSequences;
size_t blockSize;
size_t litSize;
} BlockSummary;
BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs);
/* ==============================================================
* Private declarations
* These prototypes shall only be called from within lib/compress
@@ -1544,7 +1415,7 @@ BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs);
* Note: srcSizeHint == 0 means 0!
*/
ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_CParamMode_e mode);
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
/*! ZSTD_initCStream_internal() :
* Private use only. Init streaming operation.
@@ -1556,7 +1427,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
const ZSTD_CDict* cdict,
const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize);
void ZSTD_resetSeqStore(SeqStore_t* ssPtr);
void ZSTD_resetSeqStore(seqStore_t* ssPtr);
/*! ZSTD_getCParamsFromCDict() :
* as the name implies */
@@ -1609,6 +1480,33 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
*/
void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
* ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
* Note that the block delimiter must include the last literals of the block.
*/
size_t
ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
/* Returns the number of bytes to move the current read position back by.
* Only non-zero if we ended up splitting a sequence.
* Otherwise, it may return a ZSTD error if something went wrong.
*
* This function will attempt to scan through blockSize bytes
* represented by the sequences in @inSeqs,
* storing any (partial) sequences.
*
* Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
* avoid splitting a match, or to avoid splitting a match such that it would produce a match
* smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
*/
size_t
ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
/* Returns 1 if an external sequence producer is registered, otherwise returns 0. */
MEM_STATIC int ZSTD_hasExtSeqProd(const ZSTD_CCtx_params* params) {
return params->extSeqProdFunc != NULL;
+1 -1
View File
@@ -140,7 +140,7 @@ size_t ZSTD_compressLiterals (
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
BYTE* const ostart = (BYTE*)dst;
U32 singleStream = srcSize < 256;
SymbolEncodingType_e hType = set_compressed;
symbolEncodingType_e hType = set_compressed;
size_t cLitSize;
DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)",
+7 -7
View File
@@ -153,13 +153,13 @@ size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog,
return cost >> 8;
}
SymbolEncodingType_e
symbolEncodingType_e
ZSTD_selectEncodingType(
FSE_repeat* repeatMode, unsigned const* count, unsigned const max,
size_t const mostFrequent, size_t nbSeq, unsigned const FSELog,
FSE_CTable const* prevCTable,
short const* defaultNorm, U32 defaultNormLog,
ZSTD_DefaultPolicy_e const isDefaultAllowed,
ZSTD_defaultPolicy_e const isDefaultAllowed,
ZSTD_strategy const strategy)
{
ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0);
@@ -241,7 +241,7 @@ typedef struct {
size_t
ZSTD_buildCTable(void* dst, size_t dstCapacity,
FSE_CTable* nextCTable, U32 FSELog, SymbolEncodingType_e type,
FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
unsigned* count, U32 max,
const BYTE* codeTable, size_t nbSeq,
const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
@@ -293,7 +293,7 @@ ZSTD_encodeSequences_body(
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
SeqDef const* sequences, size_t nbSeq, int longOffsets)
seqDef const* sequences, size_t nbSeq, int longOffsets)
{
BIT_CStream_t blockStream;
FSE_CState_t stateMatchLength;
@@ -387,7 +387,7 @@ ZSTD_encodeSequences_default(
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
SeqDef const* sequences, size_t nbSeq, int longOffsets)
seqDef const* sequences, size_t nbSeq, int longOffsets)
{
return ZSTD_encodeSequences_body(dst, dstCapacity,
CTable_MatchLength, mlCodeTable,
@@ -405,7 +405,7 @@ ZSTD_encodeSequences_bmi2(
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
SeqDef const* sequences, size_t nbSeq, int longOffsets)
seqDef const* sequences, size_t nbSeq, int longOffsets)
{
return ZSTD_encodeSequences_body(dst, dstCapacity,
CTable_MatchLength, mlCodeTable,
@@ -421,7 +421,7 @@ size_t ZSTD_encodeSequences(
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2)
seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2)
{
DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity);
#if DYNAMIC_BMI2
+6 -7
View File
@@ -11,27 +11,26 @@
#ifndef ZSTD_COMPRESS_SEQUENCES_H
#define ZSTD_COMPRESS_SEQUENCES_H
#include "zstd_compress_internal.h" /* SeqDef */
#include "../common/fse.h" /* FSE_repeat, FSE_CTable */
#include "../common/zstd_internal.h" /* SymbolEncodingType_e, ZSTD_strategy */
#include "../common/zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */
typedef enum {
ZSTD_defaultDisallowed = 0,
ZSTD_defaultAllowed = 1
} ZSTD_DefaultPolicy_e;
} ZSTD_defaultPolicy_e;
SymbolEncodingType_e
symbolEncodingType_e
ZSTD_selectEncodingType(
FSE_repeat* repeatMode, unsigned const* count, unsigned const max,
size_t const mostFrequent, size_t nbSeq, unsigned const FSELog,
FSE_CTable const* prevCTable,
short const* defaultNorm, U32 defaultNormLog,
ZSTD_DefaultPolicy_e const isDefaultAllowed,
ZSTD_defaultPolicy_e const isDefaultAllowed,
ZSTD_strategy const strategy);
size_t
ZSTD_buildCTable(void* dst, size_t dstCapacity,
FSE_CTable* nextCTable, U32 FSELog, SymbolEncodingType_e type,
FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type,
unsigned* count, U32 max,
const BYTE* codeTable, size_t nbSeq,
const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax,
@@ -43,7 +42,7 @@ size_t ZSTD_encodeSequences(
FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable,
FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable,
FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable,
SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2);
seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2);
size_t ZSTD_fseBitCost(
FSE_CTable const* ctable,
+17 -17
View File
@@ -51,7 +51,7 @@ ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
BYTE* const oend = ostart + dstSize;
BYTE* op = ostart + lhSize;
U32 const singleStream = lhSize == 3;
SymbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
size_t cLitSize = 0;
DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy);
@@ -126,15 +126,15 @@ ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
}
static size_t
ZSTD_seqDecompressedSize(SeqStore_t const* seqStore,
const SeqDef* sequences, size_t nbSeqs,
ZSTD_seqDecompressedSize(seqStore_t const* seqStore,
const seqDef* sequences, size_t nbSeqs,
size_t litSize, int lastSubBlock)
{
size_t matchLengthSum = 0;
size_t litLengthSum = 0;
size_t n;
for (n=0; n<nbSeqs; n++) {
const ZSTD_SequenceLength seqLen = ZSTD_getSequenceLength(seqStore, sequences+n);
const ZSTD_sequenceLength seqLen = ZSTD_getSequenceLength(seqStore, sequences+n);
litLengthSum += seqLen.litLength;
matchLengthSum += seqLen.matchLength;
}
@@ -162,7 +162,7 @@ ZSTD_seqDecompressedSize(SeqStore_t const* seqStore,
static size_t
ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
const ZSTD_fseCTablesMetadata_t* fseMetadata,
const SeqDef* sequences, size_t nbSeq,
const seqDef* sequences, size_t nbSeq,
const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
const ZSTD_CCtx_params* cctxParams,
void* dst, size_t dstCapacity,
@@ -262,7 +262,7 @@ ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
* Or 0 if it failed to compress. */
static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
const SeqDef* sequences, size_t nbSeq,
const seqDef* sequences, size_t nbSeq,
const BYTE* literals, size_t litSize,
const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
const ZSTD_CCtx_params* cctxParams,
@@ -327,7 +327,7 @@ static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t lit
return 0;
}
static size_t ZSTD_estimateSubBlockSize_symbolType(SymbolEncodingType_e type,
static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
const BYTE* codeTable, unsigned maxCode,
size_t nbSeq, const FSE_CTable* fseCTable,
const U8* additionalBits,
@@ -426,7 +426,7 @@ static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMe
return 0;
}
static size_t countLiterals(SeqStore_t const* seqStore, const SeqDef* sp, size_t seqCount)
static size_t countLiterals(seqStore_t const* seqStore, const seqDef* sp, size_t seqCount)
{
size_t n, total = 0;
assert(sp != NULL);
@@ -439,7 +439,7 @@ static size_t countLiterals(SeqStore_t const* seqStore, const SeqDef* sp, size_t
#define BYTESCALE 256
static size_t sizeBlockSequences(const SeqDef* sp, size_t nbSeqs,
static size_t sizeBlockSequences(const seqDef* sp, size_t nbSeqs,
size_t targetBudget, size_t avgLitCost, size_t avgSeqCost,
int firstSubBlock)
{
@@ -476,7 +476,7 @@ static size_t sizeBlockSequences(const SeqDef* sp, size_t nbSeqs,
* Sub-blocks are all compressed, except the last one when beneficial.
* @return : compressed size of the super block (which features multiple ZSTD blocks)
* or 0 if it failed to compress. */
static size_t ZSTD_compressSubBlock_multi(const SeqStore_t* seqStorePtr,
static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
const ZSTD_compressedBlockState_t* prevCBlock,
ZSTD_compressedBlockState_t* nextCBlock,
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
@@ -486,9 +486,9 @@ static size_t ZSTD_compressSubBlock_multi(const SeqStore_t* seqStorePtr,
const int bmi2, U32 lastBlock,
void* workspace, size_t wkspSize)
{
const SeqDef* const sstart = seqStorePtr->sequencesStart;
const SeqDef* const send = seqStorePtr->sequences;
const SeqDef* sp = sstart; /* tracks progresses within seqStorePtr->sequences */
const seqDef* const sstart = seqStorePtr->sequencesStart;
const seqDef* const send = seqStorePtr->sequences;
const seqDef* sp = sstart; /* tracks progresses within seqStorePtr->sequences */
size_t const nbSeqs = (size_t)(send - sstart);
const BYTE* const lstart = seqStorePtr->litStart;
const BYTE* const lend = seqStorePtr->lit;
@@ -647,8 +647,8 @@ static size_t ZSTD_compressSubBlock_multi(const SeqStore_t* seqStorePtr,
op += cSize;
/* We have to regenerate the repcodes because we've skipped some sequences */
if (sp < send) {
const SeqDef* seq;
Repcodes_t rep;
const seqDef* seq;
repcodes_t rep;
ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
for (seq = sstart; seq < sp; ++seq) {
ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
@@ -674,7 +674,7 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
&zc->blockState.nextCBlock->entropy,
&zc->appliedParams,
&entropyMetadata,
zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */), "");
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
return ZSTD_compressSubBlock_multi(&zc->seqStore,
zc->blockState.prevCBlock,
@@ -684,5 +684,5 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
dst, dstCapacity,
src, srcSize,
zc->bmi2, lastBlock,
zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */);
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
}
+23 -40
View File
@@ -17,7 +17,10 @@
#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */
#include "../common/zstd_internal.h"
#include "../common/portability_macros.h"
#include "../common/compiler.h" /* ZS2_isPower2 */
#if defined (__cplusplus)
extern "C" {
#endif
/*-*************************************
* Constants
@@ -203,9 +206,9 @@ MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
/**
* Align must be a power of 2.
*/
MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t align) {
MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
size_t const mask = align - 1;
assert(ZSTD_isPower2(align));
assert((align & mask) == 0);
return (size + mask) & ~mask;
}
@@ -219,7 +222,7 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t align) {
* to figure out how much space you need for the matchState tables. Everything
* else is though.
*
* Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned64_alloc_size().
* Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size().
*/
MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
if (size == 0)
@@ -231,16 +234,12 @@ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
#endif
}
MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size, size_t alignment) {
return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, alignment));
}
/**
* Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes.
* Used to determine the number of bytes required for a given "aligned".
*/
MEM_STATIC size_t ZSTD_cwksp_aligned64_alloc_size(size_t size) {
return ZSTD_cwksp_aligned_alloc_size(size, ZSTD_CWKSP_ALIGNMENT_BYTES);
MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) {
return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES));
}
/**
@@ -263,7 +262,7 @@ MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) {
MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) {
size_t const alignBytesMask = alignBytes - 1;
size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask;
assert(ZSTD_isPower2(alignBytes));
assert((alignBytes & alignBytesMask) == 0);
assert(bytes < alignBytes);
return bytes;
}
@@ -272,12 +271,8 @@ MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignByt
* Returns the initial value for allocStart which is used to determine the position from
* which we can allocate from the end of the workspace.
*/
MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws)
{
char* endPtr = (char*)ws->workspaceEnd;
assert(ZSTD_isPower2(ZSTD_CWKSP_ALIGNMENT_BYTES));
endPtr = endPtr - ((size_t)endPtr % ZSTD_CWKSP_ALIGNMENT_BYTES);
return (void*)endPtr;
MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) {
return (void*)((size_t)ws->workspaceEnd & ~(ZSTD_CWKSP_ALIGNMENT_BYTES-1));
}
/**
@@ -292,7 +287,7 @@ ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes)
{
void* const alloc = (BYTE*)ws->allocStart - bytes;
void* const bottom = ws->tableEnd;
DEBUGLOG(5, "cwksp: reserving [0x%p]:%zd bytes; %zd bytes remaining",
DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
ZSTD_cwksp_assert_internal_consistency(ws);
assert(alloc >= bottom);
@@ -409,7 +404,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t byt
{
size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES);
void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once);
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
if(ptr && ptr < ws->initOnceStart) {
/* We assume the memory following the current allocation is either:
* 1. Not usable as initOnce memory (end of workspace)
@@ -429,12 +424,11 @@ MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t byt
/**
* Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
*/
MEM_STATIC void* ZSTD_cwksp_reserve_aligned64(ZSTD_cwksp* ws, size_t bytes)
MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes)
{
void* const ptr = ZSTD_cwksp_reserve_internal(ws,
ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES),
void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES),
ZSTD_cwksp_alloc_aligned);
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
return ptr;
}
@@ -480,7 +474,7 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes)
#endif
assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
return alloc;
}
@@ -526,20 +520,6 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes)
return alloc;
}
/**
* with alignment control
* Note : should happen only once, at workspace first initialization
*/
MEM_STATIC void* ZSTD_cwksp_reserve_object_aligned(ZSTD_cwksp* ws, size_t byteSize, size_t alignment)
{
size_t const mask = alignment - 1;
size_t const surplus = (alignment > sizeof(void*)) ? alignment - sizeof(void*) : 0;
void* const start = ZSTD_cwksp_reserve_object(ws, byteSize + surplus);
if (start == NULL) return NULL;
if (surplus == 0) return start;
assert(ZSTD_isPower2(alignment));
return (void*)(((size_t)start + surplus) & ~mask);
}
MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws)
{
@@ -597,8 +577,7 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
* Invalidates table allocations.
* All other allocations remain valid.
*/
MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws)
{
MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
DEBUGLOG(4, "cwksp: clearing tables!");
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
@@ -762,4 +741,8 @@ MEM_STATIC void ZSTD_cwksp_bump_oversized_duration(
}
}
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_CWKSP_H */
+42 -50
View File
@@ -15,7 +15,7 @@
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillDoubleHashTableForCDict(ZSTD_MatchState_t* ms,
void ZSTD_fillDoubleHashTableForCDict(ZSTD_matchState_t* ms,
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
@@ -53,7 +53,7 @@ void ZSTD_fillDoubleHashTableForCDict(ZSTD_MatchState_t* ms,
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillDoubleHashTableForCCtx(ZSTD_MatchState_t* ms,
void ZSTD_fillDoubleHashTableForCCtx(ZSTD_matchState_t* ms,
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
@@ -87,7 +87,7 @@ void ZSTD_fillDoubleHashTableForCCtx(ZSTD_MatchState_t* ms,
} }
}
void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms,
void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
const void* const end,
ZSTD_dictTableLoadMethod_e dtlm,
ZSTD_tableFillPurpose_e tfp)
@@ -103,7 +103,7 @@ void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms,
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_doubleFast_noDict_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, U32 const mls /* template */)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
@@ -142,14 +142,9 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
const BYTE* matchl0; /* the long match for ip */
const BYTE* matchs0; /* the short match for ip */
const BYTE* matchl1; /* the long match for ip1 */
const BYTE* matchs0_safe; /* matchs0 or safe address */
const BYTE* ip = istart; /* the current position */
const BYTE* ip1; /* the next position */
/* Array of ~random data, should have low probability of matching data
* we load from here instead of from tables, if matchl0/matchl1 are
* invalid indices. Used to avoid unpredictable branches. */
const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0,0xe2,0xb4};
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic");
@@ -196,30 +191,25 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
hl1 = ZSTD_hashPtr(ip1, hBitsL, 8);
/* idxl0 > prefixLowestIndex is a (somewhat) unpredictable branch.
* However expression below complies into conditional move. Since
* match is unlikely and we only *branch* on idxl0 > prefixLowestIndex
* if there is a match, all branches become predictable. */
{ const BYTE* const matchl0_safe = ZSTD_selectAddr(idxl0, prefixLowestIndex, matchl0, &dummy[0]);
if (idxl0 > prefixLowestIndex) {
/* check prefix long match */
if (MEM_read64(matchl0_safe) == MEM_read64(ip) && matchl0_safe == matchl0) {
if (MEM_read64(matchl0) == MEM_read64(ip)) {
mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8;
offset = (U32)(ip-matchl0);
while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */
goto _match_found;
} }
}
}
idxl1 = hashLong[hl1];
matchl1 = base + idxl1;
/* Same optimization as matchl0 above */
matchs0_safe = ZSTD_selectAddr(idxs0, prefixLowestIndex, matchs0, &dummy[0]);
if (idxs0 > prefixLowestIndex) {
/* check prefix short match */
if(MEM_read32(matchs0_safe) == MEM_read32(ip) && matchs0_safe == matchs0) {
if (MEM_read32(matchs0) == MEM_read32(ip)) {
goto _search_next_long;
}
}
if (ip1 >= nextStep) {
PREFETCH_L1(ip1 + 64);
@@ -252,23 +242,21 @@ _cleanup:
_search_next_long:
/* short match found: let's check for a longer one */
/* check prefix long +1 match */
if (idxl1 > prefixLowestIndex) {
if (MEM_read64(matchl1) == MEM_read64(ip1)) {
ip = ip1;
mLength = ZSTD_count(ip+8, matchl1+8, iend) + 8;
offset = (U32)(ip-matchl1);
while (((ip>anchor) & (matchl1>prefixLowest)) && (ip[-1] == matchl1[-1])) { ip--; matchl1--; mLength++; } /* catch up */
goto _match_found;
}
}
/* if no long +1 match, explore the short match we found */
mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
offset = (U32)(ip - matchs0);
/* check long match at +1 position */
if ((idxl1 > prefixLowestIndex) && (MEM_read64(matchl1) == MEM_read64(ip1))) {
size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8;
if (l1len > mLength) {
/* use the long match instead */
ip = ip1;
mLength = l1len;
offset = (U32)(ip-matchl1);
matchs0 = matchl1;
}
}
while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* complete backward */
while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* catch up */
/* fall-through */
@@ -326,7 +314,7 @@ _match_stored:
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize,
U32 const mls /* template */)
{
@@ -347,7 +335,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
const BYTE* const ilimit = iend - HASH_READ_SIZE;
U32 offset_1=rep[0], offset_2=rep[1];
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const ZSTD_matchState_t* const dms = ms->dictMatchState;
const ZSTD_compressionParameters* const dictCParams = &dms->cParams;
const U32* const dictHashLong = dms->hashTable;
const U32* const dictHashSmall = dms->chainTable;
@@ -404,7 +392,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
/* check repcode */
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -413,12 +401,14 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
goto _match_stored;
}
if ((matchIndexL >= prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
if (matchIndexL > prefixLowestIndex) {
/* check prefix long match */
if (MEM_read64(matchLong) == MEM_read64(ip)) {
mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
offset = (U32)(ip-matchLong);
while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
goto _match_found;
}
} else if (dictTagsMatchL) {
/* check dictMatchState long match */
U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS;
@@ -433,7 +423,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
} }
if (matchIndexS > prefixLowestIndex) {
/* short match candidate */
/* check prefix short match */
if (MEM_read32(match) == MEM_read32(ip)) {
goto _search_next_long;
}
@@ -463,12 +453,14 @@ _search_next_long:
hashLong[hl3] = curr + 1;
/* check prefix long +1 match */
if ((matchIndexL3 >= prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1))) {
if (matchIndexL3 > prefixLowestIndex) {
if (MEM_read64(matchL3) == MEM_read64(ip+1)) {
mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
ip++;
offset = (U32)(ip-matchL3);
while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
goto _match_found;
}
} else if (dictTagsMatchL3) {
/* check dict long +1 match */
U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS;
@@ -521,7 +513,7 @@ _match_stored:
const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ?
dictBase + repIndex2 - dictIndexDelta :
base + repIndex2;
if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex2))
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
@@ -548,7 +540,7 @@ _match_stored:
#define ZSTD_GEN_DFAST_FN(dictMode, mls) \
static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
void const* src, size_t srcSize) \
{ \
return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \
@@ -566,7 +558,7 @@ ZSTD_GEN_DFAST_FN(dictMatchState, 7)
size_t ZSTD_compressBlock_doubleFast(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
const U32 mls = ms->cParams.minMatch;
@@ -586,7 +578,7 @@ size_t ZSTD_compressBlock_doubleFast(
size_t ZSTD_compressBlock_doubleFast_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
const U32 mls = ms->cParams.minMatch;
@@ -608,7 +600,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState(
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_doubleFast_extDict_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize,
U32 const mls /* template */)
{
@@ -659,7 +651,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic(
size_t mLength;
hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
if (((ZSTD_index_overlap_check(prefixStartIndex, repIndex))
if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */
& (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
@@ -727,7 +719,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict_generic(
U32 const current2 = (U32)(ip-base);
U32 const repIndex2 = current2 - offset_2;
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2))
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */
& (offset_2 <= current2 - dictStartIndex))
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
@@ -757,7 +749,7 @@ ZSTD_GEN_DFAST_FN(extDict, 6)
ZSTD_GEN_DFAST_FN(extDict, 7)
size_t ZSTD_compressBlock_doubleFast_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
U32 const mls = ms->cParams.minMatch;
+12 -4
View File
@@ -11,23 +11,27 @@
#ifndef ZSTD_DOUBLE_FAST_H
#define ZSTD_DOUBLE_FAST_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "../common/mem.h" /* U32 */
#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms,
void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
void const* end, ZSTD_dictTableLoadMethod_e dtlm,
ZSTD_tableFillPurpose_e tfp);
size_t ZSTD_compressBlock_doubleFast(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_doubleFast_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_doubleFast_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST ZSTD_compressBlock_doubleFast
@@ -39,4 +43,8 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT NULL
#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_DOUBLE_FAST_H */
+72 -89
View File
@@ -13,7 +13,7 @@
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillHashTableForCDict(ZSTD_MatchState_t* ms,
void ZSTD_fillHashTableForCDict(ZSTD_matchState_t* ms,
const void* const end,
ZSTD_dictTableLoadMethod_e dtlm)
{
@@ -50,7 +50,7 @@ void ZSTD_fillHashTableForCDict(ZSTD_MatchState_t* ms,
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillHashTableForCCtx(ZSTD_MatchState_t* ms,
void ZSTD_fillHashTableForCCtx(ZSTD_matchState_t* ms,
const void* const end,
ZSTD_dictTableLoadMethod_e dtlm)
{
@@ -84,7 +84,7 @@ void ZSTD_fillHashTableForCCtx(ZSTD_MatchState_t* ms,
} } } }
}
void ZSTD_fillHashTable(ZSTD_MatchState_t* ms,
void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
const void* const end,
ZSTD_dictTableLoadMethod_e dtlm,
ZSTD_tableFillPurpose_e tfp)
@@ -97,50 +97,6 @@ void ZSTD_fillHashTable(ZSTD_MatchState_t* ms,
}
typedef int (*ZSTD_match4Found) (const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit);
static int
ZSTD_match4Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit)
{
/* Array of ~random data, should have low probability of matching data.
* Load from here if the index is invalid.
* Used to avoid unpredictable branches. */
static const BYTE dummy[] = {0x12,0x34,0x56,0x78};
/* currentIdx >= lowLimit is a (somewhat) unpredictable branch.
* However expression below compiles into conditional move.
*/
const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy);
/* Note: this used to be written as : return test1 && test2;
* Unfortunately, once inlined, these tests become branches,
* in which case it becomes critical that they are executed in the right order (test1 then test2).
* So we have to write these tests in a specific manner to ensure their ordering.
*/
if (MEM_read32(currentPtr) != MEM_read32(mvalAddr)) return 0;
/* force ordering of these tests, which matters once the function is inlined, as they become branches */
#if defined(__GNUC__)
__asm__("");
#endif
return matchIdx >= idxLowLimit;
}
static int
ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit)
{
/* using a branch instead of a cmov,
* because it's faster in scenarios where matchIdx >= idxLowLimit is generally true,
* aka almost all candidates are within range */
U32 mval;
if (matchIdx >= idxLowLimit) {
mval = MEM_read32(matchAddress);
} else {
mval = MEM_read32(currentPtr) ^ 1; /* guaranteed to not match. */
}
return (MEM_read32(currentPtr) == mval);
}
/**
* If you squint hard enough (and ignore repcodes), the search operation at any
* given position is broken into 4 stages:
@@ -190,14 +146,15 @@ ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 ma
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_fast_noDict_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize,
U32 const mls, int useCmov)
U32 const mls, U32 const hasStep)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
U32 const hlog = cParams->hashLog;
size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; /* min 2 */
/* support stepSize of 0 */
size_t const stepSize = hasStep ? (cParams->targetLength + !(cParams->targetLength) + 1) : 2;
const BYTE* const base = ms->window.base;
const BYTE* const istart = (const BYTE*)src;
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
@@ -219,7 +176,8 @@ size_t ZSTD_compressBlock_fast_noDict_generic(
size_t hash0; /* hash for ip0 */
size_t hash1; /* hash for ip1 */
U32 matchIdx; /* match idx for ip0 */
U32 idx; /* match idx for ip0 */
U32 mval; /* src value at match idx */
U32 offcode;
const BYTE* match0;
@@ -232,7 +190,6 @@ size_t ZSTD_compressBlock_fast_noDict_generic(
size_t step;
const BYTE* nextStep;
const size_t kStepIncr = (1 << (kSearchStrength - 1));
const ZSTD_match4Found matchFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch;
DEBUGLOG(5, "ZSTD_compressBlock_fast_generic");
ip0 += (ip0 == prefixStart);
@@ -261,7 +218,7 @@ _start: /* Requires: ip0 */
hash0 = ZSTD_hashPtr(ip0, hlog, mls);
hash1 = ZSTD_hashPtr(ip1, hlog, mls);
matchIdx = hashTable[hash0];
idx = hashTable[hash0];
do {
/* load repcode match for ip[2]*/
@@ -281,25 +238,35 @@ _start: /* Requires: ip0 */
offcode = REPCODE1_TO_OFFBASE;
mLength += 4;
/* Write next hash table entry: it's already calculated.
* This write is known to be safe because ip1 is before the
/* First write next hash table entry; we've already calculated it.
* This write is known to be safe because the ip1 is before the
* repcode (ip2). */
hashTable[hash1] = (U32)(ip1 - base);
goto _match;
}
if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) {
/* Write next hash table entry (it's already calculated).
* This write is known to be safe because the ip1 == ip0 + 1,
* so searching will resume after ip1 */
/* load match for ip[0] */
if (idx >= prefixStartIndex) {
mval = MEM_read32(base + idx);
} else {
mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */
}
/* check match at ip[0] */
if (MEM_read32(ip0) == mval) {
/* found a match! */
/* First write next hash table entry; we've already calculated it.
* This write is known to be safe because the ip1 == ip0 + 1, so
* we know we will resume searching after ip1 */
hashTable[hash1] = (U32)(ip1 - base);
goto _offset;
}
/* lookup ip[1] */
matchIdx = hashTable[hash1];
idx = hashTable[hash1];
/* hash ip[2] */
hash0 = hash1;
@@ -314,19 +281,36 @@ _start: /* Requires: ip0 */
current0 = (U32)(ip0 - base);
hashTable[hash0] = current0;
if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) {
/* Write next hash table entry, since it's already calculated */
/* load match for ip[0] */
if (idx >= prefixStartIndex) {
mval = MEM_read32(base + idx);
} else {
mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */
}
/* check match at ip[0] */
if (MEM_read32(ip0) == mval) {
/* found a match! */
/* first write next hash table entry; we've already calculated it */
if (step <= 4) {
/* Avoid writing an index if it's >= position where search will resume.
* The minimum possible match has length 4, so search can resume at ip0 + 4.
/* We need to avoid writing an index into the hash table >= the
* position at which we will pick up our searching after we've
* taken this match.
*
* The minimum possible match has length 4, so the earliest ip0
* can be after we take this match will be the current ip0 + 4.
* ip1 is ip0 + step - 1. If ip1 is >= ip0 + 4, we can't safely
* write this position.
*/
hashTable[hash1] = (U32)(ip1 - base);
}
goto _offset;
}
/* lookup ip[1] */
matchIdx = hashTable[hash1];
idx = hashTable[hash1];
/* hash ip[2] */
hash0 = hash1;
@@ -348,7 +332,7 @@ _start: /* Requires: ip0 */
} while (ip3 < ilimit);
_cleanup:
/* Note that there are probably still a couple positions one could search.
/* Note that there are probably still a couple positions we could search.
* However, it seems to be a meaningful performance hit to try to search
* them. So let's not. */
@@ -377,7 +361,7 @@ _cleanup:
_offset: /* Requires: ip0, idx */
/* Compute the offset code. */
match0 = base + matchIdx;
match0 = base + idx;
rep_offset2 = rep_offset1;
rep_offset1 = (U32)(ip0-match0);
offcode = OFFSET_TO_OFFBASE(rep_offset1);
@@ -422,12 +406,12 @@ _match: /* Requires: ip0, match0, offcode */
goto _start;
}
#define ZSTD_GEN_FAST_FN(dictMode, mml, cmov) \
static size_t ZSTD_compressBlock_fast_##dictMode##_##mml##_##cmov( \
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
#define ZSTD_GEN_FAST_FN(dictMode, mls, step) \
static size_t ZSTD_compressBlock_fast_##dictMode##_##mls##_##step( \
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
void const* src, size_t srcSize) \
{ \
return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mml, cmov); \
return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, step); \
}
ZSTD_GEN_FAST_FN(noDict, 4, 1)
@@ -441,15 +425,13 @@ ZSTD_GEN_FAST_FN(noDict, 6, 0)
ZSTD_GEN_FAST_FN(noDict, 7, 0)
size_t ZSTD_compressBlock_fast(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
U32 const mml = ms->cParams.minMatch;
/* use cmov when "candidate in range" branch is likely unpredictable */
int const useCmov = ms->cParams.windowLog < 19;
U32 const mls = ms->cParams.minMatch;
assert(ms->dictMatchState == NULL);
if (useCmov) {
switch(mml)
if (ms->cParams.targetLength > 1) {
switch(mls)
{
default: /* includes case 3 */
case 4 :
@@ -462,8 +444,7 @@ size_t ZSTD_compressBlock_fast(
return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize);
}
} else {
/* use a branch instead */
switch(mml)
switch(mls)
{
default: /* includes case 3 */
case 4 :
@@ -475,13 +456,14 @@ size_t ZSTD_compressBlock_fast(
case 7 :
return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize);
}
}
}
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_fast_dictMatchState_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
@@ -500,7 +482,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
const BYTE* const ilimit = iend - HASH_READ_SIZE;
U32 offset_1=rep[0], offset_2=rep[1];
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const ZSTD_matchState_t* const dms = ms->dictMatchState;
const ZSTD_compressionParameters* const dictCParams = &dms->cParams ;
const U32* const dictHashTable = dms->hashTable;
const U32 dictStartIndex = dms->window.dictLimit;
@@ -564,7 +546,8 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls);
hashTable[hash0] = curr; /* update hash table */
if ((ZSTD_index_overlap_check(prefixStartIndex, repIndex))
if (((U32) ((prefixStartIndex - 1) - repIndex) >=
3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
&& (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) {
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4;
@@ -597,8 +580,8 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
}
}
if (ZSTD_match4Found_cmov(ip0, match, matchIndex, prefixStartIndex)) {
/* found a regular match of size >= 4 */
if (matchIndex > prefixStartIndex && MEM_read32(match) == MEM_read32(ip0)) {
/* found a regular match */
U32 const offset = (U32) (ip0 - match);
mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4;
while (((ip0 > anchor) & (match > prefixStart))
@@ -648,7 +631,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
dictBase - dictIndexDelta + repIndex2 :
base + repIndex2;
if ( (ZSTD_index_overlap_check(prefixStartIndex, repIndex2))
if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
&& (MEM_read32(repMatch2) == MEM_read32(ip0))) {
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
@@ -684,7 +667,7 @@ ZSTD_GEN_FAST_FN(dictMatchState, 6, 0)
ZSTD_GEN_FAST_FN(dictMatchState, 7, 0)
size_t ZSTD_compressBlock_fast_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
U32 const mls = ms->cParams.minMatch;
@@ -707,7 +690,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_fast_extDict_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
@@ -942,7 +925,7 @@ _match: /* Requires: ip0, match0, offcode, matchEnd */
while (ip0 <= ilimit) {
U32 const repIndex2 = (U32)(ip0-base) - offset_2;
const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) & (offset_2 > 0))
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 > 0)) /* intentional underflow */
&& (MEM_read32(repMatch2) == MEM_read32(ip0)) ) {
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
@@ -965,7 +948,7 @@ ZSTD_GEN_FAST_FN(extDict, 6, 0)
ZSTD_GEN_FAST_FN(extDict, 7, 0)
size_t ZSTD_compressBlock_fast_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
U32 const mls = ms->cParams.minMatch;
+12 -4
View File
@@ -11,20 +11,28 @@
#ifndef ZSTD_FAST_H
#define ZSTD_FAST_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "../common/mem.h" /* U32 */
#include "zstd_compress_internal.h"
void ZSTD_fillHashTable(ZSTD_MatchState_t* ms,
void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
void const* end, ZSTD_dictTableLoadMethod_e dtlm,
ZSTD_tableFillPurpose_e tfp);
size_t ZSTD_compressBlock_fast(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_fast_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_fast_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_FAST_H */
+67 -111
View File
@@ -26,7 +26,7 @@
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_updateDUBT(ZSTD_MatchState_t* ms,
void ZSTD_updateDUBT(ZSTD_matchState_t* ms,
const BYTE* ip, const BYTE* iend,
U32 mls)
{
@@ -71,7 +71,7 @@ void ZSTD_updateDUBT(ZSTD_MatchState_t* ms,
* doesn't fail */
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_insertDUBT1(const ZSTD_MatchState_t* ms,
void ZSTD_insertDUBT1(const ZSTD_matchState_t* ms,
U32 curr, const BYTE* inputEnd,
U32 nbCompares, U32 btLow,
const ZSTD_dictMode_e dictMode)
@@ -162,7 +162,7 @@ void ZSTD_insertDUBT1(const ZSTD_MatchState_t* ms,
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_DUBT_findBetterDictMatch (
const ZSTD_MatchState_t* ms,
const ZSTD_matchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
size_t* offsetPtr,
size_t bestLength,
@@ -170,7 +170,7 @@ size_t ZSTD_DUBT_findBetterDictMatch (
U32 const mls,
const ZSTD_dictMode_e dictMode)
{
const ZSTD_MatchState_t * const dms = ms->dictMatchState;
const ZSTD_matchState_t * const dms = ms->dictMatchState;
const ZSTD_compressionParameters* const dmsCParams = &dms->cParams;
const U32 * const dictHashTable = dms->hashTable;
U32 const hashLog = dmsCParams->hashLog;
@@ -240,7 +240,7 @@ size_t ZSTD_DUBT_findBetterDictMatch (
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_DUBT_findBestMatch(ZSTD_MatchState_t* ms,
size_t ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
size_t* offBasePtr,
U32 const mls,
@@ -392,7 +392,7 @@ size_t ZSTD_DUBT_findBestMatch(ZSTD_MatchState_t* ms,
/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_BtFindBestMatch( ZSTD_MatchState_t* ms,
size_t ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offBasePtr,
const U32 mls /* template */,
@@ -408,7 +408,7 @@ size_t ZSTD_BtFindBestMatch( ZSTD_MatchState_t* ms,
* Dedicated dict search
***********************************/
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_MatchState_t* ms, const BYTE* const ip)
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip)
{
const BYTE* const base = ms->window.base;
U32 const target = (U32)(ip - base);
@@ -527,7 +527,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_MatchState_t* ms, const B
*/
FORCE_INLINE_TEMPLATE
size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts,
const ZSTD_MatchState_t* const dms,
const ZSTD_matchState_t* const dms,
const BYTE* const ip, const BYTE* const iLimit,
const BYTE* const prefixStart, const U32 curr,
const U32 dictLimit, const size_t ddsIdx) {
@@ -630,7 +630,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_insertAndFindFirstIndex_internal(
ZSTD_MatchState_t* ms,
ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams,
const BYTE* ip, U32 const mls, U32 const lazySkipping)
{
@@ -656,7 +656,7 @@ U32 ZSTD_insertAndFindFirstIndex_internal(
return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
}
U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip) {
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
const ZSTD_compressionParameters* const cParams = &ms->cParams;
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch, /* lazySkipping*/ 0);
}
@@ -665,7 +665,7 @@ U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip) {
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_HcFindBestMatch(
ZSTD_MatchState_t* ms,
ZSTD_matchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 mls, const ZSTD_dictMode_e dictMode)
@@ -689,7 +689,7 @@ size_t ZSTD_HcFindBestMatch(
U32 nbAttempts = 1U << cParams->searchLog;
size_t ml=4-1;
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const ZSTD_matchState_t* const dms = ms->dictMatchState;
const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch
? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0;
const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch
@@ -834,7 +834,7 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, BYTE const* t
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_row_fillHashCache(ZSTD_MatchState_t* ms, const BYTE* base,
void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const BYTE* base,
U32 const rowLog, U32 const mls,
U32 idx, const BYTE* const iLimit)
{
@@ -882,7 +882,7 @@ U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable,
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_row_update_internalImpl(ZSTD_MatchState_t* ms,
void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms,
U32 updateStartIdx, U32 const updateEndIdx,
U32 const mls, U32 const rowLog,
U32 const rowMask, U32 const useCache)
@@ -913,7 +913,7 @@ void ZSTD_row_update_internalImpl(ZSTD_MatchState_t* ms,
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_row_update_internal(ZSTD_MatchState_t* ms, const BYTE* ip,
void ZSTD_row_update_internal(ZSTD_matchState_t* ms, const BYTE* ip,
U32 const mls, U32 const rowLog,
U32 const rowMask, U32 const useCache)
{
@@ -946,7 +946,7 @@ void ZSTD_row_update_internal(ZSTD_MatchState_t* ms, const BYTE* ip,
* External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary
* processing.
*/
void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* ip) {
void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) {
const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
const U32 rowMask = (1u << rowLog) - 1;
const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */);
@@ -1050,44 +1050,6 @@ ZSTD_row_getNEONMask(const U32 rowEntries, const BYTE* const src, const BYTE tag
}
}
#endif
#if defined(ZSTD_ARCH_RISCV_RVV) && (__riscv_xlen == 64)
FORCE_INLINE_TEMPLATE ZSTD_VecMask
ZSTD_row_getRVVMask(int rowEntries, const BYTE* const src, const BYTE tag, const U32 head)
{
ZSTD_VecMask matches;
size_t vl;
if (rowEntries == 16) {
vl = __riscv_vsetvl_e8m1(16);
{
vuint8m1_t chunk = __riscv_vle8_v_u8m1(src, vl);
vbool8_t mask = __riscv_vmseq_vx_u8m1_b8(chunk, tag, vl);
vuint16m1_t mask_u16 = __riscv_vreinterpret_v_b8_u16m1(mask);
matches = __riscv_vmv_x_s_u16m1_u16(mask_u16);
return ZSTD_rotateRight_U16((U16)matches, head);
}
} else if (rowEntries == 32) {
vl = __riscv_vsetvl_e8m2(32);
{
vuint8m2_t chunk = __riscv_vle8_v_u8m2(src, vl);
vbool4_t mask = __riscv_vmseq_vx_u8m2_b4(chunk, tag, vl);
vuint32m1_t mask_u32 = __riscv_vreinterpret_v_b4_u32m1(mask);
matches = __riscv_vmv_x_s_u32m1_u32(mask_u32);
return ZSTD_rotateRight_U32((U32)matches, head);
}
} else { // rowEntries = 64
vl = __riscv_vsetvl_e8m4(64);
{
vuint8m4_t chunk = __riscv_vle8_v_u8m4(src, vl);
vbool2_t mask = __riscv_vmseq_vx_u8m4_b2(chunk, tag, vl);
vuint64m1_t mask_u64 = __riscv_vreinterpret_v_b2_u64m1(mask);
matches = __riscv_vmv_x_s_u64m1_u64(mask_u64);
return ZSTD_rotateRight_U64(matches, head);
}
}
}
#endif
/* Returns a ZSTD_VecMask (U64) that has the nth group (determined by
* ZSTD_row_matchMaskGroupWidth) of bits set to 1 if the newly-computed "tag"
@@ -1107,20 +1069,14 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr
return ZSTD_row_getSSEMask(rowEntries / 16, src, tag, headGrouped);
#elif defined(ZSTD_ARCH_RISCV_RVV) && (__riscv_xlen == 64)
#else /* SW or NEON-LE */
return ZSTD_row_getRVVMask(rowEntries, src, tag, headGrouped);
#else
#if defined(ZSTD_ARCH_ARM_NEON)
# if defined(ZSTD_ARCH_ARM_NEON)
/* This NEON path only works for little endian - otherwise use SWAR below */
if (MEM_isLittleEndian()) {
return ZSTD_row_getNEONMask(rowEntries, src, tag, headGrouped);
}
#endif
# endif /* ZSTD_ARCH_ARM_NEON */
/* SWAR */
{ const int chunkSize = sizeof(size_t);
const size_t shiftAmount = ((chunkSize * 8) - chunkSize);
@@ -1167,9 +1123,9 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr
/* The high-level approach of the SIMD row based match finder is as follows:
* - Figure out where to insert the new entry:
* - Generate a hash for current input position and split it into a one byte of tag and `rowHashLog` bits of index.
* - The hash is salted by a value that changes on every context reset, so when the same table is used
* we will avoid collisions that would otherwise slow us down by introducing phantom matches.
* - Generate a hash for current input posistion and split it into a one byte of tag and `rowHashLog` bits of index.
* - The hash is salted by a value that changes on every contex reset, so when the same table is used
* we will avoid collisions that would otherwise slow us down by intorducing phantom matches.
* - The hashTable is effectively split into groups or "rows" of 15 or 31 entries of U32, and the index determines
* which row to insert into.
* - Determine the correct position within the row to insert the entry into. Each row of 15 or 31 can
@@ -1183,7 +1139,7 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGr
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_RowFindBestMatch(
ZSTD_MatchState_t* ms,
ZSTD_matchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 mls, const ZSTD_dictMode_e dictMode,
@@ -1215,7 +1171,7 @@ size_t ZSTD_RowFindBestMatch(
U32 hash;
/* DMS/DDS variables that may be referenced laster */
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const ZSTD_matchState_t* const dms = ms->dictMatchState;
/* Initialize the following variables to satisfy static analyzer */
size_t ddsIdx = 0;
@@ -1384,7 +1340,7 @@ size_t ZSTD_RowFindBestMatch(
* ZSTD_searchMax() dispatches to the correct implementation function.
*
* TODO: The start of the search function involves loading and calculating a
* bunch of constants from the ZSTD_MatchState_t. These computations could be
* bunch of constants from the ZSTD_matchState_t. These computations could be
* done in an initialization function, and saved somewhere in the match state.
* Then we could pass a pointer to the saved state instead of the match state,
* and avoid duplicate computations.
@@ -1408,7 +1364,7 @@ size_t ZSTD_RowFindBestMatch(
#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls) \
ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)( \
ZSTD_MatchState_t* ms, \
ZSTD_matchState_t* ms, \
const BYTE* ip, const BYTE* const iLimit, \
size_t* offBasePtr) \
{ \
@@ -1418,7 +1374,7 @@ size_t ZSTD_RowFindBestMatch(
#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls) \
ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)( \
ZSTD_MatchState_t* ms, \
ZSTD_matchState_t* ms, \
const BYTE* ip, const BYTE* const iLimit, \
size_t* offsetPtr) \
{ \
@@ -1428,7 +1384,7 @@ size_t ZSTD_RowFindBestMatch(
#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) \
ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)( \
ZSTD_MatchState_t* ms, \
ZSTD_matchState_t* ms, \
const BYTE* ip, const BYTE* const iLimit, \
size_t* offsetPtr) \
{ \
@@ -1529,7 +1485,7 @@ typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searc
* If a match is found its offset is stored in @p offsetPtr.
*/
FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax(
ZSTD_MatchState_t* ms,
ZSTD_matchState_t* ms,
const BYTE* ip,
const BYTE* iend,
size_t* offsetPtr,
@@ -1558,7 +1514,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax(
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_lazy_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore,
ZSTD_matchState_t* ms, seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const searchMethod_e searchMethod, const U32 depth,
@@ -1581,7 +1537,7 @@ size_t ZSTD_compressBlock_lazy_generic(
const int isDMS = dictMode == ZSTD_dictMatchState;
const int isDDS = dictMode == ZSTD_dedicatedDictSearch;
const int isDxS = isDMS || isDDS;
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const ZSTD_matchState_t* const dms = ms->dictMatchState;
const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0;
const BYTE* const dictBase = isDxS ? dms->window.base : NULL;
const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL;
@@ -1634,7 +1590,7 @@ size_t ZSTD_compressBlock_lazy_generic(
&& repIndex < prefixLowestIndex) ?
dictBase + (repIndex - dictIndexDelta) :
base + repIndex;
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -1686,7 +1642,7 @@ size_t ZSTD_compressBlock_lazy_generic(
const BYTE* repMatch = repIndex < prefixLowestIndex ?
dictBase + (repIndex - dictIndexDelta) :
base + repIndex;
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
&& (MEM_read32(repMatch) == MEM_read32(ip)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -1722,7 +1678,7 @@ size_t ZSTD_compressBlock_lazy_generic(
const BYTE* repMatch = repIndex < prefixLowestIndex ?
dictBase + (repIndex - dictIndexDelta) :
base + repIndex;
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */)
&& (MEM_read32(repMatch) == MEM_read32(ip)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
@@ -1784,7 +1740,7 @@ _storeSequence:
const BYTE* repMatch = repIndex < prefixLowestIndex ?
dictBase - dictIndexDelta + repIndex :
base + repIndex;
if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */)
&& (MEM_read32(repMatch) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4;
@@ -1826,42 +1782,42 @@ _storeSequence:
#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_greedy(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict);
}
size_t ZSTD_compressBlock_greedy_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch);
}
size_t ZSTD_compressBlock_greedy_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict);
}
size_t ZSTD_compressBlock_greedy_dictMatchState_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch);
@@ -1870,42 +1826,42 @@ size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch);
}
size_t ZSTD_compressBlock_lazy_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy_dictMatchState_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch);
@@ -1914,42 +1870,42 @@ size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy2(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy2_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch);
}
size_t ZSTD_compressBlock_lazy2_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch);
@@ -1958,14 +1914,14 @@ size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btlazy2(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict);
}
size_t ZSTD_compressBlock_btlazy2_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState);
@@ -1979,7 +1935,7 @@ size_t ZSTD_compressBlock_btlazy2_dictMatchState(
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_lazy_extDict_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore,
ZSTD_matchState_t* ms, seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const searchMethod_e searchMethod, const U32 depth)
@@ -2030,7 +1986,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
const U32 repIndex = (U32)(curr+1 - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
& (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */
if (MEM_read32(ip+1) == MEM_read32(repMatch)) {
/* repcode detected we should take it */
@@ -2071,7 +2027,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
const U32 repIndex = (U32)(curr - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
& (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected */
@@ -2103,7 +2059,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
const U32 repIndex = (U32)(curr - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
& (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected */
@@ -2157,7 +2113,7 @@ _storeSequence:
const U32 repIndex = repCurrent - offset_2;
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
& (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected we should take it */
@@ -2183,14 +2139,14 @@ _storeSequence:
#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_greedy_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0);
}
size_t ZSTD_compressBlock_greedy_extDict_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0);
@@ -2199,7 +2155,7 @@ size_t ZSTD_compressBlock_greedy_extDict_row(
#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
@@ -2207,7 +2163,7 @@ size_t ZSTD_compressBlock_lazy_extDict(
}
size_t ZSTD_compressBlock_lazy_extDict_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
@@ -2217,7 +2173,7 @@ size_t ZSTD_compressBlock_lazy_extDict_row(
#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy2_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
@@ -2225,7 +2181,7 @@ size_t ZSTD_compressBlock_lazy2_extDict(
}
size_t ZSTD_compressBlock_lazy2_extDict_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2);
@@ -2234,7 +2190,7 @@ size_t ZSTD_compressBlock_lazy2_extDict_row(
#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btlazy2_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
+39 -30
View File
@@ -11,6 +11,10 @@
#ifndef ZSTD_LAZY_H
#define ZSTD_LAZY_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "zstd_compress_internal.h"
/**
@@ -27,38 +31,38 @@
|| !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR)
U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip);
void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* ip);
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_MatchState_t* ms, const BYTE* const ip);
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip);
void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */
#endif
#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_greedy(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_dictMatchState_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_extDict_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_GREEDY ZSTD_compressBlock_greedy
@@ -82,28 +86,28 @@ size_t ZSTD_compressBlock_greedy_extDict_row(
#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_dictMatchState_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_extDict_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_LAZY ZSTD_compressBlock_lazy
@@ -127,28 +131,28 @@ size_t ZSTD_compressBlock_lazy_extDict_row(
#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy2(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_extDict_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_LAZY2 ZSTD_compressBlock_lazy2
@@ -172,13 +176,13 @@ size_t ZSTD_compressBlock_lazy2_extDict_row(
#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btlazy2(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btlazy2_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btlazy2_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_BTLAZY2 ZSTD_compressBlock_btlazy2
@@ -190,4 +194,9 @@ size_t ZSTD_compressBlock_btlazy2_extDict(
#define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT NULL
#endif
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_LAZY_H */
+33 -52
View File
@@ -16,7 +16,7 @@
#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */
#include "zstd_ldm_geartab.h"
#define LDM_BUCKET_SIZE_LOG 4
#define LDM_BUCKET_SIZE_LOG 3
#define LDM_MIN_MATCH_LENGTH 64
#define LDM_HASH_RLOG 7
@@ -133,39 +133,21 @@ done:
}
void ZSTD_ldm_adjustParameters(ldmParams_t* params,
const ZSTD_compressionParameters* cParams)
ZSTD_compressionParameters const* cParams)
{
params->windowLog = cParams->windowLog;
ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
DEBUGLOG(4, "ZSTD_ldm_adjustParameters");
if (params->hashRateLog == 0) {
if (params->hashLog > 0) {
/* if params->hashLog is set, derive hashRateLog from it */
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
if (params->windowLog > params->hashLog) {
params->hashRateLog = params->windowLog - params->hashLog;
}
} else {
assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9);
/* mapping from [fast, rate7] to [btultra2, rate4] */
params->hashRateLog = 7 - (cParams->strategy/3);
}
}
if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH;
if (params->hashLog == 0) {
if (params->windowLog <= params->hashRateLog) {
params->hashLog = ZSTD_HASHLOG_MIN;
} else {
params->hashLog = BOUNDED(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog, ZSTD_HASHLOG_MAX);
params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
}
}
if (params->minMatchLength == 0) {
params->minMatchLength = LDM_MIN_MATCH_LENGTH;
if (cParams->strategy >= ZSTD_btultra)
params->minMatchLength /= 2;
}
if (params->bucketSizeLog==0) {
assert(1 <= (int)cParams->strategy && (int)cParams->strategy <= 9);
params->bucketSizeLog = BOUNDED(LDM_BUCKET_SIZE_LOG, (U32)cParams->strategy, ZSTD_LDM_BUCKETSIZELOG_MAX);
if (params->hashRateLog == 0) {
params->hashRateLog = params->windowLog < params->hashLog
? 0
: params->windowLog - params->hashLog;
}
params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog);
}
@@ -188,22 +170,22 @@ size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize)
/** ZSTD_ldm_getBucket() :
* Returns a pointer to the start of the bucket associated with hash. */
static ldmEntry_t* ZSTD_ldm_getBucket(
const ldmState_t* ldmState, size_t hash, U32 const bucketSizeLog)
ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams)
{
return ldmState->hashTable + (hash << bucketSizeLog);
return ldmState->hashTable + (hash << ldmParams.bucketSizeLog);
}
/** ZSTD_ldm_insertEntry() :
* Insert the entry with corresponding hash into the hash table */
static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
size_t const hash, const ldmEntry_t entry,
U32 const bucketSizeLog)
ldmParams_t const ldmParams)
{
BYTE* const pOffset = ldmState->bucketOffsets + hash;
unsigned const offset = *pOffset;
*(ZSTD_ldm_getBucket(ldmState, hash, bucketSizeLog) + offset) = entry;
*pOffset = (BYTE)((offset + 1) & ((1u << bucketSizeLog) - 1));
*(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry;
*pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1));
}
@@ -252,7 +234,7 @@ static size_t ZSTD_ldm_countBackwardsMatch_2segments(
*
* The tables for the other strategies are filled within their
* block compressors. */
static size_t ZSTD_ldm_fillFastTables(ZSTD_MatchState_t* ms,
static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
void const* end)
{
const BYTE* const iend = (const BYTE*)end;
@@ -291,8 +273,7 @@ void ZSTD_ldm_fillHashTable(
const BYTE* iend, ldmParams_t const* params)
{
U32 const minMatchLength = params->minMatchLength;
U32 const bucketSizeLog = params->bucketSizeLog;
U32 const hBits = params->hashLog - bucketSizeLog;
U32 const hBits = params->hashLog - params->bucketSizeLog;
BYTE const* const base = ldmState->window.base;
BYTE const* const istart = ip;
ldmRollingHashState_t hashState;
@@ -307,7 +288,7 @@ void ZSTD_ldm_fillHashTable(
unsigned n;
numSplits = 0;
hashed = ZSTD_ldm_gear_feed(&hashState, ip, (size_t)(iend - ip), splits, &numSplits);
hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits);
for (n = 0; n < numSplits; n++) {
if (ip + splits[n] >= istart + minMatchLength) {
@@ -318,7 +299,7 @@ void ZSTD_ldm_fillHashTable(
entry.offset = (U32)(split - base);
entry.checksum = (U32)(xxhash >> 32);
ZSTD_ldm_insertEntry(ldmState, hash, entry, params->bucketSizeLog);
ZSTD_ldm_insertEntry(ldmState, hash, entry, *params);
}
}
@@ -332,7 +313,7 @@ void ZSTD_ldm_fillHashTable(
* Sets cctx->nextToUpdate to a position corresponding closer to anchor
* if it is far way
* (after a long match, only update tables a limited amount). */
static void ZSTD_ldm_limitTableUpdate(ZSTD_MatchState_t* ms, const BYTE* anchor)
static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor)
{
U32 const curr = (U32)(anchor - ms->window.base);
if (curr > ms->nextToUpdate + 1024) {
@@ -344,7 +325,7 @@ static void ZSTD_ldm_limitTableUpdate(ZSTD_MatchState_t* ms, const BYTE* anchor)
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_ldm_generateSequences_internal(
ldmState_t* ldmState, RawSeqStore_t* rawSeqStore,
ldmState_t* ldmState, rawSeqStore_t* rawSeqStore,
ldmParams_t const* params, void const* src, size_t srcSize)
{
/* LDM parameters */
@@ -398,7 +379,7 @@ size_t ZSTD_ldm_generateSequences_internal(
candidates[n].split = split;
candidates[n].hash = hash;
candidates[n].checksum = (U32)(xxhash >> 32);
candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, params->bucketSizeLog);
candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params);
PREFETCH_L1(candidates[n].bucket);
}
@@ -421,7 +402,7 @@ size_t ZSTD_ldm_generateSequences_internal(
* the previous one, we merely register it in the hash table and
* move on */
if (split < anchor) {
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog);
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
continue;
}
@@ -468,7 +449,7 @@ size_t ZSTD_ldm_generateSequences_internal(
/* No match found -- insert an entry into the hash table
* and process the next candidate match */
if (bestEntry == NULL) {
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog);
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
continue;
}
@@ -489,7 +470,7 @@ size_t ZSTD_ldm_generateSequences_internal(
/* Insert the current entry into the hash table --- it must be
* done after the previous block to avoid clobbering bestEntry */
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, params->bucketSizeLog);
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
anchor = split + forwardMatchLength;
@@ -528,7 +509,7 @@ static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size,
}
size_t ZSTD_ldm_generateSequences(
ldmState_t* ldmState, RawSeqStore_t* sequences,
ldmState_t* ldmState, rawSeqStore_t* sequences,
ldmParams_t const* params, void const* src, size_t srcSize)
{
U32 const maxDist = 1U << params->windowLog;
@@ -605,7 +586,7 @@ size_t ZSTD_ldm_generateSequences(
}
void
ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch)
ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch)
{
while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) {
rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos;
@@ -641,7 +622,7 @@ ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize, U32 const min
* Returns the current sequence to handle, or if the rest of the block should
* be literals, it returns a sequence with offset == 0.
*/
static rawSeq maybeSplitSequence(RawSeqStore_t* rawSeqStore,
static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
U32 const remaining, U32 const minMatch)
{
rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos];
@@ -665,7 +646,7 @@ static rawSeq maybeSplitSequence(RawSeqStore_t* rawSeqStore,
return sequence;
}
void ZSTD_ldm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes) {
void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
while (currPos && rawSeqStore->pos < rawSeqStore->size) {
rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
@@ -682,14 +663,14 @@ void ZSTD_ldm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes) {
}
}
size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_ParamSwitch_e useRowMatchFinder,
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_paramSwitch_e useRowMatchFinder,
void const* src, size_t srcSize)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
unsigned const minMatch = cParams->minMatch;
ZSTD_BlockCompressor_f const blockCompressor =
ZSTD_blockCompressor const blockCompressor =
ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms));
/* Input bounds */
BYTE const* const istart = (BYTE const*)src;
+14 -6
View File
@@ -11,6 +11,10 @@
#ifndef ZSTD_LDM_H
#define ZSTD_LDM_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "zstd_compress_internal.h" /* ldmParams_t, U32 */
#include "../zstd.h" /* ZSTD_CCtx, size_t */
@@ -39,7 +43,7 @@ void ZSTD_ldm_fillHashTable(
* sequences.
*/
size_t ZSTD_ldm_generateSequences(
ldmState_t* ldms, RawSeqStore_t* sequences,
ldmState_t* ldms, rawSeqStore_t* sequences,
ldmParams_t const* params, void const* src, size_t srcSize);
/**
@@ -60,9 +64,9 @@ size_t ZSTD_ldm_generateSequences(
* two. We handle that case correctly, and update `rawSeqStore` appropriately.
* NOTE: This function does not return any errors.
*/
size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_ParamSwitch_e useRowMatchFinder,
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_paramSwitch_e useRowMatchFinder,
void const* src, size_t srcSize);
/**
@@ -72,7 +76,7 @@ size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore,
* Avoids emitting matches less than `minMatch` bytes.
* Must be called for data that is not passed to ZSTD_ldm_blockCompress().
*/
void ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize,
void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize,
U32 const minMatch);
/* ZSTD_ldm_skipRawSeqStoreBytes():
@@ -80,7 +84,7 @@ void ZSTD_ldm_skipSequences(RawSeqStore_t* rawSeqStore, size_t srcSize,
* Not to be used in conjunction with ZSTD_ldm_skipSequences().
* Must be called for data with is not passed to ZSTD_ldm_blockCompress().
*/
void ZSTD_ldm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes);
void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes);
/** ZSTD_ldm_getTableSize() :
* Estimate the space needed for long distance matching tables or 0 if LDM is
@@ -106,4 +110,8 @@ size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize);
void ZSTD_ldm_adjustParameters(ldmParams_t* params,
ZSTD_compressionParameters const* cParams);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_FAST_H */
+59 -55
View File
@@ -408,7 +408,7 @@ MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length)
Assumption : always within prefix (i.e. not within extDict) */
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_MatchState_t* ms,
U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_matchState_t* ms,
U32* nextToUpdate3,
const BYTE* const ip)
{
@@ -440,7 +440,7 @@ U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_MatchState_t* ms,
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_insertBt1(
const ZSTD_MatchState_t* ms,
const ZSTD_matchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
U32 const target,
U32 const mls, const int extDict)
@@ -560,7 +560,7 @@ U32 ZSTD_insertBt1(
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_updateTree_internal(
ZSTD_MatchState_t* ms,
ZSTD_matchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
const U32 mls, const ZSTD_dictMode_e dictMode)
{
@@ -580,7 +580,7 @@ void ZSTD_updateTree_internal(
ms->nextToUpdate = target;
}
void ZSTD_updateTree(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend) {
void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
}
@@ -589,7 +589,7 @@ ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32
ZSTD_insertBtAndGetAllMatches (
ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */
ZSTD_MatchState_t* ms,
ZSTD_matchState_t* ms,
U32* nextToUpdate3,
const BYTE* const ip, const BYTE* const iLimit,
const ZSTD_dictMode_e dictMode,
@@ -625,7 +625,7 @@ ZSTD_insertBtAndGetAllMatches (
U32 mnum = 0;
U32 nbCompares = 1U << cParams->searchLog;
const ZSTD_MatchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL;
const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL;
const ZSTD_compressionParameters* const dmsCParams =
dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL;
const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL;
@@ -664,13 +664,13 @@ ZSTD_insertBtAndGetAllMatches (
assert(curr >= windowLow);
if ( dictMode == ZSTD_extDict
&& ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */
& (ZSTD_index_overlap_check(dictLimit, repIndex)) )
& (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */)
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch;
}
if (dictMode == ZSTD_dictMatchState
&& ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */
& (ZSTD_index_overlap_check(dictLimit, repIndex)) )
& ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch;
} }
@@ -819,7 +819,7 @@ ZSTD_insertBtAndGetAllMatches (
typedef U32 (*ZSTD_getAllMatchesFn)(
ZSTD_match_t*,
ZSTD_MatchState_t*,
ZSTD_matchState_t*,
U32*,
const BYTE*,
const BYTE*,
@@ -831,7 +831,7 @@ FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_btGetAllMatches_internal(
ZSTD_match_t* matches,
ZSTD_MatchState_t* ms,
ZSTD_matchState_t* ms,
U32* nextToUpdate3,
const BYTE* ip,
const BYTE* const iHighLimit,
@@ -854,7 +854,7 @@ U32 ZSTD_btGetAllMatches_internal(
#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \
static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \
ZSTD_match_t* matches, \
ZSTD_MatchState_t* ms, \
ZSTD_matchState_t* ms, \
U32* nextToUpdate3, \
const BYTE* ip, \
const BYTE* const iHighLimit, \
@@ -886,7 +886,7 @@ GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState)
}
static ZSTD_getAllMatchesFn
ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_dictMode_e const dictMode)
ZSTD_selectBtGetAllMatches(ZSTD_matchState_t const* ms, ZSTD_dictMode_e const dictMode)
{
ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = {
ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict),
@@ -905,7 +905,7 @@ ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_dictMode_e const di
/* Struct containing info needed to make decision about ldm inclusion */
typedef struct {
RawSeqStore_t seqStore; /* External match candidates store for this block */
rawSeqStore_t seqStore; /* External match candidates store for this block */
U32 startPosInBlock; /* Start position of the current match candidate */
U32 endPosInBlock; /* End position of the current match candidate */
U32 offset; /* Offset of the match candidate */
@@ -915,7 +915,7 @@ typedef struct {
* Moves forward in @rawSeqStore by @nbBytes,
* which will update the fields 'pos' and 'posInSequence'.
*/
static void ZSTD_optLdm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes)
static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes)
{
U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
while (currPos && rawSeqStore->pos < rawSeqStore->size) {
@@ -972,7 +972,7 @@ ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock
return;
}
/* Matches may be < minMatch by this process. In that case, we will reject them
/* Matches may be < MINMATCH by this process. In that case, we will reject them
when we are deciding whether or not to add the ldm */
optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining;
optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining;
@@ -994,8 +994,7 @@ ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock
* into 'matches'. Maintains the correct ordering of 'matches'.
*/
static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
const ZSTD_optLdm_t* optLdm, U32 currPosInBlock,
U32 minMatch)
const ZSTD_optLdm_t* optLdm, U32 currPosInBlock)
{
U32 const posDiff = currPosInBlock - optLdm->startPosInBlock;
/* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */
@@ -1004,7 +1003,7 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
/* Ensure that current block position is not outside of the match */
if (currPosInBlock < optLdm->startPosInBlock
|| currPosInBlock >= optLdm->endPosInBlock
|| candidateMatchLength < minMatch) {
|| candidateMatchLength < MINMATCH) {
return;
}
@@ -1024,8 +1023,7 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
static void
ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm,
ZSTD_match_t* matches, U32* nbMatches,
U32 currPosInBlock, U32 remainingBytes,
U32 minMatch)
U32 currPosInBlock, U32 remainingBytes)
{
if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
return;
@@ -1042,7 +1040,7 @@ ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm,
}
ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes);
}
ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock, minMatch);
ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock);
}
@@ -1074,8 +1072,8 @@ listStats(const U32* table, int lastEltID)
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t
ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
SeqStore_t* seqStore,
ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const int optLevel,
@@ -1124,8 +1122,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
U32 const ll0 = !litlen;
U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch);
ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
(U32)(ip-istart), (U32)(iend-ip),
minMatch);
(U32)(ip-istart), (U32)(iend-ip));
if (!nbMatches) {
DEBUGLOG(8, "no match found at cPos %u", (unsigned)(ip-istart));
ip++;
@@ -1200,7 +1197,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
for (cur = 1; cur <= last_pos; cur++) {
const BYTE* const inr = ip + cur;
assert(cur <= ZSTD_OPT_NUM);
DEBUGLOG(7, "cPos:%i==rPos:%u", (int)(inr-istart), cur);
DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur);
/* Fix current position with one literal if cheaper */
{ U32 const litlen = opt[cur-1].litlen + 1;
@@ -1210,8 +1207,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
assert(price < 1000000000); /* overflow check */
if (price <= opt[cur].price) {
ZSTD_optimal_t const prevMatch = opt[cur];
DEBUGLOG(7, "cPos:%i==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)",
(int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen,
DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)",
inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen,
opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]);
opt[cur] = opt[cur-1];
opt[cur].litlen = litlen;
@@ -1230,34 +1227,34 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
&& (with1literal < opt[cur+1].price) ) {
/* update offset history - before it disappears */
U32 const prev = cur - prevMatch.mlen;
Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, prevMatch.off, opt[prev].litlen==0);
repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, prevMatch.off, opt[prev].litlen==0);
assert(cur >= prevMatch.mlen);
DEBUGLOG(7, "==> match+1lit is cheaper (%.2f < %.2f) (hist:%u,%u,%u) !",
ZSTD_fCost(with1literal), ZSTD_fCost(withMoreLiterals),
newReps.rep[0], newReps.rep[1], newReps.rep[2] );
opt[cur+1] = prevMatch; /* mlen & offbase */
ZSTD_memcpy(opt[cur+1].rep, &newReps, sizeof(Repcodes_t));
ZSTD_memcpy(opt[cur+1].rep, &newReps, sizeof(repcodes_t));
opt[cur+1].litlen = 1;
opt[cur+1].price = with1literal;
if (last_pos < cur+1) last_pos = cur+1;
}
}
} else {
DEBUGLOG(7, "cPos:%i==rPos:%u : literal would cost more (%.2f>%.2f)",
(int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price));
DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f)",
inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price));
}
}
/* Offset history is not updated during match comparison.
* Do it here, now that the match is selected and confirmed.
*/
ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(Repcodes_t));
ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t));
assert(cur >= opt[cur].mlen);
if (opt[cur].litlen == 0) {
/* just finished a match => alter offset history */
U32 const prev = cur - opt[cur].mlen;
Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0);
ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(Repcodes_t));
repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0);
ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t));
}
/* last match must start at a minimum distance of 8 from oend */
@@ -1279,8 +1276,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
U32 matchNb;
ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
(U32)(inr-istart), (U32)(iend-inr),
minMatch);
(U32)(inr-istart), (U32)(iend-inr));
if (!nbMatches) {
DEBUGLOG(7, "rPos:%u : no match found", cur);
@@ -1288,8 +1284,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
}
{ U32 const longestML = matches[nbMatches-1].len;
DEBUGLOG(7, "cPos:%i==rPos:%u, found %u matches, of longest ML=%u",
(int)(inr-istart), cur, nbMatches, longestML);
DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of longest ML=%u",
inr-istart, cur, nbMatches, longestML);
if ( (longestML > sufficient_len)
|| (cur + longestML >= ZSTD_OPT_NUM)
@@ -1357,10 +1353,10 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
/* Update offset history */
if (lastStretch.litlen == 0) {
/* finishing on a match : update offset history */
Repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastStretch.off, opt[cur].litlen==0);
ZSTD_memcpy(rep, &reps, sizeof(Repcodes_t));
repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastStretch.off, opt[cur].litlen==0);
ZSTD_memcpy(rep, &reps, sizeof(repcodes_t));
} else {
ZSTD_memcpy(rep, lastStretch.rep, sizeof(Repcodes_t));
ZSTD_memcpy(rep, lastStretch.rep, sizeof(repcodes_t));
assert(cur >= lastStretch.litlen);
cur -= lastStretch.litlen;
}
@@ -1382,8 +1378,16 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
assert(storeEnd < ZSTD_OPT_SIZE);
DEBUGLOG(6, "last stretch copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
storeEnd, lastStretch.litlen, lastStretch.mlen, lastStretch.off);
if (lastStretch.litlen > 0) {
/* last "sequence" is unfinished: just a bunch of literals */
opt[storeEnd].litlen = lastStretch.litlen;
opt[storeEnd].mlen = 0;
storeStart = storeEnd-1;
opt[storeStart] = lastStretch;
} {
opt[storeEnd] = lastStretch; /* note: litlen will be fixed */
storeStart = storeEnd;
}
while (1) {
ZSTD_optimal_t nextStretch = opt[stretchPos];
opt[storeStart].litlen = nextStretch.litlen;
@@ -1407,8 +1411,8 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
U32 const mlen = opt[storePos].mlen;
U32 const offBase = opt[storePos].off;
U32 const advance = llen + mlen;
DEBUGLOG(6, "considering seq starting at %i, llen=%u, mlen=%u",
(int)(anchor - istart), (unsigned)llen, (unsigned)mlen);
DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u",
anchor - istart, (unsigned)llen, (unsigned)mlen);
if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */
assert(storePos == storeEnd); /* must be last sequence */
@@ -1436,7 +1440,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
static size_t ZSTD_compressBlock_opt0(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
{
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode);
@@ -1445,7 +1449,7 @@ static size_t ZSTD_compressBlock_opt0(
#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
static size_t ZSTD_compressBlock_opt2(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
{
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode);
@@ -1454,7 +1458,7 @@ static size_t ZSTD_compressBlock_opt2(
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btopt(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_btopt");
@@ -1473,8 +1477,8 @@ size_t ZSTD_compressBlock_btopt(
*/
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_initStats_ultra(ZSTD_MatchState_t* ms,
SeqStore_t* seqStore,
void ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
@@ -1499,7 +1503,7 @@ void ZSTD_initStats_ultra(ZSTD_MatchState_t* ms,
}
size_t ZSTD_compressBlock_btultra(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
@@ -1507,7 +1511,7 @@ size_t ZSTD_compressBlock_btultra(
}
size_t ZSTD_compressBlock_btultra2(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
U32 const curr = (U32)((const BYTE*)src - ms->window.base);
@@ -1537,14 +1541,14 @@ size_t ZSTD_compressBlock_btultra2(
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btopt_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_btopt_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict);
@@ -1553,14 +1557,14 @@ size_t ZSTD_compressBlock_btopt_extDict(
#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btultra_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_btultra_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict);
+16 -8
View File
@@ -11,24 +11,28 @@
#ifndef ZSTD_OPT_H
#define ZSTD_OPT_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "zstd_compress_internal.h"
#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR)
/* used in ZSTD_loadDictionaryContent() */
void ZSTD_updateTree(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend);
void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend);
#endif
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btopt(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btopt_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btopt_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_BTOPT ZSTD_compressBlock_btopt
@@ -42,20 +46,20 @@ size_t ZSTD_compressBlock_btopt_extDict(
#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btultra(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
/* note : no btultra2 variant for extDict nor dictMatchState,
* because btultra2 is not meant to work with dictionaries
* and is only specific for the first block (no prefix) */
size_t ZSTD_compressBlock_btultra2(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#define ZSTD_COMPRESSBLOCK_BTULTRA ZSTD_compressBlock_btultra
@@ -69,4 +73,8 @@ size_t ZSTD_compressBlock_btultra2(
#define ZSTD_COMPRESSBLOCK_BTULTRA2 NULL
#endif
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_OPT_H */
-238
View File
@@ -1,238 +0,0 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "../common/compiler.h" /* ZSTD_ALIGNOF */
#include "../common/mem.h" /* S64 */
#include "../common/zstd_deps.h" /* ZSTD_memset */
#include "../common/zstd_internal.h" /* ZSTD_STATIC_ASSERT */
#include "hist.h" /* HIST_add */
#include "zstd_preSplit.h"
#define BLOCKSIZE_MIN 3500
#define THRESHOLD_PENALTY_RATE 16
#define THRESHOLD_BASE (THRESHOLD_PENALTY_RATE - 2)
#define THRESHOLD_PENALTY 3
#define HASHLENGTH 2
#define HASHLOG_MAX 10
#define HASHTABLESIZE (1 << HASHLOG_MAX)
#define HASHMASK (HASHTABLESIZE - 1)
#define KNUTH 0x9e3779b9
/* for hashLog > 8, hash 2 bytes.
* for hashLog == 8, just take the byte, no hashing.
* The speed of this method relies on compile-time constant propagation */
FORCE_INLINE_TEMPLATE unsigned hash2(const void *p, unsigned hashLog)
{
assert(hashLog >= 8);
if (hashLog == 8) return (U32)((const BYTE*)p)[0];
assert(hashLog <= HASHLOG_MAX);
return (U32)(MEM_read16(p)) * KNUTH >> (32 - hashLog);
}
typedef struct {
unsigned events[HASHTABLESIZE];
size_t nbEvents;
} Fingerprint;
typedef struct {
Fingerprint pastEvents;
Fingerprint newEvents;
} FPStats;
static void initStats(FPStats* fpstats)
{
ZSTD_memset(fpstats, 0, sizeof(FPStats));
}
FORCE_INLINE_TEMPLATE void
addEvents_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog)
{
const char* p = (const char*)src;
size_t limit = srcSize - HASHLENGTH + 1;
size_t n;
assert(srcSize >= HASHLENGTH);
for (n = 0; n < limit; n+=samplingRate) {
fp->events[hash2(p+n, hashLog)]++;
}
fp->nbEvents += limit/samplingRate;
}
FORCE_INLINE_TEMPLATE void
recordFingerprint_generic(Fingerprint* fp, const void* src, size_t srcSize, size_t samplingRate, unsigned hashLog)
{
ZSTD_memset(fp, 0, sizeof(unsigned) * ((size_t)1 << hashLog));
fp->nbEvents = 0;
addEvents_generic(fp, src, srcSize, samplingRate, hashLog);
}
typedef void (*RecordEvents_f)(Fingerprint* fp, const void* src, size_t srcSize);
#define FP_RECORD(_rate) ZSTD_recordFingerprint_##_rate
#define ZSTD_GEN_RECORD_FINGERPRINT(_rate, _hSize) \
static void FP_RECORD(_rate)(Fingerprint* fp, const void* src, size_t srcSize) \
{ \
recordFingerprint_generic(fp, src, srcSize, _rate, _hSize); \
}
ZSTD_GEN_RECORD_FINGERPRINT(1, 10)
ZSTD_GEN_RECORD_FINGERPRINT(5, 10)
ZSTD_GEN_RECORD_FINGERPRINT(11, 9)
ZSTD_GEN_RECORD_FINGERPRINT(43, 8)
static U64 abs64(S64 s64) { return (U64)((s64 < 0) ? -s64 : s64); }
static U64 fpDistance(const Fingerprint* fp1, const Fingerprint* fp2, unsigned hashLog)
{
U64 distance = 0;
size_t n;
assert(hashLog <= HASHLOG_MAX);
for (n = 0; n < ((size_t)1 << hashLog); n++) {
distance +=
abs64((S64)fp1->events[n] * (S64)fp2->nbEvents - (S64)fp2->events[n] * (S64)fp1->nbEvents);
}
return distance;
}
/* Compare newEvents with pastEvents
* return 1 when considered "too different"
*/
static int compareFingerprints(const Fingerprint* ref,
const Fingerprint* newfp,
int penalty,
unsigned hashLog)
{
assert(ref->nbEvents > 0);
assert(newfp->nbEvents > 0);
{ U64 p50 = (U64)ref->nbEvents * (U64)newfp->nbEvents;
U64 deviation = fpDistance(ref, newfp, hashLog);
U64 threshold = p50 * (U64)(THRESHOLD_BASE + penalty) / THRESHOLD_PENALTY_RATE;
return deviation >= threshold;
}
}
static void mergeEvents(Fingerprint* acc, const Fingerprint* newfp)
{
size_t n;
for (n = 0; n < HASHTABLESIZE; n++) {
acc->events[n] += newfp->events[n];
}
acc->nbEvents += newfp->nbEvents;
}
static void flushEvents(FPStats* fpstats)
{
size_t n;
for (n = 0; n < HASHTABLESIZE; n++) {
fpstats->pastEvents.events[n] = fpstats->newEvents.events[n];
}
fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents;
ZSTD_memset(&fpstats->newEvents, 0, sizeof(fpstats->newEvents));
}
static void removeEvents(Fingerprint* acc, const Fingerprint* slice)
{
size_t n;
for (n = 0; n < HASHTABLESIZE; n++) {
assert(acc->events[n] >= slice->events[n]);
acc->events[n] -= slice->events[n];
}
acc->nbEvents -= slice->nbEvents;
}
#define CHUNKSIZE (8 << 10)
static size_t ZSTD_splitBlock_byChunks(const void* blockStart, size_t blockSize,
int level,
void* workspace, size_t wkspSize)
{
static const RecordEvents_f records_fs[] = {
FP_RECORD(43), FP_RECORD(11), FP_RECORD(5), FP_RECORD(1)
};
static const unsigned hashParams[] = { 8, 9, 10, 10 };
const RecordEvents_f record_f = (assert(0<=level && level<=3), records_fs[level]);
FPStats* const fpstats = (FPStats*)workspace;
const char* p = (const char*)blockStart;
int penalty = THRESHOLD_PENALTY;
size_t pos = 0;
assert(blockSize == (128 << 10));
assert(workspace != NULL);
assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0);
ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats));
assert(wkspSize >= sizeof(FPStats)); (void)wkspSize;
initStats(fpstats);
record_f(&fpstats->pastEvents, p, CHUNKSIZE);
for (pos = CHUNKSIZE; pos <= blockSize - CHUNKSIZE; pos += CHUNKSIZE) {
record_f(&fpstats->newEvents, p + pos, CHUNKSIZE);
if (compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, penalty, hashParams[level])) {
return pos;
} else {
mergeEvents(&fpstats->pastEvents, &fpstats->newEvents);
if (penalty > 0) penalty--;
}
}
assert(pos == blockSize);
return blockSize;
(void)flushEvents; (void)removeEvents;
}
/* ZSTD_splitBlock_fromBorders(): very fast strategy :
* compare fingerprint from beginning and end of the block,
* derive from their difference if it's preferable to split in the middle,
* repeat the process a second time, for finer grained decision.
* 3 times did not brought improvements, so I stopped at 2.
* Benefits are good enough for a cheap heuristic.
* More accurate splitting saves more, but speed impact is also more perceptible.
* For better accuracy, use more elaborate variant *_byChunks.
*/
static size_t ZSTD_splitBlock_fromBorders(const void* blockStart, size_t blockSize,
void* workspace, size_t wkspSize)
{
#define SEGMENT_SIZE 512
FPStats* const fpstats = (FPStats*)workspace;
Fingerprint* middleEvents = (Fingerprint*)(void*)((char*)workspace + 512 * sizeof(unsigned));
assert(blockSize == (128 << 10));
assert(workspace != NULL);
assert((size_t)workspace % ZSTD_ALIGNOF(FPStats) == 0);
ZSTD_STATIC_ASSERT(ZSTD_SLIPBLOCK_WORKSPACESIZE >= sizeof(FPStats));
assert(wkspSize >= sizeof(FPStats)); (void)wkspSize;
initStats(fpstats);
HIST_add(fpstats->pastEvents.events, blockStart, SEGMENT_SIZE);
HIST_add(fpstats->newEvents.events, (const char*)blockStart + blockSize - SEGMENT_SIZE, SEGMENT_SIZE);
fpstats->pastEvents.nbEvents = fpstats->newEvents.nbEvents = SEGMENT_SIZE;
if (!compareFingerprints(&fpstats->pastEvents, &fpstats->newEvents, 0, 8))
return blockSize;
HIST_add(middleEvents->events, (const char*)blockStart + blockSize/2 - SEGMENT_SIZE/2, SEGMENT_SIZE);
middleEvents->nbEvents = SEGMENT_SIZE;
{ U64 const distFromBegin = fpDistance(&fpstats->pastEvents, middleEvents, 8);
U64 const distFromEnd = fpDistance(&fpstats->newEvents, middleEvents, 8);
U64 const minDistance = SEGMENT_SIZE * SEGMENT_SIZE / 3;
if (abs64((S64)distFromBegin - (S64)distFromEnd) < minDistance)
return 64 KB;
return (distFromBegin > distFromEnd) ? 32 KB : 96 KB;
}
}
size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize,
int level,
void* workspace, size_t wkspSize)
{
DEBUGLOG(6, "ZSTD_splitBlock (level=%i)", level);
assert(0<=level && level<=4);
if (level == 0)
return ZSTD_splitBlock_fromBorders(blockStart, blockSize, workspace, wkspSize);
/* level >= 1*/
return ZSTD_splitBlock_byChunks(blockStart, blockSize, level-1, workspace, wkspSize);
}
-33
View File
@@ -1,33 +0,0 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef ZSTD_PRESPLIT_H
#define ZSTD_PRESPLIT_H
#include <stddef.h> /* size_t */
#define ZSTD_SLIPBLOCK_WORKSPACESIZE 8208
/* ZSTD_splitBlock():
* @level must be a value between 0 and 4.
* higher levels spend more energy to detect block boundaries.
* @workspace must be aligned for size_t.
* @wkspSize must be at least >= ZSTD_SLIPBLOCK_WORKSPACESIZE
* note:
* For the time being, this function only accepts full 128 KB blocks.
* Therefore, @blockSize must be == 128 KB.
* While this could be extended to smaller sizes in the future,
* it is not yet clear if this would be useful. TBD.
*/
size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize,
int level,
void* workspace, size_t wkspSize);
#endif /* ZSTD_PRESPLIT_H */
+80 -122
View File
@@ -90,9 +90,9 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
typedef struct buffer_s {
void* start;
size_t capacity;
} Buffer;
} buffer_t;
static const Buffer g_nullBuffer = { NULL, 0 };
static const buffer_t g_nullBuffer = { NULL, 0 };
typedef struct ZSTDMT_bufferPool_s {
ZSTD_pthread_mutex_t poolMutex;
@@ -100,7 +100,7 @@ typedef struct ZSTDMT_bufferPool_s {
unsigned totalBuffers;
unsigned nbBuffers;
ZSTD_customMem cMem;
Buffer* buffers;
buffer_t* buffers;
} ZSTDMT_bufferPool;
static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
@@ -128,7 +128,7 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned maxNbBuffers, ZSTD_cu
ZSTD_customFree(bufPool, cMem);
return NULL;
}
bufPool->buffers = (Buffer*)ZSTD_customCalloc(maxNbBuffers * sizeof(Buffer), cMem);
bufPool->buffers = (buffer_t*)ZSTD_customCalloc(maxNbBuffers * sizeof(buffer_t), cMem);
if (bufPool->buffers==NULL) {
ZSTDMT_freeBufferPool(bufPool);
return NULL;
@@ -144,7 +144,7 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned maxNbBuffers, ZSTD_cu
static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
{
size_t const poolSize = sizeof(*bufPool);
size_t const arraySize = bufPool->totalBuffers * sizeof(Buffer);
size_t const arraySize = bufPool->totalBuffers * sizeof(buffer_t);
unsigned u;
size_t totalBufferSize = 0;
ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
@@ -189,13 +189,13 @@ static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool,
* assumption : bufPool must be valid
* @return : a buffer, with start pointer and size
* note: allocation may fail, in this case, start==NULL and size==0 */
static Buffer ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
{
size_t const bSize = bufPool->bufferSize;
DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize);
ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
if (bufPool->nbBuffers) { /* try to use an existing buffer */
Buffer const buf = bufPool->buffers[--(bufPool->nbBuffers)];
buffer_t const buf = bufPool->buffers[--(bufPool->nbBuffers)];
size_t const availBufferSize = buf.capacity;
bufPool->buffers[bufPool->nbBuffers] = g_nullBuffer;
if ((availBufferSize >= bSize) & ((availBufferSize>>3) <= bSize)) {
@@ -212,7 +212,7 @@ static Buffer ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
/* create new buffer */
DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer");
{ Buffer buffer;
{ buffer_t buffer;
void* const start = ZSTD_customMalloc(bSize, bufPool->cMem);
buffer.start = start; /* note : start can be NULL if malloc fails ! */
buffer.capacity = (start==NULL) ? 0 : bSize;
@@ -231,12 +231,12 @@ static Buffer ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
* @return : a buffer that is at least the buffer pool buffer size.
* If a reallocation happens, the data in the input buffer is copied.
*/
static Buffer ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, Buffer buffer)
static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer)
{
size_t const bSize = bufPool->bufferSize;
if (buffer.capacity < bSize) {
void* const start = ZSTD_customMalloc(bSize, bufPool->cMem);
Buffer newBuffer;
buffer_t newBuffer;
newBuffer.start = start;
newBuffer.capacity = start == NULL ? 0 : bSize;
if (start != NULL) {
@@ -252,7 +252,7 @@ static Buffer ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, Buffer buffer)
#endif
/* store buffer for later re-use, up to pool capacity */
static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, Buffer buf)
static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
{
DEBUGLOG(5, "ZSTDMT_releaseBuffer");
if (buf.start == NULL) return; /* compatible with release on NULL */
@@ -290,23 +290,23 @@ static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool)
return ZSTDMT_sizeof_bufferPool(seqPool);
}
static RawSeqStore_t bufferToSeq(Buffer buffer)
static rawSeqStore_t bufferToSeq(buffer_t buffer)
{
RawSeqStore_t seq = kNullRawSeqStore;
rawSeqStore_t seq = kNullRawSeqStore;
seq.seq = (rawSeq*)buffer.start;
seq.capacity = buffer.capacity / sizeof(rawSeq);
return seq;
}
static Buffer seqToBuffer(RawSeqStore_t seq)
static buffer_t seqToBuffer(rawSeqStore_t seq)
{
Buffer buffer;
buffer_t buffer;
buffer.start = seq.seq;
buffer.capacity = seq.capacity * sizeof(rawSeq);
return buffer;
}
static RawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool)
static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool)
{
if (seqPool->bufferSize == 0) {
return kNullRawSeqStore;
@@ -315,13 +315,13 @@ static RawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool)
}
#if ZSTD_RESIZE_SEQPOOL
static RawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, RawSeqStore_t seq)
static rawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq)
{
return bufferToSeq(ZSTDMT_resizeBuffer(seqPool, seqToBuffer(seq)));
}
#endif
static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, RawSeqStore_t seq)
static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq)
{
ZSTDMT_releaseBuffer(seqPool, seqToBuffer(seq));
}
@@ -466,7 +466,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
typedef struct {
void const* start;
size_t size;
} Range;
} range_t;
typedef struct {
/* All variables in the struct are protected by mutex. */
@@ -482,10 +482,10 @@ typedef struct {
ZSTD_pthread_mutex_t ldmWindowMutex;
ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is updated */
ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */
} SerialState;
} serialState_t;
static int
ZSTDMT_serialState_reset(SerialState* serialState,
ZSTDMT_serialState_reset(serialState_t* serialState,
ZSTDMT_seqPool* seqPool,
ZSTD_CCtx_params params,
size_t jobSize,
@@ -555,7 +555,7 @@ ZSTDMT_serialState_reset(SerialState* serialState,
return 0;
}
static int ZSTDMT_serialState_init(SerialState* serialState)
static int ZSTDMT_serialState_init(serialState_t* serialState)
{
int initError = 0;
ZSTD_memset(serialState, 0, sizeof(*serialState));
@@ -566,7 +566,7 @@ static int ZSTDMT_serialState_init(SerialState* serialState)
return initError;
}
static void ZSTDMT_serialState_free(SerialState* serialState)
static void ZSTDMT_serialState_free(serialState_t* serialState)
{
ZSTD_customMem cMem = serialState->params.customMem;
ZSTD_pthread_mutex_destroy(&serialState->mutex);
@@ -577,10 +577,9 @@ static void ZSTDMT_serialState_free(SerialState* serialState)
ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem);
}
static void
ZSTDMT_serialState_genSequences(SerialState* serialState,
RawSeqStore_t* seqStore,
Range src, unsigned jobID)
static void ZSTDMT_serialState_update(serialState_t* serialState,
ZSTD_CCtx* jobCCtx, rawSeqStore_t seqStore,
range_t src, unsigned jobID)
{
/* Wait for our turn */
ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex);
@@ -593,13 +592,12 @@ ZSTDMT_serialState_genSequences(SerialState* serialState,
/* It is now our turn, do any processing necessary */
if (serialState->params.ldmParams.enableLdm == ZSTD_ps_enable) {
size_t error;
DEBUGLOG(6, "ZSTDMT_serialState_genSequences: LDM update");
assert(seqStore->seq != NULL && seqStore->pos == 0 &&
seqStore->size == 0 && seqStore->capacity > 0);
assert(seqStore.seq != NULL && seqStore.pos == 0 &&
seqStore.size == 0 && seqStore.capacity > 0);
assert(src.size <= serialState->params.jobSize);
ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0);
error = ZSTD_ldm_generateSequences(
&serialState->ldmState, seqStore,
&serialState->ldmState, &seqStore,
&serialState->params.ldmParams, src.start, src.size);
/* We provide a large enough buffer to never fail. */
assert(!ZSTD_isError(error)); (void)error;
@@ -618,22 +616,14 @@ ZSTDMT_serialState_genSequences(SerialState* serialState,
serialState->nextJobID++;
ZSTD_pthread_cond_broadcast(&serialState->cond);
ZSTD_pthread_mutex_unlock(&serialState->mutex);
}
static void
ZSTDMT_serialState_applySequences(const SerialState* serialState, /* just for an assert() check */
ZSTD_CCtx* jobCCtx,
const RawSeqStore_t* seqStore)
{
if (seqStore->size > 0) {
DEBUGLOG(5, "ZSTDMT_serialState_applySequences: uploading %u external sequences", (unsigned)seqStore->size);
assert(serialState->params.ldmParams.enableLdm == ZSTD_ps_enable); (void)serialState;
assert(jobCCtx);
ZSTD_referenceExternalSequences(jobCCtx, seqStore->seq, seqStore->size);
if (seqStore.size > 0) {
ZSTD_referenceExternalSequences(jobCCtx, seqStore.seq, seqStore.size);
assert(serialState->params.ldmParams.enableLdm == ZSTD_ps_enable);
}
}
static void ZSTDMT_serialState_ensureFinished(SerialState* serialState,
static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState,
unsigned jobID, size_t cSize)
{
ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex);
@@ -657,7 +647,7 @@ static void ZSTDMT_serialState_ensureFinished(SerialState* serialState,
/* ===== Worker thread ===== */
/* ------------------------------------------ */
static const Range kNullRange = { NULL, 0 };
static const range_t kNullRange = { NULL, 0 };
typedef struct {
size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */
@@ -667,10 +657,10 @@ typedef struct {
ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */
ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */
ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */
SerialState* serial; /* Thread-safe - used by mtctx and (all) workers */
Buffer dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */
Range prefix; /* set by mtctx, then read by worker & mtctx => no barrier */
Range src; /* set by mtctx, then read by worker & mtctx => no barrier */
serialState_t* serial; /* Thread-safe - used by mtctx and (all) workers */
buffer_t dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */
range_t prefix; /* set by mtctx, then read by worker & mtctx => no barrier */
range_t src; /* set by mtctx, then read by worker & mtctx => no barrier */
unsigned jobID; /* set by mtctx, then read by worker => no barrier */
unsigned firstJob; /* set by mtctx, then read by worker => no barrier */
unsigned lastJob; /* set by mtctx, then read by worker => no barrier */
@@ -695,11 +685,10 @@ static void ZSTDMT_compressionJob(void* jobDescription)
ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */
ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool);
RawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool);
Buffer dstBuff = job->dstBuff;
rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool);
buffer_t dstBuff = job->dstBuff;
size_t lastCBlockSize = 0;
DEBUGLOG(5, "ZSTDMT_compressionJob: job %u", job->jobID);
/* resources */
if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation));
if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */
@@ -721,15 +710,11 @@ static void ZSTDMT_compressionJob(void* jobDescription)
/* init */
/* Perform serial step as early as possible */
ZSTDMT_serialState_genSequences(job->serial, &rawSeqStore, job->src, job->jobID);
if (job->cdict) {
size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize);
assert(job->firstJob); /* only allowed for first job */
if (ZSTD_isError(initError)) JOB_ERROR(initError);
} else {
} else { /* srcStart points at reloaded section */
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
{ size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
@@ -738,17 +723,16 @@ static void ZSTDMT_compressionJob(void* jobDescription)
size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0);
if (ZSTD_isError(err)) JOB_ERROR(err);
}
DEBUGLOG(6, "ZSTDMT_compressionJob: job %u: loading prefix of size %zu", job->jobID, job->prefix.size);
{ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
job->prefix.start, job->prefix.size, ZSTD_dct_rawContent,
job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */
ZSTD_dtlm_fast,
NULL, /*cdict*/
&jobParams, pledgedSrcSize);
if (ZSTD_isError(initError)) JOB_ERROR(initError);
} }
/* External Sequences can only be applied after CCtx initialization */
ZSTDMT_serialState_applySequences(job->serial, cctx, &rawSeqStore);
/* Perform serial step as early as possible, but after CCtx initialization */
ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID);
if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */
size_t const hSize = ZSTD_compressContinue_public(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0);
@@ -757,7 +741,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
ZSTD_invalidateRepCodes(cctx);
}
/* compress the entire job by smaller chunks, for better granularity */
/* compress */
{ size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX;
int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize);
const BYTE* ip = (const BYTE*) job->src.start;
@@ -825,10 +809,10 @@ _endJob:
/* ------------------------------------------ */
typedef struct {
Range prefix; /* read-only non-owned prefix buffer */
Buffer buffer;
range_t prefix; /* read-only non-owned prefix buffer */
buffer_t buffer;
size_t filled;
} InBuff_t;
} inBuff_t;
typedef struct {
BYTE* buffer; /* The round input buffer. All jobs get references
@@ -842,9 +826,9 @@ typedef struct {
* the inBuff is sent to the worker thread.
* pos <= capacity.
*/
} RoundBuff_t;
} roundBuff_t;
static const RoundBuff_t kNullRoundBuff = {NULL, 0, 0};
static const roundBuff_t kNullRoundBuff = {NULL, 0, 0};
#define RSYNC_LENGTH 32
/* Don't create chunks smaller than the zstd block size.
@@ -861,7 +845,7 @@ typedef struct {
U64 hash;
U64 hitMask;
U64 primePower;
} RSyncState_t;
} rsyncState_t;
struct ZSTDMT_CCtx_s {
POOL_ctx* factory;
@@ -873,10 +857,10 @@ struct ZSTDMT_CCtx_s {
size_t targetSectionSize;
size_t targetPrefixSize;
int jobReady; /* 1 => one job is already prepared, but pool has shortage of workers. Don't create a new job. */
InBuff_t inBuff;
RoundBuff_t roundBuff;
SerialState serial;
RSyncState_t rsync;
inBuff_t inBuff;
roundBuff_t roundBuff;
serialState_t serial;
rsyncState_t rsync;
unsigned jobIDMask;
unsigned doneJobID;
unsigned nextJobID;
@@ -1008,7 +992,6 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
{
unsigned jobID;
DEBUGLOG(3, "ZSTDMT_releaseAllJobResources");
if (mtctx->jobs) {
for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) {
/* Copy the mutex/cond out */
ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex;
@@ -1022,7 +1005,6 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
mtctx->jobs[jobID].job_mutex = mutex;
mtctx->jobs[jobID].job_cond = cond;
}
}
mtctx->inBuff.buffer = g_nullBuffer;
mtctx->inBuff.filled = 0;
mtctx->allJobsCompleted = 1;
@@ -1263,26 +1245,30 @@ size_t ZSTDMT_initCStream_internal(
/* init */
if (params.nbWorkers != mtctx->params.nbWorkers)
FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, (unsigned)params.nbWorkers) , "");
FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , "");
if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX;
DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers);
if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */
ZSTDMT_waitForAllJobsCompleted(mtctx);
ZSTDMT_releaseAllJobResources(mtctx); /* Will set allJobsCompleted to 1 */
ZSTDMT_releaseAllJobResources(mtctx);
mtctx->allJobsCompleted = 1;
}
mtctx->params = params;
mtctx->frameContentSize = pledgedSrcSize;
ZSTD_freeCDict(mtctx->cdictLocal);
if (dict) {
ZSTD_freeCDict(mtctx->cdictLocal);
mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */
params.cParams, mtctx->cMem);
mtctx->cdict = mtctx->cdictLocal;
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
} else {
ZSTD_freeCDict(mtctx->cdictLocal);
mtctx->cdictLocal = NULL;
mtctx->cdict = cdict;
}
@@ -1348,32 +1334,9 @@ size_t ZSTDMT_initCStream_internal(
mtctx->allJobsCompleted = 0;
mtctx->consumed = 0;
mtctx->produced = 0;
/* update dictionary */
ZSTD_freeCDict(mtctx->cdictLocal);
mtctx->cdictLocal = NULL;
mtctx->cdict = NULL;
if (dict) {
if (dictContentType == ZSTD_dct_rawContent) {
mtctx->inBuff.prefix.start = (const BYTE*)dict;
mtctx->inBuff.prefix.size = dictSize;
} else {
/* note : a loadPrefix becomes an internal CDict */
mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byRef, dictContentType,
params.cParams, mtctx->cMem);
mtctx->cdict = mtctx->cdictLocal;
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
}
} else {
mtctx->cdict = cdict;
}
if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize,
dict, dictSize, dictContentType))
return ERROR(memory_allocation);
return 0;
}
@@ -1440,7 +1403,7 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZS
mtctx->roundBuff.pos += srcSize;
mtctx->inBuff.buffer = g_nullBuffer;
mtctx->inBuff.filled = 0;
/* Set the prefix for next job */
/* Set the prefix */
if (!endFrame) {
size_t const newPrefixSize = MIN(srcSize, mtctx->targetPrefixSize);
mtctx->inBuff.prefix.start = src + srcSize - newPrefixSize;
@@ -1577,17 +1540,12 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
* If the data of the first job is broken up into two segments, we cover both
* sections.
*/
static Range ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx)
static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx)
{
unsigned const firstJobID = mtctx->doneJobID;
unsigned const lastJobID = mtctx->nextJobID;
unsigned jobID;
/* no need to check during first round */
size_t roundBuffCapacity = mtctx->roundBuff.capacity;
size_t nbJobs1stRoundMin = roundBuffCapacity / mtctx->targetSectionSize;
if (lastJobID < nbJobs1stRoundMin) return kNullRange;
for (jobID = firstJobID; jobID < lastJobID; ++jobID) {
unsigned const wJobID = jobID & mtctx->jobIDMask;
size_t consumed;
@@ -1597,7 +1555,7 @@ static Range ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx)
ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
if (consumed < mtctx->jobs[wJobID].src.size) {
Range range = mtctx->jobs[wJobID].prefix;
range_t range = mtctx->jobs[wJobID].prefix;
if (range.size == 0) {
/* Empty prefix */
range = mtctx->jobs[wJobID].src;
@@ -1613,7 +1571,7 @@ static Range ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx)
/**
* Returns non-zero iff buffer and range overlap.
*/
static int ZSTDMT_isOverlapped(Buffer buffer, Range range)
static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range)
{
BYTE const* const bufferStart = (BYTE const*)buffer.start;
BYTE const* const rangeStart = (BYTE const*)range.start;
@@ -1633,10 +1591,10 @@ static int ZSTDMT_isOverlapped(Buffer buffer, Range range)
}
}
static int ZSTDMT_doesOverlapWindow(Buffer buffer, ZSTD_window_t window)
static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window)
{
Range extDict;
Range prefix;
range_t extDict;
range_t prefix;
DEBUGLOG(5, "ZSTDMT_doesOverlapWindow");
extDict.start = window.dictBase + window.lowLimit;
@@ -1655,7 +1613,7 @@ static int ZSTDMT_doesOverlapWindow(Buffer buffer, ZSTD_window_t window)
|| ZSTDMT_isOverlapped(buffer, prefix);
}
static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, Buffer buffer)
static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer)
{
if (mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable) {
ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex;
@@ -1680,16 +1638,16 @@ static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, Buffer buffer)
*/
static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
{
Range const inUse = ZSTDMT_getInputDataInUse(mtctx);
range_t const inUse = ZSTDMT_getInputDataInUse(mtctx);
size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos;
size_t const spaceNeeded = mtctx->targetSectionSize;
Buffer buffer;
size_t const target = mtctx->targetSectionSize;
buffer_t buffer;
DEBUGLOG(5, "ZSTDMT_tryGetInputRange");
assert(mtctx->inBuff.buffer.start == NULL);
assert(mtctx->roundBuff.capacity >= spaceNeeded);
assert(mtctx->roundBuff.capacity >= target);
if (spaceLeft < spaceNeeded) {
if (spaceLeft < target) {
/* ZSTD_invalidateRepCodes() doesn't work for extDict variants.
* Simply copy the prefix to the beginning in that case.
*/
@@ -1708,7 +1666,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
mtctx->roundBuff.pos = prefixSize;
}
buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos;
buffer.capacity = spaceNeeded;
buffer.capacity = target;
if (ZSTDMT_isOverlapped(buffer, inUse)) {
DEBUGLOG(5, "Waiting for buffer...");
@@ -1735,7 +1693,7 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
typedef struct {
size_t toLoad; /* The number of bytes to load from the input. */
int flush; /* Boolean declaring if we must flush because we found a synchronization point. */
} SyncPoint;
} syncPoint_t;
/**
* Searches through the input for a synchronization point. If one is found, we
@@ -1743,14 +1701,14 @@ typedef struct {
* Otherwise, we will load as many bytes as possible and instruct the caller
* to continue as normal.
*/
static SyncPoint
static syncPoint_t
findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input)
{
BYTE const* const istart = (BYTE const*)input.src + input.pos;
U64 const primePower = mtctx->rsync.primePower;
U64 const hitMask = mtctx->rsync.hitMask;
SyncPoint syncPoint;
syncPoint_t syncPoint;
U64 hash;
BYTE const* prev;
size_t pos;
@@ -1882,7 +1840,7 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start);
}
if (mtctx->inBuff.buffer.start != NULL) {
SyncPoint const syncPoint = findSynchronizationPoint(mtctx, *input);
syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input);
if (syncPoint.flush && endOp == ZSTD_e_continue) {
endOp = ZSTD_e_flush;
}
+15 -4
View File
@@ -11,10 +11,10 @@
#ifndef ZSTDMT_COMPRESS_H
#define ZSTDMT_COMPRESS_H
/* === Dependencies === */
#include "../common/zstd_deps.h" /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
#include "../zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
#if defined (__cplusplus)
extern "C" {
#endif
/* Note : This is an internal API.
* These APIs used to be exposed with ZSTDLIB_API,
@@ -25,6 +25,12 @@
* otherwise ZSTDMT_createCCtx*() will fail.
*/
/* === Dependencies === */
#include "../common/zstd_deps.h" /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
#include "../zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
/* === Constants === */
#ifndef ZSTDMT_NBWORKERS_MAX /* a different value can be selected at compile time */
# define ZSTDMT_NBWORKERS_MAX ((sizeof(void*)==4) /*32-bit*/ ? 64 : 256)
@@ -99,4 +105,9 @@ void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_p
*/
ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTDMT_COMPRESS_H */
+40 -49
View File
@@ -15,7 +15,6 @@
/* **************************************************************
* Dependencies
****************************************************************/
#include <stddef.h> /* size_t */
#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */
#include "../common/compiler.h"
#include "../common/bitstream.h" /* BIT_* */
@@ -196,7 +195,7 @@ static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* ds
const BYTE* const istart = (const BYTE*)src;
BYTE* const oend = (BYTE*)ZSTD_maybeNullPtrAdd(dst, (ptrdiff_t)dstSize);
BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize);
/* The fast decoding loop assumes 64-bit little-endian.
* This condition is false on x32.
@@ -579,7 +578,7 @@ HUF_decompress1X1_usingDTable_internal_body(
const HUF_DTable* DTable)
{
BYTE* op = (BYTE*)dst;
BYTE* const oend = (BYTE*)ZSTD_maybeNullPtrAdd(op, (ptrdiff_t)dstSize);
BYTE* const oend = ZSTD_maybeNullPtrAdd(op, dstSize);
const void* dtPtr = DTable + 1;
const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
BIT_DStream_t bitD;
@@ -787,17 +786,17 @@ void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
#define HUF_4X1_DECODE_SYMBOL(_stream, _symbol) \
do { \
U64 const index = bits[(_stream)] >> 53; \
U16 const entry = dtable[index]; \
bits[(_stream)] <<= entry & 0x3F; \
op[(_stream)][(_symbol)] = (BYTE)(entry >> 8); \
int const index = (int)(bits[(_stream)] >> 53); \
int const entry = (int)dtable[index]; \
bits[(_stream)] <<= (entry & 0x3F); \
op[(_stream)][(_symbol)] = (BYTE)((entry >> 8) & 0xFF); \
} while (0)
#define HUF_5X1_RELOAD_STREAM(_stream) \
#define HUF_4X1_RELOAD_STREAM(_stream) \
do { \
U64 const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \
U64 const nbBits = ctz & 7; \
U64 const nbBytes = ctz >> 3; \
int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \
int const nbBits = ctz & 7; \
int const nbBytes = ctz >> 3; \
op[(_stream)] += 5; \
ip[(_stream)] -= nbBytes; \
bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \
@@ -816,11 +815,11 @@ void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4);
/* Reload each of the 4 the bitstreams */
HUF_4X_FOR_EACH_STREAM(HUF_5X1_RELOAD_STREAM);
HUF_4X_FOR_EACH_STREAM(HUF_4X1_RELOAD_STREAM);
} while (op[3] < olimit);
#undef HUF_4X1_DECODE_SYMBOL
#undef HUF_5X1_RELOAD_STREAM
#undef HUF_4X1_RELOAD_STREAM
}
_out:
@@ -846,7 +845,7 @@ HUF_decompress4X1_usingDTable_internal_fast(
{
void const* dt = DTable + 1;
BYTE const* const ilowest = (BYTE const*)cSrc;
BYTE* const oend = (BYTE*)ZSTD_maybeNullPtrAdd(dst, (ptrdiff_t)dstSize);
BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize);
HUF_DecompressFastArgs args;
{ size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
FORWARD_IF_ERROR(ret, "Failed to init fast loop args");
@@ -1363,7 +1362,7 @@ HUF_decompress1X2_usingDTable_internal_body(
/* decode */
{ BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = (BYTE*)ZSTD_maybeNullPtrAdd(ostart, (ptrdiff_t)dstSize);
BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, dstSize);
const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */
const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
DTableDesc const dtd = HUF_getDTableDesc(DTable);
@@ -1606,62 +1605,54 @@ void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
#define HUF_4X2_DECODE_SYMBOL(_stream, _decode3) \
do { \
if ((_decode3) || (_stream) != 3) { \
U64 const index = bits[(_stream)] >> 53; \
size_t const entry = MEM_readLE32(&dtable[index]); \
MEM_write16(op[(_stream)], (U16)entry); \
bits[(_stream)] <<= (entry >> 16) & 0x3F; \
op[(_stream)] += entry >> 24; \
int const index = (int)(bits[(_stream)] >> 53); \
HUF_DEltX2 const entry = dtable[index]; \
MEM_write16(op[(_stream)], entry.sequence); \
bits[(_stream)] <<= (entry.nbBits) & 0x3F; \
op[(_stream)] += (entry.length); \
} \
} while (0)
#define HUF_5X2_RELOAD_STREAM(_stream, _decode3) \
#define HUF_4X2_RELOAD_STREAM(_stream) \
do { \
if (_decode3) HUF_4X2_DECODE_SYMBOL(3, 1); \
HUF_4X2_DECODE_SYMBOL(3, 1); \
{ \
U64 const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \
U64 const nbBits = ctz & 7; \
U64 const nbBytes = ctz >> 3; \
int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \
int const nbBits = ctz & 7; \
int const nbBytes = ctz >> 3; \
ip[(_stream)] -= nbBytes; \
bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \
bits[(_stream)] <<= nbBits; \
} \
} while (0)
#if defined(__aarch64__)
# define HUF_4X2_4WAY 1
#else
# define HUF_4X2_4WAY 0
#endif
#define HUF_4X2_3WAY !HUF_4X2_4WAY
/* Manually unroll the loop because compilers don't consistently
* unroll the inner loops, which destroys performance.
*/
do {
/* Decode 5 symbols from each of the first 3 or 4 streams.
* In the 3-way case the final stream will be decoded during
* the reload phase to reduce register pressure.
/* Decode 5 symbols from each of the first 3 streams.
* The final stream will be decoded during the reload phase
* to reduce register pressure.
*/
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, HUF_4X2_4WAY);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, HUF_4X2_4WAY);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, HUF_4X2_4WAY);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, HUF_4X2_4WAY);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, HUF_4X2_4WAY);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0);
/* In the 3-way case decode one symbol from the final stream. */
HUF_4X2_DECODE_SYMBOL(3, HUF_4X2_3WAY);
/* Decode one symbol from the final stream */
HUF_4X2_DECODE_SYMBOL(3, 1);
/* In the 3-way case decode 4 symbols from the final stream &
* reload bitstreams. The final stream is reloaded last, meaning
* that all 5 symbols are decoded from the final stream before
* it is reloaded.
/* Decode 4 symbols from the final stream & reload bitstreams.
* The final stream is reloaded last, meaning that all 5 symbols
* are decoded from the final stream before it is reloaded.
*/
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_5X2_RELOAD_STREAM, HUF_4X2_3WAY);
HUF_4X_FOR_EACH_STREAM(HUF_4X2_RELOAD_STREAM);
} while (op[3] < olimit);
}
#undef HUF_4X2_DECODE_SYMBOL
#undef HUF_5X2_RELOAD_STREAM
#undef HUF_4X2_RELOAD_STREAM
_out:
@@ -1680,7 +1671,7 @@ HUF_decompress4X2_usingDTable_internal_fast(
HUF_DecompressFastLoopFn loopFn) {
void const* dt = DTable + 1;
const BYTE* const ilowest = (const BYTE*)cSrc;
BYTE* const oend = (BYTE*)ZSTD_maybeNullPtrAdd(dst, (ptrdiff_t)dstSize);
BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize);
HUF_DecompressFastArgs args;
{
size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+21 -192
View File
@@ -38,25 +38,17 @@
#endif
// There appears to be an unreconcilable syntax difference between Linux and Darwin assemblers.
// Name of a private label (i.e. not exported to symbol table) on Darwin has to start with "L",
// on Linux has to start with ".". There's no way to have a name start with both "." and "L", so
// we have to use a macro.
#if defined(__APPLE__)
#define LOCAL_LABEL(label) L_ ## label
#else
#define LOCAL_LABEL(label) .L_ ## label
#endif
#if ZSTD_ENABLE_ASM_X86_64_BMI2
/* Calling convention:
*
* %rdi (or %rcx on Windows) contains the first argument: HUF_DecompressAsmArgs*.
* %rdi contains the first argument: HUF_DecompressAsmArgs*.
* %rbp isn't maintained (no frame pointer).
* %rsp contains the stack pointer that grows down.
* No red-zone is assumed, only addresses >= %rsp are used.
* All register contents are preserved.
*
* TODO: Support Windows calling convention.
*/
ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_fast_asm_loop)
@@ -127,62 +119,25 @@ ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_fast_asm_loop)
_HUF_decompress4X1_usingDTable_internal_fast_asm_loop:
HUF_decompress4X1_usingDTable_internal_fast_asm_loop:
ZSTD_CET_ENDBRANCH
.cfi_startproc
.cfi_def_cfa_offset 8
.cfi_offset %rip, -8
/* Save all registers - even if they are callee saved for simplicity. */
push %rax
.cfi_def_cfa_offset 16
.cfi_offset rax, -16
push %rbx
.cfi_def_cfa_offset 24
.cfi_offset rbx, -24
push %rcx
.cfi_def_cfa_offset 32
.cfi_offset rcx, -32
push %rdx
.cfi_def_cfa_offset 40
.cfi_offset rdx, -40
push %rbp
.cfi_def_cfa_offset 48
.cfi_offset rbp, -48
push %rsi
.cfi_def_cfa_offset 56
.cfi_offset rsi, -56
push %rdi
.cfi_def_cfa_offset 64
.cfi_offset rdi, -64
push %r8
.cfi_def_cfa_offset 72
.cfi_offset r8, -72
push %r9
.cfi_def_cfa_offset 80
.cfi_offset r9, -80
push %r10
.cfi_def_cfa_offset 88
.cfi_offset r10, -88
push %r11
.cfi_def_cfa_offset 96
.cfi_offset r11, -96
push %r12
.cfi_def_cfa_offset 104
.cfi_offset r12, -104
push %r13
.cfi_def_cfa_offset 112
.cfi_offset r13, -112
push %r14
.cfi_def_cfa_offset 120
.cfi_offset r14, -120
push %r15
.cfi_def_cfa_offset 128
.cfi_offset r15, -128
/* Read HUF_DecompressAsmArgs* args from %rax */
#if defined(_WIN32)
movq %rcx, %rax
#else
movq %rdi, %rax
#endif
movq 0(%rax), %ip0
movq 8(%rax), %ip1
movq 16(%rax), %ip2
@@ -197,18 +152,13 @@ HUF_decompress4X1_usingDTable_internal_fast_asm_loop:
movq 88(%rax), %bits3
movq 96(%rax), %dtable
push %rax /* argument */
.cfi_def_cfa_offset 136
push 104(%rax) /* ilowest */
.cfi_def_cfa_offset 144
push 112(%rax) /* oend */
.cfi_def_cfa_offset 152
push %olimit /* olimit space */
.cfi_def_cfa_offset 160
subq $24, %rsp
.cfi_def_cfa_offset 184
LOCAL_LABEL(4X1_compute_olimit):
.L_4X1_compute_olimit:
/* Computes how many iterations we can do safely
* %r15, %rax may be clobbered
* rbx, rdx must be saved
@@ -255,19 +205,19 @@ LOCAL_LABEL(4X1_compute_olimit):
/* If (op3 + 20 > olimit) */
movq %op3, %rax /* rax = op3 */
cmpq %rax, %olimit /* op3 == olimit */
je LOCAL_LABEL(4X1_exit)
je .L_4X1_exit
/* If (ip1 < ip0) go to exit */
cmpq %ip0, %ip1
jb LOCAL_LABEL(4X1_exit)
jb .L_4X1_exit
/* If (ip2 < ip1) go to exit */
cmpq %ip1, %ip2
jb LOCAL_LABEL(4X1_exit)
jb .L_4X1_exit
/* If (ip3 < ip2) go to exit */
cmpq %ip2, %ip3
jb LOCAL_LABEL(4X1_exit)
jb .L_4X1_exit
/* Reads top 11 bits from bits[n]
* Loads dt[bits[n]] into var[n]
@@ -328,7 +278,7 @@ LOCAL_LABEL(4X1_compute_olimit):
.p2align 6
LOCAL_LABEL(4X1_loop_body):
.L_4X1_loop_body:
/* Decode 5 symbols in each of the 4 streams (20 total)
* Must have called GET_NEXT_DELT for each stream
*/
@@ -366,7 +316,7 @@ LOCAL_LABEL(4X1_loop_body):
/* If op3 < olimit: continue the loop */
cmp %op3, 24(%rsp)
ja LOCAL_LABEL(4X1_loop_body)
ja .L_4X1_loop_body
/* Reload ip[1,2,3] from stack */
movq 0(%rsp), %ip1
@@ -374,25 +324,20 @@ LOCAL_LABEL(4X1_loop_body):
movq 16(%rsp), %ip3
/* Re-compute olimit */
jmp LOCAL_LABEL(4X1_compute_olimit)
jmp .L_4X1_compute_olimit
#undef GET_NEXT_DELT
#undef DECODE_FROM_DELT
#undef DECODE
#undef RELOAD_BITS
LOCAL_LABEL(4X1_exit):
.L_4X1_exit:
addq $24, %rsp
.cfi_def_cfa_offset 160
/* Restore stack (oend & olimit) */
pop %rax /* olimit */
.cfi_def_cfa_offset 152
pop %rax /* oend */
.cfi_def_cfa_offset 144
pop %rax /* ilowest */
.cfi_def_cfa_offset 136
pop %rax /* arg */
.cfi_def_cfa_offset 128
/* Save ip / op / bits */
movq %ip0, 0(%rax)
@@ -410,112 +355,43 @@ LOCAL_LABEL(4X1_exit):
/* Restore registers */
pop %r15
.cfi_restore r15
.cfi_def_cfa_offset 120
pop %r14
.cfi_restore r14
.cfi_def_cfa_offset 112
pop %r13
.cfi_restore r13
.cfi_def_cfa_offset 104
pop %r12
.cfi_restore r12
.cfi_def_cfa_offset 96
pop %r11
.cfi_restore r11
.cfi_def_cfa_offset 88
pop %r10
.cfi_restore r10
.cfi_def_cfa_offset 80
pop %r9
.cfi_restore r9
.cfi_def_cfa_offset 72
pop %r8
.cfi_restore r8
.cfi_def_cfa_offset 64
pop %rdi
.cfi_restore rdi
.cfi_def_cfa_offset 56
pop %rsi
.cfi_restore rsi
.cfi_def_cfa_offset 48
pop %rbp
.cfi_restore rbp
.cfi_def_cfa_offset 40
pop %rdx
.cfi_restore rdx
.cfi_def_cfa_offset 32
pop %rcx
.cfi_restore rcx
.cfi_def_cfa_offset 24
pop %rbx
.cfi_restore rbx
.cfi_def_cfa_offset 16
pop %rax
.cfi_restore rax
.cfi_def_cfa_offset 8
ret
.cfi_endproc
_HUF_decompress4X2_usingDTable_internal_fast_asm_loop:
HUF_decompress4X2_usingDTable_internal_fast_asm_loop:
ZSTD_CET_ENDBRANCH
.cfi_startproc
.cfi_def_cfa_offset 8
.cfi_offset %rip, -8
/* Save all registers - even if they are callee saved for simplicity. */
push %rax
.cfi_def_cfa_offset 16
.cfi_offset rax, -16
push %rbx
.cfi_def_cfa_offset 24
.cfi_offset rbx, -24
push %rcx
.cfi_def_cfa_offset 32
.cfi_offset rcx, -32
push %rdx
.cfi_def_cfa_offset 40
.cfi_offset rdx, -40
push %rbp
.cfi_def_cfa_offset 48
.cfi_offset rbp, -48
push %rsi
.cfi_def_cfa_offset 56
.cfi_offset rsi, -56
push %rdi
.cfi_def_cfa_offset 64
.cfi_offset rdi, -64
push %r8
.cfi_def_cfa_offset 72
.cfi_offset r8, -72
push %r9
.cfi_def_cfa_offset 80
.cfi_offset r9, -80
push %r10
.cfi_def_cfa_offset 88
.cfi_offset r10, -88
push %r11
.cfi_def_cfa_offset 96
.cfi_offset r11, -96
push %r12
.cfi_def_cfa_offset 104
.cfi_offset r12, -104
push %r13
.cfi_def_cfa_offset 112
.cfi_offset r13, -112
push %r14
.cfi_def_cfa_offset 120
.cfi_offset r14, -120
push %r15
.cfi_def_cfa_offset 128
.cfi_offset r15, -128
/* Read HUF_DecompressAsmArgs* args from %rax */
#if defined(_WIN32)
movq %rcx, %rax
#else
movq %rdi, %rax
#endif
movq 0(%rax), %ip0
movq 8(%rax), %ip1
movq 16(%rax), %ip2
@@ -530,33 +406,25 @@ HUF_decompress4X2_usingDTable_internal_fast_asm_loop:
movq 88(%rax), %bits3
movq 96(%rax), %dtable
push %rax /* argument */
.cfi_def_cfa_offset 136
push %rax /* olimit */
.cfi_def_cfa_offset 144
push 104(%rax) /* ilowest */
.cfi_def_cfa_offset 152
movq 112(%rax), %rax
push %rax /* oend3 */
.cfi_def_cfa_offset 160
movq %op3, %rax
push %rax /* oend2 */
.cfi_def_cfa_offset 168
movq %op2, %rax
push %rax /* oend1 */
.cfi_def_cfa_offset 176
movq %op1, %rax
push %rax /* oend0 */
.cfi_def_cfa_offset 184
/* Scratch space */
subq $8, %rsp
.cfi_def_cfa_offset 192
LOCAL_LABEL(4X2_compute_olimit):
.L_4X2_compute_olimit:
/* Computes how many iterations we can do safely
* %r15, %rax may be clobbered
* rdx must be saved
@@ -620,19 +488,19 @@ LOCAL_LABEL(4X2_compute_olimit):
/* If (op3 + 10 > olimit) */
movq %op3, %rax /* rax = op3 */
cmpq %rax, %olimit /* op3 == olimit */
je LOCAL_LABEL(4X2_exit)
je .L_4X2_exit
/* If (ip1 < ip0) go to exit */
cmpq %ip0, %ip1
jb LOCAL_LABEL(4X2_exit)
jb .L_4X2_exit
/* If (ip2 < ip1) go to exit */
cmpq %ip1, %ip2
jb LOCAL_LABEL(4X2_exit)
jb .L_4X2_exit
/* If (ip3 < ip2) go to exit */
cmpq %ip2, %ip3
jb LOCAL_LABEL(4X2_exit)
jb .L_4X2_exit
#define DECODE(n, idx) \
movq %bits##n, %rax; \
@@ -659,7 +527,7 @@ LOCAL_LABEL(4X2_compute_olimit):
.p2align 6
LOCAL_LABEL(4X2_loop_body):
.L_4X2_loop_body:
/* We clobber r8, so store it on the stack */
movq %r8, 0(%rsp)
@@ -676,29 +544,21 @@ LOCAL_LABEL(4X2_loop_body):
FOR_EACH_STREAM(RELOAD_BITS)
cmp %op3, 48(%rsp)
ja LOCAL_LABEL(4X2_loop_body)
jmp LOCAL_LABEL(4X2_compute_olimit)
ja .L_4X2_loop_body
jmp .L_4X2_compute_olimit
#undef DECODE
#undef RELOAD_BITS
LOCAL_LABEL(4X2_exit):
.L_4X2_exit:
addq $8, %rsp
.cfi_def_cfa_offset 184
/* Restore stack (oend & olimit) */
pop %rax /* oend0 */
.cfi_def_cfa_offset 176
pop %rax /* oend1 */
.cfi_def_cfa_offset 168
pop %rax /* oend2 */
.cfi_def_cfa_offset 160
pop %rax /* oend3 */
.cfi_def_cfa_offset 152
pop %rax /* ilowest */
.cfi_def_cfa_offset 144
pop %rax /* olimit */
.cfi_def_cfa_offset 136
pop %rax /* arg */
.cfi_def_cfa_offset 128
/* Save ip / op / bits */
movq %ip0, 0(%rax)
@@ -716,51 +576,20 @@ LOCAL_LABEL(4X2_exit):
/* Restore registers */
pop %r15
.cfi_restore r15
.cfi_def_cfa_offset 120
pop %r14
.cfi_restore r14
.cfi_def_cfa_offset 112
pop %r13
.cfi_restore r13
.cfi_def_cfa_offset 104
pop %r12
.cfi_restore r12
.cfi_def_cfa_offset 96
pop %r11
.cfi_restore r11
.cfi_def_cfa_offset 88
pop %r10
.cfi_restore r10
.cfi_def_cfa_offset 80
pop %r9
.cfi_restore r9
.cfi_def_cfa_offset 72
pop %r8
.cfi_restore r8
.cfi_def_cfa_offset 64
pop %rdi
.cfi_restore rdi
.cfi_def_cfa_offset 56
pop %rsi
.cfi_restore rsi
.cfi_def_cfa_offset 48
pop %rbp
.cfi_restore rbp
.cfi_def_cfa_offset 40
pop %rdx
.cfi_restore rdx
.cfi_def_cfa_offset 32
pop %rcx
.cfi_restore rcx
.cfi_def_cfa_offset 24
pop %rbx
.cfi_restore rbx
.cfi_def_cfa_offset 16
pop %rax
.cfi_restore rax
.cfi_def_cfa_offset 8
ret
.cfi_endproc
#endif
+11 -14
View File
@@ -444,7 +444,7 @@ size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
* @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
** or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
{
const BYTE* ip = (const BYTE*)src;
size_t const minInputSize = ZSTD_startingInputLength(format);
@@ -484,10 +484,8 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, s
if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));
zfhPtr->frameType = ZSTD_skippableFrame;
zfhPtr->dictID = MEM_readLE32(src) - ZSTD_MAGIC_SKIPPABLE_START;
zfhPtr->headerSize = ZSTD_SKIPPABLEHEADERSIZE;
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
zfhPtr->frameType = ZSTD_skippableFrame;
return 0;
}
RETURN_ERROR(prefix_unknown, "");
@@ -556,7 +554,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, s
* @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
* or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize)
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
{
return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);
}
@@ -574,7 +572,7 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
}
#endif
{ ZSTD_FrameHeader zfh;
{ ZSTD_frameHeader zfh;
if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0)
return ZSTD_CONTENTSIZE_ERROR;
if (zfh.frameType == ZSTD_skippableFrame) {
@@ -752,7 +750,7 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
const BYTE* const ipstart = ip;
size_t remainingSize = srcSize;
size_t nbBlocks = 0;
ZSTD_FrameHeader zfh;
ZSTD_frameHeader zfh;
/* Extract Frame Header */
{ size_t const ret = ZSTD_getFrameHeader_advanced(&zfh, src, srcSize, format);
@@ -813,7 +811,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
/** ZSTD_decompressBound() :
* compatible with legacy mode
* `src` must point to the start of a ZSTD frame or a skippable frame
* `src` must point to the start of a ZSTD frame or a skippeable frame
* `srcSize` must be at least as large as the frame contained
* @return : the maximum decompressed size of the compressed source
*/
@@ -845,7 +843,7 @@ size_t ZSTD_decompressionMargin(void const* src, size_t srcSize)
ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1);
size_t const compressedSize = frameSizeInfo.compressedSize;
unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
ZSTD_FrameHeader zfh;
ZSTD_frameHeader zfh;
FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), "");
if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
@@ -919,7 +917,7 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
return regenSize;
}
static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, int streaming)
static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
{
#if ZSTD_TRACE
if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) {
@@ -1059,7 +1057,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
}
ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
/* Allow caller to get size read */
DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %i, consuming %i bytes of input", (int)(op-ostart), (int)(ip - (const BYTE*)*srcPtr));
DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr);
*srcPtr = ip;
*srcSizePtr = remainingSrcSize;
return (size_t)(op-ostart);
@@ -1643,7 +1641,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
* ZSTD_getFrameHeader(), which will provide a more precise error code. */
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
{
ZSTD_FrameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 };
ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 };
size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
if (ZSTD_isError(hError)) return 0;
return zfp.dictID;
@@ -2001,7 +1999,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize)
size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
{
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */
ZSTD_FrameHeader zfh;
ZSTD_frameHeader zfh;
size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
if (ZSTD_isError(err)) return err;
RETURN_ERROR_IF(err>0, srcSize_wrong, "");
@@ -2096,7 +2094,6 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
U32 someMoreWork = 1;
DEBUGLOG(5, "ZSTD_decompressStream");
assert(zds != NULL);
RETURN_ERROR_IF(
input->pos > input->size,
srcSize_wrong,
+65 -161
View File
@@ -16,13 +16,14 @@
*********************************************************/
#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
#include "../common/compiler.h" /* prefetch */
#include "../common/cpu.h" /* bmi2 */
#include "../common/mem.h" /* low level memory routines */
#include <stddef.h>
#define FSE_STATIC_LINKING_ONLY
#include "../common/fse.h"
#include "../common/huf.h"
#include "../common/zstd_internal.h"
#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
#include "zstd_decompress_block.h"
#include "../common/bits.h" /* ZSTD_highbit32 */
@@ -138,7 +139,7 @@ static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");
{ const BYTE* const istart = (const BYTE*) src;
SymbolEncodingType_e const litEncType = (SymbolEncodingType_e)(istart[0] & 3);
symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
size_t const blockSizeMax = ZSTD_blockSizeMax(dctx);
switch(litEncType)
@@ -357,7 +358,7 @@ size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx,
* - start from default distributions, present in /lib/common/zstd_internal.h
* - generate tables normally, using ZSTD_buildFSETable()
* - printout the content of tables
* - prettify output, report below, test with fuzzer to ensure it's correct */
* - pretify output, report below, test with fuzzer to ensure it's correct */
/* Default FSE distribution table for Literal Lengths */
static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {
@@ -644,7 +645,7 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
* @return : nb bytes read from src,
* or an error code if it fails */
static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr,
SymbolEncodingType_e type, unsigned max, U32 maxLog,
symbolEncodingType_e type, unsigned max, U32 maxLog,
const void* src, size_t srcSize,
const U32* baseValue, const U8* nbAdditionalBits,
const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable,
@@ -727,16 +728,15 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
/* FSE table descriptors */
RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */
RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */
{ SymbolEncodingType_e const LLtype = (SymbolEncodingType_e)(*ip >> 6);
SymbolEncodingType_e const OFtype = (SymbolEncodingType_e)((*ip >> 4) & 3);
SymbolEncodingType_e const MLtype = (SymbolEncodingType_e)((*ip >> 2) & 3);
{ symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
ip++;
/* Build DTables */
assert(ip <= iend);
{ size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
LLtype, MaxLL, LLFSELog,
ip, (size_t)(iend-ip),
ip, iend-ip,
LL_base, LL_bits,
LL_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq,
@@ -746,10 +746,9 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
ip += llhSize;
}
assert(ip <= iend);
{ size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
OFtype, MaxOff, OffFSELog,
ip, (size_t)(iend-ip),
ip, iend-ip,
OF_base, OF_bits,
OF_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq,
@@ -759,10 +758,9 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
ip += ofhSize;
}
assert(ip <= iend);
{ size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
MLtype, MaxML, MLFSELog,
ip, (size_t)(iend-ip),
ip, iend-ip,
ML_base, ML_bits,
ML_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq,
@@ -773,7 +771,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
}
}
return (size_t)(ip-istart);
return ip-istart;
}
@@ -803,8 +801,7 @@ typedef struct {
* Precondition: *ip <= *op
* Postcondition: *op - *op >= 8
*/
HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset)
{
HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) {
assert(*ip <= *op);
if (offset < 8) {
/* close range match, overlap */
@@ -837,9 +834,7 @@ HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset)
* - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
* The src buffer must be before the dst buffer.
*/
static void
ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, size_t length, ZSTD_overlap_e ovtype)
{
static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) {
ptrdiff_t const diff = op - ip;
BYTE* const oend = op + length;
@@ -854,8 +849,7 @@ ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, size_t length,
if (ovtype == ZSTD_overlap_src_before_dst) {
/* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */
assert(length >= 8);
assert(diff > 0);
ZSTD_overlapCopy8(&op, &ip, (size_t)diff);
ZSTD_overlapCopy8(&op, &ip, diff);
length -= 8;
assert(op - ip >= 8);
assert(op <= oend);
@@ -869,7 +863,7 @@ ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, size_t length,
if (op <= oend_w) {
/* Wildcopy until we get close to the end. */
assert(oend > oend_w);
ZSTD_wildcopy(op, ip, (size_t)(oend_w - op), ovtype);
ZSTD_wildcopy(op, ip, oend_w - op, ovtype);
ip += oend_w - op;
op += oend_w - op;
}
@@ -880,8 +874,7 @@ ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, size_t length,
/* ZSTD_safecopyDstBeforeSrc():
* This version allows overlap with dst before src, or handles the non-overlap case with dst after src
* Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */
static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, size_t length)
{
static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) {
ptrdiff_t const diff = op - ip;
BYTE* const oend = op + length;
@@ -892,7 +885,7 @@ static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, size_t length)
}
if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) {
ZSTD_wildcopy(op, ip, (size_t)(oend - WILDCOPY_OVERLENGTH - op), ZSTD_no_overlap);
ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap);
ip += oend - WILDCOPY_OVERLENGTH - op;
op += oend - WILDCOPY_OVERLENGTH - op;
}
@@ -943,7 +936,7 @@ size_t ZSTD_execSequenceEnd(BYTE* op,
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = (size_t)(dictEnd - match);
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
@@ -992,7 +985,7 @@ size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op,
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = (size_t)(dictEnd - match);
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
@@ -1065,7 +1058,7 @@ size_t ZSTD_execSequence(BYTE* op,
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = (size_t)(dictEnd - match);
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
@@ -1086,7 +1079,7 @@ size_t ZSTD_execSequence(BYTE* op,
* longer than literals (in general). In silesia, ~10% of matches are longer
* than 16 bytes.
*/
ZSTD_wildcopy(op, match, sequence.matchLength, ZSTD_no_overlap);
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
return sequenceLength;
}
assert(sequence.offset < WILDCOPY_VECLEN);
@@ -1097,7 +1090,7 @@ size_t ZSTD_execSequence(BYTE* op,
/* If the match length is > 8 bytes, then continue with the wildcopy. */
if (sequence.matchLength > 8) {
assert(op < oMatchEnd);
ZSTD_wildcopy(op, match, sequence.matchLength - 8, ZSTD_overlap_src_before_dst);
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst);
}
return sequenceLength;
}
@@ -1158,7 +1151,7 @@ size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op,
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = (size_t)(dictEnd - match);
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
@@ -1178,7 +1171,7 @@ size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op,
* longer than literals (in general). In silesia, ~10% of matches are longer
* than 16 bytes.
*/
ZSTD_wildcopy(op, match, sequence.matchLength, ZSTD_no_overlap);
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
return sequenceLength;
}
assert(sequence.offset < WILDCOPY_VECLEN);
@@ -1189,7 +1182,7 @@ size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op,
/* If the match length is > 8 bytes, then continue with the wildcopy. */
if (sequence.matchLength > 8) {
assert(op < oMatchEnd);
ZSTD_wildcopy(op, match, sequence.matchLength-8, ZSTD_overlap_src_before_dst);
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst);
}
return sequenceLength;
}
@@ -1236,10 +1229,6 @@ FORCE_INLINE_TEMPLATE seq_t
ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq)
{
seq_t seq;
#if defined(__aarch64__)
size_t prevOffset0 = seqState->prevOffset[0];
size_t prevOffset1 = seqState->prevOffset[1];
size_t prevOffset2 = seqState->prevOffset[2];
/*
* ZSTD_seqSymbol is a 64 bits wide structure.
* It can be loaded in one operation
@@ -1248,7 +1237,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c
* operations that cause performance drop. This can be avoided by using this
* ZSTD_memcpy hack.
*/
# if defined(__GNUC__) && !defined(__clang__)
#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__))
ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS;
ZSTD_seqSymbol* const llDInfo = &llDInfoS;
ZSTD_seqSymbol* const mlDInfo = &mlDInfoS;
@@ -1256,12 +1245,11 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c
ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol));
ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol));
ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol));
# else
#else
const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
# endif
(void)longOffsets;
#endif
seq.matchLength = mlDInfo->baseValue;
seq.litLength = llDInfo->baseValue;
{ U32 const ofBase = ofDInfo->baseValue;
@@ -1280,98 +1268,10 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c
assert(llBits <= MaxLLBits);
assert(mlBits <= MaxMLBits);
assert(ofBits <= MaxOff);
/* As GCC has better branch and block analyzers, sometimes it is only
* valuable to mark likeliness for Clang.
*/
/* sequence */
{ size_t offset;
if (ofBits > 1) {
ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32);
ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits);
offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
prevOffset2 = prevOffset1;
prevOffset1 = prevOffset0;
prevOffset0 = offset;
} else {
U32 const ll0 = (llDInfo->baseValue == 0);
if (LIKELY((ofBits == 0))) {
if (ll0) {
offset = prevOffset1;
prevOffset1 = prevOffset0;
prevOffset0 = offset;
} else {
offset = prevOffset0;
}
} else {
offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1);
{ size_t temp = (offset == 1) ? prevOffset1
: (offset == 3) ? prevOffset0 - 1
: (offset >= 2) ? prevOffset2
: prevOffset0;
/* 0 is not valid: input corrupted => force offset to -1 =>
* corruption detected at execSequence.
*/
temp -= !temp;
prevOffset2 = (offset == 1) ? prevOffset2 : prevOffset1;
prevOffset1 = prevOffset0;
prevOffset0 = offset = temp;
} } }
seq.offset = offset;
}
if (mlBits > 0)
seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/);
if (UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
BIT_reloadDStream(&seqState->DStream);
/* Ensure there are enough bits to read the rest of data in 64-bit mode. */
ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
if (llBits > 0)
seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
if (!isLastSeq) {
/* Don't update FSE state for last sequence. */
ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */
ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */
ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */
BIT_reloadDStream(&seqState->DStream);
}
}
seqState->prevOffset[0] = prevOffset0;
seqState->prevOffset[1] = prevOffset1;
seqState->prevOffset[2] = prevOffset2;
#else /* !defined(__aarch64__) */
const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
seq.matchLength = mlDInfo->baseValue;
seq.litLength = llDInfo->baseValue;
{ U32 const ofBase = ofDInfo->baseValue;
BYTE const llBits = llDInfo->nbAdditionalBits;
BYTE const mlBits = mlDInfo->nbAdditionalBits;
BYTE const ofBits = ofDInfo->nbAdditionalBits;
BYTE const totalBits = llBits+mlBits+ofBits;
U16 const llNext = llDInfo->nextState;
U16 const mlNext = mlDInfo->nextState;
U16 const ofNext = ofDInfo->nextState;
U32 const llnbBits = llDInfo->nbBits;
U32 const mlnbBits = mlDInfo->nbBits;
U32 const ofnbBits = ofDInfo->nbBits;
assert(llBits <= MaxLLBits);
assert(mlBits <= MaxMLBits);
assert(ofBits <= MaxOff);
/* As GCC has better branch and block analyzers, sometimes it is only
* valuable to mark likeliness for Clang.
/*
* As gcc has better branch and block analyzers, sometimes it is only
* valuable to mark likeliness for clang, it gives around 3-4% of
* performance.
*/
/* sequence */
@@ -1433,7 +1333,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c
(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
if (!isLastSeq) {
/* Don't update FSE state for last sequence. */
/* don't update FSE state for last Sequence */
ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */
ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */
if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
@@ -1441,7 +1341,6 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, c
BIT_reloadDStream(&seqState->DStream);
}
}
#endif /* defined(__aarch64__) */
return seq;
}
@@ -1506,8 +1405,10 @@ ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = (BYTE*)ZSTD_maybeNullPtrAdd(ostart, (ptrdiff_t)maxDstSize);
BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize);
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* litBufferEnd = dctx->litBufferEnd;
@@ -1522,7 +1423,7 @@ ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
RETURN_ERROR_IF(
ERR_isError(BIT_initDStream(&seqState.DStream, seqStart, seqSize)),
ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
corruption_detected, "");
ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@@ -1614,8 +1515,7 @@ ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
/* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */
if (nbSeq > 0) {
const size_t leftoverLit = (size_t)(dctx->litBufferEnd - litPtr);
assert(dctx->litBufferEnd >= litPtr);
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength);
if (leftoverLit) {
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
@@ -1717,10 +1617,10 @@ ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = (dctx->litBufferLocation == ZSTD_not_in_dst) ?
(BYTE*)ZSTD_maybeNullPtrAdd(ostart, (ptrdiff_t)maxDstSize) :
dctx->litBuffer;
BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer;
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litEnd = litPtr + dctx->litSize;
@@ -1735,7 +1635,7 @@ ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
dctx->fseEntropy = 1;
{ U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
RETURN_ERROR_IF(
ERR_isError(BIT_initDStream(&seqState.DStream, seqStart, seqSize)),
ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)),
corruption_detected, "");
ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@@ -1819,8 +1719,8 @@ size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence,
{ const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart;
/* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted.
* No consequence though : memory address is only used for prefetching, not for dereferencing */
const BYTE* const match = (const BYTE*)ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, (ptrdiff_t)prefetchPos), (ptrdiff_t)sequence.offset);
PREFETCH_L1(match); PREFETCH_L1(ZSTD_wrappedPtrAdd(match, CACHELINE_SIZE)); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset);
PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
}
return prefetchPos + sequence.matchLength;
}
@@ -1836,10 +1736,10 @@ ZSTD_decompressSequencesLong_body(
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = (dctx->litBufferLocation == ZSTD_in_dst) ?
dctx->litBuffer :
(BYTE*)ZSTD_maybeNullPtrAdd(ostart, (ptrdiff_t)maxDstSize);
BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize);
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* litBufferEnd = dctx->litBufferEnd;
@@ -1861,8 +1761,9 @@ ZSTD_decompressSequencesLong_body(
dctx->fseEntropy = 1;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
assert(dst != NULL);
assert(iend >= ip);
RETURN_ERROR_IF(
ERR_isError(BIT_initDStream(&seqState.DStream, seqStart, seqSize)),
ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
corruption_detected, "");
ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@@ -1881,9 +1782,9 @@ ZSTD_decompressSequencesLong_body(
if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) {
/* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */
const size_t leftoverLit = (size_t)(dctx->litBufferEnd - litPtr);
assert(dctx->litBufferEnd >= litPtr);
if (leftoverLit) {
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
if (leftoverLit)
{
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit;
@@ -1927,8 +1828,7 @@ ZSTD_decompressSequencesLong_body(
for ( ; seqNb<nbSeq ; seqNb++) {
seq_t *sequence = &(sequences[seqNb&STORED_SEQS_MASK]);
if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) {
const size_t leftoverLit = (size_t)(dctx->litBufferEnd - litPtr);
assert(dctx->litBufferEnd >= litPtr);
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
if (leftoverLit) {
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
@@ -1967,8 +1867,7 @@ ZSTD_decompressSequencesLong_body(
/* last literal segment */
if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */
size_t const lastLLSize = (size_t)(litBufferEnd - litPtr);
assert(litBufferEnd >= litPtr);
size_t const lastLLSize = litBufferEnd - litPtr;
RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memmove(op, litPtr, lastLLSize);
@@ -1977,8 +1876,7 @@ ZSTD_decompressSequencesLong_body(
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
}
{ size_t const lastLLSize = (size_t)(litBufferEnd - litPtr);
assert(litBufferEnd >= litPtr);
{ size_t const lastLLSize = litBufferEnd - litPtr;
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memmove(op, litPtr, lastLLSize);
@@ -2037,6 +1935,12 @@ ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx,
#endif /* DYNAMIC_BMI2 */
typedef size_t (*ZSTD_decompressSequences_t)(
ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset);
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
static size_t
ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
@@ -2095,9 +1999,9 @@ ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
* both the prefix and the extDict. At @p op any offset larger than this
* is invalid.
*/
static size_t ZSTD_totalHistorySize(void* curPtr, const void* virtualStart)
static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart)
{
return (size_t)((char*)curPtr - (const char*)virtualStart);
return (size_t)(op - virtualStart);
}
typedef struct {
@@ -2196,7 +2100,7 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
* Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t.
*/
size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx));
size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd(dst, (ptrdiff_t)blockSizeMax), (BYTE const*)dctx->virtualStart);
size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart);
/* isLongOffset must be true if there are long offsets.
* Offsets are long if they are larger than ZSTD_maxShortOffset().
* We don't expect that to be the case in 64-bit mode.
+3 -3
View File
@@ -136,7 +136,7 @@ struct ZSTD_DCtx_s
const void* virtualStart; /* virtual start of previous segment if it was just before current one */
const void* dictEnd; /* end of previous segment */
size_t expected;
ZSTD_FrameHeader fParams;
ZSTD_frameHeader fParams;
U64 processedCSize;
U64 decodedSize;
blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
@@ -154,7 +154,7 @@ struct ZSTD_DCtx_s
size_t rleSize;
size_t staticSize;
int isFrameDecompression;
#if DYNAMIC_BMI2
#if DYNAMIC_BMI2 != 0
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
#endif
@@ -211,7 +211,7 @@ struct ZSTD_DCtx_s
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) {
#if DYNAMIC_BMI2
#if DYNAMIC_BMI2 != 0
return dctx->bmi2;
#else
(void)dctx;
+57 -129
View File
@@ -21,21 +21,8 @@
/*-*************************************
* Dependencies
***************************************/
/* qsort_r is an extension.
*
* Android NDK does not ship qsort_r().
*/
#if (defined(__linux__) && !defined(__ANDROID__)) || defined(__CYGWIN__) || defined(__MSYS__)
# ifndef _GNU_SOURCE
# define _GNU_SOURCE
# endif
#endif
#define __STDC_WANT_LIB_EXT1__ 1 /* request C11 Annex K, which includes qsort_s() */
#include <stdio.h> /* fprintf */
#include <stdlib.h> /* malloc, free, qsort_r */
#include <stdlib.h> /* malloc, free, qsort */
#include <string.h> /* memset */
#include <time.h> /* clock */
@@ -43,7 +30,6 @@
# define ZDICT_STATIC_LINKING_ONLY
#endif
#include "../common/debug.h" /* DEBUG_STATIC_ASSERT */
#include "../common/mem.h" /* read */
#include "../common/pool.h" /* POOL_ctx */
#include "../common/threading.h" /* ZSTD_pthread_mutex_t */
@@ -65,58 +51,40 @@
#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB))
#define COVER_DEFAULT_SPLITPOINT 1.0
/**
* Select the qsort() variant used by cover
*/
#define ZDICT_QSORT_MIN 0
#define ZDICT_QSORT_C90 ZDICT_QSORT_MIN
#define ZDICT_QSORT_GNU 1
#define ZDICT_QSORT_APPLE 2
#define ZDICT_QSORT_MSVC 3
#define ZDICT_QSORT_C11 ZDICT_QSORT_MAX
#define ZDICT_QSORT_MAX 4
#ifndef ZDICT_QSORT
# if defined(__APPLE__)
# define ZDICT_QSORT ZDICT_QSORT_APPLE /* uses qsort_r() with a different order for parameters */
# elif (defined(__linux__) && !defined(__ANDROID__)) || defined(__CYGWIN__) || defined(__MSYS__)
# define ZDICT_QSORT ZDICT_QSORT_GNU /* uses qsort_r() */
# elif defined(_WIN32) && defined(_MSC_VER)
# define ZDICT_QSORT ZDICT_QSORT_MSVC /* uses qsort_s() with a different order for parameters */
# elif defined(STDC_LIB_EXT1) && (STDC_LIB_EXT1 > 0) /* C11 Annex K */
# define ZDICT_QSORT ZDICT_QSORT_C11 /* uses qsort_s() */
# else
# define ZDICT_QSORT ZDICT_QSORT_C90 /* uses standard qsort() which is not re-entrant (requires global variable) */
# endif
#endif
/*-*************************************
* Console display
*
* Captures the `displayLevel` variable in the local scope.
***************************************/
#ifndef LOCALDISPLAYLEVEL
static int g_displayLevel = 0;
#endif
#undef DISPLAY
#define DISPLAY(...) \
{ \
fprintf(stderr, __VA_ARGS__); \
fflush(stderr); \
}
#undef DISPLAYLEVEL
#define DISPLAYLEVEL(l, ...) \
#undef LOCALDISPLAYLEVEL
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
if (displayLevel >= l) { \
DISPLAY(__VA_ARGS__); \
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
#undef DISPLAYLEVEL
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
#undef DISPLAYUPDATE
#define DISPLAYUPDATE(lastUpdateTime, l, ...) \
#ifndef LOCALDISPLAYUPDATE
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
static clock_t g_time = 0;
#endif
#undef LOCALDISPLAYUPDATE
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
if (displayLevel >= l) { \
const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; \
if ((clock() - lastUpdateTime > refreshRate) || (displayLevel >= 4)) { \
lastUpdateTime = clock(); \
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
g_time = clock(); \
DISPLAY(__VA_ARGS__); \
} \
}
#undef DISPLAYUPDATE
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
/*-*************************************
* Hash table
@@ -211,7 +179,7 @@ static U32 *COVER_map_at(COVER_map_t *map, U32 key) {
*/
static void COVER_map_remove(COVER_map_t *map, U32 key) {
U32 i = COVER_map_index(map, key);
COVER_map_pair_t* del = &map->data[i];
COVER_map_pair_t *del = &map->data[i];
U32 shift = 1;
if (del->value == MAP_EMPTY_VALUE) {
return;
@@ -262,13 +230,10 @@ typedef struct {
U32 *freqs;
U32 *dmerAt;
unsigned d;
int displayLevel;
} COVER_ctx_t;
#if ZDICT_QSORT == ZDICT_QSORT_C90
/* Use global context for non-reentrant sort functions */
/* We need a global context for qsort... */
static COVER_ctx_t *g_coverCtx = NULL;
#endif
/*-*************************************
* Helper functions
@@ -311,15 +276,11 @@ static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) {
/**
* Same as COVER_cmp() except ties are broken by pointer value
* NOTE: g_coverCtx must be set to call this function. A global is required because
* qsort doesn't take an opaque pointer.
*/
#if (ZDICT_QSORT == ZDICT_QSORT_MSVC) || (ZDICT_QSORT == ZDICT_QSORT_APPLE)
static int WIN_CDECL COVER_strict_cmp(void* g_coverCtx, const void* lp, const void* rp) {
#elif (ZDICT_QSORT == ZDICT_QSORT_GNU) || (ZDICT_QSORT == ZDICT_QSORT_C11)
static int COVER_strict_cmp(const void *lp, const void *rp, void *g_coverCtx) {
#else /* C90 fallback.*/
static int COVER_strict_cmp(const void *lp, const void *rp) {
#endif
int result = COVER_cmp((COVER_ctx_t*)g_coverCtx, lp, rp);
static int WIN_CDECL COVER_strict_cmp(const void *lp, const void *rp) {
int result = COVER_cmp(g_coverCtx, lp, rp);
if (result == 0) {
result = lp < rp ? -1 : 1;
}
@@ -328,52 +289,14 @@ static int COVER_strict_cmp(const void *lp, const void *rp) {
/**
* Faster version for d <= 8.
*/
#if (ZDICT_QSORT == ZDICT_QSORT_MSVC) || (ZDICT_QSORT == ZDICT_QSORT_APPLE)
static int WIN_CDECL COVER_strict_cmp8(void* g_coverCtx, const void* lp, const void* rp) {
#elif (ZDICT_QSORT == ZDICT_QSORT_GNU) || (ZDICT_QSORT == ZDICT_QSORT_C11)
static int COVER_strict_cmp8(const void *lp, const void *rp, void *g_coverCtx) {
#else /* C90 fallback.*/
static int COVER_strict_cmp8(const void *lp, const void *rp) {
#endif
int result = COVER_cmp8((COVER_ctx_t*)g_coverCtx, lp, rp);
static int WIN_CDECL COVER_strict_cmp8(const void *lp, const void *rp) {
int result = COVER_cmp8(g_coverCtx, lp, rp);
if (result == 0) {
result = lp < rp ? -1 : 1;
}
return result;
}
/**
* Abstract away divergence of qsort_r() parameters.
* Hopefully when C11 become the norm, we will be able
* to clean it up.
*/
static void stableSort(COVER_ctx_t *ctx)
{
DEBUG_STATIC_ASSERT(ZDICT_QSORT_MIN <= ZDICT_QSORT && ZDICT_QSORT <= ZDICT_QSORT_MAX);
#if (ZDICT_QSORT == ZDICT_QSORT_APPLE)
qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
ctx,
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
#elif (ZDICT_QSORT == ZDICT_QSORT_GNU)
qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
ctx);
#elif (ZDICT_QSORT == ZDICT_QSORT_MSVC)
qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
ctx);
#elif (ZDICT_QSORT == ZDICT_QSORT_C11)
qsort_s(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
ctx);
#else /* C90 fallback.*/
g_coverCtx = ctx;
/* TODO(cavalcanti): implement a reentrant qsort() when _r is not available. */
qsort(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
#endif
}
/**
* Returns the first pointer in [first, last) whose element does not compare
* less than value. If no such element exists it returns last.
@@ -627,7 +550,7 @@ static void COVER_ctx_destroy(COVER_ctx_t *ctx) {
*/
static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
const size_t *samplesSizes, unsigned nbSamples,
unsigned d, double splitPoint, int displayLevel)
unsigned d, double splitPoint)
{
const BYTE *const samples = (const BYTE *)samplesBuffer;
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
@@ -636,7 +559,6 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
ctx->displayLevel = displayLevel;
/* Checks */
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
@@ -698,7 +620,17 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
for (i = 0; i < ctx->suffixSize; ++i) {
ctx->suffix[i] = i;
}
stableSort(ctx);
/* qsort doesn't take an opaque pointer, so pass as a global.
* On OpenBSD qsort() is not guaranteed to be stable, their mergesort() is.
*/
g_coverCtx = ctx;
#if defined(__OpenBSD__)
mergesort(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
#else
qsort(ctx->suffix, ctx->suffixSize, sizeof(U32),
(ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
#endif
}
DISPLAYLEVEL(2, "Computing frequencies\n");
/* For each dmer group (group of positions with the same first d bytes):
@@ -721,7 +653,7 @@ void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLeve
if (ratio >= 10) {
return;
}
DISPLAYLEVEL(1,
LOCALDISPLAYLEVEL(displayLevel, 1,
"WARNING: The maximum dictionary size %u is too large "
"compared to the source size %u! "
"size(source)/size(dictionary) = %f, but it should be >= "
@@ -763,8 +695,6 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
const size_t maxZeroScoreRun = MAX(10, MIN(100, epochs.num >> 3));
size_t zeroScoreRun = 0;
size_t epoch;
clock_t lastUpdateTime = 0;
const int displayLevel = ctx->displayLevel;
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
(U32)epochs.num, (U32)epochs.size);
/* Loop through the epochs until there are no more segments or the dictionary
@@ -798,7 +728,6 @@ static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs,
tail -= segmentSize;
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
DISPLAYUPDATE(
lastUpdateTime,
2, "\r%u%% ",
(unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
}
@@ -814,8 +743,9 @@ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
BYTE* const dict = (BYTE*)dictBuffer;
COVER_ctx_t ctx;
COVER_map_t activeDmers;
const int displayLevel = (int)parameters.zParams.notificationLevel;
parameters.splitPoint = 1.0;
/* Initialize global data */
g_displayLevel = (int)parameters.zParams.notificationLevel;
/* Checks */
if (!COVER_checkParameters(parameters, dictBufferCapacity)) {
DISPLAYLEVEL(1, "Cover parameters incorrect\n");
@@ -833,12 +763,12 @@ ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover(
/* Initialize context and activeDmers */
{
size_t const initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
parameters.d, parameters.splitPoint, displayLevel);
parameters.d, parameters.splitPoint);
if (ZSTD_isError(initVal)) {
return initVal;
}
}
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, displayLevel);
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, g_displayLevel);
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
COVER_ctx_destroy(&ctx);
@@ -1161,7 +1091,6 @@ static void COVER_tryParameters(void *opaque)
BYTE* const dict = (BYTE*)malloc(dictBufferCapacity);
COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
U32* const freqs = (U32*)malloc(ctx->suffixSize * sizeof(U32));
const int displayLevel = ctx->displayLevel;
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
goto _cleanup;
@@ -1213,22 +1142,21 @@ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
(1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
const unsigned shrinkDict = 0;
/* Local variables */
int displayLevel = (int)parameters->zParams.notificationLevel;
const int displayLevel = parameters->zParams.notificationLevel;
unsigned iteration = 1;
unsigned d;
unsigned k;
COVER_best_t best;
POOL_ctx *pool = NULL;
int warned = 0;
clock_t lastUpdateTime = 0;
/* Checks */
if (splitPoint <= 0 || splitPoint > 1) {
DISPLAYLEVEL(1, "Incorrect parameters\n");
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
return ERROR(parameter_outOfBound);
}
if (kMinK < kMaxD || kMaxK < kMinK) {
DISPLAYLEVEL(1, "Incorrect parameters\n");
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
return ERROR(parameter_outOfBound);
}
if (nbSamples == 0) {
@@ -1248,19 +1176,19 @@ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
}
/* Initialization */
COVER_best_init(&best);
/* Turn down global display level to clean up display at level 2 and below */
g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
/* Loop through d first because each new value needs a new context */
DISPLAYLEVEL(2, "Trying %u different sets of parameters\n",
LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
kIterations);
for (d = kMinD; d <= kMaxD; d += 2) {
/* Initialize the context for this value of d */
COVER_ctx_t ctx;
DISPLAYLEVEL(3, "d=%u\n", d);
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
{
/* Turn down global display level to clean up display at level 2 and below */
const int childDisplayLevel = (displayLevel == 0) ? 0 : displayLevel - 1;
const size_t initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, childDisplayLevel);
const size_t initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint);
if (ZSTD_isError(initVal)) {
DISPLAYLEVEL(1, "Failed to initialize context\n");
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
COVER_best_destroy(&best);
POOL_free(pool);
return initVal;
@@ -1275,9 +1203,9 @@ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
/* Prepare the arguments */
COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc(
sizeof(COVER_tryParameters_data_t));
DISPLAYLEVEL(3, "k=%u\n", k);
LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
if (!data) {
DISPLAYLEVEL(1, "Failed to allocate parameters\n");
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
COVER_best_destroy(&best);
COVER_ctx_destroy(&ctx);
POOL_free(pool);
@@ -1292,7 +1220,7 @@ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps;
data->parameters.shrinkDict = shrinkDict;
data->parameters.zParams.notificationLevel = (unsigned)ctx.displayLevel;
data->parameters.zParams.notificationLevel = g_displayLevel;
/* Check the parameters */
if (!COVER_checkParameters(data->parameters, dictBufferCapacity)) {
DISPLAYLEVEL(1, "Cover parameters incorrect\n");
@@ -1307,14 +1235,14 @@ ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover(
COVER_tryParameters(data);
}
/* Print status */
DISPLAYUPDATE(lastUpdateTime, 2, "\r%u%% ",
LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
(unsigned)((iteration * 100) / kIterations));
++iteration;
}
COVER_best_wait(&best);
COVER_ctx_destroy(&ctx);
}
DISPLAYLEVEL(2, "\r%79s\r", "");
LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
/* Fill the output buffer and parameters with output of the best parameters */
{
const size_t dictSize = best.dictSize;
+77 -77
View File
@@ -25,22 +25,23 @@
*/
/*- Compiler specifics -*/
#ifdef __clang__
#pragma clang diagnostic ignored "-Wshorten-64-to-32"
#endif
#if defined(_MSC_VER)
# pragma warning(disable : 4244)
# pragma warning(disable : 4127) /* C4127 : Condition expression is constant */
#endif
/*- Dependencies -*/
#include <assert.h>
#include <limits.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include "divsufsort.h"
#define PTRDIFF_TO_INT(x) (assert((x) <= INT_MAX && (x) >= INT_MIN), (int)(x))
/*- Constants -*/
#if defined(INLINE)
# undef INLINE
@@ -198,8 +199,8 @@ ss_isqrt(int x) {
int y, e;
if(x >= (SS_BLOCKSIZE * SS_BLOCKSIZE)) { return SS_BLOCKSIZE; }
e = ((unsigned)x & 0xffff0000) ?
(((unsigned)x & 0xff000000) ?
e = (x & 0xffff0000) ?
((x & 0xff000000) ?
24 + lg_table[(x >> 24) & 0xff] :
16 + lg_table[(x >> 16) & 0xff]) :
((x & 0x0000ff00) ?
@@ -353,7 +354,7 @@ ss_pivot(const unsigned char *Td, const int *PA, int *first, int *last) {
int *middle;
int t;
t = PTRDIFF_TO_INT(last - first);
t = last - first;
middle = first + t / 2;
if(t <= 512) {
@@ -408,7 +409,7 @@ ss_mintrosort(const unsigned char *T, const int *PA,
int limit;
int v, x = 0;
for(ssize = 0, limit = ss_ilg(PTRDIFF_TO_INT(last - first));;) {
for(ssize = 0, limit = ss_ilg(last - first);;) {
if((last - first) <= SS_INSERTIONSORT_THRESHOLD) {
#if 1 < SS_INSERTIONSORT_THRESHOLD
@@ -419,7 +420,7 @@ ss_mintrosort(const unsigned char *T, const int *PA,
}
Td = T + depth;
if(limit-- == 0) { ss_heapsort(Td, PA, first, PTRDIFF_TO_INT(last - first)); }
if(limit-- == 0) { ss_heapsort(Td, PA, first, last - first); }
if(limit < 0) {
for(a = first + 1, v = Td[PA[*first]]; a < last; ++a) {
if((x = Td[PA[*a]]) != v) {
@@ -434,16 +435,16 @@ ss_mintrosort(const unsigned char *T, const int *PA,
if((a - first) <= (last - a)) {
if(1 < (a - first)) {
STACK_PUSH(a, last, depth, -1);
last = a, depth += 1, limit = ss_ilg(PTRDIFF_TO_INT(a - first));
last = a, depth += 1, limit = ss_ilg(a - first);
} else {
first = a, limit = -1;
}
} else {
if(1 < (last - a)) {
STACK_PUSH(first, a, depth + 1, ss_ilg(PTRDIFF_TO_INT(a - first)));
STACK_PUSH(first, a, depth + 1, ss_ilg(a - first));
first = a, limit = -1;
} else {
last = a, depth += 1, limit = ss_ilg(PTRDIFF_TO_INT(a - first));
last = a, depth += 1, limit = ss_ilg(a - first);
}
}
continue;
@@ -480,9 +481,9 @@ ss_mintrosort(const unsigned char *T, const int *PA,
if(a <= d) {
c = b - 1;
if((s = PTRDIFF_TO_INT(a - first)) > (t = PTRDIFF_TO_INT(b - a))) { s = t; }
if((s = a - first) > (t = b - a)) { s = t; }
for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
if((s = PTRDIFF_TO_INT(d - c)) > (t = PTRDIFF_TO_INT(last - d - 1))) { s = t; }
if((s = d - c) > (t = last - d - 1)) { s = t; }
for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
a = first + (b - a), c = last - (d - c);
@@ -490,38 +491,38 @@ ss_mintrosort(const unsigned char *T, const int *PA,
if((a - first) <= (last - c)) {
if((last - c) <= (c - b)) {
STACK_PUSH(b, c, depth + 1, ss_ilg(PTRDIFF_TO_INT(c - b)));
STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
STACK_PUSH(c, last, depth, limit);
last = a;
} else if((a - first) <= (c - b)) {
STACK_PUSH(c, last, depth, limit);
STACK_PUSH(b, c, depth + 1, ss_ilg(PTRDIFF_TO_INT(c - b)));
STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
last = a;
} else {
STACK_PUSH(c, last, depth, limit);
STACK_PUSH(first, a, depth, limit);
first = b, last = c, depth += 1, limit = ss_ilg(PTRDIFF_TO_INT(c - b));
first = b, last = c, depth += 1, limit = ss_ilg(c - b);
}
} else {
if((a - first) <= (c - b)) {
STACK_PUSH(b, c, depth + 1, ss_ilg(PTRDIFF_TO_INT(c - b)));
STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
STACK_PUSH(first, a, depth, limit);
first = c;
} else if((last - c) <= (c - b)) {
STACK_PUSH(first, a, depth, limit);
STACK_PUSH(b, c, depth + 1, ss_ilg(PTRDIFF_TO_INT(c - b)));
STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
first = c;
} else {
STACK_PUSH(first, a, depth, limit);
STACK_PUSH(c, last, depth, limit);
first = b, last = c, depth += 1, limit = ss_ilg(PTRDIFF_TO_INT(c - b));
first = b, last = c, depth += 1, limit = ss_ilg(c - b);
}
}
} else {
limit += 1;
if(Td[PA[*first] - 1] < v) {
first = ss_partition(PA, first, last, depth);
limit = ss_ilg(PTRDIFF_TO_INT(last - first));
limit = ss_ilg(last - first);
}
depth += 1;
}
@@ -550,8 +551,7 @@ void
ss_rotate(int *first, int *middle, int *last) {
int *a, *b, t;
int l, r;
l = PTRDIFF_TO_INT(middle - first);
r = PTRDIFF_TO_INT(last - middle);
l = middle - first, r = last - middle;
for(; (0 < l) && (0 < r);) {
if(l == r) { ss_blockswap(first, middle, l); break; }
if(l < r) {
@@ -601,7 +601,7 @@ ss_inplacemerge(const unsigned char *T, const int *PA,
for(;;) {
if(*(last - 1) < 0) { x = 1; p = PA + ~*(last - 1); }
else { x = 0; p = PA + *(last - 1); }
for(a = first, len = PTRDIFF_TO_INT(middle - first), half = len >> 1, r = -1;
for(a = first, len = middle - first, half = len >> 1, r = -1;
0 < len;
len = half, half >>= 1) {
b = a + half;
@@ -640,7 +640,7 @@ ss_mergeforward(const unsigned char *T, const int *PA,
int r;
bufend = buf + (middle - first) - 1;
ss_blockswap(buf, first, PTRDIFF_TO_INT(middle - first));
ss_blockswap(buf, first, middle - first);
for(t = *(a = first), b = buf, c = middle;;) {
r = ss_compare(T, PA + *b, PA + *c, depth);
@@ -692,7 +692,7 @@ ss_mergebackward(const unsigned char *T, const int *PA,
int x;
bufend = buf + (last - middle) - 1;
ss_blockswap(buf, middle, PTRDIFF_TO_INT(last - middle));
ss_blockswap(buf, middle, last - middle);
x = 0;
if(*bufend < 0) { p1 = PA + ~*bufend; x |= 1; }
@@ -781,7 +781,7 @@ ss_swapmerge(const unsigned char *T, const int *PA,
continue;
}
for(m = 0, len = PTRDIFF_TO_INT(MIN(middle - first, last - middle)), half = len >> 1;
for(m = 0, len = MIN(middle - first, last - middle), half = len >> 1;
0 < len;
len = half, half >>= 1) {
if(ss_compare(T, PA + GETIDX(*(middle + m + half)),
@@ -850,8 +850,8 @@ sssort(const unsigned char *T, const int *PA,
ss_mintrosort(T, PA, first, last, depth);
#else
if((bufsize < SS_BLOCKSIZE) &&
(bufsize < PTRDIFF_TO_INT(last - first)) &&
(bufsize < (limit = ss_isqrt(PTRDIFF_TO_INT(last - first))))) {
(bufsize < (last - first)) &&
(bufsize < (limit = ss_isqrt(last - first)))) {
if(SS_BLOCKSIZE < limit) { limit = SS_BLOCKSIZE; }
buf = middle = last - limit, bufsize = limit;
} else {
@@ -863,7 +863,7 @@ sssort(const unsigned char *T, const int *PA,
#elif 1 < SS_BLOCKSIZE
ss_insertionsort(T, PA, a, a + SS_BLOCKSIZE, depth);
#endif
curbufsize = PTRDIFF_TO_INT(last - (a + SS_BLOCKSIZE));
curbufsize = last - (a + SS_BLOCKSIZE);
curbuf = a + SS_BLOCKSIZE;
if(curbufsize <= bufsize) { curbufsize = bufsize, curbuf = buf; }
for(b = a, k = SS_BLOCKSIZE, j = i; j & 1; b -= k, k <<= 1, j >>= 1) {
@@ -1016,7 +1016,7 @@ tr_pivot(const int *ISAd, int *first, int *last) {
int *middle;
int t;
t = PTRDIFF_TO_INT(last - first);
t = last - first;
middle = first + t / 2;
if(t <= 512) {
@@ -1098,9 +1098,9 @@ tr_partition(const int *ISAd,
if(a <= d) {
c = b - 1;
if((s = PTRDIFF_TO_INT(a - first)) > (t = PTRDIFF_TO_INT(b - a))) { s = t; }
if((s = a - first) > (t = b - a)) { s = t; }
for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
if((s = PTRDIFF_TO_INT(d - c)) > (t = PTRDIFF_TO_INT(last - d - 1))) { s = t; }
if((s = d - c) > (t = last - d - 1)) { s = t; }
for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
first += (b - a), last -= (d - c);
}
@@ -1117,17 +1117,17 @@ tr_copy(int *ISA, const int *SA,
int *c, *d, *e;
int s, v;
v = PTRDIFF_TO_INT(b - SA - 1);
v = b - SA - 1;
for(c = first, d = a - 1; c <= d; ++c) {
if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
*++d = s;
ISA[s] = PTRDIFF_TO_INT(d - SA);
ISA[s] = d - SA;
}
}
for(c = last - 1, e = d + 1, d = b; e < d; --c) {
if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
*--d = s;
ISA[s] = PTRDIFF_TO_INT(d - SA);
ISA[s] = d - SA;
}
}
}
@@ -1141,13 +1141,13 @@ tr_partialcopy(int *ISA, const int *SA,
int s, v;
int rank, lastrank, newrank = -1;
v = PTRDIFF_TO_INT(b - SA - 1);
v = b - SA - 1;
lastrank = -1;
for(c = first, d = a - 1; c <= d; ++c) {
if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
*++d = s;
rank = ISA[s + depth];
if(lastrank != rank) { lastrank = rank; newrank = PTRDIFF_TO_INT(d - SA); }
if(lastrank != rank) { lastrank = rank; newrank = d - SA; }
ISA[s] = newrank;
}
}
@@ -1155,7 +1155,7 @@ tr_partialcopy(int *ISA, const int *SA,
lastrank = -1;
for(e = d; first <= e; --e) {
rank = ISA[*e];
if(lastrank != rank) { lastrank = rank; newrank = PTRDIFF_TO_INT(e - SA); }
if(lastrank != rank) { lastrank = rank; newrank = e - SA; }
if(newrank != rank) { ISA[*e] = newrank; }
}
@@ -1164,7 +1164,7 @@ tr_partialcopy(int *ISA, const int *SA,
if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
*--d = s;
rank = ISA[s + depth];
if(lastrank != rank) { lastrank = rank; newrank = PTRDIFF_TO_INT(d - SA); }
if(lastrank != rank) { lastrank = rank; newrank = d - SA; }
ISA[s] = newrank;
}
}
@@ -1180,23 +1180,23 @@ tr_introsort(int *ISA, const int *ISAd,
int *a, *b, *c;
int t;
int v, x = 0;
int incr = PTRDIFF_TO_INT(ISAd - ISA);
int incr = ISAd - ISA;
int limit, next;
int ssize, trlink = -1;
for(ssize = 0, limit = tr_ilg(PTRDIFF_TO_INT(last - first));;) {
for(ssize = 0, limit = tr_ilg(last - first);;) {
if(limit < 0) {
if(limit == -1) {
/* tandem repeat partition */
tr_partition(ISAd - incr, first, first, last, &a, &b, PTRDIFF_TO_INT(last - SA - 1));
tr_partition(ISAd - incr, first, first, last, &a, &b, last - SA - 1);
/* update ranks */
if(a < last) {
for(c = first, v = PTRDIFF_TO_INT(a - SA - 1); c < a; ++c) { ISA[*c] = v; }
for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; }
}
if(b < last) {
for(c = a, v = PTRDIFF_TO_INT(b - SA - 1); c < b; ++c) { ISA[*c] = v; }
for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; }
}
/* push */
@@ -1207,19 +1207,19 @@ tr_introsort(int *ISA, const int *ISAd,
}
if((a - first) <= (last - b)) {
if(1 < (a - first)) {
STACK_PUSH5(ISAd, b, last, tr_ilg(PTRDIFF_TO_INT(last - b)), trlink);
last = a, limit = tr_ilg(PTRDIFF_TO_INT(a - first));
STACK_PUSH5(ISAd, b, last, tr_ilg(last - b), trlink);
last = a, limit = tr_ilg(a - first);
} else if(1 < (last - b)) {
first = b, limit = tr_ilg(PTRDIFF_TO_INT(last - b));
first = b, limit = tr_ilg(last - b);
} else {
STACK_POP5(ISAd, first, last, limit, trlink);
}
} else {
if(1 < (last - b)) {
STACK_PUSH5(ISAd, first, a, tr_ilg(PTRDIFF_TO_INT(a - first)), trlink);
first = b, limit = tr_ilg(PTRDIFF_TO_INT(last - b));
STACK_PUSH5(ISAd, first, a, tr_ilg(a - first), trlink);
first = b, limit = tr_ilg(last - b);
} else if(1 < (a - first)) {
last = a, limit = tr_ilg(PTRDIFF_TO_INT(a - first));
last = a, limit = tr_ilg(a - first);
} else {
STACK_POP5(ISAd, first, last, limit, trlink);
}
@@ -1228,26 +1228,26 @@ tr_introsort(int *ISA, const int *ISAd,
/* tandem repeat copy */
a = stack[--ssize].b, b = stack[ssize].c;
if(stack[ssize].d == 0) {
tr_copy(ISA, SA, first, a, b, last, PTRDIFF_TO_INT(ISAd - ISA));
tr_copy(ISA, SA, first, a, b, last, ISAd - ISA);
} else {
if(0 <= trlink) { stack[trlink].d = -1; }
tr_partialcopy(ISA, SA, first, a, b, last, PTRDIFF_TO_INT(ISAd - ISA));
tr_partialcopy(ISA, SA, first, a, b, last, ISAd - ISA);
}
STACK_POP5(ISAd, first, last, limit, trlink);
} else {
/* sorted partition */
if(0 <= *first) {
a = first;
do { ISA[*a] = PTRDIFF_TO_INT(a - SA); } while((++a < last) && (0 <= *a));
do { ISA[*a] = a - SA; } while((++a < last) && (0 <= *a));
first = a;
}
if(first < last) {
a = first; do { *a = ~*a; } while(*++a < 0);
next = (ISA[*a] != ISAd[*a]) ? tr_ilg(PTRDIFF_TO_INT(a - first + 1)) : -1;
if(++a < last) { for(b = first, v = PTRDIFF_TO_INT(a - SA - 1); b < a; ++b) { ISA[*b] = v; } }
next = (ISA[*a] != ISAd[*a]) ? tr_ilg(a - first + 1) : -1;
if(++a < last) { for(b = first, v = a - SA - 1; b < a; ++b) { ISA[*b] = v; } }
/* push */
if(trbudget_check(budget, PTRDIFF_TO_INT(a - first))) {
if(trbudget_check(budget, a - first)) {
if((a - first) <= (last - a)) {
STACK_PUSH5(ISAd, a, last, -3, trlink);
ISAd += incr, last = a, limit = next;
@@ -1281,7 +1281,7 @@ tr_introsort(int *ISA, const int *ISAd,
}
if(limit-- == 0) {
tr_heapsort(ISAd, first, PTRDIFF_TO_INT(last - first));
tr_heapsort(ISAd, first, last - first);
for(a = last - 1; first < a; a = b) {
for(x = ISAd[*a], b = a - 1; (first <= b) && (ISAd[*b] == x); --b) { *b = ~*b; }
}
@@ -1297,14 +1297,14 @@ tr_introsort(int *ISA, const int *ISAd,
/* partition */
tr_partition(ISAd, first, first + 1, last, &a, &b, v);
if((last - first) != (b - a)) {
next = (ISA[*a] != v) ? tr_ilg(PTRDIFF_TO_INT(b - a)) : -1;
next = (ISA[*a] != v) ? tr_ilg(b - a) : -1;
/* update ranks */
for(c = first, v = PTRDIFF_TO_INT(a - SA - 1); c < a; ++c) { ISA[*c] = v; }
if(b < last) { for(c = a, v = PTRDIFF_TO_INT(b - SA - 1); c < b; ++c) { ISA[*c] = v; } }
for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; }
if(b < last) { for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } }
/* push */
if((1 < (b - a)) && (trbudget_check(budget, PTRDIFF_TO_INT(b - a)))) {
if((1 < (b - a)) && (trbudget_check(budget, b - a))) {
if((a - first) <= (last - b)) {
if((last - b) <= (b - a)) {
if(1 < (a - first)) {
@@ -1381,8 +1381,8 @@ tr_introsort(int *ISA, const int *ISAd,
}
}
} else {
if(trbudget_check(budget, PTRDIFF_TO_INT(last - first))) {
limit = tr_ilg(PTRDIFF_TO_INT(last - first)), ISAd += incr;
if(trbudget_check(budget, last - first)) {
limit = tr_ilg(last - first), ISAd += incr;
} else {
if(0 <= trlink) { stack[trlink].d = -1; }
STACK_POP5(ISAd, first, last, limit, trlink);
@@ -1420,7 +1420,7 @@ trsort(int *ISA, int *SA, int n, int depth) {
budget.count = 0;
tr_introsort(ISA, ISAd, SA, first, last, &budget);
if(budget.count != 0) { unsorted += budget.count; }
else { skip = PTRDIFF_TO_INT(first - last); }
else { skip = first - last; }
} else if((last - first) == 1) {
skip = -1;
}
@@ -1634,7 +1634,7 @@ construct_SA(const unsigned char *T, int *SA,
c0 = T[--s];
if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
if(c0 != c2) {
if(0 <= c2) { BUCKET_B(c2, c1) = PTRDIFF_TO_INT(k - SA); }
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
k = SA + BUCKET_B(c2 = c0, c1);
}
assert(k < j); assert(k != NULL);
@@ -1658,7 +1658,7 @@ construct_SA(const unsigned char *T, int *SA,
c0 = T[--s];
if((s == 0) || (T[s - 1] < c0)) { s = ~s; }
if(c0 != c2) {
BUCKET_A(c2) = PTRDIFF_TO_INT(k - SA);
BUCKET_A(c2) = k - SA;
k = SA + BUCKET_A(c2 = c0);
}
assert(i < k);
@@ -1698,7 +1698,7 @@ construct_BWT(const unsigned char *T, int *SA,
*j = ~((int)c0);
if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
if(c0 != c2) {
if(0 <= c2) { BUCKET_B(c2, c1) = PTRDIFF_TO_INT(k - SA); }
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
k = SA + BUCKET_B(c2 = c0, c1);
}
assert(k < j); assert(k != NULL);
@@ -1726,7 +1726,7 @@ construct_BWT(const unsigned char *T, int *SA,
*i = c0;
if((0 < s) && (T[s - 1] < c0)) { s = ~((int)T[s - 1]); }
if(c0 != c2) {
BUCKET_A(c2) = PTRDIFF_TO_INT(k - SA);
BUCKET_A(c2) = k - SA;
k = SA + BUCKET_A(c2 = c0);
}
assert(i < k);
@@ -1738,7 +1738,7 @@ construct_BWT(const unsigned char *T, int *SA,
}
}
return PTRDIFF_TO_INT(orig - SA);
return orig - SA;
}
/* Constructs the burrows-wheeler transformed string directly
@@ -1776,13 +1776,13 @@ construct_BWT_indexes(const unsigned char *T, int *SA,
assert(((s + 1) < n) && (T[s] <= T[s + 1]));
assert(T[s - 1] <= T[s]);
if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = PTRDIFF_TO_INT(j - SA);
if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = j - SA;
c0 = T[--s];
*j = ~((int)c0);
if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
if(c0 != c2) {
if(0 <= c2) { BUCKET_B(c2, c1) = PTRDIFF_TO_INT(k - SA); }
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
k = SA + BUCKET_B(c2 = c0, c1);
}
assert(k < j); assert(k != NULL);
@@ -1802,7 +1802,7 @@ construct_BWT_indexes(const unsigned char *T, int *SA,
the sorted order of type B suffixes. */
k = SA + BUCKET_A(c2 = T[n - 1]);
if (T[n - 2] < c2) {
if (((n - 1) & mod) == 0) indexes[(n - 1) / (mod + 1) - 1] = PTRDIFF_TO_INT(k - SA);
if (((n - 1) & mod) == 0) indexes[(n - 1) / (mod + 1) - 1] = k - SA;
*k++ = ~((int)T[n - 2]);
}
else {
@@ -1814,17 +1814,17 @@ construct_BWT_indexes(const unsigned char *T, int *SA,
if(0 < (s = *i)) {
assert(T[s - 1] >= T[s]);
if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = PTRDIFF_TO_INT(i - SA);
if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = i - SA;
c0 = T[--s];
*i = c0;
if(c0 != c2) {
BUCKET_A(c2) = PTRDIFF_TO_INT(k - SA);
BUCKET_A(c2) = k - SA;
k = SA + BUCKET_A(c2 = c0);
}
assert(i < k);
if((0 < s) && (T[s - 1] < c0)) {
if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = PTRDIFF_TO_INT(k - SA);
if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = k - SA;
*k++ = ~((int)T[s - 1]);
} else
*k++ = s;
@@ -1835,7 +1835,7 @@ construct_BWT_indexes(const unsigned char *T, int *SA,
}
}
return PTRDIFF_TO_INT(orig - SA);
return orig - SA;
}
+10
View File
@@ -27,6 +27,11 @@
#ifndef _DIVSUFSORT_H
#define _DIVSUFSORT_H 1
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
/*- Prototypes -*/
/**
@@ -54,4 +59,9 @@ divsufsort(const unsigned char *T, int *SA, int n, int openMP);
int
divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP);
#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
#endif /* _DIVSUFSORT_H */
+39 -38
View File
@@ -49,30 +49,38 @@
/*-*************************************
* Console display
*
* Captures the `displayLevel` variable in the local scope.
***************************************/
#ifndef LOCALDISPLAYLEVEL
static int g_displayLevel = 0;
#endif
#undef DISPLAY
#define DISPLAY(...) \
{ \
fprintf(stderr, __VA_ARGS__); \
fflush(stderr); \
}
#undef DISPLAYLEVEL
#define DISPLAYLEVEL(l, ...) \
#undef LOCALDISPLAYLEVEL
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
if (displayLevel >= l) { \
DISPLAY(__VA_ARGS__); \
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
#undef DISPLAYLEVEL
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
#undef DISPLAYUPDATE
#define DISPLAYUPDATE(lastUpdateTime, l, ...) \
#ifndef LOCALDISPLAYUPDATE
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
static clock_t g_time = 0;
#endif
#undef LOCALDISPLAYUPDATE
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
if (displayLevel >= l) { \
const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; \
if ((clock() - lastUpdateTime > refreshRate) || (displayLevel >= 4)) { \
lastUpdateTime = clock(); \
if ((clock() - g_time > g_refreshRate) || (displayLevel >= 4)) { \
g_time = clock(); \
DISPLAY(__VA_ARGS__); \
} \
}
#undef DISPLAYUPDATE
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
/*-*************************************
@@ -128,7 +136,6 @@ typedef struct {
unsigned d;
unsigned f;
FASTCOVER_accel_t accelParams;
int displayLevel;
} FASTCOVER_ctx_t;
@@ -307,8 +314,7 @@ FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
const void* samplesBuffer,
const size_t* samplesSizes, unsigned nbSamples,
unsigned d, double splitPoint, unsigned f,
FASTCOVER_accel_t accelParams,
int displayLevel)
FASTCOVER_accel_t accelParams)
{
const BYTE* const samples = (const BYTE*)samplesBuffer;
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
@@ -317,7 +323,6 @@ FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
ctx->displayLevel = displayLevel;
/* Checks */
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
@@ -404,9 +409,7 @@ FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
const COVER_epoch_info_t epochs = COVER_computeEpochs(
(U32)dictBufferCapacity, (U32)ctx->nbDmers, parameters.k, 1);
const size_t maxZeroScoreRun = 10;
const int displayLevel = ctx->displayLevel;
size_t zeroScoreRun = 0;
clock_t lastUpdateTime = 0;
size_t epoch;
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n",
(U32)epochs.num, (U32)epochs.size);
@@ -444,7 +447,6 @@ FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
tail -= segmentSize;
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
DISPLAYUPDATE(
lastUpdateTime,
2, "\r%u%% ",
(unsigned)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
}
@@ -482,7 +484,6 @@ static void FASTCOVER_tryParameters(void* opaque)
BYTE *const dict = (BYTE*)malloc(dictBufferCapacity);
COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
U32* freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32));
const int displayLevel = ctx->displayLevel;
if (!segmentFreqs || !dict || !freqs) {
DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
goto _cleanup;
@@ -554,7 +555,8 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
FASTCOVER_ctx_t ctx;
ZDICT_cover_params_t coverParams;
FASTCOVER_accel_t accelParams;
const int displayLevel = (int)parameters.zParams.notificationLevel;
/* Initialize global data */
g_displayLevel = (int)parameters.zParams.notificationLevel;
/* Assign splitPoint and f if not provided */
parameters.splitPoint = 1.0;
parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f;
@@ -583,13 +585,13 @@ ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
{
size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
coverParams.d, parameters.splitPoint, parameters.f,
accelParams, displayLevel);
accelParams);
if (ZSTD_isError(initVal)) {
DISPLAYLEVEL(1, "Failed to initialize context\n");
return initVal;
}
}
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, displayLevel);
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, g_displayLevel);
/* Build the dictionary */
DISPLAYLEVEL(2, "Building dictionary\n");
{
@@ -644,26 +646,25 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
COVER_best_t best;
POOL_ctx *pool = NULL;
int warned = 0;
clock_t lastUpdateTime = 0;
/* Checks */
if (splitPoint <= 0 || splitPoint > 1) {
DISPLAYLEVEL(1, "Incorrect splitPoint\n");
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n");
return ERROR(parameter_outOfBound);
}
if (accel == 0 || accel > FASTCOVER_MAX_ACCEL) {
DISPLAYLEVEL(1, "Incorrect accel\n");
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect accel\n");
return ERROR(parameter_outOfBound);
}
if (kMinK < kMaxD || kMaxK < kMinK) {
DISPLAYLEVEL(1, "Incorrect k\n");
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n");
return ERROR(parameter_outOfBound);
}
if (nbSamples == 0) {
DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
LOCALDISPLAYLEVEL(displayLevel, 1, "FASTCOVER must have at least one input file\n");
return ERROR(srcSize_wrong);
}
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
LOCALDISPLAYLEVEL(displayLevel, 1, "dictBufferCapacity must be at least %u\n",
ZDICT_DICTSIZE_MIN);
return ERROR(dstSize_tooSmall);
}
@@ -678,18 +679,19 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
memset(&coverParams, 0 , sizeof(coverParams));
FASTCOVER_convertToCoverParams(*parameters, &coverParams);
accelParams = FASTCOVER_defaultAccelParameters[accel];
/* Turn down global display level to clean up display at level 2 and below */
g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
/* Loop through d first because each new value needs a new context */
DISPLAYLEVEL(2, "Trying %u different sets of parameters\n", kIterations);
LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
kIterations);
for (d = kMinD; d <= kMaxD; d += 2) {
/* Initialize the context for this value of d */
FASTCOVER_ctx_t ctx;
DISPLAYLEVEL(3, "d=%u\n", d);
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
{
/* Turn down global display level to clean up display at level 2 and below */
const int childDisplayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams, childDisplayLevel);
size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams);
if (ZSTD_isError(initVal)) {
DISPLAYLEVEL(1, "Failed to initialize context\n");
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
COVER_best_destroy(&best);
POOL_free(pool);
return initVal;
@@ -704,9 +706,9 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
/* Prepare the arguments */
FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc(
sizeof(FASTCOVER_tryParameters_data_t));
DISPLAYLEVEL(3, "k=%u\n", k);
LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
if (!data) {
DISPLAYLEVEL(1, "Failed to allocate parameters\n");
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
COVER_best_destroy(&best);
FASTCOVER_ctx_destroy(&ctx);
POOL_free(pool);
@@ -721,7 +723,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps;
data->parameters.shrinkDict = shrinkDict;
data->parameters.zParams.notificationLevel = (unsigned)ctx.displayLevel;
data->parameters.zParams.notificationLevel = (unsigned)g_displayLevel;
/* Check the parameters */
if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity,
data->ctx->f, accel)) {
@@ -737,15 +739,14 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
FASTCOVER_tryParameters(data);
}
/* Print status */
DISPLAYUPDATE(lastUpdateTime,
2, "\r%u%% ",
LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
(unsigned)((iteration * 100) / kIterations));
++iteration;
}
COVER_best_wait(&best);
FASTCOVER_ctx_destroy(&ctx);
}
DISPLAYLEVEL(2, "\r%79s\r", "");
LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
/* Fill the output buffer and parameters with output of the best parameters */
{
const size_t dictSize = best.dictSize;
+16 -20
View File
@@ -168,7 +168,7 @@ static void ZDICT_initDictItem(dictItem* d)
#define MINMATCHLENGTH 7 /* heuristic determined experimentally */
static dictItem ZDICT_analyzePos(
BYTE* doneMarks,
const unsigned* suffix, U32 start,
const int* suffix, U32 start,
const void* buffer, U32 minRatio, U32 notificationLevel)
{
U32 lengthList[LLIMIT] = {0};
@@ -293,10 +293,8 @@ static dictItem ZDICT_analyzePos(
for (i=(int)(maxLength-2); i>=0; i--)
cumulLength[i] = cumulLength[i+1] + lengthList[i];
{ unsigned u;
for (u=LLIMIT-1; u>=MINMATCHLENGTH; u--) if (cumulLength[u]>=minRatio) break;
maxLength = u;
}
for (i=LLIMIT-1; i>=MINMATCHLENGTH; i--) if (cumulLength[i]>=minRatio) break;
maxLength = i;
/* reduce maxLength in case of final into repetitive data */
{ U32 l = (U32)maxLength;
@@ -308,10 +306,8 @@ static dictItem ZDICT_analyzePos(
/* calculate savings */
savings[5] = 0;
{ unsigned u;
for (u=MINMATCHLENGTH; u<=maxLength; u++)
savings[u] = savings[u-1] + (lengthList[u] * (u-3));
}
for (i=MINMATCHLENGTH; i<=(int)maxLength; i++)
savings[i] = savings[i-1] + (lengthList[i] * (i-3));
DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n",
(unsigned)pos, (unsigned)maxLength, (unsigned)savings[maxLength], (double)savings[maxLength] / (double)maxLength);
@@ -387,11 +383,11 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const
if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */
/* append */
int const addedLength = (int)eltEnd - (int)(table[u].pos + table[u].length); /* note: can be negative */
int const addedLength = (int)eltEnd - (int)(table[u].pos + table[u].length);
table[u].savings += elt.length / 8; /* rough approx bonus */
if (addedLength > 0) { /* otherwise, elt fully included into existing */
table[u].length += (unsigned)addedLength;
table[u].savings += elt.savings * (unsigned)addedLength / elt.length; /* rough approx */
table[u].length += addedLength;
table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */
}
/* sort : improve rank */
elt = table[u];
@@ -403,7 +399,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const
if (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) {
if (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) {
size_t const addedLength = MAX( elt.length - table[u].length , 1 );
size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 );
table[u].pos = elt.pos;
table[u].savings += (U32)(elt.savings * addedLength / elt.length);
table[u].length = MIN(elt.length, table[u].length + 1);
@@ -471,8 +467,8 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
const size_t* fileSizes, unsigned nbFiles,
unsigned minRatio, U32 notificationLevel)
{
unsigned* const suffix0 = (unsigned*)malloc((bufferSize+2)*sizeof(*suffix0));
unsigned* const suffix = suffix0+1;
int* const suffix0 = (int*)malloc((bufferSize+2)*sizeof(*suffix0));
int* const suffix = suffix0+1;
U32* reverseSuffix = (U32*)malloc((bufferSize)*sizeof(*reverseSuffix));
BYTE* doneMarks = (BYTE*)malloc((bufferSize+16)*sizeof(*doneMarks)); /* +16 for overflow security */
U32* filePos = (U32*)malloc(nbFiles * sizeof(*filePos));
@@ -507,11 +503,11 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize,
/* sort */
DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (unsigned)(bufferSize>>20));
{ int const divSuftSortResult = divsufsort((const unsigned char*)buffer, (int*)suffix, (int)bufferSize, 0);
{ int const divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0);
if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; }
}
suffix[bufferSize] = (unsigned)bufferSize; /* leads into noise */
suffix0[0] = (unsigned)bufferSize; /* leads into noise */
suffix[bufferSize] = (int)bufferSize; /* leads into noise */
suffix0[0] = (int)bufferSize; /* leads into noise */
/* build reverse suffix sort */
{ size_t pos;
for (pos=0; pos < bufferSize; pos++)
@@ -584,7 +580,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; }
if (cSize) { /* if == 0; block is not compressible */
const SeqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc);
const seqStore_t* const seqStorePtr = ZSTD_getSeqStore(esr.zc);
/* literals stats */
{ const BYTE* bytePtr;
@@ -612,7 +608,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params,
}
if (nbSeq >= 2) { /* rep offsets */
const SeqDef* const seq = seqStorePtr->sequencesStart;
const seqDef* const seq = seqStorePtr->sequencesStart;
U32 offset1 = seq[0].offBase - ZSTD_REP_NUM;
U32 offset2 = seq[1].offBase - ZSTD_REP_NUM;
if (offset1 >= MAXREPOFFSET) offset1 = 0;
+20 -55
View File
@@ -1,55 +1,20 @@
@echo off
setlocal
rem Detect build type based on available files
set BUILD_TYPE=make
if exist "build\cmake\build\lib\Release\zstd_static.lib" set BUILD_TYPE=cmake
echo Detected build type: %BUILD_TYPE%
rem Create required directories
mkdir bin\dll bin\static bin\example bin\include
rem Copy common files using a subroutine. Exits immediately on failure.
call :copyFile "tests\fullbench.c" "bin\example\"
call :copyFile "programs\datagen.c" "bin\example\"
call :copyFile "programs\datagen.h" "bin\example\"
call :copyFile "programs\util.h" "bin\example\"
call :copyFile "programs\platform.h" "bin\example\"
call :copyFile "lib\common\mem.h" "bin\example\"
call :copyFile "lib\common\zstd_internal.h" "bin\example\"
call :copyFile "lib\common\error_private.h" "bin\example\"
call :copyFile "lib\common\xxhash.h" "bin\example\"
call :copyFile "lib\dll\example\Makefile" "bin\example\"
call :copyFile "lib\dll\example\fullbench-dll.*" "bin\example\"
call :copyFile "lib\zstd.h" "bin\include\"
call :copyFile "lib\zstd_errors.h" "bin\include\"
call :copyFile "lib\zdict.h" "bin\include\"
rem Copy build-specific files
if "%BUILD_TYPE%"=="cmake" (
echo Copying CMake build artifacts...
call :copyFile "build\cmake\build\lib\Release\zstd_static.lib" "bin\static\libzstd_static.lib"
call :copyFile "build\cmake\build\lib\Release\zstd.dll" "bin\dll\libzstd.dll"
call :copyFile "build\cmake\build\lib\Release\zstd.lib" "bin\dll\zstd.lib"
call :copyFile "build\cmake\build\programs\Release\zstd.exe" "bin\zstd.exe"
call :copyFile "lib\dll\example\README.md" "bin\README.md"
) else (
echo Copying Make build artifacts...
call :copyFile "lib\libzstd.a" "bin\static\libzstd_static.lib"
call :copyFile "lib\dll\libzstd.*" "bin\dll\"
call :copyFile "programs\zstd.exe" "bin\zstd.exe"
call :copyFile "lib\dll\example\README.md" "bin\"
)
echo Build package created successfully for %BUILD_TYPE% build!
endlocal
exit /b 0
:copyFile
copy "%~1" "%~2"
if errorlevel 1 (
echo Failed to copy "%~1"
exit 1
)
exit /b
@ECHO OFF
MKDIR bin\dll bin\static bin\example bin\include
COPY tests\fullbench.c bin\example\
COPY programs\datagen.c bin\example\
COPY programs\datagen.h bin\example\
COPY programs\util.h bin\example\
COPY programs\platform.h bin\example\
COPY lib\common\mem.h bin\example\
COPY lib\common\zstd_internal.h bin\example\
COPY lib\common\error_private.h bin\example\
COPY lib\common\xxhash.h bin\example\
COPY lib\libzstd.a bin\static\libzstd_static.lib
COPY lib\dll\libzstd.* bin\dll\
COPY lib\dll\example\Makefile bin\example\
COPY lib\dll\example\fullbench-dll.* bin\example\
COPY lib\dll\example\README.md bin\
COPY lib\zstd.h bin\include\
COPY lib\common\zstd_errors.h bin\include\
COPY lib\dictBuilder\zdict.h bin\include\
COPY programs\zstd.exe bin\zstd.exe
-17
View File
@@ -1,17 +0,0 @@
# ################################################################
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
# ################################################################
# This included Makefile provides the following variables :
# UNAME, INSTALL_OS_LIST
UNAME := $(shell sh -c 'MSYSTEM="MSYS" uname')
# List of OSes for which target install is supported
INSTALL_OS_LIST ?= Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT% CYGWIN_NT%
+7 -8
View File
@@ -430,7 +430,7 @@ static unsigned FSE_isError(size_t code) { return (code > (size_t)(-FSE_ERROR_ma
static short FSE_abs(short a)
{
return a<0? (short)-a : a;
return a<0? -a : a;
}
@@ -1383,7 +1383,7 @@ typedef struct {
BYTE* matchLength;
BYTE* dumpsStart;
BYTE* dumps;
} SeqStore_t;
} seqStore_t;
typedef struct ZSTD_Cctx_s
@@ -1391,7 +1391,7 @@ typedef struct ZSTD_Cctx_s
const BYTE* base;
U32 current;
U32 nextUpdate;
SeqStore_t seqStore;
seqStore_t seqStore;
#ifdef __AVX2__
__m256i hashTable[HASH_TABLESIZE>>3];
#else
@@ -1823,12 +1823,11 @@ static size_t ZSTD_decompressSequences(
BYTE* const ostart = (BYTE* const)dst;
BYTE* op = ostart;
BYTE* const oend = ostart + maxDstSize;
size_t errorCode = 0;
size_t dumpsLength = 0;
size_t errorCode, dumpsLength;
const BYTE* litPtr = litStart;
const BYTE* const litEnd = litStart + litSize;
int nbSeq = 0;
const BYTE* dumps = NULL;
int nbSeq;
const BYTE* dumps;
U32* DTableLL = dctx->LLTable;
U32* DTableML = dctx->MLTable;
U32* DTableOffb = dctx->OffTable;
@@ -1916,7 +1915,7 @@ size_t ZSTDv01_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const voi
size_t remainingSize = srcSize;
U32 magicNumber;
size_t errorCode=0;
blockProperties_t blockProperties = { 0 };
blockProperties_t blockProperties;
/* Frame Header */
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
+1 -1
View File
@@ -2722,7 +2722,7 @@ typedef struct {
BYTE* matchLength;
BYTE* dumpsStart;
BYTE* dumps;
} SeqStore_t;
} seqStore_t;
/* *************************************
+2 -2
View File
@@ -1111,7 +1111,7 @@ static unsigned FSE_isError(size_t code) { return ERR_isError(code); }
****************************************************************/
static short FSE_abs(short a)
{
return a<0 ? (short)-a : a;
return a<0 ? -a : a;
}
static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
@@ -2362,7 +2362,7 @@ typedef struct {
BYTE* matchLength;
BYTE* dumpsStart;
BYTE* dumps;
} SeqStore_t;
} seqStore_t;
/* *************************************
+1 -1
View File
@@ -1094,7 +1094,7 @@ static unsigned FSE_isError(size_t code) { return ERR_isError(code); }
****************************************************************/
static short FSE_abs(short a)
{
return a<0 ? (short)-a : a;
return a<0 ? -a : a;
}
static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
+4 -11
View File
@@ -491,7 +491,7 @@ typedef struct {
U32 litLengthSum;
U32 litSum;
U32 offCodeSum;
} SeqStore_t;
} seqStore_t;
@@ -1224,7 +1224,7 @@ const char* FSEv05_getErrorName(size_t code) { return ERR_getErrorName(code); }
/*-**************************************************************
* FSEv05 NCount encoding-decoding
****************************************************************/
static short FSEv05_abs(short a) { return a<0 ? (short)-a : a; }
static short FSEv05_abs(short a) { return a<0 ? -a : a; }
size_t FSEv05_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
@@ -3972,15 +3972,8 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
zbc->outStart += flushedSize;
if (flushedSize == toFlushSize) {
zbc->stage = ZBUFFv05ds_read;
if (zbc->outStart + BLOCKSIZE > zbc->outBuffSize) {
/* Not enough room for next block - need to wrap buffer.
* Preserve history: copy the last windowSize bytes to the
* beginning so that back-references can still find valid data. */
size_t const windowSize = (size_t)1 << zbc->params.windowLog;
size_t const preserveSize = MIN(zbc->outEnd, windowSize);
memmove(zbc->outBuff, zbc->outBuff + zbc->outEnd - preserveSize, preserveSize);
zbc->outStart = zbc->outEnd = preserveSize;
}
if (zbc->outStart + BLOCKSIZE > zbc->outBuffSize)
zbc->outStart = zbc->outEnd = 0;
break;
}
/* cannot flush everything */
+3 -7
View File
@@ -552,9 +552,9 @@ typedef struct {
U32 cachedLitLength;
const BYTE* cachedLiterals;
ZSTDv06_stats_t stats;
} SeqStore_t;
} seqStore_t;
void ZSTDv06_seqToCodes(const SeqStore_t* seqStorePtr, size_t const nbSeq);
void ZSTDv06_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
#endif /* ZSTDv06_CCOMMON_H_MODULE */
@@ -1202,7 +1202,7 @@ static unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
/*-**************************************************************
* FSE NCount encoding-decoding
****************************************************************/
static short FSEv06_abs(short a) { return a<0 ? (short)-a : a; }
static short FSEv06_abs(short a) { return a<0 ? -a : a; }
size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
const void* headerBuffer, size_t hbSize)
@@ -3919,10 +3919,6 @@ ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void)
if (zbd==NULL) return NULL;
memset(zbd, 0, sizeof(*zbd));
zbd->zd = ZSTDv06_createDCtx();
if (zbd->zd==NULL) {
ZBUFFv06_freeDCtx(zbd); /* avoid leaking the context */
return NULL;
}
zbd->stage = ZBUFFds_init;
return zbd;
}
+2 -2
View File
@@ -2787,9 +2787,9 @@ typedef struct {
U32 cachedLitLength;
const BYTE* cachedLiterals;
ZSTDv07_stats_t stats;
} SeqStore_t;
} seqStore_t;
void ZSTDv07_seqToCodes(const SeqStore_t* seqStorePtr, size_t const nbSeq);
void ZSTDv07_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
/* custom memory allocation functions */
static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction, ZSTDv07_defaultFreeFunction, NULL };
+10 -7
View File
@@ -22,14 +22,18 @@ LIBZSTD_MK_INCLUDED := 1
# By default, library's directory is same as this included makefile
LIB_SRCDIR ?= $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
LIB_BINDIR ?= $(LIB_SRCDIR)
LIB_BINDIR ?= $(LIBSRC_DIR)
# ZSTD_LIB_MINIFY is a helper variable that
# configures a bunch of other variables to space-optimized defaults.
ZSTD_LIB_MINIFY ?= 0
# Legacy support disabled by default
ZSTD_LEGACY_SUPPORT ?= 0
# Legacy support
ifneq ($(ZSTD_LIB_MINIFY), 0)
ZSTD_LEGACY_SUPPORT ?= 0
else
ZSTD_LEGACY_SUPPORT ?= 5
endif
ZSTD_LEGACY_MULTITHREADED_API ?= 0
# Build size optimizations
@@ -202,16 +206,15 @@ endif
endif
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
# Include install_oses.mk from the same directory
include $(dir $(lastword $(MAKEFILE_LIST)))/install_oses.mk
LN ?= ln
CP ?= cp -f
UNAME := $(shell uname)
ifndef BUILD_DIR
ifeq ($(UNAME), Darwin)
ifeq ($(shell md5 < /dev/null > /dev/null; echo $$?), 0)
HASH ?= md5
endif
else ifeq ($(UNAME), FreeBSD)
HASH ?= gmd5sum
else ifeq ($(UNAME), NetBSD)
HASH ?= md5 -n
else ifeq ($(UNAME), OpenBSD)
+2 -3
View File
@@ -11,7 +11,6 @@ Name: zstd
Description: fast lossless compression algorithm library
URL: https://facebook.github.io/zstd/
Version: @VERSION@
Libs: -L${libdir} -lzstd @LIBS_MT@
Libs: -L${libdir} -lzstd
Libs.private: @LIBS_PRIVATE@
Cflags: -I${includedir} @LIBS_MT@
License: BSD-3-Clause OR GPL-2.0-only
Cflags: -I${includedir}
+21
View File
@@ -1,6 +1,27 @@
module libzstd [extern_c] {
header "zstd.h"
export *
config_macros [exhaustive] \
/* zstd.h */ \
ZSTD_STATIC_LINKING_ONLY, \
ZSTDLIB_VISIBILITY, \
ZSTDLIB_VISIBLE, \
ZSTDLIB_HIDDEN, \
ZSTD_DLL_EXPORT, \
ZSTDLIB_STATIC_API, \
ZSTD_DISABLE_DEPRECATE_WARNINGS, \
ZSTD_CLEVEL_DEFAULT, \
/* zdict.h */ \
ZDICT_STATIC_LINKING_ONLY, \
ZDICTLIB_VISIBLE, \
ZDICTLIB_HIDDEN, \
ZDICTLIB_VISIBILITY, \
ZDICTLIB_STATIC_API, \
ZDICT_DISABLE_DEPRECATE_WARNINGS, \
/* zstd_errors.h */ \
ZSTDERRORLIB_VISIBLE, \
ZSTDERRORLIB_HIDDEN, \
ZSTDERRORLIB_VISIBILITY
module dictbuilder [extern_c] {
header "zdict.h"
+8 -15
View File
@@ -8,16 +8,16 @@
* You may select, at your option, one of the above-listed licenses.
*/
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef ZSTD_ZDICT_H
#define ZSTD_ZDICT_H
/*====== Dependencies ======*/
#include <stddef.h> /* size_t */
#if defined (__cplusplus)
extern "C" {
#endif
/* ===== ZDICTLIB_API : control library symbols visibility ===== */
#ifndef ZDICTLIB_VISIBLE
@@ -248,7 +248,7 @@ typedef struct {
* is presumed that the most profitable content is at the end of the dictionary,
* since that is the cheapest to reference.
*
* `maxDictSize` must be >= max(dictContentSize, ZDICT_DICTSIZE_MIN).
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
*
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
* or an error code, which can be tested by ZDICT_isError().
@@ -271,19 +271,11 @@ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictS
ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_ZDICT_H */
#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC)
#define ZSTD_ZDICT_H_STATIC
#if defined (__cplusplus)
extern "C" {
#endif
/* This can be overridden externally to hide static symbols. */
#ifndef ZDICTLIB_STATIC_API
# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
@@ -474,8 +466,9 @@ ZDICTLIB_STATIC_API
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
#endif /* ZSTD_ZDICT_H_STATIC */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_ZDICT_H_STATIC */
+124 -244
View File
@@ -7,22 +7,17 @@
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef ZSTD_H_235446
#define ZSTD_H_235446
/* ====== Dependencies ======*/
#include <limits.h> /* INT_MAX */
#include <stddef.h> /* size_t */
#include "zstd_errors.h" /* list of errors */
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
#include <limits.h> /* INT_MAX */
#endif /* ZSTD_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
extern "C" {
#endif
/* ===== ZSTDLIB_API : control library symbols visibility ===== */
#ifndef ZSTDLIB_VISIBLE
@@ -62,7 +57,7 @@ extern "C" {
#else
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZSTD_DEPRECATED(message) [[deprecated(message)]]
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) || defined(__IAR_SYSTEMS_ICC__)
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZSTD_DEPRECATED(message) __attribute__((deprecated))
@@ -110,8 +105,8 @@ extern "C" {
/*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 6
#define ZSTD_VERSION_RELEASE 0
#define ZSTD_VERSION_MINOR 5
#define ZSTD_VERSION_RELEASE 6
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
/*! ZSTD_versionNumber() :
@@ -149,7 +144,7 @@ ZSTDLIB_API const char* ZSTD_versionString(void);
/***************************************
* Simple Core API
* Simple API
***************************************/
/*! ZSTD_compress() :
* Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
@@ -163,79 +158,67 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
/*! ZSTD_decompress() :
* `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
* Multiple compressed frames can be decompressed at once with this method.
* The result will be the concatenation of all decompressed frames, back to back.
* `dstCapacity` is an upper bound of originalSize to regenerate.
* First frame's decompressed size can be extracted using ZSTD_getFrameContentSize().
* If maximum upper bound isn't known, prefer using streaming mode to decompress data.
* If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
* @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
* or an errorCode if it fails (which can be tested using ZSTD_isError()). */
ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/*====== Decompression helper functions ======*/
/*! @brief Returns the decompressed content size stored in a ZSTD frame header.
*
* @since v1.3.0
*
* @param src Pointer to the beginning of a ZSTD encoded frame.
* @param srcSize Size of the buffer pointed to by @p src. It must be at least as large as the frame header.
* Any value greater than or equal to `ZSTD_frameHeaderSize_max` is sufficient.
* @return The decompressed size in bytes when the value is available in the frame header.
* @retval ZSTD_CONTENTSIZE_UNKNOWN The frame does not encode a decompressed size (typical for streaming).
* @retval ZSTD_CONTENTSIZE_ERROR An error occurred (e.g. invalid magic number, @p srcSize too small).
*
* @note The return value is not compatible with `ZSTD_isError()`.
* @note A return value of 0 denotes a valid but empty frame. Skippable frames also report 0.
* @note The decompressed size field is optional. When it is absent (the function returns @c ZSTD_CONTENTSIZE_UNKNOWN),
* the caller must rely on streaming decompression or an application-specific upper bound. `ZSTD_decompress()`
* only requires an upper bound, so applications may enforce their own block limits (for example 16 KB).
* @note The decompressed size is guaranteed to be present when compression was performed with single-pass APIs such as
* `ZSTD_compress()`, `ZSTD_compressCCtx()`, `ZSTD_compress_usingDict()`, or `ZSTD_compress_usingCDict()`.
* @note The decompressed size is a 64-bit value and may exceed the addressable space of the system. Use streaming
* decompression when the value is too large to materialize in contiguous memory.
* @warning When processing untrusted input, validate the returned size against the application's limits; attackers may
* forge an arbitrarily large value.
* @note This function replaces `ZSTD_getDecompressedSize()`.
*/
/*! ZSTD_getFrameContentSize() : requires v1.3.0+
* `src` should point to the start of a ZSTD encoded frame.
* `srcSize` must be at least as large as the frame header.
* hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
* @return : - decompressed size of `src` frame content, if known
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
* note 1 : a 0 return value means the frame is valid but "empty".
* note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application can rely on some implicit limit,
* as ZSTD_decompress() only needs an upper bound of decompressed size.
* (For example, data could be necessarily cut into blocks <= 16 KB).
* note 3 : decompressed size is always present when compression is completed using single-pass functions,
* such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
* note 4 : decompressed size can be very large (64-bits value),
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
* Always ensure return value fits within application's authorized limits.
* Each application can set its own limits.
* note 6 : This function replaces ZSTD_getDecompressedSize() */
#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
/*! ZSTD_getDecompressedSize() (obsolete):
* This function is now obsolete, in favor of ZSTD_getFrameContentSize().
/*! ZSTD_getDecompressedSize() :
* NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
* Both functions work the same way, but ZSTD_getDecompressedSize() blends
* "empty", "unknown" and "error" results to the same return value (0),
* while ZSTD_getFrameContentSize() gives them separate return values.
* @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
ZSTDLIB_API
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
* `src` should point to the start of a ZSTD frame or skippable frame.
* `srcSize` must be >= first frame size
* @return : the compressed size of the first frame starting at `src`,
* suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
* or an error code if input is invalid
* Note 1: this method is called _find*() because it's not enough to read the header,
* it may have to scan through the frame's content, to reach its end.
* Note 2: this method also works with Skippable Frames. In which case,
* it returns the size of the complete skippable frame,
* which is always equal to its content size + 8 bytes for headers. */
* or an error code if input is invalid */
ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
/*====== Compression helper functions ======*/
/*! ZSTD_compressBound() :
/*====== Helper functions ======*/
/* ZSTD_compressBound() :
* maximum compressed size in worst case single-pass scenario.
* When invoking `ZSTD_compress()`, or any other one-pass compression function,
* When invoking `ZSTD_compress()` or any other one-pass compression function,
* it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
* as it eliminates one potential failure scenario,
* aka not enough room in dst buffer to write the compressed frame.
* Note : ZSTD_compressBound() itself can fail, if @srcSize >= ZSTD_MAX_INPUT_SIZE .
* Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
* In which case, ZSTD_compressBound() will return an error code
* which can be tested using ZSTD_isError().
*
@@ -243,22 +226,18 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
* same as ZSTD_compressBound(), but as a macro.
* It can be used to produce constants, which can be useful for static allocation,
* for example to size a static array on stack.
* Will produce constant value 0 if srcSize is too large.
* Will produce constant value 0 if srcSize too large.
*/
#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U)
#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
/*====== Error helper functions ======*/
/* ZSTD_isError() :
* Most ZSTD_* functions returning a size_t value can be tested for error,
* using ZSTD_isError().
* @return 1 if error, 0 otherwise
*/
ZSTDLIB_API unsigned ZSTD_isError(size_t result); /*!< tells if a `size_t` function result is an error code */
ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); /* convert a result into an error code, which can be compared to error enum list */
ZSTDLIB_API const char* ZSTD_getErrorName(size_t result); /*!< provides readable string from a function result */
ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */
ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */
@@ -269,17 +248,17 @@ ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compress
***************************************/
/*= Compression context
* When compressing many times,
* it is recommended to allocate a compression context just once,
* it is recommended to allocate a context just once,
* and reuse it for each successive compression operation.
* This will make the workload easier for system's memory.
* This will make workload friendlier for system's memory.
* Note : re-using context is just a speed / resource optimization.
* It doesn't change the compression ratio, which remains identical.
* Note 2: For parallel execution in multi-threaded environments,
* use one different context per thread .
* Note 2 : In multi-threaded environments,
* use one different context per thread for parallel execution.
*/
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* compatible with NULL pointer */
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */
/*! ZSTD_compressCCtx() :
* Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
@@ -287,7 +266,7 @@ ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* compatible with NULL
* this function compresses at the requested compression level,
* __ignoring any other advanced parameter__ .
* If any advanced parameter was set using the advanced API,
* they will all be reset. Only @compressionLevel remains.
* they will all be reset. Only `compressionLevel` remains.
*/
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
@@ -413,7 +392,7 @@ typedef enum {
* Special: value 0 means "use default strategy". */
ZSTD_c_targetCBlockSize=130, /* v1.5.6+
* Attempts to fit compressed block size into approximately targetCBlockSize.
* Attempts to fit compressed block size into approximatively targetCBlockSize.
* Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
* Note that it's not a guarantee, just a convergence target (default:0).
* No target when targetCBlockSize == 0.
@@ -509,8 +488,7 @@ typedef enum {
* ZSTD_c_stableOutBuffer
* ZSTD_c_blockDelimiters
* ZSTD_c_validateSequences
* ZSTD_c_blockSplitterLevel
* ZSTD_c_splitAfterSequences
* ZSTD_c_useBlockSplitter
* ZSTD_c_useRowMatchFinder
* ZSTD_c_prefetchCDictTables
* ZSTD_c_enableSeqProducerFallback
@@ -537,8 +515,7 @@ typedef enum {
ZSTD_c_experimentalParam16=1013,
ZSTD_c_experimentalParam17=1014,
ZSTD_c_experimentalParam18=1015,
ZSTD_c_experimentalParam19=1016,
ZSTD_c_experimentalParam20=1017
ZSTD_c_experimentalParam19=1016
} ZSTD_cParameter;
typedef struct {
@@ -878,7 +855,7 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
*
* A ZSTD_DStream object is required to track streaming operations.
* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
* ZSTD_DStream objects can be re-employed multiple times.
* ZSTD_DStream objects can be reused multiple times.
*
* Use ZSTD_initDStream() to start a new decompression operation.
* @return : recommended first input size
@@ -888,21 +865,16 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
* The function will update both `pos` fields.
* If `input.pos < input.size`, some input has not been consumed.
* It's up to the caller to present again remaining data.
*
* The function tries to flush all data decoded immediately, respecting output buffer size.
* If `output.pos < output.size`, decoder has flushed everything it could.
*
* However, when `output.pos == output.size`, it's more difficult to know.
* If @return > 0, the frame is not complete, meaning
* either there is still some data left to flush within internal buffers,
* or there is more input to read to complete the frame (or both).
* But if `output.pos == output.size`, there might be some data left within internal buffers.,
* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
* @return : 0 when a frame is completely decoded and fully flushed,
* or an error code, which can be tested using ZSTD_isError(),
* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
* the return value is a suggested next input size (just a hint for better latency)
* that will never request more than the remaining content of the compressed frame.
* that will never request more than the remaining frame size.
* *******************************************************************************/
typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
@@ -929,10 +901,9 @@ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
* Function will update both input and output `pos` fields exposing current state via these fields:
* - `input.pos < input.size`, some input remaining and caller should provide remaining input
* on the next call.
* - `output.pos < output.size`, decoder flushed internal output buffer.
* - `output.pos == output.size`, unflushed data potentially present in the internal buffers,
* check ZSTD_decompressStream() @return value,
* if > 0, invoke it again to flush remaining data to output.
* - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
* - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
* call ZSTD_decompressStream() again to flush remaining data to output.
* Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
*
* @return : 0 when a frame is completely decoded and fully flushed,
@@ -1210,10 +1181,6 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_H_235446 */
@@ -1229,10 +1196,6 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
#if defined (__cplusplus)
extern "C" {
#endif
/* This can be overridden externally to hide static symbols. */
#ifndef ZSTDLIB_STATIC_API
# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
@@ -1344,7 +1307,7 @@ typedef struct {
*
* Note: This field is optional. ZSTD_generateSequences() will calculate the value of
* 'rep', but repeat offsets do not necessarily need to be calculated from an external
* sequence provider perspective. For example, ZSTD_compressSequences() does not
* sequence provider's perspective. For example, ZSTD_compressSequences() does not
* use this 'rep' field at all (as of now).
*/
} ZSTD_Sequence;
@@ -1449,15 +1412,14 @@ typedef enum {
} ZSTD_literalCompressionMode_e;
typedef enum {
/* Note: This enum controls features which are conditionally beneficial.
* Zstd can take a decision on whether or not to enable the feature (ZSTD_ps_auto),
* but setting the switch to ZSTD_ps_enable or ZSTD_ps_disable force enable/disable the feature.
/* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
* decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
* or ZSTD_ps_disable allow for a force enable/disable the feature.
*/
ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */
ZSTD_ps_enable = 1, /* Force-enable the feature */
ZSTD_ps_disable = 2 /* Do not use the feature */
} ZSTD_ParamSwitch_e;
#define ZSTD_paramSwitch_e ZSTD_ParamSwitch_e /* old name */
} ZSTD_paramSwitch_e;
/***************************************
* Frame header and size functions
@@ -1502,36 +1464,34 @@ ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src,
ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
/*! ZSTD_frameHeaderSize() :
* srcSize must be large enough, aka >= ZSTD_FRAMEHEADERSIZE_PREFIX.
* srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
* @return : size of the Frame Header,
* or an error code (if srcSize is too small) */
ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_FrameType_e;
#define ZSTD_frameType_e ZSTD_FrameType_e /* old name */
typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
typedef struct {
unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
unsigned blockSizeMax;
ZSTD_FrameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
unsigned headerSize;
unsigned dictID; /* for ZSTD_skippableFrame, contains the skippable magic variant [0-15] */
unsigned dictID;
unsigned checksumFlag;
unsigned _reserved1;
unsigned _reserved2;
} ZSTD_FrameHeader;
#define ZSTD_frameHeader ZSTD_FrameHeader /* old name */
} ZSTD_frameHeader;
/*! ZSTD_getFrameHeader() :
* decode Frame Header into `zfhPtr`, or requires larger `srcSize`.
* @return : 0 => header is complete, `zfhPtr` is correctly filled,
* >0 => `srcSize` is too small, @return value is the wanted `srcSize` amount, `zfhPtr` is not filled,
* decode Frame Header, or requires larger `srcSize`.
* @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
* or an error code, which can be tested using ZSTD_isError() */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize);
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */
/*! ZSTD_getFrameHeader_advanced() :
* same as ZSTD_getFrameHeader(),
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_FrameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
/*! ZSTD_decompressionMargin() :
* Zstd supports in-place decompression, where the input and output buffers overlap.
@@ -1579,10 +1539,9 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSi
))
typedef enum {
ZSTD_sf_noBlockDelimiters = 0, /* ZSTD_Sequence[] has no block delimiters, just sequences */
ZSTD_sf_explicitBlockDelimiters = 1 /* ZSTD_Sequence[] contains explicit block delimiters */
} ZSTD_SequenceFormat_e;
#define ZSTD_sequenceFormat_e ZSTD_SequenceFormat_e /* old name */
ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
} ZSTD_sequenceFormat_e;
/*! ZSTD_sequenceBound() :
* `srcSize` : size of the input buffer
@@ -1606,7 +1565,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
* @param zc The compression context to be used for ZSTD_compress2(). Set any
* compression parameters you need on this context.
* @param outSeqs The output sequences buffer of size @p outSeqsSize
* @param outSeqsCapacity The size of the output sequences buffer.
* @param outSeqsSize The size of the output sequences buffer.
* ZSTD_sequenceBound(srcSize) is an upper bound on the number
* of sequences that can be generated.
* @param src The source buffer to generate sequences from of size @p srcSize.
@@ -1624,7 +1583,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()")
ZSTDLIB_STATIC_API size_t
ZSTD_generateSequences(ZSTD_CCtx* zc,
ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
ZSTD_Sequence* outSeqs, size_t outSeqsSize,
const void* src, size_t srcSize);
/*! ZSTD_mergeBlockDelimiters() :
@@ -1644,7 +1603,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, si
* Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
* @src contains the entire input (not just the literals).
* If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
* If a dictionary is included, then the cctx should reference the dict (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.).
* If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
* The entire source is compressed into a single frame.
*
* The compression behavior changes based on cctx params. In particular:
@@ -1653,17 +1612,11 @@ ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, si
* the block size derived from the cctx, and sequences may be split. This is the default setting.
*
* If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
* valid block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
* block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
*
* When ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, it's possible to decide generating repcodes
* using the advanced parameter ZSTD_c_repcodeResolution. Repcodes will improve compression ratio, though the benefit
* can vary greatly depending on Sequences. On the other hand, repcode resolution is an expensive operation.
* By default, it's disabled at low (<10) compression levels, and enabled above the threshold (>=10).
* ZSTD_c_repcodeResolution makes it possible to directly manage this processing in either direction.
*
* If ZSTD_c_validateSequences == 0, this function blindly accepts the Sequences provided. Invalid Sequences cause undefined
* behavior. If ZSTD_c_validateSequences == 1, then the function will detect invalid Sequences (see doc/zstd_compression_format.md for
* specifics regarding offset/matchlength requirements) and then bail out and return an error.
* If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
* behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
* specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
*
* In addition to the two adjustable experimental params, there are other important cctx params.
* - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
@@ -1671,51 +1624,24 @@ ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, si
* - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
* is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
*
* Note: Repcodes are, as of now, always re-calculated within this function, ZSTD_Sequence.rep is effectively unused.
* Dev Note: Once ability to ingest repcodes become available, the explicit block delims mode must respect those repcodes exactly,
* and cannot emit an RLE block that disagrees with the repcode history.
* Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
* Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
* and cannot emit an RLE block that disagrees with the repcode history
* @return : final compressed size, or a ZSTD error code.
*/
ZSTDLIB_STATIC_API size_t
ZSTD_compressSequences(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* src, size_t srcSize);
/*! ZSTD_compressSequencesAndLiterals() :
* This is a variant of ZSTD_compressSequences() which,
* instead of receiving (src,srcSize) as input parameter, receives (literals,litSize),
* aka all the literals, already extracted and laid out into a single continuous buffer.
* This can be useful if the process generating the sequences also happens to generate the buffer of literals,
* thus skipping an extraction + caching stage.
* It's a speed optimization, useful when the right conditions are met,
* but it also features the following limitations:
* - Only supports explicit delimiter mode
* - Currently does not support Sequences validation (so input Sequences are trusted)
* - Not compatible with frame checksum, which must be disabled
* - If any block is incompressible, will fail and return an error
* - @litSize must be == sum of all @.litLength fields in @inSeqs. Any discrepancy will generate an error.
* - @litBufCapacity is the size of the underlying buffer into which literals are written, starting at address @literals.
* @litBufCapacity must be at least 8 bytes larger than @litSize.
* - @decompressedSize must be correct, and correspond to the sum of all Sequences. Any discrepancy will generate an error.
* @return : final compressed size, or a ZSTD error code.
*/
ZSTDLIB_STATIC_API size_t
ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t nbSequences,
const void* literals, size_t litSize, size_t litBufCapacity,
size_t decompressedSize);
/*! ZSTD_writeSkippableFrame() :
* Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
*
* Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
* ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
* As such, the parameter magicVariant controls the exact skippable frame magic number variant used,
* so the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
* As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
* the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
*
* Returns an error if destination buffer is not large enough, if the source size is not representable
* with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
@@ -1723,28 +1649,26 @@ ZSTD_compressSequencesAndLiterals(ZSTD_CCtx* cctx,
* @return : number of bytes written or a ZSTD error.
*/
ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
unsigned magicVariant);
const void* src, size_t srcSize, unsigned magicVariant);
/*! ZSTD_readSkippableFrame() :
* Retrieves the content of a zstd skippable frame starting at @src, and writes it to @dst buffer.
* Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
*
* The parameter @magicVariant will receive the magicVariant that was supplied when the frame was written,
* i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.
* This can be NULL if the caller is not interested in the magicVariant.
* The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
* i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
* in the magicVariant.
*
* Returns an error if destination buffer is not large enough, or if the frame is not skippable.
*
* @return : number of bytes written or a ZSTD error.
*/
ZSTDLIB_STATIC_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity,
unsigned* magicVariant,
ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
const void* src, size_t srcSize);
/*! ZSTD_isSkippableFrame() :
* Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
*/
ZSTDLIB_STATIC_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
@@ -1868,18 +1792,11 @@ ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict(
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
#if defined(__clang__) && __clang_major__ >= 5
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
#endif
static
#ifdef __GNUC__
__attribute__((__unused__))
#endif
ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */
#if defined(__clang__) && __clang_major__ >= 5
#pragma clang diagnostic pop
#endif
ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
@@ -2059,7 +1976,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* See the comments on that enum for an explanation of the feature. */
#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
/* Controlled with ZSTD_ParamSwitch_e enum.
/* Controlled with ZSTD_paramSwitch_e enum.
* Default is ZSTD_ps_auto.
* Set to ZSTD_ps_disable to never compress literals.
* Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
@@ -2200,46 +2117,22 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
/* ZSTD_c_validateSequences
* Default is 0 == disabled. Set to 1 to enable sequence validation.
*
* For use with sequence compression API: ZSTD_compressSequences*().
* Designates whether or not provided sequences are validated within ZSTD_compressSequences*()
* For use with sequence compression API: ZSTD_compressSequences().
* Designates whether or not we validate sequences provided to ZSTD_compressSequences()
* during function execution.
*
* When Sequence validation is disabled (default), Sequences are compressed as-is,
* so they must correct, otherwise it would result in a corruption error.
* Without validation, providing a sequence that does not conform to the zstd spec will cause
* undefined behavior, and may produce a corrupted block.
*
* Sequence validation adds some protection, by ensuring that all values respect boundary conditions.
* If a Sequence is detected invalid (see doc/zstd_compression_format.md for
* With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for
* specifics regarding offset/matchlength requirements) then the function will bail out and
* return an error.
*
*/
#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
/* ZSTD_c_blockSplitterLevel
* note: this parameter only influences the first splitter stage,
* which is active before producing the sequences.
* ZSTD_c_splitAfterSequences controls the next splitter stage,
* which is active after sequence production.
* Note that both can be combined.
* Allowed values are between 0 and ZSTD_BLOCKSPLITTER_LEVEL_MAX included.
* 0 means "auto", which will select a value depending on current ZSTD_c_strategy.
* 1 means no splitting.
* Then, values from 2 to 6 are sorted in increasing cpu load order.
*
* Note that currently the first block is never split,
* to ensure expansion guarantees in presence of incompressible data.
*/
#define ZSTD_BLOCKSPLITTER_LEVEL_MAX 6
#define ZSTD_c_blockSplitterLevel ZSTD_c_experimentalParam20
/* ZSTD_c_splitAfterSequences
* This is a stronger splitter algorithm,
* based on actual sequences previously produced by the selected parser.
* It's also slower, and as a consequence, mostly used for high compression levels.
* While the post-splitter does overlap with the pre-splitter,
* both can nonetheless be combined,
* notably with ZSTD_c_blockSplitterLevel at ZSTD_BLOCKSPLITTER_LEVEL_MAX,
* resulting in higher compression ratio than just one of them.
*
/* ZSTD_c_useBlockSplitter
* Controlled with ZSTD_paramSwitch_e enum.
* Default is ZSTD_ps_auto.
* Set to ZSTD_ps_disable to never use block splitter.
* Set to ZSTD_ps_enable to always use block splitter.
@@ -2247,10 +2140,10 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
* block splitting based on the compression parameters.
*/
#define ZSTD_c_splitAfterSequences ZSTD_c_experimentalParam13
#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13
/* ZSTD_c_useRowMatchFinder
* Controlled with ZSTD_ParamSwitch_e enum.
* Controlled with ZSTD_paramSwitch_e enum.
* Default is ZSTD_ps_auto.
* Set to ZSTD_ps_disable to never use row-based matchfinder.
* Set to ZSTD_ps_enable to force usage of row-based matchfinder.
@@ -2282,7 +2175,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
/* ZSTD_c_prefetchCDictTables
* Controlled with ZSTD_ParamSwitch_e enum. Default is ZSTD_ps_auto.
* Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto.
*
* In some situations, zstd uses CDict tables in-place rather than copying them
* into the working context. (See docs on ZSTD_dictAttachPref_e above for details).
@@ -2326,21 +2219,19 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
* bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
* compressBound() inaccurate). Only currently meant to be used for testing.
*
*/
#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
/* ZSTD_c_repcodeResolution
* This parameter only has an effect if ZSTD_c_blockDelimiters is
* set to ZSTD_sf_explicitBlockDelimiters (may change in the future).
/* ZSTD_c_searchForExternalRepcodes
* This parameter affects how zstd parses external sequences, such as sequences
* provided through the compressSequences() API or from an external block-level
* sequence producer.
*
* This parameter affects how zstd parses external sequences,
* provided via the ZSTD_compressSequences*() API
* or from an external block-level sequence producer.
*
* If set to ZSTD_ps_enable, the library will check for repeated offsets within
* If set to ZSTD_ps_enable, the library will check for repeated offsets in
* external sequences, even if those repcodes are not explicitly indicated in
* the "rep" field. Note that this is the only way to exploit repcode matches
* while using compressSequences*() or an external sequence producer, since zstd
* while using compressSequences() or an external sequence producer, since zstd
* currently ignores the "rep" field of external sequences.
*
* If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
@@ -2349,11 +2240,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
* compression ratio.
*
* The default value is ZSTD_ps_auto, for which the library will enable/disable
* based on compression level (currently: level<10 disables, level>=10 enables).
* based on compression level.
*
* Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
* set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
*/
#define ZSTD_c_repcodeResolution ZSTD_c_experimentalParam19
#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 /* older name */
#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
/*! ZSTD_CCtx_getParameter() :
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
@@ -3060,7 +2952,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
>0 : `srcSize` is too small, please provide at least result bytes on next attempt.
errorCode, which can be tested using ZSTD_isError().
It fills a ZSTD_FrameHeader structure with important information to correctly decode the frame,
It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
As a consequence, check that values remain within valid application range.
@@ -3137,18 +3029,6 @@ ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
/*! ZSTD_isDeterministicBuild() :
* Returns 1 if the library is built using standard compilation flags,
* and participates in determinism guarantees with other builds of the
* same version.
* If this function returns 0, it means the library was compiled with
* non-standard compilation flags that change the output of the
* compressor.
* This is mainly used for Zstd's determinism test suite, which is only
* run when this function returns 1.
*/
ZSTDLIB_API int ZSTD_isDeterministicBuild(void);
/* ========================================= */
/** Block level API (DEPRECATED) */
@@ -3202,8 +3082,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_
ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
+8 -1
View File
@@ -15,6 +15,10 @@
extern "C" {
#endif
/*===== dependency =====*/
#include <stddef.h> /* size_t */
/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
#ifndef ZSTDERRORLIB_VISIBLE
/* Backwards compatibility with old macro name */
@@ -76,7 +80,6 @@ typedef enum {
ZSTD_error_tableLog_tooLarge = 44,
ZSTD_error_maxSymbolValue_tooLarge = 46,
ZSTD_error_maxSymbolValue_tooSmall = 48,
ZSTD_error_cannotProduce_uncompressedBlock = 49,
ZSTD_error_stabilityCondition_notRespected = 50,
ZSTD_error_stage_wrong = 60,
ZSTD_error_init_missing = 62,
@@ -97,6 +100,10 @@ typedef enum {
ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
} ZSTD_ErrorCode;
/*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare with enum list published above */
ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */
+11 -11
View File
@@ -170,7 +170,7 @@ endif
zstd : $(BUILD_DIR)/zstd
if [ $(BIN_ISDIFFERENT) -eq 1 ]; then \
$(CP) $<$(EXT) $@$(EXT); \
cp -f $<$(EXT) $@$(EXT); \
echo zstd build completed; \
else \
echo zstd already built; \
@@ -232,7 +232,7 @@ zstd-dll : zstd
.PHONY: zstd-pgo
zstd-pgo : LLVM_PROFDATA?=llvm-profdata
zstd-pgo : PROF_GENERATE_FLAGS=-fprofile-generate $(if $(findstring gcc,$(CC)),-fprofile-dir=.)
zstd-pgo : PROF_USE_FLAGS=-fprofile-use $(if $(findstring gcc,$(CC)),-fprofile-dir=. -Wno-error=missing-profile -Wno-error=coverage-mismatch)
zstd-pgo : PROF_USE_FLAGS=-fprofile-use $(if $(findstring gcc,$(CC)),-fprofile-dir=. -Werror=missing-profile -Wno-error=coverage-mismatch)
zstd-pgo :
$(MAKE) clean HASH_DIR=$(HASH_DIR)
$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_GENERATE_FLAGS)"
@@ -271,7 +271,7 @@ zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zst
CLEAN += zstdmt
zstdmt: zstd
$(LN) -sf zstd zstdmt
ln -sf zstd zstdmt
.PHONY: generate_res
generate_res: $(RES64_FILE) $(RES32_FILE)
@@ -345,7 +345,7 @@ include $(wildcard $(DEPFILES))
#-----------------------------------------------------------------------------
# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(INSTALL_OS_LIST),$(UNAME)))
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT CYGWIN_NT))
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
EGREP_OPTIONS ?=
@@ -388,14 +388,14 @@ datarootdir ?= $(PREFIX)/share
mandir ?= $(datarootdir)/man
man1dir ?= $(mandir)/man1
ifneq (,$(filter OpenBSD NetBSD DragonFly SunOS,$(UNAME)))
ifneq (,$(filter $(UNAME),OpenBSD FreeBSD NetBSD DragonFly SunOS))
MANDIR ?= $(PREFIX)/man
MAN1DIR ?= $(MANDIR)/man1
else
MAN1DIR ?= $(man1dir)
endif
ifneq (,$(filter SunOS,$(UNAME)))
ifneq (,$(filter $(UNAME),SunOS))
INSTALL ?= ginstall
else
INSTALL ?= install
@@ -414,15 +414,15 @@ install:
[ -e $(DESTDIR)$(MAN1DIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(MAN1DIR)/
@echo Installing binaries
$(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT)
$(LN) -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT)
$(LN) -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT)
$(LN) -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT)
ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT)
ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT)
ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT)
$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
@echo Installing man pages
$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1
$(LN) -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1
$(LN) -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1
ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1
ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1
$(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1
$(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1
@echo zstd installation completed
+68 -117
View File
@@ -79,13 +79,6 @@ There are however other Makefile targets that create different variations of CLI
This can be useful to produce smaller binaries.
A corresponding `Makefile` target using this ability is `zstd-compress`.
- __ZSTD_DISPLAY_LEVEL_DEFAULT__ : Controls the default verbosity level of `zstd` output.
The default value is `2`. Lower values (e.g., `1`) reduce output verbosity,
while higher values (e.g., `3`) increase it.
This allows setting preferred verbosity at compile time,
rather than passing `-q` or `-v` flags at runtime.
Example : `CPPFLAGS="-DZSTD_DISPLAY_LEVEL_DEFAULT=1" make`
- __BACKTRACE__ : `zstd` can display a stack backtrace when execution
generates a runtime exception. By default, this feature may be
degraded/disabled on some platforms unless additional compiler directives are
@@ -136,131 +129,89 @@ CLI includes in-memory compression benchmark module for zstd.
The benchmark is conducted using given filenames. The files are read into memory and joined together.
It makes benchmark more precise as it eliminates I/O overhead.
Multiple filenames can be supplied, as multiple parameters, with wildcards,
or directory names can be used with `-r` option.
If no file is provided, the benchmark will use a procedurally generated "lorem ipsum" content.
or names of directories can be used as parameters with `-r` option.
The benchmark measures ratio, compressed size, compression and decompression speed.
One can select compression levels starting from `-b` and ending with `-e`.
The `-i` parameter selects minimal time used for each of tested levels.
The benchmark can also be used to test specific parameters,
such as number of threads (`-T#`), or advanced parameters (`--zstd=#`), or dictionary compression (`-D DICTIONARY`),
and many others available on command for regular compression and decompression.
### Usage of Command Line Interface
The full list of options can be obtained with `-h` or `-H` parameter:
```
*** Zstandard CLI (64-bit) v1.5.6, by Yann Collet ***
Usage :
zstd [args] [FILE(s)] [-o file]
Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.
FILE : a filename
with no FILE, or when FILE is - , read standard input
Arguments :
-# : # compression level (1-19, default: 3)
-d : decompression
-D DICT: use DICT as Dictionary for compression or decompression
-o file: result stored into `file` (only 1 output file)
-f : overwrite output without prompting, also (de)compress links
--rm : remove source file(s) after successful de/compression
-k : preserve source file(s) (default)
-h/-H : display help/long help and exit
Usage: zstd [OPTIONS...] [INPUT... | -] [-o OUTPUT]
Advanced arguments :
-V : display Version number and exit
-c : write to standard output (even if it is the console)
-v : verbose mode; specify multiple times to increase verbosity
-q : suppress warnings; specify twice to suppress errors too
--no-progress : do not display the progress counter
-r : operate recursively on directories
--filelist FILE : read list of files to operate upon from FILE
--output-dir-flat DIR : processed files are stored into DIR
--output-dir-mirror DIR : processed files are stored into DIR respecting original directory structure
--[no-]asyncio : use asynchronous IO (default: enabled)
--[no-]check : during compression, add XXH64 integrity checksum to frame (default: enabled). If specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate).
-- : All arguments after "--" are treated as files
Options:
-o OUTPUT Write output to a single file, OUTPUT.
-k, --keep Preserve INPUT file(s). [Default]
--rm Remove INPUT file(s) after successful (de)compression to file.
Advanced compression arguments :
--ultra : enable levels beyond 19, up to 22 (requires more memory)
--long[=#]: enable long distance matching with given window log (default: 27)
--fast[=#]: switch to very fast compression levels (default: 1)
--adapt : dynamically adapt compression level to I/O conditions
--patch-from=FILE : specify the file to be used as a reference point for zstd's diff engine
-T# : spawns # compression threads (default: 1, 0==# cores)
-B# : select size of each job (default: 0==automatic)
--single-thread : use a single thread for both I/O and compression (result slightly different than -T1)
--rsyncable : compress using a rsync-friendly method (-B sets block size)
--exclude-compressed: only compress files that are not already compressed
--stream-size=# : specify size of streaming input from `stdin`
--size-hint=# optimize compression parameters for streaming input of approximately this size
--target-compressed-block-size=# : generate compressed block of approximately targeted size
--no-dictID : don't write dictID into header (dictionary compression only)
--[no-]compress-literals : force (un)compressed literals
--format=zstd : compress files to the .zst format (default)
--format=gzip : compress files to the .gz format
--format=xz : compress files to the .xz format
--format=lzma : compress files to the .lzma format
--format=lz4 : compress files to the .lz4 format
-# Desired compression level, where `#` is a number between 1 and 19;
lower numbers provide faster compression, higher numbers yield
better compression ratios. [Default: 3]
Advanced decompression arguments :
-l : print information about zstd compressed files
--test : test compressed file integrity
-M# : Set a memory usage limit for decompression
--[no-]sparse : sparse mode (default: disabled)
-d, --decompress Perform decompression.
-D DICT Use DICT as the dictionary for compression or decompression.
Dictionary builder :
--train ## : create a dictionary from a training set of files
--train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] : use the cover algorithm with optional args
--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] : use the fast cover algorithm with optional args
--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9)
-o DICT : DICT is dictionary name (default: dictionary)
--maxdict=# : limit dictionary to specified size (default: 112640)
--dictID=# : force dictionary ID to specified value (default: random)
-f, --force Disable input and output checks. Allows overwriting existing files,
receiving input from the console, printing output to STDOUT, and
operating on links, block devices, etc. Unrecognized formats will be
passed-through through as-is.
-h Display short usage and exit.
-H, --help Display full help and exit.
-V, --version Display the program version and exit.
Advanced options:
-c, --stdout Write to STDOUT (even if it is a console) and keep the INPUT file(s).
-v, --verbose Enable verbose output; pass multiple times to increase verbosity.
-q, --quiet Suppress warnings; pass twice to suppress errors.
--trace LOG Log tracing information to LOG.
--[no-]progress Forcibly show/hide the progress counter. NOTE: Any (de)compressed
output to terminal will mix with progress counter text.
-r Operate recursively on directories.
--filelist LIST Read a list of files to operate on from LIST.
--output-dir-flat DIR Store processed files in DIR.
--output-dir-mirror DIR Store processed files in DIR, respecting original directory structure.
--[no-]asyncio Use asynchronous IO. [Default: Enabled]
--[no-]check Add XXH64 integrity checksums during compression. [Default: Add, Validate]
If `-d` is present, ignore/validate checksums during decompression.
-- Treat remaining arguments after `--` as files.
Advanced compression options:
--ultra Enable levels beyond 19, up to 22; requires more memory.
--fast[=#] Use to very fast compression levels. [Default: 1]
--adapt Dynamically adapt compression level to I/O conditions.
--long[=#] Enable long distance matching with window log #. [Default: 27]
--patch-from=REF Use REF as the reference point for Zstandard's diff engine.
-T# Spawn # compression threads. [Default: 1; pass 0 for core count.]
--single-thread Share a single thread for I/O and compression (slightly different than `-T1`).
--auto-threads={physical|logical}
Use physical/logical cores when using `-T0`. [Default: Physical]
-B# Set job size to #. [Default: 0 (automatic)]
--rsyncable Compress using a rsync-friendly method (`-B` sets block size).
--exclude-compressed Only compress files that are not already compressed.
--stream-size=# Specify size of streaming input from STDIN.
--size-hint=# Optimize compression parameters for streaming input of approximately size #.
--target-compressed-block-size=#
Generate compressed blocks of approximately # size.
--no-dictID Don't write `dictID` into the header (dictionary compression only).
--[no-]compress-literals Force (un)compressed literals.
--[no-]row-match-finder Explicitly enable/disable the fast, row-based matchfinder for
the 'greedy', 'lazy', and 'lazy2' strategies.
--format=zstd Compress files to the `.zst` format. [Default]
--[no-]mmap-dict Memory-map dictionary file rather than mallocing and loading all at once
--format=gzip Compress files to the `.gz` format.
--format=xz Compress files to the `.xz` format.
--format=lzma Compress files to the `.lzma` format.
--format=lz4 Compress files to the `.lz4` format.
Advanced decompression options:
-l Print information about Zstandard-compressed files.
--test Test compressed file integrity.
-M# Set the memory usage limit to # megabytes.
--[no-]sparse Enable sparse mode. [Default: Enabled for files, disabled for STDOUT.]
--[no-]pass-through Pass through uncompressed files as-is. [Default: Disabled]
Dictionary builder:
--train Create a dictionary from a training set of files.
--train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]]
Use the cover algorithm (with optional arguments).
--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]]
Use the fast cover algorithm (with optional arguments).
--train-legacy[=s=#] Use the legacy algorithm with selectivity #. [Default: 9]
-o NAME Use NAME as dictionary name. [Default: dictionary]
--maxdict=# Limit dictionary to specified size #. [Default: 112640]
--dictID=# Force dictionary ID to #. [Default: Random]
Benchmark options:
-b# Perform benchmarking with compression level #. [Default: 3]
-e# Test all compression levels up to #; starting level is `-b#`. [Default: 1]
-i# Set the minimum evaluation to time # seconds. [Default: 3]
-B# Cut file into independent chunks of size #. [Default: No chunking]
-S Output one benchmark result per input file. [Default: Consolidated result]
-D dictionary Benchmark using dictionary
--priority=rt Set process priority to real-time.
Benchmark arguments :
-b# : benchmark file(s), using # compression level (default: 3)
-e# : test all compression levels successively from -b# to -e# (default: 1)
-i# : minimum evaluation time in seconds (default: 3s)
-B# : cut file into independent chunks of size # (default: no chunking)
-S : output one benchmark result per input file (default: consolidated result)
--priority=rt : set process priority to real-time
```
### Passing parameters through Environment Variables
+10
View File
@@ -15,12 +15,17 @@
* or detecting and returning an error
*/
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef BENCH_FN_H_23876
#define BENCH_FN_H_23876
/* === Dependencies === */
#include <stddef.h> /* size_t */
/* ==== Benchmark any function, iterated on a set of blocks ==== */
/* BMK_runTime_t: valid result return type */
@@ -170,4 +175,9 @@ typedef union {
} BMK_timedFnState_shell;
BMK_timedFnState_t* BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms);
#endif /* BENCH_FN_H_23876 */
#if defined (__cplusplus)
}
#endif
+142 -148
View File
@@ -139,61 +139,52 @@ static const size_t maxMemory = (sizeof(size_t) == 4)
return r; \
}
static size_t uintSize(unsigned value)
{
size_t size = 1;
while (value >= 10) {
size++;
value /= 10;
}
return size;
}
/* Note: presume @buffer is large enough */
static void writeUint_varLen(char* buffer, size_t capacity, unsigned value)
{
int endPos = (int)uintSize(value) - 1;
assert(uintSize(value) >= 1);
assert(uintSize(value) < capacity); (void)capacity;
while (endPos >= 0) {
char c = '0' + (char)(value % 10);
buffer[endPos--] = c;
value /= 10;
}
}
/* replacement for snprintf(), which is not supported by C89.
* sprintf() would be the supported one, but it's labelled unsafe:
* modern static analyzer will flag sprintf() as dangerous, making it unusable.
* formatString_u() replaces snprintf() for the specific case where there is only one %u argument */
/* replacement for snprintf(), which is not supported by C89
* sprintf() would be the supported one, but it's labelled unsafe,
* so some modern static analyzer will flag it as such, making it unusable.
* formatString_u() replaces snprintf() for the specific case where there are only %u arguments */
static int formatString_u(char* buffer, size_t buffer_size, const char* formatString, unsigned int value)
{
size_t const valueSize = uintSize(value);
size_t written = 0;
int i;
assert(value <= 100);
for (i = 0; formatString[i] != '\0' && written < buffer_size - 1; i++) {
for (i = 0; formatString[i] != '\0' && written < buffer_size - 1; ++i) {
if (formatString[i] != '%') {
buffer[written++] = formatString[i];
continue;
}
i++;
if (formatString[i] == 'u') {
if (written + valueSize >= buffer_size) abort(); /* buffer not large enough */
writeUint_varLen(buffer + written, buffer_size - written, value);
written += valueSize;
if (formatString[++i] == 'u') {
/* Handle single digit */
if (value < 10) {
buffer[written++] = '0' + (char)value;
} else if (value < 100) {
/* Handle two digits */
if (written >= buffer_size - 2) {
return -1; /* buffer overflow */
}
buffer[written++] = '0' + (char)(value / 10);
buffer[written++] = '0' + (char)(value % 10);
} else { /* 100 */
if (written >= buffer_size - 3) {
return -1; /* buffer overflow */
}
buffer[written++] = '1';
buffer[written++] = '0';
buffer[written++] = '0';
}
} else if (formatString[i] == '%') { /* Check for escaped percent sign */
buffer[written++] = '%';
} else {
abort(); /* unsupported format */
return -1; /* unsupported format */
}
}
if (written < buffer_size) {
buffer[written] = '\0';
} else {
abort(); /* buffer not large enough */
buffer[0] = '\0'; /* Handle truncation */
}
return (int)written;
@@ -208,7 +199,7 @@ BMK_advancedParams_t BMK_initAdvancedParams(void)
BMK_advancedParams_t const res = {
BMK_both, /* mode */
BMK_TIMETEST_DEFAULT_S, /* nbSeconds */
0, /* chunkSizeMax */
0, /* blockSize */
0, /* targetCBlockSize */
0, /* nbWorkers */
0, /* realTime */
@@ -227,6 +218,16 @@ BMK_advancedParams_t BMK_initAdvancedParams(void)
/* ********************************************************
* Bench functions
**********************************************************/
typedef struct {
const void* srcPtr;
size_t srcSize;
void* cPtr;
size_t cRoom;
size_t cSize;
void* resPtr;
size_t resSize;
} blockParam_t;
#undef MIN
#undef MAX
#define MIN(a, b) ((a) < (b) ? (a) : (b))
@@ -425,16 +426,16 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
const char* displayName,
const BMK_advancedParams_t* adv)
{
size_t const chunkSizeMax =
((adv->chunkSizeMax >= 32 && (adv->mode != BMK_decodeOnly))
? adv->chunkSizeMax
size_t const blockSize =
((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly))
? adv->blockSize
: srcSize)
+ (!srcSize); /* avoid div by 0 */
BMK_benchResult_t benchResult;
size_t const loadedCompressedSize = srcSize;
size_t cSize = 0;
double ratio = 0.;
U32 nbChunks = 0;
U32 nbBlocks;
assert(cctx != NULL);
assert(dctx != NULL);
@@ -490,42 +491,41 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
}
}
/* Init data chunks */
/* Init data blocks */
{
const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)(*resultBufferPtr);
U32 fileNb, chunkID;
for (chunkID = 0, fileNb = 0; fileNb < nbFiles; fileNb++) {
U32 fileNb;
for (nbBlocks = 0, fileNb = 0; fileNb < nbFiles; fileNb++) {
size_t remaining = fileSizes[fileNb];
U32 const nbChunksforThisFile = (adv->mode == BMK_decodeOnly)
U32 const nbBlocksforThisFile = (adv->mode == BMK_decodeOnly)
? 1
: (U32)((remaining + (chunkSizeMax - 1)) / chunkSizeMax);
U32 const chunkIdEnd = chunkID + nbChunksforThisFile;
for (; chunkID < chunkIdEnd; chunkID++) {
size_t const chunkSize = MIN(remaining, chunkSizeMax);
srcPtrs[chunkID] = srcPtr;
srcSizes[chunkID] = chunkSize;
cPtrs[chunkID] = cPtr;
cCapacities[chunkID] = (adv->mode == BMK_decodeOnly)
? chunkSize
: ZSTD_compressBound(chunkSize);
resPtrs[chunkID] = resPtr;
resSizes[chunkID] = (adv->mode == BMK_decodeOnly)
: (U32)((remaining + (blockSize - 1)) / blockSize);
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
for (; nbBlocks < blockEnd; nbBlocks++) {
size_t const thisBlockSize = MIN(remaining, blockSize);
srcPtrs[nbBlocks] = srcPtr;
srcSizes[nbBlocks] = thisBlockSize;
cPtrs[nbBlocks] = cPtr;
cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly)
? thisBlockSize
: ZSTD_compressBound(thisBlockSize);
resPtrs[nbBlocks] = resPtr;
resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly)
? (size_t)ZSTD_findDecompressedSize(
srcPtr, chunkSize)
: chunkSize;
srcPtr += chunkSize;
cPtr += cCapacities[chunkID];
resPtr += chunkSize;
remaining -= chunkSize;
srcPtr, thisBlockSize)
: thisBlockSize;
srcPtr += thisBlockSize;
cPtr += cCapacities[nbBlocks];
resPtr += thisBlockSize;
remaining -= thisBlockSize;
if (adv->mode == BMK_decodeOnly) {
cSizes[chunkID] = chunkSize;
benchResult.cSize = chunkSize;
cSizes[nbBlocks] = thisBlockSize;
benchResult.cSize = thisBlockSize;
}
}
}
nbChunks = chunkID;
}
/* warming up `compressedBuffer` */
@@ -560,7 +560,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
cbp.initFn = local_initCCtx; /* BMK_initCCtx */
cbp.initPayload = &cctxprep;
cbp.errorFn = ZSTD_isError;
cbp.blockCount = nbChunks;
cbp.blockCount = nbBlocks;
cbp.srcBuffers = srcPtrs;
cbp.srcSizes = srcSizes;
cbp.dstBuffers = cPtrs;
@@ -579,7 +579,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
dbp.initFn = local_initDCtx;
dbp.initPayload = &dctxprep;
dbp.errorFn = ZSTD_isError;
dbp.blockCount = nbChunks;
dbp.blockCount = nbBlocks;
dbp.srcBuffers = (const void* const*)cPtrs;
dbp.srcSizes = cSizes;
dbp.dstBuffers = resPtrs;
@@ -624,7 +624,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
}
{
int const ratioDigits = 1 + (ratio < 100.) + (ratio < 10.);
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
assert(cSize < UINT_MAX);
OUTPUTLEVEL(
2,
@@ -633,7 +633,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
displayName,
(unsigned)srcSize,
(unsigned)cSize,
ratioDigits,
ratioAccuracy,
ratio,
benchResult.cSpeed < (10 * MB_UNIT) ? 2 : 1,
(double)benchResult.cSpeed / MB_UNIT);
@@ -660,7 +660,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
}
{
int const ratioDigits = 1 + (ratio < 100.) + (ratio < 10.);
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
OUTPUTLEVEL(
2,
"%2s-%-17.17s :%10u ->%10u (x%5.*f), %6.*f MB/s, %6.1f MB/s\r",
@@ -668,7 +668,7 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
displayName,
(unsigned)srcSize,
(unsigned)cSize,
ratioDigits,
ratioAccuracy,
ratio,
benchResult.cSpeed < (10 * MB_UNIT) ? 2 : 1,
(double)benchResult.cSpeed / MB_UNIT,
@@ -681,7 +681,8 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
} /* while (!(compressionCompleted && decompressionCompleted)) */
/* CRC Checking */
{ const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
{
const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr);
U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if ((adv->mode == BMK_both) && (crcOrig != crcCheck)) {
size_t u;
@@ -694,14 +695,14 @@ static BMK_benchOutcome_t BMK_benchMemAdvancedNoAlloc(
unsigned segNb, bNb, pos;
size_t bacc = 0;
DISPLAY("Decoding error at pos %u ", (unsigned)u);
for (segNb = 0; segNb < nbChunks; segNb++) {
for (segNb = 0; segNb < nbBlocks; segNb++) {
if (bacc + srcSizes[segNb] > u)
break;
bacc += srcSizes[segNb];
}
pos = (U32)(u - bacc);
bNb = pos / (128 KB);
DISPLAY("(sample %u, chunk %u, pos %u) \n",
DISPLAY("(sample %u, block %u, pos %u) \n",
segNb,
bNb,
pos);
@@ -785,24 +786,25 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(
int const dstParamsError =
!dstBuffer ^ !dstCapacity; /* must be both NULL or none */
size_t const chunkSize =
((adv->chunkSizeMax >= 32 && (adv->mode != BMK_decodeOnly))
? adv->chunkSizeMax
size_t const blockSize =
((adv->blockSize >= 32 && (adv->mode != BMK_decodeOnly))
? adv->blockSize
: srcSize)
+ (!srcSize) /* avoid div by 0 */;
U32 const nbChunksMax =
(U32)((srcSize + (chunkSize - 1)) / chunkSize) + nbFiles;
U32 const maxNbBlocks =
(U32)((srcSize + (blockSize - 1)) / blockSize) + nbFiles;
/* these are the blockTable parameters, just split up */
const void** const srcPtrs =
(const void**)malloc(nbChunksMax * sizeof(void*));
size_t* const srcSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
(const void**)malloc(maxNbBlocks * sizeof(void*));
size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
void** const cPtrs = (void**)malloc(nbChunksMax * sizeof(void*));
size_t* const cSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
size_t* const cCapacities = (size_t*)malloc(nbChunksMax * sizeof(size_t));
void** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
void** const resPtrs = (void**)malloc(nbChunksMax * sizeof(void*));
size_t* const resSizes = (size_t*)malloc(nbChunksMax * sizeof(size_t));
void** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(
adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS);
@@ -814,7 +816,7 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(
const size_t maxCompressedSize = dstCapacity
? dstCapacity
: ZSTD_compressBound(srcSize) + (nbChunksMax * 1024);
: ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);
void* const internalDstBuffer =
dstBuffer ? NULL : malloc(maxCompressedSize);
@@ -917,13 +919,12 @@ BMK_benchOutcome_t BMK_benchMem(
&adv);
}
/* @return: 0 on success, !0 if error */
static int BMK_benchCLevels(
static BMK_benchOutcome_t BMK_benchCLevel(
const void* srcBuffer,
size_t benchedSize,
const size_t* fileSizes,
unsigned nbFiles,
int startCLevel, int endCLevel,
int cLevel,
const ZSTD_compressionParameters* comprParams,
const void* dictBuffer,
size_t dictBufferSize,
@@ -931,70 +932,63 @@ static int BMK_benchCLevels(
const char* displayName,
BMK_advancedParams_t const* const adv)
{
int level;
const char* pch = strrchr(displayName, '\\'); /* Windows */
if (!pch)
pch = strrchr(displayName, '/'); /* Linux */
if (pch)
displayName = pch + 1;
if (endCLevel > ZSTD_maxCLevel()) {
DISPLAYLEVEL(1, "Invalid Compression Level \n");
return 15;
}
if (endCLevel < startCLevel) {
DISPLAYLEVEL(1, "Invalid Compression Level Range \n");
return 15;
}
if (adv->realTime) {
DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
SET_REALTIME_PRIORITY;
}
if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */
OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB chunks\n",
OUTPUT("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n",
ZSTD_VERSION_STRING,
ZSTD_GIT_COMMIT_STRING,
(unsigned)benchedSize,
adv->nbSeconds,
(unsigned)(adv->chunkSizeMax >> 10));
(unsigned)(adv->blockSize >> 10));
for (level = startCLevel; level <= endCLevel; level++) {
BMK_benchOutcome_t res = BMK_benchMemAdvanced(
return BMK_benchMemAdvanced(
srcBuffer,
benchedSize,
NULL,
0,
fileSizes,
nbFiles,
level,
cLevel,
comprParams,
dictBuffer,
dictBufferSize,
displayLevel,
displayName,
adv);
if (!BMK_isSuccessful_benchOutcome(res)) return 1;
}
return 0;
}
int BMK_syntheticTest(
int cLevel,
double compressibility,
int startingCLevel, int endCLevel,
const ZSTD_compressionParameters* compressionParams,
int displayLevel,
const BMK_advancedParams_t* adv)
{
char nameBuff[20] = { 0 };
const char* name = nameBuff;
size_t const benchedSize = adv->chunkSizeMax ? adv->chunkSizeMax : 10000000;
size_t const benchedSize = adv->blockSize ? adv->blockSize : 10000000;
void* srcBuffer;
BMK_benchOutcome_t res;
if (cLevel > ZSTD_maxCLevel()) {
DISPLAYLEVEL(1, "Invalid Compression Level");
return 15;
}
/* Memory allocation */
void* const srcBuffer = malloc(benchedSize);
srcBuffer = malloc(benchedSize);
if (!srcBuffer) {
DISPLAYLEVEL(1, "allocation error : not enough memory \n");
DISPLAYLEVEL(1, "allocation error : not enough memory");
return 16;
}
@@ -1012,21 +1006,23 @@ int BMK_syntheticTest(
}
/* Bench */
{ int res = BMK_benchCLevels(
res = BMK_benchCLevel(
srcBuffer,
benchedSize,
&benchedSize,
1,
startingCLevel, endCLevel,
&benchedSize /* ? */,
1 /* ? */,
cLevel,
compressionParams,
NULL,
0, /* dictionary */
displayLevel,
name,
adv);
/* clean up */
free(srcBuffer);
return res;
}
return !BMK_isSuccessful_benchOutcome(res);
}
static size_t BMK_findMaxMem(U64 requiredMem)
@@ -1062,12 +1058,11 @@ static int BMK_loadFiles(
size_t pos = 0, totalSize = 0;
unsigned n;
for (n = 0; n < nbFiles; n++) {
const char* const filename = fileNamesTable[n];
U64 fileSize = UTIL_getFileSize(
filename); /* last file may be shortened */
if (UTIL_isDirectory(filename)) {
fileNamesTable[n]); /* last file may be shortened */
if (UTIL_isDirectory(fileNamesTable[n])) {
DISPLAYLEVEL(
2, "Ignoring %s directory... \n", filename);
2, "Ignoring %s directory... \n", fileNamesTable[n]);
fileSizes[n] = 0;
continue;
}
@@ -1075,29 +1070,25 @@ static int BMK_loadFiles(
DISPLAYLEVEL(
2,
"Cannot evaluate size of %s, ignoring ... \n",
filename);
fileNamesTable[n]);
fileSizes[n] = 0;
continue;
}
if (fileSize > bufferSize - pos) {
/* buffer too small - limit quantity loaded */
fileSize = bufferSize - pos;
nbFiles = n; /* stop after this file */
}
{ FILE* const f = fopen(filename, "rb");
if (f == NULL) {
{
FILE* const f = fopen(fileNamesTable[n], "rb");
if (f == NULL)
RETURN_ERROR_INT(
10, "cannot open file %s", filename);
}
OUTPUTLEVEL(2, "Loading %s... \r", filename);
{ size_t const readSize =
10, "impossible to open file %s", fileNamesTable[n]);
OUTPUTLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize - pos)
fileSize = bufferSize - pos,
nbFiles = n; /* buffer too small - stop after this file */
{
size_t const readSize =
fread(((char*)buffer) + pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) {
fclose(f);
if (readSize != (size_t)fileSize)
RETURN_ERROR_INT(
11, "invalid read %s", filename);
}
11, "could not read %s", fileNamesTable[n]);
pos += readSize;
}
fileSizes[n] = (size_t)fileSize;
@@ -1115,7 +1106,7 @@ int BMK_benchFilesAdvanced(
const char* const* fileNamesTable,
unsigned nbFiles,
const char* dictFileName,
int startCLevel, int endCLevel,
int cLevel,
const ZSTD_compressionParameters* compressionParams,
int displayLevel,
const BMK_advancedParams_t* adv)
@@ -1125,7 +1116,7 @@ int BMK_benchFilesAdvanced(
void* dictBuffer = NULL;
size_t dictBufferSize = 0;
size_t* fileSizes = NULL;
int res = 1;
BMK_benchOutcome_t res;
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
if (!nbFiles) {
@@ -1133,7 +1124,7 @@ int BMK_benchFilesAdvanced(
return 13;
}
if (endCLevel > ZSTD_maxCLevel()) {
if (cLevel > ZSTD_maxCLevel()) {
DISPLAYLEVEL(1, "Invalid Compression Level");
return 14;
}
@@ -1187,6 +1178,7 @@ int BMK_benchFilesAdvanced(
1 /*?*/,
displayLevel);
if (errorCode) {
res = BMK_benchOutcome_error();
goto _cleanUp;
}
}
@@ -1218,6 +1210,7 @@ int BMK_benchFilesAdvanced(
nbFiles,
displayLevel);
if (errorCode) {
res = BMK_benchOutcome_error();
goto _cleanUp;
}
}
@@ -1226,14 +1219,15 @@ int BMK_benchFilesAdvanced(
{
char mfName[20] = { 0 };
formatString_u(mfName, sizeof(mfName), " %u files", nbFiles);
{ const char* const displayName =
{
const char* const displayName =
(nbFiles > 1) ? mfName : fileNamesTable[0];
res = BMK_benchCLevels(
res = BMK_benchCLevel(
srcBuffer,
benchedSize,
fileSizes,
nbFiles,
startCLevel, endCLevel,
cLevel,
compressionParams,
dictBuffer,
dictBufferSize,
@@ -1247,7 +1241,7 @@ _cleanUp:
free(srcBuffer);
free(dictBuffer);
free(fileSizes);
return res;
return !BMK_isSuccessful_benchOutcome(res);
}
int BMK_benchFiles(
@@ -1263,7 +1257,7 @@ int BMK_benchFiles(
fileNamesTable,
nbFiles,
dictFileName,
cLevel, cLevel,
cLevel,
compressionParams,
displayLevel,
&adv);
+16 -9
View File
@@ -14,6 +14,10 @@
* and display progress result and final summary
*/
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef BENCH_ZSTD_H_3242387
#define BENCH_ZSTD_H_3242387
@@ -22,6 +26,7 @@
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "../lib/zstd.h" /* ZSTD_compressionParameters */
/* === Constants === */
#define MB_UNIT 1000000
@@ -92,9 +97,9 @@ typedef enum {
} BMK_mode_t;
typedef struct {
BMK_mode_t mode; /* 0: both, 1: compress only 2: decode only */
BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */
unsigned nbSeconds; /* default timing is in nbSeconds */
size_t chunkSizeMax; /* Maximum size of each independent chunk */
size_t blockSize; /* Maximum size of each block*/
size_t targetCBlockSize;/* Approximative size of compressed blocks */
int nbWorkers; /* multithreading */
unsigned realTime; /* real time priority */
@@ -104,7 +109,7 @@ typedef struct {
int ldmHashLog;
int ldmBucketSizeLog;
int ldmHashRateLog;
ZSTD_ParamSwitch_e literalCompressionMode;
ZSTD_paramSwitch_e literalCompressionMode;
int useRowMatchFinder; /* use row-based matchfinder if possible */
} BMK_advancedParams_t;
@@ -117,13 +122,12 @@ BMK_advancedParams_t BMK_initAdvancedParams(void);
int BMK_benchFilesAdvanced(
const char* const * fileNamesTable, unsigned nbFiles,
const char* dictFileName,
int startCLevel, int endCLevel,
const ZSTD_compressionParameters* compressionParams,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
/*! BMK_syntheticTest() -- called from zstdcli */
/* Generates a sample with datagen, using @compressibility argument
* @cLevel - compression level to benchmark, errors if invalid
/* Generates a sample with datagen, using compressibility argument */
/* @cLevel - compression level to benchmark, errors if invalid
* @compressibility - determines compressibility of sample, range [0.0 - 1.0]
* if @compressibility < 0.0, uses the lorem ipsum generator
* @compressionParams - basic compression Parameters
@@ -131,8 +135,7 @@ int BMK_benchFilesAdvanced(
* @adv - see advanced_Params_t
* @return: 0 on success, !0 on error
*/
int BMK_syntheticTest(double compressibility,
int startingCLevel, int endCLevel,
int BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
@@ -189,3 +192,7 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
#endif /* BENCH_ZSTD_H_3242387 */
#if defined (__cplusplus)
}
#endif
-8
View File
@@ -14,10 +14,6 @@
#include <stddef.h> /* size_t */
#if defined (__cplusplus)
extern "C" {
#endif
void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed);
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed);
/*!RDG_genBuffer
@@ -31,8 +27,4 @@ void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba
Same as RDG_genBuffer, but generates data into stdout
*/
#if defined (__cplusplus)
} /* extern "C" */
#endif
#endif
+6 -13
View File
@@ -280,15 +280,8 @@ static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t
for (n=0; n<nbFiles; n++) {
S64 const fileSize = DiB_getFileSize(fileNamesTable[n]);
/* TODO: is there a minimum sample size? What if the file is 1-byte? */
/* Skip empty or invalid files */
if (fileSize <= 0) {
if (fileSize < 0) {
DISPLAYLEVEL(3, "Sample file '%s' is unreadable or stat failed, skipping...\n",
fileNamesTable[n]);
} else {
DISPLAYLEVEL(3, "Sample file '%s' has zero size, skipping...\n",
fileNamesTable[n]);
}
if (fileSize == 0) {
DISPLAYLEVEL(3, "Sample file '%s' has zero size, skipping...\n", fileNamesTable[n]);
continue;
}
@@ -305,7 +298,7 @@ static fileStats DiB_fileStats(const char** fileNamesTable, int nbFiles, size_t
fs.oneSampleTooLarge |= (fileSize > 2*SAMPLESIZE_MAX);
/* Limit to the first SAMPLESIZE_MAX (128kB) of the file */
DISPLAYLEVEL(3, "Sample file '%s' is too large, limiting to %d KB\n",
DISPLAYLEVEL(3, "Sample file '%s' is too large, limiting to %d KB",
fileNamesTable[n], SAMPLESIZE_MAX / (1 KB));
}
fs.nbSamples += 1;
@@ -369,9 +362,9 @@ int DiB_trainFromFiles(const char* dictFileName, size_t maxDictSize,
DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX);
}
if (fs.nbSamples < 5) {
DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing !\n");
DISPLAYLEVEL(2, "! Please provide _one file per sample_.\n");
DISPLAYLEVEL(2, "! Alternatively, split file(s) into fixed-size samples, with --split=#\n");
DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n");
DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n");
DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n");
EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */
}
if (fs.totalSizeToLoad < (S64)maxDictSize * 8) {
+177 -428
View File
@@ -125,271 +125,6 @@ char const* FIO_lzmaVersion(void)
#define TEMPORARY_FILE_PERMISSIONS (0600)
#endif
#ifndef ZSTD_NOCOMPRESS
/* *************************************
* Synchronous compression IO helpers
* Lightweight wrapper used by compression paths to manage buffered
* reads/writes without the async job machinery.
***************************************/
typedef struct {
const FIO_prefs_t* prefs;
FILE* srcFile;
FILE* dstFile;
unsigned storedSkips;
U8* inBuffer;
size_t inCapacity;
U8* srcBuffer;
size_t srcBufferLoaded;
U8* outBuffer;
size_t outCapacity;
} FIO_SyncCompressIO;
static void FIO_SyncCompressIO_init(FIO_SyncCompressIO* io,
const FIO_prefs_t* prefs,
size_t inCapacity,
size_t outCapacity);
static void FIO_SyncCompressIO_destroy(FIO_SyncCompressIO* io);
static void FIO_SyncCompressIO_setSrc(FIO_SyncCompressIO* io, FILE* file);
static void FIO_SyncCompressIO_clearSrc(FIO_SyncCompressIO* io);
static void FIO_SyncCompressIO_setDst(FIO_SyncCompressIO* io, FILE* file);
static int FIO_SyncCompressIO_closeDst(FIO_SyncCompressIO* io);
static size_t FIO_SyncCompressIO_fillBuffer(FIO_SyncCompressIO* io, size_t minToHave);
static void FIO_SyncCompressIO_consumeBytes(FIO_SyncCompressIO* io, size_t n);
static void FIO_SyncCompressIO_commitOut(FIO_SyncCompressIO* io, const void* buffer, size_t size);
static void FIO_SyncCompressIO_finish(FIO_SyncCompressIO* io);
static unsigned FIO_sparseWrite(FILE* file,
const void* buffer, size_t bufferSize,
const FIO_prefs_t* const prefs,
unsigned storedSkips)
{
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
size_t bufferSizeT = bufferSize / sizeof(size_t);
const size_t* const bufferTEnd = bufferT + bufferSizeT;
const size_t* ptrT = bufferT;
static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */
if (prefs->testMode) return 0; /* do not output anything in test mode */
if (!prefs->sparseFileSupport) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
if (sizeCheck != bufferSize)
EXM_THROW(70, "Write error : cannot write block : %s",
strerror(errno));
return 0;
}
/* avoid int overflow */
if (storedSkips > 1 GB) {
if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0)
EXM_THROW(91, "1 GB skip error (sparse file support)");
storedSkips -= 1 GB;
}
while (ptrT < bufferTEnd) {
size_t nb0T;
/* adjust last segment if < 32 KB */
size_t seg0SizeT = segmentSizeT;
if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT;
bufferSizeT -= seg0SizeT;
/* count leading zeroes */
for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ;
storedSkips += (unsigned)(nb0T * sizeof(size_t));
if (nb0T != seg0SizeT) { /* not all 0s */
size_t const nbNon0ST = seg0SizeT - nb0T;
/* skip leading zeros */
if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0)
EXM_THROW(92, "Sparse skip error ; try --no-sparse");
storedSkips = 0;
/* write the rest */
if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST)
EXM_THROW(93, "Write error : cannot write block : %s",
strerror(errno));
}
ptrT += seg0SizeT;
}
{ static size_t const maskT = sizeof(size_t)-1;
if (bufferSize & maskT) {
/* size not multiple of sizeof(size_t) : implies end of block */
const char* const restStart = (const char*)bufferTEnd;
const char* restPtr = restStart;
const char* const restEnd = (const char*)buffer + bufferSize;
assert(restEnd > restStart && restEnd < restStart + sizeof(size_t));
for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
storedSkips += (unsigned) (restPtr - restStart);
if (restPtr != restEnd) {
/* not all remaining bytes are 0 */
size_t const restSize = (size_t)(restEnd - restPtr);
if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0)
EXM_THROW(92, "Sparse skip error ; try --no-sparse");
if (fwrite(restPtr, 1, restSize, file) != restSize)
EXM_THROW(95, "Write error : cannot write end of decoded block : %s",
strerror(errno));
storedSkips = 0;
} } }
return storedSkips;
}
static void FIO_sparseWriteEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
{
if (file == NULL) return;
if (prefs->testMode) {
assert(storedSkips == 0);
return;
}
if (storedSkips>0) {
assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */
if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0)
EXM_THROW(69, "Final skip error (sparse file support)");
/* last zero must be explicitly written,
* so that skipped ones get implicitly translated as zero by FS */
{ const char lastZeroByte[1] = { 0 };
if (fwrite(lastZeroByte, 1, 1, file) != 1)
EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno));
}
}
}
static void FIO_SyncCompressIO_init(FIO_SyncCompressIO* io,
const FIO_prefs_t* prefs,
size_t inCapacity,
size_t outCapacity)
{
memset(io, 0, sizeof(*io));
io->prefs = prefs;
io->inCapacity = inCapacity;
io->outCapacity = outCapacity;
io->inBuffer = (U8*)malloc(inCapacity);
if (!io->inBuffer)
EXM_THROW(101, "Allocation error : not enough memory");
io->outBuffer = (U8*)malloc(outCapacity);
if (!io->outBuffer) {
free(io->inBuffer);
io->inBuffer = NULL;
EXM_THROW(101, "Allocation error : not enough memory");
}
io->srcBuffer = io->inBuffer;
io->srcBufferLoaded = 0;
}
static void FIO_SyncCompressIO_destroy(FIO_SyncCompressIO* io)
{
if (!io) return;
free(io->inBuffer);
free(io->outBuffer);
io->inBuffer = NULL;
io->outBuffer = NULL;
io->srcBuffer = NULL;
io->srcBufferLoaded = 0;
io->srcFile = NULL;
io->dstFile = NULL;
io->storedSkips = 0;
}
static void FIO_SyncCompressIO_setSrc(FIO_SyncCompressIO* io, FILE* file)
{
io->srcFile = file;
io->srcBuffer = io->inBuffer;
io->srcBufferLoaded = 0;
}
static void FIO_SyncCompressIO_clearSrc(FIO_SyncCompressIO* io)
{
io->srcFile = NULL;
io->srcBuffer = io->inBuffer;
io->srcBufferLoaded = 0;
}
static void FIO_SyncCompressIO_setDst(FIO_SyncCompressIO* io, FILE* file)
{
io->dstFile = file;
io->storedSkips = 0;
}
static int FIO_SyncCompressIO_closeDst(FIO_SyncCompressIO* io)
{
int result = 0;
if (io->dstFile != NULL) {
FIO_SyncCompressIO_finish(io);
result = fclose(io->dstFile);
io->dstFile = NULL;
}
return result;
}
static size_t FIO_SyncCompressIO_fillBuffer(FIO_SyncCompressIO* io, size_t minToHave)
{
size_t added = 0;
if (io->srcFile == NULL)
return 0;
if (minToHave > io->inCapacity)
minToHave = io->inCapacity;
if (io->srcBufferLoaded >= minToHave)
return 0;
if (io->srcBuffer != io->inBuffer) {
if (io->srcBufferLoaded > 0)
memmove(io->inBuffer, io->srcBuffer, io->srcBufferLoaded);
io->srcBuffer = io->inBuffer;
}
while (io->srcBufferLoaded < minToHave) {
size_t const toRead = io->inCapacity - io->srcBufferLoaded;
size_t const readBytes = fread(io->inBuffer + io->srcBufferLoaded, 1, toRead, io->srcFile);
if (readBytes == 0) {
if (ferror(io->srcFile))
EXM_THROW(37, "Read error");
break; /* EOF */
}
io->srcBufferLoaded += readBytes;
added += readBytes;
if (readBytes < toRead)
break;
}
return added;
}
static void FIO_SyncCompressIO_consumeBytes(FIO_SyncCompressIO* io, size_t n)
{
assert(n <= io->srcBufferLoaded);
io->srcBuffer += n;
io->srcBufferLoaded -= n;
if (io->srcBufferLoaded == 0)
io->srcBuffer = io->inBuffer;
}
static void FIO_SyncCompressIO_commitOut(FIO_SyncCompressIO* io, const void* buffer, size_t size)
{
if (size == 0)
return;
if (io->dstFile == NULL) {
assert(io->prefs->testMode);
return;
}
io->storedSkips = FIO_sparseWrite(io->dstFile, buffer, size, io->prefs, io->storedSkips);
}
static void FIO_SyncCompressIO_finish(FIO_SyncCompressIO* io)
{
if (io->dstFile == NULL)
return;
FIO_sparseWriteEnd(io->prefs, io->dstFile, io->storedSkips);
io->storedSkips = 0;
}
#endif /* ZSTD_NOCOMPRESS */
/*-************************************
* Signal (Ctrl-C trapping)
**************************************/
@@ -553,7 +288,7 @@ FIO_prefs_t* FIO_createPreferences(void)
ret->removeSrcFile = 0;
ret->memLimit = 0;
ret->nbWorkers = 1;
ret->jobSize = 0;
ret->blockSize = 0;
ret->overlapLog = FIO_OVERLAP_LOG_NOTSET;
ret->adaptiveMode = 0;
ret->rsyncable = 0;
@@ -642,10 +377,10 @@ void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompresse
void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; }
void FIO_setJobSize(FIO_prefs_t* const prefs, int jobSize) {
if (jobSize && prefs->nbWorkers==0)
void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
if (blockSize && prefs->nbWorkers==0)
DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
prefs->jobSize = jobSize;
prefs->blockSize = blockSize;
}
void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
@@ -688,7 +423,7 @@ void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) {
void FIO_setLiteralCompressionMode(
FIO_prefs_t* const prefs,
ZSTD_ParamSwitch_e mode) {
ZSTD_paramSwitch_e mode) {
prefs->literalCompressionMode = mode;
}
@@ -750,7 +485,7 @@ void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
prefs->passThrough = (value != 0);
}
void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_ParamSwitch_e value)
void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_paramSwitch_e value)
{
prefs->mmapDict = value;
}
@@ -803,16 +538,14 @@ static int FIO_removeFile(const char* path)
}
/** FIO_openSrcFile() :
* condition : `srcFileName` must be non-NULL.
* optional: `prefs` may be NULL.
* condition : `srcFileName` must be non-NULL. `prefs` may be NULL.
* @result : FILE* to `srcFileName`, or NULL if it fails */
static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName, stat_t* statbuf)
{
int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0;
assert(srcFileName != NULL);
assert(statbuf != NULL);
if (!strcmp(srcFileName, stdinmark)) {
if (!strcmp (srcFileName, stdinmark)) {
DISPLAYLEVEL(4,"Using stdin for input \n");
SET_BINARY_MODE(stdin);
return stdin;
@@ -824,10 +557,8 @@ static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFile
return NULL;
}
/* Accept regular files, FIFOs, and process substitution file descriptors */
if (!UTIL_isRegularFileStat(statbuf)
&& !UTIL_isFIFOStat(statbuf)
&& !UTIL_isFileDescriptorPipe(srcFileName) /* Process substitution support */
&& !(allowBlockDevices && UTIL_isBlockDevStat(statbuf))
) {
DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n",
@@ -850,6 +581,8 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
const char* srcFileName, const char* dstFileName,
const int mode)
{
int isDstRegFile;
if (prefs->testMode) return NULL; /* do not open file in test mode */
assert(dstFileName != NULL);
@@ -869,7 +602,16 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
return NULL;
}
if (UTIL_isRegularFile(dstFileName)) {
isDstRegFile = UTIL_isRegularFile(dstFileName); /* invoke once */
if (prefs->sparseFileSupport == 1) {
prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
if (!isDstRegFile) {
prefs->sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is disabled when output is not a file \n");
}
}
if (isDstRegFile) {
/* Check if destination file already exists */
#if !defined(_WIN32)
/* this test does not work on Windows :
@@ -895,7 +637,6 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
}
{
int isDstRegFile;
#if defined(_WIN32)
/* Windows requires opening the file as a "binary" file to avoid
* mangling. This macro doesn't exist on unix. */
@@ -913,23 +654,8 @@ FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
f = fdopen(fd, "wb");
}
#endif
/* Check regular file after opening with O_CREAT */
isDstRegFile = UTIL_isFdRegularFile(fd);
if (prefs->sparseFileSupport == 1) {
prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
if (!isDstRegFile) {
prefs->sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is disabled when output is not a file \n");
}
}
if (f == NULL) {
if (UTIL_isFileDescriptorPipe(dstFileName)) {
DISPLAYLEVEL(1, "zstd: error: no output specified (use -o or -c). \n");
} else {
DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno));
}
} else {
/* An increased buffer size can provide a significant performance
* boost on some platforms. Note that providing a NULL buf with a
@@ -1050,9 +776,7 @@ static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_
}
*bufferPtr = mmap(NULL, (size_t)fileSize, PROT_READ, MAP_PRIVATE, fileHandle, 0);
if (*bufferPtr == MAP_FAILED) {
EXM_THROW(34, "%s", strerror(errno))
}
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
close(fileHandle);
return (size_t)fileSize;
@@ -1345,7 +1069,8 @@ typedef struct {
const char* dictFileName;
stat_t dictFileStat;
ZSTD_CStream* cctx;
FIO_SyncCompressIO io;
WritePoolCtx_t *writeCtx;
ReadPoolCtx_t *readCtx;
} cRess_t;
/** ZSTD_cycleLog() :
@@ -1375,12 +1100,11 @@ static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
FIO_setLdmFlag(prefs, 1);
}
if (cParams.strategy >= ZSTD_btopt) {
DISPLAYLEVEL(4, "[Optimal parser notes] Consider the following to improve patch size at the cost of speed:\n");
DISPLAYLEVEL(4, "- Set a larger targetLength (e.g. --zstd=targetLength=4096)\n");
DISPLAYLEVEL(4, "- Set a larger chainLog (e.g. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX);
DISPLAYLEVEL(4, "- Set a larger LDM hashLog (e.g. --zstd=ldmHashLog=%u)\n", ZSTD_LDM_HASHLOG_MAX);
DISPLAYLEVEL(4, "- Set a smaller LDM rateLog (e.g. --zstd=ldmHashRateLog=%u)\n", ZSTD_LDM_HASHRATELOG_MIN);
DISPLAYLEVEL(4, "Also consider playing around with searchLog and hashLog\n");
DISPLAYLEVEL(3, "[Optimal parser notes] Consider the following to improve patch size at the cost of speed:\n");
DISPLAYLEVEL(3, "- Use --single-thread mode in the zstd cli\n");
DISPLAYLEVEL(3, "- Set a larger targetLength (e.g. --zstd=targetLength=4096)\n");
DISPLAYLEVEL(3, "- Set a larger chainLog (e.g. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX);
DISPLAYLEVEL(3, "Also consider playing around with searchLog and hashLog\n");
}
}
@@ -1413,7 +1137,8 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
FIO_initDict(&ress.dict, dictFileName, prefs, &ress.dictFileStat, dictBufferType); /* works with dictFileName==NULL */
FIO_SyncCompressIO_init(&ress.io, prefs, ZSTD_CStreamInSize(), ZSTD_CStreamOutSize());
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
/* Advanced parameters, including dictionary */
if (dictFileName && (ress.dict.dictBuffer==NULL))
@@ -1457,7 +1182,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
#ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->jobSize) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) );
if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) {
DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) );
@@ -1477,20 +1202,21 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
static void FIO_freeCResources(cRess_t* const ress)
{
FIO_freeDict(&(ress->dict));
FIO_SyncCompressIO_destroy(&ress->io);
AIO_WritePool_free(ress->writeCtx);
AIO_ReadPool_free(ress->readCtx);
ZSTD_freeCStream(ress->cctx); /* never fails */
}
#ifdef ZSTD_GZCOMPRESS
static unsigned long long
FIO_compressGzFrame(cRess_t* ress,
FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but not changed */
const char* srcFileName, U64 const srcFileSize,
int compressionLevel, U64* readsize)
{
FIO_SyncCompressIO* const syncIO = &ress->io;
unsigned long long inFileSize = 0, outFileSize = 0;
z_stream strm;
IOJob_t *writeJob = NULL;
if (compressionLevel > Z_BEST_COMPRESSION)
compressionLevel = Z_BEST_COMPRESSION;
@@ -1506,36 +1232,37 @@ FIO_compressGzFrame(cRess_t* ress,
EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
} }
writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
strm.next_in = 0;
strm.avail_in = 0;
strm.next_out = (Bytef*)syncIO->outBuffer;
strm.avail_out = (uInt)syncIO->outCapacity;
strm.next_out = (Bytef*)writeJob->buffer;
strm.avail_out = (uInt)writeJob->bufferSize;
while (1) {
int ret;
if (strm.avail_in == 0) {
size_t const added = FIO_SyncCompressIO_fillBuffer(syncIO, ZSTD_CStreamInSize());
if (syncIO->srcBufferLoaded == 0) break;
inFileSize += added;
*readsize += added;
strm.next_in = (z_const unsigned char*)syncIO->srcBuffer;
strm.avail_in = (uInt)syncIO->srcBufferLoaded;
AIO_ReadPool_fillBuffer(ress->readCtx, ZSTD_CStreamInSize());
if (ress->readCtx->srcBufferLoaded == 0) break;
inFileSize += ress->readCtx->srcBufferLoaded;
strm.next_in = (z_const unsigned char*)ress->readCtx->srcBuffer;
strm.avail_in = (uInt)ress->readCtx->srcBufferLoaded;
}
{
size_t const availBefore = strm.avail_in;
ret = deflate(&strm, Z_NO_FLUSH);
FIO_SyncCompressIO_consumeBytes(syncIO, availBefore - strm.avail_in);
AIO_ReadPool_consumeBytes(ress->readCtx, availBefore - strm.avail_in);
}
if (ret != Z_OK)
EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret);
{ size_t const cSize = (size_t)((uInt)syncIO->outCapacity - strm.avail_out);
{ size_t const cSize = writeJob->bufferSize - strm.avail_out;
if (cSize) {
FIO_SyncCompressIO_commitOut(syncIO, syncIO->outBuffer, cSize);
writeJob->usedBufferSize = cSize;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
outFileSize += cSize;
strm.next_out = (Bytef*)syncIO->outBuffer;
strm.avail_out = (uInt)syncIO->outCapacity;
strm.next_out = (Bytef*)writeJob->buffer;
strm.avail_out = (uInt)writeJob->bufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
DISPLAYUPDATE_PROGRESS(
@@ -1551,12 +1278,13 @@ FIO_compressGzFrame(cRess_t* ress,
while (1) {
int const ret = deflate(&strm, Z_FINISH);
{ size_t const cSize = (size_t)((uInt)syncIO->outCapacity - strm.avail_out);
{ size_t const cSize = writeJob->bufferSize - strm.avail_out;
if (cSize) {
FIO_SyncCompressIO_commitOut(syncIO, syncIO->outBuffer, cSize);
writeJob->usedBufferSize = cSize;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
outFileSize += cSize;
strm.next_out = (Bytef*)syncIO->outBuffer;
strm.avail_out = (uInt)syncIO->outCapacity;
strm.next_out = (Bytef*)writeJob->buffer;
strm.avail_out = (uInt)writeJob->bufferSize;
} }
if (ret == Z_STREAM_END) break;
if (ret != Z_BUF_ERROR)
@@ -1568,7 +1296,8 @@ FIO_compressGzFrame(cRess_t* ress,
EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
} }
*readsize = inFileSize;
FIO_SyncCompressIO_finish(syncIO);
AIO_WritePool_releaseIoJob(writeJob);
AIO_WritePool_sparseWriteEnd(ress->writeCtx);
return outFileSize;
}
#endif
@@ -1580,11 +1309,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
const char* srcFileName, U64 const srcFileSize,
int compressionLevel, U64* readsize, int plain_lzma)
{
FIO_SyncCompressIO* const syncIO = &ress->io;
unsigned long long inFileSize = 0, outFileSize = 0;
lzma_stream strm = LZMA_STREAM_INIT;
lzma_action action = LZMA_RUN;
lzma_ret ret;
IOJob_t *writeJob = NULL;
if (compressionLevel < 0) compressionLevel = 0;
if (compressionLevel > 9) compressionLevel = 9;
@@ -1602,35 +1331,37 @@ FIO_compressLzmaFrame(cRess_t* ress,
EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
}
strm.next_out = (BYTE*)syncIO->outBuffer;
strm.avail_out = syncIO->outCapacity;
writeJob =AIO_WritePool_acquireJob(ress->writeCtx);
strm.next_out = (BYTE*)writeJob->buffer;
strm.avail_out = writeJob->bufferSize;
strm.next_in = 0;
strm.avail_in = 0;
while (1) {
if (strm.avail_in == 0) {
size_t const added = FIO_SyncCompressIO_fillBuffer(syncIO, ZSTD_CStreamInSize());
if (syncIO->srcBufferLoaded == 0) action = LZMA_FINISH;
inFileSize += added;
*readsize += added;
strm.next_in = (BYTE const*)syncIO->srcBuffer;
strm.avail_in = syncIO->srcBufferLoaded;
size_t const inSize = AIO_ReadPool_fillBuffer(ress->readCtx, ZSTD_CStreamInSize());
if (ress->readCtx->srcBufferLoaded == 0) action = LZMA_FINISH;
inFileSize += inSize;
strm.next_in = (BYTE const*)ress->readCtx->srcBuffer;
strm.avail_in = ress->readCtx->srcBufferLoaded;
}
{
size_t const availBefore = strm.avail_in;
ret = lzma_code(&strm, action);
FIO_SyncCompressIO_consumeBytes(syncIO, availBefore - strm.avail_in);
AIO_ReadPool_consumeBytes(ress->readCtx, availBefore - strm.avail_in);
}
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
{ size_t const compBytes = syncIO->outCapacity - strm.avail_out;
{ size_t const compBytes = writeJob->bufferSize - strm.avail_out;
if (compBytes) {
FIO_SyncCompressIO_commitOut(syncIO, syncIO->outBuffer, compBytes);
writeJob->usedBufferSize = compBytes;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
outFileSize += compBytes;
strm.next_out = (BYTE*)syncIO->outBuffer;
strm.avail_out = syncIO->outCapacity;
strm.next_out = (BYTE*)writeJob->buffer;
strm.avail_out = writeJob->bufferSize;
} }
if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
DISPLAYUPDATE_PROGRESS("\rRead : %u MB ==> %.2f%%",
@@ -1646,7 +1377,8 @@ FIO_compressLzmaFrame(cRess_t* ress,
lzma_end(&strm);
*readsize = inFileSize;
FIO_SyncCompressIO_finish(syncIO);
AIO_WritePool_releaseIoJob(writeJob);
AIO_WritePool_sparseWriteEnd(ress->writeCtx);
return outFileSize;
}
@@ -1667,20 +1399,21 @@ FIO_compressLz4Frame(cRess_t* ress,
int compressionLevel, int checksumFlag,
U64* readsize)
{
FIO_SyncCompressIO* const syncIO = &ress->io;
const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB);
unsigned long long inFileSize = 0, outFileSize = 0;
LZ4F_preferences_t prefs;
LZ4F_compressionContext_t ctx;
IOJob_t* writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
if (LZ4F_isError(errorCode))
EXM_THROW(31, "zstd: failed to create lz4 compression context");
memset(&prefs, 0, sizeof(prefs));
assert(blockSize <= syncIO->inCapacity);
assert(blockSize <= ress->readCtx->base.jobBufferSize);
/* autoflush off to mitigate a bug in lz4<=1.9.3 for compression level 12 */
prefs.autoFlush = 0;
@@ -1691,26 +1424,25 @@ FIO_compressLz4Frame(cRess_t* ress,
#if LZ4_VERSION_NUMBER >= 10600
prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize;
#endif
assert(LZ4F_compressBound(blockSize, &prefs) <= syncIO->outCapacity);
assert(LZ4F_compressBound(blockSize, &prefs) <= writeJob->bufferSize);
{
size_t headerSize = LZ4F_compressBegin(ctx, syncIO->outBuffer, syncIO->outCapacity, &prefs);
size_t headerSize = LZ4F_compressBegin(ctx, writeJob->buffer, writeJob->bufferSize, &prefs);
if (LZ4F_isError(headerSize))
EXM_THROW(33, "File header generation failed : %s",
LZ4F_getErrorName(headerSize));
FIO_SyncCompressIO_commitOut(syncIO, syncIO->outBuffer, headerSize);
writeJob->usedBufferSize = headerSize;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
outFileSize += headerSize;
{
size_t const added = FIO_SyncCompressIO_fillBuffer(syncIO, blockSize);
inFileSize += added;
*readsize += added;
}
/* Read first block */
inFileSize += AIO_ReadPool_fillBuffer(ress->readCtx, blockSize);
while (syncIO->srcBufferLoaded) {
size_t const inSize = MIN(blockSize, syncIO->srcBufferLoaded);
size_t const outSize = LZ4F_compressUpdate(ctx, syncIO->outBuffer, syncIO->outCapacity,
syncIO->srcBuffer, inSize, NULL);
/* Main Loop */
while (ress->readCtx->srcBufferLoaded) {
size_t inSize = MIN(blockSize, ress->readCtx->srcBufferLoaded);
size_t const outSize = LZ4F_compressUpdate(ctx, writeJob->buffer, writeJob->bufferSize,
ress->readCtx->srcBuffer, inSize, NULL);
if (LZ4F_isError(outSize))
EXM_THROW(35, "zstd: %s: lz4 compression failed : %s",
srcFileName, LZ4F_getErrorName(outSize));
@@ -1725,27 +1457,30 @@ FIO_compressLz4Frame(cRess_t* ress,
(double)outFileSize/(double)inFileSize*100);
}
FIO_SyncCompressIO_commitOut(syncIO, syncIO->outBuffer, outSize);
/* Write Block */
writeJob->usedBufferSize = outSize;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
FIO_SyncCompressIO_consumeBytes(syncIO, inSize);
{
size_t const added = FIO_SyncCompressIO_fillBuffer(syncIO, blockSize);
inFileSize += added;
*readsize += added;
}
/* Read next block */
AIO_ReadPool_consumeBytes(ress->readCtx, inSize);
inFileSize += AIO_ReadPool_fillBuffer(ress->readCtx, blockSize);
}
headerSize = LZ4F_compressEnd(ctx, syncIO->outBuffer, syncIO->outCapacity, NULL);
/* End of Stream mark */
headerSize = LZ4F_compressEnd(ctx, writeJob->buffer, writeJob->bufferSize, NULL);
if (LZ4F_isError(headerSize))
EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s",
srcFileName, LZ4F_getErrorName(headerSize));
FIO_SyncCompressIO_commitOut(syncIO, syncIO->outBuffer, headerSize);
writeJob->usedBufferSize = headerSize;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
outFileSize += headerSize;
}
*readsize = inFileSize;
LZ4F_freeCompressionContext(ctx);
FIO_SyncCompressIO_finish(syncIO);
AIO_WritePool_releaseIoJob(writeJob);
AIO_WritePool_sparseWriteEnd(ress->writeCtx);
return outFileSize;
}
@@ -1754,11 +1489,12 @@ FIO_compressLz4Frame(cRess_t* ress,
static unsigned long long
FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
cRess_t* ress,
const cRess_t* ressPtr,
const char* srcFileName, U64 fileSize,
int compressionLevel, U64* readsize)
{
FIO_SyncCompressIO* const syncIO = &ress->io;
cRess_t const ress = *ressPtr;
IOJob_t *writeJob = AIO_WritePool_acquireJob(ressPtr->writeCtx);
U64 compressedfilesize = 0;
ZSTD_EndDirective directive = ZSTD_e_continue;
@@ -1783,16 +1519,17 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
/* init */
if (fileSize != UTIL_FILESIZE_UNKNOWN) {
pledgedSrcSize = fileSize;
CHECK(ZSTD_CCtx_setPledgedSrcSize(ress->cctx, fileSize));
CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize));
} else if (prefs->streamSrcSize > 0) {
/* unknown source size; use the declared stream size */
pledgedSrcSize = prefs->streamSrcSize;
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress->cctx, prefs->streamSrcSize) );
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) );
}
{ int windowLog;
{
int windowLog;
UTIL_HumanReadableSize_t windowSize;
CHECK(ZSTD_CCtx_getParameter(ress->cctx, ZSTD_c_windowLog, &windowLog));
CHECK(ZSTD_CCtx_getParameter(ress.cctx, ZSTD_c_windowLog, &windowLog));
if (windowLog == 0) {
if (prefs->ldmFlag) {
/* If long mode is set without a window size libzstd will set this size internally */
@@ -1805,17 +1542,18 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
windowSize = UTIL_makeHumanReadableSize(MAX(1ULL, MIN(1ULL << windowLog, pledgedSrcSize)));
DISPLAYLEVEL(4, "Decompression will require %.*f%s of memory\n", windowSize.precision, windowSize.value, windowSize.suffix);
}
(void)srcFileName;
/* Main compression loop */
do {
size_t stillToFlush;
/* Fill input Buffer */
size_t const inSize = FIO_SyncCompressIO_fillBuffer(syncIO, ZSTD_CStreamInSize());
ZSTD_inBuffer inBuff = setInBuffer( syncIO->srcBuffer, syncIO->srcBufferLoaded, 0 );
size_t const inSize = AIO_ReadPool_fillBuffer(ress.readCtx, ZSTD_CStreamInSize());
ZSTD_inBuffer inBuff = setInBuffer( ress.readCtx->srcBuffer, ress.readCtx->srcBufferLoaded, 0 );
DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize);
*readsize += inSize;
if ((syncIO->srcBufferLoaded == 0) || (*readsize == fileSize))
if ((ress.readCtx->srcBufferLoaded == 0) || (*readsize == fileSize))
directive = ZSTD_e_end;
stillToFlush = 1;
@@ -1823,10 +1561,10 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
|| (directive == ZSTD_e_end && stillToFlush != 0) ) {
size_t const oldIPos = inBuff.pos;
ZSTD_outBuffer outBuff = setOutBuffer( syncIO->outBuffer, syncIO->outCapacity, 0 );
size_t const toFlushNow = ZSTD_toFlushNow(ress->cctx);
CHECK_V(stillToFlush, ZSTD_compressStream2(ress->cctx, &outBuff, &inBuff, directive));
FIO_SyncCompressIO_consumeBytes(syncIO, inBuff.pos - oldIPos);
ZSTD_outBuffer outBuff = setOutBuffer( writeJob->buffer, writeJob->bufferSize, 0 );
size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx);
CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive));
AIO_ReadPool_consumeBytes(ress.readCtx, inBuff.pos - oldIPos);
/* count stats */
inputPresented++;
@@ -1837,13 +1575,14 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n",
(unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos);
if (outBuff.pos) {
FIO_SyncCompressIO_commitOut(syncIO, syncIO->outBuffer, outBuff.pos);
writeJob->usedBufferSize = outBuff.pos;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
compressedfilesize += outBuff.pos;
}
/* adaptive mode : statistics measurement and speed correction */
if (prefs->adaptiveMode && UTIL_clockSpanMicro(lastAdaptTime) > adaptEveryMicro) {
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress->cctx);
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
lastAdaptTime = UTIL_getTime();
@@ -1916,14 +1655,14 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel;
compressionLevel += (compressionLevel == 0); /* skip 0 */
ZSTD_CCtx_setParameter(ress->cctx, ZSTD_c_compressionLevel, compressionLevel);
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
if (speedChange == faster) {
DISPLAYLEVEL(6, "faster speed , lighter compression \n")
compressionLevel --;
if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel;
compressionLevel -= (compressionLevel == 0); /* skip 0 */
ZSTD_CCtx_setParameter(ress->cctx, ZSTD_c_compressionLevel, compressionLevel);
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
}
speedChange = noChange;
@@ -1933,7 +1672,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
/* display notification */
if (SHOULD_DISPLAY_PROGRESS() && READY_FOR_UPDATE()) {
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress->cctx);
ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx);
double const cShare = (double)zfp.produced / (double)(zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
UTIL_HumanReadableSize_t const buffered_hrs = UTIL_makeHumanReadableSize(zfp.ingested - zfp.consumed);
UTIL_HumanReadableSize_t const consumed_hrs = UTIL_makeHumanReadableSize(zfp.consumed);
@@ -1980,7 +1719,8 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
(unsigned long long)*readsize, (unsigned long long)fileSize);
}
FIO_SyncCompressIO_finish(syncIO);
AIO_WritePool_releaseIoJob(writeJob);
AIO_WritePool_sparseWriteEnd(ressPtr->writeCtx);
return compressedfilesize;
}
@@ -1993,7 +1733,7 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
static int
FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
cRess_t* ress,
cRess_t ress,
const char* dstFileName, const char* srcFileName,
int compressionLevel)
{
@@ -2008,12 +1748,12 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
switch (prefs->compressionType) {
default:
case FIO_zstdCompression:
compressedfilesize = FIO_compressZstdFrame(fCtx, prefs, ress, srcFileName, fileSize, compressionLevel, &readsize);
compressedfilesize = FIO_compressZstdFrame(fCtx, prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize);
break;
case FIO_gzipCompression:
#ifdef ZSTD_GZCOMPRESS
compressedfilesize = FIO_compressGzFrame(ress, srcFileName, fileSize, compressionLevel, &readsize);
compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
#else
(void)compressionLevel;
EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n",
@@ -2024,7 +1764,7 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
case FIO_xzCompression:
case FIO_lzmaCompression:
#ifdef ZSTD_LZMACOMPRESS
compressedfilesize = FIO_compressLzmaFrame(ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO_lzmaCompression);
compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO_lzmaCompression);
#else
(void)compressionLevel;
EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n",
@@ -2034,7 +1774,7 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
case FIO_lz4Compression:
#ifdef ZSTD_LZ4COMPRESS
compressedfilesize = FIO_compressLz4Frame(ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &readsize);
compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &readsize);
#else
(void)compressionLevel;
EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n",
@@ -2090,7 +1830,7 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
*/
static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
cRess_t* ress,
cRess_t ress,
const char* dstFileName,
const char* srcFileName,
const stat_t* srcFileStat,
@@ -2101,7 +1841,8 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
int transferStat = 0;
int dstFd = -1;
if (ress->io.dstFile == NULL) {
assert(AIO_ReadPool_getFile(ress.readCtx) != NULL);
if (AIO_WritePool_getFile(ress.writeCtx) == NULL) {
int dstFileInitialPermissions = DEFAULT_FILE_PERMISSIONS;
if ( strcmp (srcFileName, stdinmark)
&& strcmp (dstFileName, stdoutmark)
@@ -2112,13 +1853,15 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
closeDstFile = 1;
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName);
{
FILE *dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFileInitialPermissions);
{ FILE *dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFileInitialPermissions);
if (dstFile==NULL) return 1; /* could not open dstFileName */
dstFd = fileno(dstFile);
FIO_SyncCompressIO_setDst(&ress->io, dstFile);
AIO_WritePool_setFile(ress.writeCtx, dstFile);
}
/* Must only be added after FIO_openDstFile() succeeds. */
/* Must only be added after FIO_openDstFile() succeeds.
* Otherwise we may delete the destination file if it already exists,
* and the user presses Ctrl-C when asked if they wish to overwrite.
*/
addHandler(dstFileName);
}
@@ -2132,7 +1875,7 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
}
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName);
if (FIO_SyncCompressIO_closeDst(&ress->io)) { /* error closing file */
if (AIO_WritePool_closeFile(ress.writeCtx)) { /* error closing file */
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result=1;
}
@@ -2141,9 +1884,10 @@ static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
UTIL_utime(dstFileName, srcFileStat);
}
if ( (result != 0)
&& strcmp(dstFileName, stdoutmark) ) {
FIO_removeFile(dstFileName);
if ( (result != 0) /* operation failure */
&& strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */
) {
FIO_removeFile(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */
}
}
@@ -2277,7 +2021,7 @@ static const char *compressedFileExtensions[] = {
static int
FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
cRess_t* ress,
cRess_t ress,
const char* dstFileName,
const char* srcFileName,
int compressionLevel)
@@ -2299,7 +2043,7 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
}
/* ensure src is not the same as dict (if present) */
if (ress->dictFileName != NULL && UTIL_isSameFileStat(srcFileName, ress->dictFileName, &srcFileStat, &ress->dictFileStat)) {
if (ress.dictFileName != NULL && UTIL_isSameFileStat(srcFileName, ress.dictFileName, &srcFileStat, &ress.dictFileStat)) {
DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
return 1;
}
@@ -2318,21 +2062,23 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
if (srcFile == NULL) return 1; /* srcFile could not be opened */
/* Don't use AsyncIO for small files */
if (strcmp(srcFileName, stdinmark)) /* Stdin doesn't have stats */
fileSize = UTIL_getFileSizeStat(&srcFileStat);
(void)fileSize;
if(fileSize != UTIL_FILESIZE_UNKNOWN && fileSize < ZSTD_BLOCKSIZE_MAX * 3) {
AIO_ReadPool_setAsync(ress.readCtx, 0);
AIO_WritePool_setAsync(ress.writeCtx, 0);
} else {
AIO_ReadPool_setAsync(ress.readCtx, 1);
AIO_WritePool_setAsync(ress.writeCtx, 1);
}
FIO_SyncCompressIO_setSrc(&ress->io, srcFile);
AIO_ReadPool_setFile(ress.readCtx, srcFile);
result = FIO_compressFilename_dstFile(
fCtx, prefs, ress,
dstFileName, srcFileName,
&srcFileStat, compressionLevel);
FIO_SyncCompressIO_clearSrc(&ress->io);
if (srcFile != NULL && fclose(srcFile)) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
return 1;
}
AIO_ReadPool_closeFile(ress.readCtx);
if ( prefs->removeSrcFile /* --rm */
&& result == 0 /* success */
@@ -2373,7 +2119,7 @@ void FIO_displayCompressionParameters(const FIO_prefs_t* prefs)
DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport));
DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID");
DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag));
DISPLAY(" --jobsize=%d", prefs->jobSize);
DISPLAY(" --block-size=%d", prefs->blockSize);
if (prefs->adaptiveMode)
DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel);
DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder));
@@ -2399,11 +2145,9 @@ int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const
int compressionLevel, ZSTD_compressionParameters comprParams)
{
cRess_t ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
int const result = FIO_compressFilename_srcFile(fCtx, prefs, &ress, dstFileName, srcFileName, compressionLevel);
int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
#ifndef ZSTD_DISPLAY_LEVEL_DEFAULT
# define ZSTD_DISPLAY_LEVEL_DEFAULT 2
#endif
#define DISPLAY_LEVEL_DEFAULT 2
FIO_freeCResources(&ress);
return result;
@@ -2498,13 +2242,13 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
if (dstFile == NULL) { /* could not open outFileName */
error = 1;
} else {
FIO_SyncCompressIO_setDst(&ress.io, dstFile);
AIO_WritePool_setFile(ress.writeCtx, dstFile);
for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) {
status = FIO_compressFilename_srcFile(fCtx, prefs, &ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel);
status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel);
if (!status) fCtx->nbFilesProcessed++;
error |= status;
}
if (FIO_SyncCompressIO_closeDst(&ress.io))
if (AIO_WritePool_closeFile(ress.writeCtx))
EXM_THROW(29, "Write error (%s) : cannot properly close %s",
strerror(errno), outFileName);
}
@@ -2528,7 +2272,7 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
} else {
dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */
}
status = FIO_compressFilename_srcFile(fCtx, prefs, &ress, dstFileName, srcFileName, compressionLevel);
status = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
if (!status) fCtx->nbFilesProcessed++;
error |= status;
}
@@ -2659,7 +2403,7 @@ FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
size_t err,
const char* srcFileName)
{
ZSTD_FrameHeader header;
ZSTD_frameHeader header;
/* Help message only for one specific error */
if (ZSTD_getErrorCode(err) != ZSTD_error_frameParameter_windowTooLarge)
@@ -2695,14 +2439,12 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
U64 alreadyDecoded) /* for multi-frames streams */
{
U64 frameSize = 0;
const char* srcFName20 = srcFileName;
IOJob_t* writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
assert(writeJob);
IOJob_t *writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
/* display last 20 characters only when not --verbose */
{ size_t const srcFileLength = strlen(srcFileName);
if ((srcFileLength>20) && (g_display_prefs.displayLevel<3))
srcFName20 += srcFileLength-20;
srcFileName += srcFileLength-20;
}
ZSTD_DCtx_reset(ress->dctx, ZSTD_reset_session_only);
@@ -2729,12 +2471,19 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
frameSize += outBuff.pos;
if (fCtx->nbFilesTotal > 1) {
size_t srcFileNameSize = strlen(srcFileName);
if (srcFileNameSize > 18) {
const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15;
DISPLAYUPDATE_PROGRESS(
"\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFName20, hrs.precision, hrs.value, hrs.suffix);
"\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix);
} else {
DISPLAYUPDATE_PROGRESS("\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix);
}
} else {
DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s... ",
srcFName20, hrs.precision, hrs.value, hrs.suffix);
srcFileName, hrs.precision, hrs.value, hrs.suffix);
}
AIO_ReadPool_consumeBytes(ress->readCtx, inBuff.pos);
@@ -3459,7 +3208,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
{ U32 const magicNumber = MEM_readLE32(headerBuffer);
/* Zstandard frame */
if (magicNumber == ZSTD_MAGICNUMBER) {
ZSTD_FrameHeader header;
ZSTD_frameHeader header;
U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead);
if ( frameContentSize == ZSTD_CONTENTSIZE_ERROR
|| frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ) {
+13 -3
View File
@@ -17,6 +17,11 @@
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "../lib/zstd.h" /* ZSTD_* */
#if defined (__cplusplus)
extern "C" {
#endif
/* *************************************
* Special i/o constants
**************************************/
@@ -70,7 +75,7 @@ void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt);
void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder);
void FIO_setJobSize(FIO_prefs_t* const prefs, int jobSize);
void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize);
void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag);
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag);
void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog);
@@ -90,7 +95,7 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint);
void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode);
void FIO_setLiteralCompressionMode(
FIO_prefs_t* const prefs,
ZSTD_ParamSwitch_e mode);
ZSTD_paramSwitch_e mode);
void FIO_setProgressSetting(FIO_progressSetting_e progressSetting);
void FIO_setNotificationLevel(int level);
@@ -101,7 +106,7 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs);
void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value);
void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value);
void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_ParamSwitch_e value);
void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_paramSwitch_e value);
/* FIO_ctx_t functions */
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);
@@ -168,4 +173,9 @@ char const* FIO_zlibVersion(void);
char const* FIO_lz4Version(void);
char const* FIO_lzmaVersion(void);
#if defined (__cplusplus)
}
#endif
#endif /* FILEIO_H_23981798732 */
+1 -1
View File
@@ -268,7 +268,7 @@ static void AIO_IOPool_destroy(IOPoolCtx_t* ctx) {
/* AIO_IOPool_acquireJob:
* Returns an available io job to be used for a future io. */
static IOJob_t* AIO_IOPool_acquireJob(IOPoolCtx_t* ctx) {
IOJob_t* job;
IOJob_t *job;
assert(ctx->file != NULL || ctx->prefs->testMode);
AIO_IOPool_lockJobsMutex(ctx);
assert(ctx->availableJobsCount > 0);
+8
View File
@@ -22,6 +22,10 @@
#ifndef ZSTD_FILEIO_ASYNCIO_H
#define ZSTD_FILEIO_ASYNCIO_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "../lib/common/mem.h" /* U32, U64 */
#include "fileio_types.h"
#include "platform.h"
@@ -192,4 +196,8 @@ FILE* AIO_ReadPool_getFile(const ReadPoolCtx_t *ctx);
* Closes the current set file. Waits for all current enqueued tasks to complete and resets state. */
int AIO_ReadPool_closeFile(ReadPoolCtx_t *ctx);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_FILEIO_ASYNCIO_H */
+14 -10
View File
@@ -11,6 +11,10 @@
#ifndef ZSTD_FILEIO_COMMON_H
#define ZSTD_FILEIO_COMMON_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "../lib/common/mem.h" /* U32, U64 */
#include "fileio_types.h"
#include "platform.h"
@@ -24,14 +28,11 @@
#define GB *(1U<<30)
#undef MAX
#define MAX(a,b) ((a)>(b) ? (a) : (b))
#undef MIN /* in case it would be already defined */
#define MIN(a,b) ((a) < (b) ? (a) : (b))
extern FIO_display_prefs_t g_display_prefs;
#define DISPLAY_F(f, ...) fprintf((f), __VA_ARGS__)
#define DISPLAYOUT(...) DISPLAY_F(stdout, __VA_ARGS__)
#define DISPLAY(...) DISPLAY_F(stderr, __VA_ARGS__)
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
extern UTIL_time_t g_displayClock;
@@ -55,6 +56,10 @@ extern UTIL_time_t g_displayClock;
#define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }}
#define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } }
#undef MIN /* in case it would be already defined */
#define MIN(a,b) ((a) < (b) ? (a) : (b))
#define EXM_THROW(error, ...) \
{ \
DISPLAYLEVEL(1, "zstd: "); \
@@ -75,11 +80,7 @@ extern UTIL_time_t g_displayClock;
/* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW */
#if defined(LIBC_NO_FSEEKO)
/* Some older libc implementations don't include these functions (e.g. Bionic < 24) */
# define LONG_SEEK fseek
# define LONG_TELL ftell
#elif defined(_MSC_VER) && _MSC_VER >= 1400
#if defined(_MSC_VER) && _MSC_VER >= 1400
# define LONG_SEEK _fseeki64
# define LONG_TELL _ftelli64
#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
@@ -118,4 +119,7 @@ extern UTIL_time_t g_displayClock;
# define LONG_TELL ftell
#endif
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_FILEIO_COMMON_H */
+3 -3
View File
@@ -37,7 +37,7 @@ typedef struct FIO_prefs_s {
int sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
int dictIDFlag;
int checksumFlag;
int jobSize;
int blockSize;
int overlapLog;
int adaptiveMode;
int useRowMatchFinder;
@@ -53,7 +53,7 @@ typedef struct FIO_prefs_s {
size_t targetCBlockSize;
int srcSizeHint;
int testMode;
ZSTD_ParamSwitch_e literalCompressionMode;
ZSTD_paramSwitch_e literalCompressionMode;
/* IO preferences */
int removeSrcFile;
@@ -69,7 +69,7 @@ typedef struct FIO_prefs_s {
int contentSize;
int allowBlockDevices;
int passThrough;
ZSTD_ParamSwitch_e mmapDict;
ZSTD_paramSwitch_e mmapDict;
} FIO_prefs_t;
typedef enum {FIO_mallocDict, FIO_mmapDict} FIO_dictBufferType_t;
+12 -12
View File
@@ -11,6 +11,12 @@
#ifndef PLATFORM_H_MODULE
#define PLATFORM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* **************************************
* Compiler Options
****************************************/
@@ -32,10 +38,9 @@
#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \
|| defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \
|| (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \
|| defined __x86_64__ || defined _M_X64 /* x86 64-bit */ \
|| defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \
|| defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \
|| (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \
|| (defined(__riscv) && __riscv_xlen == 64) /* RISC-V 64-bit */ \
|| defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \
|| (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */
# if !defined(__64BIT__)
@@ -139,20 +144,10 @@
# include <io.h> /* _isatty */
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
# include <stdio.h> /* FILE */
#if defined (__cplusplus)
extern "C" {
#endif
static __inline int IS_CONSOLE(FILE* stdStream) {
DWORD dummy;
return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy);
}
#if defined (__cplusplus)
}
#endif
#else
# define IS_CONSOLE(stdStream) 0
#endif
@@ -215,4 +210,9 @@ static __inline int IS_CONSOLE(FILE* stdStream) {
# endif
#endif
#if defined (__cplusplus)
}
#endif
#endif /* PLATFORM_H_MODULE */
+1 -3
View File
@@ -28,20 +28,18 @@
UTIL_time_t UTIL_getTime(void)
{
static LARGE_INTEGER ticksPerSecond;
static double nsFactor = 1.0;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond)) {
perror("timefn::QueryPerformanceFrequency");
abort();
}
nsFactor = 1000000000.0 / (double)ticksPerSecond.QuadPart;
init = 1;
}
{ UTIL_time_t r;
LARGE_INTEGER x;
QueryPerformanceCounter(&x);
r.t = (PTime)((double)x.QuadPart * nsFactor);
r.t = (PTime)(x.QuadPart * 1000000000ULL / ticksPerSecond.QuadPart);
return r;
}
}
+11
View File
@@ -11,6 +11,12 @@
#ifndef TIME_FN_H_MODULE_287987
#define TIME_FN_H_MODULE_287987
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Types
******************************************/
@@ -56,4 +62,9 @@ PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
#define SEC_TO_MICRO ((PTime)1000000) /* nb of microseconds in a second */
#if defined (__cplusplus)
}
#endif
#endif /* TIME_FN_H_MODULE_287987 */
+76 -155
View File
@@ -8,6 +8,11 @@
* You may select, at your option, one of the above-listed licenses.
*/
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
@@ -151,8 +156,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg,
/*-*************************************
* Constants
***************************************/
#define KB * (1 << 10)
#define LIST_SIZE_INCREASE (8 KB)
#define LIST_SIZE_INCREASE (8*1024)
#define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50
@@ -197,16 +201,6 @@ int UTIL_stat(const char* filename, stat_t* statbuf)
return UTIL_fstat(-1, filename, statbuf);
}
int UTIL_isFdRegularFile(int fd)
{
stat_t statbuf;
int ret;
UTIL_TRACE_CALL("UTIL_isFdRegularFile(%d)", fd);
ret = fd >= 0 && UTIL_fstat(fd, "", &statbuf) && UTIL_isRegularFileStat(&statbuf);
UTIL_TRACE_RET(ret);
return ret;
}
int UTIL_isRegularFile(const char* infilename)
{
stat_t statbuf;
@@ -459,26 +453,6 @@ int UTIL_isFIFOStat(const stat_t* statbuf)
return 0;
}
/* process substitution */
int UTIL_isFileDescriptorPipe(const char* filename)
{
UTIL_TRACE_CALL("UTIL_isFileDescriptorPipe(%s)", filename);
/* Check if the filename is a /dev/fd/ path which indicates a file descriptor */
if (filename[0] == '/' && strncmp(filename, "/dev/fd/", 8) == 0) {
UTIL_TRACE_RET(1);
return 1;
}
/* Check for alternative process substitution formats on different systems */
if (filename[0] == '/' && strncmp(filename, "/proc/self/fd/", 14) == 0) {
UTIL_TRACE_RET(1);
return 1;
}
UTIL_TRACE_RET(0);
return 0; /* Not recognized as a file descriptor pipe */
}
/* UTIL_isBlockDevStat : distinguish named pipes */
int UTIL_isBlockDevStat(const stat_t* statbuf)
{
@@ -645,157 +619,101 @@ U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles)
}
/* Read the entire content of a file into a buffer with progressive resizing */
static char* UTIL_readFileContent(FILE* inFile, size_t* totalReadPtr)
/* condition : @file must be valid, and not have reached its end.
* @return : length of line written into @buf, ended with `\0` instead of '\n',
* or 0, if there is no new line */
static size_t readLineFromFile(char* buf, size_t len, FILE* file)
{
size_t bufSize = 64 KB; /* Start with a reasonable buffer size */
size_t totalRead = 0;
size_t bytesRead = 0;
char* buf = (char*)malloc(bufSize);
if (buf == NULL) return NULL;
/* Read the file incrementally */
while ((bytesRead = fread(buf + totalRead, 1, bufSize - totalRead - 1, inFile)) > 0) {
totalRead += bytesRead;
/* If buffer is nearly full, expand it */
if (bufSize - totalRead < 1 KB) {
if (bufSize >= MAX_FILE_OF_FILE_NAMES_SIZE) {
/* Too large, abort */
free(buf);
return NULL;
assert(!feof(file));
if ( fgets(buf, (int) len, file) == NULL ) return 0;
{ size_t linelen = strlen(buf);
if (strlen(buf)==0) return 0;
if (buf[linelen-1] == '\n') linelen--;
buf[linelen] = '\0';
return linelen+1;
}
{ size_t newBufSize = bufSize * 2;
if (newBufSize > MAX_FILE_OF_FILE_NAMES_SIZE)
newBufSize = MAX_FILE_OF_FILE_NAMES_SIZE;
{ char* newBuf = (char*)realloc(buf, newBufSize);
if (newBuf == NULL) {
free(buf);
return NULL;
}
buf = newBuf;
bufSize = newBufSize;
} } }
}
/* Add null terminator to the end */
buf[totalRead] = '\0';
*totalReadPtr = totalRead;
return buf;
}
/* Process a buffer containing multiple lines and count the number of lines */
static size_t UTIL_processLines(char* buffer, size_t bufferSize)
/* Conditions :
* size of @inputFileName file must be < @dstCapacity
* @dst must be initialized
* @return : nb of lines
* or -1 if there's an error
*/
static int
readLinesFromFile(void* dst, size_t dstCapacity,
const char* inputFileName)
{
size_t lineCount = 0;
size_t i = 0;
/* Convert newlines to null terminators and count lines */
while (i < bufferSize) {
if (buffer[i] == '\n') {
buffer[i] = '\0'; /* Replace newlines with null terminators */
lineCount++;
}
i++;
}
/* Count the last line if it doesn't end with a newline */
if (bufferSize > 0 && (i == 0 || buffer[i-1] != '\0')) {
lineCount++;
}
return lineCount;
}
/* Create an array of pointers to the lines in a buffer */
static const char** UTIL_createLinePointers(char* buffer, size_t numLines, size_t bufferSize)
{
size_t lineIndex = 0;
int nbFiles = 0;
size_t pos = 0;
void* const bufferPtrs = malloc(numLines * sizeof(const char**));
const char** const linePointers = (const char**)bufferPtrs;
if (bufferPtrs == NULL) return NULL;
char* const buf = (char*)dst;
FILE* const inputFile = fopen(inputFileName, "r");
while (lineIndex < numLines && pos < bufferSize) {
size_t len = 0;
linePointers[lineIndex++] = buffer+pos;
assert(dst != NULL);
/* Find the next null terminator, being careful not to go past the buffer */
while ((pos + len < bufferSize) && buffer[pos + len] != '\0') {
len++;
if(!inputFile) {
if (g_utilDisplayLevel >= 1) perror("zstd:util:readLinesFromFile");
return -1;
}
/* Move past this string and its null terminator */
pos += len;
if (pos < bufferSize) pos++; /* Skip the null terminator if we're not at buffer end */
while ( !feof(inputFile) ) {
size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile);
if (lineLength == 0) break;
assert(pos + lineLength <= dstCapacity); /* '=' for inputFile not terminated with '\n' */
pos += lineLength;
++nbFiles;
}
/* Verify we processed the expected number of lines */
if (lineIndex != numLines) {
/* Something went wrong - we didn't find as many lines as expected */
free(bufferPtrs);
return NULL;
}
CONTROL( fclose(inputFile) == 0 );
return linePointers;
return nbFiles;
}
/*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/
FileNamesTable*
UTIL_createFileNamesTable_fromFileList(const char* fileList)
UTIL_createFileNamesTable_fromFileName(const char* inputFileName)
{
size_t nbFiles = 0;
char* buf;
size_t bufSize;
stat_t statbuf;
char* buffer = NULL;
size_t numLines = 0;
size_t bufferSize = 0;
/* Check if the input is a valid file */
if (!UTIL_stat(fileList, &statbuf)) {
if (!UTIL_stat(inputFileName, &statbuf) || !UTIL_isRegularFileStat(&statbuf))
return NULL;
{ U64 const inputFileSize = UTIL_getFileSizeStat(&statbuf);
if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE)
return NULL;
bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */
}
buf = (char*) malloc(bufSize);
CONTROL( buf != NULL );
{ int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName);
if (ret_nbFiles <= 0) {
free(buf);
return NULL;
}
/* Check if the input is a supported type */
if (!UTIL_isRegularFileStat(&statbuf) &&
!UTIL_isFIFOStat(&statbuf) &&
!UTIL_isFileDescriptorPipe(fileList)) {
return NULL;
nbFiles = (size_t)ret_nbFiles;
}
/* Open the input file */
{ FILE* const inFile = fopen(fileList, "rb");
if (inFile == NULL) return NULL;
{ const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable));
CONTROL(filenamesTable != NULL);
/* Read the file content */
buffer = UTIL_readFileContent(inFile, &bufferSize);
fclose(inFile);
{ size_t fnb, pos = 0;
for (fnb = 0; fnb < nbFiles; fnb++) {
filenamesTable[fnb] = buf+pos;
pos += strlen(buf+pos)+1; /* +1 for the finishing `\0` */
}
assert(pos <= bufSize);
}
if (buffer == NULL) return NULL;
/* Process lines */
numLines = UTIL_processLines(buffer, bufferSize);
if (numLines == 0) {
free(buffer);
return NULL;
}
/* Create line pointers */
{ const char** linePointers = UTIL_createLinePointers(buffer, numLines, bufferSize);
if (linePointers == NULL) {
free(buffer);
return NULL;
}
/* Create the final table */
return UTIL_assembleFileNamesTable(linePointers, numLines, buffer);
return UTIL_assembleFileNamesTable(filenamesTable, nbFiles, buf);
}
}
static FileNamesTable*
UTIL_assembleFileNamesTable2(const char** filenames, size_t tableSize, size_t tableCapacity, char* buf)
{
@@ -928,7 +846,6 @@ static int UTIL_prepareFileList(const char* dirName,
hFile=FindFirstFileA(path, &cFile);
if (hFile == INVALID_HANDLE_VALUE) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
free(path);
return 0;
}
free(path);
@@ -1729,3 +1646,7 @@ int UTIL_countLogicalCores(void)
{
return UTIL_countCores(1);
}
#if defined (__cplusplus)
}
#endif
+12 -17
View File
@@ -11,26 +11,25 @@
#ifndef UTIL_H_MODULE
#define UTIL_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
#include "platform.h" /* PLATFORM_POSIX_VERSION, ZSTD_NANOSLEEP_SUPPORT, ZSTD_SETPRIORITY_SUPPORT */
#include <stddef.h> /* size_t, ptrdiff_t */
#include <stdio.h> /* FILE */
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat, chmod */
#include "../lib/common/mem.h" /* U64 */
#if !(defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__))
#include <libgen.h>
#endif
/*-************************************************************
* Fix fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW
***************************************************************/
#if defined(LIBC_NO_FSEEKO)
/* Some older libc implementations don't include these functions (e.g. Bionic < 24) */
# define UTIL_fseek fseek
#elif defined(_MSC_VER) && (_MSC_VER >= 1400)
#if defined(_MSC_VER) && (_MSC_VER >= 1400)
# define UTIL_fseek _fseeki64
#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
# define UTIL_fseek fseeko
@@ -40,6 +39,7 @@
# define UTIL_fseek fseek
#endif
/*-*************************************************
* Sleep & priority functions: Windows - Posix - others
***************************************************/
@@ -88,10 +88,6 @@
#endif
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Console log
******************************************/
@@ -122,6 +118,7 @@ int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, const
#define STRDUP(s) _strdup(s)
#else
#define PATH_SEP '/'
#include <libgen.h>
#define STRDUP(s) strdup(s)
#endif
@@ -184,7 +181,6 @@ int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_
* compute the needed information.
*/
int UTIL_isFdRegularFile(int fd);
int UTIL_isRegularFile(const char* infilename);
int UTIL_isDirectory(const char* infilename);
int UTIL_isSameFile(const char* file1, const char* file2);
@@ -192,7 +188,6 @@ int UTIL_isSameFileStat(const char* file1, const char* file2, const stat_t* file
int UTIL_isCompressedFile(const char* infilename, const char *extensionList[]);
int UTIL_isLink(const char* infilename);
int UTIL_isFIFO(const char* infilename);
int UTIL_isFileDescriptorPipe(const char* filename);
/**
* Returns with the given file descriptor is a console.
@@ -252,13 +247,13 @@ typedef struct
size_t tableCapacity;
} FileNamesTable;
/*! UTIL_createFileNamesTable_fromFileList() :
/*! UTIL_createFileNamesTable_fromFileName() :
* read filenames from @inputFileName, and store them into returned object.
* @return : a FileNamesTable*, or NULL in case of error (ex: @inputFileName doesn't exist).
* Note: inputFileSize must be less than 50MB
*/
FileNamesTable*
UTIL_createFileNamesTable_fromFileList(const char* inputFileName);
UTIL_createFileNamesTable_fromFileName(const char* inputFileName);
/*! UTIL_assembleFileNamesTable() :
* This function takes ownership of its arguments, @filenames and @buf,
+40 -54
View File
@@ -1,5 +1,5 @@
.
.TH "ZSTD" "1" "February 2025" "zstd 1.5.7" "User Commands"
.TH "ZSTD" "1" "March 2024" "zstd 1.5.6" "User Commands"
.
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -108,7 +108,7 @@ Display information related to a zstd compressed file, such as size, ratio, and
\fB\-\-fast[=#]\fR: switch to ultra\-fast compression levels\. If \fB=#\fR is not present, it defaults to \fB1\fR\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. This setting overwrites compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\.
.
.IP "\(bu" 4
\fB\-T#\fR, \fB\-\-threads=#\fR: Compress using \fB#\fR working threads (default: between 1 and 4 depending on physical CPU cores; see \fBZSTD_NBTHREADS\fR below)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to \fBZSTDMT_NBWORKERS_MAX\fR, which is either 64 in 32\-bit mode, or 256 for 64\-bit environments\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
\fB\-T#\fR, \fB\-\-threads=#\fR: Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to \fBZSTDMT_NBWORKERS_MAX\fR, which is either 64 in 32\-bit mode, or 256 for 64\-bit environments\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
.
.IP "\(bu" 4
\fB\-\-single\-thread\fR: Use a single thread for both I/O and compression\. As compression is serialized with I/O, this can be slightly slower\. Single\-thread mode features significantly lower memory usage, which can be useful for systems with limited amount of memory, such as 32\-bit systems\.
@@ -135,9 +135,6 @@ Note 2: this mode is different from \fB\-T1\fR, which spawns 1 compression threa
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
.
.IP "\(bu" 4
\fB\-\-max\fR: set advanced parameters to maximum compression\. warning: this setting is very slow and uses a lot of resources\. It\'s inappropriate for 32\-bit mode and therefore disabled in this mode\.
.
.IP "\(bu" 4
\fB\-D DICT\fR: use \fBDICT\fR as Dictionary to compress or decompress FILE(s)
.
.IP "\(bu" 4
@@ -150,7 +147,7 @@ Note: cannot use both this and \fB\-D\fR together\.
Note: \fB\-\-long\fR mode will be automatically activated if \fIchainLog\fR < \fIfileLog\fR (\fIfileLog\fR being the \fIwindowLog\fR required to cover the whole file)\. You can also manually force it\.
.
.IP
Note: up to level 15, you can use \fB\-\-patch\-from\fR in \fB\-\-single\-thread\fR mode to improve compression ratio marginally at the cost of speed\. Using \'\-\-single\-thread\' above level 15 will lead to lower compression ratios\.
Note: for all levels, you can use \fB\-\-patch\-from\fR in \fB\-\-single\-thread\fR mode to improve compression ratio at the cost of speed\.
.
.IP
Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e\. 4096), and by setting a large \fB\-\-zstd=chainLog=\fR\.
@@ -268,7 +265,7 @@ Employing environment variables to set parameters has security implications\. Th
\fBZSTD_CLEVEL\fR can be used to set the level between 1 and 19 (the "normal" range)\. If the value of \fBZSTD_CLEVEL\fR is not a valid integer, it will be ignored with a warning message\. \fBZSTD_CLEVEL\fR just replaces the default compression level (\fB3\fR)\.
.
.P
\fBZSTD_NBTHREADS\fR can be used to set the number of threads \fBzstd\fR will attempt to use during compression\. If the value of \fBZSTD_NBTHREADS\fR is not a valid unsigned integer, it will be ignored with a warning message\. \fBZSTD_NBTHREADS\fR has a default value of \fBmax(1, min(4, nbCores/4))\fR, and is capped at ZSTDMT_NBWORKERS_MAX==200\. \fBzstd\fR must be compiled with multithread support for this variable to have any effect\.
\fBZSTD_NBTHREADS\fR can be used to set the number of threads \fBzstd\fR will attempt to use during compression\. If the value of \fBZSTD_NBTHREADS\fR is not a valid unsigned integer, it will be ignored with a warning message\. \fBZSTD_NBTHREADS\fR has a default value of (\fB1\fR), and is capped at ZSTDMT_NBWORKERS_MAX==200\. \fBzstd\fR must be compiled with multithread support for this variable to have any effect\.
.
.P
They can both be overridden by corresponding command line arguments: \fB\-#\fR for compression level and \fB\-T#\fR for number of compression threads\.
@@ -360,19 +357,6 @@ Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This pa
The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default": \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
.
.TP
\fBldmHashRateLog\fR=\fIlhrlog\fR, \fBlhrlog\fR=\fIlhrlog\fR
Specify the frequency of inserting entries into the long distance matching hash table\.
.
.IP
This option is ignored unless long distance matching is enabled\.
.
.IP
Larger values will improve compression speed\. Deviating far from the default value will likely result in a decrease in compression ratio\.
.
.IP
The default value varies between 4 and 7, depending on \fBstrategy\fR\.
.
.TP
\fBldmHashLog\fR=\fIlhlog\fR, \fBlhlog\fR=\fIlhlog\fR
Specify the maximum size for a hash table used for long distance matching\.
.
@@ -383,7 +367,7 @@ This option is ignored unless long distance matching is enabled\.
Bigger hash tables usually improve compression ratio at the expense of more memory during compression and a decrease in compression speed\.
.
.IP
The minimum \fIlhlog\fR is 6 and the maximum is 30 (default: \fBwindowLog \- ldmHashRateLog\fR)\.
The minimum \fIlhlog\fR is 6 and the maximum is 30 (default: 20)\.
.
.TP
\fBldmMinMatch\fR=\fIlmml\fR, \fBlmml\fR=\fIlmml\fR
@@ -396,7 +380,7 @@ This option is ignored unless long distance matching is enabled\.
Larger/very small values usually decrease compression ratio\.
.
.IP
The minimum \fIlmml\fR is 4 and the maximum is 4096 (default: 32 to 64, depending on \fBstrategy\fR)\.
The minimum \fIlmml\fR is 4 and the maximum is 4096 (default: 64)\.
.
.TP
\fBldmBucketSizeLog\fR=\fIlblog\fR, \fBlblog\fR=\fIlblog\fR
@@ -409,7 +393,20 @@ This option is ignored unless long distance matching is enabled\.
Larger bucket sizes improve collision resolution but decrease compression speed\.
.
.IP
The minimum \fIlblog\fR is 1 and the maximum is 8 (default: 4 to 8, depending on \fBstrategy\fR)\.
The minimum \fIlblog\fR is 1 and the maximum is 8 (default: 3)\.
.
.TP
\fBldmHashRateLog\fR=\fIlhrlog\fR, \fBlhrlog\fR=\fIlhrlog\fR
Specify the frequency of inserting entries into the long distance matching hash table\.
.
.IP
This option is ignored unless long distance matching is enabled\.
.
.IP
Larger values will improve compression speed\. Deviating far from the default value will likely result in a decrease in compression ratio\.
.
.IP
The default value is \fBwlog \- lhlog\fR\.
.
.SS "Example"
The following parameters sets advanced compression options to something similar to predefined level 19 for files bigger than 256 KB:
@@ -524,48 +521,37 @@ Examples:
\fBzstd \-\-train\-legacy=selectivity=8 FILEs\fR
.
.SH "BENCHMARK"
The \fBzstd\fR CLI provides a benchmarking mode that can be used to easily find suitable compression parameters, or alternatively to benchmark a computer\'s performance\. \fBzstd \-b [FILE(s)]\fR will benchmark \fBzstd\fR for both compression and decompression using default compression level\. Note that results are very dependent on the content being compressed\.
The \fBzstd\fR CLI provides a benchmarking mode that can be used to easily find suitable compression parameters, or alternatively to benchmark a computer\'s performance\. Note that the results are highly dependent on the content being compressed\.
.
.P
It\'s possible to pass multiple files to the benchmark, and even a directory with \fB\-r DIRECTORY\fR\. When no \fBFILE\fR is provided, the benchmark will use a procedurally generated \fBlorem ipsum\fR text\.
.TP
\fB\-b#\fR
benchmark file(s) using compression level #
.
.P
Benchmarking will employ \fBmax(1, min(4, nbCores/4))\fR worker threads by default in order to match the behavior of the normal CLI I/O\.
.TP
\fB\-e#\fR
benchmark file(s) using multiple compression levels, from \fB\-b#\fR to \fB\-e#\fR (inclusive)
.
.IP "\(bu" 4
\fB\-b#\fR: benchmark file(s) using compression level #
.TP
\fB\-d\fR
benchmark decompression speed only (requires providing an already zstd\-compressed content)
.
.IP "\(bu" 4
\fB\-e#\fR: benchmark file(s) using multiple compression levels, from \fB\-b#\fR to \fB\-e#\fR (inclusive)
.TP
\fB\-i#\fR
minimum evaluation time, in seconds (default: 3s), benchmark mode only
.
.IP "\(bu" 4
\fB\-d\fR: benchmark decompression speed only (requires providing a zstd\-compressed content)
.TP
\fB\-B#\fR, \fB\-\-block\-size=#\fR
cut file(s) into independent chunks of size # (default: no chunking)
.
.IP "\(bu" 4
\fB\-i#\fR: minimum evaluation time, in seconds (default: 3s), benchmark mode only
.
.IP "\(bu" 4
\fB\-B#\fR, \fB\-\-block\-size=#\fR: cut file(s) into independent chunks of size # (default: no chunking)
.
.IP "\(bu" 4
\fB\-S\fR: output one benchmark result per input file (default: consolidated result)
.
.IP "\(bu" 4
\fB\-D dictionary\fR benchmark using dictionary
.
.IP "\(bu" 4
\fB\-\-priority=rt\fR: set process priority to real\-time (Windows)
.
.IP "" 0
.
.P
Beyond compression levels, benchmarking is also compatible with other parameters, such as number of threads (\fB\-T#\fR), advanced compression parameters (\fB\-\-zstd=###\fR), dictionary compression (\fB\-D dictionary\fR), or even disabling checksum verification for example\.
.TP
\fB\-\-priority=rt\fR
set process priority to real\-time (Windows)
.
.P
\fBOutput Format:\fR CompressionLevel#Filename: InputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
.
.P
\fBMethodology:\fR For speed measurement, the entire input is compressed/decompressed in\-memory to measure speed\. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy\.
\fBMethodology:\fR For both compression and decompression speed, the entire input is compressed/decompressed in\-memory to measure speed\. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy\.
.
.SH "SEE ALSO"
\fBzstdgrep\fR(1), \fBzstdless\fR(1), \fBgzip\fR(1), \fBxz\fR(1)
+30 -50
View File
@@ -113,11 +113,7 @@ the last one takes effect.
Because the compressor's behavior highly depends on the content to compress, there's no guarantee of a smooth progression from one level to another.
* `--ultra`:
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
Decompression will also need more memory when using these levels.
* `--max`:
set advanced parameters to reach maximum compression.
warning: this setting is very slow and uses a lot of resources.
It's inappropriate for 32-bit mode and therefore disabled in this mode.
Note that decompression will also require more memory when using these levels.
* `--fast[=#]`:
switch to ultra-fast compression levels.
If `=#` is not present, it defaults to `1`.
@@ -126,12 +122,11 @@ the last one takes effect.
This setting overwrites compression level if one was set previously.
Similarly, if a compression level is set after `--fast`, it overrides it.
* `-T#`, `--threads=#`:
Compress using `#` working threads (default: between 1 and 4 depending on physical CPU cores; see `ZSTD_NBTHREADS` below).
Compress using `#` working threads (default: 1).
If `#` is 0, attempt to detect and use the number of physical CPU cores.
In all cases, the nb of threads is capped to `ZSTDMT_NBWORKERS_MAX`,
which is either 64 in 32-bit mode, or 256 for 64-bit environments.
This modifier does nothing if `zstd` is compiled without multithread support.
Note that memory usage increases with each thread.
* `--single-thread`:
Use a single thread for both I/O and compression.
As compression is serialized with I/O, this can be slightly slower.
@@ -158,7 +153,7 @@ the last one takes effect.
when combined with multiple worker threads (>=2).
* `--long[=#]`:
enables long distance matching with `#` `windowLog`, if `#` is not
present it defaults to `27`. The highest possible value is 31.
present it defaults to `27`.
This increases the window size (`windowLog`) and memory usage for both the
compressor and decompressor.
This setting is designed to improve the compression ratio for files with
@@ -179,10 +174,8 @@ the last one takes effect.
(_fileLog_ being the _windowLog_ required to cover the whole file). You
can also manually force it.
Note: up to level 15, you can use `--patch-from` in `--single-thread` mode
to improve compression ratio marginally at the cost of speed. Using
'--single-thread' above level 15 will lead to lower compression
ratios.
Note: for all levels, you can use `--patch-from` in `--single-thread` mode
to improve compression ratio at the cost of speed.
Note: for level 19, you can get increased compression ratio at the cost
of speed by specifying `--zstd=targetLength=` to be something large
@@ -348,7 +341,7 @@ If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a
`ZSTD_NBTHREADS` can be used to set the number of threads `zstd` will attempt to use during compression.
If the value of `ZSTD_NBTHREADS` is not a valid unsigned integer, it will be ignored with a warning message.
`ZSTD_NBTHREADS` has a default value of `max(1, min(4, nbCores/4))`, and is capped at ZSTDMT_NBWORKERS_MAX==200.
`ZSTD_NBTHREADS` has a default value of (`1`), and is capped at ZSTDMT_NBWORKERS_MAX==200.
`zstd` must be compiled with multithread support for this variable to have any effect.
They can both be overridden by corresponding command line arguments:
@@ -456,17 +449,6 @@ The list of available _options_:
Value 0 is special and means "default": _ovlog_ is automatically determined by `zstd`.
In which case, _ovlog_ will range from 6 to 9, depending on selected _strat_.
- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
Specify the frequency of inserting entries into the long distance matching
hash table.
This option is ignored unless long distance matching is enabled.
Larger values will improve compression speed. Deviating far from the
default value will likely result in a decrease in compression ratio.
The default value varies between 4 and 7, depending on `strategy`.
- `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_:
Specify the maximum size for a hash table used for long distance matching.
@@ -475,7 +457,7 @@ The list of available _options_:
Bigger hash tables usually improve compression ratio at the expense of more
memory during compression and a decrease in compression speed.
The minimum _lhlog_ is 6 and the maximum is 30 (default: `windowLog - ldmHashRateLog`).
The minimum _lhlog_ is 6 and the maximum is 30 (default: 20).
- `ldmMinMatch`=_lmml_, `lmml`=_lmml_:
Specify the minimum searched length of a match for long distance matching.
@@ -484,7 +466,7 @@ The list of available _options_:
Larger/very small values usually decrease compression ratio.
The minimum _lmml_ is 4 and the maximum is 4096 (default: 32 to 64, depending on `strategy`).
The minimum _lmml_ is 4 and the maximum is 4096 (default: 64).
- `ldmBucketSizeLog`=_lblog_, `lblog`=_lblog_:
Specify the size of each bucket for the hash table used for long distance
@@ -495,8 +477,18 @@ The list of available _options_:
Larger bucket sizes improve collision resolution but decrease compression
speed.
The minimum _lblog_ is 1 and the maximum is 8 (default: 4 to 8, depending on `strategy`).
The minimum _lblog_ is 1 and the maximum is 8 (default: 3).
- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
Specify the frequency of inserting entries into the long distance matching
hash table.
This option is ignored unless long distance matching is enabled.
Larger values will improve compression speed. Deviating far from the
default value will likely result in a decrease in compression ratio.
The default value is `wlog - lhlog`.
### Example
The following parameters sets advanced compression options to something
@@ -504,12 +496,12 @@ similar to predefined level 19 for files bigger than 256 KB:
`--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
### --jobsize=#:
### -B#:
Specify the size of each compression job.
This parameter is only meaningful when multi-threading is enabled.
Each compression job is run in parallel, so this value can indirectly impact the nb of active threads.
This parameter is only available when multi-threading is enabled.
Each compression job is run in parallel, so this value indirectly impacts the nb of active threads.
Default job size varies depending on compression level (generally `4 * windowSize`).
`--jobsize=#` makes it possible to manually select a custom size.
`-B#` makes it possible to manually select a custom size.
Note that job size must respect a minimum value which is enforced transparently.
This minimum is either 512 KB, or `overlapSize`, whichever is largest.
Different job sizes will lead to non-identical compressed frames.
@@ -555,8 +547,8 @@ Compression of small files similar to the sample set will be greatly improved.
Use `#` compression level during training (optional).
Will generate statistics more tuned for selected compression level,
resulting in a _small_ compression ratio improvement for this level.
* `--split=#`:
Split input files into independent chunks of size # (default: no split)
* `-B#`:
Split input files into blocks of size # (default: no split)
* `-M#`, `--memory=#`:
Limit the amount of sample data loaded for training (default: 2 GB).
Note that the default (2 GB) is also the maximum.
@@ -668,36 +660,24 @@ Compression of small files similar to the sample set will be greatly improved.
BENCHMARK
---------
The `zstd` CLI provides a benchmarking mode that can be used to easily find suitable compression parameters, or alternatively to benchmark a computer's performance.
`zstd -b [FILE(s)]` will benchmark `zstd` for both compression and decompression using default compression level.
Note that results are very dependent on the content being compressed.
It's possible to pass multiple files to the benchmark, and even a directory with `-r DIRECTORY`.
When no `FILE` is provided, the benchmark will use a procedurally generated `lorem ipsum` text.
Benchmarking will employ `max(1, min(4, nbCores/4))` worker threads by default in order to match the behavior of the normal CLI I/O.
Note that the results are highly dependent on the content being compressed.
* `-b#`:
benchmark file(s) using compression level #
* `-e#`:
benchmark file(s) using multiple compression levels, from `-b#` to `-e#` (inclusive)
* `-d`:
benchmark decompression speed only (requires providing a zstd-compressed content)
benchmark decompression speed only (requires providing an already zstd-compressed content)
* `-i#`:
minimum evaluation time, in seconds (default: 3s), benchmark mode only
* `--split=#`:
split input file(s) into independent chunks of size # (default: no chunking)
* `-S`:
output one benchmark result per input file (default: consolidated result)
* `-D dictionary`
benchmark using dictionary
* `-B#`, `--block-size=#`:
cut file(s) into independent chunks of size # (default: no chunking)
* `--priority=rt`:
set process priority to real-time (Windows)
Beyond compression levels, benchmarking is also compatible with other parameters, such as number of threads (`-T#`), advanced compression parameters (`--zstd=###`), dictionary compression (`-D dictionary`), or even disabling checksum verification for example.
**Output Format:** CompressionLevel#Filename: InputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
**Methodology:** For speed measurement, the entire input is compressed/decompressed in-memory to measure speed. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy.
**Methodology:** For both compression and decompression speed, the entire input is compressed/decompressed in-memory to measure speed. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy.
SEE ALSO
+112 -183
View File
@@ -8,6 +8,22 @@
* You may select, at your option, one of the above-listed licenses.
*/
/*-************************************
* Tuning parameters
**************************************/
#ifndef ZSTDCLI_CLEVEL_DEFAULT
# define ZSTDCLI_CLEVEL_DEFAULT 3
#endif
#ifndef ZSTDCLI_CLEVEL_MAX
# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */
#endif
#ifndef ZSTDCLI_NBTHREADS_DEFAULT
# define ZSTDCLI_NBTHREADS_DEFAULT 1
#endif
/*-************************************
* Dependencies
**************************************/
@@ -16,6 +32,7 @@
#include <stdlib.h> /* getenv */
#include <string.h> /* strcmp, strlen */
#include <stdio.h> /* fprintf(), stdin, stdout, stderr */
#include <errno.h> /* errno */
#include <assert.h> /* assert */
#include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
@@ -30,24 +47,7 @@
#endif
#include "../lib/zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
#include "fileio_asyncio.h"
#include "fileio_common.h"
/*-************************************
* Tuning parameters
**************************************/
#ifndef ZSTDCLI_CLEVEL_DEFAULT
# define ZSTDCLI_CLEVEL_DEFAULT 3
#endif
#ifndef ZSTDCLI_CLEVEL_MAX
# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */
#endif
#ifndef ZSTDCLI_NBTHREADS_DEFAULT
#define ZSTDCLI_NBTHREADS_DEFAULT (unsigned)(MAX(1, MIN(4, UTIL_countLogicalCores() / 4)))
#endif
static unsigned init_nbWorkers(unsigned defaultNbWorkers);
/*-************************************
* Constants
@@ -77,9 +77,7 @@ static unsigned init_nbWorkers(unsigned defaultNbWorkers);
#define MB *(1 <<20)
#define GB *(1U<<30)
#ifndef ZSTD_DISPLAY_LEVEL_DEFAULT
# define ZSTD_DISPLAY_LEVEL_DEFAULT 2
#endif
#define DISPLAY_LEVEL_DEFAULT 2
static const char* g_defaultDictName = "dictionary";
static const unsigned g_defaultMaxDictSize = 110 KB;
@@ -96,17 +94,17 @@ static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
#define DEFAULT_ACCEL 1
#define NBWORKERS_AUTOCPU 0
#define NBWORKERS_UNSET UINT_MAX
typedef enum { cover, fastCover, legacy } dictType;
/*-************************************
* Display Macros
**************************************/
#undef DISPLAYLEVEL
#define DISPLAY_F(f, ...) fprintf((f), __VA_ARGS__)
#define DISPLAYOUT(...) DISPLAY_F(stdout, __VA_ARGS__)
#define DISPLAY(...) DISPLAY_F(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
static int g_displayLevel = ZSTD_DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
/*-************************************
@@ -149,9 +147,8 @@ static void usage(FILE* f, const char* programName)
DISPLAY_F(f, "Usage: %s [OPTIONS...] [INPUT... | -] [-o OUTPUT]\n\n", programName);
DISPLAY_F(f, "Options:\n");
DISPLAY_F(f, " -o OUTPUT Write output to a single file, OUTPUT.\n");
DISPLAY_F(f, " -c, --stdout Write to STDOUT (even if it is a console) and keep the INPUT file(s).\n");
DISPLAY_F(f, " -k, --keep Preserve INPUT file(s). [Default] \n");
DISPLAY_F(f, " --rm Remove INPUT file(s) after successful (de)compression to file.\n");
DISPLAY_F(f, " --rm Remove INPUT file(s) after successful (de)compression.\n");
#ifdef ZSTD_GZCOMPRESS
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
DISPLAY_F(f, " -n, --no-name Do not store original filename when compressing.\n\n");
@@ -184,6 +181,7 @@ static void usageAdvanced(const char* programName)
DISPLAYOUT("\n");
usage(stdout, programName);
DISPLAYOUT("Advanced options:\n");
DISPLAYOUT(" -c, --stdout Write to STDOUT (even if it is a console) and keep the INPUT file(s).\n\n");
DISPLAYOUT(" -v, --verbose Enable verbose output; pass multiple times to increase verbosity.\n");
DISPLAYOUT(" -q, --quiet Suppress warnings; pass twice to suppress errors.\n");
@@ -233,15 +231,14 @@ static void usageAdvanced(const char* programName)
#endif
DISPLAYOUT(" --adapt Dynamically adapt compression level to I/O conditions.\n");
DISPLAYOUT(" --long[=#] Enable long distance matching with window log #. [Default: %u]\n", g_defaultMaxWindowLog);
DISPLAYOUT(" --patch-from=REF Use REF as the reference point for Zstandard's diff engine. \n");
DISPLAYOUT(" --patch-apply Equivalent for `-d --patch-from` \n\n");
DISPLAYOUT(" --patch-from=REF Use REF as the reference point for Zstandard's diff engine. \n\n");
# ifdef ZSTD_MULTITHREAD
DISPLAYOUT(" -T# Spawn # compression threads. [Default: %u; pass 0 for core count.]\n", init_nbWorkers(ZSTDCLI_NBTHREADS_DEFAULT));
DISPLAYOUT(" -T# Spawn # compression threads. [Default: 1; pass 0 for core count.]\n");
DISPLAYOUT(" --single-thread Share a single thread for I/O and compression (slightly different than `-T1`).\n");
DISPLAYOUT(" --auto-threads={physical|logical}\n");
DISPLAYOUT(" Use physical/logical cores when using `-T0`. [Default: Physical]\n\n");
DISPLAYOUT(" --jobsize=# Set job size to #. [Default: 0 (automatic)]\n");
DISPLAYOUT(" --rsyncable Compress using a rsync-friendly method (`--jobsize=#` sets unit size). \n");
DISPLAYOUT(" -B# Set job size to #. [Default: 0 (automatic)]\n");
DISPLAYOUT(" --rsyncable Compress using a rsync-friendly method (`-B` sets block size). \n");
DISPLAYOUT("\n");
# endif
DISPLAYOUT(" --exclude-compressed Only compress files that are not already compressed.\n\n");
@@ -312,9 +309,8 @@ static void usageAdvanced(const char* programName)
DISPLAYOUT(" -b# Perform benchmarking with compression level #. [Default: %d]\n", ZSTDCLI_CLEVEL_DEFAULT);
DISPLAYOUT(" -e# Test all compression levels up to #; starting level is `-b#`. [Default: 1]\n");
DISPLAYOUT(" -i# Set the minimum evaluation to time # seconds. [Default: 3]\n");
DISPLAYOUT(" --split=# Split input into independent chunks of size #. [Default: No chunking]\n");
DISPLAYOUT(" -B# Cut file into independent chunks of size #. [Default: No chunking]\n");
DISPLAYOUT(" -S Output one benchmark result per input file. [Default: Consolidated result]\n");
DISPLAYOUT(" -D dictionary Benchmark using dictionary \n");
DISPLAYOUT(" --priority=rt Set process priority to real-time.\n");
#endif
@@ -349,7 +345,7 @@ static void errorOut(const char* msg)
/*! readU32FromCharChecked() :
* @return 0 if success, and store the result in *value.
* allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
* @return 1 if an overflow error occurs */
static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
@@ -364,22 +360,15 @@ static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
if (result < last) return 1; /* overflow error */
(*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M') || (**stringPtr=='G')) {
switch (**stringPtr) {
case 'K':
if (result > (((unsigned)-1) >> 10)) return 1; /* overflow error */
if ((**stringPtr=='K') || (**stringPtr=='M')) {
unsigned const maxK = ((unsigned)(-1)) >> 10;
if (result > maxK) return 1; /* overflow error */
result <<= 10;
if (**stringPtr=='M') {
if (result > maxK) return 1; /* overflow error */
result <<= 10;
break;
case 'M':
if (result > (((unsigned)-1) >> 20)) return 1; /* overflow error */
result <<= 20;
break;
case 'G':
if (result > (((unsigned)-1) >> 30)) return 1; /* overflow error */
result <<= 30;
break;
}
(*stringPtr)++; /* skip `K`, `M` or `G` */
(*stringPtr)++; /* skip `K` or `M` */
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
@@ -389,7 +378,7 @@ static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
/*! readU32FromChar() :
* @return : unsigned integer value read from input in `char` format.
* allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
* Note : function will exit() program if digit sequence overflows */
static unsigned readU32FromChar(const char** stringPtr) {
@@ -401,7 +390,7 @@ static unsigned readU32FromChar(const char** stringPtr) {
/*! readIntFromChar() :
* @return : signed integer value read from input in `char` format.
* allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
* Note : function will exit() program if digit sequence overflows */
static int readIntFromChar(const char** stringPtr) {
@@ -418,7 +407,7 @@ static int readIntFromChar(const char** stringPtr) {
/*! readSizeTFromCharChecked() :
* @return 0 if success, and store the result in *value.
* allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
* @return 1 if an overflow error occurs */
static int readSizeTFromCharChecked(const char** stringPtr, size_t* value)
@@ -433,22 +422,15 @@ static int readSizeTFromCharChecked(const char** stringPtr, size_t* value)
if (result < last) return 1; /* overflow error */
(*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M') || (**stringPtr=='G')) {
switch (**stringPtr) {
case 'K':
if (result > (((size_t)-1) >> 10)) return 1; /* overflow error */
if ((**stringPtr=='K') || (**stringPtr=='M')) {
size_t const maxK = ((size_t)(-1)) >> 10;
if (result > maxK) return 1; /* overflow error */
result <<= 10;
if (**stringPtr=='M') {
if (result > maxK) return 1; /* overflow error */
result <<= 10;
break;
case 'M':
if (result > (((size_t)-1) >> 20)) return 1; /* overflow error */
result <<= 20;
break;
case 'G':
if (result > (((size_t)-1) >> 30)) return 1; /* overflow error */
result <<= 30;
break;
}
(*stringPtr)++; /* skip `K`, `M` or `G` */
(*stringPtr)++; /* skip `K` or `M` */
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
@@ -458,7 +440,7 @@ static int readSizeTFromCharChecked(const char** stringPtr, size_t* value)
/*! readSizeTFromChar() :
* @return : size_t value read from input in `char` format.
* allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
* Note : function will exit() program if digit sequence overflows */
static size_t readSizeTFromChar(const char** stringPtr) {
@@ -654,29 +636,15 @@ static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressi
return 0;
}
DISPLAYLEVEL(4, "windowLog=%d, chainLog=%d, hashLog=%d, searchLog=%d \n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
DISPLAYLEVEL(4, "minMatch=%d, targetLength=%d, strategy=%d \n", params->minMatch, params->targetLength, params->strategy);
if (stringPtr[0] != 0) return 0; /* check the end of string */
return 1;
}
static void setMaxCompression(ZSTD_compressionParameters* params)
{
params->windowLog = ZSTD_WINDOWLOG_MAX;
params->chainLog = ZSTD_CHAINLOG_MAX;
params->hashLog = ZSTD_HASHLOG_MAX;
params->searchLog = ZSTD_SEARCHLOG_MAX;
params->minMatch = ZSTD_MINMATCH_MIN;
params->targetLength = ZSTD_TARGETLENGTH_MAX;
params->strategy = ZSTD_STRATEGY_MAX;
g_overlapLog = ZSTD_OVERLAPLOG_MAX;
g_ldmHashLog = ZSTD_LDM_HASHLOG_MAX;
g_ldmHashRateLog = 0; /* automatically derived */
g_ldmMinMatch = 16; /* heuristic */
g_ldmBucketSizeLog = ZSTD_LDM_BUCKETSIZELOG_MAX;
}
static void printVersion(void)
{
if (g_displayLevel < ZSTD_DISPLAY_LEVEL_DEFAULT) {
if (g_displayLevel < DISPLAY_LEVEL_DEFAULT) {
DISPLAYOUT("%s\n", ZSTD_VERSION_STRING);
return;
}
@@ -704,12 +672,6 @@ static void printVersion(void)
DISPLAYOUT("lz4 version %s\n", FIO_lz4Version());
DISPLAYOUT("lzma version %s\n", FIO_lzmaVersion());
#ifdef ZSTD_MULTITHREAD
DISPLAYOUT("supports Multithreading \n");
#else
DISPLAYOUT("single-thread operations only \n");
#endif
/* posix support */
#ifdef _POSIX_C_SOURCE
DISPLAYOUT("_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
@@ -720,10 +682,7 @@ static void printVersion(void)
#ifdef PLATFORM_POSIX_VERSION
DISPLAYOUT("PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
#endif
if (!ZSTD_isDeterministicBuild()) {
DISPLAYOUT("non-deterministic build\n");
} } }
} }
}
#define ZSTD_NB_STRATEGIES 9
@@ -737,7 +696,7 @@ static void printDefaultCParams(const char* filename, const char* dictFileName,
unsigned long long fileSize = UTIL_getFileSize(filename);
const size_t dictSize = dictFileName != NULL ? (size_t)UTIL_getFileSize(dictFileName) : 0;
const ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, fileSize, dictSize);
if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAY("%s (%llu bytes)\n", filename, fileSize);
if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAY("%s (%u bytes)\n", filename, (unsigned)fileSize);
else DISPLAY("%s (src size unknown)\n", filename);
DISPLAY(" - windowLog : %u\n", cParams.windowLog);
DISPLAY(" - chainLog : %u\n", cParams.chainLog);
@@ -770,7 +729,7 @@ static void printActualCParams(const char* filename, const char* dictFileName, i
/* Environment variables for parameter setting */
#define ENV_CLEVEL "ZSTD_CLEVEL"
#define ENV_NBWORKERS "ZSTD_NBTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */
#define ENV_NBTHREADS "ZSTD_NBTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */
/* pick up environment variable */
static int init_cLevel(void) {
@@ -800,29 +759,26 @@ static int init_cLevel(void) {
return ZSTDCLI_CLEVEL_DEFAULT;
}
static unsigned init_nbWorkers(unsigned defaultNbWorkers) {
#ifdef ZSTD_MULTITHREAD
const char* const env = getenv(ENV_NBWORKERS);
static unsigned init_nbThreads(void) {
const char* const env = getenv(ENV_NBTHREADS);
if (env != NULL) {
const char* ptr = env;
if ((*ptr>='0') && (*ptr<='9')) {
unsigned nbThreads;
if (readU32FromCharChecked(&ptr, &nbThreads)) {
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBWORKERS, env);
return defaultNbWorkers;
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBTHREADS, env);
return ZSTDCLI_NBTHREADS_DEFAULT;
} else if (*ptr == 0) {
return nbThreads;
}
}
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBWORKERS, env);
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBTHREADS, env);
}
return defaultNbWorkers;
#else
(void)defaultNbWorkers;
return 1;
#endif
return ZSTDCLI_NBTHREADS_DEFAULT;
}
#endif
#define NEXT_FIELD(ptr) { \
if (*argument == '=') { \
@@ -841,21 +797,21 @@ static unsigned init_nbWorkers(unsigned defaultNbWorkers) {
CLEAN_RETURN(1); \
} } }
#define NEXT_UINT32(_varu32) { \
#define NEXT_UINT32(val32) { \
const char* __nb; \
NEXT_FIELD(__nb); \
_varu32 = readU32FromChar(&__nb); \
val32 = readU32FromChar(&__nb); \
if(*__nb != 0) { \
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed"); \
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
} \
}
#define NEXT_TSIZE(_varTsize) { \
#define NEXT_TSIZE(valTsize) { \
const char* __nb; \
NEXT_FIELD(__nb); \
_varTsize = readSizeTFromChar(&__nb); \
valTsize = readSizeTFromChar(&__nb); \
if(*__nb != 0) { \
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed"); \
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
} \
}
@@ -893,25 +849,24 @@ int main(int argCount, const char* argv[])
singleThread = 0,
defaultLogicalCores = 0,
showDefaultCParams = 0,
contentSize = 1,
removeSrcFile = 0,
cLevel = init_cLevel(),
ultra = 0,
cLevelLast = MINCLEVEL - 1, /* for benchmark range */
setThreads_non1 = 0;
unsigned nbWorkers = init_nbWorkers(NBWORKERS_UNSET);
ZSTD_ParamSwitch_e mmapDict = ZSTD_ps_auto;
ZSTD_ParamSwitch_e useRowMatchFinder = ZSTD_ps_auto;
ultra=0,
contentSize=1,
removeSrcFile=0;
ZSTD_paramSwitch_e mmapDict=ZSTD_ps_auto;
ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
FIO_compressionType_t cType = FIO_zstdCompression;
unsigned nbWorkers = 0;
double compressibility = -1.0; /* lorem ipsum generator */
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t chunkSize = 0;
size_t blockSize = 0;
FIO_prefs_t* const prefs = FIO_createPreferences();
FIO_ctx_t* const fCtx = FIO_createContext();
FIO_progressSetting_e progress = FIO_ps_auto;
zstd_operation_mode operation = zom_compress;
ZSTD_compressionParameters compressionParams;
int cLevel = init_cLevel();
int cLevelLast = MINCLEVEL - 1; /* lower than minimum */
unsigned recursive = 0;
unsigned memLimit = 0;
FileNamesTable* filenames = UTIL_allocateFileNamesTable((size_t)argCount); /* argCount >= 1 */
@@ -939,7 +894,8 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NOBENCH
BMK_advancedParams_t benchParams = BMK_initAdvancedParams();
#endif
ZSTD_ParamSwitch_e literalCompressionMode = ZSTD_ps_auto;
ZSTD_paramSwitch_e literalCompressionMode = ZSTD_ps_auto;
/* init */
checkLibVersion();
@@ -948,9 +904,12 @@ int main(int argCount, const char* argv[])
assert(argCount >= 1);
if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAYLEVEL(1, "zstd: allocation error \n"); exit(1); }
programName = lastNameFromPath(programName);
#ifdef ZSTD_MULTITHREAD
nbWorkers = init_nbThreads();
#endif
/* preset behaviors */
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=NBWORKERS_AUTOCPU, singleThread=0;
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; FIO_setPassThroughFlag(prefs, 1); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
@@ -1055,16 +1014,6 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--fake-stderr-is-console")) { UTIL_fakeStderrIsConsole(); continue; }
if (!strcmp(argument, "--trace-file-stat")) { UTIL_traceFileStat(); continue; }
if (!strcmp(argument, "--max")) {
if (sizeof(void*)==4) {
DISPLAYLEVEL(2, "--max is incompatible with 32-bit mode \n");
badUsage(programName, originalArgument);
CLEAN_RETURN(1);
}
ultra=1; ldmFlag = 1; setMaxCompression(&compressionParams);
continue;
}
/* long commands with arguments */
#ifndef ZSTD_NODICT
if (longCommandWArg(&argument, "--train-cover")) {
@@ -1101,13 +1050,11 @@ int main(int argCount, const char* argv[])
continue;
}
#endif
if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); setThreads_non1 = (nbWorkers != 1); continue; }
if (longCommandWArg(&argument, "--threads")) { NEXT_UINT32(nbWorkers); continue; }
if (longCommandWArg(&argument, "--memlimit")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memory")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--memlimit-decompress")) { NEXT_UINT32(memLimit); continue; }
if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(chunkSize); continue; } /* hidden command, prefer --split below */
if (longCommandWArg(&argument, "--split")) { NEXT_TSIZE(chunkSize); continue; }
if (longCommandWArg(&argument, "--jobsize")) { NEXT_TSIZE(chunkSize); continue; } /* note: overloaded variable */
if (longCommandWArg(&argument, "--block-size")) { NEXT_TSIZE(blockSize); continue; }
if (longCommandWArg(&argument, "--maxdict")) { NEXT_UINT32(maxDictSize); continue; }
if (longCommandWArg(&argument, "--dictID")) { NEXT_UINT32(dictID); continue; }
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) { badUsage(programName, originalArgument); CLEAN_RETURN(1); } ; cType = FIO_zstdCompression; continue; }
@@ -1142,12 +1089,10 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NOTRACE
if (longCommandWArg(&argument, "--trace")) { char const* traceFile; NEXT_FIELD(traceFile); TRACE_enable(traceFile); continue; }
#endif
if (longCommandWArg(&argument, "--patch-from")) { NEXT_FIELD(patchFromDictFileName); ultra = 1; continue; }
if (longCommandWArg(&argument, "--patch-apply")) { operation=zom_decompress; NEXT_FIELD(patchFromDictFileName); memLimit= 1U << ZSTD_WINDOWLOG_MAX; continue; }
if (longCommandWArg(&argument, "--patch-from")) { NEXT_FIELD(patchFromDictFileName); continue; }
if (longCommandWArg(&argument, "--long")) {
unsigned ldmWindowLog = 0;
ldmFlag = 1;
ultra = 1;
/* Parse optional window log */
if (*argument == '=') {
++argument;
@@ -1291,10 +1236,10 @@ int main(int argCount, const char* argv[])
bench_nbSeconds = readU32FromChar(&argument);
break;
/* hidden shortcut for --split=# and --jobsize=# */
/* cut input into blocks (benchmark only) */
case 'B':
argument++;
chunkSize = readU32FromChar(&argument);
blockSize = readU32FromChar(&argument);
break;
/* benchmark files separately (hidden option) */
@@ -1309,7 +1254,6 @@ int main(int argCount, const char* argv[])
case 'T':
argument++;
nbWorkers = readU32FromChar(&argument);
setThreads_non1 = (nbWorkers != 1);
break;
/* Dictionary Selection level */
@@ -1354,11 +1298,11 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(3, WELCOME_MESSAGE);
#ifdef ZSTD_MULTITHREAD
if ((operation==zom_decompress) && (setThreads_non1)) {
if ((operation==zom_decompress) && (!singleThread) && (nbWorkers > 1)) {
DISPLAYLEVEL(2, "Warning : decompression does not support multi-threading\n");
}
if ((nbWorkers==NBWORKERS_AUTOCPU) && (!singleThread)) {
/* automatically set # workers based on # of reported cpu cores */
if ((nbWorkers==0) && (!singleThread)) {
/* automatically set # workers based on # of reported cpus */
if (defaultLogicalCores) {
nbWorkers = (unsigned)UTIL_countLogicalCores();
DISPLAYLEVEL(3, "Note: %d logical core(s) detected \n", nbWorkers);
@@ -1367,10 +1311,8 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
}
}
if (operation == zom_compress)
DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers);
#else
(void)singleThread; (void)nbWorkers; (void)defaultLogicalCores; (void)setThreads_non1;
(void)singleThread; (void)nbWorkers; (void)defaultLogicalCores;
#endif
g_utilDisplayLevel = g_displayLevel;
@@ -1397,7 +1339,7 @@ int main(int argCount, const char* argv[])
size_t const nbFileLists = file_of_names->tableSize;
size_t flNb;
for (flNb=0; flNb < nbFileLists; flNb++) {
FileNamesTable* const fnt = UTIL_createFileNamesTable_fromFileList(file_of_names->fileNames[flNb]);
FileNamesTable* const fnt = UTIL_createFileNamesTable_fromFileName(file_of_names->fileNames[flNb]);
if (fnt==NULL) {
DISPLAYLEVEL(1, "zstd: error reading %s \n", file_of_names->fileNames[flNb]);
CLEAN_RETURN(1);
@@ -1428,14 +1370,11 @@ int main(int argCount, const char* argv[])
/* Check if benchmark is selected */
if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH
int benchThreadsMsgLevel = 2;
if (nbWorkers == NBWORKERS_UNSET) nbWorkers = 1;
if (nbWorkers <= 1) benchThreadsMsgLevel = 3;
if (cType != FIO_zstdCompression) {
DISPLAYLEVEL(1, "benchmark mode is only compatible with zstd format \n");
CLEAN_RETURN(1);
}
benchParams.chunkSizeMax = chunkSize;
benchParams.blockSize = blockSize;
benchParams.targetCBlockSize = targetCBlockSize;
benchParams.nbWorkers = (int)nbWorkers;
benchParams.realTime = (unsigned)setRealTimePrio;
@@ -1452,34 +1391,31 @@ int main(int argCount, const char* argv[])
}
benchParams.literalCompressionMode = literalCompressionMode;
if (benchParams.mode == BMK_decodeOnly) cLevel = cLevelLast = 0;
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
if (cLevelLast < cLevel) cLevelLast = cLevel;
DISPLAYLEVEL(benchThreadsMsgLevel, "Benchmarking ");
if (filenames->tableSize > 1)
DISPLAYLEVEL(3, "%u files ", (unsigned)filenames->tableSize);
if (cLevelLast > cLevel) {
DISPLAYLEVEL(3, "from level %d to %d ", cLevel, cLevelLast);
} else {
DISPLAYLEVEL(3, "at level %d ", cLevel);
}
DISPLAYLEVEL(benchThreadsMsgLevel, "using %i threads \n", nbWorkers);
if (cLevelLast > cLevel)
DISPLAYLEVEL(3, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (filenames->tableSize > 0) {
if(separateFiles) {
unsigned i;
for(i = 0; i < filenames->tableSize; i++) {
operationResult = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel, &benchParams);
}
int c;
DISPLAYLEVEL(3, "Benchmarking %s \n", filenames->fileNames[i]);
for(c = cLevel; c <= cLevelLast; c++) {
operationResult = BMK_benchFilesAdvanced(&filenames->fileNames[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
} }
} else {
operationResult = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel, &benchParams);
}
for(; cLevel <= cLevelLast; cLevel++) {
operationResult = BMK_benchFilesAdvanced(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
} }
} else {
operationResult = BMK_syntheticTest(compressibility, cLevel, cLevelLast, &compressionParams, g_displayLevel, &benchParams);
}
for(; cLevel <= cLevelLast; cLevel++) {
operationResult = BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
} }
#else
(void)bench_nbSeconds; (void)chunkSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
#endif
goto _end;
}
@@ -1491,23 +1427,22 @@ int main(int argCount, const char* argv[])
zParams.compressionLevel = dictCLevel;
zParams.notificationLevel = (unsigned)g_displayLevel;
zParams.dictID = dictID;
if (nbWorkers == NBWORKERS_UNSET) nbWorkers = init_nbWorkers(ZSTDCLI_NBTHREADS_DEFAULT);
if (dict == cover) {
int const optimize = !coverParams.k || !coverParams.d;
coverParams.nbThreads = (unsigned)nbWorkers;
coverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, NULL, &coverParams, NULL, optimize, memLimit);
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, &coverParams, NULL, optimize, memLimit);
} else if (dict == fastCover) {
int const optimize = !fastCoverParams.k || !fastCoverParams.d;
fastCoverParams.nbThreads = (unsigned)nbWorkers;
fastCoverParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, NULL, NULL, &fastCoverParams, optimize, memLimit);
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, NULL, NULL, &fastCoverParams, optimize, memLimit);
} else {
ZDICT_legacy_params_t dictParams;
memset(&dictParams, 0, sizeof(dictParams));
dictParams.selectivityLevel = dictSelect;
dictParams.zParams = zParams;
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, chunkSize, &dictParams, NULL, NULL, 0, memLimit);
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenames->fileNames, (int)filenames->tableSize, blockSize, &dictParams, NULL, NULL, 0, memLimit);
}
#else
(void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */
@@ -1556,12 +1491,7 @@ int main(int argCount, const char* argv[])
/* check compression level limits */
{ int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
if (cLevel > maxCLevel) {
DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i. ", maxCLevel);
DISPLAYLEVEL(2, "Specify --ultra to raise the limit to 22 and use "
"--long=31 for maximum compression. Note that this "
"requires high amounts of memory, and the resulting data "
"might be rejected by third-party decoders and is "
"therefore only recommended for archival purposes. \n");
DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel);
cLevel = maxCLevel;
} }
#endif
@@ -1619,9 +1549,8 @@ int main(int argCount, const char* argv[])
#ifndef ZSTD_NOCOMPRESS
FIO_setCompressionType(prefs, cType);
FIO_setContentSize(prefs, contentSize);
if (nbWorkers == NBWORKERS_UNSET) nbWorkers = init_nbWorkers(ZSTDCLI_NBTHREADS_DEFAULT);
FIO_setNbWorkers(prefs, (int)nbWorkers);
FIO_setJobSize(prefs, (int)chunkSize);
FIO_setBlockSize(prefs, (int)blockSize);
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
+1 -2
View File
@@ -81,9 +81,8 @@ static void TRACE_log(char const* method, PTime duration, ZSTD_Trace const* trac
{
int level = 0;
int workers = 0;
double const durationAsDouble = duration ? (double)duration : .1;
double const ratio = (double)trace->uncompressedSize / (double)trace->compressedSize;
double const speed = ((double)trace->uncompressedSize * 1000) / durationAsDouble;
double const speed = ((double)trace->uncompressedSize * 1000) / (double)duration;
if (trace->params) {
ZSTD_CCtxParams_getParameter(trace->params, ZSTD_c_compressionLevel, &level);
ZSTD_CCtxParams_getParameter(trace->params, ZSTD_c_nbWorkers, &workers);
+1 -1
View File
@@ -1,5 +1,5 @@
.
.TH "ZSTDGREP" "1" "February 2025" "zstd 1.5.7" "User Commands"
.TH "ZSTDGREP" "1" "March 2024" "zstd 1.5.6" "User Commands"
.
.SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
+1 -1
View File
@@ -1,5 +1,5 @@
.
.TH "ZSTDLESS" "1" "February 2025" "zstd 1.5.7" "User Commands"
.TH "ZSTDLESS" "1" "March 2024" "zstd 1.5.6" "User Commands"
.
.SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files
-1
View File
@@ -5,7 +5,6 @@ fullbench-lib
fuzzer
fuzzer32
fuzzer-dll
largeDictionary
zbufftest
zbufftest32
zbufftest-dll
+25 -33
View File
@@ -20,15 +20,12 @@
# zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode
# ##########################################################################
ZSTD_LEGACY_SUPPORT ?= 0
ZSTD_LEGACY_SUPPORT ?= 5
export ZSTD_LEGACY_SUPPORT
DEBUGLEVEL ?= 2
export DEBUGLEVEL # transmit value to sub-makefiles
.PHONY: default
default: fullbench
LIBZSTD_MK_DIR := ../lib
include $(LIBZSTD_MK_DIR)/libzstd.mk
@@ -81,6 +78,9 @@ FUZZERTEST ?= -T200s
ZSTDRTTEST = --test-large-data
DECODECORPUS_TESTTIME ?= -T30
.PHONY: default
default: fullbench
.PHONY: all
all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash poolTests
@@ -148,14 +148,13 @@ fullbench32: CPPFLAGS += -m32
$(FULLBENCHS) : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
$(FULLBENCHS) : LDFLAGS += $(MULTITHREAD_LD)
$(FULLBENCHS) : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
$(FULLBENCHS) : DEBUGLEVEL = 0 # turn off assert() for speed measurements
$(FULLBENCHS) : $(ZSTD_FILES)
$(FULLBENCHS) : $(PRGDIR)/datagen.c $(PRGDIR)/lorem.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
$(FULLBENCHS) : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
$(LINK.c) $^ -o $@$(EXT)
CLEAN += fullbench-lib
fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/lorem.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(LIB_SRCDIR)/libzstd.a fullbench.c
fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(LIB_SRCDIR)/libzstd.a fullbench.c
$(LINK.c) $^ -o $@$(EXT)
# note : broken : requires symbols unavailable from dynamic library
@@ -225,9 +224,9 @@ roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c
CLEAN += longmatch
longmatch : $(ZSTD_OBJECTS) longmatch.c
CLEAN += largeDictionary
largeDictionary: CFLAGS += $(MULTITHREAD)
largeDictionary: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c largeDictionary.c
CLEAN += bigdict
bigdict: CFLAGS += $(MULTITHREAD)
bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c
CLEAN += invalidDictionaries
invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c
@@ -272,8 +271,8 @@ clean:
#----------------------------------------------------------------------------------
# valgrind tests validated only for some posix platforms
#----------------------------------------------------------------------------------
UNAME := $(shell sh -c 'MSYSTEM="MSYS" uname')
ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS AIX CYGWIN_NT%,$(UNAME)))
UNAME := $(shell uname)
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS AIX CYGWIN_NT))
HOST_OS = POSIX
.PHONY: test-valgrind
@@ -281,7 +280,7 @@ test-valgrind: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --err
test-valgrind: zstd datagen fuzzer fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
$(VALGRIND) ./datagen -g50M > $(VOID)
$(VALGRIND) $(PRGDIR)/zstd --fake-stdin-is-console ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
$(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
./datagen -g80 | $(VALGRIND) $(PRGDIR)/zstd - -c > $(VOID)
./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp
@@ -301,10 +300,10 @@ endif
#-----------------------------------------------------------------------------
# make tests validated only for below targets
#-----------------------------------------------------------------------------
ifneq (,$(filter MSYS POSIX,$(HOST_OS)))
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
DIFF:=diff
ifneq (,$(filter SunOS,$(UNAME)))
ifneq (,$(filter $(UNAME),SunOS))
DIFF:=gdiff
endif
@@ -312,12 +311,12 @@ endif
list:
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
.PHONY: shortest
shortest: ZSTDRTTEST= # remove long tests
shortest: test-zstd
.PHONY: check
check: ZSTDRTTEST= # remove long tests
check: test-zstd
@echo "\n******************************"
@echo "All tests completed successfully"
@echo "******************************"
check: shortest
.PHONY: fuzztest
fuzztest: test-fuzzer test-zstream test-decodecorpus
@@ -327,9 +326,6 @@ test: test-zstd test-cli-tests test-fullbench test-fuzzer test-zstream test-inva
ifeq ($(QEMU_SYS),)
test: test-pool
endif
@echo "\n******************************"
@echo "All tests completed successfully"
@echo "******************************"
.PHONY: test32
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
@@ -359,11 +355,7 @@ test-zstd test-zstd32 test-zstd-nolegacy test-zstd-dll: datagen
test-cli-tests: ZSTD = $(PRGDIR)/zstd
test-cli-tests: zstd datagen
file $(ZSTD)
./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen $(CLI_TEST_ARGS)
.PHONY: update-cli-tests
update-cli-tests: CLI_TEST_ARGS = --set-exact-output
update-cli-tests: test-cli-tests
./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen
.PHONY: test-fullbench
test-fullbench: fullbench datagen
@@ -399,8 +391,8 @@ test-zstream32: zstreamtest32
test-longmatch: longmatch
$(QEMU_SYS) ./longmatch
test-largeDictionary: largeDictionary
$(QEMU_SYS) ./largeDictionary
test-bigdict: bigdict
$(QEMU_SYS) ./bigdict
test-invalidDictionaries: invalidDictionaries
$(QEMU_SYS) ./invalidDictionaries
@@ -447,9 +439,9 @@ test-decodecorpus-cli: decodecorpus
test-pool: poolTests
$(QEMU_SYS) ./poolTests
test-lz4: ZSTD = $(PRGDIR)/zstd
test-lz4: ZSTD_LZ4 = ./lz4
test-lz4: ZSTD_UNLZ4 = ./unlz4
test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd
test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4
test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4
test-lz4: zstd decodecorpus datagen
[ -f lz4 ] || ln -s $(PRGDIR)/zstd lz4
[ -f unlz4 ] || ln -s $(PRGDIR)/zstd unlz4
+2 -2
View File
@@ -225,7 +225,7 @@ def get_regressions_dictionary(baseline_build, test_build, filenames_directory,
for j, level in enumerate(levels):
old_cspeed, old_dspeed = old[j]
new_cspeed, new_dspeed = new[j]
cspeed_reg, dspeed_reg, baseline_label, test_label = parse_regressions_and_labels(
cspeed_reg, dspeed_reg, baesline_label, test_label = parse_regressions_and_labels(
old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build
)
if cspeed_reg > CSPEED_REGRESSION_TOLERANCE:
@@ -315,7 +315,7 @@ if __name__ == "__main__":
quit()
if mode == "onetime":
main(filenames, levels, iterations, frequency=frequency, dictionary_filename=dictionary_filename)
main(filenames, levels, iterations, frequency=frequenc, dictionary_filename=dictionary_filename)
elif mode == "current":
builds = [{"user": None, "branch": "None", "hash": None}]
main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)
+15 -18
View File
@@ -70,38 +70,37 @@ int main(int argc, const char** argv)
char* buffer = (char*)malloc(bufferSize);
void* out = malloc(outSize);
void* roundtrip = malloc(dataSize);
int _exit_code = 1;
(void)argc;
(void)argv;
if (!buffer || !out || !roundtrip || !cctx || !dctx) {
fprintf(stderr, "Allocation failure\n");
goto cleanup;
return 1;
}
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 31)))
goto cleanup;
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)))
goto cleanup;
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_overlapLog, 9)))
goto cleanup;
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)))
goto cleanup;
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt)))
goto cleanup;
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, 7)))
goto cleanup;
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7)))
goto cleanup;
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, 1)))
goto cleanup;
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 10)))
goto cleanup;
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, 10)))
goto cleanup;
return 1;
if (ZSTD_isError(ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 31)))
goto cleanup;
return 1;
RDG_genBuffer(buffer, bufferSize, 1.0, 0.0, 0xbeefcafe);
@@ -111,21 +110,19 @@ int main(int argc, const char** argv)
for (i = 0; i < 10; ++i) {
fprintf(stderr, "Compressing 1 GB\n");
if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_continue))
goto cleanup;
return 1;
}
}
fprintf(stderr, "Compressing 1 GB\n");
if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_end))
goto cleanup;
return 1;
_exit_code = 0;
fprintf(stderr, "Success!\n");
cleanup:
free(roundtrip);
free(out);
free(buffer);
ZSTD_freeDCtx(dctx);
ZSTD_freeCCtx(cctx);
return _exit_code;
return 0;
}
+1 -2
View File
@@ -5,9 +5,8 @@ Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT*
Options:
-o OUTPUT Write output to a single file, OUTPUT.
-c, --stdout Write to STDOUT (even if it is a console) and keep the INPUT file(s).
-k, --keep Preserve INPUT file(s). *Default*
--rm Remove INPUT file(s) after successful (de)compression to file.
--rm Remove INPUT file(s) after successful (de)compression.
-# Desired compression level, where `#` is a number between 1 and 19;
lower numbers provide faster compression, higher numbers yield
-24
View File
@@ -35,30 +35,6 @@ rm file.zst
println "+ zstd --memory=1MiB file"
zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1G file"
zstd -q --memory=1G file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1GB file"
zstd -q --memory=1GB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1GiB file"
zstd -q --memory=1GiB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=3G file"
zstd -q --memory=3G file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=3GB file"
zstd -q --memory=3GB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=3GiB file"
zstd -q --memory=3GiB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=4G file"
zstd --memory=4G file && die "Should not allow out-of-bound numeric parameter"
println "+ zstd --memory=4GB file"
zstd --memory=4GB file && die "Should not allow out-of-bound numeric parameter"
println "+ zstd --memory=4GiB file"
zstd --memory=4GiB file && die "Should not allow out-of-bound numeric parameter"
rm file
exit 0
+6 -15
View File
@@ -1,9 +1,9 @@
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
Should allow numeric parameter without suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
@@ -11,12 +11,3 @@ Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
error: numeric value overflows 32-bit unsigned int
error: numeric value overflows 32-bit unsigned int
error: numeric value overflows 32-bit unsigned int
@@ -11,12 +11,3 @@
+ zstd --memory=1M file
+ zstd --memory=1MB file
+ zstd --memory=1MiB file
+ zstd --memory=1G file
+ zstd --memory=1GB file
+ zstd --memory=1GiB file
+ zstd --memory=3G file
+ zstd --memory=3GB file
+ zstd --memory=3GiB file
+ zstd --memory=4G file
+ zstd --memory=4GB file
+ zstd --memory=4GiB file
+1 -11
View File
@@ -18,16 +18,12 @@ esac
case "$UNAME" in
Darwin) MD5SUM="md5 -r" ;;
FreeBSD) MD5SUM="gmd5sum" ;;
NetBSD) MD5SUM="md5 -n" ;;
OpenBSD) MD5SUM="md5" ;;
*) MD5SUM="md5sum" ;;
esac
md5hash() {
$MD5SUM | dd bs=1 count=32 status=none
echo
}
DIFF="diff"
case "$UNAME" in
SunOS) DIFF="gdiff" ;;
@@ -39,9 +35,3 @@ then
else
hasMT="true"
fi
if zstd -vv --version | grep -q 'non-deterministic'; then
NON_DETERMINISTIC="true"
else
NON_DETERMINISTIC=""
fi
+1 -14
View File
@@ -5,27 +5,14 @@ set -v
datagen > file
# Retrieve the program's version information
# Note: command echoing differs between macos and linux, so it's disabled below
set +v
version_info=$(zstd -V)
set -v
# Compress with various levels and ensure that their sizes are ordered
zstd --fast=10 file -o file-f10.zst -q
zstd --fast=1 file -o file-f1.zst -q
zstd -1 file -o file-1.zst -q
zstd -19 file -o file-19.zst -q
if echo "$version_info" | grep -q '32-bit'; then
# skip --max test: not enough address space
cp file-19.zst file-max.zst
else
zstd --max file -o file-max.zst -q
fi
zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-max.zst
zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
cmp_size -le file-max.zst file-19.zst
cmp_size -lt file-19.zst file-1.zst
cmp_size -lt file-1.zst file-f1.zst
cmp_size -lt file-f1.zst file-f10.zst
@@ -1,26 +1,15 @@
datagen > file
# Retrieve the program's version information
# Note: command echoing differs between macos and linux, so it's disabled below
set +v
# Compress with various levels and ensure that their sizes are ordered
zstd --fast=10 file -o file-f10.zst -q
zstd --fast=1 file -o file-f1.zst -q
zstd -1 file -o file-1.zst -q
zstd -19 file -o file-19.zst -q
if echo "$version_info" | grep -q '32-bit'; then
# skip --max test: not enough address space
cp file-19.zst file-max.zst
else
zstd --max file -o file-max.zst -q
fi
zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-max.zst
5 files decompressed : 327685 bytes total
zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
4 files decompressed : 262148 bytes total
cmp_size -le file-max.zst file-19.zst
cmp_size -lt file-19.zst file-1.zst
cmp_size -lt file-1.zst file-f1.zst
cmp_size -lt file-f1.zst file-f10.zst
@@ -9,14 +9,7 @@ zstd --rsyncable -f file -q ; zstd -t file.zst
zstd -T0 -f file -q ; zstd -t file.zst
zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst
zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst
zstd -T0 --jobsize=1M -f file -q ; zstd -t file.zst
# multi-thread decompression warning test
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -d file.zst -o file4
# setting multi-thread via environment variable does not trigger decompression warning
zstd -T0 -f file -q ; zstd -t file.zst; ZSTD_NBTHREADS=0 zstd -df file.zst -o file3
zstd -T0 -f file -q ; zstd -t file.zst; ZSTD_NBTHREADS=2 zstd -df file.zst -o file4
# setting nbThreads==1 does not trigger decompression warning
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T1 -df file.zst -o file3
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -T1 -df file.zst -o file4
@@ -6,16 +6,6 @@ file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
Warning : decompression does not support multi-threading
file.zst : 65537 bytes
file.zst : 65537 bytes
Warning : decompression does not support multi-threading
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
file.zst : 65537 bytes
-36
View File
@@ -1,36 +0,0 @@
#!/bin/sh
. "$COMMON/platform.sh"
set -e
# To update checksums on version change run this from the tests/ directory
# make update-cli-tests
if [ -n "$NON_DETERMINISTIC" ]; then
# Skip tests if we have a non-deterministic build
cat "$CLI_TESTS/determinism/basic.sh.stdout.exact"
exit 0
fi
for level in $(seq 1 19); do
for file in $(ls files/); do
file="files/$file"
echo "level $level, file $file"
zstd --single-thread -q -$level $file -c | md5hash
done
done
for file in $(ls files/); do
file="files/$file"
echo "level 1, long=18, file $file"
zstd --long=18 --single-thread -q -1 $file -c | md5hash
echo "level 19, long=18, file $file"
zstd --long=18 --single-thread -q -19 $file -c | md5hash
done
for file in $(ls files/); do
file="files/$file"
echo "level -1, file $file"
zstd -q --single-thread --fast=1 $file -c | md5hash
done
@@ -1,880 +0,0 @@
level 1, file files/g0
5d80401e01d33084c65e94f93351e94c
level 1, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 1, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 1, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 1, file files/g1000
6bf2f4b179864fd8db4676037465feed
level 1, file files/g10000
2ae44c4053b2b47724c8f612dfb60d24
level 1, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 1, file files/g10000-P10
2d0eeab6a966098583a1dfeafb5090c1
level 1, file files/g10000-P100
93436482b4da30ce2300d356448c8990
level 1, file files/g10000-P25
c64b5f512c44b6a647da81753791b9a7
level 1, file files/g10000-P50
3982325490d90c8307e734c1b25790df
level 1, file files/g10000-P75
b7504f80fee16b5ba6a0f46571eb563a
level 1, file files/g10000-P90
350892bec7f7ad6a7d6af01c5d8b07c7
level 1, file files/g100000
daa38a869130494c077290cf54f2d895
level 1, file files/g1000000
a1d548531221d408b95dc5d9c600b3f0
level 1, file files/g20000
1da5b56511e8693867c0cdd962c521aa
level 1, file files/g200000
41fb3b3d46d4221f2f0b072c65dd6e0a
level 1, file files/g30000
788bc5abca5e33d79bb79a4eca98b9cd
level 1, file files/g50000
0bf9fafd84a2d56a788a9159f1f23f26
level 1, file files/g500000
73401d6df0657e091de20468f32579a9
level 2, file files/g0
5d80401e01d33084c65e94f93351e94c
level 2, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 2, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 2, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 2, file files/g1000
a14988fb331bc6f8b33d9eba3a0416dc
level 2, file files/g10000
a8656ebab20efa8a3fb281327aab5d58
level 2, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 2, file files/g10000-P10
9db9877242632964c232ebd4485dba07
level 2, file files/g10000-P100
93436482b4da30ce2300d356448c8990
level 2, file files/g10000-P25
d7a7223240a607fe28a2e8fd641a969d
level 2, file files/g10000-P50
86478908f9caa5ca4029454005063078
level 2, file files/g10000-P75
14f5dcdb347e87790d49c1eb99ac5069
level 2, file files/g10000-P90
05bdb42a2eeb5788d6288dc241cde12c
level 2, file files/g100000
12c6a4c50fad3e479e0f0ffe1d4df324
level 2, file files/g1000000
4f9cdc0de37b22d657fd9d3c61ec5b44
level 2, file files/g20000
1e70c3fe429c1af41e9cb8a536fb6df1
level 2, file files/g200000
bb1cc37142783345db29c3fd5838ce4e
level 2, file files/g30000
52528017fe0eec85b6c3244e6fceb4ed
level 2, file files/g50000
a38bdd671b6d3bd76b479dfc01d1c7fb
level 2, file files/g500000
cd2090a38bbd677b385238355c996c01
level 3, file files/g0
5d80401e01d33084c65e94f93351e94c
level 3, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 3, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 3, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 3, file files/g1000
3ec47dcb2d606b9fdef3f19b1304f8fe
level 3, file files/g10000
69a9d518b84fe2a66b57dfb4ab8905ae
level 3, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 3, file files/g10000-P10
ac9866ac355c4ed8939deb9fbeec1aef
level 3, file files/g10000-P100
93436482b4da30ce2300d356448c8990
level 3, file files/g10000-P25
5e1fe7a3831f6632bc8c9873ab8b633d
level 3, file files/g10000-P50
76295f181396a98565eb9cc69b96dc75
level 3, file files/g10000-P75
1f751dc70508e81fef197311e8583e99
level 3, file files/g10000-P90
47c7b061c299dc253c6aaaf489e936cb
level 3, file files/g100000
4b30b2be3394f03f1cf1f37a08dcec12
level 3, file files/g1000000
4301dea72cc4dd6162e46caa59788a09
level 3, file files/g20000
30456361833d27c0962c2faaa254e615
level 3, file files/g200000
0f01c07c57d60298dd54c6b6b197c3d3
level 3, file files/g30000
0b3c506a1b1b6ccbb54a852c370f5cdc
level 3, file files/g50000
81368c0b96bf1a2f318940b836b46074
level 3, file files/g500000
0c1ef9c6d3d75bfa0dd5d893f65da47c
level 4, file files/g0
5d80401e01d33084c65e94f93351e94c
level 4, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 4, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 4, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 4, file files/g1000
6a1214c19ab13d15934244a4655fa327
level 4, file files/g10000
09bbd556d9ee9f74f36b36bfa6b17325
level 4, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 4, file files/g10000-P10
1adb177b7a441c3585ffd781654758ba
level 4, file files/g10000-P100
93436482b4da30ce2300d356448c8990
level 4, file files/g10000-P25
3a62170477d7f17d64dc49b971388ff7
level 4, file files/g10000-P50
de93403eafd68786410fa52a95752514
level 4, file files/g10000-P75
f5466b3cfc3ba1e5afecb8d8bb91250f
level 4, file files/g10000-P90
132869db4091a9b9bdbc5236435a1d98
level 4, file files/g100000
dd92f22c593f40e0ccd6b31ad09d9d18
level 4, file files/g1000000
cfe7c063909c635f22438f83eb824082
level 4, file files/g20000
f8be159b057c6ee827f36cdd2ee5c43f
level 4, file files/g200000
71362cc7c28dcc730b591e3002fe888c
level 4, file files/g30000
f0b899995c6ba86066bd7f407fc708ef
level 4, file files/g50000
2f293e665d5161e68d8e1f86ac6841be
level 4, file files/g500000
42c5c45c595af60d5cf28f4cdd8ab392
level 5, file files/g0
5d80401e01d33084c65e94f93351e94c
level 5, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 5, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 5, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 5, file files/g1000
d47284586e181d455017d821bd3de1ef
level 5, file files/g10000
d6cd223f1c215d332654ca241a376549
level 5, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 5, file files/g10000-P10
66b935a7666ae668ac990d5c9aaac652
level 5, file files/g10000-P100
15fede9faa3d7c484dead66071fb8c5d
level 5, file files/g10000-P25
b292339cacea797a184554a2de3c1ed3
level 5, file files/g10000-P50
10b65af5125bf21a6810b8fb7a30b8b1
level 5, file files/g10000-P75
04fc60ea83db4b6101778f91a73d6040
level 5, file files/g10000-P90
302c8c7355082a7f40efe4ea40912411
level 5, file files/g100000
338f5e822a469774462f144d7ae371a0
level 5, file files/g1000000
46b8608a1f833477387e2fc844a817d5
level 5, file files/g20000
b7a468d363a7797d3d7a48577cd20213
level 5, file files/g200000
5ed09b3fd58ba540128319c02f9259cb
level 5, file files/g30000
fd4c0e5acb85271c101fd8ae1f411232
level 5, file files/g50000
2a14c76d629615bd50a370e587d13eb8
level 5, file files/g500000
cb1122167d4a747f3f8e0dd18bce6fa3
level 6, file files/g0
5d80401e01d33084c65e94f93351e94c
level 6, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 6, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 6, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 6, file files/g1000
19187f900e7540efdcab3735f489f718
level 6, file files/g10000
70c03e642fb98b896f2d96c0fa0eca74
level 6, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 6, file files/g10000-P10
70e2df33337095c818051d86ea68f188
level 6, file files/g10000-P100
15fede9faa3d7c484dead66071fb8c5d
level 6, file files/g10000-P25
dd4ed2ced4ae48672d5c8c0d4a7d58a5
level 6, file files/g10000-P50
9182b845f45f4990c8993e1c54e1e2e6
level 6, file files/g10000-P75
d3e6bb5a8e9790d23dda0a38b3def532
level 6, file files/g10000-P90
21da6c8aefa00b3e16f7674c642aa043
level 6, file files/g100000
350a0f07d7ddbcd0c629da0bcf734f02
level 6, file files/g1000000
d40c700beee149ab272b94c7ef0f3811
level 6, file files/g20000
335bc3f5767a7d4ad479408b0e289459
level 6, file files/g200000
032e3313ca247a8f4687059fa708284c
level 6, file files/g30000
a4b9fed2e1005298617125ee3b00cb06
level 6, file files/g50000
d7b3dab8d19fb992795c440d5f953c94
level 6, file files/g500000
44e59ccbf7a9794d4ea03f3f59f61ffd
level 7, file files/g0
5d80401e01d33084c65e94f93351e94c
level 7, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 7, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 7, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 7, file files/g1000
19187f900e7540efdcab3735f489f718
level 7, file files/g10000
495f1e5d30de3b34c5a76301875fa231
level 7, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 7, file files/g10000-P10
8a91fa2460ec9bd0387baa31beb81060
level 7, file files/g10000-P100
15fede9faa3d7c484dead66071fb8c5d
level 7, file files/g10000-P25
7dc0563f2e66a2e57c0f119e4d5e636e
level 7, file files/g10000-P50
e93a6eb179f7b26daa896328b2583b04
level 7, file files/g10000-P75
cb1e5facfc34fca7323000c6188ad657
level 7, file files/g10000-P90
e9984c6557f5a392f16e5c139dbe4994
level 7, file files/g100000
6e493dbe2985fec69860ef34728ac01a
level 7, file files/g1000000
455023a055f3a4de76c7295408ce3c8f
level 7, file files/g20000
2359efcd587e36ac9b74a8344bbf0e53
level 7, file files/g200000
ca80dbc88183a5aabb4aae955c449dff
level 7, file files/g30000
7e1be7e841c9fddd776dbe94b50db0ab
level 7, file files/g50000
09b80b58d70622de6f2354a06c15cb2a
level 7, file files/g500000
c90609f0558f0979eccb572ce0af4aa3
level 8, file files/g0
5d80401e01d33084c65e94f93351e94c
level 8, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 8, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 8, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 8, file files/g1000
19187f900e7540efdcab3735f489f718
level 8, file files/g10000
495f1e5d30de3b34c5a76301875fa231
level 8, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 8, file files/g10000-P10
8a91fa2460ec9bd0387baa31beb81060
level 8, file files/g10000-P100
15fede9faa3d7c484dead66071fb8c5d
level 8, file files/g10000-P25
526c95fd77cbcc2079720bb1b5515a54
level 8, file files/g10000-P50
50d4e0154067af438e56729716c3eda7
level 8, file files/g10000-P75
c94a94cce795566b065f2d3dca554c28
level 8, file files/g10000-P90
8f83ce03eb280cba48bafafbd066b752
level 8, file files/g100000
dbc4c43d9dc0937e67e92a908688a51e
level 8, file files/g1000000
5a4e6e44c89c3538fdb09823e649c8f6
level 8, file files/g20000
3653750b7ea2e8dc97d1d24ccec72153
level 8, file files/g200000
e978eb15e94cf221f1c7c55a059bbc4a
level 8, file files/g30000
9668f8c1292931955e14b81d67c2c648
level 8, file files/g50000
36a864d5bb5d9167d19ac237ce3bef51
level 8, file files/g500000
eaaac1f3c4861d4d501d9c92fa67cc08
level 9, file files/g0
5d80401e01d33084c65e94f93351e94c
level 9, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 9, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 9, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 9, file files/g1000
42d1f8aa9ee42135013077f09b77fd1a
level 9, file files/g10000
ae4166250554922e9ed27e436c13bf6b
level 9, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 9, file files/g10000-P10
bd9ede6738524ef732c9516146536acd
level 9, file files/g10000-P100
93436482b4da30ce2300d356448c8990
level 9, file files/g10000-P25
526c95fd77cbcc2079720bb1b5515a54
level 9, file files/g10000-P50
af6f029bd0d9baee1a78b8f7ffef5df3
level 9, file files/g10000-P75
f56895eec07a63269ea3eaeb59e87f40
level 9, file files/g10000-P90
c5cef2890886bfaf2d747b37ededc261
level 9, file files/g100000
f11c301dcf8c791e4f2db29e1b388eba
level 9, file files/g1000000
f90563e0d0962955d31af45fc8f2b19d
level 9, file files/g20000
49820710f3fbc6627b60071e91dba56d
level 9, file files/g200000
4689b0eb28c34619d8a9087cfa33fc88
level 9, file files/g30000
c14b029711fa6abc627ed3a051902b4c
level 9, file files/g50000
2d1cde6271f46081fbeaaba10f1c48bb
level 9, file files/g500000
eaaac1f3c4861d4d501d9c92fa67cc08
level 10, file files/g0
5d80401e01d33084c65e94f93351e94c
level 10, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 10, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 10, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 10, file files/g1000
42d1f8aa9ee42135013077f09b77fd1a
level 10, file files/g10000
ae4166250554922e9ed27e436c13bf6b
level 10, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 10, file files/g10000-P10
bd9ede6738524ef732c9516146536acd
level 10, file files/g10000-P100
93436482b4da30ce2300d356448c8990
level 10, file files/g10000-P25
526c95fd77cbcc2079720bb1b5515a54
level 10, file files/g10000-P50
6f821eef670933208d85daad6a9a2791
level 10, file files/g10000-P75
9c944c50c800d72294bdc9a9a452dc90
level 10, file files/g10000-P90
838a22c0731a67d185cac135b2bac5c7
level 10, file files/g100000
bbb2e91bdee192ae6f23404542a35bfb
level 10, file files/g1000000
1e52803b3311c2d4428c56d059901721
level 10, file files/g20000
939b50f472cf67c6dfbc4d3074e04011
level 10, file files/g200000
adc5f0c58c75e79e4bc4800c015d2401
level 10, file files/g30000
43e46ec5aae76c55c890a18f49b0163b
level 10, file files/g50000
65c57573aef37a38bd6c587ef4388800
level 10, file files/g500000
d3c45fce79c26bce889c3fa6a5480200
level 11, file files/g0
5d80401e01d33084c65e94f93351e94c
level 11, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 11, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 11, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 11, file files/g1000
4aa08662527d6d7f996705929e0dd8e0
level 11, file files/g10000
330f75029558d7cb6fa2756d85a998bd
level 11, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 11, file files/g10000-P10
990879c5b3d1c99aa5ef39f6049cebc9
level 11, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 11, file files/g10000-P25
26a10f96c2008608a33a731ff39cd9c8
level 11, file files/g10000-P50
129c6082daea0da5f52358c71b252750
level 11, file files/g10000-P75
7ff21b2c10548b9fc8ec5b61f86b9db0
level 11, file files/g10000-P90
0def9775620838420b295c07fd50a196
level 11, file files/g100000
c115057c51f9bb603a7d0c6bdf76e422
level 11, file files/g1000000
d28d3aed914ba7368dd004a9431c0173
level 11, file files/g20000
650cf27ffca49ed98f0e5dbb8997d7eb
level 11, file files/g200000
3210b89239e35e559be0ccd842577bb4
level 11, file files/g30000
09fe5a276e3fec09490c85962263fb26
level 11, file files/g50000
a9fa2677e4faab0eaa2334f278b3ec7d
level 11, file files/g500000
86815b1a35a46312a081f1af4946a62e
level 12, file files/g0
5d80401e01d33084c65e94f93351e94c
level 12, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 12, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 12, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 12, file files/g1000
6cb98f0115baf96a720638cf92b7803b
level 12, file files/g10000
b73485abce0df9802dcfeee652c45a8d
level 12, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 12, file files/g10000-P10
d00ac554fefc8a0f40420f908ff6fe69
level 12, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 12, file files/g10000-P25
9e20110ea128e4201c9a09eb48ad16c8
level 12, file files/g10000-P50
07287871c8b4ec3b5a925563a97451ce
level 12, file files/g10000-P75
2a47bb2eb7fb84a157f4525da9f78a77
level 12, file files/g10000-P90
d9c87f3f0faf212ec2d60cb611dac8a9
level 12, file files/g100000
d5d45068ded1319342ac3b7867d8c8fd
level 12, file files/g1000000
d28d3aed914ba7368dd004a9431c0173
level 12, file files/g20000
650cf27ffca49ed98f0e5dbb8997d7eb
level 12, file files/g200000
077d0f0ef9e9af86b1a298624fc70062
level 12, file files/g30000
09fe5a276e3fec09490c85962263fb26
level 12, file files/g50000
a9fa2677e4faab0eaa2334f278b3ec7d
level 12, file files/g500000
86815b1a35a46312a081f1af4946a62e
level 13, file files/g0
5d80401e01d33084c65e94f93351e94c
level 13, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 13, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 13, file files/g100
e6e88e42b891fbc82c87a60928d88e97
level 13, file files/g1000
773b6b59d472db0932b5ad1ad75eac64
level 13, file files/g10000
dfd1e66b36c80b191c338d0b14813920
level 13, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 13, file files/g10000-P10
40b5f47e143e235198f6408a099208e8
level 13, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 13, file files/g10000-P25
deaa99c2458ca8ea661b358ef837ef58
level 13, file files/g10000-P50
dd575cdc1ac009a7f2407013063d02e3
level 13, file files/g10000-P75
0d411eb9beee65c6f5dd7764b639097b
level 13, file files/g10000-P90
61db327f8fdd9a735e68259f05f71f0d
level 13, file files/g100000
dece3af6a1b7ef4ab80cad69119a3b61
level 13, file files/g1000000
916989f734410afce6b3ae72a3352e68
level 13, file files/g20000
7ce4e5c10392cbae28ed79b0707f627f
level 13, file files/g200000
ab3d3f2a0fb5d4e54f8e799213d7e995
level 13, file files/g30000
61f3ce2b87a584169e92791fa54c7361
level 13, file files/g50000
aba34c68e1d812fab8ce9863169e050b
level 13, file files/g500000
9f16a159f5fa88b875e5c1e0574a999c
level 14, file files/g0
5d80401e01d33084c65e94f93351e94c
level 14, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 14, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 14, file files/g100
e6e88e42b891fbc82c87a60928d88e97
level 14, file files/g1000
773b6b59d472db0932b5ad1ad75eac64
level 14, file files/g10000
dfd1e66b36c80b191c338d0b14813920
level 14, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 14, file files/g10000-P10
40b5f47e143e235198f6408a099208e8
level 14, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 14, file files/g10000-P25
deaa99c2458ca8ea661b358ef837ef58
level 14, file files/g10000-P50
246b1782db44f4226d32c8718e39ac8c
level 14, file files/g10000-P75
023a4c9c041fbcd4f584d5e65f9b4444
level 14, file files/g10000-P90
d57d3ce213e81d7ce80930115afc9f71
level 14, file files/g100000
29c1d79c3dc1dc51ec4ee3fd19fdbd10
level 14, file files/g1000000
7721785c341760a66ac9e2fb39cfeb33
level 14, file files/g20000
1055a0a808598d1eedf0442fceff44e0
level 14, file files/g200000
87df6235943fd09ba3f6a1f24cbc3a3a
level 14, file files/g30000
74c32e0cd8bcd62bd4e1cf599c193abf
level 14, file files/g50000
10d3bbb5e9822fa5c31fc2e79aeae7e9
level 14, file files/g500000
f8865d65f1790f723184007b2f940127
level 15, file files/g0
5d80401e01d33084c65e94f93351e94c
level 15, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 15, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 15, file files/g100
e6e88e42b891fbc82c87a60928d88e97
level 15, file files/g1000
773b6b59d472db0932b5ad1ad75eac64
level 15, file files/g10000
dfd1e66b36c80b191c338d0b14813920
level 15, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 15, file files/g10000-P10
40b5f47e143e235198f6408a099208e8
level 15, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 15, file files/g10000-P25
deaa99c2458ca8ea661b358ef837ef58
level 15, file files/g10000-P50
91cc04148eef18ae4222038ad05f0586
level 15, file files/g10000-P75
aa5e9e619fe89644098c9ad17a8b9ad4
level 15, file files/g10000-P90
28d6d71a33f507d1e7acd56d586e3981
level 15, file files/g100000
29c1d79c3dc1dc51ec4ee3fd19fdbd10
level 15, file files/g1000000
de1091b6021db006820019edfcb1384c
level 15, file files/g20000
1055a0a808598d1eedf0442fceff44e0
level 15, file files/g200000
0096718cf88b77da67fc2211bf85e3ca
level 15, file files/g30000
74c32e0cd8bcd62bd4e1cf599c193abf
level 15, file files/g50000
10d3bbb5e9822fa5c31fc2e79aeae7e9
level 15, file files/g500000
53aa49e42ffac4621397b88d4749c9c0
level 16, file files/g0
5d80401e01d33084c65e94f93351e94c
level 16, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 16, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 16, file files/g100
e6e88e42b891fbc82c87a60928d88e97
level 16, file files/g1000
f3eeeae11e293292fe1810d8745d0ae1
level 16, file files/g10000
23c5d58cca20ab65ff8a5d780f09e075
level 16, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 16, file files/g10000-P10
cced9c0f5709262da932fc8540bb72f4
level 16, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 16, file files/g10000-P25
6676c9b599408491f7abc00947782054
level 16, file files/g10000-P50
9af82247b7fe18900592ae4fd5f1890a
level 16, file files/g10000-P75
d44f2305f023fbff3d1a872ab740958a
level 16, file files/g10000-P90
517f98319c717526b5e211079f073fbc
level 16, file files/g100000
95ec61667f371a3621b34a042f1f4a0a
level 16, file files/g1000000
b71641629b58754f1961ac394f177d7c
level 16, file files/g20000
7bd4e03d703ead21e15fddd4423ef79e
level 16, file files/g200000
fd1c701e9872f304a71f0e7f7aa9f918
level 16, file files/g30000
bc2211b250b5f4ed0a7bd97d447132e0
level 16, file files/g50000
a6d98ebd2ed96688482c7d1181dded42
level 16, file files/g500000
44e339bb9e57842b77476f6d75ff3fbd
level 17, file files/g0
5d80401e01d33084c65e94f93351e94c
level 17, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 17, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 17, file files/g100
e6e88e42b891fbc82c87a60928d88e97
level 17, file files/g1000
f3eeeae11e293292fe1810d8745d0ae1
level 17, file files/g10000
23c5d58cca20ab65ff8a5d780f09e075
level 17, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 17, file files/g10000-P10
cced9c0f5709262da932fc8540bb72f4
level 17, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 17, file files/g10000-P25
6676c9b599408491f7abc00947782054
level 17, file files/g10000-P50
607968cd37d41631312307f6c8a37ab4
level 17, file files/g10000-P75
818c2b0124e0dad26f9d95d753b11140
level 17, file files/g10000-P90
2d01bb79185ec88e5d7e684d3983eff8
level 17, file files/g100000
95ec61667f371a3621b34a042f1f4a0a
level 17, file files/g1000000
cbd03f012664a02441271c50ba1330bf
level 17, file files/g20000
7bd4e03d703ead21e15fddd4423ef79e
level 17, file files/g200000
fd1c701e9872f304a71f0e7f7aa9f918
level 17, file files/g30000
bc2211b250b5f4ed0a7bd97d447132e0
level 17, file files/g50000
a6d98ebd2ed96688482c7d1181dded42
level 17, file files/g500000
01223076923dc7e3e06eeb3d71c467d2
level 18, file files/g0
5d80401e01d33084c65e94f93351e94c
level 18, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 18, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 18, file files/g100
e6e88e42b891fbc82c87a60928d88e97
level 18, file files/g1000
f3eeeae11e293292fe1810d8745d0ae1
level 18, file files/g10000
23c5d58cca20ab65ff8a5d780f09e075
level 18, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 18, file files/g10000-P10
cced9c0f5709262da932fc8540bb72f4
level 18, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 18, file files/g10000-P25
6676c9b599408491f7abc00947782054
level 18, file files/g10000-P50
c61ad9309da86f7229498e46daacd6d9
level 18, file files/g10000-P75
8bf1c42a6a1f46ff8549e87f7b6daa54
level 18, file files/g10000-P90
29077b7a282975006c20701bca0d439d
level 18, file files/g100000
95ec61667f371a3621b34a042f1f4a0a
level 18, file files/g1000000
1e598049810de86924756bcda8941085
level 18, file files/g20000
7bd4e03d703ead21e15fddd4423ef79e
level 18, file files/g200000
ab03b5ab84b6ebe79fa325118a895c6b
level 18, file files/g30000
bc2211b250b5f4ed0a7bd97d447132e0
level 18, file files/g50000
a6d98ebd2ed96688482c7d1181dded42
level 18, file files/g500000
1b5374095c0bdc676655fe4db94286d1
level 19, file files/g0
5d80401e01d33084c65e94f93351e94c
level 19, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 19, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 19, file files/g100
e6e88e42b891fbc82c87a60928d88e97
level 19, file files/g1000
f3eeeae11e293292fe1810d8745d0ae1
level 19, file files/g10000
23c5d58cca20ab65ff8a5d780f09e075
level 19, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 19, file files/g10000-P10
cced9c0f5709262da932fc8540bb72f4
level 19, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 19, file files/g10000-P25
6676c9b599408491f7abc00947782054
level 19, file files/g10000-P50
c61ad9309da86f7229498e46daacd6d9
level 19, file files/g10000-P75
d3171f297dfd08e444d481b30b1bbefa
level 19, file files/g10000-P90
87c4a59ec6e15f6da86b26969044a8a4
level 19, file files/g100000
7ae11dc1919d94e80979d41d12a0b578
level 19, file files/g1000000
d4b9ee8879d7f30bed3ec9e70520ca67
level 19, file files/g20000
70028792166e24282f5497592b632192
level 19, file files/g200000
ab03b5ab84b6ebe79fa325118a895c6b
level 19, file files/g30000
d698feea5119e141a08656439c4c1508
level 19, file files/g50000
b81384753f0db76004ac97681f6ef757
level 19, file files/g500000
0a25ba39483255a2c16899ab80a3ed8f
level 1, long=18, file files/g0
5d80401e01d33084c65e94f93351e94c
level 19, long=18, file files/g0
5d80401e01d33084c65e94f93351e94c
level 1, long=18, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 19, long=18, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 1, long=18, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 19, long=18, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 1, long=18, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 19, long=18, file files/g100
e6e88e42b891fbc82c87a60928d88e97
level 1, long=18, file files/g1000
6bf2f4b179864fd8db4676037465feed
level 19, long=18, file files/g1000
f3eeeae11e293292fe1810d8745d0ae1
level 1, long=18, file files/g10000
2ae44c4053b2b47724c8f612dfb60d24
level 19, long=18, file files/g10000
23c5d58cca20ab65ff8a5d780f09e075
level 1, long=18, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 19, long=18, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 1, long=18, file files/g10000-P10
6a8f6e75ab538eb08a422febc8b68006
level 19, long=18, file files/g10000-P10
cced9c0f5709262da932fc8540bb72f4
level 1, long=18, file files/g10000-P100
5db6fbe04a2de772ad4e49f5357d0543
level 19, long=18, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 1, long=18, file files/g10000-P25
6fd603b31365845346faab9cd64fd647
level 19, long=18, file files/g10000-P25
6676c9b599408491f7abc00947782054
level 1, long=18, file files/g10000-P50
d1de679e9a8c962ed1cc668a96bf930a
level 19, long=18, file files/g10000-P50
c61ad9309da86f7229498e46daacd6d9
level 1, long=18, file files/g10000-P75
dcf9822423e2ad42a569985707cd5ab4
level 19, long=18, file files/g10000-P75
f0ea19930e3866ff7a1932931346e0ac
level 1, long=18, file files/g10000-P90
87d249fd8c1ad0064d067a1e5ad97647
level 19, long=18, file files/g10000-P90
582e2c4c5a06e452d8dae22e7210c070
level 1, long=18, file files/g100000
daa38a869130494c077290cf54f2d895
level 19, long=18, file files/g100000
7ae11dc1919d94e80979d41d12a0b578
level 1, long=18, file files/g1000000
81b9ffd138f3b25254a7078b06950e5f
level 19, long=18, file files/g1000000
db54f06bf3abdbfd57892ab3aad18233
level 1, long=18, file files/g20000
1da5b56511e8693867c0cdd962c521aa
level 19, long=18, file files/g20000
70028792166e24282f5497592b632192
level 1, long=18, file files/g200000
17eb9ff8d912da9c445def85cb748e54
level 19, long=18, file files/g200000
ab03b5ab84b6ebe79fa325118a895c6b
level 1, long=18, file files/g30000
788bc5abca5e33d79bb79a4eca98b9cd
level 19, long=18, file files/g30000
d698feea5119e141a08656439c4c1508
level 1, long=18, file files/g50000
0bf9fafd84a2d56a788a9159f1f23f26
level 19, long=18, file files/g50000
b81384753f0db76004ac97681f6ef757
level 1, long=18, file files/g500000
f035223dc02f581daf5dd79773c4dc2b
level 19, long=18, file files/g500000
97093f718faf7b9094e27ea822fd1a29
level -1, file files/g0
5d80401e01d33084c65e94f93351e94c
level -1, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level -1, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level -1, file files/g100
eb805e7fdb95bc42231b20d0e12e2a53
level -1, file files/g1000
ac7c900e31b9e4fe37b7846a898c7d48
level -1, file files/g10000
55e1e6ed7fcf5581a17f72f3a32e0834
level -1, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level -1, file files/g10000-P10
0bf549bdfc20b644c37901918de0d0eb
level -1, file files/g10000-P100
93436482b4da30ce2300d356448c8990
level -1, file files/g10000-P25
b65475ee206e480b3adc9fcc7b79c54b
level -1, file files/g10000-P50
ae4bf85c8343052ecb1d666bd9766fbc
level -1, file files/g10000-P75
b0674333f519e71708f226f935ae63d5
level -1, file files/g10000-P90
2daa80aa34a8259fed7e04ae41397d6c
level -1, file files/g100000
586e12a3aa574f35aedf0afadee3040c
level -1, file files/g1000000
1cc8eb9ee1e4c20373fe3c1b0aa1a982
level -1, file files/g20000
b220e6387b1f5036c203f8be5f1a571c
level -1, file files/g200000
fc42db710c19a4f4a27f004fb4678d9c
level -1, file files/g30000
5350a7346d815f4ad42216b91ba8749a
level -1, file files/g50000
4f00a5a94e8f98c6972053057f6ac971
level -1, file files/g500000
4abb2e46c936c273955778c00f3eb492
@@ -1,45 +0,0 @@
#!/bin/sh
. "$COMMON/platform.sh"
set -e
# To update checksums on version change run this from the tests/ directory
# make update-cli-tests
if [ -n "$NON_DETERMINISTIC" ] || [ -z "$hasMT" ]; then
# Skip tests if we have a non-deterministic build
cat "$CLI_TESTS/determinism/multithread.sh.stdout.exact"
exit 0
fi
for level in 1 3 7 19; do
for file in $(ls files/); do
file="files/$file"
echo "level $level, file $file"
zstd -T2 -q -$level $file -c | md5hash
done
done
for file in $(ls files/); do
file="files/$file"
echo "level 1, long=18, file $file"
zstd --long=18 -T2 -q -1 $file -c | md5hash
echo "level 19, long=18, file $file"
zstd --long=18 -T2 -q -19 $file -c | md5hash
done
for file in $(ls files/); do
file="files/$file"
echo "Vary number of threads on $file"
zstd -qf -1 $file -o $file.zst.good
zstd -qf -T1 -1 $file
$DIFF $file.zst $file.zst.good
zstd -qf -T2 -1 $file
$DIFF $file.zst $file.zst.good
zstd -qf -T4 -1 $file
$DIFF $file.zst $file.zst.good
done
@@ -1,260 +0,0 @@
level 1, file files/g0
5d80401e01d33084c65e94f93351e94c
level 1, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 1, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 1, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 1, file files/g1000
6bf2f4b179864fd8db4676037465feed
level 1, file files/g10000
2ae44c4053b2b47724c8f612dfb60d24
level 1, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 1, file files/g10000-P10
2d0eeab6a966098583a1dfeafb5090c1
level 1, file files/g10000-P100
93436482b4da30ce2300d356448c8990
level 1, file files/g10000-P25
c64b5f512c44b6a647da81753791b9a7
level 1, file files/g10000-P50
3982325490d90c8307e734c1b25790df
level 1, file files/g10000-P75
b7504f80fee16b5ba6a0f46571eb563a
level 1, file files/g10000-P90
350892bec7f7ad6a7d6af01c5d8b07c7
level 1, file files/g100000
daa38a869130494c077290cf54f2d895
level 1, file files/g1000000
a1d548531221d408b95dc5d9c600b3f0
level 1, file files/g20000
1da5b56511e8693867c0cdd962c521aa
level 1, file files/g200000
41fb3b3d46d4221f2f0b072c65dd6e0a
level 1, file files/g30000
788bc5abca5e33d79bb79a4eca98b9cd
level 1, file files/g50000
0bf9fafd84a2d56a788a9159f1f23f26
level 1, file files/g500000
73401d6df0657e091de20468f32579a9
level 3, file files/g0
5d80401e01d33084c65e94f93351e94c
level 3, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 3, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 3, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 3, file files/g1000
3ec47dcb2d606b9fdef3f19b1304f8fe
level 3, file files/g10000
69a9d518b84fe2a66b57dfb4ab8905ae
level 3, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 3, file files/g10000-P10
ac9866ac355c4ed8939deb9fbeec1aef
level 3, file files/g10000-P100
93436482b4da30ce2300d356448c8990
level 3, file files/g10000-P25
5e1fe7a3831f6632bc8c9873ab8b633d
level 3, file files/g10000-P50
76295f181396a98565eb9cc69b96dc75
level 3, file files/g10000-P75
1f751dc70508e81fef197311e8583e99
level 3, file files/g10000-P90
47c7b061c299dc253c6aaaf489e936cb
level 3, file files/g100000
4b30b2be3394f03f1cf1f37a08dcec12
level 3, file files/g1000000
4301dea72cc4dd6162e46caa59788a09
level 3, file files/g20000
30456361833d27c0962c2faaa254e615
level 3, file files/g200000
0f01c07c57d60298dd54c6b6b197c3d3
level 3, file files/g30000
0b3c506a1b1b6ccbb54a852c370f5cdc
level 3, file files/g50000
81368c0b96bf1a2f318940b836b46074
level 3, file files/g500000
0c1ef9c6d3d75bfa0dd5d893f65da47c
level 7, file files/g0
5d80401e01d33084c65e94f93351e94c
level 7, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 7, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 7, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 7, file files/g1000
19187f900e7540efdcab3735f489f718
level 7, file files/g10000
495f1e5d30de3b34c5a76301875fa231
level 7, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 7, file files/g10000-P10
8a91fa2460ec9bd0387baa31beb81060
level 7, file files/g10000-P100
15fede9faa3d7c484dead66071fb8c5d
level 7, file files/g10000-P25
7dc0563f2e66a2e57c0f119e4d5e636e
level 7, file files/g10000-P50
e93a6eb179f7b26daa896328b2583b04
level 7, file files/g10000-P75
cb1e5facfc34fca7323000c6188ad657
level 7, file files/g10000-P90
e9984c6557f5a392f16e5c139dbe4994
level 7, file files/g100000
6e493dbe2985fec69860ef34728ac01a
level 7, file files/g1000000
455023a055f3a4de76c7295408ce3c8f
level 7, file files/g20000
2359efcd587e36ac9b74a8344bbf0e53
level 7, file files/g200000
ca80dbc88183a5aabb4aae955c449dff
level 7, file files/g30000
7e1be7e841c9fddd776dbe94b50db0ab
level 7, file files/g50000
09b80b58d70622de6f2354a06c15cb2a
level 7, file files/g500000
c90609f0558f0979eccb572ce0af4aa3
level 19, file files/g0
5d80401e01d33084c65e94f93351e94c
level 19, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 19, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 19, file files/g100
e6e88e42b891fbc82c87a60928d88e97
level 19, file files/g1000
f3eeeae11e293292fe1810d8745d0ae1
level 19, file files/g10000
23c5d58cca20ab65ff8a5d780f09e075
level 19, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 19, file files/g10000-P10
cced9c0f5709262da932fc8540bb72f4
level 19, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 19, file files/g10000-P25
6676c9b599408491f7abc00947782054
level 19, file files/g10000-P50
c61ad9309da86f7229498e46daacd6d9
level 19, file files/g10000-P75
d3171f297dfd08e444d481b30b1bbefa
level 19, file files/g10000-P90
87c4a59ec6e15f6da86b26969044a8a4
level 19, file files/g100000
7ae11dc1919d94e80979d41d12a0b578
level 19, file files/g1000000
d4b9ee8879d7f30bed3ec9e70520ca67
level 19, file files/g20000
70028792166e24282f5497592b632192
level 19, file files/g200000
ab03b5ab84b6ebe79fa325118a895c6b
level 19, file files/g30000
d698feea5119e141a08656439c4c1508
level 19, file files/g50000
b81384753f0db76004ac97681f6ef757
level 19, file files/g500000
0a25ba39483255a2c16899ab80a3ed8f
level 1, long=18, file files/g0
5d80401e01d33084c65e94f93351e94c
level 19, long=18, file files/g0
5d80401e01d33084c65e94f93351e94c
level 1, long=18, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 19, long=18, file files/g1
8436b07a7eb916ac9c48af847b7e612f
level 1, long=18, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 19, long=18, file files/g10
6cf177a3e6494ac3c76c857df6a9b31d
level 1, long=18, file files/g100
c94d1ef6bbec8b4899486b06207ee829
level 19, long=18, file files/g100
e6e88e42b891fbc82c87a60928d88e97
level 1, long=18, file files/g1000
6bf2f4b179864fd8db4676037465feed
level 19, long=18, file files/g1000
f3eeeae11e293292fe1810d8745d0ae1
level 1, long=18, file files/g10000
2ae44c4053b2b47724c8f612dfb60d24
level 19, long=18, file files/g10000
23c5d58cca20ab65ff8a5d780f09e075
level 1, long=18, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 19, long=18, file files/g10000-P0
6c2641d3b83775c50b766793b39967c4
level 1, long=18, file files/g10000-P10
6a8f6e75ab538eb08a422febc8b68006
level 19, long=18, file files/g10000-P10
cced9c0f5709262da932fc8540bb72f4
level 1, long=18, file files/g10000-P100
5db6fbe04a2de772ad4e49f5357d0543
level 19, long=18, file files/g10000-P100
049015191cf579b24653c8730fcc10f5
level 1, long=18, file files/g10000-P25
6fd603b31365845346faab9cd64fd647
level 19, long=18, file files/g10000-P25
6676c9b599408491f7abc00947782054
level 1, long=18, file files/g10000-P50
d1de679e9a8c962ed1cc668a96bf930a
level 19, long=18, file files/g10000-P50
c61ad9309da86f7229498e46daacd6d9
level 1, long=18, file files/g10000-P75
dcf9822423e2ad42a569985707cd5ab4
level 19, long=18, file files/g10000-P75
f0ea19930e3866ff7a1932931346e0ac
level 1, long=18, file files/g10000-P90
87d249fd8c1ad0064d067a1e5ad97647
level 19, long=18, file files/g10000-P90
582e2c4c5a06e452d8dae22e7210c070
level 1, long=18, file files/g100000
daa38a869130494c077290cf54f2d895
level 19, long=18, file files/g100000
7ae11dc1919d94e80979d41d12a0b578
level 1, long=18, file files/g1000000
2c410773bb02367d20d20569393faf1e
level 19, long=18, file files/g1000000
e3f5ef0204aedbac03e80d4300b3afc7
level 1, long=18, file files/g20000
1da5b56511e8693867c0cdd962c521aa
level 19, long=18, file files/g20000
70028792166e24282f5497592b632192
level 1, long=18, file files/g200000
17eb9ff8d912da9c445def85cb748e54
level 19, long=18, file files/g200000
ab03b5ab84b6ebe79fa325118a895c6b
level 1, long=18, file files/g30000
788bc5abca5e33d79bb79a4eca98b9cd
level 19, long=18, file files/g30000
d698feea5119e141a08656439c4c1508
level 1, long=18, file files/g50000
0bf9fafd84a2d56a788a9159f1f23f26
level 19, long=18, file files/g50000
b81384753f0db76004ac97681f6ef757
level 1, long=18, file files/g500000
f035223dc02f581daf5dd79773c4dc2b
level 19, long=18, file files/g500000
97093f718faf7b9094e27ea822fd1a29
Vary number of threads on files/g0
Vary number of threads on files/g1
Vary number of threads on files/g10
Vary number of threads on files/g100
Vary number of threads on files/g1000
Vary number of threads on files/g10000
Vary number of threads on files/g10000-P0
Vary number of threads on files/g10000-P10
Vary number of threads on files/g10000-P100
Vary number of threads on files/g10000-P25
Vary number of threads on files/g10000-P50
Vary number of threads on files/g10000-P75
Vary number of threads on files/g10000-P90
Vary number of threads on files/g100000
Vary number of threads on files/g1000000
Vary number of threads on files/g20000
Vary number of threads on files/g200000
Vary number of threads on files/g30000
Vary number of threads on files/g50000
Vary number of threads on files/g500000
-44
View File
@@ -1,44 +0,0 @@
#!/bin/sh
. "$COMMON/platform.sh"
set -e
# To update checksums on version change run this from the tests/ directory
# make update-cli-tests
if [ -n "$NON_DETERMINISTIC" ]; then
# Skip tests if we have a non-deterministic build
cat "$CLI_TESTS/determinism/reuse.sh.stdout.exact"
exit 0
fi
datagen -g0 > file0
datagen -g1 > file1
datagen -g1000 > file1000
datagen -g100000 > file100000
validate() {
$DIFF file0.zst file0.zst.good
$DIFF file1.zst file1.zst.good
$DIFF file1000.zst file1000.zst.good
$DIFF file100000.zst file100000.zst.good
}
# Check that context reuse doesn't impact determinism
for level in $(seq 1 19); do
echo $level
zstd -qf --single-thread -$level file0 -o file0.zst.good
zstd -qf --single-thread -$level file1 -o file1.zst.good
zstd -qf --single-thread -$level file1000 -o file1000.zst.good
zstd -qf --single-thread -$level file100000 -o file100000.zst.good
zstd -qf --single-thread -$level file0 file1 file1000 file100000
validate
zstd -qf --single-thread -$level file1 file0 file1000 file100000
validate
zstd -qf --single-thread -$level file1000 file1 file0 file100000
validate
zstd -qf --single-thread -$level file100000 file1000 file1 file0
validate
done
@@ -1,19 +0,0 @@
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
-5
View File
@@ -1,5 +0,0 @@
#!/bin/sh
set -e
cp -r ../files .
-30
View File
@@ -1,30 +0,0 @@
#!/bin/sh
set -e
. "$COMMON/platform.sh"
mkdir files/
datagen -g0 > files/g0
datagen -g1 > files/g1
datagen -g10 > files/g10
datagen -g100 > files/g100
datagen -g1000 > files/g1000
datagen -g10000 > files/g10000
datagen -g20000 > files/g20000
datagen -g30000 > files/g30000
datagen -g50000 > files/g50000
datagen -g100000 > files/g100000
datagen -g200000 > files/g200000
datagen -g500000 > files/g500000
datagen -g1000000 > files/g1000000
datagen -g10000 -P0 > files/g10000-P0
datagen -g10000 -P10 > files/g10000-P10
datagen -g10000 -P25 > files/g10000-P25
datagen -g10000 -P50 > files/g10000-P50
datagen -g10000 -P75 > files/g10000-P75
datagen -g10000 -P90 > files/g10000-P90
datagen -g10000 -P100 > files/g10000-P100
@@ -1,5 +1,5 @@
zstd --train
! Warning : nb of samples too low for proper processing !
! Please provide _one file per sample_.
! Alternatively, split file(s) into fixed-size samples, with --split=#
! Alternatively, split files into fixed-size blocks representative of samples, with -B#
Error 14 : nb of samples too low
@@ -22,9 +22,5 @@ Trace:FileStat: > UTIL_isRegularFile(out/file.zst)
Trace:FileStat: > UTIL_stat(-1, out/file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isFdRegularFile(-1)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isFileDescriptorPipe(out/file.zst)
Trace:FileStat: < 0
zstd: out/file.zst: Permission denied
zstd: can't stat out/file.zst : Permission denied -- ignored
@@ -22,10 +22,6 @@ Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isFdRegularFile(*)
Trace:FileStat: > UTIL_stat(*, )
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
@@ -14,10 +14,6 @@ Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isFdRegularFile(*)
Trace:FileStat: > UTIL_stat(*, )
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(-1, file.zst)
Trace:FileStat: < 1
@@ -22,10 +22,6 @@ Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isFdRegularFile(*)
Trace:FileStat: > UTIL_stat(*, )
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
@@ -14,10 +14,6 @@ Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isFdRegularFile(*)
Trace:FileStat: > UTIL_stat(*, )
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(-1, file)
Trace:FileStat: < 1
+1 -2
View File
@@ -640,7 +640,7 @@ if __name__ == "__main__":
help="Preserve the scratch directory TEST_DIR/scratch/ for debugging purposes."
)
parser.add_argument("--verbose", action="store_true", help="Verbose test output.")
parser.add_argument("--timeout", default=800, type=int, help="Test case timeout in seconds. Set to 0 to disable timeouts.")
parser.add_argument("--timeout", default=200, type=int, help="Test case timeout in seconds. Set to 0 to disable timeouts.")
parser.add_argument(
"--exec-prefix",
default=None,
@@ -705,7 +705,6 @@ if __name__ == "__main__":
env["DATAGEN_BIN"] = os.path.abspath(args.datagen)
env["ZSTDGREP_BIN"] = os.path.abspath(args.zstdgrep)
env["ZSTDLESS_BIN"] = os.path.abspath(args.zstdless)
env["CLI_TESTS"] = os.path.abspath(args.test_dir)
env["COMMON"] = os.path.abspath(os.path.join(args.test_dir, "common"))
env["PATH"] = bin_dir + ":" + os.getenv("PATH", "")
env["LC_ALL"] = "C"
+12 -74
View File
@@ -182,7 +182,7 @@ BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE];
BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2];
BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX];
SeqDef SEQUENCE_BUFFER[MAX_NB_SEQ];
seqDef SEQUENCE_BUFFER[MAX_NB_SEQ];
BYTE SEQUENCE_LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX]; /* storeSeq expects a place to copy literals to */
BYTE SEQUENCE_LLCODE[ZSTD_BLOCKSIZE_MAX];
BYTE SEQUENCE_MLCODE[ZSTD_BLOCKSIZE_MAX];
@@ -247,12 +247,6 @@ typedef enum {
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#endif
typedef enum {
lt_raw,
lt_rle,
lt_compressed,
} literalType_e;
/*-*******************************************************
* Global variables (set from command line)
*********************************************************/
@@ -265,11 +259,7 @@ U32 g_maxBlockSize = ZSTD_BLOCKSIZE_MAX; /* <= 128 KB */
struct {
int contentSize; /* force the content size to be present */
blockType_e *blockType; /* force specific block type */
literalType_e *literalType; /* force specific literals type */
int frame_header_only; /* generate only frame header */
int no_magic; /* do not generate magic number */
} opts;
} opts; /* advanced options on generation */
/* Generate and write a random frame header */
static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info)
@@ -298,19 +288,10 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info)
{
/* Generate random content size */
int force_block_type = opts.blockType != NULL;
size_t highBit;
if (RAND(seed) & 7 && g_maxDecompressedSizeLog > 7) {
/* do content of at least 128 bytes */
highBit = 1ULL << RAND_range(seed, 7, g_maxDecompressedSizeLog);
} else if (force_block_type) {
if ((RAND(seed) & 3) || (*(opts.blockType) == bt_rle)) {
/* do small content */
highBit = 1ULL << RAND_range(seed, 0, MIN(7, 1U << g_maxDecompressedSizeLog));
} else {
/* 0 size frame */
highBit = 0;
}
} else if (RAND(seed) & 3) {
/* do small content */
highBit = 1ULL << RAND_range(seed, 0, MIN(7, 1U << g_maxDecompressedSizeLog));
@@ -343,10 +324,8 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info)
}
/* write out the header */
if (!opts.no_magic) {
MEM_writeLE32(op + pos, ZSTD_MAGICNUMBER);
pos += 4;
}
{
/*
@@ -391,10 +370,8 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info)
/* Write a literal block in either raw or RLE form, return the literals size */
static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t contentSize)
{
int force_literal_type = opts.literalType != NULL;
int const type = (force_literal_type) ? *(opts.literalType) : RAND(seed) % 2;
BYTE* op = (BYTE*)frame->data;
int const type = RAND(seed) % 2;
int const sizeFormatDesc = RAND(seed) % 8;
size_t litSize;
size_t maxLitSize = MIN(contentSize, g_maxBlockSize);
@@ -505,7 +482,7 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
size_t compressedSize = 0;
size_t maxLitSize = MIN(contentSize-3, g_maxBlockSize);
SymbolEncodingType_e hType;
symbolEncodingType_e hType;
if (contentSize < 64) {
/* make sure we get reasonably-sized literals for compression */
@@ -642,22 +619,15 @@ static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t con
static size_t writeLiteralsBlock(U32* seed, frame_t* frame, size_t contentSize)
{
int select_compressed = 0;
if (opts.literalType) {
select_compressed = *(opts.literalType) == lt_compressed;
} else {
/* only do compressed for larger segments to avoid compressibility issues */
select_compressed = RAND(seed) & 7 && contentSize >= 64;
}
if (select_compressed) {
if (RAND(seed) & 7 && contentSize >= 64) {
return writeLiteralsBlockCompressed(seed, frame, contentSize);
} else {
return writeLiteralsBlockSimple(seed, frame, contentSize);
}
}
static inline void initSeqStore(SeqStore_t *seqStore) {
static inline void initSeqStore(seqStore_t *seqStore) {
seqStore->maxNbSeq = MAX_NB_SEQ;
seqStore->maxNbLit = ZSTD_BLOCKSIZE_MAX;
seqStore->sequencesStart = SEQUENCE_BUFFER;
@@ -671,7 +641,7 @@ static inline void initSeqStore(SeqStore_t *seqStore) {
/* Randomly generate sequence commands */
static U32
generateSequences(U32* seed, frame_t* frame, SeqStore_t* seqStore,
generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
size_t contentSize, size_t literalsSize, dictInfo info)
{
/* The total length of all the matches */
@@ -832,7 +802,7 @@ static int isSymbolSubset(const BYTE* symbols, size_t len, const BYTE* set, BYTE
return 1;
}
static size_t writeSequences(U32* seed, frame_t* frame, SeqStore_t* seqStorePtr,
static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
size_t nbSeq)
{
/* This code is mostly copied from ZSTD_compressSequences in zstd_compress.c */
@@ -842,7 +812,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, SeqStore_t* seqStorePtr,
FSE_CTable* CTable_OffsetBits = frame->stats.offcodeCTable;
FSE_CTable* CTable_MatchLength = frame->stats.matchlengthCTable;
U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
const SeqDef* const sequences = seqStorePtr->sequencesStart;
const seqDef* const sequences = seqStorePtr->sequencesStart;
const BYTE* const ofCodeTable = seqStorePtr->ofCode;
const BYTE* const llCodeTable = seqStorePtr->llCode;
const BYTE* const mlCodeTable = seqStorePtr->mlCode;
@@ -1028,7 +998,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, SeqStore_t* seqStorePtr,
static size_t writeSequencesBlock(U32* seed, frame_t* frame, size_t contentSize,
size_t literalsSize, dictInfo info)
{
SeqStore_t seqStore;
seqStore_t seqStore;
size_t numSequences;
@@ -1064,8 +1034,7 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize
static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
int lastBlock, dictInfo info)
{
int force_block_type = opts.blockType != NULL;
int const blockTypeDesc = (force_block_type) ? *(opts.blockType) : RAND(seed) % 8;
int const blockTypeDesc = RAND(seed) % 8;
size_t blockSize;
int blockType;
@@ -1104,7 +1073,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
frame->data = op;
compressedSize = writeCompressedBlock(seed, frame, contentSize, info);
if (compressedSize >= contentSize && !force_block_type) { /* compressed block must be strictly smaller than uncompressed one */
if (compressedSize >= contentSize) { /* compressed block must be strictly smaller than uncompressed one */
blockType = 0;
memcpy(op, frame->src, contentSize);
@@ -1275,11 +1244,7 @@ static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info)
DISPLAYLEVEL(3, "frame seed: %u\n", (unsigned)seed);
initFrame(fr);
writeFrameHeader(&seed, fr, info);
if (opts.frame_header_only)
return seed;
writeBlocks(&seed, fr, info);
writeChecksum(fr);
@@ -1807,9 +1772,6 @@ static void advancedUsage(const char* programName)
DISPLAY( " --max-block-size-log=# : max block size log, must be in range [2, 17]\n");
DISPLAY( " --max-content-size-log=# : max content size log, must be <= 20\n");
DISPLAY( " (this is ignored with gen-blocks)\n");
DISPLAY( " --block-type=# : force certain block type (raw=0, rle=1, compressed=2)\n");
DISPLAY( " --frame-header-only : dump only frame header\n");
DISPLAY( " --no-magic : do not add magic number\n");
}
/*! readU32FromChar() :
@@ -1931,18 +1893,6 @@ int main(int argc, char** argv)
U32 value = readU32FromChar(&argument);
g_maxDecompressedSizeLog =
MIN(MAX_DECOMPRESSED_SIZE_LOG, value);
} else if (longCommandWArg(&argument, "block-type=")) {
U32 value = readU32FromChar(&argument);
opts.blockType = malloc(sizeof(blockType_e));
*(opts.blockType) = value;
} else if (longCommandWArg(&argument, "literal-type=")) {
U32 value = readU32FromChar(&argument);
opts.literalType = malloc(sizeof(literalType_e));
*(opts.literalType) = value;
} else if (strcmp(argument, "frame-header-only") == 0) {
opts.frame_header_only = 1;
} else if (strcmp(argument, "no-magic") == 0) {
opts.no_magic = 1;
} else {
advancedUsage(argv[0]);
return 1;
@@ -1954,18 +1904,6 @@ int main(int argc, char** argv)
return 1;
} } } } /* for (argNb=1; argNb<argc; argNb++) */
if (opts.blockType) {
if ((opts.contentSize == 0) && (*(opts.blockType) == bt_rle)) {
DISPLAY("Error: content-size has to be used together with blockType=1 (rle block)\n");
return 1;
}
if (opts.literalType && (*(opts.blockType) != bt_compressed)) {
DISPLAY("Error: literal-type can be used only with blockType=2 (compressed block)\n");
return 1;
}
}
if (!seedset) {
seed = makeSeed();
}
+241 -471
View File
@@ -12,16 +12,14 @@
/*_************************************
* Includes
**************************************/
#define _CRT_SECURE_NO_WARNINGS /* disable Visual warning that it doesn't like fopen() */
#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still bench some deprecated functions */
#include <limits.h>
#include "util.h" /* Compiler options, UTIL_GetFileSize */
#include <stdlib.h> /* malloc */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <assert.h>
#include "timefn.h" /* UTIL_clockSpanNano, UTIL_getTime */
#include "mem.h" /* U32 */
#include "compress/zstd_compress_internal.h"
#ifndef ZSTD_DLL_IMPORT
#include "zstd_internal.h" /* ZSTD_decodeSeqHeaders, ZSTD_blockHeaderSize, ZSTD_getcBlockSize, blockType_e, KB, MB */
#include "decompress/zstd_decompress_internal.h" /* ZSTD_DCtx struct */
@@ -35,10 +33,10 @@
#include "zstd.h" /* ZSTD_versionString */
#include "util.h" /* time functions */
#include "datagen.h"
#include "lorem.h"
#include "benchfn.h" /* CustomBench */
#include "benchzstd.h" /* MB_UNIT */
/*_************************************
* Constants
**************************************/
@@ -53,7 +51,7 @@
#define DEFAULT_CLEVEL 1
#define COMPRESSIBILITY_DEFAULT (-1.0)
#define COMPRESSIBILITY_DEFAULT 0.50
static const size_t kSampleSizeDefault = 10000000;
#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */
@@ -66,7 +64,6 @@ static const size_t kSampleSizeDefault = 10000000;
#define CONTROL(c) { if (!(c)) { abort(); } } /* like assert(), but cannot be disabled */
/*_************************************
* Benchmark Parameters
**************************************/
@@ -100,6 +97,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
*********************************************************/
static ZSTD_CCtx* g_zcc = NULL;
static size_t
local_ZSTD_compress(const void* src, size_t srcSize,
void* dst, size_t dstSize,
@@ -131,88 +129,40 @@ local_ZSTD_compress_freshCCtx(const void* src, size_t srcSize,
}
}
typedef struct {
void* prepBuffer;
size_t prepSize;
void* dst;
size_t dstCapacity;
size_t fixedOrigSize; /* optional, 0 means "no modification" */
} PrepResult;
#define PREPRESULT_INIT { NULL, 0, NULL, 0, 0 }
static PrepResult prepDecompress(const void* src, size_t srcSize, int cLevel)
{
size_t prepCapacity = ZSTD_compressBound(srcSize);
void* prepBuffer = malloc(prepCapacity);
size_t cSize = ZSTD_compress(prepBuffer, prepCapacity, src, srcSize, cLevel);
void* dst = malloc(srcSize);
PrepResult r = PREPRESULT_INIT;
assert(dst != NULL);
r.prepBuffer = prepBuffer;
r.prepSize = cSize;
r.dst = dst;
r.dstCapacity = srcSize;
return r;
}
static size_t g_cSize = 0;
static size_t local_ZSTD_decompress(const void* src, size_t srcSize,
void* dst, size_t dstSize,
void* unused)
void* buff2)
{
(void)unused;
return ZSTD_decompress(dst, dstSize, src, srcSize);
(void)src; (void)srcSize;
return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
}
static ZSTD_DCtx* g_zdc = NULL; /* will be initialized within benchMem */
static size_t local_ZSTD_decompressDCtx(const void* src, size_t srcSize,
void* dst, size_t dstSize,
void* unused)
void* buff2)
{
(void)unused;
return ZSTD_decompressDCtx(g_zdc, dst, dstSize, src, srcSize);
(void)src; (void)srcSize;
return ZSTD_decompressDCtx(g_zdc, dst, dstSize, buff2, g_cSize);
}
#ifndef ZSTD_DLL_IMPORT
static PrepResult prepLiterals(const void* src, size_t srcSize, int cLevel)
{
PrepResult r = PREPRESULT_INIT;
size_t dstCapacity = srcSize;
void* dst = malloc(dstCapacity);
void* prepBuffer;
size_t prepSize = ZSTD_compress(dst, dstCapacity, src, srcSize, cLevel);
size_t frameHeaderSize = ZSTD_frameHeaderSize(dst, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
CONTROL(!ZSTD_isError(frameHeaderSize));
/* check block is compressible, hence contains a literals section */
{ blockProperties_t bp;
ZSTD_getcBlockSize((char*)dst+frameHeaderSize, dstCapacity, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("no compressed literals\n");
return r;
} }
{ size_t const skippedSize = frameHeaderSize + ZSTD_blockHeaderSize;
prepSize -= skippedSize;
prepBuffer = malloc(prepSize);
CONTROL(prepBuffer != NULL);
memmove(prepBuffer, (char*)dst+skippedSize, prepSize);
}
ZSTD_decompressBegin(g_zdc);
r.prepBuffer = prepBuffer;
r.prepSize = prepSize;
r.dst = dst;
r.dstCapacity = dstCapacity;
r.fixedOrigSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
return r;
}
extern size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx,
const void* src, size_t srcSize,
void* dst, size_t dstCapacity);
static size_t
local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* unused)
static size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
{
(void)unused;
return ZSTD_decodeLiteralsBlock_wrapper(g_zdc, src, srcSize, dst, dstCapacity);
(void)src; (void)srcSize; (void)dst; (void)dstSize;
return ZSTD_decodeLiteralsBlock_wrapper(g_zdc, buff2, g_cSize, dst, dstSize);
}
static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
{
int nbSeq;
(void)src; (void)srcSize; (void)dst; (void)dstSize;
return ZSTD_decodeSeqHeaders(g_zdc, &nbSeq, buff2, g_cSize);
}
FORCE_NOINLINE size_t ZSTD_decodeLiteralsHeader(ZSTD_DCtx* dctx, void const* src, size_t srcSize)
@@ -220,7 +170,7 @@ FORCE_NOINLINE size_t ZSTD_decodeLiteralsHeader(ZSTD_DCtx* dctx, void const* src
RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");
{
BYTE const* istart = (BYTE const*)src;
SymbolEncodingType_e const litEncType = (SymbolEncodingType_e)(istart[0] & 3);
symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
if (litEncType == set_compressed) {
RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3");
{
@@ -269,59 +219,11 @@ FORCE_NOINLINE size_t ZSTD_decodeLiteralsHeader(ZSTD_DCtx* dctx, void const* src
return 0;
}
static size_t
local_ZSTD_decodeLiteralsHeader(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* unused)
static size_t local_ZSTD_decodeLiteralsHeader(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
{
(void)dst; (void)dstCapacity; (void)unused;
return ZSTD_decodeLiteralsHeader(g_zdc, src, srcSize);
(void)dst, (void)dstSize, (void)src, (void)srcSize;
return ZSTD_decodeLiteralsHeader(g_zdc, buff2, g_cSize);
}
static PrepResult prepSequences1stBlock(const void* src, size_t srcSize, int cLevel)
{
PrepResult r = PREPRESULT_INIT;
size_t const dstCapacity = srcSize;
void* dst = malloc(dstCapacity);
const BYTE* ip = dst;
const BYTE* iend;
{ size_t const cSize = ZSTD_compress(dst, dstCapacity, src, srcSize, cLevel);
CONTROL(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
}
/* Skip frame Header */
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(dst, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
CONTROL(!ZSTD_isError(frameHeaderSize));
ip += frameHeaderSize;
}
/* Find end of block */
{ blockProperties_t bp;
size_t const cBlockSize = ZSTD_getcBlockSize(ip, dstCapacity, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("no compressed sequences\n");
return r;
}
iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */
}
ip += ZSTD_blockHeaderSize; /* skip block header */
ZSTD_decompressBegin(g_zdc);
CONTROL(iend > ip);
ip += ZSTD_decodeLiteralsBlock_wrapper(g_zdc, ip, (size_t)(iend-ip), dst, dstCapacity); /* skip literal segment */
r.prepSize = (size_t)(iend-ip);
r.prepBuffer = malloc(r.prepSize);
CONTROL(r.prepBuffer != NULL);
memmove(r.prepBuffer, ip, r.prepSize); /* copy rest of block (it starts by SeqHeader) */
r.dst = dst;
r.dstCapacity = dstCapacity;
r.fixedOrigSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
return r;
}
static size_t
local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* unused)
{
int nbSeq;
(void)unused; (void)dst; (void)dstCapacity;
return ZSTD_decodeSeqHeaders(g_zdc, &nbSeq, src, srcSize);
}
#endif
static ZSTD_CStream* g_cstream= NULL;
@@ -444,22 +346,23 @@ static ZSTD_DStream* g_dstream= NULL;
static size_t
local_ZSTD_decompressStream(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* unused)
void* buff2)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)unused;
(void)src; (void)srcSize;
ZSTD_initDStream(g_dstream);
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
buffIn.src = src;
buffIn.size = srcSize;
buffIn.src = buff2;
buffIn.size = g_cSize;
buffIn.pos = 0;
ZSTD_decompressStream(g_dstream, &buffOut, &buffIn);
return buffOut.pos;
}
#ifndef ZSTD_DLL_IMPORT
static size_t local_ZSTD_compressContinue(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* payload)
@@ -505,15 +408,15 @@ local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize,
static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* unused)
void* buff2)
{
size_t regeneratedSize = 0;
const BYTE* ip = (const BYTE*)src;
const BYTE* const iend = ip + srcSize;
const BYTE* ip = (const BYTE*)buff2;
const BYTE* const iend = ip + g_cSize;
BYTE* op = (BYTE*)dst;
size_t remainingCapacity = dstCapacity;
(void)unused;
(void)src; (void)srcSize; /* unused */
ZSTD_decompressBegin(g_zdc);
while (ip < iend) {
size_t const iSize = ZSTD_nextSrcSizeToDecompress(g_zdc);
@@ -526,300 +429,99 @@ static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize,
return regeneratedSize;
}
static PrepResult prepSequences(const void* src, size_t srcSize, int cLevel)
{
PrepResult r = PREPRESULT_INIT;
size_t const dstCapacity = ZSTD_compressBound(srcSize);
void* const dst = malloc(dstCapacity);
size_t const prepCapacity = dstCapacity * 4;
void* prepBuffer = malloc(prepCapacity);
void* sequencesStart = (char*)prepBuffer + 2*sizeof(unsigned);
ZSTD_Sequence* const seqs = sequencesStart;
size_t const seqsCapacity = prepCapacity / sizeof(ZSTD_Sequence);
size_t nbSeqs;
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel);
nbSeqs = ZSTD_generateSequences(g_zcc, seqs, seqsCapacity, src, srcSize);
CONTROL(srcSize < UINT_MAX);
MEM_write32(prepBuffer, (U32)srcSize);
MEM_write32((char*)prepBuffer+4, (U32)nbSeqs);
memcpy(seqs + nbSeqs, src, srcSize);
r.prepBuffer = prepBuffer;
r.prepSize = 8 + sizeof(ZSTD_Sequence)*nbSeqs + srcSize;
r.dst = dst;
r.dstCapacity = dstCapacity;
return r;
}
static size_t local_compressSequences(const void* input, size_t inputSize,
void* dst, size_t dstCapacity,
void* payload)
{
const char* ip = input;
size_t srcSize = MEM_read32(ip);
size_t nbSeqs = MEM_read32(ip+=4);
const ZSTD_Sequence* seqs = (const ZSTD_Sequence*)(const void*)(ip+=4);
const void* src = (ip+=nbSeqs * sizeof(ZSTD_Sequence));
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
assert(8 + nbSeqs * sizeof(ZSTD_Sequence) + srcSize == inputSize); (void)inputSize;
(void)payload;
return ZSTD_compressSequences(g_zcc, dst, dstCapacity, seqs, nbSeqs, src, srcSize);
}
static PrepResult prepSequencesAndLiterals(const void* src, size_t srcSize, int cLevel)
{
PrepResult r = PREPRESULT_INIT;
size_t const dstCapacity = ZSTD_compressBound(srcSize);
void* const dst = malloc(dstCapacity);
size_t const prepCapacity = dstCapacity * 4;
void* prepBuffer = malloc(prepCapacity);
void* sequencesStart = (char*)prepBuffer + 3*sizeof(unsigned);
ZSTD_Sequence* const seqs = sequencesStart;
size_t const seqsCapacity = prepCapacity / sizeof(ZSTD_Sequence);
size_t nbSeqs;
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel);
nbSeqs = ZSTD_generateSequences(g_zcc, seqs, seqsCapacity, src, srcSize);
CONTROL(srcSize < UINT_MAX);
MEM_write32(prepBuffer, (U32)srcSize);
MEM_write32((char*)prepBuffer+4, (U32)nbSeqs);
/* copy literals */
{ char* const litStart = (char*)(seqs + nbSeqs);
size_t nbLiterals = 0;
const char* ip = src;
size_t n;
for (n=0; n<nbSeqs; n++) {
size_t const litSize = seqs[n].litLength;
memcpy(litStart + nbLiterals, ip, litSize);
ip += litSize + seqs[n].matchLength;
nbLiterals += litSize;
}
MEM_write32((char*)prepBuffer+8, (U32)nbLiterals);
r.prepBuffer = prepBuffer;
r.prepSize = 12 + sizeof(ZSTD_Sequence)*nbSeqs + nbLiterals;
r.dst = dst;
r.dstCapacity = dstCapacity;
}
return r;
}
static size_t
local_compressSequencesAndLiterals(const void* input, size_t inputSize,
void* dst, size_t dstCapacity,
void* payload)
{
const char* ip = input;
size_t decompressedSize = MEM_read32(ip);
size_t nbSeqs = MEM_read32(ip+=4);
size_t nbLiterals = MEM_read32(ip+=4);
const ZSTD_Sequence* seqs = (const ZSTD_Sequence*)(const void*)(ip+=4);
const void* literals = (ip+=nbSeqs * sizeof(ZSTD_Sequence));
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
# if 0 /* for tests */
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_repcodeResolution, ZSTD_ps_enable);
#endif
assert(12 + nbSeqs * sizeof(ZSTD_Sequence) + nbLiterals == inputSize); (void)inputSize;
(void)payload;
return ZSTD_compressSequencesAndLiterals(g_zcc, dst, dstCapacity, seqs, nbSeqs, literals, nbLiterals, nbLiterals + 8, decompressedSize);
}
static PrepResult prepConvertSequences(const void* src, size_t srcSize, int cLevel)
{
PrepResult r = PREPRESULT_INIT;
size_t const prepCapacity = srcSize * 4;
void* prepBuffer = malloc(prepCapacity);
void* sequencesStart = (char*)prepBuffer + 2*sizeof(unsigned);
ZSTD_Sequence* const seqs = sequencesStart;
size_t const seqsCapacity = prepCapacity / sizeof(ZSTD_Sequence);
size_t totalNbSeqs, nbSeqs, blockSize=0;
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel);
totalNbSeqs = ZSTD_generateSequences(g_zcc, seqs, seqsCapacity, src, srcSize);
CONTROL(!ZSTD_isError(totalNbSeqs));
/* find nb sequences in first block */
{ size_t n;
for (n=0; n<totalNbSeqs; n++) {
if (seqs[n].matchLength == 0) break;
blockSize += seqs[n].litLength + seqs[n].matchLength;
}
blockSize += seqs[n].litLength;
nbSeqs = n+1;
#if 0
printf("found %zu sequences representing a first block of size %zu\n", nbSeqs, blockSize);
#endif
}
/* generate benchmarked input */
CONTROL(blockSize < UINT_MAX);
MEM_write32(prepBuffer, (U32)blockSize);
MEM_write32((char*)prepBuffer+4, (U32)nbSeqs);
memcpy(seqs + nbSeqs, src, srcSize);
r.prepBuffer = prepBuffer;
r.prepSize = 8 + sizeof(ZSTD_Sequence) * nbSeqs;
r.fixedOrigSize = blockSize;
return r;
}
static size_t
local_convertSequences(const void* input, size_t inputSize,
void* dst, size_t dstCapacity,
void* payload)
{
const char* ip = input;
size_t const blockSize = MEM_read32(ip);
size_t const nbSeqs = MEM_read32(ip+=4);
const ZSTD_Sequence* seqs = (const ZSTD_Sequence*)(const void*)(ip+=4);
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
ZSTD_resetSeqStore(&g_zcc->seqStore);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
# if 0 /* for tests */
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_repcodeResolution, ZSTD_ps_enable);
#endif
assert(8 + nbSeqs * sizeof(ZSTD_Sequence) == inputSize); (void)inputSize;
(void)dst; (void)dstCapacity;
(void)payload; (void)blockSize;
(void)ZSTD_convertBlockSequences(g_zcc, seqs, nbSeqs, 0);
return nbSeqs;
}
static size_t
check_compressedSequences(const void* compressed, size_t cSize, const void* orig, size_t origSize)
{
size_t decSize;
int diff;
void* decompressed = malloc(origSize);
if (decompressed == NULL) return 2;
decSize = ZSTD_decompress(decompressed, origSize, compressed, cSize);
if (decSize != origSize) { free(decompressed); DISPLAY("ZSTD_decompress failed (%u) ", (unsigned)decSize); return 1; }
diff = memcmp(decompressed, orig, origSize);
if (diff) { free(decompressed); return 1; }
free(decompressed);
return 0;
}
static size_t
local_get1BlockSummary(const void* input, size_t inputSize,
void* dst, size_t dstCapacity,
void* payload)
{
const char* ip = input;
size_t const blockSize = MEM_read32(ip);
size_t const nbSeqs = MEM_read32(ip+=4);
const ZSTD_Sequence* seqs = (const ZSTD_Sequence*)(const void*)(ip+=4);
ZSTD_CCtx_reset(g_zcc, ZSTD_reset_session_and_parameters);
ZSTD_resetSeqStore(&g_zcc->seqStore);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
assert(8 + nbSeqs * sizeof(ZSTD_Sequence) == inputSize); (void)inputSize;
(void)dst; (void)dstCapacity;
(void)payload; (void)blockSize;
(void)ZSTD_get1BlockSummary(seqs, nbSeqs);
return nbSeqs;
}
static PrepResult prepCopy(const void* src, size_t srcSize, int cLevel)
{
PrepResult r = PREPRESULT_INIT;
(void)cLevel;
r.prepSize = srcSize;
r.prepBuffer = malloc(srcSize);
CONTROL(r.prepBuffer != NULL);
memcpy(r.prepBuffer, src, srcSize);
r.dstCapacity = ZSTD_compressBound(srcSize);
r.dst = malloc(r.dstCapacity);
CONTROL(r.dst != NULL);
return r;
}
static PrepResult prepShorterDstCapacity(const void* src, size_t srcSize, int cLevel)
{
PrepResult r = prepCopy(src, srcSize, cLevel);
assert(r.dstCapacity > 1);
r.dstCapacity -= 1;
return r;
}
/*_*******************************************************
* List of Scenarios
* Bench functions
*********************************************************/
/* if PrepFunction_f returns PrepResult.prepBuffSize == 0, benchmarking is cancelled */
typedef PrepResult (*PrepFunction_f)(const void* src, size_t srcSize, int cLevel);
typedef size_t (*BenchedFunction_f)(const void* src, size_t srcSize, void* dst, size_t dstSize, void* opaque);
/* must return 0, otherwise verification is considered failed */
typedef size_t (*VerifFunction_f)(const void* processed, size_t procSize, const void* input, size_t inputSize);
typedef struct {
const char* name;
PrepFunction_f preparation_f;
BenchedFunction_f benched_f;
VerifFunction_f verif_f; /* optional */
} BenchScenario;
static BenchScenario kScenarios[] = {
{ "compress", NULL, local_ZSTD_compress, check_compressedSequences },
{ "decompress", prepDecompress, local_ZSTD_decompress, NULL },
{ "compress_freshCCtx", NULL, local_ZSTD_compress_freshCCtx, check_compressedSequences },
{ "decompressDCtx", prepDecompress, local_ZSTD_decompressDCtx, NULL },
{ "compressContinue", NULL, local_ZSTD_compressContinue, check_compressedSequences },
{ "compressContinue_extDict", NULL, local_ZSTD_compressContinue_extDict, NULL },
{ "decompressContinue", prepDecompress, local_ZSTD_decompressContinue, NULL },
{ "compressStream", NULL, local_ZSTD_compressStream, check_compressedSequences },
{ "compressStream_freshCCtx", NULL, local_ZSTD_compressStream_freshCCtx, check_compressedSequences },
{ "decompressStream", prepDecompress, local_ZSTD_decompressStream, NULL },
{ "compress2", NULL, local_ZSTD_compress2, check_compressedSequences },
{ "compressStream2, end", NULL, local_ZSTD_compressStream2_end, check_compressedSequences },
{ "compressStream2, end & short", prepShorterDstCapacity, local_ZSTD_compressStream2_end, check_compressedSequences },
{ "compressStream2, continue", NULL, local_ZSTD_compressStream2_continue, check_compressedSequences },
{ "compressStream2, -T2, continue", NULL, local_ZSTD_compress_generic_T2_continue, check_compressedSequences },
{ "compressStream2, -T2, end", NULL, local_ZSTD_compress_generic_T2_end, check_compressedSequences },
{ "compressSequences", prepSequences, local_compressSequences, check_compressedSequences },
{ "compressSequencesAndLiterals", prepSequencesAndLiterals, local_compressSequencesAndLiterals, check_compressedSequences },
{ "convertSequences (1st block)", prepConvertSequences, local_convertSequences, NULL },
{ "get1BlockSummary (1st block)", prepConvertSequences, local_get1BlockSummary, NULL },
#ifndef ZSTD_DLL_IMPORT
{ "decodeLiteralsHeader (1st block)", prepLiterals, local_ZSTD_decodeLiteralsHeader, NULL },
{ "decodeLiteralsBlock (1st block)", prepLiterals, local_ZSTD_decodeLiteralsBlock, NULL },
{ "decodeSeqHeaders (1st block)", prepSequences1stBlock, local_ZSTD_decodeSeqHeaders, NULL },
#endif
};
#define NB_SCENARIOS (sizeof(kScenarios) / sizeof(kScenarios[0]))
/*_*******************************************************
* Bench loop
*********************************************************/
static int benchMem(unsigned scenarioID,
const void* origSrc, size_t origSrcSize,
static int benchMem(unsigned benchNb,
const void* src, size_t srcSize,
int cLevel, ZSTD_compressionParameters cparams)
{
size_t dstCapacity = 0;
void* dst = NULL;
void* prepBuff = NULL;
size_t prepBuffSize = 0;
size_t dstBuffSize = ZSTD_compressBound(srcSize);
BYTE* dstBuff;
void* dstBuff2;
void* payload;
const char* benchName;
BMK_benchFn_t benchFunction;
PrepFunction_f prep_f;
VerifFunction_f verif_f;
int errorcode = 0;
if (scenarioID >= NB_SCENARIOS) return 0; /* scenario doesn't exist */
/* Selection */
switch(benchNb)
{
case 1:
benchFunction = local_ZSTD_compress; benchName = "compress";
break;
case 2:
benchFunction = local_ZSTD_decompress; benchName = "decompress";
break;
case 3:
benchFunction = local_ZSTD_compress_freshCCtx; benchName = "compress_freshCCtx";
break;
case 4:
benchFunction = local_ZSTD_decompressDCtx; benchName = "decompressDCtx";
break;
#ifndef ZSTD_DLL_IMPORT
case 11:
benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue";
break;
case 12:
benchFunction = local_ZSTD_compressContinue_extDict; benchName = "compressContinue_extDict";
break;
case 13:
benchFunction = local_ZSTD_decompressContinue; benchName = "decompressContinue";
break;
case 30:
benchFunction = local_ZSTD_decodeLiteralsHeader; benchName = "decodeLiteralsHeader";
break;
case 31:
benchFunction = local_ZSTD_decodeLiteralsBlock; benchName = "decodeLiteralsBlock";
break;
case 32:
benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "decodeSeqHeaders";
break;
#endif
case 41:
benchFunction = local_ZSTD_compressStream; benchName = "compressStream";
break;
case 42:
benchFunction = local_ZSTD_decompressStream; benchName = "decompressStream";
break;
case 43:
benchFunction = local_ZSTD_compressStream_freshCCtx; benchName = "compressStream_freshCCtx";
break;
case 50:
benchFunction = local_ZSTD_compress2; benchName = "compress2";
break;
case 51:
benchFunction = local_ZSTD_compressStream2_end; benchName = "compressStream2, end";
break;
case 52:
benchFunction = local_ZSTD_compressStream2_end; benchName = "compressStream2, end & short";
break;
case 53:
benchFunction = local_ZSTD_compressStream2_continue; benchName = "compressStream2, continue";
break;
case 61:
benchFunction = local_ZSTD_compress_generic_T2_continue; benchName = "compress_generic, -T2, continue";
break;
case 62:
benchFunction = local_ZSTD_compress_generic_T2_end; benchName = "compress_generic, -T2, end";
break;
default :
return 0;
}
benchName = kScenarios[scenarioID].name;
benchFunction = kScenarios[scenarioID].benched_f;
prep_f = kScenarios[scenarioID].preparation_f;
verif_f = kScenarios[scenarioID].verif_f;
if (prep_f == NULL) prep_f = prepCopy; /* default */
/* Initialization */
/* Allocation */
dstBuff = (BYTE*)malloc(dstBuffSize);
dstBuff2 = malloc(dstBuffSize);
if ((!dstBuff) || (!dstBuff2)) {
DISPLAY("\nError: not enough memory!\n");
free(dstBuff); free(dstBuff2);
return 12;
}
payload = dstBuff2;
if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
@@ -836,7 +538,7 @@ static int benchMem(unsigned scenarioID,
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, (int)cparams.searchLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, (int)cparams.minMatch);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, (int)cparams.targetLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_strategy, (int)cparams.strategy);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_strategy, cparams.strategy);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, (int)cparams.windowLog);
@@ -845,26 +547,114 @@ static int benchMem(unsigned scenarioID,
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, (int)cparams.searchLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, (int)cparams.minMatch);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, (int)cparams.targetLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_strategy, (int)cparams.strategy);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_strategy, cparams.strategy);
/* Preparation */
switch(benchNb)
{
case 1:
payload = &cparams;
{ PrepResult pr = prep_f(origSrc, origSrcSize, cLevel);
dst = pr.dst;
dstCapacity = pr.dstCapacity;
prepBuff = pr.prepBuffer;
prepBuffSize = pr.prepSize;
if (pr.fixedOrigSize) origSrcSize = pr.fixedOrigSize;
break;
case 2:
g_cSize = ZSTD_compress(dstBuff2, dstBuffSize, src, srcSize, cLevel);
break;
case 3:
payload = &cparams;
break;
case 4:
g_cSize = ZSTD_compress(dstBuff2, dstBuffSize, src, srcSize, cLevel);
break;
#ifndef ZSTD_DLL_IMPORT
case 11:
payload = &cparams;
break;
case 12:
payload = &cparams;
break;
case 13 :
g_cSize = ZSTD_compress(dstBuff2, dstBuffSize, src, srcSize, cLevel);
break;
case 30: /* ZSTD_decodeLiteralsHeader */
/* fall-through */
case 31: /* ZSTD_decodeLiteralsBlock : starts literals block in dstBuff2 */
{ size_t frameHeaderSize;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
CONTROL(!ZSTD_isError(frameHeaderSize));
/* check block is compressible, hence contains a literals section */
{ blockProperties_t bp;
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
goto _cleanOut;
} }
{ size_t const skippedSize = frameHeaderSize + ZSTD_blockHeaderSize;
memcpy(dstBuff2, dstBuff+skippedSize, g_cSize-skippedSize);
}
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
ZSTD_decompressBegin(g_zdc);
break;
}
case 32: /* ZSTD_decodeSeqHeaders */
{ blockProperties_t bp;
const BYTE* ip = dstBuff;
const BYTE* iend;
{ size_t const cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
CONTROL(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
}
/* Skip frame Header */
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1));
CONTROL(!ZSTD_isError(frameHeaderSize));
ip += frameHeaderSize;
}
/* Find end of block */
{ size_t const cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
goto _cleanOut;
}
iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */
}
ip += ZSTD_blockHeaderSize; /* skip block header */
ZSTD_decompressBegin(g_zdc);
CONTROL(iend > ip);
ip += ZSTD_decodeLiteralsBlock_wrapper(g_zdc, ip, (size_t)(iend-ip), dstBuff, dstBuffSize); /* skip literal segment */
g_cSize = (size_t)(iend-ip);
memcpy(dstBuff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
break;
}
#else
case 31:
goto _cleanOut;
#endif
case 41 :
payload = &cparams;
break;
case 42 :
g_cSize = ZSTD_compress(payload, dstBuffSize, src, srcSize, cLevel);
break;
case 43 :
payload = &cparams;
break;
case 52 :
/* compressStream2, short dstCapacity */
dstBuffSize--;
break;
/* test functions */
/* convention: test functions have ID > 100 */
default : ;
}
if (prepBuffSize==0) goto _cleanOut; /* failed preparation */
/* warming up dstBuff */
{ size_t i; for (i=0; i<dstCapacity; i++) ((BYTE*)dst)[i]=(BYTE)i; }
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; }
/* benchmark loop */
{ BMK_timedFnState_t* const tfs = BMK_createTimedFnState(g_nbIterations * 1000, 1000);
void* const avoidStrictAliasingPtr = &dst;
const void* prepSrc = prepBuff;
void* const avoidStrictAliasingPtr = &dstBuff;
BMK_benchParams_t bp;
BMK_runTime_t bestResult;
bestResult.sumOfReturn = 0;
@@ -877,38 +667,29 @@ static int benchMem(unsigned scenarioID,
bp.initPayload = NULL;
bp.errorFn = ZSTD_isError;
bp.blockCount = 1;
bp.srcBuffers = &prepSrc;
bp.srcSizes = &prepBuffSize;
bp.srcBuffers = &src;
bp.srcSizes = &srcSize;
bp.dstBuffers = (void* const*) avoidStrictAliasingPtr; /* circumvent strict aliasing warning on gcc-8,
* because gcc considers that `void* const *` and `void**` are 2 different types */
bp.dstCapacities = &dstCapacity;
bp.dstCapacities = &dstBuffSize;
bp.blockResults = NULL;
for (;;) {
BMK_runOutcome_t const bOutcome = BMK_benchTimedFn(tfs, bp);
if (!BMK_isSuccessful_runOutcome(bOutcome)) {
DISPLAY("ERROR: Scenario %u: %s \n", scenarioID, ZSTD_getErrorName(BMK_extract_errorResult(bOutcome)));
DISPLAY("ERROR benchmarking function ! ! \n");
errorcode = 1;
BMK_freeTimedFnState(tfs);
goto _cleanOut;
}
{ BMK_runTime_t const newResult = BMK_extract_runTime(bOutcome);
if (newResult.nanoSecPerRun < bestResult.nanoSecPerRun )
bestResult.nanoSecPerRun = newResult.nanoSecPerRun;
DISPLAY("\r%2u#%-31.31s:%8.1f MB/s (%8u) ",
scenarioID, benchName,
(double)origSrcSize * TIMELOOP_NANOSEC / bestResult.nanoSecPerRun / MB_UNIT,
DISPLAY("\r%2u#%-29.29s:%8.1f MB/s (%8u) ",
benchNb, benchName,
(double)srcSize * TIMELOOP_NANOSEC / bestResult.nanoSecPerRun / MB_UNIT,
(unsigned)newResult.sumOfReturn );
if (verif_f) {
size_t const vRes = verif_f(dst, newResult.sumOfReturn, origSrc, origSrcSize);
if (vRes) {
DISPLAY(" validation failed ! (%u)\n", (unsigned)vRes);
break;
}
}
}
if ( BMK_isCompleted_TimedFn(tfs) ) break;
@@ -918,8 +699,8 @@ static int benchMem(unsigned scenarioID,
DISPLAY("\n");
_cleanOut:
free(prepBuff);
free(dst);
free(dstBuff);
free(dstBuff2);
ZSTD_freeCCtx(g_zcc); g_zcc=NULL;
ZSTD_freeDCtx(g_zdc); g_zdc=NULL;
ZSTD_freeCStream(g_cstream); g_cstream=NULL;
@@ -928,13 +709,7 @@ _cleanOut:
}
#define BENCH_ALL_SCENARIOS 999
/*
* if @compressibility < 0.0, use Lorem Ipsum generator
* otherwise, @compressibility is expected to be between 0.0 and 1.0
* if scenarioID == BENCH_ALL_SCENARIOS, all scenarios will be run on the sample
*/
static int benchSample(U32 scenarioID,
static int benchSample(U32 benchNb,
size_t benchedSize, double compressibility,
int cLevel, ZSTD_compressionParameters cparams)
{
@@ -943,30 +718,24 @@ static int benchSample(U32 scenarioID,
if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; }
/* Fill buffer */
if (compressibility < 0.0) {
LOREM_genBuffer(origBuff, benchedSize, 0);
} else {
RDG_genBuffer(origBuff, benchedSize, compressibility, 0.0, 0);
}
/* bench */
DISPLAY("\r%70s\r", "");
DISPLAY(" Sample %u bytes : \n", (unsigned)benchedSize);
if (scenarioID == BENCH_ALL_SCENARIOS) {
for (scenarioID=0; scenarioID<100; scenarioID++) {
benchMem(scenarioID, origBuff, benchedSize, cLevel, cparams);
}
} else {
benchMem(scenarioID, origBuff, benchedSize, cLevel, cparams);
}
if (benchNb) {
benchMem(benchNb, origBuff, benchedSize, cLevel, cparams);
} else { /* 0 == run all tests */
for (benchNb=0; benchNb<100; benchNb++) {
benchMem(benchNb, origBuff, benchedSize, cLevel, cparams);
} }
free(origBuff);
return 0;
}
static int benchFiles(U32 scenarioID,
static int benchFiles(U32 benchNb,
const char** fileNamesTable, const int nbFiles,
int cLevel, ZSTD_compressionParameters cparams)
{
@@ -1012,12 +781,13 @@ static int benchFiles(U32 scenarioID,
/* bench */
DISPLAY("\r%70s\r", ""); /* blank line */
DISPLAY(" %s : \n", inFileName);
if (scenarioID == BENCH_ALL_SCENARIOS) {
for (scenarioID=0; scenarioID<100; scenarioID++) {
benchMem(scenarioID, origBuff, benchedSize, cLevel, cparams);
}
if (benchNb) {
benchMem(benchNb, origBuff, benchedSize, cLevel, cparams);
} else {
benchMem(scenarioID, origBuff, benchedSize, cLevel, cparams);
for (benchNb=0; benchNb<100; benchNb++) {
benchMem(benchNb, origBuff, benchedSize, cLevel, cparams);
}
benchNb = 0;
}
free(origBuff);
@@ -1107,7 +877,7 @@ int main(int argc, const char** argv)
int argNb, filenamesStart=0, result;
const char* const exename = argv[0];
const char* input_filename = NULL;
U32 scenarioID = BENCH_ALL_SCENARIOS, main_pause = 0;
U32 benchNb = 0, main_pause = 0;
int cLevel = DEFAULT_CLEVEL;
ZSTD_compressionParameters cparams = ZSTD_getCParams(cLevel, 0, 0);
size_t sampleSize = kSampleSizeDefault;
@@ -1158,7 +928,7 @@ int main(int argc, const char** argv)
/* Select specific algorithm to bench */
case 'b':
argument++;
scenarioID = readU32FromChar(&argument);
benchNb = readU32FromChar(&argument);
break;
/* Select compression level to use */
@@ -1200,9 +970,9 @@ int main(int argc, const char** argv)
if (filenamesStart==0) /* no input file */
result = benchSample(scenarioID, sampleSize, compressibility, cLevel, cparams);
result = benchSample(benchNb, sampleSize, compressibility, cLevel, cparams);
else
result = benchFiles(scenarioID, argv+filenamesStart, argc-filenamesStart, cLevel, cparams);
result = benchFiles(benchNb, argv+filenamesStart, argc-filenamesStart, cLevel, cparams);
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
-2
View File
@@ -23,8 +23,6 @@ fuzz-*.log
rt_lib_*
d_lib_*
crash-*
decompress_cross_format
generate_sequences
# misc
trace
+7 -8
View File
@@ -17,7 +17,7 @@ LDFLAGS ?=
ARFLAGS ?=
LIB_FUZZING_ENGINE ?= libregression.a
PYTHON ?= python
ifeq ($(shell sh -c 'MSYSTEM="MSYS" uname') , Darwin)
ifeq ($(shell uname), Darwin)
DOWNLOAD?=curl -L -o
else
DOWNLOAD?=wget -O
@@ -28,9 +28,6 @@ LIBZSTD_MK_DIR = ../../lib
DEBUGLEVEL ?= 2
ZSTD_LEGACY_SUPPORT ?= 1
.PHONY: default
default: all
include $(LIBZSTD_MK_DIR)/libzstd.mk
PRGDIR = ../../programs
@@ -104,6 +101,10 @@ FUZZ_RT_OBJ9 := $(FUZZ_RT_OBJ8:.c=.o)
FUZZ_RT_OBJ10 := $(THIRD_PARTY_SEQ_PROD_OBJ) $(FUZZ_RT_OBJ9)
FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ10:.S=.o)
.PHONY: default all clean cleanall
default: all
FUZZ_TARGETS := \
simple_round_trip \
stream_round_trip \
@@ -127,7 +128,6 @@ FUZZ_TARGETS := \
decompress_cross_format \
generate_sequences
.PHONY: all clean cleanall
all: libregression.a $(FUZZ_TARGETS)
rt_lib_common_%.o: $(LIB_SRCDIR)/common/%.c
@@ -263,10 +263,9 @@ corpora: $(patsubst %,corpora/%,$(FUZZ_TARGETS))
.PHONY: seedcorpora
seedcorpora: $(patsubst %,corpora/%_seed_corpus.zip,$(FUZZ_TARGETS))
REGRESSION_TARGET ?= all
regressiontest: corpora
CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build $(REGRESSION_TARGET) --debug=$(DEBUGLEVEL)
$(PYTHON) ./fuzz.py regression $(REGRESSION_TARGET)
CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build all --debug=$(DEBUGLEVEL)
$(PYTHON) ./fuzz.py regression all
clean:
@$(RM) *.a *.o $(FUZZ_TARGETS)
+3 -3
View File
@@ -77,9 +77,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
FUZZ_dataProducer_uint32Range(producer, 0, 2);
size = FUZZ_dataProducer_remainingBytes(producer);
DEBUGLOG(4, "Dict load method %d", dlm);
DEBUGLOG(4, "Dict content type %d", dct);
DEBUGLOG(4, "Dict size %u", (unsigned)size);
DEBUGLOG(2, "Dict load method %d", dlm);
DEBUGLOG(2, "Dict content type %d", dct);
DEBUGLOG(2, "Dict size %u", (unsigned)size);
void* const rBuf = FUZZ_malloc(size);
size_t const cBufSize = ZSTD_compressBound(size);
+1
View File
@@ -408,6 +408,7 @@ def build(args):
mflags = shlex.split(args.mflags)
# Flags to be added to both cflags and cxxflags
common_flags = [
'-Werror',
'-Wno-error=declaration-after-statement',
'-Wno-error=c++-compat',
'-Wno-error=deprecated' # C files are sometimes compiled with CXX
+3 -2
View File
@@ -31,11 +31,12 @@ void* FUZZ_malloc_rand(size_t size, FUZZ_dataProducer_t *producer)
return mem;
} else {
uintptr_t ptr = 0;
/* Return junk pointer 50% of the time */
/* Add +- 1M 50% of the time */
if (FUZZ_dataProducer_uint32Range(producer, 0, 1))
ptr += FUZZ_dataProducer_int32Range(producer, -1000000, 1000000);
FUZZ_dataProducer_int32Range(producer, -1000000, 1000000);
return (void*)ptr;
}
}
int FUZZ_memcmp(void const* lhs, void const* rhs, size_t size)
-1
View File
@@ -66,7 +66,6 @@ void* FUZZ_malloc(size_t size);
/**
* malloc except returns random pointer for zero sized data and FUZZ_ASSERT
* that malloc doesn't fail.
* WARNING: Only free the returned pointer if size > 0!
*/
void* FUZZ_malloc_rand(size_t size, FUZZ_dataProducer_t *producer);
+15 -86
View File
@@ -16,7 +16,6 @@
*/
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd_errors.h"
#include <stddef.h>
#include <stdlib.h>
@@ -77,7 +76,7 @@ static char* generatePseudoRandomString(char* str, size_t size, FUZZ_dataProduce
static size_t decodeSequences(void* dst, size_t nbSequences,
size_t literalsSize,
const void* dict, size_t dictSize,
ZSTD_SequenceFormat_e mode)
ZSTD_sequenceFormat_e mode)
{
const uint8_t* litPtr = literalsBuffer;
const uint8_t* const litBegin = literalsBuffer;
@@ -128,7 +127,7 @@ static size_t decodeSequences(void* dst, size_t nbSequences,
FUZZ_ASSERT(litPtr <= litEnd);
if (mode == ZSTD_sf_noBlockDelimiters) {
const uint32_t lastLLSize = (uint32_t)(litEnd - litPtr);
if (lastLLSize <= (uint32_t)(oend - op)) {
if (lastLLSize <= oend - op) {
memcpy(op, litPtr, lastLLSize);
generatedSrcBufferSize += lastLLSize;
} }
@@ -142,7 +141,7 @@ static size_t decodeSequences(void* dst, size_t nbSequences,
*/
static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
size_t literalsSizeLimit, size_t dictSize,
size_t windowLog, ZSTD_SequenceFormat_e mode)
size_t windowLog, ZSTD_sequenceFormat_e mode)
{
const uint32_t repCode = 0; /* not used by sequence ingestion api */
size_t windowSize = 1ULL << windowLog;
@@ -156,7 +155,7 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
if (mode == ZSTD_sf_explicitBlockDelimiters) {
/* ensure that no sequence can be larger than one block */
literalsSizeLimit = MIN(literalsSizeLimit, blockSizeMax/2);
matchLengthMax = MIN(matchLengthMax, (uint32_t)blockSizeMax/2);
matchLengthMax = MIN(matchLengthMax, blockSizeMax/2);
}
while ( nbSeqGenerated < ZSTD_FUZZ_MAX_NBSEQ - 3 /* extra room for explicit delimiters */
@@ -172,7 +171,7 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
if (bytesGenerated > ZSTD_FUZZ_GENERATED_SRC_MAXSIZE) {
break;
}
offsetBound = (bytesGenerated > windowSize) ? (uint32_t)windowSize : bytesGenerated + (uint32_t)dictSize;
offsetBound = (bytesGenerated > windowSize) ? windowSize : bytesGenerated + (uint32_t)dictSize;
offset = FUZZ_dataProducer_uint32Range(producer, 1, offsetBound);
if (dictSize > 0 && bytesGenerated <= windowSize) {
/* Prevent match length from being such that it would be associated with an offset too large
@@ -181,7 +180,7 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
*/
const size_t bytesToReachWindowSize = windowSize - bytesGenerated;
if (bytesToReachWindowSize < ZSTD_MINMATCH_MIN) {
const uint32_t newOffsetBound = offsetBound > windowSize ? (uint32_t)windowSize : offsetBound;
const uint32_t newOffsetBound = offsetBound > windowSize ? windowSize : offsetBound;
offset = FUZZ_dataProducer_uint32Range(producer, 1, newOffsetBound);
} else {
matchBound = MIN(matchLengthMax, (uint32_t)bytesToReachWindowSize);
@@ -202,14 +201,14 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
if (blockSize + seqSize > blockSizeMax) { /* reaching limit : must end block now */
const ZSTD_Sequence endBlock = {0, 0, 0, 0};
generatedSequences[nbSeqGenerated++] = endBlock;
blockSize = (uint32_t)seqSize;
blockSize = seqSize;
}
if (split) {
const ZSTD_Sequence endBlock = {0, lastLits, 0, 0};
generatedSequences[nbSeqGenerated++] = endBlock;
assert(lastLits <= seq.litLength);
seq.litLength -= lastLits;
blockSize = (uint32_t)(seqSize - lastLits);
blockSize = seqSize - lastLits;
} else {
blockSize += seqSize;
}
@@ -228,73 +227,12 @@ static size_t generateRandomSequences(FUZZ_dataProducer_t* producer,
return nbSeqGenerated;
}
static size_t
transferLiterals(void* dst, size_t dstCapacity, const ZSTD_Sequence* seqs, size_t nbSeqs, const void* src, size_t srcSize)
{
size_t n;
char* op = dst;
char* const oend = op + dstCapacity;
const char* ip = src;
const char* const iend = ip + srcSize;
for (n=0; n<nbSeqs; n++) {
size_t litLen = seqs[n].litLength;
size_t mlen = seqs[n].matchLength;
assert(op + litLen < oend); (void)oend;
assert(ip + litLen + mlen <= iend); (void)iend;
memcpy(op, ip, litLen);
op += litLen;
ip += litLen + mlen;
}
assert(oend - op >= 8);
return (size_t)(op - (char*)dst);
}
static size_t roundTripTest_compressSequencesAndLiterals(
void* result, size_t resultCapacity,
void* compressed, size_t compressedCapacity,
const void* src, size_t srcSize,
const ZSTD_Sequence* seqs, size_t nbSeqs)
{
size_t const litCapacity = srcSize + 8;
void* literals = malloc(litCapacity);
size_t cSize, litSize;
assert(literals);
litSize = transferLiterals(literals, litCapacity, seqs, nbSeqs, src, srcSize);
cSize = ZSTD_compressSequencesAndLiterals(cctx,
compressed, compressedCapacity,
seqs, nbSeqs,
literals, litSize, litCapacity, srcSize);
free(literals);
if (ZSTD_getErrorCode(cSize) == ZSTD_error_cannotProduce_uncompressedBlock) {
/* Valid scenario : ZSTD_compressSequencesAndLiterals cannot generate uncompressed blocks */
return 0;
}
if (ZSTD_getErrorCode(cSize) == ZSTD_error_dstSize_tooSmall) {
/* Valid scenario : in explicit delimiter mode,
* it might be possible for the compressed size to outgrow dstCapacity.
* In which case, it's still a valid fuzzer scenario,
* but no roundtrip shall be possible */
return 0;
}
/* round-trip */
FUZZ_ZASSERT(cSize);
{ size_t const dSize = ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
FUZZ_ZASSERT(dSize);
FUZZ_ASSERT_MSG(dSize == srcSize, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, result, srcSize), "Corruption!");
return dSize;
}
}
static size_t roundTripTest(void* result, size_t resultCapacity,
void* compressed, size_t compressedCapacity,
const void* src, size_t srcSize,
const ZSTD_Sequence* seqs, size_t nbSeqs,
const ZSTD_Sequence* seqs, size_t seqSize,
unsigned hasDict,
ZSTD_SequenceFormat_e mode)
ZSTD_sequenceFormat_e mode)
{
size_t cSize;
size_t dSize;
@@ -304,17 +242,8 @@ static size_t roundTripTest(void* result, size_t resultCapacity,
FUZZ_ZASSERT(ZSTD_DCtx_refDDict(dctx, ddict));
}
{ int blockMode, validation;
/* compressSequencesAndLiterals() only supports explicitBlockDelimiters and no validation */
FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_blockDelimiters, &blockMode));
FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_validateSequences, &validation));
if ((blockMode == ZSTD_sf_explicitBlockDelimiters) && (!validation)) {
FUZZ_ZASSERT(roundTripTest_compressSequencesAndLiterals(result, resultCapacity, compressed, compressedCapacity, src, srcSize, seqs, nbSeqs));
}
}
cSize = ZSTD_compressSequences(cctx, compressed, compressedCapacity,
seqs, nbSeqs,
seqs, seqSize,
src, srcSize);
if ( (ZSTD_getErrorCode(cSize) == ZSTD_error_dstSize_tooSmall)
&& (mode == ZSTD_sf_explicitBlockDelimiters) ) {
@@ -347,7 +276,7 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
unsigned hasDict;
unsigned wLog;
int cLevel;
ZSTD_SequenceFormat_e mode;
ZSTD_sequenceFormat_e mode;
FUZZ_dataProducer_t* const producer = FUZZ_dataProducer_create(src, size);
FUZZ_ASSERT(producer);
@@ -364,15 +293,15 @@ int LLVMFuzzerTestOneInput(const uint8_t* src, size_t size)
/* Generate window log first so we don't generate offsets too large */
wLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
cLevel = FUZZ_dataProducer_int32Range(producer, -3, 22);
mode = (ZSTD_SequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
mode = (ZSTD_sequenceFormat_e)FUZZ_dataProducer_int32Range(producer, 0, 1);
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)wLog);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, wLog);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, ZSTD_MINMATCH_MIN);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, (int)mode);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, mode);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach);
if (!literalsBuffer) {
+1 -1
View File
@@ -34,7 +34,7 @@ static size_t getDecompressionMargin(void const* compressed, size_t cSize, size_
/* The macro should be correct in this case, but it may be smaller
* because of e.g. block splitting, so take the smaller of the two.
*/
ZSTD_FrameHeader zfh;
ZSTD_frameHeader zfh;
size_t marginM;
FUZZ_ZASSERT(ZSTD_getFrameHeader(&zfh, compressed, cSize));
if (maxBlockSize == 0) {
+1 -1
View File
@@ -21,7 +21,7 @@
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
{
ZSTD_FrameHeader zfh;
ZSTD_frameHeader zfh;
if (size == 0) {
src = NULL;
}
+2 -3
View File
@@ -140,13 +140,12 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, producer);
setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer);
setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer);
setRand(cctx, ZSTD_c_blockSplitterLevel, 0, ZSTD_BLOCKSPLITTER_LEVEL_MAX, producer);
setRand(cctx, ZSTD_c_splitAfterSequences, 0, 2, producer);
setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer);
setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer);
setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer);
setRand(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN, ZSTD_BLOCKSIZE_MAX, producer);
setRand(cctx, ZSTD_c_validateSequences, 0, 1, producer);
setRand(cctx, ZSTD_c_repcodeResolution, 0, 2, producer);
setRand(cctx, ZSTD_c_searchForExternalRepcodes, 0, 2, producer);
if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
}
+39 -638
View File
@@ -40,18 +40,11 @@
#include "datagen.h" /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH64 */
#include "util.h"
#include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
/* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
#include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */
#include "compress/hist.h" /* HIST_count_wksp */
#include "compress/zstd_compress_internal.h" /* ZSTD_get1BlockSummary */
/*-************************************
* Macros
**************************************/
#define COUNTOF(array) (sizeof(array) / sizeof(*(array)))
/*-************************************
@@ -317,7 +310,7 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
#endif
static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
BYTE* src, size_t size, ZSTD_SequenceFormat_e format)
BYTE* src, size_t size, ZSTD_sequenceFormat_e format)
{
size_t i;
size_t j;
@@ -346,35 +339,6 @@ static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
}
}
static size_t FUZ_getLitSize(const ZSTD_Sequence* seqs, size_t nbSeqs)
{
size_t n, litSize = 0;
assert(seqs != NULL);
for (n=0; n<nbSeqs; n++) {
litSize += seqs[n].litLength;
}
return litSize;
}
static void
FUZ_transferLiterals(void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_Sequence* seqs, size_t nbSeqs)
{
size_t n;
const char* ip = (const char*)src;
char* op = (char*)dst;
size_t const litSize = FUZ_getLitSize(seqs, nbSeqs);
assert(litSize <= dstCapacity);
for (n=0; n<nbSeqs; n++) {
size_t const ll = seqs[n].litLength;
memcpy(op, ip, ll);
op += ll;
ip += ll + seqs[n].matchLength;
}
assert((size_t)(ip - (const char*)src) == srcSize);
}
#ifdef ZSTD_MULTITHREAD
typedef struct {
@@ -575,123 +539,6 @@ static void test_decompressBound(unsigned tnb)
DISPLAYLEVEL(3, "OK \n");
}
static unsigned test_histCountWksp(unsigned seed, unsigned testNb)
{
static const unsigned symLowLimits[] = { 0, 27, 0, 0, 27, 42, 0, 0, 27, 42, 27, 42 };
static const unsigned symHighLimits[] = { 255, 255, 210, 110, 42, 42, 210, 110, 42, 42, 42, 42 };
static const unsigned symMaxLimits[] = { 255, 255, 255, 255, 255, 255, 230, 130, 99, 99, 42, 42 };
static const size_t inputSizes[] = { 3367, 1761, 893, 117 };
unsigned workspace[HIST_WKSP_SIZE_U32];
size_t res, i, is, il;
DISPLAYLEVEL(3, "test%3u : HIST_count_wksp with empty source : ", testNb++);
{
/* With NULL source UBSan of older Clang could fail: applying zero offset to null pointer. */
static const unsigned char source[4] = { 0 };
unsigned count[1] = { 0 };
unsigned maxSym = 0;
res = HIST_count_wksp(count, &maxSym, source, 0, workspace, sizeof(workspace));
CHECK_EQ(res, 0);
CHECK_EQ(maxSym, 0);
CHECK_EQ(count[0], 0);
}
DISPLAYLEVEL(3, "OK \n");
#if HIST_WKSP_SIZE_U32
DISPLAYLEVEL(3, "test%3u : HIST_count_wksp with small workspace : ", testNb++);
{
unsigned count[1] = { 0 };
unsigned maxSym = 0;
res = HIST_count_wksp(count, &maxSym, NULL, 0, workspace, sizeof(workspace) - 1);
CHECK_EQ(res, ERROR(workSpace_tooSmall));
CHECK_EQ(maxSym, 0);
CHECK_EQ(count[0], 0);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3u : HIST_count_wksp with wrong workspace alignment : ", testNb++);
{
unsigned count[1] = { 0 };
unsigned maxSym = 0;
res = HIST_count_wksp(count, &maxSym, NULL, 0, (unsigned*)(void*)((char*)workspace + 1), sizeof(workspace));
CHECK_EQ(res, ERROR(GENERIC));
CHECK_EQ(maxSym, 0);
CHECK_EQ(count[0], 0);
}
DISPLAYLEVEL(3, "OK \n");
#endif
DISPLAYLEVEL(3, "test%3u : HIST_count_wksp with symbol out of range, small size : ", testNb++);
{
/* For less elements HIST_count_parallel_wksp would fail. */
static const unsigned char source[4] = { 1, 4, 0, 2 };
static const unsigned expected[6] = { 0 };
unsigned count[6] = { 0 };
unsigned maxSym = 2;
res = HIST_count_wksp(count, &maxSym, source, sizeof(source), workspace, sizeof(workspace));
CHECK_EQ(res, ERROR(maxSymbolValue_tooSmall));
CHECK_EQ(maxSym, 2);
for (i = 0; i < COUNTOF(expected); ++i) CHECK_EQ(count[i], expected[i]);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3u : HIST_count_wksp with symbol out of range, medium size : ", testNb++);
{
unsigned char source[3407];
unsigned count[6] = { 0 };
unsigned maxSym = 2;
for (i = 0; i < COUNTOF(source); ++i) {
source[i] = (48271 * (i + 1)) & 3;
}
res = HIST_count_wksp(count, &maxSym, source, sizeof(source), workspace, sizeof(workspace));
CHECK_EQ(res, ERROR(maxSymbolValue_tooSmall));
CHECK_EQ(maxSym, 2);
for (i = 0; i < COUNTOF(count); ++i) CHECK_EQ(count[i], 0);
}
DISPLAYLEVEL(3, "OK \n");
for (il = 0; il < COUNTOF(symMaxLimits); ++il) {
unsigned symMax = symMaxLimits[il];
unsigned symLow = symLowLimits[il];
unsigned symHigh = symHighLimits[il];
unsigned symRange = symHigh - symLow + 1;
for (is = 0; is < COUNTOF(inputSizes); ++is) {
unsigned char source[4000];
size_t inputSize = inputSizes[is];
assert(inputSize <= sizeof(source));
DISPLAYLEVEL(3, "test%3u : HIST_count_wksp test in [%u..%u], symMax: %u, inputSize: %u : ",
testNb++, symLow, symHigh, symMax, (unsigned)inputSize);
{
unsigned count[260] = { 0 };
unsigned expected[COUNTOF(count)] = { 0 };
unsigned maxSym = symMax;
unsigned realMaxSym = symMax;
unsigned maxCount = 0;
for (i = 0; i < inputSize; ++i) {
unsigned prng = (48271 * (i + seed)) % symRange + symLow;
source[i] = (unsigned char)prng;
++expected[prng];
}
/* for basic buffer overwrite checks */
for (i = maxSym + 1; i < COUNTOF(count); ++i) expected[i] = count[i] = ~0u;
for (i = 0; i <= maxSym; ++i) maxCount = MAX(maxCount, expected[i]);
for (i = realMaxSym; i > 0; --i) {
if (expected[i]) break;
--realMaxSym;
}
res = HIST_count_wksp(count, &maxSym, source, inputSize, workspace, sizeof(workspace));
CHECK_EQ(res, maxCount);
CHECK_EQ(maxSym, realMaxSym);
for (i = 0; i < COUNTOF(expected); ++i) CHECK_EQ(count[i], expected[i]);
}
DISPLAYLEVEL(3, "OK \n");
}
}
return testNb;
}
static void test_setCParams(unsigned tnb)
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
@@ -712,270 +559,6 @@ static void test_setCParams(unsigned tnb)
DISPLAYLEVEL(3, "OK \n");
}
static void test_blockSplitter_incompressibleExpansionProtection(unsigned testNb, unsigned seed)
{
DISPLAYLEVEL(3, "test%3i : Check block splitter doesn't oversplit incompressible data (seed %u): ", testNb, seed);
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
size_t const srcSize = 256 * 1024; /* needs to be at least 2 blocks */
void* incompressible = malloc(srcSize);
size_t const dstCapacity = ZSTD_compressBound(srcSize);
void* cBuffer = malloc(dstCapacity);
size_t const chunkSize = 8 KB;
size_t const nbChunks = srcSize / chunkSize;
size_t chunkNb, cSizeNoSplit, cSizeWithSplit;
assert(cctx != NULL);
assert(incompressible != NULL);
assert(cBuffer != NULL);
/* let's fill input with random noise (incompressible) */
RDG_genBuffer(incompressible, srcSize, 0.0, 0.0, seed);
/* this pattern targets the fastest _byChunk variant's sampling (level 3).
* manually checked that, without the @savings protection, it would over-split.
*/
for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
BYTE* const p = (BYTE*)incompressible + chunkNb * chunkSize;
size_t const samplingRate = 43;
int addOrRemove = chunkNb % 2;
size_t n;
for (n=0; n<chunkSize; n+=samplingRate) {
if (addOrRemove) {
p[n] &= 0x80;
} else {
p[n] |= 0x80;
}
}
}
/* run first without splitting */
ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockSplitterLevel, 1 /* no split */);
cSizeNoSplit = ZSTD_compress2(cctx, cBuffer, dstCapacity, incompressible, srcSize);
/* run with sample43 splitter, check it's still the same */
ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockSplitterLevel, 3 /* sample43, fastest _byChunk variant */);
cSizeWithSplit = ZSTD_compress2(cctx, cBuffer, dstCapacity, incompressible, srcSize);
if (cSizeWithSplit != cSizeNoSplit) {
DISPLAYLEVEL(1, "invalid compressed size: cSizeWithSplit %u != %u cSizeNoSplit \n",
(unsigned)cSizeWithSplit, (unsigned)cSizeNoSplit);
abort();
}
DISPLAYLEVEL(4, "compressed size: cSizeWithSplit %u == %u cSizeNoSplit : ",
(unsigned)cSizeWithSplit, (unsigned)cSizeNoSplit);
free(incompressible);
free(cBuffer);
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
}
size_t convertSequences_noRepcodes(SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs,
size_t nbSequences);
static size_t convertSequences_noRepcodes_ref(
SeqDef* dstSeqs,
const ZSTD_Sequence* inSeqs,
size_t nbSequences)
{
size_t longLen = 0;
size_t n;
for (n=0; n<nbSequences; n++) {
dstSeqs[n].offBase = OFFSET_TO_OFFBASE(inSeqs[n].offset);
dstSeqs[n].litLength = (U16)inSeqs[n].litLength;
dstSeqs[n].mlBase = (U16)(inSeqs[n].matchLength - MINMATCH);
/* Check for long length > 65535. */
if (UNLIKELY(inSeqs[n].matchLength > 65535+MINMATCH)) {
assert(longLen == 0);
longLen = n + 1;
}
if (UNLIKELY(inSeqs[n].litLength > 65535)) {
assert(longLen == 0);
longLen = n + nbSequences + 1;
}
}
return longLen;
}
static unsigned test_convertSequences_noRepcodes(unsigned seed, unsigned testNb)
{
ZSTD_Sequence nsrc[12];
SeqDef ndst[12], rdst[12];
size_t ref, ret, i, j;
seed += 0xDEADBEEF;
for (i = 0; i < COUNTOF(nsrc); ++i) {
seed = 48271 * ((unsigned)i + seed);
nsrc[i].offset = (seed & 0xFFFF) | 1; /* Offset shall not be zero. */
seed = 48271 * ((unsigned)i + seed);
nsrc[i].litLength = seed & 0xFFFF;
seed = 48271 * ((unsigned)i + seed);
nsrc[i].matchLength = (seed & 0xFFFFFF) % (65536 + MINMATCH);
seed = 48271 * ((unsigned)i + seed);
nsrc[i].rep = seed & 0xFF;
}
/* For near overflow and proper negative value handling. */
nsrc[5].matchLength = 65535 + MINMATCH;
nsrc[6].litLength = 65535;
nsrc[6].matchLength = 0;
nsrc[7].litLength = 0;
nsrc[7].matchLength = MINMATCH;
for (i = 0; i <= COUNTOF(nsrc); ++i) {
DISPLAYLEVEL(3, "test%3u : convertSequences_noRepcodes with %u inputs : ",
testNb++, (unsigned)i);
memset(ndst, 0, sizeof(ndst));
memset(rdst, 0, sizeof(rdst));
ref = convertSequences_noRepcodes_ref(rdst, nsrc, i);
ret = convertSequences_noRepcodes(ndst, nsrc, i);
CHECK_EQ(ret, ref);
CHECK_EQ(memcmp(rdst, ndst, sizeof(ndst)), 0);
DISPLAYLEVEL(3, "OK \n");
}
nsrc[7].matchLength = 65536 + MINMATCH;
for (i = 8; i <= COUNTOF(nsrc); ++i) {
DISPLAYLEVEL(3, "test%3u : convertSequences_noRepcodes with %u inputs and "
"matchLength overflow : ",
testNb++, (unsigned)i);
memset(ndst, 0, sizeof(ndst));
memset(rdst, 0, sizeof(rdst));
ref = convertSequences_noRepcodes_ref(rdst, nsrc, i);
ret = convertSequences_noRepcodes(ndst, nsrc, i);
CHECK_EQ(ret, ref);
CHECK_EQ(memcmp(rdst, ndst, sizeof(ndst)), 0);
DISPLAYLEVEL(3, "OK \n");
assert(COUNTOF(nsrc) > 8);
for (j = 4; j < 8; ++j) {
DISPLAYLEVEL(3, "test%3u : convertSequences_noRepcodes with %u inputs and "
"matchLength overflow #%u : ",
testNb++, (unsigned)i, (unsigned)(i - j));
memset(ndst, 0, sizeof(ndst));
memset(rdst, 0, sizeof(rdst));
ref = convertSequences_noRepcodes_ref(rdst, nsrc + j, i - j);
ret = convertSequences_noRepcodes(ndst, nsrc + j, i - j);
CHECK_EQ(ret, ref);
CHECK_EQ(memcmp(rdst, ndst, sizeof(ndst)), 0);
DISPLAYLEVEL(3, "OK \n");
}
}
nsrc[7].matchLength = 1;
nsrc[7].litLength = 65536;
for (i = 8; i <= COUNTOF(nsrc); ++i) {
DISPLAYLEVEL(3, "test%3u : convertSequences_noRepcodes with %u inputs and "
"litLength overflow: ",
testNb++, (unsigned)i);
memset(ndst, 0, sizeof(ndst));
memset(rdst, 0, sizeof(rdst));
ref = convertSequences_noRepcodes_ref(rdst, nsrc, i);
ret = convertSequences_noRepcodes(ndst, nsrc, i);
CHECK_EQ(ret, ref);
CHECK_EQ(memcmp(rdst, ndst, sizeof(ndst)), 0);
DISPLAYLEVEL(3, "OK \n");
assert(COUNTOF(nsrc) > 8);
for (j = 4; j < 8; ++j) {
DISPLAYLEVEL(3, "test%3u : convertSequences_noRepcodes with %u inputs and "
"litLength overflow #%u: ",
testNb++, (unsigned)i, (unsigned)(i - j));
memset(ndst, 0, sizeof(ndst));
memset(rdst, 0, sizeof(rdst));
ref = convertSequences_noRepcodes_ref(rdst, nsrc + j, i - j);
ret = convertSequences_noRepcodes(ndst, nsrc + j, i - j);
CHECK_EQ(ret, ref);
CHECK_EQ(memcmp(rdst, ndst, sizeof(ndst)), 0);
DISPLAYLEVEL(3, "OK \n");
}
}
return testNb;
}
static unsigned test_get1BlockSummary(unsigned testNb)
{
static const ZSTD_Sequence nseqs[] = {
{ 10, 2, 4, 1 },
{ 20, 3, 5, 2 },
{ 30, 6, 8, 3 },
{ 40, 7, 9, 4 },
{ 50, 10, 12, 5 },
{ 60, 11, 13, 6 },
{ 0, 14, 0, 7 },
{ 70, 15, 17, 8 },
{ 80, 16, 18, 9 },
{ 90, 19, 21, 1 },
{ 99, 20, 22, 2 },
};
static const BlockSummary blocks[] = {
{ 7, 104, 53 },
{ 6, 98, 51 },
{ 5, 90, 48 },
{ 4, 76, 42 },
{ 3, 60, 35 },
{ 2, 38, 25 },
{ 1, 14, 14 },
};
size_t i;
DISPLAYLEVEL(3, "test%3u : ZSTD_get1BlockSummary with empty array : ", testNb++);
{
BlockSummary bs = ZSTD_get1BlockSummary(nseqs, 0);
CHECK_EQ(bs.nbSequences, ERROR(externalSequences_invalid));
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3u : ZSTD_get1BlockSummary with 1 literal only : ", testNb++);
{
static const ZSTD_Sequence seqs[] = { { 0, 5, 0, 0 } };
BlockSummary bs = ZSTD_get1BlockSummary(seqs, 1);
CHECK_EQ(bs.nbSequences, 1);
CHECK_EQ(bs.litSize, 5);
CHECK_EQ(bs.blockSize, 5);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3u : ZSTD_get1BlockSummary with no terminator : ", testNb++);
{
static const ZSTD_Sequence seqs[] = { { 10, 2, 4, 0 }, { 20, 3, 5, 0 } };
BlockSummary bs = ZSTD_get1BlockSummary(seqs, 2);
CHECK_EQ(bs.nbSequences, ERROR(externalSequences_invalid));
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3u : ZSTD_get1BlockSummary with rep ignored : ", testNb++);
{
static const ZSTD_Sequence seqs[] = {
{ 10, 2, 4, 2 },
{ 10, 3, 5, 2 },
{ 0, 7, 0, 3 },
};
BlockSummary bs = ZSTD_get1BlockSummary(seqs, 3);
CHECK_EQ(bs.nbSequences, 3);
CHECK_EQ(bs.litSize, 2 + 3 + 7);
CHECK_EQ(bs.blockSize, (4 + 5) + (2 + 3 + 7));
}
DISPLAYLEVEL(3, "OK \n");
assert(COUNTOF(nseqs) > COUNTOF(blocks));
for (i = 0; i < COUNTOF(blocks); ++i) {
BlockSummary bs;
DISPLAYLEVEL(3, "test%3u : ZSTD_get1BlockSummary with %u inputs : ",
testNb++, (unsigned)(COUNTOF(nseqs) - i));
bs = ZSTD_get1BlockSummary(nseqs + i, COUNTOF(nseqs) - i);
CHECK_EQ(bs.nbSequences, blocks[i].nbSequences);
CHECK_EQ(bs.litSize, blocks[i].litSize);
CHECK_EQ(bs.blockSize, blocks[i].blockSize);
DISPLAYLEVEL(3, "OK \n");
}
return testNb;
}
/* ============================================================= */
static int basicUnitTests(U32 const seed, double compressibility)
{
size_t const CNBuffSize = 5 MB;
@@ -1041,8 +624,6 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
testNb = test_histCountWksp(seed, testNb);
DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
if (cctx==NULL) goto _output_error;
@@ -1286,7 +867,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
ZSTD_freeCCtx(cctx);
}
{ /* ensure frame content size is missing */
ZSTD_FrameHeader zfh;
ZSTD_frameHeader zfh;
size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
}
@@ -1414,37 +995,6 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ldm hashRateLog > windowLog underflow check : ", testNb++);
{
/* Test that when windowLog < hashRateLog, we don't get excessive memory usage
* due to underflow in hashLog calculation (windowLog - hashRateLog). */
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
size_t const size = (1U << 10); // 1 KB
size_t const dstCapacity = ZSTD_compressBound(size);
void* src = (void*)malloc(size);
void* dst = (void*)malloc(dstCapacity);
RDG_genBuffer(src, size, 0.5, 0.5, seed);
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 0));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 12));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 13));
CHECK_Z(ZSTD_compress2(cctx, dst, dstCapacity, src, size));
{ size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
DISPLAYLEVEL(3, "CCtx size: %u bytes ", (unsigned)cctxSize);
CHECK_LT(cctxSize, 50 MB);
}
ZSTD_freeCCtx(cctx);
free(src);
free(dst);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++);
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
@@ -1810,7 +1360,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitAfterSequences, ZSTD_ps_enable));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize);
CHECK_Z(cSize);
@@ -1824,8 +1374,6 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
test_blockSplitter_incompressibleExpansionProtection(testNb++, seed);
DISPLAYLEVEL(3, "test%3d : superblock uncompressible data: too many nocompress superblocks : ", testNb++);
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
@@ -2127,8 +1675,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
int value;
ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0);
cparams.strategy = (ZSTD_strategy)-1; /* set invalid value, on purpose */
/* Set invalid cParams == error out, and no change. */
cparams.strategy = -1;
/* Set invalid cParams == no change. */
CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
@@ -2191,12 +1739,12 @@ static int basicUnitTests(U32 const seed, double compressibility)
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setParams() : ", testNb++);
DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCarams() : ", testNb++);
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
int value;
ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
params.cParams.strategy = (ZSTD_strategy)-1; /* set invalid value, on purpose */
/* Set invalid params == error out, and no change. */
params.cParams.strategy = -1;
/* Set invalid params == no change. */
CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params)));
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
@@ -2280,7 +1828,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
params.fParams.contentSizeFlag = 0;
params.cParams.windowLog = ZSTD_WINDOWLOG_MAX;
for (cnb = 0; cnb < nbCompressions; ++cnb) {
DISPLAYLEVEL(6, "run %u / %u \n", (unsigned)cnb, (unsigned)nbCompressions);
DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions);
CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */
CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) );
}
@@ -2298,8 +1846,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
assert(smallCCtx != NULL);
CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
{ size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
DISPLAYLEVEL(5, "(large) %uKB > 32*%uKB (small) : ",
(unsigned)(largeCCtxSize>>10), (unsigned)(smallCCtxSize>>10));
DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
largeCCtxSize>>10, smallCCtxSize>>10);
assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c .
* make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
}
@@ -2546,25 +2094,6 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : estimation functions with LDM enabled (issue #4590) : ", testNb++);
{
/* ZSTD_estimateCCtxSize_usingCCtxParams must adjust zeroed-out
* LDM parameters when LDM is enabled, to avoid division by zero
* in ZSTD_ldm_getMaxNbSeq. */
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
size_t cctxSize;
CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 22));
CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
if (ZSTD_isError(cctxSize)) goto _output_error;
if (cctxSize == 0) goto _output_error;
cctxSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
if (ZSTD_isError(cctxSize)) goto _output_error;
if (cctxSize == 0) goto _output_error;
ZSTD_freeCCtxParams(params);
}
DISPLAYLEVEL(3, "OK \n");
}
free(staticCCtxBuffer);
free(staticDCtxBuffer);
@@ -2661,7 +2190,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitAfterSequences, ZSTD_ps_enable) );
CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK_Z(cSize);
ZSTD_freeCCtx(cctx);
@@ -2893,7 +2422,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
(const char*)CNBuffer + dictSize, testSize) );
{ ZSTD_FrameHeader zfh;
{ ZSTD_frameHeader zfh;
if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
} }
@@ -2921,7 +2450,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
3742, 3675, 3674, 3665, 3664,
3663, 3662, 3661, 3660, 3660,
3660, 3660, 3660 };
size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2840, 2950,
size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2820, 2940,
2950, 2950, 2925, 2900, 2892,
2910, 2910, 2910, 2780, 2775,
2765, 2760, 2755, 2754, 2753,
@@ -3861,7 +3390,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
{ size_t const compressionResult = ZSTD_compress2(cctx,
compressedBuffer, compressedBufferSize,
CNBuffer, srcSize);
DISPLAYLEVEL(5, "simple=%u vs %u=advanced : ", (unsigned)cSize_1pass, (unsigned)compressionResult);
DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult);
if (ZSTD_isError(compressionResult)) goto _output_error;
if (compressionResult != cSize_1pass) goto _output_error;
} }
@@ -3966,7 +3495,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
{ ZSTD_FrameHeader zfh;
{ ZSTD_frameHeader zfh;
size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
if (zfhrt != 0) goto _output_error;
}
@@ -4124,19 +3653,16 @@ static int basicUnitTests(U32 const seed, double compressibility)
ZSTD_freeDCtx(dctx);
}
/* rle detection test: must compress better blocks with a single identical byte repeated */
/* long rle test */
{ size_t sampleSize = 0;
size_t maxCompressedSize = 46; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
DISPLAYLEVEL(3, "test%3i : RLE detection test : ", testNb++);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 2);
sampleSize += 256 KB - 2;
memset((char*)CNBuffer+sampleSize, 'A', 100 KB);
sampleSize += 100 KB;
size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
sampleSize += 256 KB - 1;
memset((char*)CNBuffer+sampleSize, 'A', 96 KB);
sampleSize += 96 KB;
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
if (ZSTD_isError(cSize) || cSize > maxCompressedSize) {
DISPLAYLEVEL(4, "error: cSize %u > %u expected ! \n", (unsigned)cSize, (unsigned)maxCompressedSize);
goto _output_error;
}
if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error;
{ CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
if (regenSize!=sampleSize) goto _output_error; }
DISPLAYLEVEL(3, "OK \n");
@@ -4217,7 +3743,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
if (seqs == NULL) goto _output_error;
assert(cctx != NULL);
/* Populate src with compressible random data */
/* Populate src with random data */
RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
/* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */
@@ -4259,112 +3785,6 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
testNb = test_convertSequences_noRepcodes(seed, testNb);
testNb = test_get1BlockSummary(testNb);
DISPLAYLEVEL(3, "test%3i : ZSTD_compressSequencesAndLiterals : ", testNb++);
{
const size_t srcSize = 497000;
const BYTE* const src = (BYTE*)CNBuffer;
BYTE* const dst = (BYTE*)compressedBuffer;
const size_t dstCapacity = ZSTD_compressBound(srcSize);
const size_t decompressSize = srcSize;
char* const decompressBuffer = (char*)malloc(decompressSize);
char* const litBuffer = (char*)malloc(decompressSize);
size_t compressedSize;
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
size_t nbSeqs;
if (litBuffer == NULL) goto _output_error;
if (decompressBuffer == NULL) goto _output_error;
if (seqs == NULL) goto _output_error;
assert(cctx != NULL);
/* Populate src with compressible random data */
RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
/* Roundtrip Test using the AndLiterals() variant */
nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
{ size_t const litSize = FUZ_getLitSize(seqs, nbSeqs);
FUZ_transferLiterals(litBuffer, decompressSize, CNBuffer, srcSize, seqs, nbSeqs);
/* not enough literals: must fail */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1, decompressSize, srcSize);
if (!ZSTD_isError(compressedSize)) {
DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: not enough literals provided\n");
goto _output_error;
}
/* too many literals: must fail */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1, decompressSize, srcSize);
if (!ZSTD_isError(compressedSize)) {
DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: too many literals provided\n");
goto _output_error;
}
/* srcSize too large: must fail */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize+1);
if (!ZSTD_isError(compressedSize)) {
DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too large\n");
goto _output_error;
}
/* srcSize too small: must fail */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize-1);
if (!ZSTD_isError(compressedSize)) {
DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too small\n");
goto _output_error;
}
/* correct amount of literals: should compress successfully */
compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize);
if (ZSTD_isError(compressedSize)) {
DISPLAY("Error in ZSTD_compressSequencesAndLiterals()\n");
goto _output_error;
}
}
{ ZSTD_FrameHeader zfh;
size_t const zfhStatus = ZSTD_getFrameHeader(&zfh, dst, compressedSize);
if (zfhStatus != 0) {
DISPLAY("Error reading frame header\n");
goto _output_error;
}
if (zfh.frameContentSize != srcSize) {
DISPLAY("Error: ZSTD_compressSequencesAndLiterals() did not report srcSize in the frame header\n");
goto _output_error;
}
if (zfh.windowSize > srcSize) {
DISPLAY("Error: ZSTD_compressSequencesAndLiterals() did not resized window size to smaller contentSize\n");
goto _output_error;
}
}
{ size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
if (ZSTD_isError(dSize)) {
DISPLAY("Error during decompression of frame produced by ZSTD_compressSequencesAndLiterals()\n");
goto _output_error;
}
if (dSize != srcSize) {
DISPLAY("Error: decompression of frame produced by ZSTD_compressSequencesAndLiterals() has different size\n");
goto _output_error;
}
if (memcmp(decompressBuffer, src, srcSize)) {
DISPLAY("Error: decompression of frame produced by ZSTD_compressSequencesAndLiterals() produces a different content (of same size)\n");
goto _output_error;
}
}
ZSTD_freeCCtx(cctx);
free(litBuffer);
free(decompressBuffer);
free(seqs);
}
DISPLAYLEVEL(3, "OK \n");
/* Multiple blocks of zeros test */
#define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
@@ -4495,32 +3915,13 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
/* frame operations on skippable frames */
{ const char skippableFrame[] = "\x52\x2a\x4d\x18\x05\x0\x0\0abcde";
size_t const skippableFrameSize = sizeof(skippableFrame) - 1 /* remove the terminating /0 */;
DISPLAYLEVEL(3, "test%3i : ZSTD_findFrameCompressedSize on skippable frame : ", testNb++);
CHECK(ZSTD_findFrameCompressedSize(skippableFrame, skippableFrameSize) == skippableFrameSize);
/* findFrameCompressedSize on skippable frames */
DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++);
{ const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
size_t const frameSrcSize = 13;
if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize on skippable frame : ", testNb++);
CHECK(ZSTD_getFrameContentSize(skippableFrame, skippableFrameSize) == 0);
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameHeader on skippable frame : ", testNb++);
{ ZSTD_FrameHeader zfh;
size_t const s = ZSTD_getFrameHeader(&zfh, skippableFrame, skippableFrameSize);
CHECK_Z(s);
CHECK(s == 0); /* success */
CHECK(zfh.frameType == ZSTD_skippableFrame);
CHECK(zfh.headerSize == ZSTD_SKIPPABLEHEADERSIZE);
CHECK(zfh.dictID == 2); /* magic variant */
assert(skippableFrameSize >= ZSTD_SKIPPABLEHEADERSIZE);
CHECK(zfh.frameContentSize == skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE);
}
DISPLAYLEVEL(3, "OK \n");
}
/* error string tests */
DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++);
if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
@@ -4538,8 +3939,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1);
ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1);
ZSTD_FrameHeader lgHeader;
ZSTD_FrameHeader smHeader;
ZSTD_frameHeader lgHeader;
ZSTD_frameHeader smHeader;
CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict));
CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize));
@@ -4746,8 +4147,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
for (; level < ZSTD_maxCLevel(); ++level) {
size_t const currSize = ZSTD_estimateCCtxSize(level);
if (prevSize > currSize) {
DISPLAYLEVEL(3, "Error! previous cctx size: %u at level: %d is larger than current cctx size: %u at level: %d",
(unsigned)prevSize, level-1, (unsigned)currSize, level);
DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d",
prevSize, level-1, currSize, level);
goto _output_error;
}
prevSize = currSize;
@@ -4771,8 +4172,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
if (cctxSizeUsingLevel < cctxSizeUsingCParams
|| ZSTD_isError(cctxSizeUsingCParams)
|| ZSTD_isError(cctxSizeUsingLevel)) {
DISPLAYLEVEL(3, "error! l: %d dict: %u srcSize: %u cctx size cpar: %u, cctx size level: %u\n",
level, (unsigned)dictSize, (unsigned)srcSize, (unsigned)cctxSizeUsingCParams, (unsigned)cctxSizeUsingLevel);
DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
goto _output_error;
} } } } }
DISPLAYLEVEL(3, "OK \n");
@@ -5158,7 +4559,7 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const
} }
/* frame header decompression test */
{ ZSTD_FrameHeader zfh;
{ ZSTD_frameHeader zfh;
CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) );
CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect");
}
@@ -5300,7 +4701,7 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const
/* streaming decompression test */
DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
/* ensure memory requirement is good enough (should always be true) */
{ ZSTD_FrameHeader zfh;
{ ZSTD_frameHeader zfh;
CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX),
"ZSTD_getFrameHeader(): error retrieving frame information");
{ size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize);
+1 -1
View File
@@ -36,7 +36,7 @@ clean:
#------------------------------------------------------------------------------
# validated only for Linux, macOS, Hurd and some BSD targets
#------------------------------------------------------------------------------
ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD,$(shell sh -c 'MSYSTEM="MSYS" uname') ))
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
test-%: zstd
@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh
-2
View File
@@ -56,7 +56,6 @@ static int testSimpleAPI(void)
} else {
DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret));
}
free(output);
return 1;
}
if (ret != size) {
@@ -65,7 +64,6 @@ static int testSimpleAPI(void)
}
if (memcmp(EXPECTED, output, size) != 0) {
DISPLAY("ERROR: Wrong decoded output produced\n");
free(output);
return 1;
}
+6 -17
View File
@@ -21,14 +21,14 @@ mustBeAbsent() {
$ECHO "$@ correctly not present" # for some reason, this $ECHO must exist, otherwise mustBeAbsent() always fails (??)
}
# default compilation : all features enabled - no zbuff, no legacy
# default compilation : all features enabled - no zbuff
$ECHO "testing default library compilation"
CFLAGS= make -C $DIR/../lib libzstd libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
mustBeAbsent "zstd_v07.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM tmplog
@@ -44,7 +44,7 @@ nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
mustBeAbsent "zstd_compress.o"
isPresent "zstd_decompress.o"
mustBeAbsent "zdict.o"
mustBeAbsent "zstd_v07.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
@@ -66,7 +66,7 @@ nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
mustBeAbsent "zstd_v07.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
@@ -77,7 +77,7 @@ nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
mustBeAbsent "zstd_v07.o"
isPresent "zstd_v07.o"
isPresent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
@@ -88,7 +88,7 @@ nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
mustBeAbsent "zdict.o"
mustBeAbsent "zstd_v07.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
@@ -102,14 +102,3 @@ mustBeAbsent "zdict.o"
mustBeAbsent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# legacy support explicitly enabled
$ECHO "testing with legacy support explicitly enabled"
ZSTD_LEGACY_SUPPORT=5 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
+17 -38
View File
@@ -39,39 +39,24 @@ int main(int argc, const char** argv)
unsigned windowLog = 18;
(void)argc;
(void)argv;
int _exit_code = 0;
/* Create stream */
ctx = ZSTD_createCCtx();
if (!ctx) { return 1; }
/* Set parameters */
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, windowLog))) {
_exit_code = 2;
goto _clean_ctx;
}
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, 13))) {
_exit_code = 2;
goto _clean_ctx;
}
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, 14))) {
_exit_code = 2;
goto _clean_ctx;
}
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, 1))) {
_exit_code = 2;
goto _clean_ctx;
}
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, 7))) {
_exit_code = 2;
goto _clean_ctx;
}
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, 16))) {
_exit_code = 2;
goto _clean_ctx;
}
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, ZSTD_fast))) {
_exit_code = 2;
goto _clean_ctx;
}
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, windowLog)))
return 2;
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, 13)))
return 2;
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, 14)))
return 2;
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, 1)))
return 2;
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, 7)))
return 2;
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, 16)))
return 2;
if (ZSTD_isError(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, ZSTD_fast)))
return 2;
{
U64 compressed = 0;
const U64 toCompress = ((U64)1) << 33;
@@ -96,28 +81,22 @@ int main(int argc, const char** argv)
}
printf("Compressing, trying to generate a segfault \n");
if (compress(ctx, out, srcBuffer, size)) {
_exit_code = 1;
goto _clean_buffer;
return 1;
}
compressed += size;
while (compressed < toCompress) {
const size_t block = rand() % (size - pos + 1);
if (pos == size) { pos = 0; }
if (compress(ctx, out, srcBuffer + pos, block)) {
_exit_code = 1;
goto _clean_buffer;
return 1;
}
pos += block;
compressed += block;
}
printf("Compression completed successfully (no error triggered)\n");
_clean_buffer:
free(srcBuffer);
free(dstBuffer);
}
_clean_ctx:
ZSTD_freeCCtx(ctx);
return _exit_code;
return 0;
}
+4 -3
View File
@@ -1242,7 +1242,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
size_t n;
size_t totalSizeToLoad = (size_t)UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
size_t* fileSizes = calloc(nbFiles, sizeof(size_t));
size_t* fileSizes = calloc(sizeof(size_t), nbFiles);
void* srcBuffer = NULL;
int ret = 0;
@@ -1273,6 +1273,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab
f = fopen(fileNamesTable[n], "rb");
if (f==NULL) {
DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
fclose(f);
ret = 10;
goto _cleanUp;
}
@@ -1457,7 +1458,7 @@ createMemoTableArray(const paramValues_t p,
const size_t varyLen,
const U32 memoTableLog)
{
memoTable_t* const mtAll = (memoTable_t*)calloc((ZSTD_STRATEGY_MAX + 1), sizeof(memoTable_t));
memoTable_t* const mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_STRATEGY_MAX + 1));
ZSTD_strategy i, stratMin = ZSTD_STRATEGY_MIN, stratMax = ZSTD_STRATEGY_MAX;
if(mtAll == NULL) {
@@ -1494,7 +1495,7 @@ createMemoTableArray(const paramValues_t p,
mtl = ((size_t)1 << memoTableLog);
}
mtAll[i].table = (BYTE*)calloc(mtl, sizeof(BYTE));
mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl);
mtAll[i].tableLen = mtl;
if(mtAll[i].table == NULL) {
+37 -43
View File
@@ -1,7 +1,7 @@
#!/bin/sh
set -e # exit immediately on error
set -x # print commands before execution (debug)
# set -x # print commands before execution (debug)
unset ZSTD_CLEVEL
unset ZSTD_NBTHREADS
@@ -16,7 +16,13 @@ datagen() {
"$DATAGEN_BIN" "$@"
}
alias zstd='$EXE_PREFIX $ZSTD_BIN'
zstd() {
if [ -z "$EXE_PREFIX" ]; then
"$ZSTD_BIN" "$@"
else
"$EXE_PREFIX" "$ZSTD_BIN" "$@"
fi
}
sudoZstd() {
if [ -z "$EXE_PREFIX" ]; then
@@ -43,7 +49,7 @@ roundTripTest() {
rm -f tmp1 tmp2
println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel"
datagen $1 $proba | $MD5SUM > tmp1
datagen $1 $proba | zstd -T0 --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2
datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2
$DIFF -q tmp1 tmp2
}
@@ -65,7 +71,7 @@ fileRoundTripTest() {
println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d"
datagen $1 $local_p > tmp
< tmp $MD5SUM > tmp.md5.1
zstd -T0 --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2
zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2
$DIFF -q tmp.md5.1 tmp.md5.2
}
@@ -103,7 +109,10 @@ isTerminal=${isTerminal:-$detectedTerminal}
isWindows=false
INTOVOID="/dev/null"
DEVDEVICE="/dev/zero"
case "$UNAME" in
GNU) DEVDEVICE="/dev/random" ;;
*) DEVDEVICE="/dev/zero" ;;
esac
case "$OS" in
Windows*)
isWindows=true
@@ -114,6 +123,7 @@ esac
case "$UNAME" in
Darwin) MD5SUM="md5 -r" ;;
FreeBSD) MD5SUM="gmd5sum" ;;
NetBSD) MD5SUM="md5 -n" ;;
OpenBSD) MD5SUM="md5" ;;
*) MD5SUM="md5sum" ;;
@@ -848,7 +858,6 @@ ls tmp* > tmpList
zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal file list
rm -rf tmp*
println "\n===> --[no-]content-size tests"
datagen > tmp_contentsize
@@ -1075,27 +1084,21 @@ cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high
cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low
cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high
cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low
println "test : allows and interprets K,KB,KiB,M,MB,MiB,G,GB and GiB suffix"
println "test : allows and interprets K,KB,KiB,M,MB and MiB suffix"
cat tmp | zstd -14 -f --size-hint=11K | zstd -t
cat tmp | zstd -14 -f --size-hint=11KB | zstd -t
cat tmp | zstd -14 -f --size-hint=11KiB | zstd -t
cat tmp | zstd -14 -f --size-hint=1M | zstd -t
cat tmp | zstd -14 -f --size-hint=1MB | zstd -t
cat tmp | zstd -14 -f --size-hint=1MiB | zstd -t
cat tmp | zstd -14 -f --size-hint=1G | zstd -t
cat tmp | zstd -14 -f --size-hint=1GB | zstd -t
cat tmp | zstd -14 -f --size-hint=1GiB | zstd -t
cat tmp | zstd -14 -f --size-hint=3G | zstd -t
cat tmp | zstd -14 -f --size-hint=3GB | zstd -t
cat tmp | zstd -14 -f --size-hint=3GiB | zstd -t
println "\n===> dictionary tests "
println "- Test high/low compressibility corpus training"
datagen -g12M -P90 > tmpCorpusHighCompress
datagen -g12M -P5 > tmpCorpusLowCompress
zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress
zstd --train --split=2K tmpCorpusLowCompress -o tmpDictLowCompress
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress
zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress
rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress
println "- Test with raw dict (content only) "
datagen > tmpDict
@@ -1180,8 +1183,8 @@ rm -f tmp* dictionary
println "- Test --memory for dictionary compression"
datagen -g12M -P90 > tmpCorpusHighCompress
zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)"
zstd --train --split=2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)"
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr
cat zstTrainWithMemLimitStdErr | $GREP "setting manual memory limit for dictionary training data at 5 MB"
cat zstTrainWithMemLimitStdErr | $GREP "Training samples set too large (12 MB); training on 5 MB only..."
rm zstTrainWithMemLimitStdErr
@@ -1555,8 +1558,9 @@ then
roundTripTest -g4M "1 -T0 --auto-threads=physical"
roundTripTest -g4M "1 -T0 --auto-threads=logical"
roundTripTest -g8M "3 -T2"
roundTripTest -g8M "19 --long"
roundTripTest -g8000K "2 --threads=2"
fileRoundTripTest -g4M "19 -T2 --split=1M"
fileRoundTripTest -g4M "19 -T2 -B1M"
println "\n===> zstdmt long distance matching round-trip tests "
roundTripTest -g8M "3 --long=24 -T2"
@@ -1571,9 +1575,7 @@ then
ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting=
ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage
ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single worker
ZSTD_NBTHREADS=4 zstd -f mt_tmp -vv 2>&1 | $GREP "4 worker threads" # check message
ZSTD_NBTHREADS=0 zstd -f mt_tmp -vv 2>&1 | $GREP "core(s) detected" # check core count autodetection is triggered
ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread
# temporary envvar changes in the above tests would actually persist in macos /bin/sh
unset ZSTD_NBTHREADS
rm -f mt_tmp*
@@ -1690,8 +1692,6 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB"
roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB"
roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB"
roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB"
roundTripTest -g1M -P50 "1 --single-thread --long=30" " --memory=1GB"
roundTripTest -g1M -P50 "1 --single-thread --long=30" " --zstd=wlog=29 --memory=1GB"
if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -ne "1" ]; then
@@ -1775,7 +1775,7 @@ then
println "\n===> rsyncable mode "
roundTripTest -g10M " --rsyncable"
roundTripTest -g10M " --rsyncable --split=100K"
roundTripTest -g10M " --rsyncable -B100K"
println "===> test: --rsyncable must fail with --single-thread"
zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread"
fi
@@ -1786,15 +1786,11 @@ datagen -g1000 -P10 > tmp_patch
zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff
zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch
zstd -f --patch-apply=tmp_dict tmp_patch_diff -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch
println "\n===> alternate syntax: patch-from origin"
zstd -f --patch-from tmp_dict tmp_patch -o tmp_patch_diff
zstd -df --patch-from tmp_dict tmp_patch_diff -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch
zstd -f --patch-apply tmp_dict tmp_patch_diff -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch
rm -rf tmp_*
println "\n===> patch-from recursive tests"
@@ -1858,8 +1854,6 @@ roundTripTest -g18000017 -P88 17
roundTripTest -g18000018 -P94 18
roundTripTest -g18000019 -P96 19
roundTripTest -g8M "19 --long"
roundTripTest -g5000000000 -P99 "1 --zstd=wlog=25"
roundTripTest -g3700000000 -P0 "1 --zstd=strategy=6,wlog=25" # ensure btlazy2 can survive an overflow rescale
@@ -1898,36 +1892,36 @@ println "\n===> cover dictionary builder : advanced options "
TESTFILE="$PRGDIR"/zstdcli.c
datagen > tmpDict
println "- Create first dictionary"
zstd -T0 --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
cp "$TESTFILE" tmp
zstd -f tmp -D tmpDict
zstd -f tmp -D tmpDict --patch-from=tmpDict && die "error: can't use -D and --patch-from=#at the same time"
zstd -d tmp.zst -D tmpDict -fo result
$DIFF "$TESTFILE" result
zstd -T0 --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)"
zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)"
println "- Create second (different) dictionary"
zstd -T0 --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
println "- Create dictionary using shrink-dict flag"
zstd -T0 --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict
zstd -T0 --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1
zstd -T0 --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2
zstd -T0 --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3
zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict
zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1
zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2
zstd --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3
println "- Create dictionary with short dictID"
zstd -T0 --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
println "- Create dictionary with size limit"
zstd -T0 --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Compare size of dictionary from 90% training samples with 80% training samples"
zstd -T0 --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd -T0 --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using all samples for both training and testing"
zstd -T0 --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Test -o before --train-cover"
rm -f tmpDict dictionary
zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f tmpDict
zstd -T0 --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f dictionary
rm -f tmp* dictionary
+3 -3
View File
@@ -159,15 +159,15 @@ static result_t compress_cctx_compress(
return result_error(result_error_skip);
int const level = config_get_level(config);
result_t result;
ZSTD_CCtx* cctx = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
if (cctx == NULL || dctx == NULL) {
fprintf(stderr, "context creation failed\n");
result = result_error(result_error_system_error);
goto out;
return result_error(result_error_system_error);
}
result_t result;
result_data_t data = {.total_size = 0};
for (size_t i = 0; i < state->inputs.size; ++i) {
data_buffer_t const input = state->inputs.buffers[i];
+1 -3
View File
@@ -10,7 +10,7 @@
#include "result.h"
const char* result_get_error_string(result_t result) {
char const* result_get_error_string(result_t result) {
switch (result_get_error(result)) {
case result_error_ok:
return "okay";
@@ -24,7 +24,5 @@ const char* result_get_error_string(result_t result) {
return "decompression error";
case result_error_round_trip_error:
return "round trip error";
default:
return "unknown error";
}
}
+1 -1
View File
@@ -94,7 +94,7 @@ static result_error_t result_get_error(result_t result) {
return result.internal_error;
}
const char* result_get_error_string(result_t result);
char const* result_get_error_string(result_t result);
static result_data_t result_get_data(result_t result) {
return result.internal_data;
+650 -650
View File
@@ -1,28 +1,28 @@
Data, Config, Method, Total compressed size
silesia.tar, level -5, compress simple, 6858730
silesia.tar, level -3, compress simple, 6502944
silesia.tar, level -1, compress simple, 6175652
silesia.tar, level 0, compress simple, 4829268
silesia.tar, level 1, compress simple, 5307443
silesia.tar, level 3, compress simple, 4829268
silesia.tar, level 4, compress simple, 4767074
silesia.tar, level 5, compress simple, 4662847
silesia.tar, level 6, compress simple, 4597877
silesia.tar, level 7, compress simple, 4563998
silesia.tar, level 9, compress simple, 4537558
silesia.tar, level 13, compress simple, 4484732
silesia.tar, level 16, compress simple, 4355572
silesia.tar, level 19, compress simple, 4257629
silesia.tar, uncompressed literals, compress simple, 4829268
silesia.tar, uncompressed literals optimal, compress simple, 4257629
silesia.tar, huffman literals, compress simple, 6175652
github.tar, level -5, compress simple, 52173
github.tar, level -3, compress simple, 45783
github.tar, level -1, compress simple, 42606
github.tar, level 0, compress simple, 38884
silesia.tar, level -5, compress simple, 6861055
silesia.tar, level -3, compress simple, 6505483
silesia.tar, level -1, compress simple, 6179047
silesia.tar, level 0, compress simple, 4854086
silesia.tar, level 1, compress simple, 5327717
silesia.tar, level 3, compress simple, 4854086
silesia.tar, level 4, compress simple, 4791503
silesia.tar, level 5, compress simple, 4679004
silesia.tar, level 6, compress simple, 4614561
silesia.tar, level 7, compress simple, 4579828
silesia.tar, level 9, compress simple, 4555448
silesia.tar, level 13, compress simple, 4502956
silesia.tar, level 16, compress simple, 4360385
silesia.tar, level 19, compress simple, 4260939
silesia.tar, uncompressed literals, compress simple, 4854086
silesia.tar, uncompressed literals optimal, compress simple, 4260939
silesia.tar, huffman literals, compress simple, 6179047
github.tar, level -5, compress simple, 52115
github.tar, level -3, compress simple, 45678
github.tar, level -1, compress simple, 42560
github.tar, level 0, compress simple, 38831
github.tar, level 1, compress simple, 39200
github.tar, level 3, compress simple, 38884
github.tar, level 4, compress simple, 38880
github.tar, level 3, compress simple, 38831
github.tar, level 4, compress simple, 38893
github.tar, level 5, compress simple, 39651
github.tar, level 6, compress simple, 39282
github.tar, level 7, compress simple, 38005
@@ -30,45 +30,45 @@ github.tar, level 9, compress
github.tar, level 13, compress simple, 35501
github.tar, level 16, compress simple, 40466
github.tar, level 19, compress simple, 32262
github.tar, uncompressed literals, compress simple, 38884
github.tar, uncompressed literals, compress simple, 38831
github.tar, uncompressed literals optimal, compress simple, 32262
github.tar, huffman literals, compress simple, 42606
silesia, level -5, compress cctx, 6854688
silesia, level -3, compress cctx, 6502839
silesia, level -1, compress cctx, 6173625
silesia, level 0, compress cctx, 4832054
silesia, level 1, compress cctx, 5304296
silesia, level 3, compress cctx, 4832054
silesia, level 4, compress cctx, 4768799
silesia, level 5, compress cctx, 4663718
silesia, level 6, compress cctx, 4600034
silesia, level 7, compress cctx, 4566069
silesia, level 9, compress cctx, 4540520
silesia, level 13, compress cctx, 4488969
silesia, level 16, compress cctx, 4356799
silesia, level 19, compress cctx, 4265851
silesia, long distance mode, compress cctx, 4832054
silesia, multithreaded, compress cctx, 4832054
silesia, multithreaded long distance mode, compress cctx, 4832054
silesia, small window log, compress cctx, 7082907
silesia, small hash log, compress cctx, 6525510
silesia, small chain log, compress cctx, 4912248
silesia, explicit params, compress cctx, 4789676
silesia, uncompressed literals, compress cctx, 4832054
silesia, uncompressed literals optimal, compress cctx, 4265851
silesia, huffman literals, compress cctx, 6173625
silesia, multithreaded with advanced params, compress cctx, 4832054
github.tar, huffman literals, compress simple, 42560
silesia, level -5, compress cctx, 6857372
silesia, level -3, compress cctx, 6503412
silesia, level -1, compress cctx, 6172202
silesia, level 0, compress cctx, 4842075
silesia, level 1, compress cctx, 5306632
silesia, level 3, compress cctx, 4842075
silesia, level 4, compress cctx, 4779186
silesia, level 5, compress cctx, 4667668
silesia, level 6, compress cctx, 4604351
silesia, level 7, compress cctx, 4570271
silesia, level 9, compress cctx, 4545850
silesia, level 13, compress cctx, 4493990
silesia, level 16, compress cctx, 4359652
silesia, level 19, compress cctx, 4266582
silesia, long distance mode, compress cctx, 4842075
silesia, multithreaded, compress cctx, 4842075
silesia, multithreaded long distance mode, compress cctx, 4842075
silesia, small window log, compress cctx, 7082951
silesia, small hash log, compress cctx, 6526141
silesia, small chain log, compress cctx, 4912197
silesia, explicit params, compress cctx, 4794318
silesia, uncompressed literals, compress cctx, 4842075
silesia, uncompressed literals optimal, compress cctx, 4266582
silesia, huffman literals, compress cctx, 6172202
silesia, multithreaded with advanced params, compress cctx, 4842075
github, level -5, compress cctx, 204407
github, level -5 with dict, compress cctx, 47581
github, level -3, compress cctx, 193253
github, level -3 with dict, compress cctx, 43043
github, level -1, compress cctx, 175468
github, level -1 with dict, compress cctx, 42044
github, level 0, compress cctx, 136331
github, level 0, compress cctx, 136332
github, level 0 with dict, compress cctx, 41534
github, level 1, compress cctx, 142365
github, level 1 with dict, compress cctx, 41715
github, level 3, compress cctx, 136331
github, level 3, compress cctx, 136332
github, level 3 with dict, compress cctx, 41534
github, level 4, compress cctx, 136199
github, level 4 with dict, compress cctx, 41725
@@ -93,75 +93,75 @@ github, small window log, compress
github, small hash log, compress cctx, 138949
github, small chain log, compress cctx, 139242
github, explicit params, compress cctx, 140932
github, uncompressed literals, compress cctx, 136331
github, uncompressed literals, compress cctx, 136332
github, uncompressed literals optimal, compress cctx, 132879
github, huffman literals, compress cctx, 175468
github, multithreaded with advanced params, compress cctx, 141069
silesia, level -5, zstdcli, 6854509
silesia, level -3, zstdcli, 6502336
silesia, level -1, zstdcli, 6171366
silesia, level 0, zstdcli, 4833113
silesia, level 1, zstdcli, 5302161
silesia, level 3, zstdcli, 4833113
silesia, level 4, zstdcli, 4770061
silesia, level 5, zstdcli, 4663332
silesia, level 6, zstdcli, 4599601
silesia, level 7, zstdcli, 4565601
silesia, level 9, zstdcli, 4540082
silesia, level 13, zstdcli, 4488438
silesia, level 16, zstdcli, 4358150
silesia, level 19, zstdcli, 4265929
silesia, long distance mode, zstdcli, 4824341
silesia, multithreaded, zstdcli, 4833113
silesia, multithreaded long distance mode, zstdcli, 4824341
silesia, small window log, zstdcli, 7094528
silesia, small hash log, zstdcli, 6527214
silesia, small chain log, zstdcli, 4911647
silesia, explicit params, zstdcli, 4790803
silesia, uncompressed literals, zstdcli, 5118235
silesia, uncompressed literals optimal, zstdcli, 4316761
silesia, huffman literals, zstdcli, 5316827
silesia, multithreaded with advanced params, zstdcli, 5118235
silesia.tar, level -5, zstdcli, 6859945
silesia.tar, level -3, zstdcli, 6504296
silesia.tar, level -1, zstdcli, 6176520
silesia.tar, level 0, zstdcli, 4836004
silesia.tar, level 1, zstdcli, 5309074
silesia.tar, level 3, zstdcli, 4836004
silesia.tar, level 4, zstdcli, 4774061
silesia.tar, level 5, zstdcli, 4667310
silesia.tar, level 6, zstdcli, 4602398
silesia.tar, level 7, zstdcli, 4568891
silesia.tar, level 9, zstdcli, 4541098
silesia.tar, level 13, zstdcli, 4488484
silesia.tar, level 16, zstdcli, 4357018
silesia.tar, level 19, zstdcli, 4259593
silesia.tar, no source size, zstdcli, 4836000
silesia.tar, long distance mode, zstdcli, 4827830
silesia.tar, multithreaded, zstdcli, 4836004
silesia.tar, multithreaded long distance mode, zstdcli, 4827830
silesia.tar, small window log, zstdcli, 7100110
silesia.tar, small hash log, zstdcli, 6530127
silesia.tar, small chain log, zstdcli, 4915865
silesia.tar, explicit params, zstdcli, 4808370
silesia.tar, uncompressed literals, zstdcli, 5116583
silesia.tar, uncompressed literals optimal, zstdcli, 4306520
silesia.tar, huffman literals, zstdcli, 5324019
silesia.tar, multithreaded with advanced params, zstdcli, 5116583
silesia, level -5, zstdcli, 6857420
silesia, level -3, zstdcli, 6503460
silesia, level -1, zstdcli, 6172250
silesia, level 0, zstdcli, 4842123
silesia, level 1, zstdcli, 5306680
silesia, level 3, zstdcli, 4842123
silesia, level 4, zstdcli, 4779234
silesia, level 5, zstdcli, 4667716
silesia, level 6, zstdcli, 4604399
silesia, level 7, zstdcli, 4570319
silesia, level 9, zstdcli, 4545898
silesia, level 13, zstdcli, 4494038
silesia, level 16, zstdcli, 4359700
silesia, level 19, zstdcli, 4266630
silesia, long distance mode, zstdcli, 4833785
silesia, multithreaded, zstdcli, 4842123
silesia, multithreaded long distance mode, zstdcli, 4833785
silesia, small window log, zstdcli, 7095048
silesia, small hash log, zstdcli, 6526189
silesia, small chain log, zstdcli, 4912245
silesia, explicit params, zstdcli, 4795840
silesia, uncompressed literals, zstdcli, 5120614
silesia, uncompressed literals optimal, zstdcli, 4316928
silesia, huffman literals, zstdcli, 5321417
silesia, multithreaded with advanced params, zstdcli, 5120614
silesia.tar, level -5, zstdcli, 6862049
silesia.tar, level -3, zstdcli, 6506509
silesia.tar, level -1, zstdcli, 6179789
silesia.tar, level 0, zstdcli, 4854164
silesia.tar, level 1, zstdcli, 5329010
silesia.tar, level 3, zstdcli, 4854164
silesia.tar, level 4, zstdcli, 4792352
silesia.tar, level 5, zstdcli, 4679860
silesia.tar, level 6, zstdcli, 4615355
silesia.tar, level 7, zstdcli, 4581791
silesia.tar, level 9, zstdcli, 4555452
silesia.tar, level 13, zstdcli, 4502960
silesia.tar, level 16, zstdcli, 4360389
silesia.tar, level 19, zstdcli, 4260943
silesia.tar, no source size, zstdcli, 4854160
silesia.tar, long distance mode, zstdcli, 4845745
silesia.tar, multithreaded, zstdcli, 4854164
silesia.tar, multithreaded long distance mode, zstdcli, 4845745
silesia.tar, small window log, zstdcli, 7100701
silesia.tar, small hash log, zstdcli, 6529264
silesia.tar, small chain log, zstdcli, 4917022
silesia.tar, explicit params, zstdcli, 4821112
silesia.tar, uncompressed literals, zstdcli, 5122571
silesia.tar, uncompressed literals optimal, zstdcli, 4308455
silesia.tar, huffman literals, zstdcli, 5342074
silesia.tar, multithreaded with advanced params, zstdcli, 5122571
github, level -5, zstdcli, 206407
github, level -5 with dict, zstdcli, 47832
github, level -3, zstdcli, 195253
github, level -3 with dict, zstdcli, 46671
github, level -1, zstdcli, 177468
github, level -1 with dict, zstdcli, 43825
github, level 0, zstdcli, 138331
github, level 0 with dict, zstdcli, 43118
github, level 0, zstdcli, 138332
github, level 0 with dict, zstdcli, 43148
github, level 1, zstdcli, 144365
github, level 1 with dict, zstdcli, 43266
github, level 3, zstdcli, 138331
github, level 3 with dict, zstdcli, 43118
github, level 3, zstdcli, 138332
github, level 3 with dict, zstdcli, 43148
github, level 4, zstdcli, 138199
github, level 4 with dict, zstdcli, 43229
github, level 4 with dict, zstdcli, 43251
github, level 5, zstdcli, 137121
github, level 5 with dict, zstdcli, 40728
github, level 6, zstdcli, 137122
@@ -176,30 +176,30 @@ github, level 16, zstdcli,
github, level 16 with dict, zstdcli, 39902
github, level 19, zstdcli, 134879
github, level 19 with dict, zstdcli, 39916
github, long distance mode, zstdcli, 138331
github, multithreaded, zstdcli, 138331
github, multithreaded long distance mode, zstdcli, 138331
github, small window log, zstdcli, 138331
github, long distance mode, zstdcli, 138332
github, multithreaded, zstdcli, 138332
github, multithreaded long distance mode, zstdcli, 138332
github, small window log, zstdcli, 138332
github, small hash log, zstdcli, 137590
github, small chain log, zstdcli, 138341
github, explicit params, zstdcli, 136197
github, uncompressed literals, zstdcli, 167909
github, uncompressed literals, zstdcli, 167911
github, uncompressed literals optimal, zstdcli, 154667
github, huffman literals, zstdcli, 144365
github, multithreaded with advanced params, zstdcli, 167909
github.tar, level -5, zstdcli, 52231
github.tar, level -5 with dict, zstdcli, 51249
github.tar, level -3, zstdcli, 45778
github.tar, level -3 with dict, zstdcli, 44847
github.tar, level -1, zstdcli, 42680
github.tar, level -1 with dict, zstdcli, 41486
github.tar, level 0, zstdcli, 38888
github, multithreaded with advanced params, zstdcli, 167911
github.tar, level -5, zstdcli, 52119
github.tar, level -5 with dict, zstdcli, 51101
github.tar, level -3, zstdcli, 45682
github.tar, level -3 with dict, zstdcli, 44738
github.tar, level -1, zstdcli, 42564
github.tar, level -1 with dict, zstdcli, 41357
github.tar, level 0, zstdcli, 38835
github.tar, level 0 with dict, zstdcli, 37999
github.tar, level 1, zstdcli, 39340
github.tar, level 1 with dict, zstdcli, 38230
github.tar, level 3, zstdcli, 38888
github.tar, level 1, zstdcli, 39204
github.tar, level 1 with dict, zstdcli, 38123
github.tar, level 3, zstdcli, 38835
github.tar, level 3 with dict, zstdcli, 37999
github.tar, level 4, zstdcli, 38884
github.tar, level 4, zstdcli, 38897
github.tar, level 4 with dict, zstdcli, 37952
github.tar, level 5, zstdcli, 39655
github.tar, level 5 with dict, zstdcli, 39073
@@ -215,97 +215,97 @@ github.tar, level 16, zstdcli,
github.tar, level 16 with dict, zstdcli, 33379
github.tar, level 19, zstdcli, 32266
github.tar, level 19 with dict, zstdcli, 32705
github.tar, no source size, zstdcli, 38885
github.tar, no source size with dict, zstdcli, 38115
github.tar, long distance mode, zstdcli, 40143
github.tar, multithreaded, zstdcli, 38888
github.tar, multithreaded long distance mode, zstdcli, 40143
github.tar, small window log, zstdcli, 198539
github.tar, no source size, zstdcli, 38832
github.tar, no source size with dict, zstdcli, 38004
github.tar, long distance mode, zstdcli, 40236
github.tar, multithreaded, zstdcli, 38835
github.tar, multithreaded long distance mode, zstdcli, 40236
github.tar, small window log, zstdcli, 198544
github.tar, small hash log, zstdcli, 129874
github.tar, small chain log, zstdcli, 41673
github.tar, explicit params, zstdcli, 41385
github.tar, uncompressed literals, zstdcli, 41566
github.tar, uncompressed literals, zstdcli, 41529
github.tar, uncompressed literals optimal, zstdcli, 35360
github.tar, huffman literals, zstdcli, 38989
github.tar, multithreaded with advanced params, zstdcli, 41566
silesia, level -5, advanced one pass, 6854688
silesia, level -3, advanced one pass, 6502839
silesia, level -1, advanced one pass, 6173625
silesia, level 0, advanced one pass, 4832054
silesia, level 1, advanced one pass, 5304296
silesia, level 3, advanced one pass, 4832054
silesia, level 4, advanced one pass, 4768799
silesia, level 5 row 1, advanced one pass, 4663718
silesia, level 5 row 2, advanced one pass, 4666272
silesia, level 5, advanced one pass, 4663718
silesia, level 6, advanced one pass, 4600034
silesia, level 7 row 1, advanced one pass, 4566069
silesia, level 7 row 2, advanced one pass, 4560893
silesia, level 7, advanced one pass, 4566069
silesia, level 9, advanced one pass, 4540520
silesia, level 11 row 1, advanced one pass, 4500472
silesia, level 11 row 2, advanced one pass, 4498174
silesia, level 12 row 1, advanced one pass, 4500472
silesia, level 12 row 2, advanced one pass, 4498174
silesia, level 13, advanced one pass, 4488969
silesia, level 16, advanced one pass, 4356799
silesia, level 19, advanced one pass, 4265851
silesia, no source size, advanced one pass, 4832054
silesia, long distance mode, advanced one pass, 4823264
silesia, multithreaded, advanced one pass, 4833065
silesia, multithreaded long distance mode, advanced one pass, 4824293
silesia, small window log, advanced one pass, 7094480
silesia, small hash log, advanced one pass, 6525510
silesia, small chain log, advanced one pass, 4912248
silesia, explicit params, advanced one pass, 4791219
silesia, uncompressed literals, advanced one pass, 5117526
silesia, uncompressed literals optimal, advanced one pass, 4316644
silesia, huffman literals, advanced one pass, 5319104
silesia, multithreaded with advanced params, advanced one pass, 5118187
silesia.tar, level -5, advanced one pass, 6858730
silesia.tar, level -3, advanced one pass, 6502944
silesia.tar, level -1, advanced one pass, 6175652
silesia.tar, level 0, advanced one pass, 4829268
silesia.tar, level 1, advanced one pass, 5307443
silesia.tar, level 3, advanced one pass, 4829268
silesia.tar, level 4, advanced one pass, 4767074
silesia.tar, level 5 row 1, advanced one pass, 4662847
silesia.tar, level 5 row 2, advanced one pass, 4666825
silesia.tar, level 5, advanced one pass, 4662847
silesia.tar, level 6, advanced one pass, 4597877
silesia.tar, level 7 row 1, advanced one pass, 4563998
silesia.tar, level 7 row 2, advanced one pass, 4559520
silesia.tar, level 7, advanced one pass, 4563998
silesia.tar, level 9, advanced one pass, 4537558
silesia.tar, level 11 row 1, advanced one pass, 4496590
silesia.tar, level 11 row 2, advanced one pass, 4495225
silesia.tar, level 12 row 1, advanced one pass, 4496084
silesia.tar, level 12 row 2, advanced one pass, 4495434
silesia.tar, level 13, advanced one pass, 4484732
silesia.tar, level 16, advanced one pass, 4355572
silesia.tar, level 19, advanced one pass, 4257629
silesia.tar, no source size, advanced one pass, 4829268
silesia.tar, long distance mode, advanced one pass, 4815868
silesia.tar, multithreaded, advanced one pass, 4836000
silesia.tar, multithreaded long distance mode, advanced one pass, 4827826
silesia.tar, small window log, advanced one pass, 7100064
silesia.tar, small hash log, advanced one pass, 6530222
silesia.tar, small chain log, advanced one pass, 4915689
silesia.tar, explicit params, advanced one pass, 4790421
silesia.tar, uncompressed literals, advanced one pass, 5114702
silesia.tar, uncompressed literals optimal, advanced one pass, 4306289
silesia.tar, huffman literals, advanced one pass, 5323421
silesia.tar, multithreaded with advanced params, advanced one pass, 5116579
github.tar, huffman literals, zstdcli, 38857
github.tar, multithreaded with advanced params, zstdcli, 41529
silesia, level -5, advanced one pass, 6857372
silesia, level -3, advanced one pass, 6503412
silesia, level -1, advanced one pass, 6172202
silesia, level 0, advanced one pass, 4842075
silesia, level 1, advanced one pass, 5306632
silesia, level 3, advanced one pass, 4842075
silesia, level 4, advanced one pass, 4779186
silesia, level 5 row 1, advanced one pass, 4667668
silesia, level 5 row 2, advanced one pass, 4670326
silesia, level 5, advanced one pass, 4667668
silesia, level 6, advanced one pass, 4604351
silesia, level 7 row 1, advanced one pass, 4570271
silesia, level 7 row 2, advanced one pass, 4565169
silesia, level 7, advanced one pass, 4570271
silesia, level 9, advanced one pass, 4545850
silesia, level 11 row 1, advanced one pass, 4505658
silesia, level 11 row 2, advanced one pass, 4503429
silesia, level 12 row 1, advanced one pass, 4505658
silesia, level 12 row 2, advanced one pass, 4503429
silesia, level 13, advanced one pass, 4493990
silesia, level 16, advanced one pass, 4359652
silesia, level 19, advanced one pass, 4266582
silesia, no source size, advanced one pass, 4842075
silesia, long distance mode, advanced one pass, 4833710
silesia, multithreaded, advanced one pass, 4842075
silesia, multithreaded long distance mode, advanced one pass, 4833737
silesia, small window log, advanced one pass, 7095000
silesia, small hash log, advanced one pass, 6526141
silesia, small chain log, advanced one pass, 4912197
silesia, explicit params, advanced one pass, 4795840
silesia, uncompressed literals, advanced one pass, 5120566
silesia, uncompressed literals optimal, advanced one pass, 4316880
silesia, huffman literals, advanced one pass, 5321369
silesia, multithreaded with advanced params, advanced one pass, 5120566
silesia.tar, level -5, advanced one pass, 6861055
silesia.tar, level -3, advanced one pass, 6505483
silesia.tar, level -1, advanced one pass, 6179047
silesia.tar, level 0, advanced one pass, 4854086
silesia.tar, level 1, advanced one pass, 5327717
silesia.tar, level 3, advanced one pass, 4854086
silesia.tar, level 4, advanced one pass, 4791503
silesia.tar, level 5 row 1, advanced one pass, 4679004
silesia.tar, level 5 row 2, advanced one pass, 4682334
silesia.tar, level 5, advanced one pass, 4679004
silesia.tar, level 6, advanced one pass, 4614561
silesia.tar, level 7 row 1, advanced one pass, 4579828
silesia.tar, level 7 row 2, advanced one pass, 4575602
silesia.tar, level 7, advanced one pass, 4579828
silesia.tar, level 9, advanced one pass, 4555448
silesia.tar, level 11 row 1, advanced one pass, 4514962
silesia.tar, level 11 row 2, advanced one pass, 4513816
silesia.tar, level 12 row 1, advanced one pass, 4514517
silesia.tar, level 12 row 2, advanced one pass, 4514007
silesia.tar, level 13, advanced one pass, 4502956
silesia.tar, level 16, advanced one pass, 4360385
silesia.tar, level 19, advanced one pass, 4260939
silesia.tar, no source size, advanced one pass, 4854086
silesia.tar, long distance mode, advanced one pass, 4840452
silesia.tar, multithreaded, advanced one pass, 4854160
silesia.tar, multithreaded long distance mode, advanced one pass, 4845741
silesia.tar, small window log, advanced one pass, 7100655
silesia.tar, small hash log, advanced one pass, 6529206
silesia.tar, small chain log, advanced one pass, 4917041
silesia.tar, explicit params, advanced one pass, 4807274
silesia.tar, uncompressed literals, advanced one pass, 5122473
silesia.tar, uncompressed literals optimal, advanced one pass, 4308451
silesia.tar, huffman literals, advanced one pass, 5341705
silesia.tar, multithreaded with advanced params, advanced one pass, 5122567
github, level -5, advanced one pass, 204407
github, level -5 with dict, advanced one pass, 45832
github, level -3, advanced one pass, 193253
github, level -3 with dict, advanced one pass, 44671
github, level -1, advanced one pass, 175468
github, level -1 with dict, advanced one pass, 41825
github, level 0, advanced one pass, 136331
github, level 0 with dict, advanced one pass, 41118
github, level 0 with dict dms, advanced one pass, 41118
github, level 0 with dict dds, advanced one pass, 41118
github, level 0, advanced one pass, 136332
github, level 0 with dict, advanced one pass, 41148
github, level 0 with dict dms, advanced one pass, 41148
github, level 0 with dict dds, advanced one pass, 41148
github, level 0 with dict copy, advanced one pass, 41124
github, level 0 with dict load, advanced one pass, 41847
github, level 1, advanced one pass, 142365
@@ -314,16 +314,16 @@ github, level 1 with dict dms, advanced
github, level 1 with dict dds, advanced one pass, 41266
github, level 1 with dict copy, advanced one pass, 41279
github, level 1 with dict load, advanced one pass, 43331
github, level 3, advanced one pass, 136331
github, level 3 with dict, advanced one pass, 41118
github, level 3 with dict dms, advanced one pass, 41118
github, level 3 with dict dds, advanced one pass, 41118
github, level 3, advanced one pass, 136332
github, level 3 with dict, advanced one pass, 41148
github, level 3 with dict dms, advanced one pass, 41148
github, level 3 with dict dds, advanced one pass, 41148
github, level 3 with dict copy, advanced one pass, 41124
github, level 3 with dict load, advanced one pass, 41847
github, level 4, advanced one pass, 136199
github, level 4 with dict, advanced one pass, 41229
github, level 4 with dict dms, advanced one pass, 41229
github, level 4 with dict dds, advanced one pass, 41229
github, level 4 with dict, advanced one pass, 41251
github, level 4 with dict dms, advanced one pass, 41251
github, level 4 with dict dds, advanced one pass, 41251
github, level 4 with dict copy, advanced one pass, 41216
github, level 4 with dict load, advanced one pass, 41548
github, level 5 row 1, advanced one pass, 134584
@@ -395,60 +395,60 @@ github, level 13 with dict, advanced
github, level 13 with dict dms, advanced one pass, 39900
github, level 13 with dict dds, advanced one pass, 39900
github, level 13 with dict copy, advanced one pass, 39948
github, level 13 with dict load, advanced one pass, 42643
github, level 13 with dict load, advanced one pass, 42624
github, level 16, advanced one pass, 133209
github, level 16 with dict, advanced one pass, 37902
github, level 16 with dict dms, advanced one pass, 37902
github, level 16 with dict dds, advanced one pass, 37902
github, level 16 with dict copy, advanced one pass, 37892
github, level 16 with dict load, advanced one pass, 42434
github, level 16 with dict load, advanced one pass, 42402
github, level 19, advanced one pass, 132879
github, level 19 with dict, advanced one pass, 37916
github, level 19 with dict dms, advanced one pass, 37916
github, level 19 with dict dds, advanced one pass, 37916
github, level 19 with dict copy, advanced one pass, 37906
github, level 19 with dict load, advanced one pass, 40405
github, no source size, advanced one pass, 136331
github, no source size with dict, advanced one pass, 41118
github, long distance mode, advanced one pass, 136331
github, multithreaded, advanced one pass, 136331
github, multithreaded long distance mode, advanced one pass, 136331
github, small window log, advanced one pass, 136331
github, level 19 with dict load, advanced one pass, 39770
github, no source size, advanced one pass, 136332
github, no source size with dict, advanced one pass, 41148
github, long distance mode, advanced one pass, 136332
github, multithreaded, advanced one pass, 136332
github, multithreaded long distance mode, advanced one pass, 136332
github, small window log, advanced one pass, 136332
github, small hash log, advanced one pass, 135590
github, small chain log, advanced one pass, 136341
github, explicit params, advanced one pass, 137727
github, uncompressed literals, advanced one pass, 165909
github, uncompressed literals, advanced one pass, 165911
github, uncompressed literals optimal, advanced one pass, 152667
github, huffman literals, advanced one pass, 142365
github, multithreaded with advanced params, advanced one pass, 165909
github.tar, level -5, advanced one pass, 52173
github.tar, level -5 with dict, advanced one pass, 51161
github.tar, level -3, advanced one pass, 45783
github.tar, level -3 with dict, advanced one pass, 44768
github.tar, level -1, advanced one pass, 42606
github.tar, level -1 with dict, advanced one pass, 41397
github.tar, level 0, advanced one pass, 38884
github, multithreaded with advanced params, advanced one pass, 165911
github.tar, level -5, advanced one pass, 52115
github.tar, level -5 with dict, advanced one pass, 51097
github.tar, level -3, advanced one pass, 45678
github.tar, level -3 with dict, advanced one pass, 44734
github.tar, level -1, advanced one pass, 42560
github.tar, level -1 with dict, advanced one pass, 41353
github.tar, level 0, advanced one pass, 38831
github.tar, level 0 with dict, advanced one pass, 37995
github.tar, level 0 with dict dms, advanced one pass, 38114
github.tar, level 0 with dict dds, advanced one pass, 38114
github.tar, level 0 with dict dms, advanced one pass, 38003
github.tar, level 0 with dict dds, advanced one pass, 38003
github.tar, level 0 with dict copy, advanced one pass, 37995
github.tar, level 0 with dict load, advanced one pass, 37956
github.tar, level 1, advanced one pass, 39200
github.tar, level 1 with dict, advanced one pass, 38189
github.tar, level 1 with dict dms, advanced one pass, 38398
github.tar, level 1 with dict dds, advanced one pass, 38398
github.tar, level 1 with dict copy, advanced one pass, 38189
github.tar, level 1 with dict load, advanced one pass, 38393
github.tar, level 3, advanced one pass, 38884
github.tar, level 1 with dict, advanced one pass, 38119
github.tar, level 1 with dict dms, advanced one pass, 38406
github.tar, level 1 with dict dds, advanced one pass, 38406
github.tar, level 1 with dict copy, advanced one pass, 38119
github.tar, level 1 with dict load, advanced one pass, 38364
github.tar, level 3, advanced one pass, 38831
github.tar, level 3 with dict, advanced one pass, 37995
github.tar, level 3 with dict dms, advanced one pass, 38114
github.tar, level 3 with dict dds, advanced one pass, 38114
github.tar, level 3 with dict dms, advanced one pass, 38003
github.tar, level 3 with dict dds, advanced one pass, 38003
github.tar, level 3 with dict copy, advanced one pass, 37995
github.tar, level 3 with dict load, advanced one pass, 37956
github.tar, level 4, advanced one pass, 38880
github.tar, level 4, advanced one pass, 38893
github.tar, level 4 with dict, advanced one pass, 37948
github.tar, level 4 with dict dms, advanced one pass, 37995
github.tar, level 4 with dict dds, advanced one pass, 37995
github.tar, level 4 with dict dms, advanced one pass, 37954
github.tar, level 4 with dict dds, advanced one pass, 37954
github.tar, level 4 with dict copy, advanced one pass, 37948
github.tar, level 4 with dict load, advanced one pass, 37927
github.tar, level 5 row 1, advanced one pass, 39651
@@ -533,97 +533,97 @@ github.tar, level 19 with dict dms, advanced
github.tar, level 19 with dict dds, advanced one pass, 32565
github.tar, level 19 with dict copy, advanced one pass, 32701
github.tar, level 19 with dict load, advanced one pass, 32428
github.tar, no source size, advanced one pass, 38884
github.tar, no source size, advanced one pass, 38831
github.tar, no source size with dict, advanced one pass, 37995
github.tar, long distance mode, advanced one pass, 40156
github.tar, multithreaded, advanced one pass, 38884
github.tar, multithreaded long distance mode, advanced one pass, 40139
github.tar, small window log, advanced one pass, 198535
github.tar, long distance mode, advanced one pass, 40252
github.tar, multithreaded, advanced one pass, 38831
github.tar, multithreaded long distance mode, advanced one pass, 40232
github.tar, small window log, advanced one pass, 198540
github.tar, small hash log, advanced one pass, 129870
github.tar, small chain log, advanced one pass, 41669
github.tar, explicit params, advanced one pass, 41385
github.tar, uncompressed literals, advanced one pass, 41562
github.tar, uncompressed literals, advanced one pass, 41525
github.tar, uncompressed literals optimal, advanced one pass, 35356
github.tar, huffman literals, advanced one pass, 38921
github.tar, multithreaded with advanced params, advanced one pass, 41562
silesia, level -5, advanced one pass small out, 6854688
silesia, level -3, advanced one pass small out, 6502839
silesia, level -1, advanced one pass small out, 6173625
silesia, level 0, advanced one pass small out, 4832054
silesia, level 1, advanced one pass small out, 5304296
silesia, level 3, advanced one pass small out, 4832054
silesia, level 4, advanced one pass small out, 4768799
silesia, level 5 row 1, advanced one pass small out, 4663718
silesia, level 5 row 2, advanced one pass small out, 4666272
silesia, level 5, advanced one pass small out, 4663718
silesia, level 6, advanced one pass small out, 4600034
silesia, level 7 row 1, advanced one pass small out, 4566069
silesia, level 7 row 2, advanced one pass small out, 4560893
silesia, level 7, advanced one pass small out, 4566069
silesia, level 9, advanced one pass small out, 4540520
silesia, level 11 row 1, advanced one pass small out, 4500472
silesia, level 11 row 2, advanced one pass small out, 4498174
silesia, level 12 row 1, advanced one pass small out, 4500472
silesia, level 12 row 2, advanced one pass small out, 4498174
silesia, level 13, advanced one pass small out, 4488969
silesia, level 16, advanced one pass small out, 4356799
silesia, level 19, advanced one pass small out, 4265851
silesia, no source size, advanced one pass small out, 4832054
silesia, long distance mode, advanced one pass small out, 4823264
silesia, multithreaded, advanced one pass small out, 4833065
silesia, multithreaded long distance mode, advanced one pass small out, 4824293
silesia, small window log, advanced one pass small out, 7094480
silesia, small hash log, advanced one pass small out, 6525510
silesia, small chain log, advanced one pass small out, 4912248
silesia, explicit params, advanced one pass small out, 4791219
silesia, uncompressed literals, advanced one pass small out, 5117526
silesia, uncompressed literals optimal, advanced one pass small out, 4316644
silesia, huffman literals, advanced one pass small out, 5319104
silesia, multithreaded with advanced params, advanced one pass small out, 5118187
silesia.tar, level -5, advanced one pass small out, 6858730
silesia.tar, level -3, advanced one pass small out, 6502944
silesia.tar, level -1, advanced one pass small out, 6175652
silesia.tar, level 0, advanced one pass small out, 4829268
silesia.tar, level 1, advanced one pass small out, 5307443
silesia.tar, level 3, advanced one pass small out, 4829268
silesia.tar, level 4, advanced one pass small out, 4767074
silesia.tar, level 5 row 1, advanced one pass small out, 4662847
silesia.tar, level 5 row 2, advanced one pass small out, 4666825
silesia.tar, level 5, advanced one pass small out, 4662847
silesia.tar, level 6, advanced one pass small out, 4597877
silesia.tar, level 7 row 1, advanced one pass small out, 4563998
silesia.tar, level 7 row 2, advanced one pass small out, 4559520
silesia.tar, level 7, advanced one pass small out, 4563998
silesia.tar, level 9, advanced one pass small out, 4537558
silesia.tar, level 11 row 1, advanced one pass small out, 4496590
silesia.tar, level 11 row 2, advanced one pass small out, 4495225
silesia.tar, level 12 row 1, advanced one pass small out, 4496084
silesia.tar, level 12 row 2, advanced one pass small out, 4495434
silesia.tar, level 13, advanced one pass small out, 4484732
silesia.tar, level 16, advanced one pass small out, 4355572
silesia.tar, level 19, advanced one pass small out, 4257629
silesia.tar, no source size, advanced one pass small out, 4829268
silesia.tar, long distance mode, advanced one pass small out, 4815868
silesia.tar, multithreaded, advanced one pass small out, 4836000
silesia.tar, multithreaded long distance mode, advanced one pass small out, 4827826
silesia.tar, small window log, advanced one pass small out, 7100064
silesia.tar, small hash log, advanced one pass small out, 6530222
silesia.tar, small chain log, advanced one pass small out, 4915689
silesia.tar, explicit params, advanced one pass small out, 4790421
silesia.tar, uncompressed literals, advanced one pass small out, 5114702
silesia.tar, uncompressed literals optimal, advanced one pass small out, 4306289
silesia.tar, huffman literals, advanced one pass small out, 5323421
silesia.tar, multithreaded with advanced params, advanced one pass small out, 5116579
github.tar, huffman literals, advanced one pass, 38853
github.tar, multithreaded with advanced params, advanced one pass, 41525
silesia, level -5, advanced one pass small out, 6857372
silesia, level -3, advanced one pass small out, 6503412
silesia, level -1, advanced one pass small out, 6172202
silesia, level 0, advanced one pass small out, 4842075
silesia, level 1, advanced one pass small out, 5306632
silesia, level 3, advanced one pass small out, 4842075
silesia, level 4, advanced one pass small out, 4779186
silesia, level 5 row 1, advanced one pass small out, 4667668
silesia, level 5 row 2, advanced one pass small out, 4670326
silesia, level 5, advanced one pass small out, 4667668
silesia, level 6, advanced one pass small out, 4604351
silesia, level 7 row 1, advanced one pass small out, 4570271
silesia, level 7 row 2, advanced one pass small out, 4565169
silesia, level 7, advanced one pass small out, 4570271
silesia, level 9, advanced one pass small out, 4545850
silesia, level 11 row 1, advanced one pass small out, 4505658
silesia, level 11 row 2, advanced one pass small out, 4503429
silesia, level 12 row 1, advanced one pass small out, 4505658
silesia, level 12 row 2, advanced one pass small out, 4503429
silesia, level 13, advanced one pass small out, 4493990
silesia, level 16, advanced one pass small out, 4359652
silesia, level 19, advanced one pass small out, 4266582
silesia, no source size, advanced one pass small out, 4842075
silesia, long distance mode, advanced one pass small out, 4833710
silesia, multithreaded, advanced one pass small out, 4842075
silesia, multithreaded long distance mode, advanced one pass small out, 4833737
silesia, small window log, advanced one pass small out, 7095000
silesia, small hash log, advanced one pass small out, 6526141
silesia, small chain log, advanced one pass small out, 4912197
silesia, explicit params, advanced one pass small out, 4795840
silesia, uncompressed literals, advanced one pass small out, 5120566
silesia, uncompressed literals optimal, advanced one pass small out, 4316880
silesia, huffman literals, advanced one pass small out, 5321369
silesia, multithreaded with advanced params, advanced one pass small out, 5120566
silesia.tar, level -5, advanced one pass small out, 6861055
silesia.tar, level -3, advanced one pass small out, 6505483
silesia.tar, level -1, advanced one pass small out, 6179047
silesia.tar, level 0, advanced one pass small out, 4854086
silesia.tar, level 1, advanced one pass small out, 5327717
silesia.tar, level 3, advanced one pass small out, 4854086
silesia.tar, level 4, advanced one pass small out, 4791503
silesia.tar, level 5 row 1, advanced one pass small out, 4679004
silesia.tar, level 5 row 2, advanced one pass small out, 4682334
silesia.tar, level 5, advanced one pass small out, 4679004
silesia.tar, level 6, advanced one pass small out, 4614561
silesia.tar, level 7 row 1, advanced one pass small out, 4579828
silesia.tar, level 7 row 2, advanced one pass small out, 4575602
silesia.tar, level 7, advanced one pass small out, 4579828
silesia.tar, level 9, advanced one pass small out, 4555448
silesia.tar, level 11 row 1, advanced one pass small out, 4514962
silesia.tar, level 11 row 2, advanced one pass small out, 4513816
silesia.tar, level 12 row 1, advanced one pass small out, 4514517
silesia.tar, level 12 row 2, advanced one pass small out, 4514007
silesia.tar, level 13, advanced one pass small out, 4502956
silesia.tar, level 16, advanced one pass small out, 4360385
silesia.tar, level 19, advanced one pass small out, 4260939
silesia.tar, no source size, advanced one pass small out, 4854086
silesia.tar, long distance mode, advanced one pass small out, 4840452
silesia.tar, multithreaded, advanced one pass small out, 4854160
silesia.tar, multithreaded long distance mode, advanced one pass small out, 4845741
silesia.tar, small window log, advanced one pass small out, 7100655
silesia.tar, small hash log, advanced one pass small out, 6529206
silesia.tar, small chain log, advanced one pass small out, 4917041
silesia.tar, explicit params, advanced one pass small out, 4807274
silesia.tar, uncompressed literals, advanced one pass small out, 5122473
silesia.tar, uncompressed literals optimal, advanced one pass small out, 4308451
silesia.tar, huffman literals, advanced one pass small out, 5341705
silesia.tar, multithreaded with advanced params, advanced one pass small out, 5122567
github, level -5, advanced one pass small out, 204407
github, level -5 with dict, advanced one pass small out, 45832
github, level -3, advanced one pass small out, 193253
github, level -3 with dict, advanced one pass small out, 44671
github, level -1, advanced one pass small out, 175468
github, level -1 with dict, advanced one pass small out, 41825
github, level 0, advanced one pass small out, 136331
github, level 0 with dict, advanced one pass small out, 41118
github, level 0 with dict dms, advanced one pass small out, 41118
github, level 0 with dict dds, advanced one pass small out, 41118
github, level 0, advanced one pass small out, 136332
github, level 0 with dict, advanced one pass small out, 41148
github, level 0 with dict dms, advanced one pass small out, 41148
github, level 0 with dict dds, advanced one pass small out, 41148
github, level 0 with dict copy, advanced one pass small out, 41124
github, level 0 with dict load, advanced one pass small out, 41847
github, level 1, advanced one pass small out, 142365
@@ -632,16 +632,16 @@ github, level 1 with dict dms, advanced
github, level 1 with dict dds, advanced one pass small out, 41266
github, level 1 with dict copy, advanced one pass small out, 41279
github, level 1 with dict load, advanced one pass small out, 43331
github, level 3, advanced one pass small out, 136331
github, level 3 with dict, advanced one pass small out, 41118
github, level 3 with dict dms, advanced one pass small out, 41118
github, level 3 with dict dds, advanced one pass small out, 41118
github, level 3, advanced one pass small out, 136332
github, level 3 with dict, advanced one pass small out, 41148
github, level 3 with dict dms, advanced one pass small out, 41148
github, level 3 with dict dds, advanced one pass small out, 41148
github, level 3 with dict copy, advanced one pass small out, 41124
github, level 3 with dict load, advanced one pass small out, 41847
github, level 4, advanced one pass small out, 136199
github, level 4 with dict, advanced one pass small out, 41229
github, level 4 with dict dms, advanced one pass small out, 41229
github, level 4 with dict dds, advanced one pass small out, 41229
github, level 4 with dict, advanced one pass small out, 41251
github, level 4 with dict dms, advanced one pass small out, 41251
github, level 4 with dict dds, advanced one pass small out, 41251
github, level 4 with dict copy, advanced one pass small out, 41216
github, level 4 with dict load, advanced one pass small out, 41548
github, level 5 row 1, advanced one pass small out, 134584
@@ -713,60 +713,60 @@ github, level 13 with dict, advanced
github, level 13 with dict dms, advanced one pass small out, 39900
github, level 13 with dict dds, advanced one pass small out, 39900
github, level 13 with dict copy, advanced one pass small out, 39948
github, level 13 with dict load, advanced one pass small out, 42643
github, level 13 with dict load, advanced one pass small out, 42624
github, level 16, advanced one pass small out, 133209
github, level 16 with dict, advanced one pass small out, 37902
github, level 16 with dict dms, advanced one pass small out, 37902
github, level 16 with dict dds, advanced one pass small out, 37902
github, level 16 with dict copy, advanced one pass small out, 37892
github, level 16 with dict load, advanced one pass small out, 42434
github, level 16 with dict load, advanced one pass small out, 42402
github, level 19, advanced one pass small out, 132879
github, level 19 with dict, advanced one pass small out, 37916
github, level 19 with dict dms, advanced one pass small out, 37916
github, level 19 with dict dds, advanced one pass small out, 37916
github, level 19 with dict copy, advanced one pass small out, 37906
github, level 19 with dict load, advanced one pass small out, 40405
github, no source size, advanced one pass small out, 136331
github, no source size with dict, advanced one pass small out, 41118
github, long distance mode, advanced one pass small out, 136331
github, multithreaded, advanced one pass small out, 136331
github, multithreaded long distance mode, advanced one pass small out, 136331
github, small window log, advanced one pass small out, 136331
github, level 19 with dict load, advanced one pass small out, 39770
github, no source size, advanced one pass small out, 136332
github, no source size with dict, advanced one pass small out, 41148
github, long distance mode, advanced one pass small out, 136332
github, multithreaded, advanced one pass small out, 136332
github, multithreaded long distance mode, advanced one pass small out, 136332
github, small window log, advanced one pass small out, 136332
github, small hash log, advanced one pass small out, 135590
github, small chain log, advanced one pass small out, 136341
github, explicit params, advanced one pass small out, 137727
github, uncompressed literals, advanced one pass small out, 165909
github, uncompressed literals, advanced one pass small out, 165911
github, uncompressed literals optimal, advanced one pass small out, 152667
github, huffman literals, advanced one pass small out, 142365
github, multithreaded with advanced params, advanced one pass small out, 165909
github.tar, level -5, advanced one pass small out, 52173
github.tar, level -5 with dict, advanced one pass small out, 51161
github.tar, level -3, advanced one pass small out, 45783
github.tar, level -3 with dict, advanced one pass small out, 44768
github.tar, level -1, advanced one pass small out, 42606
github.tar, level -1 with dict, advanced one pass small out, 41397
github.tar, level 0, advanced one pass small out, 38884
github, multithreaded with advanced params, advanced one pass small out, 165911
github.tar, level -5, advanced one pass small out, 52115
github.tar, level -5 with dict, advanced one pass small out, 51097
github.tar, level -3, advanced one pass small out, 45678
github.tar, level -3 with dict, advanced one pass small out, 44734
github.tar, level -1, advanced one pass small out, 42560
github.tar, level -1 with dict, advanced one pass small out, 41353
github.tar, level 0, advanced one pass small out, 38831
github.tar, level 0 with dict, advanced one pass small out, 37995
github.tar, level 0 with dict dms, advanced one pass small out, 38114
github.tar, level 0 with dict dds, advanced one pass small out, 38114
github.tar, level 0 with dict dms, advanced one pass small out, 38003
github.tar, level 0 with dict dds, advanced one pass small out, 38003
github.tar, level 0 with dict copy, advanced one pass small out, 37995
github.tar, level 0 with dict load, advanced one pass small out, 37956
github.tar, level 1, advanced one pass small out, 39200
github.tar, level 1 with dict, advanced one pass small out, 38189
github.tar, level 1 with dict dms, advanced one pass small out, 38398
github.tar, level 1 with dict dds, advanced one pass small out, 38398
github.tar, level 1 with dict copy, advanced one pass small out, 38189
github.tar, level 1 with dict load, advanced one pass small out, 38393
github.tar, level 3, advanced one pass small out, 38884
github.tar, level 1 with dict, advanced one pass small out, 38119
github.tar, level 1 with dict dms, advanced one pass small out, 38406
github.tar, level 1 with dict dds, advanced one pass small out, 38406
github.tar, level 1 with dict copy, advanced one pass small out, 38119
github.tar, level 1 with dict load, advanced one pass small out, 38364
github.tar, level 3, advanced one pass small out, 38831
github.tar, level 3 with dict, advanced one pass small out, 37995
github.tar, level 3 with dict dms, advanced one pass small out, 38114
github.tar, level 3 with dict dds, advanced one pass small out, 38114
github.tar, level 3 with dict dms, advanced one pass small out, 38003
github.tar, level 3 with dict dds, advanced one pass small out, 38003
github.tar, level 3 with dict copy, advanced one pass small out, 37995
github.tar, level 3 with dict load, advanced one pass small out, 37956
github.tar, level 4, advanced one pass small out, 38880
github.tar, level 4, advanced one pass small out, 38893
github.tar, level 4 with dict, advanced one pass small out, 37948
github.tar, level 4 with dict dms, advanced one pass small out, 37995
github.tar, level 4 with dict dds, advanced one pass small out, 37995
github.tar, level 4 with dict dms, advanced one pass small out, 37954
github.tar, level 4 with dict dds, advanced one pass small out, 37954
github.tar, level 4 with dict copy, advanced one pass small out, 37948
github.tar, level 4 with dict load, advanced one pass small out, 37927
github.tar, level 5 row 1, advanced one pass small out, 39651
@@ -851,97 +851,97 @@ github.tar, level 19 with dict dms, advanced
github.tar, level 19 with dict dds, advanced one pass small out, 32565
github.tar, level 19 with dict copy, advanced one pass small out, 32701
github.tar, level 19 with dict load, advanced one pass small out, 32428
github.tar, no source size, advanced one pass small out, 38884
github.tar, no source size, advanced one pass small out, 38831
github.tar, no source size with dict, advanced one pass small out, 37995
github.tar, long distance mode, advanced one pass small out, 40156
github.tar, multithreaded, advanced one pass small out, 38884
github.tar, multithreaded long distance mode, advanced one pass small out, 40139
github.tar, small window log, advanced one pass small out, 198535
github.tar, long distance mode, advanced one pass small out, 40252
github.tar, multithreaded, advanced one pass small out, 38831
github.tar, multithreaded long distance mode, advanced one pass small out, 40232
github.tar, small window log, advanced one pass small out, 198540
github.tar, small hash log, advanced one pass small out, 129870
github.tar, small chain log, advanced one pass small out, 41669
github.tar, explicit params, advanced one pass small out, 41385
github.tar, uncompressed literals, advanced one pass small out, 41562
github.tar, uncompressed literals, advanced one pass small out, 41525
github.tar, uncompressed literals optimal, advanced one pass small out, 35356
github.tar, huffman literals, advanced one pass small out, 38921
github.tar, multithreaded with advanced params, advanced one pass small out, 41562
silesia, level -5, advanced streaming, 6853462
silesia, level -3, advanced streaming, 6502349
silesia, level -1, advanced streaming, 6172125
silesia, level 0, advanced streaming, 4835804
silesia, level 1, advanced streaming, 5301644
silesia, level 3, advanced streaming, 4835804
silesia, level 4, advanced streaming, 4773049
silesia, level 5 row 1, advanced streaming, 4664679
silesia, level 5 row 2, advanced streaming, 4667307
silesia, level 5, advanced streaming, 4664679
silesia, level 6, advanced streaming, 4601116
silesia, level 7 row 1, advanced streaming, 4567082
silesia, level 7 row 2, advanced streaming, 4561992
silesia, level 7, advanced streaming, 4567082
silesia, level 9, advanced streaming, 4542474
silesia, level 11 row 1, advanced streaming, 4502322
silesia, level 11 row 2, advanced streaming, 4500050
silesia, level 12 row 1, advanced streaming, 4502322
silesia, level 12 row 2, advanced streaming, 4500050
silesia, level 13, advanced streaming, 4490650
silesia, level 16, advanced streaming, 4358094
silesia, level 19, advanced streaming, 4265908
silesia, no source size, advanced streaming, 4835768
silesia, long distance mode, advanced streaming, 4827032
silesia, multithreaded, advanced streaming, 4833065
silesia, multithreaded long distance mode, advanced streaming, 4824293
silesia, small window log, advanced streaming, 7110591
silesia, small hash log, advanced streaming, 6525259
silesia, small chain log, advanced streaming, 4911577
silesia, explicit params, advanced streaming, 4792505
silesia, uncompressed literals, advanced streaming, 5116404
silesia, uncompressed literals optimal, advanced streaming, 4316533
silesia, huffman literals, advanced streaming, 5317620
silesia, multithreaded with advanced params, advanced streaming, 5118187
silesia.tar, level -5, advanced streaming, 6853184
silesia.tar, level -3, advanced streaming, 6503455
silesia.tar, level -1, advanced streaming, 6175761
silesia.tar, level 0, advanced streaming, 4846783
silesia.tar, level 1, advanced streaming, 5306719
silesia.tar, level 3, advanced streaming, 4846783
silesia.tar, level 4, advanced streaming, 4785332
silesia.tar, level 5 row 1, advanced streaming, 4664523
silesia.tar, level 5 row 2, advanced streaming, 4668292
silesia.tar, level 5, advanced streaming, 4664523
silesia.tar, level 6, advanced streaming, 4599420
silesia.tar, level 7 row 1, advanced streaming, 4565332
silesia.tar, level 7 row 2, advanced streaming, 4561064
silesia.tar, level 7, advanced streaming, 4565332
silesia.tar, level 9, advanced streaming, 4539391
silesia.tar, level 11 row 1, advanced streaming, 4498530
silesia.tar, level 11 row 2, advanced streaming, 4497297
silesia.tar, level 12 row 1, advanced streaming, 4498097
silesia.tar, level 12 row 2, advanced streaming, 4497497
silesia.tar, level 13, advanced streaming, 4486652
silesia.tar, level 16, advanced streaming, 4358029
silesia.tar, level 19, advanced streaming, 4258228
silesia.tar, no source size, advanced streaming, 4846779
silesia.tar, long distance mode, advanced streaming, 4825842
silesia.tar, multithreaded, advanced streaming, 4836000
silesia.tar, multithreaded long distance mode, advanced streaming, 4827826
silesia.tar, small window log, advanced streaming, 7117024
silesia.tar, small hash log, advanced streaming, 6529503
silesia.tar, small chain log, advanced streaming, 4915956
silesia.tar, explicit params, advanced streaming, 4791739
silesia.tar, uncompressed literals, advanced streaming, 5123274
silesia.tar, uncompressed literals optimal, advanced streaming, 4306968
silesia.tar, huffman literals, advanced streaming, 5323245
silesia.tar, multithreaded with advanced params, advanced streaming, 5116579
github.tar, huffman literals, advanced one pass small out, 38853
github.tar, multithreaded with advanced params, advanced one pass small out, 41525
silesia, level -5, advanced streaming, 6854744
silesia, level -3, advanced streaming, 6503319
silesia, level -1, advanced streaming, 6172207
silesia, level 0, advanced streaming, 4842075
silesia, level 1, advanced streaming, 5306388
silesia, level 3, advanced streaming, 4842075
silesia, level 4, advanced streaming, 4779186
silesia, level 5 row 1, advanced streaming, 4667668
silesia, level 5 row 2, advanced streaming, 4670326
silesia, level 5, advanced streaming, 4667668
silesia, level 6, advanced streaming, 4604351
silesia, level 7 row 1, advanced streaming, 4570271
silesia, level 7 row 2, advanced streaming, 4565169
silesia, level 7, advanced streaming, 4570271
silesia, level 9, advanced streaming, 4545850
silesia, level 11 row 1, advanced streaming, 4505658
silesia, level 11 row 2, advanced streaming, 4503429
silesia, level 12 row 1, advanced streaming, 4505658
silesia, level 12 row 2, advanced streaming, 4503429
silesia, level 13, advanced streaming, 4493990
silesia, level 16, advanced streaming, 4359652
silesia, level 19, advanced streaming, 4266582
silesia, no source size, advanced streaming, 4842039
silesia, long distance mode, advanced streaming, 4833710
silesia, multithreaded, advanced streaming, 4842075
silesia, multithreaded long distance mode, advanced streaming, 4833737
silesia, small window log, advanced streaming, 7111103
silesia, small hash log, advanced streaming, 6526141
silesia, small chain log, advanced streaming, 4912197
silesia, explicit params, advanced streaming, 4795857
silesia, uncompressed literals, advanced streaming, 5120566
silesia, uncompressed literals optimal, advanced streaming, 4316880
silesia, huffman literals, advanced streaming, 5321370
silesia, multithreaded with advanced params, advanced streaming, 5120566
silesia.tar, level -5, advanced streaming, 6856523
silesia.tar, level -3, advanced streaming, 6505954
silesia.tar, level -1, advanced streaming, 6179056
silesia.tar, level 0, advanced streaming, 4859271
silesia.tar, level 1, advanced streaming, 5327708
silesia.tar, level 3, advanced streaming, 4859271
silesia.tar, level 4, advanced streaming, 4797470
silesia.tar, level 5 row 1, advanced streaming, 4679020
silesia.tar, level 5 row 2, advanced streaming, 4682355
silesia.tar, level 5, advanced streaming, 4679020
silesia.tar, level 6, advanced streaming, 4614558
silesia.tar, level 7 row 1, advanced streaming, 4579823
silesia.tar, level 7 row 2, advanced streaming, 4575601
silesia.tar, level 7, advanced streaming, 4579823
silesia.tar, level 9, advanced streaming, 4555445
silesia.tar, level 11 row 1, advanced streaming, 4514959
silesia.tar, level 11 row 2, advanced streaming, 4513810
silesia.tar, level 12 row 1, advanced streaming, 4514514
silesia.tar, level 12 row 2, advanced streaming, 4514003
silesia.tar, level 13, advanced streaming, 4502956
silesia.tar, level 16, advanced streaming, 4360385
silesia.tar, level 19, advanced streaming, 4260939
silesia.tar, no source size, advanced streaming, 4859267
silesia.tar, long distance mode, advanced streaming, 4840452
silesia.tar, multithreaded, advanced streaming, 4854160
silesia.tar, multithreaded long distance mode, advanced streaming, 4845741
silesia.tar, small window log, advanced streaming, 7117559
silesia.tar, small hash log, advanced streaming, 6529209
silesia.tar, small chain log, advanced streaming, 4917021
silesia.tar, explicit params, advanced streaming, 4807288
silesia.tar, uncompressed literals, advanced streaming, 5127423
silesia.tar, uncompressed literals optimal, advanced streaming, 4308451
silesia.tar, huffman literals, advanced streaming, 5341712
silesia.tar, multithreaded with advanced params, advanced streaming, 5122567
github, level -5, advanced streaming, 204407
github, level -5 with dict, advanced streaming, 45832
github, level -3, advanced streaming, 193253
github, level -3 with dict, advanced streaming, 44671
github, level -1, advanced streaming, 175468
github, level -1 with dict, advanced streaming, 41825
github, level 0, advanced streaming, 136331
github, level 0 with dict, advanced streaming, 41118
github, level 0 with dict dms, advanced streaming, 41118
github, level 0 with dict dds, advanced streaming, 41118
github, level 0, advanced streaming, 136332
github, level 0 with dict, advanced streaming, 41148
github, level 0 with dict dms, advanced streaming, 41148
github, level 0 with dict dds, advanced streaming, 41148
github, level 0 with dict copy, advanced streaming, 41124
github, level 0 with dict load, advanced streaming, 41847
github, level 1, advanced streaming, 142365
@@ -950,16 +950,16 @@ github, level 1 with dict dms, advanced
github, level 1 with dict dds, advanced streaming, 41266
github, level 1 with dict copy, advanced streaming, 41279
github, level 1 with dict load, advanced streaming, 43331
github, level 3, advanced streaming, 136331
github, level 3 with dict, advanced streaming, 41118
github, level 3 with dict dms, advanced streaming, 41118
github, level 3 with dict dds, advanced streaming, 41118
github, level 3, advanced streaming, 136332
github, level 3 with dict, advanced streaming, 41148
github, level 3 with dict dms, advanced streaming, 41148
github, level 3 with dict dds, advanced streaming, 41148
github, level 3 with dict copy, advanced streaming, 41124
github, level 3 with dict load, advanced streaming, 41847
github, level 4, advanced streaming, 136199
github, level 4 with dict, advanced streaming, 41229
github, level 4 with dict dms, advanced streaming, 41229
github, level 4 with dict dds, advanced streaming, 41229
github, level 4 with dict, advanced streaming, 41251
github, level 4 with dict dms, advanced streaming, 41251
github, level 4 with dict dds, advanced streaming, 41251
github, level 4 with dict copy, advanced streaming, 41216
github, level 4 with dict load, advanced streaming, 41548
github, level 5 row 1, advanced streaming, 134584
@@ -1031,60 +1031,60 @@ github, level 13 with dict, advanced
github, level 13 with dict dms, advanced streaming, 39900
github, level 13 with dict dds, advanced streaming, 39900
github, level 13 with dict copy, advanced streaming, 39948
github, level 13 with dict load, advanced streaming, 42643
github, level 13 with dict load, advanced streaming, 42624
github, level 16, advanced streaming, 133209
github, level 16 with dict, advanced streaming, 37902
github, level 16 with dict dms, advanced streaming, 37902
github, level 16 with dict dds, advanced streaming, 37902
github, level 16 with dict copy, advanced streaming, 37892
github, level 16 with dict load, advanced streaming, 42434
github, level 16 with dict load, advanced streaming, 42402
github, level 19, advanced streaming, 132879
github, level 19 with dict, advanced streaming, 37916
github, level 19 with dict dms, advanced streaming, 37916
github, level 19 with dict dds, advanced streaming, 37916
github, level 19 with dict copy, advanced streaming, 37906
github, level 19 with dict load, advanced streaming, 40405
github, no source size, advanced streaming, 136331
github, no source size with dict, advanced streaming, 41118
github, long distance mode, advanced streaming, 136331
github, multithreaded, advanced streaming, 136331
github, multithreaded long distance mode, advanced streaming, 136331
github, small window log, advanced streaming, 136331
github, level 19 with dict load, advanced streaming, 39770
github, no source size, advanced streaming, 136332
github, no source size with dict, advanced streaming, 41148
github, long distance mode, advanced streaming, 136332
github, multithreaded, advanced streaming, 136332
github, multithreaded long distance mode, advanced streaming, 136332
github, small window log, advanced streaming, 136332
github, small hash log, advanced streaming, 135590
github, small chain log, advanced streaming, 136341
github, explicit params, advanced streaming, 137727
github, uncompressed literals, advanced streaming, 165909
github, uncompressed literals, advanced streaming, 165911
github, uncompressed literals optimal, advanced streaming, 152667
github, huffman literals, advanced streaming, 142365
github, multithreaded with advanced params, advanced streaming, 165909
github.tar, level -5, advanced streaming, 52273
github.tar, level -5 with dict, advanced streaming, 51297
github.tar, level -3, advanced streaming, 45783
github.tar, level -3 with dict, advanced streaming, 44853
github.tar, level -1, advanced streaming, 42687
github.tar, level -1 with dict, advanced streaming, 41486
github.tar, level 0, advanced streaming, 38884
github, multithreaded with advanced params, advanced streaming, 165911
github.tar, level -5, advanced streaming, 52152
github.tar, level -5 with dict, advanced streaming, 51181
github.tar, level -3, advanced streaming, 45678
github.tar, level -3 with dict, advanced streaming, 44734
github.tar, level -1, advanced streaming, 42560
github.tar, level -1 with dict, advanced streaming, 41353
github.tar, level 0, advanced streaming, 38831
github.tar, level 0 with dict, advanced streaming, 37995
github.tar, level 0 with dict dms, advanced streaming, 38114
github.tar, level 0 with dict dds, advanced streaming, 38114
github.tar, level 0 with dict dms, advanced streaming, 38003
github.tar, level 0 with dict dds, advanced streaming, 38003
github.tar, level 0 with dict copy, advanced streaming, 37995
github.tar, level 0 with dict load, advanced streaming, 37956
github.tar, level 1, advanced streaming, 39346
github.tar, level 1 with dict, advanced streaming, 38251
github.tar, level 1 with dict dms, advanced streaming, 38557
github.tar, level 1 with dict dds, advanced streaming, 38557
github.tar, level 1 with dict copy, advanced streaming, 38251
github.tar, level 1 with dict load, advanced streaming, 38503
github.tar, level 3, advanced streaming, 38884
github.tar, level 1, advanced streaming, 39200
github.tar, level 1 with dict, advanced streaming, 38119
github.tar, level 1 with dict dms, advanced streaming, 38406
github.tar, level 1 with dict dds, advanced streaming, 38406
github.tar, level 1 with dict copy, advanced streaming, 38119
github.tar, level 1 with dict load, advanced streaming, 38364
github.tar, level 3, advanced streaming, 38831
github.tar, level 3 with dict, advanced streaming, 37995
github.tar, level 3 with dict dms, advanced streaming, 38114
github.tar, level 3 with dict dds, advanced streaming, 38114
github.tar, level 3 with dict dms, advanced streaming, 38003
github.tar, level 3 with dict dds, advanced streaming, 38003
github.tar, level 3 with dict copy, advanced streaming, 37995
github.tar, level 3 with dict load, advanced streaming, 37956
github.tar, level 4, advanced streaming, 38880
github.tar, level 4, advanced streaming, 38893
github.tar, level 4 with dict, advanced streaming, 37948
github.tar, level 4 with dict dms, advanced streaming, 37995
github.tar, level 4 with dict dds, advanced streaming, 37995
github.tar, level 4 with dict dms, advanced streaming, 37954
github.tar, level 4 with dict dds, advanced streaming, 37954
github.tar, level 4 with dict copy, advanced streaming, 37948
github.tar, level 4 with dict load, advanced streaming, 37927
github.tar, level 5 row 1, advanced streaming, 39651
@@ -1169,69 +1169,69 @@ github.tar, level 19 with dict dms, advanced
github.tar, level 19 with dict dds, advanced streaming, 32565
github.tar, level 19 with dict copy, advanced streaming, 32701
github.tar, level 19 with dict load, advanced streaming, 32428
github.tar, no source size, advanced streaming, 38881
github.tar, no source size with dict, advanced streaming, 38111
github.tar, long distance mode, advanced streaming, 40156
github.tar, multithreaded, advanced streaming, 38884
github.tar, multithreaded long distance mode, advanced streaming, 40139
github.tar, small window log, advanced streaming, 199553
github.tar, no source size, advanced streaming, 38828
github.tar, no source size with dict, advanced streaming, 38000
github.tar, long distance mode, advanced streaming, 40252
github.tar, multithreaded, advanced streaming, 38831
github.tar, multithreaded long distance mode, advanced streaming, 40232
github.tar, small window log, advanced streaming, 199558
github.tar, small hash log, advanced streaming, 129870
github.tar, small chain log, advanced streaming, 41669
github.tar, explicit params, advanced streaming, 41385
github.tar, uncompressed literals, advanced streaming, 41562
github.tar, uncompressed literals, advanced streaming, 41525
github.tar, uncompressed literals optimal, advanced streaming, 35356
github.tar, huffman literals, advanced streaming, 38998
github.tar, multithreaded with advanced params, advanced streaming, 41562
silesia, level -5, old streaming, 6853462
silesia, level -3, old streaming, 6502349
silesia, level -1, old streaming, 6172125
silesia, level 0, old streaming, 4835804
silesia, level 1, old streaming, 5301644
silesia, level 3, old streaming, 4835804
silesia, level 4, old streaming, 4773049
silesia, level 5, old streaming, 4664679
silesia, level 6, old streaming, 4601116
silesia, level 7, old streaming, 4567082
silesia, level 9, old streaming, 4542474
silesia, level 13, old streaming, 4490650
silesia, level 16, old streaming, 4358094
silesia, level 19, old streaming, 4265908
silesia, no source size, old streaming, 4835768
silesia, uncompressed literals, old streaming, 4835804
silesia, uncompressed literals optimal, old streaming, 4265908
silesia, huffman literals, old streaming, 6172125
silesia.tar, level -5, old streaming, 6853184
silesia.tar, level -3, old streaming, 6503455
silesia.tar, level -1, old streaming, 6175761
silesia.tar, level 0, old streaming, 4846783
silesia.tar, level 1, old streaming, 5306719
silesia.tar, level 3, old streaming, 4846783
silesia.tar, level 4, old streaming, 4785332
silesia.tar, level 5, old streaming, 4664523
silesia.tar, level 6, old streaming, 4599420
silesia.tar, level 7, old streaming, 4565332
silesia.tar, level 9, old streaming, 4539391
silesia.tar, level 13, old streaming, 4486652
silesia.tar, level 16, old streaming, 4358029
silesia.tar, level 19, old streaming, 4258228
silesia.tar, no source size, old streaming, 4846779
silesia.tar, uncompressed literals, old streaming, 4846783
silesia.tar, uncompressed literals optimal, old streaming, 4258228
silesia.tar, huffman literals, old streaming, 6175761
github.tar, huffman literals, advanced streaming, 38853
github.tar, multithreaded with advanced params, advanced streaming, 41525
silesia, level -5, old streaming, 6854744
silesia, level -3, old streaming, 6503319
silesia, level -1, old streaming, 6172207
silesia, level 0, old streaming, 4842075
silesia, level 1, old streaming, 5306388
silesia, level 3, old streaming, 4842075
silesia, level 4, old streaming, 4779186
silesia, level 5, old streaming, 4667668
silesia, level 6, old streaming, 4604351
silesia, level 7, old streaming, 4570271
silesia, level 9, old streaming, 4545850
silesia, level 13, old streaming, 4493990
silesia, level 16, old streaming, 4359652
silesia, level 19, old streaming, 4266582
silesia, no source size, old streaming, 4842039
silesia, uncompressed literals, old streaming, 4842075
silesia, uncompressed literals optimal, old streaming, 4266582
silesia, huffman literals, old streaming, 6172207
silesia.tar, level -5, old streaming, 6856523
silesia.tar, level -3, old streaming, 6505954
silesia.tar, level -1, old streaming, 6179056
silesia.tar, level 0, old streaming, 4859271
silesia.tar, level 1, old streaming, 5327708
silesia.tar, level 3, old streaming, 4859271
silesia.tar, level 4, old streaming, 4797470
silesia.tar, level 5, old streaming, 4679020
silesia.tar, level 6, old streaming, 4614558
silesia.tar, level 7, old streaming, 4579823
silesia.tar, level 9, old streaming, 4555445
silesia.tar, level 13, old streaming, 4502956
silesia.tar, level 16, old streaming, 4360385
silesia.tar, level 19, old streaming, 4260939
silesia.tar, no source size, old streaming, 4859267
silesia.tar, uncompressed literals, old streaming, 4859271
silesia.tar, uncompressed literals optimal, old streaming, 4260939
silesia.tar, huffman literals, old streaming, 6179056
github, level -5, old streaming, 204407
github, level -5 with dict, old streaming, 45832
github, level -3, old streaming, 193253
github, level -3 with dict, old streaming, 44671
github, level -1, old streaming, 175468
github, level -1 with dict, old streaming, 41825
github, level 0, old streaming, 136331
github, level 0 with dict, old streaming, 41118
github, level 0, old streaming, 136332
github, level 0 with dict, old streaming, 41148
github, level 1, old streaming, 142365
github, level 1 with dict, old streaming, 41266
github, level 3, old streaming, 136331
github, level 3 with dict, old streaming, 41118
github, level 3, old streaming, 136332
github, level 3 with dict, old streaming, 41148
github, level 4, old streaming, 136199
github, level 4 with dict, old streaming, 41229
github, level 4 with dict, old streaming, 41251
github, level 5, old streaming, 135121
github, level 5 with dict, old streaming, 38754
github, level 6, old streaming, 135122
@@ -1247,23 +1247,23 @@ github, level 16 with dict, old stre
github, level 19, old streaming, 132879
github, level 19 with dict, old streaming, 37916
github, no source size, old streaming, 140599
github, no source size with dict, old streaming, 40652
github, uncompressed literals, old streaming, 136331
github, no source size with dict, old streaming, 40654
github, uncompressed literals, old streaming, 136332
github, uncompressed literals optimal, old streaming, 132879
github, huffman literals, old streaming, 175468
github.tar, level -5, old streaming, 52273
github.tar, level -5 with dict, old streaming, 51297
github.tar, level -3, old streaming, 45783
github.tar, level -3 with dict, old streaming, 44853
github.tar, level -1, old streaming, 42687
github.tar, level -1 with dict, old streaming, 41486
github.tar, level 0, old streaming, 38884
github.tar, level -5, old streaming, 52152
github.tar, level -5 with dict, old streaming, 51181
github.tar, level -3, old streaming, 45678
github.tar, level -3 with dict, old streaming, 44734
github.tar, level -1, old streaming, 42560
github.tar, level -1 with dict, old streaming, 41353
github.tar, level 0, old streaming, 38831
github.tar, level 0 with dict, old streaming, 37995
github.tar, level 1, old streaming, 39346
github.tar, level 1 with dict, old streaming, 38251
github.tar, level 3, old streaming, 38884
github.tar, level 1, old streaming, 39200
github.tar, level 1 with dict, old streaming, 38119
github.tar, level 3, old streaming, 38831
github.tar, level 3 with dict, old streaming, 37995
github.tar, level 4, old streaming, 38880
github.tar, level 4, old streaming, 38893
github.tar, level 4 with dict, old streaming, 37948
github.tar, level 5, old streaming, 39651
github.tar, level 5 with dict, old streaming, 39145
@@ -1279,77 +1279,77 @@ github.tar, level 16, old stre
github.tar, level 16 with dict, old streaming, 33375
github.tar, level 19, old streaming, 32262
github.tar, level 19 with dict, old streaming, 32701
github.tar, no source size, old streaming, 38881
github.tar, no source size with dict, old streaming, 38111
github.tar, uncompressed literals, old streaming, 38884
github.tar, no source size, old streaming, 38828
github.tar, no source size with dict, old streaming, 38000
github.tar, uncompressed literals, old streaming, 38831
github.tar, uncompressed literals optimal, old streaming, 32262
github.tar, huffman literals, old streaming, 42687
silesia, level -5, old streaming advanced, 6853462
silesia, level -3, old streaming advanced, 6502349
silesia, level -1, old streaming advanced, 6172125
silesia, level 0, old streaming advanced, 4835804
silesia, level 1, old streaming advanced, 5301644
silesia, level 3, old streaming advanced, 4835804
silesia, level 4, old streaming advanced, 4773049
silesia, level 5, old streaming advanced, 4664679
silesia, level 6, old streaming advanced, 4601116
silesia, level 7, old streaming advanced, 4567082
silesia, level 9, old streaming advanced, 4542474
silesia, level 13, old streaming advanced, 4490650
silesia, level 16, old streaming advanced, 4358094
silesia, level 19, old streaming advanced, 4265908
silesia, no source size, old streaming advanced, 4835768
silesia, long distance mode, old streaming advanced, 4835804
silesia, multithreaded, old streaming advanced, 4835804
silesia, multithreaded long distance mode, old streaming advanced, 4835804
silesia, small window log, old streaming advanced, 7110591
silesia, small hash log, old streaming advanced, 6525259
silesia, small chain log, old streaming advanced, 4911577
silesia, explicit params, old streaming advanced, 4792505
silesia, uncompressed literals, old streaming advanced, 4835804
silesia, uncompressed literals optimal, old streaming advanced, 4265908
silesia, huffman literals, old streaming advanced, 6172125
silesia, multithreaded with advanced params, old streaming advanced, 4835804
silesia.tar, level -5, old streaming advanced, 6853184
silesia.tar, level -3, old streaming advanced, 6503455
silesia.tar, level -1, old streaming advanced, 6175761
silesia.tar, level 0, old streaming advanced, 4846783
silesia.tar, level 1, old streaming advanced, 5306719
silesia.tar, level 3, old streaming advanced, 4846783
silesia.tar, level 4, old streaming advanced, 4785332
silesia.tar, level 5, old streaming advanced, 4664523
silesia.tar, level 6, old streaming advanced, 4599420
silesia.tar, level 7, old streaming advanced, 4565332
silesia.tar, level 9, old streaming advanced, 4539391
silesia.tar, level 13, old streaming advanced, 4486652
silesia.tar, level 16, old streaming advanced, 4358029
silesia.tar, level 19, old streaming advanced, 4258228
silesia.tar, no source size, old streaming advanced, 4846779
silesia.tar, long distance mode, old streaming advanced, 4846783
silesia.tar, multithreaded, old streaming advanced, 4846783
silesia.tar, multithreaded long distance mode, old streaming advanced, 4846783
silesia.tar, small window log, old streaming advanced, 7117027
silesia.tar, small hash log, old streaming advanced, 6529503
silesia.tar, small chain log, old streaming advanced, 4915956
silesia.tar, explicit params, old streaming advanced, 4791739
silesia.tar, uncompressed literals, old streaming advanced, 4846783
silesia.tar, uncompressed literals optimal, old streaming advanced, 4258228
silesia.tar, huffman literals, old streaming advanced, 6175761
silesia.tar, multithreaded with advanced params, old streaming advanced, 4846783
github.tar, huffman literals, old streaming, 42560
silesia, level -5, old streaming advanced, 6854744
silesia, level -3, old streaming advanced, 6503319
silesia, level -1, old streaming advanced, 6172207
silesia, level 0, old streaming advanced, 4842075
silesia, level 1, old streaming advanced, 5306388
silesia, level 3, old streaming advanced, 4842075
silesia, level 4, old streaming advanced, 4779186
silesia, level 5, old streaming advanced, 4667668
silesia, level 6, old streaming advanced, 4604351
silesia, level 7, old streaming advanced, 4570271
silesia, level 9, old streaming advanced, 4545850
silesia, level 13, old streaming advanced, 4493990
silesia, level 16, old streaming advanced, 4359652
silesia, level 19, old streaming advanced, 4266582
silesia, no source size, old streaming advanced, 4842039
silesia, long distance mode, old streaming advanced, 4842075
silesia, multithreaded, old streaming advanced, 4842075
silesia, multithreaded long distance mode, old streaming advanced, 4842075
silesia, small window log, old streaming advanced, 7111103
silesia, small hash log, old streaming advanced, 6526141
silesia, small chain log, old streaming advanced, 4912197
silesia, explicit params, old streaming advanced, 4795857
silesia, uncompressed literals, old streaming advanced, 4842075
silesia, uncompressed literals optimal, old streaming advanced, 4266582
silesia, huffman literals, old streaming advanced, 6172207
silesia, multithreaded with advanced params, old streaming advanced, 4842075
silesia.tar, level -5, old streaming advanced, 6856523
silesia.tar, level -3, old streaming advanced, 6505954
silesia.tar, level -1, old streaming advanced, 6179056
silesia.tar, level 0, old streaming advanced, 4859271
silesia.tar, level 1, old streaming advanced, 5327708
silesia.tar, level 3, old streaming advanced, 4859271
silesia.tar, level 4, old streaming advanced, 4797470
silesia.tar, level 5, old streaming advanced, 4679020
silesia.tar, level 6, old streaming advanced, 4614558
silesia.tar, level 7, old streaming advanced, 4579823
silesia.tar, level 9, old streaming advanced, 4555445
silesia.tar, level 13, old streaming advanced, 4502956
silesia.tar, level 16, old streaming advanced, 4360385
silesia.tar, level 19, old streaming advanced, 4260939
silesia.tar, no source size, old streaming advanced, 4859267
silesia.tar, long distance mode, old streaming advanced, 4859271
silesia.tar, multithreaded, old streaming advanced, 4859271
silesia.tar, multithreaded long distance mode, old streaming advanced, 4859271
silesia.tar, small window log, old streaming advanced, 7117562
silesia.tar, small hash log, old streaming advanced, 6529209
silesia.tar, small chain log, old streaming advanced, 4917021
silesia.tar, explicit params, old streaming advanced, 4807288
silesia.tar, uncompressed literals, old streaming advanced, 4859271
silesia.tar, uncompressed literals optimal, old streaming advanced, 4260939
silesia.tar, huffman literals, old streaming advanced, 6179056
silesia.tar, multithreaded with advanced params, old streaming advanced, 4859271
github, level -5, old streaming advanced, 213265
github, level -5 with dict, old streaming advanced, 46708
github, level -3, old streaming advanced, 196126
github, level -3 with dict, old streaming advanced, 45476
github, level -1, old streaming advanced, 181107
github, level -1 with dict, old streaming advanced, 42060
github, level 0, old streaming advanced, 141101
github, level 0 with dict, old streaming advanced, 41074
github, level 0, old streaming advanced, 141104
github, level 0 with dict, old streaming advanced, 41113
github, level 1, old streaming advanced, 143693
github, level 1 with dict, old streaming advanced, 42430
github, level 3, old streaming advanced, 141101
github, level 3 with dict, old streaming advanced, 41074
github, level 4, old streaming advanced, 141101
github, level 4 with dict, old streaming advanced, 41046
github, level 3, old streaming advanced, 141104
github, level 3 with dict, old streaming advanced, 41113
github, level 4, old streaming advanced, 141104
github, level 4 with dict, old streaming advanced, 41084
github, level 5, old streaming advanced, 139402
github, level 5 with dict, old streaming advanced, 38723
github, level 6, old streaming advanced, 138676
@@ -1366,30 +1366,30 @@ github, level 19, old stre
github, level 19 with dict, old streaming advanced, 37916
github, no source size, old streaming advanced, 140599
github, no source size with dict, old streaming advanced, 40608
github, long distance mode, old streaming advanced, 141101
github, multithreaded, old streaming advanced, 141101
github, multithreaded long distance mode, old streaming advanced, 141101
github, small window log, old streaming advanced, 141101
github, long distance mode, old streaming advanced, 141104
github, multithreaded, old streaming advanced, 141104
github, multithreaded long distance mode, old streaming advanced, 141104
github, small window log, old streaming advanced, 141104
github, small hash log, old streaming advanced, 141597
github, small chain log, old streaming advanced, 139275
github, explicit params, old streaming advanced, 140937
github, uncompressed literals, old streaming advanced, 141101
github, uncompressed literals, old streaming advanced, 141104
github, uncompressed literals optimal, old streaming advanced, 132879
github, huffman literals, old streaming advanced, 181107
github, multithreaded with advanced params, old streaming advanced, 141101
github.tar, level -5, old streaming advanced, 52273
github.tar, level -5 with dict, old streaming advanced, 51249
github.tar, level -3, old streaming advanced, 45783
github.tar, level -3 with dict, old streaming advanced, 45093
github.tar, level -1, old streaming advanced, 42687
github.tar, level -1 with dict, old streaming advanced, 41762
github.tar, level 0, old streaming advanced, 38884
github, multithreaded with advanced params, old streaming advanced, 141104
github.tar, level -5, old streaming advanced, 52152
github.tar, level -5 with dict, old streaming advanced, 51129
github.tar, level -3, old streaming advanced, 45678
github.tar, level -3 with dict, old streaming advanced, 44986
github.tar, level -1, old streaming advanced, 42560
github.tar, level -1 with dict, old streaming advanced, 41650
github.tar, level 0, old streaming advanced, 38831
github.tar, level 0 with dict, old streaming advanced, 38013
github.tar, level 1, old streaming advanced, 39346
github.tar, level 1 with dict, old streaming advanced, 38507
github.tar, level 3, old streaming advanced, 38884
github.tar, level 1, old streaming advanced, 39200
github.tar, level 1 with dict, old streaming advanced, 38359
github.tar, level 3, old streaming advanced, 38831
github.tar, level 3 with dict, old streaming advanced, 38013
github.tar, level 4, old streaming advanced, 38880
github.tar, level 4, old streaming advanced, 38893
github.tar, level 4 with dict, old streaming advanced, 38063
github.tar, level 5, old streaming advanced, 39651
github.tar, level 5 with dict, old streaming advanced, 39018
@@ -1405,26 +1405,26 @@ github.tar, level 16, old stre
github.tar, level 16 with dict, old streaming advanced, 38578
github.tar, level 19, old streaming advanced, 32262
github.tar, level 19 with dict, old streaming advanced, 32678
github.tar, no source size, old streaming advanced, 38881
github.tar, no source size with dict, old streaming advanced, 38076
github.tar, long distance mode, old streaming advanced, 38884
github.tar, multithreaded, old streaming advanced, 38884
github.tar, multithreaded long distance mode, old streaming advanced, 38884
github.tar, small window log, old streaming advanced, 199556
github.tar, no source size, old streaming advanced, 38828
github.tar, no source size with dict, old streaming advanced, 38015
github.tar, long distance mode, old streaming advanced, 38831
github.tar, multithreaded, old streaming advanced, 38831
github.tar, multithreaded long distance mode, old streaming advanced, 38831
github.tar, small window log, old streaming advanced, 199561
github.tar, small hash log, old streaming advanced, 129870
github.tar, small chain log, old streaming advanced, 41669
github.tar, explicit params, old streaming advanced, 41385
github.tar, uncompressed literals, old streaming advanced, 38884
github.tar, uncompressed literals, old streaming advanced, 38831
github.tar, uncompressed literals optimal, old streaming advanced, 32262
github.tar, huffman literals, old streaming advanced, 42687
github.tar, multithreaded with advanced params, old streaming advanced, 38884
github.tar, huffman literals, old streaming advanced, 42560
github.tar, multithreaded with advanced params, old streaming advanced, 38831
github, level -5 with dict, old streaming cdict, 45832
github, level -3 with dict, old streaming cdict, 44671
github, level -1 with dict, old streaming cdict, 41825
github, level 0 with dict, old streaming cdict, 41118
github, level 0 with dict, old streaming cdict, 41148
github, level 1 with dict, old streaming cdict, 41266
github, level 3 with dict, old streaming cdict, 41118
github, level 4 with dict, old streaming cdict, 41229
github, level 3 with dict, old streaming cdict, 41148
github, level 4 with dict, old streaming cdict, 41251
github, level 5 with dict, old streaming cdict, 38754
github, level 6 with dict, old streaming cdict, 38669
github, level 7 with dict, old streaming cdict, 38765
@@ -1432,12 +1432,12 @@ github, level 9 with dict, old stre
github, level 13 with dict, old streaming cdict, 39900
github, level 16 with dict, old streaming cdict, 37902
github, level 19 with dict, old streaming cdict, 37916
github, no source size with dict, old streaming cdict, 40652
github.tar, level -5 with dict, old streaming cdict, 51407
github.tar, level -3 with dict, old streaming cdict, 45254
github.tar, level -1 with dict, old streaming cdict, 41973
github, no source size with dict, old streaming cdict, 40654
github.tar, level -5 with dict, old streaming cdict, 51286
github.tar, level -3 with dict, old streaming cdict, 45147
github.tar, level -1 with dict, old streaming cdict, 41865
github.tar, level 0 with dict, old streaming cdict, 37956
github.tar, level 1 with dict, old streaming cdict, 38503
github.tar, level 1 with dict, old streaming cdict, 38364
github.tar, level 3 with dict, old streaming cdict, 37956
github.tar, level 4 with dict, old streaming cdict, 37927
github.tar, level 5 with dict, old streaming cdict, 39000
@@ -1447,14 +1447,14 @@ github.tar, level 9 with dict, old stre
github.tar, level 13 with dict, old streaming cdict, 36010
github.tar, level 16 with dict, old streaming cdict, 39081
github.tar, level 19 with dict, old streaming cdict, 32428
github.tar, no source size with dict, old streaming cdict, 38111
github.tar, no source size with dict, old streaming cdict, 38000
github, level -5 with dict, old streaming advanced cdict, 46708
github, level -3 with dict, old streaming advanced cdict, 45476
github, level -1 with dict, old streaming advanced cdict, 42060
github, level 0 with dict, old streaming advanced cdict, 41074
github, level 0 with dict, old streaming advanced cdict, 41113
github, level 1 with dict, old streaming advanced cdict, 42430
github, level 3 with dict, old streaming advanced cdict, 41074
github, level 4 with dict, old streaming advanced cdict, 41046
github, level 3 with dict, old streaming advanced cdict, 41113
github, level 4 with dict, old streaming advanced cdict, 41084
github, level 5 with dict, old streaming advanced cdict, 38723
github, level 6 with dict, old streaming advanced cdict, 38744
github, level 7 with dict, old streaming advanced cdict, 38875
@@ -1463,11 +1463,11 @@ github, level 13 with dict, old stre
github, level 16 with dict, old streaming advanced cdict, 40804
github, level 19 with dict, old streaming advanced cdict, 37916
github, no source size with dict, old streaming advanced cdict, 40608
github.tar, level -5 with dict, old streaming advanced cdict, 50907
github.tar, level -3 with dict, old streaming advanced cdict, 45032
github.tar, level -1 with dict, old streaming advanced cdict, 41589
github.tar, level -5 with dict, old streaming advanced cdict, 50791
github.tar, level -3 with dict, old streaming advanced cdict, 44926
github.tar, level -1 with dict, old streaming advanced cdict, 41482
github.tar, level 0 with dict, old streaming advanced cdict, 38013
github.tar, level 1 with dict, old streaming advanced cdict, 38294
github.tar, level 1 with dict, old streaming advanced cdict, 38168
github.tar, level 3 with dict, old streaming advanced cdict, 38013
github.tar, level 4 with dict, old streaming advanced cdict, 38063
github.tar, level 5 with dict, old streaming advanced cdict, 39018
@@ -1477,4 +1477,4 @@ github.tar, level 9 with dict, old stre
github.tar, level 13 with dict, old streaming advanced cdict, 35807
github.tar, level 16 with dict, old streaming advanced cdict, 38578
github.tar, level 19 with dict, old streaming advanced cdict, 32678
github.tar, no source size with dict, old streaming advanced cdict, 38076
github.tar, no source size with dict, old streaming advanced cdict, 38015
1 Data Config Method Total compressed size
2 silesia.tar level -5 compress simple 6858730 6861055
3 silesia.tar level -3 compress simple 6502944 6505483
4 silesia.tar level -1 compress simple 6175652 6179047
5 silesia.tar level 0 compress simple 4829268 4854086
6 silesia.tar level 1 compress simple 5307443 5327717
7 silesia.tar level 3 compress simple 4829268 4854086
8 silesia.tar level 4 compress simple 4767074 4791503
9 silesia.tar level 5 compress simple 4662847 4679004
10 silesia.tar level 6 compress simple 4597877 4614561
11 silesia.tar level 7 compress simple 4563998 4579828
12 silesia.tar level 9 compress simple 4537558 4555448
13 silesia.tar level 13 compress simple 4484732 4502956
14 silesia.tar level 16 compress simple 4355572 4360385
15 silesia.tar level 19 compress simple 4257629 4260939
16 silesia.tar uncompressed literals compress simple 4829268 4854086
17 silesia.tar uncompressed literals optimal compress simple 4257629 4260939
18 silesia.tar huffman literals compress simple 6175652 6179047
19 github.tar level -5 compress simple 52173 52115
20 github.tar level -3 compress simple 45783 45678
21 github.tar level -1 compress simple 42606 42560
22 github.tar level 0 compress simple 38884 38831
23 github.tar level 1 compress simple 39200
24 github.tar level 3 compress simple 38884 38831
25 github.tar level 4 compress simple 38880 38893
26 github.tar level 5 compress simple 39651
27 github.tar level 6 compress simple 39282
28 github.tar level 7 compress simple 38005
30 github.tar level 13 compress simple 35501
31 github.tar level 16 compress simple 40466
32 github.tar level 19 compress simple 32262
33 github.tar uncompressed literals compress simple 38884 38831
34 github.tar uncompressed literals optimal compress simple 32262
35 github.tar huffman literals compress simple 42606 42560
36 silesia level -5 compress cctx 6854688 6857372
37 silesia level -3 compress cctx 6502839 6503412
38 silesia level -1 compress cctx 6173625 6172202
39 silesia level 0 compress cctx 4832054 4842075
40 silesia level 1 compress cctx 5304296 5306632
41 silesia level 3 compress cctx 4832054 4842075
42 silesia level 4 compress cctx 4768799 4779186
43 silesia level 5 compress cctx 4663718 4667668
44 silesia level 6 compress cctx 4600034 4604351
45 silesia level 7 compress cctx 4566069 4570271
46 silesia level 9 compress cctx 4540520 4545850
47 silesia level 13 compress cctx 4488969 4493990
48 silesia level 16 compress cctx 4356799 4359652
49 silesia level 19 compress cctx 4265851 4266582
50 silesia long distance mode compress cctx 4832054 4842075
51 silesia multithreaded compress cctx 4832054 4842075
52 silesia multithreaded long distance mode compress cctx 4832054 4842075
53 silesia small window log compress cctx 7082907 7082951
54 silesia small hash log compress cctx 6525510 6526141
55 silesia small chain log compress cctx 4912248 4912197
56 silesia explicit params compress cctx 4789676 4794318
57 silesia uncompressed literals compress cctx 4832054 4842075
58 silesia uncompressed literals optimal compress cctx 4265851 4266582
59 silesia huffman literals compress cctx 6173625 6172202
60 silesia multithreaded with advanced params compress cctx 4832054 4842075
61 github level -5 compress cctx 204407
62 github level -5 with dict compress cctx 47581
63 github level -3 compress cctx 193253
64 github level -3 with dict compress cctx 43043
65 github level -1 compress cctx 175468
66 github level -1 with dict compress cctx 42044
67 github level 0 compress cctx 136331 136332
68 github level 0 with dict compress cctx 41534
69 github level 1 compress cctx 142365
70 github level 1 with dict compress cctx 41715
71 github level 3 compress cctx 136331 136332
72 github level 3 with dict compress cctx 41534
73 github level 4 compress cctx 136199
74 github level 4 with dict compress cctx 41725
93 github small hash log compress cctx 138949
94 github small chain log compress cctx 139242
95 github explicit params compress cctx 140932
96 github uncompressed literals compress cctx 136331 136332
97 github uncompressed literals optimal compress cctx 132879
98 github huffman literals compress cctx 175468
99 github multithreaded with advanced params compress cctx 141069
100 silesia level -5 zstdcli 6854509 6857420
101 silesia level -3 zstdcli 6502336 6503460
102 silesia level -1 zstdcli 6171366 6172250
103 silesia level 0 zstdcli 4833113 4842123
104 silesia level 1 zstdcli 5302161 5306680
105 silesia level 3 zstdcli 4833113 4842123
106 silesia level 4 zstdcli 4770061 4779234
107 silesia level 5 zstdcli 4663332 4667716
108 silesia level 6 zstdcli 4599601 4604399
109 silesia level 7 zstdcli 4565601 4570319
110 silesia level 9 zstdcli 4540082 4545898
111 silesia level 13 zstdcli 4488438 4494038
112 silesia level 16 zstdcli 4358150 4359700
113 silesia level 19 zstdcli 4265929 4266630
114 silesia long distance mode zstdcli 4824341 4833785
115 silesia multithreaded zstdcli 4833113 4842123
116 silesia multithreaded long distance mode zstdcli 4824341 4833785
117 silesia small window log zstdcli 7094528 7095048
118 silesia small hash log zstdcli 6527214 6526189
119 silesia small chain log zstdcli 4911647 4912245
120 silesia explicit params zstdcli 4790803 4795840
121 silesia uncompressed literals zstdcli 5118235 5120614
122 silesia uncompressed literals optimal zstdcli 4316761 4316928
123 silesia huffman literals zstdcli 5316827 5321417
124 silesia multithreaded with advanced params zstdcli 5118235 5120614
125 silesia.tar level -5 zstdcli 6859945 6862049
126 silesia.tar level -3 zstdcli 6504296 6506509
127 silesia.tar level -1 zstdcli 6176520 6179789
128 silesia.tar level 0 zstdcli 4836004 4854164
129 silesia.tar level 1 zstdcli 5309074 5329010
130 silesia.tar level 3 zstdcli 4836004 4854164
131 silesia.tar level 4 zstdcli 4774061 4792352
132 silesia.tar level 5 zstdcli 4667310 4679860
133 silesia.tar level 6 zstdcli 4602398 4615355
134 silesia.tar level 7 zstdcli 4568891 4581791
135 silesia.tar level 9 zstdcli 4541098 4555452
136 silesia.tar level 13 zstdcli 4488484 4502960
137 silesia.tar level 16 zstdcli 4357018 4360389
138 silesia.tar level 19 zstdcli 4259593 4260943
139 silesia.tar no source size zstdcli 4836000 4854160
140 silesia.tar long distance mode zstdcli 4827830 4845745
141 silesia.tar multithreaded zstdcli 4836004 4854164
142 silesia.tar multithreaded long distance mode zstdcli 4827830 4845745
143 silesia.tar small window log zstdcli 7100110 7100701
144 silesia.tar small hash log zstdcli 6530127 6529264
145 silesia.tar small chain log zstdcli 4915865 4917022
146 silesia.tar explicit params zstdcli 4808370 4821112
147 silesia.tar uncompressed literals zstdcli 5116583 5122571
148 silesia.tar uncompressed literals optimal zstdcli 4306520 4308455
149 silesia.tar huffman literals zstdcli 5324019 5342074
150 silesia.tar multithreaded with advanced params zstdcli 5116583 5122571
151 github level -5 zstdcli 206407
152 github level -5 with dict zstdcli 47832
153 github level -3 zstdcli 195253
154 github level -3 with dict zstdcli 46671
155 github level -1 zstdcli 177468
156 github level -1 with dict zstdcli 43825
157 github level 0 zstdcli 138331 138332
158 github level 0 with dict zstdcli 43118 43148
159 github level 1 zstdcli 144365
160 github level 1 with dict zstdcli 43266
161 github level 3 zstdcli 138331 138332
162 github level 3 with dict zstdcli 43118 43148
163 github level 4 zstdcli 138199
164 github level 4 with dict zstdcli 43229 43251
165 github level 5 zstdcli 137121
166 github level 5 with dict zstdcli 40728
167 github level 6 zstdcli 137122
176 github level 16 with dict zstdcli 39902
177 github level 19 zstdcli 134879
178 github level 19 with dict zstdcli 39916
179 github long distance mode zstdcli 138331 138332
180 github multithreaded zstdcli 138331 138332
181 github multithreaded long distance mode zstdcli 138331 138332
182 github small window log zstdcli 138331 138332
183 github small hash log zstdcli 137590
184 github small chain log zstdcli 138341
185 github explicit params zstdcli 136197
186 github uncompressed literals zstdcli 167909 167911
187 github uncompressed literals optimal zstdcli 154667
188 github huffman literals zstdcli 144365
189 github multithreaded with advanced params zstdcli 167909 167911
190 github.tar level -5 zstdcli 52231 52119
191 github.tar level -5 with dict zstdcli 51249 51101
192 github.tar level -3 zstdcli 45778 45682
193 github.tar level -3 with dict zstdcli 44847 44738
194 github.tar level -1 zstdcli 42680 42564
195 github.tar level -1 with dict zstdcli 41486 41357
196 github.tar level 0 zstdcli 38888 38835
197 github.tar level 0 with dict zstdcli 37999
198 github.tar level 1 zstdcli 39340 39204
199 github.tar level 1 with dict zstdcli 38230 38123
200 github.tar level 3 zstdcli 38888 38835
201 github.tar level 3 with dict zstdcli 37999
202 github.tar level 4 zstdcli 38884 38897
203 github.tar level 4 with dict zstdcli 37952
204 github.tar level 5 zstdcli 39655
205 github.tar level 5 with dict zstdcli 39073
215 github.tar level 16 with dict zstdcli 33379
216 github.tar level 19 zstdcli 32266
217 github.tar level 19 with dict zstdcli 32705
218 github.tar no source size zstdcli 38885 38832
219 github.tar no source size with dict zstdcli 38115 38004
220 github.tar long distance mode zstdcli 40143 40236
221 github.tar multithreaded zstdcli 38888 38835
222 github.tar multithreaded long distance mode zstdcli 40143 40236
223 github.tar small window log zstdcli 198539 198544
224 github.tar small hash log zstdcli 129874
225 github.tar small chain log zstdcli 41673
226 github.tar explicit params zstdcli 41385
227 github.tar uncompressed literals zstdcli 41566 41529
228 github.tar uncompressed literals optimal zstdcli 35360
229 github.tar huffman literals zstdcli 38989 38857
230 github.tar multithreaded with advanced params zstdcli 41566 41529
231 silesia level -5 advanced one pass 6854688 6857372
232 silesia level -3 advanced one pass 6502839 6503412
233 silesia level -1 advanced one pass 6173625 6172202
234 silesia level 0 advanced one pass 4832054 4842075
235 silesia level 1 advanced one pass 5304296 5306632
236 silesia level 3 advanced one pass 4832054 4842075
237 silesia level 4 advanced one pass 4768799 4779186
238 silesia level 5 row 1 advanced one pass 4663718 4667668
239 silesia level 5 row 2 advanced one pass 4666272 4670326
240 silesia level 5 advanced one pass 4663718 4667668
241 silesia level 6 advanced one pass 4600034 4604351
242 silesia level 7 row 1 advanced one pass 4566069 4570271
243 silesia level 7 row 2 advanced one pass 4560893 4565169
244 silesia level 7 advanced one pass 4566069 4570271
245 silesia level 9 advanced one pass 4540520 4545850
246 silesia level 11 row 1 advanced one pass 4500472 4505658
247 silesia level 11 row 2 advanced one pass 4498174 4503429
248 silesia level 12 row 1 advanced one pass 4500472 4505658
249 silesia level 12 row 2 advanced one pass 4498174 4503429
250 silesia level 13 advanced one pass 4488969 4493990
251 silesia level 16 advanced one pass 4356799 4359652
252 silesia level 19 advanced one pass 4265851 4266582
253 silesia no source size advanced one pass 4832054 4842075
254 silesia long distance mode advanced one pass 4823264 4833710
255 silesia multithreaded advanced one pass 4833065 4842075
256 silesia multithreaded long distance mode advanced one pass 4824293 4833737
257 silesia small window log advanced one pass 7094480 7095000
258 silesia small hash log advanced one pass 6525510 6526141
259 silesia small chain log advanced one pass 4912248 4912197
260 silesia explicit params advanced one pass 4791219 4795840
261 silesia uncompressed literals advanced one pass 5117526 5120566
262 silesia uncompressed literals optimal advanced one pass 4316644 4316880
263 silesia huffman literals advanced one pass 5319104 5321369
264 silesia multithreaded with advanced params advanced one pass 5118187 5120566
265 silesia.tar level -5 advanced one pass 6858730 6861055
266 silesia.tar level -3 advanced one pass 6502944 6505483
267 silesia.tar level -1 advanced one pass 6175652 6179047
268 silesia.tar level 0 advanced one pass 4829268 4854086
269 silesia.tar level 1 advanced one pass 5307443 5327717
270 silesia.tar level 3 advanced one pass 4829268 4854086
271 silesia.tar level 4 advanced one pass 4767074 4791503
272 silesia.tar level 5 row 1 advanced one pass 4662847 4679004
273 silesia.tar level 5 row 2 advanced one pass 4666825 4682334
274 silesia.tar level 5 advanced one pass 4662847 4679004
275 silesia.tar level 6 advanced one pass 4597877 4614561
276 silesia.tar level 7 row 1 advanced one pass 4563998 4579828
277 silesia.tar level 7 row 2 advanced one pass 4559520 4575602
278 silesia.tar level 7 advanced one pass 4563998 4579828
279 silesia.tar level 9 advanced one pass 4537558 4555448
280 silesia.tar level 11 row 1 advanced one pass 4496590 4514962
281 silesia.tar level 11 row 2 advanced one pass 4495225 4513816
282 silesia.tar level 12 row 1 advanced one pass 4496084 4514517
283 silesia.tar level 12 row 2 advanced one pass 4495434 4514007
284 silesia.tar level 13 advanced one pass 4484732 4502956
285 silesia.tar level 16 advanced one pass 4355572 4360385
286 silesia.tar level 19 advanced one pass 4257629 4260939
287 silesia.tar no source size advanced one pass 4829268 4854086
288 silesia.tar long distance mode advanced one pass 4815868 4840452
289 silesia.tar multithreaded advanced one pass 4836000 4854160
290 silesia.tar multithreaded long distance mode advanced one pass 4827826 4845741
291 silesia.tar small window log advanced one pass 7100064 7100655
292 silesia.tar small hash log advanced one pass 6530222 6529206
293 silesia.tar small chain log advanced one pass 4915689 4917041
294 silesia.tar explicit params advanced one pass 4790421 4807274
295 silesia.tar uncompressed literals advanced one pass 5114702 5122473
296 silesia.tar uncompressed literals optimal advanced one pass 4306289 4308451
297 silesia.tar huffman literals advanced one pass 5323421 5341705
298 silesia.tar multithreaded with advanced params advanced one pass 5116579 5122567
299 github level -5 advanced one pass 204407
300 github level -5 with dict advanced one pass 45832
301 github level -3 advanced one pass 193253
302 github level -3 with dict advanced one pass 44671
303 github level -1 advanced one pass 175468
304 github level -1 with dict advanced one pass 41825
305 github level 0 advanced one pass 136331 136332
306 github level 0 with dict advanced one pass 41118 41148
307 github level 0 with dict dms advanced one pass 41118 41148
308 github level 0 with dict dds advanced one pass 41118 41148
309 github level 0 with dict copy advanced one pass 41124
310 github level 0 with dict load advanced one pass 41847
311 github level 1 advanced one pass 142365
314 github level 1 with dict dds advanced one pass 41266
315 github level 1 with dict copy advanced one pass 41279
316 github level 1 with dict load advanced one pass 43331
317 github level 3 advanced one pass 136331 136332
318 github level 3 with dict advanced one pass 41118 41148
319 github level 3 with dict dms advanced one pass 41118 41148
320 github level 3 with dict dds advanced one pass 41118 41148
321 github level 3 with dict copy advanced one pass 41124
322 github level 3 with dict load advanced one pass 41847
323 github level 4 advanced one pass 136199
324 github level 4 with dict advanced one pass 41229 41251
325 github level 4 with dict dms advanced one pass 41229 41251
326 github level 4 with dict dds advanced one pass 41229 41251
327 github level 4 with dict copy advanced one pass 41216
328 github level 4 with dict load advanced one pass 41548
329 github level 5 row 1 advanced one pass 134584
395 github level 13 with dict dms advanced one pass 39900
396 github level 13 with dict dds advanced one pass 39900
397 github level 13 with dict copy advanced one pass 39948
398 github level 13 with dict load advanced one pass 42643 42624
399 github level 16 advanced one pass 133209
400 github level 16 with dict advanced one pass 37902
401 github level 16 with dict dms advanced one pass 37902
402 github level 16 with dict dds advanced one pass 37902
403 github level 16 with dict copy advanced one pass 37892
404 github level 16 with dict load advanced one pass 42434 42402
405 github level 19 advanced one pass 132879
406 github level 19 with dict advanced one pass 37916
407 github level 19 with dict dms advanced one pass 37916
408 github level 19 with dict dds advanced one pass 37916
409 github level 19 with dict copy advanced one pass 37906
410 github level 19 with dict load advanced one pass 40405 39770
411 github no source size advanced one pass 136331 136332
412 github no source size with dict advanced one pass 41118 41148
413 github long distance mode advanced one pass 136331 136332
414 github multithreaded advanced one pass 136331 136332
415 github multithreaded long distance mode advanced one pass 136331 136332
416 github small window log advanced one pass 136331 136332
417 github small hash log advanced one pass 135590
418 github small chain log advanced one pass 136341
419 github explicit params advanced one pass 137727
420 github uncompressed literals advanced one pass 165909 165911
421 github uncompressed literals optimal advanced one pass 152667
422 github huffman literals advanced one pass 142365
423 github multithreaded with advanced params advanced one pass 165909 165911
424 github.tar level -5 advanced one pass 52173 52115
425 github.tar level -5 with dict advanced one pass 51161 51097
426 github.tar level -3 advanced one pass 45783 45678
427 github.tar level -3 with dict advanced one pass 44768 44734
428 github.tar level -1 advanced one pass 42606 42560
429 github.tar level -1 with dict advanced one pass 41397 41353
430 github.tar level 0 advanced one pass 38884 38831
431 github.tar level 0 with dict advanced one pass 37995
432 github.tar level 0 with dict dms advanced one pass 38114 38003
433 github.tar level 0 with dict dds advanced one pass 38114 38003
434 github.tar level 0 with dict copy advanced one pass 37995
435 github.tar level 0 with dict load advanced one pass 37956
436 github.tar level 1 advanced one pass 39200
437 github.tar level 1 with dict advanced one pass 38189 38119
438 github.tar level 1 with dict dms advanced one pass 38398 38406
439 github.tar level 1 with dict dds advanced one pass 38398 38406
440 github.tar level 1 with dict copy advanced one pass 38189 38119
441 github.tar level 1 with dict load advanced one pass 38393 38364
442 github.tar level 3 advanced one pass 38884 38831
443 github.tar level 3 with dict advanced one pass 37995
444 github.tar level 3 with dict dms advanced one pass 38114 38003
445 github.tar level 3 with dict dds advanced one pass 38114 38003
446 github.tar level 3 with dict copy advanced one pass 37995
447 github.tar level 3 with dict load advanced one pass 37956
448 github.tar level 4 advanced one pass 38880 38893
449 github.tar level 4 with dict advanced one pass 37948
450 github.tar level 4 with dict dms advanced one pass 37995 37954
451 github.tar level 4 with dict dds advanced one pass 37995 37954
452 github.tar level 4 with dict copy advanced one pass 37948
453 github.tar level 4 with dict load advanced one pass 37927
454 github.tar level 5 row 1 advanced one pass 39651
533 github.tar level 19 with dict dds advanced one pass 32565
534 github.tar level 19 with dict copy advanced one pass 32701
535 github.tar level 19 with dict load advanced one pass 32428
536 github.tar no source size advanced one pass 38884 38831
537 github.tar no source size with dict advanced one pass 37995
538 github.tar long distance mode advanced one pass 40156 40252
539 github.tar multithreaded advanced one pass 38884 38831
540 github.tar multithreaded long distance mode advanced one pass 40139 40232
541 github.tar small window log advanced one pass 198535 198540
542 github.tar small hash log advanced one pass 129870
543 github.tar small chain log advanced one pass 41669
544 github.tar explicit params advanced one pass 41385
545 github.tar uncompressed literals advanced one pass 41562 41525
546 github.tar uncompressed literals optimal advanced one pass 35356
547 github.tar huffman literals advanced one pass 38921 38853
548 github.tar multithreaded with advanced params advanced one pass 41562 41525
549 silesia level -5 advanced one pass small out 6854688 6857372
550 silesia level -3 advanced one pass small out 6502839 6503412
551 silesia level -1 advanced one pass small out 6173625 6172202
552 silesia level 0 advanced one pass small out 4832054 4842075
553 silesia level 1 advanced one pass small out 5304296 5306632
554 silesia level 3 advanced one pass small out 4832054 4842075
555 silesia level 4 advanced one pass small out 4768799 4779186
556 silesia level 5 row 1 advanced one pass small out 4663718 4667668
557 silesia level 5 row 2 advanced one pass small out 4666272 4670326
558 silesia level 5 advanced one pass small out 4663718 4667668
559 silesia level 6 advanced one pass small out 4600034 4604351
560 silesia level 7 row 1 advanced one pass small out 4566069 4570271
561 silesia level 7 row 2 advanced one pass small out 4560893 4565169
562 silesia level 7 advanced one pass small out 4566069 4570271
563 silesia level 9 advanced one pass small out 4540520 4545850
564 silesia level 11 row 1 advanced one pass small out 4500472 4505658
565 silesia level 11 row 2 advanced one pass small out 4498174 4503429
566 silesia level 12 row 1 advanced one pass small out 4500472 4505658
567 silesia level 12 row 2 advanced one pass small out 4498174 4503429
568 silesia level 13 advanced one pass small out 4488969 4493990
569 silesia level 16 advanced one pass small out 4356799 4359652
570 silesia level 19 advanced one pass small out 4265851 4266582
571 silesia no source size advanced one pass small out 4832054 4842075
572 silesia long distance mode advanced one pass small out 4823264 4833710
573 silesia multithreaded advanced one pass small out 4833065 4842075
574 silesia multithreaded long distance mode advanced one pass small out 4824293 4833737
575 silesia small window log advanced one pass small out 7094480 7095000
576 silesia small hash log advanced one pass small out 6525510 6526141
577 silesia small chain log advanced one pass small out 4912248 4912197
578 silesia explicit params advanced one pass small out 4791219 4795840
579 silesia uncompressed literals advanced one pass small out 5117526 5120566
580 silesia uncompressed literals optimal advanced one pass small out 4316644 4316880
581 silesia huffman literals advanced one pass small out 5319104 5321369
582 silesia multithreaded with advanced params advanced one pass small out 5118187 5120566
583 silesia.tar level -5 advanced one pass small out 6858730 6861055
584 silesia.tar level -3 advanced one pass small out 6502944 6505483
585 silesia.tar level -1 advanced one pass small out 6175652 6179047
586 silesia.tar level 0 advanced one pass small out 4829268 4854086
587 silesia.tar level 1 advanced one pass small out 5307443 5327717
588 silesia.tar level 3 advanced one pass small out 4829268 4854086
589 silesia.tar level 4 advanced one pass small out 4767074 4791503
590 silesia.tar level 5 row 1 advanced one pass small out 4662847 4679004
591 silesia.tar level 5 row 2 advanced one pass small out 4666825 4682334
592 silesia.tar level 5 advanced one pass small out 4662847 4679004
593 silesia.tar level 6 advanced one pass small out 4597877 4614561
594 silesia.tar level 7 row 1 advanced one pass small out 4563998 4579828
595 silesia.tar level 7 row 2 advanced one pass small out 4559520 4575602
596 silesia.tar level 7 advanced one pass small out 4563998 4579828
597 silesia.tar level 9 advanced one pass small out 4537558 4555448
598 silesia.tar level 11 row 1 advanced one pass small out 4496590 4514962
599 silesia.tar level 11 row 2 advanced one pass small out 4495225 4513816
600 silesia.tar level 12 row 1 advanced one pass small out 4496084 4514517
601 silesia.tar level 12 row 2 advanced one pass small out 4495434 4514007
602 silesia.tar level 13 advanced one pass small out 4484732 4502956
603 silesia.tar level 16 advanced one pass small out 4355572 4360385
604 silesia.tar level 19 advanced one pass small out 4257629 4260939
605 silesia.tar no source size advanced one pass small out 4829268 4854086
606 silesia.tar long distance mode advanced one pass small out 4815868 4840452
607 silesia.tar multithreaded advanced one pass small out 4836000 4854160
608 silesia.tar multithreaded long distance mode advanced one pass small out 4827826 4845741
609 silesia.tar small window log advanced one pass small out 7100064 7100655
610 silesia.tar small hash log advanced one pass small out 6530222 6529206
611 silesia.tar small chain log advanced one pass small out 4915689 4917041
612 silesia.tar explicit params advanced one pass small out 4790421 4807274
613 silesia.tar uncompressed literals advanced one pass small out 5114702 5122473
614 silesia.tar uncompressed literals optimal advanced one pass small out 4306289 4308451
615 silesia.tar huffman literals advanced one pass small out 5323421 5341705
616 silesia.tar multithreaded with advanced params advanced one pass small out 5116579 5122567
617 github level -5 advanced one pass small out 204407
618 github level -5 with dict advanced one pass small out 45832
619 github level -3 advanced one pass small out 193253
620 github level -3 with dict advanced one pass small out 44671
621 github level -1 advanced one pass small out 175468
622 github level -1 with dict advanced one pass small out 41825
623 github level 0 advanced one pass small out 136331 136332
624 github level 0 with dict advanced one pass small out 41118 41148
625 github level 0 with dict dms advanced one pass small out 41118 41148
626 github level 0 with dict dds advanced one pass small out 41118 41148
627 github level 0 with dict copy advanced one pass small out 41124
628 github level 0 with dict load advanced one pass small out 41847
629 github level 1 advanced one pass small out 142365
632 github level 1 with dict dds advanced one pass small out 41266
633 github level 1 with dict copy advanced one pass small out 41279
634 github level 1 with dict load advanced one pass small out 43331
635 github level 3 advanced one pass small out 136331 136332
636 github level 3 with dict advanced one pass small out 41118 41148
637 github level 3 with dict dms advanced one pass small out 41118 41148
638 github level 3 with dict dds advanced one pass small out 41118 41148
639 github level 3 with dict copy advanced one pass small out 41124
640 github level 3 with dict load advanced one pass small out 41847
641 github level 4 advanced one pass small out 136199
642 github level 4 with dict advanced one pass small out 41229 41251
643 github level 4 with dict dms advanced one pass small out 41229 41251
644 github level 4 with dict dds advanced one pass small out 41229 41251
645 github level 4 with dict copy advanced one pass small out 41216
646 github level 4 with dict load advanced one pass small out 41548
647 github level 5 row 1 advanced one pass small out 134584
713 github level 13 with dict dms advanced one pass small out 39900
714 github level 13 with dict dds advanced one pass small out 39900
715 github level 13 with dict copy advanced one pass small out 39948
716 github level 13 with dict load advanced one pass small out 42643 42624
717 github level 16 advanced one pass small out 133209
718 github level 16 with dict advanced one pass small out 37902
719 github level 16 with dict dms advanced one pass small out 37902
720 github level 16 with dict dds advanced one pass small out 37902
721 github level 16 with dict copy advanced one pass small out 37892
722 github level 16 with dict load advanced one pass small out 42434 42402
723 github level 19 advanced one pass small out 132879
724 github level 19 with dict advanced one pass small out 37916
725 github level 19 with dict dms advanced one pass small out 37916
726 github level 19 with dict dds advanced one pass small out 37916
727 github level 19 with dict copy advanced one pass small out 37906
728 github level 19 with dict load advanced one pass small out 40405 39770
729 github no source size advanced one pass small out 136331 136332
730 github no source size with dict advanced one pass small out 41118 41148
731 github long distance mode advanced one pass small out 136331 136332
732 github multithreaded advanced one pass small out 136331 136332
733 github multithreaded long distance mode advanced one pass small out 136331 136332
734 github small window log advanced one pass small out 136331 136332
735 github small hash log advanced one pass small out 135590
736 github small chain log advanced one pass small out 136341
737 github explicit params advanced one pass small out 137727
738 github uncompressed literals advanced one pass small out 165909 165911
739 github uncompressed literals optimal advanced one pass small out 152667
740 github huffman literals advanced one pass small out 142365
741 github multithreaded with advanced params advanced one pass small out 165909 165911
742 github.tar level -5 advanced one pass small out 52173 52115
743 github.tar level -5 with dict advanced one pass small out 51161 51097
744 github.tar level -3 advanced one pass small out 45783 45678
745 github.tar level -3 with dict advanced one pass small out 44768 44734
746 github.tar level -1 advanced one pass small out 42606 42560
747 github.tar level -1 with dict advanced one pass small out 41397 41353
748 github.tar level 0 advanced one pass small out 38884 38831
749 github.tar level 0 with dict advanced one pass small out 37995
750 github.tar level 0 with dict dms advanced one pass small out 38114 38003
751 github.tar level 0 with dict dds advanced one pass small out 38114 38003
752 github.tar level 0 with dict copy advanced one pass small out 37995
753 github.tar level 0 with dict load advanced one pass small out 37956
754 github.tar level 1 advanced one pass small out 39200
755 github.tar level 1 with dict advanced one pass small out 38189 38119
756 github.tar level 1 with dict dms advanced one pass small out 38398 38406
757 github.tar level 1 with dict dds advanced one pass small out 38398 38406
758 github.tar level 1 with dict copy advanced one pass small out 38189 38119
759 github.tar level 1 with dict load advanced one pass small out 38393 38364
760 github.tar level 3 advanced one pass small out 38884 38831
761 github.tar level 3 with dict advanced one pass small out 37995
762 github.tar level 3 with dict dms advanced one pass small out 38114 38003
763 github.tar level 3 with dict dds advanced one pass small out 38114 38003
764 github.tar level 3 with dict copy advanced one pass small out 37995
765 github.tar level 3 with dict load advanced one pass small out 37956
766 github.tar level 4 advanced one pass small out 38880 38893
767 github.tar level 4 with dict advanced one pass small out 37948
768 github.tar level 4 with dict dms advanced one pass small out 37995 37954
769 github.tar level 4 with dict dds advanced one pass small out 37995 37954
770 github.tar level 4 with dict copy advanced one pass small out 37948
771 github.tar level 4 with dict load advanced one pass small out 37927
772 github.tar level 5 row 1 advanced one pass small out 39651
851 github.tar level 19 with dict dds advanced one pass small out 32565
852 github.tar level 19 with dict copy advanced one pass small out 32701
853 github.tar level 19 with dict load advanced one pass small out 32428
854 github.tar no source size advanced one pass small out 38884 38831
855 github.tar no source size with dict advanced one pass small out 37995
856 github.tar long distance mode advanced one pass small out 40156 40252
857 github.tar multithreaded advanced one pass small out 38884 38831
858 github.tar multithreaded long distance mode advanced one pass small out 40139 40232
859 github.tar small window log advanced one pass small out 198535 198540
860 github.tar small hash log advanced one pass small out 129870
861 github.tar small chain log advanced one pass small out 41669
862 github.tar explicit params advanced one pass small out 41385
863 github.tar uncompressed literals advanced one pass small out 41562 41525
864 github.tar uncompressed literals optimal advanced one pass small out 35356
865 github.tar huffman literals advanced one pass small out 38921 38853
866 github.tar multithreaded with advanced params advanced one pass small out 41562 41525
867 silesia level -5 advanced streaming 6853462 6854744
868 silesia level -3 advanced streaming 6502349 6503319
869 silesia level -1 advanced streaming 6172125 6172207
870 silesia level 0 advanced streaming 4835804 4842075
871 silesia level 1 advanced streaming 5301644 5306388
872 silesia level 3 advanced streaming 4835804 4842075
873 silesia level 4 advanced streaming 4773049 4779186
874 silesia level 5 row 1 advanced streaming 4664679 4667668
875 silesia level 5 row 2 advanced streaming 4667307 4670326
876 silesia level 5 advanced streaming 4664679 4667668
877 silesia level 6 advanced streaming 4601116 4604351
878 silesia level 7 row 1 advanced streaming 4567082 4570271
879 silesia level 7 row 2 advanced streaming 4561992 4565169
880 silesia level 7 advanced streaming 4567082 4570271
881 silesia level 9 advanced streaming 4542474 4545850
882 silesia level 11 row 1 advanced streaming 4502322 4505658
883 silesia level 11 row 2 advanced streaming 4500050 4503429
884 silesia level 12 row 1 advanced streaming 4502322 4505658
885 silesia level 12 row 2 advanced streaming 4500050 4503429
886 silesia level 13 advanced streaming 4490650 4493990
887 silesia level 16 advanced streaming 4358094 4359652
888 silesia level 19 advanced streaming 4265908 4266582
889 silesia no source size advanced streaming 4835768 4842039
890 silesia long distance mode advanced streaming 4827032 4833710
891 silesia multithreaded advanced streaming 4833065 4842075
892 silesia multithreaded long distance mode advanced streaming 4824293 4833737
893 silesia small window log advanced streaming 7110591 7111103
894 silesia small hash log advanced streaming 6525259 6526141
895 silesia small chain log advanced streaming 4911577 4912197
896 silesia explicit params advanced streaming 4792505 4795857
897 silesia uncompressed literals advanced streaming 5116404 5120566
898 silesia uncompressed literals optimal advanced streaming 4316533 4316880
899 silesia huffman literals advanced streaming 5317620 5321370
900 silesia multithreaded with advanced params advanced streaming 5118187 5120566
901 silesia.tar level -5 advanced streaming 6853184 6856523
902 silesia.tar level -3 advanced streaming 6503455 6505954
903 silesia.tar level -1 advanced streaming 6175761 6179056
904 silesia.tar level 0 advanced streaming 4846783 4859271
905 silesia.tar level 1 advanced streaming 5306719 5327708
906 silesia.tar level 3 advanced streaming 4846783 4859271
907 silesia.tar level 4 advanced streaming 4785332 4797470
908 silesia.tar level 5 row 1 advanced streaming 4664523 4679020
909 silesia.tar level 5 row 2 advanced streaming 4668292 4682355
910 silesia.tar level 5 advanced streaming 4664523 4679020
911 silesia.tar level 6 advanced streaming 4599420 4614558
912 silesia.tar level 7 row 1 advanced streaming 4565332 4579823
913 silesia.tar level 7 row 2 advanced streaming 4561064 4575601
914 silesia.tar level 7 advanced streaming 4565332 4579823
915 silesia.tar level 9 advanced streaming 4539391 4555445
916 silesia.tar level 11 row 1 advanced streaming 4498530 4514959
917 silesia.tar level 11 row 2 advanced streaming 4497297 4513810
918 silesia.tar level 12 row 1 advanced streaming 4498097 4514514
919 silesia.tar level 12 row 2 advanced streaming 4497497 4514003
920 silesia.tar level 13 advanced streaming 4486652 4502956
921 silesia.tar level 16 advanced streaming 4358029 4360385
922 silesia.tar level 19 advanced streaming 4258228 4260939
923 silesia.tar no source size advanced streaming 4846779 4859267
924 silesia.tar long distance mode advanced streaming 4825842 4840452
925 silesia.tar multithreaded advanced streaming 4836000 4854160
926 silesia.tar multithreaded long distance mode advanced streaming 4827826 4845741
927 silesia.tar small window log advanced streaming 7117024 7117559
928 silesia.tar small hash log advanced streaming 6529503 6529209
929 silesia.tar small chain log advanced streaming 4915956 4917021
930 silesia.tar explicit params advanced streaming 4791739 4807288
931 silesia.tar uncompressed literals advanced streaming 5123274 5127423
932 silesia.tar uncompressed literals optimal advanced streaming 4306968 4308451
933 silesia.tar huffman literals advanced streaming 5323245 5341712
934 silesia.tar multithreaded with advanced params advanced streaming 5116579 5122567
935 github level -5 advanced streaming 204407
936 github level -5 with dict advanced streaming 45832
937 github level -3 advanced streaming 193253
938 github level -3 with dict advanced streaming 44671
939 github level -1 advanced streaming 175468
940 github level -1 with dict advanced streaming 41825
941 github level 0 advanced streaming 136331 136332
942 github level 0 with dict advanced streaming 41118 41148
943 github level 0 with dict dms advanced streaming 41118 41148
944 github level 0 with dict dds advanced streaming 41118 41148
945 github level 0 with dict copy advanced streaming 41124
946 github level 0 with dict load advanced streaming 41847
947 github level 1 advanced streaming 142365
950 github level 1 with dict dds advanced streaming 41266
951 github level 1 with dict copy advanced streaming 41279
952 github level 1 with dict load advanced streaming 43331
953 github level 3 advanced streaming 136331 136332
954 github level 3 with dict advanced streaming 41118 41148
955 github level 3 with dict dms advanced streaming 41118 41148
956 github level 3 with dict dds advanced streaming 41118 41148
957 github level 3 with dict copy advanced streaming 41124
958 github level 3 with dict load advanced streaming 41847
959 github level 4 advanced streaming 136199
960 github level 4 with dict advanced streaming 41229 41251
961 github level 4 with dict dms advanced streaming 41229 41251
962 github level 4 with dict dds advanced streaming 41229 41251
963 github level 4 with dict copy advanced streaming 41216
964 github level 4 with dict load advanced streaming 41548
965 github level 5 row 1 advanced streaming 134584
1031 github level 13 with dict dms advanced streaming 39900
1032 github level 13 with dict dds advanced streaming 39900
1033 github level 13 with dict copy advanced streaming 39948
1034 github level 13 with dict load advanced streaming 42643 42624
1035 github level 16 advanced streaming 133209
1036 github level 16 with dict advanced streaming 37902
1037 github level 16 with dict dms advanced streaming 37902
1038 github level 16 with dict dds advanced streaming 37902
1039 github level 16 with dict copy advanced streaming 37892
1040 github level 16 with dict load advanced streaming 42434 42402
1041 github level 19 advanced streaming 132879
1042 github level 19 with dict advanced streaming 37916
1043 github level 19 with dict dms advanced streaming 37916
1044 github level 19 with dict dds advanced streaming 37916
1045 github level 19 with dict copy advanced streaming 37906
1046 github level 19 with dict load advanced streaming 40405 39770
1047 github no source size advanced streaming 136331 136332
1048 github no source size with dict advanced streaming 41118 41148
1049 github long distance mode advanced streaming 136331 136332
1050 github multithreaded advanced streaming 136331 136332
1051 github multithreaded long distance mode advanced streaming 136331 136332
1052 github small window log advanced streaming 136331 136332
1053 github small hash log advanced streaming 135590
1054 github small chain log advanced streaming 136341
1055 github explicit params advanced streaming 137727
1056 github uncompressed literals advanced streaming 165909 165911
1057 github uncompressed literals optimal advanced streaming 152667
1058 github huffman literals advanced streaming 142365
1059 github multithreaded with advanced params advanced streaming 165909 165911
1060 github.tar level -5 advanced streaming 52273 52152
1061 github.tar level -5 with dict advanced streaming 51297 51181
1062 github.tar level -3 advanced streaming 45783 45678
1063 github.tar level -3 with dict advanced streaming 44853 44734
1064 github.tar level -1 advanced streaming 42687 42560
1065 github.tar level -1 with dict advanced streaming 41486 41353
1066 github.tar level 0 advanced streaming 38884 38831
1067 github.tar level 0 with dict advanced streaming 37995
1068 github.tar level 0 with dict dms advanced streaming 38114 38003
1069 github.tar level 0 with dict dds advanced streaming 38114 38003
1070 github.tar level 0 with dict copy advanced streaming 37995
1071 github.tar level 0 with dict load advanced streaming 37956
1072 github.tar level 1 advanced streaming 39346 39200
1073 github.tar level 1 with dict advanced streaming 38251 38119
1074 github.tar level 1 with dict dms advanced streaming 38557 38406
1075 github.tar level 1 with dict dds advanced streaming 38557 38406
1076 github.tar level 1 with dict copy advanced streaming 38251 38119
1077 github.tar level 1 with dict load advanced streaming 38503 38364
1078 github.tar level 3 advanced streaming 38884 38831
1079 github.tar level 3 with dict advanced streaming 37995
1080 github.tar level 3 with dict dms advanced streaming 38114 38003
1081 github.tar level 3 with dict dds advanced streaming 38114 38003
1082 github.tar level 3 with dict copy advanced streaming 37995
1083 github.tar level 3 with dict load advanced streaming 37956
1084 github.tar level 4 advanced streaming 38880 38893
1085 github.tar level 4 with dict advanced streaming 37948
1086 github.tar level 4 with dict dms advanced streaming 37995 37954
1087 github.tar level 4 with dict dds advanced streaming 37995 37954
1088 github.tar level 4 with dict copy advanced streaming 37948
1089 github.tar level 4 with dict load advanced streaming 37927
1090 github.tar level 5 row 1 advanced streaming 39651
1169 github.tar level 19 with dict dds advanced streaming 32565
1170 github.tar level 19 with dict copy advanced streaming 32701
1171 github.tar level 19 with dict load advanced streaming 32428
1172 github.tar no source size advanced streaming 38881 38828
1173 github.tar no source size with dict advanced streaming 38111 38000
1174 github.tar long distance mode advanced streaming 40156 40252
1175 github.tar multithreaded advanced streaming 38884 38831
1176 github.tar multithreaded long distance mode advanced streaming 40139 40232
1177 github.tar small window log advanced streaming 199553 199558
1178 github.tar small hash log advanced streaming 129870
1179 github.tar small chain log advanced streaming 41669
1180 github.tar explicit params advanced streaming 41385
1181 github.tar uncompressed literals advanced streaming 41562 41525
1182 github.tar uncompressed literals optimal advanced streaming 35356
1183 github.tar huffman literals advanced streaming 38998 38853
1184 github.tar multithreaded with advanced params advanced streaming 41562 41525
1185 silesia level -5 old streaming 6853462 6854744
1186 silesia level -3 old streaming 6502349 6503319
1187 silesia level -1 old streaming 6172125 6172207
1188 silesia level 0 old streaming 4835804 4842075
1189 silesia level 1 old streaming 5301644 5306388
1190 silesia level 3 old streaming 4835804 4842075
1191 silesia level 4 old streaming 4773049 4779186
1192 silesia level 5 old streaming 4664679 4667668
1193 silesia level 6 old streaming 4601116 4604351
1194 silesia level 7 old streaming 4567082 4570271
1195 silesia level 9 old streaming 4542474 4545850
1196 silesia level 13 old streaming 4490650 4493990
1197 silesia level 16 old streaming 4358094 4359652
1198 silesia level 19 old streaming 4265908 4266582
1199 silesia no source size old streaming 4835768 4842039
1200 silesia uncompressed literals old streaming 4835804 4842075
1201 silesia uncompressed literals optimal old streaming 4265908 4266582
1202 silesia huffman literals old streaming 6172125 6172207
1203 silesia.tar level -5 old streaming 6853184 6856523
1204 silesia.tar level -3 old streaming 6503455 6505954
1205 silesia.tar level -1 old streaming 6175761 6179056
1206 silesia.tar level 0 old streaming 4846783 4859271
1207 silesia.tar level 1 old streaming 5306719 5327708
1208 silesia.tar level 3 old streaming 4846783 4859271
1209 silesia.tar level 4 old streaming 4785332 4797470
1210 silesia.tar level 5 old streaming 4664523 4679020
1211 silesia.tar level 6 old streaming 4599420 4614558
1212 silesia.tar level 7 old streaming 4565332 4579823
1213 silesia.tar level 9 old streaming 4539391 4555445
1214 silesia.tar level 13 old streaming 4486652 4502956
1215 silesia.tar level 16 old streaming 4358029 4360385
1216 silesia.tar level 19 old streaming 4258228 4260939
1217 silesia.tar no source size old streaming 4846779 4859267
1218 silesia.tar uncompressed literals old streaming 4846783 4859271
1219 silesia.tar uncompressed literals optimal old streaming 4258228 4260939
1220 silesia.tar huffman literals old streaming 6175761 6179056
1221 github level -5 old streaming 204407
1222 github level -5 with dict old streaming 45832
1223 github level -3 old streaming 193253
1224 github level -3 with dict old streaming 44671
1225 github level -1 old streaming 175468
1226 github level -1 with dict old streaming 41825
1227 github level 0 old streaming 136331 136332
1228 github level 0 with dict old streaming 41118 41148
1229 github level 1 old streaming 142365
1230 github level 1 with dict old streaming 41266
1231 github level 3 old streaming 136331 136332
1232 github level 3 with dict old streaming 41118 41148
1233 github level 4 old streaming 136199
1234 github level 4 with dict old streaming 41229 41251
1235 github level 5 old streaming 135121
1236 github level 5 with dict old streaming 38754
1237 github level 6 old streaming 135122
1247 github level 19 old streaming 132879
1248 github level 19 with dict old streaming 37916
1249 github no source size old streaming 140599
1250 github no source size with dict old streaming 40652 40654
1251 github uncompressed literals old streaming 136331 136332
1252 github uncompressed literals optimal old streaming 132879
1253 github huffman literals old streaming 175468
1254 github.tar level -5 old streaming 52273 52152
1255 github.tar level -5 with dict old streaming 51297 51181
1256 github.tar level -3 old streaming 45783 45678
1257 github.tar level -3 with dict old streaming 44853 44734
1258 github.tar level -1 old streaming 42687 42560
1259 github.tar level -1 with dict old streaming 41486 41353
1260 github.tar level 0 old streaming 38884 38831
1261 github.tar level 0 with dict old streaming 37995
1262 github.tar level 1 old streaming 39346 39200
1263 github.tar level 1 with dict old streaming 38251 38119
1264 github.tar level 3 old streaming 38884 38831
1265 github.tar level 3 with dict old streaming 37995
1266 github.tar level 4 old streaming 38880 38893
1267 github.tar level 4 with dict old streaming 37948
1268 github.tar level 5 old streaming 39651
1269 github.tar level 5 with dict old streaming 39145
1279 github.tar level 16 with dict old streaming 33375
1280 github.tar level 19 old streaming 32262
1281 github.tar level 19 with dict old streaming 32701
1282 github.tar no source size old streaming 38881 38828
1283 github.tar no source size with dict old streaming 38111 38000
1284 github.tar uncompressed literals old streaming 38884 38831
1285 github.tar uncompressed literals optimal old streaming 32262
1286 github.tar huffman literals old streaming 42687 42560
1287 silesia level -5 old streaming advanced 6853462 6854744
1288 silesia level -3 old streaming advanced 6502349 6503319
1289 silesia level -1 old streaming advanced 6172125 6172207
1290 silesia level 0 old streaming advanced 4835804 4842075
1291 silesia level 1 old streaming advanced 5301644 5306388
1292 silesia level 3 old streaming advanced 4835804 4842075
1293 silesia level 4 old streaming advanced 4773049 4779186
1294 silesia level 5 old streaming advanced 4664679 4667668
1295 silesia level 6 old streaming advanced 4601116 4604351
1296 silesia level 7 old streaming advanced 4567082 4570271
1297 silesia level 9 old streaming advanced 4542474 4545850
1298 silesia level 13 old streaming advanced 4490650 4493990
1299 silesia level 16 old streaming advanced 4358094 4359652
1300 silesia level 19 old streaming advanced 4265908 4266582
1301 silesia no source size old streaming advanced 4835768 4842039
1302 silesia long distance mode old streaming advanced 4835804 4842075
1303 silesia multithreaded old streaming advanced 4835804 4842075
1304 silesia multithreaded long distance mode old streaming advanced 4835804 4842075
1305 silesia small window log old streaming advanced 7110591 7111103
1306 silesia small hash log old streaming advanced 6525259 6526141
1307 silesia small chain log old streaming advanced 4911577 4912197
1308 silesia explicit params old streaming advanced 4792505 4795857
1309 silesia uncompressed literals old streaming advanced 4835804 4842075
1310 silesia uncompressed literals optimal old streaming advanced 4265908 4266582
1311 silesia huffman literals old streaming advanced 6172125 6172207
1312 silesia multithreaded with advanced params old streaming advanced 4835804 4842075
1313 silesia.tar level -5 old streaming advanced 6853184 6856523
1314 silesia.tar level -3 old streaming advanced 6503455 6505954
1315 silesia.tar level -1 old streaming advanced 6175761 6179056
1316 silesia.tar level 0 old streaming advanced 4846783 4859271
1317 silesia.tar level 1 old streaming advanced 5306719 5327708
1318 silesia.tar level 3 old streaming advanced 4846783 4859271
1319 silesia.tar level 4 old streaming advanced 4785332 4797470
1320 silesia.tar level 5 old streaming advanced 4664523 4679020
1321 silesia.tar level 6 old streaming advanced 4599420 4614558
1322 silesia.tar level 7 old streaming advanced 4565332 4579823
1323 silesia.tar level 9 old streaming advanced 4539391 4555445
1324 silesia.tar level 13 old streaming advanced 4486652 4502956
1325 silesia.tar level 16 old streaming advanced 4358029 4360385
1326 silesia.tar level 19 old streaming advanced 4258228 4260939
1327 silesia.tar no source size old streaming advanced 4846779 4859267
1328 silesia.tar long distance mode old streaming advanced 4846783 4859271
1329 silesia.tar multithreaded old streaming advanced 4846783 4859271
1330 silesia.tar multithreaded long distance mode old streaming advanced 4846783 4859271
1331 silesia.tar small window log old streaming advanced 7117027 7117562
1332 silesia.tar small hash log old streaming advanced 6529503 6529209
1333 silesia.tar small chain log old streaming advanced 4915956 4917021
1334 silesia.tar explicit params old streaming advanced 4791739 4807288
1335 silesia.tar uncompressed literals old streaming advanced 4846783 4859271
1336 silesia.tar uncompressed literals optimal old streaming advanced 4258228 4260939
1337 silesia.tar huffman literals old streaming advanced 6175761 6179056
1338 silesia.tar multithreaded with advanced params old streaming advanced 4846783 4859271
1339 github level -5 old streaming advanced 213265
1340 github level -5 with dict old streaming advanced 46708
1341 github level -3 old streaming advanced 196126
1342 github level -3 with dict old streaming advanced 45476
1343 github level -1 old streaming advanced 181107
1344 github level -1 with dict old streaming advanced 42060
1345 github level 0 old streaming advanced 141101 141104
1346 github level 0 with dict old streaming advanced 41074 41113
1347 github level 1 old streaming advanced 143693
1348 github level 1 with dict old streaming advanced 42430
1349 github level 3 old streaming advanced 141101 141104
1350 github level 3 with dict old streaming advanced 41074 41113
1351 github level 4 old streaming advanced 141101 141104
1352 github level 4 with dict old streaming advanced 41046 41084
1353 github level 5 old streaming advanced 139402
1354 github level 5 with dict old streaming advanced 38723
1355 github level 6 old streaming advanced 138676
1366 github level 19 with dict old streaming advanced 37916
1367 github no source size old streaming advanced 140599
1368 github no source size with dict old streaming advanced 40608
1369 github long distance mode old streaming advanced 141101 141104
1370 github multithreaded old streaming advanced 141101 141104
1371 github multithreaded long distance mode old streaming advanced 141101 141104
1372 github small window log old streaming advanced 141101 141104
1373 github small hash log old streaming advanced 141597
1374 github small chain log old streaming advanced 139275
1375 github explicit params old streaming advanced 140937
1376 github uncompressed literals old streaming advanced 141101 141104
1377 github uncompressed literals optimal old streaming advanced 132879
1378 github huffman literals old streaming advanced 181107
1379 github multithreaded with advanced params old streaming advanced 141101 141104
1380 github.tar level -5 old streaming advanced 52273 52152
1381 github.tar level -5 with dict old streaming advanced 51249 51129
1382 github.tar level -3 old streaming advanced 45783 45678
1383 github.tar level -3 with dict old streaming advanced 45093 44986
1384 github.tar level -1 old streaming advanced 42687 42560
1385 github.tar level -1 with dict old streaming advanced 41762 41650
1386 github.tar level 0 old streaming advanced 38884 38831
1387 github.tar level 0 with dict old streaming advanced 38013
1388 github.tar level 1 old streaming advanced 39346 39200
1389 github.tar level 1 with dict old streaming advanced 38507 38359
1390 github.tar level 3 old streaming advanced 38884 38831
1391 github.tar level 3 with dict old streaming advanced 38013
1392 github.tar level 4 old streaming advanced 38880 38893
1393 github.tar level 4 with dict old streaming advanced 38063
1394 github.tar level 5 old streaming advanced 39651
1395 github.tar level 5 with dict old streaming advanced 39018
1405 github.tar level 16 with dict old streaming advanced 38578
1406 github.tar level 19 old streaming advanced 32262
1407 github.tar level 19 with dict old streaming advanced 32678
1408 github.tar no source size old streaming advanced 38881 38828
1409 github.tar no source size with dict old streaming advanced 38076 38015
1410 github.tar long distance mode old streaming advanced 38884 38831
1411 github.tar multithreaded old streaming advanced 38884 38831
1412 github.tar multithreaded long distance mode old streaming advanced 38884 38831
1413 github.tar small window log old streaming advanced 199556 199561
1414 github.tar small hash log old streaming advanced 129870
1415 github.tar small chain log old streaming advanced 41669
1416 github.tar explicit params old streaming advanced 41385
1417 github.tar uncompressed literals old streaming advanced 38884 38831
1418 github.tar uncompressed literals optimal old streaming advanced 32262
1419 github.tar huffman literals old streaming advanced 42687 42560
1420 github.tar multithreaded with advanced params old streaming advanced 38884 38831
1421 github level -5 with dict old streaming cdict 45832
1422 github level -3 with dict old streaming cdict 44671
1423 github level -1 with dict old streaming cdict 41825
1424 github level 0 with dict old streaming cdict 41118 41148
1425 github level 1 with dict old streaming cdict 41266
1426 github level 3 with dict old streaming cdict 41118 41148
1427 github level 4 with dict old streaming cdict 41229 41251
1428 github level 5 with dict old streaming cdict 38754
1429 github level 6 with dict old streaming cdict 38669
1430 github level 7 with dict old streaming cdict 38765
1432 github level 13 with dict old streaming cdict 39900
1433 github level 16 with dict old streaming cdict 37902
1434 github level 19 with dict old streaming cdict 37916
1435 github no source size with dict old streaming cdict 40652 40654
1436 github.tar level -5 with dict old streaming cdict 51407 51286
1437 github.tar level -3 with dict old streaming cdict 45254 45147
1438 github.tar level -1 with dict old streaming cdict 41973 41865
1439 github.tar level 0 with dict old streaming cdict 37956
1440 github.tar level 1 with dict old streaming cdict 38503 38364
1441 github.tar level 3 with dict old streaming cdict 37956
1442 github.tar level 4 with dict old streaming cdict 37927
1443 github.tar level 5 with dict old streaming cdict 39000
1447 github.tar level 13 with dict old streaming cdict 36010
1448 github.tar level 16 with dict old streaming cdict 39081
1449 github.tar level 19 with dict old streaming cdict 32428
1450 github.tar no source size with dict old streaming cdict 38111 38000
1451 github level -5 with dict old streaming advanced cdict 46708
1452 github level -3 with dict old streaming advanced cdict 45476
1453 github level -1 with dict old streaming advanced cdict 42060
1454 github level 0 with dict old streaming advanced cdict 41074 41113
1455 github level 1 with dict old streaming advanced cdict 42430
1456 github level 3 with dict old streaming advanced cdict 41074 41113
1457 github level 4 with dict old streaming advanced cdict 41046 41084
1458 github level 5 with dict old streaming advanced cdict 38723
1459 github level 6 with dict old streaming advanced cdict 38744
1460 github level 7 with dict old streaming advanced cdict 38875
1463 github level 16 with dict old streaming advanced cdict 40804
1464 github level 19 with dict old streaming advanced cdict 37916
1465 github no source size with dict old streaming advanced cdict 40608
1466 github.tar level -5 with dict old streaming advanced cdict 50907 50791
1467 github.tar level -3 with dict old streaming advanced cdict 45032 44926
1468 github.tar level -1 with dict old streaming advanced cdict 41589 41482
1469 github.tar level 0 with dict old streaming advanced cdict 38013
1470 github.tar level 1 with dict old streaming advanced cdict 38294 38168
1471 github.tar level 3 with dict old streaming advanced cdict 38013
1472 github.tar level 4 with dict old streaming advanced cdict 38063
1473 github.tar level 5 with dict old streaming advanced cdict 39018
1477 github.tar level 13 with dict old streaming advanced cdict 35807
1478 github.tar level 16 with dict old streaming advanced cdict 38578
1479 github.tar level 19 with dict old streaming advanced cdict 32678
1480 github.tar no source size with dict old streaming advanced cdict 38076 38015
+2 -12
View File
@@ -259,23 +259,13 @@ if __name__ == '__main__':
shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag))
os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest/<TAG>/programs
make(['clean'], False) # separate 'clean' target to allow parallel build
# Enable legacy support for cross-version compatibility testing.
# Use ZSTD_LEGACY_SUPPORT=1 for v0.6.x due to a bug where headers
# check for ==1 but code checks for >=1.
# Use ZSTD_LEGACY_SUPPORT=5 for v1.2.0+ because =1 includes old
# legacy files (v01-v04) that have missing includes in newer versions.
if tag < 'v1.2.0':
make(['zstd', 'ZSTD_LEGACY_SUPPORT=1'], False)
else:
make(['zstd', 'ZSTD_LEGACY_SUPPORT=5'], False)
make(['zstd'], False)
else:
os.chdir(programs_dir)
print('-----------------------------------------------')
print('compiling head')
print('-----------------------------------------------')
# Enable legacy support for head to test cross-version compatibility
# (legacy support is disabled by default since v1.6.0)
make(['zstd', 'ZSTD_LEGACY_SUPPORT=5'], False)
make(['zstd'], False)
shutil.copy2('zstd', dst_zstd)
# remove any remaining *.zst and *.dec from previous test
-92
View File
@@ -1,92 +0,0 @@
#!/usr/bin/env bash
# test_process_substitution.bash
# Test zstd's support for process substitution with --filelist
# Process arguments
ZSTD_PATH="zstd" # Default to using zstd from PATH
if [ $# -ge 1 ]; then
ZSTD_PATH="$1"
fi
echo "Using zstd executable: $ZSTD_PATH"
set -e # Exit on error
# Set up test directory and files
echo "Setting up test environment..."
TEST_DIR="tmp_process_substit"
rm -rf "$TEST_DIR"
mkdir -p "$TEST_DIR"
echo "Content of file 1" > "$TEST_DIR/file1.txt"
echo "Content of file 2" > "$TEST_DIR/file2.txt"
echo "Content of file 3" > "$TEST_DIR/file3.txt"
# Clean up any previous test artifacts
rm -f "$TEST_DIR/output.zst" "$TEST_DIR/output_echo.zst" "$TEST_DIR/output_cat.zst"
rm -rf "$TEST_DIR/extracted"
mkdir -p "$TEST_DIR/extracted"
echo "=== Testing process substitution with --filelist ==="
# Test 1: Basic process substitution with find
echo "Test 1: Basic process substitution (find command)"
"$ZSTD_PATH" --filelist=<(find "$TEST_DIR" -name "*.txt" | sort) -c > "$TEST_DIR/output.zst"
if [ -f "$TEST_DIR/output.zst" ]; then
echo "✓ Test 1 PASSED: Output file was created"
else
echo "✗ Test 1 FAILED: Output file was not created"
exit 1
fi
# Test 2: Process substitution with echo
echo "Test 2: Process substitution (echo command)"
"$ZSTD_PATH" --filelist=<(echo -e "$TEST_DIR/file1.txt\n$TEST_DIR/file2.txt") -c > "$TEST_DIR/output_echo.zst"
if [ -f "$TEST_DIR/output_echo.zst" ]; then
echo "✓ Test 2 PASSED: Output file was created"
else
echo "✗ Test 2 FAILED: Output file was not created"
exit 1
fi
# Test 3: Process substitution with cat
echo "Test 3: Process substitution (cat command)"
echo -e "$TEST_DIR/file1.txt\n$TEST_DIR/file3.txt" > "$TEST_DIR/filelist.txt"
"$ZSTD_PATH" --filelist=<(cat "$TEST_DIR/filelist.txt") -c > "$TEST_DIR/output_cat.zst"
if [ -f "$TEST_DIR/output_cat.zst" ]; then
echo "✓ Test 3 PASSED: Output file was created"
else
echo "✗ Test 3 FAILED: Output file was not created"
exit 1
fi
# Test 4: Verify contents of archives
echo "Test 4: Verifying archive contents"
"$ZSTD_PATH" -d "$TEST_DIR/output.zst" -o "$TEST_DIR/extracted/combined.out"
if grep -q "Content of file 1" "$TEST_DIR/extracted/combined.out" &&
grep -q "Content of file 2" "$TEST_DIR/extracted/combined.out" &&
grep -q "Content of file 3" "$TEST_DIR/extracted/combined.out"; then
echo "✓ Test 4 PASSED: All files were correctly archived and extracted"
else
echo "✗ Test 4 FAILED: Not all expected content was found in the extracted file"
exit 1
fi
# Test 5: Edge case with empty list
echo "Test 5: Process substitution with empty input"
"$ZSTD_PATH" --filelist=<(echo "") -c > "$TEST_DIR/output_empty.zst" 2>/dev/null || true
if [ -f "$TEST_DIR/output_empty.zst" ]; then
echo "✓ Test 5 PASSED: Handled empty input gracefully"
else
echo "✓ Test 5 PASSED: Properly rejected empty input"
fi
# cleanup
rm -rf "$TEST_DIR"
echo "All tests completed successfully!"
+31 -91
View File
@@ -436,12 +436,12 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
/* context size functions */
DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
{ ZSTD_FrameHeader fhi;
{ ZSTD_frameHeader fhi;
const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
if (gfhError!=0) goto _output_error;
DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize);
{ size_t const s = ZSTD_estimateDStreamSize((size_t)fhi.windowSize)
{ size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
/* uses ZSTD_initDStream_usingDict() */
+ ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
if (ZSTD_isError(s)) goto _output_error;
@@ -537,7 +537,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
{ size_t const r = ZSTD_endStream(zc, &outBuff);
CHECK(r != 0, "Error or some data not flushed (ret=%i)", ZSTD_getErrorCode(r));
CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
}
inBuff.src = outBuff.dst;
inBuff.size = outBuff.pos;
@@ -567,7 +567,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
outBuff.size = compressedBufferSize;
outBuff.pos = 0;
{ size_t const r = ZSTD_endStream(zc, &outBuff);
CHECK(r != 0, "Error or some data not flushed (ret=%i)", ZSTD_getErrorCode(r));
CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
}
inBuff.src = outBuff.dst;
inBuff.size = outBuff.pos;
@@ -595,7 +595,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
{ size_t const r = ZSTD_endStream(zc, &outBuff);
CHECK(r != 0, "Error or some data not flushed (ret=%i)", ZSTD_getErrorCode(r));
CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
}
{ unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
@@ -777,7 +777,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
CHECK(streamingSize < singlePassSize + (1 << 18) + 3 * ZSTD_BLOCKSIZE_MAX, "Streaming doesn't use the right amount of memory");
CHECK(streamingSize != streaming2KSize + 3 * (ZSTD_BLOCKSIZE_MAX - 2048), "ZSTD_d_blockSizeMax didn't save the right amount of memory");
DISPLAYLEVEL(3, "| %u | %u | %u | ", (unsigned)singlePassSize, (unsigned)streaming2KSize, (unsigned)streamingSize);
DISPLAYLEVEL(3, "| %zu | %zu | %zu | ", singlePassSize, streaming2KSize, streamingSize);
ZSTD_freeDCtx(dctx);
}
@@ -884,8 +884,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() uses stable input and output : ", testNb++);
CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test");
/* check that compression fits with just a 8-bytes margin */
CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize+8, CNBuffer, CNBufferSize));
CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize));
CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx));
/* @cctxSize2 : sizeof_CCtx when doing full streaming (no stable in/out) */
{ ZSTD_CCtx* const cctx2 = ZSTD_createCCtx();
@@ -958,7 +957,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
break;
out.size = MIN(out.size + cSize / 4, compressedBufferSize);
}
CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, out.pos));
CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer modify buffer : ", testNb++);
@@ -999,7 +998,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
CHECK_Z(decSize);
CHECK(decSize != inputSize, "regenerated %u bytes, instead of %u", (unsigned)decSize, (unsigned)inputSize);
CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
} }
DISPLAYLEVEL(3, "OK \n");
@@ -1029,14 +1028,14 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
CHECK_Z(decSize);
CHECK(decSize != inputSize, "regenerated %u bytes, instead of %u", (unsigned)decSize, (unsigned)inputSize);
CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
} }
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer: context size : ", testNb++);
{ size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
DISPLAYLEVEL(4, "cctxSize1=%u; cctxSize=%u; cctxSize2=%u : ", (unsigned)cctxSize1, (unsigned)cctxSize, (unsigned)cctxSize2);
DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass");
CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
cctxSize1 = cctxSize;
@@ -1071,7 +1070,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer: context size : ", testNb++);
{ size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
DISPLAYLEVEL(4, "cctxSize1=%u; cctxSize=%u; cctxSize2=%u : ", (unsigned)cctxSize1, (unsigned)cctxSize, (unsigned)cctxSize2);
DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer");
CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
}
@@ -1505,7 +1504,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (unsigned)srcSize);
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, (int)jobSize) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) );
assert(start > offset);
assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
@@ -1530,7 +1529,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
{ ZSTD_DStream* const dstream = ZSTD_createDCtx();
ZSTD_FrameHeader zfh;
ZSTD_frameHeader zfh;
ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
DISPLAYLEVEL(5, "frame windowsize = %u : ", (unsigned)zfh.windowSize);
outBuff.dst = decodedBuffer;
@@ -1551,7 +1550,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
{ unsigned const kMaxWindowLog = 24;
unsigned value;
ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1ULL << kMaxWindowLog, 1024);
ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
ZSTD_CDict* cdict;
ZSTD_DDict* ddict;
SEQ_stream seq = SEQ_initStream(0x87654321);
@@ -1654,7 +1653,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
int windowLog;
int const kMaxWindowLog = bigTests ? 29 : 26;
size_t const kNbSequences = 10000;
size_t const kMaxSrcSize = ((size_t)1 << kMaxWindowLog) + 10 * kNbSequences;
size_t const kMaxSrcSize = (1u << kMaxWindowLog) + 10 * kNbSequences;
char* src = calloc(kMaxSrcSize, 1);
ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
for (windowLog = ZSTD_WINDOWLOG_MIN; windowLog <= kMaxWindowLog; ++windowLog) {
@@ -1896,13 +1895,13 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
/* Write a bunch of 6 byte blocks */
while (remainingInput > 0) {
char testBuffer[6] = { 0x66, 0x66, 0x66, 0x66, 0x66, 0x66 };
char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
const size_t kSmallBlockSize = sizeof(testBuffer);
ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
CHECK(in.pos != in.size, "input not fully consumed");
remainingInput -= (int)kSmallBlockSize;
remainingInput -= kSmallBlockSize;
}
/* Write several very long offset matches into the dictionary */
for (offset = 1024; offset >= 0; offset -= 128) {
@@ -2357,7 +2356,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Testing external sequence producer with static CCtx (one-shot): ", testNb++);
DISPLAYLEVEL(3, "test%3i : Testing external sequence producer with static CCtx: ", testNb++);
{
size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
BYTE* const dstBuf = (BYTE*)malloc(dstBufSize);
@@ -2376,7 +2375,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
size_t const cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
cctxBuf = malloc(cctxSize);
staticCCtx = ZSTD_initStaticCCtx(cctxBuf, cctxSize);
CHECK_Z(ZSTD_CCtx_setParametersUsingCCtxParams(staticCCtx, params));
ZSTD_CCtx_setParametersUsingCCtxParams(staticCCtx, params);
}
// Check that compression with external sequence producer succeeds when expected
@@ -2409,65 +2408,6 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Testing external sequence producer with static CCtx (streaming): ", testNb++);
{
size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
BYTE* const dstBuf = (BYTE*)malloc(dstBufSize);
size_t const checkBufSize = CNBufferSize;
BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
ZSTD_CCtx* staticCCtx;
void* cctxBuf;
EMF_testCase seqProdState;
CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_validateSequences, 1));
CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableSeqProducerFallback, 0));
ZSTD_CCtxParams_registerSequenceProducer(params, &seqProdState, zstreamSequenceProducer);
{
size_t const cctxSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
cctxBuf = malloc(cctxSize);
staticCCtx = ZSTD_initStaticCCtx(cctxBuf, cctxSize);
CHECK_Z(ZSTD_CCtx_setParametersUsingCCtxParams(staticCCtx, params));
}
// Check that compression with external sequence producer succeeds when expected
seqProdState = EMF_LOTS_OF_SEQS;
{
ZSTD_inBuffer inBuf = { CNBuffer, CNBufferSize, 0 };
ZSTD_outBuffer outBuf = { dstBuf, dstBufSize, 0 };
size_t dResult;
CHECK_Z(ZSTD_compressStream(staticCCtx, &outBuf, &inBuf));
CHECK_Z(ZSTD_endStream(staticCCtx, &outBuf));
CHECK(inBuf.pos != inBuf.size, "EMF: inBuf.pos != inBuf.size");
dResult = ZSTD_decompress(checkBuf, checkBufSize, outBuf.dst, outBuf.pos);
CHECK(ZSTD_isError(dResult), "EMF: Decompression error: %s", ZSTD_getErrorName(dResult));
CHECK(dResult != CNBufferSize, "EMF: Corruption!");
CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
}
CHECK_Z(ZSTD_CCtx_reset(staticCCtx, ZSTD_reset_session_only));
// Check that compression with external sequence producer fails when expected
seqProdState = EMF_BIG_ERROR;
{
ZSTD_inBuffer inBuf = { CNBuffer, CNBufferSize, 0 };
ZSTD_outBuffer outBuf = { dstBuf, dstBufSize, 0 };
size_t const cResult = ZSTD_compressStream(staticCCtx, &outBuf, &inBuf);
CHECK(!ZSTD_isError(cResult), "EMF: Should have raised an error!");
CHECK(
ZSTD_getErrorCode(cResult) != ZSTD_error_sequenceProducer_failed,
"EMF: Wrong error code: %s", ZSTD_getErrorName(cResult)
);
}
free(dstBuf);
free(checkBuf);
free(cctxBuf);
ZSTD_freeCCtxParams(params);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : Decoder should reject invalid frame header on legacy frames: ", testNb++);
{
const unsigned char compressed[] = { 0x26,0xb5,0x2f,0xfd,0x50,0x91,0xfd,0xd8,0xb5 };
@@ -2479,7 +2419,7 @@ static int basicUnitTests(U32 seed, double compressibility, int bigTests)
DISPLAYLEVEL(3, "test%3i : Test single-shot fallback for magicless mode: ", testNb++);
{
// Acquire resources
// Aquire resources
size_t const srcSize = COMPRESSIBLE_NOISE_LENGTH;
void* src = malloc(srcSize);
size_t const dstSize = ZSTD_compressBound(srcSize);
@@ -2582,13 +2522,13 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co
static const U32 maxSampleLog = 19;
size_t const srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* cNoiseBuffer[5];
size_t const copyBufferSize = srcBufferSize + ((size_t)1 << maxSampleLog);
size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog);
BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
size_t const dstBufferSize = srcBufferSize;
BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
int result = 0;
U32 result = 0;
unsigned testNb = 0;
U32 coreSeed = seed;
ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */
@@ -2685,7 +2625,7 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevelCandidate = ( FUZ_rand(&lseed) %
((unsigned)ZSTD_maxCLevel() -
(ZSTD_maxCLevel() -
(MAX(testLog, dictLog) / 3)))
+ 1;
U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
@@ -2698,7 +2638,7 @@ static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double co
}
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, (int)cLevel) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, cLevel) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, FUZ_rand(&lseed) & 1) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1) );
CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1) );
@@ -2837,9 +2777,9 @@ static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
int useOpaqueAPI)
{
if (useOpaqueAPI) {
return ZSTD_CCtxParams_setParameter(cctxParams, param, (int)value);
return ZSTD_CCtxParams_setParameter(cctxParams, param, value);
} else {
return ZSTD_CCtx_setParameter(zc, param, (int)value);
return ZSTD_CCtx_setParameter(zc, param, value);
}
}
@@ -2851,7 +2791,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
static const U32 maxSampleLog = 19;
size_t const srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* cNoiseBuffer[5];
size_t const copyBufferSize= srcBufferSize + ((size_t)1 << maxSampleLog);
size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
@@ -2983,7 +2923,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
const U32 windowLogMax = bigTests ? 24 : 20;
const U32 searchLogMax = bigTests ? 15 : 13;
if (dictSize)
DISPLAYLEVEL(5, "t%u: with dictionary of size : %u \n", testNb, (unsigned)dictSize);
DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize);
/* mess with compression parameters */
cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
@@ -3178,7 +3118,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
}
cSize = outBuff.pos;
compressedCrcs[iter] = XXH64(cBuffer, cSize, 0);
DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (unsigned)cSize);
DISPLAYLEVEL(5, "Frame completed : %zu bytes \n", cSize);
}
CHECK(!(compressedCrcs[0] == compressedCrcs[1]), "Compression is not deterministic!");
}
@@ -3194,7 +3134,7 @@ static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
CHECK_Z( ZSTD_resetDStream(zd) );
} else {
if (dictSize)
DISPLAYLEVEL(5, "using dictionary of size %u \n", (unsigned)dictSize);
DISPLAYLEVEL(5, "using dictionary of size %zu \n", dictSize);
CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
}
if (FUZ_rand(&lseed) & 1) {
+2 -2
View File
@@ -442,8 +442,8 @@ void test_sync(Byte *compr, uLong comprLen, Byte *uncompr, uLong uncomprLen) {
CHECK_ERR(err, "inflateSync");
err = inflate(&d_stream, Z_FINISH);
if (err != Z_STREAM_END) {
fprintf(stderr, "inflate reported %i != %i (Z_STREAM_END)\n", err, Z_STREAM_END);
if (err != Z_DATA_ERROR) {
fprintf(stderr, "inflate should report DATA_ERROR\n");
/* Because of incorrect adler32 */
exit(1);
}
+2 -2
View File
@@ -234,7 +234,7 @@ int gzwrite _Z_OF((gzFile, const void *, unsigned));
int gzwrite(gzFile gz, const void *buf, unsigned len) {
z_stream *strm;
unsigned char out[BUFLEN] = { 0 };
unsigned char out[BUFLEN];
if (gz == NULL || !gz->write)
return 0;
@@ -287,7 +287,7 @@ int gzclose _Z_OF((gzFile));
int gzclose(gzFile gz) {
z_stream *strm;
unsigned char out[BUFLEN] = { 0 };
unsigned char out[BUFLEN];
if (gz == NULL)
return Z_STREAM_ERROR;
+1 -2
View File
@@ -64,7 +64,6 @@ local int gz_init(gz_statep state) {
strm->next_out = state.state->out;
state.state->x.next = strm->next_out;
}
return 0;
}
@@ -224,7 +223,7 @@ local z_size_t gz_write(gz_statep state, voidpc buf, z_size_t len) {
z_size_t n = (unsigned)-1;
if (n > len)
n = len;
state.state->strm.avail_in = (uInt)n;
state.state->strm.avail_in = (z_uInt)n;
state.state->x.pos += n;
if (gz_comp(state, Z_NO_FLUSH) == -1)
return 0;
+5 -5
View File
@@ -11,6 +11,11 @@
#ifndef ZSTD_ZLIBWRAPPER_H
#define ZSTD_ZLIBWRAPPER_H
#if defined (__cplusplus)
extern "C" {
#endif
#define ZLIB_CONST
#define Z_PREFIX
#define ZLIB_INTERNAL /* disables gz*64 functions but fixes zlib 1.2.4 with Z_PREFIX */
@@ -24,11 +29,6 @@
#define _Z_OF OF
#endif
#if defined (__cplusplus)
extern "C" {
#endif
/* returns a string with version of zstd library */
const char * zstdVersion(void);