Merge branch 'dev' of github.com:facebook/zstd into dev
This commit is contained in:
Jennifer Liu
@@ -0,0 +1,122 @@
|
|||||||
|
# This configuration was automatically generated from a CircleCI 1.0 config.
|
||||||
|
# It should include any build commands you had along with commands that CircleCI
|
||||||
|
# inferred from your project structure. We strongly recommend you read all the
|
||||||
|
# comments in this file to understand the structure of CircleCI 2.0, as the idiom
|
||||||
|
# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather
|
||||||
|
# than the prescribed lifecycle of 1.0. In general, we recommend using this generated
|
||||||
|
# configuration as a reference rather than using it in production, though in most
|
||||||
|
# cases it should duplicate the execution of your original 1.0 config.
|
||||||
|
version: 2
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
working_directory: ~/facebook/zstd
|
||||||
|
parallelism: 1
|
||||||
|
shell: /bin/bash --login
|
||||||
|
# CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
|
||||||
|
# If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables .
|
||||||
|
environment:
|
||||||
|
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
|
||||||
|
CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
|
||||||
|
# In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages.
|
||||||
|
# In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images.
|
||||||
|
# The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job.
|
||||||
|
# We have selected a pre-built image that mirrors the build environment we use on
|
||||||
|
# the 1.0 platform, but we recommend you choose an image more tailored to the needs
|
||||||
|
# of each job. For more information on choosing an image (or alternatively using a
|
||||||
|
# VM instead of a container) see https://circleci.com/docs/2.0/executor-types/
|
||||||
|
# To see the list of pre-built images that CircleCI provides for most common languages see
|
||||||
|
# https://circleci.com/docs/2.0/circleci-images/
|
||||||
|
docker:
|
||||||
|
- image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37
|
||||||
|
command: /sbin/init
|
||||||
|
steps:
|
||||||
|
# Machine Setup
|
||||||
|
# If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
|
||||||
|
# The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out.
|
||||||
|
- checkout
|
||||||
|
# Prepare for artifact and test results collection equivalent to how it was done on 1.0.
|
||||||
|
# In many cases you can simplify this from what is generated here.
|
||||||
|
# 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
|
||||||
|
- run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
|
||||||
|
# Dependencies
|
||||||
|
# This would typically go in either a build or a build-and-test job when using workflows
|
||||||
|
# Restore the dependency cache
|
||||||
|
- restore_cache:
|
||||||
|
keys:
|
||||||
|
# This branch if available
|
||||||
|
- v1-dep-{{ .Branch }}-
|
||||||
|
# Default branch if not
|
||||||
|
- v1-dep-dev-
|
||||||
|
# Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
|
||||||
|
- v1-dep-
|
||||||
|
# This is based on your 1.0 configuration file or project settings
|
||||||
|
- run: sudo dpkg --add-architecture i386
|
||||||
|
- run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
|
||||||
|
- run: sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
|
||||||
|
# Save dependency cache
|
||||||
|
- save_cache:
|
||||||
|
key: v1-dep-{{ .Branch }}-{{ epoch }}
|
||||||
|
paths:
|
||||||
|
# This is a broad list of cache paths to include many possible development environments
|
||||||
|
# You can probably delete some of these entries
|
||||||
|
- vendor/bundle
|
||||||
|
- ~/virtualenvs
|
||||||
|
- ~/.m2
|
||||||
|
- ~/.ivy2
|
||||||
|
- ~/.bundle
|
||||||
|
- ~/.go_workspace
|
||||||
|
- ~/.gradle
|
||||||
|
- ~/.cache/bower
|
||||||
|
# Test
|
||||||
|
# This would typically be a build job when using workflows, possibly combined with build
|
||||||
|
# This is based on your 1.0 configuration file or project settings
|
||||||
|
- run: |
|
||||||
|
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; CFLAGS="-O0 -Werror" make all && make clean; fi &&
|
||||||
|
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi
|
||||||
|
- run: |
|
||||||
|
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c99build && make clean; fi &&
|
||||||
|
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build && make clean; fi
|
||||||
|
- run: |
|
||||||
|
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi &&
|
||||||
|
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build && make clean; fi
|
||||||
|
- run: |
|
||||||
|
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make aarch64build && make clean; fi &&
|
||||||
|
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi
|
||||||
|
- run: |
|
||||||
|
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi &&
|
||||||
|
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi
|
||||||
|
- run: |
|
||||||
|
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make shortest && make clean; fi &&
|
||||||
|
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
|
||||||
|
- run: |
|
||||||
|
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi &&
|
||||||
|
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi
|
||||||
|
# This is based on your 1.0 configuration file or project settings
|
||||||
|
- run: echo Circle CI tests finished
|
||||||
|
# Teardown
|
||||||
|
# If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
|
||||||
|
# Save test results
|
||||||
|
- store_test_results:
|
||||||
|
path: /tmp/circleci-test-results
|
||||||
|
# Save artifacts
|
||||||
|
- store_artifacts:
|
||||||
|
path: /tmp/circleci-artifacts
|
||||||
|
- store_artifacts:
|
||||||
|
path: /tmp/circleci-test-results
|
||||||
|
|
||||||
|
# Longer tests
|
||||||
|
#- make -C tests test-zstd-nolegacy && make clean
|
||||||
|
#- pyenv global 3.4.4; make -C tests versionsTest && make clean
|
||||||
|
#- make zlibwrapper && make clean
|
||||||
|
#- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
|
||||||
|
#- make uasan && make clean
|
||||||
|
#- make asan32 && make clean
|
||||||
|
#- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
|
||||||
|
# Valgrind tests
|
||||||
|
#- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
|
||||||
|
#- make -C tests valgrindTest && make clean
|
||||||
|
# ARM, AArch64, PowerPC, PowerPC64 tests
|
||||||
|
#- make ppctest && make clean
|
||||||
|
#- make ppc64test && make clean
|
||||||
|
#- make armtest && make clean
|
||||||
|
#- make aarch64test && make clean
|
||||||
+21
-15
@@ -10,33 +10,38 @@ addons:
|
|||||||
matrix:
|
matrix:
|
||||||
include:
|
include:
|
||||||
# Ubuntu 14.04
|
# Ubuntu 14.04
|
||||||
- env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd </dev/null' # also test when stdin is not a tty
|
- env: Cmd='make test'
|
||||||
- env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32'
|
|
||||||
- env: Cmd='make gcc7install && CC=gcc-7 make clean uasan-test-zstd'
|
- env: Cmd='make gcc6install && CC=gcc-6 CFLAGS=-Werror make -j all
|
||||||
|
&& make clean && CC=gcc-6 make -j uasan-test-zstd </dev/null' # test when stdin is not a tty
|
||||||
|
- env: Cmd='make gcc6install libc6install
|
||||||
|
&& make clean && CC=gcc-6 make -j uasan-test-zstd32'
|
||||||
|
- env: Cmd='make gcc7install && make clean && CC=gcc-7 make -j uasan-test-zstd'
|
||||||
|
- env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all'
|
||||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
|
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
|
||||||
|
|
||||||
|
- env: Cmd='make staticAnalyze'
|
||||||
|
|
||||||
- env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'
|
- env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'
|
||||||
- env: Cmd='make gcc6install libc6install && CC=gcc-6 CFLAGS=-m32 make clean uasan-fuzztest'
|
- env: Cmd='make gcc6install libc6install
|
||||||
|
&& make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest'
|
||||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest'
|
- env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest'
|
||||||
- env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
|
- env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
|
||||||
|
|
||||||
- env: Cmd='make -C tests test-fuzzer-stackmode'
|
- env: Cmd='make -j uasanregressiontest
|
||||||
|
&& make clean && make -j msanregressiontest'
|
||||||
|
|
||||||
- env: Cmd='make valgrindinstall && make -C tests clean valgrindTest'
|
- env: Cmd='make valgrindinstall && make -C tests clean valgrindTest
|
||||||
|
&& make clean && make -C tests test-fuzzer-stackmode'
|
||||||
|
|
||||||
- env: Cmd='make arminstall && make armfuzz'
|
- env: Cmd='make arminstall && make armfuzz'
|
||||||
|
- env: Cmd='make arminstall && make aarch64fuzz'
|
||||||
# Following test is disabled, as there is a bug in Travis' ld
|
|
||||||
# preventing aarch64 compilation to complete.
|
|
||||||
# > collect2: error: ld terminated with signal 11 [Segmentation fault], core dumped
|
|
||||||
# to be re-enabled in a few commit, as it's possible that a random code change circumvent the ld bug
|
|
||||||
# - env: Cmd='make arminstall && make aarch64fuzz'
|
|
||||||
|
|
||||||
- env: Cmd='make ppcinstall && make ppcfuzz'
|
- env: Cmd='make ppcinstall && make ppcfuzz'
|
||||||
- env: Cmd='make ppcinstall && make ppc64fuzz'
|
- env: Cmd='make ppcinstall && make ppc64fuzz'
|
||||||
- env: Cmd='make -j uasanregressiontest && make clean && make -j msanregressiontest'
|
|
||||||
|
|
||||||
- env: Cmd='make lz4install && make -C tests test-lz4 test-pool && make clean && bash tests/libzstd_partial_builds.sh'
|
- env: Cmd='make lz4install && make -C tests test-lz4
|
||||||
|
&& make clean && make -C tests test-pool
|
||||||
|
&& make clean && bash tests/libzstd_partial_builds.sh'
|
||||||
|
|
||||||
# tag-specific test
|
# tag-specific test
|
||||||
- if: tag =~ ^v[0-9]\.[0-9]
|
- if: tag =~ ^v[0-9]\.[0-9]
|
||||||
@@ -49,6 +54,7 @@ branches:
|
|||||||
only:
|
only:
|
||||||
- dev
|
- dev
|
||||||
- master
|
- master
|
||||||
|
- travisTest
|
||||||
|
|
||||||
script:
|
script:
|
||||||
- JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
|
- JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
|
||||||
|
|||||||
@@ -30,10 +30,9 @@ default: lib-release zstd-release
|
|||||||
all: allmost examples manual contrib
|
all: allmost examples manual contrib
|
||||||
|
|
||||||
.PHONY: allmost
|
.PHONY: allmost
|
||||||
allmost: allzstd
|
allmost: allzstd zlibwrapper
|
||||||
$(MAKE) -C $(ZWRAPDIR) all
|
|
||||||
|
|
||||||
#skip zwrapper, can't build that on alternate architectures without the proper zlib installed
|
# skip zwrapper, can't build that on alternate architectures without the proper zlib installed
|
||||||
.PHONY: allzstd
|
.PHONY: allzstd
|
||||||
allzstd: lib
|
allzstd: lib
|
||||||
$(MAKE) -C $(PRGDIR) all
|
$(MAKE) -C $(PRGDIR) all
|
||||||
@@ -44,8 +43,8 @@ all32:
|
|||||||
$(MAKE) -C $(PRGDIR) zstd32
|
$(MAKE) -C $(PRGDIR) zstd32
|
||||||
$(MAKE) -C $(TESTDIR) all32
|
$(MAKE) -C $(TESTDIR) all32
|
||||||
|
|
||||||
.PHONY: lib lib-release
|
.PHONY: lib lib-release libzstd.a
|
||||||
lib lib-release:
|
lib lib-release :
|
||||||
@$(MAKE) -C $(ZSTDDIR) $@
|
@$(MAKE) -C $(ZSTDDIR) $@
|
||||||
|
|
||||||
.PHONY: zstd zstd-release
|
.PHONY: zstd zstd-release
|
||||||
@@ -59,12 +58,13 @@ zstdmt:
|
|||||||
cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
|
cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
|
||||||
|
|
||||||
.PHONY: zlibwrapper
|
.PHONY: zlibwrapper
|
||||||
zlibwrapper:
|
zlibwrapper: lib
|
||||||
$(MAKE) -C $(ZWRAPDIR) test
|
$(MAKE) -C $(ZWRAPDIR) all
|
||||||
|
|
||||||
.PHONY: test
|
.PHONY: test
|
||||||
|
test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
|
||||||
test:
|
test:
|
||||||
MOREFLAGS+="-g -DDEBUGLEVEL=1 -Werror" $(MAKE) -C $(PRGDIR) allVariants
|
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
|
||||||
$(MAKE) -C $(TESTDIR) $@
|
$(MAKE) -C $(TESTDIR) $@
|
||||||
|
|
||||||
.PHONY: shortest
|
.PHONY: shortest
|
||||||
@@ -113,7 +113,7 @@ clean:
|
|||||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT))
|
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT))
|
||||||
|
|
||||||
HOST_OS = POSIX
|
HOST_OS = POSIX
|
||||||
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
|
CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release
|
||||||
|
|
||||||
.PHONY: list
|
.PHONY: list
|
||||||
list:
|
list:
|
||||||
@@ -286,6 +286,9 @@ gcc6install: apt-add-repo
|
|||||||
gcc7install: apt-add-repo
|
gcc7install: apt-add-repo
|
||||||
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install
|
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install
|
||||||
|
|
||||||
|
gcc8install: apt-add-repo
|
||||||
|
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-8 gcc-8-multilib" $(MAKE) apt-install
|
||||||
|
|
||||||
gpp6install: apt-add-repo
|
gpp6install: apt-add-repo
|
||||||
APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
|
APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
|
||||||
|
|
||||||
@@ -347,7 +350,10 @@ bmi32build: clean
|
|||||||
$(CC) -v
|
$(CC) -v
|
||||||
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
|
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
|
||||||
|
|
||||||
staticAnalyze: clean
|
# static analyzer test uses clang's scan-build
|
||||||
|
# does not analyze zlibWrapper, due to detected issues in zlib source code
|
||||||
|
staticAnalyze: SCANBUILD ?= scan-build
|
||||||
|
staticAnalyze:
|
||||||
$(CC) -v
|
$(CC) -v
|
||||||
CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
|
CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib
|
||||||
endif
|
endif
|
||||||
|
|||||||
@@ -121,6 +121,8 @@ A `cmake` project generator is provided within `build/cmake`.
|
|||||||
It can generate Makefiles or other build scripts
|
It can generate Makefiles or other build scripts
|
||||||
to create `zstd` binary, and `libzstd` dynamic and static libraries.
|
to create `zstd` binary, and `libzstd` dynamic and static libraries.
|
||||||
|
|
||||||
|
By default, `CMAKE_BUILD_TYPE` is set to `Release`.
|
||||||
|
|
||||||
#### Meson
|
#### Meson
|
||||||
|
|
||||||
A Meson project is provided within `contrib/meson`.
|
A Meson project is provided within `contrib/meson`.
|
||||||
|
|||||||
+3
-3
@@ -181,15 +181,15 @@
|
|||||||
- COMPILER: "gcc"
|
- COMPILER: "gcc"
|
||||||
HOST: "mingw"
|
HOST: "mingw"
|
||||||
PLATFORM: "x64"
|
PLATFORM: "x64"
|
||||||
SCRIPT: "make allzstd"
|
SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
|
||||||
- COMPILER: "gcc"
|
- COMPILER: "gcc"
|
||||||
HOST: "mingw"
|
HOST: "mingw"
|
||||||
PLATFORM: "x86"
|
PLATFORM: "x86"
|
||||||
SCRIPT: "make allzstd"
|
SCRIPT: "CFLAGS=-Werror make -j allzstd"
|
||||||
- COMPILER: "clang"
|
- COMPILER: "clang"
|
||||||
HOST: "mingw"
|
HOST: "mingw"
|
||||||
PLATFORM: "x64"
|
PLATFORM: "x64"
|
||||||
SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd"
|
SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd"
|
||||||
|
|
||||||
- COMPILER: "visual"
|
- COMPILER: "visual"
|
||||||
HOST: "visual"
|
HOST: "visual"
|
||||||
|
|||||||
@@ -18,3 +18,14 @@ Studio*
|
|||||||
|
|
||||||
# CMake
|
# CMake
|
||||||
cmake/build/
|
cmake/build/
|
||||||
|
CMakeCache.txt
|
||||||
|
CMakeFiles
|
||||||
|
CMakeScripts
|
||||||
|
Testing
|
||||||
|
Makefile
|
||||||
|
cmake_install.cmake
|
||||||
|
install_manifest.txt
|
||||||
|
compile_commands.json
|
||||||
|
CTestTestfile.cmake
|
||||||
|
build
|
||||||
|
lib
|
||||||
|
|||||||
@@ -336,6 +336,10 @@
|
|||||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\dictBuilder\fastcover.c"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||||
>
|
>
|
||||||
@@ -482,6 +486,10 @@
|
|||||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\dictBuilder\cover.h"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -348,6 +348,10 @@
|
|||||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\dictBuilder\fastcover.c"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||||
>
|
>
|
||||||
@@ -522,6 +526,10 @@
|
|||||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\dictBuilder\cover.h"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -332,6 +332,10 @@
|
|||||||
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
RelativePath="..\..\..\lib\dictBuilder\cover.c"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\dictBuilder\fastcover.c"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||||
>
|
>
|
||||||
@@ -502,6 +506,10 @@
|
|||||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||||
>
|
>
|
||||||
</File>
|
</File>
|
||||||
|
<File
|
||||||
|
RelativePath="..\..\..\lib\dictBuilder\cover.h"
|
||||||
|
>
|
||||||
|
</File>
|
||||||
<File
|
<File
|
||||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||||
>
|
>
|
||||||
|
|||||||
@@ -176,6 +176,7 @@
|
|||||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||||
|
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||||
@@ -199,6 +200,7 @@
|
|||||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
||||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
||||||
|
<ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
|
||||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||||
<ClInclude Include="..\..\..\programs\util.h" />
|
<ClInclude Include="..\..\..\programs\util.h" />
|
||||||
|
|||||||
@@ -43,6 +43,7 @@
|
|||||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||||
|
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||||
|
|||||||
@@ -43,6 +43,7 @@
|
|||||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||||
|
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||||
|
|||||||
@@ -40,6 +40,7 @@
|
|||||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
|
||||||
|
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||||
@@ -61,6 +62,7 @@
|
|||||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||||
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
|
||||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
||||||
|
<ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
|
||||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\fse.h" />
|
<ClInclude Include="..\..\..\lib\common\fse.h" />
|
||||||
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
||||||
|
|||||||
@@ -10,6 +10,10 @@
|
|||||||
PROJECT(zstd)
|
PROJECT(zstd)
|
||||||
CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
|
CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9)
|
||||||
SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
|
SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
|
||||||
|
|
||||||
|
# Ensure Release build even if not invoked via Makefile
|
||||||
|
SET(CMAKE_BUILD_TYPE "Release")
|
||||||
|
|
||||||
LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
|
LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
|
||||||
INCLUDE(GNUInstallDirs)
|
INCLUDE(GNUInstallDirs)
|
||||||
|
|
||||||
|
|||||||
@@ -27,4 +27,4 @@ ADD_CUSTOM_TARGET(zstd_manual.html ALL
|
|||||||
${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html"
|
${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html"
|
||||||
DEPENDS gen_html COMMENT "Update zstd manual")
|
DEPENDS gen_html COMMENT "Update zstd manual")
|
||||||
|
|
||||||
INSTALL(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_PREFIX}/${DOC_INSTALL_DIR}")
|
INSTALL(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}")
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ OPTION(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON)
|
|||||||
OPTION(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
|
OPTION(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON)
|
||||||
|
|
||||||
IF(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
|
IF(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC)
|
||||||
MESSAGE(SEND_ERROR "You need to build at least one flavor of libstd")
|
MESSAGE(SEND_ERROR "You need to build at least one flavor of libzstd")
|
||||||
ENDIF()
|
ENDIF()
|
||||||
|
|
||||||
# Define library directory, where sources and header files are located
|
# Define library directory, where sources and header files are located
|
||||||
@@ -47,6 +47,7 @@ SET(Sources
|
|||||||
${LIBRARY_DIR}/decompress/huf_decompress.c
|
${LIBRARY_DIR}/decompress/huf_decompress.c
|
||||||
${LIBRARY_DIR}/decompress/zstd_decompress.c
|
${LIBRARY_DIR}/decompress/zstd_decompress.c
|
||||||
${LIBRARY_DIR}/dictBuilder/cover.c
|
${LIBRARY_DIR}/dictBuilder/cover.c
|
||||||
|
${LIBRARY_DIR}/dictBuilder/fastcover.c
|
||||||
${LIBRARY_DIR}/dictBuilder/divsufsort.c
|
${LIBRARY_DIR}/dictBuilder/divsufsort.c
|
||||||
${LIBRARY_DIR}/dictBuilder/zdict.c
|
${LIBRARY_DIR}/dictBuilder/zdict.c
|
||||||
${LIBRARY_DIR}/deprecated/zbuff_common.c
|
${LIBRARY_DIR}/deprecated/zbuff_common.c
|
||||||
@@ -74,6 +75,7 @@ SET(Headers
|
|||||||
${LIBRARY_DIR}/compress/zstd_ldm.h
|
${LIBRARY_DIR}/compress/zstd_ldm.h
|
||||||
${LIBRARY_DIR}/compress/zstdmt_compress.h
|
${LIBRARY_DIR}/compress/zstdmt_compress.h
|
||||||
${LIBRARY_DIR}/dictBuilder/zdict.h
|
${LIBRARY_DIR}/dictBuilder/zdict.h
|
||||||
|
${LIBRARY_DIR}/dictBuilder/cover.h
|
||||||
${LIBRARY_DIR}/deprecated/zbuff.h)
|
${LIBRARY_DIR}/deprecated/zbuff.h)
|
||||||
|
|
||||||
IF (ZSTD_LEGACY_SUPPORT)
|
IF (ZSTD_LEGACY_SUPPORT)
|
||||||
@@ -178,6 +180,7 @@ INSTALL(FILES
|
|||||||
${LIBRARY_DIR}/zstd.h
|
${LIBRARY_DIR}/zstd.h
|
||||||
${LIBRARY_DIR}/deprecated/zbuff.h
|
${LIBRARY_DIR}/deprecated/zbuff.h
|
||||||
${LIBRARY_DIR}/dictBuilder/zdict.h
|
${LIBRARY_DIR}/dictBuilder/zdict.h
|
||||||
|
${LIBRARY_DIR}/dictBuilder/cover.h
|
||||||
${LIBRARY_DIR}/common/zstd_errors.h
|
${LIBRARY_DIR}/common/zstd_errors.h
|
||||||
DESTINATION "include")
|
DESTINATION "include")
|
||||||
|
|
||||||
|
|||||||
-63
@@ -1,63 +0,0 @@
|
|||||||
dependencies:
|
|
||||||
override:
|
|
||||||
- sudo dpkg --add-architecture i386
|
|
||||||
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
|
|
||||||
- sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
|
|
||||||
|
|
||||||
test:
|
|
||||||
override:
|
|
||||||
- ? |
|
|
||||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; CFLAGS="-O0 -Werror" make all && make clean; fi &&
|
|
||||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi
|
|
||||||
:
|
|
||||||
parallel: true
|
|
||||||
- ? |
|
|
||||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c99build && make clean; fi &&
|
|
||||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build && make clean; fi
|
|
||||||
:
|
|
||||||
parallel: true
|
|
||||||
- ? |
|
|
||||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi &&
|
|
||||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build && make clean; fi
|
|
||||||
:
|
|
||||||
parallel: true
|
|
||||||
- ? |
|
|
||||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make aarch64build && make clean; fi &&
|
|
||||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi
|
|
||||||
:
|
|
||||||
parallel: true
|
|
||||||
- ? |
|
|
||||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi &&
|
|
||||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi
|
|
||||||
:
|
|
||||||
parallel: true
|
|
||||||
- ? |
|
|
||||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make shortest && make clean; fi &&
|
|
||||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
|
|
||||||
:
|
|
||||||
parallel: true
|
|
||||||
- ? |
|
|
||||||
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi &&
|
|
||||||
if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi
|
|
||||||
:
|
|
||||||
parallel: true
|
|
||||||
|
|
||||||
post:
|
|
||||||
- echo Circle CI tests finished
|
|
||||||
|
|
||||||
# Longer tests
|
|
||||||
#- make -C tests test-zstd-nolegacy && make clean
|
|
||||||
#- pyenv global 3.4.4; make -C tests versionsTest && make clean
|
|
||||||
#- make zlibwrapper && make clean
|
|
||||||
#- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
|
|
||||||
#- make uasan && make clean
|
|
||||||
#- make asan32 && make clean
|
|
||||||
#- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
|
|
||||||
# Valgrind tests
|
|
||||||
#- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
|
|
||||||
#- make -C tests valgrindTest && make clean
|
|
||||||
# ARM, AArch64, PowerPC, PowerPC64 tests
|
|
||||||
#- make ppctest && make clean
|
|
||||||
#- make ppc64test && make clean
|
|
||||||
#- make armtest && make clean
|
|
||||||
#- make aarch64test && make clean
|
|
||||||
@@ -2,10 +2,9 @@ ARG :=
|
|||||||
|
|
||||||
CC ?= gcc
|
CC ?= gcc
|
||||||
CFLAGS ?= -O3
|
CFLAGS ?= -O3
|
||||||
INCLUDES := -I ../randomDictBuilder -I ../fastCover -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
|
INCLUDES := -I ../randomDictBuilder -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder
|
||||||
|
|
||||||
RANDOM_FILE := ../randomDictBuilder/random.c
|
RANDOM_FILE := ../randomDictBuilder/random.c
|
||||||
FAST_FILE := ../fastCover/fastCover.c
|
|
||||||
IO_FILE := ../randomDictBuilder/io.c
|
IO_FILE := ../randomDictBuilder/io.c
|
||||||
|
|
||||||
all: run clean
|
all: run clean
|
||||||
@@ -22,8 +21,8 @@ test: benchmarkTest clean
|
|||||||
benchmarkTest: benchmark test.sh
|
benchmarkTest: benchmark test.sh
|
||||||
sh test.sh
|
sh test.sh
|
||||||
|
|
||||||
benchmark: benchmark.o io.o random.o fastCover.o libzstd.a
|
benchmark: benchmark.o io.o random.o libzstd.a
|
||||||
$(CC) $(CFLAGS) benchmark.o io.o random.o fastCover.o libzstd.a -o benchmark
|
$(CC) $(CFLAGS) benchmark.o io.o random.o libzstd.a -o benchmark
|
||||||
|
|
||||||
benchmark.o: benchmark.c
|
benchmark.o: benchmark.c
|
||||||
$(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c
|
$(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c
|
||||||
@@ -31,9 +30,6 @@ benchmark.o: benchmark.c
|
|||||||
random.o: $(RANDOM_FILE)
|
random.o: $(RANDOM_FILE)
|
||||||
$(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE)
|
$(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE)
|
||||||
|
|
||||||
fastCover.o: $(FAST_FILE)
|
|
||||||
$(CC) $(CFLAGS) $(INCLUDES) -c $(FAST_FILE)
|
|
||||||
|
|
||||||
io.o: $(IO_FILE)
|
io.o: $(IO_FILE)
|
||||||
$(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE)
|
$(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE)
|
||||||
|
|
||||||
|
|||||||
@@ -14,113 +14,836 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress"
|
|||||||
|
|
||||||
###Benchmarking Result:
|
###Benchmarking Result:
|
||||||
- First Cover is optimize cover, second Cover uses optimized d and k from first one.
|
- First Cover is optimize cover, second Cover uses optimized d and k from first one.
|
||||||
- For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one.
|
- For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. This is run for accel values from 1 to 10.
|
||||||
- Fourth column is chosen d and fifth column is chosen k
|
- Fourth column is chosen d and fifth column is chosen k
|
||||||
|
|
||||||
github:
|
github:
|
||||||
NODICT 0.000025 2.999642
|
NODICT 0.000004 2.999642
|
||||||
RANDOM 0.030101 8.791189
|
RANDOM 0.024560 8.791189
|
||||||
LEGACY 0.913108 8.173529
|
LEGACY 0.727109 8.173529
|
||||||
COVER 59.234160 10.652243 8 1298
|
COVER 40.565676 10.652243 8 1298
|
||||||
COVER 6.258459 10.652243 8 1298
|
COVER 3.608284 10.652243 8 1298
|
||||||
FAST15 9.959246 10.555630 8 1874
|
FAST f=15 a=1 4.181024 10.570882 8 1154
|
||||||
FAST15 0.077719 10.555630 8 1874
|
FAST f=15 a=1 0.040788 10.570882 8 1154
|
||||||
FAST16 10.028343 10.701698 8 1106
|
FAST f=15 a=2 3.548352 10.574287 6 1970
|
||||||
FAST16 0.078117 10.701698 8 1106
|
FAST f=15 a=2 0.035535 10.574287 6 1970
|
||||||
FAST17 10.567355 10.650652 8 1106
|
FAST f=15 a=3 3.287364 10.613950 6 1010
|
||||||
FAST17 0.124833 10.650652 8 1106
|
FAST f=15 a=3 0.032182 10.613950 6 1010
|
||||||
FAST18 11.795287 10.499142 8 1826
|
FAST f=15 a=4 3.184976 10.573883 6 1058
|
||||||
FAST18 0.086992 10.499142 8 1826
|
FAST f=15 a=4 0.029878 10.573883 6 1058
|
||||||
FAST19 13.132451 10.527140 8 1826
|
FAST f=15 a=5 3.045513 10.580640 8 1154
|
||||||
FAST19 0.134716 10.527140 8 1826
|
FAST f=15 a=5 0.022162 10.580640 8 1154
|
||||||
FAST20 14.366314 10.494710 8 1826
|
FAST f=15 a=6 3.003296 10.583677 6 1010
|
||||||
FAST20 0.128844 10.494710 8 1826
|
FAST f=15 a=6 0.028091 10.583677 6 1010
|
||||||
FAST21 14.941238 10.503488 8 1778
|
FAST f=15 a=7 2.952655 10.622551 6 1106
|
||||||
FAST21 0.134975 10.503488 8 1778
|
FAST f=15 a=7 0.02724 10.622551 6 1106
|
||||||
FAST22 15.146226 10.509284 8 1826
|
FAST f=15 a=8 2.945674 10.614657 6 1010
|
||||||
FAST22 0.146918 10.509284 8 1826
|
FAST f=15 a=8 0.027264 10.614657 6 1010
|
||||||
FAST23 16.260552 10.509284 8 1826
|
FAST f=15 a=9 3.153439 10.564018 8 1154
|
||||||
FAST23 0.158494 10.509284 8 1826
|
FAST f=15 a=9 0.020635 10.564018 8 1154
|
||||||
FAST24 16.806037 10.512369 8 1826
|
FAST f=15 a=10 2.950416 10.511454 6 1010
|
||||||
FAST24 0.190464 10.512369 8 1826
|
FAST f=15 a=10 0.026606 10.511454 6 1010
|
||||||
|
FAST f=16 a=1 3.970029 10.681035 8 1154
|
||||||
|
FAST f=16 a=1 0.038188 10.681035 8 1154
|
||||||
|
FAST f=16 a=2 3.422892 10.484978 6 1874
|
||||||
|
FAST f=16 a=2 0.034702 10.484978 6 1874
|
||||||
|
FAST f=16 a=3 3.215836 10.632631 8 1154
|
||||||
|
FAST f=16 a=3 0.026084 10.632631 8 1154
|
||||||
|
FAST f=16 a=4 3.081353 10.626533 6 1106
|
||||||
|
FAST f=16 a=4 0.030032 10.626533 6 1106
|
||||||
|
FAST f=16 a=5 3.041241 10.545027 8 1922
|
||||||
|
FAST f=16 a=5 0.022882 10.545027 8 1922
|
||||||
|
FAST f=16 a=6 2.989390 10.638284 6 1874
|
||||||
|
FAST f=16 a=6 0.028308 10.638284 6 1874
|
||||||
|
FAST f=16 a=7 3.001581 10.797136 6 1106
|
||||||
|
FAST f=16 a=7 0.027479 10.797136 6 1106
|
||||||
|
FAST f=16 a=8 2.984107 10.658356 8 1058
|
||||||
|
FAST f=16 a=8 0.021099 10.658356 8 1058
|
||||||
|
FAST f=16 a=9 2.925788 10.523869 6 1010
|
||||||
|
FAST f=16 a=9 0.026905 10.523869 6 1010
|
||||||
|
FAST f=16 a=10 2.889605 10.745841 6 1874
|
||||||
|
FAST f=16 a=10 0.026846 10.745841 6 1874
|
||||||
|
FAST f=17 a=1 4.031953 10.672080 8 1202
|
||||||
|
FAST f=17 a=1 0.040658 10.672080 8 1202
|
||||||
|
FAST f=17 a=2 3.458107 10.589352 8 1106
|
||||||
|
FAST f=17 a=2 0.02926 10.589352 8 1106
|
||||||
|
FAST f=17 a=3 3.291189 10.662714 8 1154
|
||||||
|
FAST f=17 a=3 0.026531 10.662714 8 1154
|
||||||
|
FAST f=17 a=4 3.154950 10.549456 8 1346
|
||||||
|
FAST f=17 a=4 0.024991 10.549456 8 1346
|
||||||
|
FAST f=17 a=5 3.092271 10.541670 6 1202
|
||||||
|
FAST f=17 a=5 0.038285 10.541670 6 1202
|
||||||
|
FAST f=17 a=6 3.166146 10.729112 6 1874
|
||||||
|
FAST f=17 a=6 0.038217 10.729112 6 1874
|
||||||
|
FAST f=17 a=7 3.035467 10.810485 6 1106
|
||||||
|
FAST f=17 a=7 0.036655 10.810485 6 1106
|
||||||
|
FAST f=17 a=8 3.035668 10.530532 6 1058
|
||||||
|
FAST f=17 a=8 0.037715 10.530532 6 1058
|
||||||
|
FAST f=17 a=9 2.987917 10.589802 8 1922
|
||||||
|
FAST f=17 a=9 0.02217 10.589802 8 1922
|
||||||
|
FAST f=17 a=10 2.981647 10.722579 8 1106
|
||||||
|
FAST f=17 a=10 0.021948 10.722579 8 1106
|
||||||
|
FAST f=18 a=1 4.067144 10.634943 8 1154
|
||||||
|
FAST f=18 a=1 0.041386 10.634943 8 1154
|
||||||
|
FAST f=18 a=2 3.507377 10.546230 6 1970
|
||||||
|
FAST f=18 a=2 0.037572 10.546230 6 1970
|
||||||
|
FAST f=18 a=3 3.323015 10.648061 8 1154
|
||||||
|
FAST f=18 a=3 0.028306 10.648061 8 1154
|
||||||
|
FAST f=18 a=4 3.216735 10.705402 6 1010
|
||||||
|
FAST f=18 a=4 0.030755 10.705402 6 1010
|
||||||
|
FAST f=18 a=5 3.175794 10.588154 8 1874
|
||||||
|
FAST f=18 a=5 0.025315 10.588154 8 1874
|
||||||
|
FAST f=18 a=6 3.127459 10.751104 8 1106
|
||||||
|
FAST f=18 a=6 0.023897 10.751104 8 1106
|
||||||
|
FAST f=18 a=7 3.083017 10.780402 6 1106
|
||||||
|
FAST f=18 a=7 0.029158 10.780402 6 1106
|
||||||
|
FAST f=18 a=8 3.069700 10.547226 8 1346
|
||||||
|
FAST f=18 a=8 0.024046 10.547226 8 1346
|
||||||
|
FAST f=18 a=9 3.056591 10.674759 6 1010
|
||||||
|
FAST f=18 a=9 0.028496 10.674759 6 1010
|
||||||
|
FAST f=18 a=10 3.063588 10.737578 8 1106
|
||||||
|
FAST f=18 a=10 0.023033 10.737578 8 1106
|
||||||
|
FAST f=19 a=1 4.164041 10.650333 8 1154
|
||||||
|
FAST f=19 a=1 0.042906 10.650333 8 1154
|
||||||
|
FAST f=19 a=2 3.585409 10.577066 6 1058
|
||||||
|
FAST f=19 a=2 0.038994 10.577066 6 1058
|
||||||
|
FAST f=19 a=3 3.439643 10.639403 8 1154
|
||||||
|
FAST f=19 a=3 0.028427 10.639403 8 1154
|
||||||
|
FAST f=19 a=4 3.268869 10.554410 8 1298
|
||||||
|
FAST f=19 a=4 0.026866 10.554410 8 1298
|
||||||
|
FAST f=19 a=5 3.238225 10.615109 6 1010
|
||||||
|
FAST f=19 a=5 0.03078 10.615109 6 1010
|
||||||
|
FAST f=19 a=6 3.199558 10.609782 6 1874
|
||||||
|
FAST f=19 a=6 0.030099 10.609782 6 1874
|
||||||
|
FAST f=19 a=7 3.132395 10.794753 6 1106
|
||||||
|
FAST f=19 a=7 0.028964 10.794753 6 1106
|
||||||
|
FAST f=19 a=8 3.148446 10.554842 8 1298
|
||||||
|
FAST f=19 a=8 0.024277 10.554842 8 1298
|
||||||
|
FAST f=19 a=9 3.108324 10.668763 6 1010
|
||||||
|
FAST f=19 a=9 0.02896 10.668763 6 1010
|
||||||
|
FAST f=19 a=10 3.159863 10.757347 8 1106
|
||||||
|
FAST f=19 a=10 0.023351 10.757347 8 1106
|
||||||
|
FAST f=20 a=1 4.462698 10.661788 8 1154
|
||||||
|
FAST f=20 a=1 0.047174 10.661788 8 1154
|
||||||
|
FAST f=20 a=2 3.820269 10.678612 6 1106
|
||||||
|
FAST f=20 a=2 0.040807 10.678612 6 1106
|
||||||
|
FAST f=20 a=3 3.644955 10.648424 8 1154
|
||||||
|
FAST f=20 a=3 0.031398 10.648424 8 1154
|
||||||
|
FAST f=20 a=4 3.546257 10.559756 8 1298
|
||||||
|
FAST f=20 a=4 0.029856 10.559756 8 1298
|
||||||
|
FAST f=20 a=5 3.485248 10.646637 6 1010
|
||||||
|
FAST f=20 a=5 0.033756 10.646637 6 1010
|
||||||
|
FAST f=20 a=6 3.490438 10.775824 8 1106
|
||||||
|
FAST f=20 a=6 0.028338 10.775824 8 1106
|
||||||
|
FAST f=20 a=7 3.631289 10.801795 6 1106
|
||||||
|
FAST f=20 a=7 0.035228 10.801795 6 1106
|
||||||
|
FAST f=20 a=8 3.758936 10.545116 8 1346
|
||||||
|
FAST f=20 a=8 0.027495 10.545116 8 1346
|
||||||
|
FAST f=20 a=9 3.707024 10.677454 6 1010
|
||||||
|
FAST f=20 a=9 0.031326 10.677454 6 1010
|
||||||
|
FAST f=20 a=10 3.586593 10.756017 8 1106
|
||||||
|
FAST f=20 a=10 0.027122 10.756017 8 1106
|
||||||
|
FAST f=21 a=1 5.701396 10.655398 8 1154
|
||||||
|
FAST f=21 a=1 0.067744 10.655398 8 1154
|
||||||
|
FAST f=21 a=2 5.270542 10.650743 6 1106
|
||||||
|
FAST f=21 a=2 0.052999 10.650743 6 1106
|
||||||
|
FAST f=21 a=3 4.945294 10.652380 8 1154
|
||||||
|
FAST f=21 a=3 0.052678 10.652380 8 1154
|
||||||
|
FAST f=21 a=4 4.894079 10.543185 8 1298
|
||||||
|
FAST f=21 a=4 0.04997 10.543185 8 1298
|
||||||
|
FAST f=21 a=5 4.785417 10.630321 6 1010
|
||||||
|
FAST f=21 a=5 0.045294 10.630321 6 1010
|
||||||
|
FAST f=21 a=6 4.789381 10.664477 6 1874
|
||||||
|
FAST f=21 a=6 0.046578 10.664477 6 1874
|
||||||
|
FAST f=21 a=7 4.302955 10.805179 6 1106
|
||||||
|
FAST f=21 a=7 0.041205 10.805179 6 1106
|
||||||
|
FAST f=21 a=8 4.034630 10.551211 8 1298
|
||||||
|
FAST f=21 a=8 0.040121 10.551211 8 1298
|
||||||
|
FAST f=21 a=9 4.523868 10.799114 6 1010
|
||||||
|
FAST f=21 a=9 0.043592 10.799114 6 1010
|
||||||
|
FAST f=21 a=10 4.760736 10.750255 8 1106
|
||||||
|
FAST f=21 a=10 0.043483 10.750255 8 1106
|
||||||
|
FAST f=22 a=1 6.743064 10.640537 8 1154
|
||||||
|
FAST f=22 a=1 0.086967 10.640537 8 1154
|
||||||
|
FAST f=22 a=2 6.121739 10.626638 6 1970
|
||||||
|
FAST f=22 a=2 0.066337 10.626638 6 1970
|
||||||
|
FAST f=22 a=3 5.248851 10.640688 8 1154
|
||||||
|
FAST f=22 a=3 0.054935 10.640688 8 1154
|
||||||
|
FAST f=22 a=4 5.436579 10.588333 8 1298
|
||||||
|
FAST f=22 a=4 0.064113 10.588333 8 1298
|
||||||
|
FAST f=22 a=5 5.812815 10.652653 6 1010
|
||||||
|
FAST f=22 a=5 0.058189 10.652653 6 1010
|
||||||
|
FAST f=22 a=6 5.745472 10.666437 6 1874
|
||||||
|
FAST f=22 a=6 0.057188 10.666437 6 1874
|
||||||
|
FAST f=22 a=7 5.716393 10.806911 6 1106
|
||||||
|
FAST f=22 a=7 0.056 10.806911 6 1106
|
||||||
|
FAST f=22 a=8 5.698799 10.530784 8 1298
|
||||||
|
FAST f=22 a=8 0.0583 10.530784 8 1298
|
||||||
|
FAST f=22 a=9 5.710533 10.777391 6 1010
|
||||||
|
FAST f=22 a=9 0.054945 10.777391 6 1010
|
||||||
|
FAST f=22 a=10 5.685395 10.745023 8 1106
|
||||||
|
FAST f=22 a=10 0.056526 10.745023 8 1106
|
||||||
|
FAST f=23 a=1 7.836923 10.638828 8 1154
|
||||||
|
FAST f=23 a=1 0.099522 10.638828 8 1154
|
||||||
|
FAST f=23 a=2 6.627834 10.631061 6 1970
|
||||||
|
FAST f=23 a=2 0.066769 10.631061 6 1970
|
||||||
|
FAST f=23 a=3 5.602533 10.647288 8 1154
|
||||||
|
FAST f=23 a=3 0.064513 10.647288 8 1154
|
||||||
|
FAST f=23 a=4 6.005580 10.568747 8 1298
|
||||||
|
FAST f=23 a=4 0.062022 10.568747 8 1298
|
||||||
|
FAST f=23 a=5 5.481816 10.676921 6 1010
|
||||||
|
FAST f=23 a=5 0.058959 10.676921 6 1010
|
||||||
|
FAST f=23 a=6 5.460444 10.666194 6 1874
|
||||||
|
FAST f=23 a=6 0.057687 10.666194 6 1874
|
||||||
|
FAST f=23 a=7 5.659822 10.800377 6 1106
|
||||||
|
FAST f=23 a=7 0.06783 10.800377 6 1106
|
||||||
|
FAST f=23 a=8 6.826940 10.522167 8 1298
|
||||||
|
FAST f=23 a=8 0.070533 10.522167 8 1298
|
||||||
|
FAST f=23 a=9 6.804757 10.577799 8 1682
|
||||||
|
FAST f=23 a=9 0.069949 10.577799 8 1682
|
||||||
|
FAST f=23 a=10 6.774933 10.742093 8 1106
|
||||||
|
FAST f=23 a=10 0.068395 10.742093 8 1106
|
||||||
|
FAST f=24 a=1 8.444110 10.632783 8 1154
|
||||||
|
FAST f=24 a=1 0.094357 10.632783 8 1154
|
||||||
|
FAST f=24 a=2 7.289578 10.631061 6 1970
|
||||||
|
FAST f=24 a=2 0.098515 10.631061 6 1970
|
||||||
|
FAST f=24 a=3 8.619780 10.646289 8 1154
|
||||||
|
FAST f=24 a=3 0.098041 10.646289 8 1154
|
||||||
|
FAST f=24 a=4 8.508455 10.555199 8 1298
|
||||||
|
FAST f=24 a=4 0.093885 10.555199 8 1298
|
||||||
|
FAST f=24 a=5 8.471145 10.674363 6 1010
|
||||||
|
FAST f=24 a=5 0.088676 10.674363 6 1010
|
||||||
|
FAST f=24 a=6 8.426727 10.667228 6 1874
|
||||||
|
FAST f=24 a=6 0.087247 10.667228 6 1874
|
||||||
|
FAST f=24 a=7 8.356826 10.803027 6 1106
|
||||||
|
FAST f=24 a=7 0.085835 10.803027 6 1106
|
||||||
|
FAST f=24 a=8 6.756811 10.522049 8 1298
|
||||||
|
FAST f=24 a=8 0.07107 10.522049 8 1298
|
||||||
|
FAST f=24 a=9 6.548169 10.571882 8 1682
|
||||||
|
FAST f=24 a=9 0.0713 10.571882 8 1682
|
||||||
|
FAST f=24 a=10 8.238079 10.736453 8 1106
|
||||||
|
FAST f=24 a=10 0.07004 10.736453 8 1106
|
||||||
|
|
||||||
|
|
||||||
hg-commands:
|
hg-commands:
|
||||||
NODICT 0.000026 2.425291
|
NODICT 0.000005 2.425276
|
||||||
RANDOM 0.046270 3.490331
|
RANDOM 0.046332 3.490331
|
||||||
LEGACY 0.847904 3.911682
|
LEGACY 0.720351 3.911682
|
||||||
COVER 71.691804 4.132653 8 386
|
COVER 45.507731 4.132653 8 386
|
||||||
COVER 3.187085 4.132653 8 386
|
COVER 1.868810 4.132653 8 386
|
||||||
FAST15 11.593687 3.920720 6 1106
|
FAST f=15 a=1 4.561427 3.866894 8 1202
|
||||||
FAST15 0.082431 3.920720 6 1106
|
FAST f=15 a=1 0.048946 3.866894 8 1202
|
||||||
FAST16 11.775958 4.033306 8 674
|
FAST f=15 a=2 3.574462 3.892119 8 1538
|
||||||
FAST16 0.092587 4.033306 8 674
|
FAST f=15 a=2 0.033677 3.892119 8 1538
|
||||||
FAST17 11.965064 4.064132 8 1490
|
FAST f=15 a=3 3.230227 3.888791 6 1346
|
||||||
FAST17 0.106382 4.064132 8 1490
|
FAST f=15 a=3 0.034312 3.888791 6 1346
|
||||||
FAST18 11.438197 4.086714 8 290
|
FAST f=15 a=4 3.042388 3.899739 8 1010
|
||||||
FAST18 0.097293 4.086714 8 290
|
FAST f=15 a=4 0.024307 3.899739 8 1010
|
||||||
FAST19 12.292512 4.097947 8 578
|
FAST f=15 a=5 2.800148 3.896220 8 818
|
||||||
FAST19 0.104406 4.097947 8 578
|
FAST f=15 a=5 0.022331 3.896220 8 818
|
||||||
FAST20 13.857857 4.102851 8 434
|
FAST f=15 a=6 2.706518 3.882039 8 578
|
||||||
FAST20 0.139467 4.102851 8 434
|
FAST f=15 a=6 0.020955 3.882039 8 578
|
||||||
FAST21 14.599613 4.105350 8 530
|
FAST f=15 a=7 2.701820 3.885430 6 866
|
||||||
FAST21 0.189416 4.105350 8 530
|
FAST f=15 a=7 0.026074 3.885430 6 866
|
||||||
FAST22 15.966109 4.104100 8 530
|
FAST f=15 a=8 2.604445 3.906932 8 1826
|
||||||
FAST22 0.183817 4.104100 8 530
|
FAST f=15 a=8 0.021789 3.906932 8 1826
|
||||||
FAST23 18.033645 4.098110 8 914
|
FAST f=15 a=9 2.598568 3.870324 6 1682
|
||||||
FAST23 0.246641 4.098110 8 914
|
FAST f=15 a=9 0.026004 3.870324 6 1682
|
||||||
FAST24 22.992891 4.117367 8 722
|
FAST f=15 a=10 2.575920 3.920783 8 1442
|
||||||
FAST24 0.285994 4.117367 8 722
|
FAST f=15 a=10 0.020228 3.920783 8 1442
|
||||||
|
FAST f=16 a=1 4.630623 4.001430 8 770
|
||||||
|
FAST f=16 a=1 0.047497 4.001430 8 770
|
||||||
|
FAST f=16 a=2 3.674721 3.974431 8 1874
|
||||||
|
FAST f=16 a=2 0.035761 3.974431 8 1874
|
||||||
|
FAST f=16 a=3 3.338384 3.978703 8 1010
|
||||||
|
FAST f=16 a=3 0.029436 3.978703 8 1010
|
||||||
|
FAST f=16 a=4 3.004412 3.983035 8 1010
|
||||||
|
FAST f=16 a=4 0.025744 3.983035 8 1010
|
||||||
|
FAST f=16 a=5 2.881892 3.987710 8 770
|
||||||
|
FAST f=16 a=5 0.023211 3.987710 8 770
|
||||||
|
FAST f=16 a=6 2.807410 3.952717 8 1298
|
||||||
|
FAST f=16 a=6 0.023199 3.952717 8 1298
|
||||||
|
FAST f=16 a=7 2.819623 3.994627 8 770
|
||||||
|
FAST f=16 a=7 0.021806 3.994627 8 770
|
||||||
|
FAST f=16 a=8 2.740092 3.954032 8 1826
|
||||||
|
FAST f=16 a=8 0.0226 3.954032 8 1826
|
||||||
|
FAST f=16 a=9 2.682564 3.969879 6 1442
|
||||||
|
FAST f=16 a=9 0.026324 3.969879 6 1442
|
||||||
|
FAST f=16 a=10 2.657959 3.969755 8 674
|
||||||
|
FAST f=16 a=10 0.020413 3.969755 8 674
|
||||||
|
FAST f=17 a=1 4.729228 4.046000 8 530
|
||||||
|
FAST f=17 a=1 0.049703 4.046000 8 530
|
||||||
|
FAST f=17 a=2 3.764510 3.991519 8 1970
|
||||||
|
FAST f=17 a=2 0.038195 3.991519 8 1970
|
||||||
|
FAST f=17 a=3 3.416992 4.006296 6 914
|
||||||
|
FAST f=17 a=3 0.036244 4.006296 6 914
|
||||||
|
FAST f=17 a=4 3.145626 3.979182 8 1970
|
||||||
|
FAST f=17 a=4 0.028676 3.979182 8 1970
|
||||||
|
FAST f=17 a=5 2.995070 4.050070 8 770
|
||||||
|
FAST f=17 a=5 0.025707 4.050070 8 770
|
||||||
|
FAST f=17 a=6 2.911833 4.040024 8 770
|
||||||
|
FAST f=17 a=6 0.02453 4.040024 8 770
|
||||||
|
FAST f=17 a=7 2.894796 4.015884 8 818
|
||||||
|
FAST f=17 a=7 0.023956 4.015884 8 818
|
||||||
|
FAST f=17 a=8 2.789962 4.039303 8 530
|
||||||
|
FAST f=17 a=8 0.023219 4.039303 8 530
|
||||||
|
FAST f=17 a=9 2.787625 3.996762 8 1634
|
||||||
|
FAST f=17 a=9 0.023651 3.996762 8 1634
|
||||||
|
FAST f=17 a=10 2.754796 4.005059 8 1058
|
||||||
|
FAST f=17 a=10 0.022537 4.005059 8 1058
|
||||||
|
FAST f=18 a=1 4.779117 4.038214 8 242
|
||||||
|
FAST f=18 a=1 0.048814 4.038214 8 242
|
||||||
|
FAST f=18 a=2 3.829753 4.045768 8 722
|
||||||
|
FAST f=18 a=2 0.036541 4.045768 8 722
|
||||||
|
FAST f=18 a=3 3.495053 4.021497 8 770
|
||||||
|
FAST f=18 a=3 0.032648 4.021497 8 770
|
||||||
|
FAST f=18 a=4 3.221395 4.039623 8 770
|
||||||
|
FAST f=18 a=4 0.027818 4.039623 8 770
|
||||||
|
FAST f=18 a=5 3.059369 4.050414 8 530
|
||||||
|
FAST f=18 a=5 0.026296 4.050414 8 530
|
||||||
|
FAST f=18 a=6 3.019292 4.010714 6 962
|
||||||
|
FAST f=18 a=6 0.031104 4.010714 6 962
|
||||||
|
FAST f=18 a=7 2.949322 4.031439 6 770
|
||||||
|
FAST f=18 a=7 0.030745 4.031439 6 770
|
||||||
|
FAST f=18 a=8 2.876425 4.032088 6 386
|
||||||
|
FAST f=18 a=8 0.027407 4.032088 6 386
|
||||||
|
FAST f=18 a=9 2.850958 4.053372 8 674
|
||||||
|
FAST f=18 a=9 0.023799 4.053372 8 674
|
||||||
|
FAST f=18 a=10 2.884352 4.020148 8 1730
|
||||||
|
FAST f=18 a=10 0.024401 4.020148 8 1730
|
||||||
|
FAST f=19 a=1 4.815669 4.061203 8 674
|
||||||
|
FAST f=19 a=1 0.051425 4.061203 8 674
|
||||||
|
FAST f=19 a=2 3.951356 4.013822 8 1442
|
||||||
|
FAST f=19 a=2 0.039968 4.013822 8 1442
|
||||||
|
FAST f=19 a=3 3.554682 4.050425 8 722
|
||||||
|
FAST f=19 a=3 0.032725 4.050425 8 722
|
||||||
|
FAST f=19 a=4 3.242585 4.054677 8 722
|
||||||
|
FAST f=19 a=4 0.028194 4.054677 8 722
|
||||||
|
FAST f=19 a=5 3.105909 4.064524 8 818
|
||||||
|
FAST f=19 a=5 0.02675 4.064524 8 818
|
||||||
|
FAST f=19 a=6 3.059901 4.036857 8 1250
|
||||||
|
FAST f=19 a=6 0.026396 4.036857 8 1250
|
||||||
|
FAST f=19 a=7 3.016151 4.068234 6 770
|
||||||
|
FAST f=19 a=7 0.031501 4.068234 6 770
|
||||||
|
FAST f=19 a=8 2.962902 4.077509 8 530
|
||||||
|
FAST f=19 a=8 0.023333 4.077509 8 530
|
||||||
|
FAST f=19 a=9 2.899607 4.067328 8 530
|
||||||
|
FAST f=19 a=9 0.024553 4.067328 8 530
|
||||||
|
FAST f=19 a=10 2.950978 4.059901 8 434
|
||||||
|
FAST f=19 a=10 0.023852 4.059901 8 434
|
||||||
|
FAST f=20 a=1 5.259834 4.027579 8 1634
|
||||||
|
FAST f=20 a=1 0.061123 4.027579 8 1634
|
||||||
|
FAST f=20 a=2 4.382150 4.025093 8 1634
|
||||||
|
FAST f=20 a=2 0.048009 4.025093 8 1634
|
||||||
|
FAST f=20 a=3 4.104323 4.060842 8 530
|
||||||
|
FAST f=20 a=3 0.040965 4.060842 8 530
|
||||||
|
FAST f=20 a=4 3.853340 4.023504 6 914
|
||||||
|
FAST f=20 a=4 0.041072 4.023504 6 914
|
||||||
|
FAST f=20 a=5 3.728841 4.018089 6 1634
|
||||||
|
FAST f=20 a=5 0.037469 4.018089 6 1634
|
||||||
|
FAST f=20 a=6 3.683045 4.069138 8 578
|
||||||
|
FAST f=20 a=6 0.028011 4.069138 8 578
|
||||||
|
FAST f=20 a=7 3.726973 4.063160 8 722
|
||||||
|
FAST f=20 a=7 0.028437 4.063160 8 722
|
||||||
|
FAST f=20 a=8 3.555073 4.057690 8 386
|
||||||
|
FAST f=20 a=8 0.027588 4.057690 8 386
|
||||||
|
FAST f=20 a=9 3.551095 4.067253 8 482
|
||||||
|
FAST f=20 a=9 0.025976 4.067253 8 482
|
||||||
|
FAST f=20 a=10 3.490127 4.068518 8 530
|
||||||
|
FAST f=20 a=10 0.025971 4.068518 8 530
|
||||||
|
FAST f=21 a=1 7.343816 4.064945 8 770
|
||||||
|
FAST f=21 a=1 0.085035 4.064945 8 770
|
||||||
|
FAST f=21 a=2 5.930894 4.048206 8 386
|
||||||
|
FAST f=21 a=2 0.067349 4.048206 8 386
|
||||||
|
FAST f=21 a=3 6.770775 4.063417 8 578
|
||||||
|
FAST f=21 a=3 0.077104 4.063417 8 578
|
||||||
|
FAST f=21 a=4 6.889409 4.066761 8 626
|
||||||
|
FAST f=21 a=4 0.0717 4.066761 8 626
|
||||||
|
FAST f=21 a=5 6.714896 4.051813 8 914
|
||||||
|
FAST f=21 a=5 0.071026 4.051813 8 914
|
||||||
|
FAST f=21 a=6 6.539890 4.047263 8 1922
|
||||||
|
FAST f=21 a=6 0.07127 4.047263 8 1922
|
||||||
|
FAST f=21 a=7 6.511052 4.068373 8 482
|
||||||
|
FAST f=21 a=7 0.065467 4.068373 8 482
|
||||||
|
FAST f=21 a=8 6.458788 4.071597 8 482
|
||||||
|
FAST f=21 a=8 0.063817 4.071597 8 482
|
||||||
|
FAST f=21 a=9 6.377591 4.052905 8 434
|
||||||
|
FAST f=21 a=9 0.063112 4.052905 8 434
|
||||||
|
FAST f=21 a=10 6.360752 4.047773 8 530
|
||||||
|
FAST f=21 a=10 0.063606 4.047773 8 530
|
||||||
|
FAST f=22 a=1 10.523471 4.040812 8 962
|
||||||
|
FAST f=22 a=1 0.14214 4.040812 8 962
|
||||||
|
FAST f=22 a=2 9.454758 4.059396 8 914
|
||||||
|
FAST f=22 a=2 0.118343 4.059396 8 914
|
||||||
|
FAST f=22 a=3 9.043197 4.043019 8 1922
|
||||||
|
FAST f=22 a=3 0.109798 4.043019 8 1922
|
||||||
|
FAST f=22 a=4 8.716261 4.044819 8 770
|
||||||
|
FAST f=22 a=4 0.099687 4.044819 8 770
|
||||||
|
FAST f=22 a=5 8.529472 4.070576 8 530
|
||||||
|
FAST f=22 a=5 0.093127 4.070576 8 530
|
||||||
|
FAST f=22 a=6 8.424241 4.070565 8 722
|
||||||
|
FAST f=22 a=6 0.093703 4.070565 8 722
|
||||||
|
FAST f=22 a=7 8.403391 4.070591 8 578
|
||||||
|
FAST f=22 a=7 0.089763 4.070591 8 578
|
||||||
|
FAST f=22 a=8 8.285221 4.089171 8 530
|
||||||
|
FAST f=22 a=8 0.087716 4.089171 8 530
|
||||||
|
FAST f=22 a=9 8.282506 4.047470 8 722
|
||||||
|
FAST f=22 a=9 0.089773 4.047470 8 722
|
||||||
|
FAST f=22 a=10 8.241809 4.064151 8 818
|
||||||
|
FAST f=22 a=10 0.090413 4.064151 8 818
|
||||||
|
FAST f=23 a=1 12.389208 4.051635 6 530
|
||||||
|
FAST f=23 a=1 0.147796 4.051635 6 530
|
||||||
|
FAST f=23 a=2 11.300910 4.042835 6 914
|
||||||
|
FAST f=23 a=2 0.133178 4.042835 6 914
|
||||||
|
FAST f=23 a=3 10.879455 4.047415 8 626
|
||||||
|
FAST f=23 a=3 0.129571 4.047415 8 626
|
||||||
|
FAST f=23 a=4 10.522718 4.038269 6 914
|
||||||
|
FAST f=23 a=4 0.118121 4.038269 6 914
|
||||||
|
FAST f=23 a=5 10.348043 4.066884 8 434
|
||||||
|
FAST f=23 a=5 0.112098 4.066884 8 434
|
||||||
|
FAST f=23 a=6 10.238630 4.048635 8 1010
|
||||||
|
FAST f=23 a=6 0.120281 4.048635 8 1010
|
||||||
|
FAST f=23 a=7 10.213255 4.061809 8 530
|
||||||
|
FAST f=23 a=7 0.1121 4.061809 8 530
|
||||||
|
FAST f=23 a=8 10.107879 4.074104 8 818
|
||||||
|
FAST f=23 a=8 0.116544 4.074104 8 818
|
||||||
|
FAST f=23 a=9 10.063424 4.064811 8 674
|
||||||
|
FAST f=23 a=9 0.109045 4.064811 8 674
|
||||||
|
FAST f=23 a=10 10.035801 4.054918 8 530
|
||||||
|
FAST f=23 a=10 0.108735 4.054918 8 530
|
||||||
|
FAST f=24 a=1 14.963878 4.073490 8 722
|
||||||
|
FAST f=24 a=1 0.206344 4.073490 8 722
|
||||||
|
FAST f=24 a=2 13.833472 4.036100 8 962
|
||||||
|
FAST f=24 a=2 0.17486 4.036100 8 962
|
||||||
|
FAST f=24 a=3 13.404631 4.026281 6 1106
|
||||||
|
FAST f=24 a=3 0.153961 4.026281 6 1106
|
||||||
|
FAST f=24 a=4 13.041164 4.065448 8 674
|
||||||
|
FAST f=24 a=4 0.155509 4.065448 8 674
|
||||||
|
FAST f=24 a=5 12.879412 4.054636 8 674
|
||||||
|
FAST f=24 a=5 0.148282 4.054636 8 674
|
||||||
|
FAST f=24 a=6 12.773736 4.081376 8 530
|
||||||
|
FAST f=24 a=6 0.142563 4.081376 8 530
|
||||||
|
FAST f=24 a=7 12.711310 4.059834 8 770
|
||||||
|
FAST f=24 a=7 0.149321 4.059834 8 770
|
||||||
|
FAST f=24 a=8 12.635459 4.052050 8 1298
|
||||||
|
FAST f=24 a=8 0.15095 4.052050 8 1298
|
||||||
|
FAST f=24 a=9 12.558104 4.076516 8 722
|
||||||
|
FAST f=24 a=9 0.144361 4.076516 8 722
|
||||||
|
FAST f=24 a=10 10.661348 4.062137 8 818
|
||||||
|
FAST f=24 a=10 0.108232 4.062137 8 818
|
||||||
|
|
||||||
|
|
||||||
hg-changelog:
|
hg-changelog:
|
||||||
NODICT 0.000007 1.377613
|
NODICT 0.000017 1.377590
|
||||||
RANDOM 0.297345 2.097487
|
RANDOM 0.186171 2.097487
|
||||||
LEGACY 2.633992 2.058907
|
LEGACY 1.670867 2.058907
|
||||||
COVER 219.179786 2.189685 8 98
|
COVER 173.561948 2.189685 8 98
|
||||||
COVER 6.620852 2.189685 8 98
|
COVER 4.811180 2.189685 8 98
|
||||||
FAST15 47.635082 2.130794 6 386
|
FAST f=15 a=1 18.685906 2.129682 8 434
|
||||||
FAST15 0.321297 2.130794 6 386
|
FAST f=15 a=1 0.173376 2.129682 8 434
|
||||||
FAST16 43.837676 2.144845 8 194
|
FAST f=15 a=2 12.928259 2.131890 8 482
|
||||||
FAST16 0.312640 2.144845 8 194
|
FAST f=15 a=2 0.102582 2.131890 8 482
|
||||||
FAST17 49.349017 2.156099 8 242
|
FAST f=15 a=3 11.132343 2.128027 8 386
|
||||||
FAST17 0.348459 2.156099 8 242
|
FAST f=15 a=3 0.077122 2.128027 8 386
|
||||||
FAST18 51.153784 2.172439 6 98
|
FAST f=15 a=4 10.120683 2.125797 8 434
|
||||||
FAST18 0.353106 2.172439 6 98
|
FAST f=15 a=4 0.065175 2.125797 8 434
|
||||||
FAST19 52.627045 2.180321 6 98
|
FAST f=15 a=5 9.479092 2.127697 8 386
|
||||||
FAST19 0.390612 2.180321 6 98
|
FAST f=15 a=5 0.057905 2.127697 8 386
|
||||||
FAST20 63.748782 2.187431 6 98
|
FAST f=15 a=6 9.159523 2.127132 8 1682
|
||||||
FAST20 0.489544 2.187431 6 98
|
FAST f=15 a=6 0.058604 2.127132 8 1682
|
||||||
FAST21 68.709198 2.184185 6 146
|
FAST f=15 a=7 8.724003 2.129914 8 434
|
||||||
FAST21 0.530852 2.184185 6 146
|
FAST f=15 a=7 0.0493 2.129914 8 434
|
||||||
FAST22 68.491639 2.182830 6 98
|
FAST f=15 a=8 8.595001 2.127137 8 338
|
||||||
FAST22 0.645699 2.182830 6 98
|
FAST f=15 a=8 0.0474 2.127137 8 338
|
||||||
FAST23 72.558688 2.186399 8 98
|
FAST f=15 a=9 8.356405 2.125512 8 482
|
||||||
FAST23 0.593539 2.186399 8 98
|
FAST f=15 a=9 0.046126 2.125512 8 482
|
||||||
FAST24 76.137195 2.185608 6 98
|
FAST f=15 a=10 8.207111 2.126066 8 338
|
||||||
FAST24 0.680132 2.185608 6 98
|
FAST f=15 a=10 0.043292 2.126066 8 338
|
||||||
|
FAST f=16 a=1 18.464436 2.144040 8 242
|
||||||
|
FAST f=16 a=1 0.172156 2.144040 8 242
|
||||||
|
FAST f=16 a=2 12.844825 2.148171 8 194
|
||||||
|
FAST f=16 a=2 0.099619 2.148171 8 194
|
||||||
|
FAST f=16 a=3 11.082568 2.140837 8 290
|
||||||
|
FAST f=16 a=3 0.079165 2.140837 8 290
|
||||||
|
FAST f=16 a=4 10.066749 2.144405 8 386
|
||||||
|
FAST f=16 a=4 0.068411 2.144405 8 386
|
||||||
|
FAST f=16 a=5 9.501121 2.140720 8 386
|
||||||
|
FAST f=16 a=5 0.061316 2.140720 8 386
|
||||||
|
FAST f=16 a=6 9.179332 2.139478 8 386
|
||||||
|
FAST f=16 a=6 0.056322 2.139478 8 386
|
||||||
|
FAST f=16 a=7 8.849438 2.142412 8 194
|
||||||
|
FAST f=16 a=7 0.050493 2.142412 8 194
|
||||||
|
FAST f=16 a=8 8.810919 2.143454 8 434
|
||||||
|
FAST f=16 a=8 0.051304 2.143454 8 434
|
||||||
|
FAST f=16 a=9 8.553900 2.140339 8 194
|
||||||
|
FAST f=16 a=9 0.047285 2.140339 8 194
|
||||||
|
FAST f=16 a=10 8.398027 2.143130 8 386
|
||||||
|
FAST f=16 a=10 0.046386 2.143130 8 386
|
||||||
|
FAST f=17 a=1 18.644657 2.157192 8 98
|
||||||
|
FAST f=17 a=1 0.173884 2.157192 8 98
|
||||||
|
FAST f=17 a=2 13.071242 2.159830 8 146
|
||||||
|
FAST f=17 a=2 0.10388 2.159830 8 146
|
||||||
|
FAST f=17 a=3 11.332366 2.153654 6 194
|
||||||
|
FAST f=17 a=3 0.08983 2.153654 6 194
|
||||||
|
FAST f=17 a=4 10.362413 2.156813 8 242
|
||||||
|
FAST f=17 a=4 0.070389 2.156813 8 242
|
||||||
|
FAST f=17 a=5 9.808159 2.155098 6 338
|
||||||
|
FAST f=17 a=5 0.072661 2.155098 6 338
|
||||||
|
FAST f=17 a=6 9.451165 2.153845 6 146
|
||||||
|
FAST f=17 a=6 0.064959 2.153845 6 146
|
||||||
|
FAST f=17 a=7 9.163097 2.155424 6 242
|
||||||
|
FAST f=17 a=7 0.064323 2.155424 6 242
|
||||||
|
FAST f=17 a=8 9.047276 2.156640 8 242
|
||||||
|
FAST f=17 a=8 0.053382 2.156640 8 242
|
||||||
|
FAST f=17 a=9 8.807671 2.152396 8 146
|
||||||
|
FAST f=17 a=9 0.049617 2.152396 8 146
|
||||||
|
FAST f=17 a=10 8.649827 2.152370 8 146
|
||||||
|
FAST f=17 a=10 0.047849 2.152370 8 146
|
||||||
|
FAST f=18 a=1 18.809502 2.168116 8 98
|
||||||
|
FAST f=18 a=1 0.175226 2.168116 8 98
|
||||||
|
FAST f=18 a=2 13.756502 2.170870 6 242
|
||||||
|
FAST f=18 a=2 0.119507 2.170870 6 242
|
||||||
|
FAST f=18 a=3 12.059748 2.163094 6 98
|
||||||
|
FAST f=18 a=3 0.093912 2.163094 6 98
|
||||||
|
FAST f=18 a=4 11.410294 2.172372 8 98
|
||||||
|
FAST f=18 a=4 0.073048 2.172372 8 98
|
||||||
|
FAST f=18 a=5 10.560297 2.166388 8 98
|
||||||
|
FAST f=18 a=5 0.065136 2.166388 8 98
|
||||||
|
FAST f=18 a=6 10.071390 2.162672 8 98
|
||||||
|
FAST f=18 a=6 0.059402 2.162672 8 98
|
||||||
|
FAST f=18 a=7 10.084214 2.166624 6 194
|
||||||
|
FAST f=18 a=7 0.073276 2.166624 6 194
|
||||||
|
FAST f=18 a=8 9.953226 2.167454 8 98
|
||||||
|
FAST f=18 a=8 0.053659 2.167454 8 98
|
||||||
|
FAST f=18 a=9 8.982461 2.161593 6 146
|
||||||
|
FAST f=18 a=9 0.05955 2.161593 6 146
|
||||||
|
FAST f=18 a=10 8.986092 2.164373 6 242
|
||||||
|
FAST f=18 a=10 0.059135 2.164373 6 242
|
||||||
|
FAST f=19 a=1 18.908277 2.176021 8 98
|
||||||
|
FAST f=19 a=1 0.177316 2.176021 8 98
|
||||||
|
FAST f=19 a=2 13.471313 2.176103 8 98
|
||||||
|
FAST f=19 a=2 0.106344 2.176103 8 98
|
||||||
|
FAST f=19 a=3 11.571406 2.172812 8 98
|
||||||
|
FAST f=19 a=3 0.083293 2.172812 8 98
|
||||||
|
FAST f=19 a=4 10.632775 2.177770 6 146
|
||||||
|
FAST f=19 a=4 0.079864 2.177770 6 146
|
||||||
|
FAST f=19 a=5 10.030190 2.175574 6 146
|
||||||
|
FAST f=19 a=5 0.07223 2.175574 6 146
|
||||||
|
FAST f=19 a=6 9.717818 2.169997 8 98
|
||||||
|
FAST f=19 a=6 0.060049 2.169997 8 98
|
||||||
|
FAST f=19 a=7 9.397531 2.172770 8 146
|
||||||
|
FAST f=19 a=7 0.057188 2.172770 8 146
|
||||||
|
FAST f=19 a=8 9.281061 2.175822 8 98
|
||||||
|
FAST f=19 a=8 0.053711 2.175822 8 98
|
||||||
|
FAST f=19 a=9 9.165242 2.169849 6 146
|
||||||
|
FAST f=19 a=9 0.059898 2.169849 6 146
|
||||||
|
FAST f=19 a=10 9.048763 2.173394 8 98
|
||||||
|
FAST f=19 a=10 0.049757 2.173394 8 98
|
||||||
|
FAST f=20 a=1 21.166917 2.183923 6 98
|
||||||
|
FAST f=20 a=1 0.205425 2.183923 6 98
|
||||||
|
FAST f=20 a=2 15.642753 2.182349 6 98
|
||||||
|
FAST f=20 a=2 0.135957 2.182349 6 98
|
||||||
|
FAST f=20 a=3 14.053730 2.173544 6 98
|
||||||
|
FAST f=20 a=3 0.11266 2.173544 6 98
|
||||||
|
FAST f=20 a=4 15.270019 2.183656 8 98
|
||||||
|
FAST f=20 a=4 0.107892 2.183656 8 98
|
||||||
|
FAST f=20 a=5 15.497927 2.174661 6 98
|
||||||
|
FAST f=20 a=5 0.100305 2.174661 6 98
|
||||||
|
FAST f=20 a=6 13.973505 2.172391 8 98
|
||||||
|
FAST f=20 a=6 0.087565 2.172391 8 98
|
||||||
|
FAST f=20 a=7 14.083296 2.172443 8 98
|
||||||
|
FAST f=20 a=7 0.078062 2.172443 8 98
|
||||||
|
FAST f=20 a=8 12.560048 2.175581 8 98
|
||||||
|
FAST f=20 a=8 0.070282 2.175581 8 98
|
||||||
|
FAST f=20 a=9 13.078645 2.173975 6 146
|
||||||
|
FAST f=20 a=9 0.081041 2.173975 6 146
|
||||||
|
FAST f=20 a=10 12.823328 2.177778 8 98
|
||||||
|
FAST f=20 a=10 0.074522 2.177778 8 98
|
||||||
|
FAST f=21 a=1 29.825370 2.183057 6 98
|
||||||
|
FAST f=21 a=1 0.334453 2.183057 6 98
|
||||||
|
FAST f=21 a=2 29.476474 2.182752 8 98
|
||||||
|
FAST f=21 a=2 0.286602 2.182752 8 98
|
||||||
|
FAST f=21 a=3 25.937186 2.175867 8 98
|
||||||
|
FAST f=21 a=3 0.17626 2.175867 8 98
|
||||||
|
FAST f=21 a=4 20.413865 2.179780 8 98
|
||||||
|
FAST f=21 a=4 0.206085 2.179780 8 98
|
||||||
|
FAST f=21 a=5 20.541889 2.178328 6 146
|
||||||
|
FAST f=21 a=5 0.199157 2.178328 6 146
|
||||||
|
FAST f=21 a=6 21.090670 2.174443 6 146
|
||||||
|
FAST f=21 a=6 0.190645 2.174443 6 146
|
||||||
|
FAST f=21 a=7 20.221569 2.177384 6 146
|
||||||
|
FAST f=21 a=7 0.184278 2.177384 6 146
|
||||||
|
FAST f=21 a=8 20.322357 2.179456 6 98
|
||||||
|
FAST f=21 a=8 0.178458 2.179456 6 98
|
||||||
|
FAST f=21 a=9 20.683912 2.174396 6 146
|
||||||
|
FAST f=21 a=9 0.190829 2.174396 6 146
|
||||||
|
FAST f=21 a=10 20.840865 2.174905 8 98
|
||||||
|
FAST f=21 a=10 0.172515 2.174905 8 98
|
||||||
|
FAST f=22 a=1 36.822827 2.181612 6 98
|
||||||
|
FAST f=22 a=1 0.437389 2.181612 6 98
|
||||||
|
FAST f=22 a=2 30.616902 2.183142 8 98
|
||||||
|
FAST f=22 a=2 0.324284 2.183142 8 98
|
||||||
|
FAST f=22 a=3 28.472482 2.178130 8 98
|
||||||
|
FAST f=22 a=3 0.236538 2.178130 8 98
|
||||||
|
FAST f=22 a=4 25.847028 2.181878 8 98
|
||||||
|
FAST f=22 a=4 0.263744 2.181878 8 98
|
||||||
|
FAST f=22 a=5 27.095881 2.180775 8 98
|
||||||
|
FAST f=22 a=5 0.24988 2.180775 8 98
|
||||||
|
FAST f=22 a=6 25.939172 2.170916 8 98
|
||||||
|
FAST f=22 a=6 0.240033 2.170916 8 98
|
||||||
|
FAST f=22 a=7 27.064194 2.177849 8 98
|
||||||
|
FAST f=22 a=7 0.242383 2.177849 8 98
|
||||||
|
FAST f=22 a=8 25.140221 2.178216 8 98
|
||||||
|
FAST f=22 a=8 0.237601 2.178216 8 98
|
||||||
|
FAST f=22 a=9 25.505283 2.177455 6 146
|
||||||
|
FAST f=22 a=9 0.223217 2.177455 6 146
|
||||||
|
FAST f=22 a=10 24.529362 2.176705 6 98
|
||||||
|
FAST f=22 a=10 0.222876 2.176705 6 98
|
||||||
|
FAST f=23 a=1 39.127310 2.183006 6 98
|
||||||
|
FAST f=23 a=1 0.417338 2.183006 6 98
|
||||||
|
FAST f=23 a=2 32.468161 2.183524 6 98
|
||||||
|
FAST f=23 a=2 0.351645 2.183524 6 98
|
||||||
|
FAST f=23 a=3 31.577620 2.172604 6 98
|
||||||
|
FAST f=23 a=3 0.319659 2.172604 6 98
|
||||||
|
FAST f=23 a=4 30.129247 2.183932 6 98
|
||||||
|
FAST f=23 a=4 0.307239 2.183932 6 98
|
||||||
|
FAST f=23 a=5 29.103376 2.183529 6 146
|
||||||
|
FAST f=23 a=5 0.285533 2.183529 6 146
|
||||||
|
FAST f=23 a=6 29.776045 2.174367 8 98
|
||||||
|
FAST f=23 a=6 0.276846 2.174367 8 98
|
||||||
|
FAST f=23 a=7 28.940407 2.178022 6 146
|
||||||
|
FAST f=23 a=7 0.274082 2.178022 6 146
|
||||||
|
FAST f=23 a=8 29.256009 2.179462 6 98
|
||||||
|
FAST f=23 a=8 0.26949 2.179462 6 98
|
||||||
|
FAST f=23 a=9 29.347312 2.170407 8 98
|
||||||
|
FAST f=23 a=9 0.265034 2.170407 8 98
|
||||||
|
FAST f=23 a=10 29.140081 2.171762 8 98
|
||||||
|
FAST f=23 a=10 0.259183 2.171762 8 98
|
||||||
|
FAST f=24 a=1 44.871179 2.182115 6 98
|
||||||
|
FAST f=24 a=1 0.509433 2.182115 6 98
|
||||||
|
FAST f=24 a=2 38.694867 2.180549 8 98
|
||||||
|
FAST f=24 a=2 0.406695 2.180549 8 98
|
||||||
|
FAST f=24 a=3 38.363769 2.172821 8 98
|
||||||
|
FAST f=24 a=3 0.359581 2.172821 8 98
|
||||||
|
FAST f=24 a=4 36.580797 2.184142 8 98
|
||||||
|
FAST f=24 a=4 0.340614 2.184142 8 98
|
||||||
|
FAST f=24 a=5 33.125701 2.183301 8 98
|
||||||
|
FAST f=24 a=5 0.324874 2.183301 8 98
|
||||||
|
FAST f=24 a=6 34.776068 2.173019 6 146
|
||||||
|
FAST f=24 a=6 0.340397 2.173019 6 146
|
||||||
|
FAST f=24 a=7 34.417625 2.176561 6 146
|
||||||
|
FAST f=24 a=7 0.308223 2.176561 6 146
|
||||||
|
FAST f=24 a=8 35.470291 2.182161 6 98
|
||||||
|
FAST f=24 a=8 0.307724 2.182161 6 98
|
||||||
|
FAST f=24 a=9 34.927252 2.172682 6 146
|
||||||
|
FAST f=24 a=9 0.300598 2.172682 6 146
|
||||||
|
FAST f=24 a=10 33.238355 2.173395 6 98
|
||||||
|
FAST f=24 a=10 0.249916 2.173395 6 98
|
||||||
|
|
||||||
|
|
||||||
hg-manifest:
|
hg-manifest:
|
||||||
NODICT 0.000026 1.866385
|
NODICT 0.000004 1.866377
|
||||||
RANDOM 0.784554 2.309436
|
RANDOM 0.696346 2.309436
|
||||||
LEGACY 10.193714 2.506977
|
LEGACY 7.064527 2.506977
|
||||||
COVER 988.206583 2.582528 8 434
|
COVER 876.312865 2.582528 8 434
|
||||||
COVER 39.726199 2.582528 8 434
|
COVER 35.684533 2.582528 8 434
|
||||||
FAST15 168.388819 2.392920 6 1826
|
FAST f=15 a=1 76.618201 2.404013 8 1202
|
||||||
FAST15 1.272178 2.392920 6 1826
|
FAST f=15 a=1 0.700722 2.404013 8 1202
|
||||||
FAST16 161.822607 2.480762 6 1922
|
FAST f=15 a=2 49.213058 2.409248 6 1826
|
||||||
FAST16 1.164908 2.480762 6 1922
|
FAST f=15 a=2 0.473393 2.409248 6 1826
|
||||||
FAST17 157.688544 2.548285 6 1682
|
FAST f=15 a=3 41.753197 2.409677 8 1490
|
||||||
FAST17 1.222439 2.548285 6 1682
|
FAST f=15 a=3 0.336848 2.409677 8 1490
|
||||||
FAST18 154.529585 2.567634 6 386
|
FAST f=15 a=4 38.648295 2.407996 8 1538
|
||||||
FAST18 1.217596 2.567634 6 386
|
FAST f=15 a=4 0.283952 2.407996 8 1538
|
||||||
FAST19 160.244979 2.581653 8 338
|
FAST f=15 a=5 36.144936 2.402895 8 1874
|
||||||
FAST19 1.282450 2.581653 8 338
|
FAST f=15 a=5 0.270128 2.402895 8 1874
|
||||||
FAST20 191.503297 2.586881 8 194
|
FAST f=15 a=6 35.484675 2.394873 8 1586
|
||||||
FAST20 2.009748 2.586881 8 194
|
FAST f=15 a=6 0.251637 2.394873 8 1586
|
||||||
FAST21 226.389709 2.590051 6 242
|
FAST f=15 a=7 34.280599 2.397311 8 1778
|
||||||
FAST21 2.494543 2.590051 6 242
|
FAST f=15 a=7 0.23984 2.397311 8 1778
|
||||||
FAST22 217.859055 2.591376 6 194
|
FAST f=15 a=8 32.122572 2.396089 6 1490
|
||||||
FAST22 2.295693 2.591376 6 194
|
FAST f=15 a=8 0.251508 2.396089 6 1490
|
||||||
FAST23 236.819791 2.591131 8 434
|
FAST f=15 a=9 29.909842 2.390092 6 1970
|
||||||
FAST23 2.744711 2.591131 8 434
|
FAST f=15 a=9 0.251233 2.390092 6 1970
|
||||||
FAST24 269.187800 2.591548 6 290
|
FAST f=15 a=10 30.102938 2.400086 6 1682
|
||||||
FAST24 2.923671 2.591548 6 290
|
FAST f=15 a=10 0.23688 2.400086 6 1682
|
||||||
|
FAST f=16 a=1 67.750401 2.475460 6 1346
|
||||||
|
FAST f=16 a=1 0.796035 2.475460 6 1346
|
||||||
|
FAST f=16 a=2 52.812027 2.480860 6 1730
|
||||||
|
FAST f=16 a=2 0.480384 2.480860 6 1730
|
||||||
|
FAST f=16 a=3 44.179259 2.469304 8 1970
|
||||||
|
FAST f=16 a=3 0.332657 2.469304 8 1970
|
||||||
|
FAST f=16 a=4 37.612728 2.478208 6 1970
|
||||||
|
FAST f=16 a=4 0.32498 2.478208 6 1970
|
||||||
|
FAST f=16 a=5 35.056222 2.475568 6 1298
|
||||||
|
FAST f=16 a=5 0.302824 2.475568 6 1298
|
||||||
|
FAST f=16 a=6 34.713012 2.486079 8 1730
|
||||||
|
FAST f=16 a=6 0.24755 2.486079 8 1730
|
||||||
|
FAST f=16 a=7 33.713687 2.477180 6 1682
|
||||||
|
FAST f=16 a=7 0.280358 2.477180 6 1682
|
||||||
|
FAST f=16 a=8 31.571412 2.475418 8 1538
|
||||||
|
FAST f=16 a=8 0.241241 2.475418 8 1538
|
||||||
|
FAST f=16 a=9 31.608069 2.478263 8 1922
|
||||||
|
FAST f=16 a=9 0.241764 2.478263 8 1922
|
||||||
|
FAST f=16 a=10 31.358002 2.472263 8 1442
|
||||||
|
FAST f=16 a=10 0.221661 2.472263 8 1442
|
||||||
|
FAST f=17 a=1 66.185775 2.536085 6 1346
|
||||||
|
FAST f=17 a=1 0.713549 2.536085 6 1346
|
||||||
|
FAST f=17 a=2 50.365000 2.546105 8 1298
|
||||||
|
FAST f=17 a=2 0.467846 2.546105 8 1298
|
||||||
|
FAST f=17 a=3 42.712843 2.536250 8 1298
|
||||||
|
FAST f=17 a=3 0.34047 2.536250 8 1298
|
||||||
|
FAST f=17 a=4 39.514227 2.535555 8 1442
|
||||||
|
FAST f=17 a=4 0.302989 2.535555 8 1442
|
||||||
|
FAST f=17 a=5 35.189292 2.524925 8 1202
|
||||||
|
FAST f=17 a=5 0.273451 2.524925 8 1202
|
||||||
|
FAST f=17 a=6 35.791683 2.523466 8 1202
|
||||||
|
FAST f=17 a=6 0.268261 2.523466 8 1202
|
||||||
|
FAST f=17 a=7 37.416136 2.526625 6 1010
|
||||||
|
FAST f=17 a=7 0.277558 2.526625 6 1010
|
||||||
|
FAST f=17 a=8 37.084707 2.533274 6 1250
|
||||||
|
FAST f=17 a=8 0.285104 2.533274 6 1250
|
||||||
|
FAST f=17 a=9 34.183814 2.532765 8 1298
|
||||||
|
FAST f=17 a=9 0.235133 2.532765 8 1298
|
||||||
|
FAST f=17 a=10 31.149235 2.528722 8 1346
|
||||||
|
FAST f=17 a=10 0.232679 2.528722 8 1346
|
||||||
|
FAST f=18 a=1 72.942176 2.559857 6 386
|
||||||
|
FAST f=18 a=1 0.718618 2.559857 6 386
|
||||||
|
FAST f=18 a=2 51.690440 2.559572 8 290
|
||||||
|
FAST f=18 a=2 0.403978 2.559572 8 290
|
||||||
|
FAST f=18 a=3 45.344908 2.561040 8 962
|
||||||
|
FAST f=18 a=3 0.357205 2.561040 8 962
|
||||||
|
FAST f=18 a=4 39.804522 2.558446 8 1010
|
||||||
|
FAST f=18 a=4 0.310526 2.558446 8 1010
|
||||||
|
FAST f=18 a=5 38.134888 2.561811 8 626
|
||||||
|
FAST f=18 a=5 0.273743 2.561811 8 626
|
||||||
|
FAST f=18 a=6 35.091890 2.555518 8 722
|
||||||
|
FAST f=18 a=6 0.260135 2.555518 8 722
|
||||||
|
FAST f=18 a=7 34.639523 2.562938 8 290
|
||||||
|
FAST f=18 a=7 0.234294 2.562938 8 290
|
||||||
|
FAST f=18 a=8 36.076431 2.563567 8 1586
|
||||||
|
FAST f=18 a=8 0.274075 2.563567 8 1586
|
||||||
|
FAST f=18 a=9 36.376433 2.560950 8 722
|
||||||
|
FAST f=18 a=9 0.240106 2.560950 8 722
|
||||||
|
FAST f=18 a=10 32.624790 2.559340 8 578
|
||||||
|
FAST f=18 a=10 0.234704 2.559340 8 578
|
||||||
|
FAST f=19 a=1 70.513761 2.572441 8 194
|
||||||
|
FAST f=19 a=1 0.726112 2.572441 8 194
|
||||||
|
FAST f=19 a=2 59.263032 2.574560 8 482
|
||||||
|
FAST f=19 a=2 0.451554 2.574560 8 482
|
||||||
|
FAST f=19 a=3 51.509594 2.571546 6 194
|
||||||
|
FAST f=19 a=3 0.393014 2.571546 6 194
|
||||||
|
FAST f=19 a=4 55.393906 2.573386 8 482
|
||||||
|
FAST f=19 a=4 0.38819 2.573386 8 482
|
||||||
|
FAST f=19 a=5 43.201736 2.567589 8 674
|
||||||
|
FAST f=19 a=5 0.292155 2.567589 8 674
|
||||||
|
FAST f=19 a=6 42.911687 2.572666 6 434
|
||||||
|
FAST f=19 a=6 0.303988 2.572666 6 434
|
||||||
|
FAST f=19 a=7 44.687591 2.573613 6 290
|
||||||
|
FAST f=19 a=7 0.308721 2.573613 6 290
|
||||||
|
FAST f=19 a=8 37.372868 2.571039 6 194
|
||||||
|
FAST f=19 a=8 0.287137 2.571039 6 194
|
||||||
|
FAST f=19 a=9 36.074230 2.566473 6 482
|
||||||
|
FAST f=19 a=9 0.280721 2.566473 6 482
|
||||||
|
FAST f=19 a=10 33.731720 2.570306 8 194
|
||||||
|
FAST f=19 a=10 0.224073 2.570306 8 194
|
||||||
|
FAST f=20 a=1 79.670634 2.581146 6 290
|
||||||
|
FAST f=20 a=1 0.899986 2.581146 6 290
|
||||||
|
FAST f=20 a=2 58.827141 2.579782 8 386
|
||||||
|
FAST f=20 a=2 0.602288 2.579782 8 386
|
||||||
|
FAST f=20 a=3 51.289004 2.579627 8 722
|
||||||
|
FAST f=20 a=3 0.446091 2.579627 8 722
|
||||||
|
FAST f=20 a=4 47.711068 2.581508 8 722
|
||||||
|
FAST f=20 a=4 0.473007 2.581508 8 722
|
||||||
|
FAST f=20 a=5 47.402929 2.578062 6 434
|
||||||
|
FAST f=20 a=5 0.497131 2.578062 6 434
|
||||||
|
FAST f=20 a=6 54.797102 2.577365 8 482
|
||||||
|
FAST f=20 a=6 0.515061 2.577365 8 482
|
||||||
|
FAST f=20 a=7 51.370877 2.583050 8 386
|
||||||
|
FAST f=20 a=7 0.402878 2.583050 8 386
|
||||||
|
FAST f=20 a=8 51.437931 2.574875 6 242
|
||||||
|
FAST f=20 a=8 0.453094 2.574875 6 242
|
||||||
|
FAST f=20 a=9 44.105456 2.576700 6 242
|
||||||
|
FAST f=20 a=9 0.456633 2.576700 6 242
|
||||||
|
FAST f=20 a=10 44.447580 2.578305 8 338
|
||||||
|
FAST f=20 a=10 0.409121 2.578305 8 338
|
||||||
|
FAST f=21 a=1 113.031686 2.582449 6 242
|
||||||
|
FAST f=21 a=1 1.456971 2.582449 6 242
|
||||||
|
FAST f=21 a=2 97.700932 2.582124 8 194
|
||||||
|
FAST f=21 a=2 1.072078 2.582124 8 194
|
||||||
|
FAST f=21 a=3 96.563648 2.585479 8 434
|
||||||
|
FAST f=21 a=3 0.949528 2.585479 8 434
|
||||||
|
FAST f=21 a=4 90.597813 2.582366 6 386
|
||||||
|
FAST f=21 a=4 0.76944 2.582366 6 386
|
||||||
|
FAST f=21 a=5 86.815980 2.579043 8 434
|
||||||
|
FAST f=21 a=5 0.858167 2.579043 8 434
|
||||||
|
FAST f=21 a=6 91.235820 2.578378 8 530
|
||||||
|
FAST f=21 a=6 0.684274 2.578378 8 530
|
||||||
|
FAST f=21 a=7 84.392788 2.581243 8 386
|
||||||
|
FAST f=21 a=7 0.814386 2.581243 8 386
|
||||||
|
FAST f=21 a=8 82.052310 2.582547 8 338
|
||||||
|
FAST f=21 a=8 0.822633 2.582547 8 338
|
||||||
|
FAST f=21 a=9 74.696074 2.579319 8 194
|
||||||
|
FAST f=21 a=9 0.811028 2.579319 8 194
|
||||||
|
FAST f=21 a=10 76.211170 2.578766 8 290
|
||||||
|
FAST f=21 a=10 0.809715 2.578766 8 290
|
||||||
|
FAST f=22 a=1 138.976871 2.580478 8 194
|
||||||
|
FAST f=22 a=1 1.748932 2.580478 8 194
|
||||||
|
FAST f=22 a=2 120.164097 2.583633 8 386
|
||||||
|
FAST f=22 a=2 1.333239 2.583633 8 386
|
||||||
|
FAST f=22 a=3 111.986474 2.582566 6 194
|
||||||
|
FAST f=22 a=3 1.305734 2.582566 6 194
|
||||||
|
FAST f=22 a=4 108.548148 2.583068 6 194
|
||||||
|
FAST f=22 a=4 1.314026 2.583068 6 194
|
||||||
|
FAST f=22 a=5 103.173017 2.583495 6 290
|
||||||
|
FAST f=22 a=5 1.228664 2.583495 6 290
|
||||||
|
FAST f=22 a=6 108.421262 2.582349 8 530
|
||||||
|
FAST f=22 a=6 1.076773 2.582349 8 530
|
||||||
|
FAST f=22 a=7 103.284127 2.581022 8 386
|
||||||
|
FAST f=22 a=7 1.112117 2.581022 8 386
|
||||||
|
FAST f=22 a=8 96.330279 2.581073 8 290
|
||||||
|
FAST f=22 a=8 1.109303 2.581073 8 290
|
||||||
|
FAST f=22 a=9 97.651348 2.580075 6 194
|
||||||
|
FAST f=22 a=9 0.933032 2.580075 6 194
|
||||||
|
FAST f=22 a=10 101.660621 2.584886 8 194
|
||||||
|
FAST f=22 a=10 0.796823 2.584886 8 194
|
||||||
|
FAST f=23 a=1 159.322978 2.581474 6 242
|
||||||
|
FAST f=23 a=1 2.015878 2.581474 6 242
|
||||||
|
FAST f=23 a=2 134.331775 2.581619 8 194
|
||||||
|
FAST f=23 a=2 1.545845 2.581619 8 194
|
||||||
|
FAST f=23 a=3 127.724552 2.579888 6 338
|
||||||
|
FAST f=23 a=3 1.444496 2.579888 6 338
|
||||||
|
FAST f=23 a=4 126.077675 2.578137 6 242
|
||||||
|
FAST f=23 a=4 1.364394 2.578137 6 242
|
||||||
|
FAST f=23 a=5 124.914027 2.580843 8 338
|
||||||
|
FAST f=23 a=5 1.116059 2.580843 8 338
|
||||||
|
FAST f=23 a=6 122.874153 2.577637 6 338
|
||||||
|
FAST f=23 a=6 1.164584 2.577637 6 338
|
||||||
|
FAST f=23 a=7 123.099257 2.582715 6 386
|
||||||
|
FAST f=23 a=7 1.354042 2.582715 6 386
|
||||||
|
FAST f=23 a=8 122.026753 2.577681 8 194
|
||||||
|
FAST f=23 a=8 1.210966 2.577681 8 194
|
||||||
|
FAST f=23 a=9 121.164312 2.584599 6 290
|
||||||
|
FAST f=23 a=9 1.174859 2.584599 6 290
|
||||||
|
FAST f=23 a=10 117.462222 2.580358 8 194
|
||||||
|
FAST f=23 a=10 1.075258 2.580358 8 194
|
||||||
|
FAST f=24 a=1 169.539659 2.581642 6 194
|
||||||
|
FAST f=24 a=1 1.916804 2.581642 6 194
|
||||||
|
FAST f=24 a=2 160.539270 2.580421 6 290
|
||||||
|
FAST f=24 a=2 1.71087 2.580421 6 290
|
||||||
|
FAST f=24 a=3 155.455874 2.580449 6 242
|
||||||
|
FAST f=24 a=3 1.60307 2.580449 6 242
|
||||||
|
FAST f=24 a=4 147.630320 2.582953 6 338
|
||||||
|
FAST f=24 a=4 1.396364 2.582953 6 338
|
||||||
|
FAST f=24 a=5 133.767428 2.580589 6 290
|
||||||
|
FAST f=24 a=5 1.19933 2.580589 6 290
|
||||||
|
FAST f=24 a=6 146.437535 2.579453 8 194
|
||||||
|
FAST f=24 a=6 1.385405 2.579453 8 194
|
||||||
|
FAST f=24 a=7 147.227507 2.584155 8 386
|
||||||
|
FAST f=24 a=7 1.48942 2.584155 8 386
|
||||||
|
FAST f=24 a=8 138.005773 2.584115 8 194
|
||||||
|
FAST f=24 a=8 1.352 2.584115 8 194
|
||||||
|
FAST f=24 a=9 141.442625 2.582902 8 290
|
||||||
|
FAST f=24 a=9 1.39647 2.582902 8 290
|
||||||
|
FAST f=24 a=10 142.157446 2.582701 8 434
|
||||||
|
FAST f=24 a=10 1.498889 2.582701 8 434
|
||||||
|
|||||||
@@ -5,7 +5,6 @@
|
|||||||
#include <ctype.h>
|
#include <ctype.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
#include "random.h"
|
#include "random.h"
|
||||||
#include "fastCover.h"
|
|
||||||
#include "dictBuilder.h"
|
#include "dictBuilder.h"
|
||||||
#include "zstd_internal.h" /* includes zstd.h */
|
#include "zstd_internal.h" /* includes zstd.h */
|
||||||
#include "io.h"
|
#include "io.h"
|
||||||
@@ -149,7 +148,7 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev
|
|||||||
/* Allocate dst with enough space to compress the maximum sized sample */
|
/* Allocate dst with enough space to compress the maximum sized sample */
|
||||||
{
|
{
|
||||||
size_t maxSampleSize = 0;
|
size_t maxSampleSize = 0;
|
||||||
for (int i = 0; i < srcInfo->nbSamples; i++) {
|
for (i = 0; i < srcInfo->nbSamples; i++) {
|
||||||
maxSampleSize = MAX(srcInfo->samplesSizes[i], maxSampleSize);
|
maxSampleSize = MAX(srcInfo->samplesSizes[i], maxSampleSize);
|
||||||
}
|
}
|
||||||
dstCapacity = ZSTD_compressBound(maxSampleSize);
|
dstCapacity = ZSTD_compressBound(maxSampleSize);
|
||||||
@@ -291,6 +290,9 @@ int main(int argCount, const char* argv[])
|
|||||||
/* Initialize arguments to default values */
|
/* Initialize arguments to default values */
|
||||||
unsigned k = 200;
|
unsigned k = 200;
|
||||||
unsigned d = 8;
|
unsigned d = 8;
|
||||||
|
unsigned f;
|
||||||
|
unsigned accel;
|
||||||
|
unsigned i;
|
||||||
const unsigned cLevel = DEFAULT_CLEVEL;
|
const unsigned cLevel = DEFAULT_CLEVEL;
|
||||||
const unsigned dictID = 0;
|
const unsigned dictID = 0;
|
||||||
const unsigned maxDictSize = g_defaultMaxDictSize;
|
const unsigned maxDictSize = g_defaultMaxDictSize;
|
||||||
@@ -305,7 +307,7 @@ int main(int argCount, const char* argv[])
|
|||||||
const char** extendedFileList = NULL;
|
const char** extendedFileList = NULL;
|
||||||
|
|
||||||
/* Parse arguments */
|
/* Parse arguments */
|
||||||
for (int i = 1; i < argCount; i++) {
|
for (i = 1; i < argCount; i++) {
|
||||||
const char* argument = argv[i];
|
const char* argument = argv[i];
|
||||||
if (longCommandWArg(&argument, "in=")) {
|
if (longCommandWArg(&argument, "in=")) {
|
||||||
filenameTable[filenameIdx] = argument;
|
filenameTable[filenameIdx] = argument;
|
||||||
@@ -375,6 +377,7 @@ int main(int argCount, const char* argv[])
|
|||||||
|
|
||||||
/* for cover */
|
/* for cover */
|
||||||
{
|
{
|
||||||
|
/* for cover (optimizing k and d) */
|
||||||
ZDICT_cover_params_t coverParam;
|
ZDICT_cover_params_t coverParam;
|
||||||
memset(&coverParam, 0, sizeof(coverParam));
|
memset(&coverParam, 0, sizeof(coverParam));
|
||||||
coverParam.zParams = zParams;
|
coverParam.zParams = zParams;
|
||||||
@@ -388,6 +391,7 @@ int main(int argCount, const char* argv[])
|
|||||||
goto _cleanup;
|
goto _cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* for cover (with k and d provided) */
|
||||||
const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL);
|
const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL);
|
||||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100));
|
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100));
|
||||||
if(coverResult) {
|
if(coverResult) {
|
||||||
@@ -398,31 +402,34 @@ int main(int argCount, const char* argv[])
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* for fastCover */
|
/* for fastCover */
|
||||||
for (unsigned f = 15; f < 25; f++){
|
for (f = 15; f < 25; f++){
|
||||||
DISPLAYLEVEL(2, "current f is %u\n", f);
|
DISPLAYLEVEL(2, "current f is %u\n", f);
|
||||||
/* for fastCover (optimizing k and d) */
|
for (accel = 1; accel < 11; accel++) {
|
||||||
ZDICT_fastCover_params_t fastParam;
|
DISPLAYLEVEL(2, "current accel is %u\n", accel);
|
||||||
memset(&fastParam, 0, sizeof(fastParam));
|
/* for fastCover (optimizing k and d) */
|
||||||
fastParam.zParams = zParams;
|
ZDICT_fastCover_params_t fastParam;
|
||||||
fastParam.splitPoint = 1.0;
|
memset(&fastParam, 0, sizeof(fastParam));
|
||||||
fastParam.f = f;
|
fastParam.zParams = zParams;
|
||||||
fastParam.steps = 40;
|
fastParam.f = f;
|
||||||
fastParam.nbThreads = 1;
|
fastParam.steps = 40;
|
||||||
const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
|
fastParam.nbThreads = 1;
|
||||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100));
|
fastParam.accel = accel;
|
||||||
if(fastOptResult) {
|
const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
|
||||||
result = 1;
|
DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel);
|
||||||
goto _cleanup;
|
if(fastOptResult) {
|
||||||
}
|
|
||||||
|
|
||||||
/* for fastCover (with k and d provided) */
|
|
||||||
for (int i = 0; i < 5; i++) {
|
|
||||||
const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
|
|
||||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100));
|
|
||||||
if(fastResult) {
|
|
||||||
result = 1;
|
result = 1;
|
||||||
goto _cleanup;
|
goto _cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* for fastCover (with k and d provided) */
|
||||||
|
for (i = 0; i < 5; i++) {
|
||||||
|
const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam);
|
||||||
|
DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel);
|
||||||
|
if(fastResult) {
|
||||||
|
result = 1;
|
||||||
|
goto _cleanup;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
|
|||||||
free(buffOut);
|
free(buffOut);
|
||||||
}
|
}
|
||||||
|
|
||||||
static const char* createOutFilename_orDie(const char* filename)
|
static char* createOutFilename_orDie(const char* filename)
|
||||||
{
|
{
|
||||||
size_t const inL = strlen(filename);
|
size_t const inL = strlen(filename);
|
||||||
size_t const outL = inL + 5;
|
size_t const outL = inL + 5;
|
||||||
@@ -109,7 +109,7 @@ static const char* createOutFilename_orDie(const char* filename)
|
|||||||
memset(outSpace, 0, outL);
|
memset(outSpace, 0, outL);
|
||||||
strcat(outSpace, filename);
|
strcat(outSpace, filename);
|
||||||
strcat(outSpace, ".zst");
|
strcat(outSpace, ".zst");
|
||||||
return (const char*)outSpace;
|
return (char*)outSpace;
|
||||||
}
|
}
|
||||||
|
|
||||||
int main(int argc, const char** argv) {
|
int main(int argc, const char** argv) {
|
||||||
@@ -124,8 +124,9 @@ int main(int argc, const char** argv) {
|
|||||||
{ const char* const inFileName = argv[1];
|
{ const char* const inFileName = argv[1];
|
||||||
unsigned const frameSize = (unsigned)atoi(argv[2]);
|
unsigned const frameSize = (unsigned)atoi(argv[2]);
|
||||||
|
|
||||||
const char* const outFileName = createOutFilename_orDie(inFileName);
|
char* const outFileName = createOutFilename_orDie(inFileName);
|
||||||
compressFile_orDie(inFileName, outFileName, 5, frameSize);
|
compressFile_orDie(inFileName, outFileName, 5, frameSize);
|
||||||
|
free(outFileName);
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|||||||
@@ -56,6 +56,7 @@
|
|||||||
|
|
||||||
#include <stdlib.h> /* malloc, free */
|
#include <stdlib.h> /* malloc, free */
|
||||||
#include <stdio.h> /* FILE* */
|
#include <stdio.h> /* FILE* */
|
||||||
|
#include <assert.h>
|
||||||
|
|
||||||
#define XXH_STATIC_LINKING_ONLY
|
#define XXH_STATIC_LINKING_ONLY
|
||||||
#define XXH_NAMESPACE ZSTD_
|
#define XXH_NAMESPACE ZSTD_
|
||||||
@@ -112,7 +113,7 @@ static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n)
|
|||||||
|
|
||||||
static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
|
static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
|
||||||
{
|
{
|
||||||
buffWrapper_t* buff = (buffWrapper_t*) opaque;
|
buffWrapper_t* const buff = (buffWrapper_t*) opaque;
|
||||||
unsigned long long newOffset;
|
unsigned long long newOffset;
|
||||||
switch (origin) {
|
switch (origin) {
|
||||||
case SEEK_SET:
|
case SEEK_SET:
|
||||||
@@ -124,6 +125,8 @@ static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
|
|||||||
case SEEK_END:
|
case SEEK_END:
|
||||||
newOffset = (unsigned long long)buff->size - offset;
|
newOffset = (unsigned long long)buff->size - offset;
|
||||||
break;
|
break;
|
||||||
|
default:
|
||||||
|
assert(0); /* not possible */
|
||||||
}
|
}
|
||||||
if (newOffset > buff->size) {
|
if (newOffset > buff->size) {
|
||||||
return -1;
|
return -1;
|
||||||
|
|||||||
@@ -158,7 +158,8 @@ int main(int argc, const char** argv)
|
|||||||
}
|
}
|
||||||
|
|
||||||
freeResources(ress);
|
freeResources(ress);
|
||||||
/* success */
|
free(ofnBuffer);
|
||||||
|
|
||||||
printf("compressed %i files \n", argc-1);
|
printf("compressed %i files \n", argc-1);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|||||||
@@ -73,7 +73,11 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
|
|||||||
ZSTD_CStream* const cstream = ZSTD_createCStream();
|
ZSTD_CStream* const cstream = ZSTD_createCStream();
|
||||||
if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
|
if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
|
||||||
size_t const initResult = ZSTD_initCStream(cstream, cLevel);
|
size_t const initResult = ZSTD_initCStream(cstream, cLevel);
|
||||||
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
|
if (ZSTD_isError(initResult)) {
|
||||||
|
fprintf(stderr, "ZSTD_initCStream() error : %s \n",
|
||||||
|
ZSTD_getErrorName(initResult));
|
||||||
|
exit(11);
|
||||||
|
}
|
||||||
|
|
||||||
size_t read, toRead = buffInSize;
|
size_t read, toRead = buffInSize;
|
||||||
while( (read = fread_orDie(buffIn, toRead, fin)) ) {
|
while( (read = fread_orDie(buffIn, toRead, fin)) ) {
|
||||||
@@ -81,7 +85,11 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
|
|||||||
while (input.pos < input.size) {
|
while (input.pos < input.size) {
|
||||||
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
|
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
|
||||||
toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
|
toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
|
||||||
if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
|
if (ZSTD_isError(toRead)) {
|
||||||
|
fprintf(stderr, "ZSTD_compressStream() error : %s \n",
|
||||||
|
ZSTD_getErrorName(toRead));
|
||||||
|
exit(12);
|
||||||
|
}
|
||||||
if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
|
if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
|
||||||
fwrite_orDie(buffOut, output.pos, fout);
|
fwrite_orDie(buffOut, output.pos, fout);
|
||||||
}
|
}
|
||||||
@@ -100,15 +108,15 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static const char* createOutFilename_orDie(const char* filename)
|
static char* createOutFilename_orDie(const char* filename)
|
||||||
{
|
{
|
||||||
size_t const inL = strlen(filename);
|
size_t const inL = strlen(filename);
|
||||||
size_t const outL = inL + 5;
|
size_t const outL = inL + 5;
|
||||||
void* outSpace = malloc_orDie(outL);
|
void* const outSpace = malloc_orDie(outL);
|
||||||
memset(outSpace, 0, outL);
|
memset(outSpace, 0, outL);
|
||||||
strcat(outSpace, filename);
|
strcat(outSpace, filename);
|
||||||
strcat(outSpace, ".zst");
|
strcat(outSpace, ".zst");
|
||||||
return (const char*)outSpace;
|
return (char*)outSpace;
|
||||||
}
|
}
|
||||||
|
|
||||||
int main(int argc, const char** argv)
|
int main(int argc, const char** argv)
|
||||||
@@ -124,8 +132,10 @@ int main(int argc, const char** argv)
|
|||||||
|
|
||||||
const char* const inFilename = argv[1];
|
const char* const inFilename = argv[1];
|
||||||
|
|
||||||
const char* const outFilename = createOutFilename_orDie(inFilename);
|
char* const outFilename = createOutFilename_orDie(inFilename);
|
||||||
compressFile_orDie(inFilename, outFilename, 1);
|
compressFile_orDie(inFilename, outFilename, 1);
|
||||||
|
|
||||||
|
free(outFilename); /* not strictly required, since program execution stops there,
|
||||||
|
* but some static analyzer main complain otherwise */
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ cxx_library(
|
|||||||
]),
|
]),
|
||||||
headers=subdir_glob([
|
headers=subdir_glob([
|
||||||
('dictBuilder', 'divsufsort.h'),
|
('dictBuilder', 'divsufsort.h'),
|
||||||
|
('dictBuilder', 'cover.h'),
|
||||||
]),
|
]),
|
||||||
srcs=glob(['dictBuilder/*.c']),
|
srcs=glob(['dictBuilder/*.c']),
|
||||||
deps=[':common'],
|
deps=[':common'],
|
||||||
|
|||||||
@@ -19,6 +19,9 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
|
|||||||
VERSION?= $(LIBVER)
|
VERSION?= $(LIBVER)
|
||||||
|
|
||||||
CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
|
CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
|
||||||
|
ifeq ($(OS),Windows_NT) # MinGW assumed
|
||||||
|
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
|
||||||
|
endif
|
||||||
CFLAGS ?= -O3
|
CFLAGS ?= -O3
|
||||||
DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||||
|
|||||||
@@ -79,8 +79,7 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
|||||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||||
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||||
|
|
||||||
#define ZSTD_FRAMEIDSIZE 4
|
#define ZSTD_FRAMEIDSIZE 4 /* magic number size */
|
||||||
static const size_t ZSTD_frameIdSize = ZSTD_FRAMEIDSIZE; /* magic number size */
|
|
||||||
|
|
||||||
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
|
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
|
||||||
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
|
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
|
||||||
|
|||||||
+55
-35
@@ -83,7 +83,9 @@
|
|||||||
* wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
|
* wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
|
||||||
* workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
|
* workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
|
||||||
*/
|
*/
|
||||||
size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
|
size_t FSE_buildCTable_wksp(FSE_CTable* ct,
|
||||||
|
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
|
||||||
|
void* workSpace, size_t wkspSize)
|
||||||
{
|
{
|
||||||
U32 const tableSize = 1 << tableLog;
|
U32 const tableSize = 1 << tableLog;
|
||||||
U32 const tableMask = tableSize - 1;
|
U32 const tableMask = tableSize - 1;
|
||||||
@@ -101,10 +103,14 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
|
|||||||
if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
|
if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge);
|
||||||
tableU16[-2] = (U16) tableLog;
|
tableU16[-2] = (U16) tableLog;
|
||||||
tableU16[-1] = (U16) maxSymbolValue;
|
tableU16[-1] = (U16) maxSymbolValue;
|
||||||
assert(tableLog < 16); /* required for the threshold strategy to work */
|
assert(tableLog < 16); /* required for threshold strategy to work */
|
||||||
|
|
||||||
/* For explanations on how to distribute symbol values over the table :
|
/* For explanations on how to distribute symbol values over the table :
|
||||||
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
|
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
|
||||||
|
|
||||||
|
#ifdef __clang_analyzer__
|
||||||
|
memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
|
||||||
|
#endif
|
||||||
|
|
||||||
/* symbol start positions */
|
/* symbol start positions */
|
||||||
{ U32 u;
|
{ U32 u;
|
||||||
@@ -124,13 +130,15 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
|
|||||||
U32 symbol;
|
U32 symbol;
|
||||||
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
|
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
|
||||||
int nbOccurences;
|
int nbOccurences;
|
||||||
for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++) {
|
int const freq = normalizedCounter[symbol];
|
||||||
|
for (nbOccurences=0; nbOccurences<freq; nbOccurences++) {
|
||||||
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
|
||||||
position = (position + step) & tableMask;
|
position = (position + step) & tableMask;
|
||||||
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */
|
while (position > highThreshold)
|
||||||
|
position = (position + step) & tableMask; /* Low proba area */
|
||||||
} }
|
} }
|
||||||
|
|
||||||
if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */
|
assert(position==0); /* Must have initialized all positions */
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Build table */
|
/* Build table */
|
||||||
@@ -201,9 +209,10 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
|
|||||||
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
|
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
static size_t
|
||||||
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
|
FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
||||||
unsigned writeIsSafe)
|
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
|
||||||
|
unsigned writeIsSafe)
|
||||||
{
|
{
|
||||||
BYTE* const ostart = (BYTE*) header;
|
BYTE* const ostart = (BYTE*) header;
|
||||||
BYTE* out = ostart;
|
BYTE* out = ostart;
|
||||||
@@ -212,13 +221,12 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
|||||||
const int tableSize = 1 << tableLog;
|
const int tableSize = 1 << tableLog;
|
||||||
int remaining;
|
int remaining;
|
||||||
int threshold;
|
int threshold;
|
||||||
U32 bitStream;
|
U32 bitStream = 0;
|
||||||
int bitCount;
|
int bitCount = 0;
|
||||||
unsigned charnum = 0;
|
unsigned symbol = 0;
|
||||||
int previous0 = 0;
|
unsigned const alphabetSize = maxSymbolValue + 1;
|
||||||
|
int previousIs0 = 0;
|
||||||
|
|
||||||
bitStream = 0;
|
|
||||||
bitCount = 0;
|
|
||||||
/* Table Size */
|
/* Table Size */
|
||||||
bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
|
bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
|
||||||
bitCount += 4;
|
bitCount += 4;
|
||||||
@@ -228,48 +236,53 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
|||||||
threshold = tableSize;
|
threshold = tableSize;
|
||||||
nbBits = tableLog+1;
|
nbBits = tableLog+1;
|
||||||
|
|
||||||
while (remaining>1) { /* stops at 1 */
|
while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */
|
||||||
if (previous0) {
|
if (previousIs0) {
|
||||||
unsigned start = charnum;
|
unsigned start = symbol;
|
||||||
while (!normalizedCounter[charnum]) charnum++;
|
while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++;
|
||||||
while (charnum >= start+24) {
|
if (symbol == alphabetSize) break; /* incorrect distribution */
|
||||||
|
while (symbol >= start+24) {
|
||||||
start+=24;
|
start+=24;
|
||||||
bitStream += 0xFFFFU << bitCount;
|
bitStream += 0xFFFFU << bitCount;
|
||||||
if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
if ((!writeIsSafe) && (out > oend-2))
|
||||||
|
return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||||
out[0] = (BYTE) bitStream;
|
out[0] = (BYTE) bitStream;
|
||||||
out[1] = (BYTE)(bitStream>>8);
|
out[1] = (BYTE)(bitStream>>8);
|
||||||
out+=2;
|
out+=2;
|
||||||
bitStream>>=16;
|
bitStream>>=16;
|
||||||
}
|
}
|
||||||
while (charnum >= start+3) {
|
while (symbol >= start+3) {
|
||||||
start+=3;
|
start+=3;
|
||||||
bitStream += 3 << bitCount;
|
bitStream += 3 << bitCount;
|
||||||
bitCount += 2;
|
bitCount += 2;
|
||||||
}
|
}
|
||||||
bitStream += (charnum-start) << bitCount;
|
bitStream += (symbol-start) << bitCount;
|
||||||
bitCount += 2;
|
bitCount += 2;
|
||||||
if (bitCount>16) {
|
if (bitCount>16) {
|
||||||
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
if ((!writeIsSafe) && (out > oend - 2))
|
||||||
|
return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||||
out[0] = (BYTE)bitStream;
|
out[0] = (BYTE)bitStream;
|
||||||
out[1] = (BYTE)(bitStream>>8);
|
out[1] = (BYTE)(bitStream>>8);
|
||||||
out += 2;
|
out += 2;
|
||||||
bitStream >>= 16;
|
bitStream >>= 16;
|
||||||
bitCount -= 16;
|
bitCount -= 16;
|
||||||
} }
|
} }
|
||||||
{ int count = normalizedCounter[charnum++];
|
{ int count = normalizedCounter[symbol++];
|
||||||
int const max = (2*threshold-1)-remaining;
|
int const max = (2*threshold-1) - remaining;
|
||||||
remaining -= count < 0 ? -count : count;
|
remaining -= count < 0 ? -count : count;
|
||||||
count++; /* +1 for extra accuracy */
|
count++; /* +1 for extra accuracy */
|
||||||
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
|
if (count>=threshold)
|
||||||
|
count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
|
||||||
bitStream += count << bitCount;
|
bitStream += count << bitCount;
|
||||||
bitCount += nbBits;
|
bitCount += nbBits;
|
||||||
bitCount -= (count<max);
|
bitCount -= (count<max);
|
||||||
previous0 = (count==1);
|
previousIs0 = (count==1);
|
||||||
if (remaining<1) return ERROR(GENERIC);
|
if (remaining<1) return ERROR(GENERIC);
|
||||||
while (remaining<threshold) { nbBits--; threshold>>=1; }
|
while (remaining<threshold) { nbBits--; threshold>>=1; }
|
||||||
}
|
}
|
||||||
if (bitCount>16) {
|
if (bitCount>16) {
|
||||||
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
if ((!writeIsSafe) && (out > oend - 2))
|
||||||
|
return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||||
out[0] = (BYTE)bitStream;
|
out[0] = (BYTE)bitStream;
|
||||||
out[1] = (BYTE)(bitStream>>8);
|
out[1] = (BYTE)(bitStream>>8);
|
||||||
out += 2;
|
out += 2;
|
||||||
@@ -277,19 +290,23 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
|||||||
bitCount -= 16;
|
bitCount -= 16;
|
||||||
} }
|
} }
|
||||||
|
|
||||||
|
if (remaining != 1)
|
||||||
|
return ERROR(GENERIC); /* incorrect normalized distribution */
|
||||||
|
assert(symbol <= alphabetSize);
|
||||||
|
|
||||||
/* flush remaining bitStream */
|
/* flush remaining bitStream */
|
||||||
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
if ((!writeIsSafe) && (out > oend - 2))
|
||||||
|
return ERROR(dstSize_tooSmall); /* Buffer overflow */
|
||||||
out[0] = (BYTE)bitStream;
|
out[0] = (BYTE)bitStream;
|
||||||
out[1] = (BYTE)(bitStream>>8);
|
out[1] = (BYTE)(bitStream>>8);
|
||||||
out+= (bitCount+7) /8;
|
out+= (bitCount+7) /8;
|
||||||
|
|
||||||
if (charnum > maxSymbolValue + 1) return ERROR(GENERIC);
|
|
||||||
|
|
||||||
return (out-ostart);
|
return (out-ostart);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
size_t FSE_writeNCount (void* buffer, size_t bufferSize,
|
||||||
|
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||||
{
|
{
|
||||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */
|
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */
|
||||||
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */
|
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */
|
||||||
@@ -297,7 +314,7 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized
|
|||||||
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
|
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
|
||||||
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
|
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
|
||||||
|
|
||||||
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
|
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -331,7 +348,7 @@ void FSE_freeCTable (FSE_CTable* ct) { free(ct); }
|
|||||||
/* provides the minimum logSize to safely represent a distribution */
|
/* provides the minimum logSize to safely represent a distribution */
|
||||||
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
|
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
|
||||||
{
|
{
|
||||||
U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1;
|
U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1;
|
||||||
U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
|
U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
|
||||||
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
|
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
|
||||||
assert(srcSize > 1); /* Not supported, RLE should be used instead */
|
assert(srcSize > 1); /* Not supported, RLE should be used instead */
|
||||||
@@ -394,6 +411,9 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
|
|||||||
}
|
}
|
||||||
ToDistribute = (1 << tableLog) - distributed;
|
ToDistribute = (1 << tableLog) - distributed;
|
||||||
|
|
||||||
|
if (ToDistribute == 0)
|
||||||
|
return 0;
|
||||||
|
|
||||||
if ((total / ToDistribute) > lowOne) {
|
if ((total / ToDistribute) > lowOne) {
|
||||||
/* risk of rounding to zero */
|
/* risk of rounding to zero */
|
||||||
lowOne = (U32)((total * 3) / (ToDistribute * 2));
|
lowOne = (U32)((total * 3) / (ToDistribute * 2));
|
||||||
|
|||||||
@@ -1032,6 +1032,9 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
|
|||||||
|
|
||||||
ms->hashLog3 = hashLog3;
|
ms->hashLog3 = hashLog3;
|
||||||
memset(&ms->window, 0, sizeof(ms->window));
|
memset(&ms->window, 0, sizeof(ms->window));
|
||||||
|
ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */
|
||||||
|
ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */
|
||||||
|
ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */
|
||||||
ZSTD_invalidateMatchState(ms);
|
ZSTD_invalidateMatchState(ms);
|
||||||
|
|
||||||
/* opt parser space */
|
/* opt parser space */
|
||||||
@@ -1144,7 +1147,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
|
|||||||
if (zc->workSpace == NULL) return ERROR(memory_allocation);
|
if (zc->workSpace == NULL) return ERROR(memory_allocation);
|
||||||
zc->workSpaceSize = neededSpace;
|
zc->workSpaceSize = neededSpace;
|
||||||
zc->workSpaceOversizedDuration = 0;
|
zc->workSpaceOversizedDuration = 0;
|
||||||
ptr = zc->workSpace;
|
|
||||||
|
|
||||||
/* Statically sized space.
|
/* Statically sized space.
|
||||||
* entropyWorkspace never moves,
|
* entropyWorkspace never moves,
|
||||||
@@ -1281,8 +1283,9 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (attachDict) {
|
if (attachDict) {
|
||||||
const U32 cdictLen = (U32)( cdict->matchState.window.nextSrc
|
const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc
|
||||||
- cdict->matchState.window.base);
|
- cdict->matchState.window.base);
|
||||||
|
const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit;
|
||||||
if (cdictLen == 0) {
|
if (cdictLen == 0) {
|
||||||
/* don't even attach dictionaries with no contents */
|
/* don't even attach dictionaries with no contents */
|
||||||
DEBUGLOG(4, "skipping attaching empty dictionary");
|
DEBUGLOG(4, "skipping attaching empty dictionary");
|
||||||
@@ -1292,9 +1295,9 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx,
|
|||||||
|
|
||||||
/* prep working match state so dict matches never have negative indices
|
/* prep working match state so dict matches never have negative indices
|
||||||
* when they are translated to the working context's index space. */
|
* when they are translated to the working context's index space. */
|
||||||
if (cctx->blockState.matchState.window.dictLimit < cdictLen) {
|
if (cctx->blockState.matchState.window.dictLimit < cdictEnd) {
|
||||||
cctx->blockState.matchState.window.nextSrc =
|
cctx->blockState.matchState.window.nextSrc =
|
||||||
cctx->blockState.matchState.window.base + cdictLen;
|
cctx->blockState.matchState.window.base + cdictEnd;
|
||||||
ZSTD_window_clear(&cctx->blockState.matchState.window);
|
ZSTD_window_clear(&cctx->blockState.matchState.window);
|
||||||
}
|
}
|
||||||
cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit;
|
cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit;
|
||||||
|
|||||||
@@ -970,7 +970,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
|
|||||||
U32 seqPos = cur;
|
U32 seqPos = cur;
|
||||||
|
|
||||||
DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)",
|
DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)",
|
||||||
last_pos, cur);
|
last_pos, cur); (void)last_pos;
|
||||||
assert(storeEnd < ZSTD_OPT_NUM);
|
assert(storeEnd < ZSTD_OPT_NUM);
|
||||||
DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
|
DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
|
||||||
storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off);
|
storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off);
|
||||||
|
|||||||
@@ -320,7 +320,8 @@ static void ZSTDMT_setNbSeq(ZSTDMT_seqPool* const seqPool, size_t const nbSeq)
|
|||||||
|
|
||||||
static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem)
|
static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem)
|
||||||
{
|
{
|
||||||
ZSTDMT_seqPool* seqPool = ZSTDMT_createBufferPool(nbWorkers, cMem);
|
ZSTDMT_seqPool* const seqPool = ZSTDMT_createBufferPool(nbWorkers, cMem);
|
||||||
|
if (seqPool == NULL) return NULL;
|
||||||
ZSTDMT_setNbSeq(seqPool, 0);
|
ZSTDMT_setNbSeq(seqPool, 0);
|
||||||
return seqPool;
|
return seqPool;
|
||||||
}
|
}
|
||||||
@@ -1080,9 +1081,9 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx)
|
|||||||
{
|
{
|
||||||
ZSTD_frameProgression fps;
|
ZSTD_frameProgression fps;
|
||||||
DEBUGLOG(6, "ZSTDMT_getFrameProgression");
|
DEBUGLOG(6, "ZSTDMT_getFrameProgression");
|
||||||
|
fps.ingested = mtctx->consumed + mtctx->inBuff.filled;
|
||||||
fps.consumed = mtctx->consumed;
|
fps.consumed = mtctx->consumed;
|
||||||
fps.produced = mtctx->produced;
|
fps.produced = mtctx->produced;
|
||||||
fps.ingested = mtctx->consumed + mtctx->inBuff.filled;
|
|
||||||
{ unsigned jobNb;
|
{ unsigned jobNb;
|
||||||
unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1);
|
unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1);
|
||||||
DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)",
|
DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)",
|
||||||
@@ -1092,8 +1093,8 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx)
|
|||||||
ZSTD_pthread_mutex_lock(&mtctx->jobs[wJobID].job_mutex);
|
ZSTD_pthread_mutex_lock(&mtctx->jobs[wJobID].job_mutex);
|
||||||
{ size_t const cResult = mtctx->jobs[wJobID].cSize;
|
{ size_t const cResult = mtctx->jobs[wJobID].cSize;
|
||||||
size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
|
size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
|
||||||
fps.consumed += mtctx->jobs[wJobID].consumed;
|
|
||||||
fps.ingested += mtctx->jobs[wJobID].src.size;
|
fps.ingested += mtctx->jobs[wJobID].src.size;
|
||||||
|
fps.consumed += mtctx->jobs[wJobID].consumed;
|
||||||
fps.produced += produced;
|
fps.produced += produced;
|
||||||
}
|
}
|
||||||
ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
|
ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex);
|
||||||
@@ -1545,6 +1546,8 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZS
|
|||||||
|
|
||||||
|
|
||||||
/*! ZSTDMT_flushProduced() :
|
/*! ZSTDMT_flushProduced() :
|
||||||
|
* flush whatever data has been produced but not yet flushed in current job.
|
||||||
|
* move to next job if current one is fully flushed.
|
||||||
* `output` : `pos` will be updated with amount of data flushed .
|
* `output` : `pos` will be updated with amount of data flushed .
|
||||||
* `blockToFlush` : if >0, the function will block and wait if there is no data available to flush .
|
* `blockToFlush` : if >0, the function will block and wait if there is no data available to flush .
|
||||||
* @return : amount of data remaining within internal buffer, 0 if no more, 1 if unknown but > 0, or an error code */
|
* @return : amount of data remaining within internal buffer, 0 if no more, 1 if unknown but > 0, or an error code */
|
||||||
@@ -1593,6 +1596,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
|
|||||||
mtctx->jobs[wJobID].cSize += 4; /* can write this shared value, as worker is no longer active */
|
mtctx->jobs[wJobID].cSize += 4; /* can write this shared value, as worker is no longer active */
|
||||||
mtctx->jobs[wJobID].frameChecksumNeeded = 0;
|
mtctx->jobs[wJobID].frameChecksumNeeded = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (cSize > 0) { /* compression is ongoing or completed */
|
if (cSize > 0) { /* compression is ongoing or completed */
|
||||||
size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos);
|
size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos);
|
||||||
DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)",
|
DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)",
|
||||||
@@ -1606,7 +1610,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u
|
|||||||
output->pos += toFlush;
|
output->pos += toFlush;
|
||||||
mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */
|
mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */
|
||||||
|
|
||||||
if ( (srcConsumed == srcSize) /* job completed */
|
if ( (srcConsumed == srcSize) /* job is completed */
|
||||||
&& (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */
|
&& (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */
|
||||||
DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one",
|
DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one",
|
||||||
mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed);
|
mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed);
|
||||||
|
|||||||
@@ -185,7 +185,7 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
|
|||||||
static size_t ZSTD_startingInputLength(ZSTD_format_e format)
|
static size_t ZSTD_startingInputLength(ZSTD_format_e format)
|
||||||
{
|
{
|
||||||
size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ?
|
size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ?
|
||||||
ZSTD_frameHeaderSize_prefix - ZSTD_frameIdSize :
|
ZSTD_frameHeaderSize_prefix - ZSTD_FRAMEIDSIZE :
|
||||||
ZSTD_frameHeaderSize_prefix;
|
ZSTD_frameHeaderSize_prefix;
|
||||||
ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE);
|
ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE);
|
||||||
/* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
|
/* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
|
||||||
@@ -278,7 +278,7 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
|||||||
* Note 3 : Skippable Frame Identifiers are considered valid. */
|
* Note 3 : Skippable Frame Identifiers are considered valid. */
|
||||||
unsigned ZSTD_isFrame(const void* buffer, size_t size)
|
unsigned ZSTD_isFrame(const void* buffer, size_t size)
|
||||||
{
|
{
|
||||||
if (size < ZSTD_frameIdSize) return 0;
|
if (size < ZSTD_FRAMEIDSIZE) return 0;
|
||||||
{ U32 const magic = MEM_readLE32(buffer);
|
{ U32 const magic = MEM_readLE32(buffer);
|
||||||
if (magic == ZSTD_MAGICNUMBER) return 1;
|
if (magic == ZSTD_MAGICNUMBER) return 1;
|
||||||
if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
|
if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
|
||||||
@@ -330,7 +330,9 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
|
|||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
size_t const minInputSize = ZSTD_startingInputLength(format);
|
size_t const minInputSize = ZSTD_startingInputLength(format);
|
||||||
|
|
||||||
|
memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
|
||||||
if (srcSize < minInputSize) return minInputSize;
|
if (srcSize < minInputSize) return minInputSize;
|
||||||
|
if (src==NULL) return ERROR(GENERIC); /* invalid parameter */
|
||||||
|
|
||||||
if ( (format != ZSTD_f_zstd1_magicless)
|
if ( (format != ZSTD_f_zstd1_magicless)
|
||||||
&& (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
|
&& (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
|
||||||
@@ -339,7 +341,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
|
|||||||
if (srcSize < ZSTD_skippableHeaderSize)
|
if (srcSize < ZSTD_skippableHeaderSize)
|
||||||
return ZSTD_skippableHeaderSize; /* magic number + frame length */
|
return ZSTD_skippableHeaderSize; /* magic number + frame length */
|
||||||
memset(zfhPtr, 0, sizeof(*zfhPtr));
|
memset(zfhPtr, 0, sizeof(*zfhPtr));
|
||||||
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_frameIdSize);
|
zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
|
||||||
zfhPtr->frameType = ZSTD_skippableFrame;
|
zfhPtr->frameType = ZSTD_skippableFrame;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
@@ -451,7 +453,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
|||||||
size_t skippableSize;
|
size_t skippableSize;
|
||||||
if (srcSize < ZSTD_skippableHeaderSize)
|
if (srcSize < ZSTD_skippableHeaderSize)
|
||||||
return ERROR(srcSize_wrong);
|
return ERROR(srcSize_wrong);
|
||||||
skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_frameIdSize)
|
skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_FRAMEIDSIZE)
|
||||||
+ ZSTD_skippableHeaderSize;
|
+ ZSTD_skippableHeaderSize;
|
||||||
if (srcSize < skippableSize) {
|
if (srcSize < skippableSize) {
|
||||||
return ZSTD_CONTENTSIZE_ERROR;
|
return ZSTD_CONTENTSIZE_ERROR;
|
||||||
@@ -540,6 +542,7 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
|
|||||||
static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
|
static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
|
||||||
const void* src, size_t srcSize)
|
const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
|
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
||||||
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||||
memcpy(dst, src, srcSize);
|
memcpy(dst, src, srcSize);
|
||||||
return srcSize;
|
return srcSize;
|
||||||
@@ -1763,7 +1766,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
|||||||
#endif
|
#endif
|
||||||
if ( (srcSize >= ZSTD_skippableHeaderSize)
|
if ( (srcSize >= ZSTD_skippableHeaderSize)
|
||||||
&& (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) {
|
&& (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) {
|
||||||
return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize);
|
return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE);
|
||||||
} else {
|
} else {
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
const BYTE* const ipstart = ip;
|
const BYTE* const ipstart = ip;
|
||||||
@@ -1797,7 +1800,6 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
|
|||||||
if (zfh.checksumFlag) { /* Final frame content checksum */
|
if (zfh.checksumFlag) { /* Final frame content checksum */
|
||||||
if (remainingSize < 4) return ERROR(srcSize_wrong);
|
if (remainingSize < 4) return ERROR(srcSize_wrong);
|
||||||
ip += 4;
|
ip += 4;
|
||||||
remainingSize -= 4;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return ip - ipstart;
|
return ip - ipstart;
|
||||||
@@ -1932,7 +1934,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|||||||
size_t skippableSize;
|
size_t skippableSize;
|
||||||
if (srcSize < ZSTD_skippableHeaderSize)
|
if (srcSize < ZSTD_skippableHeaderSize)
|
||||||
return ERROR(srcSize_wrong);
|
return ERROR(srcSize_wrong);
|
||||||
skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize)
|
skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE)
|
||||||
+ ZSTD_skippableHeaderSize;
|
+ ZSTD_skippableHeaderSize;
|
||||||
if (srcSize < skippableSize) return ERROR(srcSize_wrong);
|
if (srcSize < skippableSize) return ERROR(srcSize_wrong);
|
||||||
|
|
||||||
@@ -2057,7 +2059,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|||||||
case ZSTDds_getFrameHeaderSize :
|
case ZSTDds_getFrameHeaderSize :
|
||||||
assert(src != NULL);
|
assert(src != NULL);
|
||||||
if (dctx->format == ZSTD_f_zstd1) { /* allows header */
|
if (dctx->format == ZSTD_f_zstd1) { /* allows header */
|
||||||
assert(srcSize >= ZSTD_frameIdSize); /* to read skippable magic number */
|
assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */
|
||||||
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
||||||
memcpy(dctx->headerBuffer, src, srcSize);
|
memcpy(dctx->headerBuffer, src, srcSize);
|
||||||
dctx->expected = ZSTD_skippableHeaderSize - srcSize; /* remaining to load to get full skippable frame header */
|
dctx->expected = ZSTD_skippableHeaderSize - srcSize; /* remaining to load to get full skippable frame header */
|
||||||
@@ -2167,7 +2169,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|||||||
assert(src != NULL);
|
assert(src != NULL);
|
||||||
assert(srcSize <= ZSTD_skippableHeaderSize);
|
assert(srcSize <= ZSTD_skippableHeaderSize);
|
||||||
memcpy(dctx->headerBuffer + (ZSTD_skippableHeaderSize - srcSize), src, srcSize); /* complete skippable header */
|
memcpy(dctx->headerBuffer + (ZSTD_skippableHeaderSize - srcSize), src, srcSize); /* complete skippable header */
|
||||||
dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_frameIdSize); /* note : dctx->expected can grow seriously large, beyond local buffer size */
|
dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */
|
||||||
dctx->stage = ZSTDds_skipFrame;
|
dctx->stage = ZSTDds_skipFrame;
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
@@ -2268,7 +2270,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
|
|||||||
if (magic != ZSTD_MAGIC_DICTIONARY) {
|
if (magic != ZSTD_MAGIC_DICTIONARY) {
|
||||||
return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
|
return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
|
||||||
} }
|
} }
|
||||||
dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_frameIdSize);
|
dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
|
||||||
|
|
||||||
/* load entropy tables */
|
/* load entropy tables */
|
||||||
{ size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
|
{ size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
|
||||||
@@ -2381,7 +2383,7 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, ZSTD_dictContentType_e
|
|||||||
return 0; /* pure content mode */
|
return 0; /* pure content mode */
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize);
|
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
|
||||||
|
|
||||||
/* load entropy tables */
|
/* load entropy tables */
|
||||||
CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted );
|
CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted );
|
||||||
@@ -2398,6 +2400,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
|
|||||||
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
|
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
|
||||||
ddict->dictBuffer = NULL;
|
ddict->dictBuffer = NULL;
|
||||||
ddict->dictContent = dict;
|
ddict->dictContent = dict;
|
||||||
|
if (!dict) dictSize = 0;
|
||||||
} else {
|
} else {
|
||||||
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
|
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
|
||||||
ddict->dictBuffer = internalBuffer;
|
ddict->dictBuffer = internalBuffer;
|
||||||
@@ -2510,7 +2513,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
|
|||||||
{
|
{
|
||||||
if (dictSize < 8) return 0;
|
if (dictSize < 8) return 0;
|
||||||
if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
|
if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
|
||||||
return MEM_readLE32((const char*)dict + ZSTD_frameIdSize);
|
return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*! ZSTD_getDictID_fromDDict() :
|
/*! ZSTD_getDictID_fromDDict() :
|
||||||
@@ -2855,7 +2858,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|||||||
CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
|
CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
|
||||||
|
|
||||||
if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
|
||||||
zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_frameIdSize);
|
zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
|
||||||
zds->stage = ZSTDds_skipFrame;
|
zds->stage = ZSTDds_skipFrame;
|
||||||
} else {
|
} else {
|
||||||
CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
|
CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize));
|
||||||
|
|||||||
+68
-79
@@ -29,6 +29,7 @@
|
|||||||
#include "mem.h" /* read */
|
#include "mem.h" /* read */
|
||||||
#include "pool.h"
|
#include "pool.h"
|
||||||
#include "threading.h"
|
#include "threading.h"
|
||||||
|
#include "cover.h"
|
||||||
#include "zstd_internal.h" /* includes zstd.h */
|
#include "zstd_internal.h" /* includes zstd.h */
|
||||||
#ifndef ZDICT_STATIC_LINKING_ONLY
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||||
#define ZDICT_STATIC_LINKING_ONLY
|
#define ZDICT_STATIC_LINKING_ONLY
|
||||||
@@ -185,7 +186,7 @@ static void COVER_map_remove(COVER_map_t *map, U32 key) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Destroyes a map that is inited with COVER_map_init().
|
* Destroys a map that is inited with COVER_map_init().
|
||||||
*/
|
*/
|
||||||
static void COVER_map_destroy(COVER_map_t *map) {
|
static void COVER_map_destroy(COVER_map_t *map) {
|
||||||
if (map->data) {
|
if (map->data) {
|
||||||
@@ -223,7 +224,7 @@ static COVER_ctx_t *g_ctx = NULL;
|
|||||||
/**
|
/**
|
||||||
* Returns the sum of the sample sizes.
|
* Returns the sum of the sample sizes.
|
||||||
*/
|
*/
|
||||||
static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
|
size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) {
|
||||||
size_t sum = 0;
|
size_t sum = 0;
|
||||||
unsigned i;
|
unsigned i;
|
||||||
for (i = 0; i < nbSamples; ++i) {
|
for (i = 0; i < nbSamples; ++i) {
|
||||||
@@ -380,14 +381,6 @@ static void COVER_group(COVER_ctx_t *ctx, const void *group,
|
|||||||
ctx->suffix[dmerId] = freq;
|
ctx->suffix[dmerId] = freq;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* A segment is a range in the source as well as the score of the segment.
|
|
||||||
*/
|
|
||||||
typedef struct {
|
|
||||||
U32 begin;
|
|
||||||
U32 end;
|
|
||||||
U32 score;
|
|
||||||
} COVER_segment_t;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Selects the best segment in an epoch.
|
* Selects the best segment in an epoch.
|
||||||
@@ -691,7 +684,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
|||||||
BYTE* const dict = (BYTE*)dictBuffer;
|
BYTE* const dict = (BYTE*)dictBuffer;
|
||||||
COVER_ctx_t ctx;
|
COVER_ctx_t ctx;
|
||||||
COVER_map_t activeDmers;
|
COVER_map_t activeDmers;
|
||||||
parameters.splitPoint = 1.0;
|
parameters.splitPoint = parameters.splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters.splitPoint;
|
||||||
/* Initialize global data */
|
/* Initialize global data */
|
||||||
g_displayLevel = parameters.zParams.notificationLevel;
|
g_displayLevel = parameters.zParams.notificationLevel;
|
||||||
/* Checks */
|
/* Checks */
|
||||||
@@ -737,28 +730,65 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* COVER_best_t is used for two purposes:
|
|
||||||
* 1. Synchronizing threads.
|
size_t COVER_checkTotalCompressedSize(const ZDICT_cover_params_t parameters,
|
||||||
* 2. Saving the best parameters and dictionary.
|
const size_t *samplesSizes, const BYTE *samples,
|
||||||
*
|
size_t *offsets,
|
||||||
* All of the methods except COVER_best_init() are thread safe if zstd is
|
size_t nbTrainSamples, size_t nbSamples,
|
||||||
* compiled with multithreaded support.
|
BYTE *const dict, size_t dictBufferCapacity) {
|
||||||
*/
|
size_t totalCompressedSize = ERROR(GENERIC);
|
||||||
typedef struct COVER_best_s {
|
/* Pointers */
|
||||||
ZSTD_pthread_mutex_t mutex;
|
ZSTD_CCtx *cctx;
|
||||||
ZSTD_pthread_cond_t cond;
|
ZSTD_CDict *cdict;
|
||||||
size_t liveJobs;
|
void *dst;
|
||||||
void *dict;
|
/* Local variables */
|
||||||
size_t dictSize;
|
size_t dstCapacity;
|
||||||
ZDICT_cover_params_t parameters;
|
size_t i;
|
||||||
size_t compressedSize;
|
/* Allocate dst with enough space to compress the maximum sized sample */
|
||||||
} COVER_best_t;
|
{
|
||||||
|
size_t maxSampleSize = 0;
|
||||||
|
i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0;
|
||||||
|
for (; i < nbSamples; ++i) {
|
||||||
|
maxSampleSize = MAX(samplesSizes[i], maxSampleSize);
|
||||||
|
}
|
||||||
|
dstCapacity = ZSTD_compressBound(maxSampleSize);
|
||||||
|
dst = malloc(dstCapacity);
|
||||||
|
}
|
||||||
|
/* Create the cctx and cdict */
|
||||||
|
cctx = ZSTD_createCCtx();
|
||||||
|
cdict = ZSTD_createCDict(dict, dictBufferCapacity,
|
||||||
|
parameters.zParams.compressionLevel);
|
||||||
|
if (!dst || !cctx || !cdict) {
|
||||||
|
goto _compressCleanup;
|
||||||
|
}
|
||||||
|
/* Compress each sample and sum their sizes (or error) */
|
||||||
|
totalCompressedSize = dictBufferCapacity;
|
||||||
|
i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0;
|
||||||
|
for (; i < nbSamples; ++i) {
|
||||||
|
const size_t size = ZSTD_compress_usingCDict(
|
||||||
|
cctx, dst, dstCapacity, samples + offsets[i],
|
||||||
|
samplesSizes[i], cdict);
|
||||||
|
if (ZSTD_isError(size)) {
|
||||||
|
totalCompressedSize = ERROR(GENERIC);
|
||||||
|
goto _compressCleanup;
|
||||||
|
}
|
||||||
|
totalCompressedSize += size;
|
||||||
|
}
|
||||||
|
_compressCleanup:
|
||||||
|
ZSTD_freeCCtx(cctx);
|
||||||
|
ZSTD_freeCDict(cdict);
|
||||||
|
if (dst) {
|
||||||
|
free(dst);
|
||||||
|
}
|
||||||
|
return totalCompressedSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize the `COVER_best_t`.
|
* Initialize the `COVER_best_t`.
|
||||||
*/
|
*/
|
||||||
static void COVER_best_init(COVER_best_t *best) {
|
void COVER_best_init(COVER_best_t *best) {
|
||||||
if (best==NULL) return; /* compatible with init on NULL */
|
if (best==NULL) return; /* compatible with init on NULL */
|
||||||
(void)ZSTD_pthread_mutex_init(&best->mutex, NULL);
|
(void)ZSTD_pthread_mutex_init(&best->mutex, NULL);
|
||||||
(void)ZSTD_pthread_cond_init(&best->cond, NULL);
|
(void)ZSTD_pthread_cond_init(&best->cond, NULL);
|
||||||
@@ -772,7 +802,7 @@ static void COVER_best_init(COVER_best_t *best) {
|
|||||||
/**
|
/**
|
||||||
* Wait until liveJobs == 0.
|
* Wait until liveJobs == 0.
|
||||||
*/
|
*/
|
||||||
static void COVER_best_wait(COVER_best_t *best) {
|
void COVER_best_wait(COVER_best_t *best) {
|
||||||
if (!best) {
|
if (!best) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -786,7 +816,7 @@ static void COVER_best_wait(COVER_best_t *best) {
|
|||||||
/**
|
/**
|
||||||
* Call COVER_best_wait() and then destroy the COVER_best_t.
|
* Call COVER_best_wait() and then destroy the COVER_best_t.
|
||||||
*/
|
*/
|
||||||
static void COVER_best_destroy(COVER_best_t *best) {
|
void COVER_best_destroy(COVER_best_t *best) {
|
||||||
if (!best) {
|
if (!best) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -802,7 +832,7 @@ static void COVER_best_destroy(COVER_best_t *best) {
|
|||||||
* Called when a thread is about to be launched.
|
* Called when a thread is about to be launched.
|
||||||
* Increments liveJobs.
|
* Increments liveJobs.
|
||||||
*/
|
*/
|
||||||
static void COVER_best_start(COVER_best_t *best) {
|
void COVER_best_start(COVER_best_t *best) {
|
||||||
if (!best) {
|
if (!best) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -816,7 +846,7 @@ static void COVER_best_start(COVER_best_t *best) {
|
|||||||
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
|
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
|
||||||
* If this dictionary is the best so far save it and its parameters.
|
* If this dictionary is the best so far save it and its parameters.
|
||||||
*/
|
*/
|
||||||
static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
|
void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
|
||||||
ZDICT_cover_params_t parameters, void *dict,
|
ZDICT_cover_params_t parameters, void *dict,
|
||||||
size_t dictSize) {
|
size_t dictSize) {
|
||||||
if (!best) {
|
if (!best) {
|
||||||
@@ -847,10 +877,10 @@ static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
|
|||||||
best->parameters = parameters;
|
best->parameters = parameters;
|
||||||
best->compressedSize = compressedSize;
|
best->compressedSize = compressedSize;
|
||||||
}
|
}
|
||||||
ZSTD_pthread_mutex_unlock(&best->mutex);
|
|
||||||
if (liveJobs == 0) {
|
if (liveJobs == 0) {
|
||||||
ZSTD_pthread_cond_broadcast(&best->cond);
|
ZSTD_pthread_cond_broadcast(&best->cond);
|
||||||
}
|
}
|
||||||
|
ZSTD_pthread_mutex_unlock(&best->mutex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -904,51 +934,10 @@ static void COVER_tryParameters(void *opaque) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
/* Check total compressed size */
|
/* Check total compressed size */
|
||||||
{
|
totalCompressedSize = COVER_checkTotalCompressedSize(parameters, ctx->samplesSizes,
|
||||||
/* Pointers */
|
ctx->samples, ctx->offsets,
|
||||||
ZSTD_CCtx *cctx;
|
ctx->nbTrainSamples, ctx->nbSamples,
|
||||||
ZSTD_CDict *cdict;
|
dict, dictBufferCapacity);
|
||||||
void *dst;
|
|
||||||
/* Local variables */
|
|
||||||
size_t dstCapacity;
|
|
||||||
size_t i;
|
|
||||||
/* Allocate dst with enough space to compress the maximum sized sample */
|
|
||||||
{
|
|
||||||
size_t maxSampleSize = 0;
|
|
||||||
i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
|
|
||||||
for (; i < ctx->nbSamples; ++i) {
|
|
||||||
maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize);
|
|
||||||
}
|
|
||||||
dstCapacity = ZSTD_compressBound(maxSampleSize);
|
|
||||||
dst = malloc(dstCapacity);
|
|
||||||
}
|
|
||||||
/* Create the cctx and cdict */
|
|
||||||
cctx = ZSTD_createCCtx();
|
|
||||||
cdict = ZSTD_createCDict(dict, dictBufferCapacity,
|
|
||||||
parameters.zParams.compressionLevel);
|
|
||||||
if (!dst || !cctx || !cdict) {
|
|
||||||
goto _compressCleanup;
|
|
||||||
}
|
|
||||||
/* Compress each sample and sum their sizes (or error) */
|
|
||||||
totalCompressedSize = dictBufferCapacity;
|
|
||||||
i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0;
|
|
||||||
for (; i < ctx->nbSamples; ++i) {
|
|
||||||
const size_t size = ZSTD_compress_usingCDict(
|
|
||||||
cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i],
|
|
||||||
ctx->samplesSizes[i], cdict);
|
|
||||||
if (ZSTD_isError(size)) {
|
|
||||||
totalCompressedSize = ERROR(GENERIC);
|
|
||||||
goto _compressCleanup;
|
|
||||||
}
|
|
||||||
totalCompressedSize += size;
|
|
||||||
}
|
|
||||||
_compressCleanup:
|
|
||||||
ZSTD_freeCCtx(cctx);
|
|
||||||
ZSTD_freeCDict(cdict);
|
|
||||||
if (dst) {
|
|
||||||
free(dst);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_cleanup:
|
_cleanup:
|
||||||
COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
|
COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
#include <stdio.h> /* fprintf */
|
||||||
|
#include <stdlib.h> /* malloc, free, qsort */
|
||||||
|
#include <string.h> /* memset */
|
||||||
|
#include <time.h> /* clock */
|
||||||
|
#include "mem.h" /* read */
|
||||||
|
#include "pool.h"
|
||||||
|
#include "threading.h"
|
||||||
|
#include "zstd_internal.h" /* includes zstd.h */
|
||||||
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||||
|
#define ZDICT_STATIC_LINKING_ONLY
|
||||||
|
#endif
|
||||||
|
#include "zdict.h"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* COVER_best_t is used for two purposes:
|
||||||
|
* 1. Synchronizing threads.
|
||||||
|
* 2. Saving the best parameters and dictionary.
|
||||||
|
*
|
||||||
|
* All of the methods except COVER_best_init() are thread safe if zstd is
|
||||||
|
* compiled with multithreaded support.
|
||||||
|
*/
|
||||||
|
typedef struct COVER_best_s {
|
||||||
|
ZSTD_pthread_mutex_t mutex;
|
||||||
|
ZSTD_pthread_cond_t cond;
|
||||||
|
size_t liveJobs;
|
||||||
|
void *dict;
|
||||||
|
size_t dictSize;
|
||||||
|
ZDICT_cover_params_t parameters;
|
||||||
|
size_t compressedSize;
|
||||||
|
} COVER_best_t;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A segment is a range in the source as well as the score of the segment.
|
||||||
|
*/
|
||||||
|
typedef struct {
|
||||||
|
U32 begin;
|
||||||
|
U32 end;
|
||||||
|
U32 score;
|
||||||
|
} COVER_segment_t;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks total compressed size of a dictionary
|
||||||
|
*/
|
||||||
|
size_t COVER_checkTotalCompressedSize(const ZDICT_cover_params_t parameters,
|
||||||
|
const size_t *samplesSizes, const BYTE *samples,
|
||||||
|
size_t *offsets,
|
||||||
|
size_t nbTrainSamples, size_t nbSamples,
|
||||||
|
BYTE *const dict, size_t dictBufferCapacity);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the sum of the sample sizes.
|
||||||
|
*/
|
||||||
|
size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) ;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize the `COVER_best_t`.
|
||||||
|
*/
|
||||||
|
void COVER_best_init(COVER_best_t *best);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait until liveJobs == 0.
|
||||||
|
*/
|
||||||
|
void COVER_best_wait(COVER_best_t *best);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Call COVER_best_wait() and then destroy the COVER_best_t.
|
||||||
|
*/
|
||||||
|
void COVER_best_destroy(COVER_best_t *best);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called when a thread is about to be launched.
|
||||||
|
* Increments liveJobs.
|
||||||
|
*/
|
||||||
|
void COVER_best_start(COVER_best_t *best);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called when a thread finishes executing, both on error or success.
|
||||||
|
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
|
||||||
|
* If this dictionary is the best so far save it and its parameters.
|
||||||
|
*/
|
||||||
|
void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
|
||||||
|
ZDICT_cover_params_t parameters, void *dict,
|
||||||
|
size_t dictSize);
|
||||||
@@ -1637,7 +1637,7 @@ construct_SA(const unsigned char *T, int *SA,
|
|||||||
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
||||||
k = SA + BUCKET_B(c2 = c0, c1);
|
k = SA + BUCKET_B(c2 = c0, c1);
|
||||||
}
|
}
|
||||||
assert(k < j);
|
assert(k < j); assert(k != NULL);
|
||||||
*k-- = s;
|
*k-- = s;
|
||||||
} else {
|
} else {
|
||||||
assert(((s == 0) && (T[s] == c1)) || (s < 0));
|
assert(((s == 0) && (T[s] == c1)) || (s < 0));
|
||||||
@@ -1701,7 +1701,7 @@ construct_BWT(const unsigned char *T, int *SA,
|
|||||||
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
||||||
k = SA + BUCKET_B(c2 = c0, c1);
|
k = SA + BUCKET_B(c2 = c0, c1);
|
||||||
}
|
}
|
||||||
assert(k < j);
|
assert(k < j); assert(k != NULL);
|
||||||
*k-- = s;
|
*k-- = s;
|
||||||
} else if(s != 0) {
|
} else if(s != 0) {
|
||||||
*j = ~s;
|
*j = ~s;
|
||||||
@@ -1785,7 +1785,7 @@ construct_BWT_indexes(const unsigned char *T, int *SA,
|
|||||||
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
|
||||||
k = SA + BUCKET_B(c2 = c0, c1);
|
k = SA + BUCKET_B(c2 = c0, c1);
|
||||||
}
|
}
|
||||||
assert(k < j);
|
assert(k < j); assert(k != NULL);
|
||||||
*k-- = s;
|
*k-- = s;
|
||||||
} else if(s != 0) {
|
} else if(s != 0) {
|
||||||
*j = ~s;
|
*j = ~s;
|
||||||
|
|||||||
@@ -0,0 +1,701 @@
|
|||||||
|
/*-*************************************
|
||||||
|
* Dependencies
|
||||||
|
***************************************/
|
||||||
|
#include <stdio.h> /* fprintf */
|
||||||
|
#include <stdlib.h> /* malloc, free, qsort */
|
||||||
|
#include <string.h> /* memset */
|
||||||
|
#include <time.h> /* clock */
|
||||||
|
|
||||||
|
#include "mem.h" /* read */
|
||||||
|
#include "pool.h"
|
||||||
|
#include "threading.h"
|
||||||
|
#include "cover.h"
|
||||||
|
#include "zstd_internal.h" /* includes zstd.h */
|
||||||
|
#ifndef ZDICT_STATIC_LINKING_ONLY
|
||||||
|
#define ZDICT_STATIC_LINKING_ONLY
|
||||||
|
#endif
|
||||||
|
#include "zdict.h"
|
||||||
|
|
||||||
|
|
||||||
|
/*-*************************************
|
||||||
|
* Constants
|
||||||
|
***************************************/
|
||||||
|
#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
|
||||||
|
#define FASTCOVER_MAX_F 31
|
||||||
|
#define FASTCOVER_MAX_ACCEL 10
|
||||||
|
#define DEFAULT_SPLITPOINT 0.75
|
||||||
|
#define DEFAULT_F 18
|
||||||
|
#define DEFAULT_ACCEL 1
|
||||||
|
|
||||||
|
|
||||||
|
/*-*************************************
|
||||||
|
* Console display
|
||||||
|
***************************************/
|
||||||
|
static int g_displayLevel = 2;
|
||||||
|
#define DISPLAY(...) \
|
||||||
|
{ \
|
||||||
|
fprintf(stderr, __VA_ARGS__); \
|
||||||
|
fflush(stderr); \
|
||||||
|
}
|
||||||
|
#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
|
||||||
|
if (displayLevel >= l) { \
|
||||||
|
DISPLAY(__VA_ARGS__); \
|
||||||
|
} /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
|
||||||
|
#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
|
||||||
|
|
||||||
|
#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
|
||||||
|
if (displayLevel >= l) { \
|
||||||
|
if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
|
||||||
|
g_time = clock(); \
|
||||||
|
DISPLAY(__VA_ARGS__); \
|
||||||
|
} \
|
||||||
|
}
|
||||||
|
#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
|
||||||
|
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||||
|
static clock_t g_time = 0;
|
||||||
|
|
||||||
|
|
||||||
|
/*-*************************************
|
||||||
|
* Hash Functions
|
||||||
|
***************************************/
|
||||||
|
static const U64 prime6bytes = 227718039650203ULL;
|
||||||
|
static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
|
||||||
|
static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
|
||||||
|
|
||||||
|
static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
|
||||||
|
static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
|
||||||
|
static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hash the d-byte value pointed to by p and mod 2^f
|
||||||
|
*/
|
||||||
|
static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) {
|
||||||
|
if (d == 6) {
|
||||||
|
return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1);
|
||||||
|
}
|
||||||
|
return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/*-*************************************
|
||||||
|
* Acceleration
|
||||||
|
***************************************/
|
||||||
|
typedef struct {
|
||||||
|
unsigned finalize; /* Percentage of training samples used for ZDICT_finalizeDictionary */
|
||||||
|
unsigned skip; /* Number of dmer skipped between each dmer counted in computeFrequency */
|
||||||
|
} FASTCOVER_accel_t;
|
||||||
|
|
||||||
|
|
||||||
|
static const FASTCOVER_accel_t FASTCOVER_defaultAccelParameters[FASTCOVER_MAX_ACCEL+1] = {
|
||||||
|
{ 100, 0 }, /* accel = 0, should not happen because accel = 0 defaults to accel = 1 */
|
||||||
|
{ 100, 0 }, /* accel = 1 */
|
||||||
|
{ 50, 1 }, /* accel = 2 */
|
||||||
|
{ 34, 2 }, /* accel = 3 */
|
||||||
|
{ 25, 3 }, /* accel = 4 */
|
||||||
|
{ 20, 4 }, /* accel = 5 */
|
||||||
|
{ 17, 5 }, /* accel = 6 */
|
||||||
|
{ 14, 6 }, /* accel = 7 */
|
||||||
|
{ 13, 7 }, /* accel = 8 */
|
||||||
|
{ 11, 8 }, /* accel = 9 */
|
||||||
|
{ 10, 9 }, /* accel = 10 */
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
/*-*************************************
|
||||||
|
* Context
|
||||||
|
***************************************/
|
||||||
|
typedef struct {
|
||||||
|
const BYTE *samples;
|
||||||
|
size_t *offsets;
|
||||||
|
const size_t *samplesSizes;
|
||||||
|
size_t nbSamples;
|
||||||
|
size_t nbTrainSamples;
|
||||||
|
size_t nbTestSamples;
|
||||||
|
size_t nbDmers;
|
||||||
|
U32 *freqs;
|
||||||
|
unsigned d;
|
||||||
|
unsigned f;
|
||||||
|
FASTCOVER_accel_t accelParams;
|
||||||
|
} FASTCOVER_ctx_t;
|
||||||
|
|
||||||
|
|
||||||
|
/*-*************************************
|
||||||
|
* Helper functions
|
||||||
|
***************************************/
|
||||||
|
/**
|
||||||
|
* Selects the best segment in an epoch.
|
||||||
|
* Segments of are scored according to the function:
|
||||||
|
*
|
||||||
|
* Let F(d) be the frequency of all dmers with hash value d.
|
||||||
|
* Let S_i be hash value of the dmer at position i of segment S which has length k.
|
||||||
|
*
|
||||||
|
* Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
|
||||||
|
*
|
||||||
|
* Once the dmer with hash value d is in the dictionay we set F(d) = 0.
|
||||||
|
*/
|
||||||
|
static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
|
||||||
|
U32 *freqs, U32 begin, U32 end,
|
||||||
|
ZDICT_cover_params_t parameters,
|
||||||
|
U16* segmentFreqs) {
|
||||||
|
/* Constants */
|
||||||
|
const U32 k = parameters.k;
|
||||||
|
const U32 d = parameters.d;
|
||||||
|
const U32 f = ctx->f;
|
||||||
|
const U32 dmersInK = k - d + 1;
|
||||||
|
|
||||||
|
/* Try each segment (activeSegment) and save the best (bestSegment) */
|
||||||
|
COVER_segment_t bestSegment = {0, 0, 0};
|
||||||
|
COVER_segment_t activeSegment;
|
||||||
|
|
||||||
|
/* Reset the activeDmers in the segment */
|
||||||
|
/* The activeSegment starts at the beginning of the epoch. */
|
||||||
|
activeSegment.begin = begin;
|
||||||
|
activeSegment.end = begin;
|
||||||
|
activeSegment.score = 0;
|
||||||
|
|
||||||
|
/* Slide the activeSegment through the whole epoch.
|
||||||
|
* Save the best segment in bestSegment.
|
||||||
|
*/
|
||||||
|
while (activeSegment.end < end) {
|
||||||
|
/* Get hash value of current dmer */
|
||||||
|
const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d);
|
||||||
|
|
||||||
|
/* Add frequency of this index to score if this is the first occurence of index in active segment */
|
||||||
|
if (segmentFreqs[index] == 0) {
|
||||||
|
activeSegment.score += freqs[index];
|
||||||
|
}
|
||||||
|
/* Increment end of segment and segmentFreqs*/
|
||||||
|
activeSegment.end += 1;
|
||||||
|
segmentFreqs[index] += 1;
|
||||||
|
/* If the window is now too large, drop the first position */
|
||||||
|
if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
|
||||||
|
/* Get hash value of the dmer to be eliminated from active segment */
|
||||||
|
const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d);
|
||||||
|
segmentFreqs[delIndex] -= 1;
|
||||||
|
/* Subtract frequency of this index from score if this is the last occurrence of this index in active segment */
|
||||||
|
if (segmentFreqs[delIndex] == 0) {
|
||||||
|
activeSegment.score -= freqs[delIndex];
|
||||||
|
}
|
||||||
|
/* Increment start of segment */
|
||||||
|
activeSegment.begin += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* If this segment is the best so far save it */
|
||||||
|
if (activeSegment.score > bestSegment.score) {
|
||||||
|
bestSegment = activeSegment;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Zero out rest of segmentFreqs array */
|
||||||
|
while (activeSegment.begin < end) {
|
||||||
|
const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d);
|
||||||
|
segmentFreqs[delIndex] -= 1;
|
||||||
|
activeSegment.begin += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
/* Zero the frequency of hash value of each dmer covered by the chosen segment. */
|
||||||
|
U32 pos;
|
||||||
|
for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
|
||||||
|
const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, f, d);
|
||||||
|
freqs[i] = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return bestSegment;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static int FASTCOVER_checkParameters(ZDICT_cover_params_t parameters,
|
||||||
|
size_t maxDictSize, unsigned f,
|
||||||
|
unsigned accel) {
|
||||||
|
/* k, d, and f are required parameters */
|
||||||
|
if (parameters.d == 0 || parameters.k == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* d has to be 6 or 8 */
|
||||||
|
if (parameters.d != 6 && parameters.d != 8) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* k <= maxDictSize */
|
||||||
|
if (parameters.k > maxDictSize) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* d <= k */
|
||||||
|
if (parameters.d > parameters.k) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* 0 < f <= FASTCOVER_MAX_F*/
|
||||||
|
if (f > FASTCOVER_MAX_F || f == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* 0 < splitPoint <= 1 */
|
||||||
|
if (parameters.splitPoint <= 0 || parameters.splitPoint > 1) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* 0 < accel <= 10 */
|
||||||
|
if (accel > 10 || accel == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clean up a context initialized with `FASTCOVER_ctx_init()`.
|
||||||
|
*/
|
||||||
|
static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) {
|
||||||
|
if (!ctx) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
free(ctx->freqs);
|
||||||
|
ctx->freqs = NULL;
|
||||||
|
|
||||||
|
free(ctx->offsets);
|
||||||
|
ctx->offsets = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate for frequency of hash value of each dmer in ctx->samples
|
||||||
|
*/
|
||||||
|
static void FASTCOVER_computeFrequency(U32 *freqs, FASTCOVER_ctx_t *ctx){
|
||||||
|
const unsigned f = ctx->f;
|
||||||
|
const unsigned d = ctx->d;
|
||||||
|
const unsigned skip = ctx->accelParams.skip;
|
||||||
|
const unsigned readLength = MAX(d, 8);
|
||||||
|
size_t start; /* start of current dmer */
|
||||||
|
size_t i;
|
||||||
|
for (i = 0; i < ctx->nbTrainSamples; i++) {
|
||||||
|
size_t currSampleStart = ctx->offsets[i];
|
||||||
|
size_t currSampleEnd = ctx->offsets[i+1];
|
||||||
|
start = currSampleStart;
|
||||||
|
while (start + readLength <= currSampleEnd) {
|
||||||
|
const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, d);
|
||||||
|
freqs[dmerIndex]++;
|
||||||
|
start = start + skip + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prepare a context for dictionary building.
|
||||||
|
* The context is only dependent on the parameter `d` and can used multiple
|
||||||
|
* times.
|
||||||
|
* Returns 1 on success or zero on error.
|
||||||
|
* The context must be destroyed with `FASTCOVER_ctx_destroy()`.
|
||||||
|
*/
|
||||||
|
static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer,
|
||||||
|
const size_t *samplesSizes, unsigned nbSamples,
|
||||||
|
unsigned d, double splitPoint, unsigned f,
|
||||||
|
FASTCOVER_accel_t accelParams) {
|
||||||
|
const BYTE *const samples = (const BYTE *)samplesBuffer;
|
||||||
|
const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
|
||||||
|
/* Split samples into testing and training sets */
|
||||||
|
const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
|
||||||
|
const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
|
||||||
|
const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
|
||||||
|
const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
|
||||||
|
/* Checks */
|
||||||
|
if (totalSamplesSize < MAX(d, sizeof(U64)) ||
|
||||||
|
totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
|
||||||
|
DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
|
||||||
|
(U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* Check if there are at least 5 training samples */
|
||||||
|
if (nbTrainSamples < 5) {
|
||||||
|
DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid\n", nbTrainSamples);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* Check if there's testing sample */
|
||||||
|
if (nbTestSamples < 1) {
|
||||||
|
DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.\n", nbTestSamples);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* Zero the context */
|
||||||
|
memset(ctx, 0, sizeof(*ctx));
|
||||||
|
DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
|
||||||
|
(U32)trainingSamplesSize);
|
||||||
|
DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
|
||||||
|
(U32)testSamplesSize);
|
||||||
|
|
||||||
|
ctx->samples = samples;
|
||||||
|
ctx->samplesSizes = samplesSizes;
|
||||||
|
ctx->nbSamples = nbSamples;
|
||||||
|
ctx->nbTrainSamples = nbTrainSamples;
|
||||||
|
ctx->nbTestSamples = nbTestSamples;
|
||||||
|
ctx->nbDmers = trainingSamplesSize - MAX(d, sizeof(U64)) + 1;
|
||||||
|
ctx->d = d;
|
||||||
|
ctx->f = f;
|
||||||
|
ctx->accelParams = accelParams;
|
||||||
|
|
||||||
|
/* The offsets of each file */
|
||||||
|
ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t));
|
||||||
|
if (!ctx->offsets) {
|
||||||
|
DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n");
|
||||||
|
FASTCOVER_ctx_destroy(ctx);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Fill offsets from the samplesSizes */
|
||||||
|
{
|
||||||
|
U32 i;
|
||||||
|
ctx->offsets[0] = 0;
|
||||||
|
for (i = 1; i <= nbSamples; ++i) {
|
||||||
|
ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Initialize frequency array of size 2^f */
|
||||||
|
ctx->freqs = (U32 *)calloc(((U64)1 << f), sizeof(U32));
|
||||||
|
|
||||||
|
DISPLAYLEVEL(2, "Computing frequencies\n");
|
||||||
|
FASTCOVER_computeFrequency(ctx->freqs, ctx);
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Given the prepared context build the dictionary.
|
||||||
|
*/
|
||||||
|
static size_t FASTCOVER_buildDictionary(const FASTCOVER_ctx_t *ctx, U32 *freqs,
|
||||||
|
void *dictBuffer, size_t dictBufferCapacity,
|
||||||
|
ZDICT_cover_params_t parameters, U16* segmentFreqs){
|
||||||
|
BYTE *const dict = (BYTE *)dictBuffer;
|
||||||
|
size_t tail = dictBufferCapacity;
|
||||||
|
/* Divide the data up into epochs of equal size.
|
||||||
|
* We will select at least one segment from each epoch.
|
||||||
|
*/
|
||||||
|
const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k));
|
||||||
|
const U32 epochSize = (U32)(ctx->nbDmers / epochs);
|
||||||
|
size_t epoch;
|
||||||
|
DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
|
||||||
|
epochSize);
|
||||||
|
/* Loop through the epochs until there are no more segments or the dictionary
|
||||||
|
* is full.
|
||||||
|
*/
|
||||||
|
for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
|
||||||
|
const U32 epochBegin = (U32)(epoch * epochSize);
|
||||||
|
const U32 epochEnd = epochBegin + epochSize;
|
||||||
|
size_t segmentSize;
|
||||||
|
/* Select a segment */
|
||||||
|
COVER_segment_t segment = FASTCOVER_selectSegment(
|
||||||
|
ctx, freqs, epochBegin, epochEnd, parameters, segmentFreqs);
|
||||||
|
|
||||||
|
/* If the segment covers no dmers, then we are out of content */
|
||||||
|
if (segment.score == 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Trim the segment if necessary and if it is too small then we are done */
|
||||||
|
segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
|
||||||
|
if (segmentSize < parameters.d) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* We fill the dictionary from the back to allow the best segments to be
|
||||||
|
* referenced with the smallest offsets.
|
||||||
|
*/
|
||||||
|
tail -= segmentSize;
|
||||||
|
memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
|
||||||
|
DISPLAYUPDATE(
|
||||||
|
2, "\r%u%% ",
|
||||||
|
(U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||||
|
return tail;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parameters for FASTCOVER_tryParameters().
|
||||||
|
*/
|
||||||
|
typedef struct FASTCOVER_tryParameters_data_s {
|
||||||
|
const FASTCOVER_ctx_t *ctx;
|
||||||
|
COVER_best_t *best;
|
||||||
|
size_t dictBufferCapacity;
|
||||||
|
ZDICT_cover_params_t parameters;
|
||||||
|
} FASTCOVER_tryParameters_data_t;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tries a set of parameters and updates the COVER_best_t with the results.
|
||||||
|
* This function is thread safe if zstd is compiled with multithreaded support.
|
||||||
|
* It takes its parameters as an *OWNING* opaque pointer to support threading.
|
||||||
|
*/
|
||||||
|
static void FASTCOVER_tryParameters(void *opaque) {
|
||||||
|
/* Save parameters as local variables */
|
||||||
|
FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque;
|
||||||
|
const FASTCOVER_ctx_t *const ctx = data->ctx;
|
||||||
|
const ZDICT_cover_params_t parameters = data->parameters;
|
||||||
|
size_t dictBufferCapacity = data->dictBufferCapacity;
|
||||||
|
size_t totalCompressedSize = ERROR(GENERIC);
|
||||||
|
/* Initialize array to keep track of frequency of dmer within activeSegment */
|
||||||
|
U16* segmentFreqs = (U16 *)calloc(((U64)1 << ctx->f), sizeof(U16));
|
||||||
|
/* Allocate space for hash table, dict, and freqs */
|
||||||
|
BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
|
||||||
|
U32 *freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32));
|
||||||
|
if (!segmentFreqs || !dict || !freqs) {
|
||||||
|
DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
|
||||||
|
goto _cleanup;
|
||||||
|
}
|
||||||
|
/* Copy the frequencies because we need to modify them */
|
||||||
|
memcpy(freqs, ctx->freqs, ((U64)1 << ctx->f) * sizeof(U32));
|
||||||
|
/* Build the dictionary */
|
||||||
|
{
|
||||||
|
const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, dictBufferCapacity,
|
||||||
|
parameters, segmentFreqs);
|
||||||
|
const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
|
||||||
|
dictBufferCapacity = ZDICT_finalizeDictionary(
|
||||||
|
dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
|
||||||
|
ctx->samples, ctx->samplesSizes, nbFinalizeSamples, parameters.zParams);
|
||||||
|
if (ZDICT_isError(dictBufferCapacity)) {
|
||||||
|
DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
|
||||||
|
goto _cleanup;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* Check total compressed size */
|
||||||
|
totalCompressedSize = COVER_checkTotalCompressedSize(parameters, ctx->samplesSizes,
|
||||||
|
ctx->samples, ctx->offsets,
|
||||||
|
ctx->nbTrainSamples, ctx->nbSamples,
|
||||||
|
dict, dictBufferCapacity);
|
||||||
|
_cleanup:
|
||||||
|
COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
|
||||||
|
dictBufferCapacity);
|
||||||
|
free(data);
|
||||||
|
free(segmentFreqs);
|
||||||
|
free(dict);
|
||||||
|
free(freqs);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
static void FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverParams,
|
||||||
|
ZDICT_cover_params_t *coverParams) {
|
||||||
|
coverParams->k = fastCoverParams.k;
|
||||||
|
coverParams->d = fastCoverParams.d;
|
||||||
|
coverParams->steps = fastCoverParams.steps;
|
||||||
|
coverParams->nbThreads = fastCoverParams.nbThreads;
|
||||||
|
coverParams->splitPoint = fastCoverParams.splitPoint;
|
||||||
|
coverParams->zParams = fastCoverParams.zParams;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static void FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams,
|
||||||
|
ZDICT_fastCover_params_t *fastCoverParams,
|
||||||
|
unsigned f, unsigned accel) {
|
||||||
|
fastCoverParams->k = coverParams.k;
|
||||||
|
fastCoverParams->d = coverParams.d;
|
||||||
|
fastCoverParams->steps = coverParams.steps;
|
||||||
|
fastCoverParams->nbThreads = coverParams.nbThreads;
|
||||||
|
fastCoverParams->splitPoint = coverParams.splitPoint;
|
||||||
|
fastCoverParams->f = f;
|
||||||
|
fastCoverParams->accel = accel;
|
||||||
|
fastCoverParams->zParams = coverParams.zParams;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(
|
||||||
|
void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
|
||||||
|
const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters) {
|
||||||
|
BYTE* const dict = (BYTE*)dictBuffer;
|
||||||
|
FASTCOVER_ctx_t ctx;
|
||||||
|
ZDICT_cover_params_t coverParams;
|
||||||
|
FASTCOVER_accel_t accelParams;
|
||||||
|
/* Initialize global data */
|
||||||
|
g_displayLevel = parameters.zParams.notificationLevel;
|
||||||
|
/* Assign splitPoint and f if not provided */
|
||||||
|
parameters.splitPoint = parameters.splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters.splitPoint;
|
||||||
|
parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f;
|
||||||
|
parameters.accel = parameters.accel == 0 ? DEFAULT_ACCEL : parameters.accel;
|
||||||
|
/* Convert to cover parameter */
|
||||||
|
memset(&coverParams, 0 , sizeof(coverParams));
|
||||||
|
FASTCOVER_convertToCoverParams(parameters, &coverParams);
|
||||||
|
/* Checks */
|
||||||
|
if (!FASTCOVER_checkParameters(coverParams, dictBufferCapacity, parameters.f,
|
||||||
|
parameters.accel)) {
|
||||||
|
DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n");
|
||||||
|
return ERROR(GENERIC);
|
||||||
|
}
|
||||||
|
if (nbSamples == 0) {
|
||||||
|
DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
|
||||||
|
return ERROR(GENERIC);
|
||||||
|
}
|
||||||
|
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
|
||||||
|
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
|
||||||
|
ZDICT_DICTSIZE_MIN);
|
||||||
|
return ERROR(dstSize_tooSmall);
|
||||||
|
}
|
||||||
|
/* Assign corresponding FASTCOVER_accel_t to accelParams*/
|
||||||
|
accelParams = FASTCOVER_defaultAccelParameters[parameters.accel];
|
||||||
|
/* Initialize context */
|
||||||
|
if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
|
||||||
|
coverParams.d, parameters.splitPoint, parameters.f,
|
||||||
|
accelParams)) {
|
||||||
|
DISPLAYLEVEL(1, "Failed to initialize context\n");
|
||||||
|
return ERROR(GENERIC);
|
||||||
|
}
|
||||||
|
/* Build the dictionary */
|
||||||
|
DISPLAYLEVEL(2, "Building dictionary\n");
|
||||||
|
{
|
||||||
|
/* Initialize array to keep track of frequency of dmer within activeSegment */
|
||||||
|
U16* segmentFreqs = (U16 *)calloc(((U64)1 << parameters.f), sizeof(U16));
|
||||||
|
const size_t tail = FASTCOVER_buildDictionary(&ctx, ctx.freqs, dictBuffer,
|
||||||
|
dictBufferCapacity, coverParams, segmentFreqs);
|
||||||
|
const unsigned nbFinalizeSamples = (unsigned)(ctx.nbTrainSamples * ctx.accelParams.finalize / 100);
|
||||||
|
const size_t dictionarySize = ZDICT_finalizeDictionary(
|
||||||
|
dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
|
||||||
|
samplesBuffer, samplesSizes, nbFinalizeSamples, coverParams.zParams);
|
||||||
|
if (!ZSTD_isError(dictionarySize)) {
|
||||||
|
DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
|
||||||
|
(U32)dictionarySize);
|
||||||
|
}
|
||||||
|
FASTCOVER_ctx_destroy(&ctx);
|
||||||
|
free(segmentFreqs);
|
||||||
|
return dictionarySize;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(
|
||||||
|
void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer,
|
||||||
|
const size_t *samplesSizes, unsigned nbSamples,
|
||||||
|
ZDICT_fastCover_params_t *parameters) {
|
||||||
|
ZDICT_cover_params_t coverParams;
|
||||||
|
FASTCOVER_accel_t accelParams;
|
||||||
|
/* constants */
|
||||||
|
const unsigned nbThreads = parameters->nbThreads;
|
||||||
|
const double splitPoint =
|
||||||
|
parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
|
||||||
|
const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
|
||||||
|
const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
|
||||||
|
const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
|
||||||
|
const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k;
|
||||||
|
const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps;
|
||||||
|
const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
|
||||||
|
const unsigned kIterations =
|
||||||
|
(1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
|
||||||
|
const unsigned f = parameters->f == 0 ? DEFAULT_F : parameters->f;
|
||||||
|
const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel;
|
||||||
|
/* Local variables */
|
||||||
|
const int displayLevel = parameters->zParams.notificationLevel;
|
||||||
|
unsigned iteration = 1;
|
||||||
|
unsigned d;
|
||||||
|
unsigned k;
|
||||||
|
COVER_best_t best;
|
||||||
|
POOL_ctx *pool = NULL;
|
||||||
|
/* Checks */
|
||||||
|
if (splitPoint <= 0 || splitPoint > 1) {
|
||||||
|
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n");
|
||||||
|
return ERROR(GENERIC);
|
||||||
|
}
|
||||||
|
if (accel == 0 || accel > FASTCOVER_MAX_ACCEL) {
|
||||||
|
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect accel\n");
|
||||||
|
return ERROR(GENERIC);
|
||||||
|
}
|
||||||
|
if (kMinK < kMaxD || kMaxK < kMinK) {
|
||||||
|
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n");
|
||||||
|
return ERROR(GENERIC);
|
||||||
|
}
|
||||||
|
if (nbSamples == 0) {
|
||||||
|
LOCALDISPLAYLEVEL(displayLevel, 1, "FASTCOVER must have at least one input file\n");
|
||||||
|
return ERROR(GENERIC);
|
||||||
|
}
|
||||||
|
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
|
||||||
|
LOCALDISPLAYLEVEL(displayLevel, 1, "dictBufferCapacity must be at least %u\n",
|
||||||
|
ZDICT_DICTSIZE_MIN);
|
||||||
|
return ERROR(dstSize_tooSmall);
|
||||||
|
}
|
||||||
|
if (nbThreads > 1) {
|
||||||
|
pool = POOL_create(nbThreads, 1);
|
||||||
|
if (!pool) {
|
||||||
|
return ERROR(memory_allocation);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* Initialization */
|
||||||
|
COVER_best_init(&best);
|
||||||
|
memset(&coverParams, 0 , sizeof(coverParams));
|
||||||
|
FASTCOVER_convertToCoverParams(*parameters, &coverParams);
|
||||||
|
accelParams = FASTCOVER_defaultAccelParameters[accel];
|
||||||
|
/* Turn down global display level to clean up display at level 2 and below */
|
||||||
|
g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
|
||||||
|
/* Loop through d first because each new value needs a new context */
|
||||||
|
LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
|
||||||
|
kIterations);
|
||||||
|
for (d = kMinD; d <= kMaxD; d += 2) {
|
||||||
|
/* Initialize the context for this value of d */
|
||||||
|
FASTCOVER_ctx_t ctx;
|
||||||
|
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
|
||||||
|
if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams)) {
|
||||||
|
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
|
||||||
|
COVER_best_destroy(&best);
|
||||||
|
POOL_free(pool);
|
||||||
|
return ERROR(GENERIC);
|
||||||
|
}
|
||||||
|
/* Loop through k reusing the same context */
|
||||||
|
for (k = kMinK; k <= kMaxK; k += kStepSize) {
|
||||||
|
/* Prepare the arguments */
|
||||||
|
FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc(
|
||||||
|
sizeof(FASTCOVER_tryParameters_data_t));
|
||||||
|
LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
|
||||||
|
if (!data) {
|
||||||
|
LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
|
||||||
|
COVER_best_destroy(&best);
|
||||||
|
FASTCOVER_ctx_destroy(&ctx);
|
||||||
|
POOL_free(pool);
|
||||||
|
return ERROR(GENERIC);
|
||||||
|
}
|
||||||
|
data->ctx = &ctx;
|
||||||
|
data->best = &best;
|
||||||
|
data->dictBufferCapacity = dictBufferCapacity;
|
||||||
|
data->parameters = coverParams;
|
||||||
|
data->parameters.k = k;
|
||||||
|
data->parameters.d = d;
|
||||||
|
data->parameters.splitPoint = splitPoint;
|
||||||
|
data->parameters.steps = kSteps;
|
||||||
|
data->parameters.zParams.notificationLevel = g_displayLevel;
|
||||||
|
/* Check the parameters */
|
||||||
|
if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity,
|
||||||
|
data->ctx->f, accel)) {
|
||||||
|
DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n");
|
||||||
|
free(data);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
/* Call the function and pass ownership of data to it */
|
||||||
|
COVER_best_start(&best);
|
||||||
|
if (pool) {
|
||||||
|
POOL_add(pool, &FASTCOVER_tryParameters, data);
|
||||||
|
} else {
|
||||||
|
FASTCOVER_tryParameters(data);
|
||||||
|
}
|
||||||
|
/* Print status */
|
||||||
|
LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
|
||||||
|
(U32)((iteration * 100) / kIterations));
|
||||||
|
++iteration;
|
||||||
|
}
|
||||||
|
COVER_best_wait(&best);
|
||||||
|
FASTCOVER_ctx_destroy(&ctx);
|
||||||
|
}
|
||||||
|
LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
|
||||||
|
/* Fill the output buffer and parameters with output of the best parameters */
|
||||||
|
{
|
||||||
|
const size_t dictSize = best.dictSize;
|
||||||
|
if (ZSTD_isError(best.compressedSize)) {
|
||||||
|
const size_t compressedSize = best.compressedSize;
|
||||||
|
COVER_best_destroy(&best);
|
||||||
|
POOL_free(pool);
|
||||||
|
return compressedSize;
|
||||||
|
}
|
||||||
|
FASTCOVER_convertToFastCoverParams(best.parameters, parameters, f, accel);
|
||||||
|
memcpy(dictBuffer, best.dict, dictSize);
|
||||||
|
COVER_best_destroy(&best);
|
||||||
|
POOL_free(pool);
|
||||||
|
return dictSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -698,7 +698,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
|||||||
short litLengthNCount[MaxLL+1];
|
short litLengthNCount[MaxLL+1];
|
||||||
U32 repOffset[MAXREPOFFSET];
|
U32 repOffset[MAXREPOFFSET];
|
||||||
offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
|
offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
|
||||||
EStats_ress_t esr;
|
EStats_ress_t esr = { NULL, NULL, NULL };
|
||||||
ZSTD_parameters params;
|
ZSTD_parameters params;
|
||||||
U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total;
|
U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total;
|
||||||
size_t pos = 0, errorCode;
|
size_t pos = 0, errorCode;
|
||||||
@@ -863,8 +863,8 @@ _cleanup:
|
|||||||
|
|
||||||
size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
|
||||||
const void* customDictContent, size_t dictContentSize,
|
const void* customDictContent, size_t dictContentSize,
|
||||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
const void* samplesBuffer, const size_t* samplesSizes,
|
||||||
ZDICT_params_t params)
|
unsigned nbSamples, ZDICT_params_t params)
|
||||||
{
|
{
|
||||||
size_t hSize;
|
size_t hSize;
|
||||||
#define HBUFFSIZE 256 /* should prove large enough for all entropy headers */
|
#define HBUFFSIZE 256 /* should prove large enough for all entropy headers */
|
||||||
@@ -1078,17 +1078,17 @@ size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity,
|
|||||||
size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
||||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples)
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples)
|
||||||
{
|
{
|
||||||
ZDICT_cover_params_t params;
|
ZDICT_fastCover_params_t params;
|
||||||
DEBUGLOG(3, "ZDICT_trainFromBuffer");
|
DEBUGLOG(3, "ZDICT_trainFromBuffer");
|
||||||
memset(¶ms, 0, sizeof(params));
|
memset(¶ms, 0, sizeof(params));
|
||||||
params.d = 8;
|
params.d = 8;
|
||||||
params.steps = 4;
|
params.steps = 4;
|
||||||
/* Default to level 6 since no compression level information is available */
|
/* Default to level 6 since no compression level information is available */
|
||||||
params.zParams.compressionLevel = 6;
|
params.zParams.compressionLevel = 3;
|
||||||
#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1)
|
#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1)
|
||||||
params.zParams.notificationLevel = DEBUGLEVEL;
|
params.zParams.notificationLevel = DEBUGLEVEL;
|
||||||
#endif
|
#endif
|
||||||
return ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, dictBufferCapacity,
|
return ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, dictBufferCapacity,
|
||||||
samplesBuffer, samplesSizes, nbSamples,
|
samplesBuffer, samplesSizes, nbSamples,
|
||||||
¶ms);
|
¶ms);
|
||||||
}
|
}
|
||||||
|
|||||||
+56
-3
@@ -39,7 +39,8 @@ extern "C" {
|
|||||||
|
|
||||||
/*! ZDICT_trainFromBuffer():
|
/*! ZDICT_trainFromBuffer():
|
||||||
* Train a dictionary from an array of samples.
|
* Train a dictionary from an array of samples.
|
||||||
* Redirect towards ZDICT_optimizeTrainFromBuffer_cover() single-threaded, with d=8 and steps=4.
|
* Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4,
|
||||||
|
* f=18, and accel=1.
|
||||||
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
||||||
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
||||||
* The resulting dictionary will be saved into `dictBuffer`.
|
* The resulting dictionary will be saved into `dictBuffer`.
|
||||||
@@ -90,6 +91,16 @@ typedef struct {
|
|||||||
ZDICT_params_t zParams;
|
ZDICT_params_t zParams;
|
||||||
} ZDICT_cover_params_t;
|
} ZDICT_cover_params_t;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */
|
||||||
|
unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */
|
||||||
|
unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(18)*/
|
||||||
|
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */
|
||||||
|
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
|
||||||
|
double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
|
||||||
|
unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
|
||||||
|
ZDICT_params_t zParams;
|
||||||
|
} ZDICT_fastCover_params_t;
|
||||||
|
|
||||||
/*! ZDICT_trainFromBuffer_cover():
|
/*! ZDICT_trainFromBuffer_cover():
|
||||||
* Train a dictionary from an array of samples using the COVER algorithm.
|
* Train a dictionary from an array of samples using the COVER algorithm.
|
||||||
@@ -116,9 +127,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
|||||||
* dictionary constructed with those parameters is stored in `dictBuffer`.
|
* dictionary constructed with those parameters is stored in `dictBuffer`.
|
||||||
*
|
*
|
||||||
* All of the parameters d, k, steps are optional.
|
* All of the parameters d, k, steps are optional.
|
||||||
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}.
|
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
|
||||||
* if steps is zero it defaults to its default value.
|
* if steps is zero it defaults to its default value.
|
||||||
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048].
|
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
|
||||||
*
|
*
|
||||||
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
||||||
* or an error code, which can be tested with ZDICT_isError().
|
* or an error code, which can be tested with ZDICT_isError().
|
||||||
@@ -130,6 +141,48 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
|
|||||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||||
ZDICT_cover_params_t* parameters);
|
ZDICT_cover_params_t* parameters);
|
||||||
|
|
||||||
|
/*! ZDICT_trainFromBuffer_fastCover():
|
||||||
|
* Train a dictionary from an array of samples using a modified version of COVER algorithm.
|
||||||
|
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
|
||||||
|
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
|
||||||
|
* d and k are required.
|
||||||
|
* All other parameters are optional, will use default values if not provided
|
||||||
|
* The resulting dictionary will be saved into `dictBuffer`.
|
||||||
|
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
||||||
|
* or an error code, which can be tested with ZDICT_isError().
|
||||||
|
* Note: ZDICT_trainFromBuffer_fastCover() requires about 1 bytes of memory for each input byte and additionally another 6 * 2^f bytes of memory .
|
||||||
|
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
|
||||||
|
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
|
||||||
|
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
|
||||||
|
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
|
||||||
|
*/
|
||||||
|
ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer,
|
||||||
|
size_t dictBufferCapacity, const void *samplesBuffer,
|
||||||
|
const size_t *samplesSizes, unsigned nbSamples,
|
||||||
|
ZDICT_fastCover_params_t parameters);
|
||||||
|
|
||||||
|
/*! ZDICT_optimizeTrainFromBuffer_fastCover():
|
||||||
|
* The same requirements as above hold for all the parameters except `parameters`.
|
||||||
|
* This function tries many parameter combinations (specifically, k and d combinations)
|
||||||
|
* and picks the best parameters. `*parameters` is filled with the best parameters found,
|
||||||
|
* dictionary constructed with those parameters is stored in `dictBuffer`.
|
||||||
|
* All of the parameters d, k, steps, f, and accel are optional.
|
||||||
|
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
|
||||||
|
* if steps is zero it defaults to its default value.
|
||||||
|
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
|
||||||
|
* If f is zero, default value of 18 is used.
|
||||||
|
* If accel is zero, default value of 1 is used.
|
||||||
|
*
|
||||||
|
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
|
||||||
|
* or an error code, which can be tested with ZDICT_isError().
|
||||||
|
* On success `*parameters` contains the parameters selected.
|
||||||
|
* Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 1 byte of memory for each input byte and additionally another 6 * 2^f bytes of memory for each thread.
|
||||||
|
*/
|
||||||
|
ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer,
|
||||||
|
size_t dictBufferCapacity, const void* samplesBuffer,
|
||||||
|
const size_t* samplesSizes, unsigned nbSamples,
|
||||||
|
ZDICT_fastCover_params_t* parameters);
|
||||||
|
|
||||||
/*! ZDICT_finalizeDictionary():
|
/*! ZDICT_finalizeDictionary():
|
||||||
* Given a custom content as a basis for dictionary, and a set of samples,
|
* Given a custom content as a basis for dictionary, and a set of samples,
|
||||||
* finalize dictionary by adding headers and statistics.
|
* finalize dictionary by adding headers and statistics.
|
||||||
|
|||||||
@@ -1093,6 +1093,7 @@ static size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, un
|
|||||||
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||||
|
|
||||||
/* Init, lay down lowprob symbols */
|
/* Init, lay down lowprob symbols */
|
||||||
|
memset(tableDecode, 0, sizeof(FSE_DECODE_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
|
||||||
DTableH.tableLog = (U16)tableLog;
|
DTableH.tableLog = (U16)tableLog;
|
||||||
for (s=0; s<=maxSymbolValue; s++)
|
for (s=0; s<=maxSymbolValue; s++)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1224,6 +1224,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns
|
|||||||
if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||||
|
|
||||||
/* Init, lay down lowprob symbols */
|
/* Init, lay down lowprob symbols */
|
||||||
|
memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */
|
||||||
DTableH.tableLog = (U16)tableLog;
|
DTableH.tableLog = (U16)tableLog;
|
||||||
for (s=0; s<=maxSymbolValue; s++) {
|
for (s=0; s<=maxSymbolValue; s++) {
|
||||||
if (normalizedCounter[s]==-1) {
|
if (normalizedCounter[s]==-1) {
|
||||||
@@ -2845,6 +2846,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|||||||
|
|
||||||
static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
|
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
||||||
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
||||||
memcpy(dst, src, srcSize);
|
memcpy(dst, src, srcSize);
|
||||||
return srcSize;
|
return srcSize;
|
||||||
|
|||||||
@@ -3041,6 +3041,7 @@ size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
|||||||
|
|
||||||
static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
|
if (dst==NULL) return ERROR(dstSize_tooSmall);
|
||||||
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||||
memcpy(dst, src, srcSize);
|
memcpy(dst, src, srcSize);
|
||||||
return srcSize;
|
return srcSize;
|
||||||
@@ -4006,7 +4007,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
|||||||
if (ZSTDv06_isError(hSize)) return hSize;
|
if (ZSTDv06_isError(hSize)) return hSize;
|
||||||
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
||||||
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
|
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
|
||||||
zbd->lhSize += iend-ip; ip = iend; notDone = 0;
|
zbd->lhSize += iend-ip;
|
||||||
*dstCapacityPtr = 0;
|
*dstCapacityPtr = 0;
|
||||||
return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */
|
return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */
|
||||||
}
|
}
|
||||||
|
|||||||
+11
-9
@@ -3150,10 +3150,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
|
|
||||||
if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min;
|
if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min;
|
||||||
|
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
||||||
if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
|
if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
|
||||||
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) {
|
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) {
|
||||||
if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */
|
if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */
|
||||||
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
|
||||||
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
||||||
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
|
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
|
||||||
return 0;
|
return 0;
|
||||||
@@ -3175,11 +3175,13 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|||||||
U32 windowSize = 0;
|
U32 windowSize = 0;
|
||||||
U32 dictID = 0;
|
U32 dictID = 0;
|
||||||
U64 frameContentSize = 0;
|
U64 frameContentSize = 0;
|
||||||
if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
|
if ((fhdByte & 0x08) != 0) /* reserved bits, which must be zero */
|
||||||
|
return ERROR(frameParameter_unsupported);
|
||||||
if (!directMode) {
|
if (!directMode) {
|
||||||
BYTE const wlByte = ip[pos++];
|
BYTE const wlByte = ip[pos++];
|
||||||
U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN;
|
U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN;
|
||||||
if (windowLog > ZSTDv07_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
|
if (windowLog > ZSTDv07_WINDOWLOG_MAX)
|
||||||
|
return ERROR(frameParameter_unsupported);
|
||||||
windowSize = (1U << windowLog);
|
windowSize = (1U << windowLog);
|
||||||
windowSize += (windowSize >> 3) * (wlByte&7);
|
windowSize += (windowSize >> 3) * (wlByte&7);
|
||||||
}
|
}
|
||||||
@@ -3201,7 +3203,8 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|||||||
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
|
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
|
||||||
}
|
}
|
||||||
if (!windowSize) windowSize = (U32)frameContentSize;
|
if (!windowSize) windowSize = (U32)frameContentSize;
|
||||||
if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
|
if (windowSize > windowSizeMax)
|
||||||
|
return ERROR(frameParameter_unsupported);
|
||||||
fparamsPtr->frameContentSize = frameContentSize;
|
fparamsPtr->frameContentSize = frameContentSize;
|
||||||
fparamsPtr->windowSize = windowSize;
|
fparamsPtr->windowSize = windowSize;
|
||||||
fparamsPtr->dictID = dictID;
|
fparamsPtr->dictID = dictID;
|
||||||
@@ -3220,11 +3223,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src,
|
|||||||
- frame header not completely provided (`srcSize` too small) */
|
- frame header not completely provided (`srcSize` too small) */
|
||||||
unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize)
|
unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
{ ZSTDv07_frameParams fparams;
|
ZSTDv07_frameParams fparams;
|
||||||
size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize);
|
size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize);
|
||||||
if (frResult!=0) return 0;
|
if (frResult!=0) return 0;
|
||||||
return fparams.frameContentSize;
|
return fparams.frameContentSize;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -38,6 +38,9 @@ endif
|
|||||||
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
||||||
-I$(ZSTDDIR)/dictBuilder \
|
-I$(ZSTDDIR)/dictBuilder \
|
||||||
-DXXH_NAMESPACE=ZSTD_
|
-DXXH_NAMESPACE=ZSTD_
|
||||||
|
ifeq ($(OS),Windows_NT) # MinGW assumed
|
||||||
|
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
|
||||||
|
endif
|
||||||
CFLAGS ?= -O3
|
CFLAGS ?= -O3
|
||||||
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||||
|
|||||||
+23
-8
@@ -151,6 +151,7 @@ Advanced arguments :
|
|||||||
Dictionary builder :
|
Dictionary builder :
|
||||||
--train ## : create a dictionary from a training set of files
|
--train ## : create a dictionary from a training set of files
|
||||||
--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args
|
--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args
|
||||||
|
--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fastcover algorithm with optional args
|
||||||
--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9)
|
--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9)
|
||||||
-o file : `file` is dictionary name (default: dictionary)
|
-o file : `file` is dictionary name (default: dictionary)
|
||||||
--maxdict=# : limit dictionary to specified size (default: 112640)
|
--maxdict=# : limit dictionary to specified size (default: 112640)
|
||||||
@@ -185,7 +186,7 @@ version is less than `128 MiB`).
|
|||||||
|
|
||||||
Compression Speed vs Ratio | Decompression Speed
|
Compression Speed vs Ratio | Decompression Speed
|
||||||
---------------------------|---------------------
|
---------------------------|---------------------
|
||||||
 | 
|
 | 
|
||||||
|
|
||||||
| Method | Compression ratio | Compression speed | Decompression speed |
|
| Method | Compression ratio | Compression speed | Decompression speed |
|
||||||
|:-------|------------------:|-------------------------:|---------------------------:|
|
|:-------|------------------:|-------------------------:|---------------------------:|
|
||||||
@@ -208,10 +209,24 @@ The below table illustrates this on the [Silesia compression corpus].
|
|||||||
[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
|
[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
|
||||||
|
|
||||||
| Method | Compression ratio | Compression speed | Decompression speed |
|
| Method | Compression ratio | Compression speed | Decompression speed |
|
||||||
|:-------|------------------:|-------------------------:|---------------------------:|
|
|:-------|------------------:|------------------:|---------------------:|
|
||||||
| `zstd -1` | `2.878` | `231.7 MB/s` | `594.4 MB/s` |
|
| `zstd -1` | `2.878` | `231.7 MB/s` | `594.4 MB/s` |
|
||||||
| `zstd -1 --long` | `2.929` | `106.5 MB/s` | `517.9 MB/s` |
|
| `zstd -1 --long` | `2.929` | `106.5 MB/s` | `517.9 MB/s` |
|
||||||
| `zstd -5` | `3.274` | `77.1 MB/s` | `464.2 MB/s` |
|
| `zstd -5` | `3.274` | `77.1 MB/s` | `464.2 MB/s` |
|
||||||
| `zstd -5 --long` | `3.319` | `51.7 MB/s` | `371.9 MB/s` |
|
| `zstd -5 --long` | `3.319` | `51.7 MB/s` | `371.9 MB/s` |
|
||||||
| `zstd -10` | `3.523` | `16.4 MB/s` | `489.2 MB/s` |
|
| `zstd -10` | `3.523` | `16.4 MB/s` | `489.2 MB/s` |
|
||||||
| `zstd -10 --long`| `3.566` | `16.2 MB/s` | `415.7 MB/s` |
|
| `zstd -10 --long`| `3.566` | `16.2 MB/s` | `415.7 MB/s` |
|
||||||
|
|
||||||
|
|
||||||
|
#### zstdgrep
|
||||||
|
|
||||||
|
`zstdgrep` is a utility which makes it possible to `grep` directly a `.zst` compressed file.
|
||||||
|
It's used the same way as normal `grep`, for example :
|
||||||
|
`zstdgrep pattern file.zst`
|
||||||
|
|
||||||
|
`zstdgrep` is _not_ compatible with dictionary compression.
|
||||||
|
|
||||||
|
To search into a file compressed with a dictionary,
|
||||||
|
it's necessary to decompress it using `zstd` or `zstdcat`,
|
||||||
|
and then pipe the result to `grep`. For example :
|
||||||
|
`zstdcat -D dictionary -qc -- file.zst | grep pattern`
|
||||||
|
|||||||
+123
-113
@@ -171,6 +171,8 @@ struct BMK_timeState_t{
|
|||||||
static void BMK_initCCtx(ZSTD_CCtx* ctx,
|
static void BMK_initCCtx(ZSTD_CCtx* ctx,
|
||||||
const void* dictBuffer, size_t dictBufferSize, int cLevel,
|
const void* dictBuffer, size_t dictBufferSize, int cLevel,
|
||||||
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
|
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
|
||||||
|
ZSTD_CCtx_reset(ctx);
|
||||||
|
ZSTD_CCtx_resetParameters(ctx);
|
||||||
if (adv->nbWorkers==1) {
|
if (adv->nbWorkers==1) {
|
||||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
|
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0);
|
||||||
} else {
|
} else {
|
||||||
@@ -195,6 +197,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
|
|||||||
|
|
||||||
static void BMK_initDCtx(ZSTD_DCtx* dctx,
|
static void BMK_initDCtx(ZSTD_DCtx* dctx,
|
||||||
const void* dictBuffer, size_t dictBufferSize) {
|
const void* dictBuffer, size_t dictBufferSize) {
|
||||||
|
ZSTD_DCtx_reset(dctx);
|
||||||
ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
|
ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -291,34 +294,34 @@ BMK_customReturn_t BMK_benchFunction(
|
|||||||
BMK_initFn_t initFn, void* initPayload,
|
BMK_initFn_t initFn, void* initPayload,
|
||||||
size_t blockCount,
|
size_t blockCount,
|
||||||
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
|
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
|
||||||
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities,
|
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResult,
|
||||||
unsigned nbLoops) {
|
unsigned nbLoops) {
|
||||||
size_t srcSize = 0, dstSize = 0, ind = 0;
|
size_t dstSize = 0;
|
||||||
U64 totalTime;
|
U64 totalTime;
|
||||||
|
|
||||||
BMK_customReturn_t retval;
|
BMK_customReturn_t retval;
|
||||||
UTIL_time_t clockStart;
|
UTIL_time_t clockStart;
|
||||||
|
|
||||||
{
|
|
||||||
unsigned i;
|
|
||||||
for(i = 0; i < blockCount; i++) {
|
|
||||||
memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */
|
|
||||||
}
|
|
||||||
|
|
||||||
UTIL_sleepMilli(5); /* give processor time to other processes */
|
|
||||||
UTIL_waitForNextTick();
|
|
||||||
}
|
|
||||||
|
|
||||||
if(!nbLoops) {
|
if(!nbLoops) {
|
||||||
EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n");
|
EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n");
|
||||||
}
|
}
|
||||||
|
|
||||||
for(ind = 0; ind < blockCount; ind++) {
|
{
|
||||||
srcSize += srcBlockSizes[ind];
|
size_t i;
|
||||||
|
for(i = 0; i < blockCount; i++) {
|
||||||
|
memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */
|
||||||
|
}
|
||||||
|
#if 0
|
||||||
|
/* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
|
||||||
|
* (Makes former slower)
|
||||||
|
*/
|
||||||
|
UTIL_sleepMilli(5); /* give processor time to other processes */
|
||||||
|
UTIL_waitForNextTick();
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
unsigned i, j, firstIter = 1;
|
unsigned i, j;
|
||||||
clockStart = UTIL_getTime();
|
clockStart = UTIL_getTime();
|
||||||
if(initFn != NULL) { initFn(initPayload); }
|
if(initFn != NULL) { initFn(initPayload); }
|
||||||
for(i = 0; i < nbLoops; i++) {
|
for(i = 0; i < nbLoops; i++) {
|
||||||
@@ -327,11 +330,13 @@ BMK_customReturn_t BMK_benchFunction(
|
|||||||
if(ZSTD_isError(res)) {
|
if(ZSTD_isError(res)) {
|
||||||
EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n",
|
EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n",
|
||||||
j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res));
|
j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res));
|
||||||
} else if(firstIter) {
|
} else if(i == nbLoops - 1) {
|
||||||
dstSize += res;
|
dstSize += res;
|
||||||
|
if(blockResult != NULL) {
|
||||||
|
blockResult[j] = res;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
firstIter = 0;
|
|
||||||
}
|
}
|
||||||
totalTime = UTIL_clockSpanNano(clockStart);
|
totalTime = UTIL_clockSpanNano(clockStart);
|
||||||
}
|
}
|
||||||
@@ -353,6 +358,9 @@ void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) {
|
|||||||
|
|
||||||
BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) {
|
BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) {
|
||||||
BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t));
|
BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t));
|
||||||
|
if(r == NULL) {
|
||||||
|
return r;
|
||||||
|
}
|
||||||
BMK_resetTimeState(r, nbSeconds);
|
BMK_resetTimeState(r, nbSeconds);
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
@@ -361,13 +369,14 @@ void BMK_freeTimeState(BMK_timedFnState_t* state) {
|
|||||||
free(state);
|
free(state);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* make option for dstBlocks to be */
|
||||||
BMK_customTimedReturn_t BMK_benchFunctionTimed(
|
BMK_customTimedReturn_t BMK_benchFunctionTimed(
|
||||||
BMK_timedFnState_t* cont,
|
BMK_timedFnState_t* cont,
|
||||||
BMK_benchFn_t benchFn, void* benchPayload,
|
BMK_benchFn_t benchFn, void* benchPayload,
|
||||||
BMK_initFn_t initFn, void* initPayload,
|
BMK_initFn_t initFn, void* initPayload,
|
||||||
size_t blockCount,
|
size_t blockCount,
|
||||||
const void* const* const srcBlockBuffers, const size_t* srcBlockSizes,
|
const void* const* const srcBlockBuffers, const size_t* srcBlockSizes,
|
||||||
void* const* const dstBlockBuffers, const size_t* dstBlockCapacities)
|
void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* blockResults)
|
||||||
{
|
{
|
||||||
U64 fastest = cont->fastestTime;
|
U64 fastest = cont->fastestTime;
|
||||||
int completed = 0;
|
int completed = 0;
|
||||||
@@ -382,9 +391,9 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
|
|||||||
UTIL_sleep(COOLPERIOD_SEC);
|
UTIL_sleep(COOLPERIOD_SEC);
|
||||||
cont->coolTime = UTIL_getTime();
|
cont->coolTime = UTIL_getTime();
|
||||||
}
|
}
|
||||||
|
/* reinitialize capacity */
|
||||||
r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload,
|
r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload,
|
||||||
blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->nbLoops);
|
blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, blockResults, cont->nbLoops);
|
||||||
if(r.result.error) { /* completed w/ error */
|
if(r.result.error) { /* completed w/ error */
|
||||||
r.completed = 1;
|
r.completed = 1;
|
||||||
return r;
|
return r;
|
||||||
@@ -393,7 +402,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
|
|||||||
{ U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops;
|
{ U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops;
|
||||||
r.completed = (cont->timeRemaining <= loopDuration);
|
r.completed = (cont->timeRemaining <= loopDuration);
|
||||||
cont->timeRemaining -= loopDuration;
|
cont->timeRemaining -= loopDuration;
|
||||||
if (loopDuration > 0) {
|
if (loopDuration > (TIMELOOP_NANOSEC / 100)) {
|
||||||
fastest = MIN(fastest, r.result.result.nanoSecPerRun);
|
fastest = MIN(fastest, r.result.result.nanoSecPerRun);
|
||||||
if(loopDuration >= MINUSABLETIME) {
|
if(loopDuration >= MINUSABLETIME) {
|
||||||
r.result.result.nanoSecPerRun = fastest;
|
r.result.result.nanoSecPerRun = fastest;
|
||||||
@@ -418,9 +427,9 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(
|
|||||||
/* benchMem with no allocation */
|
/* benchMem with no allocation */
|
||||||
static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
||||||
const void ** const srcPtrs, size_t* const srcSizes,
|
const void ** const srcPtrs, size_t* const srcSizes,
|
||||||
void** const cPtrs, size_t* const cSizes,
|
void** const cPtrs, size_t* const cCapacities, size_t* const cSizes,
|
||||||
void** const resPtrs, size_t* const resSizes,
|
void** const resPtrs, size_t* const resSizes,
|
||||||
void* resultBuffer, void* compressedBuffer,
|
void** resultBufferPtr, void* compressedBuffer,
|
||||||
const size_t maxCompressedSize,
|
const size_t maxCompressedSize,
|
||||||
BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress,
|
BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress,
|
||||||
|
|
||||||
@@ -432,7 +441,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
|||||||
int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
|
int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
|
||||||
{
|
{
|
||||||
size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
|
size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
|
||||||
BMK_return_t results;
|
BMK_return_t results = { { 0, 0, 0, 0 }, 0 } ;
|
||||||
size_t const loadedCompressedSize = srcSize;
|
size_t const loadedCompressedSize = srcSize;
|
||||||
size_t cSize = 0;
|
size_t cSize = 0;
|
||||||
double ratio = 0.;
|
double ratio = 0.;
|
||||||
@@ -454,13 +463,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
|||||||
srcPtr += fileSizes[fileNb];
|
srcPtr += fileSizes[fileNb];
|
||||||
}
|
}
|
||||||
{ size_t const decodedSize = (size_t)totalDSize64;
|
{ size_t const decodedSize = (size_t)totalDSize64;
|
||||||
free(resultBuffer);
|
free(*resultBufferPtr);
|
||||||
resultBuffer = malloc(decodedSize);
|
*resultBufferPtr = malloc(decodedSize);
|
||||||
if (!resultBuffer) {
|
if (!(*resultBufferPtr)) {
|
||||||
EXM_THROW(33, BMK_return_t, "not enough memory");
|
EXM_THROW(33, BMK_return_t, "not enough memory");
|
||||||
}
|
}
|
||||||
if (totalDSize64 > decodedSize) {
|
if (totalDSize64 > decodedSize) {
|
||||||
free(resultBuffer);
|
free(*resultBufferPtr);
|
||||||
EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */
|
EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */
|
||||||
}
|
}
|
||||||
cSize = srcSize;
|
cSize = srcSize;
|
||||||
@@ -472,7 +481,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
|||||||
/* Init data blocks */
|
/* Init data blocks */
|
||||||
{ const char* srcPtr = (const char*)srcBuffer;
|
{ const char* srcPtr = (const char*)srcBuffer;
|
||||||
char* cPtr = (char*)compressedBuffer;
|
char* cPtr = (char*)compressedBuffer;
|
||||||
char* resPtr = (char*)resultBuffer;
|
char* resPtr = (char*)(*resultBufferPtr);
|
||||||
U32 fileNb;
|
U32 fileNb;
|
||||||
for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
|
for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
|
||||||
size_t remaining = fileSizes[fileNb];
|
size_t remaining = fileSizes[fileNb];
|
||||||
@@ -483,11 +492,11 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
|||||||
srcPtrs[nbBlocks] = (const void*)srcPtr;
|
srcPtrs[nbBlocks] = (const void*)srcPtr;
|
||||||
srcSizes[nbBlocks] = thisBlockSize;
|
srcSizes[nbBlocks] = thisBlockSize;
|
||||||
cPtrs[nbBlocks] = (void*)cPtr;
|
cPtrs[nbBlocks] = (void*)cPtr;
|
||||||
cSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
|
cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
|
||||||
resPtrs[nbBlocks] = (void*)resPtr;
|
resPtrs[nbBlocks] = (void*)resPtr;
|
||||||
resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
|
resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
|
||||||
srcPtr += thisBlockSize;
|
srcPtr += thisBlockSize;
|
||||||
cPtr += cSizes[nbBlocks];
|
cPtr += cCapacities[nbBlocks];
|
||||||
resPtr += thisBlockSize;
|
resPtr += thisBlockSize;
|
||||||
remaining -= thisBlockSize;
|
remaining -= thisBlockSize;
|
||||||
}
|
}
|
||||||
@@ -538,7 +547,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
|||||||
while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) {
|
while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) {
|
||||||
if(!intermediateResultCompress.completed) {
|
if(!intermediateResultCompress.completed) {
|
||||||
intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
|
intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
|
||||||
nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes);
|
nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes);
|
||||||
if(intermediateResultCompress.result.error) {
|
if(intermediateResultCompress.result.error) {
|
||||||
results.error = intermediateResultCompress.result.error;
|
results.error = intermediateResultCompress.result.error;
|
||||||
return results;
|
return results;
|
||||||
@@ -546,9 +555,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
|||||||
ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn);
|
ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn);
|
||||||
{
|
{
|
||||||
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||||
double const compressionSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000;
|
results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / intermediateResultCompress.result.result.nanoSecPerRun);
|
||||||
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
|
|
||||||
results.result.cSpeed = compressionSpeed * 1000000;
|
|
||||||
cSize = intermediateResultCompress.result.result.sumOfReturn;
|
cSize = intermediateResultCompress.result.result.sumOfReturn;
|
||||||
results.result.cSize = cSize;
|
results.result.cSize = cSize;
|
||||||
ratio = (double)srcSize / results.result.cSize;
|
ratio = (double)srcSize / results.result.cSize;
|
||||||
@@ -556,13 +563,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
|||||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
|
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
|
||||||
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
||||||
ratioAccuracy, ratio,
|
ratioAccuracy, ratio,
|
||||||
cSpeedAccuracy, compressionSpeed);
|
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if(!intermediateResultDecompress.completed) {
|
if(!intermediateResultDecompress.completed) {
|
||||||
intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
|
intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
|
||||||
nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes);
|
nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL);
|
||||||
if(intermediateResultDecompress.result.error) {
|
if(intermediateResultDecompress.result.error) {
|
||||||
results.error = intermediateResultDecompress.result.error;
|
results.error = intermediateResultDecompress.result.error;
|
||||||
return results;
|
return results;
|
||||||
@@ -570,79 +577,79 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
|||||||
|
|
||||||
{
|
{
|
||||||
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||||
double const compressionSpeed = results.result.cSpeed / 1000000;
|
results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ intermediateResultDecompress.result.result.nanoSecPerRun);
|
||||||
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
|
|
||||||
double const decompressionSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000;
|
|
||||||
results.result.dSpeed = decompressionSpeed * 1000000;
|
|
||||||
markNb = (markNb+1) % NB_MARKS;
|
markNb = (markNb+1) % NB_MARKS;
|
||||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
|
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
|
||||||
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
||||||
ratioAccuracy, ratio,
|
ratioAccuracy, ratio,
|
||||||
cSpeedAccuracy, compressionSpeed,
|
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB),
|
||||||
decompressionSpeed);
|
(double)results.result.dSpeed / (1 MB));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} else {
|
} else { //iterMode;
|
||||||
if(adv->mode != BMK_decodeOnly) {
|
if(adv->mode != BMK_decodeOnly) {
|
||||||
|
|
||||||
BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
|
BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep,
|
||||||
nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds);
|
nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, adv->nbSeconds);
|
||||||
if(compressionResults.error) {
|
if(compressionResults.error) {
|
||||||
results.error = compressionResults.error;
|
results.error = compressionResults.error;
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(compressionResults.result.nanoSecPerRun == 0) {
|
if(compressionResults.result.nanoSecPerRun == 0) {
|
||||||
results.result.cSpeed = 0;
|
results.result.cSpeed = 0;
|
||||||
} else {
|
} else {
|
||||||
results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC;
|
results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun;
|
||||||
}
|
}
|
||||||
|
|
||||||
results.result.cSize = compressionResults.result.sumOfReturn;
|
results.result.cSize = compressionResults.result.sumOfReturn;
|
||||||
{
|
{
|
||||||
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||||
double const compressionSpeed = results.result.cSpeed / 1000000;
|
cSize = compressionResults.result.sumOfReturn;
|
||||||
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
|
results.result.cSize = cSize;
|
||||||
ratio = (double)srcSize / results.result.cSize;
|
ratio = (double)srcSize / results.result.cSize;
|
||||||
markNb = (markNb+1) % NB_MARKS;
|
markNb = (markNb+1) % NB_MARKS;
|
||||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
|
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
|
||||||
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
||||||
ratioAccuracy, ratio,
|
ratioAccuracy, ratio,
|
||||||
cSpeedAccuracy, compressionSpeed);
|
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(adv->mode != BMK_compressOnly) {
|
if(adv->mode != BMK_compressOnly) {
|
||||||
BMK_customReturn_t decompressionResults = BMK_benchFunction(
|
BMK_customReturn_t decompressionResults = BMK_benchFunction(
|
||||||
&local_defaultDecompress, (void*)(dctx),
|
&local_defaultDecompress, (void*)(dctx),
|
||||||
&local_initDCtx, (void*)&dctxprep, nbBlocks,
|
&local_initDCtx, (void*)&dctxprep, nbBlocks,
|
||||||
(const void* const*)cPtrs, cSizes, resPtrs, resSizes,
|
(const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL,
|
||||||
adv->nbSeconds);
|
adv->nbSeconds);
|
||||||
if(decompressionResults.error) {
|
if(decompressionResults.error) {
|
||||||
results.error = decompressionResults.error;
|
results.error = decompressionResults.error;
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(decompressionResults.result.nanoSecPerRun == 0) {
|
if(decompressionResults.result.nanoSecPerRun == 0) {
|
||||||
results.result.dSpeed = 0;
|
results.result.dSpeed = 0;
|
||||||
} else {
|
} else {
|
||||||
results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC;
|
results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun;
|
||||||
}
|
}
|
||||||
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
|
||||||
double const compressionSpeed = results.result.cSpeed / 1000000;
|
{
|
||||||
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
|
int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
|
||||||
double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000;
|
|
||||||
results.result.dSpeed = decompressionSpeed * 1000000;
|
|
||||||
markNb = (markNb+1) % NB_MARKS;
|
markNb = (markNb+1) % NB_MARKS;
|
||||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
|
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
|
||||||
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
|
||||||
ratioAccuracy, ratio,
|
ratioAccuracy, ratio,
|
||||||
cSpeedAccuracy, compressionSpeed,
|
results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB),
|
||||||
decompressionSpeed);
|
(double)results.result.dSpeed / (1 MB));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* CRC Checking */
|
/* CRC Checking */
|
||||||
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
|
{ void* resultBuffer = *resultBufferPtr;
|
||||||
|
U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||||
/* adv->mode == 0 -> compress + decompress */
|
/* adv->mode == 0 -> compress + decompress */
|
||||||
if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
|
if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) {
|
||||||
size_t u;
|
size_t u;
|
||||||
@@ -682,8 +689,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
|||||||
} /* CRC Checking */
|
} /* CRC Checking */
|
||||||
|
|
||||||
if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
|
if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */
|
||||||
double const cSpeed = results.result.cSpeed / 1000000;
|
double const cSpeed = (double)results.result.cSpeed / (1 MB);
|
||||||
double const dSpeed = results.result.dSpeed / 1000000;
|
double const dSpeed = (double)results.result.dSpeed / (1 MB);
|
||||||
if (adv->additionalParam) {
|
if (adv->additionalParam) {
|
||||||
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam);
|
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam);
|
||||||
} else {
|
} else {
|
||||||
@@ -692,14 +699,16 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc(
|
|||||||
}
|
}
|
||||||
DISPLAYLEVEL(2, "%2i#\n", cLevel);
|
DISPLAYLEVEL(2, "%2i#\n", cLevel);
|
||||||
} /* Bench */
|
} /* Bench */
|
||||||
|
results.result.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx);
|
||||||
|
results.error = 0;
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
|
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
|
||||||
|
void* dstBuffer, size_t dstCapacity,
|
||||||
const size_t* fileSizes, unsigned nbFiles,
|
const size_t* fileSizes, unsigned nbFiles,
|
||||||
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
||||||
const void* dictBuffer, size_t dictBufferSize,
|
const void* dictBuffer, size_t dictBufferSize,
|
||||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
|
||||||
int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
|
int displayLevel, const char* displayName, const BMK_advancedParams_t* adv)
|
||||||
|
|
||||||
{
|
{
|
||||||
@@ -707,47 +716,72 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
|
|||||||
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
||||||
|
|
||||||
/* these are the blockTable parameters, just split up */
|
/* these are the blockTable parameters, just split up */
|
||||||
const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*));
|
const void ** const srcPtrs = (const void**)malloc(maxNbBlocks * sizeof(void*));
|
||||||
size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
|
size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||||
|
|
||||||
void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*));
|
|
||||||
size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
|
|
||||||
|
|
||||||
void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*));
|
void ** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
|
||||||
size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t));
|
size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||||
|
size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||||
|
|
||||||
|
void ** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*));
|
||||||
|
size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||||
|
|
||||||
const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
|
|
||||||
void* compressedBuffer = malloc(maxCompressedSize);
|
|
||||||
void* resultBuffer = malloc(srcSize);
|
|
||||||
BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds);
|
BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds);
|
||||||
BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds);
|
BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds);
|
||||||
|
|
||||||
BMK_return_t results;
|
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
||||||
int allocationincomplete = !compressedBuffer || !resultBuffer ||
|
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||||
!srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes;
|
|
||||||
if (!allocationincomplete) {
|
const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024);
|
||||||
results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes,
|
|
||||||
resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress,
|
void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize);
|
||||||
|
void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer;
|
||||||
|
|
||||||
|
BMK_return_t results = { { 0, 0, 0, 0 }, 0 };
|
||||||
|
|
||||||
|
int parametersConflict = !dstBuffer ^ !dstCapacity;
|
||||||
|
|
||||||
|
void* resultBuffer = srcSize ? malloc(srcSize) : NULL;
|
||||||
|
|
||||||
|
int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs ||
|
||||||
|
!cSizes || !cCapacities || !resPtrs || !resSizes ||
|
||||||
|
!timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if (!allocationincomplete && !parametersConflict) {
|
||||||
|
results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes,
|
||||||
|
resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress,
|
||||||
srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams,
|
srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams,
|
||||||
dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv);
|
dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* clean up */
|
/* clean up */
|
||||||
BMK_freeTimeState(timeStateCompress);
|
BMK_freeTimeState(timeStateCompress);
|
||||||
BMK_freeTimeState(timeStateDecompress);
|
BMK_freeTimeState(timeStateDecompress);
|
||||||
free(compressedBuffer);
|
|
||||||
|
ZSTD_freeCCtx(ctx);
|
||||||
|
ZSTD_freeDCtx(dctx);
|
||||||
|
|
||||||
|
free(internalDstBuffer);
|
||||||
free(resultBuffer);
|
free(resultBuffer);
|
||||||
|
|
||||||
free((void*)srcPtrs);
|
free((void*)srcPtrs);
|
||||||
free(srcSizes);
|
free(srcSizes);
|
||||||
free(cPtrs);
|
free(cPtrs);
|
||||||
free(cSizes);
|
free(cSizes);
|
||||||
|
free(cCapacities);
|
||||||
free(resPtrs);
|
free(resPtrs);
|
||||||
free(resSizes);
|
free(resSizes);
|
||||||
|
|
||||||
if(allocationincomplete) {
|
if(allocationincomplete) {
|
||||||
EXM_THROW(31, BMK_return_t, "allocation error : not enough memory");
|
EXM_THROW(31, BMK_return_t, "allocation error : not enough memory");
|
||||||
}
|
}
|
||||||
results.error = 0;
|
|
||||||
|
if(parametersConflict) {
|
||||||
|
EXM_THROW(32, BMK_return_t, "Conflicting input results");
|
||||||
|
}
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -755,42 +789,17 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
|||||||
const size_t* fileSizes, unsigned nbFiles,
|
const size_t* fileSizes, unsigned nbFiles,
|
||||||
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
||||||
const void* dictBuffer, size_t dictBufferSize,
|
const void* dictBuffer, size_t dictBufferSize,
|
||||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
|
||||||
int displayLevel, const char* displayName) {
|
int displayLevel, const char* displayName) {
|
||||||
|
|
||||||
const BMK_advancedParams_t adv = BMK_initAdvancedParams();
|
const BMK_advancedParams_t adv = BMK_initAdvancedParams();
|
||||||
return BMK_benchMemAdvanced(srcBuffer, srcSize,
|
return BMK_benchMemAdvanced(srcBuffer, srcSize,
|
||||||
|
NULL, 0,
|
||||||
fileSizes, nbFiles,
|
fileSizes, nbFiles,
|
||||||
cLevel, comprParams,
|
cLevel, comprParams,
|
||||||
dictBuffer, dictBufferSize,
|
dictBuffer, dictBufferSize,
|
||||||
ctx, dctx,
|
|
||||||
displayLevel, displayName, &adv);
|
displayLevel, displayName, &adv);
|
||||||
}
|
}
|
||||||
|
|
||||||
static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize,
|
|
||||||
const size_t* fileSizes, unsigned nbFiles,
|
|
||||||
int cLevel, const ZSTD_compressionParameters* const comprParams,
|
|
||||||
const void* dictBuffer, size_t dictBufferSize,
|
|
||||||
int displayLevel, const char* displayName,
|
|
||||||
const BMK_advancedParams_t* const adv)
|
|
||||||
{
|
|
||||||
BMK_return_t res;
|
|
||||||
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
|
||||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
|
||||||
if(ctx == NULL || dctx == NULL) {
|
|
||||||
EXM_THROW(12, BMK_return_t, "not enough memory for contexts");
|
|
||||||
}
|
|
||||||
res = BMK_benchMemAdvanced(srcBuffer, srcSize,
|
|
||||||
fileSizes, nbFiles,
|
|
||||||
cLevel, comprParams,
|
|
||||||
dictBuffer, dictBufferSize,
|
|
||||||
ctx, dctx,
|
|
||||||
displayLevel, displayName, adv);
|
|
||||||
ZSTD_freeCCtx(ctx);
|
|
||||||
ZSTD_freeDCtx(dctx);
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||||
{
|
{
|
||||||
size_t const step = 64 MB;
|
size_t const step = 64 MB;
|
||||||
@@ -803,7 +812,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
|||||||
do {
|
do {
|
||||||
testmem = (BYTE*)malloc((size_t)requiredMem);
|
testmem = (BYTE*)malloc((size_t)requiredMem);
|
||||||
requiredMem -= step;
|
requiredMem -= step;
|
||||||
} while (!testmem);
|
} while (!testmem && requiredMem > 0);
|
||||||
|
|
||||||
free(testmem);
|
free(testmem);
|
||||||
return (size_t)(requiredMem);
|
return (size_t)(requiredMem);
|
||||||
@@ -831,12 +840,12 @@ static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
|
|||||||
if (displayLevel == 1 && !adv->additionalParam)
|
if (displayLevel == 1 && !adv->additionalParam)
|
||||||
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
|
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
|
||||||
|
|
||||||
res = BMK_benchMemCtxless(srcBuffer, benchedSize,
|
res = BMK_benchMemAdvanced(srcBuffer, benchedSize,
|
||||||
fileSizes, nbFiles,
|
NULL, 0,
|
||||||
cLevel, comprParams,
|
fileSizes, nbFiles,
|
||||||
dictBuffer, dictBufferSize,
|
cLevel, comprParams,
|
||||||
displayLevel, displayName,
|
dictBuffer, dictBufferSize,
|
||||||
adv);
|
displayLevel, displayName, adv);
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
@@ -931,7 +940,8 @@ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, un
|
|||||||
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
||||||
if (benchedSize < totalSizeToLoad)
|
if (benchedSize < totalSizeToLoad)
|
||||||
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
||||||
srcBuffer = malloc(benchedSize);
|
|
||||||
|
srcBuffer = benchedSize ? malloc(benchedSize) : NULL;
|
||||||
if (!srcBuffer) {
|
if (!srcBuffer) {
|
||||||
free(dictBuffer);
|
free(dictBuffer);
|
||||||
free(fileSizes);
|
free(fileSizes);
|
||||||
|
|||||||
+14
-12
@@ -32,8 +32,9 @@ extern "C" {
|
|||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
size_t cSize;
|
size_t cSize;
|
||||||
double cSpeed; /* bytes / sec */
|
U64 cSpeed; /* bytes / sec */
|
||||||
double dSpeed;
|
U64 dSpeed;
|
||||||
|
size_t cMem;
|
||||||
} BMK_result_t;
|
} BMK_result_t;
|
||||||
|
|
||||||
ERROR_STRUCT(BMK_result_t, BMK_return_t);
|
ERROR_STRUCT(BMK_result_t, BMK_return_t);
|
||||||
@@ -125,8 +126,6 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
|
|||||||
* comprParams - basic compression parameters
|
* comprParams - basic compression parameters
|
||||||
* dictBuffer - a dictionary if used, null otherwise
|
* dictBuffer - a dictionary if used, null otherwise
|
||||||
* dictBufferSize - size of dictBuffer, 0 otherwise
|
* dictBufferSize - size of dictBuffer, 0 otherwise
|
||||||
* ctx - Compression Context (must be provided)
|
|
||||||
* dctx - Decompression Context (must be provided)
|
|
||||||
* diplayLevel - see BMK_benchFiles
|
* diplayLevel - see BMK_benchFiles
|
||||||
* displayName - name used by display
|
* displayName - name used by display
|
||||||
* return
|
* return
|
||||||
@@ -138,15 +137,17 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
|||||||
const size_t* fileSizes, unsigned nbFiles,
|
const size_t* fileSizes, unsigned nbFiles,
|
||||||
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
||||||
const void* dictBuffer, size_t dictBufferSize,
|
const void* dictBuffer, size_t dictBufferSize,
|
||||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
|
||||||
int displayLevel, const char* displayName);
|
int displayLevel, const char* displayName);
|
||||||
|
|
||||||
/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */
|
/* See benchMem for normal parameter uses and return, see advancedParams_t for adv
|
||||||
|
* dstBuffer - destination buffer to write compressed output in, NULL if none provided.
|
||||||
|
* dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL
|
||||||
|
*/
|
||||||
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
|
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
|
||||||
|
void* dstBuffer, size_t dstCapacity,
|
||||||
const size_t* fileSizes, unsigned nbFiles,
|
const size_t* fileSizes, unsigned nbFiles,
|
||||||
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
const int cLevel, const ZSTD_compressionParameters* comprParams,
|
||||||
const void* dictBuffer, size_t dictBufferSize,
|
const void* dictBuffer, size_t dictBufferSize,
|
||||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
|
||||||
int displayLevel, const char* displayName,
|
int displayLevel, const char* displayName,
|
||||||
const BMK_advancedParams_t* adv);
|
const BMK_advancedParams_t* adv);
|
||||||
|
|
||||||
@@ -170,8 +171,10 @@ typedef size_t (*BMK_initFn_t)(void*);
|
|||||||
* srcBuffers - an array of buffers to be operated on by benchFn
|
* srcBuffers - an array of buffers to be operated on by benchFn
|
||||||
* srcSizes - an array of the sizes of above buffers
|
* srcSizes - an array of the sizes of above buffers
|
||||||
* dstBuffers - an array of buffers to be written into by benchFn
|
* dstBuffers - an array of buffers to be written into by benchFn
|
||||||
* dstCapacities - an array of the capacities of above buffers.
|
* dstCapacities - an array of the capacities of above buffers
|
||||||
|
* blockResults - the return value of benchFn called on each block.
|
||||||
* nbLoops - defines number of times benchFn is run.
|
* nbLoops - defines number of times benchFn is run.
|
||||||
|
* assumed array of size blockCount, will have compressed size of each block written to it.
|
||||||
* return
|
* return
|
||||||
* .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return
|
* .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return
|
||||||
* of the calls to initFn and benchFn, or if benchFunction errors internally
|
* of the calls to initFn and benchFn, or if benchFunction errors internally
|
||||||
@@ -182,12 +185,11 @@ typedef size_t (*BMK_initFn_t)(void*);
|
|||||||
* into dstBuffer, hence this value will be the total amount of bytes written to
|
* into dstBuffer, hence this value will be the total amount of bytes written to
|
||||||
* dstBuffer.
|
* dstBuffer.
|
||||||
*/
|
*/
|
||||||
BMK_customReturn_t BMK_benchFunction(
|
BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload,
|
||||||
BMK_benchFn_t benchFn, void* benchPayload,
|
|
||||||
BMK_initFn_t initFn, void* initPayload,
|
BMK_initFn_t initFn, void* initPayload,
|
||||||
size_t blockCount,
|
size_t blockCount,
|
||||||
const void* const * const srcBuffers, const size_t* srcSizes,
|
const void* const * const srcBuffers, const size_t* srcSizes,
|
||||||
void* const * const dstBuffers, const size_t* dstCapacities,
|
void * const * const dstBuffers, const size_t* dstCapacities, size_t* blockResults,
|
||||||
unsigned nbLoops);
|
unsigned nbLoops);
|
||||||
|
|
||||||
|
|
||||||
@@ -216,7 +218,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont,
|
|||||||
BMK_initFn_t initFn, void* initPayload,
|
BMK_initFn_t initFn, void* initPayload,
|
||||||
size_t blockCount,
|
size_t blockCount,
|
||||||
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
|
const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
|
||||||
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities);
|
void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResults);
|
||||||
|
|
||||||
#endif /* BENCH_H_121279284357 */
|
#endif /* BENCH_H_121279284357 */
|
||||||
|
|
||||||
|
|||||||
+39
-13
@@ -27,6 +27,7 @@
|
|||||||
#include <string.h> /* memset */
|
#include <string.h> /* memset */
|
||||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||||
#include <errno.h> /* errno */
|
#include <errno.h> /* errno */
|
||||||
|
#include <assert.h>
|
||||||
|
|
||||||
#include "mem.h" /* read */
|
#include "mem.h" /* read */
|
||||||
#include "error_private.h"
|
#include "error_private.h"
|
||||||
@@ -43,6 +44,7 @@
|
|||||||
#define SAMPLESIZE_MAX (128 KB)
|
#define SAMPLESIZE_MAX (128 KB)
|
||||||
#define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */
|
#define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */
|
||||||
#define COVER_MEMMULT 9 /* rough estimation : memory cost to analyze 1 byte of sample */
|
#define COVER_MEMMULT 9 /* rough estimation : memory cost to analyze 1 byte of sample */
|
||||||
|
#define FASTCOVER_MEMMULT 1 /* rough estimation : memory cost to analyze 1 byte of sample */
|
||||||
static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
|
static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
|
||||||
|
|
||||||
#define NOISELENGTH 32
|
#define NOISELENGTH 32
|
||||||
@@ -165,6 +167,7 @@ static U32 DiB_rand(U32* src)
|
|||||||
static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) {
|
static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) {
|
||||||
U32 seed = 0xFD2FB528;
|
U32 seed = 0xFD2FB528;
|
||||||
unsigned i;
|
unsigned i;
|
||||||
|
assert(nbFiles >= 1);
|
||||||
for (i = nbFiles - 1; i > 0; --i) {
|
for (i = nbFiles - 1; i > 0; --i) {
|
||||||
unsigned const j = DiB_rand(&seed) % (i + 1);
|
unsigned const j = DiB_rand(&seed) % (i + 1);
|
||||||
const char* const tmp = fileNamesTable[j];
|
const char* const tmp = fileNamesTable[j];
|
||||||
@@ -269,16 +272,19 @@ size_t ZDICT_trainFromBuffer_unsafe_legacy(void* dictBuffer, size_t dictBufferCa
|
|||||||
|
|
||||||
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||||
const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
|
const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
|
||||||
ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams,
|
ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
|
||||||
int optimizeCover)
|
ZDICT_fastCover_params_t* fastCoverParams, int optimize)
|
||||||
{
|
{
|
||||||
unsigned const displayLevel = params ? params->zParams.notificationLevel :
|
unsigned const displayLevel = params ? params->zParams.notificationLevel :
|
||||||
coverParams ? coverParams->zParams.notificationLevel :
|
coverParams ? coverParams->zParams.notificationLevel :
|
||||||
|
fastCoverParams ? fastCoverParams->zParams.notificationLevel :
|
||||||
0; /* should never happen */
|
0; /* should never happen */
|
||||||
void* const dictBuffer = malloc(maxDictSize);
|
void* const dictBuffer = malloc(maxDictSize);
|
||||||
fileStats const fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel);
|
fileStats const fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel);
|
||||||
size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t));
|
size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t));
|
||||||
size_t const memMult = params ? MEMMULT : COVER_MEMMULT;
|
size_t const memMult = params ? MEMMULT :
|
||||||
|
coverParams ? COVER_MEMMULT:
|
||||||
|
FASTCOVER_MEMMULT;
|
||||||
size_t const maxMem = DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult;
|
size_t const maxMem = DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult;
|
||||||
size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad);
|
size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad);
|
||||||
void* const srcBuffer = malloc(loadedSize+NOISELENGTH);
|
void* const srcBuffer = malloc(loadedSize+NOISELENGTH);
|
||||||
@@ -310,7 +316,8 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
|||||||
/* Load input buffer */
|
/* Load input buffer */
|
||||||
DISPLAYLEVEL(3, "Shuffling input files\n");
|
DISPLAYLEVEL(3, "Shuffling input files\n");
|
||||||
DiB_shuffle(fileNamesTable, nbFiles);
|
DiB_shuffle(fileNamesTable, nbFiles);
|
||||||
nbFiles = DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel);
|
|
||||||
|
DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel);
|
||||||
|
|
||||||
{ size_t dictSize;
|
{ size_t dictSize;
|
||||||
if (params) {
|
if (params) {
|
||||||
@@ -318,17 +325,36 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
|||||||
dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize,
|
dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize,
|
||||||
srcBuffer, sampleSizes, fs.nbSamples,
|
srcBuffer, sampleSizes, fs.nbSamples,
|
||||||
*params);
|
*params);
|
||||||
} else if (optimizeCover) {
|
} else if (coverParams) {
|
||||||
dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
|
if (optimize) {
|
||||||
srcBuffer, sampleSizes, fs.nbSamples,
|
dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize,
|
||||||
coverParams);
|
srcBuffer, sampleSizes, fs.nbSamples,
|
||||||
if (!ZDICT_isError(dictSize)) {
|
coverParams);
|
||||||
unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);
|
if (!ZDICT_isError(dictSize)) {
|
||||||
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage);
|
unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100);
|
||||||
|
DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d,
|
||||||
|
coverParams->steps, splitPercentage);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
|
||||||
|
sampleSizes, fs.nbSamples, *coverParams);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer,
|
assert(fastCoverParams != NULL);
|
||||||
sampleSizes, fs.nbSamples, *coverParams);
|
if (optimize) {
|
||||||
|
dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize,
|
||||||
|
srcBuffer, sampleSizes, fs.nbSamples,
|
||||||
|
fastCoverParams);
|
||||||
|
if (!ZDICT_isError(dictSize)) {
|
||||||
|
unsigned splitPercentage = (unsigned)(fastCoverParams->splitPoint * 100);
|
||||||
|
DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastCoverParams->k,
|
||||||
|
fastCoverParams->d, fastCoverParams->f, fastCoverParams->steps, splitPercentage,
|
||||||
|
fastCoverParams->accel);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, srcBuffer,
|
||||||
|
sampleSizes, fs.nbSamples, *fastCoverParams);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (ZDICT_isError(dictSize)) {
|
if (ZDICT_isError(dictSize)) {
|
||||||
DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
|
DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
|
||||||
|
|||||||
+2
-2
@@ -33,7 +33,7 @@
|
|||||||
*/
|
*/
|
||||||
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||||
const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
|
const char** fileNamesTable, unsigned nbFiles, size_t chunkSize,
|
||||||
ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams,
|
ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams,
|
||||||
int optimizeCover);
|
ZDICT_fastCover_params_t* fastCoverParams, int optimize);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+6
-9
@@ -727,11 +727,6 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress,
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
/*! FIO_compressFilename_internal() :
|
|
||||||
* same as FIO_compressFilename_extRess(), with `ress.desFile` already opened.
|
|
||||||
* @return : 0 : compression completed correctly,
|
|
||||||
* 1 : missing or pb opening srcFileName
|
|
||||||
*/
|
|
||||||
static unsigned long long
|
static unsigned long long
|
||||||
FIO_compressZstdFrame(const cRess_t* ressPtr,
|
FIO_compressZstdFrame(const cRess_t* ressPtr,
|
||||||
const char* srcFileName, U64 fileSize,
|
const char* srcFileName, U64 fileSize,
|
||||||
@@ -763,7 +758,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
|
|||||||
directive = ZSTD_e_end;
|
directive = ZSTD_e_end;
|
||||||
|
|
||||||
result = 1;
|
result = 1;
|
||||||
while (inBuff.pos != inBuff.size || (directive == ZSTD_e_end && result != 0)) {
|
while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */
|
||||||
|
|| (directive == ZSTD_e_end && result != 0) ) {
|
||||||
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
|
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
|
||||||
CHECK_V(result, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive));
|
CHECK_V(result, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive));
|
||||||
|
|
||||||
@@ -786,7 +782,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
|
|||||||
(U32)(zfp.consumed >> 20),
|
(U32)(zfp.consumed >> 20),
|
||||||
(U32)(zfp.produced >> 20),
|
(U32)(zfp.produced >> 20),
|
||||||
cShare );
|
cShare );
|
||||||
} else { /* g_displayLevel == 2 */
|
} else {
|
||||||
|
/* g_displayLevel <= 2; only display notifications if == 2; */
|
||||||
DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20));
|
DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20));
|
||||||
if (fileSize != UTIL_FILESIZE_UNKNOWN)
|
if (fileSize != UTIL_FILESIZE_UNKNOWN)
|
||||||
DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20));
|
DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20));
|
||||||
@@ -1014,8 +1011,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
|||||||
if (!dstFileName) {
|
if (!dstFileName) {
|
||||||
EXM_THROW(30, "zstd: %s", strerror(errno));
|
EXM_THROW(30, "zstd: %s", strerror(errno));
|
||||||
} }
|
} }
|
||||||
strcpy(dstFileName, inFileNamesTable[u]);
|
strncpy(dstFileName, inFileNamesTable[u], ifnSize+1 /* Include null */);
|
||||||
strcat(dstFileName, suffix);
|
strncat(dstFileName, suffix, suffixSize);
|
||||||
missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
|
missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel);
|
||||||
} }
|
} }
|
||||||
|
|
||||||
|
|||||||
+20
-1
@@ -194,7 +194,7 @@ All arguments after \fB\-\-\fR are treated as files
|
|||||||
Use FILEs as training set to create a dictionary\. The training set should contain a lot of small files (> 100), and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)\.
|
Use FILEs as training set to create a dictionary\. The training set should contain a lot of small files (> 100), and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)\.
|
||||||
.
|
.
|
||||||
.IP
|
.IP
|
||||||
Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-cover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. Equivalent to \fB\-\-train\-cover=d=8,steps=4\fR\.
|
Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-cover\fR\ or \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. Equivalent to \fB\-\-train\-fastCover=d=8,steps=4\fR\.
|
||||||
.
|
.
|
||||||
.TP
|
.TP
|
||||||
\fB\-o file\fR
|
\fB\-o file\fR
|
||||||
@@ -240,6 +240,25 @@ Examples:
|
|||||||
\fBzstd \-\-train\-cover=k=50 FILEs\fR
|
\fBzstd \-\-train\-cover=k=50 FILEs\fR
|
||||||
.
|
.
|
||||||
.TP
|
.TP
|
||||||
|
\fB\-\-train\-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]\fR
|
||||||
|
Same as cover but with extra parameters \fIf\fR and \fIaccel\fR and different default value of split
|
||||||
|
.
|
||||||
|
.IP
|
||||||
|
If \fIsplit\fR is not specified, then it tries \fIsplit\fR = 75. If \fIf\fR is not specified, then it tries \fIf\fR = 18. Requires that 0 < \fIf\fR < 32. If \fIaccel\fR is not specified, then it tries \fIaccel\fR = 1. Requires that 0 < \fIaccel\fR <= 10. Requires that \fId\fR = 6 or \fId\fR = 8.
|
||||||
|
.
|
||||||
|
.IP
|
||||||
|
\fIf\fR is log of size of array that keeps track of frequency of subsegments of size \fId\fR. The subsegment is hashed to an index in the range [0,2^\fIf\fR - 1]. It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency. Using a higher \fIf\fR reduces collision but takes longer.
|
||||||
|
.
|
||||||
|
.IP
|
||||||
|
Examples:
|
||||||
|
.
|
||||||
|
.IP
|
||||||
|
\fBzstd \-\-train\-fastcover FILEs\fR
|
||||||
|
.
|
||||||
|
.IP
|
||||||
|
\fBzstd \-\-train\-fastcover=d=8,f=15,accel=2 FILEs\fR
|
||||||
|
.
|
||||||
|
.TP
|
||||||
\fB\-\-train\-legacy[=selectivity=#]\fR
|
\fB\-\-train\-legacy[=selectivity=#]\fR
|
||||||
Use legacy dictionary builder algorithm with the given dictionary \fIselectivity\fR (default: 9)\. The smaller the \fIselectivity\fR value, the denser the dictionary, improving its efficiency but reducing its possible maximum size\. \fB\-\-train\-legacy=s=#\fR is also accepted\.
|
Use legacy dictionary builder algorithm with the given dictionary \fIselectivity\fR (default: 9)\. The smaller the \fIselectivity\fR value, the denser the dictionary, improving its efficiency but reducing its possible maximum size\. \fB\-\-train\-legacy=s=#\fR is also accepted\.
|
||||||
.
|
.
|
||||||
|
|||||||
@@ -254,6 +254,26 @@ Compression of small files similar to the sample set will be greatly improved.
|
|||||||
|
|
||||||
`zstd --train-cover=k=50,split=60 FILEs`
|
`zstd --train-cover=k=50,split=60 FILEs`
|
||||||
|
|
||||||
|
* `--train-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]`:
|
||||||
|
Same as cover but with extra parameters _f_ and _accel_ and different default value of split
|
||||||
|
If _split_ is not specified, then it tries _split_ = 75.
|
||||||
|
If _f_ is not specified, then it tries _f_ = 18.
|
||||||
|
Requires that 0 < _f_ < 32.
|
||||||
|
If _accel_ is not specified, then it tries _accel_ = 1.
|
||||||
|
Requires that 0 < _accel_ <= 10.
|
||||||
|
Requires that _d_ = 6 or _d_ = 8.
|
||||||
|
|
||||||
|
_f_ is log of size of array that keeps track of frequency of subsegments of size _d_.
|
||||||
|
The subsegment is hashed to an index in the range [0,2^_f_ - 1].
|
||||||
|
It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency.
|
||||||
|
Using a higher _f_ reduces collision but takes longer.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
`zstd --train-fastcover FILEs`
|
||||||
|
|
||||||
|
`zstd --train-fastcover=d=8,f=15,accel=2 FILEs`
|
||||||
|
|
||||||
* `--train-legacy[=selectivity=#]`:
|
* `--train-legacy[=selectivity=#]`:
|
||||||
Use legacy dictionary builder algorithm with the given dictionary
|
Use legacy dictionary builder algorithm with the given dictionary
|
||||||
_selectivity_ (default: 9).
|
_selectivity_ (default: 9).
|
||||||
|
|||||||
+86
-24
@@ -84,7 +84,10 @@ static U32 g_ldmMinMatch = 0;
|
|||||||
static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
|
static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
|
||||||
static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
|
static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
|
||||||
|
|
||||||
#define DEFAULT_SPLITPOINT 1.0
|
|
||||||
|
#define DEFAULT_ACCEL 1
|
||||||
|
|
||||||
|
typedef enum { cover, fastCover, legacy } dictType;
|
||||||
|
|
||||||
/*-************************************
|
/*-************************************
|
||||||
* Display Macros
|
* Display Macros
|
||||||
@@ -172,6 +175,7 @@ static int usage_advanced(const char* programName)
|
|||||||
DISPLAY( "Dictionary builder : \n");
|
DISPLAY( "Dictionary builder : \n");
|
||||||
DISPLAY( "--train ## : create a dictionary from a training set of files \n");
|
DISPLAY( "--train ## : create a dictionary from a training set of files \n");
|
||||||
DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n");
|
DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n");
|
||||||
|
DISPLAY( "--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fast cover algorithm with optional args\n");
|
||||||
DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
|
DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
|
||||||
DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
|
DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
|
||||||
DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
|
DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
|
||||||
@@ -295,6 +299,33 @@ static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* parseFastCoverParameters() :
|
||||||
|
* reads fastcover parameters from *stringPtr (e.g. "--train-fastcover=k=48,d=8,f=20,steps=32,accel=2") into *params
|
||||||
|
* @return 1 means that fastcover parameters were correct
|
||||||
|
* @return 0 in case of malformed parameters
|
||||||
|
*/
|
||||||
|
static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_params_t* params)
|
||||||
|
{
|
||||||
|
memset(params, 0, sizeof(*params));
|
||||||
|
for (; ;) {
|
||||||
|
if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&stringPtr, "f=")) { params->f = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&stringPtr, "accel=")) { params->accel = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&stringPtr, "split=")) {
|
||||||
|
unsigned splitPercentage = readU32FromChar(&stringPtr);
|
||||||
|
params->splitPoint = (double)splitPercentage / 100.0;
|
||||||
|
if (stringPtr[0]==',') { stringPtr++; continue; } else break;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (stringPtr[0] != 0) return 0;
|
||||||
|
DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint * 100), params->accel);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* parseLegacyParameters() :
|
* parseLegacyParameters() :
|
||||||
* reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
|
* reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
|
||||||
@@ -316,7 +347,19 @@ static ZDICT_cover_params_t defaultCoverParams(void)
|
|||||||
memset(¶ms, 0, sizeof(params));
|
memset(¶ms, 0, sizeof(params));
|
||||||
params.d = 8;
|
params.d = 8;
|
||||||
params.steps = 4;
|
params.steps = 4;
|
||||||
params.splitPoint = DEFAULT_SPLITPOINT;
|
params.splitPoint = 1.0;
|
||||||
|
return params;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ZDICT_fastCover_params_t defaultFastCoverParams(void)
|
||||||
|
{
|
||||||
|
ZDICT_fastCover_params_t params;
|
||||||
|
memset(¶ms, 0, sizeof(params));
|
||||||
|
params.d = 8;
|
||||||
|
params.f = 18;
|
||||||
|
params.steps = 4;
|
||||||
|
params.splitPoint = 0.75; /* different from default splitPoint of cover */
|
||||||
|
params.accel = DEFAULT_ACCEL;
|
||||||
return params;
|
return params;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
@@ -406,7 +449,6 @@ int main(int argCount, const char* argv[])
|
|||||||
singleThread = 0,
|
singleThread = 0,
|
||||||
ultra=0;
|
ultra=0;
|
||||||
double compressibility = 0.5;
|
double compressibility = 0.5;
|
||||||
BMK_advancedParams_t adv = BMK_initAdvancedParams();
|
|
||||||
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
|
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
|
||||||
size_t blockSize = 0;
|
size_t blockSize = 0;
|
||||||
zstd_operation_mode operation = zom_compress;
|
zstd_operation_mode operation = zom_compress;
|
||||||
@@ -432,7 +474,11 @@ int main(int argCount, const char* argv[])
|
|||||||
#endif
|
#endif
|
||||||
#ifndef ZSTD_NODICT
|
#ifndef ZSTD_NODICT
|
||||||
ZDICT_cover_params_t coverParams = defaultCoverParams();
|
ZDICT_cover_params_t coverParams = defaultCoverParams();
|
||||||
int cover = 1;
|
ZDICT_fastCover_params_t fastCoverParams = defaultFastCoverParams();
|
||||||
|
dictType dict = fastCover;
|
||||||
|
#endif
|
||||||
|
#ifndef ZSTD_NOBENCH
|
||||||
|
BMK_advancedParams_t benchParams = BMK_initAdvancedParams();
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|
||||||
@@ -528,18 +574,29 @@ int main(int argCount, const char* argv[])
|
|||||||
operation = zom_train;
|
operation = zom_train;
|
||||||
if (outFileName == NULL)
|
if (outFileName == NULL)
|
||||||
outFileName = g_defaultDictName;
|
outFileName = g_defaultDictName;
|
||||||
cover = 1;
|
dict = cover;
|
||||||
/* Allow optional arguments following an = */
|
/* Allow optional arguments following an = */
|
||||||
if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
|
if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
|
||||||
else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
|
else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
|
||||||
else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
|
else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (longCommandWArg(&argument, "--train-fastcover")) {
|
||||||
|
operation = zom_train;
|
||||||
|
if (outFileName == NULL)
|
||||||
|
outFileName = g_defaultDictName;
|
||||||
|
dict = fastCover;
|
||||||
|
/* Allow optional arguments following an = */
|
||||||
|
if (*argument == 0) { memset(&fastCoverParams, 0, sizeof(fastCoverParams)); }
|
||||||
|
else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
|
||||||
|
else if (!parseFastCoverParameters(argument, &fastCoverParams)) { CLEAN_RETURN(badusage(programName)); }
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (longCommandWArg(&argument, "--train-legacy")) {
|
if (longCommandWArg(&argument, "--train-legacy")) {
|
||||||
operation = zom_train;
|
operation = zom_train;
|
||||||
if (outFileName == NULL)
|
if (outFileName == NULL)
|
||||||
outFileName = g_defaultDictName;
|
outFileName = g_defaultDictName;
|
||||||
cover = 0;
|
dict = legacy;
|
||||||
/* Allow optional arguments following an = */
|
/* Allow optional arguments following an = */
|
||||||
if (*argument == 0) { continue; }
|
if (*argument == 0) { continue; }
|
||||||
else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
|
else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
|
||||||
@@ -620,7 +677,7 @@ int main(int argCount, const char* argv[])
|
|||||||
/* Decoding */
|
/* Decoding */
|
||||||
case 'd':
|
case 'd':
|
||||||
#ifndef ZSTD_NOBENCH
|
#ifndef ZSTD_NOBENCH
|
||||||
adv.mode = BMK_decodeOnly;
|
benchParams.mode = BMK_decodeOnly;
|
||||||
if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */
|
if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */
|
||||||
#endif
|
#endif
|
||||||
operation=zom_decompress; argument++; break;
|
operation=zom_decompress; argument++; break;
|
||||||
@@ -713,7 +770,7 @@ int main(int argCount, const char* argv[])
|
|||||||
case 'p': argument++;
|
case 'p': argument++;
|
||||||
#ifndef ZSTD_NOBENCH
|
#ifndef ZSTD_NOBENCH
|
||||||
if ((*argument>='0') && (*argument<='9')) {
|
if ((*argument>='0') && (*argument<='9')) {
|
||||||
adv.additionalParam = (int)readU32FromChar(&argument);
|
benchParams.additionalParam = (int)readU32FromChar(&argument);
|
||||||
} else
|
} else
|
||||||
#endif
|
#endif
|
||||||
main_pause=1;
|
main_pause=1;
|
||||||
@@ -826,18 +883,18 @@ int main(int argCount, const char* argv[])
|
|||||||
/* Check if benchmark is selected */
|
/* Check if benchmark is selected */
|
||||||
if (operation==zom_bench) {
|
if (operation==zom_bench) {
|
||||||
#ifndef ZSTD_NOBENCH
|
#ifndef ZSTD_NOBENCH
|
||||||
adv.blockSize = blockSize;
|
benchParams.blockSize = blockSize;
|
||||||
adv.nbWorkers = nbWorkers;
|
benchParams.nbWorkers = nbWorkers;
|
||||||
adv.realTime = setRealTimePrio;
|
benchParams.realTime = setRealTimePrio;
|
||||||
adv.nbSeconds = bench_nbSeconds;
|
benchParams.nbSeconds = bench_nbSeconds;
|
||||||
adv.ldmFlag = ldmFlag;
|
benchParams.ldmFlag = ldmFlag;
|
||||||
adv.ldmMinMatch = g_ldmMinMatch;
|
benchParams.ldmMinMatch = g_ldmMinMatch;
|
||||||
adv.ldmHashLog = g_ldmHashLog;
|
benchParams.ldmHashLog = g_ldmHashLog;
|
||||||
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
|
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
|
||||||
adv.ldmBucketSizeLog = g_ldmBucketSizeLog;
|
benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
|
||||||
}
|
}
|
||||||
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
|
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
|
||||||
adv.ldmHashEveryLog = g_ldmHashEveryLog;
|
benchParams.ldmHashEveryLog = g_ldmHashEveryLog;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
|
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
|
||||||
@@ -852,22 +909,22 @@ int main(int argCount, const char* argv[])
|
|||||||
int c;
|
int c;
|
||||||
DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]);
|
DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]);
|
||||||
for(c = cLevel; c <= cLevelLast; c++) {
|
for(c = cLevel; c <= cLevelLast; c++) {
|
||||||
BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &adv);
|
BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
for(; cLevel <= cLevelLast; cLevel++) {
|
for(; cLevel <= cLevelLast; cLevel++) {
|
||||||
BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv);
|
BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
for(; cLevel <= cLevelLast; cLevel++) {
|
for(; cLevel <= cLevelLast; cLevel++) {
|
||||||
BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &adv);
|
BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#else
|
#else
|
||||||
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles;
|
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
|
||||||
#endif
|
#endif
|
||||||
goto _end;
|
goto _end;
|
||||||
}
|
}
|
||||||
@@ -879,17 +936,22 @@ int main(int argCount, const char* argv[])
|
|||||||
zParams.compressionLevel = dictCLevel;
|
zParams.compressionLevel = dictCLevel;
|
||||||
zParams.notificationLevel = g_displayLevel;
|
zParams.notificationLevel = g_displayLevel;
|
||||||
zParams.dictID = dictID;
|
zParams.dictID = dictID;
|
||||||
if (cover) {
|
if (dict == cover) {
|
||||||
int const optimize = !coverParams.k || !coverParams.d;
|
int const optimize = !coverParams.k || !coverParams.d;
|
||||||
coverParams.nbThreads = nbWorkers;
|
coverParams.nbThreads = nbWorkers;
|
||||||
coverParams.zParams = zParams;
|
coverParams.zParams = zParams;
|
||||||
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, optimize);
|
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
|
||||||
|
} else if (dict == fastCover) {
|
||||||
|
int const optimize = !fastCoverParams.k || !fastCoverParams.d;
|
||||||
|
fastCoverParams.nbThreads = nbWorkers;
|
||||||
|
fastCoverParams.zParams = zParams;
|
||||||
|
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
|
||||||
} else {
|
} else {
|
||||||
ZDICT_legacy_params_t dictParams;
|
ZDICT_legacy_params_t dictParams;
|
||||||
memset(&dictParams, 0, sizeof(dictParams));
|
memset(&dictParams, 0, sizeof(dictParams));
|
||||||
dictParams.selectivityLevel = dictSelect;
|
dictParams.selectivityLevel = dictSelect;
|
||||||
dictParams.zParams = zParams;
|
dictParams.zParams = zParams;
|
||||||
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, 0);
|
operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, NULL, 0);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
goto _end;
|
goto _end;
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ invalidDictionaries
|
|||||||
checkTag
|
checkTag
|
||||||
zcat
|
zcat
|
||||||
zstdcat
|
zstdcat
|
||||||
|
tm
|
||||||
|
|
||||||
# Tmp test directory
|
# Tmp test directory
|
||||||
zstdtest
|
zstdtest
|
||||||
|
|||||||
+5
-2
@@ -27,6 +27,9 @@ DEBUGLEVEL ?= 1
|
|||||||
DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL)
|
DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL)
|
||||||
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
||||||
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
|
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
|
||||||
|
ifeq ($(OS),Windows_NT) # MinGW assumed
|
||||||
|
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
|
||||||
|
endif
|
||||||
CFLAGS ?= -O3
|
CFLAGS ?= -O3
|
||||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||||
@@ -126,7 +129,7 @@ zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c
|
|||||||
fullbench32: CPPFLAGS += -m32
|
fullbench32: CPPFLAGS += -m32
|
||||||
fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
|
fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
|
||||||
fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
|
fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
|
||||||
fullbench fullbench32 : DEBUGFLAGS = # turn off assert() for speed measurements
|
fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
|
||||||
fullbench fullbench32 : $(ZSTD_FILES)
|
fullbench fullbench32 : $(ZSTD_FILES)
|
||||||
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
|
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
|
||||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||||
@@ -197,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll
|
|||||||
zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
|
zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
|
||||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
|
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
|
||||||
|
|
||||||
paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements
|
paramgrill : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
|
||||||
paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c
|
paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c
|
||||||
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
|
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
|
||||||
|
|
||||||
|
|||||||
@@ -88,3 +88,56 @@ as well as the 10,000 original files for more detailed comparison of decompressi
|
|||||||
will choose a random seed, and for 1 minute,
|
will choose a random seed, and for 1 minute,
|
||||||
generate random test frames and ensure that the
|
generate random test frames and ensure that the
|
||||||
zstd library correctly decompresses them in both simple and streaming modes.
|
zstd library correctly decompresses them in both simple and streaming modes.
|
||||||
|
|
||||||
|
#### `paramgrill` - tool for generating compression table parameters and optimizing parameters on file given constraints
|
||||||
|
|
||||||
|
Full list of arguments
|
||||||
|
```
|
||||||
|
-T# : set level 1 speed objective
|
||||||
|
-B# : cut input into blocks of size # (default : single block)
|
||||||
|
-S : benchmarks a single run (example command: -Sl3w10h12)
|
||||||
|
w# - windowLog
|
||||||
|
h# - hashLog
|
||||||
|
c# - chainLog
|
||||||
|
s# - searchLog
|
||||||
|
l# - searchLength
|
||||||
|
t# - targetLength
|
||||||
|
S# - strategy
|
||||||
|
L# - level
|
||||||
|
--zstd= : Single run, parameter selection syntax same as zstdcli with more parameters
|
||||||
|
(Added forceAttachDictionary / fadt)
|
||||||
|
When invoked with --optimize, this represents the sample to exceed.
|
||||||
|
--optimize= : find parameters to maximize compression ratio given parameters
|
||||||
|
Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
|
||||||
|
cSpeed= : Minimum compression speed
|
||||||
|
dSpeed= : Minimum decompression speed
|
||||||
|
cMem= : Maximum compression memory
|
||||||
|
lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
|
||||||
|
stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
|
||||||
|
: In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
|
||||||
|
(Lower value will begin with stronger strategies) (Default 90%)
|
||||||
|
speedRatio= (accepts decimals)
|
||||||
|
: determines value of gains in speed vs gains in ratio
|
||||||
|
when determining overall winner (default 5 (1% ratio = 5% speed)).
|
||||||
|
tries= : Maximum number of random restarts on a single strategy before switching (Default 5)
|
||||||
|
Higher values will make optimizer run longer, more chances to find better solution.
|
||||||
|
memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
|
||||||
|
Setting memLog = 0 turns off memoization
|
||||||
|
--display= : specifiy which parameters are included in the output
|
||||||
|
can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
|
||||||
|
(Default: display all params available)
|
||||||
|
-P# : generated sample compressibility (when no file is provided)
|
||||||
|
-t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours ))
|
||||||
|
-v : Prints Benchmarking output
|
||||||
|
-D : Next argument dictionary file
|
||||||
|
-s : Benchmark all files separately
|
||||||
|
-q : Quiet, repeat for more quiet
|
||||||
|
-q Prints parameters + results whenever a new best is found
|
||||||
|
-qq Only prints parameters whenever a new best is found, prints final parameters + results
|
||||||
|
-qqq Only print final parameters + results
|
||||||
|
-qqqq Only prints final parameter set in the form --zstd=
|
||||||
|
-v : Verbose, cancels quiet, repeat for more volume
|
||||||
|
-v Prints all candidate parameters and results
|
||||||
|
|
||||||
|
```
|
||||||
|
Any inputs afterwards are treated as files to benchmark.
|
||||||
|
|||||||
+3
-2
@@ -336,7 +336,7 @@ size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst,
|
|||||||
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
|
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
|
||||||
{
|
{
|
||||||
BYTE* dstBuff;
|
BYTE* dstBuff;
|
||||||
size_t const dstBuffSize = ZSTD_compressBound(srcSize);
|
size_t dstBuffSize = ZSTD_compressBound(srcSize);
|
||||||
void* buff2, *buff1;
|
void* buff2, *buff1;
|
||||||
const char* benchName;
|
const char* benchName;
|
||||||
BMK_benchFn_t benchFunction;
|
BMK_benchFn_t benchFunction;
|
||||||
@@ -514,9 +514,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel,
|
|||||||
|
|
||||||
/* benchmark loop */
|
/* benchmark loop */
|
||||||
{
|
{
|
||||||
|
void* dstBuffv = (void*)dstBuff;
|
||||||
r = BMK_benchFunction(benchFunction, buff2,
|
r = BMK_benchFunction(benchFunction, buff2,
|
||||||
NULL, NULL, 1, &src, &srcSize,
|
NULL, NULL, 1, &src, &srcSize,
|
||||||
(void * const * const)&dstBuff, &dstBuffSize, g_nbIterations);
|
&dstBuffv, &dstBuffSize, NULL, g_nbIterations);
|
||||||
if(r.error) {
|
if(r.error) {
|
||||||
DISPLAY("ERROR %d ! ! \n", r.error);
|
DISPLAY("ERROR %d ! ! \n", r.error);
|
||||||
errorcode = r.error;
|
errorcode = r.error;
|
||||||
|
|||||||
+69
-14
@@ -27,6 +27,7 @@
|
|||||||
#include <string.h> /* strcmp */
|
#include <string.h> /* strcmp */
|
||||||
#include <assert.h>
|
#include <assert.h>
|
||||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
|
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
|
||||||
|
#include "fse.h"
|
||||||
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||||
#include "zstd_errors.h" /* ZSTD_getErrorCode */
|
#include "zstd_errors.h" /* ZSTD_getErrorCode */
|
||||||
#include "zstdmt_compress.h"
|
#include "zstdmt_compress.h"
|
||||||
@@ -178,13 +179,9 @@ static void FUZ_displayMallocStats(mallocCounter_t count)
|
|||||||
(U32)(count.totalMalloc >> 10));
|
(U32)(count.totalMalloc >> 10));
|
||||||
}
|
}
|
||||||
|
|
||||||
static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part,
|
||||||
|
void* inBuffer, size_t inSize, void* outBuffer, size_t outSize)
|
||||||
{
|
{
|
||||||
size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
|
|
||||||
size_t const outSize = ZSTD_compressBound(inSize);
|
|
||||||
void* const inBuffer = malloc(inSize);
|
|
||||||
void* const outBuffer = malloc(outSize);
|
|
||||||
|
|
||||||
/* test only played in verbose mode, as they are long */
|
/* test only played in verbose mode, as they are long */
|
||||||
if (g_displayLevel<3) return 0;
|
if (g_displayLevel<3) return 0;
|
||||||
|
|
||||||
@@ -269,6 +266,28 @@ static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
||||||
|
{
|
||||||
|
size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
|
||||||
|
size_t const outSize = ZSTD_compressBound(inSize);
|
||||||
|
void* const inBuffer = malloc(inSize);
|
||||||
|
void* const outBuffer = malloc(outSize);
|
||||||
|
int result;
|
||||||
|
|
||||||
|
/* Create compressible noise */
|
||||||
|
if (!inBuffer || !outBuffer) {
|
||||||
|
DISPLAY("Not enough memory, aborting \n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
result = FUZ_mallocTests_internal(seed, compressibility, part,
|
||||||
|
inBuffer, inSize, outBuffer, outSize);
|
||||||
|
|
||||||
|
free(inBuffer);
|
||||||
|
free(outBuffer);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
#else
|
#else
|
||||||
|
|
||||||
static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
|
||||||
@@ -411,6 +430,26 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
}
|
}
|
||||||
DISPLAYLEVEL(3, "OK \n");
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++);
|
||||||
|
{ int i;
|
||||||
|
for (i=0; i<20; i++)
|
||||||
|
((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */
|
||||||
|
memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */
|
||||||
|
for (i=1; i<=19; i++) {
|
||||||
|
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||||
|
size_t size1, size2;
|
||||||
|
DISPLAYLEVEL(5, "l%i ", i);
|
||||||
|
size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 30, i);
|
||||||
|
CHECK_Z(size1);
|
||||||
|
size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 30, i);
|
||||||
|
CHECK_Z(size2);
|
||||||
|
CHECK_EQ(size1, size2);
|
||||||
|
|
||||||
|
ZSTD_freeCCtx(cctx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++);
|
DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++);
|
||||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||||
ZSTD_outBuffer out = {NULL, 0, 0};
|
ZSTD_outBuffer out = {NULL, 0, 0};
|
||||||
@@ -1423,6 +1462,24 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
}
|
}
|
||||||
DISPLAYLEVEL(3, "OK \n");
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
|
DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++);
|
||||||
|
{
|
||||||
|
short norm[32];
|
||||||
|
unsigned count[32];
|
||||||
|
unsigned const tableLog = 5;
|
||||||
|
size_t const nbSeq = 32;
|
||||||
|
unsigned const maxSymbolValue = 31;
|
||||||
|
size_t i;
|
||||||
|
|
||||||
|
for (i = 0; i < 32; ++i)
|
||||||
|
count[i] = 1;
|
||||||
|
/* Calling FSE_normalizeCount() on a uniform distribution should not
|
||||||
|
* cause a division by zero.
|
||||||
|
*/
|
||||||
|
FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue);
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(3, "OK \n");
|
||||||
|
|
||||||
_end:
|
_end:
|
||||||
free(CNBuffer);
|
free(CNBuffer);
|
||||||
free(compressedBuffer);
|
free(compressedBuffer);
|
||||||
@@ -1496,7 +1553,6 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
size_t const dstBufferSize = (size_t)1<<maxSampleLog;
|
size_t const dstBufferSize = (size_t)1<<maxSampleLog;
|
||||||
size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
|
size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
|
||||||
BYTE* cNoiseBuffer[5];
|
BYTE* cNoiseBuffer[5];
|
||||||
BYTE* srcBuffer; /* jumping pointer */
|
|
||||||
BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
|
BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
|
||||||
BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
|
BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
|
||||||
BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
|
BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
|
||||||
@@ -1505,7 +1561,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||||
U32 result = 0;
|
U32 result = 0;
|
||||||
U32 testNb = 0;
|
U32 testNb = 0;
|
||||||
U32 coreSeed = seed, lseed = 0;
|
U32 coreSeed = seed;
|
||||||
UTIL_time_t const startClock = UTIL_getTime();
|
UTIL_time_t const startClock = UTIL_getTime();
|
||||||
U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
|
U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
|
||||||
int const cLevelLimiter = bigTests ? 3 : 2;
|
int const cLevelLimiter = bigTests ? 3 : 2;
|
||||||
@@ -1526,13 +1582,14 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
|
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
|
||||||
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
|
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
|
||||||
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
|
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
|
||||||
srcBuffer = cNoiseBuffer[2];
|
|
||||||
|
|
||||||
/* catch up testNb */
|
/* catch up testNb */
|
||||||
for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
|
for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
|
||||||
|
|
||||||
/* main test loop */
|
/* main test loop */
|
||||||
for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
|
for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
|
||||||
|
BYTE* srcBuffer; /* jumping pointer */
|
||||||
|
U32 lseed;
|
||||||
size_t sampleSize, maxTestSize, totalTestSize;
|
size_t sampleSize, maxTestSize, totalTestSize;
|
||||||
size_t cSize, totalCSize, totalGenSize;
|
size_t cSize, totalCSize, totalGenSize;
|
||||||
U64 crcOrig;
|
U64 crcOrig;
|
||||||
@@ -1763,11 +1820,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
|||||||
CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
|
CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
|
||||||
CHECK (totalCSize != cSize, "compressed data should be fully read")
|
CHECK (totalCSize != cSize, "compressed data should be fully read")
|
||||||
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
||||||
if (crcDest!=crcOrig) {
|
CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)",
|
||||||
size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
|
(U32)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (U32)totalTestSize);
|
||||||
CHECK (1, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
|
}
|
||||||
(U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]);
|
|
||||||
} }
|
|
||||||
} /* for ( ; (testNb <= nbTests) */
|
} /* for ( ; (testNb <= nbTests) */
|
||||||
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
|
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
|
||||||
|
|
||||||
|
|||||||
+2245
-488
@@ -17,7 +17,6 @@
|
|||||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||||
#include <string.h> /* strcmp */
|
#include <string.h> /* strcmp */
|
||||||
#include <math.h> /* log */
|
#include <math.h> /* log */
|
||||||
#include <time.h>
|
|
||||||
#include <assert.h>
|
#include <assert.h>
|
||||||
|
|
||||||
#include "mem.h"
|
#include "mem.h"
|
||||||
@@ -27,7 +26,8 @@
|
|||||||
#include "xxhash.h"
|
#include "xxhash.h"
|
||||||
#include "util.h"
|
#include "util.h"
|
||||||
#include "bench.h"
|
#include "bench.h"
|
||||||
|
#include "zstd_errors.h"
|
||||||
|
#include "zstd_internal.h"
|
||||||
|
|
||||||
/*-************************************
|
/*-************************************
|
||||||
* Constants
|
* Constants
|
||||||
@@ -36,13 +36,7 @@
|
|||||||
#define AUTHOR "Yann Collet"
|
#define AUTHOR "Yann Collet"
|
||||||
#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR
|
#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR
|
||||||
|
|
||||||
|
#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */
|
||||||
#define KB *(1<<10)
|
|
||||||
#define MB *(1<<20)
|
|
||||||
#define GB *(1ULL<<30)
|
|
||||||
|
|
||||||
#define NBLOOPS 2
|
|
||||||
#define TIMELOOP (2 * SEC_TO_MICRO)
|
|
||||||
#define NB_LEVELS_TRACKED 22 /* ensured being >= ZSTD_maxCLevel() in BMK_init_level_constraints() */
|
#define NB_LEVELS_TRACKED 22 /* ensured being >= ZSTD_maxCLevel() in BMK_init_level_constraints() */
|
||||||
|
|
||||||
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
|
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
|
||||||
@@ -56,39 +50,302 @@ static const int g_maxNbVariations = 64;
|
|||||||
* Macros
|
* Macros
|
||||||
**************************************/
|
**************************************/
|
||||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||||
|
#define DISPLAYLEVEL(n, ...) if(g_displayLevel >= n) { fprintf(stderr, __VA_ARGS__); }
|
||||||
|
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
|
||||||
|
|
||||||
|
#define TIMED 0
|
||||||
|
#ifndef DEBUG
|
||||||
|
# define DEBUG 0
|
||||||
|
#endif
|
||||||
|
|
||||||
#undef MIN
|
#undef MIN
|
||||||
#undef MAX
|
#undef MAX
|
||||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||||
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
|
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
|
||||||
#define CUSTOM_LEVEL 99
|
#define CUSTOM_LEVEL 99
|
||||||
|
#define BASE_CLEVEL 1
|
||||||
|
|
||||||
|
#define FADT_MIN 0
|
||||||
|
#define FADT_MAX ((U32)-1)
|
||||||
|
|
||||||
|
#define ZSTD_TARGETLENGTH_MIN 0
|
||||||
|
#define ZSTD_TARGETLENGTH_MAX 999
|
||||||
|
|
||||||
|
#define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1)
|
||||||
|
#define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1)
|
||||||
|
#define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1)
|
||||||
|
#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1)
|
||||||
|
#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1)
|
||||||
|
#define TLEN_RANGE 17
|
||||||
|
#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1)
|
||||||
|
#define FADT_RANGE 3
|
||||||
|
|
||||||
|
#define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } }
|
||||||
|
#define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } }
|
||||||
|
|
||||||
|
#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt uses -1 */
|
||||||
|
|
||||||
|
static const char* g_stratName[ZSTD_btultra+1] = {
|
||||||
|
"(none) ", "ZSTD_fast ", "ZSTD_dfast ",
|
||||||
|
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
|
||||||
|
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
|
||||||
|
|
||||||
|
|
||||||
|
static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 };
|
||||||
|
|
||||||
/*-************************************
|
/*-************************************
|
||||||
* Benchmark Parameters
|
* Setup for Adding new params
|
||||||
**************************************/
|
**************************************/
|
||||||
|
|
||||||
static double g_grillDuration_s = 99999; /* about 27 hours */
|
/* indices for each of the variables */
|
||||||
static U32 g_nbIterations = NBLOOPS;
|
typedef enum {
|
||||||
static double g_compressibility = COMPRESSIBILITY_DEFAULT;
|
wlog_ind = 0,
|
||||||
static U32 g_blockSize = 0;
|
clog_ind = 1,
|
||||||
static U32 g_rand = 1;
|
hlog_ind = 2,
|
||||||
static U32 g_singleRun = 0;
|
slog_ind = 3,
|
||||||
static U32 g_target = 0;
|
slen_ind = 4,
|
||||||
static U32 g_noSeed = 0;
|
tlen_ind = 5,
|
||||||
static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
|
strt_ind = 6,
|
||||||
|
fadt_ind = 7, /* forceAttachDict */
|
||||||
|
NUM_PARAMS = 8
|
||||||
|
} varInds_t;
|
||||||
|
|
||||||
void BMK_SetNbIterations(int nbLoops)
|
typedef struct {
|
||||||
{
|
U32 vals[NUM_PARAMS];
|
||||||
g_nbIterations = nbLoops;
|
} paramValues_t;
|
||||||
DISPLAY("- %u iterations -\n", g_nbIterations);
|
|
||||||
|
/* maximum value of parameters */
|
||||||
|
static const U32 mintable[NUM_PARAMS] =
|
||||||
|
{ ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN };
|
||||||
|
|
||||||
|
/* minimum value of parameters */
|
||||||
|
static const U32 maxtable[NUM_PARAMS] =
|
||||||
|
{ ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX };
|
||||||
|
|
||||||
|
/* # of values parameters can take on */
|
||||||
|
static const U32 rangetable[NUM_PARAMS] =
|
||||||
|
{ WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE };
|
||||||
|
|
||||||
|
/* ZSTD_cctxSetParameter() index to set */
|
||||||
|
static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] =
|
||||||
|
{ ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict };
|
||||||
|
|
||||||
|
/* names of parameters */
|
||||||
|
static const char* g_paramNames[NUM_PARAMS] =
|
||||||
|
{ "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" };
|
||||||
|
|
||||||
|
/* shortened names of parameters */
|
||||||
|
static const char* g_shortParamNames[NUM_PARAMS] =
|
||||||
|
{ "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" };
|
||||||
|
|
||||||
|
/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
|
||||||
|
static U32 rangeMap(varInds_t param, int ind) {
|
||||||
|
ind = MAX(MIN(ind, (int)rangetable[param] - 1), 0);
|
||||||
|
switch(param) {
|
||||||
|
case tlen_ind:
|
||||||
|
return tlen_table[ind];
|
||||||
|
case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */
|
||||||
|
return ind - 1;
|
||||||
|
case wlog_ind: /* using default: triggers -Wswitch-enum */
|
||||||
|
case clog_ind:
|
||||||
|
case hlog_ind:
|
||||||
|
case slog_ind:
|
||||||
|
case slen_ind:
|
||||||
|
case strt_ind:
|
||||||
|
return mintable[param] + ind;
|
||||||
|
case NUM_PARAMS:
|
||||||
|
DISPLAY("Error, not a valid param\n ");
|
||||||
|
return (U32)-1;
|
||||||
|
}
|
||||||
|
return 0; /* should never happen, stop compiler warnings */
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* inverse of rangeMap */
|
||||||
|
static int invRangeMap(varInds_t param, U32 value) {
|
||||||
|
value = MIN(MAX(mintable[param], value), maxtable[param]);
|
||||||
|
switch(param) {
|
||||||
|
case tlen_ind: /* bin search */
|
||||||
|
{
|
||||||
|
int lo = 0;
|
||||||
|
int hi = TLEN_RANGE;
|
||||||
|
while(lo < hi) {
|
||||||
|
int mid = (lo + hi) / 2;
|
||||||
|
if(tlen_table[mid] < value) {
|
||||||
|
lo = mid + 1;
|
||||||
|
} if(tlen_table[mid] == value) {
|
||||||
|
return mid;
|
||||||
|
} else {
|
||||||
|
hi = mid;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return lo;
|
||||||
|
}
|
||||||
|
case fadt_ind:
|
||||||
|
return (int)value + 1;
|
||||||
|
case wlog_ind:
|
||||||
|
case clog_ind:
|
||||||
|
case hlog_ind:
|
||||||
|
case slog_ind:
|
||||||
|
case slen_ind:
|
||||||
|
case strt_ind:
|
||||||
|
return value - mintable[param];
|
||||||
|
case NUM_PARAMS:
|
||||||
|
DISPLAY("Error, not a valid param\n ");
|
||||||
|
return -2;
|
||||||
|
}
|
||||||
|
return 0; /* should never happen, stop compiler warnings */
|
||||||
|
}
|
||||||
|
|
||||||
|
/* display of params */
|
||||||
|
static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) {
|
||||||
|
switch(param) {
|
||||||
|
case fadt_ind: if(width) { fprintf(f, "%*d", width, (int)value); } else { fprintf(f, "%d", (int)value); } break;
|
||||||
|
case strt_ind: if(width) { fprintf(f, "%*s", width, g_stratName[value]); } else { fprintf(f, "%s", g_stratName[value]); } break;
|
||||||
|
case wlog_ind:
|
||||||
|
case clog_ind:
|
||||||
|
case hlog_ind:
|
||||||
|
case slog_ind:
|
||||||
|
case slen_ind:
|
||||||
|
case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break;
|
||||||
|
case NUM_PARAMS:
|
||||||
|
DISPLAY("Error, not a valid param\n "); break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Benchmark Parameters/Global Variables
|
||||||
|
**************************************/
|
||||||
|
|
||||||
|
typedef BYTE U8;
|
||||||
|
|
||||||
|
/* General Utility */
|
||||||
|
static U32 g_timeLimit_s = 99999; /* about 27 hours */
|
||||||
|
static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */
|
||||||
|
static U32 g_blockSize = 0;
|
||||||
|
static U32 g_rand = 1;
|
||||||
|
|
||||||
|
/* Display */
|
||||||
|
static int g_displayLevel = 3;
|
||||||
|
static BYTE g_silenceParams[NUM_PARAMS];
|
||||||
|
|
||||||
|
/* Mode Selection */
|
||||||
|
static U32 g_singleRun = 0;
|
||||||
|
static U32 g_optimizer = 0;
|
||||||
|
static int g_optmode = 0;
|
||||||
|
|
||||||
|
/* For cLevel Table generation */
|
||||||
|
static U32 g_target = 0;
|
||||||
|
static U32 g_noSeed = 0;
|
||||||
|
|
||||||
|
/* For optimizer */
|
||||||
|
static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */
|
||||||
|
static double g_ratioMultiplier = 5.;
|
||||||
|
static U32 g_strictness = PARAM_UNSET; /* range 1 - 100, measure of how strict */
|
||||||
|
static BMK_result_t g_lvltarget;
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
directMap,
|
||||||
|
xxhashMap,
|
||||||
|
noMemo
|
||||||
|
} memoTableType_t;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
memoTableType_t tableType;
|
||||||
|
BYTE* table;
|
||||||
|
size_t tableLen;
|
||||||
|
varInds_t varArray[NUM_PARAMS];
|
||||||
|
size_t varLen;
|
||||||
|
} memoTable_t;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
BMK_result_t result;
|
||||||
|
paramValues_t params;
|
||||||
|
} winnerInfo_t;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
U32 cSpeed; /* bytes / sec */
|
||||||
|
U32 dSpeed;
|
||||||
|
U32 cMem; /* bytes */
|
||||||
|
} constraint_t;
|
||||||
|
|
||||||
|
typedef struct winner_ll_node winner_ll_node;
|
||||||
|
struct winner_ll_node {
|
||||||
|
winnerInfo_t res;
|
||||||
|
winner_ll_node* next;
|
||||||
|
};
|
||||||
|
|
||||||
|
static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Additional Global Variables (Defined Above Use)
|
||||||
|
* g_level_constraint
|
||||||
|
* g_alreadyTested
|
||||||
|
* g_maxTries
|
||||||
|
* g_clockGranularity
|
||||||
|
*/
|
||||||
|
|
||||||
/*-*******************************************************
|
/*-*******************************************************
|
||||||
* Private functions
|
* General Util Functions
|
||||||
*********************************************************/
|
*********************************************************/
|
||||||
|
|
||||||
/* accuracy in seconds only, span can be multiple years */
|
/* nullified useless params, to ensure count stats */
|
||||||
static double BMK_timeSpan(time_t tStart) { return difftime(time(NULL), tStart); }
|
/* cleans up params for memoizing / display */
|
||||||
|
static paramValues_t sanitizeParams(paramValues_t params)
|
||||||
|
{
|
||||||
|
if (params.vals[strt_ind] == ZSTD_fast)
|
||||||
|
params.vals[clog_ind] = 0, params.vals[slog_ind] = 0;
|
||||||
|
if (params.vals[strt_ind] == ZSTD_dfast)
|
||||||
|
params.vals[slog_ind] = 0;
|
||||||
|
if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast)
|
||||||
|
params.vals[tlen_ind] = 0;
|
||||||
|
|
||||||
|
return params;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) {
|
||||||
|
ZSTD_compressionParameters c;
|
||||||
|
memset(&c, 0, sizeof(ZSTD_compressionParameters));
|
||||||
|
c.windowLog = p.vals[wlog_ind];
|
||||||
|
c.chainLog = p.vals[clog_ind];
|
||||||
|
c.hashLog = p.vals[hlog_ind];
|
||||||
|
c.searchLog = p.vals[slog_ind];
|
||||||
|
c.searchLength = p.vals[slen_ind];
|
||||||
|
c.targetLength = p.vals[tlen_ind];
|
||||||
|
c.strategy = p.vals[strt_ind];
|
||||||
|
/* no forceAttachDict */
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
|
||||||
|
static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) {
|
||||||
|
paramValues_t p;
|
||||||
|
varInds_t i;
|
||||||
|
p.vals[wlog_ind] = c.windowLog;
|
||||||
|
p.vals[clog_ind] = c.chainLog;
|
||||||
|
p.vals[hlog_ind] = c.hashLog;
|
||||||
|
p.vals[slog_ind] = c.searchLog;
|
||||||
|
p.vals[slen_ind] = c.searchLength;
|
||||||
|
p.vals[tlen_ind] = c.targetLength;
|
||||||
|
p.vals[strt_ind] = c.strategy;
|
||||||
|
|
||||||
|
/* set all other params to their minimum value */
|
||||||
|
for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
|
||||||
|
p.vals[i] = mintable[i];
|
||||||
|
}
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* equivalent of ZSTD_adjustCParams for paramValues_t */
|
||||||
|
static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, const size_t dictSize) {
|
||||||
|
paramValues_t ot = p;
|
||||||
|
varInds_t i;
|
||||||
|
p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize));
|
||||||
|
if(!dictSize) { p.vals[fadt_ind] = 0; }
|
||||||
|
/* retain value of all other parameters */
|
||||||
|
for(i = strt_ind + 1; i < NUM_PARAMS; i++) {
|
||||||
|
p.vals[i] = ot.vals[i];
|
||||||
|
}
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||||
{
|
{
|
||||||
@@ -98,16 +355,18 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
|||||||
requiredMem = (((requiredMem >> 26) + 1) << 26);
|
requiredMem = (((requiredMem >> 26) + 1) << 26);
|
||||||
if (requiredMem > maxMemory) requiredMem = maxMemory;
|
if (requiredMem > maxMemory) requiredMem = maxMemory;
|
||||||
|
|
||||||
requiredMem += 2*step;
|
requiredMem += 2 * step;
|
||||||
while (!testmem) {
|
while (!testmem && requiredMem > 0) {
|
||||||
requiredMem -= step;
|
|
||||||
testmem = malloc ((size_t)requiredMem);
|
testmem = malloc ((size_t)requiredMem);
|
||||||
|
requiredMem -= step;
|
||||||
}
|
}
|
||||||
|
|
||||||
free (testmem);
|
free (testmem);
|
||||||
return (size_t) (requiredMem - step);
|
return (size_t) requiredMem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* accuracy in seconds only, span can be multiple years */
|
||||||
|
static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); }
|
||||||
|
|
||||||
static U32 FUZ_rotl32(U32 x, U32 r)
|
static U32 FUZ_rotl32(U32 x, U32 r)
|
||||||
{
|
{
|
||||||
@@ -126,75 +385,467 @@ U32 FUZ_rand(U32* src)
|
|||||||
return rand32 >> 5;
|
return rand32 >> 5;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** longCommandWArg() :
|
/* allows zeros */
|
||||||
* check if *stringPtr is the same as longCommand.
|
#define CLAMPCHECK(val,min,max) { \
|
||||||
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
|
if (((val)<(min)) | ((val)>(max))) { \
|
||||||
* @return 0 and doesn't modify *stringPtr otherwise.
|
DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \
|
||||||
* from zstdcli.c
|
return 0; \
|
||||||
|
} }
|
||||||
|
|
||||||
|
static int paramValid(const paramValues_t paramTarget) {
|
||||||
|
U32 i;
|
||||||
|
for(i = 0; i < NUM_PARAMS; i++) {
|
||||||
|
CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]);
|
||||||
|
}
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static paramValues_t cParamUnsetMin(paramValues_t paramTarget) {
|
||||||
|
varInds_t i;
|
||||||
|
for(i = 0; i < NUM_PARAMS; i++) {
|
||||||
|
if(paramTarget.vals[i] == PARAM_UNSET) {
|
||||||
|
paramTarget.vals[i] = mintable[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return paramTarget;
|
||||||
|
}
|
||||||
|
|
||||||
|
static paramValues_t emptyParams(void) {
|
||||||
|
U32 i;
|
||||||
|
paramValues_t p;
|
||||||
|
for(i = 0; i < NUM_PARAMS; i++) {
|
||||||
|
p.vals[i] = PARAM_UNSET;
|
||||||
|
}
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
static winnerInfo_t initWinnerInfo(const paramValues_t p) {
|
||||||
|
winnerInfo_t w1;
|
||||||
|
w1.result.cSpeed = 0.;
|
||||||
|
w1.result.dSpeed = 0.;
|
||||||
|
w1.result.cMem = (size_t)-1;
|
||||||
|
w1.result.cSize = (size_t)-1;
|
||||||
|
w1.params = p;
|
||||||
|
return w1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) {
|
||||||
|
U32 i;
|
||||||
|
for(i = 0; i < NUM_PARAMS; i++) {
|
||||||
|
if(mask.vals[i] != PARAM_UNSET) {
|
||||||
|
base.vals[i] = mask.vals[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return base;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) {
|
||||||
|
ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* varies ptr by nbChanges respecting varyParams*/
|
||||||
|
static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges)
|
||||||
|
{
|
||||||
|
paramValues_t p;
|
||||||
|
U32 validated = 0;
|
||||||
|
while (!validated) {
|
||||||
|
U32 i;
|
||||||
|
p = *ptr;
|
||||||
|
for (i = 0 ; i < nbChanges ; i++) {
|
||||||
|
const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1);
|
||||||
|
paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
|
||||||
|
}
|
||||||
|
validated = paramValid(p);
|
||||||
|
}
|
||||||
|
*ptr = p;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Completely random parameter selection */
|
||||||
|
static paramValues_t randomParams(void)
|
||||||
|
{
|
||||||
|
varInds_t v; paramValues_t p;
|
||||||
|
for(v = 0; v <= NUM_PARAMS; v++) {
|
||||||
|
p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
|
||||||
|
}
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
static U64 g_clockGranularity = 100000000ULL;
|
||||||
|
|
||||||
|
static void findClockGranularity(void) {
|
||||||
|
UTIL_time_t clockStart = UTIL_getTime();
|
||||||
|
U64 el1 = 0, el2 = 0;
|
||||||
|
int i = 0;
|
||||||
|
do {
|
||||||
|
el1 = el2;
|
||||||
|
el2 = UTIL_clockSpanNano(clockStart);
|
||||||
|
if(el1 < el2) {
|
||||||
|
U64 iv = el2 - el1;
|
||||||
|
if(g_clockGranularity > iv) {
|
||||||
|
g_clockGranularity = iv;
|
||||||
|
i = 0;
|
||||||
|
} else {
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} while(i < 10);
|
||||||
|
DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Optimizer Util Functions
|
||||||
|
**************************************/
|
||||||
|
|
||||||
|
/* checks results are feasible */
|
||||||
|
static int feasible(const BMK_result_t results, const constraint_t target) {
|
||||||
|
return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_optmode || results.cSize <= g_lvltarget.cSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* hill climbing value for part 1 */
|
||||||
|
/* Scoring here is a linear reward for all set constraints normalized between 0 to 1
|
||||||
|
* (with 0 at 0 and 1 being fully fulfilling the constraint), summed with a logarithmic
|
||||||
|
* bonus to exceeding the constraint value. We also give linear ratio for compression ratio.
|
||||||
|
* The constant factors are experimental.
|
||||||
*/
|
*/
|
||||||
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
|
static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) {
|
||||||
{
|
double cs = 0., ds = 0., rt, cm = 0.;
|
||||||
size_t const comSize = strlen(longCommand);
|
const double r1 = 1, r2 = 0.1, rtr = 0.5;
|
||||||
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
double ret;
|
||||||
if (result) *stringPtr += comSize;
|
if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; }
|
||||||
return result;
|
if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; }
|
||||||
|
if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; }
|
||||||
|
rt = ((double)srcSize / res.cSize);
|
||||||
|
|
||||||
|
ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr +
|
||||||
|
(MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2;
|
||||||
|
|
||||||
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*-*******************************************************
|
/* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */
|
||||||
* Bench functions
|
static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) {
|
||||||
*********************************************************/
|
double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1;
|
||||||
|
double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1;
|
||||||
typedef struct
|
if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) {
|
||||||
{
|
return 0.0;
|
||||||
const char* srcPtr;
|
}
|
||||||
size_t srcSize;
|
return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1;
|
||||||
char* cPtr;
|
|
||||||
size_t cRoom;
|
|
||||||
size_t cSize;
|
|
||||||
char* resPtr;
|
|
||||||
size_t resSize;
|
|
||||||
} blockParam_t;
|
|
||||||
|
|
||||||
|
|
||||||
const char* g_stratName[ZSTD_btultra+1] = {
|
|
||||||
"(none) ", "ZSTD_fast ", "ZSTD_dfast ",
|
|
||||||
"ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ",
|
|
||||||
"ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "};
|
|
||||||
|
|
||||||
/* TODO: support additional parameters (more files, fileSizes) */
|
|
||||||
static size_t
|
|
||||||
BMK_benchParam(BMK_result_t* resultPtr,
|
|
||||||
const void* srcBuffer, size_t srcSize,
|
|
||||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
|
|
||||||
const ZSTD_compressionParameters cParams) {
|
|
||||||
|
|
||||||
BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File");
|
|
||||||
*resultPtr = res.result;
|
|
||||||
return res.error;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize)
|
/* return true if r2 strictly better than r1 */
|
||||||
|
static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) {
|
||||||
|
if(feasible(result1, target) && feasible(result2, target)) {
|
||||||
|
if(g_optmode) {
|
||||||
|
return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget);
|
||||||
|
} else {
|
||||||
|
return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed)
|
||||||
|
|| (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target)));
|
||||||
|
}
|
||||||
|
|
||||||
|
static constraint_t relaxTarget(constraint_t target) {
|
||||||
|
target.cMem = (U32)-1;
|
||||||
|
target.cSpeed *= ((double)g_strictness) / 100;
|
||||||
|
target.dSpeed *= ((double)g_strictness) / 100;
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) {
|
||||||
|
varInds_t v;
|
||||||
|
for(v = 0; v < NUM_PARAMS; v++) {
|
||||||
|
if(pc->vals[v] != PARAM_UNSET) {
|
||||||
|
U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]);
|
||||||
|
if(newval != pc->vals[v]) {
|
||||||
|
pc->vals[v] = newval;
|
||||||
|
DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(pc->vals[wlog_ind] != PARAM_UNSET) {
|
||||||
|
|
||||||
|
U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1;
|
||||||
|
/* edge case of highBit not working for 0 */
|
||||||
|
|
||||||
|
if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) {
|
||||||
|
U32 adjust = MAX(mintable[wlog_ind], sshb);
|
||||||
|
if(adjust != pc->vals[wlog_ind]) {
|
||||||
|
pc->vals[wlog_ind] = adjust;
|
||||||
|
DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
|
||||||
|
U32 maxclog;
|
||||||
|
if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) {
|
||||||
|
maxclog = pc->vals[wlog_ind] + 1;
|
||||||
|
} else {
|
||||||
|
maxclog = pc->vals[wlog_ind];
|
||||||
|
}
|
||||||
|
|
||||||
|
if(pc->vals[clog_ind] > maxclog) {
|
||||||
|
pc->vals[clog_ind] = maxclog;
|
||||||
|
DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) {
|
||||||
|
if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) {
|
||||||
|
pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1;
|
||||||
|
DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) {
|
||||||
|
if(pc->vals[slog_ind] > pc->vals[clog_ind]) {
|
||||||
|
pc->vals[clog_ind] = pc->vals[slog_ind];
|
||||||
|
DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) {
|
||||||
|
return
|
||||||
|
(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */
|
||||||
|
|| ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */
|
||||||
|
|| (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/
|
||||||
|
|| (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */
|
||||||
|
|| (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Display Functions
|
||||||
|
**************************************/
|
||||||
|
|
||||||
|
static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) {
|
||||||
|
varInds_t v;
|
||||||
|
int first = 1;
|
||||||
|
fprintf(f,"--zstd=");
|
||||||
|
for(v = 0; v < NUM_PARAMS; v++) {
|
||||||
|
if(g_silenceParams[v]) { continue; }
|
||||||
|
if(!first) { fprintf(f, ","); }
|
||||||
|
fprintf(f,"%s=", g_paramNames[v]);
|
||||||
|
|
||||||
|
if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); }
|
||||||
|
else { displayParamVal(f, v, params.vals[v], 0); }
|
||||||
|
first = 0;
|
||||||
|
}
|
||||||
|
fprintf(f, "\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) {
|
||||||
|
varInds_t v;
|
||||||
|
int first = 1;
|
||||||
|
res.params = cParamUnsetMin(res.params);
|
||||||
|
fprintf(f," {");
|
||||||
|
for(v = 0; v < NUM_PARAMS; v++) {
|
||||||
|
if(g_silenceParams[v]) { continue; }
|
||||||
|
if(!first) { fprintf(f, ","); }
|
||||||
|
displayParamVal(f, v, res.params.vals[v], 3);
|
||||||
|
first = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
|
||||||
|
(double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Writes to f the results of a parameter benchmark */
|
||||||
|
/* when used with --optimize, will only print results better than previously discovered */
|
||||||
|
static void BMK_printWinner(FILE* f, const int cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize)
|
||||||
{
|
{
|
||||||
char lvlstr[15] = "Custom Level";
|
char lvlstr[15] = "Custom Level";
|
||||||
DISPLAY("\r%79s\r", "");
|
winnerInfo_t w;
|
||||||
fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ",
|
w.params = params;
|
||||||
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength,
|
w.result = result;
|
||||||
params.targetLength, g_stratName[(U32)(params.strategy)]);
|
|
||||||
|
fprintf(f, "\r%79s\r", "");
|
||||||
|
|
||||||
if(cLevel != CUSTOM_LEVEL) {
|
if(cLevel != CUSTOM_LEVEL) {
|
||||||
snprintf(lvlstr, 15, " Level %2u ", cLevel);
|
snprintf(lvlstr, 15, " Level %2d ", cLevel);
|
||||||
}
|
}
|
||||||
fprintf(f,
|
|
||||||
"/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n",
|
if(TIMED) {
|
||||||
lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.);
|
const U64 time = UTIL_clockSpanNano(g_time);
|
||||||
|
const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC);
|
||||||
|
fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC);
|
||||||
|
}
|
||||||
|
|
||||||
|
fprintf(f, "/* %s */ ", lvlstr);
|
||||||
|
BMK_displayOneResult(f, w, srcSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* comparison function: */
|
||||||
|
/* strictly better, strictly worse, equal, speed-side adv, size-side adv */
|
||||||
|
#define WORSE_RESULT 0
|
||||||
|
#define BETTER_RESULT 1
|
||||||
|
#define ERROR_RESULT 2
|
||||||
|
|
||||||
typedef struct {
|
#define SPEED_RESULT 4
|
||||||
BMK_result_t result;
|
#define SIZE_RESULT 5
|
||||||
ZSTD_compressionParameters params;
|
/* maybe have epsilon-eq to limit table size? */
|
||||||
} winnerInfo_t;
|
static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) {
|
||||||
|
if(r1.cSpeed < r2.cSpeed) {
|
||||||
|
if(r1.cSize >= r2.cSize) {
|
||||||
|
return BETTER_RESULT;
|
||||||
|
}
|
||||||
|
return SPEED_RESULT; /* r2 is smaller but not faster. */
|
||||||
|
} else {
|
||||||
|
if(r1.cSize <= r2.cSize) {
|
||||||
|
return WORSE_RESULT;
|
||||||
|
}
|
||||||
|
return SIZE_RESULT; /* r2 is faster but not smaller */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
|
/* 0 for insertion, 1 for no insert */
|
||||||
|
/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */
|
||||||
|
static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) {
|
||||||
|
BMK_result_t r = w.result;
|
||||||
|
winner_ll_node* cur_node = g_winners;
|
||||||
|
/* first node to insert */
|
||||||
|
if(!feasible(r, targetConstraints)) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(g_winners == NULL) {
|
||||||
|
winner_ll_node* first_node = malloc(sizeof(winner_ll_node));
|
||||||
|
if(first_node == NULL) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
first_node->next = NULL;
|
||||||
|
first_node->res = w;
|
||||||
|
g_winners = first_node;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
while(cur_node->next != NULL) {
|
||||||
|
switch(speedSizeCompare(cur_node->res.result, r)) {
|
||||||
|
case WORSE_RESULT:
|
||||||
|
{
|
||||||
|
return 1; /* never insert if better */
|
||||||
|
}
|
||||||
|
case BETTER_RESULT:
|
||||||
|
{
|
||||||
|
winner_ll_node* tmp;
|
||||||
|
cur_node->res = cur_node->next->res;
|
||||||
|
tmp = cur_node->next;
|
||||||
|
cur_node->next = cur_node->next->next;
|
||||||
|
free(tmp);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case SIZE_RESULT:
|
||||||
|
{
|
||||||
|
cur_node = cur_node->next;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case SPEED_RESULT: /* insert after first size result, then return */
|
||||||
|
{
|
||||||
|
winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
|
||||||
|
if(newnode == NULL) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
newnode->res = cur_node->res;
|
||||||
|
cur_node->res = w;
|
||||||
|
newnode->next = cur_node->next;
|
||||||
|
cur_node->next = newnode;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
assert(cur_node->next == NULL);
|
||||||
|
switch(speedSizeCompare(cur_node->res.result, r)) {
|
||||||
|
case WORSE_RESULT:
|
||||||
|
{
|
||||||
|
return 1; /* never insert if better */
|
||||||
|
}
|
||||||
|
case BETTER_RESULT:
|
||||||
|
{
|
||||||
|
cur_node->res = w;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
case SIZE_RESULT:
|
||||||
|
{
|
||||||
|
winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
|
||||||
|
if(newnode == NULL) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
newnode->res = w;
|
||||||
|
newnode->next = NULL;
|
||||||
|
cur_node->next = newnode;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
case SPEED_RESULT: /* insert before first size result, then return */
|
||||||
|
{
|
||||||
|
winner_ll_node* newnode = malloc(sizeof(winner_ll_node));
|
||||||
|
if(newnode == NULL) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
newnode->res = cur_node->res;
|
||||||
|
cur_node->res = w;
|
||||||
|
newnode->next = cur_node->next;
|
||||||
|
cur_node->next = newnode;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize)
|
||||||
|
{
|
||||||
|
/* global winner used for constraints */
|
||||||
|
/* cSize, cSpeed, dSpeed, cMem */
|
||||||
|
static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } };
|
||||||
|
if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) {
|
||||||
|
if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
|
||||||
|
DISPLAY("New Winner: \n");
|
||||||
|
}
|
||||||
|
|
||||||
|
if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); }
|
||||||
|
|
||||||
|
if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) {
|
||||||
|
if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); }
|
||||||
|
g_winner.result = result;
|
||||||
|
g_winner.params = params;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) {
|
||||||
|
winnerInfo_t w;
|
||||||
|
winner_ll_node* n;
|
||||||
|
w.result = result;
|
||||||
|
w.params = params;
|
||||||
|
insertWinner(w, targetConstraints);
|
||||||
|
|
||||||
|
if(!DEBUG) { fprintf(f, "\033c"); }
|
||||||
|
fprintf(f, "\n");
|
||||||
|
|
||||||
|
/* the table */
|
||||||
|
fprintf(f, "================================\n");
|
||||||
|
for(n = g_winners; n != NULL; n = n->next) {
|
||||||
|
BMK_displayOneResult(f, n->res, srcSize);
|
||||||
|
}
|
||||||
|
fprintf(f, "================================\n");
|
||||||
|
fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n",
|
||||||
|
(double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB));
|
||||||
|
|
||||||
|
|
||||||
|
fprintf(f, "Overall Winner: \n");
|
||||||
|
BMK_displayOneResult(f, g_winner, srcSize);
|
||||||
|
BMK_translateAdvancedParams(f, g_winner.params);
|
||||||
|
|
||||||
|
fprintf(f, "Latest BMK: \n");\
|
||||||
|
BMK_displayOneResult(f, w, srcSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
|
||||||
{
|
{
|
||||||
int cLevel;
|
int cLevel;
|
||||||
|
|
||||||
@@ -206,7 +857,7 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSize)
|
static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize)
|
||||||
{
|
{
|
||||||
fseek(f, 0, SEEK_SET);
|
fseek(f, 0, SEEK_SET);
|
||||||
BMK_printWinners2(f, winners, srcSize);
|
BMK_printWinners2(f, winners, srcSize);
|
||||||
@@ -215,9 +866,774 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/*-*******************************************************
|
||||||
|
* Functions to Benchmark
|
||||||
|
*********************************************************/
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
double cSpeed_min;
|
ZSTD_CCtx* ctx;
|
||||||
double dSpeed_min;
|
const void* dictBuffer;
|
||||||
|
size_t dictBufferSize;
|
||||||
|
int cLevel;
|
||||||
|
const paramValues_t* comprParams;
|
||||||
|
} BMK_initCCtxArgs;
|
||||||
|
|
||||||
|
static size_t local_initCCtx(void* payload) {
|
||||||
|
const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload;
|
||||||
|
varInds_t i;
|
||||||
|
ZSTD_CCtx_reset(ag->ctx);
|
||||||
|
ZSTD_CCtx_resetParameters(ag->ctx);
|
||||||
|
ZSTD_CCtx_setParameter(ag->ctx, ZSTD_p_compressionLevel, ag->cLevel);
|
||||||
|
|
||||||
|
for(i = 0; i < NUM_PARAMS; i++) {
|
||||||
|
if(ag->comprParams->vals[i] != PARAM_UNSET)
|
||||||
|
ZSTD_CCtx_setParameter(ag->ctx, cctxSetParamTable[i], ag->comprParams->vals[i]);
|
||||||
|
}
|
||||||
|
ZSTD_CCtx_loadDictionary(ag->ctx, ag->dictBuffer, ag->dictBufferSize);
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
ZSTD_DCtx* dctx;
|
||||||
|
const void* dictBuffer;
|
||||||
|
size_t dictBufferSize;
|
||||||
|
} BMK_initDCtxArgs;
|
||||||
|
|
||||||
|
static size_t local_initDCtx(void* payload) {
|
||||||
|
const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload;
|
||||||
|
ZSTD_DCtx_reset(ag->dctx);
|
||||||
|
ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* additional argument is just the context */
|
||||||
|
static size_t local_defaultCompress(
|
||||||
|
const void* srcBuffer, size_t srcSize,
|
||||||
|
void* dstBuffer, size_t dstSize,
|
||||||
|
void* addArgs) {
|
||||||
|
size_t moreToFlush = 1;
|
||||||
|
ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs;
|
||||||
|
ZSTD_inBuffer in;
|
||||||
|
ZSTD_outBuffer out;
|
||||||
|
in.src = srcBuffer;
|
||||||
|
in.size = srcSize;
|
||||||
|
in.pos = 0;
|
||||||
|
out.dst = dstBuffer;
|
||||||
|
out.size = dstSize;
|
||||||
|
out.pos = 0;
|
||||||
|
assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */
|
||||||
|
while (moreToFlush) {
|
||||||
|
if(out.pos == out.size) {
|
||||||
|
return (size_t)-ZSTD_error_dstSize_tooSmall;
|
||||||
|
}
|
||||||
|
moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end);
|
||||||
|
if (ZSTD_isError(moreToFlush)) {
|
||||||
|
return moreToFlush;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out.pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* additional argument is just the context */
|
||||||
|
static size_t local_defaultDecompress(
|
||||||
|
const void* srcBuffer, size_t srcSize,
|
||||||
|
void* dstBuffer, size_t dstSize,
|
||||||
|
void* addArgs) {
|
||||||
|
size_t moreToFlush = 1;
|
||||||
|
ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs;
|
||||||
|
ZSTD_inBuffer in;
|
||||||
|
ZSTD_outBuffer out;
|
||||||
|
in.src = srcBuffer;
|
||||||
|
in.size = srcSize;
|
||||||
|
in.pos = 0;
|
||||||
|
out.dst = dstBuffer;
|
||||||
|
out.size = dstSize;
|
||||||
|
out.pos = 0;
|
||||||
|
while (moreToFlush) {
|
||||||
|
if(out.pos == out.size) {
|
||||||
|
return (size_t)-ZSTD_error_dstSize_tooSmall;
|
||||||
|
}
|
||||||
|
moreToFlush = ZSTD_decompress_generic(dctx,
|
||||||
|
&out, &in);
|
||||||
|
if (ZSTD_isError(moreToFlush)) {
|
||||||
|
return moreToFlush;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out.pos;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Data Initialization Functions
|
||||||
|
**************************************/
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
void* srcBuffer;
|
||||||
|
size_t srcSize;
|
||||||
|
const void** srcPtrs;
|
||||||
|
size_t* srcSizes;
|
||||||
|
void** dstPtrs;
|
||||||
|
size_t* dstCapacities;
|
||||||
|
size_t* dstSizes;
|
||||||
|
void** resPtrs;
|
||||||
|
size_t* resSizes;
|
||||||
|
size_t nbBlocks;
|
||||||
|
size_t maxBlockSize;
|
||||||
|
} buffers_t;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
size_t dictSize;
|
||||||
|
void* dictBuffer;
|
||||||
|
ZSTD_CCtx* cctx;
|
||||||
|
ZSTD_DCtx* dctx;
|
||||||
|
} contexts_t;
|
||||||
|
|
||||||
|
static void freeNonSrcBuffers(const buffers_t b) {
|
||||||
|
free(b.srcPtrs);
|
||||||
|
free(b.srcSizes);
|
||||||
|
|
||||||
|
if(b.dstPtrs != NULL) {
|
||||||
|
free(b.dstPtrs[0]);
|
||||||
|
}
|
||||||
|
free(b.dstPtrs);
|
||||||
|
free(b.dstCapacities);
|
||||||
|
free(b.dstSizes);
|
||||||
|
|
||||||
|
if(b.resPtrs != NULL) {
|
||||||
|
free(b.resPtrs[0]);
|
||||||
|
}
|
||||||
|
free(b.resPtrs);
|
||||||
|
free(b.resSizes);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void freeBuffers(const buffers_t b) {
|
||||||
|
if(b.srcPtrs != NULL) {
|
||||||
|
free(b.srcBuffer);
|
||||||
|
}
|
||||||
|
freeNonSrcBuffers(b);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* srcBuffer will be freed by freeBuffers now */
|
||||||
|
static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size_t nbFiles,
|
||||||
|
const size_t* fileSizes)
|
||||||
|
{
|
||||||
|
size_t pos = 0, n, blockSize;
|
||||||
|
U32 maxNbBlocks, blockNb = 0;
|
||||||
|
buff->srcSize = 0;
|
||||||
|
for(n = 0; n < nbFiles; n++) {
|
||||||
|
buff->srcSize += fileSizes[n];
|
||||||
|
}
|
||||||
|
|
||||||
|
if(buff->srcSize == 0) {
|
||||||
|
DISPLAY("No data to bench\n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
blockSize = g_blockSize ? g_blockSize : buff->srcSize;
|
||||||
|
maxNbBlocks = (U32) ((buff->srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles;
|
||||||
|
|
||||||
|
buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*));
|
||||||
|
buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||||
|
|
||||||
|
buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
|
||||||
|
buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||||
|
buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||||
|
|
||||||
|
buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*));
|
||||||
|
buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t));
|
||||||
|
|
||||||
|
if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) {
|
||||||
|
DISPLAY("alloc error\n");
|
||||||
|
freeNonSrcBuffers(*buff);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
buff->srcBuffer = srcBuffer;
|
||||||
|
buff->srcPtrs[0] = (const void*)buff->srcBuffer;
|
||||||
|
buff->dstPtrs[0] = malloc(ZSTD_compressBound(buff->srcSize) + (maxNbBlocks * 1024));
|
||||||
|
buff->resPtrs[0] = malloc(buff->srcSize);
|
||||||
|
|
||||||
|
if(!buff->dstPtrs[0] || !buff->resPtrs[0]) {
|
||||||
|
DISPLAY("alloc error\n");
|
||||||
|
freeNonSrcBuffers(*buff);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
for(n = 0; n < nbFiles; n++) {
|
||||||
|
size_t pos_end = pos + fileSizes[n];
|
||||||
|
for(; pos < pos_end; blockNb++) {
|
||||||
|
buff->srcPtrs[blockNb] = (const void*)((char*)srcBuffer + pos);
|
||||||
|
buff->srcSizes[blockNb] = blockSize;
|
||||||
|
pos += blockSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(fileSizes[n] > 0) { buff->srcSizes[blockNb - 1] = ((fileSizes[n] - 1) % blockSize) + 1; }
|
||||||
|
pos = pos_end;
|
||||||
|
}
|
||||||
|
|
||||||
|
buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]);
|
||||||
|
buff->dstSizes[0] = buff->dstCapacities[0];
|
||||||
|
buff->resSizes[0] = buff->srcSizes[0];
|
||||||
|
buff->maxBlockSize = buff->srcSizes[0];
|
||||||
|
|
||||||
|
for(n = 1; n < blockNb; n++) {
|
||||||
|
buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1];
|
||||||
|
buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1];
|
||||||
|
buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]);
|
||||||
|
buff->dstSizes[n] = buff->dstCapacities[n];
|
||||||
|
buff->resSizes[n] = buff->srcSizes[n];
|
||||||
|
|
||||||
|
buff->maxBlockSize = MAX(buff->maxBlockSize, buff->srcSizes[n]);
|
||||||
|
}
|
||||||
|
|
||||||
|
buff->nbBlocks = blockNb;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* allocates buffer's arguments. returns success / failuere */
|
||||||
|
static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable,
|
||||||
|
size_t nbFiles) {
|
||||||
|
size_t pos = 0;
|
||||||
|
size_t n;
|
||||||
|
size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles);
|
||||||
|
size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad);
|
||||||
|
size_t* fileSizes = calloc(sizeof(size_t), nbFiles);
|
||||||
|
void* srcBuffer = NULL;
|
||||||
|
int ret = 0;
|
||||||
|
|
||||||
|
if(!totalSizeToLoad || !benchedSize) {
|
||||||
|
ret = 1;
|
||||||
|
DISPLAY("Nothing to Bench\n");
|
||||||
|
goto _cleanUp;
|
||||||
|
}
|
||||||
|
|
||||||
|
srcBuffer = malloc(benchedSize);
|
||||||
|
|
||||||
|
if(!fileSizes || !srcBuffer) {
|
||||||
|
ret = 1;
|
||||||
|
goto _cleanUp;
|
||||||
|
}
|
||||||
|
|
||||||
|
for(n = 0; n < nbFiles; n++) {
|
||||||
|
FILE* f;
|
||||||
|
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
||||||
|
if (UTIL_isDirectory(fileNamesTable[n])) {
|
||||||
|
DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (fileSize == UTIL_FILESIZE_UNKNOWN) {
|
||||||
|
DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
f = fopen(fileNamesTable[n], "rb");
|
||||||
|
if (f==NULL) {
|
||||||
|
DISPLAY("impossible to open file %s\n", fileNamesTable[n]);
|
||||||
|
fclose(f);
|
||||||
|
ret = 10;
|
||||||
|
goto _cleanUp;
|
||||||
|
}
|
||||||
|
|
||||||
|
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
|
||||||
|
|
||||||
|
if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */
|
||||||
|
{
|
||||||
|
char* buffer = (char*)(srcBuffer);
|
||||||
|
size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f);
|
||||||
|
fclose(f);
|
||||||
|
if (readSize != (size_t)fileSize) {
|
||||||
|
DISPLAY("could not read %s", fileNamesTable[n]);
|
||||||
|
ret = 1;
|
||||||
|
goto _cleanUp;
|
||||||
|
}
|
||||||
|
|
||||||
|
fileSizes[n] = readSize;
|
||||||
|
pos += readSize;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes);
|
||||||
|
|
||||||
|
_cleanUp:
|
||||||
|
if(ret) { free(srcBuffer); }
|
||||||
|
free(fileSizes);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void freeContexts(const contexts_t ctx) {
|
||||||
|
free(ctx.dictBuffer);
|
||||||
|
ZSTD_freeCCtx(ctx.cctx);
|
||||||
|
ZSTD_freeDCtx(ctx.dctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int createContexts(contexts_t* ctx, const char* dictFileName) {
|
||||||
|
FILE* f;
|
||||||
|
size_t readSize;
|
||||||
|
ctx->cctx = ZSTD_createCCtx();
|
||||||
|
ctx->dctx = ZSTD_createDCtx();
|
||||||
|
if(dictFileName == NULL) {
|
||||||
|
ctx->dictSize = 0;
|
||||||
|
ctx->dictBuffer = NULL;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
ctx->dictSize = UTIL_getFileSize(dictFileName);
|
||||||
|
ctx->dictBuffer = malloc(ctx->dictSize);
|
||||||
|
|
||||||
|
f = fopen(dictFileName, "rb");
|
||||||
|
|
||||||
|
if(!f) {
|
||||||
|
DISPLAY("unable to open file\n");
|
||||||
|
fclose(f);
|
||||||
|
freeContexts(*ctx);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) {
|
||||||
|
DISPLAY("dictionary too large\n");
|
||||||
|
fclose(f);
|
||||||
|
freeContexts(*ctx);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f);
|
||||||
|
if(readSize != ctx->dictSize) {
|
||||||
|
DISPLAY("unable to read file\n");
|
||||||
|
fclose(f);
|
||||||
|
freeContexts(*ctx);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
fclose(f);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Optimizer Memoization Functions
|
||||||
|
**************************************/
|
||||||
|
|
||||||
|
/* return: new length */
|
||||||
|
/* keep old array, will need if iter over strategy. */
|
||||||
|
/* prunes useless params */
|
||||||
|
static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) {
|
||||||
|
size_t i, j = 0;
|
||||||
|
for(i = 0; i < varLength; i++) {
|
||||||
|
if( !((varArray[i] == clog_ind && strat == ZSTD_fast)
|
||||||
|
|| (varArray[i] == slog_ind && strat == ZSTD_fast)
|
||||||
|
|| (varArray[i] == slog_ind && strat == ZSTD_dfast)
|
||||||
|
|| (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) {
|
||||||
|
varNew[j] = varArray[i];
|
||||||
|
j++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return j;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* res should be NUM_PARAMS size */
|
||||||
|
/* constructs varArray from paramValues_t style parameter */
|
||||||
|
/* pass in using dict. */
|
||||||
|
static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) {
|
||||||
|
varInds_t i;
|
||||||
|
size_t j = 0;
|
||||||
|
for(i = 0; i < NUM_PARAMS; i++) {
|
||||||
|
if(paramConstraints.vals[i] == PARAM_UNSET) {
|
||||||
|
if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */
|
||||||
|
res[j] = i; j++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return j;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* length of memo table given free variables */
|
||||||
|
static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) {
|
||||||
|
size_t arrayLen = 1;
|
||||||
|
size_t i;
|
||||||
|
for(i = 0; i < varyLen; i++) {
|
||||||
|
if(varyParams[i] == strt_ind) continue; /* strategy separated by table */
|
||||||
|
arrayLen *= rangetable[varyParams[i]];
|
||||||
|
}
|
||||||
|
return arrayLen;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* returns unique index in memotable of compression parameters */
|
||||||
|
static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) {
|
||||||
|
size_t i;
|
||||||
|
unsigned ind = 0;
|
||||||
|
for(i = 0; i < varyLen; i++) {
|
||||||
|
varInds_t v = varyParams[i];
|
||||||
|
if(v == strt_ind) continue; /* exclude strategy from memotable */
|
||||||
|
ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]);
|
||||||
|
}
|
||||||
|
return ind;
|
||||||
|
}
|
||||||
|
|
||||||
|
static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) {
|
||||||
|
const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
|
||||||
|
switch(mt.tableType) {
|
||||||
|
case directMap:
|
||||||
|
return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)];
|
||||||
|
case xxhashMap:
|
||||||
|
return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen];
|
||||||
|
case noMemo:
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return 0; /* should never happen, stop compiler warnings */
|
||||||
|
}
|
||||||
|
|
||||||
|
static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) {
|
||||||
|
const memoTable_t mt = memoTableArray[p.vals[strt_ind]];
|
||||||
|
switch(mt.tableType) {
|
||||||
|
case directMap:
|
||||||
|
mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break;
|
||||||
|
case xxhashMap:
|
||||||
|
mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break;
|
||||||
|
case noMemo:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* frees all allocated memotables */
|
||||||
|
static void freeMemoTableArray(memoTable_t* const mtAll) {
|
||||||
|
int i;
|
||||||
|
if(mtAll == NULL) { return; }
|
||||||
|
for(i = 1; i <= (int)ZSTD_btultra; i++) {
|
||||||
|
free(mtAll[i].table);
|
||||||
|
}
|
||||||
|
free(mtAll);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* inits memotables for all (including mallocs), all strategies */
|
||||||
|
/* takes unsanitized varyParams */
|
||||||
|
static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) {
|
||||||
|
memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1));
|
||||||
|
ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra;
|
||||||
|
|
||||||
|
if(mtAll == NULL) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
for(i = 1; i <= (int)ZSTD_btultra; i++) {
|
||||||
|
mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* no memoization */
|
||||||
|
if(memoTableLog == 0) {
|
||||||
|
for(i = 1; i <= (int)ZSTD_btultra; i++) {
|
||||||
|
mtAll[i].tableType = noMemo;
|
||||||
|
mtAll[i].table = NULL;
|
||||||
|
mtAll[i].tableLen = 0;
|
||||||
|
}
|
||||||
|
return mtAll;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if(p.vals[strt_ind] != PARAM_UNSET) {
|
||||||
|
stratMin = p.vals[strt_ind];
|
||||||
|
stratMax = p.vals[strt_ind];
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
for(i = stratMin; i <= stratMax; i++) {
|
||||||
|
size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen);
|
||||||
|
mtAll[i].tableType = directMap;
|
||||||
|
|
||||||
|
if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */
|
||||||
|
mtAll[i].tableType = xxhashMap;
|
||||||
|
mtl = (1ULL << memoTableLog);
|
||||||
|
}
|
||||||
|
|
||||||
|
mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl);
|
||||||
|
mtAll[i].tableLen = mtl;
|
||||||
|
|
||||||
|
if(mtAll[i].table == NULL) {
|
||||||
|
freeMemoTableArray(mtAll);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return mtAll;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Sets pc to random unmeasured set of parameters */
|
||||||
|
/* specifiy strategy */
|
||||||
|
static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st)
|
||||||
|
{
|
||||||
|
size_t j;
|
||||||
|
const memoTable_t mt = memoTableArray[st];
|
||||||
|
pc->vals[strt_ind] = st;
|
||||||
|
for(j = 0; j < mt.tableLen; j++) {
|
||||||
|
int i;
|
||||||
|
for(i = 0; i < NUM_PARAMS; i++) {
|
||||||
|
varInds_t v = mt.varArray[i];
|
||||||
|
if(v == strt_ind) continue;
|
||||||
|
pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Benchmarking Functions
|
||||||
|
**************************************/
|
||||||
|
|
||||||
|
/* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */
|
||||||
|
/* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */
|
||||||
|
/* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */
|
||||||
|
/* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */
|
||||||
|
|
||||||
|
/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs */
|
||||||
|
/* dictionary nullable, nothing else though. */
|
||||||
|
static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx,
|
||||||
|
const int cLevel, const paramValues_t* comprParams,
|
||||||
|
const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) {
|
||||||
|
|
||||||
|
U32 i;
|
||||||
|
BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ;
|
||||||
|
const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs;
|
||||||
|
size_t const *const srcSizes = buf.srcSizes;
|
||||||
|
void** const dstPtrs = buf.dstPtrs;
|
||||||
|
size_t const *const dstCapacities = buf.dstCapacities;
|
||||||
|
size_t* const dstSizes = buf.dstSizes;
|
||||||
|
void** const resPtrs = buf.resPtrs;
|
||||||
|
size_t const *const resSizes = buf.resSizes;
|
||||||
|
const void* dictBuffer = ctx.dictBuffer;
|
||||||
|
const size_t dictBufferSize = ctx.dictSize;
|
||||||
|
const size_t nbBlocks = buf.nbBlocks;
|
||||||
|
const size_t srcSize = buf.srcSize;
|
||||||
|
ZSTD_CCtx* cctx = ctx.cctx;
|
||||||
|
ZSTD_DCtx* dctx = ctx.dctx;
|
||||||
|
|
||||||
|
/* warmimg up memory */
|
||||||
|
for(i = 0; i < buf.nbBlocks; i++) {
|
||||||
|
if(mode != BMK_decodeOnly) {
|
||||||
|
RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1);
|
||||||
|
} else {
|
||||||
|
RDG_genBuffer(resPtrs[i], resSizes[i], 0.10, 0.50, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Bench */
|
||||||
|
{
|
||||||
|
/* init args */
|
||||||
|
BMK_initCCtxArgs cctxprep;
|
||||||
|
BMK_initDCtxArgs dctxprep;
|
||||||
|
cctxprep.ctx = cctx;
|
||||||
|
cctxprep.dictBuffer = dictBuffer;
|
||||||
|
cctxprep.dictBufferSize = dictBufferSize;
|
||||||
|
cctxprep.cLevel = cLevel;
|
||||||
|
cctxprep.comprParams = comprParams;
|
||||||
|
dctxprep.dctx = dctx;
|
||||||
|
dctxprep.dictBuffer = dictBuffer;
|
||||||
|
dctxprep.dictBufferSize = dictBufferSize;
|
||||||
|
|
||||||
|
if(loopMode == BMK_timeMode) {
|
||||||
|
BMK_customTimedReturn_t intermediateResultCompress;
|
||||||
|
BMK_customTimedReturn_t intermediateResultDecompress;
|
||||||
|
BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds);
|
||||||
|
BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds);
|
||||||
|
if(mode == BMK_compressOnly) {
|
||||||
|
intermediateResultCompress.completed = 0;
|
||||||
|
intermediateResultDecompress.completed = 1;
|
||||||
|
} else if (mode == BMK_decodeOnly) {
|
||||||
|
intermediateResultCompress.completed = 1;
|
||||||
|
intermediateResultDecompress.completed = 0;
|
||||||
|
} else { /* both */
|
||||||
|
intermediateResultCompress.completed = 0;
|
||||||
|
intermediateResultDecompress.completed = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
while(!intermediateResultCompress.completed) {
|
||||||
|
intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep,
|
||||||
|
nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes);
|
||||||
|
|
||||||
|
if(intermediateResultCompress.result.error) {
|
||||||
|
results.error = intermediateResultCompress.result.error;
|
||||||
|
BMK_freeTimeState(timeStateCompress);
|
||||||
|
BMK_freeTimeState(timeStateDecompress);
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultCompress.result.result.nanoSecPerRun;
|
||||||
|
results.result.cSize = intermediateResultCompress.result.result.sumOfReturn;
|
||||||
|
}
|
||||||
|
|
||||||
|
while(!intermediateResultDecompress.completed) {
|
||||||
|
intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep,
|
||||||
|
nbBlocks, (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL);
|
||||||
|
|
||||||
|
if(intermediateResultDecompress.result.error) {
|
||||||
|
results.error = intermediateResultDecompress.result.error;
|
||||||
|
BMK_freeTimeState(timeStateCompress);
|
||||||
|
BMK_freeTimeState(timeStateDecompress);
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultDecompress.result.result.nanoSecPerRun;
|
||||||
|
}
|
||||||
|
|
||||||
|
BMK_freeTimeState(timeStateCompress);
|
||||||
|
BMK_freeTimeState(timeStateDecompress);
|
||||||
|
|
||||||
|
} else { /* iterMode; */
|
||||||
|
if(mode != BMK_decodeOnly) {
|
||||||
|
|
||||||
|
BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep,
|
||||||
|
nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes, nbSeconds);
|
||||||
|
if(compressionResults.error) {
|
||||||
|
results.error = compressionResults.error;
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
if(compressionResults.result.nanoSecPerRun == 0) {
|
||||||
|
results.result.cSpeed = 0;
|
||||||
|
} else {
|
||||||
|
results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun;
|
||||||
|
}
|
||||||
|
results.result.cSize = compressionResults.result.sumOfReturn;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(mode != BMK_compressOnly) {
|
||||||
|
BMK_customReturn_t decompressionResults;
|
||||||
|
decompressionResults = BMK_benchFunction(
|
||||||
|
&local_defaultDecompress, (void*)(dctx),
|
||||||
|
&local_initDCtx, (void*)&dctxprep, nbBlocks,
|
||||||
|
(const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL,
|
||||||
|
nbSeconds);
|
||||||
|
|
||||||
|
if(decompressionResults.error) {
|
||||||
|
results.error = decompressionResults.error;
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(decompressionResults.result.nanoSecPerRun == 0) {
|
||||||
|
results.result.dSpeed = 0;
|
||||||
|
} else {
|
||||||
|
results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* Bench */
|
||||||
|
results.result.cMem = (1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx);
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int BMK_benchParam(BMK_result_t* resultPtr,
|
||||||
|
const buffers_t buf, const contexts_t ctx,
|
||||||
|
const paramValues_t cParams) {
|
||||||
|
BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3);
|
||||||
|
*resultPtr = res.result;
|
||||||
|
return res.error;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) { \
|
||||||
|
if(conditional) { \
|
||||||
|
BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec); \
|
||||||
|
if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; } \
|
||||||
|
if(mode != BMK_decodeOnly) { \
|
||||||
|
resultvar.cSpeed = tmpret.result.cSpeed; \
|
||||||
|
resultvar.cSize = tmpret.result.cSize; \
|
||||||
|
resultvar.cMem = tmpret.result.cMem; \
|
||||||
|
} \
|
||||||
|
if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; } \
|
||||||
|
} \
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */
|
||||||
|
#define VARIANCE 1.2
|
||||||
|
static int allBench(BMK_result_t* resultPtr,
|
||||||
|
const buffers_t buf, const contexts_t ctx,
|
||||||
|
const paramValues_t cParams,
|
||||||
|
const constraint_t target,
|
||||||
|
BMK_result_t* winnerResult, int feas) {
|
||||||
|
BMK_result_t resultMax, benchres;
|
||||||
|
U64 loopDurationC = 0, loopDurationD = 0;
|
||||||
|
double uncertaintyConstantC = 3., uncertaintyConstantD = 3.;
|
||||||
|
double winnerRS;
|
||||||
|
/* initial benchmarking, gives exact ratio and memory, warms up future runs */
|
||||||
|
CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1);
|
||||||
|
|
||||||
|
winnerRS = resultScore(*winnerResult, buf.srcSize, target);
|
||||||
|
DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS);
|
||||||
|
|
||||||
|
*resultPtr = benchres;
|
||||||
|
|
||||||
|
/* calculate uncertainty in compression / decompression runs */
|
||||||
|
if(benchres.cSpeed) {
|
||||||
|
loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed);
|
||||||
|
uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(benchres.dSpeed) {
|
||||||
|
loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed);
|
||||||
|
uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* anything with worse ratio in feas is definitely worse, discard */
|
||||||
|
if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) {
|
||||||
|
return WORSE_RESULT;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* second run, if first run is too short, gives approximate cSpeed + dSpeed */
|
||||||
|
CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1);
|
||||||
|
CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly, BMK_iterMode, 1);
|
||||||
|
|
||||||
|
*resultPtr = benchres;
|
||||||
|
|
||||||
|
/* optimistic assumption of benchres */
|
||||||
|
resultMax = benchres;
|
||||||
|
resultMax.cSpeed *= uncertaintyConstantC * VARIANCE;
|
||||||
|
resultMax.dSpeed *= uncertaintyConstantD * VARIANCE;
|
||||||
|
|
||||||
|
/* disregard infeasible results in feas mode */
|
||||||
|
/* disregard if resultMax < winner in infeas mode */
|
||||||
|
if((feas && !feasible(resultMax, target)) ||
|
||||||
|
(!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) {
|
||||||
|
return WORSE_RESULT;
|
||||||
|
}
|
||||||
|
|
||||||
|
CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1);
|
||||||
|
CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly, BMK_timeMode, 1);
|
||||||
|
|
||||||
|
*resultPtr = benchres;
|
||||||
|
|
||||||
|
/* compare by resultScore when in infeas */
|
||||||
|
/* compare by compareResultLT when in feas */
|
||||||
|
if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) ||
|
||||||
|
(feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) ) {
|
||||||
|
return BETTER_RESULT;
|
||||||
|
} else {
|
||||||
|
return WORSE_RESULT;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#define INFEASIBLE_THRESHOLD 200
|
||||||
|
/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */
|
||||||
|
static int benchMemo(BMK_result_t* resultPtr,
|
||||||
|
const buffers_t buf, const contexts_t ctx,
|
||||||
|
const paramValues_t cParams,
|
||||||
|
const constraint_t target,
|
||||||
|
BMK_result_t* winnerResult, memoTable_t* const memoTableArray,
|
||||||
|
const int feas) {
|
||||||
|
static int bmcount = 0;
|
||||||
|
int res;
|
||||||
|
|
||||||
|
if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; }
|
||||||
|
|
||||||
|
res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas);
|
||||||
|
|
||||||
|
if(DEBUG && !(bmcount % 250)) {
|
||||||
|
DISPLAY("Count: %d\n", bmcount);
|
||||||
|
bmcount++;
|
||||||
|
}
|
||||||
|
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize);
|
||||||
|
|
||||||
|
if(res == BETTER_RESULT || feas) {
|
||||||
|
memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
U64 cSpeed_min;
|
||||||
|
U64 dSpeed_min;
|
||||||
U32 windowLog_max;
|
U32 windowLog_max;
|
||||||
ZSTD_strategy strategy_max;
|
ZSTD_strategy strategy_max;
|
||||||
} level_constraints_t;
|
} level_constraints_t;
|
||||||
@@ -243,15 +1659,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1)
|
|||||||
} }
|
} }
|
||||||
}
|
}
|
||||||
|
|
||||||
static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params,
|
static int BMK_seed(winnerInfo_t* winners, const paramValues_t params,
|
||||||
const void* srcBuffer, size_t srcSize,
|
const buffers_t buf, const contexts_t ctx)
|
||||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx)
|
|
||||||
{
|
{
|
||||||
BMK_result_t testResult;
|
BMK_result_t testResult;
|
||||||
int better = 0;
|
int better = 0;
|
||||||
int cLevel;
|
int cLevel;
|
||||||
|
|
||||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, dctx, params);
|
BMK_benchParam(&testResult, buf, ctx, params);
|
||||||
|
|
||||||
|
|
||||||
for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
|
for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) {
|
||||||
@@ -259,32 +1674,32 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
|
|||||||
continue; /* not fast enough for this level */
|
continue; /* not fast enough for this level */
|
||||||
if (testResult.dSpeed < g_level_constraint[cLevel].dSpeed_min)
|
if (testResult.dSpeed < g_level_constraint[cLevel].dSpeed_min)
|
||||||
continue; /* not fast enough for this level */
|
continue; /* not fast enough for this level */
|
||||||
if (params.windowLog > g_level_constraint[cLevel].windowLog_max)
|
if (params.vals[wlog_ind] > g_level_constraint[cLevel].windowLog_max)
|
||||||
continue; /* too much memory for this level */
|
continue; /* too much memory for this level */
|
||||||
if (params.strategy > g_level_constraint[cLevel].strategy_max)
|
if (params.vals[strt_ind] > g_level_constraint[cLevel].strategy_max)
|
||||||
continue; /* forbidden strategy for this level */
|
continue; /* forbidden strategy for this level */
|
||||||
if (winners[cLevel].result.cSize==0) {
|
if (winners[cLevel].result.cSize==0) {
|
||||||
/* first solution for this cLevel */
|
/* first solution for this cLevel */
|
||||||
winners[cLevel].result = testResult;
|
winners[cLevel].result = testResult;
|
||||||
winners[cLevel].params = params;
|
winners[cLevel].params = params;
|
||||||
BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
|
BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
|
||||||
better = 1;
|
better = 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
|
if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) {
|
||||||
/* Validate solution is "good enough" */
|
/* Validate solution is "good enough" */
|
||||||
double W_ratio = (double)srcSize / testResult.cSize;
|
double W_ratio = (double)buf.srcSize / testResult.cSize;
|
||||||
double O_ratio = (double)srcSize / winners[cLevel].result.cSize;
|
double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize;
|
||||||
double W_ratioNote = log (W_ratio);
|
double W_ratioNote = log (W_ratio);
|
||||||
double O_ratioNote = log (O_ratio);
|
double O_ratioNote = log (O_ratio);
|
||||||
size_t W_DMemUsed = (1 << params.windowLog) + (16 KB);
|
size_t W_DMemUsed = (1 << params.vals[wlog_ind]) + (16 KB);
|
||||||
size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB);
|
size_t O_DMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + (16 KB);
|
||||||
double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
|
double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
|
||||||
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
|
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
|
||||||
|
|
||||||
size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params);
|
size_t W_CMemUsed = (1 << params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(params));
|
||||||
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params);
|
size_t O_CMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(winners[cLevel].params));
|
||||||
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
|
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
|
||||||
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
|
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
|
||||||
|
|
||||||
@@ -314,16 +1729,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
|
|||||||
/* too large compression speed difference for the compression benefit */
|
/* too large compression speed difference for the compression benefit */
|
||||||
if (W_ratio > O_ratio)
|
if (W_ratio > O_ratio)
|
||||||
DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
|
DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
|
||||||
W_ratio, testResult.cSpeed / 1000000,
|
W_ratio, (double)testResult.cSpeed / (1 MB),
|
||||||
O_ratio, winners[cLevel].result.cSpeed / 1000000., cLevel);
|
O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB), cLevel);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (W_DSpeed_note < O_DSpeed_note ) {
|
if (W_DSpeed_note < O_DSpeed_note ) {
|
||||||
/* too large decompression speed difference for the compression benefit */
|
/* too large decompression speed difference for the compression benefit */
|
||||||
if (W_ratio > O_ratio)
|
if (W_ratio > O_ratio)
|
||||||
DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
|
DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n",
|
||||||
W_ratio, testResult.dSpeed / 1000000.,
|
W_ratio, (double)testResult.dSpeed / (1 MB),
|
||||||
O_ratio, winners[cLevel].result.dSpeed / 1000000., cLevel);
|
O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB), cLevel);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,7 +1747,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
|
|||||||
|
|
||||||
winners[cLevel].result = testResult;
|
winners[cLevel].result = testResult;
|
||||||
winners[cLevel].params = params;
|
winners[cLevel].params = params;
|
||||||
BMK_printWinner(stdout, cLevel, testResult, params, srcSize);
|
BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize);
|
||||||
|
|
||||||
better = 1;
|
better = 1;
|
||||||
} }
|
} }
|
||||||
@@ -340,158 +1755,72 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para
|
|||||||
return better;
|
return better;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
/* nullified useless params, to ensure count stats */
|
* Compression Level Table Generation Functions
|
||||||
static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params)
|
**************************************/
|
||||||
{
|
|
||||||
g_params = params;
|
|
||||||
if (params.strategy == ZSTD_fast)
|
|
||||||
g_params.chainLog = 0, g_params.searchLog = 0;
|
|
||||||
if (params.strategy == ZSTD_dfast)
|
|
||||||
g_params.searchLog = 0;
|
|
||||||
if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra)
|
|
||||||
g_params.targetLength = 0;
|
|
||||||
return &g_params;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static void paramVariation(ZSTD_compressionParameters* ptr)
|
|
||||||
{
|
|
||||||
ZSTD_compressionParameters p;
|
|
||||||
U32 validated = 0;
|
|
||||||
while (!validated) {
|
|
||||||
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
|
|
||||||
p = *ptr;
|
|
||||||
for ( ; nbChanges ; nbChanges--) {
|
|
||||||
const U32 changeID = FUZ_rand(&g_rand) % 14;
|
|
||||||
switch(changeID)
|
|
||||||
{
|
|
||||||
case 0:
|
|
||||||
p.chainLog++; break;
|
|
||||||
case 1:
|
|
||||||
p.chainLog--; break;
|
|
||||||
case 2:
|
|
||||||
p.hashLog++; break;
|
|
||||||
case 3:
|
|
||||||
p.hashLog--; break;
|
|
||||||
case 4:
|
|
||||||
p.searchLog++; break;
|
|
||||||
case 5:
|
|
||||||
p.searchLog--; break;
|
|
||||||
case 6:
|
|
||||||
p.windowLog++; break;
|
|
||||||
case 7:
|
|
||||||
p.windowLog--; break;
|
|
||||||
case 8:
|
|
||||||
p.searchLength++; break;
|
|
||||||
case 9:
|
|
||||||
p.searchLength--; break;
|
|
||||||
case 10:
|
|
||||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break;
|
|
||||||
case 11:
|
|
||||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break;
|
|
||||||
case 12:
|
|
||||||
p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
|
|
||||||
case 13:
|
|
||||||
p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
validated = !ZSTD_isError(ZSTD_checkCParams(p));
|
|
||||||
}
|
|
||||||
*ptr = p;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
#define PARAMTABLELOG 25
|
#define PARAMTABLELOG 25
|
||||||
#define PARAMTABLESIZE (1<<PARAMTABLELOG)
|
#define PARAMTABLESIZE (1<<PARAMTABLELOG)
|
||||||
#define PARAMTABLEMASK (PARAMTABLESIZE-1)
|
#define PARAMTABLEMASK (PARAMTABLESIZE-1)
|
||||||
static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
|
static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
|
||||||
|
|
||||||
#define NB_TESTS_PLAYED(p) \
|
static BYTE* NB_TESTS_PLAYED(paramValues_t p) {
|
||||||
g_alreadyTested[(XXH64(sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK]
|
ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(p));
|
||||||
|
return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK];
|
||||||
|
}
|
||||||
|
|
||||||
static void playAround(FILE* f, winnerInfo_t* winners,
|
static void playAround(FILE* f, winnerInfo_t* winners,
|
||||||
ZSTD_compressionParameters params,
|
paramValues_t p,
|
||||||
const void* srcBuffer, size_t srcSize,
|
const buffers_t buf, const contexts_t ctx)
|
||||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx)
|
|
||||||
{
|
{
|
||||||
int nbVariations = 0;
|
int nbVariations = 0, i;
|
||||||
UTIL_time_t const clockStart = UTIL_getTime();
|
UTIL_time_t const clockStart = UTIL_getTime();
|
||||||
|
|
||||||
while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
|
while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) {
|
||||||
ZSTD_compressionParameters p = params;
|
BYTE* b;
|
||||||
|
|
||||||
if (nbVariations++ > g_maxNbVariations) break;
|
if (nbVariations++ > g_maxNbVariations) break;
|
||||||
paramVariation(&p);
|
|
||||||
|
do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } }
|
||||||
|
while(!paramValid(p));
|
||||||
|
|
||||||
/* exclude faster if already played params */
|
/* exclude faster if already played params */
|
||||||
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1))
|
if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
/* test */
|
/* test */
|
||||||
NB_TESTS_PLAYED(p)++;
|
b = NB_TESTS_PLAYED(p);
|
||||||
if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx, dctx)) continue;
|
(*b)++;
|
||||||
|
if (!BMK_seed(winners, p, buf, ctx)) continue;
|
||||||
|
|
||||||
/* improvement found => search more */
|
/* improvement found => search more */
|
||||||
BMK_printWinners(f, winners, srcSize);
|
BMK_printWinners(f, winners, buf.srcSize);
|
||||||
playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx);
|
playAround(f, winners, p, buf, ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static ZSTD_compressionParameters randomParams(void)
|
|
||||||
{
|
|
||||||
ZSTD_compressionParameters p;
|
|
||||||
U32 validated = 0;
|
|
||||||
while (!validated) {
|
|
||||||
/* totally random entry */
|
|
||||||
p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN;
|
|
||||||
p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN;
|
|
||||||
p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN;
|
|
||||||
p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN;
|
|
||||||
p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN;
|
|
||||||
p.targetLength=(FUZ_rand(&g_rand) % (512));
|
|
||||||
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1));
|
|
||||||
validated = !ZSTD_isError(ZSTD_checkCParams(p));
|
|
||||||
}
|
|
||||||
return p;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void BMK_selectRandomStart(
|
static void BMK_selectRandomStart(
|
||||||
FILE* f, winnerInfo_t* winners,
|
FILE* f, winnerInfo_t* winners,
|
||||||
const void* srcBuffer, size_t srcSize,
|
const buffers_t buf, const contexts_t ctx)
|
||||||
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx)
|
|
||||||
{
|
{
|
||||||
U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
|
U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1);
|
||||||
if ((id==0) || (winners[id].params.windowLog==0)) {
|
if ((id==0) || (winners[id].params.vals[wlog_ind]==0)) {
|
||||||
/* use some random entry */
|
/* use some random entry */
|
||||||
ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
|
paramValues_t const p = adjustParams(cParamsToPVals(pvalsToCParams(randomParams())), /* defaults nonCompression parameters */
|
||||||
playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx);
|
buf.srcSize, 0);
|
||||||
|
playAround(f, winners, p, buf, ctx);
|
||||||
} else {
|
} else {
|
||||||
playAround(f, winners, winners[id].params, srcBuffer, srcSize, ctx, dctx);
|
playAround(f, winners, winners[id].params, buf, ctx);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx)
|
||||||
static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize)
|
|
||||||
{
|
{
|
||||||
BMK_result_t testResult;
|
paramValues_t params;
|
||||||
g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
|
|
||||||
BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, g_params);
|
|
||||||
DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize,
|
|
||||||
testResult.cSpeed / 1000000, testResult.dSpeed / 1000000);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize)
|
|
||||||
{
|
|
||||||
ZSTD_compressionParameters params;
|
|
||||||
winnerInfo_t winners[NB_LEVELS_TRACKED+1];
|
winnerInfo_t winners[NB_LEVELS_TRACKED+1];
|
||||||
const char* const rfName = "grillResults.txt";
|
const char* const rfName = "grillResults.txt";
|
||||||
FILE* const f = fopen(rfName, "w");
|
FILE* const f = fopen(rfName, "w");
|
||||||
const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */
|
|
||||||
|
|
||||||
/* init */
|
/* init */
|
||||||
assert(g_singleRun==0);
|
assert(g_singleRun==0);
|
||||||
@@ -499,12 +1828,12 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src
|
|||||||
if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
|
if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
|
||||||
|
|
||||||
if (g_target) {
|
if (g_target) {
|
||||||
BMK_init_level_constraints(g_target*1000000);
|
BMK_init_level_constraints(g_target * (1 MB));
|
||||||
} else {
|
} else {
|
||||||
/* baseline config for level 1 */
|
/* baseline config for level 1 */
|
||||||
ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0);
|
paramValues_t const l1params = cParamsToPVals(ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize));
|
||||||
BMK_result_t testResult;
|
BMK_result_t testResult;
|
||||||
BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, l1params);
|
BMK_benchParam(&testResult, buf, ctx, l1params);
|
||||||
BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
|
BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -512,251 +1841,543 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src
|
|||||||
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
|
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
|
||||||
int i;
|
int i;
|
||||||
for (i=0; i<=maxSeeds; i++) {
|
for (i=0; i<=maxSeeds; i++) {
|
||||||
params = ZSTD_getCParams(i, blockSize, 0);
|
params = cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, 0));
|
||||||
BMK_seed(winners, params, srcBuffer, srcSize, cctx, dctx);
|
BMK_seed(winners, params, buf, ctx);
|
||||||
} }
|
} }
|
||||||
BMK_printWinners(f, winners, srcSize);
|
BMK_printWinners(f, winners, buf.srcSize);
|
||||||
|
|
||||||
/* start tests */
|
/* start tests */
|
||||||
{ const time_t grillStart = time(NULL);
|
{ const UTIL_time_t grillStart = UTIL_getTime();
|
||||||
do {
|
do {
|
||||||
BMK_selectRandomStart(f, winners, srcBuffer, srcSize, cctx, dctx);
|
BMK_selectRandomStart(f, winners, buf, ctx);
|
||||||
} while (BMK_timeSpan(grillStart) < g_grillDuration_s);
|
} while (BMK_timeSpan(grillStart) < g_timeLimit_s);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* end summary */
|
/* end summary */
|
||||||
BMK_printWinners(f, winners, srcSize);
|
BMK_printWinners(f, winners, buf.srcSize);
|
||||||
DISPLAY("grillParams operations completed \n");
|
DISPLAY("grillParams operations completed \n");
|
||||||
|
|
||||||
/* clean up*/
|
/* clean up*/
|
||||||
fclose(f);
|
fclose(f);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void BMK_benchMem_usingCCtx(ZSTD_CCtx* const cctx, ZSTD_DCtx* const dctx, const void* srcBuffer, size_t srcSize)
|
/*-************************************
|
||||||
{
|
* Single Benchmark Functions
|
||||||
if (g_singleRun)
|
**************************************/
|
||||||
return BMK_benchOnce(cctx, dctx, srcBuffer, srcSize);
|
|
||||||
else
|
|
||||||
return BMK_benchFullTable(cctx, dctx, srcBuffer, srcSize);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void BMK_benchMemCCtxInit(const void* srcBuffer, size_t srcSize)
|
static int benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel) {
|
||||||
{
|
BMK_result_t testResult;
|
||||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
|
||||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
|
||||||
if (cctx==NULL || dctx==NULL) { DISPLAY("Context Creation failed \n"); exit(1); }
|
|
||||||
BMK_benchMem_usingCCtx(cctx, dctx, srcBuffer, srcSize);
|
|
||||||
ZSTD_freeCCtx(cctx);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
if(BMK_benchParam(&testResult, buf, ctx, g_params)) {
|
||||||
|
DISPLAY("Error during benchmarking\n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
static int benchSample(void)
|
BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize);
|
||||||
{
|
|
||||||
const char* const name = "Sample 10MB";
|
|
||||||
size_t const benchedSize = 10000000;
|
|
||||||
|
|
||||||
void* origBuff = malloc(benchedSize);
|
|
||||||
if (!origBuff) { perror("not enough memory"); return 12; }
|
|
||||||
|
|
||||||
/* Fill buffer */
|
|
||||||
RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0);
|
|
||||||
|
|
||||||
/* bench */
|
|
||||||
DISPLAY("\r%79s\r", "");
|
|
||||||
DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100));
|
|
||||||
BMK_benchMemCCtxInit(origBuff, benchedSize);
|
|
||||||
|
|
||||||
free(origBuff);
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int benchSample(double compressibility, int cLevel)
|
||||||
|
{
|
||||||
|
const char* const name = "Sample 10MB";
|
||||||
|
size_t const benchedSize = 10 MB;
|
||||||
|
void* srcBuffer = malloc(benchedSize);
|
||||||
|
int ret = 0;
|
||||||
|
|
||||||
|
buffers_t buf;
|
||||||
|
contexts_t ctx;
|
||||||
|
|
||||||
|
if(srcBuffer == NULL) {
|
||||||
|
DISPLAY("Out of Memory\n");
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
|
||||||
|
|
||||||
|
if(createBuffersFromMemory(&buf, srcBuffer, 1, &benchedSize)) {
|
||||||
|
DISPLAY("Buffer Creation Error\n");
|
||||||
|
free(srcBuffer);
|
||||||
|
return 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(createContexts(&ctx, NULL)) {
|
||||||
|
DISPLAY("Context Creation Error\n");
|
||||||
|
freeBuffers(buf);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* bench */
|
||||||
|
DISPLAY("\r%79s\r", "");
|
||||||
|
DISPLAY("using %s %i%%: \n", name, (int)(compressibility*100));
|
||||||
|
|
||||||
|
if(g_singleRun) {
|
||||||
|
ret = benchOnce(buf, ctx, cLevel);
|
||||||
|
} else {
|
||||||
|
BMK_benchFullTable(buf, ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
freeBuffers(buf);
|
||||||
|
freeContexts(ctx);
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
/* benchFiles() :
|
/* benchFiles() :
|
||||||
* note: while this function takes a table of filenames,
|
* note: while this function takes a table of filenames,
|
||||||
* in practice, only the first filename will be used */
|
* in practice, only the first filename will be used */
|
||||||
int benchFiles(const char** fileNamesTable, int nbFiles)
|
int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, const int cLevel)
|
||||||
{
|
{
|
||||||
int fileIdx=0;
|
buffers_t buf;
|
||||||
|
contexts_t ctx;
|
||||||
|
int ret = 0;
|
||||||
|
|
||||||
/* Loop for each file */
|
if(createBuffers(&buf, fileNamesTable, nbFiles)) {
|
||||||
while (fileIdx<nbFiles) {
|
DISPLAY("unable to load files\n");
|
||||||
const char* const inFileName = fileNamesTable[fileIdx++];
|
return 1;
|
||||||
FILE* const inFile = fopen( inFileName, "rb" );
|
|
||||||
U64 const inFileSize = UTIL_getFileSize(inFileName);
|
|
||||||
size_t benchedSize;
|
|
||||||
void* origBuff;
|
|
||||||
|
|
||||||
/* Check file existence */
|
|
||||||
if (inFile==NULL) {
|
|
||||||
DISPLAY( "Pb opening %s\n", inFileName);
|
|
||||||
return 11;
|
|
||||||
}
|
|
||||||
if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
|
|
||||||
DISPLAY("Pb evaluating size of %s \n", inFileName);
|
|
||||||
fclose(inFile);
|
|
||||||
return 11;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Memory allocation */
|
|
||||||
benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
|
|
||||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
|
||||||
if (benchedSize < inFileSize)
|
|
||||||
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
|
|
||||||
origBuff = malloc(benchedSize);
|
|
||||||
if (origBuff==NULL) {
|
|
||||||
DISPLAY("\nError: not enough memory!\n");
|
|
||||||
fclose(inFile);
|
|
||||||
return 12;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Fill input buffer */
|
|
||||||
DISPLAY("Loading %s... \r", inFileName);
|
|
||||||
{ size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
|
|
||||||
fclose(inFile);
|
|
||||||
if(readSize != benchedSize) {
|
|
||||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
|
||||||
free(origBuff);
|
|
||||||
return 13;
|
|
||||||
} }
|
|
||||||
|
|
||||||
/* bench */
|
|
||||||
DISPLAY("\r%79s\r", "");
|
|
||||||
DISPLAY("using %s : \n", inFileName);
|
|
||||||
BMK_benchMemCCtxInit(origBuff, benchedSize);
|
|
||||||
|
|
||||||
/* clean */
|
|
||||||
free(origBuff);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
if(createContexts(&ctx, dictFileName)) {
|
||||||
}
|
DISPLAY("unable to load dictionary\n");
|
||||||
|
freeBuffers(buf);
|
||||||
|
return 2;
|
||||||
static void BMK_translateAdvancedParams(ZSTD_compressionParameters params)
|
|
||||||
{
|
|
||||||
DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n",
|
|
||||||
params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy));
|
|
||||||
}
|
|
||||||
|
|
||||||
/* optimizeForSize():
|
|
||||||
* targetSpeed : expressed in MB/s */
|
|
||||||
int optimizeForSize(const char* inFileName, U32 targetSpeed)
|
|
||||||
{
|
|
||||||
FILE* const inFile = fopen( inFileName, "rb" );
|
|
||||||
U64 const inFileSize = UTIL_getFileSize(inFileName);
|
|
||||||
size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3;
|
|
||||||
void* origBuff;
|
|
||||||
/* Init */
|
|
||||||
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
|
|
||||||
if (inFileSize == UTIL_FILESIZE_UNKNOWN) {
|
|
||||||
DISPLAY("Pb evaluatin size of %s \n", inFileName);
|
|
||||||
fclose(inFile);
|
|
||||||
return 11;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Memory allocation & restrictions */
|
|
||||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
|
||||||
if (benchedSize < inFileSize) {
|
|
||||||
DISPLAY("Not enough memory for '%s' \n", inFileName);
|
|
||||||
fclose(inFile);
|
|
||||||
return 11;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Alloc */
|
|
||||||
origBuff = malloc(benchedSize);
|
|
||||||
if(!origBuff) {
|
|
||||||
DISPLAY("\nError: not enough memory!\n");
|
|
||||||
fclose(inFile);
|
|
||||||
return 12;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Fill input buffer */
|
|
||||||
DISPLAY("Loading %s... \r", inFileName);
|
|
||||||
{ size_t const readSize = fread(origBuff, 1, benchedSize, inFile);
|
|
||||||
fclose(inFile);
|
|
||||||
if(readSize != benchedSize) {
|
|
||||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
|
||||||
free(origBuff);
|
|
||||||
return 13;
|
|
||||||
} }
|
|
||||||
|
|
||||||
/* bench */
|
|
||||||
DISPLAY("\r%79s\r", "");
|
DISPLAY("\r%79s\r", "");
|
||||||
DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed);
|
if(nbFiles == 1) {
|
||||||
targetSpeed *= 1000000;
|
DISPLAY("using %s : \n", fileNamesTable[0]);
|
||||||
{ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
|
} else {
|
||||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
DISPLAY("using %d Files : \n", nbFiles);
|
||||||
winnerInfo_t winner;
|
}
|
||||||
BMK_result_t candidate;
|
|
||||||
const size_t blockSize = g_blockSize ? g_blockSize : benchedSize;
|
|
||||||
|
|
||||||
/* init */
|
if(g_singleRun) {
|
||||||
if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;}
|
ret = benchOnce(buf, ctx, cLevel);
|
||||||
memset(&winner, 0, sizeof(winner));
|
} else {
|
||||||
winner.result.cSize = (size_t)(-1);
|
BMK_benchFullTable(buf, ctx);
|
||||||
|
}
|
||||||
|
|
||||||
/* find best solution from default params */
|
freeBuffers(buf);
|
||||||
{ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
|
freeContexts(ctx);
|
||||||
int i;
|
return ret;
|
||||||
for (i=1; i<=maxSeeds; i++) {
|
}
|
||||||
ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0);
|
|
||||||
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams);
|
|
||||||
if (candidate.cSpeed < (double)targetSpeed) {
|
/*-************************************
|
||||||
|
* Local Optimization Functions
|
||||||
|
**************************************/
|
||||||
|
|
||||||
|
/* One iteration of hill climbing. Specifically, it first tries all
|
||||||
|
* valid parameter configurations w/ manhattan distance 1 and picks the best one
|
||||||
|
* failing that, it progressively tries candidates further and further away (up to #dim + 2)
|
||||||
|
* if it finds a candidate exceeding winnerInfo, it will repeat. Otherwise, it will stop the
|
||||||
|
* current stage of hill climbing.
|
||||||
|
* Each iteration of hill climbing proceeds in 2 'phases'. Phase 1 climbs according to
|
||||||
|
* the resultScore function, which is effectively a linear increase in reward until it reaches
|
||||||
|
* the constraint-satisfying value, it which point any excess results in only logarithmic reward.
|
||||||
|
* This aims to find some constraint-satisfying point.
|
||||||
|
* Phase 2 optimizes in accordance with what the original function sets out to maximize, with
|
||||||
|
* all feasible solutions valued over all infeasible solutions.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/* sanitize all params here.
|
||||||
|
* all generation after random should be sanitized. (maybe sanitize random)
|
||||||
|
*/
|
||||||
|
static winnerInfo_t climbOnce(const constraint_t target,
|
||||||
|
memoTable_t* mtAll,
|
||||||
|
const buffers_t buf, const contexts_t ctx,
|
||||||
|
const paramValues_t init) {
|
||||||
|
/*
|
||||||
|
* cparam - currently considered 'center'
|
||||||
|
* candidate - params to benchmark/results
|
||||||
|
* winner - best option found so far.
|
||||||
|
*/
|
||||||
|
paramValues_t cparam = init;
|
||||||
|
winnerInfo_t candidateInfo, winnerInfo;
|
||||||
|
int better = 1;
|
||||||
|
int feas = 0;
|
||||||
|
|
||||||
|
winnerInfo = initWinnerInfo(init);
|
||||||
|
candidateInfo = winnerInfo;
|
||||||
|
|
||||||
|
{
|
||||||
|
winnerInfo_t bestFeasible1 = initWinnerInfo(cparam);
|
||||||
|
DEBUGOUTPUT("Climb Part 1\n");
|
||||||
|
while(better) {
|
||||||
|
|
||||||
|
int offset;
|
||||||
|
size_t i, dist;
|
||||||
|
const size_t varLen = mtAll[cparam.vals[strt_ind]].varLen;
|
||||||
|
better = 0;
|
||||||
|
DEBUGOUTPUT("Start\n");
|
||||||
|
cparam = winnerInfo.params;
|
||||||
|
candidateInfo.params = cparam;
|
||||||
|
/* all dist-1 candidates */
|
||||||
|
for(i = 0; i < varLen; i++) {
|
||||||
|
for(offset = -1; offset <= 1; offset += 2) {
|
||||||
|
CHECKTIME(winnerInfo);
|
||||||
|
candidateInfo.params = cparam;
|
||||||
|
paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params);
|
||||||
|
|
||||||
|
if(paramValid(candidateInfo.params)) {
|
||||||
|
int res;
|
||||||
|
res = benchMemo(&candidateInfo.result, buf, ctx,
|
||||||
|
sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas);
|
||||||
|
DEBUGOUTPUT("Res: %d\n", res);
|
||||||
|
if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */
|
||||||
|
winnerInfo = candidateInfo;
|
||||||
|
better = 1;
|
||||||
|
if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
|
||||||
|
bestFeasible1 = winnerInfo;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(better) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */
|
||||||
|
for(i = 0; i < (1 << varLen) / varLen + 2; i++) {
|
||||||
|
int res;
|
||||||
|
CHECKTIME(winnerInfo);
|
||||||
|
candidateInfo.params = cparam;
|
||||||
|
/* param error checking already done here */
|
||||||
|
paramVariation(&candidateInfo.params, mtAll, (U32)dist);
|
||||||
|
|
||||||
|
res = benchMemo(&candidateInfo.result, buf, ctx,
|
||||||
|
sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas);
|
||||||
|
DEBUGOUTPUT("Res: %d\n", res);
|
||||||
|
if(res == BETTER_RESULT) { /* synonymous with better in this case*/
|
||||||
|
winnerInfo = candidateInfo;
|
||||||
|
better = 1;
|
||||||
|
if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) {
|
||||||
|
bestFeasible1 = winnerInfo;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
if(better) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if ( (candidate.cSize < winner.result.cSize)
|
}
|
||||||
| ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) )
|
|
||||||
{
|
if(!better) { /* infeas -> feas -> stop */
|
||||||
winner.params = CParams;
|
if(feas) { return winnerInfo; }
|
||||||
winner.result = candidate;
|
|
||||||
BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize);
|
feas = 1;
|
||||||
} }
|
better = 1;
|
||||||
|
winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */
|
||||||
|
DEBUGOUTPUT("Climb Part 2\n");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
winnerInfo = bestFeasible1;
|
||||||
BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize);
|
|
||||||
|
|
||||||
BMK_translateAdvancedParams(winner.params);
|
|
||||||
|
|
||||||
/* start tests */
|
|
||||||
{ time_t const grillStart = time(NULL);
|
|
||||||
do {
|
|
||||||
ZSTD_compressionParameters params = winner.params;
|
|
||||||
paramVariation(¶ms);
|
|
||||||
if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */
|
|
||||||
params = ZSTD_adjustCParams(params, blockSize, 0);
|
|
||||||
|
|
||||||
/* exclude faster if already played set of params */
|
|
||||||
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
|
|
||||||
|
|
||||||
/* test */
|
|
||||||
NB_TESTS_PLAYED(params)++;
|
|
||||||
BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params);
|
|
||||||
|
|
||||||
/* improvement found => new winner */
|
|
||||||
if ( (candidate.cSpeed > targetSpeed)
|
|
||||||
& ( (candidate.cSize < winner.result.cSize)
|
|
||||||
| ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) )
|
|
||||||
{
|
|
||||||
winner.params = params;
|
|
||||||
winner.result = candidate;
|
|
||||||
BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize);
|
|
||||||
BMK_translateAdvancedParams(winner.params);
|
|
||||||
}
|
|
||||||
} while (BMK_timeSpan(grillStart) < g_grillDuration_s);
|
|
||||||
}
|
|
||||||
/* end summary */
|
|
||||||
|
|
||||||
BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize);
|
|
||||||
BMK_translateAdvancedParams(winner.params);
|
|
||||||
DISPLAY("grillParams size - optimizer completed \n");
|
|
||||||
|
|
||||||
/* clean up*/
|
|
||||||
ZSTD_freeCCtx(ctx);
|
|
||||||
ZSTD_freeDCtx(dctx);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
free(origBuff);
|
return winnerInfo;
|
||||||
return 0;
|
}
|
||||||
|
|
||||||
|
/* Optimizes for a fixed strategy */
|
||||||
|
|
||||||
|
/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visitied)
|
||||||
|
weight more on visit for bad results, less on good results/more on later results / ones with more failures.
|
||||||
|
allocate memoTable here.
|
||||||
|
*/
|
||||||
|
static winnerInfo_t optimizeFixedStrategy(
|
||||||
|
const buffers_t buf, const contexts_t ctx,
|
||||||
|
const constraint_t target, paramValues_t paramTarget,
|
||||||
|
const ZSTD_strategy strat,
|
||||||
|
memoTable_t* memoTableArray, const int tries) {
|
||||||
|
int i = 0;
|
||||||
|
|
||||||
|
paramValues_t init;
|
||||||
|
winnerInfo_t winnerInfo, candidateInfo;
|
||||||
|
winnerInfo = initWinnerInfo(emptyParams());
|
||||||
|
/* so climb is given the right fixed strategy */
|
||||||
|
paramTarget.vals[strt_ind] = strat;
|
||||||
|
/* to pass ZSTD_checkCParams */
|
||||||
|
paramTarget = cParamUnsetMin(paramTarget);
|
||||||
|
|
||||||
|
init = paramTarget;
|
||||||
|
|
||||||
|
for(i = 0; i < tries; i++) {
|
||||||
|
DEBUGOUTPUT("Restart\n");
|
||||||
|
do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize));
|
||||||
|
candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init);
|
||||||
|
if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) {
|
||||||
|
winnerInfo = candidateInfo;
|
||||||
|
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize);
|
||||||
|
i = 0;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
CHECKTIME(winnerInfo);
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
return winnerInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* goes best, best-1, best+1, best-2, ... */
|
||||||
|
/* return 0 if nothing remaining */
|
||||||
|
static int nextStrategy(const int currentStrategy, const int bestStrategy) {
|
||||||
|
if(bestStrategy <= currentStrategy) {
|
||||||
|
int candidate = 2 * bestStrategy - currentStrategy - 1;
|
||||||
|
if(candidate < 1) {
|
||||||
|
candidate = currentStrategy + 1;
|
||||||
|
if(candidate > (int)ZSTD_btultra) {
|
||||||
|
return 0;
|
||||||
|
} else {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
} else { /* bestStrategy >= currentStrategy */
|
||||||
|
int candidate = 2 * bestStrategy - currentStrategy;
|
||||||
|
if(candidate > (int)ZSTD_btultra) {
|
||||||
|
candidate = currentStrategy - 1;
|
||||||
|
if(candidate < 1) {
|
||||||
|
return 0;
|
||||||
|
} else {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* experiment with playing with this and decay value */
|
||||||
|
|
||||||
|
/* main fn called when using --optimize */
|
||||||
|
/* Does strategy selection by benchmarking default compression levels
|
||||||
|
* then optimizes by strategy, starting with the best one and moving
|
||||||
|
* progressively moving further away by number
|
||||||
|
* args:
|
||||||
|
* fileNamesTable - list of files to benchmark
|
||||||
|
* nbFiles - length of fileNamesTable
|
||||||
|
* dictFileName - name of dictionary file if one, else NULL
|
||||||
|
* target - performance constraints (cSpeed, dSpeed, cMem)
|
||||||
|
* paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast)
|
||||||
|
* cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
|
||||||
|
*/
|
||||||
|
|
||||||
|
static int g_maxTries = 5;
|
||||||
|
#define TRY_DECAY 1
|
||||||
|
|
||||||
|
static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget,
|
||||||
|
const int cLevelOpt, const int cLevelRun, const U32 memoTableLog)
|
||||||
|
{
|
||||||
|
varInds_t varArray [NUM_PARAMS];
|
||||||
|
int ret = 0;
|
||||||
|
const size_t varLen = variableParams(paramTarget, varArray, dictFileName != NULL);
|
||||||
|
winnerInfo_t winner = initWinnerInfo(emptyParams());
|
||||||
|
memoTable_t* allMT = NULL;
|
||||||
|
paramValues_t paramBase;
|
||||||
|
contexts_t ctx;
|
||||||
|
buffers_t buf;
|
||||||
|
g_time = UTIL_getTime();
|
||||||
|
|
||||||
|
if(createBuffers(&buf, fileNamesTable, nbFiles)) {
|
||||||
|
DISPLAY("unable to load files\n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(createContexts(&ctx, dictFileName)) {
|
||||||
|
DISPLAY("unable to load dictionary\n");
|
||||||
|
freeBuffers(buf);
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(nbFiles == 1) {
|
||||||
|
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[0]);
|
||||||
|
} else {
|
||||||
|
DISPLAYLEVEL(2, "Loading %lu Files... \r", (unsigned long)nbFiles);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* sanitize paramTarget */
|
||||||
|
optimizerAdjustInput(¶mTarget, buf.maxBlockSize);
|
||||||
|
paramBase = cParamUnsetMin(paramTarget);
|
||||||
|
|
||||||
|
allMT = createMemoTableArray(paramTarget, varArray, varLen, memoTableLog);
|
||||||
|
|
||||||
|
if(!allMT) {
|
||||||
|
DISPLAY("MemoTable Init Error\n");
|
||||||
|
ret = 2;
|
||||||
|
goto _cleanUp;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* default strictnesses */
|
||||||
|
if(g_strictness == PARAM_UNSET) {
|
||||||
|
if(g_optmode) {
|
||||||
|
g_strictness = 100;
|
||||||
|
} else {
|
||||||
|
g_strictness = 90;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if(0 >= g_strictness || g_strictness > 100) {
|
||||||
|
DISPLAY("Strictness Outside of Bounds\n");
|
||||||
|
ret = 4;
|
||||||
|
goto _cleanUp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* use level'ing mode instead of normal target mode */
|
||||||
|
if(g_optmode) {
|
||||||
|
winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize));
|
||||||
|
if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) {
|
||||||
|
ret = 3;
|
||||||
|
goto _cleanUp;
|
||||||
|
}
|
||||||
|
|
||||||
|
g_lvltarget = winner.result;
|
||||||
|
g_lvltarget.cSpeed *= ((double)g_strictness) / 100;
|
||||||
|
g_lvltarget.dSpeed *= ((double)g_strictness) / 100;
|
||||||
|
g_lvltarget.cSize /= ((double)g_strictness) / 100;
|
||||||
|
|
||||||
|
target.cSpeed = (U32)g_lvltarget.cSpeed;
|
||||||
|
target.dSpeed = (U32)g_lvltarget.dSpeed;
|
||||||
|
|
||||||
|
BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Don't want it to return anything worse than the best known result */
|
||||||
|
if(g_singleRun) {
|
||||||
|
BMK_result_t res;
|
||||||
|
g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize);
|
||||||
|
if(BMK_benchParam(&res, buf, ctx, g_params)) {
|
||||||
|
ret = 45;
|
||||||
|
goto _cleanUp;
|
||||||
|
}
|
||||||
|
if(compareResultLT(winner.result, res, relaxTarget(target), buf.srcSize)) {
|
||||||
|
winner.result = res;
|
||||||
|
winner.params = g_params;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* bench */
|
||||||
|
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||||
|
if(nbFiles == 1) {
|
||||||
|
DISPLAYLEVEL(2, "optimizing for %s", fileNamesTable[0]);
|
||||||
|
} else {
|
||||||
|
DISPLAYLEVEL(2, "optimizing for %lu Files", (unsigned long)nbFiles);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(target.cSpeed != 0) { DISPLAYLEVEL(2," - limit compression speed %u MB/s", target.cSpeed >> 20); }
|
||||||
|
if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", target.dSpeed >> 20); }
|
||||||
|
if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); }
|
||||||
|
|
||||||
|
DISPLAYLEVEL(2, "\n");
|
||||||
|
findClockGranularity();
|
||||||
|
|
||||||
|
{
|
||||||
|
paramValues_t CParams;
|
||||||
|
|
||||||
|
/* find best solution from default params */
|
||||||
|
{
|
||||||
|
/* strategy selection */
|
||||||
|
const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel();
|
||||||
|
DEBUGOUTPUT("Strategy Selection\n");
|
||||||
|
if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
|
||||||
|
BMK_result_t candidate;
|
||||||
|
int i;
|
||||||
|
for (i=1; i<=maxSeeds; i++) {
|
||||||
|
int ec;
|
||||||
|
CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, ctx.dictSize)), paramTarget);
|
||||||
|
ec = BMK_benchParam(&candidate, buf, ctx, CParams);
|
||||||
|
BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize);
|
||||||
|
|
||||||
|
if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) {
|
||||||
|
winner.result = candidate;
|
||||||
|
winner.params = CParams;
|
||||||
|
}
|
||||||
|
|
||||||
|
CHECKTIMEGT(ret, 0, _displayCleanUp); /* if pass time limit, stop */
|
||||||
|
/* if the current params are too slow, just stop. */
|
||||||
|
if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; }
|
||||||
|
}
|
||||||
|
|
||||||
|
BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DEBUGOUTPUT("Real Opt\n");
|
||||||
|
/* start 'real' optimization */
|
||||||
|
{
|
||||||
|
int bestStrategy = (int)winner.params.vals[strt_ind];
|
||||||
|
if(paramTarget.vals[strt_ind] == PARAM_UNSET) {
|
||||||
|
int st = bestStrategy;
|
||||||
|
int tries = g_maxTries;
|
||||||
|
|
||||||
|
{
|
||||||
|
/* one iterations of hill climbing with the level-defined parameters. */
|
||||||
|
winnerInfo_t w1 = climbOnce(target, allMT, buf, ctx, winner.params);
|
||||||
|
if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) {
|
||||||
|
winner = w1;
|
||||||
|
}
|
||||||
|
CHECKTIMEGT(ret, 0, _displayCleanUp);
|
||||||
|
}
|
||||||
|
|
||||||
|
while(st && tries > 0) {
|
||||||
|
winnerInfo_t wc;
|
||||||
|
DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]);
|
||||||
|
|
||||||
|
wc = optimizeFixedStrategy(buf, ctx, target, paramBase, st, allMT, tries);
|
||||||
|
|
||||||
|
if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) {
|
||||||
|
winner = wc;
|
||||||
|
tries = g_maxTries;
|
||||||
|
bestStrategy = st;
|
||||||
|
} else {
|
||||||
|
st = nextStrategy(st, bestStrategy);
|
||||||
|
tries -= TRY_DECAY;
|
||||||
|
}
|
||||||
|
CHECKTIMEGT(ret, 0, _displayCleanUp);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], allMT, g_maxTries);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/* no solution found */
|
||||||
|
if(winner.result.cSize == (size_t)-1) {
|
||||||
|
ret = 1;
|
||||||
|
DISPLAY("No feasible solution found\n");
|
||||||
|
goto _cleanUp;
|
||||||
|
}
|
||||||
|
/* end summary */
|
||||||
|
_displayCleanUp:
|
||||||
|
if(g_displayLevel >= 0) { BMK_displayOneResult(stdout, winner, buf.srcSize); }
|
||||||
|
BMK_translateAdvancedParams(stdout, winner.params);
|
||||||
|
DISPLAYLEVEL(1, "grillParams size - optimizer completed \n");
|
||||||
|
|
||||||
|
}
|
||||||
|
_cleanUp:
|
||||||
|
freeContexts(ctx);
|
||||||
|
freeBuffers(buf);
|
||||||
|
freeMemoTableArray(allMT);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* CLI parsing functions
|
||||||
|
**************************************/
|
||||||
|
|
||||||
|
/** longCommandWArg() :
|
||||||
|
* check if *stringPtr is the same as longCommand.
|
||||||
|
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
|
||||||
|
* @return 0 and doesn't modify *stringPtr otherwise.
|
||||||
|
* from zstdcli.c
|
||||||
|
*/
|
||||||
|
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
|
||||||
|
{
|
||||||
|
size_t const comSize = strlen(longCommand);
|
||||||
|
int const result = !strncmp(*stringPtr, longCommand, comSize);
|
||||||
|
if (result) *stringPtr += comSize;
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void errorOut(const char* msg)
|
static void errorOut(const char* msg)
|
||||||
@@ -772,7 +2393,9 @@ static void errorOut(const char* msg)
|
|||||||
static unsigned readU32FromChar(const char** stringPtr)
|
static unsigned readU32FromChar(const char** stringPtr)
|
||||||
{
|
{
|
||||||
const char errorMsg[] = "error: numeric value too large";
|
const char errorMsg[] = "error: numeric value too large";
|
||||||
|
unsigned sign = 1;
|
||||||
unsigned result = 0;
|
unsigned result = 0;
|
||||||
|
if(**stringPtr == '-') { sign = (unsigned)-1; (*stringPtr)++; }
|
||||||
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||||
unsigned const max = (((unsigned)(-1)) / 10) - 1;
|
unsigned const max = (((unsigned)(-1)) / 10) - 1;
|
||||||
if (result > max) errorOut(errorMsg);
|
if (result > max) errorOut(errorMsg);
|
||||||
@@ -790,6 +2413,22 @@ static unsigned readU32FromChar(const char** stringPtr)
|
|||||||
if (**stringPtr=='i') (*stringPtr)++;
|
if (**stringPtr=='i') (*stringPtr)++;
|
||||||
if (**stringPtr=='B') (*stringPtr)++;
|
if (**stringPtr=='B') (*stringPtr)++;
|
||||||
}
|
}
|
||||||
|
return result * sign;
|
||||||
|
}
|
||||||
|
|
||||||
|
static double readDoubleFromChar(const char** stringPtr)
|
||||||
|
{
|
||||||
|
double result = 0, divide = 10;
|
||||||
|
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||||
|
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
|
||||||
|
}
|
||||||
|
if(**stringPtr!='.') {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
(*stringPtr)++;
|
||||||
|
while ((**stringPtr >='0') && (**stringPtr <='9')) {
|
||||||
|
result += (double)(**stringPtr - '0') / divide, divide *= 10, (*stringPtr)++ ;
|
||||||
|
}
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -806,15 +2445,16 @@ static int usage(const char* exename)
|
|||||||
static int usage_advanced(void)
|
static int usage_advanced(void)
|
||||||
{
|
{
|
||||||
DISPLAY( "\nAdvanced options :\n");
|
DISPLAY( "\nAdvanced options :\n");
|
||||||
DISPLAY( " -T# : set level 1 speed objective \n");
|
DISPLAY( " -T# : set level 1 speed objective \n");
|
||||||
DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n");
|
DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n");
|
||||||
DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS);
|
DISPLAY( " --optimize= : same as -O with more verbose syntax (see README.md)\n");
|
||||||
DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n");
|
DISPLAY( " -S : Single run \n");
|
||||||
DISPLAY( " -S : Single run \n");
|
DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n");
|
||||||
DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n");
|
DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100);
|
||||||
DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100);
|
DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", g_timeLimit_s, (double)g_timeLimit_s / 3600);
|
||||||
DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600);
|
DISPLAY( " -v : Prints Benchmarking output\n");
|
||||||
DISPLAY( " -v : Prints Benchmarking output\n");
|
DISPLAY( " -D : Next argument dictionary file\n");
|
||||||
|
DISPLAY( " -s : Seperate Files\n");
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -825,41 +2465,87 @@ static int badusage(const char* exename)
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } }
|
||||||
|
/* 1 if successful parse, 0 otherwise */
|
||||||
|
static int parse_params(const char** argptr, paramValues_t* pv) {
|
||||||
|
int matched = 0;
|
||||||
|
const char* argOrig = *argptr;
|
||||||
|
varInds_t v;
|
||||||
|
for(v = 0; v < NUM_PARAMS; v++) {
|
||||||
|
if(longCommandWArg(argptr,g_shortParamNames[v]) || longCommandWArg(argptr, g_paramNames[v])) {
|
||||||
|
if(**argptr == '=') {
|
||||||
|
(*argptr)++;
|
||||||
|
pv->vals[v] = readU32FromChar(argptr);
|
||||||
|
matched = 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* reset and try again */
|
||||||
|
*argptr = argOrig;
|
||||||
|
}
|
||||||
|
return matched;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Main
|
||||||
|
**************************************/
|
||||||
|
|
||||||
int main(int argc, const char** argv)
|
int main(int argc, const char** argv)
|
||||||
{
|
{
|
||||||
int i,
|
int i,
|
||||||
filenamesStart=0,
|
filenamesStart=0,
|
||||||
result;
|
result;
|
||||||
const char* exename=argv[0];
|
const char* exename=argv[0];
|
||||||
const char* input_filename=0;
|
const char* input_filename = NULL;
|
||||||
U32 optimizer = 0;
|
const char* dictFileName = NULL;
|
||||||
U32 main_pause = 0;
|
U32 main_pause = 0;
|
||||||
U32 targetSpeed = 0;
|
int cLevelOpt = 0, cLevelRun = 0;
|
||||||
|
int seperateFiles = 0;
|
||||||
|
double compressibility = COMPRESSIBILITY_DEFAULT;
|
||||||
|
U32 memoTableLog = PARAM_UNSET;
|
||||||
|
constraint_t target = { 0, 0, (U32)-1 };
|
||||||
|
|
||||||
|
paramValues_t paramTarget = emptyParams();
|
||||||
|
g_params = emptyParams();
|
||||||
|
|
||||||
assert(argc>=1); /* for exename */
|
assert(argc>=1); /* for exename */
|
||||||
|
|
||||||
/* Welcome message */
|
|
||||||
DISPLAY(WELCOME_MESSAGE);
|
|
||||||
|
|
||||||
for(i=1; i<argc; i++) {
|
for(i=1; i<argc; i++) {
|
||||||
const char* argument = argv[i];
|
const char* argument = argv[i];
|
||||||
|
DEBUGOUTPUT("%d: %s\n", i, argument);
|
||||||
assert(argument != NULL);
|
assert(argument != NULL);
|
||||||
|
|
||||||
if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; }
|
if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; }
|
||||||
|
|
||||||
/* Decode command (note : aggregated commands are allowed) */
|
if (longCommandWArg(&argument, "--optimize=")) {
|
||||||
if (longCommandWArg(&argument, "--zstd=")) {
|
g_optimizer = 1;
|
||||||
g_singleRun = 1;
|
|
||||||
g_params = ZSTD_getCParams(2, g_blockSize, 0);
|
|
||||||
for ( ; ;) {
|
for ( ; ;) {
|
||||||
if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { g_params.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
if(parse_params(&argument, ¶mTarget)) { if(argument[0] == ',') { argument++; continue; } else break; }
|
||||||
if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { g_params.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed);
|
||||||
if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { g_params.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed);
|
||||||
if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { g_params.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem);
|
||||||
if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { g_params.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
|
||||||
if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { g_params.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
|
||||||
if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { g_params.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; }
|
PARSE_SUB_ARGS("memoLimitLog=", "memLog=", memoTableLog);
|
||||||
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; }
|
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
|
||||||
|
|
||||||
|
DISPLAY("invalid optimization parameter \n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (argument[0] != 0) {
|
||||||
|
DISPLAY("invalid --optimize= format\n");
|
||||||
|
return 1; /* check the end of string */
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
} else if (longCommandWArg(&argument, "--zstd=")) {
|
||||||
|
/* Decode command (note : aggregated commands are allowed) */
|
||||||
|
g_singleRun = 1;
|
||||||
|
for ( ; ;) {
|
||||||
|
if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; }
|
||||||
|
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
|
||||||
|
|
||||||
DISPLAY("invalid compression parameter \n");
|
DISPLAY("invalid compression parameter \n");
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
@@ -868,7 +2554,45 @@ int main(int argc, const char** argv)
|
|||||||
DISPLAY("invalid --zstd= format\n");
|
DISPLAY("invalid --zstd= format\n");
|
||||||
return 1; /* check the end of string */
|
return 1; /* check the end of string */
|
||||||
}
|
}
|
||||||
|
continue;
|
||||||
/* if not return, success */
|
/* if not return, success */
|
||||||
|
|
||||||
|
} else if (longCommandWArg(&argument, "--display=")) {
|
||||||
|
/* Decode command (note : aggregated commands are allowed) */
|
||||||
|
memset(g_silenceParams, 1, sizeof(g_silenceParams));
|
||||||
|
for ( ; ;) {
|
||||||
|
int found = 0;
|
||||||
|
varInds_t v;
|
||||||
|
for(v = 0; v < NUM_PARAMS; v++) {
|
||||||
|
if(longCommandWArg(&argument, g_shortParamNames[v]) || longCommandWArg(&argument, g_paramNames[v])) {
|
||||||
|
g_silenceParams[v] = 0;
|
||||||
|
found = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(longCommandWArg(&argument, "compressionParameters") || longCommandWArg(&argument, "cParams")) {
|
||||||
|
for(v = 0; v <= strt_ind; v++) {
|
||||||
|
g_silenceParams[v] = 0;
|
||||||
|
}
|
||||||
|
found = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if(found) {
|
||||||
|
if(argument[0]==',') {
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
DISPLAY("invalid parameter name parameter \n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (argument[0] != 0) {
|
||||||
|
DISPLAY("invalid --display format\n");
|
||||||
|
return 1; /* check the end of string */
|
||||||
|
}
|
||||||
|
continue;
|
||||||
} else if (argument[0]=='-') {
|
} else if (argument[0]=='-') {
|
||||||
argument++;
|
argument++;
|
||||||
|
|
||||||
@@ -882,66 +2606,58 @@ int main(int argc, const char** argv)
|
|||||||
|
|
||||||
/* Pause at the end (hidden option) */
|
/* Pause at the end (hidden option) */
|
||||||
case 'p': main_pause = 1; argument++; break;
|
case 'p': main_pause = 1; argument++; break;
|
||||||
/* Modify Nb Iterations */
|
|
||||||
|
|
||||||
case 'i':
|
|
||||||
argument++;
|
|
||||||
g_nbIterations = readU32FromChar(&argument);
|
|
||||||
break;
|
|
||||||
|
|
||||||
/* Sample compressibility (when no file provided) */
|
/* Sample compressibility (when no file provided) */
|
||||||
case 'P':
|
case 'P':
|
||||||
argument++;
|
argument++;
|
||||||
{ U32 const proba32 = readU32FromChar(&argument);
|
{ U32 const proba32 = readU32FromChar(&argument);
|
||||||
g_compressibility = (double)proba32 / 100.;
|
compressibility = (double)proba32 / 100.;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case 'O':
|
|
||||||
argument++;
|
|
||||||
optimizer = 1;
|
|
||||||
targetSpeed = readU32FromChar(&argument);
|
|
||||||
break;
|
|
||||||
|
|
||||||
/* Run Single conf */
|
/* Run Single conf */
|
||||||
case 'S':
|
case 'S':
|
||||||
g_singleRun = 1;
|
g_singleRun = 1;
|
||||||
argument++;
|
argument++;
|
||||||
g_params = ZSTD_getCParams(2, g_blockSize, 0);
|
|
||||||
for ( ; ; ) {
|
for ( ; ; ) {
|
||||||
switch(*argument)
|
switch(*argument)
|
||||||
{
|
{
|
||||||
case 'w':
|
case 'w':
|
||||||
argument++;
|
argument++;
|
||||||
g_params.windowLog = readU32FromChar(&argument);
|
g_params.vals[wlog_ind] = readU32FromChar(&argument);
|
||||||
continue;
|
continue;
|
||||||
case 'c':
|
case 'c':
|
||||||
argument++;
|
argument++;
|
||||||
g_params.chainLog = readU32FromChar(&argument);
|
g_params.vals[clog_ind] = readU32FromChar(&argument);
|
||||||
continue;
|
continue;
|
||||||
case 'h':
|
case 'h':
|
||||||
argument++;
|
argument++;
|
||||||
g_params.hashLog = readU32FromChar(&argument);
|
g_params.vals[hlog_ind] = readU32FromChar(&argument);
|
||||||
continue;
|
continue;
|
||||||
case 's':
|
case 's':
|
||||||
argument++;
|
argument++;
|
||||||
g_params.searchLog = readU32FromChar(&argument);
|
g_params.vals[slog_ind] = readU32FromChar(&argument);
|
||||||
continue;
|
continue;
|
||||||
case 'l': /* search length */
|
case 'l': /* search length */
|
||||||
argument++;
|
argument++;
|
||||||
g_params.searchLength = readU32FromChar(&argument);
|
g_params.vals[slen_ind] = readU32FromChar(&argument);
|
||||||
continue;
|
continue;
|
||||||
case 't': /* target length */
|
case 't': /* target length */
|
||||||
argument++;
|
argument++;
|
||||||
g_params.targetLength = readU32FromChar(&argument);
|
g_params.vals[tlen_ind] = readU32FromChar(&argument);
|
||||||
continue;
|
continue;
|
||||||
case 'S': /* strategy */
|
case 'S': /* strategy */
|
||||||
argument++;
|
argument++;
|
||||||
g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument);
|
g_params.vals[strt_ind] = readU32FromChar(&argument);
|
||||||
|
continue;
|
||||||
|
case 'f': /* forceAttachDict */
|
||||||
|
argument++;
|
||||||
|
g_params.vals[fadt_ind] = readU32FromChar(&argument);
|
||||||
continue;
|
continue;
|
||||||
case 'L':
|
case 'L':
|
||||||
{ int const cLevel = readU32FromChar(&argument);
|
{ argument++;
|
||||||
g_params = ZSTD_getCParams(cLevel, g_blockSize, 0);
|
cLevelRun = readU32FromChar(&argument);
|
||||||
|
g_params = emptyParams();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
default : ;
|
default : ;
|
||||||
@@ -967,7 +2683,32 @@ int main(int argc, const char** argv)
|
|||||||
/* caps runtime (in seconds) */
|
/* caps runtime (in seconds) */
|
||||||
case 't':
|
case 't':
|
||||||
argument++;
|
argument++;
|
||||||
g_grillDuration_s = (double)readU32FromChar(&argument);
|
g_timeLimit_s = readU32FromChar(&argument);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 's':
|
||||||
|
argument++;
|
||||||
|
seperateFiles = 1;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'q':
|
||||||
|
while (argument[0] == 'q') { argument++; g_displayLevel--; }
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'v':
|
||||||
|
while (argument[0] == 'v') { argument++; g_displayLevel++; }
|
||||||
|
break;
|
||||||
|
|
||||||
|
/* load dictionary file (only applicable for optimizer rn) */
|
||||||
|
case 'D':
|
||||||
|
if(i == argc - 1) { /* last argument, return error. */
|
||||||
|
DISPLAY("Dictionary file expected but not given : %d\n", i);
|
||||||
|
return 1;
|
||||||
|
} else {
|
||||||
|
i++;
|
||||||
|
dictFileName = argv[i];
|
||||||
|
argument += strlen(argument);
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
/* Unknown command */
|
/* Unknown command */
|
||||||
@@ -981,19 +2722,35 @@ int main(int argc, const char** argv)
|
|||||||
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
|
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Welcome message */
|
||||||
|
DISPLAYLEVEL(2, WELCOME_MESSAGE);
|
||||||
|
|
||||||
if (filenamesStart==0) {
|
if (filenamesStart==0) {
|
||||||
if (optimizer) {
|
if (g_optimizer) {
|
||||||
DISPLAY("Optimizer Expects File\n");
|
DISPLAY("Optimizer Expects File\n");
|
||||||
return 1;
|
return 1;
|
||||||
} else {
|
} else {
|
||||||
result = benchSample();
|
result = benchSample(compressibility, cLevelRun);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (optimizer) {
|
if(seperateFiles) {
|
||||||
result = optimizeForSize(input_filename, targetSpeed);
|
for(i = 0; i < argc - filenamesStart; i++) {
|
||||||
|
if (g_optimizer) {
|
||||||
|
result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog);
|
||||||
|
if(result) { DISPLAY("Error on File %d", i); return result; }
|
||||||
|
} else {
|
||||||
|
result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevelRun);
|
||||||
|
if(result) { DISPLAY("Error on File %d", i); return result; }
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
result = benchFiles(argv+filenamesStart, argc-filenamesStart);
|
if (g_optimizer) {
|
||||||
} }
|
result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog);
|
||||||
|
} else {
|
||||||
|
result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevelRun);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
|
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }
|
||||||
|
|
||||||
|
|||||||
+42
-1
@@ -427,7 +427,7 @@ $ECHO "- Create second (different) dictionary"
|
|||||||
$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
|
$ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
|
||||||
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
|
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
|
||||||
$ECHO "- Create dictionary with short dictID"
|
$ECHO "- Create dictionary with short dictID"
|
||||||
$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c --dictID=1 -o tmpDict1
|
$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1
|
||||||
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
||||||
$ECHO "- Create dictionary with size limit"
|
$ECHO "- Create dictionary with size limit"
|
||||||
$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
|
$ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
|
||||||
@@ -444,6 +444,47 @@ $ZSTD --train-cover *.c ../programs/*.c
|
|||||||
test -f dictionary
|
test -f dictionary
|
||||||
rm tmp* dictionary
|
rm tmp* dictionary
|
||||||
|
|
||||||
|
|
||||||
|
$ECHO "\n===> fastCover dictionary builder : advanced options "
|
||||||
|
|
||||||
|
TESTFILE=../programs/zstdcli.c
|
||||||
|
./datagen > tmpDict
|
||||||
|
$ECHO "- Create first dictionary"
|
||||||
|
$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 *.c ../programs/*.c -o tmpDict
|
||||||
|
cp $TESTFILE tmp
|
||||||
|
$ZSTD -f tmp -D tmpDict
|
||||||
|
$ZSTD -d tmp.zst -D tmpDict -fo result
|
||||||
|
$DIFF $TESTFILE result
|
||||||
|
$ECHO "- Create second (different) dictionary"
|
||||||
|
$ZSTD --train-fastcover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC
|
||||||
|
$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
|
||||||
|
$ECHO "- Create dictionary with short dictID"
|
||||||
|
$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1
|
||||||
|
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
||||||
|
$ECHO "- Create dictionary with size limit"
|
||||||
|
$ZSTD --train-fastcover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K
|
||||||
|
$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples"
|
||||||
|
$ZSTD --train-fastcover=split=90 -r *.c ../programs/*.c
|
||||||
|
$ZSTD --train-fastcover=split=80 -r *.c ../programs/*.c
|
||||||
|
$ECHO "- Create dictionary using all samples for both training and testing"
|
||||||
|
$ZSTD --train-fastcover=split=100 -r *.c ../programs/*.c
|
||||||
|
$ECHO "- Create dictionary using f=16"
|
||||||
|
$ZSTD --train-fastcover=f=16 -r *.c ../programs/*.c
|
||||||
|
$ECHO "- Create dictionary using accel=2"
|
||||||
|
$ZSTD --train-fastcover=accel=2 -r *.c ../programs/*.c
|
||||||
|
$ECHO "- Create dictionary using accel=10"
|
||||||
|
$ZSTD --train-fastcover=accel=10 -r *.c ../programs/*.c
|
||||||
|
$ECHO "- Create dictionary with multithreading"
|
||||||
|
$ZSTD --train-fastcover -T4 -r *.c ../programs/*.c
|
||||||
|
$ECHO "- Test -o before --train-fastcover"
|
||||||
|
rm -f tmpDict dictionary
|
||||||
|
$ZSTD -o tmpDict --train-fastcover *.c ../programs/*.c
|
||||||
|
test -f tmpDict
|
||||||
|
$ZSTD --train-fastcover *.c ../programs/*.c
|
||||||
|
test -f dictionary
|
||||||
|
rm tmp* dictionary
|
||||||
|
|
||||||
|
|
||||||
$ECHO "\n===> legacy dictionary builder "
|
$ECHO "\n===> legacy dictionary builder "
|
||||||
|
|
||||||
TESTFILE=../programs/zstdcli.c
|
TESTFILE=../programs/zstdcli.c
|
||||||
|
|||||||
@@ -212,7 +212,7 @@ static void loadFile(void* buffer, const char* fileName, size_t fileSize)
|
|||||||
static void fileCheck(const char* fileName, int testCCtxParams)
|
static void fileCheck(const char* fileName, int testCCtxParams)
|
||||||
{
|
{
|
||||||
size_t const fileSize = getFileSize(fileName);
|
size_t const fileSize = getFileSize(fileName);
|
||||||
void* buffer = malloc(fileSize);
|
void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */);
|
||||||
if (!buffer) {
|
if (!buffer) {
|
||||||
fprintf(stderr, "not enough memory \n");
|
fprintf(stderr, "not enough memory \n");
|
||||||
exit(4);
|
exit(4);
|
||||||
|
|||||||
@@ -144,6 +144,8 @@ static const void *symbols[] = {
|
|||||||
/* zdict.h: advanced functions */
|
/* zdict.h: advanced functions */
|
||||||
&ZDICT_trainFromBuffer_cover,
|
&ZDICT_trainFromBuffer_cover,
|
||||||
&ZDICT_optimizeTrainFromBuffer_cover,
|
&ZDICT_optimizeTrainFromBuffer_cover,
|
||||||
|
&ZDICT_trainFromBuffer_fastCover,
|
||||||
|
&ZDICT_optimizeTrainFromBuffer_fastCover,
|
||||||
&ZDICT_finalizeDictionary,
|
&ZDICT_finalizeDictionary,
|
||||||
&ZDICT_trainFromBuffer_legacy,
|
&ZDICT_trainFromBuffer_legacy,
|
||||||
&ZDICT_addEntropyTablesFromBuffer,
|
&ZDICT_addEntropyTablesFromBuffer,
|
||||||
|
|||||||
@@ -600,7 +600,6 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
|
size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
|
||||||
DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (U32)initError);
|
DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (U32)initError);
|
||||||
if (ZSTD_isError(initError)) goto _output_error;
|
if (ZSTD_isError(initError)) goto _output_error;
|
||||||
cSize = 0;
|
|
||||||
outBuff.dst = compressedBuffer;
|
outBuff.dst = compressedBuffer;
|
||||||
outBuff.size = compressedBufferSize;
|
outBuff.size = compressedBufferSize;
|
||||||
outBuff.pos = 0;
|
outBuff.pos = 0;
|
||||||
@@ -718,7 +717,6 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
|
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
|
||||||
size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
|
size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
|
||||||
if (ZSTD_isError(initError)) goto _output_error;
|
if (ZSTD_isError(initError)) goto _output_error;
|
||||||
cSize = 0;
|
|
||||||
outBuff.dst = compressedBuffer;
|
outBuff.dst = compressedBuffer;
|
||||||
outBuff.size = compressedBufferSize;
|
outBuff.size = compressedBufferSize;
|
||||||
outBuff.pos = 0;
|
outBuff.pos = 0;
|
||||||
|
|||||||
@@ -573,10 +573,10 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
|||||||
do {
|
do {
|
||||||
testmem = (BYTE*)malloc((size_t)requiredMem);
|
testmem = (BYTE*)malloc((size_t)requiredMem);
|
||||||
requiredMem -= step;
|
requiredMem -= step;
|
||||||
} while (!testmem);
|
} while (!testmem && requiredMem); /* do not allocate zero bytes */
|
||||||
|
|
||||||
free(testmem);
|
free(testmem);
|
||||||
return (size_t)(requiredMem);
|
return (size_t)(requiredMem+1); /* avoid zero */
|
||||||
}
|
}
|
||||||
|
|
||||||
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||||
@@ -734,7 +734,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
|||||||
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
||||||
if (benchedSize < totalSizeToLoad)
|
if (benchedSize < totalSizeToLoad)
|
||||||
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
||||||
srcBuffer = malloc(benchedSize);
|
srcBuffer = malloc(benchedSize + !benchedSize);
|
||||||
if (!srcBuffer) EXM_THROW(12, "not enough memory");
|
if (!srcBuffer) EXM_THROW(12, "not enough memory");
|
||||||
|
|
||||||
/* Load input buffer */
|
/* Load input buffer */
|
||||||
|
|||||||
+2
-2
@@ -111,7 +111,7 @@ local gzFile gz_open(path, fd, mode)
|
|||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
/* allocate gzFile structure to return */
|
/* allocate gzFile structure to return */
|
||||||
state = (gz_statep)(gz_state*)malloc(sizeof(gz_state));
|
state.state = (gz_state*)malloc(sizeof(gz_state));
|
||||||
if (state.state == NULL)
|
if (state.state == NULL)
|
||||||
return NULL;
|
return NULL;
|
||||||
state.state->size = 0; /* no buffers allocated yet */
|
state.state->size = 0; /* no buffers allocated yet */
|
||||||
@@ -266,7 +266,7 @@ local gzFile gz_open(path, fd, mode)
|
|||||||
gz_reset(state);
|
gz_reset(state);
|
||||||
|
|
||||||
/* return stream */
|
/* return stream */
|
||||||
return (gzFile)state.file;
|
return state.file;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* -- see zlib.h -- */
|
/* -- see zlib.h -- */
|
||||||
|
|||||||
@@ -6,6 +6,8 @@
|
|||||||
* For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
|
* For conditions of distribution and use, see http://www.zlib.net/zlib_license.html
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
#include <assert.h>
|
||||||
|
|
||||||
#include "gzguts.h"
|
#include "gzguts.h"
|
||||||
|
|
||||||
/* Local functions */
|
/* Local functions */
|
||||||
@@ -24,7 +26,7 @@ local int gz_init(state)
|
|||||||
z_streamp strm = &(state.state->strm);
|
z_streamp strm = &(state.state->strm);
|
||||||
|
|
||||||
/* allocate input buffer (double size for gzprintf) */
|
/* allocate input buffer (double size for gzprintf) */
|
||||||
state.state->in = (unsigned char *)malloc(state.state->want << 1);
|
state.state->in = (unsigned char*)malloc(state.state->want << 1);
|
||||||
if (state.state->in == NULL) {
|
if (state.state->in == NULL) {
|
||||||
gz_error(state, Z_MEM_ERROR, "out of memory");
|
gz_error(state, Z_MEM_ERROR, "out of memory");
|
||||||
return -1;
|
return -1;
|
||||||
@@ -33,7 +35,7 @@ local int gz_init(state)
|
|||||||
/* only need output buffer and deflate state if compressing */
|
/* only need output buffer and deflate state if compressing */
|
||||||
if (!state.state->direct) {
|
if (!state.state->direct) {
|
||||||
/* allocate output buffer */
|
/* allocate output buffer */
|
||||||
state.state->out = (unsigned char *)malloc(state.state->want);
|
state.state->out = (unsigned char*)malloc(state.state->want);
|
||||||
if (state.state->out == NULL) {
|
if (state.state->out == NULL) {
|
||||||
free(state.state->in);
|
free(state.state->in);
|
||||||
gz_error(state, Z_MEM_ERROR, "out of memory");
|
gz_error(state, Z_MEM_ERROR, "out of memory");
|
||||||
@@ -284,6 +286,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
|
|||||||
gz_statep state;
|
gz_statep state;
|
||||||
|
|
||||||
/* get internal structure */
|
/* get internal structure */
|
||||||
|
assert(size != 0);
|
||||||
if (file == NULL)
|
if (file == NULL)
|
||||||
return 0;
|
return 0;
|
||||||
state = (gz_statep)file;
|
state = (gz_statep)file;
|
||||||
@@ -294,7 +297,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
|
|||||||
|
|
||||||
/* compute bytes to read -- error on overflow */
|
/* compute bytes to read -- error on overflow */
|
||||||
len = nitems * size;
|
len = nitems * size;
|
||||||
if (size && len / size != nitems) {
|
if (size && (len / size != nitems)) {
|
||||||
gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t");
|
gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t");
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user