Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
65330cb86b | ||
|
|
6870d9e8e6 | ||
|
|
9c64937987 | ||
|
|
6e5e61344b | ||
|
|
ea04e7c471 | ||
|
|
5d7aa9af26 | ||
|
|
0813ba6845 | ||
|
|
3bfaf17013 |
@@ -12,10 +12,5 @@
|
||||
# Visual Studio
|
||||
*.sln text eol=crlf
|
||||
*.vcxproj* text eol=crlf
|
||||
*.vcproj* text eol=crlf
|
||||
*.suo binary
|
||||
*.rc binary
|
||||
|
||||
# Windows
|
||||
*.bat text eol=crlf
|
||||
*.cmd text eol=crlf
|
||||
|
||||
+4
-8
@@ -18,12 +18,7 @@
|
||||
*.app
|
||||
|
||||
# Visual C++
|
||||
projects/VS2008
|
||||
projects/VS2010
|
||||
projects/VS2012
|
||||
projects/VS2013
|
||||
projects/VS2015
|
||||
build/bin
|
||||
visual/
|
||||
|
||||
# IDEA solution files
|
||||
*.idea
|
||||
@@ -32,7 +27,8 @@ build/bin
|
||||
.directory
|
||||
_codelite/
|
||||
_zstdbench/
|
||||
.clang_complete
|
||||
zlib_wrapper/
|
||||
|
||||
# CMake
|
||||
projects/cmake/
|
||||
contrib/cmake/
|
||||
projects/
|
||||
+27
-86
@@ -1,90 +1,31 @@
|
||||
language: c
|
||||
compiler: gcc
|
||||
matrix:
|
||||
fast_finish: true
|
||||
include:
|
||||
# Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
|
||||
- os: linux
|
||||
sudo: false
|
||||
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install
|
||||
- os: linux
|
||||
sudo: false
|
||||
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest
|
||||
- os: linux
|
||||
sudo: false
|
||||
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test
|
||||
- os: linux
|
||||
sudo: false
|
||||
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy"
|
||||
- os: linux
|
||||
sudo: false
|
||||
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan
|
||||
- os: linux
|
||||
sudo: false
|
||||
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan
|
||||
- os: linux
|
||||
sudo: false
|
||||
env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper
|
||||
# Standard Ubuntu 12.04 LTS Server Edition 64 bit
|
||||
- os: linux
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest
|
||||
- os: linux
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest
|
||||
- os: linux
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test
|
||||
- os: linux
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test
|
||||
- os: linux
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test
|
||||
- os: linux
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install
|
||||
- os: linux
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32"
|
||||
- os: linux
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C tests versionsTest"
|
||||
- os: linux
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32
|
||||
- os: linux
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest"
|
||||
# Ubuntu 14.04 LTS Server Edition 64 bit
|
||||
- os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install
|
||||
- os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper
|
||||
# OS X Mavericks
|
||||
- os: osx
|
||||
env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install
|
||||
- os: osx
|
||||
env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test
|
||||
- os: osx
|
||||
env: PLATFORM="OS X Mavericks" MAKE_PARAM=test
|
||||
exclude:
|
||||
- compiler: gcc
|
||||
|
||||
before_install:
|
||||
- set -e
|
||||
- |
|
||||
if [ "$TRAVIS_OS_NAME" == "linux" ]; then
|
||||
CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l)
|
||||
echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n"
|
||||
if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then
|
||||
sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind
|
||||
fi
|
||||
fi
|
||||
- sudo apt-get update -qq
|
||||
- sudo apt-get install -qq clang
|
||||
- sudo apt-get install -qq g++-multilib
|
||||
- sudo apt-get install -qq gcc-multilib
|
||||
- sudo apt-get install -qq valgrind
|
||||
|
||||
script:
|
||||
- make $MAKE_PARAM
|
||||
env:
|
||||
- ZSTD_TRAVIS_CI_ENV=travis-install
|
||||
- ZSTD_TRAVIS_CI_ENV=cmaketest
|
||||
- ZSTD_TRAVIS_CI_ENV=clangtest
|
||||
- ZSTD_TRAVIS_CI_ENV=gpptest
|
||||
- ZSTD_TRAVIS_CI_ENV=gnu90test
|
||||
- ZSTD_TRAVIS_CI_ENV=armtest-w-install
|
||||
- ZSTD_TRAVIS_CI_ENV=test
|
||||
- ZSTD_TRAVIS_CI_ENV="-C programs test32"
|
||||
- ZSTD_TRAVIS_CI_ENV="-C programs test-zstd_nolegacy"
|
||||
- ZSTD_TRAVIS_CI_ENV=usan
|
||||
- ZSTD_TRAVIS_CI_ENV=asan
|
||||
- ZSTD_TRAVIS_CI_ENV=asan32
|
||||
- ZSTD_TRAVIS_CI_ENV="-C programs valgrindTest"
|
||||
|
||||
compiler: gcc
|
||||
|
||||
script:
|
||||
- make $ZSTD_TRAVIS_CI_ENV
|
||||
|
||||
matrix:
|
||||
fast_finish: true
|
||||
|
||||
@@ -2,19 +2,19 @@
|
||||
# zstd - Makefile
|
||||
# Copyright (C) Yann Collet 2014-2016
|
||||
# All rights reserved.
|
||||
#
|
||||
#
|
||||
# BSD license
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification,
|
||||
# are permitted provided that the following conditions are met:
|
||||
#
|
||||
#
|
||||
# * Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
#
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this
|
||||
# list of conditions and the following disclaimer in the documentation and/or
|
||||
# other materials provided with the distribution.
|
||||
#
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
@@ -25,14 +25,16 @@
|
||||
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
#
|
||||
# You can contact the author at :
|
||||
# - zstd homepage : http://www.zstd.net/
|
||||
# ################################################################
|
||||
|
||||
# force a version number : uncomment below export (otherwise, default to the one declared into zstd.h)
|
||||
#export VERSION := 0.6.1
|
||||
|
||||
PRGDIR = programs
|
||||
ZSTDDIR = lib
|
||||
ZWRAPDIR = zlibWrapper
|
||||
|
||||
# Define nul output
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
@@ -41,36 +43,30 @@ else
|
||||
VOID = /dev/null
|
||||
endif
|
||||
|
||||
.PHONY: default all zlibwrapper zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
|
||||
.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
|
||||
|
||||
default: zstdprogram
|
||||
|
||||
all:
|
||||
all:
|
||||
$(MAKE) -C $(ZSTDDIR) $@
|
||||
$(MAKE) -C $(PRGDIR) $@
|
||||
|
||||
zstdprogram:
|
||||
$(MAKE) -C $(PRGDIR)
|
||||
|
||||
zlibwrapper:
|
||||
$(MAKE) -C $(ZSTDDIR) all
|
||||
$(MAKE) -C $(ZWRAPDIR) all
|
||||
|
||||
test:
|
||||
$(MAKE) -C $(PRGDIR) $@
|
||||
|
||||
clean:
|
||||
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
|
||||
#------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
|
||||
HOST_OS = POSIX
|
||||
|
||||
install:
|
||||
$(MAKE) -C $(ZSTDDIR) $@
|
||||
$(MAKE) -C $(PRGDIR) $@
|
||||
@@ -82,95 +78,21 @@ uninstall:
|
||||
travis-install:
|
||||
$(MAKE) install PREFIX=~/install_test_dir
|
||||
|
||||
gpptest: clean
|
||||
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
|
||||
cmaketest:
|
||||
cd contrib/cmake ; cmake . ; $(MAKE)
|
||||
|
||||
clangtest: clean
|
||||
clang -v
|
||||
$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
|
||||
|
||||
armtest: clean
|
||||
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||
|
||||
# for Travis CI
|
||||
arminstall: clean
|
||||
sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi
|
||||
|
||||
# for Travis CI
|
||||
armtest-w-install: clean arminstall armtest
|
||||
|
||||
ppctest: clean
|
||||
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||
|
||||
# for Travis CI
|
||||
ppcinstall: clean
|
||||
# sudo apt-get update -y -q
|
||||
sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04
|
||||
|
||||
# for Travis CI
|
||||
ppctest-w-install: clean ppcinstall ppctest
|
||||
|
||||
ppc64test: clean
|
||||
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||
|
||||
ppc64install: clean
|
||||
sudo apt-get update -y -q
|
||||
sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu
|
||||
update-binfmts --displ
|
||||
|
||||
ppc64test-w-install: clean ppc64install ppc64test
|
||||
|
||||
usan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
|
||||
|
||||
asan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
|
||||
|
||||
msan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory" # datagen.c used to fail this test for no obvious reason
|
||||
|
||||
asan32: clean
|
||||
$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
|
||||
|
||||
uasan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
|
||||
|
||||
endif
|
||||
|
||||
|
||||
ifneq (,$(filter MSYS%,$(shell uname)))
|
||||
HOST_OS = MSYS
|
||||
CMAKE_PARAMS = -G"MSYS Makefiles"
|
||||
endif
|
||||
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
|
||||
#------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
|
||||
cmaketest:
|
||||
cmake --version
|
||||
rm -rf projects/cmake/build
|
||||
mkdir projects/cmake/build
|
||||
cd projects/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
|
||||
|
||||
c90test: clean
|
||||
CFLAGS="-std=c90" $(MAKE) all # will fail, due to // and long long
|
||||
gpptest: clean
|
||||
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
|
||||
|
||||
gnu90test: clean
|
||||
CFLAGS="-std=gnu90" $(MAKE) all
|
||||
$(MAKE) all CFLAGS="-std=gnu90 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef -Wdeclaration-after-statement -Werror"
|
||||
|
||||
c99test: clean
|
||||
CFLAGS="-std=c99" $(MAKE) all
|
||||
|
||||
gnu99test: clean
|
||||
CFLAGS="-std=gnu99" $(MAKE) all
|
||||
|
||||
c11test: clean
|
||||
CFLAGS="-std=c11" $(MAKE) all
|
||||
c90test: clean
|
||||
$(MAKE) all CFLAGS="-std=c90 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef -Werror" # will fail, due to // and long long
|
||||
|
||||
bmix64test: clean
|
||||
CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(PRGDIR) test
|
||||
@@ -180,4 +102,46 @@ bmix32test: clean
|
||||
|
||||
bmi32test: clean
|
||||
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
|
||||
|
||||
armtest: clean
|
||||
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||
|
||||
# for Travis CI
|
||||
arminstall: clean
|
||||
sudo apt-get install -q qemu
|
||||
sudo apt-get install -q binfmt-support
|
||||
sudo apt-get install -q qemu-user-static
|
||||
sudo apt-get install -q gcc-arm-linux-gnueabi
|
||||
|
||||
# for Travis CI
|
||||
armtest-w-install: clean arminstall armtest
|
||||
|
||||
ppctest: clean
|
||||
$(MAKE) -C $(PRGDIR) datagen # use native, faster
|
||||
$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
|
||||
|
||||
# for Travis CI
|
||||
ppcinstall: clean
|
||||
sudo apt-get install -q qemu
|
||||
sudo apt-get install -q binfmt-support
|
||||
sudo apt-get install -q qemu-user-static
|
||||
sudo apt-get update -q
|
||||
sudo apt-get install -q gcc-powerpc-linux-gnu # unfortunately, doesn't work on Travis CI (package not available)
|
||||
|
||||
# for Travis CI
|
||||
ppctest-w-install: clean ppcinstall ppctest
|
||||
|
||||
usan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
|
||||
|
||||
asan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
|
||||
|
||||
asan32: clean
|
||||
$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
|
||||
|
||||
uasan: clean
|
||||
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
|
||||
|
||||
endif
|
||||
|
||||
@@ -1,16 +1,3 @@
|
||||
v0.7.0
|
||||
New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
|
||||
New : Command `--rm`, to remove source file after successful de/compression
|
||||
New : Visual build scripts, by Christophe Chevalier
|
||||
New : Support for Sparse File-systems (do not use space for zero-filled sectors)
|
||||
New : Frame checksum support
|
||||
New : Support pass-through mode (when using `-df`)
|
||||
API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()`
|
||||
API : create dictionary files from custom content, by Giuseppe Ottaviano
|
||||
API : support for custom malloc/free functions
|
||||
New : controllable Dictionary ID
|
||||
New : Support for skippable frames
|
||||
|
||||
v0.6.1
|
||||
New : zlib wrapper API, thanks to Przemyslaw Skibinski
|
||||
New : Ability to compile compressor / decompressor separately
|
||||
@@ -72,7 +59,7 @@ Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
|
||||
removed `zstd.c`
|
||||
|
||||
v0.4.0
|
||||
Command line utility compatible with high compression levels
|
||||
Command line utility compatible with high compression levels
|
||||
Removed zstdhc => merged into zstd
|
||||
Added : ZBUFF API (see zstd_buffered.h)
|
||||
Rolling buffer support
|
||||
@@ -102,7 +89,7 @@ v0.2.2
|
||||
Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
|
||||
|
||||
v0.2.1
|
||||
Fix : Read errors, advanced fuzzer tests, by Hanno Böck
|
||||
Fix : Read errors, advanced fuzzer tests, by Hanno Böck
|
||||
|
||||
v0.2.0
|
||||
**Breaking format change**
|
||||
@@ -121,3 +108,4 @@ detects write-flush errors
|
||||
git@github.com:Cyan4973/zstd.git
|
||||
v0.1.0
|
||||
first release
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
**Zstd**, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level and better compression ratios.
|
||||
**Zstd**, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level compression ratio.
|
||||
|
||||
It is provided as a BSD-license package, hosted on Github.
|
||||
|
||||
@@ -7,7 +7,7 @@ It is provided as a BSD-license package, hosted on Github.
|
||||
|master | [](https://travis-ci.org/Cyan4973/zstd) |
|
||||
|dev | [](https://travis-ci.org/Cyan4973/zstd) |
|
||||
|
||||
As a reference, several fast compression algorithms were tested and compared on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with gcc 5.2.1, with the [Silesia compression corpus].
|
||||
As a reference, several fast compression algorithms were tested and compared to [zlib] on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with gcc 5.2.1, on the [Silesia compression corpus].
|
||||
|
||||
[lzbench]: https://github.com/inikep/lzbench
|
||||
[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
|
||||
@@ -16,7 +16,7 @@ As a reference, several fast compression algorithms were tested and compared on
|
||||
|Name | Ratio | C.speed | D.speed |
|
||||
|-----------------|-------|--------:|--------:|
|
||||
| | | MB/s | MB/s |
|
||||
|**zstd 0.7.0 -1**|**2.877**|**325**| **930** |
|
||||
|**zstd 0.6.0 -1**|**2.877**|**330**| **915** |
|
||||
| [zlib] 1.2.8 -1 | 2.730 | 95 | 360 |
|
||||
| brotli -0 | 2.708 | 220 | 430 |
|
||||
| QuickLZ 1.5 | 2.237 | 510 | 605 |
|
||||
@@ -28,16 +28,16 @@ As a reference, several fast compression algorithms were tested and compared on
|
||||
[zlib]:http://www.zlib.net/
|
||||
[LZ4]: http://www.lz4.org/
|
||||
|
||||
Zstd can also offer stronger compression ratios at the cost of compression speed.
|
||||
Speed vs Compression trade-off is configurable by small increment. Decompression speed is preserved and remain roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.
|
||||
Zstd can also offer stronger compression ratio at the cost of compression speed.
|
||||
Speed vs Compression trade-off is configurable by small increment. Decompression speed is preserved and remain roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib].
|
||||
|
||||
The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with gcc 5.2.1, on the [Silesia compression corpus].
|
||||
The following test is run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with gcc 5.2.1, on the [Silesia compression corpus].
|
||||
|
||||
Compression Speed vs Ratio | Decompression Speed
|
||||
---------------------------|--------------------
|
||||
 | 
|
||||
|
||||
Several algorithms can produce higher compression ratio but at slower speed, falling outside of the graph.
|
||||
Several algorithms can produce higher compression ratio at slower speed, falling outside of the graph.
|
||||
For a larger picture including very slow modes, [click on this link](images/DCspeed5.png) .
|
||||
|
||||
|
||||
@@ -74,10 +74,8 @@ Hence, deploying one dictionary per type of data will provide the greater benefi
|
||||
|
||||
### Status
|
||||
|
||||
Zstd compression format has reached "Final status". It means it is planned to become the official stable zstd format and be tagged `v1.0`. The reason it's not yet tagged `v1.0` is that it currently performs its "validation period", making sure the format holds all its promises and nothing was missed.
|
||||
Zstd library also offers legacy decoder support. Any data compressed by any version >= `v0.1` (hence including current one) remains decodable now and in the future.
|
||||
The library has been validated using strong [fuzzer tests](https://en.wikipedia.org/wiki/Fuzz_testing), including both [internal tools](programs/fuzzer.c) and [external ones](http://lcamtuf.coredump.cx/afl). It's able to withstand hazard situations, including invalid inputs.
|
||||
As a consequence, Zstandard is considered safe for, and is currently used in, production environments.
|
||||
Zstd is in development. The internal format evolves to reach better performance. "Final Format" is projected H1 2016, and will be tagged `v1.0`. Zstd offers legacy support, meaning any data compressed by any version >= 0.1 (therefore including current one) remain decodable in the future.
|
||||
The library is also quite robust, able to withstand hazards situations, including invalid inputs. Library reliability has been tested using [Fuzz Testing](https://en.wikipedia.org/wiki/Fuzz_testing), with both [internal tools](programs/fuzzer.c) and [external ones](http://lcamtuf.coredump.cx/afl). Therefore, Zstandard is considered safe for production environments.
|
||||
|
||||
### Branch Policy
|
||||
|
||||
|
||||
+19
-98
@@ -1,104 +1,25 @@
|
||||
version: 1.0.{build}
|
||||
configuration:
|
||||
- Release
|
||||
- Debug
|
||||
platform:
|
||||
- Win32
|
||||
- x64
|
||||
environment:
|
||||
matrix:
|
||||
- COMPILER: "visual"
|
||||
CONFIGURATION: "Debug"
|
||||
PLATFORM: "x64"
|
||||
- COMPILER: "visual"
|
||||
CONFIGURATION: "Debug"
|
||||
PLATFORM: "Win32"
|
||||
- COMPILER: "visual"
|
||||
CONFIGURATION: "Release"
|
||||
PLATFORM: "x64"
|
||||
- COMPILER: "visual"
|
||||
CONFIGURATION: "Release"
|
||||
PLATFORM: "Win32"
|
||||
- COMPILER: "gcc"
|
||||
MAKE_PARAMS: "test"
|
||||
PLATFORM: "mingw64"
|
||||
- COMPILER: "gcc"
|
||||
MAKE_PARAMS: "test"
|
||||
PLATFORM: "mingw32"
|
||||
|
||||
install:
|
||||
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
|
||||
- if [%COMPILER%]==[gcc] (
|
||||
SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" &&
|
||||
SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
|
||||
SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
|
||||
COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe
|
||||
) else (
|
||||
IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;")
|
||||
)
|
||||
|
||||
# - PlatformToolset: v100
|
||||
- PlatformToolset: v110
|
||||
- PlatformToolset: v120
|
||||
- PlatformToolset: v140
|
||||
build_script:
|
||||
- ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL%
|
||||
- if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
|
||||
- if [%PLATFORM%]==[mingw64] (
|
||||
make clean &&
|
||||
ECHO *** &&
|
||||
ECHO *** Building clang &&
|
||||
ECHO *** &&
|
||||
ECHO make %CLANG_PARAMS% &&
|
||||
make %CLANG_PARAMS% &&
|
||||
COPY programs\fuzzer.exe projects\fuzzer_clang.exe &&
|
||||
make clean
|
||||
)
|
||||
- if [%COMPILER%]==[gcc] (
|
||||
ECHO *** &&
|
||||
ECHO *** Building %PLATFORM% &&
|
||||
ECHO *** &&
|
||||
ECHO make %MAKE_PARAMS% &&
|
||||
make %MAKE_PARAMS% &&
|
||||
make clean
|
||||
)
|
||||
- if [%COMPILER%]==[visual] (
|
||||
ECHO *** &&
|
||||
ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% &&
|
||||
ECHO *** &&
|
||||
msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
|
||||
COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe &&
|
||||
ECHO *** &&
|
||||
ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
|
||||
ECHO *** &&
|
||||
msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
|
||||
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
|
||||
ECHO *** &&
|
||||
ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
|
||||
ECHO *** &&
|
||||
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
|
||||
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe &&
|
||||
ECHO *** &&
|
||||
ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
|
||||
ECHO *** &&
|
||||
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
|
||||
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe &&
|
||||
ECHO *** &&
|
||||
ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
|
||||
ECHO *** &&
|
||||
msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
|
||||
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
|
||||
COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\
|
||||
)
|
||||
- cmd: >-
|
||||
ECHO PlatformToolset=%PlatformToolset%
|
||||
|
||||
msbuild "visual\2013\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=%PlatformToolset% /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll"
|
||||
test_script:
|
||||
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
|
||||
- SET FUZZERTEST=-T1mn
|
||||
- if [%COMPILER%]==[gcc] (
|
||||
if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST%
|
||||
)
|
||||
- if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
|
||||
CD programs &&
|
||||
SET ZSTD=./zstd.exe &&
|
||||
sh -e playTests.sh --test-large-data &&
|
||||
fullbench.exe -i1 &&
|
||||
fullbench.exe -i1 -P0 &&
|
||||
CD .. &&
|
||||
projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
|
||||
projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
|
||||
)
|
||||
- cmd: >-
|
||||
ECHO %APPVEYOR_BUILD_FOLDER%\visual\2013\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe %FUZZERTEST%
|
||||
|
||||
SET FUZZERTEST=-T2mn
|
||||
|
||||
IF %CONFIGURATION%==Release (%APPVEYOR_BUILD_FOLDER%\visual\2013\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe %FUZZERTEST%)
|
||||
@@ -44,6 +44,13 @@ ELSE (ZSTD_LEGACY_SUPPORT)
|
||||
ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
# TARGET_INCLUDE_DIRECTORIES can use in version 2.8.11 and greate
|
||||
IF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11))
|
||||
SET(WORKAROUND_OUTDATED_CODE_STYLE TRUE)
|
||||
ELSE ()
|
||||
SET(WORKAROUND_OUTDATED_CODE_STYLE FALSE)
|
||||
ENDIF ((${CMAKE_MAJOR_VERSION} EQUAL 2) AND (${CMAKE_MINOR_VERSION} EQUAL 8) AND (${CMAKE_PATCH_VERSION} LESS 11))
|
||||
|
||||
ADD_SUBDIRECTORY(lib)
|
||||
ADD_SUBDIRECTORY(programs)
|
||||
|
||||
@@ -52,8 +59,3 @@ ADD_SUBDIRECTORY(programs)
|
||||
#-----------------------------------------------------------------------------
|
||||
INCLUDE(CMakeModules/AddExtraCompilationFlags.cmake)
|
||||
ADD_EXTRA_COMPILATION_FLAGS()
|
||||
|
||||
ADD_CUSTOM_TARGET(clean-all
|
||||
COMMAND ${CMAKE_BUILD_TOOL} clean
|
||||
COMMAND rm -rf ${CMAKE_BINARY_DIR}/
|
||||
)
|
||||
@@ -2,19 +2,19 @@
|
||||
# zstd - Makefile
|
||||
# Copyright (C) Yann Collet 2014-2016
|
||||
# All rights reserved.
|
||||
#
|
||||
#
|
||||
# BSD license
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification,
|
||||
# are permitted provided that the following conditions are met:
|
||||
#
|
||||
#
|
||||
# * Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
#
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this
|
||||
# list of conditions and the following disclaimer in the documentation and/or
|
||||
# other materials provided with the distribution.
|
||||
#
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
@@ -25,7 +25,7 @@
|
||||
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
#
|
||||
# You can contact the author at :
|
||||
# - zstd homepage : http://www.zstd.net/
|
||||
# ################################################################
|
||||
@@ -59,7 +59,6 @@ MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
|
||||
SET(Sources
|
||||
${LIBRARY_DIR}/common/entropy_common.c
|
||||
${LIBRARY_DIR}/common/zstd_common.c
|
||||
${LIBRARY_DIR}/common/xxhash.c
|
||||
${LIBRARY_DIR}/common/fse_decompress.c
|
||||
${LIBRARY_DIR}/compress/fse_compress.c
|
||||
${LIBRARY_DIR}/compress/huf_compress.c
|
||||
@@ -76,12 +75,17 @@ SET(Headers
|
||||
${LIBRARY_DIR}/common/error_private.h
|
||||
${LIBRARY_DIR}/common/error_public.h
|
||||
${LIBRARY_DIR}/common/fse.h
|
||||
${LIBRARY_DIR}/common/fse_static.h
|
||||
${LIBRARY_DIR}/common/huf.h
|
||||
${LIBRARY_DIR}/common/huf_static.h
|
||||
${LIBRARY_DIR}/common/mem.h
|
||||
${LIBRARY_DIR}/common/zbuff.h
|
||||
${LIBRARY_DIR}/common/zbuff_static.h
|
||||
${LIBRARY_DIR}/common/zstd_internal.h
|
||||
${LIBRARY_DIR}/common/zstd_static.h
|
||||
${LIBRARY_DIR}/common/zstd.h
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.h)
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.h
|
||||
${LIBRARY_DIR}/dictBuilder/zdict_static.h)
|
||||
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
|
||||
@@ -93,7 +97,8 @@ IF (ZSTD_LEGACY_SUPPORT)
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v03.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v04.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v05.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v06.c)
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v07.c
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v08.c)
|
||||
|
||||
SET(Headers ${Headers}
|
||||
${LIBRARY_LEGACY_DIR}/zstd_legacy.h
|
||||
@@ -102,12 +107,13 @@ IF (ZSTD_LEGACY_SUPPORT)
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v03.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v04.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v05.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v06.h)
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v07.h
|
||||
${LIBRARY_LEGACY_DIR}/zstd_v08.h)
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
IF (MSVC)
|
||||
SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/projects/VS2010/zstdlib)
|
||||
SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstdlib.rc)
|
||||
SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/visual/2013/zstdlib)
|
||||
SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/resource.h ${MSVC_RESOURCE_DIR}/zstdlib.rc)
|
||||
ENDIF (MSVC)
|
||||
|
||||
# Split project to static and shared libraries build
|
||||
@@ -120,12 +126,24 @@ IF (MSVC)
|
||||
SET_TARGET_PROPERTIES(libzstd_shared PROPERTIES COMPILE_DEFINITIONS "ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS")
|
||||
ENDIF (MSVC)
|
||||
|
||||
# Define include directories
|
||||
IF (NOT WORKAROUND_OUTDATED_CODE_STYLE)
|
||||
TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_DIR}/common)
|
||||
TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_DIR}/common)
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_LEGACY_DIR})
|
||||
TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_LEGACY_DIR})
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
ENDIF (NOT WORKAROUND_OUTDATED_CODE_STYLE)
|
||||
|
||||
# Define library base name
|
||||
IF (MSVC)
|
||||
IF (UNIX)
|
||||
SET(LIBRARY_BASE_NAME libzstd)
|
||||
ELSEIF (MSVC)
|
||||
SET(LIBRARY_BASE_NAME zstdlib)
|
||||
ELSE ()
|
||||
SET(LIBRARY_BASE_NAME libzstd)
|
||||
ENDIF (MSVC)
|
||||
MESSAGE(FATAL_ERROR "Unsupported build type")
|
||||
ENDIF (UNIX)
|
||||
|
||||
# Define static and shared library names
|
||||
SET(STATIC_LIBRARY_OUTPUT_NAME ${LIBRARY_BASE_NAME})
|
||||
@@ -154,15 +172,12 @@ SET_TARGET_PROPERTIES(
|
||||
OUTPUT_NAME ${SHARED_LIBRARY_OUTPUT_NAME})
|
||||
|
||||
IF (UNIX)
|
||||
IF ("${PREFIX}" STREQUAL "")
|
||||
SET(PREFIX /usr/local)
|
||||
ENDIF()
|
||||
MESSAGE("the variable PREFIX=${PREFIX}")
|
||||
SET(PREFIX /usr/local)
|
||||
SET(INSTALL_LIBRARY_DIR ${PREFIX}/lib)
|
||||
SET(INSTALL_INCLUDE_DIR ${PREFIX}/include)
|
||||
|
||||
# install target
|
||||
INSTALL(FILES ${LIBRARY_DIR}/common/zstd.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
|
||||
INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/zstd_buffered.h ${LIBRARY_DIR}/dictBuilder.h DESTINATION ${INSTALL_INCLUDE_DIR})
|
||||
INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR})
|
||||
INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR})
|
||||
|
||||
@@ -38,23 +38,33 @@ SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
SET(ROOT_DIR ../../..)
|
||||
|
||||
# Define programs directory, where sources and header files are located
|
||||
SET(LIBRARY_DIR ${ROOT_DIR}/lib)
|
||||
SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_DIR})
|
||||
|
||||
IF (WORKAROUND_OUTDATED_CODE_STYLE)
|
||||
# Define library directory, where sources and header files are located
|
||||
SET(LIBRARY_DIR ${ROOT_DIR}/lib)
|
||||
INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common)
|
||||
ENDIF (WORKAROUND_OUTDATED_CODE_STYLE)
|
||||
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
|
||||
INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR})
|
||||
|
||||
IF (WORKAROUND_OUTDATED_CODE_STYLE)
|
||||
INCLUDE_DIRECTORIES(${LIBRARY_DIR}/legacy)
|
||||
ENDIF (WORKAROUND_OUTDATED_CODE_STYLE)
|
||||
|
||||
SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c)
|
||||
ENDIF (ZSTD_LEGACY_SUPPORT)
|
||||
|
||||
ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY})
|
||||
ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${LIBRARY_DIR}/common/xxhash.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY})
|
||||
TARGET_LINK_LIBRARIES(zstd libzstd_static)
|
||||
|
||||
ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fullbench.c)
|
||||
TARGET_LINK_LIBRARIES(fullbench libzstd_static)
|
||||
|
||||
ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fuzzer.c)
|
||||
ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${LIBRARY_DIR}/common/xxhash.c ${PROGRAMS_DIR}/fuzzer.c)
|
||||
TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
|
||||
|
||||
IF (UNIX)
|
||||
@@ -62,10 +72,10 @@ IF (UNIX)
|
||||
TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
|
||||
SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
|
||||
|
||||
ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/zbufftest.c)
|
||||
ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${LIBRARY_DIR}/common/xxhash.c ${PROGRAMS_DIR}/zbufftest.c)
|
||||
TARGET_LINK_LIBRARIES(zbufftest libzstd_static)
|
||||
|
||||
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/paramgrill.c)
|
||||
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${LIBRARY_DIR}/common/xxhash.c ${PROGRAMS_DIR}/paramgrill.c)
|
||||
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
|
||||
|
||||
ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/datagencli.c)
|
||||
+16
-13
@@ -2,19 +2,19 @@
|
||||
# ZSTD library - Makefile
|
||||
# Copyright (C) Yann Collet 2015-2016
|
||||
# All rights reserved.
|
||||
#
|
||||
#
|
||||
# BSD license
|
||||
|
||||
# Redistribution and use in source and binary forms, with or without modification,
|
||||
# are permitted provided that the following conditions are met:
|
||||
#
|
||||
#
|
||||
# * Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
#
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this
|
||||
# list of conditions and the following disclaimer in the documentation and/or
|
||||
# other materials provided with the distribution.
|
||||
#
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
@@ -25,7 +25,7 @@
|
||||
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
#
|
||||
# You can contact the author at :
|
||||
# - ZSTD homepage : http://www.zstd.net
|
||||
# ################################################################
|
||||
@@ -43,21 +43,24 @@ VERSION?= $(LIBVER)
|
||||
|
||||
DESTDIR?=
|
||||
PREFIX ?= /usr/local
|
||||
CPPFLAGS= -I. -I./common -DXXH_NAMESPACE=ZSTD_
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -Wstrict-aliasing=1
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
||||
|
||||
LIBDIR ?= $(PREFIX)/lib
|
||||
INCLUDEDIR=$(PREFIX)/include
|
||||
|
||||
CPPFLAGS= -I./common -DXXH_NAMESPACE=ZSTD_
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
||||
|
||||
|
||||
ZSTD_FILES := common/*.c compress/*.c decompress/*.c dictBuilder/*.c
|
||||
ZSTDCOMP_FILES := compress/zstd_compress.c compress/fse_compress.c compress/huf_compress.c compress/zbuff_compress.c
|
||||
ZSTDDECOMP_FILES := decompress/zstd_decompress.c common/fse_decompress.c decompress/huf_decompress.c decompress/zbuff_decompress.c
|
||||
ZSTDDICT_FILES := dictBuilder/zdict.c dictBuilder/divsufsort.c
|
||||
ZSTD_FILES := $(ZSTDDECOMP_FILES) common/entropy_common.c common/zstd_common.c common/xxhash.c $(ZSTDCOMP_FILES) $(ZSTDDICT_FILES)
|
||||
ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c legacy/zstd_v07.c legacy/zstd_v08.c
|
||||
|
||||
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
|
||||
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
|
||||
else
|
||||
ZSTD_FILES+= legacy/*.c
|
||||
ZSTD_FILES+= $(ZSTD_LEGACY)
|
||||
CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
|
||||
endif
|
||||
|
||||
|
||||
+11
-3
@@ -12,22 +12,27 @@ To build the zstd library the following files are required:
|
||||
- [common/error_public.h](common/error_public.h)
|
||||
- common/fse.h
|
||||
- common/fse_decompress.c
|
||||
- common/fse_static.h
|
||||
- common/huf.h
|
||||
- common/huf_static.h
|
||||
- [common/mem.h](common/mem.h)
|
||||
- [common/zstd.h]
|
||||
- common/zstd_internal.h
|
||||
- common/zstd_static.h
|
||||
- compress/fse_compress.c
|
||||
- compress/huf_compress.c
|
||||
- compress/zstd_compress.c
|
||||
- compress/zstd_opt.h
|
||||
- compress/zstd_opt.h
|
||||
- decompress/huf_decompress.c
|
||||
- decompress/zstd_decompress.c
|
||||
|
||||
Stable API is exposed in [common/zstd.h].
|
||||
Advanced and experimental API can be enabled by defining `ZSTD_STATIC_LINKING_ONLY`.
|
||||
Never use them with a dynamic library, as their definition may change in future versions.
|
||||
Advanced and experimental API is exposed in [common/zstd_static.h].
|
||||
API elements of [common/zstd_static.h] should be used with static linking only,
|
||||
as their definition may change in future version of the library.
|
||||
|
||||
[common/zstd.h]: common/zstd.h
|
||||
[common/zstd_static.h]: common/zstd_static.h
|
||||
|
||||
|
||||
#### Separate compressor and decompressor
|
||||
@@ -42,6 +47,7 @@ This complementary API makes streaming integration easier.
|
||||
It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/projects/7-Zip-ZStd) :
|
||||
|
||||
- common/zbuff.h
|
||||
- common/zbuff_static.h
|
||||
- compress/zbuff_compress.c
|
||||
- decompress/zbuff_decompress.c
|
||||
|
||||
@@ -53,6 +59,7 @@ To create dictionaries from training sets :
|
||||
- dictBuilder/divsufsort.h
|
||||
- dictBuilder/zdict.c
|
||||
- dictBuilder/zdict.h
|
||||
- dictBuilder/zdict_static.h
|
||||
|
||||
#### Miscellaneous
|
||||
|
||||
@@ -61,3 +68,4 @@ The other files are not source code. There are :
|
||||
- LICENSE : contains the BSD license text
|
||||
- Makefile : script to compile or install zstd library (static or dynamic)
|
||||
- libzstd.pc.in : for pkg-config (make install)
|
||||
|
||||
|
||||
@@ -162,12 +162,14 @@ MEM_STATIC unsigned BIT_highbit32 (register U32 val)
|
||||
# else /* Software version */
|
||||
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
unsigned r;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
|
||||
r = DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
|
||||
return r;
|
||||
# endif
|
||||
}
|
||||
|
||||
@@ -266,8 +268,8 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
bitD->bitsConsumed = 8 - BIT_highbit32(lastByte); }
|
||||
} else {
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
bitD->ptr = bitD->start;
|
||||
@@ -283,8 +285,8 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
|
||||
default:;
|
||||
}
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
bitD->bitsConsumed = 8 - BIT_highbit32(lastByte); }
|
||||
bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
|
||||
}
|
||||
|
||||
@@ -373,7 +375,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
||||
{
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
||||
return BIT_DStream_overflow;
|
||||
|
||||
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
|
||||
|
||||
@@ -35,12 +35,12 @@
|
||||
/* *************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stdlib.h>
|
||||
#include "mem.h"
|
||||
#include "error_private.h" /* ERR_*, ERROR */
|
||||
#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
|
||||
#include "fse.h" /* FSE_isError, FSE_getErrorName */
|
||||
#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
|
||||
#include "huf.h" /* HUF_isError, HUF_getErrorName */
|
||||
#include "fse_static.h" /* FSE_MIN_TABLELOG */
|
||||
#include "error_private.h"
|
||||
#include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */
|
||||
#include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */
|
||||
|
||||
|
||||
|
||||
@@ -156,76 +156,3 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
|
||||
if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong);
|
||||
return ip-istart;
|
||||
}
|
||||
|
||||
|
||||
/*! HUF_readStats() :
|
||||
Read compact Huffman tree, saved by HUF_writeCTable().
|
||||
`huffWeight` is destination buffer.
|
||||
@return : size read from `src` , or an error Code .
|
||||
Note : Needed by HUF_readCTable() and HUF_readDTableXn() .
|
||||
*/
|
||||
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 weightTotal;
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
size_t iSize = ip[0];
|
||||
size_t oSize;
|
||||
|
||||
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
if (iSize >= 128) { /* special header */
|
||||
if (iSize >= (242)) { /* RLE */
|
||||
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
|
||||
oSize = l[iSize-242];
|
||||
memset(huffWeight, 1, hwSize);
|
||||
iSize = 0;
|
||||
}
|
||||
else { /* Incompressible */
|
||||
oSize = iSize - 127;
|
||||
iSize = ((oSize+1)/2);
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
if (oSize >= hwSize) return ERROR(corruption_detected);
|
||||
ip += 1;
|
||||
{ U32 n;
|
||||
for (n=0; n<oSize; n+=2) {
|
||||
huffWeight[n] = ip[n/2] >> 4;
|
||||
huffWeight[n+1] = ip[n/2] & 15;
|
||||
} } } }
|
||||
else { /* header compressed with FSE (normal case) */
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
|
||||
if (FSE_isError(oSize)) return oSize;
|
||||
}
|
||||
|
||||
/* collect weight stats */
|
||||
memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32));
|
||||
weightTotal = 0;
|
||||
{ U32 n; for (n=0; n<oSize; n++) {
|
||||
if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||
rankStats[huffWeight[n]]++;
|
||||
weightTotal += (1 << huffWeight[n]) >> 1;
|
||||
} }
|
||||
|
||||
/* get last non-null symbol weight (implied, total must be 2^n) */
|
||||
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
|
||||
if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||
*tableLogPtr = tableLog;
|
||||
/* determine last weight */
|
||||
{ U32 const total = 1 << tableLog;
|
||||
U32 const rest = total - weightTotal;
|
||||
U32 const verif = 1 << BIT_highbit32(rest);
|
||||
U32 const lastWeight = BIT_highbit32(rest) + 1;
|
||||
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
||||
huffWeight[oSize] = (BYTE)lastWeight;
|
||||
rankStats[lastWeight]++;
|
||||
} }
|
||||
|
||||
/* check tree construction validity */
|
||||
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
|
||||
|
||||
/* results */
|
||||
*nbSymbolsPtr = (U32)(oSize+1);
|
||||
return iSize+1;
|
||||
}
|
||||
|
||||
@@ -102,12 +102,10 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
|
||||
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
||||
case PREFIX(srcSize_wrong): return "Src size incorrect";
|
||||
case PREFIX(corruption_detected): return "Corrupted block detected";
|
||||
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
|
||||
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
||||
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
||||
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
||||
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
|
||||
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
|
||||
case PREFIX(maxCode):
|
||||
default: return notErrorCode;
|
||||
}
|
||||
|
||||
@@ -54,12 +54,10 @@ typedef enum {
|
||||
ZSTD_error_dstSize_tooSmall,
|
||||
ZSTD_error_srcSize_wrong,
|
||||
ZSTD_error_corruption_detected,
|
||||
ZSTD_error_checksum_wrong,
|
||||
ZSTD_error_tableLog_tooLarge,
|
||||
ZSTD_error_maxSymbolValue_tooLarge,
|
||||
ZSTD_error_maxSymbolValue_tooSmall,
|
||||
ZSTD_error_dictionary_corrupted,
|
||||
ZSTD_error_dictionary_wrong,
|
||||
ZSTD_error_maxCode
|
||||
} ZSTD_ErrorCode;
|
||||
|
||||
|
||||
+2
-351
@@ -132,8 +132,8 @@ size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src,
|
||||
/*! FSE_optimalTableLog():
|
||||
dynamically downsize 'tableLog' when conditions are met.
|
||||
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
|
||||
@return : recommended tableLog (necessarily <= 'maxTableLog') */
|
||||
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
||||
@return : recommended tableLog (necessarily <= initial 'tableLog') */
|
||||
unsigned FSE_optimalTableLog(unsigned tableLog, size_t srcSize, unsigned maxSymbolValue);
|
||||
|
||||
/*! FSE_normalizeCount():
|
||||
normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
|
||||
@@ -272,355 +272,6 @@ If there is an error, the function will return an error code, which can be teste
|
||||
*/
|
||||
|
||||
|
||||
#ifdef FSE_STATIC_LINKING_ONLY
|
||||
|
||||
/* *** Dependency *** */
|
||||
#include "bitstream.h"
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Static allocation
|
||||
*******************************************/
|
||||
/* FSE buffer bounds */
|
||||
#define FSE_NCOUNTBOUND 512
|
||||
#define FSE_BLOCKBOUND(size) (size + (size>>7))
|
||||
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
|
||||
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
|
||||
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE advanced API
|
||||
*******************************************/
|
||||
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
||||
/**< same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */
|
||||
|
||||
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
|
||||
/**< same as FSE_optimalTableLog(), which used `minus==2` */
|
||||
|
||||
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
|
||||
/**< build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
|
||||
|
||||
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
|
||||
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
|
||||
|
||||
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
|
||||
/**< build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
|
||||
|
||||
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
|
||||
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE symbol compression API
|
||||
*******************************************/
|
||||
/*!
|
||||
This API consists of small unitary functions, which highly benefit from being inlined.
|
||||
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
|
||||
Visual seems to do it automatically.
|
||||
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
|
||||
If none of these solutions is applicable, include "fse.c" directly.
|
||||
*/
|
||||
typedef struct
|
||||
{
|
||||
ptrdiff_t value;
|
||||
const void* stateTable;
|
||||
const void* symbolTT;
|
||||
unsigned stateLog;
|
||||
} FSE_CState_t;
|
||||
|
||||
static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
|
||||
|
||||
static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
|
||||
|
||||
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
|
||||
|
||||
/**<
|
||||
These functions are inner components of FSE_compress_usingCTable().
|
||||
They allow the creation of custom streams, mixing multiple tables and bit sources.
|
||||
|
||||
A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
|
||||
So the first symbol you will encode is the last you will decode, like a LIFO stack.
|
||||
|
||||
You will need a few variables to track your CStream. They are :
|
||||
|
||||
FSE_CTable ct; // Provided by FSE_buildCTable()
|
||||
BIT_CStream_t bitStream; // bitStream tracking structure
|
||||
FSE_CState_t state; // State tracking structure (can have several)
|
||||
|
||||
|
||||
The first thing to do is to init bitStream and state.
|
||||
size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
|
||||
FSE_initCState(&state, ct);
|
||||
|
||||
Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
|
||||
You can then encode your input data, byte after byte.
|
||||
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
|
||||
Remember decoding will be done in reverse direction.
|
||||
FSE_encodeByte(&bitStream, &state, symbol);
|
||||
|
||||
At any time, you can also add any bit sequence.
|
||||
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
|
||||
BIT_addBits(&bitStream, bitField, nbBits);
|
||||
|
||||
The above methods don't commit data to memory, they just store it into local register, for speed.
|
||||
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
Writing data to memory is a manual operation, performed by the flushBits function.
|
||||
BIT_flushBits(&bitStream);
|
||||
|
||||
Your last FSE encoding operation shall be to flush your last state value(s).
|
||||
FSE_flushState(&bitStream, &state);
|
||||
|
||||
Finally, you must close the bitStream.
|
||||
The function returns the size of CStream in bytes.
|
||||
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
|
||||
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
|
||||
size_t size = BIT_closeCStream(&bitStream);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE symbol decompression API
|
||||
*******************************************/
|
||||
typedef struct
|
||||
{
|
||||
size_t state;
|
||||
const void* table; /* precise table may vary, depending on U16 */
|
||||
} FSE_DState_t;
|
||||
|
||||
|
||||
static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
|
||||
|
||||
static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||
|
||||
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
||||
|
||||
/**<
|
||||
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
BIT_DStream_t DStream; // Stream context
|
||||
FSE_DState_t DState; // State context. Multiple ones are possible
|
||||
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = BIT_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSE_reloadDStream(&DStream);
|
||||
|
||||
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BIT_DStream_unfinished : there is still some data left into the DStream.
|
||||
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
BIT_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSE_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE unsafe API
|
||||
*******************************************/
|
||||
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Implementation of inlined functions
|
||||
*******************************************/
|
||||
typedef struct {
|
||||
int deltaFindState;
|
||||
U32 deltaNbBits;
|
||||
} FSE_symbolCompressionTransform; /* total 8 bytes */
|
||||
|
||||
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
|
||||
{
|
||||
const void* ptr = ct;
|
||||
const U16* u16ptr = (const U16*) ptr;
|
||||
const U32 tableLog = MEM_read16(ptr);
|
||||
statePtr->value = (ptrdiff_t)1<<tableLog;
|
||||
statePtr->stateTable = u16ptr+2;
|
||||
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
|
||||
statePtr->stateLog = tableLog;
|
||||
}
|
||||
|
||||
|
||||
/*! FSE_initCState2() :
|
||||
* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
|
||||
* uses the smallest state value possible, saving the cost of this symbol */
|
||||
MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
|
||||
{
|
||||
FSE_initCState(statePtr, ct);
|
||||
{ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
||||
const U16* stateTable = (const U16*)(statePtr->stateTable);
|
||||
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
|
||||
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
|
||||
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
|
||||
{
|
||||
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
||||
const U16* const stateTable = (const U16*)(statePtr->stateTable);
|
||||
U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
|
||||
BIT_addBits(bitC, statePtr->value, nbBitsOut);
|
||||
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
|
||||
{
|
||||
BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
|
||||
BIT_flushBits(bitC);
|
||||
}
|
||||
|
||||
/*<===== Decompression =====>*/
|
||||
|
||||
typedef struct {
|
||||
U16 tableLog;
|
||||
U16 fastMode;
|
||||
} FSE_DTableHeader; /* sizeof U32 */
|
||||
|
||||
typedef struct
|
||||
{
|
||||
unsigned short newState;
|
||||
unsigned char symbol;
|
||||
unsigned char nbBits;
|
||||
} FSE_decode_t; /* size == U32 */
|
||||
|
||||
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
|
||||
{
|
||||
const void* ptr = dt;
|
||||
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
|
||||
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
|
||||
BIT_reloadDStream(bitD);
|
||||
DStatePtr->table = dt + 1;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
return DInfo.symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
/*! FSE_decodeSymbolFast() :
|
||||
unsafe, only works if no symbol has a probability > 50% */
|
||||
MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
||||
{
|
||||
return DStatePtr->state == 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifndef FSE_COMMONDEFS_ONLY
|
||||
|
||||
/* **************************************************************
|
||||
* Tuning parameters
|
||||
****************************************************************/
|
||||
/*!MEMORY_USAGE :
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||
* Increasing memory usage improves compression ratio
|
||||
* Reduced memory usage can improve speed, due to cache effect
|
||||
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
|
||||
#define FSE_MAX_MEMORY_USAGE 14
|
||||
#define FSE_DEFAULT_MEMORY_USAGE 13
|
||||
|
||||
/*!FSE_MAX_SYMBOL_VALUE :
|
||||
* Maximum symbol value authorized.
|
||||
* Required for proper stack allocation */
|
||||
#define FSE_MAX_SYMBOL_VALUE 255
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* template functions type & suffix
|
||||
****************************************************************/
|
||||
#define FSE_FUNCTION_TYPE BYTE
|
||||
#define FSE_FUNCTION_EXTENSION
|
||||
#define FSE_DECODE_TYPE FSE_decode_t
|
||||
|
||||
|
||||
#endif /* !FSE_COMMONDEFS_ONLY */
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
* Constants
|
||||
*****************************************************************/
|
||||
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
|
||||
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
|
||||
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
|
||||
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
|
||||
#define FSE_MIN_TABLELOG 5
|
||||
|
||||
#define FSE_TABLELOG_ABSOLUTE_MAX 15
|
||||
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
|
||||
# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
|
||||
#endif
|
||||
|
||||
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
|
||||
|
||||
|
||||
#endif /* FSE_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -58,8 +58,7 @@
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include <stdio.h> /* printf (debug) */
|
||||
#include "bitstream.h"
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#include "fse_static.h"
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
/* ******************************************************************
|
||||
FSE : Finite State Entropy coder
|
||||
header file for static linking (only)
|
||||
Copyright (C) 2013-2015, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
#ifndef FSE_STATIC_H
|
||||
#define FSE_STATIC_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Dependencies
|
||||
*******************************************/
|
||||
#include "fse.h"
|
||||
#include "bitstream.h"
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Static allocation
|
||||
*******************************************/
|
||||
/* FSE buffer bounds */
|
||||
#define FSE_NCOUNTBOUND 512
|
||||
#define FSE_BLOCKBOUND(size) (size + (size>>7))
|
||||
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
|
||||
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
|
||||
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE advanced API
|
||||
*******************************************/
|
||||
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
||||
/* same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */
|
||||
|
||||
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
|
||||
/* build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
|
||||
|
||||
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
|
||||
/* build a fake FSE_CTable, designed to compress always the same symbolValue */
|
||||
|
||||
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
|
||||
/* build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
|
||||
|
||||
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
|
||||
/* build a fake FSE_DTable, designed to always generate the same symbolValue */
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE symbol compression API
|
||||
*******************************************/
|
||||
/*!
|
||||
This API consists of small unitary functions, which highly benefit from being inlined.
|
||||
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
|
||||
Visual seems to do it automatically.
|
||||
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
|
||||
If none of these solutions is applicable, include "fse.c" directly.
|
||||
*/
|
||||
typedef struct
|
||||
{
|
||||
ptrdiff_t value;
|
||||
const void* stateTable;
|
||||
const void* symbolTT;
|
||||
unsigned stateLog;
|
||||
} FSE_CState_t;
|
||||
|
||||
static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
|
||||
|
||||
static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
|
||||
|
||||
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
|
||||
|
||||
/*!
|
||||
These functions are inner components of FSE_compress_usingCTable().
|
||||
They allow the creation of custom streams, mixing multiple tables and bit sources.
|
||||
|
||||
A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
|
||||
So the first symbol you will encode is the last you will decode, like a LIFO stack.
|
||||
|
||||
You will need a few variables to track your CStream. They are :
|
||||
|
||||
FSE_CTable ct; // Provided by FSE_buildCTable()
|
||||
BIT_CStream_t bitStream; // bitStream tracking structure
|
||||
FSE_CState_t state; // State tracking structure (can have several)
|
||||
|
||||
|
||||
The first thing to do is to init bitStream and state.
|
||||
size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
|
||||
FSE_initCState(&state, ct);
|
||||
|
||||
Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
|
||||
You can then encode your input data, byte after byte.
|
||||
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
|
||||
Remember decoding will be done in reverse direction.
|
||||
FSE_encodeByte(&bitStream, &state, symbol);
|
||||
|
||||
At any time, you can also add any bit sequence.
|
||||
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
|
||||
BIT_addBits(&bitStream, bitField, nbBits);
|
||||
|
||||
The above methods don't commit data to memory, they just store it into local register, for speed.
|
||||
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
Writing data to memory is a manual operation, performed by the flushBits function.
|
||||
BIT_flushBits(&bitStream);
|
||||
|
||||
Your last FSE encoding operation shall be to flush your last state value(s).
|
||||
FSE_flushState(&bitStream, &state);
|
||||
|
||||
Finally, you must close the bitStream.
|
||||
The function returns the size of CStream in bytes.
|
||||
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
|
||||
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
|
||||
size_t size = BIT_closeCStream(&bitStream);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE symbol decompression API
|
||||
*******************************************/
|
||||
typedef struct
|
||||
{
|
||||
size_t state;
|
||||
const void* table; /* precise table may vary, depending on U16 */
|
||||
} FSE_DState_t;
|
||||
|
||||
|
||||
static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
|
||||
|
||||
static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||
|
||||
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
||||
|
||||
/*!
|
||||
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
BIT_DStream_t DStream; // Stream context
|
||||
FSE_DState_t DState; // State context. Multiple ones are possible
|
||||
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
|
||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = BIT_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSE_reloadDStream(&DStream);
|
||||
|
||||
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BIT_DStream_unfinished : there is still some data left into the DStream.
|
||||
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
BIT_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSE_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE unsafe API
|
||||
*******************************************/
|
||||
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Implementation of inlined functions
|
||||
*******************************************/
|
||||
typedef struct {
|
||||
int deltaFindState;
|
||||
U32 deltaNbBits;
|
||||
} FSE_symbolCompressionTransform; /* total 8 bytes */
|
||||
|
||||
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
|
||||
{
|
||||
const void* ptr = ct;
|
||||
const U16* u16ptr = (const U16*) ptr;
|
||||
const U32 tableLog = MEM_read16(ptr);
|
||||
statePtr->value = (ptrdiff_t)1<<tableLog;
|
||||
statePtr->stateTable = u16ptr+2;
|
||||
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
|
||||
statePtr->stateLog = tableLog;
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
|
||||
{
|
||||
FSE_initCState(statePtr, ct);
|
||||
{
|
||||
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
||||
const U16* stateTable = (const U16*)(statePtr->stateTable);
|
||||
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
|
||||
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
|
||||
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
|
||||
{
|
||||
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
||||
const U16* const stateTable = (const U16*)(statePtr->stateTable);
|
||||
U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
|
||||
BIT_addBits(bitC, statePtr->value, nbBitsOut);
|
||||
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
|
||||
{
|
||||
BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
|
||||
BIT_flushBits(bitC);
|
||||
}
|
||||
|
||||
/*<===== Decompression =====>*/
|
||||
|
||||
typedef struct {
|
||||
U16 tableLog;
|
||||
U16 fastMode;
|
||||
} FSE_DTableHeader; /* sizeof U32 */
|
||||
|
||||
typedef struct
|
||||
{
|
||||
unsigned short newState;
|
||||
unsigned char symbol;
|
||||
unsigned char nbBits;
|
||||
} FSE_decode_t; /* size == U32 */
|
||||
|
||||
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
|
||||
{
|
||||
const void* ptr = dt;
|
||||
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
|
||||
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
|
||||
BIT_reloadDStream(bitD);
|
||||
DStatePtr->table = dt + 1;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
return DInfo.symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
/*! FSE_decodeSymbolFast() :
|
||||
unsafe, only works if no symbol has a probability > 50% */
|
||||
MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
||||
{
|
||||
return DStatePtr->state == 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifndef FSE_COMMONDEFS_ONLY
|
||||
|
||||
/* **************************************************************
|
||||
* Tuning parameters
|
||||
****************************************************************/
|
||||
/*!MEMORY_USAGE :
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||
* Increasing memory usage improves compression ratio
|
||||
* Reduced memory usage can improve speed, due to cache effect
|
||||
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
|
||||
#define FSE_MAX_MEMORY_USAGE 14
|
||||
#define FSE_DEFAULT_MEMORY_USAGE 13
|
||||
|
||||
/*!FSE_MAX_SYMBOL_VALUE :
|
||||
* Maximum symbol value authorized.
|
||||
* Required for proper stack allocation */
|
||||
#define FSE_MAX_SYMBOL_VALUE 255
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* template functions type & suffix
|
||||
****************************************************************/
|
||||
#define FSE_FUNCTION_TYPE BYTE
|
||||
#define FSE_FUNCTION_EXTENSION
|
||||
#define FSE_DECODE_TYPE FSE_decode_t
|
||||
|
||||
|
||||
#endif /* !FSE_COMMONDEFS_ONLY */
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
* Constants
|
||||
*****************************************************************/
|
||||
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
|
||||
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
|
||||
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
|
||||
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
|
||||
#define FSE_MIN_TABLELOG 5
|
||||
|
||||
#define FSE_TABLELOG_ABSOLUTE_MAX 15
|
||||
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
|
||||
#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
|
||||
#endif
|
||||
|
||||
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* FSE_STATIC_H */
|
||||
+25
-156
@@ -31,198 +31,67 @@
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef HUF_H_298734234
|
||||
#define HUF_H_298734234
|
||||
#ifndef HUF_H
|
||||
#define HUF_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* *** Dependencies *** */
|
||||
/* ****************************************
|
||||
* Dependency
|
||||
******************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* *** simple functions *** */
|
||||
/**
|
||||
/* ****************************************
|
||||
* HUF simple functions
|
||||
******************************************/
|
||||
size_t HUF_compress(void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
size_t HUF_decompress(void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
/*
|
||||
HUF_compress() :
|
||||
Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
|
||||
'dst' buffer must be already allocated.
|
||||
Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
|
||||
`srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
|
||||
@return : size of compressed data (<= `dstCapacity`).
|
||||
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
||||
'dst' buffer must be already allocated. Compression runs faster if dstCapacity >= HUF_compressBound(srcSize).
|
||||
Note : srcSize must be <= 128 KB
|
||||
@return : size of compressed data (<= dstCapacity)
|
||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
||||
if return == 1, srcData is a single repeated byte symbol (RLE compression).
|
||||
if HUF_isError(return), compression failed (more details using HUF_getErrorName())
|
||||
*/
|
||||
size_t HUF_compress(void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/**
|
||||
HUF_decompress() :
|
||||
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated buffer 'dst', of minimum size 'dstSize'.
|
||||
`dstSize` : **must** be the ***exact*** size of original (uncompressed) data.
|
||||
into already allocated destination buffer 'dst', of size 'dstSize'.
|
||||
`dstSize` : must be the **exact** size of original (uncompressed) data.
|
||||
Note : in contrast with FSE, HUF_decompress can regenerate
|
||||
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
|
||||
because it knows size to regenerate.
|
||||
@return : size of regenerated data (== dstSize),
|
||||
@return : size of regenerated data (== dstSize)
|
||||
or an error code, which can be tested using HUF_isError()
|
||||
*/
|
||||
size_t HUF_decompress(void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Tool functions
|
||||
******************************************/
|
||||
#define HUF_BLOCKSIZE_MAX (128 * 1024)
|
||||
size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
|
||||
size_t HUF_compressBound(size_t size); /**< maximum compressed size */
|
||||
|
||||
/* Error Management */
|
||||
unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
|
||||
const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
/* *** Advanced function *** */
|
||||
|
||||
/** HUF_compress2() :
|
||||
* Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` */
|
||||
/* ****************************************
|
||||
* Advanced functions
|
||||
******************************************/
|
||||
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
|
||||
#ifdef HUF_STATIC_LINKING_ONLY
|
||||
|
||||
/* *** Dependencies *** */
|
||||
#include "mem.h" /* U32 */
|
||||
|
||||
|
||||
/* *** Constants *** */
|
||||
#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
||||
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
|
||||
#define HUF_TABLELOG_DEFAULT HUF_TABLELOG_MAX /* tableLog by default, when not specified */
|
||||
#define HUF_SYMBOLVALUE_MAX 255
|
||||
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
|
||||
# error "HUF_TABLELOG_MAX is too large !"
|
||||
#endif
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Static allocation
|
||||
******************************************/
|
||||
/* HUF buffer bounds */
|
||||
#define HUF_CTABLEBOUND 129
|
||||
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
|
||||
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* static allocation of HUF's Compression Table */
|
||||
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
|
||||
U32 name##hb[maxSymbolValue+1]; \
|
||||
void* name##hv = &(name##hb); \
|
||||
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
|
||||
|
||||
/* static allocation of HUF's DTable */
|
||||
typedef U32 HUF_DTable;
|
||||
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
|
||||
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
||||
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) }
|
||||
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
|
||||
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) }
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Advanced decompression functions
|
||||
******************************************/
|
||||
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
|
||||
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
||||
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
|
||||
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
|
||||
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
||||
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* HUF detailed API
|
||||
******************************************/
|
||||
/*!
|
||||
HUF_compress() does the following:
|
||||
1. count symbol occurrence from source[] into table count[] using FSE_count()
|
||||
2. (optional) refine tableLog using HUF_optimalTableLog()
|
||||
3. build Huffman table from count using HUF_buildCTable()
|
||||
4. save Huffman table to memory buffer using HUF_writeCTable()
|
||||
5. encode the data stream using HUF_compress4X_usingCTable()
|
||||
|
||||
The following API allows targeting specific sub-functions for advanced tasks.
|
||||
For example, it's possible to compress several blocks using the same 'CTable',
|
||||
or to save and regenerate 'CTable' using external methods.
|
||||
*/
|
||||
/* FSE_count() : find it within "fse.h" */
|
||||
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
|
||||
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
|
||||
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
|
||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||
|
||||
|
||||
/*! HUF_readStats() :
|
||||
Read compact Huffman tree, saved by HUF_writeCTable().
|
||||
`huffWeight` is destination buffer.
|
||||
@return : size read from `src` , or an error Code .
|
||||
Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
|
||||
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/** HUF_readCTable() :
|
||||
* Loading a CTable saved with HUF_writeCTable() */
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/*
|
||||
HUF_decompress() does the following:
|
||||
1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
|
||||
2. build Huffman table from save, using HUF_readDTableXn()
|
||||
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
|
||||
*/
|
||||
|
||||
/** HUF_selectDecoder() :
|
||||
* Tells which decoder is likely to decode faster,
|
||||
* based on a set of pre-determined metrics.
|
||||
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
|
||||
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
|
||||
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
|
||||
|
||||
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
||||
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize);
|
||||
|
||||
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
|
||||
|
||||
/* single stream variants */
|
||||
|
||||
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||
|
||||
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
||||
|
||||
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
|
||||
|
||||
|
||||
#endif /* HUF_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* HUF_H_298734234 */
|
||||
#endif /* HUF_H */
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
/* ******************************************************************
|
||||
Huffman codec, part of New Generation Entropy library
|
||||
header file, for static linking only
|
||||
Copyright (C) 2013-2016, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef HUF_STATIC_H
|
||||
#define HUF_STATIC_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Dependency
|
||||
******************************************/
|
||||
#include "huf.h"
|
||||
#include "fse.h"
|
||||
#include "bitstream.h"
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Static allocation
|
||||
******************************************/
|
||||
/* HUF buffer bounds */
|
||||
#define HUF_CTABLEBOUND 129
|
||||
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
|
||||
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* static allocation of HUF's Compression Table */
|
||||
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
|
||||
U32 name##hb[maxSymbolValue+1]; \
|
||||
void* name##hv = &(name##hb); \
|
||||
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
|
||||
|
||||
/* static allocation of HUF's DTable */
|
||||
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<maxTableLog))
|
||||
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
||||
unsigned short DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
|
||||
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
|
||||
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
|
||||
#define HUF_CREATE_STATIC_DTABLEX6(DTable, maxTableLog) \
|
||||
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog }
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Advanced decompression functions
|
||||
******************************************/
|
||||
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
|
||||
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* HUF detailed API
|
||||
******************************************/
|
||||
/*!
|
||||
HUF_compress() does the following:
|
||||
1. count symbol occurrence from source[] into table count[] using FSE_count()
|
||||
2. build Huffman table from count using HUF_buildCTable()
|
||||
3. save Huffman table to memory buffer using HUF_writeCTable()
|
||||
4. encode the data stream using HUF_compress4X_usingCTable()
|
||||
|
||||
The following API allows targeting specific sub-functions for advanced tasks.
|
||||
For example, it's possible to compress several blocks using the same 'CTable',
|
||||
or to save and regenerate 'CTable' using external methods.
|
||||
*/
|
||||
/* FSE_count() : find it within "fse.h" */
|
||||
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
|
||||
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
|
||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||
|
||||
|
||||
/*!
|
||||
HUF_decompress() does the following:
|
||||
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
|
||||
2. build Huffman table from save, using HUF_readDTableXn()
|
||||
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
|
||||
*/
|
||||
size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
|
||||
size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
|
||||
size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize);
|
||||
|
||||
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
|
||||
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
||||
size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
||||
|
||||
|
||||
/* single stream variants */
|
||||
|
||||
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
|
||||
|
||||
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
||||
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
|
||||
|
||||
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
|
||||
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
||||
size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
||||
|
||||
|
||||
/* Loading a CTable saved with HUF_writeCTable() */
|
||||
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Constants
|
||||
****************************************************************/
|
||||
#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
|
||||
#define HUF_MAX_TABLELOG 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
|
||||
#define HUF_DEFAULT_TABLELOG HUF_MAX_TABLELOG /* tableLog by default, when not specified */
|
||||
#define HUF_MAX_SYMBOL_VALUE 255
|
||||
#if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG)
|
||||
# error "HUF_MAX_TABLELOG is too large !"
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*! HUF_readStats() :
|
||||
Read compact Huffman tree, saved by HUF_writeCTable().
|
||||
`huffWeight` is destination buffer.
|
||||
@return : size read from `src`
|
||||
*/
|
||||
MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 weightTotal;
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
size_t iSize = ip[0];
|
||||
size_t oSize;
|
||||
|
||||
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
if (iSize >= 128) { /* special header */
|
||||
if (iSize >= (242)) { /* RLE */
|
||||
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
|
||||
oSize = l[iSize-242];
|
||||
memset(huffWeight, 1, hwSize);
|
||||
iSize = 0;
|
||||
}
|
||||
else { /* Incompressible */
|
||||
oSize = iSize - 127;
|
||||
iSize = ((oSize+1)/2);
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
if (oSize >= hwSize) return ERROR(corruption_detected);
|
||||
ip += 1;
|
||||
{ U32 n;
|
||||
for (n=0; n<oSize; n+=2) {
|
||||
huffWeight[n] = ip[n/2] >> 4;
|
||||
huffWeight[n+1] = ip[n/2] & 15;
|
||||
} } } }
|
||||
else { /* header compressed with FSE (normal case) */
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
|
||||
if (FSE_isError(oSize)) return oSize;
|
||||
}
|
||||
|
||||
/* collect weight stats */
|
||||
memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32));
|
||||
weightTotal = 0;
|
||||
{ U32 n; for (n=0; n<oSize; n++) {
|
||||
if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
|
||||
rankStats[huffWeight[n]]++;
|
||||
weightTotal += (1 << huffWeight[n]) >> 1;
|
||||
} }
|
||||
|
||||
/* get last non-null symbol weight (implied, total must be 2^n) */
|
||||
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
|
||||
if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
|
||||
*tableLogPtr = tableLog;
|
||||
/* determine last weight */
|
||||
{ U32 const total = 1 << tableLog;
|
||||
U32 const rest = total - weightTotal;
|
||||
U32 const verif = 1 << BIT_highbit32(rest);
|
||||
U32 const lastWeight = BIT_highbit32(rest) + 1;
|
||||
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
||||
huffWeight[oSize] = (BYTE)lastWeight;
|
||||
rankStats[lastWeight]++;
|
||||
} }
|
||||
|
||||
/* check tree construction validity */
|
||||
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
|
||||
|
||||
/* results */
|
||||
*nbSymbolsPtr = (U32)(oSize+1);
|
||||
return iSize+1;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* HUF_STATIC_H */
|
||||
+18
-6
@@ -44,16 +44,14 @@ extern "C" {
|
||||
******************************************/
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include <string.h> /* memcpy */
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
# include <stdlib.h> /* _byteswap_ulong */
|
||||
#endif
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Compiler specifics
|
||||
******************************************/
|
||||
#if defined(_MSC_VER)
|
||||
# include <intrin.h> /* _byteswap_ */
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
# include <stdlib.h> /* _byteswap_ulong */
|
||||
# include <intrin.h> /* _byteswap_* */
|
||||
#endif
|
||||
#if defined(__GNUC__)
|
||||
# define MEM_STATIC static __attribute__((unused))
|
||||
@@ -65,6 +63,10 @@ extern "C" {
|
||||
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
/* code only tested on 32 and 64 bits systems */
|
||||
#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
|
||||
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Basic Types
|
||||
@@ -256,6 +258,17 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
|
||||
{
|
||||
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
|
||||
{
|
||||
MEM_writeLE16(memPtr, (U16)val);
|
||||
((BYTE*)memPtr)[2] = (BYTE)(val>>16);
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
@@ -374,4 +387,3 @@ MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
|
||||
#endif
|
||||
|
||||
#endif /* MEM_H_MODULE */
|
||||
|
||||
|
||||
+4
-36
@@ -28,8 +28,8 @@
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net/
|
||||
*/
|
||||
#ifndef ZSTD_BUFFERED_H_23987
|
||||
#define ZSTD_BUFFERED_H_23987
|
||||
#ifndef ZSTD_BUFFERED_H
|
||||
#define ZSTD_BUFFERED_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
@@ -38,7 +38,7 @@ extern "C" {
|
||||
/* *************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
@@ -160,40 +160,8 @@ ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void);
|
||||
ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
|
||||
|
||||
|
||||
#ifdef ZBUFF_STATIC_LINKING_ONLY
|
||||
|
||||
/* ====================================================================================
|
||||
* The definitions in this section are considered experimental.
|
||||
* They should never be used in association with a dynamic library, as they may change in the future.
|
||||
* They are provided for advanced usages.
|
||||
* Use them only in association with static linking.
|
||||
* ==================================================================================== */
|
||||
|
||||
/*--- Dependency ---*/
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
|
||||
#include "zstd.h"
|
||||
|
||||
|
||||
/*--- External memory ---*/
|
||||
/*! ZBUFF_createCCtx_advanced() :
|
||||
* Create a ZBUFF compression context using external alloc and free functions */
|
||||
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
|
||||
|
||||
/*! ZBUFF_createDCtx_advanced() :
|
||||
* Create a ZBUFF decompression context using external alloc and free functions */
|
||||
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
|
||||
|
||||
|
||||
/*--- Advanced Streaming function ---*/
|
||||
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize);
|
||||
|
||||
#endif /* ZBUFF_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_BUFFERED_H_23987 */
|
||||
#endif /* ZSTD_BUFFERED_H */
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
/*
|
||||
zstd - buffered version of compression library
|
||||
experimental complementary API, for static linking only
|
||||
Copyright (C) 2015-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net
|
||||
*/
|
||||
#ifndef ZSTD_BUFFERED_STATIC_H
|
||||
#define ZSTD_BUFFERED_STATIC_H
|
||||
|
||||
/* The objects defined into this file should be considered experimental.
|
||||
* They are not labelled stable, as their prototype may change in the future.
|
||||
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include "zstd_static.h" /* ZSTD_parameters */
|
||||
#include "zbuff.h"
|
||||
#include "zstd_internal.h" /* MIN */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Advanced Streaming functions
|
||||
***************************************/
|
||||
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize);
|
||||
|
||||
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t length = MIN(dstCapacity, srcSize);
|
||||
memcpy(dst, src, length);
|
||||
return length;
|
||||
}
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_BUFFERED_STATIC_H */
|
||||
+11
-295
@@ -29,8 +29,8 @@
|
||||
You can contact the author at :
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
*/
|
||||
#ifndef ZSTD_H_235446
|
||||
#define ZSTD_H_235446
|
||||
#ifndef ZSTD_H
|
||||
#define ZSTD_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
@@ -60,8 +60,8 @@ extern "C" {
|
||||
* Version
|
||||
***************************************/
|
||||
#define ZSTD_VERSION_MAJOR 0
|
||||
#define ZSTD_VERSION_MINOR 7
|
||||
#define ZSTD_VERSION_RELEASE 0
|
||||
#define ZSTD_VERSION_MINOR 6
|
||||
#define ZSTD_VERSION_RELEASE 2
|
||||
|
||||
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
|
||||
#define ZSTD_QUOTE(str) #str
|
||||
@@ -126,14 +126,12 @@ ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /*!< @return : error
|
||||
ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/*-************************
|
||||
* Simple dictionary API
|
||||
***************************/
|
||||
/*-***********************
|
||||
* Dictionary API
|
||||
*************************/
|
||||
/*! ZSTD_compress_usingDict() :
|
||||
* Compression using a pre-defined Dictionary content (see dictBuilder).
|
||||
* Note 1 : This function load the dictionary, resulting in a significant startup time.
|
||||
* Note 2 : `dict` must remain valid and unmodified during compression operation.
|
||||
* Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_compressCCtx() */
|
||||
* Note : dict can be NULL, in which case, it's equivalent to ZSTD_compressCCtx() */
|
||||
ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
@@ -142,298 +140,16 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
|
||||
|
||||
/*! ZSTD_decompress_usingDict() :
|
||||
* Decompression using a pre-defined Dictionary content (see dictBuilder).
|
||||
* Dictionary must be identical to the one used during compression.
|
||||
* Note 1 : This function load the dictionary, resulting in a significant startup time
|
||||
* Note 2 : `dict` must remain valid and unmodified during compression operation.
|
||||
* Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_decompressDCtx() */
|
||||
* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.
|
||||
* Note : dict can be NULL, in which case, it's equivalent to ZSTD_decompressDCtx() */
|
||||
ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize);
|
||||
|
||||
|
||||
/*-**************************
|
||||
* Advanced Dictionary API
|
||||
****************************/
|
||||
/*! ZSTD_createCDict() :
|
||||
* Create a digested dictionary, ready to start compression operation without startup delay.
|
||||
* `dict` can be released after creation */
|
||||
typedef struct ZSTD_CDict_s ZSTD_CDict;
|
||||
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
|
||||
|
||||
/*! ZSTD_compress_usingCDict() :
|
||||
* Compression using a pre-digested Dictionary.
|
||||
* Much faster than ZSTD_compress_usingDict() when same dictionary is used multiple times.
|
||||
* Note that compression level is decided during dictionary creation */
|
||||
ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_CDict* cdict);
|
||||
|
||||
/*! ZSTD_createDDict() :
|
||||
* Create a digested dictionary, ready to start decompression operation without startup delay.
|
||||
* `dict` can be released after creation */
|
||||
typedef struct ZSTD_DDict_s ZSTD_DDict;
|
||||
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
|
||||
|
||||
/*! ZSTD_decompress_usingDDict() :
|
||||
* Decompression using a pre-digested Dictionary
|
||||
* Much faster than ZSTD_decompress_usingDict() when same dictionary is used multiple times. */
|
||||
ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_DDict* ddict);
|
||||
|
||||
|
||||
|
||||
#ifdef ZSTD_STATIC_LINKING_ONLY
|
||||
|
||||
/* ====================================================================================
|
||||
* The definitions in this section are considered experimental.
|
||||
* They should never be used in association with a dynamic library, as they may change in the future.
|
||||
* They are provided for advanced usages.
|
||||
* Use them only in association with static linking.
|
||||
* ==================================================================================== */
|
||||
|
||||
/*--- Dependency ---*/
|
||||
#include "mem.h" /* U32 */
|
||||
|
||||
|
||||
/*--- Constants ---*/
|
||||
#define ZSTD_MAGICNUMBER 0xFD2FB527 /* v0.7 */
|
||||
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
|
||||
|
||||
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27))
|
||||
#define ZSTD_WINDOWLOG_MIN 18
|
||||
#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
|
||||
#define ZSTD_CHAINLOG_MIN 4
|
||||
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
|
||||
#define ZSTD_HASHLOG_MIN 12
|
||||
#define ZSTD_HASHLOG3_MAX 17
|
||||
#define ZSTD_HASHLOG3_MIN 15
|
||||
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
|
||||
#define ZSTD_SEARCHLOG_MIN 1
|
||||
#define ZSTD_SEARCHLENGTH_MAX 7
|
||||
#define ZSTD_SEARCHLENGTH_MIN 3
|
||||
#define ZSTD_TARGETLENGTH_MIN 4
|
||||
#define ZSTD_TARGETLENGTH_MAX 999
|
||||
|
||||
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
|
||||
static const size_t ZSTD_frameHeaderSize_min = 5;
|
||||
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
|
||||
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
|
||||
|
||||
|
||||
/*--- Types ---*/
|
||||
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */
|
||||
|
||||
typedef struct {
|
||||
U32 windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */
|
||||
U32 chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
|
||||
U32 hashLog; /*< dispatch table : larger == faster, more memory */
|
||||
U32 searchLog; /*< nb of searches : larger == more compression, slower */
|
||||
U32 searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */
|
||||
U32 targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */
|
||||
ZSTD_strategy strategy;
|
||||
} ZSTD_compressionParameters;
|
||||
|
||||
typedef struct {
|
||||
U32 contentSizeFlag; /*< 1: content size will be in frame header (if known). */
|
||||
U32 checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
|
||||
U32 noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */
|
||||
} ZSTD_frameParameters;
|
||||
|
||||
typedef struct {
|
||||
ZSTD_compressionParameters cParams;
|
||||
ZSTD_frameParameters fParams;
|
||||
} ZSTD_parameters;
|
||||
|
||||
/* custom memory allocation functions */
|
||||
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
|
||||
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
|
||||
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Advanced compression functions
|
||||
***************************************/
|
||||
/*! ZSTD_createCCtx_advanced() :
|
||||
* Create a ZSTD compression context using external alloc and free functions */
|
||||
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
|
||||
|
||||
/*! ZSTD_createCDict_advanced() :
|
||||
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
|
||||
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, ZSTD_customMem customMem);
|
||||
|
||||
ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
|
||||
|
||||
/*! ZSTD_getCParams() :
|
||||
* @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
|
||||
* `srcSize` value is optional, select 0 if not known */
|
||||
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_checkParams() :
|
||||
* Ensure param values remain within authorized range */
|
||||
ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
||||
|
||||
/*! ZSTD_adjustParams() :
|
||||
* optimize params for a given `srcSize` and `dictSize`.
|
||||
* both values are optional, select `0` if unknown. */
|
||||
ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_compress_advanced() :
|
||||
* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
|
||||
ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize,
|
||||
ZSTD_parameters params);
|
||||
|
||||
|
||||
/*--- Advanced Decompression functions ---*/
|
||||
|
||||
/*! ZSTD_createDCtx_advanced() :
|
||||
* Create a ZSTD decompression context using external alloc and free functions */
|
||||
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
|
||||
|
||||
|
||||
/* ****************************************************************
|
||||
* Streaming functions (direct mode - synchronous and buffer-less)
|
||||
******************************************************************/
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize);
|
||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity);
|
||||
|
||||
/*
|
||||
A ZSTD_CCtx object is required to track streaming operations.
|
||||
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
|
||||
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
|
||||
|
||||
Start by initializing a context.
|
||||
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
|
||||
or ZSTD_compressBegin_advanced(), for finer parameter control.
|
||||
It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
|
||||
|
||||
Then, consume your input using ZSTD_compressContinue().
|
||||
ZSTD_compressContinue() presumes prior data is still accessible and unmodified (up to maximum distance size, see WindowLog).
|
||||
The interface is synchronous, so input will be entirely consumed and produce associated compressed output.
|
||||
You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
|
||||
Worst case evaluation is provided by ZSTD_compressBound().
|
||||
|
||||
Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
|
||||
Without epilogue, frames will be considered unfinished (broken) by decoders.
|
||||
|
||||
You can then reuse ZSTD_CCtx to compress some new frame.
|
||||
*/
|
||||
|
||||
typedef struct {
|
||||
U64 frameContentSize;
|
||||
U32 windowSize;
|
||||
U32 dictID;
|
||||
U32 checksumFlag;
|
||||
} ZSTD_frameParams;
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
/*
|
||||
Streaming decompression, direct mode (bufferless)
|
||||
|
||||
A ZSTD_DCtx object is required to track streaming operations.
|
||||
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
||||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
|
||||
First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input.
|
||||
It can provide the minimum size of rolling buffer required to properly decompress data,
|
||||
and optionally the final size of uncompressed content.
|
||||
(Note : content size is an optional info that may not be present. 0 means : content size unknown)
|
||||
Frame parameters are extracted from the beginning of compressed frame.
|
||||
The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max (so if `srcSize` >= ZSTD_frameHeaderSize_max, it will always work)
|
||||
If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result.
|
||||
Result : 0 when successful, it means the ZSTD_frameParams structure has been filled.
|
||||
>0 : means there is not enough data into `src`. Provides the expected size to successfully decode header.
|
||||
errorCode, which can be tested using ZSTD_isError()
|
||||
|
||||
Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
|
||||
Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
|
||||
|
||||
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
|
||||
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
|
||||
ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
|
||||
ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
|
||||
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
|
||||
|
||||
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
|
||||
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
|
||||
|
||||
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
|
||||
Context can then be reset to start a new decompression.
|
||||
|
||||
Skippable frames allow the integration of user-defined data into a flow of concatenated frames.
|
||||
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following:
|
||||
a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
|
||||
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
|
||||
c) Frame Content - any content (User Data) of length equal to Frame Size
|
||||
For skippable frames ZSTD_decompressContinue() always returns 0.
|
||||
For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
|
||||
It also returns Frame Size as fparamsPtr->frameContentSize.
|
||||
*/
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Block functions
|
||||
****************************************/
|
||||
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
|
||||
A few rules to respect :
|
||||
- Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB)
|
||||
- Compressing or decompressing requires a context structure
|
||||
+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
|
||||
- It is necessary to init context before starting
|
||||
+ compression : ZSTD_compressBegin()
|
||||
+ decompression : ZSTD_decompressBegin()
|
||||
+ variants _usingDict() are also allowed
|
||||
+ copyCCtx() and copyDCtx() work too
|
||||
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
|
||||
In which case, nothing is produced into `dst`.
|
||||
+ User must test for such outcome and deal directly with uncompressed data
|
||||
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!
|
||||
*/
|
||||
|
||||
#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
|
||||
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Error management
|
||||
***************************************/
|
||||
#include "error_public.h"
|
||||
/*! ZSTD_getErrorCode() :
|
||||
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
|
||||
which can be used to compare directly with enum list published into "error_public.h" */
|
||||
ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
|
||||
ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
|
||||
|
||||
|
||||
#endif /* ZSTD_STATIC_LINKING_ONLY */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_H_235446 */
|
||||
#endif /* ZSTD_H */
|
||||
|
||||
@@ -33,10 +33,8 @@
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include "error_private.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
|
||||
#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */
|
||||
#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
|
||||
|
||||
|
||||
@@ -72,20 +70,3 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(c
|
||||
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
|
||||
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
|
||||
|
||||
|
||||
void* ZSTD_defaultAllocFunction(void* opaque, size_t size)
|
||||
{
|
||||
void* address = malloc(size);
|
||||
(void)opaque;
|
||||
/* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
|
||||
return address;
|
||||
}
|
||||
|
||||
void ZSTD_defaultFreeFunction(void* opaque, void* address)
|
||||
{
|
||||
(void)opaque;
|
||||
/* if (address) printf("free %p opaque=%p \n", address, opaque); */
|
||||
free(address);
|
||||
}
|
||||
|
||||
+35
-18
@@ -37,8 +37,7 @@
|
||||
***************************************/
|
||||
#include "mem.h"
|
||||
#include "error_private.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "zstd_static.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
@@ -64,12 +63,11 @@
|
||||
#endif
|
||||
|
||||
#define ZSTD_OPT_NUM (1<<12)
|
||||
#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7 */
|
||||
#define ZSTD_DICT_MAGIC 0xEC30A436
|
||||
|
||||
#define ZSTD_REP_NUM 3
|
||||
#define ZSTD_REP_INIT ZSTD_REP_NUM
|
||||
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
|
||||
static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
@@ -82,11 +80,10 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
||||
#define BIT1 2
|
||||
#define BIT0 1
|
||||
|
||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
|
||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 12
|
||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 };
|
||||
|
||||
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
|
||||
#define ZSTD_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */
|
||||
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
|
||||
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
||||
|
||||
@@ -94,12 +91,17 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
||||
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
|
||||
|
||||
#define HufLog 12
|
||||
typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t;
|
||||
|
||||
#define IS_HUF 0
|
||||
#define IS_PCH 1
|
||||
#define IS_RAW 2
|
||||
#define IS_RLE 3
|
||||
|
||||
#define LONGNBSEQ 0x7F00
|
||||
|
||||
#define MINMATCH 3
|
||||
#define EQUAL_READ32 4
|
||||
#define REPCODE_STARTVALUE 1
|
||||
|
||||
#define Litbits 8
|
||||
#define MaxLit ((1<<Litbits) - 1)
|
||||
@@ -158,12 +160,32 @@ MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length)
|
||||
while (op < oend);
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned ZSTD_highbit(U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return 31 - __builtin_clz(val);
|
||||
# else /* Software version */
|
||||
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
int r;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
|
||||
return r;
|
||||
# endif
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Private interfaces
|
||||
*********************************************/
|
||||
typedef struct ZSTD_stats_s ZSTD_stats_t;
|
||||
|
||||
typedef struct {
|
||||
U32 off;
|
||||
U32 len;
|
||||
@@ -180,12 +202,12 @@ typedef struct {
|
||||
#if ZSTD_OPT_DEBUG == 3
|
||||
#include ".debug/zstd_stats.h"
|
||||
#else
|
||||
struct ZSTD_stats_s { U32 unused; };
|
||||
typedef struct { U32 unused; } ZSTD_stats_t;
|
||||
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
|
||||
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
|
||||
#endif /* #if ZSTD_OPT_DEBUG == 3 */
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
void* buffer;
|
||||
@@ -228,11 +250,6 @@ typedef struct {
|
||||
|
||||
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
|
||||
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
|
||||
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
|
||||
|
||||
/* custom memory allocation functions */
|
||||
void* ZSTD_defaultAllocFunction(void* opaque, size_t size);
|
||||
void ZSTD_defaultFreeFunction(void* opaque, void* address);
|
||||
static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };
|
||||
|
||||
#endif /* ZSTD_CCOMMON_H_MODULE */
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
/*
|
||||
zstd - standard compression library
|
||||
Header File for static linking only
|
||||
Copyright (C) 2014-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net
|
||||
*/
|
||||
#ifndef ZSTD_STATIC_H
|
||||
#define ZSTD_STATIC_H
|
||||
|
||||
/* The prototypes defined within this file are considered experimental.
|
||||
* They should not be used in the context DLL as they may change in the future.
|
||||
* Prefer static linking if you need them, to control breaking version changes issues.
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "zstd.h"
|
||||
#include "mem.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Types
|
||||
***************************************/
|
||||
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27))
|
||||
#define ZSTD_WINDOWLOG_MIN 18
|
||||
#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
|
||||
#define ZSTD_CHAINLOG_MIN 4
|
||||
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
|
||||
#define ZSTD_HASHLOG_MIN 12
|
||||
#define ZSTD_HASHLOG3_MAX 17
|
||||
#define ZSTD_HASHLOG3_MIN 15
|
||||
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
|
||||
#define ZSTD_SEARCHLOG_MIN 1
|
||||
#define ZSTD_SEARCHLENGTH_MAX 7
|
||||
#define ZSTD_SEARCHLENGTH_MIN 3
|
||||
#define ZSTD_TARGETLENGTH_MIN 4
|
||||
#define ZSTD_TARGETLENGTH_MAX 999
|
||||
|
||||
/* from faster to stronger */
|
||||
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;
|
||||
|
||||
typedef struct {
|
||||
U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
|
||||
U32 chainLog; /* fully searched segment : larger == more compression, slower, more memory (useless for fast) */
|
||||
U32 hashLog; /* dispatch table : larger == faster, more memory */
|
||||
U32 searchLog; /* nb of searches : larger == more compression, slower */
|
||||
U32 searchLength; /* match length searched : larger == faster decompression, sometimes less compression */
|
||||
U32 targetLength; /* acceptable match size for optimal parser (only) : larger == more compression, slower */
|
||||
ZSTD_strategy strategy;
|
||||
} ZSTD_compressionParameters;
|
||||
|
||||
typedef struct {
|
||||
U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */
|
||||
} ZSTD_frameParameters;
|
||||
|
||||
typedef struct {
|
||||
ZSTD_compressionParameters cParams;
|
||||
ZSTD_frameParameters fParams;
|
||||
} ZSTD_parameters;
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Advanced functions
|
||||
***************************************/
|
||||
ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
|
||||
|
||||
/*! ZSTD_getCParams() :
|
||||
* @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
|
||||
* `srcSize` value is optional, select 0 if not known */
|
||||
ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_checkParams() :
|
||||
* Ensure param values remain within authorized range */
|
||||
ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
|
||||
|
||||
/*! ZSTD_adjustParams() :
|
||||
* optimize params for a given `srcSize` and `dictSize`.
|
||||
* both values are optional, select `0` if unknown. */
|
||||
ZSTDLIB_API void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize);
|
||||
|
||||
/*! ZSTD_compress_advanced() :
|
||||
* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
|
||||
ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize,
|
||||
ZSTD_parameters params);
|
||||
|
||||
/*! ZSTD_compress_usingPreparedDCtx() :
|
||||
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
|
||||
* It avoids reloading the dictionary each time.
|
||||
* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
|
||||
* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
|
||||
ZSTDLIB_API size_t ZSTD_compress_usingPreparedCCtx(
|
||||
ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/*- Advanced Decompression functions -*/
|
||||
|
||||
/*! ZSTD_decompress_usingPreparedDCtx() :
|
||||
* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
|
||||
* It avoids reloading the dictionary each time.
|
||||
* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
|
||||
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
|
||||
ZSTDLIB_API size_t ZSTD_decompress_usingPreparedDCtx(
|
||||
ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Streaming functions (direct mode)
|
||||
****************************************/
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
|
||||
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize);
|
||||
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity);
|
||||
|
||||
/*
|
||||
Streaming compression, synchronous mode (bufferless)
|
||||
|
||||
A ZSTD_CCtx object is required to track streaming operations.
|
||||
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage it.
|
||||
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
|
||||
|
||||
Start by initializing a context.
|
||||
Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression,
|
||||
or ZSTD_compressBegin_advanced(), for finer parameter control.
|
||||
It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
|
||||
|
||||
Then, consume your input using ZSTD_compressContinue().
|
||||
The interface is synchronous, so all input will be consumed and produce a compressed output.
|
||||
You must ensure there is enough space in destination buffer to store compressed data under worst case scenario.
|
||||
Worst case evaluation is provided by ZSTD_compressBound().
|
||||
|
||||
Finish a frame with ZSTD_compressEnd(), which will write the epilogue.
|
||||
Without the epilogue, frames will be considered incomplete by decoder.
|
||||
|
||||
You can then reuse ZSTD_CCtx to compress some new frame.
|
||||
*/
|
||||
|
||||
typedef struct { U64 frameContentSize; U32 windowLog; } ZSTD_frameParams;
|
||||
|
||||
#define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */
|
||||
static const size_t ZSTD_frameHeaderSize_min = 5;
|
||||
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
|
||||
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
/*
|
||||
Streaming decompression, direct mode (bufferless)
|
||||
|
||||
A ZSTD_DCtx object is required to track streaming operations.
|
||||
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
||||
A ZSTD_DCtx object can be re-used multiple times.
|
||||
|
||||
First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input.
|
||||
It can provide the minimum size of rolling buffer required to properly decompress data,
|
||||
and optionally the final size of uncompressed content.
|
||||
(Note : content size is an optional info that may not be present. 0 means : content size unknown)
|
||||
Frame parameters are extracted from the beginning of compressed frame.
|
||||
The amount of data to read is variable, from ZSTD_frameHeaderSize_min to ZSTD_frameHeaderSize_max (so if `srcSize` >= ZSTD_frameHeaderSize_max, it will always work)
|
||||
If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result.
|
||||
Result : 0 when successful, it means the ZSTD_frameParams structure has been filled.
|
||||
>0 : means there is not enough data into `src`. Provides the expected size to successfully decode header.
|
||||
errorCode, which can be tested using ZSTD_isError()
|
||||
|
||||
Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict().
|
||||
Alternatively, you can copy a prepared context, using ZSTD_copyDCtx().
|
||||
|
||||
Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
|
||||
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
|
||||
ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail.
|
||||
ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
|
||||
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
|
||||
|
||||
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity)
|
||||
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
|
||||
|
||||
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
|
||||
Context can then be reset to start a new decompression.
|
||||
*/
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Block functions
|
||||
****************************************/
|
||||
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
|
||||
A few rules to respect :
|
||||
- Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB)
|
||||
- Compressing or decompressing requires a context structure
|
||||
+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
|
||||
- It is necessary to init context before starting
|
||||
+ compression : ZSTD_compressBegin()
|
||||
+ decompression : ZSTD_decompressBegin()
|
||||
+ variants _usingDict() are also allowed
|
||||
+ copyCCtx() and copyDCtx() work too
|
||||
- When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
|
||||
In which case, nothing is produced into `dst`.
|
||||
+ User must test for such outcome and deal directly with uncompressed data
|
||||
+ ZSTD_decompressBlock() doesn't accept uncompressed data as input !!
|
||||
*/
|
||||
|
||||
#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
|
||||
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Error management
|
||||
***************************************/
|
||||
#include "error_public.h"
|
||||
/*! ZSTD_getErrorCode() :
|
||||
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
|
||||
which can be used to compare directly with enum list published into "error_public.h" */
|
||||
ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
|
||||
ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_STATIC_H */
|
||||
@@ -37,50 +37,57 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
//#include "zstd.h"
|
||||
//#include "mem.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
//#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Types
|
||||
***************************************/
|
||||
struct ZSTD_stats_s {
|
||||
U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength;
|
||||
typedef struct {
|
||||
U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength, priceDumpsLength;
|
||||
U32 totalMatchSum, totalLitSum, totalSeqSum, totalRepSum;
|
||||
U32 litSum, matchLengthSum, litLengthSum, offCodeSum;
|
||||
U32 matchLengthFreq[MaxML+1];
|
||||
U32 litLengthFreq[MaxLL+1];
|
||||
U32 matchLengthFreq[1<<MLbits];
|
||||
U32 litLengthFreq[1<<LLbits];
|
||||
U32 litFreq[1<<Litbits];
|
||||
U32 offCodeFreq[MaxOff+1];
|
||||
};
|
||||
U32 offCodeFreq[1<<Offbits];
|
||||
} ZSTD_stats_t;
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Stats functions
|
||||
* Advanced functions
|
||||
***************************************/
|
||||
MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc() { return malloc(sizeof(ZSTD_stats_t)); }
|
||||
MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { free(stats); }
|
||||
|
||||
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength)
|
||||
{
|
||||
stats->totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4);
|
||||
printf("\navgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum);
|
||||
printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8);
|
||||
printf("avgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum);
|
||||
printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d DumpsLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength+stats->priceDumpsLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8, stats->priceDumpsLength/8);
|
||||
}
|
||||
|
||||
|
||||
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats)
|
||||
{
|
||||
stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1;
|
||||
stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = 0;
|
||||
stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = stats->priceDumpsLength = 0;
|
||||
}
|
||||
|
||||
|
||||
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats)
|
||||
{
|
||||
unsigned u;
|
||||
|
||||
stats->litSum = (2<<Litbits);
|
||||
stats->litLengthSum = MaxLL+1;
|
||||
stats->matchLengthSum = MaxML+1;
|
||||
stats->offCodeSum = (MaxOff+1);
|
||||
stats->litSum = (1<<Litbits);
|
||||
stats->litLengthSum = (1<<LLbits);
|
||||
stats->matchLengthSum = (1<<MLbits);
|
||||
stats->offCodeSum = (1<<Offbits);
|
||||
|
||||
for (u=0; u<=MaxLit; u++)
|
||||
stats->litFreq[u] = 1;
|
||||
@@ -92,69 +99,64 @@ MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats)
|
||||
stats->offCodeFreq[u] = 1;
|
||||
}
|
||||
|
||||
|
||||
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
|
||||
{
|
||||
U32 u;
|
||||
/* literals */
|
||||
stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1);
|
||||
for (u=0; u < litLength; u++)
|
||||
stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1);
|
||||
stats->litSum += litLength;
|
||||
for (u=0; u < litLength; u++)
|
||||
stats->litFreq[literals[u]]++;
|
||||
|
||||
/* literal Length */
|
||||
{ static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 16, 17, 17, 18, 18, 19, 19,
|
||||
20, 20, 20, 20, 21, 21, 21, 21,
|
||||
22, 22, 22, 22, 22, 22, 22, 22,
|
||||
23, 23, 23, 23, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||
const BYTE LL_deltaCode = 19;
|
||||
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||
if (litLength) {
|
||||
stats->priceLitLength += LL_bits[llCode] + ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[llCode]+1);
|
||||
} else {
|
||||
stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1);
|
||||
}
|
||||
stats->litLengthFreq[llCode]++;
|
||||
stats->litLengthSum++;
|
||||
}
|
||||
|
||||
/* match offset */
|
||||
{ BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
|
||||
stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1);
|
||||
stats->priceOffset += offCode;
|
||||
stats->offCodeSum++;
|
||||
stats->offCodeFreq[offCode]++;
|
||||
}
|
||||
/* offset */
|
||||
BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset+1) + 1 : 0;
|
||||
stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1);
|
||||
stats->priceOffset += (offCode-1) + (!offCode);
|
||||
|
||||
/* match Length */
|
||||
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
|
||||
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
||||
const BYTE ML_deltaCode = 36;
|
||||
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
||||
stats->priceMatchLength += ML_bits[mlCode] + ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[mlCode]+1);
|
||||
stats->matchLengthFreq[mlCode]++;
|
||||
stats->matchLengthSum++;
|
||||
stats->priceDumpsLength += ((matchLength >= MaxML)<<3) + ((matchLength >= 255+MaxML)<<4) + ((matchLength>=(1<<15))<<3);
|
||||
stats->priceMatchLength += ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[(matchLength >= MaxML) ? MaxML : matchLength]+1);
|
||||
|
||||
if (litLength) {
|
||||
/* literals */
|
||||
U32 u;
|
||||
stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1);
|
||||
for (u=0; u < litLength; u++)
|
||||
stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1);
|
||||
|
||||
/* literal Length */
|
||||
stats->priceDumpsLength += ((litLength >= MaxLL)<<3) + ((litLength >= 255+MaxLL)<<4) + ((litLength>=(1<<15))<<3);
|
||||
stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[(litLength >= MaxLL) ? MaxLL : litLength]+1);
|
||||
} else {
|
||||
stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1);
|
||||
}
|
||||
|
||||
|
||||
if (offset == 0) stats->totalRepSum++;
|
||||
stats->totalSeqSum++;
|
||||
stats->totalMatchSum += matchLength;
|
||||
stats->totalLitSum += litLength;
|
||||
|
||||
U32 u;
|
||||
/* literals */
|
||||
stats->litSum += litLength;
|
||||
for (u=0; u < litLength; u++)
|
||||
stats->litFreq[literals[u]]++;
|
||||
|
||||
/* literal Length */
|
||||
stats->litLengthSum++;
|
||||
if (litLength >= MaxLL)
|
||||
stats->litLengthFreq[MaxLL]++;
|
||||
else
|
||||
stats->litLengthFreq[litLength]++;
|
||||
|
||||
/* match offset */
|
||||
stats->offCodeSum++;
|
||||
stats->offCodeFreq[offCode]++;
|
||||
|
||||
/* match Length */
|
||||
stats->matchLengthSum++;
|
||||
if (matchLength >= MaxML)
|
||||
stats->matchLengthFreq[MaxML]++;
|
||||
else
|
||||
stats->matchLengthFreq[matchLength]++;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -57,8 +57,7 @@
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include <stdio.h> /* printf (debug) */
|
||||
#include "bitstream.h"
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#include "fse_static.h"
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
@@ -112,6 +111,8 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
|
||||
U32 highThreshold = tableSize-1;
|
||||
|
||||
/* CTable header */
|
||||
|
||||
|
||||
tableU16[-2] = (U16) tableLog;
|
||||
tableU16[-1] = (U16) maxSymbolValue;
|
||||
|
||||
@@ -146,10 +147,10 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
|
||||
}
|
||||
|
||||
/* Build table */
|
||||
{ U32 u; for (u=0; u<tableSize; u++) {
|
||||
{ U32 u; for (u=0; u<tableSize; u++) {
|
||||
FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
|
||||
tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */
|
||||
} }
|
||||
}}
|
||||
|
||||
/* Build Symbol Transformation Table */
|
||||
{ unsigned total = 0;
|
||||
@@ -256,7 +257,7 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
|
||||
bitStream += count << bitCount;
|
||||
bitCount += nbBits;
|
||||
bitCount -= (count<max);
|
||||
previous0 = (count==1);
|
||||
previous0 = (count==1);
|
||||
while (remaining<threshold) nbBits--, threshold>>=1;
|
||||
}
|
||||
if (bitCount>16) {
|
||||
@@ -443,7 +444,7 @@ FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
|
||||
return (FSE_CTable*)malloc(size);
|
||||
}
|
||||
|
||||
void FSE_freeCTable (FSE_CTable* ct) { free(ct); }
|
||||
void FSE_freeCTable (FSE_CTable* ct) { free(ct); }
|
||||
|
||||
/* provides the minimum logSize to safely represent a distribution */
|
||||
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
|
||||
@@ -454,9 +455,9 @@ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
|
||||
return minBits;
|
||||
}
|
||||
|
||||
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
|
||||
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
|
||||
{
|
||||
U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
|
||||
U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - 2;
|
||||
U32 tableLog = maxTableLog;
|
||||
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
|
||||
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
|
||||
@@ -467,11 +468,6 @@ unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsi
|
||||
return tableLog;
|
||||
}
|
||||
|
||||
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
|
||||
{
|
||||
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
|
||||
}
|
||||
|
||||
|
||||
/* Secondary normalization method.
|
||||
To be used when primary method fails. */
|
||||
|
||||
+73
-89
@@ -59,13 +59,12 @@
|
||||
/* **************************************************************
|
||||
* Includes
|
||||
****************************************************************/
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include <stdio.h> /* printf (debug) */
|
||||
#include "huf_static.h"
|
||||
#include "bitstream.h"
|
||||
#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
|
||||
#include "fse.h" /* header compression */
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
@@ -74,15 +73,6 @@
|
||||
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Utils
|
||||
****************************************************************/
|
||||
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
|
||||
{
|
||||
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
|
||||
}
|
||||
|
||||
|
||||
/* *******************************************************
|
||||
* HUF : Huffman block compression
|
||||
*********************************************************/
|
||||
@@ -104,14 +94,14 @@ typedef struct nodeElt_s {
|
||||
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
||||
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
|
||||
{
|
||||
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
|
||||
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
|
||||
BYTE bitsToWeight[HUF_MAX_TABLELOG + 1];
|
||||
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
|
||||
U32 n;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
size_t size;
|
||||
|
||||
/* check conditions */
|
||||
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX + 1)
|
||||
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1)
|
||||
return ERROR(GENERIC);
|
||||
|
||||
/* convert to weight */
|
||||
@@ -126,7 +116,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
||||
if (size >= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */
|
||||
if ((size <= 1) || (size >= maxSymbolValue/2)) {
|
||||
if (size==1) { /* RLE */
|
||||
/* only possible case : series of 1 (because there are at least 2) */
|
||||
/* only possible case : serie of 1 (because there are at least 2) */
|
||||
/* can only be 2^n or (2^n-1), otherwise not an huffman tree */
|
||||
BYTE code;
|
||||
switch(maxSymbolValue)
|
||||
@@ -169,19 +159,19 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
|
||||
|
||||
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
|
||||
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
|
||||
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
|
||||
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
|
||||
U32 tableLog = 0;
|
||||
size_t readSize;
|
||||
U32 nbSymbols = 0;
|
||||
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
/* get symbol weights */
|
||||
readSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
||||
readSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
||||
if (HUF_isError(readSize)) return readSize;
|
||||
|
||||
/* check result */
|
||||
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
|
||||
if (tableLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
|
||||
|
||||
/* Prepare base value per rank */
|
||||
@@ -199,12 +189,12 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
|
||||
}}
|
||||
|
||||
/* fill val */
|
||||
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
|
||||
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
|
||||
{ U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
U16 valPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
|
||||
/* determine stating value per rank */
|
||||
{ U16 min = 0;
|
||||
U32 n; for (n=HUF_TABLELOG_MAX; n>0; n--) {
|
||||
U32 n; for (n=HUF_MAX_TABLELOG; n>0; n--) {
|
||||
valPerRank[n] = min; /* get starting value within each rank */
|
||||
min += nbPerRank[n];
|
||||
min >>= 1;
|
||||
@@ -239,7 +229,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
|
||||
/* repay normalized cost */
|
||||
{ U32 const noSymbol = 0xF0F0F0F0;
|
||||
U32 rankLast[HUF_TABLELOG_MAX+1];
|
||||
U32 rankLast[HUF_MAX_TABLELOG+1];
|
||||
int pos;
|
||||
|
||||
/* Get pos of last (smallest) symbol per rank */
|
||||
@@ -263,7 +253,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
if (highTotal <= lowTotal) break;
|
||||
} }
|
||||
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
|
||||
while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
|
||||
while ((nBitsToDecrease<=HUF_MAX_TABLELOG) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
|
||||
nBitsToDecrease ++;
|
||||
totalCost -= 1 << (nBitsToDecrease-1);
|
||||
if (rankLast[nBitsToDecrease-1] == noSymbol)
|
||||
@@ -322,10 +312,10 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
|
||||
}
|
||||
|
||||
|
||||
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
|
||||
#define STARTNODE (HUF_MAX_SYMBOL_VALUE+1)
|
||||
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
|
||||
{
|
||||
nodeElt huffNode0[2*HUF_SYMBOLVALUE_MAX+1 +1];
|
||||
nodeElt huffNode0[2*HUF_MAX_SYMBOL_VALUE+1 +1];
|
||||
nodeElt* huffNode = huffNode0 + 1;
|
||||
U32 n, nonNullRank;
|
||||
int lowS, lowN;
|
||||
@@ -333,8 +323,8 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
|
||||
U32 nodeRoot;
|
||||
|
||||
/* safety checks */
|
||||
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
|
||||
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
|
||||
if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
|
||||
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return ERROR(GENERIC);
|
||||
memset(huffNode0, 0, sizeof(huffNode0));
|
||||
|
||||
/* sort, decreasing order */
|
||||
@@ -370,9 +360,9 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
|
||||
maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
|
||||
|
||||
/* fill result into tree (val, nbBits) */
|
||||
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
|
||||
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
|
||||
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
|
||||
{ U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
U16 valPerRank[HUF_MAX_TABLELOG+1] = {0};
|
||||
if (maxNbBits > HUF_MAX_TABLELOG) return ERROR(GENERIC); /* check fit into table */
|
||||
for (n=0; n<=nonNullRank; n++)
|
||||
nbPerRank[huffNode[n].nbBits]++;
|
||||
/* determine stating value per rank */
|
||||
@@ -401,10 +391,10 @@ size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
|
||||
#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
|
||||
|
||||
#define HUF_FLUSHBITS_1(stream) \
|
||||
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream)
|
||||
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) HUF_FLUSHBITS(stream)
|
||||
|
||||
#define HUF_FLUSHBITS_2(stream) \
|
||||
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream)
|
||||
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream)
|
||||
|
||||
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
||||
{
|
||||
@@ -451,47 +441,44 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
|
||||
|
||||
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
|
||||
{
|
||||
size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */
|
||||
size_t segmentSize = (srcSize+3)/4; /* first 3 segments */
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
const BYTE* const iend = ip + srcSize;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
size_t errorCode;
|
||||
|
||||
if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */
|
||||
if (srcSize < 12) return 0; /* no saving possible : too small input */
|
||||
op += 6; /* jumpTable */
|
||||
|
||||
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(cSize)) return cSize;
|
||||
if (cSize==0) return 0;
|
||||
MEM_writeLE16(ostart, (U16)cSize);
|
||||
op += cSize;
|
||||
}
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
MEM_writeLE16(ostart, (U16)errorCode);
|
||||
|
||||
ip += segmentSize;
|
||||
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(cSize)) return cSize;
|
||||
if (cSize==0) return 0;
|
||||
MEM_writeLE16(ostart+2, (U16)cSize);
|
||||
op += cSize;
|
||||
}
|
||||
op += errorCode;
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
MEM_writeLE16(ostart+2, (U16)errorCode);
|
||||
|
||||
ip += segmentSize;
|
||||
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(cSize)) return cSize;
|
||||
if (cSize==0) return 0;
|
||||
MEM_writeLE16(ostart+4, (U16)cSize);
|
||||
op += cSize;
|
||||
}
|
||||
op += errorCode;
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
MEM_writeLE16(ostart+4, (U16)errorCode);
|
||||
|
||||
ip += segmentSize;
|
||||
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
|
||||
if (HUF_isError(cSize)) return cSize;
|
||||
if (cSize==0) return 0;
|
||||
op += cSize;
|
||||
}
|
||||
op += errorCode;
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
|
||||
op += errorCode;
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
@@ -506,46 +493,43 @@ static size_t HUF_compress_internal (
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
BYTE* op = ostart;
|
||||
|
||||
U32 count[HUF_SYMBOLVALUE_MAX+1];
|
||||
HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
|
||||
U32 count[HUF_MAX_SYMBOL_VALUE+1];
|
||||
HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1];
|
||||
size_t errorCode;
|
||||
|
||||
/* checks & inits */
|
||||
if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
|
||||
if (!dstSize) return 0; /* cannot fit within dst budget */
|
||||
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
|
||||
if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
|
||||
if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
|
||||
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
|
||||
if (srcSize < 1) return 0; /* Uncompressed - note : 1 means rle, so first byte must be correct */
|
||||
if (dstSize < 1) return 0; /* not compressible within dst budget */
|
||||
if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */
|
||||
if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE;
|
||||
if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG;
|
||||
|
||||
/* Scan input and build symbol stats */
|
||||
{ size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
|
||||
if (HUF_isError(largest)) return largest;
|
||||
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* rle */
|
||||
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
|
||||
}
|
||||
errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }
|
||||
if (errorCode <= (srcSize >> 7)+1) return 0; /* Heuristic : not compressible enough */
|
||||
|
||||
/* Build Huffman Tree */
|
||||
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
|
||||
{ size_t const maxBits = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
|
||||
if (HUF_isError(maxBits)) return maxBits;
|
||||
huffLog = (U32)maxBits;
|
||||
}
|
||||
errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
huffLog = (U32)errorCode;
|
||||
|
||||
/* Write table description header */
|
||||
{ size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
|
||||
if (HUF_isError(hSize)) return hSize;
|
||||
if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */
|
||||
op += hSize;
|
||||
}
|
||||
errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode + 12 >= srcSize) return 0; /* not useful to try compression */
|
||||
op += errorCode;
|
||||
|
||||
/* Compress */
|
||||
{ size_t const cSize = (singleStream) ?
|
||||
HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */
|
||||
HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
|
||||
if (HUF_isError(cSize)) return cSize;
|
||||
if (cSize==0) return 0; /* uncompressible */
|
||||
op += cSize;
|
||||
}
|
||||
if (singleStream)
|
||||
errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */
|
||||
else
|
||||
errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode==0) return 0;
|
||||
op += errorCode;
|
||||
|
||||
/* check compressibility */
|
||||
if ((size_t)(op-ostart) >= srcSize-1)
|
||||
@@ -572,5 +556,5 @@ size_t HUF_compress2 (void* dst, size_t dstSize,
|
||||
|
||||
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);
|
||||
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_DEFAULT_TABLELOG);
|
||||
}
|
||||
|
||||
@@ -35,9 +35,9 @@
|
||||
***************************************/
|
||||
#include <stdlib.h>
|
||||
#include "error_private.h"
|
||||
#include "zstd_internal.h" /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */
|
||||
#define ZBUFF_STATIC_LINKING_ONLY
|
||||
#include "zbuff.h"
|
||||
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */
|
||||
#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */
|
||||
#include "zbuff_static.h"
|
||||
|
||||
|
||||
/* *************************************
|
||||
@@ -81,7 +81,7 @@ static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
|
||||
|
||||
typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
|
||||
|
||||
/* *** Resources *** */
|
||||
/* *** Ressources *** */
|
||||
struct ZBUFF_CCtx_s {
|
||||
ZSTD_CCtx* zc;
|
||||
char* inBuff;
|
||||
@@ -95,30 +95,14 @@ struct ZBUFF_CCtx_s {
|
||||
size_t outBuffContentSize;
|
||||
size_t outBuffFlushedSize;
|
||||
ZBUFF_cStage stage;
|
||||
ZSTD_customMem customMem;
|
||||
}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
|
||||
|
||||
ZBUFF_CCtx* ZBUFF_createCCtx(void)
|
||||
{
|
||||
return ZBUFF_createCCtx_advanced(defaultCustomMem);
|
||||
}
|
||||
|
||||
ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
|
||||
{
|
||||
ZBUFF_CCtx* zbc;
|
||||
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx));
|
||||
ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx));
|
||||
if (zbc==NULL) return NULL;
|
||||
memset(zbc, 0, sizeof(ZBUFF_CCtx));
|
||||
memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem));
|
||||
zbc->zc = ZSTD_createCCtx_advanced(customMem);
|
||||
if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; }
|
||||
memset(zbc, 0, sizeof(*zbc));
|
||||
zbc->zc = ZSTD_createCCtx();
|
||||
return zbc;
|
||||
}
|
||||
|
||||
@@ -126,9 +110,9 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
|
||||
{
|
||||
if (zbc==NULL) return 0; /* support free on NULL */
|
||||
ZSTD_freeCCtx(zbc->zc);
|
||||
if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff);
|
||||
if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff);
|
||||
zbc->customMem.customFree(zbc->customMem.opaque, zbc);
|
||||
free(zbc->inBuff);
|
||||
free(zbc->outBuff);
|
||||
free(zbc);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -143,16 +127,16 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
|
||||
if (zbc->inBuffSize < neededInBuffSize) {
|
||||
zbc->inBuffSize = neededInBuffSize;
|
||||
zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */
|
||||
zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize);
|
||||
free(zbc->inBuff); /* should not be necessary */
|
||||
zbc->inBuff = (char*)malloc(neededInBuffSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize);
|
||||
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2);
|
||||
}
|
||||
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
|
||||
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
|
||||
zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */
|
||||
zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize);
|
||||
free(zbc->outBuff); /* should not be necessary */
|
||||
zbc->outBuff = (char*)malloc(zbc->outBuffSize);
|
||||
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
|
||||
@@ -171,8 +155,9 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
|
||||
return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
|
||||
}
|
||||
|
||||
@@ -182,15 +167,6 @@ size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
|
||||
}
|
||||
|
||||
|
||||
/* internal util function */
|
||||
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const length = MIN(dstCapacity, srcSize);
|
||||
memcpy(dst, src, length);
|
||||
return length;
|
||||
}
|
||||
|
||||
|
||||
/* *** Compression *** */
|
||||
|
||||
static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
|
||||
+209
-430
@@ -51,15 +51,12 @@
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include <string.h> /* memset */
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <string.h> /* memset */
|
||||
#include "mem.h"
|
||||
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
#include "xxhash.h" /* XXH_reset, update, digest */
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "fse_static.h"
|
||||
#include "huf_static.h"
|
||||
#include "zstd_internal.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
@@ -73,27 +70,6 @@ static const U32 g_searchStrength = 8; /* control skip over incompressible dat
|
||||
***************************************/
|
||||
size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
|
||||
|
||||
static U32 ZSTD_highbit32(U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return 31 - __builtin_clz(val);
|
||||
# else /* Software version */
|
||||
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
int r;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
|
||||
return r;
|
||||
# endif
|
||||
}
|
||||
|
||||
/*-*************************************
|
||||
* Sequence storage
|
||||
@@ -123,16 +99,10 @@ struct ZSTD_CCtx_s
|
||||
U32 hashLog3; /* dispatch table : larger == faster, more memory */
|
||||
U32 loadedDictEnd;
|
||||
U32 stage; /* 0: created; 1: init,dictLoad; 2:started */
|
||||
U32 rep[ZSTD_REP_NUM];
|
||||
U32 savedRep[ZSTD_REP_NUM];
|
||||
U32 dictID;
|
||||
ZSTD_parameters params;
|
||||
void* workSpace;
|
||||
size_t workSpaceSize;
|
||||
size_t blockSize;
|
||||
U64 frameContentSize;
|
||||
XXH64_state_t xxhState;
|
||||
ZSTD_customMem customMem;
|
||||
|
||||
seqStore_t seqStore; /* sequences storage ptrs */
|
||||
U32* hashTable;
|
||||
@@ -147,31 +117,13 @@ struct ZSTD_CCtx_s
|
||||
|
||||
ZSTD_CCtx* ZSTD_createCCtx(void)
|
||||
{
|
||||
return ZSTD_createCCtx_advanced(defaultCustomMem);
|
||||
}
|
||||
|
||||
ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
|
||||
{
|
||||
ZSTD_CCtx* ctx;
|
||||
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
ctx = (ZSTD_CCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CCtx));
|
||||
if (!ctx) return NULL;
|
||||
memset(ctx, 0, sizeof(ZSTD_CCtx));
|
||||
memcpy(&ctx->customMem, &customMem, sizeof(ZSTD_customMem));
|
||||
return ctx;
|
||||
return (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx));
|
||||
}
|
||||
|
||||
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
|
||||
{
|
||||
if (cctx==NULL) return 0; /* support free on NULL */
|
||||
if (cctx->workSpace) cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace);
|
||||
cctx->customMem.customFree(cctx->customMem.opaque, cctx);
|
||||
free(cctx->workSpace);
|
||||
free(cctx);
|
||||
return 0; /* reserved as a potential error code in the future */
|
||||
}
|
||||
|
||||
@@ -202,6 +154,8 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
|
||||
}
|
||||
|
||||
|
||||
static unsigned ZSTD_highbit(U32 val);
|
||||
|
||||
/** ZSTD_checkCParams_advanced() :
|
||||
temporary work-around, while the compressor compatibility remains limited regarding windowLog < 18 */
|
||||
size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSize)
|
||||
@@ -215,32 +169,30 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz
|
||||
}
|
||||
|
||||
|
||||
/** ZSTD_adjustCParams() :
|
||||
optimize cPar for a given input (`srcSize` and `dictSize`).
|
||||
/** ZSTD_adjustParams() :
|
||||
optimize params for q given input (`srcSize` and `dictSize`).
|
||||
mostly downsizing to reduce memory consumption and initialization.
|
||||
Both `srcSize` and `dictSize` are optional (use 0 if unknown),
|
||||
but if both are 0, no optimization can be done.
|
||||
Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
|
||||
ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize)
|
||||
Note : params is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
|
||||
void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t dictSize)
|
||||
{
|
||||
if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */
|
||||
if (srcSize+dictSize == 0) return; /* no size information available : no adjustment */
|
||||
|
||||
/* resize params, to use less memory when necessary */
|
||||
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0;
|
||||
U64 const rSize = srcSize + dictSize + minSrcSize;
|
||||
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
|
||||
U32 const srcLog = ZSTD_highbit32((U32)(rSize)-1) + 1;
|
||||
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
|
||||
U32 const srcLog = ZSTD_highbit((U32)(rSize)-1) + 1;
|
||||
if (params->windowLog > srcLog) params->windowLog = srcLog;
|
||||
} }
|
||||
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
|
||||
{ U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) || (cPar.strategy == ZSTD_btopt);
|
||||
U32 const maxChainLog = cPar.windowLog+btPlus;
|
||||
if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
|
||||
if (params->hashLog > params->windowLog) params->hashLog = params->windowLog;
|
||||
{ U32 const btPlus = (params->strategy == ZSTD_btlazy2) || (params->strategy == ZSTD_btopt);
|
||||
U32 const maxChainLog = params->windowLog+btPlus;
|
||||
if (params->chainLog > maxChainLog) params->chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
|
||||
|
||||
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
|
||||
if ((cPar.hashLog < ZSTD_HASHLOG_MIN) && ( (U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
|
||||
|
||||
return cPar;
|
||||
if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
|
||||
if ((params->hashLog < ZSTD_HASHLOG_MIN) && ((U32)params->strategy >= (U32)ZSTD_btlazy2)) params->hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
|
||||
}
|
||||
|
||||
|
||||
@@ -248,7 +200,6 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface
|
||||
{
|
||||
ZSTD_CCtx* zc = ZSTD_createCCtx();
|
||||
ZSTD_parameters params;
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.cParams = cParams;
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0);
|
||||
@@ -260,7 +211,7 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface
|
||||
/*! ZSTD_resetCCtx_advanced() :
|
||||
note : 'params' is expected to be validated */
|
||||
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
ZSTD_parameters params, U64 frameContentSize, U32 reset)
|
||||
ZSTD_parameters params, U32 reset)
|
||||
{ /* note : params considered validated here */
|
||||
const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog);
|
||||
const U32 divider = (params.cParams.searchLength==3) ? 3 : 4;
|
||||
@@ -268,10 +219,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
const size_t tokenSpace = blockSize + 11*maxNbSeq;
|
||||
const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
|
||||
const size_t hSize = ((size_t)1) << params.cParams.hashLog;
|
||||
const U32 hashLog3 = (params.cParams.searchLength>3) ? 0 :
|
||||
( (!frameContentSize || frameContentSize >= 8192) ? ZSTD_HASHLOG3_MAX :
|
||||
((frameContentSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN) );
|
||||
const size_t h3Size = ((size_t)1) << hashLog3;
|
||||
const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0;
|
||||
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
||||
|
||||
/* Check if workSpace is large enough, alloc a new one if needed */
|
||||
@@ -280,19 +228,17 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
|
||||
+ ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
|
||||
if (zc->workSpaceSize < neededSpace) {
|
||||
zc->customMem.customFree(zc->customMem.opaque, zc->workSpace);
|
||||
zc->workSpace = zc->customMem.customAlloc(zc->customMem.opaque, neededSpace);
|
||||
free(zc->workSpace);
|
||||
zc->workSpace = malloc(neededSpace);
|
||||
if (zc->workSpace == NULL) return ERROR(memory_allocation);
|
||||
zc->workSpaceSize = neededSpace;
|
||||
} }
|
||||
|
||||
if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */
|
||||
XXH64_reset(&zc->xxhState, 0);
|
||||
zc->hashLog3 = hashLog3;
|
||||
zc->hashTable = (U32*)(zc->workSpace);
|
||||
zc->hashTable3 = (U32*)(zc->workSpace);
|
||||
zc->hashTable = zc->hashTable3 + h3Size;
|
||||
zc->chainTable = zc->hashTable + hSize;
|
||||
zc->hashTable3 = zc->chainTable + chainSize;
|
||||
zc->seqStore.buffer = zc->hashTable3 + h3Size;
|
||||
zc->seqStore.buffer = zc->chainTable + chainSize;
|
||||
zc->hufTable = (HUF_CElt*)zc->seqStore.buffer;
|
||||
zc->flagStaticTables = 0;
|
||||
zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256;
|
||||
@@ -305,8 +251,6 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
zc->lowLimit = 0;
|
||||
zc->params = params;
|
||||
zc->blockSize = blockSize;
|
||||
zc->frameContentSize = frameContentSize;
|
||||
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; }
|
||||
|
||||
if (params.cParams.strategy == ZSTD_btopt) {
|
||||
zc->seqStore.litFreq = (U32*)(zc->seqStore.buffer);
|
||||
@@ -327,7 +271,6 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
zc->seqStore.litStart = zc->seqStore.offCodeStart + maxNbSeq;
|
||||
|
||||
zc->stage = 1;
|
||||
zc->dictID = 0;
|
||||
zc->loadedDictEnd = 0;
|
||||
|
||||
return 0;
|
||||
@@ -342,19 +285,19 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
|
||||
{
|
||||
if (srcCCtx->stage!=1) return ERROR(stage_wrong);
|
||||
|
||||
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
|
||||
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0);
|
||||
dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */
|
||||
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0);
|
||||
dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */
|
||||
|
||||
/* copy tables */
|
||||
{ const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog);
|
||||
const size_t hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog;
|
||||
const size_t h3Size = (size_t)1 << srcCCtx->hashLog3;
|
||||
const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0;
|
||||
const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
|
||||
memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
|
||||
}
|
||||
|
||||
/* copy dictionary offsets */
|
||||
/* copy dictionary pointers */
|
||||
dstCCtx->nextToUpdate = srcCCtx->nextToUpdate;
|
||||
dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3;
|
||||
dstCCtx->nextSrc = srcCCtx->nextSrc;
|
||||
@@ -363,7 +306,6 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
|
||||
dstCCtx->dictLimit = srcCCtx->dictLimit;
|
||||
dstCCtx->lowLimit = srcCCtx->lowLimit;
|
||||
dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd;
|
||||
dstCCtx->dictID = srcCCtx->dictID;
|
||||
|
||||
/* copy entropy tables */
|
||||
dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
|
||||
@@ -411,9 +353,8 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
|
||||
/* Frame format description
|
||||
Frame Header - [ Block Header - Block ] - Frame End
|
||||
1) Frame Header
|
||||
- 4 bytes : Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h)
|
||||
- 1 byte : Frame Header Descriptor
|
||||
- 1-13 bytes : Optional fields
|
||||
- 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h)
|
||||
- 1 byte - Frame Descriptor
|
||||
2) Block Header
|
||||
- 3 bytes, starting with a 2-bits descriptor
|
||||
Uncompressed, Compressed, Frame End, unused
|
||||
@@ -426,38 +367,13 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
|
||||
|
||||
/* Frame descriptor
|
||||
|
||||
// old
|
||||
1 byte - Alloc :
|
||||
1 byte, using :
|
||||
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
|
||||
bit 4 : reserved for windowLog (must be zero)
|
||||
bit 4 : minmatch 4(0) or 3(1)
|
||||
bit 5 : reserved (must be zero)
|
||||
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
|
||||
|
||||
1 byte - checker :
|
||||
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||
bit 2-7 : reserved (must be zero)
|
||||
|
||||
|
||||
// new
|
||||
1 byte - FrameHeaderDescription :
|
||||
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||
bit 2-4 : reserved (must be zero)
|
||||
bit 5 : SkippedWindowLog (if 1, WindowLog byte is not present)
|
||||
bit 6-7 : FrameContentFieldsize (0, 2, 4, or 8)
|
||||
if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1;
|
||||
|
||||
Optional : WindowLog (0 or 1 byte)
|
||||
bit 0-2 : octal Fractional (1/8th)
|
||||
bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB)
|
||||
|
||||
Optional : dictID (0, 1, 2 or 4 bytes)
|
||||
Automatic adaptation
|
||||
0 : no dictID
|
||||
1 : 1 - 255
|
||||
2 : 256 - 65535
|
||||
4 : all other values
|
||||
|
||||
Optional : content size (0, 1, 2, 4 or 8 bytes)
|
||||
Optional : content size (0, 1, 2 or 8 bytes)
|
||||
0 : unknown
|
||||
1 : 0-255 bytes
|
||||
2 : 256 - 65535+256
|
||||
@@ -578,15 +494,15 @@ static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void
|
||||
switch(flSize)
|
||||
{
|
||||
case 1: /* 2 - 1 - 5 */
|
||||
ostart[0] = (BYTE)((lbt_raw<<6) + (0<<5) + srcSize);
|
||||
ostart[0] = (BYTE)((IS_RAW<<6) + (0<<5) + srcSize);
|
||||
break;
|
||||
case 2: /* 2 - 2 - 12 */
|
||||
ostart[0] = (BYTE)((lbt_raw<<6) + (2<<4) + (srcSize >> 8));
|
||||
ostart[0] = (BYTE)((IS_RAW<<6) + (2<<4) + (srcSize >> 8));
|
||||
ostart[1] = (BYTE)srcSize;
|
||||
break;
|
||||
default: /*note : should not be necessary : flSize is within {1,2,3} */
|
||||
case 3: /* 2 - 2 - 20 */
|
||||
ostart[0] = (BYTE)((lbt_raw<<6) + (3<<4) + (srcSize >> 16));
|
||||
ostart[0] = (BYTE)((IS_RAW<<6) + (3<<4) + (srcSize >> 16));
|
||||
ostart[1] = (BYTE)(srcSize>>8);
|
||||
ostart[2] = (BYTE)srcSize;
|
||||
break;
|
||||
@@ -606,15 +522,15 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, cons
|
||||
switch(flSize)
|
||||
{
|
||||
case 1: /* 2 - 1 - 5 */
|
||||
ostart[0] = (BYTE)((lbt_rle<<6) + (0<<5) + srcSize);
|
||||
ostart[0] = (BYTE)((IS_RLE<<6) + (0<<5) + srcSize);
|
||||
break;
|
||||
case 2: /* 2 - 2 - 12 */
|
||||
ostart[0] = (BYTE)((lbt_rle<<6) + (2<<4) + (srcSize >> 8));
|
||||
ostart[0] = (BYTE)((IS_RLE<<6) + (2<<4) + (srcSize >> 8));
|
||||
ostart[1] = (BYTE)srcSize;
|
||||
break;
|
||||
default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */
|
||||
case 3: /* 2 - 2 - 20 */
|
||||
ostart[0] = (BYTE)((lbt_rle<<6) + (3<<4) + (srcSize >> 16));
|
||||
ostart[0] = (BYTE)((IS_RLE<<6) + (3<<4) + (srcSize >> 16));
|
||||
ostart[1] = (BYTE)(srcSize>>8);
|
||||
ostart[2] = (BYTE)srcSize;
|
||||
break;
|
||||
@@ -635,7 +551,7 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
|
||||
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
U32 singleStream = srcSize < 256;
|
||||
litBlockType_t hType = lbt_huffman;
|
||||
U32 hType = IS_HUF;
|
||||
size_t cLitSize;
|
||||
|
||||
|
||||
@@ -647,7 +563,7 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
|
||||
|
||||
if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */
|
||||
if (zc->flagStaticTables && (lhSize==3)) {
|
||||
hType = lbt_repeat;
|
||||
hType = IS_PCH;
|
||||
singleStream = 1;
|
||||
cLitSize = HUF_compress1X_usingCTable(ostart+lhSize, dstCapacity-lhSize, src, srcSize, zc->hufTable);
|
||||
} else {
|
||||
@@ -704,7 +620,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
|
||||
size_t u;
|
||||
for (u=0; u<nbSeq; u++) {
|
||||
U32 const ll = llTable[u];
|
||||
llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit32(ll) + LL_deltaCode : LL_Code[ll];
|
||||
llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit(ll) + LL_deltaCode : LL_Code[ll];
|
||||
}
|
||||
if (seqStorePtr->longLengthID==1)
|
||||
llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
|
||||
@@ -714,7 +630,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
|
||||
{ const U32* const offsetTable = seqStorePtr->offsetStart;
|
||||
BYTE* const ofCodeTable = seqStorePtr->offCodeStart;
|
||||
size_t u;
|
||||
for (u=0; u<nbSeq; u++) ofCodeTable[u] = (BYTE)ZSTD_highbit32(offsetTable[u]);
|
||||
for (u=0; u<nbSeq; u++) ofCodeTable[u] = (BYTE)ZSTD_highbit(offsetTable[u]);
|
||||
}
|
||||
|
||||
/* ML codes */
|
||||
@@ -732,7 +648,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
|
||||
size_t u;
|
||||
for (u=0; u<nbSeq; u++) {
|
||||
U32 const ml = mlTable[u];
|
||||
mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit32(ml) + ML_deltaCode : ML_Code[ml];
|
||||
mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit(ml) + ML_deltaCode : ML_Code[ml];
|
||||
}
|
||||
if (seqStorePtr->longLengthID==2)
|
||||
mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
|
||||
@@ -921,9 +837,6 @@ _check_compressibility:
|
||||
size_t const maxCSize = srcSize - minGain;
|
||||
if ((size_t)(op-ostart) >= maxCSize) return 0; }
|
||||
|
||||
/* confirm repcodes */
|
||||
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->savedRep[i]; }
|
||||
|
||||
return op - ostart;
|
||||
}
|
||||
|
||||
@@ -933,17 +846,17 @@ _check_compressibility:
|
||||
`offsetCode` : distance to match, or 0 == repCode.
|
||||
`matchCode` : matchLength - MINMATCH
|
||||
*/
|
||||
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, size_t offsetCode, size_t matchCode)
|
||||
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
|
||||
{
|
||||
#if 0 /* for debug */
|
||||
static const BYTE* g_start = NULL;
|
||||
const U32 pos = (U32)(literals - g_start);
|
||||
if (g_start==NULL) g_start = literals;
|
||||
//if ((pos > 1) && (pos < 50000))
|
||||
if ((pos > 2587900) && (pos < 2588050))
|
||||
printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
|
||||
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
|
||||
#endif
|
||||
ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, (const BYTE*)literals, offsetCode, matchCode); /* debug only */
|
||||
ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode);
|
||||
|
||||
/* copy Literals */
|
||||
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
|
||||
@@ -1029,7 +942,7 @@ static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLim
|
||||
const BYTE* const pStart = pIn;
|
||||
|
||||
while ((pIn<pInLimit-(sizeof(size_t)-1))) {
|
||||
size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
||||
size_t diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
||||
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
|
||||
pIn += ZSTD_NbCommonBytes(diff);
|
||||
return (size_t)(pIn - pStart);
|
||||
@@ -1046,8 +959,10 @@ static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLim
|
||||
*/
|
||||
static size_t ZSTD_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart)
|
||||
{
|
||||
const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd);
|
||||
size_t matchLength = ZSTD_count(ip, match, vEnd);
|
||||
size_t matchLength;
|
||||
const BYTE* vEnd = ip + (mEnd - match);
|
||||
if (vEnd > iEnd) vEnd = iEnd;
|
||||
matchLength = ZSTD_count(ip, match, vEnd);
|
||||
if (match + matchLength == mEnd)
|
||||
matchLength += ZSTD_count(ip+matchLength, iStart, iEnd);
|
||||
return matchLength;
|
||||
@@ -1110,47 +1025,43 @@ static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls)
|
||||
|
||||
|
||||
FORCE_INLINE
|
||||
void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
|
||||
void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc,
|
||||
const void* src, size_t srcSize,
|
||||
const U32 mls)
|
||||
{
|
||||
U32* const hashTable = cctx->hashTable;
|
||||
const U32 hBits = cctx->params.cParams.hashLog;
|
||||
seqStore_t* seqStorePtr = &(cctx->seqStore);
|
||||
const BYTE* const base = cctx->base;
|
||||
U32* const hashTable = zc->hashTable;
|
||||
const U32 hBits = zc->params.cParams.hashLog;
|
||||
seqStore_t* seqStorePtr = &(zc->seqStore);
|
||||
const BYTE* const base = zc->base;
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* anchor = istart;
|
||||
const U32 lowestIndex = cctx->dictLimit;
|
||||
const BYTE* const lowest = base + lowestIndex;
|
||||
const U32 lowIndex = zc->dictLimit;
|
||||
const BYTE* const lowest = base + lowIndex;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
size_t offset_1=cctx->rep[0], offset_2=cctx->rep[1];
|
||||
size_t offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE;
|
||||
|
||||
/* init */
|
||||
ZSTD_resetSeqStore(seqStorePtr);
|
||||
ip += (ip==lowest);
|
||||
{ U32 const maxRep = (U32)(ip-lowest);
|
||||
if (offset_1 > maxRep) offset_1 = 0;
|
||||
if (offset_2 > maxRep) offset_2 = 0;
|
||||
}
|
||||
if (ip < lowest+REPCODE_STARTVALUE) ip = lowest+REPCODE_STARTVALUE;
|
||||
|
||||
/* Main Search Loop */
|
||||
while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
|
||||
while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
|
||||
size_t mlCode;
|
||||
size_t offset;
|
||||
size_t const h = ZSTD_hashPtr(ip, hBits, mls);
|
||||
U32 const current = (U32)(ip-base);
|
||||
U32 const matchIndex = hashTable[h];
|
||||
const size_t h = ZSTD_hashPtr(ip, hBits, mls);
|
||||
const U32 matchIndex = hashTable[h];
|
||||
const BYTE* match = base + matchIndex;
|
||||
const U32 current = (U32)(ip-base);
|
||||
hashTable[h] = current; /* update hash table */
|
||||
|
||||
if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */
|
||||
if (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)) { /* note : by construction, offset_1 <= current */
|
||||
mlCode = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mlCode-MINMATCH);
|
||||
} else {
|
||||
if ( (matchIndex <= lowestIndex) ||
|
||||
if ( (matchIndex <= lowIndex) ||
|
||||
(MEM_read32(match) != MEM_read32(ip)) ) {
|
||||
ip += ((ip-anchor) >> g_searchStrength) + 1;
|
||||
continue;
|
||||
@@ -1174,8 +1085,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
|
||||
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base);
|
||||
/* check immediate repcode */
|
||||
while ( (ip <= ilimit)
|
||||
&& ( (offset_2>0)
|
||||
& (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
|
||||
&& (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) {
|
||||
/* store sequence */
|
||||
size_t const rlCode = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32;
|
||||
{ size_t const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */
|
||||
@@ -1186,10 +1096,6 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
|
||||
continue; /* faster when present ... (?) */
|
||||
} } }
|
||||
|
||||
/* save reps for next block */
|
||||
cctx->savedRep[0] = offset_1 ? (U32)offset_1 : (U32)(iend-base);
|
||||
cctx->savedRep[1] = offset_2 ? (U32)offset_2 : (U32)(iend-base);
|
||||
|
||||
/* Last Literals */
|
||||
{ size_t const lastLLSize = iend - anchor;
|
||||
memcpy(seqStorePtr->lit, anchor, lastLLSize);
|
||||
@@ -1229,20 +1135,22 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* anchor = istart;
|
||||
const U32 lowestIndex = ctx->lowLimit;
|
||||
const BYTE* const dictStart = dictBase + lowestIndex;
|
||||
const U32 lowLimit = ctx->lowLimit;
|
||||
const BYTE* const dictStart = dictBase + lowLimit;
|
||||
const U32 dictLimit = ctx->dictLimit;
|
||||
const BYTE* const lowPrefixPtr = base + dictLimit;
|
||||
const BYTE* const dictEnd = dictBase + dictLimit;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1];
|
||||
|
||||
U32 offset_2=REPCODE_STARTVALUE, offset_1=REPCODE_STARTVALUE;
|
||||
|
||||
|
||||
/* init */
|
||||
ZSTD_resetSeqStore(seqStorePtr);
|
||||
/* skip first position to avoid read overflow during repcode match check */
|
||||
hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base);
|
||||
ip++;
|
||||
hashTable[ZSTD_hashPtr(ip+0, hBits, mls)] = (U32)(ip-base+0);
|
||||
ip += REPCODE_STARTVALUE;
|
||||
|
||||
/* Main Search Loop */
|
||||
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
|
||||
@@ -1258,14 +1166,14 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
|
||||
U32 offset;
|
||||
hashTable[h] = current; /* update hash table */
|
||||
|
||||
if ( (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
if ( ((repIndex >= dictLimit) || (repIndex <= dictLimit-4))
|
||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
||||
const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend;
|
||||
mlCode = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mlCode-MINMATCH);
|
||||
} else {
|
||||
if ( (matchIndex < lowestIndex) ||
|
||||
if ( (matchIndex < lowLimit) ||
|
||||
(MEM_read32(match) != MEM_read32(ip)) ) {
|
||||
ip += ((ip-anchor) >> g_searchStrength) + 1;
|
||||
continue;
|
||||
@@ -1293,8 +1201,8 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
|
||||
U32 const current2 = (U32)(ip-base);
|
||||
U32 const repIndex2 = current2 - offset_2;
|
||||
const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2;
|
||||
if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
if ( ((repIndex2 <= dictLimit-4) || (repIndex2 >= dictLimit))
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend;
|
||||
size_t repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32;
|
||||
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
||||
@@ -1307,9 +1215,6 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
|
||||
break;
|
||||
} } }
|
||||
|
||||
/* save reps for next block */
|
||||
ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2;
|
||||
|
||||
/* Last Literals */
|
||||
{ size_t const lastLLSize = iend - anchor;
|
||||
memcpy(seqStorePtr->lit, anchor, lastLLSize);
|
||||
@@ -1493,7 +1398,7 @@ static size_t ZSTD_insertBtAndFindBestMatch (
|
||||
if (matchLength > bestLength) {
|
||||
if (matchLength > matchEndIdx - matchIndex)
|
||||
matchEndIdx = matchIndex + (U32)matchLength;
|
||||
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
|
||||
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) )
|
||||
bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
|
||||
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
|
||||
break; /* drop, to guarantee consistency (miss a little bit of compression) */
|
||||
@@ -1737,19 +1642,15 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
|
||||
typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit,
|
||||
size_t* offsetPtr,
|
||||
U32 maxNbAttempts, U32 matchLengthSearch);
|
||||
searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
|
||||
U32 rep[ZSTD_REP_INIT];
|
||||
searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
|
||||
|
||||
/* init */
|
||||
ip += (ip==base);
|
||||
U32 rep[ZSTD_REP_INIT];
|
||||
{ U32 i ; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; }
|
||||
|
||||
ctx->nextToUpdate3 = ctx->nextToUpdate;
|
||||
ZSTD_resetSeqStore(seqStorePtr);
|
||||
{ U32 i;
|
||||
U32 const maxRep = (U32)(ip-base);
|
||||
for (i=0; i<ZSTD_REP_INIT; i++) {
|
||||
rep[i]=ctx->rep[i];
|
||||
if (rep[i]>maxRep) rep[i]=0;
|
||||
} }
|
||||
if ((ip-base) < REPCODE_STARTVALUE) ip = base + REPCODE_STARTVALUE;
|
||||
|
||||
/* Match Loop */
|
||||
while (ip < ilimit) {
|
||||
@@ -1758,7 +1659,7 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
|
||||
const BYTE* start=ip+1;
|
||||
|
||||
/* check repCode */
|
||||
if ((rep[0]>0) & (MEM_read32(ip+1) == MEM_read32(ip+1 - rep[0]))) {
|
||||
if (MEM_read32(ip+1) == MEM_read32(ip+1 - rep[0])) {
|
||||
/* repcode : we take it */
|
||||
matchLength = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
|
||||
if (depth==0) goto _storeSequence;
|
||||
@@ -1780,17 +1681,17 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
|
||||
if (depth>=1)
|
||||
while (ip<ilimit) {
|
||||
ip ++;
|
||||
if ((offset) && ((rep[0]>0) & (MEM_read32(ip) == MEM_read32(ip - rep[0])))) {
|
||||
if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) {
|
||||
size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
|
||||
int const gain2 = (int)(mlRep * 3);
|
||||
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1);
|
||||
int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1);
|
||||
if ((mlRep >= EQUAL_READ32) && (gain2 > gain1))
|
||||
matchLength = mlRep, offset = 0, start = ip;
|
||||
}
|
||||
{ size_t offset2=99999999;
|
||||
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
|
||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
|
||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
|
||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
|
||||
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
||||
matchLength = ml2, offset = offset2, start = ip;
|
||||
continue; /* search a better one */
|
||||
@@ -1799,17 +1700,17 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
|
||||
/* let's find an even better one */
|
||||
if ((depth==2) && (ip<ilimit)) {
|
||||
ip ++;
|
||||
if ((offset) && ((rep[0]>0) & (MEM_read32(ip) == MEM_read32(ip - rep[0])))) {
|
||||
if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) {
|
||||
size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
|
||||
int const gain2 = (int)(ml2 * 4);
|
||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1);
|
||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1);
|
||||
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1))
|
||||
matchLength = ml2, offset = 0, start = ip;
|
||||
}
|
||||
{ size_t offset2=99999999;
|
||||
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
|
||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
|
||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
|
||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
|
||||
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
||||
matchLength = ml2, offset = offset2, start = ip;
|
||||
continue;
|
||||
@@ -1833,8 +1734,7 @@ _storeSequence:
|
||||
|
||||
/* check immediate repcode */
|
||||
while ( (ip <= ilimit)
|
||||
&& ((rep[1]>0)
|
||||
& (MEM_read32(ip) == MEM_read32(ip - rep[1])) )) {
|
||||
&& (MEM_read32(ip) == MEM_read32(ip - rep[1])) ) {
|
||||
/* store sequence */
|
||||
matchLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[1], iend) + EQUAL_READ32;
|
||||
offset = rep[1]; rep[1] = rep[0]; rep[0] = (U32)offset; /* swap repcodes */
|
||||
@@ -1844,13 +1744,6 @@ _storeSequence:
|
||||
continue; /* faster when present ... (?) */
|
||||
} }
|
||||
|
||||
/* Save reps for next block */
|
||||
{ int i;
|
||||
for (i=0; i<ZSTD_REP_NUM; i++) {
|
||||
if (!rep[i]) rep[i] = (U32)(iend - ctx->base); /* in case some zero are left */
|
||||
ctx->savedRep[i] = rep[i];
|
||||
} }
|
||||
|
||||
/* Last Literals */
|
||||
{ size_t const lastLLSize = iend - anchor;
|
||||
memcpy(seqStorePtr->lit, anchor, lastLLSize);
|
||||
@@ -1894,7 +1787,6 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
const BYTE* const base = ctx->base;
|
||||
const U32 dictLimit = ctx->dictLimit;
|
||||
const U32 lowestIndex = ctx->lowLimit;
|
||||
const BYTE* const prefixStart = base + dictLimit;
|
||||
const BYTE* const dictBase = ctx->dictBase;
|
||||
const BYTE* const dictEnd = dictBase + dictLimit;
|
||||
@@ -1910,11 +1802,11 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
||||
|
||||
/* init */
|
||||
U32 rep[ZSTD_REP_INIT];
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; }
|
||||
|
||||
ctx->nextToUpdate3 = ctx->nextToUpdate;
|
||||
ZSTD_resetSeqStore(seqStorePtr);
|
||||
ip += (ip == prefixStart);
|
||||
if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
|
||||
|
||||
/* Match Loop */
|
||||
while (ip < ilimit) {
|
||||
@@ -1924,10 +1816,11 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
||||
U32 current = (U32)(ip-base);
|
||||
|
||||
/* check repCode */
|
||||
{ const U32 repIndex = (U32)(current+1 - rep[0]);
|
||||
{
|
||||
const U32 repIndex = (U32)(current+1 - rep[0]);
|
||||
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
|
||||
if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
|
||||
if (MEM_read32(ip+1) == MEM_read32(repMatch)) {
|
||||
/* repcode detected we should take it */
|
||||
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
||||
@@ -1957,13 +1850,13 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
||||
const U32 repIndex = (U32)(current - rep[0]);
|
||||
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
|
||||
if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
|
||||
if (MEM_read32(ip) == MEM_read32(repMatch)) {
|
||||
/* repcode detected */
|
||||
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
||||
size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
|
||||
int const gain2 = (int)(repLength * 3);
|
||||
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1);
|
||||
int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1);
|
||||
if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
|
||||
matchLength = repLength, offset = 0, start = ip;
|
||||
} }
|
||||
@@ -1971,8 +1864,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
||||
/* search match, depth 1 */
|
||||
{ size_t offset2=99999999;
|
||||
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
|
||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
|
||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
|
||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
|
||||
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
||||
matchLength = ml2, offset = offset2, start = ip;
|
||||
continue; /* search a better one */
|
||||
@@ -1987,13 +1880,13 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
||||
const U32 repIndex = (U32)(current - rep[0]);
|
||||
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
|
||||
if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
|
||||
if (MEM_read32(ip) == MEM_read32(repMatch)) {
|
||||
/* repcode detected */
|
||||
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
||||
size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
|
||||
int gain2 = (int)(repLength * 4);
|
||||
int gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1);
|
||||
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1);
|
||||
if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
|
||||
matchLength = repLength, offset = 0, start = ip;
|
||||
} }
|
||||
@@ -2001,8 +1894,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
||||
/* search match, depth 2 */
|
||||
{ size_t offset2=99999999;
|
||||
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
|
||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
|
||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
|
||||
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
|
||||
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
|
||||
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
|
||||
matchLength = ml2, offset = offset2, start = ip;
|
||||
continue;
|
||||
@@ -2012,7 +1905,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
|
||||
|
||||
/* catch up */
|
||||
if (offset) {
|
||||
U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
|
||||
U32 matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE));
|
||||
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
|
||||
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
|
||||
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
|
||||
@@ -2031,7 +1924,7 @@ _storeSequence:
|
||||
const U32 repIndex = (U32)((ip-base) - rep[1]);
|
||||
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */
|
||||
if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
|
||||
if (MEM_read32(ip) == MEM_read32(repMatch)) {
|
||||
/* repcode detected we should take it */
|
||||
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
||||
@@ -2045,9 +1938,6 @@ _storeSequence:
|
||||
break;
|
||||
} }
|
||||
|
||||
/* Save reps for next block */
|
||||
ctx->savedRep[0] = rep[0]; ctx->savedRep[1] = rep[1]; ctx->savedRep[2] = rep[2];
|
||||
|
||||
/* Last Literals */
|
||||
{ size_t const lastLLSize = iend - anchor;
|
||||
memcpy(seqStorePtr->lit, anchor, lastLLSize);
|
||||
@@ -2120,37 +2010,35 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCa
|
||||
|
||||
|
||||
|
||||
static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
size_t blockSize = cctx->blockSize;
|
||||
size_t blockSize = zc->blockSize;
|
||||
size_t remaining = srcSize;
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* op = ostart;
|
||||
const U32 maxDist = 1 << cctx->params.cParams.windowLog;
|
||||
ZSTD_stats_t* stats = &cctx->seqStore.stats;
|
||||
ZSTD_statsInit(stats);
|
||||
const U32 maxDist = 1 << zc->params.cParams.windowLog;
|
||||
ZSTD_stats_t* stats = &zc->seqStore.stats;
|
||||
|
||||
if (cctx->params.fParams.checksumFlag)
|
||||
XXH64_update(&cctx->xxhState, src, srcSize);
|
||||
ZSTD_statsInit(stats);
|
||||
|
||||
while (remaining) {
|
||||
size_t cSize;
|
||||
ZSTD_statsResetFreqs(stats); /* debug only */
|
||||
ZSTD_statsResetFreqs(stats);
|
||||
|
||||
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
|
||||
if (remaining < blockSize) blockSize = remaining;
|
||||
|
||||
if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
|
||||
if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) {
|
||||
/* enforce maxDist */
|
||||
U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
|
||||
if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit;
|
||||
if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit;
|
||||
U32 const newLowLimit = (U32)(ip+blockSize - zc->base) - maxDist;
|
||||
if (zc->lowLimit < newLowLimit) zc->lowLimit = newLowLimit;
|
||||
if (zc->dictLimit < zc->lowLimit) zc->dictLimit = zc->lowLimit;
|
||||
}
|
||||
|
||||
cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
|
||||
cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
|
||||
if (ZSTD_isError(cSize)) return cSize;
|
||||
|
||||
if (cSize == 0) { /* block is not compressible */
|
||||
@@ -2170,47 +2058,33 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
op += cSize;
|
||||
}
|
||||
|
||||
ZSTD_statsPrint(stats, cctx->params.cParams.searchLength);
|
||||
ZSTD_statsPrint(stats, zc->params.cParams.searchLength);
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
|
||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
||||
{ BYTE* const op = (BYTE*)dst;
|
||||
U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
|
||||
U32 const checksumFlag = params.fParams.checksumFlag>0;
|
||||
U32 const windowSize = 1U << params.cParams.windowLog;
|
||||
U32 const directModeFlag = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1));
|
||||
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||
U32 const fcsCode = params.fParams.contentSizeFlag ?
|
||||
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */
|
||||
U32 const fcsId = params.fParams.contentSizeFlag ?
|
||||
(pledgedSrcSize>0) + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) : /* 0-3 */
|
||||
0;
|
||||
BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (directModeFlag<<5) + (fcsCode<<6) );
|
||||
size_t pos;
|
||||
|
||||
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
|
||||
BYTE const fdescriptor = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */
|
||||
| (fcsId << 6) );
|
||||
size_t const hSize = ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId];
|
||||
if (hSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||
|
||||
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
|
||||
op[4] = frameHeaderDecriptionByte; pos=5;
|
||||
if (!directModeFlag) op[pos++] = windowLogByte;
|
||||
switch(dictIDSizeCode)
|
||||
op[4] = fdescriptor;
|
||||
switch(fcsId)
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : break;
|
||||
case 1 : op[pos] = (BYTE)(dictID); pos++; break;
|
||||
case 2 : MEM_writeLE16(op+pos, (U16)(dictID)); pos+=2; break;
|
||||
case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break;
|
||||
case 1 : op[5] = (BYTE)(pledgedSrcSize); break;
|
||||
case 2 : MEM_writeLE16(op+5, (U16)(pledgedSrcSize-256)); break;
|
||||
case 3 : MEM_writeLE64(op+5, (U64)(pledgedSrcSize)); break;
|
||||
}
|
||||
switch(fcsCode)
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : if (directModeFlag) op[pos++] = (BYTE)(pledgedSrcSize); break;
|
||||
case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break;
|
||||
case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break;
|
||||
case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break;
|
||||
}
|
||||
return pos;
|
||||
return hSize;
|
||||
}
|
||||
|
||||
|
||||
@@ -2224,7 +2098,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
|
||||
|
||||
if (zc->stage==0) return ERROR(stage_wrong);
|
||||
if (frame && (zc->stage==1)) { /* copy saved header */
|
||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, zc->frameContentSize, zc->dictID);
|
||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, srcSize);
|
||||
if (ZSTD_isError(fhSize)) return fhSize;
|
||||
dstCapacity -= fhSize;
|
||||
dst = (char*)dst + fhSize;
|
||||
@@ -2335,74 +2209,60 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
|
||||
/* Dictionary format :
|
||||
Magic == ZSTD_DICT_MAGIC (4 bytes)
|
||||
HUF_writeCTable(256)
|
||||
FSE_writeNCount(ml)
|
||||
FSE_writeNCount(off)
|
||||
FSE_writeNCount(ll)
|
||||
RepOffsets
|
||||
Dictionary content
|
||||
*/
|
||||
/*! ZSTD_loadDictEntropyStats() :
|
||||
@return : size read from dictionary
|
||||
note : magic number supposed already checked */
|
||||
static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
|
||||
@return : size read from dictionary */
|
||||
static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
|
||||
{
|
||||
const BYTE* dictPtr = (const BYTE*)dict;
|
||||
const BYTE* const dictEnd = dictPtr + dictSize;
|
||||
/* note : magic number already checked */
|
||||
size_t offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize, errorCode;
|
||||
short offcodeNCount[MaxOff+1];
|
||||
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
|
||||
short matchlengthNCount[MaxML+1];
|
||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||
short litlengthNCount[MaxLL+1];
|
||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||
|
||||
{ size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dict, dictSize);
|
||||
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
dictPtr += hufHeaderSize;
|
||||
}
|
||||
size_t const hufHeaderSize = HUF_readCTable(zc->hufTable, 255, dict, dictSize);
|
||||
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
zc->flagStaticTables = 1;
|
||||
dict = (const char*)dict + hufHeaderSize;
|
||||
dictSize -= hufHeaderSize;
|
||||
|
||||
{ short offcodeNCount[MaxOff+1];
|
||||
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
|
||||
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
|
||||
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += offcodeHeaderSize;
|
||||
}
|
||||
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
||||
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
errorCode = FSE_buildCTable(zc->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + offcodeHeaderSize;
|
||||
dictSize -= offcodeHeaderSize;
|
||||
|
||||
{ short matchlengthNCount[MaxML+1];
|
||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
|
||||
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += matchlengthHeaderSize;
|
||||
}
|
||||
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
||||
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
errorCode = FSE_buildCTable(zc->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + matchlengthHeaderSize;
|
||||
dictSize -= matchlengthHeaderSize;
|
||||
|
||||
{ short litlengthNCount[MaxLL+1];
|
||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
|
||||
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += litlengthHeaderSize;
|
||||
}
|
||||
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
||||
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
errorCode = FSE_buildCTable(zc->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
|
||||
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
|
||||
cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
dictPtr += 12;
|
||||
|
||||
cctx->flagStaticTables = 1;
|
||||
return dictPtr - (const BYTE*)dict;
|
||||
return hufHeaderSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
|
||||
}
|
||||
|
||||
/** ZSTD_compress_insertDictionary() :
|
||||
* @return : 0, or an error code */
|
||||
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
|
||||
{
|
||||
if ((dict==NULL) || (dictSize<=8)) return 0;
|
||||
if ((dict==NULL) || (dictSize<=4)) return 0;
|
||||
|
||||
/* default : dict is pure content */
|
||||
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize);
|
||||
zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4);
|
||||
|
||||
/* known magic number : dict is parsed for entropy stats and content */
|
||||
{ size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8) + 8;
|
||||
{ size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4 /* skip magic */, dictSize-4) + 4;
|
||||
if (ZSTD_isError(eSize)) return eSize;
|
||||
return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
|
||||
}
|
||||
@@ -2415,8 +2275,11 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
||||
{
|
||||
size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1);
|
||||
if (ZSTD_isError(resetError)) return resetError;
|
||||
{ U32 const hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN);
|
||||
zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0; }
|
||||
|
||||
{ size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, 1);
|
||||
if (ZSTD_isError(resetError)) return resetError; }
|
||||
|
||||
return ZSTD_compress_insertDictionary(zc, dict, dictSize);
|
||||
}
|
||||
@@ -2424,7 +2287,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
|
||||
|
||||
/*! ZSTD_compressBegin_advanced() :
|
||||
* @return : 0, or an error code */
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
||||
{
|
||||
@@ -2432,17 +2295,18 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
|
||||
{ size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize);
|
||||
if (ZSTD_isError(errorCode)) return errorCode; }
|
||||
|
||||
return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize);
|
||||
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, pledgedSrcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
|
||||
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", cctx->base, compressionLevel);
|
||||
return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0);
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel);
|
||||
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0);
|
||||
}
|
||||
|
||||
|
||||
@@ -2456,44 +2320,35 @@ size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
|
||||
/*! ZSTD_compressEnd() :
|
||||
* Write frame epilogue.
|
||||
* @return : nb of bytes written into dst (or an error code) */
|
||||
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
|
||||
size_t ZSTD_compressEnd(ZSTD_CCtx* zc, void* dst, size_t dstCapacity)
|
||||
{
|
||||
BYTE* op = (BYTE*)dst;
|
||||
size_t fhSize = 0;
|
||||
|
||||
/* not even init ! */
|
||||
if (cctx->stage==0) return ERROR(stage_wrong);
|
||||
if (zc->stage==0) return ERROR(stage_wrong);
|
||||
|
||||
/* special case : empty frame */
|
||||
if (cctx->stage==1) {
|
||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0);
|
||||
if (zc->stage==1) {
|
||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0);
|
||||
if (ZSTD_isError(fhSize)) return fhSize;
|
||||
dstCapacity -= fhSize;
|
||||
op += fhSize;
|
||||
cctx->stage = 2;
|
||||
zc->stage = 2;
|
||||
}
|
||||
|
||||
/* frame epilogue */
|
||||
if (dstCapacity < 3) return ERROR(dstSize_tooSmall);
|
||||
{ U32 const checksum = cctx->params.fParams.checksumFlag ?
|
||||
(U32)((XXH64_digest(&cctx->xxhState) >> 11) & ((1<<22)-1)) :
|
||||
0;
|
||||
op[0] = (BYTE)((bt_end<<6) + (checksum>>16));
|
||||
op[1] = (BYTE)(checksum>>8);
|
||||
op[2] = (BYTE)checksum;
|
||||
}
|
||||
op[0] = (BYTE)(bt_end << 6);
|
||||
op[1] = 0;
|
||||
op[2] = 0;
|
||||
|
||||
cctx->stage = 0; /* return to "created but not init" status */
|
||||
zc->stage = 0; /* return to "created by not init" status */
|
||||
return 3+fhSize;
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTD_compress_usingPreparedCCtx() :
|
||||
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
|
||||
* It avoids reloading the dictionary each time.
|
||||
* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
|
||||
* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
|
||||
static size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
|
||||
size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
@@ -2551,10 +2406,10 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
||||
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
ZSTD_adjustCParams(¶ms.cParams, srcSize, dictSize);
|
||||
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
|
||||
}
|
||||
|
||||
@@ -2569,91 +2424,15 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
|
||||
size_t result;
|
||||
ZSTD_CCtx ctxBody;
|
||||
memset(&ctxBody, 0, sizeof(ctxBody));
|
||||
memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
|
||||
result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
|
||||
ctxBody.customMem.customFree(ctxBody.customMem.opaque, ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */
|
||||
free(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
/* ===== Dictionary API ===== */
|
||||
|
||||
struct ZSTD_CDict_s {
|
||||
void* dictContent;
|
||||
size_t dictContentSize;
|
||||
ZSTD_CCtx* refContext;
|
||||
}; /* typedef'd tp ZSTD_CDict within zstd.h */
|
||||
|
||||
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem)
|
||||
{
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
{ ZSTD_CDict* const cdict = (ZSTD_CDict*) customMem.customAlloc(customMem.opaque, sizeof(*cdict));
|
||||
void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize);
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
|
||||
|
||||
if (!dictContent || !cdict || !cctx) {
|
||||
customMem.customFree(customMem.opaque, dictContent);
|
||||
customMem.customFree(customMem.opaque, cdict);
|
||||
customMem.customFree(customMem.opaque, cctx);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memcpy(dictContent, dict, dictSize);
|
||||
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
|
||||
if (ZSTD_isError(errorCode)) {
|
||||
customMem.customFree(customMem.opaque, dictContent);
|
||||
customMem.customFree(customMem.opaque, cdict);
|
||||
customMem.customFree(customMem.opaque, cctx);
|
||||
return NULL;
|
||||
} }
|
||||
|
||||
cdict->dictContent = dictContent;
|
||||
cdict->dictContentSize = dictSize;
|
||||
cdict->refContext = cctx;
|
||||
return cdict;
|
||||
}
|
||||
}
|
||||
|
||||
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_customMem const allocator = { NULL, NULL, NULL };
|
||||
ZSTD_parameters params;
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
return ZSTD_createCDict_advanced(dict, dictSize, params, allocator);
|
||||
}
|
||||
|
||||
size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
|
||||
{
|
||||
ZSTD_freeFunction const cFree = cdict->refContext->customMem.customFree;
|
||||
void* const opaque = cdict->refContext->customMem.opaque;
|
||||
ZSTD_freeCCtx(cdict->refContext);
|
||||
cFree(opaque, cdict->dictContent);
|
||||
cFree(opaque, cdict);
|
||||
return 0;
|
||||
}
|
||||
|
||||
ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_CDict* cdict)
|
||||
{
|
||||
return ZSTD_compress_usingPreparedCCtx(cctx, cdict->refContext,
|
||||
dst, dstCapacity,
|
||||
src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*-===== Pre-defined compression levels =====-*/
|
||||
|
||||
#define ZSTD_DEFAULT_CLEVEL 1
|
||||
#define ZSTD_DEFAULT_CLEVEL 5
|
||||
#define ZSTD_MAX_CLEVEL 22
|
||||
unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
|
||||
|
||||
@@ -2764,16 +2543,17 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
|
||||
},
|
||||
};
|
||||
|
||||
/*! ZSTD_getCParams() :
|
||||
* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
|
||||
* Size values are optional, provide 0 if not known or unused */
|
||||
/*! ZSTD_getParams() :
|
||||
* @return ZSTD_parameters structure for a selected compression level and srcSize.
|
||||
* `srcSize` value is optional, select 0 if not known */
|
||||
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize)
|
||||
{
|
||||
ZSTD_compressionParameters cp;
|
||||
size_t const addedSize = srcSize ? 0 : 500;
|
||||
U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1;
|
||||
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
|
||||
if (compressionLevel <= 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */
|
||||
if (compressionLevel < 0) compressionLevel = ZSTD_DEFAULT_CLEVEL;
|
||||
if (compressionLevel==0) compressionLevel = 1;
|
||||
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
|
||||
cp = ZSTD_defaultCParameters[tableID][compressionLevel];
|
||||
if (MEM_32bits()) { /* auto-correction, for 32-bits mode */
|
||||
@@ -2781,6 +2561,5 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si
|
||||
if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX;
|
||||
if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX;
|
||||
}
|
||||
cp = ZSTD_adjustCParams(cp, srcSize, dictSize);
|
||||
return cp;
|
||||
}
|
||||
|
||||
+133
-144
@@ -41,10 +41,10 @@
|
||||
***************************************/
|
||||
FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr)
|
||||
{
|
||||
ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum+1);
|
||||
ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum+1);
|
||||
ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum+1);
|
||||
ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum+1);
|
||||
ssPtr->log2matchLengthSum = ZSTD_highbit(ssPtr->matchLengthSum+1);
|
||||
ssPtr->log2litLengthSum = ZSTD_highbit(ssPtr->litLengthSum+1);
|
||||
ssPtr->log2litSum = ZSTD_highbit(ssPtr->litSum+1);
|
||||
ssPtr->log2offCodeSum = ZSTD_highbit(ssPtr->offCodeSum+1);
|
||||
ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum));
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
|
||||
U32 price, u;
|
||||
|
||||
if (litLength == 0)
|
||||
return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1);
|
||||
return ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[0]+1);
|
||||
|
||||
/* literals */
|
||||
if (ssPtr->cachedLiterals == literals) {
|
||||
@@ -114,13 +114,13 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
|
||||
const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
|
||||
price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
|
||||
for (u=0; u < additional; u++)
|
||||
price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]]+1);
|
||||
price -= ZSTD_highbit(ssPtr->litFreq[literals2[u]]+1);
|
||||
ssPtr->cachedPrice = price;
|
||||
ssPtr->cachedLitLength = litLength;
|
||||
} else {
|
||||
price = litLength * ssPtr->log2litSum;
|
||||
for (u=0; u < litLength; u++)
|
||||
price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]]+1);
|
||||
price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1);
|
||||
|
||||
if (litLength >= 12) {
|
||||
ssPtr->cachedLiterals = literals;
|
||||
@@ -139,8 +139,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||
const BYTE LL_deltaCode = 19;
|
||||
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||
price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode]+1);
|
||||
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||
price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[llCode]+1);
|
||||
}
|
||||
|
||||
return price;
|
||||
@@ -150,8 +150,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
|
||||
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
|
||||
{
|
||||
/* offset */
|
||||
BYTE offCode = (BYTE)ZSTD_highbit32(offset+1);
|
||||
U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1);
|
||||
BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
|
||||
U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1);
|
||||
|
||||
/* match Length */
|
||||
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
@@ -163,8 +163,8 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
||||
const BYTE ML_deltaCode = 36;
|
||||
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
||||
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode]+1);
|
||||
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
||||
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1);
|
||||
}
|
||||
|
||||
return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
|
||||
@@ -190,13 +190,13 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||
const BYTE LL_deltaCode = 19;
|
||||
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||
seqStorePtr->litLengthFreq[llCode]++;
|
||||
seqStorePtr->litLengthSum++;
|
||||
}
|
||||
|
||||
/* match offset */
|
||||
{ BYTE offCode = (BYTE)ZSTD_highbit32(offset+1);
|
||||
{ BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
|
||||
seqStorePtr->offCodeSum++;
|
||||
seqStorePtr->offCodeFreq[offCode]++;
|
||||
}
|
||||
@@ -211,7 +211,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
||||
const BYTE ML_deltaCode = 36;
|
||||
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
||||
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
||||
seqStorePtr->matchLengthFreq[mlCode]++;
|
||||
seqStorePtr->matchLengthSum++;
|
||||
}
|
||||
@@ -461,14 +461,15 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
ZSTD_optimal_t* opt = seqStorePtr->priceTable;
|
||||
ZSTD_match_t* matches = seqStorePtr->matchTable;
|
||||
const BYTE* inr;
|
||||
U32 offset, rep[ZSTD_REP_INIT];
|
||||
|
||||
/* init */
|
||||
U32 offset, rep[ZSTD_REP_INIT];
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; }
|
||||
|
||||
ctx->nextToUpdate3 = ctx->nextToUpdate;
|
||||
ZSTD_resetSeqStore(seqStorePtr);
|
||||
ZSTD_rescaleFreqs(seqStorePtr);
|
||||
ip += (ip==prefixStart);
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }
|
||||
if ((ip-prefixStart) < REPCODE_STARTVALUE) ip = prefixStart + REPCODE_STARTVALUE;
|
||||
|
||||
ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_GENERIC srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len);
|
||||
|
||||
@@ -481,24 +482,23 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
litlen = (U32)(ip - anchor);
|
||||
|
||||
/* check repCode */
|
||||
{ U32 i;
|
||||
for (i=0; i<ZSTD_REP_NUM; i++) {
|
||||
if ((rep[i]<(U32)(ip-prefixStart))
|
||||
&& (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - rep[i], minMatch))) {
|
||||
mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-rep[i], iend) + minMatch;
|
||||
ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen);
|
||||
if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
|
||||
best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
|
||||
goto _storeSequence;
|
||||
}
|
||||
best_off = (i<=1 && ip == anchor) ? 1-i : i;
|
||||
do {
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
if (mlen > last_pos || price < opt[mlen].price)
|
||||
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
|
||||
mlen--;
|
||||
} while (mlen >= minMatch);
|
||||
} } }
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++)
|
||||
if (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - rep[i], minMatch)) {
|
||||
/* repcode : we take it */
|
||||
mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-rep[i], iend) + minMatch;
|
||||
ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen);
|
||||
if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
|
||||
best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
|
||||
goto _storeSequence;
|
||||
}
|
||||
best_off = (i<=1 && ip == anchor) ? 1-i : i;
|
||||
do {
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
if (mlen > last_pos || price < opt[mlen].price)
|
||||
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
|
||||
mlen--;
|
||||
} while (mlen >= minMatch);
|
||||
} }
|
||||
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch);
|
||||
|
||||
@@ -516,15 +516,15 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
/* set prices using matches at position = 0 */
|
||||
best_mlen = (last_pos) ? last_pos : minMatch;
|
||||
for (u = 0; u < match_num; u++) {
|
||||
mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
|
||||
best_mlen = matches[u].len;
|
||||
ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos);
|
||||
while (mlen <= best_mlen) {
|
||||
mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
|
||||
best_mlen = matches[u].len;
|
||||
ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos);
|
||||
while (mlen <= best_mlen) {
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH);
|
||||
if (mlen > last_pos || price < opt[mlen].price)
|
||||
SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */
|
||||
mlen++;
|
||||
} }
|
||||
} }
|
||||
|
||||
if (last_pos < minMatch) { ip++; continue; }
|
||||
|
||||
@@ -572,40 +572,38 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
|
||||
|
||||
best_mlen = minMatch;
|
||||
{ U32 i;
|
||||
for (i=0; i<ZSTD_REP_NUM; i++) {
|
||||
if ((opt[cur].rep[i]<(U32)(inr-prefixStart))
|
||||
&& (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - opt[cur].rep[i], minMatch))) { /* check rep */
|
||||
mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - opt[cur].rep[i], iend) + minMatch;
|
||||
ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off);
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++)
|
||||
if (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - opt[cur].rep[i], minMatch)) { /* check rep */
|
||||
mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - opt[cur].rep[i], iend) + minMatch;
|
||||
ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off);
|
||||
|
||||
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
|
||||
ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos);
|
||||
best_mlen = mlen; best_off = i; last_pos = cur + 1;
|
||||
goto _storeSequence;
|
||||
}
|
||||
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
|
||||
ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos);
|
||||
best_mlen = mlen; best_off = i; last_pos = cur + 1;
|
||||
goto _storeSequence;
|
||||
}
|
||||
|
||||
best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i;
|
||||
if (opt[cur].mlen == 1) {
|
||||
litlen = opt[cur].litlen;
|
||||
if (cur > litlen) {
|
||||
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
|
||||
} else
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
} else {
|
||||
litlen = 0;
|
||||
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
|
||||
}
|
||||
best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i;
|
||||
if (opt[cur].mlen == 1) {
|
||||
litlen = opt[cur].litlen;
|
||||
if (cur > litlen) {
|
||||
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
|
||||
} else
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
} else {
|
||||
litlen = 0;
|
||||
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
|
||||
}
|
||||
|
||||
if (mlen > best_mlen) best_mlen = mlen;
|
||||
ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen);
|
||||
if (mlen > best_mlen) best_mlen = mlen;
|
||||
ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen);
|
||||
|
||||
do {
|
||||
if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
|
||||
SET_PRICE(cur + mlen, mlen, i, litlen, price);
|
||||
mlen--;
|
||||
} while (mlen >= minMatch);
|
||||
} } }
|
||||
do {
|
||||
if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
|
||||
SET_PRICE(cur + mlen, mlen, i, litlen, price);
|
||||
mlen--;
|
||||
} while (mlen >= minMatch);
|
||||
} }
|
||||
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen);
|
||||
ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num);
|
||||
@@ -714,11 +712,8 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
anchor = ip = ip + mlen;
|
||||
} } /* for (cur=0; cur < last_pos; ) */
|
||||
|
||||
/* Save reps for next block */
|
||||
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; }
|
||||
|
||||
/* Last Literals */
|
||||
{ size_t const lastLLSize = iend - anchor;
|
||||
{ /* Last Literals */
|
||||
size_t lastLLSize = iend - anchor;
|
||||
ZSTD_LOG_ENCODE("%d: lastLLSize literals=%u\n", (int)(ip-base), (U32)lastLLSize);
|
||||
memcpy(seqStorePtr->lit, anchor, lastLLSize);
|
||||
seqStorePtr->lit += lastLLSize;
|
||||
@@ -737,7 +732,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
const BYTE* const base = ctx->base;
|
||||
const U32 lowestIndex = ctx->lowLimit;
|
||||
const U32 dictLimit = ctx->dictLimit;
|
||||
const BYTE* const prefixStart = base + dictLimit;
|
||||
const BYTE* const dictBase = ctx->dictBase;
|
||||
@@ -754,12 +748,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
|
||||
/* init */
|
||||
U32 offset, rep[ZSTD_REP_INIT];
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; }
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; }
|
||||
|
||||
ctx->nextToUpdate3 = ctx->nextToUpdate;
|
||||
ZSTD_resetSeqStore(seqStorePtr);
|
||||
ZSTD_rescaleFreqs(seqStorePtr);
|
||||
ip += (ip==prefixStart);
|
||||
if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
|
||||
|
||||
ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_EXTDICT srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len);
|
||||
|
||||
@@ -774,32 +768,31 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
opt[0].litlen = (U32)(ip - anchor);
|
||||
|
||||
/* check repCode */
|
||||
{ U32 i;
|
||||
for (i=0; i<ZSTD_REP_NUM; i++) {
|
||||
const U32 repIndex = (U32)(current - rep[i]);
|
||||
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
if ( (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */
|
||||
&& (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
|
||||
/* repcode detected we should take it */
|
||||
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
||||
mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) {
|
||||
const U32 repIndex = (U32)(current - rep[i]);
|
||||
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
|
||||
&& (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
|
||||
/* repcode detected we should take it */
|
||||
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
||||
mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;
|
||||
|
||||
ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen);
|
||||
if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
|
||||
best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
|
||||
goto _storeSequence;
|
||||
}
|
||||
ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen);
|
||||
if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
|
||||
best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
|
||||
goto _storeSequence;
|
||||
}
|
||||
|
||||
best_off = (i<=1 && ip == anchor) ? 1-i : i;
|
||||
litlen = opt[0].litlen;
|
||||
do {
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
if (mlen > last_pos || price < opt[mlen].price)
|
||||
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
|
||||
mlen--;
|
||||
} while (mlen >= minMatch);
|
||||
} } }
|
||||
best_off = (i<=1 && ip == anchor) ? 1-i : i;
|
||||
litlen = opt[0].litlen;
|
||||
do {
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
if (mlen > last_pos || price < opt[mlen].price)
|
||||
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
|
||||
mlen--;
|
||||
} while (mlen >= minMatch);
|
||||
} } }
|
||||
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */
|
||||
|
||||
@@ -876,45 +869,44 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
|
||||
best_mlen = 0;
|
||||
|
||||
{ U32 i;
|
||||
for (i=0; i<ZSTD_REP_NUM; i++) {
|
||||
const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]);
|
||||
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
if ( (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */
|
||||
&& (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
|
||||
/* repcode detected */
|
||||
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
||||
mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;
|
||||
ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off);
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) {
|
||||
const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]);
|
||||
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
||||
const BYTE* const repMatch = repBase + repIndex;
|
||||
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
|
||||
&& (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) {
|
||||
/* repcode detected */
|
||||
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
||||
mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;
|
||||
ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off);
|
||||
|
||||
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
|
||||
ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos);
|
||||
best_mlen = mlen; best_off = i; last_pos = cur + 1;
|
||||
goto _storeSequence;
|
||||
}
|
||||
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
|
||||
ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos);
|
||||
best_mlen = mlen; best_off = i; last_pos = cur + 1;
|
||||
goto _storeSequence;
|
||||
}
|
||||
|
||||
best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i;
|
||||
if (opt[cur].mlen == 1) {
|
||||
litlen = opt[cur].litlen;
|
||||
if (cur > litlen) {
|
||||
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
|
||||
} else
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
} else {
|
||||
litlen = 0;
|
||||
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
|
||||
}
|
||||
best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i;
|
||||
if (opt[cur].mlen == 1) {
|
||||
litlen = opt[cur].litlen;
|
||||
if (cur > litlen) {
|
||||
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
|
||||
} else
|
||||
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
|
||||
} else {
|
||||
litlen = 0;
|
||||
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
|
||||
}
|
||||
|
||||
best_mlen = mlen;
|
||||
ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen);
|
||||
best_mlen = mlen;
|
||||
ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen);
|
||||
|
||||
do {
|
||||
if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
|
||||
SET_PRICE(cur + mlen, mlen, i, litlen, price);
|
||||
mlen--;
|
||||
} while (mlen >= minMatch);
|
||||
} } }
|
||||
do {
|
||||
if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
|
||||
SET_PRICE(cur + mlen, mlen, i, litlen, price);
|
||||
mlen--;
|
||||
} while (mlen >= minMatch);
|
||||
} } }
|
||||
|
||||
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
|
||||
ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num);
|
||||
@@ -1031,11 +1023,8 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
anchor = ip = ip + mlen;
|
||||
} } /* for (cur=0; cur < last_pos; ) */
|
||||
|
||||
/* Save reps for next block */
|
||||
ctx->savedRep[0] = rep[0]; ctx->savedRep[1] = rep[1]; ctx->savedRep[2] = rep[2];
|
||||
|
||||
/* Last Literals */
|
||||
{ size_t lastLLSize = iend - anchor;
|
||||
{ /* Last Literals */
|
||||
size_t lastLLSize = iend - anchor;
|
||||
ZSTD_LOG_ENCODE("%d: lastLLSize literals=%u\n", (int)(ip-base), (U32)(lastLLSize));
|
||||
memcpy(seqStorePtr->lit, anchor, lastLLSize);
|
||||
seqStorePtr->lit += lastLLSize;
|
||||
|
||||
+175
-311
@@ -59,11 +59,13 @@
|
||||
/* **************************************************************
|
||||
* Includes
|
||||
****************************************************************/
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <string.h> /* memcpy, memset */
|
||||
#include <stdio.h> /* printf (debug) */
|
||||
#include "huf_static.h"
|
||||
#include "bitstream.h"
|
||||
#include "fse.h" /* header compression */
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
@@ -72,70 +74,63 @@
|
||||
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
|
||||
|
||||
/*-***************************/
|
||||
/* generic DTableDesc */
|
||||
/*-***************************/
|
||||
|
||||
typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;
|
||||
/* *******************************************************
|
||||
* HUF : Huffman block decompression
|
||||
*********************************************************/
|
||||
typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */
|
||||
|
||||
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */
|
||||
|
||||
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
|
||||
|
||||
static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
|
||||
{
|
||||
DTableDesc dtd;
|
||||
memcpy(&dtd, table, sizeof(dtd));
|
||||
return dtd;
|
||||
}
|
||||
|
||||
|
||||
/*-***************************/
|
||||
/* single-symbol decoding */
|
||||
/*-***************************/
|
||||
|
||||
typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */
|
||||
|
||||
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
|
||||
size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
|
||||
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
|
||||
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
|
||||
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
|
||||
U32 tableLog = 0;
|
||||
U32 nbSymbols = 0;
|
||||
size_t iSize;
|
||||
U32 nbSymbols = 0;
|
||||
U32 n;
|
||||
U32 nextRankStart;
|
||||
void* const dtPtr = DTable + 1;
|
||||
HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
|
||||
|
||||
HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
|
||||
HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
|
||||
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
||||
iSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
||||
if (HUF_isError(iSize)) return iSize;
|
||||
|
||||
/* Table header */
|
||||
{ DTableDesc dtd = HUF_getDTableDesc(DTable);
|
||||
if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */
|
||||
dtd.tableType = 0;
|
||||
dtd.tableLog = (BYTE)tableLog;
|
||||
memcpy(DTable, &dtd, sizeof(dtd));
|
||||
}
|
||||
/* check result */
|
||||
if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge); /* DTable is too small */
|
||||
DTable[0] = (U16)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */
|
||||
|
||||
/* Prepare ranks */
|
||||
{ U32 n, nextRankStart = 0;
|
||||
for (n=1; n<tableLog+1; n++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += (rankVal[n] << (n-1));
|
||||
rankVal[n] = current;
|
||||
} }
|
||||
nextRankStart = 0;
|
||||
for (n=1; n<tableLog+1; n++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += (rankVal[n] << (n-1));
|
||||
rankVal[n] = current;
|
||||
}
|
||||
|
||||
/* fill DTable */
|
||||
{ U32 n;
|
||||
for (n=0; n<nbSymbols; n++) {
|
||||
U32 const w = huffWeight[n];
|
||||
U32 const length = (1 << w) >> 1;
|
||||
U32 i;
|
||||
HUF_DEltX2 D;
|
||||
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
|
||||
for (i = rankVal[w]; i < rankVal[w] + length; i++)
|
||||
dt[i] = D;
|
||||
rankVal[w] += length;
|
||||
} }
|
||||
for (n=0; n<nbSymbols; n++) {
|
||||
const U32 w = huffWeight[n];
|
||||
const U32 length = (1 << w) >> 1;
|
||||
U32 i;
|
||||
HUF_DEltX2 D;
|
||||
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
|
||||
for (i = rankVal[w]; i < rankVal[w] + length; i++)
|
||||
dt[i] = D;
|
||||
rankVal[w] += length;
|
||||
}
|
||||
|
||||
return iSize;
|
||||
}
|
||||
@@ -143,8 +138,8 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
|
||||
|
||||
static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog)
|
||||
{
|
||||
size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
|
||||
BYTE const c = dt[val].byte;
|
||||
const size_t val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
|
||||
const BYTE c = dt[val].byte;
|
||||
BIT_skipBits(Dstream, dt[val].nbBits);
|
||||
return c;
|
||||
}
|
||||
@@ -153,7 +148,7 @@ static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, con
|
||||
*ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog)
|
||||
|
||||
#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
|
||||
if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
|
||||
if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \
|
||||
HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
|
||||
|
||||
#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
|
||||
@@ -183,18 +178,17 @@ static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, B
|
||||
return pEnd-pStart;
|
||||
}
|
||||
|
||||
static size_t HUF_decompress1X2_usingDTable_internal(
|
||||
size_t HUF_decompress1X2_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUF_DTable* DTable)
|
||||
const U16* DTable)
|
||||
{
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = op + dstSize;
|
||||
const void* dtPtr = DTable + 1;
|
||||
const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
|
||||
const U32 dtLog = DTable[0];
|
||||
const void* dtPtr = DTable;
|
||||
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr)+1;
|
||||
BIT_DStream_t bitD;
|
||||
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
||||
U32 const dtLog = dtd.tableLog;
|
||||
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
@@ -207,39 +201,25 @@ static size_t HUF_decompress1X2_usingDTable_internal(
|
||||
return dstSize;
|
||||
}
|
||||
|
||||
size_t HUF_decompress1X2_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUF_DTable* DTable)
|
||||
{
|
||||
DTableDesc dtd = HUF_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 0) return ERROR(GENERIC);
|
||||
return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const hSize = HUF_readDTableX2 (DCtx, cSrc, cSrcSize);
|
||||
if (HUF_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
|
||||
}
|
||||
|
||||
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
|
||||
return HUF_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_MAX_TABLELOG);
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += errorCode;
|
||||
cSrcSize -= errorCode;
|
||||
|
||||
return HUF_decompress1X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
static size_t HUF_decompress4X2_usingDTable_internal(
|
||||
size_t HUF_decompress4X2_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUF_DTable* DTable)
|
||||
const U16* DTable)
|
||||
{
|
||||
/* Check */
|
||||
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
|
||||
@@ -247,18 +227,20 @@ static size_t HUF_decompress4X2_usingDTable_internal(
|
||||
{ const BYTE* const istart = (const BYTE*) cSrc;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
const void* const dtPtr = DTable + 1;
|
||||
const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
|
||||
const void* const dtPtr = DTable;
|
||||
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1;
|
||||
const U32 dtLog = DTable[0];
|
||||
size_t errorCode;
|
||||
|
||||
/* Init */
|
||||
BIT_DStream_t bitD1;
|
||||
BIT_DStream_t bitD2;
|
||||
BIT_DStream_t bitD3;
|
||||
BIT_DStream_t bitD4;
|
||||
size_t const length1 = MEM_readLE16(istart);
|
||||
size_t const length2 = MEM_readLE16(istart+2);
|
||||
size_t const length3 = MEM_readLE16(istart+4);
|
||||
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
|
||||
const size_t length1 = MEM_readLE16(istart);
|
||||
const size_t length2 = MEM_readLE16(istart+2);
|
||||
const size_t length3 = MEM_readLE16(istart+4);
|
||||
size_t length4;
|
||||
const BYTE* const istart1 = istart + 6; /* jumpTable */
|
||||
const BYTE* const istart2 = istart1 + length1;
|
||||
const BYTE* const istart3 = istart2 + length2;
|
||||
@@ -272,18 +254,17 @@ static size_t HUF_decompress4X2_usingDTable_internal(
|
||||
BYTE* op3 = opStart3;
|
||||
BYTE* op4 = opStart4;
|
||||
U32 endSignal;
|
||||
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
||||
U32 const dtLog = dtd.tableLog;
|
||||
|
||||
length4 = cSrcSize - (length1 + length2 + length3 + 6);
|
||||
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
errorCode = BIT_initDStream(&bitD1, istart1, length1);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
errorCode = BIT_initDStream(&bitD2, istart2, length2);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
errorCode = BIT_initDStream(&bitD3, istart3, length3);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
errorCode = BIT_initDStream(&bitD4, istart4, length4);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
|
||||
/* 16-32 symbols per loop (4-8 symbols per stream) */
|
||||
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
|
||||
@@ -329,42 +310,24 @@ static size_t HUF_decompress4X2_usingDTable_internal(
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_decompress4X2_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUF_DTable* DTable)
|
||||
{
|
||||
DTableDesc dtd = HUF_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 0) return ERROR(GENERIC);
|
||||
return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const hSize = HUF_readDTableX2 (dctx, cSrc, cSrcSize);
|
||||
if (HUF_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx);
|
||||
}
|
||||
|
||||
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
|
||||
return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_MAX_TABLELOG);
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
if (errorCode >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += errorCode;
|
||||
cSrcSize -= errorCode;
|
||||
|
||||
return HUF_decompress4X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
/* *************************/
|
||||
/* double-symbols decoding */
|
||||
/* *************************/
|
||||
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */
|
||||
|
||||
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
|
||||
|
||||
static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed,
|
||||
const U32* rankValOrigin, const int minWeight,
|
||||
@@ -372,7 +335,7 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
|
||||
U32 nbBitsBaseline, U16 baseSeq)
|
||||
{
|
||||
HUF_DEltX4 DElt;
|
||||
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
|
||||
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1];
|
||||
|
||||
/* get pre-calculated rankVal */
|
||||
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
|
||||
@@ -406,14 +369,14 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
|
||||
}}
|
||||
}
|
||||
|
||||
typedef U32 rankVal_t[HUF_TABLELOG_ABSOLUTEMAX][HUF_TABLELOG_ABSOLUTEMAX + 1];
|
||||
typedef U32 rankVal_t[HUF_ABSOLUTEMAX_TABLELOG][HUF_ABSOLUTEMAX_TABLELOG + 1];
|
||||
|
||||
static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
|
||||
const sortedSymbol_t* sortedList, const U32 sortedListSize,
|
||||
const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
|
||||
const U32 nbBitsBaseline)
|
||||
{
|
||||
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
|
||||
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1];
|
||||
const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
|
||||
const U32 minBits = nbBitsBaseline - maxWeight;
|
||||
U32 s;
|
||||
@@ -450,30 +413,29 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
|
||||
}
|
||||
}
|
||||
|
||||
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
|
||||
size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
|
||||
sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
|
||||
U32 rankStats[HUF_TABLELOG_ABSOLUTEMAX + 1] = { 0 };
|
||||
U32 rankStart0[HUF_TABLELOG_ABSOLUTEMAX + 2] = { 0 };
|
||||
BYTE weightList[HUF_MAX_SYMBOL_VALUE + 1];
|
||||
sortedSymbol_t sortedSymbol[HUF_MAX_SYMBOL_VALUE + 1];
|
||||
U32 rankStats[HUF_ABSOLUTEMAX_TABLELOG + 1] = { 0 };
|
||||
U32 rankStart0[HUF_ABSOLUTEMAX_TABLELOG + 2] = { 0 };
|
||||
U32* const rankStart = rankStart0+1;
|
||||
rankVal_t rankVal;
|
||||
U32 tableLog, maxW, sizeOfSort, nbSymbols;
|
||||
DTableDesc dtd = HUF_getDTableDesc(DTable);
|
||||
U32 const maxTableLog = dtd.maxTableLog;
|
||||
const U32 memLog = DTable[0];
|
||||
size_t iSize;
|
||||
void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
|
||||
HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr;
|
||||
void* dtPtr = DTable;
|
||||
HUF_DEltX4* const dt = ((HUF_DEltX4*)dtPtr) + 1;
|
||||
|
||||
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */
|
||||
if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
|
||||
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
|
||||
if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
||||
iSize = HUF_readStats(weightList, HUF_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
||||
if (HUF_isError(iSize)) return iSize;
|
||||
|
||||
/* check result */
|
||||
if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
|
||||
if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
|
||||
|
||||
/* find maxWeight */
|
||||
for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
|
||||
@@ -502,7 +464,7 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
|
||||
|
||||
/* Build rankVal */
|
||||
{ U32* const rankVal0 = rankVal[0];
|
||||
{ int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */
|
||||
{ int const rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
|
||||
U32 nextRankVal = 0;
|
||||
U32 w;
|
||||
for (w=1; w<maxW+1; w++) {
|
||||
@@ -512,21 +474,18 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
|
||||
} }
|
||||
{ U32 const minBits = tableLog+1 - maxW;
|
||||
U32 consumed;
|
||||
for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
|
||||
for (consumed = minBits; consumed < memLog - minBits + 1; consumed++) {
|
||||
U32* const rankValPtr = rankVal[consumed];
|
||||
U32 w;
|
||||
for (w = 1; w < maxW+1; w++) {
|
||||
rankValPtr[w] = rankVal0[w] >> consumed;
|
||||
} } } }
|
||||
|
||||
HUF_fillDTableX4(dt, maxTableLog,
|
||||
HUF_fillDTableX4(dt, memLog,
|
||||
sortedSymbol, sizeOfSort,
|
||||
rankStart0, rankVal, maxW,
|
||||
tableLog+1);
|
||||
|
||||
dtd.tableLog = (BYTE)maxTableLog;
|
||||
dtd.tableType = 1;
|
||||
memcpy(DTable, &dtd, sizeof(dtd));
|
||||
return iSize;
|
||||
}
|
||||
|
||||
@@ -558,7 +517,7 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
|
||||
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
||||
|
||||
#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
|
||||
if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
|
||||
if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \
|
||||
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
||||
|
||||
#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
|
||||
@@ -577,7 +536,7 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c
|
||||
HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
|
||||
}
|
||||
|
||||
/* closer to end : up to 2 symbols at a time */
|
||||
/* closer to the end */
|
||||
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2))
|
||||
HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
|
||||
|
||||
@@ -591,26 +550,26 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c
|
||||
}
|
||||
|
||||
|
||||
static size_t HUF_decompress1X4_usingDTable_internal(
|
||||
size_t HUF_decompress1X4_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUF_DTable* DTable)
|
||||
const U32* DTable)
|
||||
{
|
||||
BIT_DStream_t bitD;
|
||||
const BYTE* const istart = (const BYTE*) cSrc;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
|
||||
const U32 dtLog = DTable[0];
|
||||
const void* const dtPtr = DTable;
|
||||
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1;
|
||||
|
||||
/* Init */
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
}
|
||||
BIT_DStream_t bitD;
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD, istart, cSrcSize);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
|
||||
/* decode */
|
||||
{ BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */
|
||||
const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr;
|
||||
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
||||
HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog);
|
||||
}
|
||||
HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtLog);
|
||||
|
||||
/* check */
|
||||
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
|
||||
@@ -619,61 +578,49 @@ static size_t HUF_decompress1X4_usingDTable_internal(
|
||||
return dstSize;
|
||||
}
|
||||
|
||||
size_t HUF_decompress1X4_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUF_DTable* DTable)
|
||||
{
|
||||
DTableDesc dtd = HUF_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 1) return ERROR(GENERIC);
|
||||
return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const hSize = HUF_readDTableX4 (DCtx, cSrc, cSrcSize);
|
||||
if (HUF_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
|
||||
}
|
||||
|
||||
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
|
||||
return HUF_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_MAX_TABLELOG);
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const hSize = HUF_readDTableX4 (DTable, cSrc, cSrcSize);
|
||||
if (HUF_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize;
|
||||
cSrcSize -= hSize;
|
||||
|
||||
return HUF_decompress1X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
static size_t HUF_decompress4X4_usingDTable_internal(
|
||||
size_t HUF_decompress4X4_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUF_DTable* DTable)
|
||||
const U32* DTable)
|
||||
{
|
||||
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
|
||||
|
||||
{ const BYTE* const istart = (const BYTE*) cSrc;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
const void* const dtPtr = DTable+1;
|
||||
const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr;
|
||||
const void* const dtPtr = DTable;
|
||||
const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1;
|
||||
const U32 dtLog = DTable[0];
|
||||
size_t errorCode;
|
||||
|
||||
/* Init */
|
||||
BIT_DStream_t bitD1;
|
||||
BIT_DStream_t bitD2;
|
||||
BIT_DStream_t bitD3;
|
||||
BIT_DStream_t bitD4;
|
||||
size_t const length1 = MEM_readLE16(istart);
|
||||
size_t const length2 = MEM_readLE16(istart+2);
|
||||
size_t const length3 = MEM_readLE16(istart+4);
|
||||
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
|
||||
const size_t length1 = MEM_readLE16(istart);
|
||||
const size_t length2 = MEM_readLE16(istart+2);
|
||||
const size_t length3 = MEM_readLE16(istart+4);
|
||||
size_t length4;
|
||||
const BYTE* const istart1 = istart + 6; /* jumpTable */
|
||||
const BYTE* const istart2 = istart1 + length1;
|
||||
const BYTE* const istart3 = istart2 + length2;
|
||||
const BYTE* const istart4 = istart3 + length3;
|
||||
size_t const segmentSize = (dstSize+3) / 4;
|
||||
const size_t segmentSize = (dstSize+3) / 4;
|
||||
BYTE* const opStart2 = ostart + segmentSize;
|
||||
BYTE* const opStart3 = opStart2 + segmentSize;
|
||||
BYTE* const opStart4 = opStart3 + segmentSize;
|
||||
@@ -682,18 +629,17 @@ static size_t HUF_decompress4X4_usingDTable_internal(
|
||||
BYTE* op3 = opStart3;
|
||||
BYTE* op4 = opStart4;
|
||||
U32 endSignal;
|
||||
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
||||
U32 const dtLog = dtd.tableLog;
|
||||
|
||||
length4 = cSrcSize - (length1 + length2 + length3 + 6);
|
||||
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
|
||||
if (HUF_isError(errorCode)) return errorCode; }
|
||||
errorCode = BIT_initDStream(&bitD1, istart1, length1);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
errorCode = BIT_initDStream(&bitD2, istart2, length2);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
errorCode = BIT_initDStream(&bitD3, istart3, length3);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
errorCode = BIT_initDStream(&bitD4, istart4, length4);
|
||||
if (HUF_isError(errorCode)) return errorCode;
|
||||
|
||||
/* 16-32 symbols per loop (4-8 symbols per stream) */
|
||||
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
|
||||
@@ -731,8 +677,8 @@ static size_t HUF_decompress4X4_usingDTable_internal(
|
||||
HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
|
||||
|
||||
/* check */
|
||||
{ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
|
||||
if (!endCheck) return ERROR(corruption_detected); }
|
||||
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
|
||||
if (!endSignal) return ERROR(corruption_detected);
|
||||
|
||||
/* decoded size */
|
||||
return dstSize;
|
||||
@@ -740,59 +686,27 @@ static size_t HUF_decompress4X4_usingDTable_internal(
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_decompress4X4_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUF_DTable* DTable)
|
||||
{
|
||||
DTableDesc dtd = HUF_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 1) return ERROR(GENERIC);
|
||||
return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t hSize = HUF_readDTableX4 (dctx, cSrc, cSrcSize);
|
||||
if (HUF_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
|
||||
}
|
||||
|
||||
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
|
||||
return HUF_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_MAX_TABLELOG);
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t hSize = HUF_readDTableX4 (DTable, cSrc, cSrcSize);
|
||||
if (HUF_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize;
|
||||
cSrcSize -= hSize;
|
||||
|
||||
return HUF_decompress4X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/* ********************************/
|
||||
/* Generic decompression selector */
|
||||
/* ********************************/
|
||||
|
||||
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUF_DTable* DTable)
|
||||
{
|
||||
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
||||
return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) :
|
||||
HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUF_DTable* DTable)
|
||||
{
|
||||
DTableDesc const dtd = HUF_getDTableDesc(DTable);
|
||||
return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) :
|
||||
HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
|
||||
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] =
|
||||
{
|
||||
@@ -815,29 +729,12 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu
|
||||
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */
|
||||
};
|
||||
|
||||
/** HUF_selectDecoder() :
|
||||
* Tells which decoder is likely to decode faster,
|
||||
* based on a set of pre-determined metrics.
|
||||
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
|
||||
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
|
||||
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
|
||||
{
|
||||
/* decoder timing evaluation */
|
||||
U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
|
||||
U32 const D256 = (U32)(dstSize >> 8);
|
||||
U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
|
||||
U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
|
||||
DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */
|
||||
|
||||
return DTime1 < DTime0;
|
||||
}
|
||||
|
||||
|
||||
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
||||
|
||||
size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
static const decompressionAlgo decompress[2] = { HUF_decompress4X2, HUF_decompress4X4 };
|
||||
U32 Dtime[3]; /* decompression time estimation */
|
||||
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
@@ -845,50 +742,17 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS
|
||||
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
||||
/* decoder timing evaluation */
|
||||
{ U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
|
||||
U32 const D256 = (U32)(dstSize >> 8);
|
||||
U32 n; for (n=0; n<3; n++)
|
||||
Dtime[n] = algoTime[Q][n].tableTime + (algoTime[Q][n].decode256Time * D256);
|
||||
}
|
||||
|
||||
Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */
|
||||
|
||||
{ U32 algoNb = 0;
|
||||
if (Dtime[1] < Dtime[0]) algoNb = 1;
|
||||
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
//return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
|
||||
//return HUF_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
|
||||
}
|
||||
|
||||
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
|
||||
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
||||
return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
|
||||
HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
|
||||
}
|
||||
}
|
||||
|
||||
size_t HUF_decompress4X_hufOnly (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */
|
||||
|
||||
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
||||
return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
|
||||
HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
|
||||
}
|
||||
}
|
||||
|
||||
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
|
||||
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
|
||||
return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
|
||||
HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,9 +35,9 @@
|
||||
***************************************/
|
||||
#include <stdlib.h>
|
||||
#include "error_private.h"
|
||||
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */
|
||||
#define ZBUFF_STATIC_LINKING_ONLY
|
||||
#include "zbuff.h"
|
||||
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */
|
||||
#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */
|
||||
#include "zbuff_static.h"
|
||||
|
||||
|
||||
/*-***************************************************************************
|
||||
@@ -82,31 +82,15 @@ struct ZBUFF_DCtx_s {
|
||||
size_t blockSize;
|
||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
size_t lhSize;
|
||||
ZSTD_customMem customMem;
|
||||
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
|
||||
|
||||
|
||||
ZBUFF_DCtx* ZBUFF_createDCtx(void)
|
||||
{
|
||||
return ZBUFF_createDCtx_advanced(defaultCustomMem);
|
||||
}
|
||||
|
||||
ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
|
||||
{
|
||||
ZBUFF_DCtx* zbd;
|
||||
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx));
|
||||
ZBUFF_DCtx* zbd = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx));
|
||||
if (zbd==NULL) return NULL;
|
||||
memset(zbd, 0, sizeof(ZBUFF_DCtx));
|
||||
memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem));
|
||||
zbd->zd = ZSTD_createDCtx_advanced(customMem);
|
||||
if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; }
|
||||
memset(zbd, 0, sizeof(*zbd));
|
||||
zbd->zd = ZSTD_createDCtx();
|
||||
zbd->stage = ZBUFFds_init;
|
||||
return zbd;
|
||||
}
|
||||
@@ -115,9 +99,9 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
|
||||
{
|
||||
if (zbd==NULL) return 0; /* support free on null */
|
||||
ZSTD_freeDCtx(zbd->zd);
|
||||
if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
|
||||
if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd);
|
||||
free(zbd->inBuff);
|
||||
free(zbd->outBuff);
|
||||
free(zbd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -137,15 +121,6 @@ size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd)
|
||||
}
|
||||
|
||||
|
||||
/* internal util function */
|
||||
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const length = MIN(dstCapacity, srcSize);
|
||||
memcpy(dst, src, length);
|
||||
return length;
|
||||
}
|
||||
|
||||
|
||||
/* *** Decompression *** */
|
||||
|
||||
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
||||
@@ -191,22 +166,20 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
||||
if (ZSTD_isError(h2Result)) return h2Result;
|
||||
} }
|
||||
|
||||
zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
|
||||
|
||||
/* Frame header instruct buffer sizes */
|
||||
{ size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_MAX);
|
||||
{ size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX);
|
||||
zbd->blockSize = blockSize;
|
||||
if (zbd->inBuffSize < blockSize) {
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
|
||||
free(zbd->inBuff);
|
||||
zbd->inBuffSize = blockSize;
|
||||
zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
|
||||
zbd->inBuff = (char*)malloc(blockSize);
|
||||
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
{ size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
|
||||
{ size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize;
|
||||
if (zbd->outBuffSize < neededOutSize) {
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
|
||||
free(zbd->outBuff);
|
||||
zbd->outBuffSize = neededOutSize;
|
||||
zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize);
|
||||
zbd->outBuff = (char*)malloc(neededOutSize);
|
||||
if (zbd->outBuff == NULL) return ERROR(memory_allocation);
|
||||
} } }
|
||||
zbd->stage = ZBUFFds_read;
|
||||
@@ -219,13 +192,12 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
||||
break;
|
||||
}
|
||||
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
|
||||
const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
|
||||
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
|
||||
zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
|
||||
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
|
||||
ip, neededInSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
ip += neededInSize;
|
||||
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
|
||||
if (!decodedSize) break; /* this was just a header */
|
||||
zbd->outEnd = zbd->outStart + decodedSize;
|
||||
zbd->stage = ZBUFFds_flush;
|
||||
break;
|
||||
@@ -245,13 +217,12 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
||||
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
|
||||
|
||||
/* decode loaded input */
|
||||
{ const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
|
||||
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
|
||||
{ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
|
||||
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
|
||||
zbd->inBuff, neededInSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
zbd->inPos = 0; /* input is consumed */
|
||||
if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
||||
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
||||
zbd->outEnd = zbd->outStart + decodedSize;
|
||||
zbd->stage = ZBUFFds_flush;
|
||||
// break; /* ZBUFFds_flush follows */
|
||||
@@ -279,7 +250,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
|
||||
*srcSizePtr = ip-istart;
|
||||
*dstCapacityPtr = op-ostart;
|
||||
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
|
||||
// if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */
|
||||
if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */
|
||||
nextSrcSizeHint -= zbd->inPos; /* already loaded*/
|
||||
return nextSrcSizeHint;
|
||||
}
|
||||
|
||||
+113
-335
@@ -53,18 +53,15 @@
|
||||
/*-*******************************************************
|
||||
* Dependencies
|
||||
*********************************************************/
|
||||
#include <string.h> /* memcpy, memmove, memset */
|
||||
#include <stdlib.h> /* calloc */
|
||||
#include <string.h> /* memcpy, memmove */
|
||||
#include <stdio.h> /* debug only : printf */
|
||||
#include "mem.h" /* low level memory routines */
|
||||
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
#define FSE_STATIC_LINKING_ONLY
|
||||
#include "fse.h"
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
#include "zstd_internal.h"
|
||||
#include "fse_static.h"
|
||||
#include "huf_static.h"
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
# include "zstd_legacy.h"
|
||||
#endif
|
||||
|
||||
@@ -104,31 +101,25 @@ static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
||||
* Context management
|
||||
***************************************************************/
|
||||
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
|
||||
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
|
||||
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
|
||||
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage;
|
||||
|
||||
struct ZSTD_DCtx_s
|
||||
{
|
||||
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
|
||||
FSE_DTable OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
|
||||
FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
|
||||
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
|
||||
unsigned hufTableX4[HUF_DTABLE_SIZE(HufLog)];
|
||||
const void* previousDstEnd;
|
||||
const void* base;
|
||||
const void* vBase;
|
||||
const void* dictEnd;
|
||||
size_t expected;
|
||||
U32 rep[3];
|
||||
size_t headerSize;
|
||||
ZSTD_frameParams fParams;
|
||||
blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
|
||||
ZSTD_dStage stage;
|
||||
U32 litEntropy;
|
||||
U32 fseEntropy;
|
||||
XXH64_state_t xxhState;
|
||||
size_t headerSize;
|
||||
U32 dictID;
|
||||
U32 flagRepeatTable;
|
||||
const BYTE* litPtr;
|
||||
ZSTD_customMem customMem;
|
||||
size_t litBufSize;
|
||||
size_t litSize;
|
||||
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
|
||||
@@ -145,39 +136,22 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
|
||||
dctx->base = NULL;
|
||||
dctx->vBase = NULL;
|
||||
dctx->dictEnd = NULL;
|
||||
dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);
|
||||
dctx->litEntropy = dctx->fseEntropy = 0;
|
||||
dctx->dictID = 0;
|
||||
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = repStartValue[i]; }
|
||||
dctx->hufTableX4[0] = HufLog;
|
||||
dctx->flagRepeatTable = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
|
||||
{
|
||||
ZSTD_DCtx* dctx;
|
||||
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx));
|
||||
if (!dctx) return NULL;
|
||||
memcpy(&dctx->customMem, &customMem, sizeof(ZSTD_customMem));
|
||||
ZSTD_decompressBegin(dctx);
|
||||
return dctx;
|
||||
}
|
||||
|
||||
ZSTD_DCtx* ZSTD_createDCtx(void)
|
||||
{
|
||||
return ZSTD_createDCtx_advanced(defaultCustomMem);
|
||||
ZSTD_DCtx* dctx = (ZSTD_DCtx*)malloc(sizeof(ZSTD_DCtx));
|
||||
if (dctx==NULL) return NULL;
|
||||
ZSTD_decompressBegin(dctx);
|
||||
return dctx;
|
||||
}
|
||||
|
||||
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
|
||||
{
|
||||
if (dctx==NULL) return 0; /* support free on NULL */
|
||||
dctx->customMem.customFree(dctx->customMem.opaque, dctx);
|
||||
free(dctx);
|
||||
return 0; /* reserved as a potential error code in the future */
|
||||
}
|
||||
|
||||
@@ -209,44 +183,17 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
||||
|
||||
/* Frame descriptor
|
||||
|
||||
// old
|
||||
1 byte - Alloc :
|
||||
1 byte, using :
|
||||
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
|
||||
bit 4 : reserved for windowLog (must be zero)
|
||||
bit 4 : minmatch 4(0) or 3(1)
|
||||
bit 5 : reserved (must be zero)
|
||||
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
|
||||
|
||||
1 byte - checker :
|
||||
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||
bit 2-7 : reserved (must be zero)
|
||||
|
||||
// new
|
||||
1 byte - FrameHeaderDescription :
|
||||
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||
bit 2 : checksumFlag
|
||||
bit 3 : reserved (must be zero)
|
||||
bit 4 : reserved (unused, can be any value)
|
||||
bit 5 : Single Segment (if 1, WindowLog byte is not present)
|
||||
bit 6-7 : FrameContentFieldSize (0, 2, 4, or 8)
|
||||
if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1;
|
||||
|
||||
Optional : WindowLog (0 or 1 byte)
|
||||
bit 0-2 : octal Fractional (1/8th)
|
||||
bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB)
|
||||
|
||||
Optional : dictID (0, 1, 2 or 4 bytes)
|
||||
Automatic adaptation
|
||||
0 : no dictID
|
||||
1 : 1 - 255
|
||||
2 : 256 - 65535
|
||||
4 : all other values
|
||||
|
||||
Optional : content size (0, 1, 2, 4 or 8 bytes)
|
||||
0 : unknown (fcfs==0 and swl==0)
|
||||
1 : 0-255 bytes (fcfs==0 and swl==1)
|
||||
2 : 256 - 65535+256 (fcfs==1)
|
||||
4 : 0 - 4GB-1 (fcfs==2)
|
||||
8 : 0 - 16EB-1 (fcfs==3)
|
||||
Optional : content size (0, 1, 2 or 8 bytes)
|
||||
0 : unknown
|
||||
1 : 0-255 bytes
|
||||
2 : 256 - 65535+256
|
||||
8 : up to 16 exa
|
||||
*/
|
||||
|
||||
|
||||
@@ -322,18 +269,13 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
|
||||
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
|
||||
{
|
||||
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
||||
{ BYTE const fhd = ((const BYTE*)src)[4];
|
||||
U32 const dictID= fhd & 3;
|
||||
U32 const directMode = (fhd >> 5) & 1;
|
||||
U32 const fcsId = fhd >> 6;
|
||||
return ZSTD_frameHeaderSize_min + !directMode + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
|
||||
+ (directMode && !ZSTD_fcs_fieldSize[fcsId]);
|
||||
}
|
||||
{ U32 const fcsId = (((const BYTE*)src)[4]) >> 6;
|
||||
return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; }
|
||||
}
|
||||
|
||||
|
||||
/** ZSTD_getFrameParams() :
|
||||
* decode Frame Header, or require larger `srcSize`.
|
||||
* decode Frame Header, or provide expected `srcSize`.
|
||||
* @return : 0, `fparamsPtr` is correctly filled,
|
||||
* >0, `srcSize` is too small, result is expected `srcSize`,
|
||||
* or an error code, which can be tested using ZSTD_isError() */
|
||||
@@ -342,63 +284,24 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
|
||||
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min;
|
||||
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
|
||||
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
|
||||
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
||||
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
||||
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
|
||||
return 0;
|
||||
}
|
||||
return ERROR(prefix_unknown);
|
||||
}
|
||||
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
|
||||
|
||||
/* ensure there is enough `srcSize` to fully read/decode frame header */
|
||||
{ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
|
||||
if (srcSize < fhsize) return fhsize; }
|
||||
|
||||
{ BYTE const fhdByte = ip[4];
|
||||
size_t pos = 5;
|
||||
U32 const dictIDSizeCode = fhdByte&3;
|
||||
U32 const checksumFlag = (fhdByte>>2)&1;
|
||||
U32 const directMode = (fhdByte>>5)&1;
|
||||
U32 const fcsID = fhdByte>>6;
|
||||
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX;
|
||||
U32 windowSize = 0;
|
||||
U32 dictID = 0;
|
||||
U64 frameContentSize = 0;
|
||||
if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
|
||||
if (!directMode) {
|
||||
BYTE const wlByte = ip[pos++];
|
||||
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
||||
if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
|
||||
windowSize = (1U << windowLog);
|
||||
windowSize += (windowSize >> 3) * (wlByte&7);
|
||||
}
|
||||
|
||||
switch(dictIDSizeCode)
|
||||
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
||||
{ BYTE const frameDesc = ip[4];
|
||||
fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
||||
if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */
|
||||
switch(frameDesc >> 6) /* fcsId */
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : break;
|
||||
case 1 : dictID = ip[pos]; pos++; break;
|
||||
case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
|
||||
case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
|
||||
}
|
||||
switch(fcsID)
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : if (directMode) frameContentSize = ip[pos]; break;
|
||||
case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
|
||||
case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
|
||||
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
|
||||
}
|
||||
if (!windowSize) windowSize = (U32)frameContentSize;
|
||||
if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
|
||||
fparamsPtr->frameContentSize = frameContentSize;
|
||||
fparamsPtr->windowSize = windowSize;
|
||||
fparamsPtr->dictID = dictID;
|
||||
fparamsPtr->checksumFlag = checksumFlag;
|
||||
}
|
||||
case 0 : fparamsPtr->frameContentSize = 0; break;
|
||||
case 1 : fparamsPtr->frameContentSize = ip[5]; break;
|
||||
case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5)+256; break;
|
||||
case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break;
|
||||
} }
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -406,11 +309,10 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
|
||||
/** ZSTD_decodeFrameHeader() :
|
||||
* `srcSize` must be the size provided by ZSTD_frameHeaderSize().
|
||||
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
|
||||
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize)
|
||||
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize);
|
||||
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
|
||||
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
|
||||
size_t const result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize);
|
||||
if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -454,14 +356,13 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
||||
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) src;
|
||||
litBlockType_t lbt;
|
||||
|
||||
/* any compressed block with literals segment must be at least this size */
|
||||
if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected);
|
||||
lbt = (litBlockType_t)(istart[0]>> 6);
|
||||
|
||||
switch(lbt)
|
||||
switch(istart[0]>> 6)
|
||||
{
|
||||
case lbt_huffman:
|
||||
case IS_HUF:
|
||||
{ size_t litSize, litCSize, singleStream=0;
|
||||
U32 lhSize = ((istart[0]) >> 4) & 3;
|
||||
if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for lhSize, + cSize (+nbSeq) */
|
||||
@@ -491,31 +392,29 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
||||
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
|
||||
|
||||
if (HUF_isError(singleStream ?
|
||||
HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
|
||||
HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) ))
|
||||
HUF_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) :
|
||||
HUF_decompress (dctx->litBuffer, litSize, istart+lhSize, litCSize) ))
|
||||
return ERROR(corruption_detected);
|
||||
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+8;
|
||||
dctx->litSize = litSize;
|
||||
dctx->litEntropy = 1;
|
||||
return litCSize + lhSize;
|
||||
}
|
||||
case lbt_repeat:
|
||||
case IS_PCH:
|
||||
{ size_t litSize, litCSize;
|
||||
U32 lhSize = ((istart[0]) >> 4) & 3;
|
||||
if (lhSize != 1) /* only case supported for now : small litSize, single stream */
|
||||
return ERROR(corruption_detected);
|
||||
if (dctx->litEntropy==0)
|
||||
if (!dctx->flagRepeatTable)
|
||||
return ERROR(dictionary_corrupted);
|
||||
|
||||
/* 2 - 2 - 10 - 10 */
|
||||
lhSize=3;
|
||||
litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
|
||||
litCSize = ((istart[1] & 3) << 8) + istart[2];
|
||||
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
|
||||
|
||||
{ size_t const errorCode = HUF_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable);
|
||||
{ size_t const errorCode = HUF_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
|
||||
if (HUF_isError(errorCode)) return ERROR(corruption_detected);
|
||||
}
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
@@ -523,7 +422,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
||||
dctx->litSize = litSize;
|
||||
return litCSize + lhSize;
|
||||
}
|
||||
case lbt_raw:
|
||||
case IS_RAW:
|
||||
{ size_t litSize;
|
||||
U32 lhSize = ((istart[0]) >> 4) & 3;
|
||||
switch(lhSize)
|
||||
@@ -554,7 +453,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
||||
dctx->litSize = litSize;
|
||||
return lhSize+litSize;
|
||||
}
|
||||
case lbt_rle:
|
||||
case IS_RLE:
|
||||
{ size_t litSize;
|
||||
U32 lhSize = ((istart[0]) >> 4) & 3;
|
||||
switch(lhSize)
|
||||
@@ -683,10 +582,10 @@ typedef struct {
|
||||
} seqState_t;
|
||||
|
||||
|
||||
static seq_t ZSTD_decodeSequence(seqState_t* seqState)
|
||||
{
|
||||
seq_t seq;
|
||||
|
||||
static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
||||
{
|
||||
/* Literal length */
|
||||
U32 const llCode = FSE_peekSymbol(&(seqState->stateLL));
|
||||
U32 const mlCode = FSE_peekSymbol(&(seqState->stateML));
|
||||
U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
|
||||
@@ -742,13 +641,13 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
|
||||
seqState->prevOffset[1] = seqState->prevOffset[0];
|
||||
seqState->prevOffset[0] = offset;
|
||||
}
|
||||
seq.offset = offset;
|
||||
seq->offset = offset;
|
||||
}
|
||||
|
||||
seq.matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */
|
||||
seq->matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */
|
||||
if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));
|
||||
|
||||
seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */
|
||||
seq->litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */
|
||||
if (MEM_32bits() ||
|
||||
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));
|
||||
|
||||
@@ -757,8 +656,6 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
|
||||
FSE_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */
|
||||
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */
|
||||
FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */
|
||||
|
||||
return seq;
|
||||
}
|
||||
|
||||
|
||||
@@ -856,16 +753,20 @@ static size_t ZSTD_decompressSequences(
|
||||
int nbSeq;
|
||||
|
||||
/* Build Decoding Tables */
|
||||
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize);
|
||||
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->flagRepeatTable, ip, seqSize);
|
||||
if (ZSTD_isError(seqHSize)) return seqHSize;
|
||||
ip += seqHSize;
|
||||
dctx->flagRepeatTable = 0;
|
||||
}
|
||||
|
||||
/* Regen sequences */
|
||||
if (nbSeq) {
|
||||
seq_t sequence;
|
||||
seqState_t seqState;
|
||||
dctx->fseEntropy = 1;
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) seqState.prevOffset[i] = dctx->rep[i]; }
|
||||
|
||||
memset(&sequence, 0, sizeof(sequence));
|
||||
sequence.offset = REPCODE_STARTVALUE;
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) seqState.prevOffset[i] = REPCODE_STARTVALUE; }
|
||||
{ size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
|
||||
if (ERR_isError(errorCode)) return ERROR(corruption_detected); }
|
||||
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
|
||||
@@ -874,16 +775,24 @@ static size_t ZSTD_decompressSequences(
|
||||
|
||||
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
|
||||
nbSeq--;
|
||||
{ seq_t const sequence = ZSTD_decodeSequence(&seqState);
|
||||
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
|
||||
ZSTD_decodeSequence(&sequence, &seqState);
|
||||
|
||||
#if 0 /* debug */
|
||||
static BYTE* start = NULL;
|
||||
if (start==NULL) start = op;
|
||||
size_t pos = (size_t)(op-start);
|
||||
if ((pos >= 5810037) && (pos < 5810400))
|
||||
printf("Dpos %6u :%5u literals & match %3u bytes at distance %6u \n",
|
||||
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
|
||||
#endif
|
||||
|
||||
{ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
|
||||
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
||||
op += oneSeqSize;
|
||||
} }
|
||||
|
||||
/* check if reached exact end */
|
||||
if (nbSeq) return ERROR(corruption_detected);
|
||||
/* save reps for next block */
|
||||
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); }
|
||||
}
|
||||
|
||||
/* last literal segment */
|
||||
@@ -936,14 +845,6 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_generateNxByte(void* dst, size_t dstCapacity, BYTE byte, size_t length)
|
||||
{
|
||||
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||
memset(dst, byte, length);
|
||||
return length;
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTD_decompressFrame() :
|
||||
* `dctx` must be properly initialized */
|
||||
static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
@@ -953,9 +854,10 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* const iend = ip + srcSize;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
BYTE* const oend = ostart + dstCapacity;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + dstCapacity;
|
||||
size_t remainingSize = srcSize;
|
||||
blockProperties_t blockProperties = { bt_compressed, 0 };
|
||||
|
||||
/* check */
|
||||
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
@@ -970,8 +872,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
|
||||
/* Loop on each block */
|
||||
while (1) {
|
||||
size_t decodedSize;
|
||||
blockProperties_t blockProperties;
|
||||
size_t decodedSize=0;
|
||||
size_t const cBlockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties);
|
||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
@@ -988,12 +889,11 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize);
|
||||
break;
|
||||
case bt_rle :
|
||||
decodedSize = ZSTD_generateNxByte(op, oend-op, *ip, blockProperties.origSize);
|
||||
return ERROR(GENERIC); /* not yet supported */
|
||||
break;
|
||||
case bt_end :
|
||||
/* end of frame */
|
||||
if (remainingSize) return ERROR(srcSize_wrong);
|
||||
decodedSize = 0;
|
||||
break;
|
||||
default:
|
||||
return ERROR(GENERIC); /* impossible */
|
||||
@@ -1001,7 +901,6 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
if (cBlockSize == 0) break; /* bt_end */
|
||||
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize);
|
||||
op += decodedSize;
|
||||
ip += cBlockSize;
|
||||
remainingSize -= cBlockSize;
|
||||
@@ -1011,11 +910,6 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTD_decompress_usingPreparedDCtx() :
|
||||
* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
|
||||
* It avoids reloading the dictionary each time.
|
||||
* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
|
||||
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
|
||||
size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
@@ -1032,7 +926,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
|
||||
const void* dict, size_t dictSize)
|
||||
{
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
{ U32 const magicNumber = MEM_readLE32(src);
|
||||
{ const U32 magicNumber = MEM_readLE32(src);
|
||||
if (ZSTD_isLegacy(magicNumber))
|
||||
return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber);
|
||||
}
|
||||
@@ -1053,7 +947,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
|
||||
{
|
||||
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
|
||||
size_t regenSize;
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
if (dctx==NULL) return ERROR(memory_allocation);
|
||||
regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
@@ -1073,30 +967,17 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
|
||||
return dctx->expected;
|
||||
}
|
||||
|
||||
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx)
|
||||
{
|
||||
return dctx->stage == ZSTDds_skipFrame;
|
||||
}
|
||||
|
||||
/** ZSTD_decompressContinue() :
|
||||
* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
|
||||
* or an error code, which can be tested using ZSTD_isError() */
|
||||
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
/* Sanity check */
|
||||
if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
|
||||
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
|
||||
|
||||
/* Decompress : frame header; part 1 */
|
||||
switch (dctx->stage)
|
||||
{
|
||||
case ZSTDds_getFrameHeaderSize :
|
||||
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
|
||||
dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */
|
||||
dctx->stage = ZSTDds_decodeSkippableHeader;
|
||||
return 0;
|
||||
}
|
||||
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
|
||||
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
|
||||
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
|
||||
@@ -1121,13 +1002,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
||||
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
|
||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||
if (bp.blockType == bt_end) {
|
||||
if (dctx->fParams.checksumFlag) {
|
||||
U64 const h64 = XXH64_digest(&dctx->xxhState);
|
||||
U32 const h32 = (U32)(h64>>11) & ((1<<22)-1);
|
||||
const BYTE* const ip = (const BYTE*)src;
|
||||
U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16);
|
||||
if (check32 != h32) return ERROR(checksum_wrong);
|
||||
}
|
||||
dctx->expected = 0;
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
} else {
|
||||
@@ -1159,21 +1033,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
||||
dctx->stage = ZSTDds_decodeBlockHeader;
|
||||
dctx->expected = ZSTD_blockHeaderSize;
|
||||
dctx->previousDstEnd = (char*)dst + rSize;
|
||||
if (ZSTD_isError(rSize)) return rSize;
|
||||
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
|
||||
return rSize;
|
||||
}
|
||||
case ZSTDds_decodeSkippableHeader:
|
||||
{ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected);
|
||||
dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
|
||||
dctx->stage = ZSTDds_skipFrame;
|
||||
return 0;
|
||||
}
|
||||
case ZSTDds_skipFrame:
|
||||
{ dctx->expected = 0;
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
return 0;
|
||||
}
|
||||
default:
|
||||
return ERROR(GENERIC); /* impossible */
|
||||
}
|
||||
@@ -1188,78 +1049,68 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
|
||||
dctx->previousDstEnd = (const char*)dict + dictSize;
|
||||
}
|
||||
|
||||
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t const dictSize)
|
||||
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
{
|
||||
const BYTE* dictPtr = (const BYTE*)dict;
|
||||
const BYTE* const dictEnd = dictPtr + dictSize;
|
||||
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize;
|
||||
|
||||
{ size_t const hSize = HUF_readDTableX4(dctx->hufTable, dict, dictSize);
|
||||
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
|
||||
dictPtr += hSize;
|
||||
}
|
||||
hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
|
||||
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + hSize;
|
||||
dictSize -= hSize;
|
||||
|
||||
{ short offcodeNCount[MaxOff+1];
|
||||
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
|
||||
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
|
||||
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
||||
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += offcodeHeaderSize;
|
||||
dict = (const char*)dict + offcodeHeaderSize;
|
||||
dictSize -= offcodeHeaderSize;
|
||||
}
|
||||
|
||||
{ short matchlengthNCount[MaxML+1];
|
||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
|
||||
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
||||
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += matchlengthHeaderSize;
|
||||
dict = (const char*)dict + matchlengthHeaderSize;
|
||||
dictSize -= matchlengthHeaderSize;
|
||||
}
|
||||
|
||||
{ short litlengthNCount[MaxLL+1];
|
||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
|
||||
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
||||
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += litlengthHeaderSize;
|
||||
}
|
||||
|
||||
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
|
||||
dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
dictPtr += 12;
|
||||
|
||||
dctx->litEntropy = dctx->fseEntropy = 1;
|
||||
return dictPtr - (const BYTE*)dict;
|
||||
dctx->flagRepeatTable = 1;
|
||||
return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
|
||||
}
|
||||
|
||||
static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
{
|
||||
if (dictSize < 8) return ERROR(dictionary_corrupted);
|
||||
{ U32 const magic = MEM_readLE32(dict);
|
||||
if (magic != ZSTD_DICT_MAGIC) {
|
||||
/* pure content mode */
|
||||
ZSTD_refDictContent(dctx, dict, dictSize);
|
||||
return 0;
|
||||
}
|
||||
dctx->dictID = MEM_readLE32((const char*)dict + 4);
|
||||
|
||||
/* load entropy tables */
|
||||
dict = (const char*)dict + 8;
|
||||
dictSize -= 8;
|
||||
{ size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
|
||||
if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + eSize;
|
||||
dictSize -= eSize;
|
||||
}
|
||||
|
||||
/* reference dictionary content */
|
||||
size_t eSize;
|
||||
U32 const magic = MEM_readLE32(dict);
|
||||
if (magic != ZSTD_DICT_MAGIC) {
|
||||
/* pure content mode */
|
||||
ZSTD_refDictContent(dctx, dict, dictSize);
|
||||
|
||||
return 0;
|
||||
}
|
||||
/* load entropy tables */
|
||||
dict = (const char*)dict + 4;
|
||||
dictSize -= 4;
|
||||
eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
|
||||
if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
|
||||
|
||||
/* reference dictionary content */
|
||||
dict = (const char*)dict + eSize;
|
||||
dictSize -= eSize;
|
||||
ZSTD_refDictContent(dctx, dict, dictSize);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -1276,76 +1127,3 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
struct ZSTD_DDict_s {
|
||||
void* dictContent;
|
||||
size_t dictContentSize;
|
||||
ZSTD_DCtx* refContext;
|
||||
}; /* typedef'd tp ZSTD_CDict within zstd.h */
|
||||
|
||||
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem)
|
||||
{
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict));
|
||||
void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize);
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem);
|
||||
|
||||
if (!dictContent || !ddict || !dctx) {
|
||||
customMem.customFree(customMem.opaque, dictContent);
|
||||
customMem.customFree(customMem.opaque, ddict);
|
||||
customMem.customFree(customMem.opaque, dctx);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memcpy(dictContent, dict, dictSize);
|
||||
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
|
||||
if (ZSTD_isError(errorCode)) {
|
||||
customMem.customFree(customMem.opaque, dictContent);
|
||||
customMem.customFree(customMem.opaque, ddict);
|
||||
customMem.customFree(customMem.opaque, dctx);
|
||||
return NULL;
|
||||
} }
|
||||
|
||||
ddict->dictContent = dictContent;
|
||||
ddict->dictContentSize = dictSize;
|
||||
ddict->refContext = dctx;
|
||||
return ddict;
|
||||
}
|
||||
}
|
||||
|
||||
/*! ZSTD_createDDict() :
|
||||
* Create a digested dictionary, ready to start decompression operation without startup delay.
|
||||
* `dict` can be released after `ZSTD_DDict` creation */
|
||||
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
|
||||
{
|
||||
ZSTD_customMem const allocator = { NULL, NULL, NULL };
|
||||
return ZSTD_createDDict_advanced(dict, dictSize, allocator);
|
||||
}
|
||||
|
||||
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
|
||||
{
|
||||
ZSTD_freeFunction const cFree = ddict->refContext->customMem.customFree;
|
||||
void* const opaque = ddict->refContext->customMem.opaque;
|
||||
ZSTD_freeDCtx(ddict->refContext);
|
||||
cFree(opaque, ddict->dictContent);
|
||||
cFree(opaque, ddict);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*! ZSTD_decompress_usingDDict() :
|
||||
* Decompression using a pre-digested Dictionary
|
||||
* Use dictionary without significant overhead. */
|
||||
ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_DDict* ddict)
|
||||
{
|
||||
return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext,
|
||||
dst, dstCapacity,
|
||||
src, srcSize);
|
||||
}
|
||||
|
||||
+97
-181
@@ -59,13 +59,11 @@
|
||||
#include "mem.h" /* read */
|
||||
#include "error_private.h"
|
||||
#include "fse.h"
|
||||
#define HUF_STATIC_LINKING_ONLY
|
||||
#include "huf.h"
|
||||
#include "zstd_internal.h" /* includes zstd.h */
|
||||
#include "xxhash.h"
|
||||
#include "huf_static.h"
|
||||
#include "zstd_internal.h"
|
||||
#include "divsufsort.h"
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#include "zdict.h"
|
||||
#include "zdict_static.h"
|
||||
|
||||
|
||||
|
||||
/*-*************************************
|
||||
@@ -106,7 +104,8 @@ static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
|
||||
{
|
||||
const BYTE* const b = (const BYTE*)ptr;
|
||||
size_t u;
|
||||
for (u=0; u<length; u++) {
|
||||
for (u=0; u<length; u++)
|
||||
{
|
||||
BYTE c = b[u];
|
||||
if (c<32 || c>126) c = '.'; /* non-printable char */
|
||||
DISPLAYLEVEL(dlevel, "%c", c);
|
||||
@@ -199,12 +198,8 @@ static size_t ZDICT_count(const void* pIn, const void* pMatch)
|
||||
{
|
||||
const char* const pStart = (const char*)pIn;
|
||||
for (;;) {
|
||||
size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
||||
if (!diff) {
|
||||
pIn = (const char*)pIn+sizeof(size_t);
|
||||
pMatch = (const char*)pMatch+sizeof(size_t);
|
||||
continue;
|
||||
}
|
||||
size_t diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
|
||||
if (!diff) { pIn = (const char*)pIn+sizeof(size_t); pMatch = (const char*)pMatch+sizeof(size_t); continue; }
|
||||
pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff);
|
||||
return (size_t)((const char*)pIn - pStart);
|
||||
}
|
||||
@@ -351,8 +346,9 @@ static dictItem ZDICT_analyzePos(
|
||||
maxLength = i;
|
||||
|
||||
/* reduce maxLength in case of final into repetitive data */
|
||||
{ U32 l = (U32)maxLength;
|
||||
BYTE const c = b[pos + maxLength-1];
|
||||
{
|
||||
U32 l = (U32)maxLength;
|
||||
BYTE c = b[pos + maxLength-1];
|
||||
while (b[pos+l-2]==c) l--;
|
||||
maxLength = l;
|
||||
}
|
||||
@@ -371,10 +367,12 @@ static dictItem ZDICT_analyzePos(
|
||||
solution.savings = savings[maxLength];
|
||||
|
||||
/* mark positions done */
|
||||
{ U32 id;
|
||||
{
|
||||
U32 id;
|
||||
U32 testedPos;
|
||||
for (id=start; id<end; id++) {
|
||||
U32 p, pEnd;
|
||||
U32 const testedPos = suffix[id];
|
||||
testedPos = suffix[id];
|
||||
if (testedPos == pos)
|
||||
length = solution.length;
|
||||
else {
|
||||
@@ -441,7 +439,7 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
|
||||
static void ZDICT_removeDictItem(dictItem* table, U32 id)
|
||||
{
|
||||
/* convention : first element is nb of elts */
|
||||
U32 const max = table->pos;
|
||||
U32 max = table->pos;
|
||||
U32 u;
|
||||
if (!id) return; /* protection, should never happen */
|
||||
for (u=id; u<max-1; u++)
|
||||
@@ -465,7 +463,8 @@ static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt)
|
||||
}
|
||||
|
||||
/* insert */
|
||||
{ U32 current;
|
||||
{
|
||||
U32 current;
|
||||
U32 nextElt = table->pos;
|
||||
if (nextElt >= maxSize) nextElt = maxSize-1;
|
||||
current = nextElt-1;
|
||||
@@ -531,7 +530,8 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
|
||||
DISPLAYLEVEL(2, "finding patterns ... \n");
|
||||
DISPLAYLEVEL(3, "minimum ratio : %u \n", minRatio);
|
||||
|
||||
{ U32 cursor; for (cursor=0; cursor < bufferSize; ) {
|
||||
{
|
||||
U32 cursor; for (cursor=0; cursor < bufferSize; ) {
|
||||
dictItem solution;
|
||||
if (doneMarks[cursor]) { cursor++; continue; }
|
||||
solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio);
|
||||
@@ -542,7 +542,8 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
|
||||
} }
|
||||
|
||||
/* limit dictionary size */
|
||||
{ U32 const max = dictList->pos; /* convention : nb of useful elts within dictList */
|
||||
{
|
||||
U32 max = dictList->pos; /* convention : nb of useful elts within dictList */
|
||||
U32 currentSize = 0;
|
||||
U32 n; for (n=1; n<max; n++) {
|
||||
currentSize += dictList[n].length;
|
||||
@@ -578,10 +579,9 @@ typedef struct
|
||||
void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */
|
||||
} EStats_ress_t;
|
||||
|
||||
#define MAXREPOFFSET 1024
|
||||
|
||||
static void ZDICT_countEStats(EStats_ress_t esr,
|
||||
U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets,
|
||||
U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const seqStore_t* seqStorePtr;
|
||||
@@ -615,17 +615,6 @@ static void ZDICT_countEStats(EStats_ress_t esr,
|
||||
size_t u;
|
||||
for (u=0; u<nbSeq; u++) litlengthCount[codePtr[u]]++;
|
||||
} }
|
||||
|
||||
/* rep offsets */
|
||||
{ const U32* const offsetPtr = seqStorePtr->offsetStart;
|
||||
U32 offset1 = offsetPtr[0] - 3;
|
||||
U32 offset2 = offsetPtr[1] - 3;
|
||||
if (offset1 >= MAXREPOFFSET) offset1 = 0;
|
||||
if (offset2 >= MAXREPOFFSET) offset2 = 0;
|
||||
repOffsets[offset1] += 3;
|
||||
repOffsets[offset2] += 1;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -641,29 +630,12 @@ static size_t ZDICT_maxSampleSize(const size_t* fileSizes, unsigned nbFiles)
|
||||
|
||||
static size_t ZDICT_totalSampleSize(const size_t* fileSizes, unsigned nbFiles)
|
||||
{
|
||||
size_t total=0;
|
||||
size_t total;
|
||||
unsigned u;
|
||||
for (u=0; u<nbFiles; u++) total += fileSizes[u];
|
||||
for (u=0, total=0; u<nbFiles; u++) total += fileSizes[u];
|
||||
return total;
|
||||
}
|
||||
|
||||
typedef struct { U32 offset; U32 count; } offsetCount_t;
|
||||
|
||||
static void ZDICT_insertSortCount(offsetCount_t table[ZSTD_REP_NUM+1], U32 val, U32 count)
|
||||
{
|
||||
U32 u;
|
||||
table[ZSTD_REP_NUM].offset = val;
|
||||
table[ZSTD_REP_NUM].count = count;
|
||||
for (u=ZSTD_REP_NUM; u>0; u--) {
|
||||
offsetCount_t tmp;
|
||||
if (table[u-1].count >= table[u].count) break;
|
||||
tmp = table[u-1];
|
||||
table[u-1] = table[u];
|
||||
table[u] = tmp;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#define OFFCODE_MAX 18 /* only applicable to first block */
|
||||
static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
unsigned compressionLevel,
|
||||
@@ -678,8 +650,6 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
short matchLengthNCount[MaxML+1];
|
||||
U32 litLengthCount[MaxLL+1];
|
||||
short litLengthNCount[MaxLL+1];
|
||||
U32 repOffset[MAXREPOFFSET] = { 0 };
|
||||
offsetCount_t bestRepOffset[ZSTD_REP_NUM+1];
|
||||
EStats_ress_t esr;
|
||||
ZSTD_parameters params;
|
||||
U32 u, huffLog = 12, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total;
|
||||
@@ -687,15 +657,12 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
size_t eSize = 0;
|
||||
size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles);
|
||||
size_t const averageSampleSize = totalSrcSize / nbFiles;
|
||||
BYTE* dstPtr = (BYTE*)dstBuffer;
|
||||
|
||||
/* init */
|
||||
for (u=0; u<256; u++) countLit[u]=1; /* any character must be described */
|
||||
for (u=0; u<=OFFCODE_MAX; u++) offcodeCount[u]=1;
|
||||
for (u=0; u<=MaxML; u++) matchLengthCount[u]=1;
|
||||
for (u=0; u<=MaxLL; u++) litLengthCount[u]=1;
|
||||
repOffset[1] = repOffset[4] = repOffset[8] = 1;
|
||||
memset(bestRepOffset, 0, sizeof(bestRepOffset));
|
||||
esr.ref = ZSTD_createCCtx();
|
||||
esr.zc = ZSTD_createCCtx();
|
||||
esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX);
|
||||
@@ -713,7 +680,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
/* collect stats on all files */
|
||||
for (u=0; u<nbFiles; u++) {
|
||||
ZDICT_countEStats(esr,
|
||||
countLit, offcodeCount, matchLengthCount, litLengthCount, repOffset,
|
||||
countLit, offcodeCount, matchLengthCount, litLengthCount,
|
||||
(const char*)srcBuffer + pos, fileSizes[u]);
|
||||
pos += fileSizes[u];
|
||||
}
|
||||
@@ -727,13 +694,6 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
}
|
||||
huffLog = (U32)errorCode;
|
||||
|
||||
/* looking for most common first offsets */
|
||||
{ U32 offset;
|
||||
for (offset=1; offset<MAXREPOFFSET; offset++)
|
||||
ZDICT_insertSortCount(bestRepOffset, offset, repOffset[offset]);
|
||||
}
|
||||
/* note : the result of this phase should be used to better appreciate the impact on statistics */
|
||||
|
||||
total=0; for (u=0; u<=OFFCODE_MAX; u++) total+=offcodeCount[u];
|
||||
errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, OFFCODE_MAX);
|
||||
if (FSE_isError(errorCode)) {
|
||||
@@ -761,70 +721,46 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
}
|
||||
llLog = (U32)errorCode;
|
||||
|
||||
|
||||
/* write result to buffer */
|
||||
{ size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
|
||||
if (HUF_isError(hhSize)) {
|
||||
eSize = ERROR(GENERIC);
|
||||
DISPLAYLEVEL(1, "HUF_writeCTable error");
|
||||
goto _cleanup;
|
||||
}
|
||||
dstPtr += hhSize;
|
||||
maxDstSize -= hhSize;
|
||||
eSize += hhSize;
|
||||
}
|
||||
|
||||
{ size_t const ohSize = FSE_writeNCount(dstPtr, maxDstSize, offcodeNCount, OFFCODE_MAX, Offlog);
|
||||
if (FSE_isError(ohSize)) {
|
||||
eSize = ERROR(GENERIC);
|
||||
DISPLAYLEVEL(1, "FSE_writeNCount error with offcodeNCount");
|
||||
goto _cleanup;
|
||||
}
|
||||
dstPtr += ohSize;
|
||||
maxDstSize -= ohSize;
|
||||
eSize += ohSize;
|
||||
}
|
||||
|
||||
{ size_t const mhSize = FSE_writeNCount(dstPtr, maxDstSize, matchLengthNCount, MaxML, mlLog);
|
||||
if (FSE_isError(mhSize)) {
|
||||
eSize = ERROR(GENERIC);
|
||||
DISPLAYLEVEL(1, "FSE_writeNCount error with matchLengthNCount");
|
||||
goto _cleanup;
|
||||
}
|
||||
dstPtr += mhSize;
|
||||
maxDstSize -= mhSize;
|
||||
eSize += mhSize;
|
||||
}
|
||||
|
||||
{ size_t const lhSize = FSE_writeNCount(dstPtr, maxDstSize, litLengthNCount, MaxLL, llLog);
|
||||
if (FSE_isError(lhSize)) {
|
||||
eSize = ERROR(GENERIC);
|
||||
DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount");
|
||||
goto _cleanup;
|
||||
}
|
||||
dstPtr += lhSize;
|
||||
maxDstSize -= lhSize;
|
||||
eSize += lhSize;
|
||||
}
|
||||
|
||||
if (maxDstSize<12) {
|
||||
errorCode = HUF_writeCTable(dstBuffer, maxDstSize, hufTable, 255, huffLog);
|
||||
if (HUF_isError(errorCode)) {
|
||||
eSize = ERROR(GENERIC);
|
||||
DISPLAYLEVEL(1, "not enough space to write RepOffsets");
|
||||
DISPLAYLEVEL(1, "HUF_writeCTable error");
|
||||
goto _cleanup;
|
||||
}
|
||||
# if 0
|
||||
MEM_writeLE32(dstPtr+0, bestRepOffset[0].offset);
|
||||
MEM_writeLE32(dstPtr+4, bestRepOffset[1].offset);
|
||||
MEM_writeLE32(dstPtr+8, bestRepOffset[2].offset);
|
||||
#else
|
||||
/* at this stage, we don't use the result of "most common first offset",
|
||||
as the impact of statistics is not properly evaluated */
|
||||
MEM_writeLE32(dstPtr+0, repStartValue[0]);
|
||||
MEM_writeLE32(dstPtr+4, repStartValue[1]);
|
||||
MEM_writeLE32(dstPtr+8, repStartValue[2]);
|
||||
#endif
|
||||
dstPtr += 12;
|
||||
eSize += 12;
|
||||
dstBuffer = (char*)dstBuffer + errorCode;
|
||||
maxDstSize -= errorCode;
|
||||
eSize += errorCode;
|
||||
|
||||
errorCode = FSE_writeNCount(dstBuffer, maxDstSize, offcodeNCount, OFFCODE_MAX, Offlog);
|
||||
if (FSE_isError(errorCode)) {
|
||||
eSize = ERROR(GENERIC);
|
||||
DISPLAYLEVEL(1, "FSE_writeNCount error with offcodeNCount");
|
||||
goto _cleanup;
|
||||
}
|
||||
dstBuffer = (char*)dstBuffer + errorCode;
|
||||
maxDstSize -= errorCode;
|
||||
eSize += errorCode;
|
||||
|
||||
errorCode = FSE_writeNCount(dstBuffer, maxDstSize, matchLengthNCount, MaxML, mlLog);
|
||||
if (FSE_isError(errorCode)) {
|
||||
eSize = ERROR(GENERIC);
|
||||
DISPLAYLEVEL(1, "FSE_writeNCount error with matchLengthNCount");
|
||||
goto _cleanup;
|
||||
}
|
||||
dstBuffer = (char*)dstBuffer + errorCode;
|
||||
maxDstSize -= errorCode;
|
||||
eSize += errorCode;
|
||||
|
||||
errorCode = FSE_writeNCount(dstBuffer, maxDstSize, litLengthNCount, MaxLL, llLog);
|
||||
if (FSE_isError(errorCode)) {
|
||||
eSize = ERROR(GENERIC);
|
||||
DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount");
|
||||
goto _cleanup;
|
||||
}
|
||||
dstBuffer = (char*)dstBuffer + errorCode;
|
||||
maxDstSize -= errorCode;
|
||||
eSize += errorCode;
|
||||
|
||||
_cleanup:
|
||||
ZSTD_freeCCtx(esr.ref);
|
||||
@@ -849,7 +785,7 @@ static size_t ZDICT_fastSampling(void* dictBuffer, size_t dictSize,
|
||||
{
|
||||
char* dstPtr = (char*)dictBuffer + dictSize;
|
||||
const char* srcPtr = (const char*)samplesBuffer;
|
||||
size_t const nbSegments = dictSize / DIB_FASTSEGMENTSIZE;
|
||||
size_t nbSegments = dictSize / DIB_FASTSEGMENTSIZE;
|
||||
size_t segNb, interSize;
|
||||
|
||||
if (nbSegments <= 2) return ERROR(srcSize_wrong);
|
||||
@@ -874,33 +810,6 @@ static size_t ZDICT_fastSampling(void* dictBuffer, size_t dictSize,
|
||||
return nbSegments * DIB_FASTSEGMENTSIZE;
|
||||
}
|
||||
|
||||
size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_params_t params)
|
||||
{
|
||||
size_t hSize;
|
||||
unsigned const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
|
||||
|
||||
/* dictionary header */
|
||||
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
|
||||
{ U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0);
|
||||
U32 const dictID = params.dictID ? params.dictID : (U32)(randomID>>11);
|
||||
MEM_writeLE32((char*)dictBuffer+4, dictID);
|
||||
}
|
||||
hSize = 8;
|
||||
|
||||
/* entropy tables */
|
||||
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
|
||||
DISPLAYLEVEL(2, "statistics ... \n");
|
||||
hSize += ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
|
||||
compressionLevel,
|
||||
samplesBuffer, samplesSizes, nbSamples,
|
||||
(char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
|
||||
|
||||
if (hSize + dictContentSize < dictBufferCapacity)
|
||||
memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
|
||||
return MIN(dictBufferCapacity, hSize+dictContentSize);
|
||||
}
|
||||
|
||||
#define DIB_MINSAMPLESSIZE (DIB_FASTSEGMENTSIZE*3)
|
||||
/*! ZDICT_trainFromBuffer_unsafe() :
|
||||
@@ -909,13 +818,14 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
|
||||
*/
|
||||
size_t ZDICT_trainFromBuffer_unsafe(
|
||||
void* dictBuffer, size_t maxDictSize,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
const void* samplesBuffer, const size_t* sampleSizes, unsigned nbSamples,
|
||||
ZDICT_params_t params)
|
||||
{
|
||||
U32 const dictListSize = MAX( MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16));
|
||||
dictItem* const dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList));
|
||||
dictItem* dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList));
|
||||
unsigned selectivity = params.selectivityLevel;
|
||||
size_t const targetDictSize = maxDictSize;
|
||||
unsigned compressionLevel = params.compressionLevel;
|
||||
size_t targetDictSize = maxDictSize;
|
||||
size_t sBuffSize;
|
||||
size_t dictSize = 0;
|
||||
|
||||
@@ -924,17 +834,18 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
||||
if (!dictList) return ERROR(memory_allocation);
|
||||
|
||||
/* init */
|
||||
{ unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += samplesSizes[u]; }
|
||||
{ unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += sampleSizes[u]; }
|
||||
if (sBuffSize < DIB_MINSAMPLESSIZE) return 0; /* not enough source to create dictionary */
|
||||
ZDICT_initDictItem(dictList);
|
||||
g_displayLevel = params.notificationLevel;
|
||||
if (selectivity==0) selectivity = g_selectivity_default;
|
||||
if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
|
||||
|
||||
/* build dictionary */
|
||||
if (selectivity>1) { /* selectivity == 1 => fast mode */
|
||||
ZDICT_trainBuffer(dictList, dictListSize,
|
||||
samplesBuffer, sBuffSize,
|
||||
samplesSizes, nbSamples,
|
||||
sampleSizes, nbSamples,
|
||||
selectivity, (U32)targetDictSize);
|
||||
|
||||
/* display best matches */
|
||||
@@ -956,16 +867,18 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
||||
|
||||
/* create dictionary */
|
||||
{ U32 dictContentSize = ZDICT_dictSize(dictList);
|
||||
size_t hSize;
|
||||
BYTE* ptr;
|
||||
U32 u;
|
||||
|
||||
/* build dict content */
|
||||
{ U32 u;
|
||||
BYTE* ptr = (BYTE*)dictBuffer + maxDictSize;
|
||||
for (u=1; u<dictList->pos; u++) {
|
||||
U32 l = dictList[u].length;
|
||||
ptr -= l;
|
||||
if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */
|
||||
memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l);
|
||||
} }
|
||||
ptr = (BYTE*)dictBuffer + maxDictSize;
|
||||
for (u=1; u<dictList->pos; u++) {
|
||||
U32 l = dictList[u].length;
|
||||
ptr -= l;
|
||||
if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */
|
||||
memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l);
|
||||
}
|
||||
|
||||
/* fast mode dict content */
|
||||
if (selectivity==1) { /* note could also be used to complete a dictionary, but not necessarily better */
|
||||
@@ -975,9 +888,21 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
||||
samplesBuffer, sBuffSize);
|
||||
}
|
||||
|
||||
dictSize = ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, maxDictSize,
|
||||
samplesBuffer, samplesSizes, nbSamples,
|
||||
params);
|
||||
/* dictionary header */
|
||||
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
|
||||
hSize = 4;
|
||||
|
||||
/* entropic tables */
|
||||
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
|
||||
DISPLAYLEVEL(2, "statistics ... \n");
|
||||
hSize += ZDICT_analyzeEntropy((char*)dictBuffer+4, maxDictSize-4,
|
||||
compressionLevel,
|
||||
samplesBuffer, sampleSizes, nbSamples,
|
||||
(char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize);
|
||||
|
||||
if (hSize + dictContentSize < maxDictSize)
|
||||
memmove((char*)dictBuffer + hSize, (char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize);
|
||||
dictSize = MIN(maxDictSize, hSize+dictContentSize);
|
||||
}
|
||||
|
||||
/* clean up */
|
||||
@@ -989,8 +914,8 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
||||
/* issue : samplesBuffer need to be followed by a noisy guard band.
|
||||
* work around : duplicate the buffer, and add the noise */
|
||||
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_params_t params)
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_params_t params)
|
||||
{
|
||||
void* newBuff;
|
||||
size_t sBuffSize;
|
||||
@@ -1022,12 +947,3 @@ size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
||||
params);
|
||||
}
|
||||
|
||||
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples)
|
||||
{
|
||||
ZDICT_params_t params;
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
return ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, dictBufferCapacity,
|
||||
samplesBuffer, samplesSizes, nbSamples,
|
||||
params);
|
||||
}
|
||||
|
||||
@@ -52,21 +52,6 @@ extern "C" {
|
||||
size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
|
||||
|
||||
/*! ZDICT_addEntropyTablesFromBuffer() :
|
||||
|
||||
Given a content-only dictionary (built for example from common strings in
|
||||
the input), add entropy tables computed from the memory buffer
|
||||
`samplesBuffer`, where `nbSamples` samples have been stored concatenated.
|
||||
Each sample size is provided into an orderly table `samplesSizes`.
|
||||
|
||||
The input dictionary is the last `dictContentSize` bytes of `dictBuffer`. The
|
||||
resulting dictionary with added entropy tables will written back to
|
||||
`dictBuffer`.
|
||||
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`).
|
||||
*/
|
||||
size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Helper functions
|
||||
@@ -75,44 +60,6 @@ unsigned ZDICT_isError(size_t errorCode);
|
||||
const char* ZDICT_getErrorName(size_t errorCode);
|
||||
|
||||
|
||||
#ifdef ZDICT_STATIC_LINKING_ONLY
|
||||
|
||||
/* ====================================================================================
|
||||
* The definitions in this section are considered experimental.
|
||||
* They should never be used with a dynamic library, as they may change in the future.
|
||||
* They are provided for advanced usages.
|
||||
* Use them only in association with static linking.
|
||||
* ==================================================================================== */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Public type
|
||||
***************************************/
|
||||
typedef struct {
|
||||
unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */
|
||||
unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */
|
||||
unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
||||
unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */
|
||||
unsigned reserved[2]; /* space for future parameters */
|
||||
} ZDICT_params_t;
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Public functions
|
||||
***************************************/
|
||||
/*! ZDICT_trainFromBuffer_advanced() :
|
||||
Same as ZDICT_trainFromBuffer() with control over more parameters.
|
||||
`parameters` is optional and can be provided with values set to 0 to mean "default".
|
||||
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`)
|
||||
or an error code, which can be tested by ZDICT_isError().
|
||||
note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using ZDICT_setNotificationLevel()
|
||||
*/
|
||||
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_params_t parameters);
|
||||
|
||||
#endif /* ZDICT_STATIC_LINKING_ONLY */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
dictBuilder header file
|
||||
for static linking only
|
||||
Copyright (C) Yann Collet 2016
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Zstd source repository : https://www.zstd.net
|
||||
*/
|
||||
|
||||
/* This library is EXPERIMENTAL, below API is not yet stable */
|
||||
|
||||
#ifndef DICTBUILDER_STATIC_H_002
|
||||
#define DICTBUILDER_STATIC_H_002
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#include "zdict.h"
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Public type
|
||||
***************************************/
|
||||
typedef struct {
|
||||
unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */
|
||||
unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */
|
||||
unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
|
||||
unsigned reserved[3]; /* space for future parameters */
|
||||
} ZDICT_params_t;
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Public functions
|
||||
***************************************/
|
||||
/*! ZDICT_trainFromBuffer_advanced() :
|
||||
Same as ZDICT_trainFromBuffer() with control over more parameters.
|
||||
`parameters` is optional and can be provided with values set to 0 to mean "default".
|
||||
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`)
|
||||
or an error code, which can be tested by DiB_isError().
|
||||
note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using ZDICT_setNotificationLevel()
|
||||
*/
|
||||
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
|
||||
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
|
||||
ZDICT_params_t parameters);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* DICTBUILDER_STATIC_H_002 */
|
||||
+28
-17
@@ -47,7 +47,8 @@ extern "C" {
|
||||
#include "zstd_v03.h"
|
||||
#include "zstd_v04.h"
|
||||
#include "zstd_v05.h"
|
||||
#include "zstd_v06.h"
|
||||
#include "zstd_v07.h"
|
||||
#include "zstd_v08.h"
|
||||
|
||||
|
||||
/** ZSTD_isLegacy() :
|
||||
@@ -56,16 +57,17 @@ extern "C" {
|
||||
*/
|
||||
MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
|
||||
{
|
||||
switch(magicNumberLE)
|
||||
{
|
||||
case ZSTDv01_magicNumberLE:return 1;
|
||||
case ZSTDv02_magicNumber : return 2;
|
||||
case ZSTDv03_magicNumber : return 3;
|
||||
case ZSTDv04_magicNumber : return 4;
|
||||
case ZSTDv05_MAGICNUMBER : return 5;
|
||||
case ZSTDv06_MAGICNUMBER : return 6;
|
||||
default : return 0;
|
||||
}
|
||||
switch(magicNumberLE)
|
||||
{
|
||||
case ZSTDv01_magicNumberLE:return 1;
|
||||
case ZSTDv02_magicNumber : return 2;
|
||||
case ZSTDv03_magicNumber : return 3;
|
||||
case ZSTDv04_magicNumber : return 4;
|
||||
case ZSTDv05_MAGICNUMBER : return 5;
|
||||
case ZSTDv07_MAGICNUMBER : return 7;
|
||||
case ZSTDv08_MAGICNUMBER : return 8;
|
||||
default : return 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -86,19 +88,28 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
|
||||
case ZSTDv04_magicNumber :
|
||||
return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize);
|
||||
case ZSTDv05_MAGICNUMBER :
|
||||
{ size_t result;
|
||||
ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx();
|
||||
{
|
||||
size_t result;
|
||||
ZSTDv05_DCtx* zd = ZSTDv05_createDCtx();
|
||||
if (zd==NULL) return ERROR(memory_allocation);
|
||||
result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize);
|
||||
ZSTDv05_freeDCtx(zd);
|
||||
return result;
|
||||
}
|
||||
case ZSTDv06_MAGICNUMBER :
|
||||
case ZSTDv07_MAGICNUMBER :
|
||||
{ size_t result;
|
||||
ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx();
|
||||
ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx();
|
||||
if (zd==NULL) return ERROR(memory_allocation);
|
||||
result = ZSTDv06_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize);
|
||||
ZSTDv06_freeDCtx(zd);
|
||||
result = ZSTDv07_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize);
|
||||
ZSTDv07_freeDCtx(zd);
|
||||
return result;
|
||||
}
|
||||
case ZSTDv08_MAGICNUMBER :
|
||||
{ size_t result;
|
||||
ZSTDv08_DCtx* const zd = ZSTDv08_createDCtx();
|
||||
if (zd==NULL) return ERROR(memory_allocation);
|
||||
result = ZSTDv08_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize);
|
||||
ZSTDv08_freeDCtx(zd);
|
||||
return result;
|
||||
}
|
||||
default :
|
||||
|
||||
@@ -1350,7 +1350,7 @@ static unsigned FSE_isError(size_t code) { return ERR_isError(code); }
|
||||
****************************************************************/
|
||||
static short FSE_abs(short a)
|
||||
{
|
||||
return a<0 ? -a : a;
|
||||
return (short)(a<0 ? -a : a);
|
||||
}
|
||||
|
||||
static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
|
||||
+28
-33
@@ -3620,36 +3620,26 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi
|
||||
switch (ctx->stage)
|
||||
{
|
||||
case ZSTDds_getFrameHeaderSize :
|
||||
{
|
||||
/* get frame header size */
|
||||
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||
ctx->headerSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min);
|
||||
if (ZSTD_isError(ctx->headerSize)) return ctx->headerSize;
|
||||
memcpy(ctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
|
||||
if (ctx->headerSize > ZSTD_frameHeaderSize_min)
|
||||
{
|
||||
ctx->expected = ctx->headerSize - ZSTD_frameHeaderSize_min;
|
||||
ctx->stage = ZSTDds_decodeFrameHeader;
|
||||
return 0;
|
||||
}
|
||||
ctx->expected = 0; /* not necessary to copy more */
|
||||
}
|
||||
/* get frame header size */
|
||||
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||
ctx->headerSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min);
|
||||
if (ZSTD_isError(ctx->headerSize)) return ctx->headerSize;
|
||||
memcpy(ctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
|
||||
if (ctx->headerSize > ZSTD_frameHeaderSize_min) return ERROR(GENERIC); /* impossible */
|
||||
ctx->expected = 0; /* not necessary to copy more */
|
||||
/* fallthrough */
|
||||
case ZSTDds_decodeFrameHeader:
|
||||
{
|
||||
/* get frame header */
|
||||
size_t result;
|
||||
memcpy(ctx->headerBuffer + ZSTD_frameHeaderSize_min, src, ctx->expected);
|
||||
result = ZSTD_decodeFrameHeader_Part2(ctx, ctx->headerBuffer, ctx->headerSize);
|
||||
/* get frame header */
|
||||
{ size_t const result = ZSTD_decodeFrameHeader_Part2(ctx, ctx->headerBuffer, ctx->headerSize);
|
||||
if (ZSTD_isError(result)) return result;
|
||||
ctx->expected = ZSTD_blockHeaderSize;
|
||||
ctx->stage = ZSTDds_decodeBlockHeader;
|
||||
return 0;
|
||||
}
|
||||
case ZSTDds_decodeBlockHeader:
|
||||
{
|
||||
/* Decode block header */
|
||||
blockProperties_t bp;
|
||||
size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
|
||||
/* Decode block header */
|
||||
{ blockProperties_t bp;
|
||||
size_t const blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
|
||||
if (ZSTD_isError(blockSize)) return blockSize;
|
||||
if (bp.blockType == bt_end)
|
||||
{
|
||||
@@ -3864,11 +3854,9 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
|
||||
|
||||
case ZBUFFds_readHeader :
|
||||
/* read header from src */
|
||||
{
|
||||
size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr);
|
||||
{ size_t const headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr);
|
||||
if (ZSTD_isError(headerSize)) return headerSize;
|
||||
if (headerSize)
|
||||
{
|
||||
if (headerSize) {
|
||||
/* not enough input to decode header : tell how many bytes would be necessary */
|
||||
memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr);
|
||||
zbc->hPos += *srcSizePtr;
|
||||
@@ -3882,8 +3870,7 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
|
||||
|
||||
case ZBUFFds_loadHeader:
|
||||
/* complete header from src */
|
||||
{
|
||||
size_t headerSize = ZBUFF_limitCopy(
|
||||
{ size_t headerSize = ZBUFF_limitCopy(
|
||||
zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos,
|
||||
src, *srcSizePtr);
|
||||
zbc->hPos += headerSize;
|
||||
@@ -3895,12 +3882,12 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs
|
||||
*maxDstSizePtr = 0;
|
||||
return headerSize - zbc->hPos;
|
||||
} }
|
||||
/* intentional fallthrough */
|
||||
|
||||
case ZBUFFds_decodeHeader:
|
||||
/* apply header to create / resize buffers */
|
||||
{
|
||||
size_t neededOutSize = (size_t)1 << zbc->params.windowLog;
|
||||
size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
|
||||
{ size_t const neededOutSize = (size_t)1 << zbc->params.windowLog;
|
||||
size_t const neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
|
||||
if (zbc->inBuffSize < neededInSize) {
|
||||
free(zbc->inBuff);
|
||||
zbc->inBuffSize = neededInSize;
|
||||
@@ -4037,7 +4024,7 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
|
||||
return regenSize;
|
||||
#else
|
||||
ZSTD_DCtx dctx;
|
||||
return ZSTD_decompressDCtx(&dctx, dst, maxDstSize, src, srcSize);
|
||||
return ZSTDv04_decompressDCtx(&dctx, dst, maxDstSize, src, srcSize);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -4067,3 +4054,11 @@ size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDs
|
||||
{
|
||||
return ZBUFF_decompressContinue(dctx, dst, maxDstSizePtr, src, srcSizePtr);
|
||||
}
|
||||
|
||||
ZSTD_DCtx* ZSTDv04_createDCtx(void) { return ZSTD_createDCtx(); }
|
||||
size_t ZSTDv04_freeDCtx(ZSTD_DCtx* dctx) { return ZSTD_freeDCtx(dctx); }
|
||||
|
||||
size_t ZSTDv04_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_getFrameParams(params, src, srcSize);
|
||||
}
|
||||
|
||||
+10
-18
@@ -3872,25 +3872,17 @@ size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t maxDstSi
|
||||
switch (dctx->stage)
|
||||
{
|
||||
case ZSTDv05ds_getFrameHeaderSize :
|
||||
{
|
||||
/* get frame header size */
|
||||
if (srcSize != ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||
dctx->headerSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min);
|
||||
if (ZSTDv05_isError(dctx->headerSize)) return dctx->headerSize;
|
||||
memcpy(dctx->headerBuffer, src, ZSTDv05_frameHeaderSize_min);
|
||||
if (dctx->headerSize > ZSTDv05_frameHeaderSize_min) {
|
||||
dctx->expected = dctx->headerSize - ZSTDv05_frameHeaderSize_min;
|
||||
dctx->stage = ZSTDv05ds_decodeFrameHeader;
|
||||
return 0;
|
||||
}
|
||||
dctx->expected = 0; /* not necessary to copy more */
|
||||
}
|
||||
/* get frame header size */
|
||||
if (srcSize != ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||
dctx->headerSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min);
|
||||
if (ZSTDv05_isError(dctx->headerSize)) return dctx->headerSize;
|
||||
memcpy(dctx->headerBuffer, src, ZSTDv05_frameHeaderSize_min);
|
||||
if (dctx->headerSize > ZSTDv05_frameHeaderSize_min) return ERROR(GENERIC); /* should never happen */
|
||||
dctx->expected = 0; /* not necessary to copy more */
|
||||
/* fallthrough */
|
||||
case ZSTDv05ds_decodeFrameHeader:
|
||||
{
|
||||
/* get frame header */
|
||||
size_t result;
|
||||
memcpy(dctx->headerBuffer + ZSTDv05_frameHeaderSize_min, src, dctx->expected);
|
||||
result = ZSTDv05_decodeFrameHeader_Part2(dctx, dctx->headerBuffer, dctx->headerSize);
|
||||
/* get frame header */
|
||||
{ size_t const result = ZSTDv05_decodeFrameHeader_Part2(dctx, dctx->headerBuffer, dctx->headerSize);
|
||||
if (ZSTDv05_isError(result)) return result;
|
||||
dctx->expected = ZSTDv05_blockHeaderSize;
|
||||
dctx->stage = ZSTDv05ds_decodeBlockHeader;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/* ******************************************************************
|
||||
zstd_v06.c
|
||||
Decompression module for ZSTD v0.6 legacy format
|
||||
zstd_v07.c
|
||||
Decompression module for ZSTD v0.7 legacy format
|
||||
Copyright (C) 2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
@@ -33,11 +33,169 @@
|
||||
****************************************************************** */
|
||||
|
||||
/*- Dependencies -*/
|
||||
#include "zstd_v06.h"
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include <string.h> /* memcpy */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include <string.h> /* memcpy */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
#include "zstd_v07.h"
|
||||
|
||||
#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */
|
||||
#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
|
||||
#define ZSTDv07_STATIC_LINKING_ONLY
|
||||
|
||||
|
||||
#ifdef ZSTDv07_STATIC_LINKING_ONLY
|
||||
|
||||
/* ====================================================================================
|
||||
* The definitions in this section are considered experimental.
|
||||
* They should never be used with a dynamic library, as they may change in the future.
|
||||
* They are provided for advanced usages.
|
||||
* Use them only in association with static linking.
|
||||
* ==================================================================================== */
|
||||
|
||||
/*--- Constants ---*/
|
||||
#define ZSTDv07_MAGIC_SKIPPABLE_START 0x184D2A50U
|
||||
|
||||
#define ZSTDv07_WINDOWLOG_MAX_32 25
|
||||
#define ZSTDv07_WINDOWLOG_MAX_64 27
|
||||
#define ZSTDv07_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTDv07_WINDOWLOG_MAX_32 : ZSTDv07_WINDOWLOG_MAX_64))
|
||||
#define ZSTDv07_WINDOWLOG_MIN 18
|
||||
#define ZSTDv07_CHAINLOG_MAX (ZSTDv07_WINDOWLOG_MAX+1)
|
||||
#define ZSTDv07_CHAINLOG_MIN 4
|
||||
#define ZSTDv07_HASHLOG_MAX ZSTDv07_WINDOWLOG_MAX
|
||||
#define ZSTDv07_HASHLOG_MIN 12
|
||||
#define ZSTDv07_HASHLOG3_MAX 17
|
||||
#define ZSTDv07_SEARCHLOG_MAX (ZSTDv07_WINDOWLOG_MAX-1)
|
||||
#define ZSTDv07_SEARCHLOG_MIN 1
|
||||
#define ZSTDv07_SEARCHLENGTH_MAX 7
|
||||
#define ZSTDv07_SEARCHLENGTH_MIN 3
|
||||
#define ZSTDv07_TARGETLENGTH_MIN 4
|
||||
#define ZSTDv07_TARGETLENGTH_MAX 999
|
||||
|
||||
#define ZSTDv07_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
|
||||
static const size_t ZSTDv07_frameHeaderSize_min = 5;
|
||||
static const size_t ZSTDv07_frameHeaderSize_max = ZSTDv07_FRAMEHEADERSIZE_MAX;
|
||||
static const size_t ZSTDv07_skippableHeaderSize = 8; /* magic number + skippable frame length */
|
||||
|
||||
|
||||
/* custom memory allocation functions */
|
||||
typedef void* (*ZSTDv07_allocFunction) (void* opaque, size_t size);
|
||||
typedef void (*ZSTDv07_freeFunction) (void* opaque, void* address);
|
||||
typedef struct { ZSTDv07_allocFunction customAlloc; ZSTDv07_freeFunction customFree; void* opaque; } ZSTDv07_customMem;
|
||||
|
||||
|
||||
/*--- Advanced Decompression functions ---*/
|
||||
|
||||
/*! ZSTDv07_estimateDCtxSize() :
|
||||
* Gives the potential amount of memory allocated to create a ZSTDv07_DCtx */
|
||||
ZSTDLIB_API size_t ZSTDv07_estimateDCtxSize(void);
|
||||
|
||||
/*! ZSTDv07_createDCtx_advanced() :
|
||||
* Create a ZSTD decompression context using external alloc and free functions */
|
||||
ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem);
|
||||
|
||||
/*! ZSTDv07_sizeofDCtx() :
|
||||
* Gives the amount of memory used by a given ZSTDv07_DCtx */
|
||||
ZSTDLIB_API size_t ZSTDv07_sizeofDCtx(const ZSTDv07_DCtx* dctx);
|
||||
|
||||
|
||||
/* ******************************************************************
|
||||
* Buffer-less streaming functions (synchronous mode)
|
||||
********************************************************************/
|
||||
|
||||
ZSTDLIB_API size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* preparedDCtx);
|
||||
|
||||
ZSTDLIB_API size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
/*
|
||||
Buffer-less streaming decompression (synchronous mode)
|
||||
|
||||
A ZSTDv07_DCtx object is required to track streaming operations.
|
||||
Use ZSTDv07_createDCtx() / ZSTDv07_freeDCtx() to manage it.
|
||||
A ZSTDv07_DCtx object can be re-used multiple times.
|
||||
|
||||
First optional operation is to retrieve frame parameters, using ZSTDv07_getFrameParams(), which doesn't consume the input.
|
||||
It can provide the minimum size of rolling buffer required to properly decompress data (`windowSize`),
|
||||
and optionally the final size of uncompressed content.
|
||||
(Note : content size is an optional info that may not be present. 0 means : content size unknown)
|
||||
Frame parameters are extracted from the beginning of compressed frame.
|
||||
The amount of data to read is variable, from ZSTDv07_frameHeaderSize_min to ZSTDv07_frameHeaderSize_max (so if `srcSize` >= ZSTDv07_frameHeaderSize_max, it will always work)
|
||||
If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result.
|
||||
Result : 0 when successful, it means the ZSTDv07_frameParams structure has been filled.
|
||||
>0 : means there is not enough data into `src`. Provides the expected size to successfully decode header.
|
||||
errorCode, which can be tested using ZSTDv07_isError()
|
||||
|
||||
Start decompression, with ZSTDv07_decompressBegin() or ZSTDv07_decompressBegin_usingDict().
|
||||
Alternatively, you can copy a prepared context, using ZSTDv07_copyDCtx().
|
||||
|
||||
Then use ZSTDv07_nextSrcSizeToDecompress() and ZSTDv07_decompressContinue() alternatively.
|
||||
ZSTDv07_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTDv07_decompressContinue().
|
||||
ZSTDv07_decompressContinue() requires this exact amount of bytes, or it will fail.
|
||||
|
||||
@result of ZSTDv07_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
|
||||
It can be zero, which is not an error; it just means ZSTDv07_decompressContinue() has decoded some header.
|
||||
|
||||
ZSTDv07_decompressContinue() needs previous data blocks during decompression, up to `windowSize`.
|
||||
They should preferably be located contiguously, prior to current block.
|
||||
Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters.
|
||||
ZSTDv07_decompressContinue() is very sensitive to contiguity,
|
||||
if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
|
||||
or that previous contiguous segment is large enough to properly handle maximum back-reference.
|
||||
|
||||
A frame is fully decoded when ZSTDv07_nextSrcSizeToDecompress() returns zero.
|
||||
Context can then be reset to start a new decompression.
|
||||
|
||||
|
||||
== Special case : skippable frames ==
|
||||
|
||||
Skippable frames allow the integration of user-defined data into a flow of concatenated frames.
|
||||
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following:
|
||||
a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
|
||||
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
|
||||
c) Frame Content - any content (User Data) of length equal to Frame Size
|
||||
For skippable frames ZSTDv07_decompressContinue() always returns 0.
|
||||
For skippable frames ZSTDv07_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
|
||||
It also returns Frame Size as fparamsPtr->frameContentSize.
|
||||
*/
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Block functions
|
||||
****************************************/
|
||||
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
|
||||
A few rules to respect :
|
||||
- Compressing and decompressing require a context structure
|
||||
+ Use ZSTDv07_createCCtx() and ZSTDv07_createDCtx()
|
||||
- It is necessary to init context before starting
|
||||
+ compression : ZSTDv07_compressBegin()
|
||||
+ decompression : ZSTDv07_decompressBegin()
|
||||
+ variants _usingDict() are also allowed
|
||||
+ copyCCtx() and copyDCtx() work too
|
||||
- Block size is limited, it must be <= ZSTDv07_getBlockSizeMax()
|
||||
+ If you need to compress more, cut data into multiple blocks
|
||||
+ Consider using the regular ZSTDv07_compress() instead, as frame metadata costs become negligible when source size is large.
|
||||
- When a block is considered not compressible enough, ZSTDv07_compressBlock() result will be zero.
|
||||
In which case, nothing is produced into `dst`.
|
||||
+ User must test for such outcome and deal directly with uncompressed data
|
||||
+ ZSTDv07_decompressBlock() doesn't accept uncompressed data as input !!!
|
||||
+ In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history.
|
||||
Use ZSTDv07_insertBlock() in such a case.
|
||||
*/
|
||||
|
||||
#define ZSTDv07_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
|
||||
ZSTDLIB_API size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
|
||||
|
||||
|
||||
#endif /* ZSTDv07_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
/* ******************************************************************
|
||||
@@ -81,10 +239,13 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Compiler specifics
|
||||
******************************************/
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
# include <stdlib.h> /* _byteswap_ulong */
|
||||
# include <intrin.h> /* _byteswap_* */
|
||||
#endif
|
||||
#if defined(__GNUC__)
|
||||
# define MEM_STATIC static __attribute__((unused))
|
||||
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
@@ -95,6 +256,10 @@ extern "C" {
|
||||
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
/* code only tested on 32 and 64 bits systems */
|
||||
#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
|
||||
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Basic Types
|
||||
@@ -449,26 +614,32 @@ extern "C" {
|
||||
* error codes list
|
||||
******************************************/
|
||||
typedef enum {
|
||||
ZSTDv06_error_no_error,
|
||||
ZSTDv06_error_GENERIC,
|
||||
ZSTDv06_error_prefix_unknown,
|
||||
ZSTDv06_error_frameParameter_unsupported,
|
||||
ZSTDv06_error_frameParameter_unsupportedBy32bits,
|
||||
ZSTDv06_error_compressionParameter_unsupported,
|
||||
ZSTDv06_error_init_missing,
|
||||
ZSTDv06_error_memory_allocation,
|
||||
ZSTDv06_error_stage_wrong,
|
||||
ZSTDv06_error_dstSize_tooSmall,
|
||||
ZSTDv06_error_srcSize_wrong,
|
||||
ZSTDv06_error_corruption_detected,
|
||||
ZSTDv06_error_tableLog_tooLarge,
|
||||
ZSTDv06_error_maxSymbolValue_tooLarge,
|
||||
ZSTDv06_error_maxSymbolValue_tooSmall,
|
||||
ZSTDv06_error_dictionary_corrupted,
|
||||
ZSTDv06_error_maxCode
|
||||
} ZSTDv06_ErrorCode;
|
||||
ZSTDv07_error_no_error,
|
||||
ZSTDv07_error_GENERIC,
|
||||
ZSTDv07_error_prefix_unknown,
|
||||
ZSTDv07_error_frameParameter_unsupported,
|
||||
ZSTDv07_error_frameParameter_unsupportedBy32bits,
|
||||
ZSTDv07_error_compressionParameter_unsupported,
|
||||
ZSTDv07_error_init_missing,
|
||||
ZSTDv07_error_memory_allocation,
|
||||
ZSTDv07_error_stage_wrong,
|
||||
ZSTDv07_error_dstSize_tooSmall,
|
||||
ZSTDv07_error_srcSize_wrong,
|
||||
ZSTDv07_error_corruption_detected,
|
||||
ZSTDv07_error_checksum_wrong,
|
||||
ZSTDv07_error_tableLog_tooLarge,
|
||||
ZSTDv07_error_maxSymbolValue_tooLarge,
|
||||
ZSTDv07_error_maxSymbolValue_tooSmall,
|
||||
ZSTDv07_error_dictionary_corrupted,
|
||||
ZSTDv07_error_dictionary_wrong,
|
||||
ZSTDv07_error_maxCode
|
||||
} ZSTDv07_ErrorCode;
|
||||
|
||||
/* note : compare with size_t function results using ZSTDv06_getError() */
|
||||
/*! ZSTDv07_getErrorCode() :
|
||||
convert a `size_t` function result into a `ZSTDv07_ErrorCode` enum type,
|
||||
which can be used to compare directly with enum list published into "error_public.h" */
|
||||
ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t functionResult);
|
||||
const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
@@ -476,148 +647,6 @@ typedef enum {
|
||||
#endif
|
||||
|
||||
#endif /* ERROR_PUBLIC_H_MODULE */
|
||||
/*
|
||||
zstd - standard compression library
|
||||
Header File for static linking only
|
||||
Copyright (C) 2014-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net
|
||||
*/
|
||||
#ifndef ZSTDv06_STATIC_H
|
||||
#define ZSTDv06_STATIC_H
|
||||
|
||||
/* The prototypes defined within this file are considered experimental.
|
||||
* They should not be used in the context DLL as they may change in the future.
|
||||
* Prefer static linking if you need them, to control breaking version changes issues.
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*- Advanced Decompression functions -*/
|
||||
|
||||
/*! ZSTDv06_decompress_usingPreparedDCtx() :
|
||||
* Same as ZSTDv06_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
|
||||
* It avoids reloading the dictionary each time.
|
||||
* `preparedDCtx` must have been properly initialized using ZSTDv06_decompressBegin_usingDict().
|
||||
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
|
||||
ZSTDLIB_API size_t ZSTDv06_decompress_usingPreparedDCtx(
|
||||
ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
|
||||
|
||||
struct ZSTDv06_frameParams_s { U64 frameContentSize; U32 windowLog; };
|
||||
|
||||
#define ZSTDv06_FRAMEHEADERSIZE_MAX 13 /* for static allocation */
|
||||
static const size_t ZSTDv06_frameHeaderSize_min = 5;
|
||||
static const size_t ZSTDv06_frameHeaderSize_max = ZSTDv06_FRAMEHEADERSIZE_MAX;
|
||||
|
||||
ZSTDLIB_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx);
|
||||
|
||||
/*
|
||||
Streaming decompression, direct mode (bufferless)
|
||||
|
||||
A ZSTDv06_DCtx object is required to track streaming operations.
|
||||
Use ZSTDv06_createDCtx() / ZSTDv06_freeDCtx() to manage it.
|
||||
A ZSTDv06_DCtx object can be re-used multiple times.
|
||||
|
||||
First optional operation is to retrieve frame parameters, using ZSTDv06_getFrameParams(), which doesn't consume the input.
|
||||
It can provide the minimum size of rolling buffer required to properly decompress data,
|
||||
and optionally the final size of uncompressed content.
|
||||
(Note : content size is an optional info that may not be present. 0 means : content size unknown)
|
||||
Frame parameters are extracted from the beginning of compressed frame.
|
||||
The amount of data to read is variable, from ZSTDv06_frameHeaderSize_min to ZSTDv06_frameHeaderSize_max (so if `srcSize` >= ZSTDv06_frameHeaderSize_max, it will always work)
|
||||
If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result.
|
||||
Result : 0 when successful, it means the ZSTDv06_frameParams structure has been filled.
|
||||
>0 : means there is not enough data into `src`. Provides the expected size to successfully decode header.
|
||||
errorCode, which can be tested using ZSTDv06_isError()
|
||||
|
||||
Start decompression, with ZSTDv06_decompressBegin() or ZSTDv06_decompressBegin_usingDict().
|
||||
Alternatively, you can copy a prepared context, using ZSTDv06_copyDCtx().
|
||||
|
||||
Then use ZSTDv06_nextSrcSizeToDecompress() and ZSTDv06_decompressContinue() alternatively.
|
||||
ZSTDv06_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTDv06_decompressContinue().
|
||||
ZSTDv06_decompressContinue() requires this exact amount of bytes, or it will fail.
|
||||
ZSTDv06_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
|
||||
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
|
||||
|
||||
@result of ZSTDv06_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity)
|
||||
It can be zero, which is not an error; it just means ZSTDv06_decompressContinue() has decoded some header.
|
||||
|
||||
A frame is fully decoded when ZSTDv06_nextSrcSizeToDecompress() returns zero.
|
||||
Context can then be reset to start a new decompression.
|
||||
*/
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Block functions
|
||||
****************************************/
|
||||
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
|
||||
A few rules to respect :
|
||||
- Uncompressed block size must be <= ZSTDv06_BLOCKSIZE_MAX (128 KB)
|
||||
- Compressing or decompressing requires a context structure
|
||||
+ Use ZSTDv06_createCCtx() and ZSTDv06_createDCtx()
|
||||
- It is necessary to init context before starting
|
||||
+ compression : ZSTDv06_compressBegin()
|
||||
+ decompression : ZSTDv06_decompressBegin()
|
||||
+ variants _usingDict() are also allowed
|
||||
+ copyCCtx() and copyDCtx() work too
|
||||
- When a block is considered not compressible enough, ZSTDv06_compressBlock() result will be zero.
|
||||
In which case, nothing is produced into `dst`.
|
||||
+ User must test for such outcome and deal directly with uncompressed data
|
||||
+ ZSTDv06_decompressBlock() doesn't accept uncompressed data as input !!
|
||||
*/
|
||||
|
||||
#define ZSTDv06_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
|
||||
ZSTDLIB_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Error management
|
||||
***************************************/
|
||||
/*! ZSTDv06_getErrorCode() :
|
||||
convert a `size_t` function result into a `ZSTDv06_ErrorCode` enum type,
|
||||
which can be used to compare directly with enum list published into "error_public.h" */
|
||||
ZSTDLIB_API ZSTDv06_ErrorCode ZSTDv06_getErrorCode(size_t functionResult);
|
||||
ZSTDLIB_API const char* ZSTDv06_getErrorString(ZSTDv06_ErrorCode code);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTDv06_STATIC_H */
|
||||
/* ******************************************************************
|
||||
Error codes and messages
|
||||
Copyright (C) 2013-2016, Yann Collet
|
||||
@@ -660,6 +689,7 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Compiler-specific
|
||||
******************************************/
|
||||
@@ -677,8 +707,8 @@ extern "C" {
|
||||
/*-****************************************
|
||||
* Customization (error_public.h)
|
||||
******************************************/
|
||||
typedef ZSTDv06_ErrorCode ERR_enum;
|
||||
#define PREFIX(name) ZSTDv06_error_##name
|
||||
typedef ZSTDv07_ErrorCode ERR_enum;
|
||||
#define PREFIX(name) ZSTDv07_error_##name
|
||||
|
||||
|
||||
/*-****************************************
|
||||
@@ -715,10 +745,12 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
|
||||
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
||||
case PREFIX(srcSize_wrong): return "Src size incorrect";
|
||||
case PREFIX(corruption_detected): return "Corrupted block detected";
|
||||
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
|
||||
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
||||
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
||||
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
||||
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
|
||||
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
|
||||
case PREFIX(maxCode):
|
||||
default: return notErrorCode;
|
||||
}
|
||||
@@ -734,400 +766,6 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
|
||||
#endif
|
||||
|
||||
#endif /* ERROR_H_MODULE */
|
||||
/*
|
||||
zstd_internal - common functions to include
|
||||
Header File for include
|
||||
Copyright (C) 2014-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : https://www.zstd.net
|
||||
*/
|
||||
#ifndef ZSTDv06_CCOMMON_H_MODULE
|
||||
#define ZSTDv06_CCOMMON_H_MODULE
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Common macros
|
||||
***************************************/
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
#define MAX(a,b) ((a)>(b) ? (a) : (b))
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Common constants
|
||||
***************************************/
|
||||
#define ZSTDv06_OPT_DEBUG 0 // 3 = compression stats; 5 = check encoded sequences; 9 = full logs
|
||||
#include <stdio.h>
|
||||
#if defined(ZSTDv06_OPT_DEBUG) && ZSTDv06_OPT_DEBUG>=9
|
||||
#define ZSTDv06_LOG_PARSER(...) printf(__VA_ARGS__)
|
||||
#define ZSTDv06_LOG_ENCODE(...) printf(__VA_ARGS__)
|
||||
#define ZSTDv06_LOG_BLOCK(...) printf(__VA_ARGS__)
|
||||
#else
|
||||
#define ZSTDv06_LOG_PARSER(...)
|
||||
#define ZSTDv06_LOG_ENCODE(...)
|
||||
#define ZSTDv06_LOG_BLOCK(...)
|
||||
#endif
|
||||
|
||||
#define ZSTDv06_OPT_NUM (1<<12)
|
||||
#define ZSTDv06_DICT_MAGIC 0xEC30A436
|
||||
|
||||
#define ZSTDv06_REP_NUM 3
|
||||
#define ZSTDv06_REP_INIT ZSTDv06_REP_NUM
|
||||
#define ZSTDv06_REP_MOVE (ZSTDv06_REP_NUM-1)
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define BIT7 128
|
||||
#define BIT6 64
|
||||
#define BIT5 32
|
||||
#define BIT4 16
|
||||
#define BIT1 2
|
||||
#define BIT0 1
|
||||
|
||||
#define ZSTDv06_WINDOWLOG_ABSOLUTEMIN 12
|
||||
static const size_t ZSTDv06_fcs_fieldSize[4] = { 0, 1, 2, 8 };
|
||||
|
||||
#define ZSTDv06_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */
|
||||
static const size_t ZSTDv06_blockHeaderSize = ZSTDv06_BLOCKHEADERSIZE;
|
||||
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
||||
|
||||
#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
|
||||
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
|
||||
|
||||
#define HufLog 12
|
||||
|
||||
#define IS_HUF 0
|
||||
#define IS_PCH 1
|
||||
#define IS_RAW 2
|
||||
#define IS_RLE 3
|
||||
|
||||
#define LONGNBSEQ 0x7F00
|
||||
|
||||
#define MINMATCH 3
|
||||
#define EQUAL_READ32 4
|
||||
#define REPCODE_STARTVALUE 1
|
||||
|
||||
#define Litbits 8
|
||||
#define MaxLit ((1<<Litbits) - 1)
|
||||
#define MaxML 52
|
||||
#define MaxLL 35
|
||||
#define MaxOff 28
|
||||
#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
|
||||
#define MLFSELog 9
|
||||
#define LLFSELog 9
|
||||
#define OffFSELog 8
|
||||
|
||||
#define FSEv06_ENCODING_RAW 0
|
||||
#define FSEv06_ENCODING_RLE 1
|
||||
#define FSEv06_ENCODING_STATIC 2
|
||||
#define FSEv06_ENCODING_DYNAMIC 3
|
||||
|
||||
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
||||
13,14,15,16 };
|
||||
static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
|
||||
-1,-1,-1,-1 };
|
||||
static const U32 LL_defaultNormLog = 6;
|
||||
|
||||
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11,
|
||||
12,13,14,15,16 };
|
||||
static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
|
||||
-1,-1,-1,-1,-1 };
|
||||
static const U32 ML_defaultNormLog = 6;
|
||||
|
||||
static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
|
||||
static const U32 OF_defaultNormLog = 5;
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Shared functions to include for inlining
|
||||
*********************************************/
|
||||
static void ZSTDv06_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
||||
#define COPY8(d,s) { ZSTDv06_copy8(d,s); d+=8; s+=8; }
|
||||
|
||||
/*! ZSTDv06_wildcopy() :
|
||||
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
|
||||
#define WILDCOPY_OVERLENGTH 8
|
||||
MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, size_t length)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = op + length;
|
||||
do
|
||||
COPY8(op, ip)
|
||||
while (op < oend);
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned ZSTDv06_highbit(U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return 31 - __builtin_clz(val);
|
||||
# else /* Software version */
|
||||
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
int r;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
|
||||
return r;
|
||||
# endif
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Private interfaces
|
||||
*********************************************/
|
||||
typedef struct {
|
||||
U32 off;
|
||||
U32 len;
|
||||
} ZSTDv06_match_t;
|
||||
|
||||
typedef struct {
|
||||
U32 price;
|
||||
U32 off;
|
||||
U32 mlen;
|
||||
U32 litlen;
|
||||
U32 rep[ZSTDv06_REP_INIT];
|
||||
} ZSTDv06_optimal_t;
|
||||
|
||||
#if ZSTDv06_OPT_DEBUG == 3
|
||||
#include ".debug/zstd_stats.h"
|
||||
#else
|
||||
typedef struct { U32 unused; } ZSTDv06_stats_t;
|
||||
MEM_STATIC void ZSTDv06_statsPrint(ZSTDv06_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
|
||||
MEM_STATIC void ZSTDv06_statsInit(ZSTDv06_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTDv06_statsResetFreqs(ZSTDv06_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTDv06_statsUpdatePrices(ZSTDv06_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
void* buffer;
|
||||
U32* offsetStart;
|
||||
U32* offset;
|
||||
BYTE* offCodeStart;
|
||||
BYTE* litStart;
|
||||
BYTE* lit;
|
||||
U16* litLengthStart;
|
||||
U16* litLength;
|
||||
BYTE* llCodeStart;
|
||||
U16* matchLengthStart;
|
||||
U16* matchLength;
|
||||
BYTE* mlCodeStart;
|
||||
U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
|
||||
U32 longLengthPos;
|
||||
/* opt */
|
||||
ZSTDv06_optimal_t* priceTable;
|
||||
ZSTDv06_match_t* matchTable;
|
||||
U32* matchLengthFreq;
|
||||
U32* litLengthFreq;
|
||||
U32* litFreq;
|
||||
U32* offCodeFreq;
|
||||
U32 matchLengthSum;
|
||||
U32 matchSum;
|
||||
U32 litLengthSum;
|
||||
U32 litSum;
|
||||
U32 offCodeSum;
|
||||
U32 log2matchLengthSum;
|
||||
U32 log2matchSum;
|
||||
U32 log2litLengthSum;
|
||||
U32 log2litSum;
|
||||
U32 log2offCodeSum;
|
||||
U32 factor;
|
||||
U32 cachedPrice;
|
||||
U32 cachedLitLength;
|
||||
const BYTE* cachedLiterals;
|
||||
ZSTDv06_stats_t stats;
|
||||
} seqStore_t;
|
||||
|
||||
void ZSTDv06_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
|
||||
|
||||
|
||||
#endif /* ZSTDv06_CCOMMON_H_MODULE */
|
||||
/* ******************************************************************
|
||||
FSE : Finite State Entropy codec
|
||||
Public Prototypes declaration
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef FSEv06_H
|
||||
#define FSEv06_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* FSE simple functions
|
||||
******************************************/
|
||||
/*! FSEv06_decompress():
|
||||
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated destination buffer 'dst', of size 'dstCapacity'.
|
||||
@return : size of regenerated data (<= maxDstSize),
|
||||
or an error code, which can be tested using FSEv06_isError() .
|
||||
|
||||
** Important ** : FSEv06_decompress() does not decompress non-compressible nor RLE data !!!
|
||||
Why ? : making this distinction requires a header.
|
||||
Header management is intentionally delegated to the user layer, which can better manage special cases.
|
||||
*/
|
||||
size_t FSEv06_decompress(void* dst, size_t dstCapacity,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* Tool functions
|
||||
******************************************/
|
||||
size_t FSEv06_compressBound(size_t size); /* maximum compressed size */
|
||||
|
||||
/* Error Management */
|
||||
unsigned FSEv06_isError(size_t code); /* tells if a return value is an error code */
|
||||
const char* FSEv06_getErrorName(size_t code); /* provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* FSE detailed API
|
||||
******************************************/
|
||||
/*!
|
||||
|
||||
FSEv06_decompress() does the following:
|
||||
1. read normalized counters with readNCount()
|
||||
2. build decoding table 'DTable' from normalized counters
|
||||
3. decode the data stream using decoding table 'DTable'
|
||||
|
||||
The following API allows targeting specific sub-functions for advanced tasks.
|
||||
For example, it's possible to compress several blocks using the same 'CTable',
|
||||
or to save and provide normalized distribution using external method.
|
||||
*/
|
||||
|
||||
|
||||
/* *** DECOMPRESSION *** */
|
||||
|
||||
/*! FSEv06_readNCount():
|
||||
Read compactly saved 'normalizedCounter' from 'rBuffer'.
|
||||
@return : size read from 'rBuffer',
|
||||
or an errorCode, which can be tested using FSEv06_isError().
|
||||
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
|
||||
size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
|
||||
|
||||
/*! Constructor and Destructor of FSEv06_DTable.
|
||||
Note that its size depends on 'tableLog' */
|
||||
typedef unsigned FSEv06_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
||||
FSEv06_DTable* FSEv06_createDTable(unsigned tableLog);
|
||||
void FSEv06_freeDTable(FSEv06_DTable* dt);
|
||||
|
||||
/*! FSEv06_buildDTable():
|
||||
Builds 'dt', which must be already allocated, using FSEv06_createDTable().
|
||||
return : 0, or an errorCode, which can be tested using FSEv06_isError() */
|
||||
size_t FSEv06_buildDTable (FSEv06_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*! FSEv06_decompress_usingDTable():
|
||||
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
|
||||
into `dst` which must be already allocated.
|
||||
@return : size of regenerated data (necessarily <= `dstCapacity`),
|
||||
or an errorCode, which can be tested using FSEv06_isError() */
|
||||
size_t FSEv06_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSEv06_DTable* dt);
|
||||
|
||||
/*!
|
||||
Tutorial :
|
||||
----------
|
||||
(Note : these functions only decompress FSE-compressed blocks.
|
||||
If block is uncompressed, use memcpy() instead
|
||||
If block is a single repeated byte, use memset() instead )
|
||||
|
||||
The first step is to obtain the normalized frequencies of symbols.
|
||||
This can be performed by FSEv06_readNCount() if it was saved using FSEv06_writeNCount().
|
||||
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
|
||||
In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
|
||||
or size the table to handle worst case situations (typically 256).
|
||||
FSEv06_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
|
||||
The result of FSEv06_readNCount() is the number of bytes read from 'rBuffer'.
|
||||
Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
|
||||
If there is an error, the function will return an error code, which can be tested using FSEv06_isError().
|
||||
|
||||
The next step is to build the decompression tables 'FSEv06_DTable' from 'normalizedCounter'.
|
||||
This is performed by the function FSEv06_buildDTable().
|
||||
The space required by 'FSEv06_DTable' must be already allocated using FSEv06_createDTable().
|
||||
If there is an error, the function will return an error code, which can be tested using FSEv06_isError().
|
||||
|
||||
`FSEv06_DTable` can then be used to decompress `cSrc`, with FSEv06_decompress_usingDTable().
|
||||
`cSrcSize` must be strictly correct, otherwise decompression will fail.
|
||||
FSEv06_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
|
||||
If there is an error, the function will return an error code, which can be tested using FSEv06_isError(). (ex: dst buffer too small)
|
||||
*/
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* FSEv06_H */
|
||||
/* ******************************************************************
|
||||
bitstream
|
||||
Part of FSE library
|
||||
@@ -1184,8 +822,6 @@ extern "C" {
|
||||
# include <immintrin.h> /* support for bextr (experimental) */
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*-********************************************
|
||||
* bitStream decoding API (read backward)
|
||||
**********************************************/
|
||||
@@ -1195,35 +831,35 @@ typedef struct
|
||||
unsigned bitsConsumed;
|
||||
const char* ptr;
|
||||
const char* start;
|
||||
} BITv06_DStream_t;
|
||||
} BITv07_DStream_t;
|
||||
|
||||
typedef enum { BITv06_DStream_unfinished = 0,
|
||||
BITv06_DStream_endOfBuffer = 1,
|
||||
BITv06_DStream_completed = 2,
|
||||
BITv06_DStream_overflow = 3 } BITv06_DStream_status; /* result of BITv06_reloadDStream() */
|
||||
typedef enum { BITv07_DStream_unfinished = 0,
|
||||
BITv07_DStream_endOfBuffer = 1,
|
||||
BITv07_DStream_completed = 2,
|
||||
BITv07_DStream_overflow = 3 } BITv07_DStream_status; /* result of BITv07_reloadDStream() */
|
||||
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
|
||||
|
||||
MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
|
||||
MEM_STATIC size_t BITv06_readBits(BITv06_DStream_t* bitD, unsigned nbBits);
|
||||
MEM_STATIC BITv06_DStream_status BITv06_reloadDStream(BITv06_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BITv06_endOfDStream(const BITv06_DStream_t* bitD);
|
||||
MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
|
||||
MEM_STATIC size_t BITv07_readBits(BITv07_DStream_t* bitD, unsigned nbBits);
|
||||
MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* bitD);
|
||||
|
||||
|
||||
/* Start by invoking BITv06_initDStream().
|
||||
/* Start by invoking BITv07_initDStream().
|
||||
* A chunk of the bitStream is then stored into a local register.
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is explicitly reloaded from memory by the BITv06_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BITv06_DStream_unfinished.
|
||||
* Local register is explicitly reloaded from memory by the BITv07_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BITv07_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BITv06_endOfDStream().
|
||||
* Checking if DStream has reached its end can be performed with BITv07_endOfDStream().
|
||||
*/
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* unsafe API
|
||||
******************************************/
|
||||
MEM_STATIC size_t BITv06_readBitsFast(BITv06_DStream_t* bitD, unsigned nbBits);
|
||||
MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, unsigned nbBits);
|
||||
/* faster, but works only if nbBits >= 1 */
|
||||
|
||||
|
||||
@@ -1231,7 +867,7 @@ MEM_STATIC size_t BITv06_readBitsFast(BITv06_DStream_t* bitD, unsigned nbBits);
|
||||
/*-**************************************************************
|
||||
* Internal functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BITv06_highbit32 (register U32 val)
|
||||
MEM_STATIC unsigned BITv07_highbit32 (register U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
@@ -1242,32 +878,29 @@ MEM_STATIC unsigned BITv06_highbit32 (register U32 val)
|
||||
# else /* Software version */
|
||||
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
unsigned r;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
r = DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
|
||||
return r;
|
||||
return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
|
||||
# endif
|
||||
}
|
||||
|
||||
/*===== Local Constants =====*/
|
||||
static const unsigned BITv06_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
|
||||
|
||||
static const unsigned BITv07_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
|
||||
|
||||
|
||||
/*-********************************************************
|
||||
* bitStream decoding
|
||||
**********************************************************/
|
||||
/*! BITv06_initDStream() :
|
||||
* Initialize a BITv06_DStream_t.
|
||||
* `bitD` : a pointer to an already allocated BITv06_DStream_t structure.
|
||||
/*! BITv07_initDStream() :
|
||||
* Initialize a BITv07_DStream_t.
|
||||
* `bitD` : a pointer to an already allocated BITv07_DStream_t structure.
|
||||
* `srcSize` must be the *exact* size of the bitStream, in bytes.
|
||||
* @return : size of stream (== srcSize) or an errorCode if a problem is detected
|
||||
*/
|
||||
MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
|
||||
MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
|
||||
|
||||
@@ -1276,8 +909,8 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff
|
||||
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
bitD->bitsConsumed = 8 - BITv06_highbit32(lastByte); }
|
||||
bitD->bitsConsumed = lastByte ? 8 - BITv07_highbit32(lastByte) : 0;
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
|
||||
} else {
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
bitD->ptr = bitD->start;
|
||||
@@ -1293,20 +926,20 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff
|
||||
default:;
|
||||
}
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
bitD->bitsConsumed = 8 - BITv06_highbit32(lastByte); }
|
||||
bitD->bitsConsumed = lastByte ? 8 - BITv07_highbit32(lastByte) : 0;
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
|
||||
bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
|
||||
}
|
||||
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BITv06_getUpperBits(size_t bitContainer, U32 const start)
|
||||
MEM_STATIC size_t BITv07_getUpperBits(size_t bitContainer, U32 const start)
|
||||
{
|
||||
return bitContainer >> start;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BITv06_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
|
||||
MEM_STATIC size_t BITv07_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
|
||||
{
|
||||
#if defined(__BMI__) && defined(__GNUC__) /* experimental */
|
||||
# if defined(__x86_64__)
|
||||
@@ -1316,91 +949,91 @@ MEM_STATIC size_t BITv06_getMiddleBits(size_t bitContainer, U32 const start, U32
|
||||
# endif
|
||||
return _bextr_u32(bitContainer, start, nbBits);
|
||||
#else
|
||||
return (bitContainer >> start) & BITv06_mask[nbBits];
|
||||
return (bitContainer >> start) & BITv07_mask[nbBits];
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BITv06_getLowerBits(size_t bitContainer, U32 const nbBits)
|
||||
MEM_STATIC size_t BITv07_getLowerBits(size_t bitContainer, U32 const nbBits)
|
||||
{
|
||||
return bitContainer & BITv06_mask[nbBits];
|
||||
return bitContainer & BITv07_mask[nbBits];
|
||||
}
|
||||
|
||||
/*! BITv06_lookBits() :
|
||||
/*! BITv07_lookBits() :
|
||||
* Provides next n bits from local register.
|
||||
* local register is not modified.
|
||||
* On 32-bits, maxNbBits==24.
|
||||
* On 64-bits, maxNbBits==56.
|
||||
* @return : value extracted
|
||||
*/
|
||||
MEM_STATIC size_t BITv06_lookBits(const BITv06_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC size_t BITv07_lookBits(const BITv07_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
|
||||
return BITv06_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
|
||||
return BITv07_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
|
||||
#else
|
||||
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*! BITv06_lookBitsFast() :
|
||||
/*! BITv07_lookBitsFast() :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BITv06_lookBitsFast(const BITv06_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC size_t BITv07_lookBitsFast(const BITv07_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
|
||||
}
|
||||
|
||||
MEM_STATIC void BITv06_skipBits(BITv06_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC void BITv07_skipBits(BITv07_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
|
||||
/*! BITv06_readBits() :
|
||||
/*! BITv07_readBits() :
|
||||
* Read (consume) next n bits from local register and update.
|
||||
* Pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value.
|
||||
*/
|
||||
MEM_STATIC size_t BITv06_readBits(BITv06_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC size_t BITv07_readBits(BITv07_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t const value = BITv06_lookBits(bitD, nbBits);
|
||||
BITv06_skipBits(bitD, nbBits);
|
||||
size_t const value = BITv07_lookBits(bitD, nbBits);
|
||||
BITv07_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
|
||||
/*! BITv06_readBitsFast() :
|
||||
/*! BITv07_readBitsFast() :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BITv06_readBitsFast(BITv06_DStream_t* bitD, U32 nbBits)
|
||||
MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t const value = BITv06_lookBitsFast(bitD, nbBits);
|
||||
BITv06_skipBits(bitD, nbBits);
|
||||
size_t const value = BITv07_lookBitsFast(bitD, nbBits);
|
||||
BITv07_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
|
||||
/*! BITv06_reloadDStream() :
|
||||
* Refill `BITv06_DStream_t` from src buffer previously defined (see BITv06_initDStream() ).
|
||||
/*! BITv07_reloadDStream() :
|
||||
* Refill `BITv07_DStream_t` from src buffer previously defined (see BITv07_initDStream() ).
|
||||
* This function is safe, it guarantees it will not read beyond src buffer.
|
||||
* @return : status of `BITv06_DStream_t` internal register.
|
||||
* @return : status of `BITv07_DStream_t` internal register.
|
||||
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
|
||||
MEM_STATIC BITv06_DStream_status BITv06_reloadDStream(BITv06_DStream_t* bitD)
|
||||
MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD)
|
||||
{
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
||||
return BITv06_DStream_overflow;
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
|
||||
return BITv07_DStream_overflow;
|
||||
|
||||
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
|
||||
bitD->ptr -= bitD->bitsConsumed >> 3;
|
||||
bitD->bitsConsumed &= 7;
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
return BITv06_DStream_unfinished;
|
||||
return BITv07_DStream_unfinished;
|
||||
}
|
||||
if (bitD->ptr == bitD->start) {
|
||||
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BITv06_DStream_endOfBuffer;
|
||||
return BITv06_DStream_completed;
|
||||
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BITv07_DStream_endOfBuffer;
|
||||
return BITv07_DStream_completed;
|
||||
}
|
||||
{ U32 nbBytes = bitD->bitsConsumed >> 3;
|
||||
BITv06_DStream_status result = BITv06_DStream_unfinished;
|
||||
BITv07_DStream_status result = BITv07_DStream_unfinished;
|
||||
if (bitD->ptr - nbBytes < bitD->start) {
|
||||
nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
|
||||
result = BITv06_DStream_endOfBuffer;
|
||||
result = BITv07_DStream_endOfBuffer;
|
||||
}
|
||||
bitD->ptr -= nbBytes;
|
||||
bitD->bitsConsumed -= nbBytes*8;
|
||||
@@ -1409,10 +1042,10 @@ MEM_STATIC BITv06_DStream_status BITv06_reloadDStream(BITv06_DStream_t* bitD)
|
||||
}
|
||||
}
|
||||
|
||||
/*! BITv06_endOfDStream() :
|
||||
/*! BITv07_endOfDStream() :
|
||||
* @return Tells if DStream has exactly reached its end (all bits consumed).
|
||||
*/
|
||||
MEM_STATIC unsigned BITv06_endOfDStream(const BITv06_DStream_t* DStream)
|
||||
MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* DStream)
|
||||
{
|
||||
return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
|
||||
}
|
||||
@@ -1423,9 +1056,9 @@ MEM_STATIC unsigned BITv06_endOfDStream(const BITv06_DStream_t* DStream)
|
||||
|
||||
#endif /* BITSTREAM_H_MODULE */
|
||||
/* ******************************************************************
|
||||
FSE : Finite State Entropy coder
|
||||
header file for static linking (only)
|
||||
Copyright (C) 2013-2015, Yann Collet
|
||||
FSE : Finite State Entropy codec
|
||||
Public Prototypes declaration
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
@@ -1454,39 +1087,139 @@ MEM_STATIC unsigned BITv06_endOfDStream(const BITv06_DStream_t* DStream)
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
#ifndef FSEv06_STATIC_H
|
||||
#define FSEv06_STATIC_H
|
||||
#ifndef FSEv07_H
|
||||
#define FSEv07_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* FSE simple functions
|
||||
******************************************/
|
||||
|
||||
/*! FSEv07_decompress():
|
||||
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated destination buffer 'dst', of size 'dstCapacity'.
|
||||
@return : size of regenerated data (<= maxDstSize),
|
||||
or an error code, which can be tested using FSEv07_isError() .
|
||||
|
||||
** Important ** : FSEv07_decompress() does not decompress non-compressible nor RLE data !!!
|
||||
Why ? : making this distinction requires a header.
|
||||
Header management is intentionally delegated to the user layer, which can better manage special cases.
|
||||
*/
|
||||
size_t FSEv07_decompress(void* dst, size_t dstCapacity,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
|
||||
|
||||
/* Error Management */
|
||||
unsigned FSEv07_isError(size_t code); /* tells if a return value is an error code */
|
||||
const char* FSEv07_getErrorName(size_t code); /* provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* FSE detailed API
|
||||
******************************************/
|
||||
/*!
|
||||
FSEv07_decompress() does the following:
|
||||
1. read normalized counters with readNCount()
|
||||
2. build decoding table 'DTable' from normalized counters
|
||||
3. decode the data stream using decoding table 'DTable'
|
||||
|
||||
The following API allows targeting specific sub-functions for advanced tasks.
|
||||
For example, it's possible to compress several blocks using the same 'CTable',
|
||||
or to save and provide normalized distribution using external method.
|
||||
*/
|
||||
|
||||
|
||||
/* *** DECOMPRESSION *** */
|
||||
|
||||
/*! FSEv07_readNCount():
|
||||
Read compactly saved 'normalizedCounter' from 'rBuffer'.
|
||||
@return : size read from 'rBuffer',
|
||||
or an errorCode, which can be tested using FSEv07_isError().
|
||||
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
|
||||
size_t FSEv07_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
|
||||
|
||||
/*! Constructor and Destructor of FSEv07_DTable.
|
||||
Note that its size depends on 'tableLog' */
|
||||
typedef unsigned FSEv07_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
||||
FSEv07_DTable* FSEv07_createDTable(unsigned tableLog);
|
||||
void FSEv07_freeDTable(FSEv07_DTable* dt);
|
||||
|
||||
/*! FSEv07_buildDTable():
|
||||
Builds 'dt', which must be already allocated, using FSEv07_createDTable().
|
||||
return : 0, or an errorCode, which can be tested using FSEv07_isError() */
|
||||
size_t FSEv07_buildDTable (FSEv07_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*! FSEv07_decompress_usingDTable():
|
||||
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
|
||||
into `dst` which must be already allocated.
|
||||
@return : size of regenerated data (necessarily <= `dstCapacity`),
|
||||
or an errorCode, which can be tested using FSEv07_isError() */
|
||||
size_t FSEv07_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSEv07_DTable* dt);
|
||||
|
||||
/*!
|
||||
Tutorial :
|
||||
----------
|
||||
(Note : these functions only decompress FSE-compressed blocks.
|
||||
If block is uncompressed, use memcpy() instead
|
||||
If block is a single repeated byte, use memset() instead )
|
||||
|
||||
The first step is to obtain the normalized frequencies of symbols.
|
||||
This can be performed by FSEv07_readNCount() if it was saved using FSEv07_writeNCount().
|
||||
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
|
||||
In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
|
||||
or size the table to handle worst case situations (typically 256).
|
||||
FSEv07_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
|
||||
The result of FSEv07_readNCount() is the number of bytes read from 'rBuffer'.
|
||||
Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
|
||||
If there is an error, the function will return an error code, which can be tested using FSEv07_isError().
|
||||
|
||||
The next step is to build the decompression tables 'FSEv07_DTable' from 'normalizedCounter'.
|
||||
This is performed by the function FSEv07_buildDTable().
|
||||
The space required by 'FSEv07_DTable' must be already allocated using FSEv07_createDTable().
|
||||
If there is an error, the function will return an error code, which can be tested using FSEv07_isError().
|
||||
|
||||
`FSEv07_DTable` can then be used to decompress `cSrc`, with FSEv07_decompress_usingDTable().
|
||||
`cSrcSize` must be strictly correct, otherwise decompression will fail.
|
||||
FSEv07_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
|
||||
If there is an error, the function will return an error code, which can be tested using FSEv07_isError(). (ex: dst buffer too small)
|
||||
*/
|
||||
|
||||
|
||||
#ifdef FSEv07_STATIC_LINKING_ONLY
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Static allocation
|
||||
*******************************************/
|
||||
/* FSE buffer bounds */
|
||||
#define FSEv06_NCOUNTBOUND 512
|
||||
#define FSEv06_BLOCKBOUND(size) (size + (size>>7))
|
||||
#define FSEv06_COMPRESSBOUND(size) (FSEv06_NCOUNTBOUND + FSEv06_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
#define FSEv07_NCOUNTBOUND 512
|
||||
#define FSEv07_BLOCKBOUND(size) (size + (size>>7))
|
||||
|
||||
/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
|
||||
#define FSEv06_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
||||
#define FSEv07_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE advanced API
|
||||
*******************************************/
|
||||
size_t FSEv06_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
||||
/* same as FSEv06_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */
|
||||
size_t FSEv07_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
||||
/**< same as FSEv07_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */
|
||||
|
||||
size_t FSEv06_buildDTable_raw (FSEv06_DTable* dt, unsigned nbBits);
|
||||
/* build a fake FSEv06_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
|
||||
unsigned FSEv07_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
|
||||
/**< same as FSEv07_optimalTableLog(), which used `minus==2` */
|
||||
|
||||
size_t FSEv07_buildDTable_raw (FSEv07_DTable* dt, unsigned nbBits);
|
||||
/**< build a fake FSEv07_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
|
||||
|
||||
size_t FSEv07_buildDTable_rle (FSEv07_DTable* dt, unsigned char symbolValue);
|
||||
/**< build a fake FSEv07_DTable, designed to always generate the same symbolValue */
|
||||
|
||||
size_t FSEv06_buildDTable_rle (FSEv06_DTable* dt, unsigned char symbolValue);
|
||||
/* build a fake FSEv06_DTable, designed to always generate the same symbolValue */
|
||||
|
||||
|
||||
/* *****************************************
|
||||
@@ -1496,146 +1229,141 @@ typedef struct
|
||||
{
|
||||
size_t state;
|
||||
const void* table; /* precise table may vary, depending on U16 */
|
||||
} FSEv06_DState_t;
|
||||
} FSEv07_DState_t;
|
||||
|
||||
|
||||
static void FSEv06_initDState(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD, const FSEv06_DTable* dt);
|
||||
static void FSEv07_initDState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD, const FSEv07_DTable* dt);
|
||||
|
||||
static unsigned char FSEv06_decodeSymbol(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD);
|
||||
static unsigned char FSEv07_decodeSymbol(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD);
|
||||
|
||||
static unsigned FSEv06_endOfDState(const FSEv06_DState_t* DStatePtr);
|
||||
static unsigned FSEv07_endOfDState(const FSEv07_DState_t* DStatePtr);
|
||||
|
||||
/*!
|
||||
Let's now decompose FSEv06_decompress_usingDTable() into its unitary components.
|
||||
/**<
|
||||
Let's now decompose FSEv07_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
BITv06_DStream_t DStream; // Stream context
|
||||
FSEv06_DState_t DState; // State context. Multiple ones are possible
|
||||
FSEv06_DTable* DTablePtr; // Decoding table, provided by FSEv06_buildDTable()
|
||||
BITv07_DStream_t DStream; // Stream context
|
||||
FSEv07_DState_t DState; // State context. Multiple ones are possible
|
||||
FSEv07_DTable* DTablePtr; // Decoding table, provided by FSEv07_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = BITv06_initDStream(&DStream, srcBuffer, srcSize);
|
||||
errorCode = BITv07_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSEv06_initDState(&DState, &DStream, DTablePtr);
|
||||
errorCode = FSEv07_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSEv06_decodeSymbol() is 'tableLog'.
|
||||
For information the maximum number of bits read by FSEv07_decodeSymbol() is 'tableLog'.
|
||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||
unsigned char symbol = FSEv06_decodeSymbol(&DState, &DStream);
|
||||
unsigned char symbol = FSEv07_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = BITv06_readBits(&DStream, nbBits);
|
||||
size_t bitField = BITv07_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSEv06_reloadDStream(&DStream);
|
||||
endSignal = FSEv07_reloadDStream(&DStream);
|
||||
|
||||
BITv06_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BITv06_DStream_unfinished : there is still some data left into the DStream.
|
||||
BITv06_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BITv06_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BITv06_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
BITv07_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BITv07_DStream_unfinished : there is still some data left into the DStream.
|
||||
BITv07_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BITv07_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BITv07_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (BITv06_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
When reaching end of buffer (BITv07_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
BITv06_reloadDStream(&DStream) >= BITv06_DStream_completed
|
||||
BITv07_reloadDStream(&DStream) >= BITv07_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
BITv06_endOfDStream(&DStream);
|
||||
BITv07_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSEv06_endOfDState(&DState);
|
||||
FSEv07_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE unsafe API
|
||||
*******************************************/
|
||||
static unsigned char FSEv06_decodeSymbolFast(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD);
|
||||
static unsigned char FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD);
|
||||
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Implementation of inlined functions
|
||||
*******************************************/
|
||||
|
||||
|
||||
/*<===== Decompression =====>*/
|
||||
|
||||
typedef struct {
|
||||
U16 tableLog;
|
||||
U16 fastMode;
|
||||
} FSEv06_DTableHeader; /* sizeof U32 */
|
||||
} FSEv07_DTableHeader; /* sizeof U32 */
|
||||
|
||||
typedef struct
|
||||
{
|
||||
unsigned short newState;
|
||||
unsigned char symbol;
|
||||
unsigned char nbBits;
|
||||
} FSEv06_decode_t; /* size == U32 */
|
||||
} FSEv07_decode_t; /* size == U32 */
|
||||
|
||||
MEM_STATIC void FSEv06_initDState(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD, const FSEv06_DTable* dt)
|
||||
MEM_STATIC void FSEv07_initDState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD, const FSEv07_DTable* dt)
|
||||
{
|
||||
const void* ptr = dt;
|
||||
const FSEv06_DTableHeader* const DTableH = (const FSEv06_DTableHeader*)ptr;
|
||||
DStatePtr->state = BITv06_readBits(bitD, DTableH->tableLog);
|
||||
BITv06_reloadDStream(bitD);
|
||||
const FSEv07_DTableHeader* const DTableH = (const FSEv07_DTableHeader*)ptr;
|
||||
DStatePtr->state = BITv07_readBits(bitD, DTableH->tableLog);
|
||||
BITv07_reloadDStream(bitD);
|
||||
DStatePtr->table = dt + 1;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSEv06_peekSymbol(const FSEv06_DState_t* DStatePtr)
|
||||
MEM_STATIC BYTE FSEv07_peekSymbol(const FSEv07_DState_t* DStatePtr)
|
||||
{
|
||||
FSEv06_decode_t const DInfo = ((const FSEv06_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
return DInfo.symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC void FSEv06_updateState(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD)
|
||||
MEM_STATIC void FSEv07_updateState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD)
|
||||
{
|
||||
FSEv06_decode_t const DInfo = ((const FSEv06_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
size_t const lowBits = BITv06_readBits(bitD, nbBits);
|
||||
size_t const lowBits = BITv07_readBits(bitD, nbBits);
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSEv06_decodeSymbol(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD)
|
||||
MEM_STATIC BYTE FSEv07_decodeSymbol(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD)
|
||||
{
|
||||
FSEv06_decode_t const DInfo = ((const FSEv06_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BITv06_readBits(bitD, nbBits);
|
||||
size_t const lowBits = BITv07_readBits(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
/*! FSEv06_decodeSymbolFast() :
|
||||
/*! FSEv07_decodeSymbolFast() :
|
||||
unsafe, only works if no symbol has a probability > 50% */
|
||||
MEM_STATIC BYTE FSEv06_decodeSymbolFast(FSEv06_DState_t* DStatePtr, BITv06_DStream_t* bitD)
|
||||
MEM_STATIC BYTE FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD)
|
||||
{
|
||||
FSEv06_decode_t const DInfo = ((const FSEv06_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BITv06_readBitsFast(bitD, nbBits);
|
||||
size_t const lowBits = BITv07_readBitsFast(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned FSEv06_endOfDState(const FSEv06_DState_t* DStatePtr)
|
||||
MEM_STATIC unsigned FSEv07_endOfDState(const FSEv07_DState_t* DStatePtr)
|
||||
{
|
||||
return DStatePtr->state == 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifndef FSEv06_COMMONDEFS_ONLY
|
||||
#ifndef FSEv07_COMMONDEFS_ONLY
|
||||
|
||||
/* **************************************************************
|
||||
* Tuning parameters
|
||||
@@ -1645,48 +1373,225 @@ MEM_STATIC unsigned FSEv06_endOfDState(const FSEv06_DState_t* DStatePtr)
|
||||
* Increasing memory usage improves compression ratio
|
||||
* Reduced memory usage can improve speed, due to cache effect
|
||||
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
|
||||
#define FSEv06_MAX_MEMORY_USAGE 14
|
||||
#define FSEv06_DEFAULT_MEMORY_USAGE 13
|
||||
#define FSEv07_MAX_MEMORY_USAGE 14
|
||||
#define FSEv07_DEFAULT_MEMORY_USAGE 13
|
||||
|
||||
/*!FSEv06_MAX_SYMBOL_VALUE :
|
||||
/*!FSEv07_MAX_SYMBOL_VALUE :
|
||||
* Maximum symbol value authorized.
|
||||
* Required for proper stack allocation */
|
||||
#define FSEv06_MAX_SYMBOL_VALUE 255
|
||||
#define FSEv07_MAX_SYMBOL_VALUE 255
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* template functions type & suffix
|
||||
****************************************************************/
|
||||
#define FSEv06_FUNCTION_TYPE BYTE
|
||||
#define FSEv06_FUNCTION_EXTENSION
|
||||
#define FSEv06_DECODE_TYPE FSEv06_decode_t
|
||||
#define FSEv07_FUNCTION_TYPE BYTE
|
||||
#define FSEv07_FUNCTION_EXTENSION
|
||||
#define FSEv07_DECODE_TYPE FSEv07_decode_t
|
||||
|
||||
|
||||
#endif /* !FSEv06_COMMONDEFS_ONLY */
|
||||
#endif /* !FSEv07_COMMONDEFS_ONLY */
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
* Constants
|
||||
*****************************************************************/
|
||||
#define FSEv06_MAX_TABLELOG (FSEv06_MAX_MEMORY_USAGE-2)
|
||||
#define FSEv06_MAX_TABLESIZE (1U<<FSEv06_MAX_TABLELOG)
|
||||
#define FSEv06_MAXTABLESIZE_MASK (FSEv06_MAX_TABLESIZE-1)
|
||||
#define FSEv06_DEFAULT_TABLELOG (FSEv06_DEFAULT_MEMORY_USAGE-2)
|
||||
#define FSEv06_MIN_TABLELOG 5
|
||||
#define FSEv07_MAX_TABLELOG (FSEv07_MAX_MEMORY_USAGE-2)
|
||||
#define FSEv07_MAX_TABLESIZE (1U<<FSEv07_MAX_TABLELOG)
|
||||
#define FSEv07_MAXTABLESIZE_MASK (FSEv07_MAX_TABLESIZE-1)
|
||||
#define FSEv07_DEFAULT_TABLELOG (FSEv07_DEFAULT_MEMORY_USAGE-2)
|
||||
#define FSEv07_MIN_TABLELOG 5
|
||||
|
||||
#define FSEv06_TABLELOG_ABSOLUTE_MAX 15
|
||||
#if FSEv06_MAX_TABLELOG > FSEv06_TABLELOG_ABSOLUTE_MAX
|
||||
#error "FSEv06_MAX_TABLELOG > FSEv06_TABLELOG_ABSOLUTE_MAX is not supported"
|
||||
#define FSEv07_TABLELOG_ABSOLUTE_MAX 15
|
||||
#if FSEv07_MAX_TABLELOG > FSEv07_TABLELOG_ABSOLUTE_MAX
|
||||
# error "FSEv07_MAX_TABLELOG > FSEv07_TABLELOG_ABSOLUTE_MAX is not supported"
|
||||
#endif
|
||||
|
||||
#define FSEv06_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
|
||||
#define FSEv07_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
|
||||
|
||||
|
||||
#endif /* FSEv07_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* FSEv06_STATIC_H */
|
||||
#endif /* FSEv07_H */
|
||||
/* ******************************************************************
|
||||
Huffman coder, part of New Generation Entropy library
|
||||
header file
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef HUFv07_H_298734234
|
||||
#define HUFv07_H_298734234
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/* *** simple functions *** */
|
||||
/**
|
||||
HUFv07_decompress() :
|
||||
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated buffer 'dst', of minimum size 'dstSize'.
|
||||
`dstSize` : **must** be the ***exact*** size of original (uncompressed) data.
|
||||
Note : in contrast with FSE, HUFv07_decompress can regenerate
|
||||
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
|
||||
because it knows size to regenerate.
|
||||
@return : size of regenerated data (== dstSize),
|
||||
or an error code, which can be tested using HUFv07_isError()
|
||||
*/
|
||||
size_t HUFv07_decompress(void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Tool functions
|
||||
******************************************/
|
||||
#define HUFv07_BLOCKSIZE_MAX (128 * 1024)
|
||||
|
||||
/* Error Management */
|
||||
unsigned HUFv07_isError(size_t code); /**< tells if a return value is an error code */
|
||||
const char* HUFv07_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
/* *** Advanced function *** */
|
||||
|
||||
|
||||
#ifdef HUFv07_STATIC_LINKING_ONLY
|
||||
|
||||
|
||||
/* *** Constants *** */
|
||||
#define HUFv07_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUFv07_MAX_TABLELOG. Beyond that value, code does not work */
|
||||
#define HUFv07_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUFv07_ABSOLUTEMAX_TABLELOG */
|
||||
#define HUFv07_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
|
||||
#define HUFv07_SYMBOLVALUE_MAX 255
|
||||
#if (HUFv07_TABLELOG_MAX > HUFv07_TABLELOG_ABSOLUTEMAX)
|
||||
# error "HUFv07_TABLELOG_MAX is too large !"
|
||||
#endif
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Static allocation
|
||||
******************************************/
|
||||
/* HUF buffer bounds */
|
||||
#define HUFv07_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
|
||||
|
||||
/* static allocation of HUF's DTable */
|
||||
typedef U32 HUFv07_DTable;
|
||||
#define HUFv07_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
|
||||
#define HUFv07_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
||||
HUFv07_DTable DTable[HUFv07_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) }
|
||||
#define HUFv07_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
|
||||
HUFv07_DTable DTable[HUFv07_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) }
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Advanced decompression functions
|
||||
******************************************/
|
||||
size_t HUFv07_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUFv07_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
|
||||
size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
|
||||
size_t HUFv07_decompress4X_hufOnly(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
|
||||
size_t HUFv07_decompress4X2_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUFv07_decompress4X4_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
|
||||
size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
||||
size_t HUFv07_decompress1X2_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUFv07_decompress1X4_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* HUF detailed API
|
||||
******************************************/
|
||||
/*!
|
||||
The following API allows targeting specific sub-functions for advanced tasks.
|
||||
For example, it's possible to compress several blocks using the same 'CTable',
|
||||
or to save and regenerate 'CTable' using external methods.
|
||||
*/
|
||||
/* FSEv07_count() : find it within "fse.h" */
|
||||
|
||||
/*! HUFv07_readStats() :
|
||||
Read compact Huffman tree, saved by HUFv07_writeCTable().
|
||||
`huffWeight` is destination buffer.
|
||||
@return : size read from `src` , or an error Code .
|
||||
Note : Needed by HUFv07_readCTable() and HUFv07_readDTableXn() . */
|
||||
size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/*
|
||||
HUFv07_decompress() does the following:
|
||||
1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
|
||||
2. build Huffman table from save, using HUFv07_readDTableXn()
|
||||
3. decode 1 or 4 segments in parallel using HUFv07_decompressSXn_usingDTable
|
||||
*/
|
||||
|
||||
/** HUFv07_selectDecoder() :
|
||||
* Tells which decoder is likely to decode faster,
|
||||
* based on a set of pre-determined metrics.
|
||||
* @return : 0==HUFv07_decompress4X2, 1==HUFv07_decompress4X4 .
|
||||
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
|
||||
U32 HUFv07_selectDecoder (size_t dstSize, size_t cSrcSize);
|
||||
|
||||
size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSize);
|
||||
size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSize);
|
||||
|
||||
size_t HUFv07_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable);
|
||||
size_t HUFv07_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable);
|
||||
size_t HUFv07_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable);
|
||||
|
||||
|
||||
/* single stream variants */
|
||||
size_t HUFv07_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUFv07_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
||||
|
||||
size_t HUFv07_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable);
|
||||
size_t HUFv07_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable);
|
||||
size_t HUFv07_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable);
|
||||
|
||||
|
||||
#endif /* HUFv07_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* HUFv07_H_298734234 */
|
||||
/*
|
||||
Common functions of New Generation Entropy library
|
||||
Copyright (C) 2016, Yann Collet.
|
||||
@@ -1722,28 +1627,29 @@ MEM_STATIC unsigned FSEv06_endOfDState(const FSEv06_DState_t* DStatePtr)
|
||||
*************************************************************************** */
|
||||
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* FSE Error Management
|
||||
******************************************/
|
||||
unsigned FSEv06_isError(size_t code) { return ERR_isError(code); }
|
||||
unsigned FSEv07_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
const char* FSEv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
const char* FSEv07_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* HUF Error Management
|
||||
****************************************************************/
|
||||
unsigned HUFv06_isError(size_t code) { return ERR_isError(code); }
|
||||
unsigned HUFv07_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
const char* HUFv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
const char* HUFv07_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* FSE NCount encoding-decoding
|
||||
****************************************************************/
|
||||
static short FSEv06_abs(short a) { return a<0 ? -a : a; }
|
||||
static short FSEv07_abs(short a) { return (short)(a<0 ? -a : a); }
|
||||
|
||||
size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
size_t FSEv07_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) headerBuffer;
|
||||
@@ -1759,8 +1665,8 @@ size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned
|
||||
|
||||
if (hbSize < 4) return ERROR(srcSize_wrong);
|
||||
bitStream = MEM_readLE32(ip);
|
||||
nbBits = (bitStream & 0xF) + FSEv06_MIN_TABLELOG; /* extract tableLog */
|
||||
if (nbBits > FSEv06_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
|
||||
nbBits = (bitStream & 0xF) + FSEv07_MIN_TABLELOG; /* extract tableLog */
|
||||
if (nbBits > FSEv07_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
|
||||
bitStream >>= 4;
|
||||
bitCount = 4;
|
||||
*tableLogPtr = nbBits;
|
||||
@@ -1810,7 +1716,7 @@ size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned
|
||||
}
|
||||
|
||||
count--; /* extra accuracy */
|
||||
remaining -= FSEv06_abs(count);
|
||||
remaining -= FSEv07_abs(count);
|
||||
normalizedCounter[charnum++] = count;
|
||||
previous0 = !count;
|
||||
while (remaining < threshold) {
|
||||
@@ -1834,6 +1740,79 @@ size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned
|
||||
if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong);
|
||||
return ip-istart;
|
||||
}
|
||||
|
||||
|
||||
/*! HUFv07_readStats() :
|
||||
Read compact Huffman tree, saved by HUFv07_writeCTable().
|
||||
`huffWeight` is destination buffer.
|
||||
@return : size read from `src` , or an error Code .
|
||||
Note : Needed by HUFv07_readCTable() and HUFv07_readDTableXn() .
|
||||
*/
|
||||
size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 weightTotal;
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
size_t iSize = ip[0];
|
||||
size_t oSize;
|
||||
|
||||
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
if (iSize >= 128) { /* special header */
|
||||
if (iSize >= (242)) { /* RLE */
|
||||
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
|
||||
oSize = l[iSize-242];
|
||||
memset(huffWeight, 1, hwSize);
|
||||
iSize = 0;
|
||||
}
|
||||
else { /* Incompressible */
|
||||
oSize = iSize - 127;
|
||||
iSize = ((oSize+1)/2);
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
if (oSize >= hwSize) return ERROR(corruption_detected);
|
||||
ip += 1;
|
||||
{ U32 n;
|
||||
for (n=0; n<oSize; n+=2) {
|
||||
huffWeight[n] = ip[n/2] >> 4;
|
||||
huffWeight[n+1] = ip[n/2] & 15;
|
||||
} } } }
|
||||
else { /* header compressed with FSE (normal case) */
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
oSize = FSEv07_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
|
||||
if (FSEv07_isError(oSize)) return oSize;
|
||||
}
|
||||
|
||||
/* collect weight stats */
|
||||
memset(rankStats, 0, (HUFv07_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32));
|
||||
weightTotal = 0;
|
||||
{ U32 n; for (n=0; n<oSize; n++) {
|
||||
if (huffWeight[n] >= HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||
rankStats[huffWeight[n]]++;
|
||||
weightTotal += (1 << huffWeight[n]) >> 1;
|
||||
} }
|
||||
|
||||
/* get last non-null symbol weight (implied, total must be 2^n) */
|
||||
{ U32 const tableLog = BITv07_highbit32(weightTotal) + 1;
|
||||
if (tableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||
*tableLogPtr = tableLog;
|
||||
/* determine last weight */
|
||||
{ U32 const total = 1 << tableLog;
|
||||
U32 const rest = total - weightTotal;
|
||||
U32 const verif = 1 << BITv07_highbit32(rest);
|
||||
U32 const lastWeight = BITv07_highbit32(rest) + 1;
|
||||
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
||||
huffWeight[oSize] = (BYTE)lastWeight;
|
||||
rankStats[lastWeight]++;
|
||||
} }
|
||||
|
||||
/* check tree construction validity */
|
||||
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
|
||||
|
||||
/* results */
|
||||
*nbSymbolsPtr = (U32)(oSize+1);
|
||||
return iSize+1;
|
||||
}
|
||||
/* ******************************************************************
|
||||
FSE : Finite State Entropy decoder
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
@@ -1887,17 +1866,19 @@ size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
****************************************************************/
|
||||
#define FSEv06_isError ERR_isError
|
||||
#define FSEv06_STATIC_ASSERT(c) { enum { FSEv06_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
#define FSEv07_isError ERR_isError
|
||||
#define FSEv07_STATIC_ASSERT(c) { enum { FSEv07_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Complex types
|
||||
****************************************************************/
|
||||
typedef U32 DTable_max_t[FSEv06_DTABLE_SIZE_U32(FSEv06_MAX_TABLELOG)];
|
||||
typedef U32 DTable_max_t[FSEv07_DTABLE_SIZE_U32(FSEv07_MAX_TABLELOG)];
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
@@ -1910,54 +1891,54 @@ typedef U32 DTable_max_t[FSEv06_DTABLE_SIZE_U32(FSEv06_MAX_TABLELOG)];
|
||||
*/
|
||||
|
||||
/* safety checks */
|
||||
#ifndef FSEv06_FUNCTION_EXTENSION
|
||||
# error "FSEv06_FUNCTION_EXTENSION must be defined"
|
||||
#ifndef FSEv07_FUNCTION_EXTENSION
|
||||
# error "FSEv07_FUNCTION_EXTENSION must be defined"
|
||||
#endif
|
||||
#ifndef FSEv06_FUNCTION_TYPE
|
||||
# error "FSEv06_FUNCTION_TYPE must be defined"
|
||||
#ifndef FSEv07_FUNCTION_TYPE
|
||||
# error "FSEv07_FUNCTION_TYPE must be defined"
|
||||
#endif
|
||||
|
||||
/* Function names */
|
||||
#define FSEv06_CAT(X,Y) X##Y
|
||||
#define FSEv06_FUNCTION_NAME(X,Y) FSEv06_CAT(X,Y)
|
||||
#define FSEv06_TYPE_NAME(X,Y) FSEv06_CAT(X,Y)
|
||||
#define FSEv07_CAT(X,Y) X##Y
|
||||
#define FSEv07_FUNCTION_NAME(X,Y) FSEv07_CAT(X,Y)
|
||||
#define FSEv07_TYPE_NAME(X,Y) FSEv07_CAT(X,Y)
|
||||
|
||||
|
||||
/* Function templates */
|
||||
FSEv06_DTable* FSEv06_createDTable (unsigned tableLog)
|
||||
FSEv07_DTable* FSEv07_createDTable (unsigned tableLog)
|
||||
{
|
||||
if (tableLog > FSEv06_TABLELOG_ABSOLUTE_MAX) tableLog = FSEv06_TABLELOG_ABSOLUTE_MAX;
|
||||
return (FSEv06_DTable*)malloc( FSEv06_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
|
||||
if (tableLog > FSEv07_TABLELOG_ABSOLUTE_MAX) tableLog = FSEv07_TABLELOG_ABSOLUTE_MAX;
|
||||
return (FSEv07_DTable*)malloc( FSEv07_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
|
||||
}
|
||||
|
||||
void FSEv06_freeDTable (FSEv06_DTable* dt)
|
||||
void FSEv07_freeDTable (FSEv07_DTable* dt)
|
||||
{
|
||||
free(dt);
|
||||
}
|
||||
|
||||
size_t FSEv06_buildDTable(FSEv06_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
size_t FSEv07_buildDTable(FSEv07_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
|
||||
FSEv06_DECODE_TYPE* const tableDecode = (FSEv06_DECODE_TYPE*) (tdPtr);
|
||||
U16 symbolNext[FSEv06_MAX_SYMBOL_VALUE+1];
|
||||
FSEv07_DECODE_TYPE* const tableDecode = (FSEv07_DECODE_TYPE*) (tdPtr);
|
||||
U16 symbolNext[FSEv07_MAX_SYMBOL_VALUE+1];
|
||||
|
||||
U32 const maxSV1 = maxSymbolValue + 1;
|
||||
U32 const tableSize = 1 << tableLog;
|
||||
U32 highThreshold = tableSize-1;
|
||||
|
||||
/* Sanity Checks */
|
||||
if (maxSymbolValue > FSEv06_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
|
||||
if (tableLog > FSEv06_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
if (maxSymbolValue > FSEv07_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
|
||||
if (tableLog > FSEv07_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
|
||||
/* Init, lay down lowprob symbols */
|
||||
{ FSEv06_DTableHeader DTableH;
|
||||
{ FSEv07_DTableHeader DTableH;
|
||||
DTableH.tableLog = (U16)tableLog;
|
||||
DTableH.fastMode = 1;
|
||||
{ S16 const largeLimit= (S16)(1 << (tableLog-1));
|
||||
U32 s;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
if (normalizedCounter[s]==-1) {
|
||||
tableDecode[highThreshold--].symbol = (FSEv06_FUNCTION_TYPE)s;
|
||||
tableDecode[highThreshold--].symbol = (FSEv07_FUNCTION_TYPE)s;
|
||||
symbolNext[s] = 1;
|
||||
} else {
|
||||
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
|
||||
@@ -1968,12 +1949,12 @@ size_t FSEv06_buildDTable(FSEv06_DTable* dt, const short* normalizedCounter, uns
|
||||
|
||||
/* Spread symbols */
|
||||
{ U32 const tableMask = tableSize-1;
|
||||
U32 const step = FSEv06_TABLESTEP(tableSize);
|
||||
U32 const step = FSEv07_TABLESTEP(tableSize);
|
||||
U32 s, position = 0;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
int i;
|
||||
for (i=0; i<normalizedCounter[s]; i++) {
|
||||
tableDecode[position].symbol = (FSEv06_FUNCTION_TYPE)s;
|
||||
tableDecode[position].symbol = (FSEv07_FUNCTION_TYPE)s;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
|
||||
} }
|
||||
@@ -1984,9 +1965,9 @@ size_t FSEv06_buildDTable(FSEv06_DTable* dt, const short* normalizedCounter, uns
|
||||
/* Build Decoding table */
|
||||
{ U32 u;
|
||||
for (u=0; u<tableSize; u++) {
|
||||
FSEv06_FUNCTION_TYPE const symbol = (FSEv06_FUNCTION_TYPE)(tableDecode[u].symbol);
|
||||
FSEv07_FUNCTION_TYPE const symbol = (FSEv07_FUNCTION_TYPE)(tableDecode[u].symbol);
|
||||
U16 nextState = symbolNext[symbol]++;
|
||||
tableDecode[u].nbBits = (BYTE) (tableLog - BITv06_highbit32 ((U32)nextState) );
|
||||
tableDecode[u].nbBits = (BYTE) (tableLog - BITv07_highbit32 ((U32)nextState) );
|
||||
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
|
||||
} }
|
||||
|
||||
@@ -1995,17 +1976,17 @@ size_t FSEv06_buildDTable(FSEv06_DTable* dt, const short* normalizedCounter, uns
|
||||
|
||||
|
||||
|
||||
#ifndef FSEv06_COMMONDEFS_ONLY
|
||||
#ifndef FSEv07_COMMONDEFS_ONLY
|
||||
|
||||
/*-*******************************************************
|
||||
* Decompression (Byte symbols)
|
||||
*********************************************************/
|
||||
size_t FSEv06_buildDTable_rle (FSEv06_DTable* dt, BYTE symbolValue)
|
||||
size_t FSEv07_buildDTable_rle (FSEv07_DTable* dt, BYTE symbolValue)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSEv06_DTableHeader* const DTableH = (FSEv06_DTableHeader*)ptr;
|
||||
FSEv07_DTableHeader* const DTableH = (FSEv07_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSEv06_decode_t* const cell = (FSEv06_decode_t*)dPtr;
|
||||
FSEv07_decode_t* const cell = (FSEv07_decode_t*)dPtr;
|
||||
|
||||
DTableH->tableLog = 0;
|
||||
DTableH->fastMode = 0;
|
||||
@@ -2018,12 +1999,12 @@ size_t FSEv06_buildDTable_rle (FSEv06_DTable* dt, BYTE symbolValue)
|
||||
}
|
||||
|
||||
|
||||
size_t FSEv06_buildDTable_raw (FSEv06_DTable* dt, unsigned nbBits)
|
||||
size_t FSEv07_buildDTable_raw (FSEv07_DTable* dt, unsigned nbBits)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSEv06_DTableHeader* const DTableH = (FSEv06_DTableHeader*)ptr;
|
||||
FSEv07_DTableHeader* const DTableH = (FSEv07_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSEv06_decode_t* const dinfo = (FSEv06_decode_t*)dPtr;
|
||||
FSEv07_decode_t* const dinfo = (FSEv07_decode_t*)dPtr;
|
||||
const unsigned tableSize = 1 << nbBits;
|
||||
const unsigned tableMask = tableSize - 1;
|
||||
const unsigned maxSV1 = tableMask+1;
|
||||
@@ -2044,67 +2025,67 @@ size_t FSEv06_buildDTable_raw (FSEv06_DTable* dt, unsigned nbBits)
|
||||
return 0;
|
||||
}
|
||||
|
||||
FORCE_INLINE size_t FSEv06_decompress_usingDTable_generic(
|
||||
FORCE_INLINE size_t FSEv07_decompress_usingDTable_generic(
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const FSEv06_DTable* dt, const unsigned fast)
|
||||
const FSEv07_DTable* dt, const unsigned fast)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const omax = op + maxDstSize;
|
||||
BYTE* const olimit = omax-3;
|
||||
|
||||
BITv06_DStream_t bitD;
|
||||
FSEv06_DState_t state1;
|
||||
FSEv06_DState_t state2;
|
||||
BITv07_DStream_t bitD;
|
||||
FSEv07_DState_t state1;
|
||||
FSEv07_DState_t state2;
|
||||
|
||||
/* Init */
|
||||
{ size_t const errorCode = BITv06_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */
|
||||
if (FSEv06_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BITv07_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */
|
||||
if (FSEv07_isError(errorCode)) return errorCode; }
|
||||
|
||||
FSEv06_initDState(&state1, &bitD, dt);
|
||||
FSEv06_initDState(&state2, &bitD, dt);
|
||||
FSEv07_initDState(&state1, &bitD, dt);
|
||||
FSEv07_initDState(&state2, &bitD, dt);
|
||||
|
||||
#define FSEv06_GETSYMBOL(statePtr) fast ? FSEv06_decodeSymbolFast(statePtr, &bitD) : FSEv06_decodeSymbol(statePtr, &bitD)
|
||||
#define FSEv07_GETSYMBOL(statePtr) fast ? FSEv07_decodeSymbolFast(statePtr, &bitD) : FSEv07_decodeSymbol(statePtr, &bitD)
|
||||
|
||||
/* 4 symbols per loop */
|
||||
for ( ; (BITv06_reloadDStream(&bitD)==BITv06_DStream_unfinished) && (op<olimit) ; op+=4) {
|
||||
op[0] = FSEv06_GETSYMBOL(&state1);
|
||||
for ( ; (BITv07_reloadDStream(&bitD)==BITv07_DStream_unfinished) && (op<olimit) ; op+=4) {
|
||||
op[0] = FSEv07_GETSYMBOL(&state1);
|
||||
|
||||
if (FSEv06_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BITv06_reloadDStream(&bitD);
|
||||
if (FSEv07_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BITv07_reloadDStream(&bitD);
|
||||
|
||||
op[1] = FSEv06_GETSYMBOL(&state2);
|
||||
op[1] = FSEv07_GETSYMBOL(&state2);
|
||||
|
||||
if (FSEv06_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
{ if (BITv06_reloadDStream(&bitD) > BITv06_DStream_unfinished) { op+=2; break; } }
|
||||
if (FSEv07_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
{ if (BITv07_reloadDStream(&bitD) > BITv07_DStream_unfinished) { op+=2; break; } }
|
||||
|
||||
op[2] = FSEv06_GETSYMBOL(&state1);
|
||||
op[2] = FSEv07_GETSYMBOL(&state1);
|
||||
|
||||
if (FSEv06_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BITv06_reloadDStream(&bitD);
|
||||
if (FSEv07_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BITv07_reloadDStream(&bitD);
|
||||
|
||||
op[3] = FSEv06_GETSYMBOL(&state2);
|
||||
op[3] = FSEv07_GETSYMBOL(&state2);
|
||||
}
|
||||
|
||||
/* tail */
|
||||
/* note : BITv06_reloadDStream(&bitD) >= FSEv06_DStream_partiallyFilled; Ends at exactly BITv06_DStream_completed */
|
||||
/* note : BITv07_reloadDStream(&bitD) >= FSEv07_DStream_partiallyFilled; Ends at exactly BITv07_DStream_completed */
|
||||
while (1) {
|
||||
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
||||
|
||||
*op++ = FSEv06_GETSYMBOL(&state1);
|
||||
*op++ = FSEv07_GETSYMBOL(&state1);
|
||||
|
||||
if (BITv06_reloadDStream(&bitD)==BITv06_DStream_overflow) {
|
||||
*op++ = FSEv06_GETSYMBOL(&state2);
|
||||
if (BITv07_reloadDStream(&bitD)==BITv07_DStream_overflow) {
|
||||
*op++ = FSEv07_GETSYMBOL(&state2);
|
||||
break;
|
||||
}
|
||||
|
||||
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
||||
|
||||
*op++ = FSEv06_GETSYMBOL(&state2);
|
||||
*op++ = FSEv07_GETSYMBOL(&state2);
|
||||
|
||||
if (BITv06_reloadDStream(&bitD)==BITv06_DStream_overflow) {
|
||||
*op++ = FSEv06_GETSYMBOL(&state1);
|
||||
if (BITv07_reloadDStream(&bitD)==BITv07_DStream_overflow) {
|
||||
*op++ = FSEv07_GETSYMBOL(&state1);
|
||||
break;
|
||||
} }
|
||||
|
||||
@@ -2112,298 +2093,49 @@ FORCE_INLINE size_t FSEv06_decompress_usingDTable_generic(
|
||||
}
|
||||
|
||||
|
||||
size_t FSEv06_decompress_usingDTable(void* dst, size_t originalSize,
|
||||
size_t FSEv07_decompress_usingDTable(void* dst, size_t originalSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const FSEv06_DTable* dt)
|
||||
const FSEv07_DTable* dt)
|
||||
{
|
||||
const void* ptr = dt;
|
||||
const FSEv06_DTableHeader* DTableH = (const FSEv06_DTableHeader*)ptr;
|
||||
const FSEv07_DTableHeader* DTableH = (const FSEv07_DTableHeader*)ptr;
|
||||
const U32 fastMode = DTableH->fastMode;
|
||||
|
||||
/* select fast mode (static) */
|
||||
if (fastMode) return FSEv06_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
|
||||
return FSEv06_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
|
||||
if (fastMode) return FSEv07_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
|
||||
return FSEv07_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t FSEv06_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
|
||||
size_t FSEv07_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*)cSrc;
|
||||
const BYTE* ip = istart;
|
||||
short counting[FSEv06_MAX_SYMBOL_VALUE+1];
|
||||
short counting[FSEv07_MAX_SYMBOL_VALUE+1];
|
||||
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
|
||||
unsigned tableLog;
|
||||
unsigned maxSymbolValue = FSEv06_MAX_SYMBOL_VALUE;
|
||||
unsigned maxSymbolValue = FSEv07_MAX_SYMBOL_VALUE;
|
||||
|
||||
if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */
|
||||
|
||||
/* normal FSE decoding mode */
|
||||
{ size_t const NCountLength = FSEv06_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
|
||||
if (FSEv06_isError(NCountLength)) return NCountLength;
|
||||
{ size_t const NCountLength = FSEv07_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
|
||||
if (FSEv07_isError(NCountLength)) return NCountLength;
|
||||
if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */
|
||||
ip += NCountLength;
|
||||
cSrcSize -= NCountLength;
|
||||
}
|
||||
|
||||
{ size_t const errorCode = FSEv06_buildDTable (dt, counting, maxSymbolValue, tableLog);
|
||||
if (FSEv06_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = FSEv07_buildDTable (dt, counting, maxSymbolValue, tableLog);
|
||||
if (FSEv07_isError(errorCode)) return errorCode; }
|
||||
|
||||
return FSEv06_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */
|
||||
return FSEv07_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */
|
||||
}
|
||||
|
||||
|
||||
|
||||
#endif /* FSEv06_COMMONDEFS_ONLY */
|
||||
/* ******************************************************************
|
||||
Huffman coder, part of New Generation Entropy library
|
||||
header file
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
#endif /* FSEv07_COMMONDEFS_ONLY */
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef HUFv06_H
|
||||
#define HUFv06_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* HUF simple functions
|
||||
******************************************/
|
||||
size_t HUFv06_decompress(void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
/*
|
||||
HUFv06_decompress() :
|
||||
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated destination buffer 'dst', of size 'dstSize'.
|
||||
`dstSize` : must be the **exact** size of original (uncompressed) data.
|
||||
Note : in contrast with FSE, HUFv06_decompress can regenerate
|
||||
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
|
||||
because it knows size to regenerate.
|
||||
@return : size of regenerated data (== dstSize)
|
||||
or an error code, which can be tested using HUFv06_isError()
|
||||
*/
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Tool functions
|
||||
******************************************/
|
||||
size_t HUFv06_compressBound(size_t size); /**< maximum compressed size */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* HUFv06_H */
|
||||
/* ******************************************************************
|
||||
Huffman codec, part of New Generation Entropy library
|
||||
header file, for static linking only
|
||||
Copyright (C) 2013-2016, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef HUFv06_STATIC_H
|
||||
#define HUFv06_STATIC_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Static allocation
|
||||
******************************************/
|
||||
/* HUF buffer bounds */
|
||||
#define HUFv06_CTABLEBOUND 129
|
||||
#define HUFv06_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
|
||||
#define HUFv06_COMPRESSBOUND(size) (HUFv06_CTABLEBOUND + HUFv06_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* static allocation of HUF's DTable */
|
||||
#define HUFv06_DTABLE_SIZE(maxTableLog) (1 + (1<<maxTableLog))
|
||||
#define HUFv06_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
||||
unsigned short DTable[HUFv06_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
|
||||
#define HUFv06_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
|
||||
unsigned int DTable[HUFv06_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
|
||||
#define HUFv06_CREATE_STATIC_DTABLEX6(DTable, maxTableLog) \
|
||||
unsigned int DTable[HUFv06_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog }
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Advanced decompression functions
|
||||
******************************************/
|
||||
size_t HUFv06_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUFv06_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
|
||||
|
||||
|
||||
|
||||
/*!
|
||||
HUFv06_decompress() does the following:
|
||||
1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics
|
||||
2. build Huffman table from save, using HUFv06_readDTableXn()
|
||||
3. decode 1 or 4 segments in parallel using HUFv06_decompressSXn_usingDTable
|
||||
*/
|
||||
size_t HUFv06_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
|
||||
size_t HUFv06_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
|
||||
|
||||
size_t HUFv06_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
|
||||
size_t HUFv06_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
||||
|
||||
|
||||
/* single stream variants */
|
||||
size_t HUFv06_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUFv06_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
||||
|
||||
size_t HUFv06_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
|
||||
size_t HUFv06_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
|
||||
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Constants
|
||||
****************************************************************/
|
||||
#define HUFv06_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUFv06_MAX_TABLELOG. Beyond that value, code does not work */
|
||||
#define HUFv06_MAX_TABLELOG 12 /* max configured tableLog (for static allocation); can be modified up to HUFv06_ABSOLUTEMAX_TABLELOG */
|
||||
#define HUFv06_DEFAULT_TABLELOG HUFv06_MAX_TABLELOG /* tableLog by default, when not specified */
|
||||
#define HUFv06_MAX_SYMBOL_VALUE 255
|
||||
#if (HUFv06_MAX_TABLELOG > HUFv06_ABSOLUTEMAX_TABLELOG)
|
||||
# error "HUFv06_MAX_TABLELOG is too large !"
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*! HUFv06_readStats() :
|
||||
Read compact Huffman tree, saved by HUFv06_writeCTable().
|
||||
`huffWeight` is destination buffer.
|
||||
@return : size read from `src`
|
||||
*/
|
||||
MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 weightTotal;
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
size_t iSize = ip[0];
|
||||
size_t oSize;
|
||||
|
||||
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
if (iSize >= 128) { /* special header */
|
||||
if (iSize >= (242)) { /* RLE */
|
||||
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
|
||||
oSize = l[iSize-242];
|
||||
memset(huffWeight, 1, hwSize);
|
||||
iSize = 0;
|
||||
}
|
||||
else { /* Incompressible */
|
||||
oSize = iSize - 127;
|
||||
iSize = ((oSize+1)/2);
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
if (oSize >= hwSize) return ERROR(corruption_detected);
|
||||
ip += 1;
|
||||
{ U32 n;
|
||||
for (n=0; n<oSize; n+=2) {
|
||||
huffWeight[n] = ip[n/2] >> 4;
|
||||
huffWeight[n+1] = ip[n/2] & 15;
|
||||
} } } }
|
||||
else { /* header compressed with FSE (normal case) */
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
oSize = FSEv06_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
|
||||
if (FSEv06_isError(oSize)) return oSize;
|
||||
}
|
||||
|
||||
/* collect weight stats */
|
||||
memset(rankStats, 0, (HUFv06_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32));
|
||||
weightTotal = 0;
|
||||
{ U32 n; for (n=0; n<oSize; n++) {
|
||||
if (huffWeight[n] >= HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
|
||||
rankStats[huffWeight[n]]++;
|
||||
weightTotal += (1 << huffWeight[n]) >> 1;
|
||||
} }
|
||||
|
||||
/* get last non-null symbol weight (implied, total must be 2^n) */
|
||||
{ U32 const tableLog = BITv06_highbit32(weightTotal) + 1;
|
||||
if (tableLog > HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
|
||||
*tableLogPtr = tableLog;
|
||||
/* determine last weight */
|
||||
{ U32 const total = 1 << tableLog;
|
||||
U32 const rest = total - weightTotal;
|
||||
U32 const verif = 1 << BITv06_highbit32(rest);
|
||||
U32 const lastWeight = BITv06_highbit32(rest) + 1;
|
||||
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
||||
huffWeight[oSize] = (BYTE)lastWeight;
|
||||
rankStats[lastWeight]++;
|
||||
} }
|
||||
|
||||
/* check tree construction validity */
|
||||
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
|
||||
|
||||
/* results */
|
||||
*nbSymbolsPtr = (U32)(oSize+1);
|
||||
return iSize+1;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* HUFv06_STATIC_H */
|
||||
/* ******************************************************************
|
||||
Huffman decoder, part of New Generation Entropy library
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
@@ -2466,155 +2198,177 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
****************************************************************/
|
||||
#define HUFv06_STATIC_ASSERT(c) { enum { HUFv06_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
#define HUFv07_STATIC_ASSERT(c) { enum { HUFv07_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
|
||||
|
||||
/*-***************************/
|
||||
/* generic DTableDesc */
|
||||
/*-***************************/
|
||||
|
||||
/* *******************************************************
|
||||
* HUF : Huffman block decompression
|
||||
*********************************************************/
|
||||
typedef struct { BYTE byte; BYTE nbBits; } HUFv06_DEltX2; /* single-symbol decoding */
|
||||
|
||||
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUFv06_DEltX4; /* double-symbols decoding */
|
||||
|
||||
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
|
||||
typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;
|
||||
|
||||
static DTableDesc HUFv07_getDTableDesc(const HUFv07_DTable* table)
|
||||
{
|
||||
DTableDesc dtd;
|
||||
memcpy(&dtd, table, sizeof(dtd));
|
||||
return dtd;
|
||||
}
|
||||
|
||||
|
||||
/*-***************************/
|
||||
/* single-symbol decoding */
|
||||
/*-***************************/
|
||||
|
||||
size_t HUFv06_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE huffWeight[HUFv06_MAX_SYMBOL_VALUE + 1];
|
||||
U32 rankVal[HUFv06_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
|
||||
U32 tableLog = 0;
|
||||
size_t iSize;
|
||||
U32 nbSymbols = 0;
|
||||
U32 n;
|
||||
U32 nextRankStart;
|
||||
void* const dtPtr = DTable + 1;
|
||||
HUFv06_DEltX2* const dt = (HUFv06_DEltX2*)dtPtr;
|
||||
typedef struct { BYTE byte; BYTE nbBits; } HUFv07_DEltX2; /* single-symbol decoding */
|
||||
|
||||
HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */
|
||||
size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE huffWeight[HUFv07_SYMBOLVALUE_MAX + 1];
|
||||
U32 rankVal[HUFv07_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
|
||||
U32 tableLog = 0;
|
||||
U32 nbSymbols = 0;
|
||||
size_t iSize;
|
||||
void* const dtPtr = DTable + 1;
|
||||
HUFv07_DEltX2* const dt = (HUFv07_DEltX2*)dtPtr;
|
||||
|
||||
HUFv07_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv07_DTable));
|
||||
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
iSize = HUFv06_readStats(huffWeight, HUFv06_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
||||
if (HUFv06_isError(iSize)) return iSize;
|
||||
iSize = HUFv07_readStats(huffWeight, HUFv07_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
||||
if (HUFv07_isError(iSize)) return iSize;
|
||||
|
||||
/* check result */
|
||||
if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge); /* DTable is too small */
|
||||
DTable[0] = (U16)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */
|
||||
/* Table header */
|
||||
{ DTableDesc dtd = HUFv07_getDTableDesc(DTable);
|
||||
if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */
|
||||
dtd.tableType = 0;
|
||||
dtd.tableLog = (BYTE)tableLog;
|
||||
memcpy(DTable, &dtd, sizeof(dtd));
|
||||
}
|
||||
|
||||
/* Prepare ranks */
|
||||
nextRankStart = 0;
|
||||
for (n=1; n<tableLog+1; n++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += (rankVal[n] << (n-1));
|
||||
rankVal[n] = current;
|
||||
}
|
||||
{ U32 n, nextRankStart = 0;
|
||||
for (n=1; n<tableLog+1; n++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += (rankVal[n] << (n-1));
|
||||
rankVal[n] = current;
|
||||
} }
|
||||
|
||||
/* fill DTable */
|
||||
for (n=0; n<nbSymbols; n++) {
|
||||
const U32 w = huffWeight[n];
|
||||
const U32 length = (1 << w) >> 1;
|
||||
U32 i;
|
||||
HUFv06_DEltX2 D;
|
||||
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
|
||||
for (i = rankVal[w]; i < rankVal[w] + length; i++)
|
||||
dt[i] = D;
|
||||
rankVal[w] += length;
|
||||
}
|
||||
{ U32 n;
|
||||
for (n=0; n<nbSymbols; n++) {
|
||||
U32 const w = huffWeight[n];
|
||||
U32 const length = (1 << w) >> 1;
|
||||
U32 i;
|
||||
HUFv07_DEltX2 D;
|
||||
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
|
||||
for (i = rankVal[w]; i < rankVal[w] + length; i++)
|
||||
dt[i] = D;
|
||||
rankVal[w] += length;
|
||||
} }
|
||||
|
||||
return iSize;
|
||||
}
|
||||
|
||||
|
||||
static BYTE HUFv06_decodeSymbolX2(BITv06_DStream_t* Dstream, const HUFv06_DEltX2* dt, const U32 dtLog)
|
||||
static BYTE HUFv07_decodeSymbolX2(BITv07_DStream_t* Dstream, const HUFv07_DEltX2* dt, const U32 dtLog)
|
||||
{
|
||||
const size_t val = BITv06_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
|
||||
const BYTE c = dt[val].byte;
|
||||
BITv06_skipBits(Dstream, dt[val].nbBits);
|
||||
size_t const val = BITv07_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
|
||||
BYTE const c = dt[val].byte;
|
||||
BITv07_skipBits(Dstream, dt[val].nbBits);
|
||||
return c;
|
||||
}
|
||||
|
||||
#define HUFv06_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \
|
||||
*ptr++ = HUFv06_decodeSymbolX2(DStreamPtr, dt, dtLog)
|
||||
#define HUFv07_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \
|
||||
*ptr++ = HUFv07_decodeSymbolX2(DStreamPtr, dt, dtLog)
|
||||
|
||||
#define HUFv06_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
|
||||
if (MEM_64bits() || (HUFv06_MAX_TABLELOG<=12)) \
|
||||
HUFv06_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
|
||||
#define HUFv07_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
|
||||
if (MEM_64bits() || (HUFv07_TABLELOG_MAX<=12)) \
|
||||
HUFv07_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
|
||||
|
||||
#define HUFv06_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
|
||||
#define HUFv07_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
|
||||
if (MEM_64bits()) \
|
||||
HUFv06_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
|
||||
HUFv07_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
|
||||
|
||||
static inline size_t HUFv06_decodeStreamX2(BYTE* p, BITv06_DStream_t* const bitDPtr, BYTE* const pEnd, const HUFv06_DEltX2* const dt, const U32 dtLog)
|
||||
static inline size_t HUFv07_decodeStreamX2(BYTE* p, BITv07_DStream_t* const bitDPtr, BYTE* const pEnd, const HUFv07_DEltX2* const dt, const U32 dtLog)
|
||||
{
|
||||
BYTE* const pStart = p;
|
||||
|
||||
/* up to 4 symbols at a time */
|
||||
while ((BITv06_reloadDStream(bitDPtr) == BITv06_DStream_unfinished) && (p <= pEnd-4)) {
|
||||
HUFv06_DECODE_SYMBOLX2_2(p, bitDPtr);
|
||||
HUFv06_DECODE_SYMBOLX2_1(p, bitDPtr);
|
||||
HUFv06_DECODE_SYMBOLX2_2(p, bitDPtr);
|
||||
HUFv06_DECODE_SYMBOLX2_0(p, bitDPtr);
|
||||
while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p <= pEnd-4)) {
|
||||
HUFv07_DECODE_SYMBOLX2_2(p, bitDPtr);
|
||||
HUFv07_DECODE_SYMBOLX2_1(p, bitDPtr);
|
||||
HUFv07_DECODE_SYMBOLX2_2(p, bitDPtr);
|
||||
HUFv07_DECODE_SYMBOLX2_0(p, bitDPtr);
|
||||
}
|
||||
|
||||
/* closer to the end */
|
||||
while ((BITv06_reloadDStream(bitDPtr) == BITv06_DStream_unfinished) && (p < pEnd))
|
||||
HUFv06_DECODE_SYMBOLX2_0(p, bitDPtr);
|
||||
while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p < pEnd))
|
||||
HUFv07_DECODE_SYMBOLX2_0(p, bitDPtr);
|
||||
|
||||
/* no more data to retrieve from bitstream, hence no need to reload */
|
||||
while (p < pEnd)
|
||||
HUFv06_DECODE_SYMBOLX2_0(p, bitDPtr);
|
||||
HUFv07_DECODE_SYMBOLX2_0(p, bitDPtr);
|
||||
|
||||
return pEnd-pStart;
|
||||
}
|
||||
|
||||
size_t HUFv06_decompress1X2_usingDTable(
|
||||
static size_t HUFv07_decompress1X2_usingDTable_internal(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const U16* DTable)
|
||||
const HUFv07_DTable* DTable)
|
||||
{
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = op + dstSize;
|
||||
const U32 dtLog = DTable[0];
|
||||
const void* dtPtr = DTable;
|
||||
const HUFv06_DEltX2* const dt = ((const HUFv06_DEltX2*)dtPtr)+1;
|
||||
BITv06_DStream_t bitD;
|
||||
const void* dtPtr = DTable + 1;
|
||||
const HUFv07_DEltX2* const dt = (const HUFv07_DEltX2*)dtPtr;
|
||||
BITv07_DStream_t bitD;
|
||||
DTableDesc const dtd = HUFv07_getDTableDesc(DTable);
|
||||
U32 const dtLog = dtd.tableLog;
|
||||
|
||||
{ size_t const errorCode = BITv06_initDStream(&bitD, cSrc, cSrcSize);
|
||||
if (HUFv06_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BITv07_initDStream(&bitD, cSrc, cSrcSize);
|
||||
if (HUFv07_isError(errorCode)) return errorCode; }
|
||||
|
||||
HUFv06_decodeStreamX2(op, &bitD, oend, dt, dtLog);
|
||||
HUFv07_decodeStreamX2(op, &bitD, oend, dt, dtLog);
|
||||
|
||||
/* check */
|
||||
if (!BITv06_endOfDStream(&bitD)) return ERROR(corruption_detected);
|
||||
if (!BITv07_endOfDStream(&bitD)) return ERROR(corruption_detected);
|
||||
|
||||
return dstSize;
|
||||
}
|
||||
|
||||
size_t HUFv06_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
size_t HUFv07_decompress1X2_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv07_DTable* DTable)
|
||||
{
|
||||
DTableDesc dtd = HUFv07_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 0) return ERROR(GENERIC);
|
||||
return HUFv07_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
size_t HUFv07_decompress1X2_DCtx (HUFv07_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUFv06_CREATE_STATIC_DTABLEX2(DTable, HUFv06_MAX_TABLELOG);
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const errorCode = HUFv06_readDTableX2 (DTable, cSrc, cSrcSize);
|
||||
if (HUFv06_isError(errorCode)) return errorCode;
|
||||
if (errorCode >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += errorCode;
|
||||
cSrcSize -= errorCode;
|
||||
size_t const hSize = HUFv07_readDTableX2 (DCtx, cSrc, cSrcSize);
|
||||
if (HUFv07_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUFv06_decompress1X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
|
||||
return HUFv07_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
|
||||
}
|
||||
|
||||
size_t HUFv07_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUFv07_CREATE_STATIC_DTABLEX2(DTable, HUFv07_TABLELOG_MAX);
|
||||
return HUFv07_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t HUFv06_decompress4X2_usingDTable(
|
||||
static size_t HUFv07_decompress4X2_usingDTable_internal(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const U16* DTable)
|
||||
const HUFv07_DTable* DTable)
|
||||
{
|
||||
/* Check */
|
||||
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
|
||||
@@ -2622,20 +2376,18 @@ size_t HUFv06_decompress4X2_usingDTable(
|
||||
{ const BYTE* const istart = (const BYTE*) cSrc;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
const void* const dtPtr = DTable;
|
||||
const HUFv06_DEltX2* const dt = ((const HUFv06_DEltX2*)dtPtr) +1;
|
||||
const U32 dtLog = DTable[0];
|
||||
size_t errorCode;
|
||||
const void* const dtPtr = DTable + 1;
|
||||
const HUFv07_DEltX2* const dt = (const HUFv07_DEltX2*)dtPtr;
|
||||
|
||||
/* Init */
|
||||
BITv06_DStream_t bitD1;
|
||||
BITv06_DStream_t bitD2;
|
||||
BITv06_DStream_t bitD3;
|
||||
BITv06_DStream_t bitD4;
|
||||
const size_t length1 = MEM_readLE16(istart);
|
||||
const size_t length2 = MEM_readLE16(istart+2);
|
||||
const size_t length3 = MEM_readLE16(istart+4);
|
||||
size_t length4;
|
||||
BITv07_DStream_t bitD1;
|
||||
BITv07_DStream_t bitD2;
|
||||
BITv07_DStream_t bitD3;
|
||||
BITv07_DStream_t bitD4;
|
||||
size_t const length1 = MEM_readLE16(istart);
|
||||
size_t const length2 = MEM_readLE16(istart+2);
|
||||
size_t const length3 = MEM_readLE16(istart+4);
|
||||
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
|
||||
const BYTE* const istart1 = istart + 6; /* jumpTable */
|
||||
const BYTE* const istart2 = istart1 + length1;
|
||||
const BYTE* const istart3 = istart2 + length2;
|
||||
@@ -2649,38 +2401,39 @@ size_t HUFv06_decompress4X2_usingDTable(
|
||||
BYTE* op3 = opStart3;
|
||||
BYTE* op4 = opStart4;
|
||||
U32 endSignal;
|
||||
DTableDesc const dtd = HUFv07_getDTableDesc(DTable);
|
||||
U32 const dtLog = dtd.tableLog;
|
||||
|
||||
length4 = cSrcSize - (length1 + length2 + length3 + 6);
|
||||
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
|
||||
errorCode = BITv06_initDStream(&bitD1, istart1, length1);
|
||||
if (HUFv06_isError(errorCode)) return errorCode;
|
||||
errorCode = BITv06_initDStream(&bitD2, istart2, length2);
|
||||
if (HUFv06_isError(errorCode)) return errorCode;
|
||||
errorCode = BITv06_initDStream(&bitD3, istart3, length3);
|
||||
if (HUFv06_isError(errorCode)) return errorCode;
|
||||
errorCode = BITv06_initDStream(&bitD4, istart4, length4);
|
||||
if (HUFv06_isError(errorCode)) return errorCode;
|
||||
{ size_t const errorCode = BITv07_initDStream(&bitD1, istart1, length1);
|
||||
if (HUFv07_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BITv07_initDStream(&bitD2, istart2, length2);
|
||||
if (HUFv07_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BITv07_initDStream(&bitD3, istart3, length3);
|
||||
if (HUFv07_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BITv07_initDStream(&bitD4, istart4, length4);
|
||||
if (HUFv07_isError(errorCode)) return errorCode; }
|
||||
|
||||
/* 16-32 symbols per loop (4-8 symbols per stream) */
|
||||
endSignal = BITv06_reloadDStream(&bitD1) | BITv06_reloadDStream(&bitD2) | BITv06_reloadDStream(&bitD3) | BITv06_reloadDStream(&bitD4);
|
||||
for ( ; (endSignal==BITv06_DStream_unfinished) && (op4<(oend-7)) ; ) {
|
||||
HUFv06_DECODE_SYMBOLX2_2(op1, &bitD1);
|
||||
HUFv06_DECODE_SYMBOLX2_2(op2, &bitD2);
|
||||
HUFv06_DECODE_SYMBOLX2_2(op3, &bitD3);
|
||||
HUFv06_DECODE_SYMBOLX2_2(op4, &bitD4);
|
||||
HUFv06_DECODE_SYMBOLX2_1(op1, &bitD1);
|
||||
HUFv06_DECODE_SYMBOLX2_1(op2, &bitD2);
|
||||
HUFv06_DECODE_SYMBOLX2_1(op3, &bitD3);
|
||||
HUFv06_DECODE_SYMBOLX2_1(op4, &bitD4);
|
||||
HUFv06_DECODE_SYMBOLX2_2(op1, &bitD1);
|
||||
HUFv06_DECODE_SYMBOLX2_2(op2, &bitD2);
|
||||
HUFv06_DECODE_SYMBOLX2_2(op3, &bitD3);
|
||||
HUFv06_DECODE_SYMBOLX2_2(op4, &bitD4);
|
||||
HUFv06_DECODE_SYMBOLX2_0(op1, &bitD1);
|
||||
HUFv06_DECODE_SYMBOLX2_0(op2, &bitD2);
|
||||
HUFv06_DECODE_SYMBOLX2_0(op3, &bitD3);
|
||||
HUFv06_DECODE_SYMBOLX2_0(op4, &bitD4);
|
||||
endSignal = BITv06_reloadDStream(&bitD1) | BITv06_reloadDStream(&bitD2) | BITv06_reloadDStream(&bitD3) | BITv06_reloadDStream(&bitD4);
|
||||
endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4);
|
||||
for ( ; (endSignal==BITv07_DStream_unfinished) && (op4<(oend-7)) ; ) {
|
||||
HUFv07_DECODE_SYMBOLX2_2(op1, &bitD1);
|
||||
HUFv07_DECODE_SYMBOLX2_2(op2, &bitD2);
|
||||
HUFv07_DECODE_SYMBOLX2_2(op3, &bitD3);
|
||||
HUFv07_DECODE_SYMBOLX2_2(op4, &bitD4);
|
||||
HUFv07_DECODE_SYMBOLX2_1(op1, &bitD1);
|
||||
HUFv07_DECODE_SYMBOLX2_1(op2, &bitD2);
|
||||
HUFv07_DECODE_SYMBOLX2_1(op3, &bitD3);
|
||||
HUFv07_DECODE_SYMBOLX2_1(op4, &bitD4);
|
||||
HUFv07_DECODE_SYMBOLX2_2(op1, &bitD1);
|
||||
HUFv07_DECODE_SYMBOLX2_2(op2, &bitD2);
|
||||
HUFv07_DECODE_SYMBOLX2_2(op3, &bitD3);
|
||||
HUFv07_DECODE_SYMBOLX2_2(op4, &bitD4);
|
||||
HUFv07_DECODE_SYMBOLX2_0(op1, &bitD1);
|
||||
HUFv07_DECODE_SYMBOLX2_0(op2, &bitD2);
|
||||
HUFv07_DECODE_SYMBOLX2_0(op3, &bitD3);
|
||||
HUFv07_DECODE_SYMBOLX2_0(op4, &bitD4);
|
||||
endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4);
|
||||
}
|
||||
|
||||
/* check corruption */
|
||||
@@ -2690,13 +2443,13 @@ size_t HUFv06_decompress4X2_usingDTable(
|
||||
/* note : op4 supposed already verified within main loop */
|
||||
|
||||
/* finish bitStreams one by one */
|
||||
HUFv06_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
|
||||
HUFv06_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
|
||||
HUFv06_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
|
||||
HUFv06_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
|
||||
HUFv07_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
|
||||
HUFv07_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
|
||||
HUFv07_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
|
||||
HUFv07_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
|
||||
|
||||
/* check */
|
||||
endSignal = BITv06_endOfDStream(&bitD1) & BITv06_endOfDStream(&bitD2) & BITv06_endOfDStream(&bitD3) & BITv06_endOfDStream(&bitD4);
|
||||
endSignal = BITv07_endOfDStream(&bitD1) & BITv07_endOfDStream(&bitD2) & BITv07_endOfDStream(&bitD3) & BITv07_endOfDStream(&bitD4);
|
||||
if (!endSignal) return ERROR(corruption_detected);
|
||||
|
||||
/* decoded size */
|
||||
@@ -2705,32 +2458,50 @@ size_t HUFv06_decompress4X2_usingDTable(
|
||||
}
|
||||
|
||||
|
||||
size_t HUFv06_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
size_t HUFv07_decompress4X2_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv07_DTable* DTable)
|
||||
{
|
||||
DTableDesc dtd = HUFv07_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 0) return ERROR(GENERIC);
|
||||
return HUFv07_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
size_t HUFv07_decompress4X2_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUFv06_CREATE_STATIC_DTABLEX2(DTable, HUFv06_MAX_TABLELOG);
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const errorCode = HUFv06_readDTableX2 (DTable, cSrc, cSrcSize);
|
||||
if (HUFv06_isError(errorCode)) return errorCode;
|
||||
if (errorCode >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += errorCode;
|
||||
cSrcSize -= errorCode;
|
||||
size_t const hSize = HUFv07_readDTableX2 (dctx, cSrc, cSrcSize);
|
||||
if (HUFv07_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUFv06_decompress4X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
|
||||
return HUFv07_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx);
|
||||
}
|
||||
|
||||
size_t HUFv07_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUFv07_CREATE_STATIC_DTABLEX2(DTable, HUFv07_TABLELOG_MAX);
|
||||
return HUFv07_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
|
||||
/* *************************/
|
||||
/* double-symbols decoding */
|
||||
/* *************************/
|
||||
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUFv07_DEltX4; /* double-symbols decoding */
|
||||
|
||||
static void HUFv06_fillDTableX4Level2(HUFv06_DEltX4* DTable, U32 sizeLog, const U32 consumed,
|
||||
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
|
||||
|
||||
static void HUFv07_fillDTableX4Level2(HUFv07_DEltX4* DTable, U32 sizeLog, const U32 consumed,
|
||||
const U32* rankValOrigin, const int minWeight,
|
||||
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,
|
||||
U32 nbBitsBaseline, U16 baseSeq)
|
||||
{
|
||||
HUFv06_DEltX4 DElt;
|
||||
U32 rankVal[HUFv06_ABSOLUTEMAX_TABLELOG + 1];
|
||||
HUFv07_DEltX4 DElt;
|
||||
U32 rankVal[HUFv07_TABLELOG_ABSOLUTEMAX + 1];
|
||||
|
||||
/* get pre-calculated rankVal */
|
||||
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
|
||||
@@ -2764,14 +2535,14 @@ static void HUFv06_fillDTableX4Level2(HUFv06_DEltX4* DTable, U32 sizeLog, const
|
||||
}}
|
||||
}
|
||||
|
||||
typedef U32 rankVal_t[HUFv06_ABSOLUTEMAX_TABLELOG][HUFv06_ABSOLUTEMAX_TABLELOG + 1];
|
||||
typedef U32 rankVal_t[HUFv07_TABLELOG_ABSOLUTEMAX][HUFv07_TABLELOG_ABSOLUTEMAX + 1];
|
||||
|
||||
static void HUFv06_fillDTableX4(HUFv06_DEltX4* DTable, const U32 targetLog,
|
||||
static void HUFv07_fillDTableX4(HUFv07_DEltX4* DTable, const U32 targetLog,
|
||||
const sortedSymbol_t* sortedList, const U32 sortedListSize,
|
||||
const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
|
||||
const U32 nbBitsBaseline)
|
||||
{
|
||||
U32 rankVal[HUFv06_ABSOLUTEMAX_TABLELOG + 1];
|
||||
U32 rankVal[HUFv07_TABLELOG_ABSOLUTEMAX + 1];
|
||||
const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
|
||||
const U32 minBits = nbBitsBaseline - maxWeight;
|
||||
U32 s;
|
||||
@@ -2791,12 +2562,12 @@ static void HUFv06_fillDTableX4(HUFv06_DEltX4* DTable, const U32 targetLog,
|
||||
int minWeight = nbBits + scaleLog;
|
||||
if (minWeight < 1) minWeight = 1;
|
||||
sortedRank = rankStart[minWeight];
|
||||
HUFv06_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits,
|
||||
HUFv07_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits,
|
||||
rankValOrigin[nbBits], minWeight,
|
||||
sortedList+sortedRank, sortedListSize-sortedRank,
|
||||
nbBitsBaseline, symbol);
|
||||
} else {
|
||||
HUFv06_DEltX4 DElt;
|
||||
HUFv07_DEltX4 DElt;
|
||||
MEM_writeLE16(&(DElt.sequence), symbol);
|
||||
DElt.nbBits = (BYTE)(nbBits);
|
||||
DElt.length = 1;
|
||||
@@ -2808,29 +2579,30 @@ static void HUFv06_fillDTableX4(HUFv06_DEltX4* DTable, const U32 targetLog,
|
||||
}
|
||||
}
|
||||
|
||||
size_t HUFv06_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
||||
size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE weightList[HUFv06_MAX_SYMBOL_VALUE + 1];
|
||||
sortedSymbol_t sortedSymbol[HUFv06_MAX_SYMBOL_VALUE + 1];
|
||||
U32 rankStats[HUFv06_ABSOLUTEMAX_TABLELOG + 1] = { 0 };
|
||||
U32 rankStart0[HUFv06_ABSOLUTEMAX_TABLELOG + 2] = { 0 };
|
||||
BYTE weightList[HUFv07_SYMBOLVALUE_MAX + 1];
|
||||
sortedSymbol_t sortedSymbol[HUFv07_SYMBOLVALUE_MAX + 1];
|
||||
U32 rankStats[HUFv07_TABLELOG_ABSOLUTEMAX + 1] = { 0 };
|
||||
U32 rankStart0[HUFv07_TABLELOG_ABSOLUTEMAX + 2] = { 0 };
|
||||
U32* const rankStart = rankStart0+1;
|
||||
rankVal_t rankVal;
|
||||
U32 tableLog, maxW, sizeOfSort, nbSymbols;
|
||||
const U32 memLog = DTable[0];
|
||||
DTableDesc dtd = HUFv07_getDTableDesc(DTable);
|
||||
U32 const maxTableLog = dtd.maxTableLog;
|
||||
size_t iSize;
|
||||
void* dtPtr = DTable;
|
||||
HUFv06_DEltX4* const dt = ((HUFv06_DEltX4*)dtPtr) + 1;
|
||||
void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
|
||||
HUFv07_DEltX4* const dt = (HUFv07_DEltX4*)dtPtr;
|
||||
|
||||
HUFv06_STATIC_ASSERT(sizeof(HUFv06_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
|
||||
if (memLog > HUFv06_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
HUFv07_STATIC_ASSERT(sizeof(HUFv07_DEltX4) == sizeof(HUFv07_DTable)); /* if compilation fails here, assertion is false */
|
||||
if (maxTableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
|
||||
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
iSize = HUFv06_readStats(weightList, HUFv06_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
||||
if (HUFv06_isError(iSize)) return iSize;
|
||||
iSize = HUFv07_readStats(weightList, HUFv07_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
||||
if (HUFv07_isError(iSize)) return iSize;
|
||||
|
||||
/* check result */
|
||||
if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
|
||||
if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
|
||||
|
||||
/* find maxWeight */
|
||||
for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
|
||||
@@ -2859,7 +2631,7 @@ size_t HUFv06_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
||||
|
||||
/* Build rankVal */
|
||||
{ U32* const rankVal0 = rankVal[0];
|
||||
{ int const rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
|
||||
{ int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */
|
||||
U32 nextRankVal = 0;
|
||||
U32 w;
|
||||
for (w=1; w<maxW+1; w++) {
|
||||
@@ -2869,38 +2641,41 @@ size_t HUFv06_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
|
||||
} }
|
||||
{ U32 const minBits = tableLog+1 - maxW;
|
||||
U32 consumed;
|
||||
for (consumed = minBits; consumed < memLog - minBits + 1; consumed++) {
|
||||
for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
|
||||
U32* const rankValPtr = rankVal[consumed];
|
||||
U32 w;
|
||||
for (w = 1; w < maxW+1; w++) {
|
||||
rankValPtr[w] = rankVal0[w] >> consumed;
|
||||
} } } }
|
||||
|
||||
HUFv06_fillDTableX4(dt, memLog,
|
||||
HUFv07_fillDTableX4(dt, maxTableLog,
|
||||
sortedSymbol, sizeOfSort,
|
||||
rankStart0, rankVal, maxW,
|
||||
tableLog+1);
|
||||
|
||||
dtd.tableLog = (BYTE)maxTableLog;
|
||||
dtd.tableType = 1;
|
||||
memcpy(DTable, &dtd, sizeof(dtd));
|
||||
return iSize;
|
||||
}
|
||||
|
||||
|
||||
static U32 HUFv06_decodeSymbolX4(void* op, BITv06_DStream_t* DStream, const HUFv06_DEltX4* dt, const U32 dtLog)
|
||||
static U32 HUFv07_decodeSymbolX4(void* op, BITv07_DStream_t* DStream, const HUFv07_DEltX4* dt, const U32 dtLog)
|
||||
{
|
||||
const size_t val = BITv06_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
|
||||
const size_t val = BITv07_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
|
||||
memcpy(op, dt+val, 2);
|
||||
BITv06_skipBits(DStream, dt[val].nbBits);
|
||||
BITv07_skipBits(DStream, dt[val].nbBits);
|
||||
return dt[val].length;
|
||||
}
|
||||
|
||||
static U32 HUFv06_decodeLastSymbolX4(void* op, BITv06_DStream_t* DStream, const HUFv06_DEltX4* dt, const U32 dtLog)
|
||||
static U32 HUFv07_decodeLastSymbolX4(void* op, BITv07_DStream_t* DStream, const HUFv07_DEltX4* dt, const U32 dtLog)
|
||||
{
|
||||
const size_t val = BITv06_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
|
||||
const size_t val = BITv07_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
|
||||
memcpy(op, dt+val, 1);
|
||||
if (dt[val].length==1) BITv06_skipBits(DStream, dt[val].nbBits);
|
||||
if (dt[val].length==1) BITv07_skipBits(DStream, dt[val].nbBits);
|
||||
else {
|
||||
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
|
||||
BITv06_skipBits(DStream, dt[val].nbBits);
|
||||
BITv07_skipBits(DStream, dt[val].nbBits);
|
||||
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
|
||||
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
|
||||
} }
|
||||
@@ -2908,114 +2683,126 @@ static U32 HUFv06_decodeLastSymbolX4(void* op, BITv06_DStream_t* DStream, const
|
||||
}
|
||||
|
||||
|
||||
#define HUFv06_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \
|
||||
ptr += HUFv06_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
||||
#define HUFv07_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \
|
||||
ptr += HUFv07_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
||||
|
||||
#define HUFv06_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
|
||||
if (MEM_64bits() || (HUFv06_MAX_TABLELOG<=12)) \
|
||||
ptr += HUFv06_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
||||
#define HUFv07_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
|
||||
if (MEM_64bits() || (HUFv07_TABLELOG_MAX<=12)) \
|
||||
ptr += HUFv07_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
||||
|
||||
#define HUFv06_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
|
||||
#define HUFv07_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
|
||||
if (MEM_64bits()) \
|
||||
ptr += HUFv06_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
||||
ptr += HUFv07_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
||||
|
||||
static inline size_t HUFv06_decodeStreamX4(BYTE* p, BITv06_DStream_t* bitDPtr, BYTE* const pEnd, const HUFv06_DEltX4* const dt, const U32 dtLog)
|
||||
static inline size_t HUFv07_decodeStreamX4(BYTE* p, BITv07_DStream_t* bitDPtr, BYTE* const pEnd, const HUFv07_DEltX4* const dt, const U32 dtLog)
|
||||
{
|
||||
BYTE* const pStart = p;
|
||||
|
||||
/* up to 8 symbols at a time */
|
||||
while ((BITv06_reloadDStream(bitDPtr) == BITv06_DStream_unfinished) && (p < pEnd-7)) {
|
||||
HUFv06_DECODE_SYMBOLX4_2(p, bitDPtr);
|
||||
HUFv06_DECODE_SYMBOLX4_1(p, bitDPtr);
|
||||
HUFv06_DECODE_SYMBOLX4_2(p, bitDPtr);
|
||||
HUFv06_DECODE_SYMBOLX4_0(p, bitDPtr);
|
||||
while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p < pEnd-7)) {
|
||||
HUFv07_DECODE_SYMBOLX4_2(p, bitDPtr);
|
||||
HUFv07_DECODE_SYMBOLX4_1(p, bitDPtr);
|
||||
HUFv07_DECODE_SYMBOLX4_2(p, bitDPtr);
|
||||
HUFv07_DECODE_SYMBOLX4_0(p, bitDPtr);
|
||||
}
|
||||
|
||||
/* closer to the end */
|
||||
while ((BITv06_reloadDStream(bitDPtr) == BITv06_DStream_unfinished) && (p <= pEnd-2))
|
||||
HUFv06_DECODE_SYMBOLX4_0(p, bitDPtr);
|
||||
/* closer to end : up to 2 symbols at a time */
|
||||
while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p <= pEnd-2))
|
||||
HUFv07_DECODE_SYMBOLX4_0(p, bitDPtr);
|
||||
|
||||
while (p <= pEnd-2)
|
||||
HUFv06_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
|
||||
HUFv07_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
|
||||
|
||||
if (p < pEnd)
|
||||
p += HUFv06_decodeLastSymbolX4(p, bitDPtr, dt, dtLog);
|
||||
p += HUFv07_decodeLastSymbolX4(p, bitDPtr, dt, dtLog);
|
||||
|
||||
return p-pStart;
|
||||
}
|
||||
|
||||
|
||||
size_t HUFv06_decompress1X4_usingDTable(
|
||||
static size_t HUFv07_decompress1X4_usingDTable_internal(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const U32* DTable)
|
||||
const HUFv07_DTable* DTable)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) cSrc;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
|
||||
const U32 dtLog = DTable[0];
|
||||
const void* const dtPtr = DTable;
|
||||
const HUFv06_DEltX4* const dt = ((const HUFv06_DEltX4*)dtPtr) +1;
|
||||
BITv07_DStream_t bitD;
|
||||
|
||||
/* Init */
|
||||
BITv06_DStream_t bitD;
|
||||
{ size_t const errorCode = BITv06_initDStream(&bitD, istart, cSrcSize);
|
||||
if (HUFv06_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BITv07_initDStream(&bitD, cSrc, cSrcSize);
|
||||
if (HUFv07_isError(errorCode)) return errorCode;
|
||||
}
|
||||
|
||||
/* decode */
|
||||
HUFv06_decodeStreamX4(ostart, &bitD, oend, dt, dtLog);
|
||||
{ BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */
|
||||
const HUFv07_DEltX4* const dt = (const HUFv07_DEltX4*)dtPtr;
|
||||
DTableDesc const dtd = HUFv07_getDTableDesc(DTable);
|
||||
HUFv07_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog);
|
||||
}
|
||||
|
||||
/* check */
|
||||
if (!BITv06_endOfDStream(&bitD)) return ERROR(corruption_detected);
|
||||
if (!BITv07_endOfDStream(&bitD)) return ERROR(corruption_detected);
|
||||
|
||||
/* decoded size */
|
||||
return dstSize;
|
||||
}
|
||||
|
||||
size_t HUFv06_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUFv06_CREATE_STATIC_DTABLEX4(DTable, HUFv06_MAX_TABLELOG);
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const hSize = HUFv06_readDTableX4 (DTable, cSrc, cSrcSize);
|
||||
if (HUFv06_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize;
|
||||
cSrcSize -= hSize;
|
||||
|
||||
return HUFv06_decompress1X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
size_t HUFv06_decompress4X4_usingDTable(
|
||||
size_t HUFv07_decompress1X4_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const U32* DTable)
|
||||
const HUFv07_DTable* DTable)
|
||||
{
|
||||
DTableDesc dtd = HUFv07_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 1) return ERROR(GENERIC);
|
||||
return HUFv07_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
size_t HUFv07_decompress1X4_DCtx (HUFv07_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const hSize = HUFv07_readDTableX4 (DCtx, cSrc, cSrcSize);
|
||||
if (HUFv07_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUFv07_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
|
||||
}
|
||||
|
||||
size_t HUFv07_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUFv07_CREATE_STATIC_DTABLEX4(DTable, HUFv07_TABLELOG_MAX);
|
||||
return HUFv07_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
static size_t HUFv07_decompress4X4_usingDTable_internal(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv07_DTable* DTable)
|
||||
{
|
||||
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
|
||||
|
||||
{ const BYTE* const istart = (const BYTE*) cSrc;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
const void* const dtPtr = DTable;
|
||||
const HUFv06_DEltX4* const dt = ((const HUFv06_DEltX4*)dtPtr) +1;
|
||||
const U32 dtLog = DTable[0];
|
||||
size_t errorCode;
|
||||
const void* const dtPtr = DTable+1;
|
||||
const HUFv07_DEltX4* const dt = (const HUFv07_DEltX4*)dtPtr;
|
||||
|
||||
/* Init */
|
||||
BITv06_DStream_t bitD1;
|
||||
BITv06_DStream_t bitD2;
|
||||
BITv06_DStream_t bitD3;
|
||||
BITv06_DStream_t bitD4;
|
||||
const size_t length1 = MEM_readLE16(istart);
|
||||
const size_t length2 = MEM_readLE16(istart+2);
|
||||
const size_t length3 = MEM_readLE16(istart+4);
|
||||
size_t length4;
|
||||
BITv07_DStream_t bitD1;
|
||||
BITv07_DStream_t bitD2;
|
||||
BITv07_DStream_t bitD3;
|
||||
BITv07_DStream_t bitD4;
|
||||
size_t const length1 = MEM_readLE16(istart);
|
||||
size_t const length2 = MEM_readLE16(istart+2);
|
||||
size_t const length3 = MEM_readLE16(istart+4);
|
||||
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
|
||||
const BYTE* const istart1 = istart + 6; /* jumpTable */
|
||||
const BYTE* const istart2 = istart1 + length1;
|
||||
const BYTE* const istart3 = istart2 + length2;
|
||||
const BYTE* const istart4 = istart3 + length3;
|
||||
const size_t segmentSize = (dstSize+3) / 4;
|
||||
size_t const segmentSize = (dstSize+3) / 4;
|
||||
BYTE* const opStart2 = ostart + segmentSize;
|
||||
BYTE* const opStart3 = opStart2 + segmentSize;
|
||||
BYTE* const opStart4 = opStart3 + segmentSize;
|
||||
@@ -3024,39 +2811,40 @@ size_t HUFv06_decompress4X4_usingDTable(
|
||||
BYTE* op3 = opStart3;
|
||||
BYTE* op4 = opStart4;
|
||||
U32 endSignal;
|
||||
DTableDesc const dtd = HUFv07_getDTableDesc(DTable);
|
||||
U32 const dtLog = dtd.tableLog;
|
||||
|
||||
length4 = cSrcSize - (length1 + length2 + length3 + 6);
|
||||
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
|
||||
errorCode = BITv06_initDStream(&bitD1, istart1, length1);
|
||||
if (HUFv06_isError(errorCode)) return errorCode;
|
||||
errorCode = BITv06_initDStream(&bitD2, istart2, length2);
|
||||
if (HUFv06_isError(errorCode)) return errorCode;
|
||||
errorCode = BITv06_initDStream(&bitD3, istart3, length3);
|
||||
if (HUFv06_isError(errorCode)) return errorCode;
|
||||
errorCode = BITv06_initDStream(&bitD4, istart4, length4);
|
||||
if (HUFv06_isError(errorCode)) return errorCode;
|
||||
{ size_t const errorCode = BITv07_initDStream(&bitD1, istart1, length1);
|
||||
if (HUFv07_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BITv07_initDStream(&bitD2, istart2, length2);
|
||||
if (HUFv07_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BITv07_initDStream(&bitD3, istart3, length3);
|
||||
if (HUFv07_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BITv07_initDStream(&bitD4, istart4, length4);
|
||||
if (HUFv07_isError(errorCode)) return errorCode; }
|
||||
|
||||
/* 16-32 symbols per loop (4-8 symbols per stream) */
|
||||
endSignal = BITv06_reloadDStream(&bitD1) | BITv06_reloadDStream(&bitD2) | BITv06_reloadDStream(&bitD3) | BITv06_reloadDStream(&bitD4);
|
||||
for ( ; (endSignal==BITv06_DStream_unfinished) && (op4<(oend-7)) ; ) {
|
||||
HUFv06_DECODE_SYMBOLX4_2(op1, &bitD1);
|
||||
HUFv06_DECODE_SYMBOLX4_2(op2, &bitD2);
|
||||
HUFv06_DECODE_SYMBOLX4_2(op3, &bitD3);
|
||||
HUFv06_DECODE_SYMBOLX4_2(op4, &bitD4);
|
||||
HUFv06_DECODE_SYMBOLX4_1(op1, &bitD1);
|
||||
HUFv06_DECODE_SYMBOLX4_1(op2, &bitD2);
|
||||
HUFv06_DECODE_SYMBOLX4_1(op3, &bitD3);
|
||||
HUFv06_DECODE_SYMBOLX4_1(op4, &bitD4);
|
||||
HUFv06_DECODE_SYMBOLX4_2(op1, &bitD1);
|
||||
HUFv06_DECODE_SYMBOLX4_2(op2, &bitD2);
|
||||
HUFv06_DECODE_SYMBOLX4_2(op3, &bitD3);
|
||||
HUFv06_DECODE_SYMBOLX4_2(op4, &bitD4);
|
||||
HUFv06_DECODE_SYMBOLX4_0(op1, &bitD1);
|
||||
HUFv06_DECODE_SYMBOLX4_0(op2, &bitD2);
|
||||
HUFv06_DECODE_SYMBOLX4_0(op3, &bitD3);
|
||||
HUFv06_DECODE_SYMBOLX4_0(op4, &bitD4);
|
||||
endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4);
|
||||
for ( ; (endSignal==BITv07_DStream_unfinished) && (op4<(oend-7)) ; ) {
|
||||
HUFv07_DECODE_SYMBOLX4_2(op1, &bitD1);
|
||||
HUFv07_DECODE_SYMBOLX4_2(op2, &bitD2);
|
||||
HUFv07_DECODE_SYMBOLX4_2(op3, &bitD3);
|
||||
HUFv07_DECODE_SYMBOLX4_2(op4, &bitD4);
|
||||
HUFv07_DECODE_SYMBOLX4_1(op1, &bitD1);
|
||||
HUFv07_DECODE_SYMBOLX4_1(op2, &bitD2);
|
||||
HUFv07_DECODE_SYMBOLX4_1(op3, &bitD3);
|
||||
HUFv07_DECODE_SYMBOLX4_1(op4, &bitD4);
|
||||
HUFv07_DECODE_SYMBOLX4_2(op1, &bitD1);
|
||||
HUFv07_DECODE_SYMBOLX4_2(op2, &bitD2);
|
||||
HUFv07_DECODE_SYMBOLX4_2(op3, &bitD3);
|
||||
HUFv07_DECODE_SYMBOLX4_2(op4, &bitD4);
|
||||
HUFv07_DECODE_SYMBOLX4_0(op1, &bitD1);
|
||||
HUFv07_DECODE_SYMBOLX4_0(op2, &bitD2);
|
||||
HUFv07_DECODE_SYMBOLX4_0(op3, &bitD3);
|
||||
HUFv07_DECODE_SYMBOLX4_0(op4, &bitD4);
|
||||
|
||||
endSignal = BITv06_reloadDStream(&bitD1) | BITv06_reloadDStream(&bitD2) | BITv06_reloadDStream(&bitD3) | BITv06_reloadDStream(&bitD4);
|
||||
endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4);
|
||||
}
|
||||
|
||||
/* check corruption */
|
||||
@@ -3066,14 +2854,14 @@ size_t HUFv06_decompress4X4_usingDTable(
|
||||
/* note : op4 supposed already verified within main loop */
|
||||
|
||||
/* finish bitStreams one by one */
|
||||
HUFv06_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog);
|
||||
HUFv06_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog);
|
||||
HUFv06_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog);
|
||||
HUFv06_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
|
||||
HUFv07_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog);
|
||||
HUFv07_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog);
|
||||
HUFv07_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog);
|
||||
HUFv07_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
|
||||
|
||||
/* check */
|
||||
endSignal = BITv06_endOfDStream(&bitD1) & BITv06_endOfDStream(&bitD2) & BITv06_endOfDStream(&bitD3) & BITv06_endOfDStream(&bitD4);
|
||||
if (!endSignal) return ERROR(corruption_detected);
|
||||
{ U32 const endCheck = BITv07_endOfDStream(&bitD1) & BITv07_endOfDStream(&bitD2) & BITv07_endOfDStream(&bitD3) & BITv07_endOfDStream(&bitD4);
|
||||
if (!endCheck) return ERROR(corruption_detected); }
|
||||
|
||||
/* decoded size */
|
||||
return dstSize;
|
||||
@@ -3081,27 +2869,59 @@ size_t HUFv06_decompress4X4_usingDTable(
|
||||
}
|
||||
|
||||
|
||||
size_t HUFv06_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
size_t HUFv07_decompress4X4_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv07_DTable* DTable)
|
||||
{
|
||||
HUFv06_CREATE_STATIC_DTABLEX4(DTable, HUFv06_MAX_TABLELOG);
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t hSize = HUFv06_readDTableX4 (DTable, cSrc, cSrcSize);
|
||||
if (HUFv06_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize;
|
||||
cSrcSize -= hSize;
|
||||
|
||||
return HUFv06_decompress4X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
|
||||
DTableDesc dtd = HUFv07_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 1) return ERROR(GENERIC);
|
||||
return HUFv07_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
size_t HUFv07_decompress4X4_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t hSize = HUFv07_readDTableX4 (dctx, cSrc, cSrcSize);
|
||||
if (HUFv07_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUFv07_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
|
||||
}
|
||||
|
||||
size_t HUFv07_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUFv07_CREATE_STATIC_DTABLEX4(DTable, HUFv07_TABLELOG_MAX);
|
||||
return HUFv07_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
|
||||
/* ********************************/
|
||||
/* Generic decompression selector */
|
||||
/* ********************************/
|
||||
|
||||
size_t HUFv07_decompress1X_usingDTable(void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv07_DTable* DTable)
|
||||
{
|
||||
DTableDesc const dtd = HUFv07_getDTableDesc(DTable);
|
||||
return dtd.tableType ? HUFv07_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) :
|
||||
HUFv07_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
size_t HUFv07_decompress4X_usingDTable(void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv07_DTable* DTable)
|
||||
{
|
||||
DTableDesc const dtd = HUFv07_getDTableDesc(DTable);
|
||||
return dtd.tableType ? HUFv07_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) :
|
||||
HUFv07_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
|
||||
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] =
|
||||
{
|
||||
@@ -3124,12 +2944,29 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu
|
||||
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */
|
||||
};
|
||||
|
||||
/** HUFv07_selectDecoder() :
|
||||
* Tells which decoder is likely to decode faster,
|
||||
* based on a set of pre-determined metrics.
|
||||
* @return : 0==HUFv07_decompress4X2, 1==HUFv07_decompress4X4 .
|
||||
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
|
||||
U32 HUFv07_selectDecoder (size_t dstSize, size_t cSrcSize)
|
||||
{
|
||||
/* decoder timing evaluation */
|
||||
U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
|
||||
U32 const D256 = (U32)(dstSize >> 8);
|
||||
U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
|
||||
U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
|
||||
DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */
|
||||
|
||||
return DTime1 < DTime0;
|
||||
}
|
||||
|
||||
|
||||
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
||||
|
||||
size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
size_t HUFv07_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
static const decompressionAlgo decompress[3] = { HUFv06_decompress4X2, HUFv06_decompress4X4, NULL };
|
||||
U32 Dtime[3]; /* decompression time estimation */
|
||||
static const decompressionAlgo decompress[2] = { HUFv07_decompress4X2, HUFv07_decompress4X4 };
|
||||
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
@@ -3137,24 +2974,52 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
||||
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
/* decoder timing evaluation */
|
||||
{ U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
|
||||
U32 const D256 = (U32)(dstSize >> 8);
|
||||
U32 n; for (n=0; n<3; n++)
|
||||
Dtime[n] = algoTime[Q][n].tableTime + (algoTime[Q][n].decode256Time * D256);
|
||||
}
|
||||
|
||||
Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */
|
||||
|
||||
{ U32 algoNb = 0;
|
||||
if (Dtime[1] < Dtime[0]) algoNb = 1;
|
||||
// if (Dtime[2] < Dtime[algoNb]) algoNb = 2; /* current speed of HUFv06_decompress4X6 is not good */
|
||||
{ U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize);
|
||||
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
//return HUFv06_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
|
||||
//return HUFv06_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
|
||||
//return HUFv06_decompress4X6(dst, dstSize, cSrc, cSrcSize); /* multi-streams quad-symbols decoding */
|
||||
//return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
|
||||
//return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
|
||||
}
|
||||
|
||||
size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
|
||||
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
{ U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize);
|
||||
return algoNb ? HUFv07_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
|
||||
HUFv07_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
|
||||
}
|
||||
}
|
||||
|
||||
size_t HUFv07_decompress4X_hufOnly (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */
|
||||
|
||||
{ U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize);
|
||||
return algoNb ? HUFv07_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
|
||||
HUFv07_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
|
||||
}
|
||||
}
|
||||
|
||||
size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
|
||||
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
{ U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize);
|
||||
return algoNb ? HUFv07_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
|
||||
HUFv07_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
|
||||
}
|
||||
}
|
||||
/*
|
||||
Common functions of Zstd compression library
|
||||
@@ -3188,36 +3053,274 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
|
||||
*/
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Version
|
||||
******************************************/
|
||||
|
||||
/*-****************************************
|
||||
* ZSTD Error Management
|
||||
******************************************/
|
||||
/*! ZSTDv06_isError() :
|
||||
/*! ZSTDv07_isError() :
|
||||
* tells if a return value is an error code */
|
||||
unsigned ZSTDv06_isError(size_t code) { return ERR_isError(code); }
|
||||
unsigned ZSTDv07_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
/*! ZSTDv06_getErrorName() :
|
||||
/*! ZSTDv07_getErrorName() :
|
||||
* provides error code string from function result (useful for debugging) */
|
||||
const char* ZSTDv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
const char* ZSTDv07_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
/*! ZSTDv06_getError() :
|
||||
* convert a `size_t` function result into a proper ZSTDv06_errorCode enum */
|
||||
ZSTDv06_ErrorCode ZSTDv06_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
|
||||
/*! ZSTDv07_getError() :
|
||||
* convert a `size_t` function result into a proper ZSTDv07_errorCode enum */
|
||||
ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
|
||||
|
||||
/*! ZSTDv06_getErrorString() :
|
||||
/*! ZSTDv07_getErrorString() :
|
||||
* provides error code string from enum */
|
||||
const char* ZSTDv06_getErrorString(ZSTDv06_ErrorCode code) { return ERR_getErrorName(code); }
|
||||
const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* ZBUFF Error Management
|
||||
****************************************************************/
|
||||
unsigned ZBUFFv06_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
unsigned ZBUFFv07_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
|
||||
const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
const char* ZBUFFv07_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
|
||||
|
||||
|
||||
void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size)
|
||||
{
|
||||
void* address = malloc(size);
|
||||
(void)opaque;
|
||||
/* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
|
||||
return address;
|
||||
}
|
||||
|
||||
void ZSTDv07_defaultFreeFunction(void* opaque, void* address)
|
||||
{
|
||||
(void)opaque;
|
||||
/* if (address) printf("free %p opaque=%p \n", address, opaque); */
|
||||
free(address);
|
||||
}
|
||||
/*
|
||||
zstd_internal - common functions to include
|
||||
Header File for include
|
||||
Copyright (C) 2014-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : https://www.zstd.net
|
||||
*/
|
||||
#ifndef ZSTDv07_CCOMMON_H_MODULE
|
||||
#define ZSTDv07_CCOMMON_H_MODULE
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Common macros
|
||||
***************************************/
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
#define MAX(a,b) ((a)>(b) ? (a) : (b))
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Common constants
|
||||
***************************************/
|
||||
#define ZSTDv07_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */
|
||||
#include <stdio.h>
|
||||
#if defined(ZSTDv07_OPT_DEBUG) && ZSTDv07_OPT_DEBUG>=9
|
||||
#define ZSTDv07_LOG_PARSER(...) printf(__VA_ARGS__)
|
||||
#define ZSTDv07_LOG_ENCODE(...) printf(__VA_ARGS__)
|
||||
#define ZSTDv07_LOG_BLOCK(...) printf(__VA_ARGS__)
|
||||
#else
|
||||
#define ZSTDv07_LOG_PARSER(...)
|
||||
#define ZSTDv07_LOG_ENCODE(...)
|
||||
#define ZSTDv07_LOG_BLOCK(...)
|
||||
#endif
|
||||
|
||||
#define ZSTDv07_OPT_NUM (1<<12)
|
||||
#define ZSTDv07_DICT_MAGIC 0xEC30A437 /* v0.7 */
|
||||
|
||||
#define ZSTDv07_REP_NUM 3
|
||||
#define ZSTDv07_REP_INIT ZSTDv07_REP_NUM
|
||||
#define ZSTDv07_REP_MOVE (ZSTDv07_REP_NUM-1)
|
||||
static const U32 repStartValue[ZSTDv07_REP_NUM] = { 1, 4, 8 };
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define BIT7 128
|
||||
#define BIT6 64
|
||||
#define BIT5 32
|
||||
#define BIT4 16
|
||||
#define BIT1 2
|
||||
#define BIT0 1
|
||||
|
||||
#define ZSTDv07_WINDOWLOG_ABSOLUTEMIN 10
|
||||
static const size_t ZSTDv07_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||
static const size_t ZSTDv07_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||
|
||||
#define ZSTDv07_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
|
||||
static const size_t ZSTDv07_blockHeaderSize = ZSTDv07_BLOCKHEADERSIZE;
|
||||
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
||||
|
||||
#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
|
||||
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
|
||||
|
||||
#define HufLog 12
|
||||
typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t;
|
||||
|
||||
#define LONGNBSEQ 0x7F00
|
||||
|
||||
#define MINMATCH 3
|
||||
#define EQUAL_READ32 4
|
||||
|
||||
#define Litbits 8
|
||||
#define MaxLit ((1<<Litbits) - 1)
|
||||
#define MaxML 52
|
||||
#define MaxLL 35
|
||||
#define MaxOff 28
|
||||
#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
|
||||
#define MLFSELog 9
|
||||
#define LLFSELog 9
|
||||
#define OffFSELog 8
|
||||
|
||||
#define FSEv07_ENCODING_RAW 0
|
||||
#define FSEv07_ENCODING_RLE 1
|
||||
#define FSEv07_ENCODING_STATIC 2
|
||||
#define FSEv07_ENCODING_DYNAMIC 3
|
||||
|
||||
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
||||
13,14,15,16 };
|
||||
static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
|
||||
-1,-1,-1,-1 };
|
||||
static const U32 LL_defaultNormLog = 6;
|
||||
|
||||
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11,
|
||||
12,13,14,15,16 };
|
||||
static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
|
||||
-1,-1,-1,-1,-1 };
|
||||
static const U32 ML_defaultNormLog = 6;
|
||||
|
||||
static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
|
||||
static const U32 OF_defaultNormLog = 5;
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Shared functions to include for inlining
|
||||
*********************************************/
|
||||
static void ZSTDv07_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
||||
#define COPY8(d,s) { ZSTDv07_copy8(d,s); d+=8; s+=8; }
|
||||
|
||||
/*! ZSTDv07_wildcopy() :
|
||||
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
|
||||
#define WILDCOPY_OVERLENGTH 8
|
||||
MEM_STATIC void ZSTDv07_wildcopy(void* dst, const void* src, size_t length)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = op + length;
|
||||
do
|
||||
COPY8(op, ip)
|
||||
while (op < oend);
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Private interfaces
|
||||
*********************************************/
|
||||
typedef struct ZSTDv07_stats_s ZSTDv07_stats_t;
|
||||
|
||||
typedef struct {
|
||||
U32 off;
|
||||
U32 len;
|
||||
} ZSTDv07_match_t;
|
||||
|
||||
typedef struct {
|
||||
U32 price;
|
||||
U32 off;
|
||||
U32 mlen;
|
||||
U32 litlen;
|
||||
U32 rep[ZSTDv07_REP_INIT];
|
||||
} ZSTDv07_optimal_t;
|
||||
|
||||
struct ZSTDv07_stats_s { U32 unused; };
|
||||
MEM_STATIC void ZSTDv07_statsPrint(ZSTDv07_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
|
||||
MEM_STATIC void ZSTDv07_statsInit(ZSTDv07_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTDv07_statsResetFreqs(ZSTDv07_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTDv07_statsUpdatePrices(ZSTDv07_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
|
||||
|
||||
typedef struct {
|
||||
void* buffer;
|
||||
U32* offsetStart;
|
||||
U32* offset;
|
||||
BYTE* offCodeStart;
|
||||
BYTE* litStart;
|
||||
BYTE* lit;
|
||||
U16* litLengthStart;
|
||||
U16* litLength;
|
||||
BYTE* llCodeStart;
|
||||
U16* matchLengthStart;
|
||||
U16* matchLength;
|
||||
BYTE* mlCodeStart;
|
||||
U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
|
||||
U32 longLengthPos;
|
||||
/* opt */
|
||||
ZSTDv07_optimal_t* priceTable;
|
||||
ZSTDv07_match_t* matchTable;
|
||||
U32* matchLengthFreq;
|
||||
U32* litLengthFreq;
|
||||
U32* litFreq;
|
||||
U32* offCodeFreq;
|
||||
U32 matchLengthSum;
|
||||
U32 matchSum;
|
||||
U32 litLengthSum;
|
||||
U32 litSum;
|
||||
U32 offCodeSum;
|
||||
U32 log2matchLengthSum;
|
||||
U32 log2matchSum;
|
||||
U32 log2litLengthSum;
|
||||
U32 log2litSum;
|
||||
U32 log2offCodeSum;
|
||||
U32 factor;
|
||||
U32 cachedPrice;
|
||||
U32 cachedLitLength;
|
||||
const BYTE* cachedLiterals;
|
||||
ZSTDv07_stats_t stats;
|
||||
} seqStore_t;
|
||||
|
||||
void ZSTDv07_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
|
||||
|
||||
/* custom memory allocation functions */
|
||||
void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size);
|
||||
void ZSTDv07_defaultFreeFunction(void* opaque, void* address);
|
||||
static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction, ZSTDv07_defaultFreeFunction, NULL };
|
||||
|
||||
#endif /* ZSTDv07_CCOMMON_H_MODULE */
|
||||
/*
|
||||
zstd - standard compression library
|
||||
Copyright (C) 2014-2016, Yann Collet.
|
||||
@@ -3254,15 +3357,14 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er
|
||||
*****************************************************************/
|
||||
/*!
|
||||
* HEAPMODE :
|
||||
* Select how default decompression function ZSTDv06_decompress() will allocate memory,
|
||||
* Select how default decompression function ZSTDv07_decompress() will allocate memory,
|
||||
* in memory stack (0), or in memory heap (1, requires malloc())
|
||||
*/
|
||||
#ifndef ZSTDv06_HEAPMODE
|
||||
# define ZSTDv06_HEAPMODE 1
|
||||
#ifndef ZSTDv07_HEAPMODE
|
||||
# define ZSTDv07_HEAPMODE 1
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Compiler specifics
|
||||
*********************************************************/
|
||||
@@ -3283,79 +3385,106 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er
|
||||
/*-*************************************
|
||||
* Macros
|
||||
***************************************/
|
||||
#define ZSTDv06_isError ERR_isError /* for inlining */
|
||||
#define FSEv06_isError ERR_isError
|
||||
#define HUFv06_isError ERR_isError
|
||||
#define ZSTDv07_isError ERR_isError /* for inlining */
|
||||
#define FSEv07_isError ERR_isError
|
||||
#define HUFv07_isError ERR_isError
|
||||
|
||||
|
||||
/*_*******************************************************
|
||||
* Memory operations
|
||||
**********************************************************/
|
||||
static void ZSTDv06_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
||||
static void ZSTDv07_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
||||
|
||||
|
||||
/*-*************************************************************
|
||||
* Context management
|
||||
***************************************************************/
|
||||
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
|
||||
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTDv06_dStage;
|
||||
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
|
||||
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTDv07_dStage;
|
||||
|
||||
struct ZSTDv06_DCtx_s
|
||||
struct ZSTDv07_DCtx_s
|
||||
{
|
||||
FSEv06_DTable LLTable[FSEv06_DTABLE_SIZE_U32(LLFSELog)];
|
||||
FSEv06_DTable OffTable[FSEv06_DTABLE_SIZE_U32(OffFSELog)];
|
||||
FSEv06_DTable MLTable[FSEv06_DTABLE_SIZE_U32(MLFSELog)];
|
||||
unsigned hufTableX4[HUFv06_DTABLE_SIZE(HufLog)];
|
||||
FSEv07_DTable LLTable[FSEv07_DTABLE_SIZE_U32(LLFSELog)];
|
||||
FSEv07_DTable OffTable[FSEv07_DTABLE_SIZE_U32(OffFSELog)];
|
||||
FSEv07_DTable MLTable[FSEv07_DTABLE_SIZE_U32(MLFSELog)];
|
||||
HUFv07_DTable hufTable[HUFv07_DTABLE_SIZE(HufLog)]; /* can accommodate HUFv07_decompress4X */
|
||||
const void* previousDstEnd;
|
||||
const void* base;
|
||||
const void* vBase;
|
||||
const void* dictEnd;
|
||||
size_t expected;
|
||||
U32 rep[3];
|
||||
ZSTDv07_frameParams fParams;
|
||||
blockType_t bType; /* used in ZSTDv07_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
|
||||
ZSTDv07_dStage stage;
|
||||
U32 litEntropy;
|
||||
U32 fseEntropy;
|
||||
XXH64_state_t xxhState;
|
||||
size_t headerSize;
|
||||
ZSTDv06_frameParams fParams;
|
||||
blockType_t bType; /* used in ZSTDv06_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
|
||||
ZSTDv06_dStage stage;
|
||||
U32 flagRepeatTable;
|
||||
U32 dictID;
|
||||
const BYTE* litPtr;
|
||||
ZSTDv07_customMem customMem;
|
||||
size_t litBufSize;
|
||||
size_t litSize;
|
||||
BYTE litBuffer[ZSTDv06_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
|
||||
BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
|
||||
}; /* typedef'd to ZSTDv06_DCtx within "zstd_static.h" */
|
||||
BYTE litBuffer[ZSTDv07_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
|
||||
BYTE headerBuffer[ZSTDv07_FRAMEHEADERSIZE_MAX];
|
||||
}; /* typedef'd to ZSTDv07_DCtx within "zstd_static.h" */
|
||||
|
||||
size_t ZSTDv06_sizeofDCtx (void) { return sizeof(ZSTDv06_DCtx); } /* non published interface */
|
||||
int ZSTDv07_isSkipFrame(ZSTDv07_DCtx* dctx);
|
||||
|
||||
size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx)
|
||||
size_t ZSTDv07_sizeofDCtx (const ZSTDv07_DCtx* dctx) { return sizeof(*dctx); }
|
||||
|
||||
size_t ZSTDv07_estimateDCtxSize(void) { return sizeof(ZSTDv07_DCtx); }
|
||||
|
||||
size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx)
|
||||
{
|
||||
dctx->expected = ZSTDv06_frameHeaderSize_min;
|
||||
dctx->expected = ZSTDv07_frameHeaderSize_min;
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
dctx->previousDstEnd = NULL;
|
||||
dctx->base = NULL;
|
||||
dctx->vBase = NULL;
|
||||
dctx->dictEnd = NULL;
|
||||
dctx->hufTableX4[0] = HufLog;
|
||||
dctx->flagRepeatTable = 0;
|
||||
dctx->hufTable[0] = (HUFv07_DTable)((HufLog)*0x1000001);
|
||||
dctx->litEntropy = dctx->fseEntropy = 0;
|
||||
dctx->dictID = 0;
|
||||
{ int i; for (i=0; i<ZSTDv07_REP_NUM; i++) dctx->rep[i] = repStartValue[i]; }
|
||||
return 0;
|
||||
}
|
||||
|
||||
ZSTDv06_DCtx* ZSTDv06_createDCtx(void)
|
||||
ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem)
|
||||
{
|
||||
ZSTDv06_DCtx* dctx = (ZSTDv06_DCtx*)malloc(sizeof(ZSTDv06_DCtx));
|
||||
if (dctx==NULL) return NULL;
|
||||
ZSTDv06_decompressBegin(dctx);
|
||||
ZSTDv07_DCtx* dctx;
|
||||
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
dctx = (ZSTDv07_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTDv07_DCtx));
|
||||
if (!dctx) return NULL;
|
||||
memcpy(&dctx->customMem, &customMem, sizeof(ZSTDv07_customMem));
|
||||
ZSTDv07_decompressBegin(dctx);
|
||||
return dctx;
|
||||
}
|
||||
|
||||
size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx)
|
||||
ZSTDv07_DCtx* ZSTDv07_createDCtx(void)
|
||||
{
|
||||
free(dctx);
|
||||
return ZSTDv07_createDCtx_advanced(defaultCustomMem);
|
||||
}
|
||||
|
||||
size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx)
|
||||
{
|
||||
if (dctx==NULL) return 0; /* support free on NULL */
|
||||
dctx->customMem.customFree(dctx->customMem.opaque, dctx);
|
||||
return 0; /* reserved as a potential error code in the future */
|
||||
}
|
||||
|
||||
void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dstDCtx, const ZSTDv06_DCtx* srcDCtx)
|
||||
void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dstDCtx, const ZSTDv07_DCtx* srcDCtx)
|
||||
{
|
||||
memcpy(dstDCtx, srcDCtx,
|
||||
sizeof(ZSTDv06_DCtx) - (ZSTDv06_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH + ZSTDv06_frameHeaderSize_max)); /* no need to copy workspace */
|
||||
sizeof(ZSTDv07_DCtx) - (ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH + ZSTDv07_frameHeaderSize_max)); /* no need to copy workspace */
|
||||
}
|
||||
|
||||
|
||||
@@ -3366,7 +3495,7 @@ void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dstDCtx, const ZSTDv06_DCtx* srcDCtx)
|
||||
/* Frame format description
|
||||
Frame Header - [ Block Header - Block ] - Frame End
|
||||
1) Frame Header
|
||||
- 4 bytes - Magic Number : ZSTDv06_MAGICNUMBER (defined within zstd_static.h)
|
||||
- 4 bytes - Magic Number : ZSTDv07_MAGICNUMBER (defined within zstd.h)
|
||||
- 1 byte - Frame Descriptor
|
||||
2) Block Header
|
||||
- 3 bytes, starting with a 2-bits descriptor
|
||||
@@ -3378,19 +3507,34 @@ void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dstDCtx, const ZSTDv06_DCtx* srcDCtx)
|
||||
*/
|
||||
|
||||
|
||||
/* Frame descriptor
|
||||
/* Frame Header :
|
||||
|
||||
1 byte, using :
|
||||
bit 0-3 : windowLog - ZSTDv06_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
|
||||
bit 4 : minmatch 4(0) or 3(1)
|
||||
bit 5 : reserved (must be zero)
|
||||
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
|
||||
1 byte - FrameHeaderDescription :
|
||||
bit 0-1 : dictID (0, 1, 2 or 4 bytes)
|
||||
bit 2 : checksumFlag
|
||||
bit 3 : reserved (must be zero)
|
||||
bit 4 : reserved (unused, can be any value)
|
||||
bit 5 : Single Segment (if 1, WindowLog byte is not present)
|
||||
bit 6-7 : FrameContentFieldSize (0, 2, 4, or 8)
|
||||
if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1;
|
||||
|
||||
Optional : content size (0, 1, 2 or 8 bytes)
|
||||
0 : unknown
|
||||
1 : 0-255 bytes
|
||||
2 : 256 - 65535+256
|
||||
8 : up to 16 exa
|
||||
Optional : WindowLog (0 or 1 byte)
|
||||
bit 0-2 : octal Fractional (1/8th)
|
||||
bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB)
|
||||
|
||||
Optional : dictID (0, 1, 2 or 4 bytes)
|
||||
Automatic adaptation
|
||||
0 : no dictID
|
||||
1 : 1 - 255
|
||||
2 : 256 - 65535
|
||||
4 : all other values
|
||||
|
||||
Optional : content size (0, 1, 2, 4 or 8 bytes)
|
||||
0 : unknown (fcfs==0 and swl==0)
|
||||
1 : 0-255 bytes (fcfs==0 and swl==1)
|
||||
2 : 256 - 65535+256 (fcfs==1)
|
||||
4 : 0 - 4GB-1 (fcfs==2)
|
||||
8 : 0 - 16EB-1 (fcfs==3)
|
||||
*/
|
||||
|
||||
|
||||
@@ -3460,56 +3604,118 @@ void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dstDCtx, const ZSTDv06_DCtx* srcDCtx)
|
||||
TO DO
|
||||
*/
|
||||
|
||||
/** ZSTDv06_frameHeaderSize() :
|
||||
* srcSize must be >= ZSTDv06_frameHeaderSize_min.
|
||||
/** ZSTDv07_frameHeaderSize() :
|
||||
* srcSize must be >= ZSTDv07_frameHeaderSize_min.
|
||||
* @return : size of the Frame Header */
|
||||
static size_t ZSTDv06_frameHeaderSize(const void* src, size_t srcSize)
|
||||
static size_t ZSTDv07_frameHeaderSize(const void* src, size_t srcSize)
|
||||
{
|
||||
if (srcSize < ZSTDv06_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
||||
{ U32 const fcsId = (((const BYTE*)src)[4]) >> 6;
|
||||
return ZSTDv06_frameHeaderSize_min + ZSTDv06_fcs_fieldSize[fcsId]; }
|
||||
if (srcSize < ZSTDv07_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
||||
{ BYTE const fhd = ((const BYTE*)src)[4];
|
||||
U32 const dictID= fhd & 3;
|
||||
U32 const directMode = (fhd >> 5) & 1;
|
||||
U32 const fcsId = fhd >> 6;
|
||||
return ZSTDv07_frameHeaderSize_min + !directMode + ZSTDv07_did_fieldSize[dictID] + ZSTDv07_fcs_fieldSize[fcsId]
|
||||
+ (directMode && !ZSTDv07_fcs_fieldSize[fcsId]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** ZSTDv06_getFrameParams() :
|
||||
* decode Frame Header, or provide expected `srcSize`.
|
||||
/** ZSTDv07_getFrameParams() :
|
||||
* decode Frame Header, or require larger `srcSize`.
|
||||
* @return : 0, `fparamsPtr` is correctly filled,
|
||||
* >0, `srcSize` is too small, result is expected `srcSize`,
|
||||
* or an error code, which can be tested using ZSTDv06_isError() */
|
||||
size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize)
|
||||
* or an error code, which can be tested using ZSTDv07_isError() */
|
||||
size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
|
||||
if (srcSize < ZSTDv06_frameHeaderSize_min) return ZSTDv06_frameHeaderSize_min;
|
||||
if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) return ERROR(prefix_unknown);
|
||||
if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min;
|
||||
if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) {
|
||||
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) {
|
||||
if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */
|
||||
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
||||
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
||||
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
|
||||
return 0;
|
||||
}
|
||||
return ERROR(prefix_unknown);
|
||||
}
|
||||
|
||||
/* ensure there is enough `srcSize` to fully read/decode frame header */
|
||||
{ size_t const fhsize = ZSTDv06_frameHeaderSize(src, srcSize);
|
||||
{ size_t const fhsize = ZSTDv07_frameHeaderSize(src, srcSize);
|
||||
if (srcSize < fhsize) return fhsize; }
|
||||
|
||||
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
||||
{ BYTE const frameDesc = ip[4];
|
||||
fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTDv06_WINDOWLOG_ABSOLUTEMIN;
|
||||
if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */
|
||||
switch(frameDesc >> 6) /* fcsId */
|
||||
{ BYTE const fhdByte = ip[4];
|
||||
size_t pos = 5;
|
||||
U32 const dictIDSizeCode = fhdByte&3;
|
||||
U32 const checksumFlag = (fhdByte>>2)&1;
|
||||
U32 const directMode = (fhdByte>>5)&1;
|
||||
U32 const fcsID = fhdByte>>6;
|
||||
U32 const windowSizeMax = 1U << ZSTDv07_WINDOWLOG_MAX;
|
||||
U32 windowSize = 0;
|
||||
U32 dictID = 0;
|
||||
U64 frameContentSize = 0;
|
||||
if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
|
||||
if (!directMode) {
|
||||
BYTE const wlByte = ip[pos++];
|
||||
U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN;
|
||||
if (windowLog > ZSTDv07_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
|
||||
windowSize = (1U << windowLog);
|
||||
windowSize += (windowSize >> 3) * (wlByte&7);
|
||||
}
|
||||
|
||||
switch(dictIDSizeCode)
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : fparamsPtr->frameContentSize = 0; break;
|
||||
case 1 : fparamsPtr->frameContentSize = ip[5]; break;
|
||||
case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5)+256; break;
|
||||
case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break;
|
||||
} }
|
||||
case 0 : break;
|
||||
case 1 : dictID = ip[pos]; pos++; break;
|
||||
case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
|
||||
case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
|
||||
}
|
||||
switch(fcsID)
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : if (directMode) frameContentSize = ip[pos]; break;
|
||||
case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
|
||||
case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
|
||||
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
|
||||
}
|
||||
if (!windowSize) windowSize = (U32)frameContentSize;
|
||||
if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
|
||||
fparamsPtr->frameContentSize = frameContentSize;
|
||||
fparamsPtr->windowSize = windowSize;
|
||||
fparamsPtr->dictID = dictID;
|
||||
fparamsPtr->checksumFlag = checksumFlag;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/** ZSTDv06_decodeFrameHeader() :
|
||||
* `srcSize` must be the size provided by ZSTDv06_frameHeaderSize().
|
||||
* @return : 0 if success, or an error code, which can be tested using ZSTDv06_isError() */
|
||||
static size_t ZSTDv06_decodeFrameHeader(ZSTDv06_DCtx* zc, const void* src, size_t srcSize)
|
||||
/** ZSTDv07_getDecompressedSize() :
|
||||
* compatible with legacy mode
|
||||
* @return : decompressed size if known, 0 otherwise
|
||||
note : 0 can mean any of the following :
|
||||
- decompressed size is not provided within frame header
|
||||
- frame header unknown / not supported
|
||||
- frame header not completely provided (`srcSize` too small) */
|
||||
unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const result = ZSTDv06_getFrameParams(&(zc->fParams), src, srcSize);
|
||||
if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
|
||||
{ ZSTDv07_frameParams fparams;
|
||||
size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize);
|
||||
if (frResult!=0) return 0;
|
||||
return fparams.frameContentSize;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** ZSTDv07_decodeFrameHeader() :
|
||||
* `srcSize` must be the size provided by ZSTDv07_frameHeaderSize().
|
||||
* @return : 0 if success, or an error code, which can be tested using ZSTDv07_isError() */
|
||||
static size_t ZSTDv07_decodeFrameHeader(ZSTDv07_DCtx* dctx, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const result = ZSTDv07_getFrameParams(&(dctx->fParams), src, srcSize);
|
||||
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
|
||||
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -3520,14 +3726,14 @@ typedef struct
|
||||
U32 origSize;
|
||||
} blockProperties_t;
|
||||
|
||||
/*! ZSTDv06_getcBlockSize() :
|
||||
/*! ZSTDv07_getcBlockSize() :
|
||||
* Provides the size of compressed block from block header `src` */
|
||||
size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
||||
size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
||||
{
|
||||
const BYTE* const in = (const BYTE* const)src;
|
||||
U32 cSize;
|
||||
|
||||
if (srcSize < ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
if (srcSize < ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
|
||||
bpPtr->blockType = (blockType_t)((*in) >> 6);
|
||||
cSize = in[2] + (in[1]<<8) + ((in[0] & 7)<<16);
|
||||
@@ -3539,7 +3745,7 @@ size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t*
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||
memcpy(dst, src, srcSize);
|
||||
@@ -3547,21 +3753,20 @@ static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* sr
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTDv06_decodeLiteralsBlock() :
|
||||
/*! ZSTDv07_decodeLiteralsBlock() :
|
||||
@return : nb of bytes read from src (< srcSize ) */
|
||||
size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
||||
size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx,
|
||||
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) src;
|
||||
|
||||
/* any compressed block with literals segment must be at least this size */
|
||||
if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected);
|
||||
|
||||
switch(istart[0]>> 6)
|
||||
switch((litBlockType_t)(istart[0]>> 6))
|
||||
{
|
||||
case IS_HUF:
|
||||
case lbt_huffman:
|
||||
{ size_t litSize, litCSize, singleStream=0;
|
||||
U32 lhSize = ((istart[0]) >> 4) & 3;
|
||||
U32 lhSize = (istart[0] >> 4) & 3;
|
||||
if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for lhSize, + cSize (+nbSeq) */
|
||||
switch(lhSize)
|
||||
{
|
||||
@@ -3585,41 +3790,43 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
||||
litCSize = ((istart[2] & 3) << 16) + (istart[3] << 8) + istart[4];
|
||||
break;
|
||||
}
|
||||
if (litSize > ZSTDv06_BLOCKSIZE_MAX) return ERROR(corruption_detected);
|
||||
if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
|
||||
|
||||
if (HUFv06_isError(singleStream ?
|
||||
HUFv06_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) :
|
||||
HUFv06_decompress (dctx->litBuffer, litSize, istart+lhSize, litCSize) ))
|
||||
if (HUFv07_isError(singleStream ?
|
||||
HUFv07_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
|
||||
HUFv07_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) ))
|
||||
return ERROR(corruption_detected);
|
||||
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+8;
|
||||
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8;
|
||||
dctx->litSize = litSize;
|
||||
dctx->litEntropy = 1;
|
||||
return litCSize + lhSize;
|
||||
}
|
||||
case IS_PCH:
|
||||
case lbt_repeat:
|
||||
{ size_t litSize, litCSize;
|
||||
U32 lhSize = ((istart[0]) >> 4) & 3;
|
||||
if (lhSize != 1) /* only case supported for now : small litSize, single stream */
|
||||
return ERROR(corruption_detected);
|
||||
if (!dctx->flagRepeatTable)
|
||||
if (dctx->litEntropy==0)
|
||||
return ERROR(dictionary_corrupted);
|
||||
|
||||
/* 2 - 2 - 10 - 10 */
|
||||
lhSize=3;
|
||||
litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
|
||||
litCSize = ((istart[1] & 3) << 8) + istart[2];
|
||||
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
|
||||
|
||||
{ size_t const errorCode = HUFv06_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
|
||||
if (HUFv06_isError(errorCode)) return ERROR(corruption_detected);
|
||||
{ size_t const errorCode = HUFv07_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable);
|
||||
if (HUFv07_isError(errorCode)) return ERROR(corruption_detected);
|
||||
}
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
|
||||
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
|
||||
dctx->litSize = litSize;
|
||||
return litCSize + lhSize;
|
||||
}
|
||||
case IS_RAW:
|
||||
case lbt_raw:
|
||||
{ size_t litSize;
|
||||
U32 lhSize = ((istart[0]) >> 4) & 3;
|
||||
switch(lhSize)
|
||||
@@ -3640,7 +3847,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
||||
if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
|
||||
memcpy(dctx->litBuffer, istart+lhSize, litSize);
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+8;
|
||||
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8;
|
||||
dctx->litSize = litSize;
|
||||
return lhSize+litSize;
|
||||
}
|
||||
@@ -3650,7 +3857,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
||||
dctx->litSize = litSize;
|
||||
return lhSize+litSize;
|
||||
}
|
||||
case IS_RLE:
|
||||
case lbt_rle:
|
||||
{ size_t litSize;
|
||||
U32 lhSize = ((istart[0]) >> 4) & 3;
|
||||
switch(lhSize)
|
||||
@@ -3667,10 +3874,10 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
||||
if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
|
||||
break;
|
||||
}
|
||||
if (litSize > ZSTDv06_BLOCKSIZE_MAX) return ERROR(corruption_detected);
|
||||
if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||
memset(dctx->litBuffer, istart[lhSize], litSize);
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = ZSTDv06_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
|
||||
dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
|
||||
dctx->litSize = litSize;
|
||||
return lhSize+1;
|
||||
}
|
||||
@@ -3680,42 +3887,42 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTDv06_buildSeqTable() :
|
||||
/*! ZSTDv07_buildSeqTable() :
|
||||
@return : nb bytes read from src,
|
||||
or an error code if it fails, testable with ZSTDv06_isError()
|
||||
or an error code if it fails, testable with ZSTDv07_isError()
|
||||
*/
|
||||
size_t ZSTDv06_buildSeqTable(FSEv06_DTable* DTable, U32 type, U32 max, U32 maxLog,
|
||||
FORCE_INLINE size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLog,
|
||||
const void* src, size_t srcSize,
|
||||
const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
|
||||
{
|
||||
switch(type)
|
||||
{
|
||||
case FSEv06_ENCODING_RLE :
|
||||
case FSEv07_ENCODING_RLE :
|
||||
if (!srcSize) return ERROR(srcSize_wrong);
|
||||
if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected);
|
||||
FSEv06_buildDTable_rle(DTable, *(const BYTE*)src); /* if *src > max, data is corrupted */
|
||||
FSEv07_buildDTable_rle(DTable, *(const BYTE*)src); /* if *src > max, data is corrupted */
|
||||
return 1;
|
||||
case FSEv06_ENCODING_RAW :
|
||||
FSEv06_buildDTable(DTable, defaultNorm, max, defaultLog);
|
||||
case FSEv07_ENCODING_RAW :
|
||||
FSEv07_buildDTable(DTable, defaultNorm, max, defaultLog);
|
||||
return 0;
|
||||
case FSEv06_ENCODING_STATIC:
|
||||
case FSEv07_ENCODING_STATIC:
|
||||
if (!flagRepeatTable) return ERROR(corruption_detected);
|
||||
return 0;
|
||||
default : /* impossible */
|
||||
case FSEv06_ENCODING_DYNAMIC :
|
||||
case FSEv07_ENCODING_DYNAMIC :
|
||||
{ U32 tableLog;
|
||||
S16 norm[MaxSeq+1];
|
||||
size_t const headerSize = FSEv06_readNCount(norm, &max, &tableLog, src, srcSize);
|
||||
if (FSEv06_isError(headerSize)) return ERROR(corruption_detected);
|
||||
size_t const headerSize = FSEv07_readNCount(norm, &max, &tableLog, src, srcSize);
|
||||
if (FSEv07_isError(headerSize)) return ERROR(corruption_detected);
|
||||
if (tableLog > maxLog) return ERROR(corruption_detected);
|
||||
FSEv06_buildDTable(DTable, norm, max, tableLog);
|
||||
FSEv07_buildDTable(DTable, norm, max, tableLog);
|
||||
return headerSize;
|
||||
} }
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
||||
FSEv06_DTable* DTableLL, FSEv06_DTable* DTableML, FSEv06_DTable* DTableOffb, U32 flagRepeatTable,
|
||||
size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
|
||||
FSEv07_DTable* DTableLL, FSEv07_DTable* DTableML, FSEv07_DTable* DTableOffb, U32 flagRepeatTable,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE* const)src;
|
||||
@@ -3739,7 +3946,7 @@ size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
||||
|
||||
/* FSE table descriptors */
|
||||
{ U32 const LLtype = *ip >> 6;
|
||||
U32 const Offtype = (*ip >> 4) & 3;
|
||||
U32 const OFtype = (*ip >> 4) & 3;
|
||||
U32 const MLtype = (*ip >> 2) & 3;
|
||||
ip++;
|
||||
|
||||
@@ -3747,17 +3954,17 @@ size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
|
||||
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
||||
|
||||
/* Build DTables */
|
||||
{ size_t const bhSize = ZSTDv06_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
|
||||
if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
|
||||
ip += bhSize;
|
||||
{ size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
|
||||
if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected);
|
||||
ip += llhSize;
|
||||
}
|
||||
{ size_t const bhSize = ZSTDv06_buildSeqTable(DTableOffb, Offtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable);
|
||||
if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
|
||||
ip += bhSize;
|
||||
{ size_t const ofhSize = ZSTDv07_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable);
|
||||
if (ZSTDv07_isError(ofhSize)) return ERROR(corruption_detected);
|
||||
ip += ofhSize;
|
||||
}
|
||||
{ size_t const bhSize = ZSTDv06_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable);
|
||||
if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
|
||||
ip += bhSize;
|
||||
{ size_t const mlhSize = ZSTDv07_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable);
|
||||
if (ZSTDv07_isError(mlhSize)) return ERROR(corruption_detected);
|
||||
ip += mlhSize;
|
||||
} }
|
||||
|
||||
return ip-istart;
|
||||
@@ -3771,21 +3978,21 @@ typedef struct {
|
||||
} seq_t;
|
||||
|
||||
typedef struct {
|
||||
BITv06_DStream_t DStream;
|
||||
FSEv06_DState_t stateLL;
|
||||
FSEv06_DState_t stateOffb;
|
||||
FSEv06_DState_t stateML;
|
||||
size_t prevOffset[ZSTDv06_REP_INIT];
|
||||
BITv07_DStream_t DStream;
|
||||
FSEv07_DState_t stateLL;
|
||||
FSEv07_DState_t stateOffb;
|
||||
FSEv07_DState_t stateML;
|
||||
size_t prevOffset[ZSTDv07_REP_INIT];
|
||||
} seqState_t;
|
||||
|
||||
|
||||
|
||||
static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState)
|
||||
static seq_t ZSTDv07_decodeSequence(seqState_t* seqState)
|
||||
{
|
||||
/* Literal length */
|
||||
U32 const llCode = FSEv06_peekSymbol(&(seqState->stateLL));
|
||||
U32 const mlCode = FSEv06_peekSymbol(&(seqState->stateML));
|
||||
U32 const ofCode = FSEv06_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
|
||||
seq_t seq;
|
||||
|
||||
U32 const llCode = FSEv07_peekSymbol(&(seqState->stateLL));
|
||||
U32 const mlCode = FSEv07_peekSymbol(&(seqState->stateML));
|
||||
U32 const ofCode = FSEv07_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
|
||||
|
||||
U32 const llBits = LL_bits[llCode];
|
||||
U32 const mlBits = ML_bits[mlCode];
|
||||
@@ -3798,83 +4005,80 @@ static void ZSTDv06_decodeSequence(seq_t* seq, seqState_t* seqState)
|
||||
0x2000, 0x4000, 0x8000, 0x10000 };
|
||||
|
||||
static const U32 ML_base[MaxML+1] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 34, 36, 38, 40, 44, 48, 56, 64, 80, 96, 0x80, 0x100, 0x200, 0x400, 0x800,
|
||||
0x1000, 0x2000, 0x4000, 0x8000, 0x10000 };
|
||||
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
|
||||
19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
|
||||
35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
|
||||
0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
|
||||
|
||||
static const U32 OF_base[MaxOff+1] = {
|
||||
0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F,
|
||||
0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF,
|
||||
0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
|
||||
0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1 };
|
||||
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
|
||||
0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
|
||||
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
|
||||
0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
|
||||
|
||||
/* sequence */
|
||||
{ size_t offset;
|
||||
if (!ofCode)
|
||||
offset = 0;
|
||||
else {
|
||||
offset = OF_base[ofCode] + BITv06_readBits(&(seqState->DStream), ofBits); /* <= 26 bits */
|
||||
if (MEM_32bits()) BITv06_reloadDStream(&(seqState->DStream));
|
||||
offset = OF_base[ofCode] + BITv07_readBits(&(seqState->DStream), ofBits); /* <= (ZSTDv07_WINDOWLOG_MAX-1) bits */
|
||||
if (MEM_32bits()) BITv07_reloadDStream(&(seqState->DStream));
|
||||
}
|
||||
|
||||
if (offset < ZSTDv06_REP_NUM) {
|
||||
if (llCode == 0 && offset <= 1) offset = 1-offset;
|
||||
|
||||
if (offset != 0) {
|
||||
size_t temp = seqState->prevOffset[offset];
|
||||
if (offset != 1) {
|
||||
seqState->prevOffset[2] = seqState->prevOffset[1];
|
||||
}
|
||||
if (ofCode <= 1) {
|
||||
if ((llCode == 0) & (offset <= 1)) offset = 1-offset;
|
||||
if (offset) {
|
||||
size_t const temp = seqState->prevOffset[offset];
|
||||
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
|
||||
seqState->prevOffset[1] = seqState->prevOffset[0];
|
||||
seqState->prevOffset[0] = offset = temp;
|
||||
|
||||
} else {
|
||||
offset = seqState->prevOffset[0];
|
||||
}
|
||||
} else {
|
||||
offset -= ZSTDv06_REP_MOVE;
|
||||
seqState->prevOffset[2] = seqState->prevOffset[1];
|
||||
seqState->prevOffset[1] = seqState->prevOffset[0];
|
||||
seqState->prevOffset[0] = offset;
|
||||
}
|
||||
seq->offset = offset;
|
||||
seq.offset = offset;
|
||||
}
|
||||
|
||||
seq->matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BITv06_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */
|
||||
if (MEM_32bits() && (mlBits+llBits>24)) BITv06_reloadDStream(&(seqState->DStream));
|
||||
seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BITv07_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */
|
||||
if (MEM_32bits() && (mlBits+llBits>24)) BITv07_reloadDStream(&(seqState->DStream));
|
||||
|
||||
seq->litLength = LL_base[llCode] + ((llCode>15) ? BITv06_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */
|
||||
seq.litLength = LL_base[llCode] + ((llCode>15) ? BITv07_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */
|
||||
if (MEM_32bits() ||
|
||||
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BITv06_reloadDStream(&(seqState->DStream));
|
||||
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BITv07_reloadDStream(&(seqState->DStream));
|
||||
|
||||
/* ANS state update */
|
||||
FSEv06_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */
|
||||
FSEv06_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */
|
||||
if (MEM_32bits()) BITv06_reloadDStream(&(seqState->DStream)); /* <= 18 bits */
|
||||
FSEv06_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */
|
||||
FSEv07_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */
|
||||
FSEv07_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */
|
||||
if (MEM_32bits()) BITv07_reloadDStream(&(seqState->DStream)); /* <= 18 bits */
|
||||
FSEv07_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */
|
||||
|
||||
return seq;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv06_execSequence(BYTE* op,
|
||||
FORCE_INLINE
|
||||
size_t ZSTDv07_execSequence(BYTE* op,
|
||||
BYTE* const oend, seq_t sequence,
|
||||
const BYTE** litPtr, const BYTE* const litLimit_8,
|
||||
const BYTE** litPtr, const BYTE* const litLimit_w,
|
||||
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
||||
{
|
||||
BYTE* const oLitEnd = op + sequence.litLength;
|
||||
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
|
||||
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
|
||||
BYTE* const oend_8 = oend-8;
|
||||
BYTE* const oend_w = oend-WILDCOPY_OVERLENGTH;
|
||||
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
|
||||
const BYTE* match = oLitEnd - sequence.offset;
|
||||
|
||||
/* check */
|
||||
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
||||
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
||||
if (iLitEnd > litLimit_8) return ERROR(corruption_detected); /* over-read beyond lit buffer */
|
||||
if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
|
||||
if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
|
||||
|
||||
/* copy Literals */
|
||||
ZSTDv06_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
||||
ZSTDv07_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
|
||||
op = oLitEnd;
|
||||
*litPtr = iLitEnd; /* update for next sequence */
|
||||
|
||||
@@ -3906,29 +4110,29 @@ size_t ZSTDv06_execSequence(BYTE* op,
|
||||
op[2] = match[2];
|
||||
op[3] = match[3];
|
||||
match += dec32table[sequence.offset];
|
||||
ZSTDv06_copy4(op+4, match);
|
||||
ZSTDv07_copy4(op+4, match);
|
||||
match -= sub2;
|
||||
} else {
|
||||
ZSTDv06_copy8(op, match);
|
||||
ZSTDv07_copy8(op, match);
|
||||
}
|
||||
op += 8; match += 8;
|
||||
|
||||
if (oMatchEnd > oend-(16-MINMATCH)) {
|
||||
if (op < oend_8) {
|
||||
ZSTDv06_wildcopy(op, match, oend_8 - op);
|
||||
match += oend_8 - op;
|
||||
op = oend_8;
|
||||
if (op < oend_w) {
|
||||
ZSTDv07_wildcopy(op, match, oend_w - op);
|
||||
match += oend_w - op;
|
||||
op = oend_w;
|
||||
}
|
||||
while (op < oMatchEnd) *op++ = *match++;
|
||||
} else {
|
||||
ZSTDv06_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
||||
ZSTDv07_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
||||
}
|
||||
return sequenceLength;
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTDv06_decompressSequences(
|
||||
ZSTDv06_DCtx* dctx,
|
||||
static size_t ZSTDv07_decompressSequences(
|
||||
ZSTDv07_DCtx* dctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* seqStart, size_t seqSize)
|
||||
{
|
||||
@@ -3938,63 +4142,51 @@ static size_t ZSTDv06_decompressSequences(
|
||||
BYTE* const oend = ostart + maxDstSize;
|
||||
BYTE* op = ostart;
|
||||
const BYTE* litPtr = dctx->litPtr;
|
||||
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
|
||||
const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
|
||||
const BYTE* const litEnd = litPtr + dctx->litSize;
|
||||
FSEv06_DTable* DTableLL = dctx->LLTable;
|
||||
FSEv06_DTable* DTableML = dctx->MLTable;
|
||||
FSEv06_DTable* DTableOffb = dctx->OffTable;
|
||||
FSEv07_DTable* DTableLL = dctx->LLTable;
|
||||
FSEv07_DTable* DTableML = dctx->MLTable;
|
||||
FSEv07_DTable* DTableOffb = dctx->OffTable;
|
||||
const BYTE* const base = (const BYTE*) (dctx->base);
|
||||
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
||||
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
||||
int nbSeq;
|
||||
|
||||
/* Build Decoding Tables */
|
||||
{ size_t const seqHSize = ZSTDv06_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->flagRepeatTable, ip, seqSize);
|
||||
if (ZSTDv06_isError(seqHSize)) return seqHSize;
|
||||
{ size_t const seqHSize = ZSTDv07_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize);
|
||||
if (ZSTDv07_isError(seqHSize)) return seqHSize;
|
||||
ip += seqHSize;
|
||||
dctx->flagRepeatTable = 0;
|
||||
}
|
||||
|
||||
/* Regen sequences */
|
||||
if (nbSeq) {
|
||||
seq_t sequence;
|
||||
seqState_t seqState;
|
||||
|
||||
memset(&sequence, 0, sizeof(sequence));
|
||||
sequence.offset = REPCODE_STARTVALUE;
|
||||
{ U32 i; for (i=0; i<ZSTDv06_REP_INIT; i++) seqState.prevOffset[i] = REPCODE_STARTVALUE; }
|
||||
{ size_t const errorCode = BITv06_initDStream(&(seqState.DStream), ip, iend-ip);
|
||||
dctx->fseEntropy = 1;
|
||||
{ U32 i; for (i=0; i<ZSTDv07_REP_INIT; i++) seqState.prevOffset[i] = dctx->rep[i]; }
|
||||
{ size_t const errorCode = BITv07_initDStream(&(seqState.DStream), ip, iend-ip);
|
||||
if (ERR_isError(errorCode)) return ERROR(corruption_detected); }
|
||||
FSEv06_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
|
||||
FSEv06_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
|
||||
FSEv06_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
|
||||
FSEv07_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
|
||||
FSEv07_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
|
||||
FSEv07_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
|
||||
|
||||
for ( ; (BITv06_reloadDStream(&(seqState.DStream)) <= BITv06_DStream_completed) && nbSeq ; ) {
|
||||
for ( ; (BITv07_reloadDStream(&(seqState.DStream)) <= BITv07_DStream_completed) && nbSeq ; ) {
|
||||
nbSeq--;
|
||||
ZSTDv06_decodeSequence(&sequence, &seqState);
|
||||
|
||||
#if 0 /* debug */
|
||||
static BYTE* start = NULL;
|
||||
if (start==NULL) start = op;
|
||||
size_t pos = (size_t)(op-start);
|
||||
if ((pos >= 5810037) && (pos < 5810400))
|
||||
printf("Dpos %6u :%5u literals & match %3u bytes at distance %6u \n",
|
||||
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
|
||||
#endif
|
||||
|
||||
{ size_t const oneSeqSize = ZSTDv06_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
|
||||
if (ZSTDv06_isError(oneSeqSize)) return oneSeqSize;
|
||||
{ seq_t const sequence = ZSTDv07_decodeSequence(&seqState);
|
||||
size_t const oneSeqSize = ZSTDv07_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd);
|
||||
if (ZSTDv07_isError(oneSeqSize)) return oneSeqSize;
|
||||
op += oneSeqSize;
|
||||
} }
|
||||
|
||||
/* check if reached exact end */
|
||||
if (nbSeq) return ERROR(corruption_detected);
|
||||
/* save reps for next block */
|
||||
{ U32 i; for (i=0; i<ZSTDv07_REP_INIT; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); }
|
||||
}
|
||||
|
||||
/* last literal segment */
|
||||
{ size_t const lastLLSize = litEnd - litPtr;
|
||||
if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
|
||||
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
||||
//if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */
|
||||
if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
|
||||
memcpy(op, litPtr, lastLLSize);
|
||||
op += lastLLSize;
|
||||
}
|
||||
@@ -4003,7 +4195,7 @@ static size_t ZSTDv06_decompressSequences(
|
||||
}
|
||||
|
||||
|
||||
static void ZSTDv06_checkContinuity(ZSTDv06_DCtx* dctx, const void* dst)
|
||||
static void ZSTDv07_checkContinuity(ZSTDv07_DCtx* dctx, const void* dst)
|
||||
{
|
||||
if (dst != dctx->previousDstEnd) { /* not contiguous */
|
||||
dctx->dictEnd = dctx->previousDstEnd;
|
||||
@@ -4014,89 +4206,112 @@ static void ZSTDv06_checkContinuity(ZSTDv06_DCtx* dctx, const void* dst)
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTDv06_decompressBlock_internal(ZSTDv06_DCtx* dctx,
|
||||
static size_t ZSTDv07_decompressBlock_internal(ZSTDv07_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{ /* blockType == blockCompressed */
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
|
||||
if (srcSize >= ZSTDv06_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
|
||||
if (srcSize >= ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong);
|
||||
|
||||
/* Decode literals sub-block */
|
||||
{ size_t const litCSize = ZSTDv06_decodeLiteralsBlock(dctx, src, srcSize);
|
||||
if (ZSTDv06_isError(litCSize)) return litCSize;
|
||||
{ size_t const litCSize = ZSTDv07_decodeLiteralsBlock(dctx, src, srcSize);
|
||||
if (ZSTDv07_isError(litCSize)) return litCSize;
|
||||
ip += litCSize;
|
||||
srcSize -= litCSize;
|
||||
}
|
||||
return ZSTDv06_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
|
||||
return ZSTDv07_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx,
|
||||
size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTDv06_checkContinuity(dctx, dst);
|
||||
return ZSTDv06_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
|
||||
size_t dSize;
|
||||
ZSTDv07_checkContinuity(dctx, dst);
|
||||
dSize = ZSTDv07_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
|
||||
dctx->previousDstEnd = (char*)dst + dSize;
|
||||
return dSize;
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTDv06_decompressFrame() :
|
||||
/** ZSTDv07_insertBlock() :
|
||||
insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
|
||||
ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize)
|
||||
{
|
||||
ZSTDv07_checkContinuity(dctx, blockStart);
|
||||
dctx->previousDstEnd = (const char*)blockStart + blockSize;
|
||||
return blockSize;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
|
||||
{
|
||||
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||
memset(dst, byte, length);
|
||||
return length;
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTDv07_decompressFrame() :
|
||||
* `dctx` must be properly initialized */
|
||||
static size_t ZSTDv06_decompressFrame(ZSTDv06_DCtx* dctx,
|
||||
static size_t ZSTDv07_decompressFrame(ZSTDv07_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* const iend = ip + srcSize;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + dstCapacity;
|
||||
BYTE* op = ostart;
|
||||
size_t remainingSize = srcSize;
|
||||
blockProperties_t blockProperties = { bt_compressed, 0 };
|
||||
|
||||
/* check */
|
||||
if (srcSize < ZSTDv06_frameHeaderSize_min+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
|
||||
/* Frame Header */
|
||||
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
|
||||
if (ZSTDv06_isError(frameHeaderSize)) return frameHeaderSize;
|
||||
if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
if (ZSTDv06_decodeFrameHeader(dctx, src, frameHeaderSize)) return ERROR(corruption_detected);
|
||||
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
|
||||
if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize;
|
||||
if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
if (ZSTDv07_decodeFrameHeader(dctx, src, frameHeaderSize)) return ERROR(corruption_detected);
|
||||
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
||||
}
|
||||
|
||||
/* Loop on each block */
|
||||
while (1) {
|
||||
size_t decodedSize=0;
|
||||
size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, iend-ip, &blockProperties);
|
||||
if (ZSTDv06_isError(cBlockSize)) return cBlockSize;
|
||||
size_t decodedSize;
|
||||
blockProperties_t blockProperties;
|
||||
size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, iend-ip, &blockProperties);
|
||||
if (ZSTDv07_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
ip += ZSTDv06_blockHeaderSize;
|
||||
remainingSize -= ZSTDv06_blockHeaderSize;
|
||||
ip += ZSTDv07_blockHeaderSize;
|
||||
remainingSize -= ZSTDv07_blockHeaderSize;
|
||||
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
switch(blockProperties.blockType)
|
||||
{
|
||||
case bt_compressed:
|
||||
decodedSize = ZSTDv06_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize);
|
||||
decodedSize = ZSTDv07_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize);
|
||||
break;
|
||||
case bt_raw :
|
||||
decodedSize = ZSTDv06_copyRawBlock(op, oend-op, ip, cBlockSize);
|
||||
decodedSize = ZSTDv07_copyRawBlock(op, oend-op, ip, cBlockSize);
|
||||
break;
|
||||
case bt_rle :
|
||||
return ERROR(GENERIC); /* not yet supported */
|
||||
decodedSize = ZSTDv07_generateNxBytes(op, oend-op, *ip, blockProperties.origSize);
|
||||
break;
|
||||
case bt_end :
|
||||
/* end of frame */
|
||||
if (remainingSize) return ERROR(srcSize_wrong);
|
||||
decodedSize = 0;
|
||||
break;
|
||||
default:
|
||||
return ERROR(GENERIC); /* impossible */
|
||||
}
|
||||
if (cBlockSize == 0) break; /* bt_end */
|
||||
if (blockProperties.blockType == bt_end) break; /* bt_end */
|
||||
|
||||
if (ZSTDv06_isError(decodedSize)) return decodedSize;
|
||||
if (ZSTDv07_isError(decodedSize)) return decodedSize;
|
||||
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize);
|
||||
op += decodedSize;
|
||||
ip += cBlockSize;
|
||||
remainingSize -= cBlockSize;
|
||||
@@ -4106,45 +4321,50 @@ static size_t ZSTDv06_decompressFrame(ZSTDv06_DCtx* dctx,
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv06_decompress_usingPreparedDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* refDCtx,
|
||||
/*! ZSTDv07_decompress_usingPreparedDCtx() :
|
||||
* Same as ZSTDv07_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
|
||||
* It avoids reloading the dictionary each time.
|
||||
* `preparedDCtx` must have been properly initialized using ZSTDv07_decompressBegin_usingDict().
|
||||
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
|
||||
size_t ZSTDv07_decompress_usingPreparedDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* refDCtx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTDv06_copyDCtx(dctx, refDCtx);
|
||||
ZSTDv06_checkContinuity(dctx, dst);
|
||||
return ZSTDv06_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
|
||||
ZSTDv07_copyDCtx(dctx, refDCtx);
|
||||
ZSTDv07_checkContinuity(dctx, dst);
|
||||
return ZSTDv07_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx,
|
||||
size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize)
|
||||
{
|
||||
ZSTDv06_decompressBegin_usingDict(dctx, dict, dictSize);
|
||||
ZSTDv06_checkContinuity(dctx, dst);
|
||||
return ZSTDv06_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
|
||||
ZSTDv07_decompressBegin_usingDict(dctx, dict, dictSize);
|
||||
ZSTDv07_checkContinuity(dctx, dst);
|
||||
return ZSTDv07_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTDv06_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0);
|
||||
return ZSTDv07_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
#if defined(ZSTDv06_HEAPMODE) && (ZSTDv06_HEAPMODE==1)
|
||||
#if defined(ZSTDv07_HEAPMODE) && (ZSTDv07_HEAPMODE==1)
|
||||
size_t regenSize;
|
||||
ZSTDv06_DCtx* dctx = ZSTDv06_createDCtx();
|
||||
ZSTDv07_DCtx* const dctx = ZSTDv07_createDCtx();
|
||||
if (dctx==NULL) return ERROR(memory_allocation);
|
||||
regenSize = ZSTDv06_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
|
||||
ZSTDv06_freeDCtx(dctx);
|
||||
regenSize = ZSTDv07_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
|
||||
ZSTDv07_freeDCtx(dctx);
|
||||
return regenSize;
|
||||
#else /* stack mode */
|
||||
ZSTDv06_DCtx dctx;
|
||||
return ZSTDv06_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
|
||||
ZSTDv07_DCtx dctx;
|
||||
return ZSTDv07_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -4152,27 +4372,40 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
|
||||
/*_******************************
|
||||
* Streaming Decompression API
|
||||
********************************/
|
||||
size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx)
|
||||
size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx)
|
||||
{
|
||||
return dctx->expected;
|
||||
}
|
||||
|
||||
size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
int ZSTDv07_isSkipFrame(ZSTDv07_DCtx* dctx)
|
||||
{
|
||||
return dctx->stage == ZSTDds_skipFrame;
|
||||
}
|
||||
|
||||
/** ZSTDv07_decompressContinue() :
|
||||
* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
|
||||
* or an error code, which can be tested using ZSTDv07_isError() */
|
||||
size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
/* Sanity check */
|
||||
if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
|
||||
if (dstCapacity) ZSTDv06_checkContinuity(dctx, dst);
|
||||
if (dstCapacity) ZSTDv07_checkContinuity(dctx, dst);
|
||||
|
||||
/* Decompress : frame header; part 1 */
|
||||
switch (dctx->stage)
|
||||
{
|
||||
case ZSTDds_getFrameHeaderSize :
|
||||
if (srcSize != ZSTDv06_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||
dctx->headerSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
|
||||
if (ZSTDv06_isError(dctx->headerSize)) return dctx->headerSize;
|
||||
memcpy(dctx->headerBuffer, src, ZSTDv06_frameHeaderSize_min);
|
||||
if (dctx->headerSize > ZSTDv06_frameHeaderSize_min) {
|
||||
dctx->expected = dctx->headerSize - ZSTDv06_frameHeaderSize_min;
|
||||
if (srcSize != ZSTDv07_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) {
|
||||
memcpy(dctx->headerBuffer, src, ZSTDv07_frameHeaderSize_min);
|
||||
dctx->expected = ZSTDv07_skippableHeaderSize - ZSTDv07_frameHeaderSize_min; /* magic number + skippable frame length */
|
||||
dctx->stage = ZSTDds_decodeSkippableHeader;
|
||||
return 0;
|
||||
}
|
||||
dctx->headerSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
|
||||
if (ZSTDv07_isError(dctx->headerSize)) return dctx->headerSize;
|
||||
memcpy(dctx->headerBuffer, src, ZSTDv07_frameHeaderSize_min);
|
||||
if (dctx->headerSize > ZSTDv07_frameHeaderSize_min) {
|
||||
dctx->expected = dctx->headerSize - ZSTDv07_frameHeaderSize_min;
|
||||
dctx->stage = ZSTDds_decodeFrameHeader;
|
||||
return 0;
|
||||
}
|
||||
@@ -4180,18 +4413,25 @@ size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapac
|
||||
|
||||
case ZSTDds_decodeFrameHeader:
|
||||
{ size_t result;
|
||||
memcpy(dctx->headerBuffer + ZSTDv06_frameHeaderSize_min, src, dctx->expected);
|
||||
result = ZSTDv06_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
|
||||
if (ZSTDv06_isError(result)) return result;
|
||||
dctx->expected = ZSTDv06_blockHeaderSize;
|
||||
memcpy(dctx->headerBuffer + ZSTDv07_frameHeaderSize_min, src, dctx->expected);
|
||||
result = ZSTDv07_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
|
||||
if (ZSTDv07_isError(result)) return result;
|
||||
dctx->expected = ZSTDv07_blockHeaderSize;
|
||||
dctx->stage = ZSTDds_decodeBlockHeader;
|
||||
return 0;
|
||||
}
|
||||
case ZSTDds_decodeBlockHeader:
|
||||
{ blockProperties_t bp;
|
||||
size_t const cBlockSize = ZSTDv06_getcBlockSize(src, ZSTDv06_blockHeaderSize, &bp);
|
||||
if (ZSTDv06_isError(cBlockSize)) return cBlockSize;
|
||||
size_t const cBlockSize = ZSTDv07_getcBlockSize(src, ZSTDv07_blockHeaderSize, &bp);
|
||||
if (ZSTDv07_isError(cBlockSize)) return cBlockSize;
|
||||
if (bp.blockType == bt_end) {
|
||||
if (dctx->fParams.checksumFlag) {
|
||||
U64 const h64 = XXH64_digest(&dctx->xxhState);
|
||||
U32 const h32 = (U32)(h64>>11) & ((1<<22)-1);
|
||||
const BYTE* const ip = (const BYTE*)src;
|
||||
U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16);
|
||||
if (check32 != h32) return ERROR(checksum_wrong);
|
||||
}
|
||||
dctx->expected = 0;
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
} else {
|
||||
@@ -4206,10 +4446,10 @@ size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapac
|
||||
switch(dctx->bType)
|
||||
{
|
||||
case bt_compressed:
|
||||
rSize = ZSTDv06_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
|
||||
rSize = ZSTDv07_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
|
||||
break;
|
||||
case bt_raw :
|
||||
rSize = ZSTDv06_copyRawBlock(dst, dstCapacity, src, srcSize);
|
||||
rSize = ZSTDv07_copyRawBlock(dst, dstCapacity, src, srcSize);
|
||||
break;
|
||||
case bt_rle :
|
||||
return ERROR(GENERIC); /* not yet handled */
|
||||
@@ -4221,102 +4461,194 @@ size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapac
|
||||
return ERROR(GENERIC); /* impossible */
|
||||
}
|
||||
dctx->stage = ZSTDds_decodeBlockHeader;
|
||||
dctx->expected = ZSTDv06_blockHeaderSize;
|
||||
dctx->expected = ZSTDv07_blockHeaderSize;
|
||||
dctx->previousDstEnd = (char*)dst + rSize;
|
||||
if (ZSTDv07_isError(rSize)) return rSize;
|
||||
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
|
||||
return rSize;
|
||||
}
|
||||
case ZSTDds_decodeSkippableHeader:
|
||||
{ memcpy(dctx->headerBuffer + ZSTDv07_frameHeaderSize_min, src, dctx->expected);
|
||||
dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
|
||||
dctx->stage = ZSTDds_skipFrame;
|
||||
return 0;
|
||||
}
|
||||
case ZSTDds_skipFrame:
|
||||
{ dctx->expected = 0;
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
return 0;
|
||||
}
|
||||
default:
|
||||
return ERROR(GENERIC); /* impossible */
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void ZSTDv06_refDictContent(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
static size_t ZSTDv07_refDictContent(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
{
|
||||
dctx->dictEnd = dctx->previousDstEnd;
|
||||
dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
|
||||
dctx->base = dict;
|
||||
dctx->previousDstEnd = (const char*)dict + dictSize;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, size_t const dictSize)
|
||||
{
|
||||
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize;
|
||||
const BYTE* dictPtr = (const BYTE*)dict;
|
||||
const BYTE* const dictEnd = dictPtr + dictSize;
|
||||
|
||||
hSize = HUFv06_readDTableX4(dctx->hufTableX4, dict, dictSize);
|
||||
if (HUFv06_isError(hSize)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + hSize;
|
||||
dictSize -= hSize;
|
||||
{ size_t const hSize = HUFv07_readDTableX4(dctx->hufTable, dict, dictSize);
|
||||
if (HUFv07_isError(hSize)) return ERROR(dictionary_corrupted);
|
||||
dictPtr += hSize;
|
||||
}
|
||||
|
||||
{ short offcodeNCount[MaxOff+1];
|
||||
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
|
||||
offcodeHeaderSize = FSEv06_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
|
||||
if (FSEv06_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSEv06_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dict = (const char*)dict + offcodeHeaderSize;
|
||||
dictSize -= offcodeHeaderSize;
|
||||
size_t const offcodeHeaderSize = FSEv07_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
|
||||
if (FSEv07_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSEv07_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += offcodeHeaderSize;
|
||||
}
|
||||
|
||||
{ short matchlengthNCount[MaxML+1];
|
||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||
matchlengthHeaderSize = FSEv06_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
|
||||
if (FSEv06_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSEv06_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dict = (const char*)dict + matchlengthHeaderSize;
|
||||
dictSize -= matchlengthHeaderSize;
|
||||
size_t const matchlengthHeaderSize = FSEv07_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
|
||||
if (FSEv07_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSEv07_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += matchlengthHeaderSize;
|
||||
}
|
||||
|
||||
{ short litlengthNCount[MaxLL+1];
|
||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||
litlengthHeaderSize = FSEv06_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
|
||||
if (FSEv06_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSEv06_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
size_t const litlengthHeaderSize = FSEv07_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
|
||||
if (FSEv07_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSEv07_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += litlengthHeaderSize;
|
||||
}
|
||||
|
||||
dctx->flagRepeatTable = 1;
|
||||
return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
|
||||
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
|
||||
dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
dictPtr += 12;
|
||||
|
||||
dctx->litEntropy = dctx->fseEntropy = 1;
|
||||
return dictPtr - (const BYTE*)dict;
|
||||
}
|
||||
|
||||
static size_t ZSTDv06_decompress_insertDictionary(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
static size_t ZSTDv07_decompress_insertDictionary(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
{
|
||||
size_t eSize;
|
||||
U32 const magic = MEM_readLE32(dict);
|
||||
if (magic != ZSTDv06_DICT_MAGIC) {
|
||||
/* pure content mode */
|
||||
ZSTDv06_refDictContent(dctx, dict, dictSize);
|
||||
return 0;
|
||||
}
|
||||
if (dictSize < 8) return ZSTDv07_refDictContent(dctx, dict, dictSize);
|
||||
{ U32 const magic = MEM_readLE32(dict);
|
||||
if (magic != ZSTDv07_DICT_MAGIC) {
|
||||
return ZSTDv07_refDictContent(dctx, dict, dictSize); /* pure content mode */
|
||||
} }
|
||||
dctx->dictID = MEM_readLE32((const char*)dict + 4);
|
||||
|
||||
/* load entropy tables */
|
||||
dict = (const char*)dict + 4;
|
||||
dictSize -= 4;
|
||||
eSize = ZSTDv06_loadEntropy(dctx, dict, dictSize);
|
||||
if (ZSTDv06_isError(eSize)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + 8;
|
||||
dictSize -= 8;
|
||||
{ size_t const eSize = ZSTDv07_loadEntropy(dctx, dict, dictSize);
|
||||
if (ZSTDv07_isError(eSize)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + eSize;
|
||||
dictSize -= eSize;
|
||||
}
|
||||
|
||||
/* reference dictionary content */
|
||||
dict = (const char*)dict + eSize;
|
||||
dictSize -= eSize;
|
||||
ZSTDv06_refDictContent(dctx, dict, dictSize);
|
||||
|
||||
return 0;
|
||||
return ZSTDv07_refDictContent(dctx, dict, dictSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
{
|
||||
{ size_t const errorCode = ZSTDv06_decompressBegin(dctx);
|
||||
if (ZSTDv06_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = ZSTDv07_decompressBegin(dctx);
|
||||
if (ZSTDv07_isError(errorCode)) return errorCode; }
|
||||
|
||||
if (dict && dictSize) {
|
||||
size_t const errorCode = ZSTDv06_decompress_insertDictionary(dctx, dict, dictSize);
|
||||
if (ZSTDv06_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
size_t const errorCode = ZSTDv07_decompress_insertDictionary(dctx, dict, dictSize);
|
||||
if (ZSTDv07_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
struct ZSTDv07_DDict_s {
|
||||
void* dict;
|
||||
size_t dictSize;
|
||||
ZSTDv07_DCtx* refContext;
|
||||
}; /* typedef'd tp ZSTDv07_CDict within zstd.h */
|
||||
|
||||
ZSTDv07_DDict* ZSTDv07_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv07_customMem customMem)
|
||||
{
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
{ ZSTDv07_DDict* const ddict = (ZSTDv07_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict));
|
||||
void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize);
|
||||
ZSTDv07_DCtx* const dctx = ZSTDv07_createDCtx_advanced(customMem);
|
||||
|
||||
if (!dictContent || !ddict || !dctx) {
|
||||
customMem.customFree(customMem.opaque, dictContent);
|
||||
customMem.customFree(customMem.opaque, ddict);
|
||||
customMem.customFree(customMem.opaque, dctx);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memcpy(dictContent, dict, dictSize);
|
||||
{ size_t const errorCode = ZSTDv07_decompressBegin_usingDict(dctx, dictContent, dictSize);
|
||||
if (ZSTDv07_isError(errorCode)) {
|
||||
customMem.customFree(customMem.opaque, dictContent);
|
||||
customMem.customFree(customMem.opaque, ddict);
|
||||
customMem.customFree(customMem.opaque, dctx);
|
||||
return NULL;
|
||||
} }
|
||||
|
||||
ddict->dict = dictContent;
|
||||
ddict->dictSize = dictSize;
|
||||
ddict->refContext = dctx;
|
||||
return ddict;
|
||||
}
|
||||
}
|
||||
|
||||
/*! ZSTDv07_createDDict() :
|
||||
* Create a digested dictionary, ready to start decompression without startup delay.
|
||||
* `dict` can be released after `ZSTDv07_DDict` creation */
|
||||
ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize)
|
||||
{
|
||||
ZSTDv07_customMem const allocator = { NULL, NULL, NULL };
|
||||
return ZSTDv07_createDDict_advanced(dict, dictSize, allocator);
|
||||
}
|
||||
|
||||
size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict)
|
||||
{
|
||||
ZSTDv07_freeFunction const cFree = ddict->refContext->customMem.customFree;
|
||||
void* const opaque = ddict->refContext->customMem.opaque;
|
||||
ZSTDv07_freeDCtx(ddict->refContext);
|
||||
cFree(opaque, ddict->dict);
|
||||
cFree(opaque, ddict);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*! ZSTDv07_decompress_usingDDict() :
|
||||
* Decompression using a pre-digested Dictionary
|
||||
* Use dictionary without significant overhead. */
|
||||
ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTDv07_DDict* ddict)
|
||||
{
|
||||
return ZSTDv07_decompress_usingPreparedDCtx(dctx, ddict->refContext,
|
||||
dst, dstCapacity,
|
||||
src, srcSize);
|
||||
}
|
||||
/*
|
||||
Buffered version of Zstd compression library
|
||||
Copyright (C) 2015-2016, Yann Collet.
|
||||
@@ -4349,38 +4681,39 @@ size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, s
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/*-***************************************************************************
|
||||
* Streaming decompression howto
|
||||
*
|
||||
* A ZBUFFv06_DCtx object is required to track streaming operations.
|
||||
* Use ZBUFFv06_createDCtx() and ZBUFFv06_freeDCtx() to create/release resources.
|
||||
* Use ZBUFFv06_decompressInit() to start a new decompression operation,
|
||||
* or ZBUFFv06_decompressInitDictionary() if decompression requires a dictionary.
|
||||
* Note that ZBUFFv06_DCtx objects can be re-init multiple times.
|
||||
* A ZBUFFv07_DCtx object is required to track streaming operations.
|
||||
* Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources.
|
||||
* Use ZBUFFv07_decompressInit() to start a new decompression operation,
|
||||
* or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary.
|
||||
* Note that ZBUFFv07_DCtx objects can be re-init multiple times.
|
||||
*
|
||||
* Use ZBUFFv06_decompressContinue() repetitively to consume your input.
|
||||
* Use ZBUFFv07_decompressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
|
||||
* or 0 when a frame is completely decoded,
|
||||
* or an error code, which can be tested using ZBUFFv06_isError().
|
||||
* or an error code, which can be tested using ZBUFFv07_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFFv06_recommendedDInSize() and ZBUFFv06_recommendedDOutSize()
|
||||
* output : ZBUFFv06_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFFv06_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFFv06_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize()
|
||||
* output : ZBUFFv07_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFFv07_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* *******************************************************************************/
|
||||
|
||||
typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
|
||||
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFFv06_dStage;
|
||||
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFFv07_dStage;
|
||||
|
||||
/* *** Resource management *** */
|
||||
struct ZBUFFv06_DCtx_s {
|
||||
ZSTDv06_DCtx* zd;
|
||||
ZSTDv06_frameParams fParams;
|
||||
ZBUFFv06_dStage stage;
|
||||
struct ZBUFFv07_DCtx_s {
|
||||
ZSTDv07_DCtx* zd;
|
||||
ZSTDv07_frameParams fParams;
|
||||
ZBUFFv07_dStage stage;
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inPos;
|
||||
@@ -4389,51 +4722,68 @@ struct ZBUFFv06_DCtx_s {
|
||||
size_t outStart;
|
||||
size_t outEnd;
|
||||
size_t blockSize;
|
||||
BYTE headerBuffer[ZSTDv06_FRAMEHEADERSIZE_MAX];
|
||||
BYTE headerBuffer[ZSTDv07_FRAMEHEADERSIZE_MAX];
|
||||
size_t lhSize;
|
||||
}; /* typedef'd to ZBUFFv06_DCtx within "zstd_buffered.h" */
|
||||
ZSTDv07_customMem customMem;
|
||||
}; /* typedef'd to ZBUFFv07_DCtx within "zstd_buffered.h" */
|
||||
|
||||
ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem);
|
||||
|
||||
ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void)
|
||||
ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void)
|
||||
{
|
||||
ZBUFFv06_DCtx* zbd = (ZBUFFv06_DCtx*)malloc(sizeof(ZBUFFv06_DCtx));
|
||||
return ZBUFFv07_createDCtx_advanced(defaultCustomMem);
|
||||
}
|
||||
|
||||
ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem)
|
||||
{
|
||||
ZBUFFv07_DCtx* zbd;
|
||||
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
zbd = (ZBUFFv07_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFFv07_DCtx));
|
||||
if (zbd==NULL) return NULL;
|
||||
memset(zbd, 0, sizeof(*zbd));
|
||||
zbd->zd = ZSTDv06_createDCtx();
|
||||
memset(zbd, 0, sizeof(ZBUFFv07_DCtx));
|
||||
memcpy(&zbd->customMem, &customMem, sizeof(ZSTDv07_customMem));
|
||||
zbd->zd = ZSTDv07_createDCtx_advanced(customMem);
|
||||
if (zbd->zd == NULL) { ZBUFFv07_freeDCtx(zbd); return NULL; }
|
||||
zbd->stage = ZBUFFds_init;
|
||||
return zbd;
|
||||
}
|
||||
|
||||
size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* zbd)
|
||||
size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* zbd)
|
||||
{
|
||||
if (zbd==NULL) return 0; /* support free on null */
|
||||
ZSTDv06_freeDCtx(zbd->zd);
|
||||
free(zbd->inBuff);
|
||||
free(zbd->outBuff);
|
||||
free(zbd);
|
||||
ZSTDv07_freeDCtx(zbd->zd);
|
||||
if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
|
||||
if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/* *** Initialization *** */
|
||||
|
||||
size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* zbd, const void* dict, size_t dictSize)
|
||||
size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* zbd, const void* dict, size_t dictSize)
|
||||
{
|
||||
zbd->stage = ZBUFFds_loadHeader;
|
||||
zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
|
||||
return ZSTDv06_decompressBegin_usingDict(zbd->zd, dict, dictSize);
|
||||
return ZSTDv07_decompressBegin_usingDict(zbd->zd, dict, dictSize);
|
||||
}
|
||||
|
||||
size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* zbd)
|
||||
size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* zbd)
|
||||
{
|
||||
return ZBUFFv06_decompressInitDictionary(zbd, NULL, 0);
|
||||
return ZBUFFv07_decompressInitDictionary(zbd, NULL, 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
MEM_STATIC size_t ZBUFFv06_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
/* internal util function */
|
||||
MEM_STATIC size_t ZBUFFv07_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t length = MIN(dstCapacity, srcSize);
|
||||
size_t const length = MIN(dstCapacity, srcSize);
|
||||
memcpy(dst, src, length);
|
||||
return length;
|
||||
}
|
||||
@@ -4441,7 +4791,7 @@ MEM_STATIC size_t ZBUFFv06_limitCopy(void* dst, size_t dstCapacity, const void*
|
||||
|
||||
/* *** Decompression *** */
|
||||
|
||||
size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
||||
size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr)
|
||||
{
|
||||
@@ -4460,62 +4810,65 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
||||
return ERROR(init_missing);
|
||||
|
||||
case ZBUFFds_loadHeader :
|
||||
{ size_t const hSize = ZSTDv06_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
|
||||
{ size_t const hSize = ZSTDv07_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
|
||||
if (hSize != 0) {
|
||||
size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */
|
||||
if (ZSTDv06_isError(hSize)) return hSize;
|
||||
if (ZSTDv07_isError(hSize)) return hSize;
|
||||
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
||||
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
|
||||
zbd->lhSize += iend-ip; ip = iend; notDone = 0;
|
||||
zbd->lhSize += iend-ip;
|
||||
*dstCapacityPtr = 0;
|
||||
return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */
|
||||
return (hSize - zbd->lhSize) + ZSTDv07_blockHeaderSize; /* remaining header bytes + next block header */
|
||||
}
|
||||
memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
|
||||
break;
|
||||
} }
|
||||
|
||||
/* Consume header */
|
||||
{ size_t const h1Size = ZSTDv06_nextSrcSizeToDecompress(zbd->zd); /* == ZSTDv06_frameHeaderSize_min */
|
||||
size_t const h1Result = ZSTDv06_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
|
||||
if (ZSTDv06_isError(h1Result)) return h1Result;
|
||||
{ size_t const h1Size = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); /* == ZSTDv07_frameHeaderSize_min */
|
||||
size_t const h1Result = ZSTDv07_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
|
||||
if (ZSTDv07_isError(h1Result)) return h1Result;
|
||||
if (h1Size < zbd->lhSize) { /* long header */
|
||||
size_t const h2Size = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
|
||||
size_t const h2Result = ZSTDv06_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
|
||||
if (ZSTDv06_isError(h2Result)) return h2Result;
|
||||
size_t const h2Size = ZSTDv07_nextSrcSizeToDecompress(zbd->zd);
|
||||
size_t const h2Result = ZSTDv07_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
|
||||
if (ZSTDv07_isError(h2Result)) return h2Result;
|
||||
} }
|
||||
|
||||
zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTDv07_WINDOWLOG_ABSOLUTEMIN);
|
||||
|
||||
/* Frame header instruct buffer sizes */
|
||||
{ size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTDv06_BLOCKSIZE_MAX);
|
||||
{ size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTDv07_BLOCKSIZE_ABSOLUTEMAX);
|
||||
zbd->blockSize = blockSize;
|
||||
if (zbd->inBuffSize < blockSize) {
|
||||
free(zbd->inBuff);
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
|
||||
zbd->inBuffSize = blockSize;
|
||||
zbd->inBuff = (char*)malloc(blockSize);
|
||||
zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
|
||||
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
{ size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize;
|
||||
{ size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
|
||||
if (zbd->outBuffSize < neededOutSize) {
|
||||
free(zbd->outBuff);
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
|
||||
zbd->outBuffSize = neededOutSize;
|
||||
zbd->outBuff = (char*)malloc(neededOutSize);
|
||||
zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize);
|
||||
if (zbd->outBuff == NULL) return ERROR(memory_allocation);
|
||||
} } }
|
||||
zbd->stage = ZBUFFds_read;
|
||||
|
||||
case ZBUFFds_read:
|
||||
{ size_t const neededInSize = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
|
||||
{ size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd);
|
||||
if (neededInSize==0) { /* end of frame */
|
||||
zbd->stage = ZBUFFds_init;
|
||||
notDone = 0;
|
||||
break;
|
||||
}
|
||||
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
|
||||
size_t const decodedSize = ZSTDv06_decompressContinue(zbd->zd,
|
||||
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
|
||||
const int isSkipFrame = ZSTDv07_isSkipFrame(zbd->zd);
|
||||
size_t const decodedSize = ZSTDv07_decompressContinue(zbd->zd,
|
||||
zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
|
||||
ip, neededInSize);
|
||||
if (ZSTDv06_isError(decodedSize)) return decodedSize;
|
||||
if (ZSTDv07_isError(decodedSize)) return decodedSize;
|
||||
ip += neededInSize;
|
||||
if (!decodedSize) break; /* this was just a header */
|
||||
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
|
||||
zbd->outEnd = zbd->outStart + decodedSize;
|
||||
zbd->stage = ZBUFFds_flush;
|
||||
break;
|
||||
@@ -4525,22 +4878,23 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
||||
}
|
||||
|
||||
case ZBUFFds_load:
|
||||
{ size_t const neededInSize = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
|
||||
{ size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd);
|
||||
size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
|
||||
size_t loadedSize;
|
||||
if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */
|
||||
loadedSize = ZBUFFv06_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip);
|
||||
loadedSize = ZBUFFv07_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip);
|
||||
ip += loadedSize;
|
||||
zbd->inPos += loadedSize;
|
||||
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
|
||||
|
||||
/* decode loaded input */
|
||||
{ size_t const decodedSize = ZSTDv06_decompressContinue(zbd->zd,
|
||||
{ const int isSkipFrame = ZSTDv07_isSkipFrame(zbd->zd);
|
||||
size_t const decodedSize = ZSTDv07_decompressContinue(zbd->zd,
|
||||
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
|
||||
zbd->inBuff, neededInSize);
|
||||
if (ZSTDv06_isError(decodedSize)) return decodedSize;
|
||||
if (ZSTDv07_isError(decodedSize)) return decodedSize;
|
||||
zbd->inPos = 0; /* input is consumed */
|
||||
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
||||
if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
||||
zbd->outEnd = zbd->outStart + decodedSize;
|
||||
zbd->stage = ZBUFFds_flush;
|
||||
// break; /* ZBUFFds_flush follows */
|
||||
@@ -4548,7 +4902,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
||||
|
||||
case ZBUFFds_flush:
|
||||
{ size_t const toFlushSize = zbd->outEnd - zbd->outStart;
|
||||
size_t const flushedSize = ZBUFFv06_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
|
||||
size_t const flushedSize = ZBUFFv07_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
|
||||
op += flushedSize;
|
||||
zbd->outStart += flushedSize;
|
||||
if (flushedSize == toFlushSize) {
|
||||
@@ -4567,8 +4921,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
||||
/* result */
|
||||
*srcSizePtr = ip-istart;
|
||||
*dstCapacityPtr = op-ostart;
|
||||
{ size_t nextSrcSizeHint = ZSTDv06_nextSrcSizeToDecompress(zbd->zd);
|
||||
if (nextSrcSizeHint > ZSTDv06_blockHeaderSize) nextSrcSizeHint+= ZSTDv06_blockHeaderSize; /* get following block header too */
|
||||
{ size_t nextSrcSizeHint = ZSTDv07_nextSrcSizeToDecompress(zbd->zd);
|
||||
nextSrcSizeHint -= zbd->inPos; /* already loaded*/
|
||||
return nextSrcSizeHint;
|
||||
}
|
||||
@@ -4579,5 +4932,5 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd,
|
||||
/* *************************************
|
||||
* Tool functions
|
||||
***************************************/
|
||||
size_t ZBUFFv06_recommendedDInSize(void) { return ZSTDv06_BLOCKSIZE_MAX + ZSTDv06_blockHeaderSize /* block header size*/ ; }
|
||||
size_t ZBUFFv06_recommendedDOutSize(void) { return ZSTDv06_BLOCKSIZE_MAX; }
|
||||
size_t ZBUFFv07_recommendedDInSize(void) { return ZSTDv07_BLOCKSIZE_ABSOLUTEMAX + ZSTDv07_blockHeaderSize /* block header size*/ ; }
|
||||
size_t ZBUFFv07_recommendedDOutSize(void) { return ZSTDv07_BLOCKSIZE_ABSOLUTEMAX; }
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
zstd_v06 - decoder for 0.6 format
|
||||
zstd - standard compression library
|
||||
Header File
|
||||
Copyright (C) 2014-2016, Yann Collet.
|
||||
|
||||
@@ -29,27 +29,114 @@
|
||||
You can contact the author at :
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
*/
|
||||
#ifndef ZSTDv06_H
|
||||
#define ZSTDv06_H
|
||||
#ifndef ZSTDv07_H_235446
|
||||
#define ZSTDv07_H_235446
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
/*====== Dependency ======*/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/*-***************************************************************
|
||||
* Export parameters
|
||||
*****************************************************************/
|
||||
/*====== Export for Windows ======*/
|
||||
/*!
|
||||
* ZSTDv06_DLL_EXPORT :
|
||||
* ZSTDv07_DLL_EXPORT :
|
||||
* Enable exporting of functions when building a Windows DLL
|
||||
*/
|
||||
#if defined(_WIN32) && defined(ZSTDv06_DLL_EXPORT) && (ZSTDv06_DLL_EXPORT==1)
|
||||
#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1)
|
||||
# define ZSTDLIB_API __declspec(dllexport)
|
||||
#else
|
||||
# define ZSTDLIB_API
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Simple API
|
||||
***************************************/
|
||||
/*! ZSTDv07_getDecompressedSize() :
|
||||
* @return : decompressed size if known, 0 otherwise.
|
||||
note 1 : if `0`, follow up with ZSTDv07_getFrameParams() to know precise failure cause.
|
||||
note 2 : decompressed size could be wrong or intentionally modified !
|
||||
always ensure results fit within application's authorized limits */
|
||||
unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize);
|
||||
|
||||
/*! ZSTDv07_decompress() :
|
||||
`compressedSize` : must be _exact_ size of compressed input, otherwise decompression will fail.
|
||||
`dstCapacity` must be equal or larger than originalSize.
|
||||
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
|
||||
or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */
|
||||
ZSTDLIB_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
|
||||
const void* src, size_t compressedSize);
|
||||
|
||||
/*====== Helper functions ======*/
|
||||
ZSTDLIB_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
|
||||
ZSTDLIB_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Explicit memory management
|
||||
***************************************/
|
||||
/** Decompression context */
|
||||
typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx;
|
||||
ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void);
|
||||
ZSTDLIB_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */
|
||||
|
||||
/** ZSTDv07_decompressDCtx() :
|
||||
* Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */
|
||||
ZSTDLIB_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/*-************************
|
||||
* Simple dictionary API
|
||||
***************************/
|
||||
/*! ZSTDv07_decompress_usingDict() :
|
||||
* Decompression using a pre-defined Dictionary content (see dictBuilder).
|
||||
* Dictionary must be identical to the one used during compression.
|
||||
* Note : This function load the dictionary, resulting in a significant startup time */
|
||||
ZSTDLIB_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize);
|
||||
|
||||
|
||||
/*-**************************
|
||||
* Advanced Dictionary API
|
||||
****************************/
|
||||
/*! ZSTDv07_createDDict() :
|
||||
* Create a digested dictionary, ready to start decompression operation without startup delay.
|
||||
* `dict` can be released after creation */
|
||||
typedef struct ZSTDv07_DDict_s ZSTDv07_DDict;
|
||||
ZSTDLIB_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict);
|
||||
|
||||
/*! ZSTDv07_decompress_usingDDict() :
|
||||
* Decompression using a pre-digested Dictionary
|
||||
* Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */
|
||||
ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTDv07_DDict* ddict);
|
||||
|
||||
typedef struct {
|
||||
unsigned long long frameContentSize;
|
||||
unsigned windowSize;
|
||||
unsigned dictID;
|
||||
unsigned checksumFlag;
|
||||
} ZSTDv07_frameParams;
|
||||
|
||||
ZSTDLIB_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
||||
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
* Compiler specifics
|
||||
*****************************************************************/
|
||||
/* ZSTDv07_DLL_EXPORT :
|
||||
* Enable exporting of functions when building a Windows DLL */
|
||||
#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1)
|
||||
# define ZSTDLIB_API __declspec(dllexport)
|
||||
#else
|
||||
# define ZSTDLIB_API
|
||||
@@ -57,129 +144,64 @@ extern "C" {
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Simple functions
|
||||
* Streaming functions
|
||||
***************************************/
|
||||
/*! ZSTDv06_decompress() :
|
||||
`compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail.
|
||||
`dstCapacity` must be large enough, equal or larger than originalSize.
|
||||
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
|
||||
or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */
|
||||
ZSTDLIB_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
|
||||
const void* src, size_t compressedSize);
|
||||
typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx;
|
||||
ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void);
|
||||
ZSTDLIB_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx);
|
||||
|
||||
ZSTDLIB_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
|
||||
/* *************************************
|
||||
* Helper functions
|
||||
***************************************/
|
||||
ZSTDLIB_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */
|
||||
|
||||
/* Error Management */
|
||||
ZSTDLIB_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
|
||||
ZSTDLIB_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Explicit memory management
|
||||
***************************************/
|
||||
/** Decompression context */
|
||||
typedef struct ZSTDv06_DCtx_s ZSTDv06_DCtx;
|
||||
ZSTDLIB_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void);
|
||||
ZSTDLIB_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */
|
||||
|
||||
/** ZSTDv06_decompressDCtx() :
|
||||
* Same as ZSTDv06_decompress(), but requires an already allocated ZSTDv06_DCtx (see ZSTDv06_createDCtx()) */
|
||||
ZSTDLIB_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/*-***********************
|
||||
* Dictionary API
|
||||
*************************/
|
||||
/*! ZSTDv06_decompress_usingDict() :
|
||||
* Decompression using a pre-defined Dictionary content (see dictBuilder).
|
||||
* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.
|
||||
* Note : dict can be NULL, in which case, it's equivalent to ZSTDv06_decompressDCtx() */
|
||||
ZSTDLIB_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize);
|
||||
|
||||
|
||||
/*-************************
|
||||
* Advanced Streaming API
|
||||
***************************/
|
||||
|
||||
typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams;
|
||||
|
||||
ZSTDLIB_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
|
||||
ZSTDLIB_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx);
|
||||
|
||||
ZSTDLIB_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
|
||||
/* *************************************
|
||||
* ZBUFF API
|
||||
***************************************/
|
||||
|
||||
typedef struct ZBUFFv06_DCtx_s ZBUFFv06_DCtx;
|
||||
ZSTDLIB_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void);
|
||||
ZSTDLIB_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx);
|
||||
|
||||
ZSTDLIB_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
|
||||
ZSTDLIB_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx,
|
||||
ZSTDLIB_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr);
|
||||
|
||||
/*-***************************************************************************
|
||||
* Streaming decompression howto
|
||||
*
|
||||
* A ZBUFFv06_DCtx object is required to track streaming operations.
|
||||
* Use ZBUFFv06_createDCtx() and ZBUFFv06_freeDCtx() to create/release resources.
|
||||
* Use ZBUFFv06_decompressInit() to start a new decompression operation,
|
||||
* or ZBUFFv06_decompressInitDictionary() if decompression requires a dictionary.
|
||||
* Note that ZBUFFv06_DCtx objects can be re-init multiple times.
|
||||
* A ZBUFFv07_DCtx object is required to track streaming operations.
|
||||
* Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources.
|
||||
* Use ZBUFFv07_decompressInit() to start a new decompression operation,
|
||||
* or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary.
|
||||
* Note that ZBUFFv07_DCtx objects can be re-init multiple times.
|
||||
*
|
||||
* Use ZBUFFv06_decompressContinue() repetitively to consume your input.
|
||||
* Use ZBUFFv07_decompressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
|
||||
* or 0 when a frame is completely decoded,
|
||||
* or an error code, which can be tested using ZBUFFv06_isError().
|
||||
* or an error code, which can be tested using ZBUFFv07_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFFv06_recommendedDInSize() and ZBUFFv06_recommendedDOutSize()
|
||||
* output : ZBUFFv06_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFFv06_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFFv06_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize()
|
||||
* output : ZBUFFv07_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFFv07_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* *******************************************************************************/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Tool functions
|
||||
***************************************/
|
||||
ZSTDLIB_API unsigned ZBUFFv06_isError(size_t errorCode);
|
||||
ZSTDLIB_API const char* ZBUFFv06_getErrorName(size_t errorCode);
|
||||
ZSTDLIB_API unsigned ZBUFFv07_isError(size_t errorCode);
|
||||
ZSTDLIB_API const char* ZBUFFv07_getErrorName(size_t errorCode);
|
||||
|
||||
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
|
||||
* These sizes are just hints, they tend to offer better latency */
|
||||
ZSTDLIB_API size_t ZBUFFv06_recommendedDInSize(void);
|
||||
ZSTDLIB_API size_t ZBUFFv06_recommendedDOutSize(void);
|
||||
ZSTDLIB_API size_t ZBUFFv07_recommendedDInSize(void);
|
||||
ZSTDLIB_API size_t ZBUFFv07_recommendedDOutSize(void);
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define ZSTDv06_MAGICNUMBER 0xFD2FB526 /* v0.6 */
|
||||
|
||||
#define ZSTDv07_MAGICNUMBER 0xFD2FB527 /* v0.7 */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTDv06_BUFFERED_H */
|
||||
#endif /* ZSTDv07_H_235446 */
|
||||
@@ -0,0 +1,4921 @@
|
||||
/* ******************************************************************
|
||||
zstd_v08.c
|
||||
Decompression module for ZSTD v0.8 legacy format
|
||||
Copyright (C) 2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Homepage : http://www.zstd.net/
|
||||
****************************************************************** */
|
||||
|
||||
/*- Dependencies -*/
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include <string.h> /* memcpy */
|
||||
#include <stdlib.h> /* malloc, free, qsort */
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
#include "zstd_v08.h"
|
||||
|
||||
#define FSEv08_STATIC_LINKING_ONLY /* FSEv08_MIN_TABLELOG */
|
||||
#define HUFv08_STATIC_LINKING_ONLY /* HUFv08_TABLELOG_ABSOLUTEMAX */
|
||||
#define ZSTDv08_STATIC_LINKING_ONLY
|
||||
|
||||
|
||||
|
||||
#ifdef ZSTDv08_STATIC_LINKING_ONLY
|
||||
|
||||
/* ====================================================================================
|
||||
* The definitions in this section are considered experimental.
|
||||
* They should never be used with a dynamic library, as they may change in the future.
|
||||
* They are provided for advanced usages.
|
||||
* Use them only in association with static linking.
|
||||
* ==================================================================================== */
|
||||
|
||||
/*--- Constants ---*/
|
||||
#define ZSTDv08_MAGIC_SKIPPABLE_START 0x184D2A50U
|
||||
|
||||
#define ZSTDv08_WINDOWLOG_MAX_32 25
|
||||
#define ZSTDv08_WINDOWLOG_MAX_64 27
|
||||
#define ZSTDv08_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTDv08_WINDOWLOG_MAX_32 : ZSTDv08_WINDOWLOG_MAX_64))
|
||||
#define ZSTDv08_WINDOWLOG_MIN 18
|
||||
#define ZSTDv08_CHAINLOG_MAX (ZSTDv08_WINDOWLOG_MAX+1)
|
||||
#define ZSTDv08_CHAINLOG_MIN 4
|
||||
#define ZSTDv08_HASHLOG_MAX ZSTDv08_WINDOWLOG_MAX
|
||||
#define ZSTDv08_HASHLOG_MIN 12
|
||||
#define ZSTDv08_HASHLOG3_MAX 17
|
||||
#define ZSTDv08_SEARCHLOG_MAX (ZSTDv08_WINDOWLOG_MAX-1)
|
||||
#define ZSTDv08_SEARCHLOG_MIN 1
|
||||
#define ZSTDv08_SEARCHLENGTH_MAX 7
|
||||
#define ZSTDv08_SEARCHLENGTH_MIN 3
|
||||
#define ZSTDv08_TARGETLENGTH_MIN 4
|
||||
#define ZSTDv08_TARGETLENGTH_MAX 999
|
||||
|
||||
#define ZSTDv08_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
|
||||
static const size_t ZSTDv08_frameHeaderSize_min = 5;
|
||||
static const size_t ZSTDv08_frameHeaderSize_max = ZSTDv08_FRAMEHEADERSIZE_MAX;
|
||||
static const size_t ZSTDv08_skippableHeaderSize = 8; /* magic number + skippable frame length */
|
||||
|
||||
|
||||
/* custom memory allocation functions */
|
||||
typedef void* (*ZSTDv08_allocFunction) (void* opaque, size_t size);
|
||||
typedef void (*ZSTDv08_freeFunction) (void* opaque, void* address);
|
||||
typedef struct { ZSTDv08_allocFunction customAlloc; ZSTDv08_freeFunction customFree; void* opaque; } ZSTDv08_customMem;
|
||||
|
||||
|
||||
/*--- Advanced Decompression functions ---*/
|
||||
|
||||
/*! ZSTDv08_estimateDCtxSize() :
|
||||
* Gives the potential amount of memory allocated to create a ZSTDv08_DCtx */
|
||||
ZSTDLIB_API size_t ZSTDv08_estimateDCtxSize(void);
|
||||
|
||||
/*! ZSTDv08_createDCtx_advanced() :
|
||||
* Create a ZSTD decompression context using external alloc and free functions */
|
||||
ZSTDLIB_API ZSTDv08_DCtx* ZSTDv08_createDCtx_advanced(ZSTDv08_customMem customMem);
|
||||
|
||||
/*! ZSTDv08_sizeofDCtx() :
|
||||
* Gives the amount of memory used by a given ZSTDv08_DCtx */
|
||||
ZSTDLIB_API size_t ZSTDv08_sizeofDCtx(const ZSTDv08_DCtx* dctx);
|
||||
|
||||
|
||||
/* ******************************************************************
|
||||
* Buffer-less streaming functions (synchronous mode)
|
||||
********************************************************************/
|
||||
/* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
|
||||
* But it's also a complex one, with a lot of restrictions (documented below).
|
||||
* For an easier streaming API, look into common/zbuff.h
|
||||
* which removes all restrictions by allocating and managing its own internal buffer */
|
||||
|
||||
ZSTDLIB_API size_t ZSTDv08_decompressBegin(ZSTDv08_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTDv08_decompressBegin_usingDict(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API void ZSTDv08_copyDCtx(ZSTDv08_DCtx* dctx, const ZSTDv08_DCtx* preparedDCtx);
|
||||
|
||||
ZSTDLIB_API size_t ZSTDv08_nextSrcSizeToDecompress(ZSTDv08_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZSTDv08_decompressContinue(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTDv08_nextInputType_e;
|
||||
ZSTDLIB_API ZSTDv08_nextInputType_e ZSTDv08_nextInputType(ZSTDv08_DCtx* dctx);
|
||||
|
||||
/*
|
||||
Buffer-less streaming decompression (synchronous mode)
|
||||
|
||||
A ZSTDv08_DCtx object is required to track streaming operations.
|
||||
Use ZSTDv08_createDCtx() / ZSTDv08_freeDCtx() to manage it.
|
||||
A ZSTDv08_DCtx object can be re-used multiple times.
|
||||
|
||||
First typical operation is to retrieve frame parameters, using ZSTDv08_getFrameParams().
|
||||
It fills a ZSTDv08_frameParams structure which provide important information to correctly decode the frame,
|
||||
such as the minimum rolling buffer size to allocate to decompress data (`windowSize`),
|
||||
and the dictionary ID used.
|
||||
(Note : content size is optional, it may not be present. 0 means : content size unknown).
|
||||
Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information.
|
||||
As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation.
|
||||
Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB.
|
||||
Frame parameters are extracted from the beginning of the compressed frame.
|
||||
Data fragment must be large enough to ensure successful decoding, typically `ZSTDv08_frameHeaderSize_max` bytes.
|
||||
@result : 0 : successful decoding, the `ZSTDv08_frameParams` structure is correctly filled.
|
||||
>0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
|
||||
errorCode, which can be tested using ZSTDv08_isError().
|
||||
|
||||
Start decompression, with ZSTDv08_decompressBegin() or ZSTDv08_decompressBegin_usingDict().
|
||||
Alternatively, you can copy a prepared context, using ZSTDv08_copyDCtx().
|
||||
|
||||
Then use ZSTDv08_nextSrcSizeToDecompress() and ZSTDv08_decompressContinue() alternatively.
|
||||
ZSTDv08_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTDv08_decompressContinue().
|
||||
ZSTDv08_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
|
||||
|
||||
@result of ZSTDv08_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
|
||||
It can be zero, which is not an error; it just means ZSTDv08_decompressContinue() has decoded some metadata item.
|
||||
It can also be an error code, which can be tested with ZSTDv08_isError().
|
||||
|
||||
ZSTDv08_decompressContinue() needs previous data blocks during decompression, up to `windowSize`.
|
||||
They should preferably be located contiguously, prior to current block.
|
||||
Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters.
|
||||
ZSTDv08_decompressContinue() is very sensitive to contiguity,
|
||||
if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
|
||||
or that previous contiguous segment is large enough to properly handle maximum back-reference.
|
||||
|
||||
A frame is fully decoded when ZSTDv08_nextSrcSizeToDecompress() returns zero.
|
||||
Context can then be reset to start a new decompression.
|
||||
|
||||
Note : it's possible to know if next input to present is a header or a block, using ZSTDv08_nextInputType().
|
||||
This information is not required to properly decode a frame.
|
||||
|
||||
== Special case : skippable frames ==
|
||||
|
||||
Skippable frames allow integration of user-defined data into a flow of concatenated frames.
|
||||
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
|
||||
a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
|
||||
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
|
||||
c) Frame Content - any content (User Data) of length equal to Frame Size
|
||||
For skippable frames ZSTDv08_decompressContinue() always returns 0.
|
||||
For skippable frames ZSTDv08_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
|
||||
It also returns Frame Size as fparamsPtr->frameContentSize.
|
||||
*/
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Block functions
|
||||
****************************************/
|
||||
/*! Block functions produce and decode raw zstd blocks, without frame metadata.
|
||||
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
|
||||
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
|
||||
|
||||
A few rules to respect :
|
||||
- Compressing and decompressing require a context structure
|
||||
+ Use ZSTDv08_createCCtx() and ZSTDv08_createDCtx()
|
||||
- It is necessary to init context before starting
|
||||
+ compression : ZSTDv08_compressBegin()
|
||||
+ decompression : ZSTDv08_decompressBegin()
|
||||
+ variants _usingDict() are also allowed
|
||||
+ copyCCtx() and copyDCtx() work too
|
||||
- Block size is limited, it must be <= ZSTDv08_getBlockSizeMax()
|
||||
+ If you need to compress more, cut data into multiple blocks
|
||||
+ Consider using the regular ZSTDv08_compress() instead, as frame metadata costs become negligible when source size is large.
|
||||
- When a block is considered not compressible enough, ZSTDv08_compressBlock() result will be zero.
|
||||
In which case, nothing is produced into `dst`.
|
||||
+ User must test for such outcome and deal directly with uncompressed data
|
||||
+ ZSTDv08_decompressBlock() doesn't accept uncompressed data as input !!!
|
||||
+ In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history.
|
||||
Use ZSTDv08_insertBlock() in such a case.
|
||||
*/
|
||||
|
||||
#define ZSTDv08_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
|
||||
ZSTDLIB_API size_t ZSTDv08_decompressBlock(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
ZSTDLIB_API size_t ZSTDv08_insertBlock(ZSTDv08_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
|
||||
|
||||
|
||||
#endif /* ZSTDv08_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
/* ====================================================================================
|
||||
* The definitions in this section are considered experimental.
|
||||
* They should never be used in association with a dynamic library, as they may change in the future.
|
||||
* They are provided for advanced usages.
|
||||
* Use them only in association with static linking.
|
||||
* ==================================================================================== */
|
||||
|
||||
/*! ZBUFFv08_createDCtx_advanced() :
|
||||
* Create a ZBUFF decompression context using external alloc and free functions */
|
||||
ZSTDLIB_API ZBUFFv08_DCtx* ZBUFFv08_createDCtx_advanced(ZSTDv08_customMem customMem);
|
||||
|
||||
|
||||
|
||||
/* ******************************************************************
|
||||
mem.h
|
||||
low-level memory access routines
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
#ifndef MEM_H_MODULE
|
||||
#define MEM_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Compiler specifics
|
||||
******************************************/
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
# include <stdlib.h> /* _byteswap_ulong */
|
||||
# include <intrin.h> /* _byteswap_* */
|
||||
#endif
|
||||
#if defined(__GNUC__)
|
||||
# define MEM_STATIC static __inline __attribute__((unused))
|
||||
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define MEM_STATIC static inline
|
||||
#elif defined(_MSC_VER)
|
||||
# define MEM_STATIC static __inline
|
||||
#else
|
||||
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
/* code only tested on 32 and 64 bits systems */
|
||||
#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
|
||||
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Basic Types
|
||||
*****************************************************************/
|
||||
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef int16_t S16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
typedef int64_t S64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef signed short S16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
typedef signed long long S64;
|
||||
#endif
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Memory I/O
|
||||
*****************************************************************/
|
||||
/* MEM_FORCE_MEMORY_ACCESS :
|
||||
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
||||
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
||||
* The below switch allow to select different access method for improved performance.
|
||||
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
||||
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
||||
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
||||
* Method 2 : direct access. This method is portable but violate C standard.
|
||||
* It can generate buggy code on targets depending on alignment.
|
||||
* In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
||||
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
||||
* Prefer these methods in priority order (0 > 1 > 2)
|
||||
*/
|
||||
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
||||
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
||||
# define MEM_FORCE_MEMORY_ACCESS 2
|
||||
# elif defined(__INTEL_COMPILER) || \
|
||||
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
||||
# define MEM_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
|
||||
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
|
||||
|
||||
MEM_STATIC unsigned MEM_isLittleEndian(void)
|
||||
{
|
||||
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
|
||||
return one.c[0];
|
||||
}
|
||||
|
||||
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
||||
|
||||
/* violates C standard, by lying on structure alignment.
|
||||
Only use if no other choice to achieve best performance on target platform */
|
||||
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
||||
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
||||
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
||||
MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
|
||||
|
||||
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
||||
|
||||
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
||||
/* currently only defined for gcc and icc */
|
||||
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
|
||||
|
||||
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
|
||||
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
||||
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
||||
MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
|
||||
|
||||
#else
|
||||
|
||||
/* default method, safe and standard.
|
||||
can sometimes prove slower */
|
||||
|
||||
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
||||
{
|
||||
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_read32(const void* memPtr)
|
||||
{
|
||||
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_read64(const void* memPtr)
|
||||
{
|
||||
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_readST(const void* memPtr)
|
||||
{
|
||||
size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
||||
{
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
|
||||
{
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
|
||||
{
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
#endif /* MEM_FORCE_MEMORY_ACCESS */
|
||||
|
||||
MEM_STATIC U32 MEM_swap32(U32 in)
|
||||
{
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
return _byteswap_ulong(in);
|
||||
#elif defined (__GNUC__)
|
||||
return __builtin_bswap32(in);
|
||||
#else
|
||||
return ((in << 24) & 0xff000000 ) |
|
||||
((in << 8) & 0x00ff0000 ) |
|
||||
((in >> 8) & 0x0000ff00 ) |
|
||||
((in >> 24) & 0x000000ff );
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_swap64(U64 in)
|
||||
{
|
||||
#if defined(_MSC_VER) /* Visual Studio */
|
||||
return _byteswap_uint64(in);
|
||||
#elif defined (__GNUC__)
|
||||
return __builtin_bswap64(in);
|
||||
#else
|
||||
return ((in << 56) & 0xff00000000000000ULL) |
|
||||
((in << 40) & 0x00ff000000000000ULL) |
|
||||
((in << 24) & 0x0000ff0000000000ULL) |
|
||||
((in << 8) & 0x000000ff00000000ULL) |
|
||||
((in >> 8) & 0x00000000ff000000ULL) |
|
||||
((in >> 24) & 0x0000000000ff0000ULL) |
|
||||
((in >> 40) & 0x000000000000ff00ULL) |
|
||||
((in >> 56) & 0x00000000000000ffULL);
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_swapST(size_t in)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
return (size_t)MEM_swap32((U32)in);
|
||||
else
|
||||
return (size_t)MEM_swap64((U64)in);
|
||||
}
|
||||
|
||||
/*=== Little endian r/w ===*/
|
||||
|
||||
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_read16(memPtr);
|
||||
else {
|
||||
const BYTE* p = (const BYTE*)memPtr;
|
||||
return (U16)(p[0] + (p[1]<<8));
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
|
||||
{
|
||||
if (MEM_isLittleEndian()) {
|
||||
MEM_write16(memPtr, val);
|
||||
} else {
|
||||
BYTE* p = (BYTE*)memPtr;
|
||||
p[0] = (BYTE)val;
|
||||
p[1] = (BYTE)(val>>8);
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
|
||||
{
|
||||
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
|
||||
{
|
||||
MEM_writeLE16(memPtr, (U16)val);
|
||||
((BYTE*)memPtr)[2] = (BYTE)(val>>16);
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_read32(memPtr);
|
||||
else
|
||||
return MEM_swap32(MEM_read32(memPtr));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write32(memPtr, val32);
|
||||
else
|
||||
MEM_write32(memPtr, MEM_swap32(val32));
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_read64(memPtr);
|
||||
else
|
||||
return MEM_swap64(MEM_read64(memPtr));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write64(memPtr, val64);
|
||||
else
|
||||
MEM_write64(memPtr, MEM_swap64(val64));
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
return (size_t)MEM_readLE32(memPtr);
|
||||
else
|
||||
return (size_t)MEM_readLE64(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
MEM_writeLE32(memPtr, (U32)val);
|
||||
else
|
||||
MEM_writeLE64(memPtr, (U64)val);
|
||||
}
|
||||
|
||||
/*=== Big endian r/w ===*/
|
||||
|
||||
MEM_STATIC U32 MEM_readBE32(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_swap32(MEM_read32(memPtr));
|
||||
else
|
||||
return MEM_read32(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write32(memPtr, MEM_swap32(val32));
|
||||
else
|
||||
MEM_write32(memPtr, val32);
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_readBE64(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_swap64(MEM_read64(memPtr));
|
||||
else
|
||||
return MEM_read64(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
MEM_write64(memPtr, MEM_swap64(val64));
|
||||
else
|
||||
MEM_write64(memPtr, val64);
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_readBEST(const void* memPtr)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
return (size_t)MEM_readBE32(memPtr);
|
||||
else
|
||||
return (size_t)MEM_readBE64(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
MEM_writeBE32(memPtr, (U32)val);
|
||||
else
|
||||
MEM_writeBE64(memPtr, (U64)val);
|
||||
}
|
||||
|
||||
|
||||
/* function safe only for comparisons */
|
||||
MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
|
||||
{
|
||||
switch (length)
|
||||
{
|
||||
default :
|
||||
case 4 : return MEM_read32(memPtr);
|
||||
case 3 : if (MEM_isLittleEndian())
|
||||
return MEM_read32(memPtr)<<8;
|
||||
else
|
||||
return MEM_read32(memPtr)>>8;
|
||||
}
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MEM_H_MODULE */
|
||||
/* ******************************************************************
|
||||
Error codes list
|
||||
Copyright (C) 2016, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Homepage : http://www.zstd.net
|
||||
****************************************************************** */
|
||||
#ifndef ERROR_PUBLIC_H_MODULE
|
||||
#define ERROR_PUBLIC_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* error codes list
|
||||
******************************************/
|
||||
typedef enum {
|
||||
ZSTDv08_error_no_error,
|
||||
ZSTDv08_error_GENERIC,
|
||||
ZSTDv08_error_prefix_unknown,
|
||||
ZSTDv08_error_frameParameter_unsupported,
|
||||
ZSTDv08_error_frameParameter_unsupportedBy32bits,
|
||||
ZSTDv08_error_compressionParameter_unsupported,
|
||||
ZSTDv08_error_init_missing,
|
||||
ZSTDv08_error_memory_allocation,
|
||||
ZSTDv08_error_stage_wrong,
|
||||
ZSTDv08_error_dstSize_tooSmall,
|
||||
ZSTDv08_error_srcSize_wrong,
|
||||
ZSTDv08_error_corruption_detected,
|
||||
ZSTDv08_error_checksum_wrong,
|
||||
ZSTDv08_error_tableLog_tooLarge,
|
||||
ZSTDv08_error_maxSymbolValue_tooLarge,
|
||||
ZSTDv08_error_maxSymbolValue_tooSmall,
|
||||
ZSTDv08_error_dictionary_corrupted,
|
||||
ZSTDv08_error_dictionary_wrong,
|
||||
ZSTDv08_error_maxCode
|
||||
} ZSTDv08_ErrorCode;
|
||||
|
||||
/*! ZSTDv08_getErrorCode() :
|
||||
convert a `size_t` function result into a `ZSTDv08_ErrorCode` enum type,
|
||||
which can be used to compare directly with enum list published into "error_public.h" */
|
||||
ZSTDv08_ErrorCode ZSTDv08_getErrorCode(size_t functionResult);
|
||||
const char* ZSTDv08_getErrorString(ZSTDv08_ErrorCode code);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ERROR_PUBLIC_H_MODULE */
|
||||
/* ******************************************************************
|
||||
Error codes and messages
|
||||
Copyright (C) 2013-2016, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Homepage : http://www.zstd.net
|
||||
****************************************************************** */
|
||||
/* Note : this module is expected to remain private, do not expose it */
|
||||
|
||||
#ifndef ERROR_H_MODULE
|
||||
#define ERROR_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Compiler-specific
|
||||
******************************************/
|
||||
#if defined(__GNUC__)
|
||||
# define ERR_STATIC static __attribute__((unused))
|
||||
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define ERR_STATIC static inline
|
||||
#elif defined(_MSC_VER)
|
||||
# define ERR_STATIC static __inline
|
||||
#else
|
||||
# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Customization (error_public.h)
|
||||
******************************************/
|
||||
typedef ZSTDv08_ErrorCode ERR_enum;
|
||||
#define PREFIX(name) ZSTDv08_error_##name
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Error codes handling
|
||||
******************************************/
|
||||
#ifdef ERROR
|
||||
# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
|
||||
#endif
|
||||
#define ERROR(name) ((size_t)-PREFIX(name))
|
||||
|
||||
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
|
||||
|
||||
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* Error Strings
|
||||
******************************************/
|
||||
|
||||
ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
|
||||
{
|
||||
static const char* notErrorCode = "Unspecified error code";
|
||||
switch( code )
|
||||
{
|
||||
case PREFIX(no_error): return "No error detected";
|
||||
case PREFIX(GENERIC): return "Error (generic)";
|
||||
case PREFIX(prefix_unknown): return "Unknown frame descriptor";
|
||||
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
|
||||
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
|
||||
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
|
||||
case PREFIX(init_missing): return "Context should be init first";
|
||||
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
|
||||
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
|
||||
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
|
||||
case PREFIX(srcSize_wrong): return "Src size incorrect";
|
||||
case PREFIX(corruption_detected): return "Corrupted block detected";
|
||||
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
|
||||
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
|
||||
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
|
||||
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
|
||||
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
|
||||
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
|
||||
case PREFIX(maxCode):
|
||||
default: return notErrorCode;
|
||||
}
|
||||
}
|
||||
|
||||
ERR_STATIC const char* ERR_getErrorName(size_t code)
|
||||
{
|
||||
return ERR_getErrorString(ERR_getErrorCode(code));
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ERROR_H_MODULE */
|
||||
/* ******************************************************************
|
||||
bitstream
|
||||
Part of FSE library
|
||||
header file (to include)
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef BITSTREAM_H_MODULE
|
||||
#define BITSTREAM_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/*
|
||||
* This API consists of small unitary functions, which must be inlined for best performance.
|
||||
* Since link-time-optimization is not available for all compilers,
|
||||
* these functions are defined into a .h to be included.
|
||||
*/
|
||||
|
||||
|
||||
/*=========================================
|
||||
* Target specific
|
||||
=========================================*/
|
||||
#if defined(__BMI__) && defined(__GNUC__)
|
||||
# include <immintrin.h> /* support for bextr (experimental) */
|
||||
#endif
|
||||
|
||||
|
||||
/*-********************************************
|
||||
* bitStream decoding API (read backward)
|
||||
**********************************************/
|
||||
typedef struct
|
||||
{
|
||||
size_t bitContainer;
|
||||
unsigned bitsConsumed;
|
||||
const char* ptr;
|
||||
const char* start;
|
||||
} BIT_DStream_t;
|
||||
|
||||
typedef enum { BIT_DStream_unfinished = 0,
|
||||
BIT_DStream_endOfBuffer = 1,
|
||||
BIT_DStream_completed = 2,
|
||||
BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */
|
||||
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
|
||||
|
||||
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
||||
|
||||
|
||||
/* Start by invoking BIT_initDStream().
|
||||
* A chunk of the bitStream is then stored into a local register.
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
|
||||
*/
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* unsafe API
|
||||
******************************************/
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
/* faster, but works only if nbBits >= 1 */
|
||||
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* Internal functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BIT_highbit32 (register U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
_BitScanReverse ( &r, val );
|
||||
return (unsigned) r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
||||
return 31 - __builtin_clz (val);
|
||||
# else /* Software version */
|
||||
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
|
||||
# endif
|
||||
}
|
||||
|
||||
/*===== Local Constants =====*/
|
||||
static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
|
||||
|
||||
|
||||
/*-********************************************************
|
||||
* bitStream decoding
|
||||
**********************************************************/
|
||||
/*! BIT_initDStream() :
|
||||
* Initialize a BIT_DStream_t.
|
||||
* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
|
||||
* `srcSize` must be the *exact* size of the bitStream, in bytes.
|
||||
* @return : size of stream (== srcSize) or an errorCode if a problem is detected
|
||||
*/
|
||||
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
|
||||
|
||||
if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
|
||||
} else {
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
bitD->ptr = bitD->start;
|
||||
bitD->bitContainer = *(const BYTE*)(bitD->start);
|
||||
switch(srcSize)
|
||||
{
|
||||
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
|
||||
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
|
||||
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
|
||||
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
|
||||
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
|
||||
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
|
||||
default:;
|
||||
}
|
||||
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
|
||||
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
|
||||
bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
|
||||
}
|
||||
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
|
||||
{
|
||||
return bitContainer >> start;
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
|
||||
{
|
||||
#if defined(__BMI__) && defined(__GNUC__) /* experimental */
|
||||
# if defined(__x86_64__)
|
||||
if (sizeof(bitContainer)==8)
|
||||
return _bextr_u64(bitContainer, start, nbBits);
|
||||
else
|
||||
# endif
|
||||
return _bextr_u32(bitContainer, start, nbBits);
|
||||
#else
|
||||
return (bitContainer >> start) & BIT_mask[nbBits];
|
||||
#endif
|
||||
}
|
||||
|
||||
MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
|
||||
{
|
||||
return bitContainer & BIT_mask[nbBits];
|
||||
}
|
||||
|
||||
/*! BIT_lookBits() :
|
||||
* Provides next n bits from local register.
|
||||
* local register is not modified.
|
||||
* On 32-bits, maxNbBits==24.
|
||||
* On 64-bits, maxNbBits==56.
|
||||
* @return : value extracted
|
||||
*/
|
||||
MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
|
||||
return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
|
||||
#else
|
||||
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
|
||||
#endif
|
||||
}
|
||||
|
||||
/*! BIT_lookBitsFast() :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
|
||||
}
|
||||
|
||||
MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
|
||||
/*! BIT_readBits() :
|
||||
* Read (consume) next n bits from local register and update.
|
||||
* Pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value.
|
||||
*/
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t const value = BIT_lookBits(bitD, nbBits);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
|
||||
/*! BIT_readBitsFast() :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t const value = BIT_lookBitsFast(bitD, nbBits);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
|
||||
/*! BIT_reloadDStream() :
|
||||
* Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ).
|
||||
* This function is safe, it guarantees it will not read beyond src buffer.
|
||||
* @return : status of `BIT_DStream_t` internal register.
|
||||
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
||||
{
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
|
||||
return BIT_DStream_overflow;
|
||||
|
||||
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
|
||||
bitD->ptr -= bitD->bitsConsumed >> 3;
|
||||
bitD->bitsConsumed &= 7;
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
return BIT_DStream_unfinished;
|
||||
}
|
||||
if (bitD->ptr == bitD->start) {
|
||||
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
|
||||
return BIT_DStream_completed;
|
||||
}
|
||||
{ U32 nbBytes = bitD->bitsConsumed >> 3;
|
||||
BIT_DStream_status result = BIT_DStream_unfinished;
|
||||
if (bitD->ptr - nbBytes < bitD->start) {
|
||||
nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
|
||||
result = BIT_DStream_endOfBuffer;
|
||||
}
|
||||
bitD->ptr -= nbBytes;
|
||||
bitD->bitsConsumed -= nbBytes*8;
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/*! BIT_endOfDStream() :
|
||||
* @return Tells if DStream has exactly reached its end (all bits consumed).
|
||||
*/
|
||||
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
||||
{
|
||||
return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* BITSTREAM_H_MODULE */
|
||||
/* ******************************************************************
|
||||
FSE : Finite State Entropy codec
|
||||
Public Prototypes declaration
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef FSEv08_H
|
||||
#define FSEv08_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* FSE simple functions
|
||||
******************************************/
|
||||
/*! FSEv08_decompress():
|
||||
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated destination buffer 'dst', of size 'dstCapacity'.
|
||||
@return : size of regenerated data (<= maxDstSize),
|
||||
or an error code, which can be tested using FSEv08_isError() .
|
||||
|
||||
** Important ** : FSEv08_decompress() does not decompress non-compressible nor RLE data !!!
|
||||
Why ? : making this distinction requires a header.
|
||||
Header management is intentionally delegated to the user layer, which can better manage special cases.
|
||||
*/
|
||||
size_t FSEv08_decompress(void* dst, size_t dstCapacity,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* Tool functions
|
||||
******************************************/
|
||||
/* Error Management */
|
||||
unsigned FSEv08_isError(size_t code); /* tells if a return value is an error code */
|
||||
const char* FSEv08_getErrorName(size_t code); /* provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
/*-*****************************************
|
||||
* FSE detailed API
|
||||
******************************************/
|
||||
/*!
|
||||
FSEv08_decompress() does the following:
|
||||
1. read normalized counters with readNCount()
|
||||
2. build decoding table 'DTable' from normalized counters
|
||||
3. decode the data stream using decoding table 'DTable'
|
||||
|
||||
The following API allows targeting specific sub-functions for advanced tasks.
|
||||
For example, it's possible to compress several blocks using the same 'CTable',
|
||||
or to save and provide normalized distribution using external method.
|
||||
*/
|
||||
|
||||
|
||||
/* *** DECOMPRESSION *** */
|
||||
|
||||
/*! FSEv08_readNCount():
|
||||
Read compactly saved 'normalizedCounter' from 'rBuffer'.
|
||||
@return : size read from 'rBuffer',
|
||||
or an errorCode, which can be tested using FSEv08_isError().
|
||||
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
|
||||
size_t FSEv08_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
|
||||
|
||||
/*! Constructor and Destructor of FSEv08_DTable.
|
||||
Note that its size depends on 'tableLog' */
|
||||
typedef unsigned FSEv08_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
|
||||
FSEv08_DTable* FSEv08_createDTable(unsigned tableLog);
|
||||
void FSEv08_freeDTable(FSEv08_DTable* dt);
|
||||
|
||||
/*! FSEv08_buildDTable():
|
||||
Builds 'dt', which must be already allocated, using FSEv08_createDTable().
|
||||
return : 0, or an errorCode, which can be tested using FSEv08_isError() */
|
||||
size_t FSEv08_buildDTable (FSEv08_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
/*! FSEv08_decompress_usingDTable():
|
||||
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
|
||||
into `dst` which must be already allocated.
|
||||
@return : size of regenerated data (necessarily <= `dstCapacity`),
|
||||
or an errorCode, which can be tested using FSEv08_isError() */
|
||||
size_t FSEv08_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSEv08_DTable* dt);
|
||||
|
||||
/*!
|
||||
Tutorial :
|
||||
----------
|
||||
(Note : these functions only decompress FSE-compressed blocks.
|
||||
If block is uncompressed, use memcpy() instead
|
||||
If block is a single repeated byte, use memset() instead )
|
||||
|
||||
The first step is to obtain the normalized frequencies of symbols.
|
||||
This can be performed by FSEv08_readNCount() if it was saved using FSEv08_writeNCount().
|
||||
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
|
||||
In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
|
||||
or size the table to handle worst case situations (typically 256).
|
||||
FSEv08_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
|
||||
The result of FSEv08_readNCount() is the number of bytes read from 'rBuffer'.
|
||||
Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
|
||||
If there is an error, the function will return an error code, which can be tested using FSEv08_isError().
|
||||
|
||||
The next step is to build the decompression tables 'FSEv08_DTable' from 'normalizedCounter'.
|
||||
This is performed by the function FSEv08_buildDTable().
|
||||
The space required by 'FSEv08_DTable' must be already allocated using FSEv08_createDTable().
|
||||
If there is an error, the function will return an error code, which can be tested using FSEv08_isError().
|
||||
|
||||
`FSEv08_DTable` can then be used to decompress `cSrc`, with FSEv08_decompress_usingDTable().
|
||||
`cSrcSize` must be strictly correct, otherwise decompression will fail.
|
||||
FSEv08_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
|
||||
If there is an error, the function will return an error code, which can be tested using FSEv08_isError(). (ex: dst buffer too small)
|
||||
*/
|
||||
|
||||
|
||||
#ifdef FSEv08_STATIC_LINKING_ONLY
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Static allocation
|
||||
*******************************************/
|
||||
/* FSE buffer bounds */
|
||||
#define FSEv08_NCOUNTBOUND 512
|
||||
#define FSEv08_BLOCKBOUND(size) (size + (size>>7))
|
||||
|
||||
/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
|
||||
#define FSEv08_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE advanced API
|
||||
*******************************************/
|
||||
size_t FSEv08_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
|
||||
/**< same as FSEv08_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */
|
||||
|
||||
unsigned FSEv08_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
|
||||
/**< same as FSEv08_optimalTableLog(), which used `minus==2` */
|
||||
|
||||
size_t FSEv08_buildDTable_raw (FSEv08_DTable* dt, unsigned nbBits);
|
||||
/**< build a fake FSEv08_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
|
||||
|
||||
size_t FSEv08_buildDTable_rle (FSEv08_DTable* dt, unsigned char symbolValue);
|
||||
/**< build a fake FSEv08_DTable, designed to always generate the same symbolValue */
|
||||
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE symbol decompression API
|
||||
*******************************************/
|
||||
typedef struct
|
||||
{
|
||||
size_t state;
|
||||
const void* table; /* precise table may vary, depending on U16 */
|
||||
} FSEv08_DState_t;
|
||||
|
||||
|
||||
static void FSEv08_initDState(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSEv08_DTable* dt);
|
||||
|
||||
static unsigned char FSEv08_decodeSymbol(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||
|
||||
static unsigned FSEv08_endOfDState(const FSEv08_DState_t* DStatePtr);
|
||||
|
||||
/**<
|
||||
Let's now decompose FSEv08_decompress_usingDTable() into its unitary components.
|
||||
You will decode FSE-encoded symbols from the bitStream,
|
||||
and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
BIT_DStream_t DStream; // Stream context
|
||||
FSEv08_DState_t DState; // State context. Multiple ones are possible
|
||||
FSEv08_DTable* DTablePtr; // Decoding table, provided by FSEv08_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
errorCode = FSEv08_initDState(&DState, &DStream, DTablePtr);
|
||||
|
||||
You can then decode your data, symbol after symbol.
|
||||
For information the maximum number of bits read by FSEv08_decodeSymbol() is 'tableLog'.
|
||||
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
|
||||
unsigned char symbol = FSEv08_decodeSymbol(&DState, &DStream);
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = BIT_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSEv08_reloadDStream(&DStream);
|
||||
|
||||
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BIT_DStream_unfinished : there is still some data left into the DStream.
|
||||
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
BIT_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSEv08_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* FSE unsafe API
|
||||
*******************************************/
|
||||
static unsigned char FSEv08_decodeSymbolFast(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
|
||||
|
||||
|
||||
/* *****************************************
|
||||
* Implementation of inlined functions
|
||||
*******************************************/
|
||||
/*<===== Decompression =====>*/
|
||||
|
||||
typedef struct {
|
||||
U16 tableLog;
|
||||
U16 fastMode;
|
||||
} FSEv08_DTableHeader; /* sizeof U32 */
|
||||
|
||||
typedef struct
|
||||
{
|
||||
unsigned short newState;
|
||||
unsigned char symbol;
|
||||
unsigned char nbBits;
|
||||
} FSEv08_decode_t; /* size == U32 */
|
||||
|
||||
MEM_STATIC void FSEv08_initDState(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSEv08_DTable* dt)
|
||||
{
|
||||
const void* ptr = dt;
|
||||
const FSEv08_DTableHeader* const DTableH = (const FSEv08_DTableHeader*)ptr;
|
||||
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
|
||||
BIT_reloadDStream(bitD);
|
||||
DStatePtr->table = dt + 1;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSEv08_peekSymbol(const FSEv08_DState_t* DStatePtr)
|
||||
{
|
||||
FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
return DInfo.symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC void FSEv08_updateState(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSEv08_decodeSymbol(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BIT_readBits(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
/*! FSEv08_decodeSymbolFast() :
|
||||
unsafe, only works if no symbol has a probability > 50% */
|
||||
MEM_STATIC BYTE FSEv08_decodeSymbolFast(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
U32 const nbBits = DInfo.nbBits;
|
||||
BYTE const symbol = DInfo.symbol;
|
||||
size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned FSEv08_endOfDState(const FSEv08_DState_t* DStatePtr)
|
||||
{
|
||||
return DStatePtr->state == 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifndef FSEv08_COMMONDEFS_ONLY
|
||||
|
||||
/* **************************************************************
|
||||
* Tuning parameters
|
||||
****************************************************************/
|
||||
/*!MEMORY_USAGE :
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||
* Increasing memory usage improves compression ratio
|
||||
* Reduced memory usage can improve speed, due to cache effect
|
||||
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
|
||||
#define FSEv08_MAX_MEMORY_USAGE 14
|
||||
#define FSEv08_DEFAULT_MEMORY_USAGE 13
|
||||
|
||||
/*!FSEv08_MAX_SYMBOL_VALUE :
|
||||
* Maximum symbol value authorized.
|
||||
* Required for proper stack allocation */
|
||||
#define FSEv08_MAX_SYMBOL_VALUE 255
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* template functions type & suffix
|
||||
****************************************************************/
|
||||
#define FSEv08_FUNCTION_TYPE BYTE
|
||||
#define FSEv08_FUNCTION_EXTENSION
|
||||
#define FSEv08_DECODE_TYPE FSEv08_decode_t
|
||||
|
||||
|
||||
#endif /* !FSEv08_COMMONDEFS_ONLY */
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
* Constants
|
||||
*****************************************************************/
|
||||
#define FSEv08_MAX_TABLELOG (FSEv08_MAX_MEMORY_USAGE-2)
|
||||
#define FSEv08_MAX_TABLESIZE (1U<<FSEv08_MAX_TABLELOG)
|
||||
#define FSEv08_MAXTABLESIZE_MASK (FSEv08_MAX_TABLESIZE-1)
|
||||
#define FSEv08_DEFAULT_TABLELOG (FSEv08_DEFAULT_MEMORY_USAGE-2)
|
||||
#define FSEv08_MIN_TABLELOG 5
|
||||
|
||||
#define FSEv08_TABLELOG_ABSOLUTE_MAX 15
|
||||
#if FSEv08_MAX_TABLELOG > FSEv08_TABLELOG_ABSOLUTE_MAX
|
||||
# error "FSEv08_MAX_TABLELOG > FSEv08_TABLELOG_ABSOLUTE_MAX is not supported"
|
||||
#endif
|
||||
|
||||
#define FSEv08_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
|
||||
|
||||
|
||||
#endif /* FSEv08_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* FSEv08_H */
|
||||
/* ******************************************************************
|
||||
Huffman coder, part of New Generation Entropy library
|
||||
header file
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
****************************************************************** */
|
||||
#ifndef HUFv08_H_298734234
|
||||
#define HUFv08_H_298734234
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/* *** simple functions *** */
|
||||
/**
|
||||
HUFv08_decompress() :
|
||||
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated buffer 'dst', of minimum size 'dstSize'.
|
||||
`dstSize` : **must** be the ***exact*** size of original (uncompressed) data.
|
||||
Note : in contrast with FSE, HUFv08_decompress can regenerate
|
||||
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
|
||||
because it knows size to regenerate.
|
||||
@return : size of regenerated data (== dstSize),
|
||||
or an error code, which can be tested using HUFv08_isError()
|
||||
*/
|
||||
size_t HUFv08_decompress(void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Tool functions
|
||||
******************************************/
|
||||
#define HUFv08_BLOCKSIZE_MAX (128 * 1024)
|
||||
|
||||
/* Error Management */
|
||||
unsigned HUFv08_isError(size_t code); /**< tells if a return value is an error code */
|
||||
const char* HUFv08_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
/* *** Advanced function *** */
|
||||
|
||||
|
||||
#ifdef HUFv08_STATIC_LINKING_ONLY
|
||||
|
||||
|
||||
/* *** Constants *** */
|
||||
#define HUFv08_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUFv08_MAX_TABLELOG. Beyond that value, code does not work */
|
||||
#define HUFv08_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUFv08_ABSOLUTEMAX_TABLELOG */
|
||||
#define HUFv08_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
|
||||
#define HUFv08_SYMBOLVALUE_MAX 255
|
||||
#if (HUFv08_TABLELOG_MAX > HUFv08_TABLELOG_ABSOLUTEMAX)
|
||||
# error "HUFv08_TABLELOG_MAX is too large !"
|
||||
#endif
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Static allocation
|
||||
******************************************/
|
||||
/* HUF buffer bounds */
|
||||
#define HUFv08_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
|
||||
|
||||
/* static allocation of HUF's DTable */
|
||||
typedef U32 HUFv08_DTable;
|
||||
#define HUFv08_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
|
||||
#define HUFv08_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
||||
HUFv08_DTable DTable[HUFv08_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) }
|
||||
#define HUFv08_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
|
||||
HUFv08_DTable DTable[HUFv08_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) }
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* Advanced decompression functions
|
||||
******************************************/
|
||||
size_t HUFv08_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUFv08_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
|
||||
size_t HUFv08_decompress4X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
|
||||
size_t HUFv08_decompress4X_hufOnly(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
|
||||
size_t HUFv08_decompress4X2_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUFv08_decompress4X4_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
|
||||
size_t HUFv08_decompress1X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
||||
size_t HUFv08_decompress1X2_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
|
||||
size_t HUFv08_decompress1X4_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
|
||||
|
||||
|
||||
/* ****************************************
|
||||
* HUF detailed API
|
||||
******************************************/
|
||||
/*
|
||||
HUFv08_decompress() does the following:
|
||||
1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
|
||||
2. build Huffman table from save, using HUFv08_readDTableXn()
|
||||
3. decode 1 or 4 segments in parallel using HUFv08_decompressSXn_usingDTable
|
||||
*/
|
||||
|
||||
/** HUFv08_selectDecoder() :
|
||||
* Tells which decoder is likely to decode faster,
|
||||
* based on a set of pre-determined metrics.
|
||||
* @return : 0==HUFv08_decompress4X2, 1==HUFv08_decompress4X4 .
|
||||
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
|
||||
U32 HUFv08_selectDecoder (size_t dstSize, size_t cSrcSize);
|
||||
|
||||
size_t HUFv08_readDTableX2 (HUFv08_DTable* DTable, const void* src, size_t srcSize);
|
||||
size_t HUFv08_readDTableX4 (HUFv08_DTable* DTable, const void* src, size_t srcSize);
|
||||
|
||||
size_t HUFv08_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable);
|
||||
size_t HUFv08_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable);
|
||||
size_t HUFv08_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable);
|
||||
|
||||
|
||||
/* single stream variants */
|
||||
|
||||
size_t HUFv08_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUFv08_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
|
||||
|
||||
size_t HUFv08_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable);
|
||||
size_t HUFv08_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable);
|
||||
size_t HUFv08_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable);
|
||||
|
||||
|
||||
#endif /* HUFv08_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* HUFv08_H_298734234 */
|
||||
/*
|
||||
Common functions of New Generation Entropy library
|
||||
Copyright (C) 2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*************************************************************************** */
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* FSE Error Management
|
||||
******************************************/
|
||||
unsigned FSEv08_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
const char* FSEv08_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* HUF Error Management
|
||||
****************************************************************/
|
||||
unsigned HUFv08_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
const char* HUFv08_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/*-**************************************************************
|
||||
* FSE NCount encoding-decoding
|
||||
****************************************************************/
|
||||
static short FSEv08_abs(short a) { return (short)(a<0 ? -a : a); }
|
||||
|
||||
size_t FSEv08_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
|
||||
const void* headerBuffer, size_t hbSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) headerBuffer;
|
||||
const BYTE* const iend = istart + hbSize;
|
||||
const BYTE* ip = istart;
|
||||
int nbBits;
|
||||
int remaining;
|
||||
int threshold;
|
||||
U32 bitStream;
|
||||
int bitCount;
|
||||
unsigned charnum = 0;
|
||||
int previous0 = 0;
|
||||
|
||||
if (hbSize < 4) return ERROR(srcSize_wrong);
|
||||
bitStream = MEM_readLE32(ip);
|
||||
nbBits = (bitStream & 0xF) + FSEv08_MIN_TABLELOG; /* extract tableLog */
|
||||
if (nbBits > FSEv08_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
|
||||
bitStream >>= 4;
|
||||
bitCount = 4;
|
||||
*tableLogPtr = nbBits;
|
||||
remaining = (1<<nbBits)+1;
|
||||
threshold = 1<<nbBits;
|
||||
nbBits++;
|
||||
|
||||
while ((remaining>1) & (charnum<=*maxSVPtr)) {
|
||||
if (previous0) {
|
||||
unsigned n0 = charnum;
|
||||
while ((bitStream & 0xFFFF) == 0xFFFF) {
|
||||
n0 += 24;
|
||||
if (ip < iend-5) {
|
||||
ip += 2;
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
} else {
|
||||
bitStream >>= 16;
|
||||
bitCount += 16;
|
||||
} }
|
||||
while ((bitStream & 3) == 3) {
|
||||
n0 += 3;
|
||||
bitStream >>= 2;
|
||||
bitCount += 2;
|
||||
}
|
||||
n0 += bitStream & 3;
|
||||
bitCount += 2;
|
||||
if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
|
||||
while (charnum < n0) normalizedCounter[charnum++] = 0;
|
||||
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
ip += bitCount>>3;
|
||||
bitCount &= 7;
|
||||
bitStream = MEM_readLE32(ip) >> bitCount;
|
||||
} else {
|
||||
bitStream >>= 2;
|
||||
} }
|
||||
{ short const max = (short)((2*threshold-1)-remaining);
|
||||
short count;
|
||||
|
||||
if ((bitStream & (threshold-1)) < (U32)max) {
|
||||
count = (short)(bitStream & (threshold-1));
|
||||
bitCount += nbBits-1;
|
||||
} else {
|
||||
count = (short)(bitStream & (2*threshold-1));
|
||||
if (count >= threshold) count -= max;
|
||||
bitCount += nbBits;
|
||||
}
|
||||
|
||||
count--; /* extra accuracy */
|
||||
remaining -= FSEv08_abs(count);
|
||||
normalizedCounter[charnum++] = count;
|
||||
previous0 = !count;
|
||||
while (remaining < threshold) {
|
||||
nbBits--;
|
||||
threshold >>= 1;
|
||||
}
|
||||
|
||||
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
|
||||
ip += bitCount>>3;
|
||||
bitCount &= 7;
|
||||
} else {
|
||||
bitCount -= (int)(8 * (iend - 4 - ip));
|
||||
ip = iend - 4;
|
||||
}
|
||||
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
|
||||
} } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
|
||||
if (remaining != 1) return ERROR(corruption_detected);
|
||||
if (bitCount > 32) return ERROR(corruption_detected);
|
||||
*maxSVPtr = charnum-1;
|
||||
|
||||
ip += (bitCount+7)>>3;
|
||||
return ip-istart;
|
||||
}
|
||||
|
||||
|
||||
/*! HUFv08_readStats() :
|
||||
Read compact Huffman tree, saved by HUFv08_writeCTable().
|
||||
`huffWeight` is destination buffer.
|
||||
@return : size read from `src` , or an error Code .
|
||||
Note : Needed by HUFv08_readCTable() and HUFv08_readDTableX?() .
|
||||
*/
|
||||
size_t HUFv08_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
|
||||
U32* nbSymbolsPtr, U32* tableLogPtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 weightTotal;
|
||||
const BYTE* ip = (const BYTE*) src;
|
||||
size_t iSize = ip[0];
|
||||
size_t oSize;
|
||||
|
||||
/* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
if (iSize >= 128) { /* special header */
|
||||
oSize = iSize - 127;
|
||||
iSize = ((oSize+1)/2);
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
if (oSize >= hwSize) return ERROR(corruption_detected);
|
||||
ip += 1;
|
||||
{ U32 n;
|
||||
for (n=0; n<oSize; n+=2) {
|
||||
huffWeight[n] = ip[n/2] >> 4;
|
||||
huffWeight[n+1] = ip[n/2] & 15;
|
||||
} } }
|
||||
else { /* header compressed with FSE (normal case) */
|
||||
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
|
||||
oSize = FSEv08_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
|
||||
if (FSEv08_isError(oSize)) return oSize;
|
||||
}
|
||||
|
||||
/* collect weight stats */
|
||||
memset(rankStats, 0, (HUFv08_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32));
|
||||
weightTotal = 0;
|
||||
{ U32 n; for (n=0; n<oSize; n++) {
|
||||
if (huffWeight[n] >= HUFv08_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||
rankStats[huffWeight[n]]++;
|
||||
weightTotal += (1 << huffWeight[n]) >> 1;
|
||||
} }
|
||||
|
||||
/* get last non-null symbol weight (implied, total must be 2^n) */
|
||||
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
|
||||
if (tableLog > HUFv08_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||
*tableLogPtr = tableLog;
|
||||
/* determine last weight */
|
||||
{ U32 const total = 1 << tableLog;
|
||||
U32 const rest = total - weightTotal;
|
||||
U32 const verif = 1 << BIT_highbit32(rest);
|
||||
U32 const lastWeight = BIT_highbit32(rest) + 1;
|
||||
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
|
||||
huffWeight[oSize] = (BYTE)lastWeight;
|
||||
rankStats[lastWeight]++;
|
||||
} }
|
||||
|
||||
/* check tree construction validity */
|
||||
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
|
||||
|
||||
/* results */
|
||||
*nbSymbolsPtr = (U32)(oSize+1);
|
||||
return iSize+1;
|
||||
}
|
||||
/* ******************************************************************
|
||||
FSE : Finite State Entropy decoder
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Compiler specifics
|
||||
****************************************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define FORCE_INLINE static __forceinline
|
||||
# include <intrin.h> /* For Visual 2005 */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
|
||||
#else
|
||||
# ifdef __GNUC__
|
||||
# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
# define FORCE_INLINE static inline __attribute__((always_inline))
|
||||
# else
|
||||
# define FORCE_INLINE static inline
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
****************************************************************/
|
||||
#define FSEv08_isError ERR_isError
|
||||
#define FSEv08_STATIC_ASSERT(c) { enum { FSEv08_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Complex types
|
||||
****************************************************************/
|
||||
typedef U32 DTable_max_t[FSEv08_DTABLE_SIZE_U32(FSEv08_MAX_TABLELOG)];
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Templates
|
||||
****************************************************************/
|
||||
/*
|
||||
designed to be included
|
||||
for type-specific functions (template emulation in C)
|
||||
Objective is to write these functions only once, for improved maintenance
|
||||
*/
|
||||
|
||||
/* safety checks */
|
||||
#ifndef FSEv08_FUNCTION_EXTENSION
|
||||
# error "FSEv08_FUNCTION_EXTENSION must be defined"
|
||||
#endif
|
||||
#ifndef FSEv08_FUNCTION_TYPE
|
||||
# error "FSEv08_FUNCTION_TYPE must be defined"
|
||||
#endif
|
||||
|
||||
/* Function names */
|
||||
#define FSEv08_CAT(X,Y) X##Y
|
||||
#define FSEv08_FUNCTION_NAME(X,Y) FSEv08_CAT(X,Y)
|
||||
#define FSEv08_TYPE_NAME(X,Y) FSEv08_CAT(X,Y)
|
||||
|
||||
|
||||
/* Function templates */
|
||||
FSEv08_DTable* FSEv08_createDTable (unsigned tableLog)
|
||||
{
|
||||
if (tableLog > FSEv08_TABLELOG_ABSOLUTE_MAX) tableLog = FSEv08_TABLELOG_ABSOLUTE_MAX;
|
||||
return (FSEv08_DTable*)malloc( FSEv08_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
|
||||
}
|
||||
|
||||
void FSEv08_freeDTable (FSEv08_DTable* dt)
|
||||
{
|
||||
free(dt);
|
||||
}
|
||||
|
||||
size_t FSEv08_buildDTable(FSEv08_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
|
||||
{
|
||||
void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
|
||||
FSEv08_DECODE_TYPE* const tableDecode = (FSEv08_DECODE_TYPE*) (tdPtr);
|
||||
U16 symbolNext[FSEv08_MAX_SYMBOL_VALUE+1];
|
||||
|
||||
U32 const maxSV1 = maxSymbolValue + 1;
|
||||
U32 const tableSize = 1 << tableLog;
|
||||
U32 highThreshold = tableSize-1;
|
||||
|
||||
/* Sanity Checks */
|
||||
if (maxSymbolValue > FSEv08_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
|
||||
if (tableLog > FSEv08_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
|
||||
|
||||
/* Init, lay down lowprob symbols */
|
||||
{ FSEv08_DTableHeader DTableH;
|
||||
DTableH.tableLog = (U16)tableLog;
|
||||
DTableH.fastMode = 1;
|
||||
{ S16 const largeLimit= (S16)(1 << (tableLog-1));
|
||||
U32 s;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
if (normalizedCounter[s]==-1) {
|
||||
tableDecode[highThreshold--].symbol = (FSEv08_FUNCTION_TYPE)s;
|
||||
symbolNext[s] = 1;
|
||||
} else {
|
||||
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
|
||||
symbolNext[s] = normalizedCounter[s];
|
||||
} } }
|
||||
memcpy(dt, &DTableH, sizeof(DTableH));
|
||||
}
|
||||
|
||||
/* Spread symbols */
|
||||
{ U32 const tableMask = tableSize-1;
|
||||
U32 const step = FSEv08_TABLESTEP(tableSize);
|
||||
U32 s, position = 0;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
int i;
|
||||
for (i=0; i<normalizedCounter[s]; i++) {
|
||||
tableDecode[position].symbol = (FSEv08_FUNCTION_TYPE)s;
|
||||
position = (position + step) & tableMask;
|
||||
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
|
||||
} }
|
||||
|
||||
if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
|
||||
}
|
||||
|
||||
/* Build Decoding table */
|
||||
{ U32 u;
|
||||
for (u=0; u<tableSize; u++) {
|
||||
FSEv08_FUNCTION_TYPE const symbol = (FSEv08_FUNCTION_TYPE)(tableDecode[u].symbol);
|
||||
U16 nextState = symbolNext[symbol]++;
|
||||
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
|
||||
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
|
||||
} }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifndef FSEv08_COMMONDEFS_ONLY
|
||||
|
||||
/*-*******************************************************
|
||||
* Decompression (Byte symbols)
|
||||
*********************************************************/
|
||||
size_t FSEv08_buildDTable_rle (FSEv08_DTable* dt, BYTE symbolValue)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSEv08_DTableHeader* const DTableH = (FSEv08_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSEv08_decode_t* const cell = (FSEv08_decode_t*)dPtr;
|
||||
|
||||
DTableH->tableLog = 0;
|
||||
DTableH->fastMode = 0;
|
||||
|
||||
cell->newState = 0;
|
||||
cell->symbol = symbolValue;
|
||||
cell->nbBits = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
size_t FSEv08_buildDTable_raw (FSEv08_DTable* dt, unsigned nbBits)
|
||||
{
|
||||
void* ptr = dt;
|
||||
FSEv08_DTableHeader* const DTableH = (FSEv08_DTableHeader*)ptr;
|
||||
void* dPtr = dt + 1;
|
||||
FSEv08_decode_t* const dinfo = (FSEv08_decode_t*)dPtr;
|
||||
const unsigned tableSize = 1 << nbBits;
|
||||
const unsigned tableMask = tableSize - 1;
|
||||
const unsigned maxSV1 = tableMask+1;
|
||||
unsigned s;
|
||||
|
||||
/* Sanity checks */
|
||||
if (nbBits < 1) return ERROR(GENERIC); /* min size */
|
||||
|
||||
/* Build Decoding Table */
|
||||
DTableH->tableLog = (U16)nbBits;
|
||||
DTableH->fastMode = 1;
|
||||
for (s=0; s<maxSV1; s++) {
|
||||
dinfo[s].newState = 0;
|
||||
dinfo[s].symbol = (BYTE)s;
|
||||
dinfo[s].nbBits = (BYTE)nbBits;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
FORCE_INLINE size_t FSEv08_decompress_usingDTable_generic(
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const FSEv08_DTable* dt, const unsigned fast)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const omax = op + maxDstSize;
|
||||
BYTE* const olimit = omax-3;
|
||||
|
||||
BIT_DStream_t bitD;
|
||||
FSEv08_DState_t state1;
|
||||
FSEv08_DState_t state2;
|
||||
|
||||
/* Init */
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */
|
||||
if (FSEv08_isError(errorCode)) return errorCode; }
|
||||
|
||||
FSEv08_initDState(&state1, &bitD, dt);
|
||||
FSEv08_initDState(&state2, &bitD, dt);
|
||||
|
||||
#define FSEv08_GETSYMBOL(statePtr) fast ? FSEv08_decodeSymbolFast(statePtr, &bitD) : FSEv08_decodeSymbol(statePtr, &bitD)
|
||||
|
||||
/* 4 symbols per loop */
|
||||
for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op<olimit) ; op+=4) {
|
||||
op[0] = FSEv08_GETSYMBOL(&state1);
|
||||
|
||||
if (FSEv08_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BIT_reloadDStream(&bitD);
|
||||
|
||||
op[1] = FSEv08_GETSYMBOL(&state2);
|
||||
|
||||
if (FSEv08_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
{ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
|
||||
|
||||
op[2] = FSEv08_GETSYMBOL(&state1);
|
||||
|
||||
if (FSEv08_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
|
||||
BIT_reloadDStream(&bitD);
|
||||
|
||||
op[3] = FSEv08_GETSYMBOL(&state2);
|
||||
}
|
||||
|
||||
/* tail */
|
||||
/* note : BIT_reloadDStream(&bitD) >= FSEv08_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
|
||||
while (1) {
|
||||
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
||||
|
||||
*op++ = FSEv08_GETSYMBOL(&state1);
|
||||
|
||||
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
|
||||
*op++ = FSEv08_GETSYMBOL(&state2);
|
||||
break;
|
||||
}
|
||||
|
||||
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
|
||||
|
||||
*op++ = FSEv08_GETSYMBOL(&state2);
|
||||
|
||||
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
|
||||
*op++ = FSEv08_GETSYMBOL(&state1);
|
||||
break;
|
||||
} }
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
size_t FSEv08_decompress_usingDTable(void* dst, size_t originalSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const FSEv08_DTable* dt)
|
||||
{
|
||||
const void* ptr = dt;
|
||||
const FSEv08_DTableHeader* DTableH = (const FSEv08_DTableHeader*)ptr;
|
||||
const U32 fastMode = DTableH->fastMode;
|
||||
|
||||
/* select fast mode (static) */
|
||||
if (fastMode) return FSEv08_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
|
||||
return FSEv08_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t FSEv08_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*)cSrc;
|
||||
const BYTE* ip = istart;
|
||||
short counting[FSEv08_MAX_SYMBOL_VALUE+1];
|
||||
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
|
||||
unsigned tableLog;
|
||||
unsigned maxSymbolValue = FSEv08_MAX_SYMBOL_VALUE;
|
||||
|
||||
if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */
|
||||
|
||||
/* normal FSE decoding mode */
|
||||
{ size_t const NCountLength = FSEv08_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
|
||||
if (FSEv08_isError(NCountLength)) return NCountLength;
|
||||
if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */
|
||||
ip += NCountLength;
|
||||
cSrcSize -= NCountLength;
|
||||
}
|
||||
|
||||
{ size_t const errorCode = FSEv08_buildDTable (dt, counting, maxSymbolValue, tableLog);
|
||||
if (FSEv08_isError(errorCode)) return errorCode; }
|
||||
|
||||
return FSEv08_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */
|
||||
}
|
||||
|
||||
|
||||
|
||||
#endif /* FSEv08_COMMONDEFS_ONLY */
|
||||
/*
|
||||
Common functions of Zstd compression library
|
||||
Copyright (C) 2015-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net/
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/*-****************************************
|
||||
* ZSTD Error Management
|
||||
******************************************/
|
||||
/*! ZSTDv08_isError() :
|
||||
* tells if a return value is an error code */
|
||||
unsigned ZSTDv08_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
/*! ZSTDv08_getErrorName() :
|
||||
* provides error code string from function result (useful for debugging) */
|
||||
const char* ZSTDv08_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
/*! ZSTDv08_getError() :
|
||||
* convert a `size_t` function result into a proper ZSTDv08_errorCode enum */
|
||||
ZSTDv08_ErrorCode ZSTDv08_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
|
||||
|
||||
/*! ZSTDv08_getErrorString() :
|
||||
* provides error code string from enum */
|
||||
const char* ZSTDv08_getErrorString(ZSTDv08_ErrorCode code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* ZBUFF Error Management
|
||||
****************************************************************/
|
||||
unsigned ZBUFFv08_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
|
||||
const char* ZBUFFv08_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
|
||||
|
||||
|
||||
void* ZSTDv08_defaultAllocFunction(void* opaque, size_t size)
|
||||
{
|
||||
void* address = malloc(size);
|
||||
(void)opaque;
|
||||
/* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
|
||||
return address;
|
||||
}
|
||||
|
||||
void ZSTDv08_defaultFreeFunction(void* opaque, void* address)
|
||||
{
|
||||
(void)opaque;
|
||||
/* if (address) printf("free %p opaque=%p \n", address, opaque); */
|
||||
free(address);
|
||||
}
|
||||
/* ******************************************************************
|
||||
Huffman decoder, part of New Generation Entropy library
|
||||
Copyright (C) 2013-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
|
||||
/* **************************************************************
|
||||
* Compiler specifics
|
||||
****************************************************************/
|
||||
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
/* inline is defined */
|
||||
#elif defined(_MSC_VER)
|
||||
# define inline __inline
|
||||
#else
|
||||
# define inline /* disable inline */
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define FORCE_INLINE static __forceinline
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#else
|
||||
# ifdef __GNUC__
|
||||
# define FORCE_INLINE static inline __attribute__((always_inline))
|
||||
# else
|
||||
# define FORCE_INLINE static inline
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* Error Management
|
||||
****************************************************************/
|
||||
#define HUFv08_STATIC_ASSERT(c) { enum { HUFv08_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
|
||||
|
||||
|
||||
/*-***************************/
|
||||
/* generic DTableDesc */
|
||||
/*-***************************/
|
||||
|
||||
typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;
|
||||
|
||||
static DTableDesc HUFv08_getDTableDesc(const HUFv08_DTable* table)
|
||||
{
|
||||
DTableDesc dtd;
|
||||
memcpy(&dtd, table, sizeof(dtd));
|
||||
return dtd;
|
||||
}
|
||||
|
||||
|
||||
/*-***************************/
|
||||
/* single-symbol decoding */
|
||||
/*-***************************/
|
||||
|
||||
typedef struct { BYTE byte; BYTE nbBits; } HUFv08_DEltX2; /* single-symbol decoding */
|
||||
|
||||
size_t HUFv08_readDTableX2 (HUFv08_DTable* DTable, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE huffWeight[HUFv08_SYMBOLVALUE_MAX + 1];
|
||||
U32 rankVal[HUFv08_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
|
||||
U32 tableLog = 0;
|
||||
U32 nbSymbols = 0;
|
||||
size_t iSize;
|
||||
void* const dtPtr = DTable + 1;
|
||||
HUFv08_DEltX2* const dt = (HUFv08_DEltX2*)dtPtr;
|
||||
|
||||
HUFv08_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv08_DTable));
|
||||
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
iSize = HUFv08_readStats(huffWeight, HUFv08_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
|
||||
if (HUFv08_isError(iSize)) return iSize;
|
||||
|
||||
/* Table header */
|
||||
{ DTableDesc dtd = HUFv08_getDTableDesc(DTable);
|
||||
if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */
|
||||
dtd.tableType = 0;
|
||||
dtd.tableLog = (BYTE)tableLog;
|
||||
memcpy(DTable, &dtd, sizeof(dtd));
|
||||
}
|
||||
|
||||
/* Prepare ranks */
|
||||
{ U32 n, nextRankStart = 0;
|
||||
for (n=1; n<tableLog+1; n++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += (rankVal[n] << (n-1));
|
||||
rankVal[n] = current;
|
||||
} }
|
||||
|
||||
/* fill DTable */
|
||||
{ U32 n;
|
||||
for (n=0; n<nbSymbols; n++) {
|
||||
U32 const w = huffWeight[n];
|
||||
U32 const length = (1 << w) >> 1;
|
||||
U32 i;
|
||||
HUFv08_DEltX2 D;
|
||||
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
|
||||
for (i = rankVal[w]; i < rankVal[w] + length; i++)
|
||||
dt[i] = D;
|
||||
rankVal[w] += length;
|
||||
} }
|
||||
|
||||
return iSize;
|
||||
}
|
||||
|
||||
|
||||
static BYTE HUFv08_decodeSymbolX2(BIT_DStream_t* Dstream, const HUFv08_DEltX2* dt, const U32 dtLog)
|
||||
{
|
||||
size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
|
||||
BYTE const c = dt[val].byte;
|
||||
BIT_skipBits(Dstream, dt[val].nbBits);
|
||||
return c;
|
||||
}
|
||||
|
||||
#define HUFv08_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \
|
||||
*ptr++ = HUFv08_decodeSymbolX2(DStreamPtr, dt, dtLog)
|
||||
|
||||
#define HUFv08_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
|
||||
if (MEM_64bits() || (HUFv08_TABLELOG_MAX<=12)) \
|
||||
HUFv08_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
|
||||
|
||||
#define HUFv08_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
|
||||
if (MEM_64bits()) \
|
||||
HUFv08_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
|
||||
|
||||
static inline size_t HUFv08_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUFv08_DEltX2* const dt, const U32 dtLog)
|
||||
{
|
||||
BYTE* const pStart = p;
|
||||
|
||||
/* up to 4 symbols at a time */
|
||||
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) {
|
||||
HUFv08_DECODE_SYMBOLX2_2(p, bitDPtr);
|
||||
HUFv08_DECODE_SYMBOLX2_1(p, bitDPtr);
|
||||
HUFv08_DECODE_SYMBOLX2_2(p, bitDPtr);
|
||||
HUFv08_DECODE_SYMBOLX2_0(p, bitDPtr);
|
||||
}
|
||||
|
||||
/* closer to the end */
|
||||
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd))
|
||||
HUFv08_DECODE_SYMBOLX2_0(p, bitDPtr);
|
||||
|
||||
/* no more data to retrieve from bitstream, hence no need to reload */
|
||||
while (p < pEnd)
|
||||
HUFv08_DECODE_SYMBOLX2_0(p, bitDPtr);
|
||||
|
||||
return pEnd-pStart;
|
||||
}
|
||||
|
||||
static size_t HUFv08_decompress1X2_usingDTable_internal(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv08_DTable* DTable)
|
||||
{
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = op + dstSize;
|
||||
const void* dtPtr = DTable + 1;
|
||||
const HUFv08_DEltX2* const dt = (const HUFv08_DEltX2*)dtPtr;
|
||||
BIT_DStream_t bitD;
|
||||
DTableDesc const dtd = HUFv08_getDTableDesc(DTable);
|
||||
U32 const dtLog = dtd.tableLog;
|
||||
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
|
||||
if (HUFv08_isError(errorCode)) return errorCode; }
|
||||
|
||||
HUFv08_decodeStreamX2(op, &bitD, oend, dt, dtLog);
|
||||
|
||||
/* check */
|
||||
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
|
||||
|
||||
return dstSize;
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress1X2_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv08_DTable* DTable)
|
||||
{
|
||||
DTableDesc dtd = HUFv08_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 0) return ERROR(GENERIC);
|
||||
return HUFv08_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress1X2_DCtx (HUFv08_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const hSize = HUFv08_readDTableX2 (DCtx, cSrc, cSrcSize);
|
||||
if (HUFv08_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUFv08_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUFv08_CREATE_STATIC_DTABLEX2(DTable, HUFv08_TABLELOG_MAX);
|
||||
return HUFv08_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
|
||||
static size_t HUFv08_decompress4X2_usingDTable_internal(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv08_DTable* DTable)
|
||||
{
|
||||
/* Check */
|
||||
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
|
||||
|
||||
{ const BYTE* const istart = (const BYTE*) cSrc;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
const void* const dtPtr = DTable + 1;
|
||||
const HUFv08_DEltX2* const dt = (const HUFv08_DEltX2*)dtPtr;
|
||||
|
||||
/* Init */
|
||||
BIT_DStream_t bitD1;
|
||||
BIT_DStream_t bitD2;
|
||||
BIT_DStream_t bitD3;
|
||||
BIT_DStream_t bitD4;
|
||||
size_t const length1 = MEM_readLE16(istart);
|
||||
size_t const length2 = MEM_readLE16(istart+2);
|
||||
size_t const length3 = MEM_readLE16(istart+4);
|
||||
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
|
||||
const BYTE* const istart1 = istart + 6; /* jumpTable */
|
||||
const BYTE* const istart2 = istart1 + length1;
|
||||
const BYTE* const istart3 = istart2 + length2;
|
||||
const BYTE* const istart4 = istart3 + length3;
|
||||
const size_t segmentSize = (dstSize+3) / 4;
|
||||
BYTE* const opStart2 = ostart + segmentSize;
|
||||
BYTE* const opStart3 = opStart2 + segmentSize;
|
||||
BYTE* const opStart4 = opStart3 + segmentSize;
|
||||
BYTE* op1 = ostart;
|
||||
BYTE* op2 = opStart2;
|
||||
BYTE* op3 = opStart3;
|
||||
BYTE* op4 = opStart4;
|
||||
U32 endSignal;
|
||||
DTableDesc const dtd = HUFv08_getDTableDesc(DTable);
|
||||
U32 const dtLog = dtd.tableLog;
|
||||
|
||||
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
|
||||
if (HUFv08_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
|
||||
if (HUFv08_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
|
||||
if (HUFv08_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
|
||||
if (HUFv08_isError(errorCode)) return errorCode; }
|
||||
|
||||
/* 16-32 symbols per loop (4-8 symbols per stream) */
|
||||
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
|
||||
for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) {
|
||||
HUFv08_DECODE_SYMBOLX2_2(op1, &bitD1);
|
||||
HUFv08_DECODE_SYMBOLX2_2(op2, &bitD2);
|
||||
HUFv08_DECODE_SYMBOLX2_2(op3, &bitD3);
|
||||
HUFv08_DECODE_SYMBOLX2_2(op4, &bitD4);
|
||||
HUFv08_DECODE_SYMBOLX2_1(op1, &bitD1);
|
||||
HUFv08_DECODE_SYMBOLX2_1(op2, &bitD2);
|
||||
HUFv08_DECODE_SYMBOLX2_1(op3, &bitD3);
|
||||
HUFv08_DECODE_SYMBOLX2_1(op4, &bitD4);
|
||||
HUFv08_DECODE_SYMBOLX2_2(op1, &bitD1);
|
||||
HUFv08_DECODE_SYMBOLX2_2(op2, &bitD2);
|
||||
HUFv08_DECODE_SYMBOLX2_2(op3, &bitD3);
|
||||
HUFv08_DECODE_SYMBOLX2_2(op4, &bitD4);
|
||||
HUFv08_DECODE_SYMBOLX2_0(op1, &bitD1);
|
||||
HUFv08_DECODE_SYMBOLX2_0(op2, &bitD2);
|
||||
HUFv08_DECODE_SYMBOLX2_0(op3, &bitD3);
|
||||
HUFv08_DECODE_SYMBOLX2_0(op4, &bitD4);
|
||||
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
|
||||
}
|
||||
|
||||
/* check corruption */
|
||||
if (op1 > opStart2) return ERROR(corruption_detected);
|
||||
if (op2 > opStart3) return ERROR(corruption_detected);
|
||||
if (op3 > opStart4) return ERROR(corruption_detected);
|
||||
/* note : op4 supposed already verified within main loop */
|
||||
|
||||
/* finish bitStreams one by one */
|
||||
HUFv08_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
|
||||
HUFv08_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
|
||||
HUFv08_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
|
||||
HUFv08_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
|
||||
|
||||
/* check */
|
||||
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
|
||||
if (!endSignal) return ERROR(corruption_detected);
|
||||
|
||||
/* decoded size */
|
||||
return dstSize;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
size_t HUFv08_decompress4X2_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv08_DTable* DTable)
|
||||
{
|
||||
DTableDesc dtd = HUFv08_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 0) return ERROR(GENERIC);
|
||||
return HUFv08_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
size_t HUFv08_decompress4X2_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const hSize = HUFv08_readDTableX2 (dctx, cSrc, cSrcSize);
|
||||
if (HUFv08_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUFv08_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx);
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUFv08_CREATE_STATIC_DTABLEX2(DTable, HUFv08_TABLELOG_MAX);
|
||||
return HUFv08_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
|
||||
/* *************************/
|
||||
/* double-symbols decoding */
|
||||
/* *************************/
|
||||
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUFv08_DEltX4; /* double-symbols decoding */
|
||||
|
||||
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
|
||||
|
||||
static void HUFv08_fillDTableX4Level2(HUFv08_DEltX4* DTable, U32 sizeLog, const U32 consumed,
|
||||
const U32* rankValOrigin, const int minWeight,
|
||||
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,
|
||||
U32 nbBitsBaseline, U16 baseSeq)
|
||||
{
|
||||
HUFv08_DEltX4 DElt;
|
||||
U32 rankVal[HUFv08_TABLELOG_ABSOLUTEMAX + 1];
|
||||
|
||||
/* get pre-calculated rankVal */
|
||||
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
|
||||
|
||||
/* fill skipped values */
|
||||
if (minWeight>1) {
|
||||
U32 i, skipSize = rankVal[minWeight];
|
||||
MEM_writeLE16(&(DElt.sequence), baseSeq);
|
||||
DElt.nbBits = (BYTE)(consumed);
|
||||
DElt.length = 1;
|
||||
for (i = 0; i < skipSize; i++)
|
||||
DTable[i] = DElt;
|
||||
}
|
||||
|
||||
/* fill DTable */
|
||||
{ U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */
|
||||
const U32 symbol = sortedSymbols[s].symbol;
|
||||
const U32 weight = sortedSymbols[s].weight;
|
||||
const U32 nbBits = nbBitsBaseline - weight;
|
||||
const U32 length = 1 << (sizeLog-nbBits);
|
||||
const U32 start = rankVal[weight];
|
||||
U32 i = start;
|
||||
const U32 end = start + length;
|
||||
|
||||
MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
|
||||
DElt.nbBits = (BYTE)(nbBits + consumed);
|
||||
DElt.length = 2;
|
||||
do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */
|
||||
|
||||
rankVal[weight] += length;
|
||||
}}
|
||||
}
|
||||
|
||||
typedef U32 rankVal_t[HUFv08_TABLELOG_ABSOLUTEMAX][HUFv08_TABLELOG_ABSOLUTEMAX + 1];
|
||||
|
||||
static void HUFv08_fillDTableX4(HUFv08_DEltX4* DTable, const U32 targetLog,
|
||||
const sortedSymbol_t* sortedList, const U32 sortedListSize,
|
||||
const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
|
||||
const U32 nbBitsBaseline)
|
||||
{
|
||||
U32 rankVal[HUFv08_TABLELOG_ABSOLUTEMAX + 1];
|
||||
const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
|
||||
const U32 minBits = nbBitsBaseline - maxWeight;
|
||||
U32 s;
|
||||
|
||||
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
|
||||
|
||||
/* fill DTable */
|
||||
for (s=0; s<sortedListSize; s++) {
|
||||
const U16 symbol = sortedList[s].symbol;
|
||||
const U32 weight = sortedList[s].weight;
|
||||
const U32 nbBits = nbBitsBaseline - weight;
|
||||
const U32 start = rankVal[weight];
|
||||
const U32 length = 1 << (targetLog-nbBits);
|
||||
|
||||
if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */
|
||||
U32 sortedRank;
|
||||
int minWeight = nbBits + scaleLog;
|
||||
if (minWeight < 1) minWeight = 1;
|
||||
sortedRank = rankStart[minWeight];
|
||||
HUFv08_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits,
|
||||
rankValOrigin[nbBits], minWeight,
|
||||
sortedList+sortedRank, sortedListSize-sortedRank,
|
||||
nbBitsBaseline, symbol);
|
||||
} else {
|
||||
HUFv08_DEltX4 DElt;
|
||||
MEM_writeLE16(&(DElt.sequence), symbol);
|
||||
DElt.nbBits = (BYTE)(nbBits);
|
||||
DElt.length = 1;
|
||||
{ U32 u;
|
||||
const U32 end = start + length;
|
||||
for (u = start; u < end; u++) DTable[u] = DElt;
|
||||
} }
|
||||
rankVal[weight] += length;
|
||||
}
|
||||
}
|
||||
|
||||
size_t HUFv08_readDTableX4 (HUFv08_DTable* DTable, const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE weightList[HUFv08_SYMBOLVALUE_MAX + 1];
|
||||
sortedSymbol_t sortedSymbol[HUFv08_SYMBOLVALUE_MAX + 1];
|
||||
U32 rankStats[HUFv08_TABLELOG_ABSOLUTEMAX + 1] = { 0 };
|
||||
U32 rankStart0[HUFv08_TABLELOG_ABSOLUTEMAX + 2] = { 0 };
|
||||
U32* const rankStart = rankStart0+1;
|
||||
rankVal_t rankVal;
|
||||
U32 tableLog, maxW, sizeOfSort, nbSymbols;
|
||||
DTableDesc dtd = HUFv08_getDTableDesc(DTable);
|
||||
U32 const maxTableLog = dtd.maxTableLog;
|
||||
size_t iSize;
|
||||
void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
|
||||
HUFv08_DEltX4* const dt = (HUFv08_DEltX4*)dtPtr;
|
||||
|
||||
HUFv08_STATIC_ASSERT(sizeof(HUFv08_DEltX4) == sizeof(HUFv08_DTable)); /* if compilation fails here, assertion is false */
|
||||
if (maxTableLog > HUFv08_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
|
||||
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
|
||||
|
||||
iSize = HUFv08_readStats(weightList, HUFv08_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
|
||||
if (HUFv08_isError(iSize)) return iSize;
|
||||
|
||||
/* check result */
|
||||
if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
|
||||
|
||||
/* find maxWeight */
|
||||
for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
|
||||
|
||||
/* Get start index of each weight */
|
||||
{ U32 w, nextRankStart = 0;
|
||||
for (w=1; w<maxW+1; w++) {
|
||||
U32 current = nextRankStart;
|
||||
nextRankStart += rankStats[w];
|
||||
rankStart[w] = current;
|
||||
}
|
||||
rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
|
||||
sizeOfSort = nextRankStart;
|
||||
}
|
||||
|
||||
/* sort symbols by weight */
|
||||
{ U32 s;
|
||||
for (s=0; s<nbSymbols; s++) {
|
||||
U32 const w = weightList[s];
|
||||
U32 const r = rankStart[w]++;
|
||||
sortedSymbol[r].symbol = (BYTE)s;
|
||||
sortedSymbol[r].weight = (BYTE)w;
|
||||
}
|
||||
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
|
||||
}
|
||||
|
||||
/* Build rankVal */
|
||||
{ U32* const rankVal0 = rankVal[0];
|
||||
{ int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */
|
||||
U32 nextRankVal = 0;
|
||||
U32 w;
|
||||
for (w=1; w<maxW+1; w++) {
|
||||
U32 current = nextRankVal;
|
||||
nextRankVal += rankStats[w] << (w+rescale);
|
||||
rankVal0[w] = current;
|
||||
} }
|
||||
{ U32 const minBits = tableLog+1 - maxW;
|
||||
U32 consumed;
|
||||
for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
|
||||
U32* const rankValPtr = rankVal[consumed];
|
||||
U32 w;
|
||||
for (w = 1; w < maxW+1; w++) {
|
||||
rankValPtr[w] = rankVal0[w] >> consumed;
|
||||
} } } }
|
||||
|
||||
HUFv08_fillDTableX4(dt, maxTableLog,
|
||||
sortedSymbol, sizeOfSort,
|
||||
rankStart0, rankVal, maxW,
|
||||
tableLog+1);
|
||||
|
||||
dtd.tableLog = (BYTE)maxTableLog;
|
||||
dtd.tableType = 1;
|
||||
memcpy(DTable, &dtd, sizeof(dtd));
|
||||
return iSize;
|
||||
}
|
||||
|
||||
|
||||
static U32 HUFv08_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUFv08_DEltX4* dt, const U32 dtLog)
|
||||
{
|
||||
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
|
||||
memcpy(op, dt+val, 2);
|
||||
BIT_skipBits(DStream, dt[val].nbBits);
|
||||
return dt[val].length;
|
||||
}
|
||||
|
||||
static U32 HUFv08_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUFv08_DEltX4* dt, const U32 dtLog)
|
||||
{
|
||||
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
|
||||
memcpy(op, dt+val, 1);
|
||||
if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits);
|
||||
else {
|
||||
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
|
||||
BIT_skipBits(DStream, dt[val].nbBits);
|
||||
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
|
||||
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
|
||||
} }
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
#define HUFv08_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \
|
||||
ptr += HUFv08_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
||||
|
||||
#define HUFv08_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
|
||||
if (MEM_64bits() || (HUFv08_TABLELOG_MAX<=12)) \
|
||||
ptr += HUFv08_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
||||
|
||||
#define HUFv08_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
|
||||
if (MEM_64bits()) \
|
||||
ptr += HUFv08_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
|
||||
|
||||
static inline size_t HUFv08_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUFv08_DEltX4* const dt, const U32 dtLog)
|
||||
{
|
||||
BYTE* const pStart = p;
|
||||
|
||||
/* up to 8 symbols at a time */
|
||||
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd-7)) {
|
||||
HUFv08_DECODE_SYMBOLX4_2(p, bitDPtr);
|
||||
HUFv08_DECODE_SYMBOLX4_1(p, bitDPtr);
|
||||
HUFv08_DECODE_SYMBOLX4_2(p, bitDPtr);
|
||||
HUFv08_DECODE_SYMBOLX4_0(p, bitDPtr);
|
||||
}
|
||||
|
||||
/* closer to end : up to 2 symbols at a time */
|
||||
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2))
|
||||
HUFv08_DECODE_SYMBOLX4_0(p, bitDPtr);
|
||||
|
||||
while (p <= pEnd-2)
|
||||
HUFv08_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
|
||||
|
||||
if (p < pEnd)
|
||||
p += HUFv08_decodeLastSymbolX4(p, bitDPtr, dt, dtLog);
|
||||
|
||||
return p-pStart;
|
||||
}
|
||||
|
||||
|
||||
static size_t HUFv08_decompress1X4_usingDTable_internal(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv08_DTable* DTable)
|
||||
{
|
||||
BIT_DStream_t bitD;
|
||||
|
||||
/* Init */
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
|
||||
if (HUFv08_isError(errorCode)) return errorCode;
|
||||
}
|
||||
|
||||
/* decode */
|
||||
{ BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */
|
||||
const HUFv08_DEltX4* const dt = (const HUFv08_DEltX4*)dtPtr;
|
||||
DTableDesc const dtd = HUFv08_getDTableDesc(DTable);
|
||||
HUFv08_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog);
|
||||
}
|
||||
|
||||
/* check */
|
||||
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
|
||||
|
||||
/* decoded size */
|
||||
return dstSize;
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress1X4_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv08_DTable* DTable)
|
||||
{
|
||||
DTableDesc dtd = HUFv08_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 1) return ERROR(GENERIC);
|
||||
return HUFv08_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress1X4_DCtx (HUFv08_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t const hSize = HUFv08_readDTableX4 (DCtx, cSrc, cSrcSize);
|
||||
if (HUFv08_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUFv08_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUFv08_CREATE_STATIC_DTABLEX4(DTable, HUFv08_TABLELOG_MAX);
|
||||
return HUFv08_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
static size_t HUFv08_decompress4X4_usingDTable_internal(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv08_DTable* DTable)
|
||||
{
|
||||
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
|
||||
|
||||
{ const BYTE* const istart = (const BYTE*) cSrc;
|
||||
BYTE* const ostart = (BYTE*) dst;
|
||||
BYTE* const oend = ostart + dstSize;
|
||||
const void* const dtPtr = DTable+1;
|
||||
const HUFv08_DEltX4* const dt = (const HUFv08_DEltX4*)dtPtr;
|
||||
|
||||
/* Init */
|
||||
BIT_DStream_t bitD1;
|
||||
BIT_DStream_t bitD2;
|
||||
BIT_DStream_t bitD3;
|
||||
BIT_DStream_t bitD4;
|
||||
size_t const length1 = MEM_readLE16(istart);
|
||||
size_t const length2 = MEM_readLE16(istart+2);
|
||||
size_t const length3 = MEM_readLE16(istart+4);
|
||||
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
|
||||
const BYTE* const istart1 = istart + 6; /* jumpTable */
|
||||
const BYTE* const istart2 = istart1 + length1;
|
||||
const BYTE* const istart3 = istart2 + length2;
|
||||
const BYTE* const istart4 = istart3 + length3;
|
||||
size_t const segmentSize = (dstSize+3) / 4;
|
||||
BYTE* const opStart2 = ostart + segmentSize;
|
||||
BYTE* const opStart3 = opStart2 + segmentSize;
|
||||
BYTE* const opStart4 = opStart3 + segmentSize;
|
||||
BYTE* op1 = ostart;
|
||||
BYTE* op2 = opStart2;
|
||||
BYTE* op3 = opStart3;
|
||||
BYTE* op4 = opStart4;
|
||||
U32 endSignal;
|
||||
DTableDesc const dtd = HUFv08_getDTableDesc(DTable);
|
||||
U32 const dtLog = dtd.tableLog;
|
||||
|
||||
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
|
||||
if (HUFv08_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
|
||||
if (HUFv08_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
|
||||
if (HUFv08_isError(errorCode)) return errorCode; }
|
||||
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
|
||||
if (HUFv08_isError(errorCode)) return errorCode; }
|
||||
|
||||
/* 16-32 symbols per loop (4-8 symbols per stream) */
|
||||
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
|
||||
for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) {
|
||||
HUFv08_DECODE_SYMBOLX4_2(op1, &bitD1);
|
||||
HUFv08_DECODE_SYMBOLX4_2(op2, &bitD2);
|
||||
HUFv08_DECODE_SYMBOLX4_2(op3, &bitD3);
|
||||
HUFv08_DECODE_SYMBOLX4_2(op4, &bitD4);
|
||||
HUFv08_DECODE_SYMBOLX4_1(op1, &bitD1);
|
||||
HUFv08_DECODE_SYMBOLX4_1(op2, &bitD2);
|
||||
HUFv08_DECODE_SYMBOLX4_1(op3, &bitD3);
|
||||
HUFv08_DECODE_SYMBOLX4_1(op4, &bitD4);
|
||||
HUFv08_DECODE_SYMBOLX4_2(op1, &bitD1);
|
||||
HUFv08_DECODE_SYMBOLX4_2(op2, &bitD2);
|
||||
HUFv08_DECODE_SYMBOLX4_2(op3, &bitD3);
|
||||
HUFv08_DECODE_SYMBOLX4_2(op4, &bitD4);
|
||||
HUFv08_DECODE_SYMBOLX4_0(op1, &bitD1);
|
||||
HUFv08_DECODE_SYMBOLX4_0(op2, &bitD2);
|
||||
HUFv08_DECODE_SYMBOLX4_0(op3, &bitD3);
|
||||
HUFv08_DECODE_SYMBOLX4_0(op4, &bitD4);
|
||||
|
||||
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
|
||||
}
|
||||
|
||||
/* check corruption */
|
||||
if (op1 > opStart2) return ERROR(corruption_detected);
|
||||
if (op2 > opStart3) return ERROR(corruption_detected);
|
||||
if (op3 > opStart4) return ERROR(corruption_detected);
|
||||
/* note : op4 supposed already verified within main loop */
|
||||
|
||||
/* finish bitStreams one by one */
|
||||
HUFv08_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog);
|
||||
HUFv08_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog);
|
||||
HUFv08_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog);
|
||||
HUFv08_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
|
||||
|
||||
/* check */
|
||||
{ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
|
||||
if (!endCheck) return ERROR(corruption_detected); }
|
||||
|
||||
/* decoded size */
|
||||
return dstSize;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
size_t HUFv08_decompress4X4_usingDTable(
|
||||
void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv08_DTable* DTable)
|
||||
{
|
||||
DTableDesc dtd = HUFv08_getDTableDesc(DTable);
|
||||
if (dtd.tableType != 1) return ERROR(GENERIC);
|
||||
return HUFv08_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
size_t HUFv08_decompress4X4_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*) cSrc;
|
||||
|
||||
size_t hSize = HUFv08_readDTableX4 (dctx, cSrc, cSrcSize);
|
||||
if (HUFv08_isError(hSize)) return hSize;
|
||||
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
|
||||
ip += hSize; cSrcSize -= hSize;
|
||||
|
||||
return HUFv08_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
HUFv08_CREATE_STATIC_DTABLEX4(DTable, HUFv08_TABLELOG_MAX);
|
||||
return HUFv08_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
|
||||
/* ********************************/
|
||||
/* Generic decompression selector */
|
||||
/* ********************************/
|
||||
|
||||
size_t HUFv08_decompress1X_usingDTable(void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv08_DTable* DTable)
|
||||
{
|
||||
DTableDesc const dtd = HUFv08_getDTableDesc(DTable);
|
||||
return dtd.tableType ? HUFv08_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) :
|
||||
HUFv08_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress4X_usingDTable(void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize,
|
||||
const HUFv08_DTable* DTable)
|
||||
{
|
||||
DTableDesc const dtd = HUFv08_getDTableDesc(DTable);
|
||||
return dtd.tableType ? HUFv08_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) :
|
||||
HUFv08_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
|
||||
}
|
||||
|
||||
|
||||
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
|
||||
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] =
|
||||
{
|
||||
/* single, double, quad */
|
||||
{{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */
|
||||
{{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */
|
||||
{{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */
|
||||
{{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */
|
||||
{{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */
|
||||
{{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */
|
||||
{{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */
|
||||
{{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */
|
||||
{{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */
|
||||
{{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */
|
||||
{{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */
|
||||
{{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */
|
||||
{{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */
|
||||
{{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */
|
||||
{{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */
|
||||
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */
|
||||
};
|
||||
|
||||
/** HUFv08_selectDecoder() :
|
||||
* Tells which decoder is likely to decode faster,
|
||||
* based on a set of pre-determined metrics.
|
||||
* @return : 0==HUFv08_decompress4X2, 1==HUFv08_decompress4X4 .
|
||||
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
|
||||
U32 HUFv08_selectDecoder (size_t dstSize, size_t cSrcSize)
|
||||
{
|
||||
/* decoder timing evaluation */
|
||||
U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
|
||||
U32 const D256 = (U32)(dstSize >> 8);
|
||||
U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
|
||||
U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
|
||||
DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */
|
||||
|
||||
return DTime1 < DTime0;
|
||||
}
|
||||
|
||||
|
||||
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
|
||||
|
||||
size_t HUFv08_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
static const decompressionAlgo decompress[2] = { HUFv08_decompress4X2, HUFv08_decompress4X4 };
|
||||
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
|
||||
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
{ U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize);
|
||||
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
|
||||
}
|
||||
|
||||
//return HUFv08_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
|
||||
//return HUFv08_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress4X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
|
||||
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
{ U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize);
|
||||
return algoNb ? HUFv08_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
|
||||
HUFv08_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
|
||||
}
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress4X_hufOnly (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */
|
||||
|
||||
{ U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize);
|
||||
return algoNb ? HUFv08_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
|
||||
HUFv08_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
|
||||
}
|
||||
}
|
||||
|
||||
size_t HUFv08_decompress1X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
|
||||
{
|
||||
/* validation checks */
|
||||
if (dstSize == 0) return ERROR(dstSize_tooSmall);
|
||||
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
|
||||
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
|
||||
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
|
||||
|
||||
{ U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize);
|
||||
return algoNb ? HUFv08_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
|
||||
HUFv08_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
|
||||
}
|
||||
}
|
||||
/*
|
||||
zstd_internal - common functions to include
|
||||
Header File for include
|
||||
Copyright (C) 2014-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : https://www.zstd.net
|
||||
*/
|
||||
#ifndef ZSTDv08_CCOMMON_H_MODULE
|
||||
#define ZSTDv08_CCOMMON_H_MODULE
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Common macros
|
||||
***************************************/
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
#define MAX(a,b) ((a)>(b) ? (a) : (b))
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Common constants
|
||||
***************************************/
|
||||
#define ZSTDv08_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */
|
||||
#if defined(ZSTDv08_OPT_DEBUG) && ZSTDv08_OPT_DEBUG>=9
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#define ZSTDv08_LOG_PARSER(...) printf(__VA_ARGS__)
|
||||
#define ZSTDv08_LOG_ENCODE(...) printf(__VA_ARGS__)
|
||||
#define ZSTDv08_LOG_BLOCK(...) printf(__VA_ARGS__)
|
||||
#else
|
||||
#define ZSTDv08_LOG_PARSER(...)
|
||||
#define ZSTDv08_LOG_ENCODE(...)
|
||||
#define ZSTDv08_LOG_BLOCK(...)
|
||||
#endif
|
||||
|
||||
#define ZSTDv08_OPT_NUM (1<<12)
|
||||
#define ZSTDv08_DICT_MAGIC 0xEC30A437 /* v0.7+ */
|
||||
|
||||
#define ZSTDv08_REP_NUM 3 /* number of repcodes */
|
||||
#define ZSTDv08_REP_CHECK (ZSTDv08_REP_NUM-0) /* number of repcodes to check by the optimal parser */
|
||||
#define ZSTDv08_REP_MOVE (ZSTDv08_REP_NUM-1)
|
||||
static const U32 repStartValue[ZSTDv08_REP_NUM] = { 1, 4, 8 };
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define BIT7 128
|
||||
#define BIT6 64
|
||||
#define BIT5 32
|
||||
#define BIT4 16
|
||||
#define BIT1 2
|
||||
#define BIT0 1
|
||||
|
||||
#define ZSTDv08_WINDOWLOG_ABSOLUTEMIN 10
|
||||
static const size_t ZSTDv08_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||
static const size_t ZSTDv08_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||
|
||||
#define ZSTDv08_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
|
||||
static const size_t ZSTDv08_blockHeaderSize = ZSTDv08_BLOCKHEADERSIZE;
|
||||
typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
|
||||
|
||||
#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
|
||||
#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */
|
||||
|
||||
#define HufLog 12
|
||||
typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
|
||||
|
||||
#define LONGNBSEQ 0x7F00
|
||||
|
||||
#define MINMATCH 3
|
||||
#define EQUAL_READ32 4
|
||||
|
||||
#define Litbits 8
|
||||
#define MaxLit ((1<<Litbits) - 1)
|
||||
#define MaxML 52
|
||||
#define MaxLL 35
|
||||
#define MaxOff 28
|
||||
#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
|
||||
#define MLFSELog 9
|
||||
#define LLFSELog 9
|
||||
#define OffFSELog 8
|
||||
|
||||
static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
|
||||
13,14,15,16 };
|
||||
static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
|
||||
-1,-1,-1,-1 };
|
||||
static const U32 LL_defaultNormLog = 6;
|
||||
|
||||
static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11,
|
||||
12,13,14,15,16 };
|
||||
static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
|
||||
-1,-1,-1,-1,-1 };
|
||||
static const U32 ML_defaultNormLog = 6;
|
||||
|
||||
static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
|
||||
static const U32 OF_defaultNormLog = 5;
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Shared functions to include for inlining
|
||||
*********************************************/
|
||||
static void ZSTDv08_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
||||
#define COPY8(d,s) { ZSTDv08_copy8(d,s); d+=8; s+=8; }
|
||||
|
||||
/*! ZSTDv08_wildcopy() :
|
||||
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
|
||||
#define WILDCOPY_OVERLENGTH 8
|
||||
MEM_STATIC void ZSTDv08_wildcopy(void* dst, const void* src, size_t length)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = op + length;
|
||||
do
|
||||
COPY8(op, ip)
|
||||
while (op < oend);
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Private interfaces
|
||||
*********************************************/
|
||||
typedef struct ZSTDv08_stats_s ZSTDv08_stats_t;
|
||||
|
||||
typedef struct {
|
||||
U32 off;
|
||||
U32 len;
|
||||
} ZSTDv08_match_t;
|
||||
|
||||
typedef struct {
|
||||
U32 price;
|
||||
U32 off;
|
||||
U32 mlen;
|
||||
U32 litlen;
|
||||
U32 rep[ZSTDv08_REP_NUM];
|
||||
} ZSTDv08_optimal_t;
|
||||
|
||||
#if ZSTDv08_OPT_DEBUG == 3
|
||||
#include ".debug/zstd_stats.h"
|
||||
#else
|
||||
struct ZSTDv08_stats_s { U32 unused; };
|
||||
MEM_STATIC void ZSTDv08_statsPrint(ZSTDv08_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
|
||||
MEM_STATIC void ZSTDv08_statsInit(ZSTDv08_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTDv08_statsResetFreqs(ZSTDv08_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTDv08_statsUpdatePrices(ZSTDv08_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
|
||||
#endif /* #if ZSTDv08_OPT_DEBUG == 3 */
|
||||
|
||||
|
||||
typedef struct seqDef_s {
|
||||
U32 offset;
|
||||
U16 litLength;
|
||||
U16 matchLength;
|
||||
} seqDef;
|
||||
|
||||
|
||||
typedef struct {
|
||||
seqDef* sequencesStart;
|
||||
seqDef* sequences;
|
||||
BYTE* litStart;
|
||||
BYTE* lit;
|
||||
BYTE* llCode;
|
||||
BYTE* mlCode;
|
||||
BYTE* ofCode;
|
||||
U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
|
||||
U32 longLengthPos;
|
||||
/* opt */
|
||||
ZSTDv08_optimal_t* priceTable;
|
||||
ZSTDv08_match_t* matchTable;
|
||||
U32* matchLengthFreq;
|
||||
U32* litLengthFreq;
|
||||
U32* litFreq;
|
||||
U32* offCodeFreq;
|
||||
U32 matchLengthSum;
|
||||
U32 matchSum;
|
||||
U32 litLengthSum;
|
||||
U32 litSum;
|
||||
U32 offCodeSum;
|
||||
U32 log2matchLengthSum;
|
||||
U32 log2matchSum;
|
||||
U32 log2litLengthSum;
|
||||
U32 log2litSum;
|
||||
U32 log2offCodeSum;
|
||||
U32 factor;
|
||||
U32 cachedPrice;
|
||||
U32 cachedLitLength;
|
||||
const BYTE* cachedLiterals;
|
||||
ZSTDv08_stats_t stats;
|
||||
} seqStore_t;
|
||||
|
||||
void ZSTDv08_seqToCodes(const seqStore_t* seqStorePtr);
|
||||
int ZSTDv08_isSkipFrame(ZSTDv08_DCtx* dctx);
|
||||
|
||||
/* custom memory allocation functions */
|
||||
void* ZSTDv08_defaultAllocFunction(void* opaque, size_t size);
|
||||
void ZSTDv08_defaultFreeFunction(void* opaque, void* address);
|
||||
static const ZSTDv08_customMem defaultCustomMem = { ZSTDv08_defaultAllocFunction, ZSTDv08_defaultFreeFunction, NULL };
|
||||
|
||||
/*====== common function ======*/
|
||||
|
||||
MEM_STATIC U32 ZSTDv08_highbit32(U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return 31 - __builtin_clz(val);
|
||||
# else /* Software version */
|
||||
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
int r;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
|
||||
return r;
|
||||
# endif
|
||||
}
|
||||
|
||||
|
||||
#endif /* ZSTDv08_CCOMMON_H_MODULE */
|
||||
/*
|
||||
zstd - standard compression library
|
||||
Copyright (C) 2014-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net
|
||||
*/
|
||||
|
||||
/* ***************************************************************
|
||||
* Tuning parameters
|
||||
*****************************************************************/
|
||||
/*!
|
||||
* HEAPMODE :
|
||||
* Select how default decompression function ZSTDv08_decompress() will allocate memory,
|
||||
* in memory stack (0), or in memory heap (1, requires malloc())
|
||||
*/
|
||||
#ifndef ZSTDv08_HEAPMODE
|
||||
# define ZSTDv08_HEAPMODE 1
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Compiler specifics
|
||||
*********************************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define FORCE_INLINE static __forceinline
|
||||
# include <intrin.h> /* For Visual 2005 */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
|
||||
#else
|
||||
# ifdef __GNUC__
|
||||
# define FORCE_INLINE static inline __attribute__((always_inline))
|
||||
# else
|
||||
# define FORCE_INLINE static inline
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Macros
|
||||
***************************************/
|
||||
#define ZSTDv08_isError ERR_isError /* for inlining */
|
||||
#define FSEv08_isError ERR_isError
|
||||
#define HUFv08_isError ERR_isError
|
||||
|
||||
|
||||
/*_*******************************************************
|
||||
* Memory operations
|
||||
**********************************************************/
|
||||
static void ZSTDv08_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
||||
|
||||
|
||||
/*-*************************************************************
|
||||
* Context management
|
||||
***************************************************************/
|
||||
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
|
||||
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
|
||||
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
|
||||
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTDv08_dStage;
|
||||
|
||||
struct ZSTDv08_DCtx_s
|
||||
{
|
||||
FSEv08_DTable LLTable[FSEv08_DTABLE_SIZE_U32(LLFSELog)];
|
||||
FSEv08_DTable OffTable[FSEv08_DTABLE_SIZE_U32(OffFSELog)];
|
||||
FSEv08_DTable MLTable[FSEv08_DTABLE_SIZE_U32(MLFSELog)];
|
||||
HUFv08_DTable hufTable[HUFv08_DTABLE_SIZE(HufLog)]; /* can accommodate HUFv08_decompress4X */
|
||||
const void* previousDstEnd;
|
||||
const void* base;
|
||||
const void* vBase;
|
||||
const void* dictEnd;
|
||||
size_t expected;
|
||||
U32 rep[ZSTDv08_REP_NUM];
|
||||
ZSTDv08_frameParams fParams;
|
||||
blockType_e bType; /* used in ZSTDv08_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
|
||||
ZSTDv08_dStage stage;
|
||||
U32 litEntropy;
|
||||
U32 fseEntropy;
|
||||
XXH64_state_t xxhState;
|
||||
size_t headerSize;
|
||||
U32 dictID;
|
||||
const BYTE* litPtr;
|
||||
ZSTDv08_customMem customMem;
|
||||
size_t litBufSize;
|
||||
size_t litSize;
|
||||
size_t rleSize;
|
||||
BYTE litBuffer[ZSTDv08_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
|
||||
BYTE headerBuffer[ZSTDv08_FRAMEHEADERSIZE_MAX];
|
||||
}; /* typedef'd to ZSTDv08_DCtx within "zstd_static.h" */
|
||||
|
||||
size_t ZSTDv08_sizeofDCtx (const ZSTDv08_DCtx* dctx) { return sizeof(*dctx); }
|
||||
|
||||
size_t ZSTDv08_estimateDCtxSize(void) { return sizeof(ZSTDv08_DCtx); }
|
||||
|
||||
size_t ZSTDv08_decompressBegin(ZSTDv08_DCtx* dctx)
|
||||
{
|
||||
dctx->expected = ZSTDv08_frameHeaderSize_min;
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
dctx->previousDstEnd = NULL;
|
||||
dctx->base = NULL;
|
||||
dctx->vBase = NULL;
|
||||
dctx->dictEnd = NULL;
|
||||
dctx->hufTable[0] = (HUFv08_DTable)((HufLog)*0x1000001);
|
||||
dctx->litEntropy = dctx->fseEntropy = 0;
|
||||
dctx->dictID = 0;
|
||||
{ int i; for (i=0; i<ZSTDv08_REP_NUM; i++) dctx->rep[i] = repStartValue[i]; }
|
||||
return 0;
|
||||
}
|
||||
|
||||
ZSTDv08_DCtx* ZSTDv08_createDCtx_advanced(ZSTDv08_customMem customMem)
|
||||
{
|
||||
ZSTDv08_DCtx* dctx;
|
||||
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
dctx = (ZSTDv08_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTDv08_DCtx));
|
||||
if (!dctx) return NULL;
|
||||
memcpy(&dctx->customMem, &customMem, sizeof(ZSTDv08_customMem));
|
||||
ZSTDv08_decompressBegin(dctx);
|
||||
return dctx;
|
||||
}
|
||||
|
||||
ZSTDv08_DCtx* ZSTDv08_createDCtx(void)
|
||||
{
|
||||
return ZSTDv08_createDCtx_advanced(defaultCustomMem);
|
||||
}
|
||||
|
||||
size_t ZSTDv08_freeDCtx(ZSTDv08_DCtx* dctx)
|
||||
{
|
||||
if (dctx==NULL) return 0; /* support free on NULL */
|
||||
dctx->customMem.customFree(dctx->customMem.opaque, dctx);
|
||||
return 0; /* reserved as a potential error code in the future */
|
||||
}
|
||||
|
||||
void ZSTDv08_copyDCtx(ZSTDv08_DCtx* dstDCtx, const ZSTDv08_DCtx* srcDCtx)
|
||||
{
|
||||
memcpy(dstDCtx, srcDCtx,
|
||||
sizeof(ZSTDv08_DCtx) - (ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH + ZSTDv08_frameHeaderSize_max)); /* no need to copy workspace */
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************************************
|
||||
* Decompression section
|
||||
***************************************************************/
|
||||
|
||||
/* See compression format details in : zstd_compression_format.md */
|
||||
|
||||
/** ZSTDv08_frameHeaderSize() :
|
||||
* srcSize must be >= ZSTDv08_frameHeaderSize_min.
|
||||
* @return : size of the Frame Header */
|
||||
static size_t ZSTDv08_frameHeaderSize(const void* src, size_t srcSize)
|
||||
{
|
||||
if (srcSize < ZSTDv08_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
||||
{ BYTE const fhd = ((const BYTE*)src)[4];
|
||||
U32 const dictID= fhd & 3;
|
||||
U32 const singleSegment = (fhd >> 5) & 1;
|
||||
U32 const fcsId = fhd >> 6;
|
||||
return ZSTDv08_frameHeaderSize_min + !singleSegment + ZSTDv08_did_fieldSize[dictID] + ZSTDv08_fcs_fieldSize[fcsId]
|
||||
+ (singleSegment && !ZSTDv08_fcs_fieldSize[fcsId]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** ZSTDv08_getFrameParams() :
|
||||
* decode Frame Header, or require larger `srcSize`.
|
||||
* @return : 0, `fparamsPtr` is correctly filled,
|
||||
* >0, `srcSize` is too small, result is expected `srcSize`,
|
||||
* or an error code, which can be tested using ZSTDv08_isError() */
|
||||
size_t ZSTDv08_getFrameParams(ZSTDv08_frameParams* fparamsPtr, const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
|
||||
if (srcSize < ZSTDv08_frameHeaderSize_min) return ZSTDv08_frameHeaderSize_min;
|
||||
if (MEM_readLE32(src) != ZSTDv08_MAGICNUMBER) {
|
||||
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv08_MAGIC_SKIPPABLE_START) {
|
||||
if (srcSize < ZSTDv08_skippableHeaderSize) return ZSTDv08_skippableHeaderSize; /* magic number + skippable frame length */
|
||||
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
|
||||
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
|
||||
fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */
|
||||
return 0;
|
||||
}
|
||||
return ERROR(prefix_unknown);
|
||||
}
|
||||
|
||||
/* ensure there is enough `srcSize` to fully read/decode frame header */
|
||||
{ size_t const fhsize = ZSTDv08_frameHeaderSize(src, srcSize);
|
||||
if (srcSize < fhsize) return fhsize; }
|
||||
|
||||
{ BYTE const fhdByte = ip[4];
|
||||
size_t pos = 5;
|
||||
U32 const dictIDSizeCode = fhdByte&3;
|
||||
U32 const checksumFlag = (fhdByte>>2)&1;
|
||||
U32 const singleSegment = (fhdByte>>5)&1;
|
||||
U32 const fcsID = fhdByte>>6;
|
||||
U32 const windowSizeMax = 1U << ZSTDv08_WINDOWLOG_MAX;
|
||||
U32 windowSize = 0;
|
||||
U32 dictID = 0;
|
||||
U64 frameContentSize = 0;
|
||||
if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */
|
||||
if (!singleSegment) {
|
||||
BYTE const wlByte = ip[pos++];
|
||||
U32 const windowLog = (wlByte >> 3) + ZSTDv08_WINDOWLOG_ABSOLUTEMIN;
|
||||
if (windowLog > ZSTDv08_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
|
||||
windowSize = (1U << windowLog);
|
||||
windowSize += (windowSize >> 3) * (wlByte&7);
|
||||
}
|
||||
|
||||
switch(dictIDSizeCode)
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : break;
|
||||
case 1 : dictID = ip[pos]; pos++; break;
|
||||
case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
|
||||
case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
|
||||
}
|
||||
switch(fcsID)
|
||||
{
|
||||
default: /* impossible */
|
||||
case 0 : if (singleSegment) frameContentSize = ip[pos]; break;
|
||||
case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
|
||||
case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
|
||||
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
|
||||
}
|
||||
if (!windowSize) windowSize = (U32)frameContentSize;
|
||||
if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
|
||||
fparamsPtr->frameContentSize = frameContentSize;
|
||||
fparamsPtr->windowSize = windowSize;
|
||||
fparamsPtr->dictID = dictID;
|
||||
fparamsPtr->checksumFlag = checksumFlag;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/** ZSTDv08_getDecompressedSize() :
|
||||
* compatible with legacy mode
|
||||
* @return : decompressed size if known, 0 otherwise
|
||||
note : 0 can mean any of the following :
|
||||
- decompressed size is not present within frame header
|
||||
- frame header unknown / not supported
|
||||
- frame header not complete (`srcSize` too small) */
|
||||
unsigned long long ZSTDv08_getDecompressedSize(const void* src, size_t srcSize)
|
||||
{
|
||||
{ ZSTDv08_frameParams fparams;
|
||||
size_t const frResult = ZSTDv08_getFrameParams(&fparams, src, srcSize);
|
||||
if (frResult!=0) return 0;
|
||||
return fparams.frameContentSize;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** ZSTDv08_decodeFrameHeader() :
|
||||
* `srcSize` must be the size provided by ZSTDv08_frameHeaderSize().
|
||||
* @return : 0 if success, or an error code, which can be tested using ZSTDv08_isError() */
|
||||
static size_t ZSTDv08_decodeFrameHeader(ZSTDv08_DCtx* dctx, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const result = ZSTDv08_getFrameParams(&(dctx->fParams), src, srcSize);
|
||||
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
|
||||
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
typedef struct
|
||||
{
|
||||
blockType_e blockType;
|
||||
U32 lastBlock;
|
||||
U32 origSize;
|
||||
} blockProperties_t;
|
||||
|
||||
/*! ZSTDv08_getcBlockSize() :
|
||||
* Provides the size of compressed block from block header `src` */
|
||||
size_t ZSTDv08_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
||||
{
|
||||
if (srcSize < ZSTDv08_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
{ U32 const cBlockHeader = MEM_readLE24(src);
|
||||
U32 const cSize = cBlockHeader >> 3;
|
||||
bpPtr->lastBlock = cBlockHeader & 1;
|
||||
bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
|
||||
bpPtr->origSize = cSize; /* only useful for RLE */
|
||||
if (bpPtr->blockType == bt_rle) return 1;
|
||||
if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected);
|
||||
return cSize;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTDv08_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||
memcpy(dst, src, srcSize);
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTDv08_setRleBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, size_t regenSize)
|
||||
{
|
||||
if (srcSize != 1) return ERROR(srcSize_wrong);
|
||||
if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||
memset(dst, *(const BYTE*)src, regenSize);
|
||||
return regenSize;
|
||||
}
|
||||
|
||||
/*! ZSTDv08_decodeLiteralsBlock() :
|
||||
@return : nb of bytes read from src (< srcSize ) */
|
||||
size_t ZSTDv08_decodeLiteralsBlock(ZSTDv08_DCtx* dctx,
|
||||
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
||||
{
|
||||
if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected);
|
||||
|
||||
{ const BYTE* const istart = (const BYTE*) src;
|
||||
symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
|
||||
|
||||
switch(litEncType)
|
||||
{
|
||||
case set_repeat:
|
||||
if (dctx->litEntropy==0) return ERROR(dictionary_corrupted);
|
||||
/* fall-through */
|
||||
case set_compressed:
|
||||
if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */
|
||||
{ size_t lhSize, litSize, litCSize;
|
||||
U32 singleStream=0;
|
||||
U32 const lhlCode = (istart[0] >> 2) & 3;
|
||||
U32 const lhc = MEM_readLE32(istart);
|
||||
switch(lhlCode)
|
||||
{
|
||||
case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
|
||||
/* 2 - 2 - 10 - 10 */
|
||||
{ singleStream = !lhlCode;
|
||||
lhSize = 3;
|
||||
litSize = (lhc >> 4) & 0x3FF;
|
||||
litCSize = (lhc >> 14) & 0x3FF;
|
||||
break;
|
||||
}
|
||||
case 2:
|
||||
/* 2 - 2 - 14 - 14 */
|
||||
{ lhSize = 4;
|
||||
litSize = (lhc >> 4) & 0x3FFF;
|
||||
litCSize = lhc >> 18;
|
||||
break;
|
||||
}
|
||||
case 3:
|
||||
/* 2 - 2 - 18 - 18 */
|
||||
{ lhSize = 5;
|
||||
litSize = (lhc >> 4) & 0x3FFFF;
|
||||
litCSize = (lhc >> 22) + (istart[4] << 10);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (litSize > ZSTDv08_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
|
||||
|
||||
if (HUFv08_isError((litEncType==set_repeat) ?
|
||||
( singleStream ?
|
||||
HUFv08_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) :
|
||||
HUFv08_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) ) :
|
||||
( singleStream ?
|
||||
HUFv08_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
|
||||
HUFv08_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
|
||||
return ERROR(corruption_detected);
|
||||
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
|
||||
dctx->litSize = litSize;
|
||||
dctx->litEntropy = 1;
|
||||
return litCSize + lhSize;
|
||||
}
|
||||
|
||||
case set_basic:
|
||||
{ size_t litSize, lhSize;
|
||||
U32 const lhlCode = ((istart[0]) >> 2) & 3;
|
||||
switch(lhlCode)
|
||||
{
|
||||
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
|
||||
lhSize = 1;
|
||||
litSize = istart[0] >> 3;
|
||||
break;
|
||||
case 1:
|
||||
lhSize = 2;
|
||||
litSize = MEM_readLE16(istart) >> 4;
|
||||
break;
|
||||
case 3:
|
||||
lhSize = 3;
|
||||
litSize = MEM_readLE24(istart) >> 4;
|
||||
break;
|
||||
}
|
||||
|
||||
if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
|
||||
if (litSize+lhSize > srcSize) return ERROR(corruption_detected);
|
||||
memcpy(dctx->litBuffer, istart+lhSize, litSize);
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+8;
|
||||
dctx->litSize = litSize;
|
||||
return lhSize+litSize;
|
||||
}
|
||||
/* direct reference into compressed stream */
|
||||
dctx->litPtr = istart+lhSize;
|
||||
dctx->litBufSize = srcSize-lhSize;
|
||||
dctx->litSize = litSize;
|
||||
return lhSize+litSize;
|
||||
}
|
||||
|
||||
case set_rle:
|
||||
{ U32 const lhlCode = ((istart[0]) >> 2) & 3;
|
||||
size_t litSize, lhSize;
|
||||
switch(lhlCode)
|
||||
{
|
||||
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
|
||||
lhSize = 1;
|
||||
litSize = istart[0] >> 3;
|
||||
break;
|
||||
case 1:
|
||||
lhSize = 2;
|
||||
litSize = MEM_readLE16(istart) >> 4;
|
||||
break;
|
||||
case 3:
|
||||
lhSize = 3;
|
||||
litSize = MEM_readLE24(istart) >> 4;
|
||||
if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
|
||||
break;
|
||||
}
|
||||
if (litSize > ZSTDv08_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
|
||||
memset(dctx->litBuffer, istart[lhSize], litSize);
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
|
||||
dctx->litSize = litSize;
|
||||
return lhSize+1;
|
||||
}
|
||||
default:
|
||||
return ERROR(corruption_detected); /* impossible */
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTDv08_buildSeqTable() :
|
||||
@return : nb bytes read from src,
|
||||
or an error code if it fails, testable with ZSTDv08_isError()
|
||||
*/
|
||||
FORCE_INLINE size_t ZSTDv08_buildSeqTable(FSEv08_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog,
|
||||
const void* src, size_t srcSize,
|
||||
const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
|
||||
{
|
||||
switch(type)
|
||||
{
|
||||
case set_rle :
|
||||
if (!srcSize) return ERROR(srcSize_wrong);
|
||||
if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected);
|
||||
FSEv08_buildDTable_rle(DTable, *(const BYTE*)src); /* if *src > max, data is corrupted */
|
||||
return 1;
|
||||
case set_basic :
|
||||
FSEv08_buildDTable(DTable, defaultNorm, max, defaultLog);
|
||||
return 0;
|
||||
case set_repeat:
|
||||
if (!flagRepeatTable) return ERROR(corruption_detected);
|
||||
return 0;
|
||||
default : /* impossible */
|
||||
case set_compressed :
|
||||
{ U32 tableLog;
|
||||
S16 norm[MaxSeq+1];
|
||||
size_t const headerSize = FSEv08_readNCount(norm, &max, &tableLog, src, srcSize);
|
||||
if (FSEv08_isError(headerSize)) return ERROR(corruption_detected);
|
||||
if (tableLog > maxLog) return ERROR(corruption_detected);
|
||||
FSEv08_buildDTable(DTable, norm, max, tableLog);
|
||||
return headerSize;
|
||||
} }
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv08_decodeSeqHeaders(int* nbSeqPtr,
|
||||
FSEv08_DTable* DTableLL, FSEv08_DTable* DTableML, FSEv08_DTable* DTableOffb, U32 flagRepeatTable,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE* const)src;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* ip = istart;
|
||||
|
||||
/* check */
|
||||
if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong);
|
||||
|
||||
/* SeqHead */
|
||||
{ int nbSeq = *ip++;
|
||||
if (!nbSeq) { *nbSeqPtr=0; return 1; }
|
||||
if (nbSeq > 0x7F) {
|
||||
if (nbSeq == 0xFF)
|
||||
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
|
||||
else
|
||||
nbSeq = ((nbSeq-0x80)<<8) + *ip++;
|
||||
}
|
||||
*nbSeqPtr = nbSeq;
|
||||
}
|
||||
|
||||
/* FSE table descriptors */
|
||||
if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */
|
||||
{ symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
|
||||
symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
|
||||
symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
|
||||
ip++;
|
||||
|
||||
/* Build DTables */
|
||||
{ size_t const llhSize = ZSTDv08_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
|
||||
if (ZSTDv08_isError(llhSize)) return ERROR(corruption_detected);
|
||||
ip += llhSize;
|
||||
}
|
||||
{ size_t const ofhSize = ZSTDv08_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable);
|
||||
if (ZSTDv08_isError(ofhSize)) return ERROR(corruption_detected);
|
||||
ip += ofhSize;
|
||||
}
|
||||
{ size_t const mlhSize = ZSTDv08_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable);
|
||||
if (ZSTDv08_isError(mlhSize)) return ERROR(corruption_detected);
|
||||
ip += mlhSize;
|
||||
} }
|
||||
|
||||
return ip-istart;
|
||||
}
|
||||
|
||||
|
||||
typedef struct {
|
||||
size_t litLength;
|
||||
size_t matchLength;
|
||||
size_t offset;
|
||||
} seq_t;
|
||||
|
||||
typedef struct {
|
||||
BIT_DStream_t DStream;
|
||||
FSEv08_DState_t stateLL;
|
||||
FSEv08_DState_t stateOffb;
|
||||
FSEv08_DState_t stateML;
|
||||
size_t prevOffset[ZSTDv08_REP_NUM];
|
||||
} seqState_t;
|
||||
|
||||
|
||||
static seq_t ZSTDv08_decodeSequence(seqState_t* seqState)
|
||||
{
|
||||
seq_t seq;
|
||||
|
||||
U32 const llCode = FSEv08_peekSymbol(&(seqState->stateLL));
|
||||
U32 const mlCode = FSEv08_peekSymbol(&(seqState->stateML));
|
||||
U32 const ofCode = FSEv08_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */
|
||||
|
||||
U32 const llBits = LL_bits[llCode];
|
||||
U32 const mlBits = ML_bits[mlCode];
|
||||
U32 const ofBits = ofCode;
|
||||
U32 const totalBits = llBits+mlBits+ofBits;
|
||||
|
||||
static const U32 LL_base[MaxLL+1] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
|
||||
0x2000, 0x4000, 0x8000, 0x10000 };
|
||||
|
||||
static const U32 ML_base[MaxML+1] = {
|
||||
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
|
||||
19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
|
||||
35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
|
||||
0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
|
||||
|
||||
static const U32 OF_base[MaxOff+1] = {
|
||||
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
|
||||
0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
|
||||
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
|
||||
0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
|
||||
|
||||
/* sequence */
|
||||
{ size_t offset;
|
||||
if (!ofCode)
|
||||
offset = 0;
|
||||
else {
|
||||
offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= (ZSTDv08_WINDOWLOG_MAX-1) bits */
|
||||
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
|
||||
}
|
||||
|
||||
if (ofCode <= 1) {
|
||||
offset += (llCode==0);
|
||||
if (offset) {
|
||||
size_t const temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
|
||||
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
|
||||
seqState->prevOffset[1] = seqState->prevOffset[0];
|
||||
seqState->prevOffset[0] = offset = temp;
|
||||
} else {
|
||||
offset = seqState->prevOffset[0];
|
||||
}
|
||||
} else {
|
||||
seqState->prevOffset[2] = seqState->prevOffset[1];
|
||||
seqState->prevOffset[1] = seqState->prevOffset[0];
|
||||
seqState->prevOffset[0] = offset;
|
||||
}
|
||||
seq.offset = offset;
|
||||
}
|
||||
|
||||
seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */
|
||||
if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));
|
||||
|
||||
seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */
|
||||
if (MEM_32bits() ||
|
||||
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));
|
||||
|
||||
/* ANS state update */
|
||||
FSEv08_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */
|
||||
FSEv08_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */
|
||||
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */
|
||||
FSEv08_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */
|
||||
|
||||
return seq;
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE
|
||||
size_t ZSTDv08_execSequence(BYTE* op,
|
||||
BYTE* const oend, seq_t sequence,
|
||||
const BYTE** litPtr, const BYTE* const litLimit_w,
|
||||
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
|
||||
{
|
||||
BYTE* const oLitEnd = op + sequence.litLength;
|
||||
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
|
||||
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
|
||||
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
|
||||
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
|
||||
const BYTE* match = oLitEnd - sequence.offset;
|
||||
|
||||
/* check */
|
||||
if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
|
||||
if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
|
||||
|
||||
/* copy Literals */
|
||||
ZSTDv08_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
|
||||
op = oLitEnd;
|
||||
*litPtr = iLitEnd; /* update for next sequence */
|
||||
|
||||
/* copy Match */
|
||||
if (sequence.offset > (size_t)(oLitEnd - base)) {
|
||||
/* offset beyond prefix */
|
||||
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
|
||||
match = dictEnd - (base-match);
|
||||
if (match + sequence.matchLength <= dictEnd) {
|
||||
memmove(oLitEnd, match, sequence.matchLength);
|
||||
return sequenceLength;
|
||||
}
|
||||
/* span extDict & currentPrefixSegment */
|
||||
{ size_t const length1 = dictEnd - match;
|
||||
memmove(oLitEnd, match, length1);
|
||||
op = oLitEnd + length1;
|
||||
sequence.matchLength -= length1;
|
||||
match = base;
|
||||
} }
|
||||
|
||||
/* match within prefix */
|
||||
if (sequence.offset < 8) {
|
||||
/* close range match, overlap */
|
||||
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
|
||||
int const sub2 = dec64table[sequence.offset];
|
||||
op[0] = match[0];
|
||||
op[1] = match[1];
|
||||
op[2] = match[2];
|
||||
op[3] = match[3];
|
||||
match += dec32table[sequence.offset];
|
||||
ZSTDv08_copy4(op+4, match);
|
||||
match -= sub2;
|
||||
} else {
|
||||
ZSTDv08_copy8(op, match);
|
||||
}
|
||||
op += 8; match += 8;
|
||||
|
||||
if (oMatchEnd > oend-(16-MINMATCH)) {
|
||||
if (op < oend_w) {
|
||||
ZSTDv08_wildcopy(op, match, oend_w - op);
|
||||
match += oend_w - op;
|
||||
op = oend_w;
|
||||
}
|
||||
while (op < oMatchEnd) *op++ = *match++;
|
||||
} else {
|
||||
ZSTDv08_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
||||
}
|
||||
return sequenceLength;
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTDv08_decompressSequences(
|
||||
ZSTDv08_DCtx* dctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* seqStart, size_t seqSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)seqStart;
|
||||
const BYTE* const iend = ip + seqSize;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
BYTE* const oend = ostart + maxDstSize;
|
||||
BYTE* op = ostart;
|
||||
const BYTE* litPtr = dctx->litPtr;
|
||||
const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
|
||||
const BYTE* const litEnd = litPtr + dctx->litSize;
|
||||
FSEv08_DTable* DTableLL = dctx->LLTable;
|
||||
FSEv08_DTable* DTableML = dctx->MLTable;
|
||||
FSEv08_DTable* DTableOffb = dctx->OffTable;
|
||||
const BYTE* const base = (const BYTE*) (dctx->base);
|
||||
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
|
||||
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
|
||||
int nbSeq;
|
||||
|
||||
/* Build Decoding Tables */
|
||||
{ size_t const seqHSize = ZSTDv08_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize);
|
||||
if (ZSTDv08_isError(seqHSize)) return seqHSize;
|
||||
ip += seqHSize;
|
||||
}
|
||||
|
||||
/* Regen sequences */
|
||||
if (nbSeq) {
|
||||
seqState_t seqState;
|
||||
dctx->fseEntropy = 1;
|
||||
{ U32 i; for (i=0; i<ZSTDv08_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; }
|
||||
{ size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
|
||||
if (ERR_isError(errorCode)) return ERROR(corruption_detected); }
|
||||
FSEv08_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
|
||||
FSEv08_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
|
||||
FSEv08_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
|
||||
|
||||
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
|
||||
nbSeq--;
|
||||
{ seq_t const sequence = ZSTDv08_decodeSequence(&seqState);
|
||||
size_t const oneSeqSize = ZSTDv08_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd);
|
||||
if (ZSTDv08_isError(oneSeqSize)) return oneSeqSize;
|
||||
op += oneSeqSize;
|
||||
} }
|
||||
|
||||
/* check if reached exact end */
|
||||
if (nbSeq) return ERROR(corruption_detected);
|
||||
/* save reps for next block */
|
||||
{ U32 i; for (i=0; i<ZSTDv08_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); }
|
||||
}
|
||||
|
||||
/* last literal segment */
|
||||
{ size_t const lastLLSize = litEnd - litPtr;
|
||||
if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall);
|
||||
memcpy(op, litPtr, lastLLSize);
|
||||
op += lastLLSize;
|
||||
}
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
static void ZSTDv08_checkContinuity(ZSTDv08_DCtx* dctx, const void* dst)
|
||||
{
|
||||
if (dst != dctx->previousDstEnd) { /* not contiguous */
|
||||
dctx->dictEnd = dctx->previousDstEnd;
|
||||
dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
|
||||
dctx->base = dst;
|
||||
dctx->previousDstEnd = dst;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTDv08_decompressBlock_internal(ZSTDv08_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{ /* blockType == blockCompressed */
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
|
||||
if (srcSize >= ZSTDv08_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong);
|
||||
|
||||
/* Decode literals sub-block */
|
||||
{ size_t const litCSize = ZSTDv08_decodeLiteralsBlock(dctx, src, srcSize);
|
||||
if (ZSTDv08_isError(litCSize)) return litCSize;
|
||||
ip += litCSize;
|
||||
srcSize -= litCSize;
|
||||
}
|
||||
return ZSTDv08_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv08_decompressBlock(ZSTDv08_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
size_t dSize;
|
||||
ZSTDv08_checkContinuity(dctx, dst);
|
||||
dSize = ZSTDv08_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
|
||||
dctx->previousDstEnd = (char*)dst + dSize;
|
||||
return dSize;
|
||||
}
|
||||
|
||||
|
||||
/** ZSTDv08_insertBlock() :
|
||||
insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
|
||||
ZSTDLIB_API size_t ZSTDv08_insertBlock(ZSTDv08_DCtx* dctx, const void* blockStart, size_t blockSize)
|
||||
{
|
||||
ZSTDv08_checkContinuity(dctx, blockStart);
|
||||
dctx->previousDstEnd = (const char*)blockStart + blockSize;
|
||||
return blockSize;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv08_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
|
||||
{
|
||||
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||
memset(dst, byte, length);
|
||||
return length;
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTDv08_decompressFrame() :
|
||||
* `dctx` must be properly initialized */
|
||||
static size_t ZSTDv08_decompressFrame(ZSTDv08_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
BYTE* const oend = ostart + dstCapacity;
|
||||
BYTE* op = ostart;
|
||||
size_t remainingSize = srcSize;
|
||||
|
||||
/* check */
|
||||
if (srcSize < ZSTDv08_frameHeaderSize_min+ZSTDv08_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
|
||||
/* Frame Header */
|
||||
{ size_t const frameHeaderSize = ZSTDv08_frameHeaderSize(src, ZSTDv08_frameHeaderSize_min);
|
||||
size_t result;
|
||||
if (ZSTDv08_isError(frameHeaderSize)) return frameHeaderSize;
|
||||
if (srcSize < frameHeaderSize+ZSTDv08_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
result = ZSTDv08_decodeFrameHeader(dctx, src, frameHeaderSize);
|
||||
if (ZSTDv08_isError(result)) return result;
|
||||
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
||||
}
|
||||
|
||||
/* Loop on each block */
|
||||
while (1) {
|
||||
size_t decodedSize;
|
||||
blockProperties_t blockProperties;
|
||||
size_t const cBlockSize = ZSTDv08_getcBlockSize(ip, remainingSize, &blockProperties);
|
||||
if (ZSTDv08_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
ip += ZSTDv08_blockHeaderSize;
|
||||
remainingSize -= ZSTDv08_blockHeaderSize;
|
||||
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
switch(blockProperties.blockType)
|
||||
{
|
||||
case bt_compressed:
|
||||
decodedSize = ZSTDv08_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize);
|
||||
break;
|
||||
case bt_raw :
|
||||
decodedSize = ZSTDv08_copyRawBlock(op, oend-op, ip, cBlockSize);
|
||||
break;
|
||||
case bt_rle :
|
||||
decodedSize = ZSTDv08_generateNxBytes(op, oend-op, *ip, blockProperties.origSize);
|
||||
break;
|
||||
case bt_reserved :
|
||||
default:
|
||||
return ERROR(corruption_detected);
|
||||
}
|
||||
|
||||
if (ZSTDv08_isError(decodedSize)) return decodedSize;
|
||||
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize);
|
||||
op += decodedSize;
|
||||
ip += cBlockSize;
|
||||
remainingSize -= cBlockSize;
|
||||
if (blockProperties.lastBlock) break;
|
||||
}
|
||||
|
||||
if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
|
||||
U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
|
||||
U32 checkRead;
|
||||
if (remainingSize<4) return ERROR(checksum_wrong);
|
||||
checkRead = MEM_readLE32(ip);
|
||||
if (checkRead != checkCalc) return ERROR(checksum_wrong);
|
||||
remainingSize -= 4;
|
||||
}
|
||||
|
||||
if (remainingSize) return ERROR(srcSize_wrong);
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTDv08_decompress_usingPreparedDCtx() :
|
||||
* Same as ZSTDv08_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
|
||||
* It avoids reloading the dictionary each time.
|
||||
* `preparedDCtx` must have been properly initialized using ZSTDv08_decompressBegin_usingDict().
|
||||
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
|
||||
size_t ZSTDv08_decompress_usingPreparedDCtx(ZSTDv08_DCtx* dctx, const ZSTDv08_DCtx* refDCtx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTDv08_copyDCtx(dctx, refDCtx);
|
||||
ZSTDv08_checkContinuity(dctx, dst);
|
||||
return ZSTDv08_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv08_decompress_usingDict(ZSTDv08_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize)
|
||||
{
|
||||
ZSTDv08_decompressBegin_usingDict(dctx, dict, dictSize);
|
||||
ZSTDv08_checkContinuity(dctx, dst);
|
||||
return ZSTDv08_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv08_decompressDCtx(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTDv08_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv08_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
#if defined(ZSTDv08_HEAPMODE) && (ZSTDv08_HEAPMODE==1)
|
||||
size_t regenSize;
|
||||
ZSTDv08_DCtx* const dctx = ZSTDv08_createDCtx();
|
||||
if (dctx==NULL) return ERROR(memory_allocation);
|
||||
regenSize = ZSTDv08_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
|
||||
ZSTDv08_freeDCtx(dctx);
|
||||
return regenSize;
|
||||
#else /* stack mode */
|
||||
ZSTDv08_DCtx dctx;
|
||||
return ZSTDv08_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
/*-**********************************
|
||||
* Streaming Decompression API
|
||||
************************************/
|
||||
size_t ZSTDv08_nextSrcSizeToDecompress(ZSTDv08_DCtx* dctx) { return dctx->expected; }
|
||||
|
||||
ZSTDv08_nextInputType_e ZSTDv08_nextInputType(ZSTDv08_DCtx* dctx) {
|
||||
switch(dctx->stage)
|
||||
{
|
||||
default: /* should not happen */
|
||||
case ZSTDds_getFrameHeaderSize:
|
||||
case ZSTDds_decodeFrameHeader:
|
||||
return ZSTDnit_frameHeader;
|
||||
case ZSTDds_decodeBlockHeader:
|
||||
return ZSTDnit_blockHeader;
|
||||
case ZSTDds_decompressBlock:
|
||||
return ZSTDnit_block;
|
||||
case ZSTDds_decompressLastBlock:
|
||||
return ZSTDnit_lastBlock;
|
||||
case ZSTDds_checkChecksum:
|
||||
return ZSTDnit_checksum;
|
||||
case ZSTDds_decodeSkippableHeader:
|
||||
case ZSTDds_skipFrame:
|
||||
return ZSTDnit_skippableFrame;
|
||||
}
|
||||
}
|
||||
|
||||
int ZSTDv08_isSkipFrame(ZSTDv08_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */
|
||||
|
||||
/** ZSTDv08_decompressContinue() :
|
||||
* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
|
||||
* or an error code, which can be tested using ZSTDv08_isError() */
|
||||
size_t ZSTDv08_decompressContinue(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
/* Sanity check */
|
||||
if (srcSize != dctx->expected) return ERROR(srcSize_wrong);
|
||||
if (dstCapacity) ZSTDv08_checkContinuity(dctx, dst);
|
||||
|
||||
switch (dctx->stage)
|
||||
{
|
||||
case ZSTDds_getFrameHeaderSize :
|
||||
if (srcSize != ZSTDv08_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv08_MAGIC_SKIPPABLE_START) {
|
||||
memcpy(dctx->headerBuffer, src, ZSTDv08_frameHeaderSize_min);
|
||||
dctx->expected = ZSTDv08_skippableHeaderSize - ZSTDv08_frameHeaderSize_min; /* magic number + skippable frame length */
|
||||
dctx->stage = ZSTDds_decodeSkippableHeader;
|
||||
return 0;
|
||||
}
|
||||
dctx->headerSize = ZSTDv08_frameHeaderSize(src, ZSTDv08_frameHeaderSize_min);
|
||||
if (ZSTDv08_isError(dctx->headerSize)) return dctx->headerSize;
|
||||
memcpy(dctx->headerBuffer, src, ZSTDv08_frameHeaderSize_min);
|
||||
if (dctx->headerSize > ZSTDv08_frameHeaderSize_min) {
|
||||
dctx->expected = dctx->headerSize - ZSTDv08_frameHeaderSize_min;
|
||||
dctx->stage = ZSTDds_decodeFrameHeader;
|
||||
return 0;
|
||||
}
|
||||
dctx->expected = 0; /* not necessary to copy more */
|
||||
|
||||
case ZSTDds_decodeFrameHeader:
|
||||
{ size_t result;
|
||||
memcpy(dctx->headerBuffer + ZSTDv08_frameHeaderSize_min, src, dctx->expected);
|
||||
result = ZSTDv08_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize);
|
||||
if (ZSTDv08_isError(result)) return result;
|
||||
dctx->expected = ZSTDv08_blockHeaderSize;
|
||||
dctx->stage = ZSTDds_decodeBlockHeader;
|
||||
return 0;
|
||||
}
|
||||
case ZSTDds_decodeBlockHeader:
|
||||
{ blockProperties_t bp;
|
||||
size_t const cBlockSize = ZSTDv08_getcBlockSize(src, ZSTDv08_blockHeaderSize, &bp);
|
||||
if (ZSTDv08_isError(cBlockSize)) return cBlockSize;
|
||||
dctx->expected = cBlockSize;
|
||||
dctx->bType = bp.blockType;
|
||||
dctx->rleSize = bp.origSize;
|
||||
if (cBlockSize) {
|
||||
dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock;
|
||||
return 0;
|
||||
}
|
||||
/* empty block */
|
||||
if (bp.lastBlock) {
|
||||
if (dctx->fParams.checksumFlag) {
|
||||
dctx->expected = 4;
|
||||
dctx->stage = ZSTDds_checkChecksum;
|
||||
} else {
|
||||
dctx->expected = 0; /* end of frame */
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
}
|
||||
} else {
|
||||
dctx->expected = 3; /* go directly to next header */
|
||||
dctx->stage = ZSTDds_decodeBlockHeader;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
case ZSTDds_decompressLastBlock:
|
||||
case ZSTDds_decompressBlock:
|
||||
{ size_t rSize;
|
||||
switch(dctx->bType)
|
||||
{
|
||||
case bt_compressed:
|
||||
rSize = ZSTDv08_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
|
||||
break;
|
||||
case bt_raw :
|
||||
rSize = ZSTDv08_copyRawBlock(dst, dstCapacity, src, srcSize);
|
||||
break;
|
||||
case bt_rle :
|
||||
rSize = ZSTDv08_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize);
|
||||
break;
|
||||
case bt_reserved : /* should never happen */
|
||||
default:
|
||||
return ERROR(corruption_detected);
|
||||
}
|
||||
if (ZSTDv08_isError(rSize)) return rSize;
|
||||
if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize);
|
||||
|
||||
if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
|
||||
if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
|
||||
dctx->expected = 4;
|
||||
dctx->stage = ZSTDds_checkChecksum;
|
||||
} else {
|
||||
dctx->expected = 0; /* ends here */
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
}
|
||||
} else {
|
||||
dctx->stage = ZSTDds_decodeBlockHeader;
|
||||
dctx->expected = ZSTDv08_blockHeaderSize;
|
||||
dctx->previousDstEnd = (char*)dst + rSize;
|
||||
}
|
||||
return rSize;
|
||||
}
|
||||
case ZSTDds_checkChecksum:
|
||||
{ U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
|
||||
U32 const check32 = MEM_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */
|
||||
if (check32 != h32) return ERROR(checksum_wrong);
|
||||
dctx->expected = 0;
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
return 0;
|
||||
}
|
||||
case ZSTDds_decodeSkippableHeader:
|
||||
{ memcpy(dctx->headerBuffer + ZSTDv08_frameHeaderSize_min, src, dctx->expected);
|
||||
dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
|
||||
dctx->stage = ZSTDds_skipFrame;
|
||||
return 0;
|
||||
}
|
||||
case ZSTDds_skipFrame:
|
||||
{ dctx->expected = 0;
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
return 0;
|
||||
}
|
||||
default:
|
||||
return ERROR(GENERIC); /* impossible */
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTDv08_refDictContent(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
{
|
||||
dctx->dictEnd = dctx->previousDstEnd;
|
||||
dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base));
|
||||
dctx->base = dict;
|
||||
dctx->previousDstEnd = (const char*)dict + dictSize;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t ZSTDv08_loadEntropy(ZSTDv08_DCtx* dctx, const void* const dict, size_t const dictSize)
|
||||
{
|
||||
const BYTE* dictPtr = (const BYTE*)dict;
|
||||
const BYTE* const dictEnd = dictPtr + dictSize;
|
||||
|
||||
{ size_t const hSize = HUFv08_readDTableX4(dctx->hufTable, dict, dictSize);
|
||||
if (HUFv08_isError(hSize)) return ERROR(dictionary_corrupted);
|
||||
dictPtr += hSize;
|
||||
}
|
||||
|
||||
{ short offcodeNCount[MaxOff+1];
|
||||
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
|
||||
size_t const offcodeHeaderSize = FSEv08_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
|
||||
if (FSEv08_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSEv08_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
|
||||
if (FSEv08_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += offcodeHeaderSize;
|
||||
}
|
||||
|
||||
{ short matchlengthNCount[MaxML+1];
|
||||
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
|
||||
size_t const matchlengthHeaderSize = FSEv08_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
|
||||
if (FSEv08_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSEv08_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
|
||||
if (FSEv08_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += matchlengthHeaderSize;
|
||||
}
|
||||
|
||||
{ short litlengthNCount[MaxLL+1];
|
||||
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
|
||||
size_t const litlengthHeaderSize = FSEv08_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
|
||||
if (FSEv08_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
|
||||
{ size_t const errorCode = FSEv08_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
|
||||
if (FSEv08_isError(errorCode)) return ERROR(dictionary_corrupted); }
|
||||
dictPtr += litlengthHeaderSize;
|
||||
}
|
||||
|
||||
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
|
||||
dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
|
||||
dictPtr += 12;
|
||||
|
||||
dctx->litEntropy = dctx->fseEntropy = 1;
|
||||
return dictPtr - (const BYTE*)dict;
|
||||
}
|
||||
|
||||
static size_t ZSTDv08_decompress_insertDictionary(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
{
|
||||
if (dictSize < 8) return ZSTDv08_refDictContent(dctx, dict, dictSize);
|
||||
{ U32 const magic = MEM_readLE32(dict);
|
||||
if (magic != ZSTDv08_DICT_MAGIC) {
|
||||
return ZSTDv08_refDictContent(dctx, dict, dictSize); /* pure content mode */
|
||||
} }
|
||||
dctx->dictID = MEM_readLE32((const char*)dict + 4);
|
||||
|
||||
/* load entropy tables */
|
||||
dict = (const char*)dict + 8;
|
||||
dictSize -= 8;
|
||||
{ size_t const eSize = ZSTDv08_loadEntropy(dctx, dict, dictSize);
|
||||
if (ZSTDv08_isError(eSize)) return ERROR(dictionary_corrupted);
|
||||
dict = (const char*)dict + eSize;
|
||||
dictSize -= eSize;
|
||||
}
|
||||
|
||||
/* reference dictionary content */
|
||||
return ZSTDv08_refDictContent(dctx, dict, dictSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTDv08_decompressBegin_usingDict(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize)
|
||||
{
|
||||
{ size_t const errorCode = ZSTDv08_decompressBegin(dctx);
|
||||
if (ZSTDv08_isError(errorCode)) return errorCode; }
|
||||
|
||||
if (dict && dictSize) {
|
||||
size_t const errorCode = ZSTDv08_decompress_insertDictionary(dctx, dict, dictSize);
|
||||
if (ZSTDv08_isError(errorCode)) return ERROR(dictionary_corrupted);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
struct ZSTDv08_DDict_s {
|
||||
void* dict;
|
||||
size_t dictSize;
|
||||
ZSTDv08_DCtx* refContext;
|
||||
}; /* typedef'd tp ZSTDv08_CDict within zstd.h */
|
||||
|
||||
ZSTDv08_DDict* ZSTDv08_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv08_customMem customMem)
|
||||
{
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
{ ZSTDv08_DDict* const ddict = (ZSTDv08_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict));
|
||||
void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize);
|
||||
ZSTDv08_DCtx* const dctx = ZSTDv08_createDCtx_advanced(customMem);
|
||||
|
||||
if (!dictContent || !ddict || !dctx) {
|
||||
customMem.customFree(customMem.opaque, dictContent);
|
||||
customMem.customFree(customMem.opaque, ddict);
|
||||
customMem.customFree(customMem.opaque, dctx);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memcpy(dictContent, dict, dictSize);
|
||||
{ size_t const errorCode = ZSTDv08_decompressBegin_usingDict(dctx, dictContent, dictSize);
|
||||
if (ZSTDv08_isError(errorCode)) {
|
||||
customMem.customFree(customMem.opaque, dictContent);
|
||||
customMem.customFree(customMem.opaque, ddict);
|
||||
customMem.customFree(customMem.opaque, dctx);
|
||||
return NULL;
|
||||
} }
|
||||
|
||||
ddict->dict = dictContent;
|
||||
ddict->dictSize = dictSize;
|
||||
ddict->refContext = dctx;
|
||||
return ddict;
|
||||
}
|
||||
}
|
||||
|
||||
/*! ZSTDv08_createDDict() :
|
||||
* Create a digested dictionary, ready to start decompression without startup delay.
|
||||
* `dict` can be released after `ZSTDv08_DDict` creation */
|
||||
ZSTDv08_DDict* ZSTDv08_createDDict(const void* dict, size_t dictSize)
|
||||
{
|
||||
ZSTDv08_customMem const allocator = { NULL, NULL, NULL };
|
||||
return ZSTDv08_createDDict_advanced(dict, dictSize, allocator);
|
||||
}
|
||||
|
||||
size_t ZSTDv08_freeDDict(ZSTDv08_DDict* ddict)
|
||||
{
|
||||
ZSTDv08_freeFunction const cFree = ddict->refContext->customMem.customFree;
|
||||
void* const opaque = ddict->refContext->customMem.opaque;
|
||||
ZSTDv08_freeDCtx(ddict->refContext);
|
||||
cFree(opaque, ddict->dict);
|
||||
cFree(opaque, ddict);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*! ZSTDv08_decompress_usingDDict() :
|
||||
* Decompression using a pre-digested Dictionary
|
||||
* Use dictionary without significant overhead. */
|
||||
ZSTDLIB_API size_t ZSTDv08_decompress_usingDDict(ZSTDv08_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTDv08_DDict* ddict)
|
||||
{
|
||||
return ZSTDv08_decompress_usingPreparedDCtx(dctx, ddict->refContext,
|
||||
dst, dstCapacity,
|
||||
src, srcSize);
|
||||
}
|
||||
/*
|
||||
Buffered version of Zstd compression library
|
||||
Copyright (C) 2015-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net/
|
||||
*/
|
||||
|
||||
|
||||
/*-***************************************************************************
|
||||
* Streaming decompression howto
|
||||
*
|
||||
* A ZBUFFv08_DCtx object is required to track streaming operations.
|
||||
* Use ZBUFFv08_createDCtx() and ZBUFFv08_freeDCtx() to create/release resources.
|
||||
* Use ZBUFFv08_decompressInit() to start a new decompression operation,
|
||||
* or ZBUFFv08_decompressInitDictionary() if decompression requires a dictionary.
|
||||
* Note that ZBUFFv08_DCtx objects can be re-init multiple times.
|
||||
*
|
||||
* Use ZBUFFv08_decompressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
|
||||
* or 0 when a frame is completely decoded,
|
||||
* or an error code, which can be tested using ZBUFFv08_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFFv08_recommendedDInSize() and ZBUFFv08_recommendedDOutSize()
|
||||
* output : ZBUFFv08_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFFv08_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFFv08_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* *******************************************************************************/
|
||||
|
||||
typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
|
||||
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFFv08_dStage;
|
||||
|
||||
/* *** Resource management *** */
|
||||
struct ZBUFFv08_DCtx_s {
|
||||
ZSTDv08_DCtx* zd;
|
||||
ZSTDv08_frameParams fParams;
|
||||
ZBUFFv08_dStage stage;
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inPos;
|
||||
char* outBuff;
|
||||
size_t outBuffSize;
|
||||
size_t outStart;
|
||||
size_t outEnd;
|
||||
size_t blockSize;
|
||||
BYTE headerBuffer[ZSTDv08_FRAMEHEADERSIZE_MAX];
|
||||
size_t lhSize;
|
||||
ZSTDv08_customMem customMem;
|
||||
}; /* typedef'd to ZBUFFv08_DCtx within "zstd_buffered.h" */
|
||||
|
||||
|
||||
ZBUFFv08_DCtx* ZBUFFv08_createDCtx(void)
|
||||
{
|
||||
return ZBUFFv08_createDCtx_advanced(defaultCustomMem);
|
||||
}
|
||||
|
||||
ZBUFFv08_DCtx* ZBUFFv08_createDCtx_advanced(ZSTDv08_customMem customMem)
|
||||
{
|
||||
ZBUFFv08_DCtx* zbd;
|
||||
|
||||
if (!customMem.customAlloc && !customMem.customFree)
|
||||
customMem = defaultCustomMem;
|
||||
|
||||
if (!customMem.customAlloc || !customMem.customFree)
|
||||
return NULL;
|
||||
|
||||
zbd = (ZBUFFv08_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFFv08_DCtx));
|
||||
if (zbd==NULL) return NULL;
|
||||
memset(zbd, 0, sizeof(ZBUFFv08_DCtx));
|
||||
memcpy(&zbd->customMem, &customMem, sizeof(ZSTDv08_customMem));
|
||||
zbd->zd = ZSTDv08_createDCtx_advanced(customMem);
|
||||
if (zbd->zd == NULL) { ZBUFFv08_freeDCtx(zbd); return NULL; }
|
||||
zbd->stage = ZBUFFds_init;
|
||||
return zbd;
|
||||
}
|
||||
|
||||
size_t ZBUFFv08_freeDCtx(ZBUFFv08_DCtx* zbd)
|
||||
{
|
||||
if (zbd==NULL) return 0; /* support free on null */
|
||||
ZSTDv08_freeDCtx(zbd->zd);
|
||||
if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
|
||||
if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/* *** Initialization *** */
|
||||
|
||||
size_t ZBUFFv08_decompressInitDictionary(ZBUFFv08_DCtx* zbd, const void* dict, size_t dictSize)
|
||||
{
|
||||
zbd->stage = ZBUFFds_loadHeader;
|
||||
zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
|
||||
return ZSTDv08_decompressBegin_usingDict(zbd->zd, dict, dictSize);
|
||||
}
|
||||
|
||||
size_t ZBUFFv08_decompressInit(ZBUFFv08_DCtx* zbd)
|
||||
{
|
||||
return ZBUFFv08_decompressInitDictionary(zbd, NULL, 0);
|
||||
}
|
||||
|
||||
|
||||
/* internal util function */
|
||||
MEM_STATIC size_t ZBUFFv08_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const length = MIN(dstCapacity, srcSize);
|
||||
memcpy(dst, src, length);
|
||||
return length;
|
||||
}
|
||||
|
||||
|
||||
/* *** Decompression *** */
|
||||
|
||||
size_t ZBUFFv08_decompressContinue(ZBUFFv08_DCtx* zbd,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr)
|
||||
{
|
||||
const char* const istart = (const char*)src;
|
||||
const char* const iend = istart + *srcSizePtr;
|
||||
const char* ip = istart;
|
||||
char* const ostart = (char*)dst;
|
||||
char* const oend = ostart + *dstCapacityPtr;
|
||||
char* op = ostart;
|
||||
U32 someMoreWork = 1;
|
||||
|
||||
while (someMoreWork) {
|
||||
switch(zbd->stage)
|
||||
{
|
||||
case ZBUFFds_init :
|
||||
return ERROR(init_missing);
|
||||
|
||||
case ZBUFFds_loadHeader :
|
||||
{ size_t const hSize = ZSTDv08_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
|
||||
if (ZSTDv08_isError(hSize)) return hSize;
|
||||
if (hSize != 0) { /* need more input */
|
||||
size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */
|
||||
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
|
||||
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
|
||||
zbd->lhSize += iend-ip;
|
||||
*dstCapacityPtr = 0;
|
||||
return (hSize - zbd->lhSize) + ZSTDv08_blockHeaderSize; /* remaining header bytes + next block header */
|
||||
}
|
||||
memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
|
||||
break;
|
||||
} }
|
||||
|
||||
/* Consume header */
|
||||
{ size_t const h1Size = ZSTDv08_nextSrcSizeToDecompress(zbd->zd); /* == ZSTDv08_frameHeaderSize_min */
|
||||
size_t const h1Result = ZSTDv08_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
|
||||
if (ZSTDv08_isError(h1Result)) return h1Result; /* should not happen : already checked */
|
||||
if (h1Size < zbd->lhSize) { /* long header */
|
||||
size_t const h2Size = ZSTDv08_nextSrcSizeToDecompress(zbd->zd);
|
||||
size_t const h2Result = ZSTDv08_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
|
||||
if (ZSTDv08_isError(h2Result)) return h2Result;
|
||||
} }
|
||||
|
||||
zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTDv08_WINDOWLOG_ABSOLUTEMIN);
|
||||
|
||||
/* Frame header instruct buffer sizes */
|
||||
{ size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTDv08_BLOCKSIZE_ABSOLUTEMAX);
|
||||
size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
|
||||
zbd->blockSize = blockSize;
|
||||
if (zbd->inBuffSize < blockSize) {
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
|
||||
zbd->inBuffSize = blockSize;
|
||||
zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
|
||||
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
if (zbd->outBuffSize < neededOutSize) {
|
||||
zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
|
||||
zbd->outBuffSize = neededOutSize;
|
||||
zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize);
|
||||
if (zbd->outBuff == NULL) return ERROR(memory_allocation);
|
||||
} }
|
||||
zbd->stage = ZBUFFds_read;
|
||||
/* pass-through */
|
||||
|
||||
case ZBUFFds_read:
|
||||
{ size_t const neededInSize = ZSTDv08_nextSrcSizeToDecompress(zbd->zd);
|
||||
if (neededInSize==0) { /* end of frame */
|
||||
zbd->stage = ZBUFFds_init;
|
||||
someMoreWork = 0;
|
||||
break;
|
||||
}
|
||||
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
|
||||
const int isSkipFrame = ZSTDv08_isSkipFrame(zbd->zd);
|
||||
size_t const decodedSize = ZSTDv08_decompressContinue(zbd->zd,
|
||||
zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
|
||||
ip, neededInSize);
|
||||
if (ZSTDv08_isError(decodedSize)) return decodedSize;
|
||||
ip += neededInSize;
|
||||
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
|
||||
zbd->outEnd = zbd->outStart + decodedSize;
|
||||
zbd->stage = ZBUFFds_flush;
|
||||
break;
|
||||
}
|
||||
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
|
||||
zbd->stage = ZBUFFds_load;
|
||||
/* pass-through */
|
||||
}
|
||||
|
||||
case ZBUFFds_load:
|
||||
{ size_t const neededInSize = ZSTDv08_nextSrcSizeToDecompress(zbd->zd);
|
||||
size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
|
||||
size_t loadedSize;
|
||||
if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */
|
||||
loadedSize = ZBUFFv08_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip);
|
||||
ip += loadedSize;
|
||||
zbd->inPos += loadedSize;
|
||||
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
|
||||
|
||||
/* decode loaded input */
|
||||
{ const int isSkipFrame = ZSTDv08_isSkipFrame(zbd->zd);
|
||||
size_t const decodedSize = ZSTDv08_decompressContinue(zbd->zd,
|
||||
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
|
||||
zbd->inBuff, neededInSize);
|
||||
if (ZSTDv08_isError(decodedSize)) return decodedSize;
|
||||
zbd->inPos = 0; /* input is consumed */
|
||||
if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
|
||||
zbd->outEnd = zbd->outStart + decodedSize;
|
||||
zbd->stage = ZBUFFds_flush;
|
||||
/* pass-through */
|
||||
} }
|
||||
|
||||
case ZBUFFds_flush:
|
||||
{ size_t const toFlushSize = zbd->outEnd - zbd->outStart;
|
||||
size_t const flushedSize = ZBUFFv08_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
|
||||
op += flushedSize;
|
||||
zbd->outStart += flushedSize;
|
||||
if (flushedSize == toFlushSize) { /* flush completed */
|
||||
zbd->stage = ZBUFFds_read;
|
||||
if (zbd->outStart + zbd->blockSize > zbd->outBuffSize)
|
||||
zbd->outStart = zbd->outEnd = 0;
|
||||
break;
|
||||
}
|
||||
/* cannot flush everything */
|
||||
someMoreWork = 0;
|
||||
break;
|
||||
}
|
||||
default: return ERROR(GENERIC); /* impossible */
|
||||
} }
|
||||
|
||||
/* result */
|
||||
*srcSizePtr = ip-istart;
|
||||
*dstCapacityPtr = op-ostart;
|
||||
{ size_t nextSrcSizeHint = ZSTDv08_nextSrcSizeToDecompress(zbd->zd);
|
||||
if (!nextSrcSizeHint) return (zbd->outEnd != zbd->outStart); /* return 0 only if fully flushed too */
|
||||
nextSrcSizeHint += ZSTDv08_blockHeaderSize * (ZSTDv08_nextInputType(zbd->zd) == ZSTDnit_block);
|
||||
if (zbd->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
|
||||
nextSrcSizeHint -= zbd->inPos; /* already loaded*/
|
||||
return nextSrcSizeHint;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Tool functions
|
||||
***************************************/
|
||||
size_t ZBUFFv08_recommendedDInSize(void) { return ZSTDv08_BLOCKSIZE_ABSOLUTEMAX + ZSTDv08_blockHeaderSize /* block header size*/ ; }
|
||||
size_t ZBUFFv08_recommendedDOutSize(void) { return ZSTDv08_BLOCKSIZE_ABSOLUTEMAX; }
|
||||
@@ -0,0 +1,225 @@
|
||||
/*
|
||||
zstd_v08 - decoder for 0.8 format
|
||||
Header File
|
||||
Copyright (C) 2014-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
*/
|
||||
#ifndef ZSTDv08_H_235446
|
||||
#define ZSTDv08_H_235446
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*====== Dependency ======*/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/*====== Export for Windows ======*/
|
||||
/*!
|
||||
* ZSTDv08_DLL_EXPORT :
|
||||
* Enable exporting of functions when building a Windows DLL
|
||||
*/
|
||||
#if defined(_WIN32) && defined(ZSTDv08_DLL_EXPORT) && (ZSTDv08_DLL_EXPORT==1)
|
||||
# define ZSTDLIB_API __declspec(dllexport)
|
||||
#else
|
||||
# define ZSTDLIB_API
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Simple API
|
||||
***************************************/
|
||||
/*! ZSTDv08_getDecompressedSize() :
|
||||
* @return : decompressed size as a 64-bits value _if known_, 0 otherwise.
|
||||
* note 1 : decompressed size can be very large (64-bits value),
|
||||
* potentially larger than what local system can handle as a single memory segment.
|
||||
* In which case, it's necessary to use streaming mode to decompress data.
|
||||
* note 2 : decompressed size is an optional field, that may not be present.
|
||||
* When `return==0`, consider data to decompress could have any size.
|
||||
* In which case, it's necessary to use streaming mode to decompress data,
|
||||
* or rely on application's implied limits.
|
||||
* (For example, it may know that its own data is necessarily cut into blocks <= 16 KB).
|
||||
* note 3 : decompressed size could be wrong or intentionally modified !
|
||||
* Always ensure result fits within application's authorized limits !
|
||||
* Each application can have its own set of conditions.
|
||||
* If the intention is to decompress public data compressed by zstd command line utility,
|
||||
* it is recommended to support at least 8 MB for extended compatibility.
|
||||
* note 4 : when `return==0`, if precise failure cause is needed, use ZSTDv08_getFrameParams() to know more. */
|
||||
unsigned long long ZSTDv08_getDecompressedSize(const void* src, size_t srcSize);
|
||||
|
||||
/*! ZSTDv08_decompress() :
|
||||
`compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail.
|
||||
`dstCapacity` must be equal or larger than originalSize (see ZSTDv08_getDecompressedSize() ).
|
||||
If originalSize is unknown, and if there is no implied application-specific limitations,
|
||||
it's necessary to use streaming mode to decompress data.
|
||||
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
|
||||
or an errorCode if it fails (which can be tested using ZSTDv08_isError()) */
|
||||
ZSTDLIB_API size_t ZSTDv08_decompress( void* dst, size_t dstCapacity,
|
||||
const void* src, size_t compressedSize);
|
||||
|
||||
|
||||
/*====== Helper functions ======*/
|
||||
ZSTDLIB_API unsigned ZSTDv08_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
|
||||
ZSTDLIB_API const char* ZSTDv08_getErrorName(size_t code); /*!< provides readable string from an error code */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Explicit memory management
|
||||
***************************************/
|
||||
/** Decompression context */
|
||||
typedef struct ZSTDv08_DCtx_s ZSTDv08_DCtx; /*< incomplete type */
|
||||
ZSTDLIB_API ZSTDv08_DCtx* ZSTDv08_createDCtx(void);
|
||||
ZSTDLIB_API size_t ZSTDv08_freeDCtx(ZSTDv08_DCtx* dctx);
|
||||
|
||||
/** ZSTDv08_decompressDCtx() :
|
||||
* Same as ZSTDv08_decompress(), requires an allocated ZSTDv08_DCtx (see ZSTDv08_createDCtx()) */
|
||||
ZSTDLIB_API size_t ZSTDv08_decompressDCtx(ZSTDv08_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/*-************************
|
||||
* Simple dictionary API
|
||||
***************************/
|
||||
/*! ZSTDv08_decompress_usingDict() :
|
||||
* Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
|
||||
* Dictionary must be identical to the one used during compression.
|
||||
* Note : This function load the dictionary, resulting in a significant startup time */
|
||||
ZSTDLIB_API size_t ZSTDv08_decompress_usingDict(ZSTDv08_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict,size_t dictSize);
|
||||
|
||||
|
||||
/*-**************************
|
||||
* Fast Dictionary API
|
||||
****************************/
|
||||
/*! ZSTDv08_createDDict() :
|
||||
* Create a digested dictionary, ready to start decompression operation without startup delay.
|
||||
* `dict` can be released after creation */
|
||||
typedef struct ZSTDv08_DDict_s ZSTDv08_DDict;
|
||||
ZSTDLIB_API ZSTDv08_DDict* ZSTDv08_createDDict(const void* dict, size_t dictSize);
|
||||
ZSTDLIB_API size_t ZSTDv08_freeDDict(ZSTDv08_DDict* ddict);
|
||||
|
||||
/*! ZSTDv08_decompress_usingDDict() :
|
||||
* Decompression using a digested Dictionary
|
||||
* Faster startup than ZSTDv08_decompress_usingDict(), recommended when same dictionary is used multiple times. */
|
||||
ZSTDLIB_API size_t ZSTDv08_decompress_usingDDict(ZSTDv08_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTDv08_DDict* ddict);
|
||||
|
||||
typedef struct {
|
||||
unsigned long long frameContentSize;
|
||||
unsigned windowSize;
|
||||
unsigned dictID;
|
||||
unsigned checksumFlag;
|
||||
} ZSTDv08_frameParams;
|
||||
|
||||
ZSTDLIB_API size_t ZSTDv08_getFrameParams(ZSTDv08_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
|
||||
|
||||
|
||||
/* ***************************************************************
|
||||
* Compiler specifics
|
||||
*****************************************************************/
|
||||
/* ZSTDv08_DLL_EXPORT :
|
||||
* Enable exporting of functions when building a Windows DLL */
|
||||
#if defined(_WIN32) && defined(ZSTDv08_DLL_EXPORT) && (ZSTDv08_DLL_EXPORT==1)
|
||||
# define ZSTDLIB_API __declspec(dllexport)
|
||||
#else
|
||||
# define ZSTDLIB_API
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Streaming functions
|
||||
***************************************/
|
||||
/* This is the easier "buffered" streaming API,
|
||||
* using an internal buffer to lift all restrictions on user-provided buffers
|
||||
* which can be any size, any place, for both input and output.
|
||||
* ZBUFF and ZSTD are 100% interoperable,
|
||||
* frames created by one can be decoded by the other one */
|
||||
|
||||
typedef struct ZBUFFv08_DCtx_s ZBUFFv08_DCtx;
|
||||
ZSTDLIB_API ZBUFFv08_DCtx* ZBUFFv08_createDCtx(void);
|
||||
ZSTDLIB_API size_t ZBUFFv08_freeDCtx(ZBUFFv08_DCtx* dctx);
|
||||
|
||||
ZSTDLIB_API size_t ZBUFFv08_decompressInit(ZBUFFv08_DCtx* dctx);
|
||||
ZSTDLIB_API size_t ZBUFFv08_decompressInitDictionary(ZBUFFv08_DCtx* dctx, const void* dict, size_t dictSize);
|
||||
|
||||
ZSTDLIB_API size_t ZBUFFv08_decompressContinue(ZBUFFv08_DCtx* dctx,
|
||||
void* dst, size_t* dstCapacityPtr,
|
||||
const void* src, size_t* srcSizePtr);
|
||||
|
||||
/*-***************************************************************************
|
||||
* Streaming decompression howto
|
||||
*
|
||||
* A ZBUFFv08_DCtx object is required to track streaming operations.
|
||||
* Use ZBUFFv08_createDCtx() and ZBUFFv08_freeDCtx() to create/release resources.
|
||||
* Use ZBUFFv08_decompressInit() to start a new decompression operation,
|
||||
* or ZBUFFv08_decompressInitDictionary() if decompression requires a dictionary.
|
||||
* Note that ZBUFFv08_DCtx objects can be re-init multiple times.
|
||||
*
|
||||
* Use ZBUFFv08_decompressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
|
||||
* @return : 0 when a frame is completely decoded and fully flushed,
|
||||
* 1 when there is still some data left within internal buffer to flush,
|
||||
* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency),
|
||||
* or an error code, which can be tested using ZBUFFv08_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFFv08_recommendedDInSize() and ZBUFFv08_recommendedDOutSize()
|
||||
* output : ZBUFFv08_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFFv08_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFFv08_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* *******************************************************************************/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Tool functions
|
||||
***************************************/
|
||||
ZSTDLIB_API unsigned ZBUFFv08_isError(size_t errorCode);
|
||||
ZSTDLIB_API const char* ZBUFFv08_getErrorName(size_t errorCode);
|
||||
|
||||
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
|
||||
* These sizes are just hints, they tend to offer better latency */
|
||||
ZSTDLIB_API size_t ZBUFFv08_recommendedDInSize(void);
|
||||
ZSTDLIB_API size_t ZBUFFv08_recommendedDOutSize(void);
|
||||
|
||||
|
||||
|
||||
#define ZSTDv08_MAGICNUMBER 0xFD2FB528 /* v0.8 */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTDv08_H_235446 */
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# local binary (Makefile)
|
||||
zstd
|
||||
zstd32
|
||||
zstd-compress
|
||||
zstd-decompress
|
||||
fullbench
|
||||
fullbench32
|
||||
fuzzer
|
||||
@@ -11,7 +9,6 @@ zbufftest
|
||||
zbufftest32
|
||||
datagen
|
||||
paramgrill
|
||||
roundTripCrash
|
||||
|
||||
# Object files
|
||||
*.o
|
||||
@@ -40,9 +37,6 @@ roundTripCrash
|
||||
dictionary
|
||||
grillResults.txt
|
||||
_*
|
||||
tmp*
|
||||
*.zst
|
||||
result
|
||||
|
||||
# fuzzer
|
||||
afl
|
||||
|
||||
+63
-68
@@ -31,33 +31,42 @@
|
||||
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
|
||||
# ##########################################################################
|
||||
|
||||
# Version numbers
|
||||
LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/common/zstd.h`
|
||||
LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/common/zstd.h`
|
||||
LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../lib/common/zstd.h`
|
||||
LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
|
||||
LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
|
||||
LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
|
||||
LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
|
||||
LIBVER := $(shell echo $(LIBVER_SCRIPT))
|
||||
VERSION?= $(LIBVER)
|
||||
|
||||
DESTDIR?=
|
||||
PREFIX ?= /usr/local
|
||||
BINDIR = $(PREFIX)/bin
|
||||
MANDIR = $(PREFIX)/share/man/man1
|
||||
|
||||
ZSTDDIR = ../lib
|
||||
|
||||
CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_
|
||||
CPPFLAGS= -I../lib/common -DZSTD_VERSION=\"$(VERSION)\"
|
||||
CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
|
||||
CFLAGS += -std=c99 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
||||
|
||||
BINDIR = $(PREFIX)/bin
|
||||
MANDIR = $(PREFIX)/share/man/man1
|
||||
ZSTDDIR = ../lib
|
||||
|
||||
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
|
||||
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
|
||||
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/decompress/zstd_decompress.c
|
||||
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
|
||||
ZSTDCOMP_FILES := $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c
|
||||
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c
|
||||
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/zdict.c $(ZSTDDIR)/dictBuilder/divsufsort.c
|
||||
ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
|
||||
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
|
||||
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES)
|
||||
|
||||
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
|
||||
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
|
||||
ZSTDLEGACY_FILES:=
|
||||
ZSTD_FILES_LEGACY:=
|
||||
else
|
||||
ZSTD_LEGACY_SUPPORT:=1
|
||||
CPPFLAGS += -I$(ZSTDDIR)/legacy -I./legacy
|
||||
ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c legacy/fileio_legacy.c
|
||||
CPPFLAGS += -I../lib/legacy -I./legacy
|
||||
ZSTD_FILES_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v04.c $(ZSTDDIR)/legacy/zstd_v05.c $(ZSTDDIR)/legacy/zstd_v07.c $(ZSTDDIR)/legacy/zstd_v08.c legacy/fileio_legacy.c
|
||||
endif
|
||||
|
||||
|
||||
@@ -80,12 +89,12 @@ default: zstd
|
||||
|
||||
all: zstd fullbench fuzzer zbufftest paramgrill datagen zstd32 fullbench32 fuzzer32 zbufftest32
|
||||
|
||||
zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \
|
||||
zstdcli.c fileio.c bench.c datagen.c dibio.c
|
||||
zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \
|
||||
zstdcli.c fileio.c bench.c $(ZSTDDIR)/common/xxhash.c datagen.c dibio.c
|
||||
$(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
|
||||
|
||||
zstd32: $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \
|
||||
zstdcli.c fileio.c bench.c datagen.c dibio.c
|
||||
zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \
|
||||
zstdcli.c fileio.c bench.c $(ZSTDDIR)/common/xxhash.c datagen.c dibio.c
|
||||
$(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
|
||||
|
||||
zstd_nolegacy :
|
||||
@@ -105,16 +114,14 @@ zstd-pgo : clean zstd
|
||||
zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT)
|
||||
|
||||
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
|
||||
$(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c
|
||||
zstd-compress: $(ZSTDCOMP_FILES) $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
||||
|
||||
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \
|
||||
$(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c
|
||||
zstd-decompress: $(ZSTDDECOMP_FILES) $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c
|
||||
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
|
||||
|
||||
zstd-small: clean
|
||||
CFLAGS="-Os -s" $(MAKE) zstd-frugal
|
||||
zstd-small: clean
|
||||
CFLAGS="-Os -s" $(MAKE) zstd-frugal
|
||||
|
||||
fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
@@ -122,43 +129,42 @@ fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
|
||||
fullbench32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
|
||||
fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
|
||||
fuzzer : $(ZSTD_FILES) \
|
||||
datagen.c $(ZSTDDIR)/common/xxhash.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
|
||||
fuzzer32: $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
|
||||
fuzzer32: $(ZSTD_FILES) \
|
||||
datagen.c $(ZSTDDIR)/common/xxhash.c fuzzer.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
|
||||
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) \
|
||||
datagen.c $(ZSTDDIR)/common/xxhash.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
|
||||
zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) \
|
||||
datagen.c $(ZSTDDIR)/common/xxhash.c zbufftest.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c
|
||||
paramgrill : $(ZSTD_FILES) \
|
||||
datagen.c $(ZSTDDIR)/common/xxhash.c paramgrill.c
|
||||
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
|
||||
|
||||
datagen : datagen.c datagencli.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
clean:
|
||||
@rm -f core *.o tmp* result* *.gcda dictionary *.zst \
|
||||
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
|
||||
fullbench$(EXT) fullbench32$(EXT) \
|
||||
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
|
||||
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT)
|
||||
datagen$(EXT) paramgrill$(EXT)
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
|
||||
#------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
|
||||
HOST_OS = POSIX
|
||||
|
||||
install: zstd
|
||||
@echo Installing binaries
|
||||
@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
|
||||
@@ -180,42 +186,15 @@ uninstall:
|
||||
[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
|
||||
@echo zstd programs successfully uninstalled
|
||||
|
||||
valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1
|
||||
valgrindTest: zstd datagen fuzzer fullbench zbufftest
|
||||
@echo "\n ---- valgrind tests : memory analyzer ----"
|
||||
$(VALGRIND) ./datagen -g50M > $(VOID)
|
||||
$(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
|
||||
./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID)
|
||||
./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID)
|
||||
./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp
|
||||
$(VALGRIND) ./zstd -vdf tmp -o $(VOID)
|
||||
./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID)
|
||||
@rm tmp
|
||||
$(VALGRIND) ./fuzzer -T1mn -t1
|
||||
$(VALGRIND) ./fullbench -i1
|
||||
$(VALGRIND) ./zbufftest -T1mn
|
||||
|
||||
endif
|
||||
|
||||
|
||||
ifneq (,$(filter MSYS%,$(shell uname)))
|
||||
HOST_OS = MSYS
|
||||
endif
|
||||
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
|
||||
#------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
|
||||
zstd-playTests: datagen
|
||||
ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)
|
||||
|
||||
test: test-zstd test-fullbench test-fuzzer test-zbuff
|
||||
|
||||
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32
|
||||
|
||||
test-all: test test32 valgrindTest
|
||||
|
||||
zstd-playTests: datagen
|
||||
ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)
|
||||
|
||||
test-zstd: ZSTD = ./zstd
|
||||
test-zstd: zstd zstd-playTests
|
||||
|
||||
@@ -244,4 +223,20 @@ test-zbuff: zbufftest
|
||||
|
||||
test-zbuff32: zbufftest32
|
||||
./zbufftest32 $(ZBUFFTEST)
|
||||
|
||||
valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1
|
||||
valgrindTest: zstd datagen fuzzer fullbench zbufftest
|
||||
@echo "\n ---- valgrind tests : memory analyzer ----"
|
||||
$(VALGRIND) ./datagen -g50M > $(VOID)
|
||||
$(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
|
||||
./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID)
|
||||
./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID)
|
||||
./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp
|
||||
$(VALGRIND) ./zstd -vdf tmp -o $(VOID)
|
||||
./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID)
|
||||
@rm tmp
|
||||
$(VALGRIND) ./fuzzer -T1mn -t1
|
||||
$(VALGRIND) ./fullbench -i1
|
||||
$(VALGRIND) ./zbufftest -T1mn
|
||||
|
||||
endif
|
||||
|
||||
+53
-30
@@ -26,14 +26,13 @@
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */
|
||||
#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_HAS_CREATEFILELIST, UTIL_sleep */
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
#include "zstd_static.h"
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "xxhash.h"
|
||||
|
||||
@@ -189,6 +188,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
/* Bench */
|
||||
{ U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
|
||||
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
|
||||
U64 crcCheck = 0;
|
||||
UTIL_time_t coolTime;
|
||||
U32 testNb;
|
||||
size_t cSize = 0;
|
||||
@@ -216,21 +216,24 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
UTIL_getTime(&clockStart);
|
||||
|
||||
{ U32 nbLoops = 0;
|
||||
ZSTD_CDict* cdict = ZSTD_createCDict(dictBuffer, dictBufferSize, cLevel);
|
||||
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict() allocation failure");
|
||||
do {
|
||||
U32 blockNb;
|
||||
{ ZSTD_parameters params;
|
||||
params.cParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
ZSTD_adjustCParams(¶ms.cParams, blockSize, dictBufferSize);
|
||||
{ size_t const initResult = ZSTD_compressBegin_advanced(refCtx, dictBuffer, dictBufferSize, params, blockSize);
|
||||
if (ZSTD_isError(initResult)) break;
|
||||
} }
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const rSize = ZSTD_compress_usingCDict(ctx,
|
||||
size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
||||
cdict);
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
|
||||
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingPreparedCCtx() failed : %s", ZSTD_getErrorName(rSize));
|
||||
blockTable[blockNb].cSize = rSize;
|
||||
}
|
||||
nbLoops++;
|
||||
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
|
||||
ZSTD_freeCDict(cdict);
|
||||
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
|
||||
if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
|
||||
} }
|
||||
@@ -252,15 +255,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
UTIL_getTime(&clockStart);
|
||||
|
||||
{ U32 nbLoops = 0;
|
||||
ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
|
||||
if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
|
||||
do {
|
||||
U32 blockNb;
|
||||
ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize);
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const regenSize = ZSTD_decompress_usingDDict(dctx,
|
||||
size_t const regenSize = ZSTD_decompress_usingPreparedDCtx(dctx, refDCtx,
|
||||
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
|
||||
ddict);
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
|
||||
if (ZSTD_isError(regenSize)) {
|
||||
DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s \n",
|
||||
blockNb, ZSTD_getErrorName(regenSize));
|
||||
@@ -271,7 +272,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
}
|
||||
nbLoops++;
|
||||
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
|
||||
ZSTD_freeDDict(ddict);
|
||||
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
|
||||
if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
|
||||
} }
|
||||
@@ -282,7 +282,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
(double)srcSize / fastestD );
|
||||
|
||||
/* CRC Checking */
|
||||
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
{ crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
if (crcOrig!=crcCheck) {
|
||||
size_t u;
|
||||
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
|
||||
@@ -308,10 +308,12 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
#endif
|
||||
} /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
|
||||
|
||||
result->ratio = ratio;
|
||||
result->cSize = cSize;
|
||||
result->cSpeed = (double)srcSize / fastestC;
|
||||
result->dSpeed = (double)srcSize / fastestD;
|
||||
if (crcOrig == crcCheck) {
|
||||
result->ratio = ratio;
|
||||
result->cSize = cSize;
|
||||
result->cSpeed = (double)srcSize / fastestC;
|
||||
result->dSpeed = (double)srcSize / fastestD;
|
||||
}
|
||||
DISPLAYLEVEL(2, "%2i#\n", cLevel);
|
||||
} /* Bench */
|
||||
|
||||
@@ -400,9 +402,9 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
const char** fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
size_t pos = 0, totalSize = 0;
|
||||
FILE* f;
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
FILE* f;
|
||||
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
||||
if (UTIL_isDirectory(fileNamesTable[n])) {
|
||||
DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]);
|
||||
@@ -432,8 +434,9 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
||||
void* dictBuffer = NULL;
|
||||
size_t dictBufferSize = 0;
|
||||
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
||||
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
|
||||
U64 totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
|
||||
char mfName[20] = {0};
|
||||
const char* displayName = NULL;
|
||||
|
||||
if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
|
||||
|
||||
@@ -460,12 +463,13 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
||||
|
||||
/* Bench */
|
||||
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
|
||||
{ const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
|
||||
BMK_benchCLevel(srcBuffer, benchedSize,
|
||||
displayName, cLevel, cLevelLast,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize);
|
||||
}
|
||||
if (nbFiles > 1) displayName = mfName;
|
||||
else displayName = fileNamesTable[0];
|
||||
|
||||
BMK_benchCLevel(srcBuffer, benchedSize,
|
||||
displayName, cLevel, cLevelLast,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize);
|
||||
|
||||
/* clean up */
|
||||
free(srcBuffer);
|
||||
@@ -478,7 +482,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
|
||||
{
|
||||
char name[20] = {0};
|
||||
size_t benchedSize = 10000000;
|
||||
void* const srcBuffer = malloc(benchedSize);
|
||||
void* srcBuffer = malloc(benchedSize);
|
||||
|
||||
/* Memory allocation */
|
||||
if (!srcBuffer) EXM_THROW(21, "not enough memory");
|
||||
@@ -496,13 +500,32 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
|
||||
|
||||
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
||||
const char* dictFileName, int cLevel, int cLevelLast)
|
||||
const char* dictFileName, int cLevel, int cLevelLast, int recursive)
|
||||
{
|
||||
double const compressibility = (double)g_compressibilityDefault / 100;
|
||||
|
||||
if (nbFiles == 0)
|
||||
BMK_syntheticTest(cLevel, cLevelLast, compressibility);
|
||||
else
|
||||
{
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
if (recursive) {
|
||||
char* buf;
|
||||
const char** filenameTable;
|
||||
unsigned i;
|
||||
filenameTable = UTIL_createFileList(fileNamesTable, nbFiles, &buf, &nbFiles);
|
||||
if (filenameTable) {
|
||||
for (i=0; i<nbFiles; i++) DISPLAYLEVEL(3, "%d %s\n", i, filenameTable[i]);
|
||||
BMK_benchFileTable(filenameTable, nbFiles, dictFileName, cLevel, cLevelLast);
|
||||
UTIL_freeFileList(filenameTable, buf);
|
||||
}
|
||||
}
|
||||
else BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
|
||||
#else
|
||||
(void)recursive;
|
||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
|
||||
#endif
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
+3
-5
@@ -21,13 +21,12 @@
|
||||
You can contact the author at :
|
||||
- ZSTD homepage : http://www.zstd.net/
|
||||
*/
|
||||
#ifndef BENCH_H_121279284357
|
||||
#define BENCH_H_121279284357
|
||||
#pragma once
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/* Main function */
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
||||
const char* dictFileName, int cLevel, int cLevelLast);
|
||||
const char* dictFileName, int cLevel, int cLevelLast, int recursive);
|
||||
|
||||
/* Set Parameters */
|
||||
void BMK_SetNbIterations(unsigned nbLoops);
|
||||
@@ -35,4 +34,3 @@ void BMK_SetBlockSize(size_t blockSize);
|
||||
void BMK_setAdditionalParam(int additionalParam);
|
||||
void BMK_setNotificationLevel(unsigned level);
|
||||
|
||||
#endif /* BENCH_H_121279284357 */
|
||||
|
||||
+57
-70
@@ -27,9 +27,9 @@
|
||||
* Includes
|
||||
**************************************/
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <stdio.h> /* FILE, fwrite, fprintf */
|
||||
#include <stdio.h> /* FILE, fwrite */
|
||||
#include <string.h> /* memcpy */
|
||||
#include "mem.h" /* U32 */
|
||||
#include "mem.h"
|
||||
|
||||
|
||||
/*-************************************
|
||||
@@ -48,25 +48,22 @@
|
||||
* Macros
|
||||
**************************************/
|
||||
#define KB *(1 <<10)
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
#define RDG_DEBUG 0
|
||||
#define TRACE(...) if (RDG_DEBUG) fprintf(stderr, __VA_ARGS__ )
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Local constants
|
||||
* Local types
|
||||
**************************************/
|
||||
#define LTLOG 13
|
||||
#define LTSIZE (1<<LTLOG)
|
||||
#define LTMASK (LTSIZE-1)
|
||||
typedef BYTE litDistribTable[LTSIZE];
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Local Functions
|
||||
*********************************************************/
|
||||
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
static U32 RDG_rand(U32* src)
|
||||
static unsigned int RDG_rand(U32* src)
|
||||
{
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
@@ -75,136 +72,126 @@ static U32 RDG_rand(U32* src)
|
||||
rand32 ^= prime2;
|
||||
rand32 = RDG_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32 >> 5;
|
||||
return rand32;
|
||||
}
|
||||
|
||||
|
||||
static void RDG_fillLiteralDistrib(BYTE* ldt, double ld)
|
||||
static void RDG_fillLiteralDistrib(litDistribTable lt, double ld)
|
||||
{
|
||||
BYTE const firstChar = (ld<=0.0) ? 0 : '(';
|
||||
BYTE const lastChar = (ld<=0.0) ? 255 : '}';
|
||||
U32 i = 0;
|
||||
BYTE character = (ld<=0.0) ? 0 : '0';
|
||||
U32 u;
|
||||
BYTE const firstChar = (ld<=0.0) ? 0 : '(';
|
||||
BYTE const lastChar = (ld<=0.0) ?255: '}';
|
||||
|
||||
if (ld<=0.0) ld = 0.0;
|
||||
//TRACE(" percent:%5.2f%% \n", ld*100.);
|
||||
//TRACE(" start:(%c)[%02X] ", character, character);
|
||||
for (u=0; u<LTSIZE; ) {
|
||||
U32 const weight = (U32)((double)(LTSIZE - u) * ld) + 1;
|
||||
U32 const end = MIN ( u + weight , LTSIZE);
|
||||
while (u < end) ldt[u++] = character; // TRACE(" %u(%c)[%02X] ", u, character, character);
|
||||
while (i<LTSIZE) {
|
||||
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
|
||||
U32 end;
|
||||
if (weight + i > LTSIZE) weight = LTSIZE-i;
|
||||
end = i + weight;
|
||||
while (i < end) lt[i++] = character;
|
||||
character++;
|
||||
if (character > lastChar) character = firstChar;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static BYTE RDG_genChar(U32* seed, const BYTE* ldt)
|
||||
static BYTE RDG_genChar(U32* seed, const litDistribTable lt)
|
||||
{
|
||||
U32 const id = RDG_rand(seed) & LTMASK;
|
||||
//TRACE(" %u : \n", id);
|
||||
//TRACE(" %4u [%4u] ; val : %4u \n", id, id&255, ldt[id]);
|
||||
return (ldt[id]); /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with 0.0. Checked : table is fully initialized */
|
||||
return (lt[id]);
|
||||
}
|
||||
|
||||
|
||||
static U32 RDG_rand15Bits (unsigned* seedPtr)
|
||||
#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 0x7FFF)
|
||||
#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15)
|
||||
void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr)
|
||||
{
|
||||
return RDG_rand(seedPtr) & 0x7FFF;
|
||||
}
|
||||
|
||||
static U32 RDG_randLength(unsigned* seedPtr)
|
||||
{
|
||||
if (RDG_rand(seedPtr) & 7)
|
||||
return (RDG_rand(seedPtr) & 0xF);
|
||||
return (RDG_rand(seedPtr) & 0x1FF) + 0xF;
|
||||
}
|
||||
|
||||
void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr)
|
||||
{
|
||||
BYTE* const buffPtr = (BYTE*)buffer;
|
||||
U32 const matchProba32 = (U32)(32768 * matchProba);
|
||||
BYTE* buffPtr = (BYTE*)buffer;
|
||||
const U32 matchProba32 = (U32)(32768 * matchProba);
|
||||
size_t pos = prefixSize;
|
||||
U32* seed = seedPtr;
|
||||
U32 prevOffset = 1;
|
||||
|
||||
/* special case : sparse content */
|
||||
while (matchProba >= 1.0) {
|
||||
size_t size0 = RDG_rand(seedPtr) & 3;
|
||||
size_t size0 = RDG_rand(seed) & 3;
|
||||
size0 = (size_t)1 << (16 + size0 * 2);
|
||||
size0 += RDG_rand(seedPtr) & (size0-1); /* because size0 is power of 2*/
|
||||
size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/
|
||||
if (buffSize < pos + size0) {
|
||||
memset(buffPtr+pos, 0, buffSize-pos);
|
||||
return;
|
||||
}
|
||||
memset(buffPtr+pos, 0, size0);
|
||||
pos += size0;
|
||||
buffPtr[pos-1] = RDG_genChar(seedPtr, ldt);
|
||||
buffPtr[pos-1] = RDG_genChar(seed, lt);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* init */
|
||||
if (pos==0) buffPtr[0] = RDG_genChar(seedPtr, ldt), pos=1;
|
||||
if (pos==0) buffPtr[0] = RDG_genChar(seed, lt), pos=1;
|
||||
|
||||
/* Generate compressible data */
|
||||
while (pos < buffSize) {
|
||||
/* Select : Literal (char) or Match (within 32K) */
|
||||
if (RDG_rand15Bits(seedPtr) < matchProba32) {
|
||||
if (RDG_RAND15BITS < matchProba32) {
|
||||
/* Copy (within 32K) */
|
||||
U32 const length = RDG_randLength(seedPtr) + 4;
|
||||
U32 const d = (U32) MIN(pos + length , buffSize);
|
||||
U32 const repeatOffset = (RDG_rand(seedPtr) & 15) == 2;
|
||||
U32 const randOffset = RDG_rand15Bits(seedPtr) + 1;
|
||||
U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos);
|
||||
size_t match = pos - offset;
|
||||
//TRACE("pos : %u; offset: %u ; length : %u \n", (U32)pos, offset, length);
|
||||
while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */
|
||||
size_t match;
|
||||
size_t d;
|
||||
size_t const length = RDG_RANDLENGTH + 4;
|
||||
U32 offset = RDG_RAND15BITS + 1;
|
||||
U32 repeatOffset = (RDG_rand(seed) & 15) == 2;
|
||||
if (repeatOffset) offset = prevOffset;
|
||||
if (offset > pos) offset = (U32)pos;
|
||||
prevOffset = offset;
|
||||
match = pos - offset;
|
||||
d = pos + length;
|
||||
if (d > buffSize) d = buffSize;
|
||||
while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */
|
||||
} else {
|
||||
/* Literal (noise) */
|
||||
U32 const length = RDG_randLength(seedPtr);
|
||||
U32 const d = (U32) MIN(pos + length, buffSize);
|
||||
while (pos < d) buffPtr[pos++] = RDG_genChar(seedPtr, ldt);
|
||||
size_t const length = RDG_RANDLENGTH;
|
||||
size_t d = pos + length;
|
||||
if (d > buffSize) d = buffSize;
|
||||
while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt);
|
||||
} }
|
||||
}
|
||||
|
||||
|
||||
void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
|
||||
{
|
||||
BYTE ldt[LTSIZE];
|
||||
memset(ldt, '0', sizeof(ldt));
|
||||
if (litProba<=0.0) litProba = matchProba / 4.5;
|
||||
//TRACE(" percent:%5.2f%% \n", litProba*100.);
|
||||
RDG_fillLiteralDistrib(ldt, litProba);
|
||||
RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed);
|
||||
litDistribTable lt;
|
||||
if (litProba==0.0) litProba = matchProba / 4.5;
|
||||
RDG_fillLiteralDistrib(lt, litProba);
|
||||
RDG_genBlock(buffer, size, 0, matchProba, lt, &seed);
|
||||
}
|
||||
|
||||
|
||||
#define RDG_DICTSIZE (32 KB)
|
||||
#define RDG_BLOCKSIZE (128 KB)
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed)
|
||||
{
|
||||
size_t const stdBlockSize = 128 KB;
|
||||
size_t const stdDictSize = 32 KB;
|
||||
BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize);
|
||||
BYTE* buff = (BYTE*)malloc(RDG_DICTSIZE + RDG_BLOCKSIZE);
|
||||
U64 total = 0;
|
||||
BYTE ldt[LTSIZE];
|
||||
litDistribTable ldt;
|
||||
|
||||
/* init */
|
||||
if (buff==NULL) { fprintf(stdout, "not enough memory\n"); exit(1); }
|
||||
if (litProba<=0.0) litProba = matchProba / 4.5;
|
||||
memset(ldt, '0', sizeof(ldt));
|
||||
RDG_fillLiteralDistrib(ldt, litProba);
|
||||
SET_BINARY_MODE(stdout);
|
||||
|
||||
/* Generate initial dict */
|
||||
RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed);
|
||||
RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldt, &seed);
|
||||
|
||||
/* Generate compressible data */
|
||||
while (total < size) {
|
||||
size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total));
|
||||
RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed);
|
||||
size_t const genBlockSize = (size_t) (MIN (RDG_BLOCKSIZE, size-total));
|
||||
RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldt, &seed);
|
||||
total += genBlockSize;
|
||||
{ size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; }
|
||||
/* update dict */
|
||||
memcpy(buff, buff + stdBlockSize, stdDictSize);
|
||||
memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE);
|
||||
}
|
||||
|
||||
/* cleanup */
|
||||
|
||||
@@ -27,8 +27,8 @@
|
||||
/*-************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include "util.h" /* Compiler options */
|
||||
#include <stdio.h> /* fprintf, stderr */
|
||||
#include "mem.h"
|
||||
#include "datagen.h" /* RDG_generate */
|
||||
|
||||
|
||||
|
||||
+39
-39
@@ -101,30 +101,27 @@ const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCo
|
||||
/* ********************************************************
|
||||
* File related operations
|
||||
**********************************************************/
|
||||
/** DiB_loadFiles() :
|
||||
* @return : nb of files effectively loaded into `buffer` */
|
||||
static unsigned DiB_loadFiles(void* buffer, size_t bufferSize,
|
||||
size_t* fileSizes,
|
||||
const char** fileNamesTable, unsigned nbFiles)
|
||||
static void DiB_loadFiles(void* buffer, size_t bufferSize,
|
||||
size_t* fileSizes,
|
||||
const char** fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
char* const buff = (char*)buffer;
|
||||
char* buff = (char*)buffer;
|
||||
size_t pos = 0;
|
||||
unsigned n;
|
||||
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
unsigned long long const fs64 = UTIL_getFileSize(fileNamesTable[n]);
|
||||
size_t const fileSize = (size_t)(fs64 > bufferSize-pos ? 0 : fs64);
|
||||
FILE* const f = fopen(fileNamesTable[n], "rb");
|
||||
size_t readSize;
|
||||
unsigned long long fileSize = UTIL_getFileSize(fileNamesTable[n]);
|
||||
FILE* f = fopen(fileNamesTable[n], "rb");
|
||||
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
|
||||
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
{ size_t const readSize = fread(buff+pos, 1, fileSize, f);
|
||||
if (readSize != fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
|
||||
pos += readSize; }
|
||||
fileSizes[n] = fileSize;
|
||||
if (fileSize > bufferSize-pos) fileSize = 0; /* stop there, not enough memory to load all files */
|
||||
readSize = fread(buff+pos, 1, (size_t)fileSize, f);
|
||||
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
|
||||
pos += readSize;
|
||||
fileSizes[n] = (size_t)fileSize;
|
||||
fclose(f);
|
||||
if (fileSize == 0) break; /* stop there, not enough memory to load all files */
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
|
||||
@@ -133,7 +130,7 @@ static unsigned DiB_loadFiles(void* buffer, size_t bufferSize,
|
||||
**********************************************************/
|
||||
static size_t DiB_findMaxMem(unsigned long long requiredMem)
|
||||
{
|
||||
size_t const step = 8 MB;
|
||||
size_t step = 8 MB;
|
||||
void* testmem = NULL;
|
||||
|
||||
requiredMem = (((requiredMem >> 23) + 1) << 23);
|
||||
@@ -165,7 +162,7 @@ static void DiB_fillNoise(void* buffer, size_t length)
|
||||
static void DiB_saveDict(const char* dictFileName,
|
||||
const void* buff, size_t buffSize)
|
||||
{
|
||||
FILE* const f = fopen(dictFileName, "wb");
|
||||
FILE* f = fopen(dictFileName, "wb");
|
||||
if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName);
|
||||
|
||||
{ size_t const n = fwrite(buff, 1, buffSize, f);
|
||||
@@ -188,44 +185,47 @@ size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
|
||||
ZDICT_params_t parameters);
|
||||
|
||||
|
||||
#define MIN(a,b) ((a)<(b)?(a):(b))
|
||||
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
|
||||
const char** fileNamesTable, unsigned nbFiles,
|
||||
ZDICT_params_t params)
|
||||
{
|
||||
void* const dictBuffer = malloc(maxDictSize);
|
||||
size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
||||
unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
|
||||
size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
|
||||
size_t const benchedSize = MIN (maxMem, (size_t)totalSizeToLoad);
|
||||
void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
|
||||
void* srcBuffer;
|
||||
size_t benchedSize;
|
||||
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
||||
unsigned long long totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
|
||||
void* dictBuffer = malloc(maxDictSize);
|
||||
size_t dictSize;
|
||||
int result = 0;
|
||||
|
||||
/* Checks */
|
||||
if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
|
||||
|
||||
/* init */
|
||||
g_displayLevel = params.notificationLevel;
|
||||
benchedSize = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
|
||||
if ((unsigned long long)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
||||
if (benchedSize < totalSizeToLoad)
|
||||
DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20));
|
||||
|
||||
/* Memory allocation & restrictions */
|
||||
srcBuffer = malloc(benchedSize+NOISELENGTH); /* + noise */
|
||||
if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
|
||||
|
||||
/* Load input buffer */
|
||||
nbFiles = DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
|
||||
DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
|
||||
DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */
|
||||
|
||||
{ size_t const dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
|
||||
srcBuffer, fileSizes, nbFiles,
|
||||
params);
|
||||
if (ZDICT_isError(dictSize)) {
|
||||
DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
/* save dict */
|
||||
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
|
||||
DiB_saveDict(dictFileName, dictBuffer, dictSize);
|
||||
/* call buffer version */
|
||||
dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
|
||||
srcBuffer, fileSizes, nbFiles,
|
||||
params);
|
||||
if (ZDICT_isError(dictSize)) {
|
||||
DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
|
||||
/* save dict */
|
||||
DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
|
||||
DiB_saveDict(dictFileName, dictBuffer, dictSize);
|
||||
|
||||
/* clean up */
|
||||
_cleanup:
|
||||
free(srcBuffer);
|
||||
|
||||
+1
-2
@@ -32,8 +32,7 @@
|
||||
/*-*************************************
|
||||
* Dependencies
|
||||
***************************************/
|
||||
#define ZDICT_STATIC_LINKING_ONLY
|
||||
#include "zdict.h" /* ZDICT_params_t */
|
||||
#include "../lib/dictBuilder/zdict_static.h" /* ZDICT_params_t */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
|
||||
+51
-177
@@ -56,11 +56,8 @@
|
||||
|
||||
#include "mem.h"
|
||||
#include "fileio.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
|
||||
#include "zstd.h"
|
||||
#include "zstd_internal.h" /* MIN, KB, MB */
|
||||
#define ZBUFF_STATIC_LINKING_ONLY
|
||||
#include "zbuff.h"
|
||||
#include "zstd_static.h" /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
|
||||
#include "zbuff_static.h"
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
# include "zstd_legacy.h" /* ZSTD_isLegacy */
|
||||
@@ -74,7 +71,7 @@
|
||||
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
|
||||
# include <fcntl.h> /* _O_BINARY */
|
||||
# include <io.h> /* _setmode, _isatty */
|
||||
# define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); }
|
||||
# define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
|
||||
#else
|
||||
# include <unistd.h> /* isatty */
|
||||
# define SET_BINARY_MODE(file)
|
||||
@@ -99,7 +96,7 @@
|
||||
|
||||
#define CACHELINE 64
|
||||
|
||||
#define MAX_DICT_SIZE (8 MB) /* protection against large input (attack scenario) */
|
||||
#define MAX_DICT_SIZE (1 MB) /* protection against large input (attack scenario) ; can be changed */
|
||||
|
||||
#define FNSPACE 30
|
||||
|
||||
@@ -133,14 +130,6 @@ static U32 g_overwrite = 0;
|
||||
void FIO_overwriteMode(void) { g_overwrite=1; }
|
||||
static U32 g_maxWLog = 23;
|
||||
void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
|
||||
static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
|
||||
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
|
||||
static U32 g_dictIDFlag = 1;
|
||||
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
|
||||
static U32 g_checksumFlag = 0;
|
||||
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
|
||||
static U32 g_removeSrcFile = 0;
|
||||
void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
|
||||
|
||||
|
||||
/*-*************************************
|
||||
@@ -189,12 +178,8 @@ static FILE* FIO_openDstFile(const char* dstFileName)
|
||||
DISPLAYLEVEL(4,"Using stdout for output\n");
|
||||
f = stdout;
|
||||
SET_BINARY_MODE(stdout);
|
||||
if (g_sparseFileSupport==1) {
|
||||
g_sparseFileSupport = 0;
|
||||
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
|
||||
}
|
||||
} else {
|
||||
if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
|
||||
if (!g_overwrite) { /* Check if destination file already exists */
|
||||
f = fopen( dstFileName, "rb" );
|
||||
if (f != 0) { /* dest file exists, prompt for overwrite authorization */
|
||||
fclose(f);
|
||||
@@ -204,7 +189,8 @@ static FILE* FIO_openDstFile(const char* dstFileName)
|
||||
return 0;
|
||||
}
|
||||
DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
|
||||
{ int ch = getchar();
|
||||
{
|
||||
int ch = getchar();
|
||||
if ((ch!='Y') && (ch!='y')) {
|
||||
DISPLAY(" not overwritten \n");
|
||||
return 0;
|
||||
@@ -220,12 +206,12 @@ static FILE* FIO_openDstFile(const char* dstFileName)
|
||||
/*! FIO_loadFile() :
|
||||
* creates a buffer, pointed by `*bufferPtr`,
|
||||
* loads `filename` content into it,
|
||||
* up to MAX_DICT_SIZE bytes.
|
||||
* @return : loaded size
|
||||
* up to MAX_DICT_SIZE bytes
|
||||
*/
|
||||
static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
|
||||
{
|
||||
FILE* fileHandle;
|
||||
size_t readSize;
|
||||
U64 fileSize;
|
||||
|
||||
*bufferPtr = NULL;
|
||||
@@ -245,8 +231,8 @@ static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
|
||||
}
|
||||
*bufferPtr = (BYTE*)malloc((size_t)fileSize);
|
||||
if (*bufferPtr==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer");
|
||||
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
|
||||
if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
|
||||
readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
|
||||
if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName);
|
||||
fclose(fileHandle);
|
||||
return (size_t)fileSize;
|
||||
}
|
||||
@@ -316,18 +302,12 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
|
||||
/* init */
|
||||
{ ZSTD_parameters params;
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
params.fParams.checksumFlag = g_checksumFlag;
|
||||
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
||||
if ((g_maxWLog) && (params.cParams.windowLog > g_maxWLog)) {
|
||||
params.cParams.windowLog = g_maxWLog;
|
||||
params.cParams = ZSTD_adjustCParams(params.cParams, fileSize, ress.dictBufferSize);
|
||||
}
|
||||
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
|
||||
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
|
||||
} }
|
||||
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
|
||||
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
|
||||
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); }
|
||||
}
|
||||
|
||||
/* Main compression loop */
|
||||
readsize = 0;
|
||||
@@ -367,9 +347,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
|
||||
/* Status */
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
DISPLAYLEVEL(2,"%-20.20s :%6.2f%% (%6llu =>%6llu bytes, %s) \n", srcFileName,
|
||||
(double)compressedfilesize/readsize*100, (unsigned long long)readsize, (unsigned long long) compressedfilesize,
|
||||
dstFileName);
|
||||
DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
|
||||
(unsigned long long)readsize, (unsigned long long) compressedfilesize, (double)compressedfilesize/readsize*100);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -387,38 +366,36 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
|
||||
int result;
|
||||
|
||||
/* File check */
|
||||
if (UTIL_isDirectory(srcFileName)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
|
||||
return 1;
|
||||
}
|
||||
ress.srcFile = FIO_openSrcFile(srcFileName);
|
||||
if (!ress.srcFile) return 1; /* srcFile could not be opened */
|
||||
|
||||
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
|
||||
|
||||
fclose(ress.srcFile);
|
||||
if ((g_removeSrcFile) && (!result)) remove(srcFileName);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
/*! FIO_compressFilename_dstFile() :
|
||||
/*! FIO_compressFilename_extRess() :
|
||||
* @return : 0 : compression completed correctly,
|
||||
* 1 : pb
|
||||
* 1 : missing or pb opening srcFileName
|
||||
*/
|
||||
static int FIO_compressFilename_dstFile(cRess_t ress,
|
||||
static int FIO_compressFilename_extRess(cRess_t ress,
|
||||
const char* dstFileName, const char* srcFileName,
|
||||
int cLevel)
|
||||
{
|
||||
int result;
|
||||
|
||||
ress.srcFile = FIO_openSrcFile(srcFileName);
|
||||
if (ress.srcFile==0) return 1;
|
||||
ress.dstFile = FIO_openDstFile(dstFileName);
|
||||
if (ress.dstFile==0) { fclose(ress.srcFile); return 1; }
|
||||
|
||||
result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel);
|
||||
|
||||
if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
|
||||
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
|
||||
if (result!=0) remove(dstFileName); /* remove operation artefact */
|
||||
|
||||
fclose(ress.srcFile); /* no pb to expect : only reading */
|
||||
if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -426,13 +403,19 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
|
||||
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
|
||||
const char* dictFileName, int compressionLevel)
|
||||
{
|
||||
clock_t const start = clock();
|
||||
clock_t start;
|
||||
cRess_t ress;
|
||||
int issueWithSrcFile = 0;
|
||||
|
||||
/* Init */
|
||||
start = clock();
|
||||
ress = FIO_createCResources(dictFileName);
|
||||
|
||||
issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel);
|
||||
|
||||
cRess_t const ress = FIO_createCResources(dictFileName);
|
||||
int const issueWithSrcFile = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
|
||||
FIO_freeCResources(ress);
|
||||
|
||||
{ double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
|
||||
{ double seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
|
||||
DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
|
||||
}
|
||||
return issueWithSrcFile;
|
||||
@@ -456,7 +439,6 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
||||
if (!strcmp(suffix, stdoutmark)) {
|
||||
unsigned u;
|
||||
ress.dstFile = stdout;
|
||||
SET_BINARY_MODE(stdout);
|
||||
for (u=0; u<nbFiles; u++)
|
||||
missed_files += FIO_compressFilename_srcFile(ress, stdoutmark,
|
||||
inFileNamesTable[u], compressionLevel);
|
||||
@@ -468,7 +450,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
||||
if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
|
||||
strcpy(dstFileName, inFileNamesTable[u]);
|
||||
strcat(dstFileName, suffix);
|
||||
missed_files += FIO_compressFilename_dstFile(ress, dstFileName,
|
||||
missed_files += FIO_compressFilename_extRess(ress, dstFileName,
|
||||
inFileNamesTable[u], compressionLevel);
|
||||
} }
|
||||
|
||||
@@ -530,81 +512,6 @@ static void FIO_freeDResources(dRess_t ress)
|
||||
}
|
||||
|
||||
|
||||
/** FIO_fwriteSparse() :
|
||||
* @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
|
||||
static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
|
||||
{
|
||||
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
|
||||
size_t bufferSizeT = bufferSize / sizeof(size_t);
|
||||
const size_t* const bufferTEnd = bufferT + bufferSizeT;
|
||||
const size_t* ptrT = bufferT;
|
||||
static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */
|
||||
|
||||
if (!g_sparseFileSupport) { /* normal write */
|
||||
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
|
||||
if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* avoid int overflow */
|
||||
if (storedSkips > 1 GB) {
|
||||
int const seekResult = fseek(file, 1 GB, SEEK_CUR);
|
||||
if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
|
||||
storedSkips -= 1 GB;
|
||||
}
|
||||
|
||||
while (ptrT < bufferTEnd) {
|
||||
size_t seg0SizeT = segmentSizeT;
|
||||
size_t nb0T;
|
||||
|
||||
/* count leading zeros */
|
||||
if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT;
|
||||
bufferSizeT -= seg0SizeT;
|
||||
for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ;
|
||||
storedSkips += (unsigned)(nb0T * sizeof(size_t));
|
||||
|
||||
if (nb0T != seg0SizeT) { /* not all 0s */
|
||||
int const seekResult = fseek(file, storedSkips, SEEK_CUR);
|
||||
if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse");
|
||||
storedSkips = 0;
|
||||
seg0SizeT -= nb0T;
|
||||
ptrT += nb0T;
|
||||
{ size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file);
|
||||
if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block");
|
||||
} }
|
||||
ptrT += seg0SizeT;
|
||||
}
|
||||
|
||||
{ static size_t const maskT = sizeof(size_t)-1;
|
||||
if (bufferSize & maskT) { /* size not multiple of sizeof(size_t) : implies end of block */
|
||||
const char* const restStart = (const char*)bufferTEnd;
|
||||
const char* restPtr = restStart;
|
||||
size_t restSize = bufferSize & maskT;
|
||||
const char* const restEnd = restStart + restSize;
|
||||
for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
|
||||
storedSkips += (unsigned) (restPtr - restStart);
|
||||
if (restPtr != restEnd) {
|
||||
int seekResult = fseek(file, storedSkips, SEEK_CUR);
|
||||
if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
|
||||
storedSkips = 0;
|
||||
{ size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
|
||||
if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block");
|
||||
} } } }
|
||||
|
||||
return storedSkips;
|
||||
}
|
||||
|
||||
static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
|
||||
{
|
||||
if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */
|
||||
int const seekResult = fseek(file, storedSkips, SEEK_CUR);
|
||||
if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
|
||||
{ const char lastZeroByte[1] = { 0 };
|
||||
size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
|
||||
if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
|
||||
} }
|
||||
}
|
||||
|
||||
/** FIO_decompressFrame() :
|
||||
@return : size of decoded frame
|
||||
*/
|
||||
@@ -613,25 +520,26 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
|
||||
{
|
||||
U64 frameSize = 0;
|
||||
size_t readSize;
|
||||
U32 storedSkips = 0;
|
||||
|
||||
ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
|
||||
|
||||
/* Header loading (optional, saves one loop) */
|
||||
{ size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */
|
||||
{ size_t const toLoad = ZSTD_frameHeaderSize_min - alreadyLoaded; /* assumption : ZSTD_frameHeaderSize_min >= alreadyLoaded */
|
||||
size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
|
||||
readSize = alreadyLoaded + loadedSize;
|
||||
}
|
||||
|
||||
/* Main decompression Loop */
|
||||
while (1) {
|
||||
/* Decode */
|
||||
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
|
||||
size_t const toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
|
||||
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
|
||||
readSize -= inSize;
|
||||
|
||||
/* Write block */
|
||||
storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, decodedSize, storedSkips);
|
||||
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
|
||||
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block into destination"); }
|
||||
frameSize += decodedSize;
|
||||
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
|
||||
|
||||
@@ -645,34 +553,10 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
|
||||
EXM_THROW(35, "Read error");
|
||||
}
|
||||
|
||||
FIO_fwriteSparseEnd(foutput, storedSkips);
|
||||
|
||||
return frameSize;
|
||||
}
|
||||
|
||||
|
||||
/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
|
||||
@return : 0 (no error) */
|
||||
static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize)
|
||||
{
|
||||
size_t const blockSize = MIN (64 KB, bufferSize);
|
||||
size_t readFromInput = 1;
|
||||
unsigned storedSkips = 0;
|
||||
|
||||
/* assumption : first 4 bytes already loaded (magic number detection), and stored within buffer */
|
||||
{ size_t const sizeCheck = fwrite(buffer, 1, 4, foutput);
|
||||
if (sizeCheck != 4) EXM_THROW(50, "Pass-through write error"); }
|
||||
|
||||
while (readFromInput) {
|
||||
readFromInput = fread(buffer, 1, blockSize, finput);
|
||||
storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, storedSkips);
|
||||
}
|
||||
|
||||
FIO_fwriteSparseEnd(foutput, storedSkips);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/** FIO_decompressSrcFile() :
|
||||
Decompression `srcFileName` into `ress.dstFile`
|
||||
@return : 0 : OK
|
||||
@@ -682,13 +566,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
|
||||
{
|
||||
unsigned long long filesize = 0;
|
||||
FILE* const dstFile = ress.dstFile;
|
||||
FILE* srcFile;
|
||||
|
||||
if (UTIL_isDirectory(srcFileName)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
|
||||
return 1;
|
||||
}
|
||||
srcFile = FIO_openSrcFile(srcFileName);
|
||||
FILE* const srcFile = FIO_openSrcFile(srcFileName);
|
||||
if (srcFile==0) return 1;
|
||||
|
||||
/* for each frame */
|
||||
@@ -705,23 +583,19 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) && (magic != ZSTD_MAGICNUMBER)) {
|
||||
if (g_overwrite) /* -df : pass-through mode */
|
||||
return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
|
||||
else {
|
||||
DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
|
||||
return 1;
|
||||
} } }
|
||||
if (magic != ZSTD_MAGICNUMBER) {
|
||||
DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
|
||||
return 1;
|
||||
} }
|
||||
filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead);
|
||||
}
|
||||
|
||||
/* Final Status */
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
DISPLAYLEVEL(2, "%-20.20s: %llu bytes \n", srcFileName, filesize);
|
||||
DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
|
||||
|
||||
/* Close */
|
||||
fclose(srcFile);
|
||||
if (g_removeSrcFile) remove(srcFileName);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -731,7 +605,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
|
||||
@return : 0 : OK
|
||||
1 : operation aborted (src not available, dst already taken, etc.)
|
||||
*/
|
||||
static int FIO_decompressDstFile(dRess_t ress,
|
||||
static int FIO_decompressFile_extRess(dRess_t ress,
|
||||
const char* dstFileName, const char* srcFileName)
|
||||
{
|
||||
int result;
|
||||
@@ -739,9 +613,9 @@ static int FIO_decompressDstFile(dRess_t ress,
|
||||
if (ress.dstFile==0) return 1;
|
||||
|
||||
result = FIO_decompressSrcFile(ress, srcFileName);
|
||||
if (result != 0) remove(dstFileName);
|
||||
|
||||
if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
|
||||
if (result != 0) remove(dstFileName);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -752,7 +626,7 @@ int FIO_decompressFilename(const char* dstFileName, const char* srcFileName,
|
||||
int missingFiles = 0;
|
||||
dRess_t ress = FIO_createDResources(dictFileName);
|
||||
|
||||
missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName);
|
||||
missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName);
|
||||
|
||||
FIO_freeDResources(ress);
|
||||
return missingFiles;
|
||||
@@ -799,7 +673,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
|
||||
memcpy(dstFileName, srcFileName, sfnSize - suffixSize);
|
||||
dstFileName[sfnSize-suffixSize] = '\0';
|
||||
|
||||
missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName);
|
||||
missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName);
|
||||
}
|
||||
free(dstFileName);
|
||||
}
|
||||
|
||||
+1
-5
@@ -46,11 +46,7 @@ extern "C" {
|
||||
***************************************/
|
||||
void FIO_overwriteMode(void);
|
||||
void FIO_setNotificationLevel(unsigned level);
|
||||
void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */
|
||||
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
|
||||
void FIO_setDictIDFlag(unsigned dictIDFlag);
|
||||
void FIO_setChecksumFlag(unsigned checksumFlag);
|
||||
void FIO_setRemoveSrcFile(unsigned flag);
|
||||
void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */
|
||||
|
||||
|
||||
/*-*************************************
|
||||
|
||||
@@ -28,13 +28,12 @@
|
||||
#include "util.h" /* Compiler options, UTIL_GetFileSize */
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
|
||||
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressBegin, ZSTD_compressContinue, etc. */
|
||||
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||
#define FSE_STATIC_LINKING_ONLY /* FSE_DTABLE_SIZE_U32 */
|
||||
#include "fse.h"
|
||||
#include "zstd_static.h"
|
||||
#include "fse_static.h"
|
||||
#include "zbuff.h"
|
||||
#include "datagen.h"
|
||||
|
||||
@@ -43,8 +42,11 @@
|
||||
* Constants
|
||||
**************************************/
|
||||
#define PROGRAM_DESCRIPTION "Zstandard speed analyzer"
|
||||
#ifndef ZSTD_VERSION
|
||||
# define ZSTD_VERSION ""
|
||||
#endif
|
||||
#define AUTHOR "Yann Collet"
|
||||
#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR, __DATE__
|
||||
#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION, (int)(sizeof(void*)*8), AUTHOR, __DATE__
|
||||
|
||||
|
||||
#define KB *(1<<10)
|
||||
@@ -545,3 +547,4 @@ int main(int argc, const char** argv)
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
+208
-254
@@ -28,7 +28,7 @@
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define _CRT_SECURE_NO_WARNINGS /* fgets */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
|
||||
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
|
||||
#endif
|
||||
|
||||
|
||||
@@ -40,22 +40,25 @@
|
||||
#include <sys/timeb.h> /* timeb */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <time.h> /* clock_t */
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue */
|
||||
#include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */
|
||||
#include "zdict.h" /* ZDICT_trainFromBuffer */
|
||||
#include "zstd_static.h"
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "mem.h"
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
#include "mem.h"
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#ifndef ZSTD_VERSION
|
||||
# define ZSTD_VERSION ""
|
||||
#endif
|
||||
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
static const size_t COMPRESSIBLE_NOISE_LENGTH = 10 MB; /* capital, used to be a macro */
|
||||
static const U32 FUZ_compressibility_default = 50;
|
||||
static const U32 nbTestsDefault = 30000;
|
||||
|
||||
@@ -79,6 +82,7 @@ static clock_t g_displayClock = 0;
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#define MIN(a,b) ((a)<(b)?(a):(b))
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
|
||||
static clock_t FUZ_clockSpan(clock_t cStart)
|
||||
{
|
||||
@@ -86,8 +90,8 @@ static clock_t FUZ_clockSpan(clock_t cStart)
|
||||
}
|
||||
|
||||
|
||||
#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
static unsigned FUZ_rand(unsigned* src)
|
||||
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
unsigned int FUZ_rand(unsigned int* src)
|
||||
{
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
@@ -104,131 +108,142 @@ static unsigned FUZ_highbit32(U32 v32)
|
||||
{
|
||||
unsigned nbBits = 0;
|
||||
if (v32==0) return 0;
|
||||
while (v32) v32 >>= 1, nbBits++;
|
||||
while (v32) {
|
||||
v32 >>= 1;
|
||||
nbBits ++;
|
||||
}
|
||||
return nbBits;
|
||||
}
|
||||
|
||||
|
||||
#define CHECKTEST(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
|
||||
#define CHECK(fn) { CHECKTEST(err, fn); }
|
||||
#define CHECKPLUS(var, fn, more) { CHECKTEST(var, fn); more; }
|
||||
static int basicUnitTests(U32 seed, double compressibility)
|
||||
{
|
||||
size_t const CNBuffSize = 5 MB;
|
||||
void* const CNBuffer = malloc(CNBuffSize);
|
||||
void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize));
|
||||
void* const decodedBuffer = malloc(CNBuffSize);
|
||||
int testResult = 0;
|
||||
void* CNBuffer;
|
||||
void* compressedBuffer;
|
||||
void* decodedBuffer;
|
||||
U32 randState = seed;
|
||||
size_t result, cSize;
|
||||
U32 testNb=0;
|
||||
size_t cSize;
|
||||
|
||||
/* Create compressible noise */
|
||||
/* Create compressible test buffer */
|
||||
CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
|
||||
compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
|
||||
decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
|
||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
|
||||
DISPLAY("Not enough memory, aborting\n");
|
||||
testResult = 1;
|
||||
goto _end;
|
||||
}
|
||||
RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
|
||||
RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., randState);
|
||||
|
||||
/* Basic tests */
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
|
||||
CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
CNBuffer, CNBuffSize, 1),
|
||||
cSize=r );
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH);
|
||||
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
cSize = result;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
|
||||
CHECKPLUS( r , ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH);
|
||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
if (result != COMPRESSIBLE_NOISE_LENGTH) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
{ size_t u;
|
||||
for (u=0; u<CNBuffSize; u++) {
|
||||
if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;;
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
{ size_t i;
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++) {
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
|
||||
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
|
||||
if (!ZSTD_isError(r)) goto _output_error;
|
||||
if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
|
||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1);
|
||||
if (!ZSTD_isError(result)) goto _output_error;
|
||||
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
|
||||
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
|
||||
if (!ZSTD_isError(r)) goto _output_error;
|
||||
if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
|
||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1);
|
||||
if (!ZSTD_isError(result)) goto _output_error;
|
||||
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Dictionary and CCtx Duplication tests */
|
||||
{ ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
|
||||
ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
static const size_t dictSize = 551;
|
||||
{ ZSTD_CCtx* ctxOrig = ZSTD_createCCtx();
|
||||
ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
size_t const dictSize = 500;
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
|
||||
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
|
||||
if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
|
||||
if (!ZSTD_isError(copyResult)) goto _output_error; } /* error should be detected */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
|
||||
CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
|
||||
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
|
||||
{ size_t const initResult = ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2);
|
||||
if (ZSTD_isError(initResult)) goto _output_error; }
|
||||
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
|
||||
if (ZSTD_isError(copyResult)) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
|
||||
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
|
||||
cSize = 0;
|
||||
CHECKPLUS(r, ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
|
||||
cSize += r);
|
||||
CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize),
|
||||
cSize += r);
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
cSize += result;
|
||||
result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
cSize += result;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
CNBuffer, dictSize),
|
||||
if (r != CNBuffSize - dictSize) goto _output_error);
|
||||
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
|
||||
result = ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
|
||||
compressedBuffer, cSize,
|
||||
CNBuffer, dictSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
|
||||
{ size_t const cSizeOrig = cSize;
|
||||
cSize = 0;
|
||||
CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
|
||||
cSize += r);
|
||||
CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize),
|
||||
cSize += r);
|
||||
if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> have same size */
|
||||
result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
cSize += result;
|
||||
result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
cSize += result;
|
||||
if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
result = ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
|
||||
compressedBuffer, cSize,
|
||||
CNBuffer, dictSize),
|
||||
if (r != CNBuffSize - dictSize) goto _output_error);
|
||||
CNBuffer, dictSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
|
||||
{ size_t const testSize = CNBuffSize / 3;
|
||||
{ ZSTD_compressionParameters const cPar = ZSTD_getCParams(2, testSize, dictSize);
|
||||
ZSTD_frameParameters const fPar = { 1 , 0 , 0 };
|
||||
ZSTD_parameters p;
|
||||
p.cParams = cPar; p.fParams = fPar;
|
||||
CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
|
||||
}
|
||||
CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
|
||||
|
||||
CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
|
||||
(const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
|
||||
cSize = r);
|
||||
{ size_t const testSize = COMPRESSIBLE_NOISE_LENGTH / 3;
|
||||
{ ZSTD_parameters p;
|
||||
p.cParams = ZSTD_getCParams(2, testSize, dictSize);
|
||||
p.fParams.contentSizeFlag = 1;
|
||||
{ size_t const initResult = ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1);
|
||||
if (ZSTD_isError(initResult)) goto _output_error;
|
||||
} }
|
||||
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
|
||||
if (ZSTD_isError(copyResult)) goto _output_error; }
|
||||
cSize = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
{ ZSTD_frameParams fp;
|
||||
if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error;
|
||||
size_t const gfpResult = ZSTD_getFrameParams(&fp, compressedBuffer, cSize);
|
||||
if (gfpResult!=0) goto _output_error;
|
||||
if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error;
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
@@ -238,113 +253,55 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
|
||||
/* Dictionary and dictBuilder tests */
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
size_t dictSize = 16 KB;
|
||||
void* dictBuffer = malloc(dictSize);
|
||||
size_t const totalSampleSize = 1 MB;
|
||||
size_t const sampleUnitSize = 8 KB;
|
||||
U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
|
||||
size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
|
||||
|
||||
if (dictBuffer==NULL || samplesSizes==NULL) {
|
||||
free(dictBuffer);
|
||||
free(samplesSizes);
|
||||
goto _output_error;
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : dictBuilder : ", testNb++);
|
||||
{ U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
|
||||
dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
|
||||
CNBuffer, samplesSizes, nbSamples);
|
||||
if (ZDICT_isError(dictSize)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
|
||||
cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
CNBuffer, CNBuffSize,
|
||||
dictBuffer, dictSize, 4);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
|
||||
{ ZSTD_frameParameters const fParams = { 0 /*contentSize*/, 0 /*checksum*/, 1 /*NoDictID*/ };
|
||||
ZSTD_compressionParameters const cParams = ZSTD_getCParams(3, CNBuffSize, dictSize);
|
||||
ZSTD_parameters p;
|
||||
p.cParams = cParams; p.fParams = fParams;
|
||||
cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
CNBuffer, CNBuffSize,
|
||||
dictBuffer, dictSize, p);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : frame built without dictID should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
compressedBuffer, cSize,
|
||||
dictBuffer, dictSize),
|
||||
if (r != CNBuffSize) goto _output_error);
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
free(dictBuffer);
|
||||
free(samplesSizes);
|
||||
}
|
||||
|
||||
/* Decompression defense tests */
|
||||
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
|
||||
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3);
|
||||
if (!ZSTD_isError(r)) goto _output_error;
|
||||
if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; }
|
||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
|
||||
if (!ZSTD_isError(result)) goto _output_error;
|
||||
if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
|
||||
((char*)(CNBuffer))[0] = 1;
|
||||
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
|
||||
if (!ZSTD_isError(r)) goto _output_error; }
|
||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4);
|
||||
if (!ZSTD_isError(result)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* block API tests */
|
||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
static const size_t blockSize = 100 KB;
|
||||
static const size_t dictSize = 16 KB;
|
||||
const size_t blockSize = 100 KB;
|
||||
const size_t dictSize = 16 KB;
|
||||
|
||||
/* basic block compression */
|
||||
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
|
||||
CHECK( ZSTD_compressBegin(cctx, 5) );
|
||||
result = ZSTD_compressBegin(cctx, 5);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
|
||||
CHECK( ZSTD_decompressBegin(dctx) );
|
||||
{ CHECKTEST(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||
if (r != blockSize) goto _output_error; }
|
||||
result = ZSTD_decompressBegin(dctx);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
if (result != blockSize) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* dictionary block compression */
|
||||
DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++);
|
||||
CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
|
||||
result = ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
|
||||
CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
|
||||
{ CHECKTEST( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
|
||||
if (r != blockSize) goto _output_error; }
|
||||
result = ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
if (result != blockSize) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
@@ -354,29 +311,32 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
/* long rle test */
|
||||
{ size_t sampleSize = 0;
|
||||
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
|
||||
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1);
|
||||
RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState);
|
||||
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
|
||||
sampleSize += 256 KB - 1;
|
||||
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2);
|
||||
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState);
|
||||
sampleSize += 96 KB;
|
||||
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
{ CHECKTEST(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
|
||||
if (regenSize!=sampleSize) goto _output_error; }
|
||||
result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
if (result!=sampleSize) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
}
|
||||
|
||||
/* All zeroes test (test bug #137) */
|
||||
/* All zeroes test (#137 verif) */
|
||||
#define ZEROESLENGTH 100
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
|
||||
memset(CNBuffer, 0, ZEROESLENGTH);
|
||||
{ CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
|
||||
cSize = r; }
|
||||
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
cSize = result;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
|
||||
{ CHECKTEST(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
|
||||
if (r != ZEROESLENGTH) goto _output_error; }
|
||||
result = ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
if (result != ZEROESLENGTH) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* nbSeq limit test */
|
||||
@@ -396,21 +356,23 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
}}
|
||||
|
||||
/* randomly fills CNBuffer with prepared 3-bytes sequences */
|
||||
{ int i; for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
|
||||
U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
|
||||
{ int i; for (i=0; i < _3BYTESTESTLENGTH; ) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
|
||||
U32 id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
|
||||
((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
|
||||
((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
|
||||
((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
|
||||
i += 3;
|
||||
} }}
|
||||
DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++);
|
||||
{ CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
|
||||
CNBuffer, _3BYTESTESTLENGTH, 19) );
|
||||
cSize = r; }
|
||||
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), CNBuffer, _3BYTESTESTLENGTH, 19);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
cSize = result;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
|
||||
{ CHECKTEST(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
|
||||
if (r != _3BYTESTESTLENGTH) goto _output_error; }
|
||||
result = ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
if (result != _3BYTESTESTLENGTH) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
_end:
|
||||
@@ -430,11 +392,11 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
||||
{
|
||||
const BYTE* b1 = (const BYTE*)buf1;
|
||||
const BYTE* b2 = (const BYTE*)buf2;
|
||||
size_t u;
|
||||
for (u=0; u<max; u++) {
|
||||
if (b1[u] != b2[u]) break;
|
||||
size_t i;
|
||||
for (i=0; i<max; i++) {
|
||||
if (b1[i] != b2[i]) break;
|
||||
}
|
||||
return u;
|
||||
return i;
|
||||
}
|
||||
|
||||
|
||||
@@ -450,7 +412,6 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
|
||||
return FUZ_rLogLength(seed, logLength);
|
||||
}
|
||||
|
||||
#undef CHECK
|
||||
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||
|
||||
@@ -458,29 +419,35 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
{
|
||||
static const U32 maxSrcLog = 23;
|
||||
static const U32 maxSampleLog = 22;
|
||||
size_t const srcBufferSize = (size_t)1<<maxSrcLog;
|
||||
size_t const dstBufferSize = (size_t)1<<maxSampleLog;
|
||||
size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
|
||||
BYTE* cNoiseBuffer[5];
|
||||
BYTE* srcBuffer; /* jumping pointer */
|
||||
BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
|
||||
BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
|
||||
BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
|
||||
ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
|
||||
ZSTD_CCtx* const ctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
BYTE* srcBuffer;
|
||||
BYTE* cBuffer;
|
||||
BYTE* dstBuffer;
|
||||
BYTE* mirrorBuffer;
|
||||
size_t srcBufferSize = (size_t)1<<maxSrcLog;
|
||||
size_t dstBufferSize = (size_t)1<<maxSampleLog;
|
||||
size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
|
||||
U32 result = 0;
|
||||
U32 testNb = 0;
|
||||
U32 coreSeed = seed, lseed = 0;
|
||||
clock_t const startClock = clock();
|
||||
ZSTD_CCtx* refCtx;
|
||||
ZSTD_CCtx* ctx;
|
||||
ZSTD_DCtx* dctx;
|
||||
clock_t startClock = clock();
|
||||
clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
|
||||
|
||||
/* allocation */
|
||||
refCtx = ZSTD_createCCtx();
|
||||
ctx = ZSTD_createCCtx();
|
||||
dctx= ZSTD_createDCtx();
|
||||
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
|
||||
dstBuffer = (BYTE*)malloc (dstBufferSize);
|
||||
mirrorBuffer = (BYTE*)malloc (dstBufferSize);
|
||||
cBuffer = (BYTE*)malloc (cBufferSize);
|
||||
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|
||||
|| !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
|
||||
"Not enough memory, fuzzer tests cancelled");
|
||||
@@ -498,8 +465,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
|
||||
/* main test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
|
||||
size_t sampleSize, maxTestSize, totalTestSize;
|
||||
size_t cSize, totalCSize, totalGenSize;
|
||||
size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
|
||||
size_t cSize, dSize, totalCSize, totalGenSize;
|
||||
U32 sampleSizeLog, nbChunks, n;
|
||||
XXH64_state_t xxhState;
|
||||
U64 crcOrig;
|
||||
BYTE* sampleBuffer;
|
||||
@@ -529,17 +497,18 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
}
|
||||
|
||||
/* select src segment */
|
||||
sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = FUZ_rLogLength(&lseed, sampleSizeLog);
|
||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||
|
||||
/* create sample buffer (to catch read error with valgrind & sanitizers) */
|
||||
sampleBuffer = (BYTE*)malloc(sampleSize);
|
||||
CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
|
||||
{ size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
|
||||
CHECK (sampleBuffer==NULL, "not enough memory for sample buffer");
|
||||
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize);
|
||||
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
|
||||
|
||||
/* compression tests */
|
||||
{ unsigned const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize)/3))) + 1;
|
||||
{ int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (sampleSizeLog/3))) + 1;
|
||||
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
|
||||
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
|
||||
|
||||
@@ -553,7 +522,9 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); }
|
||||
{ U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
|
||||
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
|
||||
} }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* frame header decompression test */
|
||||
{ ZSTD_frameParams dParams;
|
||||
@@ -564,7 +535,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
|
||||
/* successful decompression test */
|
||||
{ size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
|
||||
size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
|
||||
dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
|
||||
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
|
||||
{ U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
|
||||
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
|
||||
@@ -575,7 +546,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
/* truncated src decompression test */
|
||||
{ size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
|
||||
size_t const tooSmallSize = cSize - missing;
|
||||
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
|
||||
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */
|
||||
CHECK(cBufferTooSmall == NULL, "not enough memory !");
|
||||
memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
|
||||
{ size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
|
||||
@@ -608,12 +579,13 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
}
|
||||
if (pos <= cSize) break;
|
||||
/* add noise */
|
||||
{ U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
|
||||
U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
|
||||
size_t const mask = (1<<nbBits) - 1;
|
||||
size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
|
||||
size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
|
||||
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
|
||||
{ U32 nbBits = FUZ_rand(&lseed) % maxNbBits;
|
||||
size_t mask, noiseStart, noiseLength;
|
||||
if (nbBits>0) nbBits--;
|
||||
mask = (1<<nbBits) - 1;
|
||||
noiseLength = (FUZ_rand(&lseed) & mask) + 1;
|
||||
if ( pos+noiseLength > cSize ) noiseLength = cSize-pos;
|
||||
noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
|
||||
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
|
||||
pos += noiseLength;
|
||||
} } }
|
||||
@@ -637,45 +609,35 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
||||
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
|
||||
|
||||
dictSize = FUZ_randomLength(&lseed, maxSampleLog); /* needed also for decompression */
|
||||
dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
|
||||
sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||
dict = srcBuffer + sampleStart;
|
||||
dictSize = sampleSize;
|
||||
|
||||
if (FUZ_rand(&lseed) & 0xF) {
|
||||
size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
|
||||
} else {
|
||||
ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
|
||||
ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
|
||||
!(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
|
||||
0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
|
||||
ZSTD_parameters p;
|
||||
size_t errorCode;
|
||||
p.cParams = cPar; p.fParams = fpar;
|
||||
errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
|
||||
}
|
||||
{ size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); }
|
||||
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); }
|
||||
}
|
||||
XXH64_reset(&xxhState, 0);
|
||||
{ U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
|
||||
U32 n;
|
||||
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
|
||||
size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
|
||||
size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
|
||||
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
|
||||
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||
|
||||
if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
|
||||
if (totalTestSize+segmentSize > maxTestSize) break;
|
||||
|
||||
{ size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
|
||||
CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
|
||||
cSize += compressResult;
|
||||
}
|
||||
XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
|
||||
memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
|
||||
totalTestSize += segmentSize;
|
||||
} }
|
||||
if (cBufferSize-cSize < ZSTD_compressBound(sampleSize)) break; /* avoid invalid dstBufferTooSmall */
|
||||
if (totalTestSize+sampleSize > maxTestSize) break;
|
||||
|
||||
{ size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
|
||||
CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
|
||||
cSize += compressResult;
|
||||
}
|
||||
XXH64_update(&xxhState, srcBuffer+sampleStart, sampleSize);
|
||||
memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
|
||||
totalTestSize += sampleSize;
|
||||
}
|
||||
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
|
||||
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
|
||||
cSize += flushResult;
|
||||
@@ -683,9 +645,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
crcOrig = XXH64_digest(&xxhState);
|
||||
|
||||
/* streaming decompression test */
|
||||
if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
|
||||
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
|
||||
CHECK (ZSTD_isError(errorCode), "ZSTD_decompressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); }
|
||||
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); }
|
||||
totalCSize = 0;
|
||||
totalGenSize = 0;
|
||||
while (totalCSize < cSize) {
|
||||
@@ -696,12 +657,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
|
||||
totalCSize += inSize;
|
||||
}
|
||||
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
|
||||
CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
|
||||
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
|
||||
CHECK (totalCSize != cSize, "compressed data should be fully read")
|
||||
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
||||
if (crcDest!=crcOrig) {
|
||||
size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
|
||||
CHECK (1, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
|
||||
CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
|
||||
(U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]);
|
||||
} }
|
||||
} /* for ( ; (testNb <= nbTests) */
|
||||
@@ -845,19 +806,12 @@ int main(int argc, const char** argv)
|
||||
} } } } /* for (argNb=1; argNb<argc; argNb++) */
|
||||
|
||||
/* Get Seed */
|
||||
DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
|
||||
|
||||
if (!seedset) {
|
||||
time_t const t = time(NULL);
|
||||
U32 const h = XXH32(&t, sizeof(t), 1);
|
||||
seed = h % 10000;
|
||||
}
|
||||
DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION);
|
||||
|
||||
if (!seedset) seed = (U32)(clock() % 10000);
|
||||
DISPLAY("Seed = %u\n", seed);
|
||||
if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %u%%\n", proba);
|
||||
|
||||
if (nbTests < testNb) nbTests = testNb;
|
||||
|
||||
if (testNb==0)
|
||||
result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
|
||||
if (!result)
|
||||
|
||||
+105
-20
@@ -473,7 +473,8 @@ unsigned long long FIOv05_decompressFrame(dRessv05_t ress,
|
||||
}
|
||||
|
||||
|
||||
/*===== v0.6.x =====*/
|
||||
|
||||
/*===== v0.7.x =====*/
|
||||
|
||||
typedef struct {
|
||||
void* srcBuffer;
|
||||
@@ -482,51 +483,127 @@ typedef struct {
|
||||
size_t dstBufferSize;
|
||||
const void* dictBuffer;
|
||||
size_t dictBufferSize;
|
||||
ZBUFFv06_DCtx* dctx;
|
||||
} dRessv06_t;
|
||||
ZBUFFv07_DCtx* dctx;
|
||||
} dRessv07_t;
|
||||
|
||||
static dRessv06_t FIOv06_createDResources(void)
|
||||
static dRessv07_t FIOv07_createDResources(void)
|
||||
{
|
||||
dRessv06_t ress;
|
||||
dRessv07_t ress;
|
||||
|
||||
/* init */
|
||||
ress.dctx = ZBUFFv06_createDCtx();
|
||||
ress.dctx = ZBUFFv07_createDCtx();
|
||||
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
|
||||
ress.dictBuffer = NULL; ress.dictBufferSize=0;
|
||||
|
||||
/* Allocate Memory */
|
||||
ress.srcBufferSize = ZBUFFv06_recommendedDInSize();
|
||||
ress.srcBufferSize = ZBUFFv07_recommendedDInSize();
|
||||
ress.srcBuffer = malloc(ress.srcBufferSize);
|
||||
ress.dstBufferSize = ZBUFFv06_recommendedDOutSize();
|
||||
ress.dstBufferSize = ZBUFFv07_recommendedDOutSize();
|
||||
ress.dstBuffer = malloc(ress.dstBufferSize);
|
||||
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
|
||||
|
||||
return ress;
|
||||
}
|
||||
|
||||
static void FIOv06_freeDResources(dRessv06_t ress)
|
||||
static void FIOv07_freeDResources(dRessv07_t ress)
|
||||
{
|
||||
size_t const errorCode = ZBUFFv06_freeDCtx(ress.dctx);
|
||||
if (ZBUFFv06_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv06_getErrorName(errorCode));
|
||||
size_t const errorCode = ZBUFFv07_freeDCtx(ress.dctx);
|
||||
if (ZBUFFv07_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv07_getErrorName(errorCode));
|
||||
free(ress.srcBuffer);
|
||||
free(ress.dstBuffer);
|
||||
}
|
||||
|
||||
|
||||
unsigned long long FIOv06_decompressFrame(dRessv06_t ress,
|
||||
unsigned long long FIOv07_decompressFrame(dRessv07_t ress,
|
||||
FILE* foutput, FILE* finput)
|
||||
{
|
||||
U64 frameSize = 0;
|
||||
size_t readSize = 4;
|
||||
|
||||
MEM_writeLE32(ress.srcBuffer, ZSTDv06_MAGICNUMBER);
|
||||
ZBUFFv06_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
|
||||
MEM_writeLE32(ress.srcBuffer, ZSTDv07_MAGICNUMBER);
|
||||
ZBUFFv07_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
|
||||
|
||||
while (1) {
|
||||
/* Decode */
|
||||
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
|
||||
size_t toRead = ZBUFFv06_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
|
||||
if (ZBUFFv06_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv06_getErrorName(toRead));
|
||||
size_t toRead = ZBUFFv07_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
|
||||
if (ZBUFFv07_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv07_getErrorName(toRead));
|
||||
readSize -= inSize;
|
||||
|
||||
/* Write block */
|
||||
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
|
||||
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); }
|
||||
frameSize += decodedSize;
|
||||
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
|
||||
|
||||
if (toRead == 0) break;
|
||||
if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
|
||||
|
||||
/* Fill input buffer */
|
||||
if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
|
||||
readSize = fread(ress.srcBuffer, 1, toRead, finput);
|
||||
if (readSize != toRead) EXM_THROW(35, "Read error");
|
||||
}
|
||||
|
||||
return frameSize;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*===== v0.8.x =====*/
|
||||
|
||||
typedef struct {
|
||||
void* srcBuffer;
|
||||
size_t srcBufferSize;
|
||||
void* dstBuffer;
|
||||
size_t dstBufferSize;
|
||||
const void* dictBuffer;
|
||||
size_t dictBufferSize;
|
||||
ZBUFFv08_DCtx* dctx;
|
||||
} dRessv08_t;
|
||||
|
||||
static dRessv08_t FIOv08_createDResources(void)
|
||||
{
|
||||
dRessv08_t ress;
|
||||
|
||||
/* init */
|
||||
ress.dctx = ZBUFFv08_createDCtx();
|
||||
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
|
||||
ress.dictBuffer = NULL; ress.dictBufferSize=0;
|
||||
|
||||
/* Allocate Memory */
|
||||
ress.srcBufferSize = ZBUFFv08_recommendedDInSize();
|
||||
ress.srcBuffer = malloc(ress.srcBufferSize);
|
||||
ress.dstBufferSize = ZBUFFv08_recommendedDOutSize();
|
||||
ress.dstBuffer = malloc(ress.dstBufferSize);
|
||||
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
|
||||
|
||||
return ress;
|
||||
}
|
||||
|
||||
static void FIOv08_freeDResources(dRessv08_t ress)
|
||||
{
|
||||
size_t const errorCode = ZBUFFv08_freeDCtx(ress.dctx);
|
||||
if (ZBUFFv08_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv08_getErrorName(errorCode));
|
||||
free(ress.srcBuffer);
|
||||
free(ress.dstBuffer);
|
||||
}
|
||||
|
||||
|
||||
unsigned long long FIOv08_decompressFrame(dRessv08_t ress,
|
||||
FILE* foutput, FILE* finput)
|
||||
{
|
||||
U64 frameSize = 0;
|
||||
size_t readSize = 4;
|
||||
|
||||
MEM_writeLE32(ress.srcBuffer, ZSTDv08_MAGICNUMBER);
|
||||
ZBUFFv08_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
|
||||
|
||||
while (1) {
|
||||
/* Decode */
|
||||
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
|
||||
size_t toRead = ZBUFFv08_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
|
||||
if (ZBUFFv08_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv08_getErrorName(toRead));
|
||||
readSize -= inSize;
|
||||
|
||||
/* Write block */
|
||||
@@ -576,12 +653,20 @@ unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput,
|
||||
FIOv05_freeDResources(r);
|
||||
return s;
|
||||
} }
|
||||
case ZSTDv06_MAGICNUMBER :
|
||||
{ dRessv06_t r = FIOv06_createDResources();
|
||||
case ZSTDv07_MAGICNUMBER :
|
||||
{ dRessv07_t r = FIOv07_createDResources();
|
||||
r.dictBuffer = dictBuffer;
|
||||
r.dictBufferSize = dictSize;
|
||||
{ unsigned long long const s = FIOv06_decompressFrame(r, foutput, finput);
|
||||
FIOv06_freeDResources(r);
|
||||
{ unsigned long long const s = FIOv07_decompressFrame(r, foutput, finput);
|
||||
FIOv07_freeDResources(r);
|
||||
return s;
|
||||
} }
|
||||
case ZSTDv08_MAGICNUMBER :
|
||||
{ dRessv08_t r = FIOv08_createDResources();
|
||||
r.dictBuffer = dictBuffer;
|
||||
r.dictBufferSize = dictSize;
|
||||
{ unsigned long long const s = FIOv08_decompressFrame(r, foutput, finput);
|
||||
FIOv08_freeDResources(r);
|
||||
return s;
|
||||
} }
|
||||
default :
|
||||
|
||||
+22
-18
@@ -48,8 +48,7 @@
|
||||
#endif
|
||||
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
|
||||
#include "zstd.h"
|
||||
#include "zstd_static.h"
|
||||
#include "datagen.h"
|
||||
#include "xxhash.h"
|
||||
|
||||
@@ -58,8 +57,11 @@
|
||||
* Constants
|
||||
**************************************/
|
||||
#define PROGRAM_DESCRIPTION "ZSTD parameters tester"
|
||||
#ifndef ZSTD_VERSION
|
||||
# define ZSTD_VERSION ""
|
||||
#endif
|
||||
#define AUTHOR "Yann Collet"
|
||||
#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR, __DATE__
|
||||
#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION, (int)(sizeof(void*)*8), AUTHOR, __DATE__
|
||||
|
||||
|
||||
#define KB *(1<<10)
|
||||
@@ -601,22 +603,22 @@ static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
}
|
||||
|
||||
|
||||
static ZSTD_compressionParameters randomParams(void)
|
||||
static void potentialRandomParams(ZSTD_compressionParameters* p, U32 inverseChance)
|
||||
{
|
||||
ZSTD_compressionParameters p;
|
||||
U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1));
|
||||
U32 validated = 0;
|
||||
if (!chance)
|
||||
while (!validated) {
|
||||
/* totally random entry */
|
||||
p.chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN;
|
||||
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
|
||||
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
|
||||
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
|
||||
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
|
||||
p.targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
|
||||
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
|
||||
validated = !ZSTD_isError(ZSTD_checkCParams(p));
|
||||
p->chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN;
|
||||
p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
|
||||
p->searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
|
||||
p->windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
|
||||
p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
|
||||
p->targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN;
|
||||
p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btopt +1));
|
||||
validated = !ZSTD_isError(ZSTD_checkCParams(*p));
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
static void BMK_selectRandomStart(
|
||||
@@ -624,10 +626,12 @@ static void BMK_selectRandomStart(
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
U32 const id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
|
||||
U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1));
|
||||
if ((id==0) || (winners[id].params.windowLog==0)) {
|
||||
/* totally random entry */
|
||||
ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0);
|
||||
ZSTD_compressionParameters p;
|
||||
potentialRandomParams(&p, 1);
|
||||
ZSTD_adjustCParams(&p, srcSize, 0);
|
||||
playAround(f, winners, p, srcBuffer, srcSize, ctx);
|
||||
}
|
||||
else
|
||||
@@ -648,7 +652,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
|
||||
if (g_singleRun) {
|
||||
BMK_result_t testResult;
|
||||
g_params = ZSTD_adjustCParams(g_params, srcSize, 0);
|
||||
ZSTD_adjustCParams(&g_params, srcSize, 0);
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
|
||||
DISPLAY("\n");
|
||||
return;
|
||||
@@ -860,7 +864,7 @@ int optimizeForSize(char* inFileName)
|
||||
do {
|
||||
params = winner.params;
|
||||
paramVariation(¶ms);
|
||||
if ((FUZ_rand(&g_rand) & 15) == 1) params = randomParams();
|
||||
potentialRandomParams(¶ms, 16);
|
||||
|
||||
/* exclude faster if already played set of params */
|
||||
if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue;
|
||||
|
||||
+52
-137
@@ -1,7 +1,7 @@
|
||||
#!/bin/sh -e
|
||||
|
||||
die() {
|
||||
$ECHO "$@" 1>&2
|
||||
echo "$@" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -14,77 +14,47 @@ roundTripTest() {
|
||||
fi
|
||||
|
||||
rm -f tmp1 tmp2
|
||||
$ECHO "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
|
||||
./datagen $1 $p | $MD5SUM > tmp1
|
||||
./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | $MD5SUM > tmp2
|
||||
echo "roundTripTest: ./datagen $1 $p | $ZSTD -v$c | $ZSTD -d"
|
||||
./datagen $1 $p | md5sum > tmp1
|
||||
./datagen $1 $p | $ZSTD -vq$c | $ZSTD -d | md5sum > tmp2
|
||||
diff -q tmp1 tmp2
|
||||
}
|
||||
|
||||
isWindows=false
|
||||
ECHO="echo"
|
||||
case "$OS" in
|
||||
Windows*)
|
||||
isWindows=true
|
||||
ECHO="echo -e"
|
||||
;;
|
||||
esac
|
||||
|
||||
MD5SUM="md5sum"
|
||||
if [ "$TRAVIS_OS_NAME" = "osx" ]; then
|
||||
MD5SUM="md5 -r"
|
||||
fi
|
||||
|
||||
$ECHO "\nStarting playTests.sh isWindows=$isWindows TRAVIS_OS_NAME=$TRAVIS_OS_NAME"
|
||||
|
||||
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
|
||||
|
||||
file $ZSTD
|
||||
$ECHO "\n**** simple tests **** "
|
||||
|
||||
echo "\n**** simple tests **** "
|
||||
./datagen > tmp
|
||||
$ZSTD -f tmp # trivial compression case, creates tmp.zst
|
||||
$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp)
|
||||
$ECHO "test : too large compression level (must fail)"
|
||||
$ZSTD -f tmp # trivial compression case, creates tmp.zst
|
||||
$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp)
|
||||
echo "test : too large compression level (must fail)"
|
||||
$ZSTD -99 tmp && die "too large compression level undetected"
|
||||
$ECHO "test : compress to stdout"
|
||||
$ZSTD tmp -c > tmpCompressed
|
||||
echo "test : compress to stdout"
|
||||
$ZSTD tmp -c > tmpCompressed
|
||||
$ZSTD tmp --stdout > tmpCompressed # long command format
|
||||
$ECHO "test : null-length file roundtrip"
|
||||
$ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
|
||||
$ECHO "test : decompress file with wrong suffix (must fail)"
|
||||
echo "test : null-length file roundtrip"
|
||||
echo -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
|
||||
echo "test : decompress file with wrong suffix (must fail)"
|
||||
$ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
|
||||
$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout
|
||||
$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout
|
||||
$ZSTD --decompress tmpCompressed -c > tmpResult
|
||||
$ZSTD --decompress tmpCompressed --stdout > tmpResult
|
||||
if [ "$isWindows" = false ] ; then
|
||||
$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout
|
||||
$ZSTD -d - < tmp.zst > /dev/null
|
||||
fi
|
||||
$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout
|
||||
$ZSTD -d - < tmp.zst > /dev/null
|
||||
$ZSTD -dc < tmp.zst > /dev/null
|
||||
$ZSTD -dc - < tmp.zst > /dev/null
|
||||
$ZSTD -q tmp && die "overwrite check failed!"
|
||||
$ZSTD -q -f tmp
|
||||
$ZSTD -q --force tmp
|
||||
$ZSTD -df tmp && die "should have refused : wrong extension"
|
||||
$ECHO "test : file removal"
|
||||
$ZSTD -f --rm tmp
|
||||
ls tmp && die "tmp should no longer be present"
|
||||
$ZSTD -f -d --rm tmp.zst
|
||||
ls tmp.zst && die "tmp.zst should no longer be present"
|
||||
rm tmp
|
||||
$ZSTD -f tmp && die "tmp not present : should have failed"
|
||||
ls tmp.zst && die "tmp.zst should not be created"
|
||||
cp tmp tmp2.zst
|
||||
$ZSTD -df tmp2.zst && die "should have failed : wrong format"
|
||||
rm tmp2.zst
|
||||
|
||||
echo "\n**** frame concatenation **** "
|
||||
|
||||
$ECHO "\n**** Pass-Through mode **** "
|
||||
$ECHO "Hello world !" | $ZSTD -df
|
||||
$ECHO "Hello world !" | $ZSTD -dcf
|
||||
|
||||
|
||||
$ECHO "\n**** frame concatenation **** "
|
||||
|
||||
$ECHO "hello " > hello.tmp
|
||||
$ECHO "world!" > world.tmp
|
||||
echo "hello " > hello.tmp
|
||||
echo "world!" > world.tmp
|
||||
cat hello.tmp world.tmp > helloworld.tmp
|
||||
$ZSTD -c hello.tmp > hello.zstd
|
||||
$ZSTD -c world.tmp > world.zstd
|
||||
@@ -93,118 +63,62 @@ $ZSTD -dc helloworld.zstd > result.tmp
|
||||
cat result.tmp
|
||||
sdiff helloworld.tmp result.tmp
|
||||
rm ./*.tmp ./*.zstd
|
||||
$ECHO "frame concatenation tests completed"
|
||||
|
||||
echo frame concatenation test completed
|
||||
|
||||
|
||||
if [ "$isWindows" = false ] ; then
|
||||
$ECHO "\n**** flush write error test **** "
|
||||
echo "\n**** flush write error test **** "
|
||||
|
||||
$ECHO "$ECHO foo | $ZSTD > /dev/full"
|
||||
$ECHO foo | $ZSTD > /dev/full && die "write error not detected!"
|
||||
$ECHO "$ECHO foo | $ZSTD | $ZSTD -d > /dev/full"
|
||||
$ECHO foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
|
||||
fi
|
||||
echo "echo foo | $ZSTD > /dev/full"
|
||||
echo foo | $ZSTD > /dev/full && die "write error not detected!"
|
||||
echo "echo foo | $ZSTD | $ZSTD -d > /dev/full"
|
||||
echo foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
|
||||
|
||||
|
||||
$ECHO "\n**** test sparse file support **** "
|
||||
echo "\n**** dictionary tests **** "
|
||||
|
||||
./datagen -g5M -P100 > tmpSparse
|
||||
$ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen
|
||||
diff -s tmpSparse tmpSparseRegen
|
||||
$ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse
|
||||
diff -s tmpSparse tmpOutSparse
|
||||
$ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse
|
||||
diff -s tmpSparse tmpOutNoSparse
|
||||
ls -ls tmpSparse*
|
||||
./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks)
|
||||
./datagen -s1 -g1200007 -P100 | diff -s - tmpSparseOdd
|
||||
ls -ls tmpSparseOdd
|
||||
$ECHO "\n Sparse Compatibility with Console :"
|
||||
$ECHO "Hello World 1 !" | $ZSTD | $ZSTD -d -c
|
||||
$ECHO "Hello World 2 !" | $ZSTD | $ZSTD -d | cat
|
||||
$ECHO "\n Sparse Compatibility with Append :"
|
||||
./datagen -P100 -g1M > tmpSparse1M
|
||||
cat tmpSparse1M tmpSparse1M > tmpSparse2M
|
||||
$ZSTD -v -f tmpSparse1M -o tmpSparseCompressed
|
||||
$ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
|
||||
$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
|
||||
ls -ls tmpSparse*
|
||||
diff tmpSparse2M tmpSparseRegenerated
|
||||
rm tmpSparse*
|
||||
./datagen > tmpDict
|
||||
./datagen -g1M | md5sum > tmp1
|
||||
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
|
||||
diff -q tmp1 tmp2
|
||||
$ZSTD --train *.c *.h -o tmpDict
|
||||
$ZSTD paramgrill.c -D tmpDict -of tmp
|
||||
$ZSTD -d tmp -D tmpDict -of result
|
||||
diff paramgrill.c result
|
||||
|
||||
|
||||
$ECHO "\n**** multiple files tests **** "
|
||||
echo "\n**** multiple files tests **** "
|
||||
|
||||
./datagen -s1 > tmp1 2> /dev/null
|
||||
./datagen -s2 -g100K > tmp2 2> /dev/null
|
||||
./datagen -s3 -g1M > tmp3 2> /dev/null
|
||||
$ZSTD -f tmp*
|
||||
$ECHO "compress tmp* : "
|
||||
echo "compress tmp* : "
|
||||
ls -ls tmp*
|
||||
rm tmp1 tmp2 tmp3
|
||||
$ECHO "decompress tmp* : "
|
||||
echo "decompress tmp* : "
|
||||
$ZSTD -df *.zst
|
||||
ls -ls tmp*
|
||||
$ECHO "compress tmp* into stdout > tmpall : "
|
||||
echo "compress tmp* into stdout > tmpall : "
|
||||
$ZSTD -c tmp1 tmp2 tmp3 > tmpall
|
||||
ls -ls tmp*
|
||||
$ECHO "decompress tmpall* into stdout > tmpdec : "
|
||||
echo "decompress tmpall* into stdout > tmpdec : "
|
||||
cp tmpall tmpall2
|
||||
$ZSTD -dc tmpall* > tmpdec
|
||||
ls -ls tmp*
|
||||
$ECHO "compress multiple files including a missing one (notHere) : "
|
||||
echo "compress multiple files including a missing one (notHere) : "
|
||||
$ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
|
||||
|
||||
|
||||
$ECHO "\n**** dictionary tests **** "
|
||||
|
||||
./datagen > tmpDict
|
||||
./datagen -g1M | $MD5SUM > tmp1
|
||||
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
|
||||
diff -q tmp1 tmp2
|
||||
$ECHO "- Create first dictionary"
|
||||
$ZSTD --train *.c -o tmpDict
|
||||
cp zstdcli.c tmp
|
||||
$ZSTD -f tmp -D tmpDict
|
||||
$ZSTD -d tmp.zst -D tmpDict -of result
|
||||
diff zstdcli.c result
|
||||
$ECHO "- Create second (different) dictionary"
|
||||
$ZSTD --train *.c *.h -o tmpDictC
|
||||
$ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!"
|
||||
$ECHO "- Create dictionary with short dictID"
|
||||
$ZSTD --train *.c --dictID 1 -o tmpDict1
|
||||
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
|
||||
$ECHO "- Compress without dictID"
|
||||
$ZSTD -f tmp -D tmpDict1 --no-dictID
|
||||
$ZSTD -d tmp.zst -D tmpDict -of result
|
||||
diff zstdcli.c result
|
||||
$ECHO "- Compress multiple files with dictionary"
|
||||
rm -rf dirTestDict
|
||||
mkdir dirTestDict
|
||||
cp *.c dirTestDict
|
||||
cp *.h dirTestDict
|
||||
cat dirTestDict/* | $MD5SUM > tmph1 # note : we expect same file order to generate same hash
|
||||
$ZSTD -f dirTestDict/* -D tmpDictC
|
||||
$ZSTD -d dirTestDict/*.zst -D tmpDictC -c | $MD5SUM > tmph2
|
||||
diff -q tmph1 tmph2
|
||||
rm -rf dirTestDict
|
||||
rm tmp*
|
||||
|
||||
|
||||
$ECHO "\n**** integrity tests **** "
|
||||
|
||||
$ECHO "test one file (tmp1.zst) "
|
||||
./datagen > tmp1
|
||||
$ZSTD tmp1
|
||||
echo "\n**** integrity tests **** "
|
||||
echo "test one file (tmp1.zst) "
|
||||
$ZSTD -t tmp1.zst
|
||||
$ZSTD --test tmp1.zst
|
||||
$ECHO "test multiple files (*.zst) "
|
||||
echo "test multiple files (*.zst) "
|
||||
$ZSTD -t *.zst
|
||||
$ECHO "test good and bad files (*) "
|
||||
echo "test good and bad files (*) "
|
||||
$ZSTD -t * && die "bad files not detected !"
|
||||
|
||||
|
||||
$ECHO "\n**** zstd round-trip tests **** "
|
||||
echo "\n**** zstd round-trip tests **** "
|
||||
|
||||
roundTripTest
|
||||
roundTripTest -g15K # TableID==3
|
||||
@@ -212,13 +126,13 @@ roundTripTest -g127K # TableID==2
|
||||
roundTripTest -g255K # TableID==1
|
||||
roundTripTest -g513K # TableID==0
|
||||
roundTripTest -g512K 6 # greedy, hash chain
|
||||
roundTripTest -g512K 16 # btlazy2
|
||||
roundTripTest -g512K 16 # btlazy2
|
||||
roundTripTest -g512K 19 # btopt
|
||||
|
||||
rm tmp*
|
||||
|
||||
if [ "$1" != "--test-large-data" ]; then
|
||||
$ECHO "Skipping large data tests"
|
||||
echo "Skipping large data tests"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -251,3 +165,4 @@ roundTripTest -g99000000 -P99 20
|
||||
roundTripTest -g6000000000 -P99 1
|
||||
|
||||
rm tmp*
|
||||
|
||||
|
||||
@@ -1,185 +0,0 @@
|
||||
/*
|
||||
roundTripCrash
|
||||
Copyright (C) Yann Collet 2013-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net
|
||||
*/
|
||||
/*
|
||||
This program takes a file in input,
|
||||
performs a zstd round-trip test (compression - decompress)
|
||||
compares the result with original
|
||||
and generates a crash (double free) on corruption detection.
|
||||
*/
|
||||
|
||||
/*===========================================
|
||||
* Dependencies
|
||||
*==========================================*/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdlib.h> /* malloc, free, exit */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <sys/types.h> /* stat */
|
||||
#include <sys/stat.h> /* stat */
|
||||
#include "zstd.h"
|
||||
|
||||
/** roundTripTest() :
|
||||
* Compresses `srcBuff` into `compressedBuff`,
|
||||
* then decompresses `compressedBuff` into `resultBuff`.
|
||||
* Compression level used is derived from first content byte.
|
||||
* @return : result of decompression, which should be == `srcSize`
|
||||
* or an error code if either compression or decompression fails.
|
||||
* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)`
|
||||
* for compression to be guaranteed to work */
|
||||
static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity,
|
||||
void* compressedBuff, size_t compressedBuffCapacity,
|
||||
const void* srcBuff, size_t srcBuffSize)
|
||||
{
|
||||
static const int maxClevel = 19;
|
||||
int const cLevel = (!srcBuffSize) ? 1 : (*(const unsigned char*)srcBuff) % maxClevel;
|
||||
size_t const cSize = ZSTD_compress(compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, cLevel);
|
||||
if (ZSTD_isError(cSize)) {
|
||||
fprintf(stderr, "Compression error : %s \n", ZSTD_getErrorName(cSize));
|
||||
return cSize;
|
||||
}
|
||||
return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, cSize);
|
||||
}
|
||||
|
||||
|
||||
static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
|
||||
{
|
||||
const char* ip1 = (const char*)buff1;
|
||||
const char* ip2 = (const char*)buff2;
|
||||
size_t pos;
|
||||
|
||||
for (pos=0; pos<buffSize; pos++)
|
||||
if (ip1[pos]!=ip2[pos])
|
||||
break;
|
||||
|
||||
return pos;
|
||||
}
|
||||
|
||||
|
||||
static void roundTripCheck(const void* srcBuff, size_t srcBuffSize)
|
||||
{
|
||||
size_t const cBuffSize = ZSTD_compressBound(srcBuffSize);
|
||||
void* cBuff = malloc(cBuffSize);
|
||||
void* rBuff = malloc(cBuffSize);
|
||||
#define CRASH { free(cBuff); free(cBuff); } /* double free, to crash program */
|
||||
|
||||
if (!cBuff || !rBuff) {
|
||||
fprintf(stderr, "not enough memory ! \n");
|
||||
exit (1);
|
||||
}
|
||||
|
||||
{ size_t const result = roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize);
|
||||
if (ZSTD_isError(result)) {
|
||||
fprintf(stderr, "roundTripTest error : %s \n", ZSTD_getErrorName(result));
|
||||
CRASH;
|
||||
}
|
||||
if (result != srcBuffSize) {
|
||||
fprintf(stderr, "Incorrect regenerated size : %u != %u\n", (unsigned)result, (unsigned)srcBuffSize);
|
||||
CRASH;
|
||||
}
|
||||
if (checkBuffers(srcBuff, rBuff, srcBuffSize) != srcBuffSize) {
|
||||
fprintf(stderr, "Silent decoding corruption !!!");
|
||||
CRASH;
|
||||
}
|
||||
}
|
||||
|
||||
free(cBuff);
|
||||
free(rBuff);
|
||||
}
|
||||
|
||||
|
||||
static size_t getFileSize(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct _stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
|
||||
#endif
|
||||
return (size_t)statbuf.st_size;
|
||||
}
|
||||
|
||||
|
||||
static int isDirectory(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct _stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/** loadFile() :
|
||||
* requirement : `buffer` size >= `fileSize` */
|
||||
static void loadFile(void* buffer, const char* fileName, size_t fileSize)
|
||||
{
|
||||
FILE* const f = fopen(fileName, "rb");
|
||||
if (isDirectory(fileName)) {
|
||||
fprintf(stderr, "Ignoring %s directory \n", fileName);
|
||||
exit(2);
|
||||
}
|
||||
if (f==NULL) {
|
||||
fprintf(stderr, "Impossible to open %s \n", fileName);
|
||||
exit(3);
|
||||
}
|
||||
{ size_t const readSize = fread(buffer, 1, fileSize, f);
|
||||
if (readSize != fileSize) {
|
||||
fprintf(stderr, "Error reading %s \n", fileName);
|
||||
exit(5);
|
||||
} }
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
|
||||
static void fileCheck(const char* fileName)
|
||||
{
|
||||
size_t const fileSize = getFileSize(fileName);
|
||||
void* buffer = malloc(fileSize);
|
||||
if (!buffer) {
|
||||
fprintf(stderr, "not enough memory \n");
|
||||
exit(4);
|
||||
}
|
||||
loadFile(buffer, fileName, fileSize);
|
||||
roundTripCheck(buffer, fileSize);
|
||||
free (buffer);
|
||||
}
|
||||
|
||||
int main(int argCount, const char** argv) {
|
||||
if (argCount < 2) {
|
||||
fprintf(stderr, "Error : no argument : need input file \n");
|
||||
exit(9);
|
||||
}
|
||||
fileCheck(argv[1]);
|
||||
fprintf(stderr, "no pb detected\n");
|
||||
return 0;
|
||||
}
|
||||
+11
-13
@@ -37,6 +37,7 @@ extern "C" {
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */
|
||||
#define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
|
||||
//#define snprintf _snprintf
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@@ -46,7 +47,7 @@ extern "C" {
|
||||
# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */
|
||||
# if defined(__sun__) /* Sun Solaris 32-bits requires specific definitions */
|
||||
# define _LARGEFILE_SOURCE /* fseeko, ftello */
|
||||
# else
|
||||
# else
|
||||
# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */
|
||||
# endif
|
||||
#endif
|
||||
@@ -90,15 +91,11 @@ extern "C" {
|
||||
# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
|
||||
# define UTIL_sleep(s) Sleep(1000*s)
|
||||
# define UTIL_sleepMilli(milli) Sleep(milli)
|
||||
#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__)))
|
||||
#elif (defined(__unix__) || defined(__unix) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__)))
|
||||
# include <unistd.h>
|
||||
# include <sys/resource.h> /* setpriority */
|
||||
# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
|
||||
# if defined(PRIO_PROCESS)
|
||||
# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
|
||||
# else
|
||||
# define SET_HIGH_PRIORITY /* disabled */
|
||||
# endif
|
||||
# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
|
||||
# define UTIL_sleep(s) sleep(s)
|
||||
# if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L)
|
||||
# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
|
||||
@@ -143,8 +140,8 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
|
||||
{
|
||||
UTIL_time_t clockStart, clockEnd;
|
||||
UTIL_getTime(&clockStart);
|
||||
do {
|
||||
UTIL_getTime(&clockEnd);
|
||||
do {
|
||||
UTIL_getTime(&clockEnd);
|
||||
} while (UTIL_getSpanTimeNano(ticksPerSecond, clockStart, clockEnd) == 0);
|
||||
}
|
||||
|
||||
@@ -283,7 +280,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
|
||||
fprintf(stderr, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp (entry->d_name, "..") == 0 ||
|
||||
strcmp (entry->d_name, ".") == 0) continue;
|
||||
@@ -327,8 +324,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
|
||||
|
||||
#endif // #ifdef _WIN32
|
||||
|
||||
/*
|
||||
* UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
|
||||
/*
|
||||
* UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
|
||||
* and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
|
||||
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
|
||||
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
|
||||
@@ -345,7 +342,8 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
|
||||
bufend = buf + LIST_SIZE_INCREASE;
|
||||
|
||||
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
|
||||
if (!UTIL_isDirectory(inputNames[i])) {
|
||||
if (UTIL_doesFileExists(inputNames[i])) {
|
||||
// printf ("UTIL_doesFileExists=[%s]\n", inputNames[i]);
|
||||
size_t len = strlen(inputNames[i]);
|
||||
if (buf + pos + len >= bufend) {
|
||||
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
|
||||
|
||||
+69
-111
@@ -35,23 +35,25 @@
|
||||
/*-************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include <stdlib.h> /* free */
|
||||
#include <stdio.h> /* fgets, sscanf */
|
||||
#include <sys/timeb.h> /* timeb */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <stdlib.h> /* free */
|
||||
#include <stdio.h> /* fgets, sscanf */
|
||||
#include <sys/timeb.h> /* timeb */
|
||||
#include <string.h> /* strcmp */
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
|
||||
#include "zstd.h" /* ZSTD_compressBound */
|
||||
#define ZBUFF_STATIC_LINKING_ONLY
|
||||
#include "zbuff.h" /* ZBUFF_createCCtx_advanced */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "zbuff.h"
|
||||
#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#include "xxhash.h" /* XXH64_* */
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#ifndef ZSTD_VERSION
|
||||
# define ZSTD_VERSION ""
|
||||
#endif
|
||||
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
@@ -98,7 +100,7 @@ static U32 FUZ_GetMilliStart(void)
|
||||
|
||||
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
|
||||
{
|
||||
U32 const nCurrent = FUZ_GetMilliStart();
|
||||
U32 nCurrent = FUZ_GetMilliStart();
|
||||
U32 nSpan = nCurrent - nTimeStart;
|
||||
if (nTimeStart > nCurrent)
|
||||
nSpan += 0x100000 * 1000;
|
||||
@@ -130,120 +132,89 @@ static unsigned FUZ_highbit32(U32 v32)
|
||||
}
|
||||
*/
|
||||
|
||||
static void* ZBUFF_allocFunction(void* opaque, size_t size)
|
||||
{
|
||||
void* address = malloc(size);
|
||||
(void)opaque;
|
||||
/* DISPLAYLEVEL(4, "alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
|
||||
return address;
|
||||
}
|
||||
|
||||
static void ZBUFF_freeFunction(void* opaque, void* address)
|
||||
{
|
||||
(void)opaque;
|
||||
/* if (address) DISPLAYLEVEL(4, "free %p opaque=%p \n", address, opaque); */
|
||||
free(address);
|
||||
}
|
||||
|
||||
static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
|
||||
static int basicUnitTests(U32 seed, double compressibility)
|
||||
{
|
||||
int testResult = 0;
|
||||
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
void* CNBuffer = malloc(CNBufferSize);
|
||||
size_t const skippableFrameSize = 11;
|
||||
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
|
||||
size_t const compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
|
||||
void* compressedBuffer = malloc(compressedBufferSize);
|
||||
size_t const decodedBufferSize = CNBufferSize;
|
||||
void* decodedBuffer = malloc(decodedBufferSize);
|
||||
size_t cSize, readSize, readSkipSize, genSize;
|
||||
U32 randState = seed;
|
||||
size_t result, cSize, readSize, genSize;
|
||||
U32 testNb=0;
|
||||
ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
|
||||
ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
|
||||
ZBUFF_CCtx* zc = ZBUFF_createCCtx();
|
||||
ZBUFF_DCtx* zd = ZBUFF_createDCtx();
|
||||
|
||||
/* Create compressible test buffer */
|
||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
|
||||
DISPLAY("Not enough memory, aborting\n");
|
||||
goto _output_error;
|
||||
}
|
||||
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
|
||||
|
||||
/* generate skippable frame */
|
||||
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
|
||||
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
|
||||
cSize = skippableFrameSize + 8;
|
||||
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., randState);
|
||||
|
||||
/* Basic compression test */
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
|
||||
readSize = CNBufferSize;
|
||||
genSize = compressedBufferSize;
|
||||
{ size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
|
||||
if (ZBUFF_isError(r)) goto _output_error; }
|
||||
result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
|
||||
if (ZBUFF_isError(result)) goto _output_error;
|
||||
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
|
||||
cSize += genSize;
|
||||
cSize = genSize;
|
||||
genSize = compressedBufferSize - cSize;
|
||||
{ size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
|
||||
if (r != 0) goto _output_error; } /*< error, or some data not flushed */
|
||||
result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
|
||||
if (result != 0) goto _output_error; /* error, or some data not flushed */
|
||||
cSize += genSize;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
/* skippable frame test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
|
||||
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||
readSkipSize = cSize;
|
||||
genSize = CNBufferSize;
|
||||
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
|
||||
if (r != 0) goto _output_error; }
|
||||
if (genSize != 0) goto _output_error; /* skippable frame len is 0 */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Basic decompression test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||
readSize = cSize - readSkipSize;
|
||||
readSize = cSize;
|
||||
genSize = CNBufferSize;
|
||||
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
|
||||
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize);
|
||||
if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */
|
||||
if (readSize != cSize) goto _output_error; /* should have read the entire frame */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* check regenerated data is byte exact */
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
{ size_t i;
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
for (i=0; i<CNBufferSize; i++) {
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
}
|
||||
|
||||
/* Byte-by-byte decompression test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
|
||||
{ size_t r, pIn=0, pOut=0;
|
||||
do
|
||||
{ ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||
r = 1;
|
||||
while (r) {
|
||||
size_t inS = 1;
|
||||
size_t outS = 1;
|
||||
r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
|
||||
pIn += inS;
|
||||
pOut += outS;
|
||||
}
|
||||
readSize = pIn;
|
||||
genSize = pOut;
|
||||
} while (genSize==0);
|
||||
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
|
||||
{ size_t r = 1, pIn=0, pOut=0;
|
||||
while (r) {
|
||||
size_t inS = 1;
|
||||
size_t outS = 1;
|
||||
r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
|
||||
pIn += inS;
|
||||
pOut += outS;
|
||||
}
|
||||
readSize = pIn;
|
||||
genSize = pOut;
|
||||
}
|
||||
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
if (readSize != cSize) goto _output_error; /* should have read the entire frame */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* check regenerated data is byte exact */
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
{ size_t i;
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
for (i=0; i<CNBufferSize; i++) {
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||
} }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
}
|
||||
|
||||
_end:
|
||||
ZBUFF_freeCCtx(zc);
|
||||
@@ -264,13 +235,19 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
||||
{
|
||||
const BYTE* b1 = (const BYTE*)buf1;
|
||||
const BYTE* b2 = (const BYTE*)buf2;
|
||||
size_t u;
|
||||
for (u=0; u<max; u++) {
|
||||
if (b1[u] != b2[u]) break;
|
||||
size_t i;
|
||||
for (i=0; i<max; i++) {
|
||||
if (b1[i] != b2[i]) break;
|
||||
}
|
||||
return u;
|
||||
return i;
|
||||
}
|
||||
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||
|
||||
|
||||
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
|
||||
{
|
||||
size_t const lengthMask = ((size_t)1 << logLength) - 1;
|
||||
@@ -283,11 +260,6 @@ static size_t FUZ_randomLength(U32* seed, U32 maxLog)
|
||||
return FUZ_rLogLength(seed, logLength);
|
||||
}
|
||||
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||
|
||||
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
|
||||
{
|
||||
static const U32 maxSrcLog = 24;
|
||||
@@ -352,8 +324,8 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
FUZ_rand(&coreSeed);
|
||||
lseed = coreSeed ^ prime1;
|
||||
|
||||
/* states full reset (unsynchronized) */
|
||||
/* some issues only happen when reusing states in a specific sequence of parameters */
|
||||
/* state total reset */
|
||||
/* some problems only happen when states are re-used in a specific order */
|
||||
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
|
||||
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
|
||||
|
||||
@@ -376,21 +348,15 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
|
||||
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
|
||||
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
|
||||
maxTestSize = FUZ_rLogLength(&lseed, testLog);
|
||||
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
|
||||
/* random dictionary selection */
|
||||
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
|
||||
dict = srcBuffer + dictStart;
|
||||
{ size_t dictStart;
|
||||
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
|
||||
dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
|
||||
dict = srcBuffer + dictStart;
|
||||
}
|
||||
{ ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
|
||||
U32 const checksum = FUZ_rand(&lseed) & 1;
|
||||
U32 const noDictIDFlag = FUZ_rand(&lseed) & 1;
|
||||
ZSTD_frameParameters const fPar = { 0, checksum, noDictIDFlag };
|
||||
ZSTD_parameters params;
|
||||
params.cParams = cPar;
|
||||
params.fParams = fPar;
|
||||
{ size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, 0);
|
||||
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
|
||||
} } }
|
||||
{ size_t const initError = ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel);
|
||||
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
|
||||
} }
|
||||
|
||||
/* multi-segments compression test */
|
||||
XXH64_reset(&xxhState, 0);
|
||||
@@ -524,8 +490,6 @@ int main(int argc, const char** argv)
|
||||
int result=0;
|
||||
U32 mainPause = 0;
|
||||
const char* programName = argv[0];
|
||||
ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL };
|
||||
ZSTD_customMem customNULL = { NULL, NULL, NULL };
|
||||
|
||||
/* Check command line */
|
||||
for(argNb=1; argNb<argc; argNb++) {
|
||||
@@ -616,7 +580,7 @@ int main(int argc, const char** argv)
|
||||
} } } /* for(argNb=1; argNb<argc; argNb++) */
|
||||
|
||||
/* Get Seed */
|
||||
DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
|
||||
DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION);
|
||||
|
||||
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
|
||||
DISPLAY("Seed = %u\n", seed);
|
||||
@@ -624,13 +588,7 @@ int main(int argc, const char** argv)
|
||||
|
||||
if (nbTests<=0) nbTests=1;
|
||||
|
||||
if (testNb==0) {
|
||||
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
|
||||
if (!result) {
|
||||
DISPLAYLEVEL(4, "Unit tests using customMem :\n")
|
||||
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
|
||||
} }
|
||||
|
||||
if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
|
||||
if (!result)
|
||||
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
|
||||
|
||||
|
||||
+5
-20
@@ -18,11 +18,11 @@
|
||||
.PP
|
||||
.B unzstd
|
||||
is equivalent to
|
||||
.BR "zstd \-d"
|
||||
.BR "zstd \-d"
|
||||
.br
|
||||
.B zstdcat
|
||||
is equivalent to
|
||||
.BR "zstd \-dc"
|
||||
.BR "zstd \-dc"
|
||||
.br
|
||||
|
||||
.SH DESCRIPTION
|
||||
@@ -43,7 +43,7 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.B \-#
|
||||
# compression level [1-22] (default:1)
|
||||
# compression level [1-21] (default:1)
|
||||
.TP
|
||||
.BR \-d ", " --decompress
|
||||
decompression
|
||||
@@ -52,7 +52,7 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
|
||||
use `file` as Dictionary to compress or decompress FILE(s)
|
||||
.TP
|
||||
.B \-o file
|
||||
save result into `file` (only possible with a single INPUT-FILE)
|
||||
save result into `file` (only possible with a single input FILE)
|
||||
.TP
|
||||
.BR \-f ", " --force
|
||||
overwrite output without prompting
|
||||
@@ -71,13 +71,6 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
|
||||
.TP
|
||||
.BR \-c ", " --stdout
|
||||
force write to standard output, even if it is the console
|
||||
.TP
|
||||
.BR \-C ", " --check
|
||||
add integrity check computed from uncompressed data
|
||||
.TP
|
||||
.BR \-t ", " --test
|
||||
Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
|
||||
No files are created or removed.
|
||||
|
||||
.SH DICTIONARY
|
||||
.PP
|
||||
@@ -97,15 +90,7 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
|
||||
dictionary saved into `file` (default: dictionary)
|
||||
.TP
|
||||
.B \--maxdict #
|
||||
limit dictionary to specified size (default : 112640)
|
||||
.TP
|
||||
.B \--dictID #
|
||||
A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary.
|
||||
By default, zstd will create a 4-bytes random number ID.
|
||||
It's possible to give a precise number instead.
|
||||
Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header,
|
||||
and an ID < 65536 will only need 2 bytes. This compares favorably to 4 bytes default.
|
||||
However, it's up to the dictionary manager to not assign twice the same ID to 2 different dictionaries.
|
||||
limit dictionary to specified size (default : 112640)
|
||||
.TP
|
||||
.B \-s#
|
||||
dictionary selectivity level (default: 9)
|
||||
|
||||
+63
-114
@@ -38,11 +38,10 @@
|
||||
#ifndef ZSTD_NOBENCH
|
||||
# include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */
|
||||
#endif
|
||||
#include "zstd_static.h" /* ZSTD_maxCLevel, ZSTD version numbers */
|
||||
#ifndef ZSTD_NODICT
|
||||
# include "dibio.h"
|
||||
#endif
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
|
||||
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||
|
||||
|
||||
|
||||
@@ -50,15 +49,15 @@
|
||||
* OS-specific Includes
|
||||
**************************************/
|
||||
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
|
||||
# include <io.h> /* _isatty */
|
||||
# include <fcntl.h> /* _O_BINARY */
|
||||
# include <io.h> /* _setmode, _isatty */
|
||||
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
|
||||
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
|
||||
#else
|
||||
#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE)
|
||||
extern int fileno(FILE *stream); /* triggers fileno() within <stdio.h> on POSIX */
|
||||
# include <unistd.h> /* isatty */
|
||||
# define SET_BINARY_MODE(file)
|
||||
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
|
||||
#else
|
||||
# define IS_CONSOLE(stdStream) 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
@@ -126,37 +125,28 @@ static int usage_advanced(const char* programName)
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Advanced arguments :\n");
|
||||
DISPLAY( " -V : display Version number and exit\n");
|
||||
DISPLAY( " -t : test compressed file integrity \n");
|
||||
DISPLAY( " -v : verbose mode\n");
|
||||
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
|
||||
DISPLAY( " -c : force write to standard output, even if it is the console\n");
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
DISPLAY( " -r : operate recursively on directories\n");
|
||||
#endif
|
||||
DISPLAY( "--rm : remove source files after successful de/compression \n");
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
|
||||
DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
|
||||
DISPLAY( "--check : enable integrity check\n");
|
||||
#endif
|
||||
#ifndef ZSTD_NODECOMPRESS
|
||||
DISPLAY( "--test : test compressed file integrity \n");
|
||||
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
|
||||
#endif
|
||||
#ifndef ZSTD_NODICT
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Dictionary builder :\n");
|
||||
DISPLAY( "--train ## : create a dictionary from a training set of files \n");
|
||||
DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
|
||||
DISPLAY( "--maxdict ## : limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
|
||||
DISPLAY( "--train : create a dictionary from a training set of files \n");
|
||||
DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName);
|
||||
DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
|
||||
DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
|
||||
DISPLAY( "--dictID ## : force dictionary ID to specified value (default: random)\n");
|
||||
#endif
|
||||
#ifndef ZSTD_NOBENCH
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Benchmark arguments :\n");
|
||||
DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
|
||||
DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n");
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : 3)\n");
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
DISPLAY( " -r : operate recursively on directories\n");
|
||||
#endif
|
||||
DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
|
||||
#endif
|
||||
return 0;
|
||||
@@ -178,24 +168,12 @@ static void waitEnter(void)
|
||||
(void)unused;
|
||||
}
|
||||
|
||||
/*! readU32FromChar() :
|
||||
@return : unsigned integer value reach from input in `char` format
|
||||
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
|
||||
Note : this function can overflow if result > MAX_UNIT */
|
||||
static unsigned readU32FromChar(const char** stringPtr)
|
||||
{
|
||||
unsigned result = 0;
|
||||
while ((**stringPtr >='0') && (**stringPtr <='9'))
|
||||
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
|
||||
|
||||
int main(int argCount, const char** argv)
|
||||
{
|
||||
int argNb,
|
||||
int i,
|
||||
bench=0,
|
||||
decode=0,
|
||||
forceStdout=0,
|
||||
@@ -204,8 +182,7 @@ int main(int argCount, const char** argv)
|
||||
operationResult=0,
|
||||
dictBuild=0,
|
||||
nextArgumentIsOutFileName=0,
|
||||
nextArgumentIsMaxDict=0,
|
||||
nextArgumentIsDictID=0;
|
||||
nextArgumentIsMaxDict=0;
|
||||
unsigned cLevel = 1;
|
||||
unsigned cLevelLast = 1;
|
||||
unsigned recursive = 0;
|
||||
@@ -216,35 +193,25 @@ int main(int argCount, const char** argv)
|
||||
const char* dictFileName = NULL;
|
||||
char* dynNameSpace = NULL;
|
||||
unsigned maxDictSize = g_defaultMaxDictSize;
|
||||
unsigned dictID = 0;
|
||||
unsigned dictCLevel = g_defaultDictCLevel;
|
||||
unsigned dictSelect = g_defaultSelectivityLevel;
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
const char** fileNamesTable = NULL;
|
||||
char* fileNamesBuf = NULL;
|
||||
unsigned fileNamesNb;
|
||||
#endif
|
||||
|
||||
/* init */
|
||||
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
|
||||
(void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */
|
||||
(void)recursive; (void)cLevelLast; (void)dictCLevel; /* not used when ZSTD_NOBENCH / ZSTD_NODICT set */
|
||||
(void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
|
||||
if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); }
|
||||
filenameTable[0] = stdinmark;
|
||||
displayOut = stderr;
|
||||
/* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
|
||||
{ size_t pos;
|
||||
for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } }
|
||||
programName += pos;
|
||||
}
|
||||
for (i = (int)strlen(programName); i > 0; i--) { if (programName[i] == '/') { i++; break; } }
|
||||
programName += i;
|
||||
|
||||
/* preset behaviors */
|
||||
if (!strcmp(programName, ZSTD_UNZSTD)) decode=1;
|
||||
if (!strcmp(programName, ZSTD_CAT)) { decode=1; forceStdout=1; displayLevel=1; outFileName=stdoutmark; }
|
||||
|
||||
/* command switches */
|
||||
for(argNb=1; argNb<argCount; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
for(i=1; i<argCount; i++) {
|
||||
const char* argument = argv[i];
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* long commands (--long-word) */
|
||||
@@ -255,17 +222,11 @@ int main(int argCount, const char** argv)
|
||||
if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
|
||||
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
|
||||
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
|
||||
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
|
||||
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
|
||||
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
|
||||
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
|
||||
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
|
||||
if (!strcmp(argument, "--test")) { decode=1; outFileName=nulmark; FIO_overwriteMode(); continue; }
|
||||
if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
|
||||
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
|
||||
if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; }
|
||||
if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */
|
||||
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
|
||||
if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
|
||||
|
||||
/* '-' means stdin/stdout */
|
||||
if (!strcmp(argument, "-")){
|
||||
@@ -280,7 +241,12 @@ int main(int argCount, const char** argv)
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
/* compression Level */
|
||||
if ((*argument>='0') && (*argument<='9')) {
|
||||
cLevel = readU32FromChar(&argument);
|
||||
cLevel = 0;
|
||||
while ((*argument >= '0') && (*argument <= '9')) {
|
||||
cLevel *= 10;
|
||||
cLevel += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
dictCLevel = cLevel;
|
||||
if (dictCLevel > ZSTD_maxCLevel())
|
||||
CLEAN_RETURN(badusage(programName));
|
||||
@@ -305,7 +271,7 @@ int main(int argCount, const char** argv)
|
||||
case 'D': nextEntryIsDictionary = 1; argument++; break;
|
||||
|
||||
/* Overwrite */
|
||||
case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break;
|
||||
case 'f': FIO_overwriteMode(); argument++; break;
|
||||
|
||||
/* Verbose mode */
|
||||
case 'v': displayLevel=4; argument++; break;
|
||||
@@ -316,18 +282,12 @@ int main(int argCount, const char** argv)
|
||||
/* keep source file (default anyway, so useless; for gzip/xz compatibility) */
|
||||
case 'k': argument++; break;
|
||||
|
||||
/* Checksum */
|
||||
case 'C': argument++; FIO_setChecksumFlag(2); break;
|
||||
|
||||
/* test compressed file */
|
||||
case 't': decode=1; outFileName=nulmark; argument++; break;
|
||||
case 't': decode=1; outFileName=nulmark; FIO_overwriteMode(); argument++; break;
|
||||
|
||||
/* dictionary name */
|
||||
case 'o': nextArgumentIsOutFileName=1; argument++; break;
|
||||
|
||||
/* recursive */
|
||||
case 'r': recursive=1; argument++; break;
|
||||
|
||||
#ifndef ZSTD_NOBENCH
|
||||
/* Benchmark */
|
||||
case 'b': bench=1; argument++; break;
|
||||
@@ -336,22 +296,33 @@ int main(int argCount, const char** argv)
|
||||
case 'e':
|
||||
/* compression Level */
|
||||
argument++;
|
||||
cLevelLast = readU32FromChar(&argument);
|
||||
if ((*argument>='0') && (*argument<='9')) {
|
||||
cLevelLast = 0;
|
||||
while ((*argument >= '0') && (*argument <= '9'))
|
||||
cLevelLast *= 10, cLevelLast += *argument++ - '0';
|
||||
}
|
||||
break;
|
||||
|
||||
/* Modify Nb Iterations (benchmark only) */
|
||||
case 'i':
|
||||
argument++;
|
||||
{ U32 const iters = readU32FromChar(&argument);
|
||||
{ U32 iters= 0;
|
||||
argument++;
|
||||
while ((*argument >='0') && (*argument <='9'))
|
||||
iters *= 10, iters += *argument++ - '0';
|
||||
BMK_setNotificationLevel(displayLevel);
|
||||
BMK_SetNbIterations(iters);
|
||||
}
|
||||
break;
|
||||
|
||||
/* recursive */
|
||||
case 'r': recursive=1; argument++; break;
|
||||
|
||||
/* cut input into blocks (benchmark only) */
|
||||
case 'B':
|
||||
argument++;
|
||||
{ size_t bSize = readU32FromChar(&argument);
|
||||
{ size_t bSize = 0;
|
||||
argument++;
|
||||
while ((*argument >='0') && (*argument <='9'))
|
||||
bSize *= 10, bSize += *argument++ - '0';
|
||||
if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */
|
||||
if (toupper(*argument)=='M') bSize<<=20, argument++;
|
||||
if (toupper(*argument)=='B') argument++;
|
||||
@@ -361,17 +332,21 @@ int main(int argCount, const char** argv)
|
||||
break;
|
||||
#endif /* ZSTD_NOBENCH */
|
||||
|
||||
/* Dictionary Selection level */
|
||||
case 's':
|
||||
argument++;
|
||||
dictSelect = readU32FromChar(&argument);
|
||||
/* Selection level */
|
||||
case 's': argument++;
|
||||
dictSelect = 0;
|
||||
while ((*argument >= '0') && (*argument <= '9'))
|
||||
dictSelect *= 10, dictSelect += *argument++ - '0';
|
||||
break;
|
||||
|
||||
/* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
|
||||
case 'p': argument++;
|
||||
#ifndef ZSTD_NOBENCH
|
||||
if ((*argument>='0') && (*argument<='9')) {
|
||||
BMK_setAdditionalParam(readU32FromChar(&argument));
|
||||
int additionalParam = 0;
|
||||
while ((*argument >= '0') && (*argument <= '9'))
|
||||
additionalParam *= 10, additionalParam += *argument++ - '0';
|
||||
BMK_setAdditionalParam(additionalParam);
|
||||
} else
|
||||
#endif
|
||||
main_pause=1;
|
||||
@@ -398,15 +373,10 @@ int main(int argCount, const char** argv)
|
||||
|
||||
if (nextArgumentIsMaxDict) {
|
||||
nextArgumentIsMaxDict = 0;
|
||||
maxDictSize = readU32FromChar(&argument);
|
||||
maxDictSize = 0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
maxDictSize = maxDictSize * 10 + (*argument - '0'), argument++;
|
||||
if (toupper(*argument)=='K') maxDictSize <<= 10;
|
||||
if (toupper(*argument)=='M') maxDictSize <<= 20;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (nextArgumentIsDictID) {
|
||||
nextArgumentIsDictID = 0;
|
||||
dictID = readU32FromChar(&argument);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -417,24 +387,11 @@ int main(int argCount, const char** argv)
|
||||
/* Welcome message (if verbose) */
|
||||
DISPLAYLEVEL(3, WELCOME_MESSAGE);
|
||||
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
if (recursive) {
|
||||
fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
|
||||
if (fileNamesTable) {
|
||||
unsigned i;
|
||||
for (i=0; i<fileNamesNb; i++) DISPLAYLEVEL(3, "%d %s\n", i, fileNamesTable[i]);
|
||||
free((void*)filenameTable);
|
||||
filenameTable = fileNamesTable;
|
||||
filenameIdx = fileNamesNb;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Check if benchmark is selected */
|
||||
if (bench) {
|
||||
#ifndef ZSTD_NOBENCH
|
||||
BMK_setNotificationLevel(displayLevel);
|
||||
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast);
|
||||
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, recursive);
|
||||
#endif
|
||||
goto _end;
|
||||
}
|
||||
@@ -446,20 +403,17 @@ int main(int argCount, const char** argv)
|
||||
dictParams.compressionLevel = dictCLevel;
|
||||
dictParams.selectivityLevel = dictSelect;
|
||||
dictParams.notificationLevel = displayLevel;
|
||||
dictParams.dictID = dictID;
|
||||
DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams);
|
||||
#endif
|
||||
goto _end;
|
||||
}
|
||||
|
||||
/* No input filename ==> use stdin and stdout */
|
||||
filenameIdx += !filenameIdx; /*< default input is stdin */
|
||||
if (!strcmp(filenameTable[0], stdinmark) && !outFileName ) outFileName = stdoutmark; /*< when input is stdin, default output is stdout */
|
||||
if(!filenameIdx) filenameIdx=1, filenameTable[0]=stdinmark, outFileName=stdoutmark;
|
||||
|
||||
/* Check if input/output defined as console; trigger an error in this case */
|
||||
if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName));
|
||||
if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !(forceStdout && decode))
|
||||
CLEAN_RETURN(badusage(programName));
|
||||
if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) CLEAN_RETURN(badusage(programName));
|
||||
|
||||
/* user-selected output filename, only possible with a single file */
|
||||
if (outFileName && strcmp(outFileName,stdoutmark) && strcmp(outFileName,nulmark) && (filenameIdx>1)) {
|
||||
@@ -484,9 +438,9 @@ int main(int argCount, const char** argv)
|
||||
{ /* decompression */
|
||||
#ifndef ZSTD_NODECOMPRESS
|
||||
if (filenameIdx==1 && outFileName)
|
||||
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
|
||||
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
|
||||
else
|
||||
operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName);
|
||||
operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName);
|
||||
#else
|
||||
DISPLAY("Decompression not supported\n");
|
||||
#endif
|
||||
@@ -495,11 +449,6 @@ int main(int argCount, const char** argv)
|
||||
_end:
|
||||
if (main_pause) waitEnter();
|
||||
free(dynNameSpace);
|
||||
#ifdef UTIL_HAS_CREATEFILELIST
|
||||
if (fileNamesTable)
|
||||
UTIL_freeFileList(fileNamesTable, fileNamesBuf);
|
||||
else
|
||||
#endif
|
||||
free((void*)filenameTable);
|
||||
free((void*)filenameTable);
|
||||
return operationResult;
|
||||
}
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
build
|
||||
*Copy
|
||||
@@ -1,9 +0,0 @@
|
||||
projects for various integrated development environments (IDE)
|
||||
================================
|
||||
|
||||
#### Included projects
|
||||
|
||||
The following projects are included with the zstd distribution:
|
||||
- cmake - CMake project contributed by Artyom Dymchenko
|
||||
- VS2008 - Visual Studio 2008 project
|
||||
- VS2010 - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015)
|
||||
@@ -1,449 +0,0 @@
|
||||
<?xml version="1.0" encoding="Windows-1252"?>
|
||||
<VisualStudioProject
|
||||
ProjectType="Visual C++"
|
||||
Version="9.00"
|
||||
Name="fullbench"
|
||||
ProjectGUID="{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}"
|
||||
RootNamespace="fullbench"
|
||||
Keyword="Win32Proj"
|
||||
TargetFrameworkVersion="196613"
|
||||
>
|
||||
<Platforms>
|
||||
<Platform
|
||||
Name="Win32"
|
||||
/>
|
||||
<Platform
|
||||
Name="x64"
|
||||
/>
|
||||
</Platforms>
|
||||
<ToolFiles>
|
||||
</ToolFiles>
|
||||
<Configurations>
|
||||
<Configuration
|
||||
Name="Debug|Win32"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
WarnAsError="true"
|
||||
DebugInformationFormat="4"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
TargetMachine="1"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Release|Win32"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
WholeProgramOptimization="1"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="2"
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
OptimizeReferences="2"
|
||||
EnableCOMDATFolding="2"
|
||||
TargetMachine="1"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Debug|x64"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
TargetEnvironment="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
WarnAsError="true"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
TargetMachine="17"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Release|x64"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
WholeProgramOptimization="1"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
TargetEnvironment="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="2"
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
OptimizeReferences="2"
|
||||
EnableCOMDATFolding="2"
|
||||
TargetMachine="17"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
</Configurations>
|
||||
<References>
|
||||
</References>
|
||||
<Files>
|
||||
<Filter
|
||||
Name="Source Files"
|
||||
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\datagen.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\fse_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\xxhash.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\fullbench.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\huf_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
<Filter
|
||||
Name="Header Files"
|
||||
Filter="h;hpp;hxx;hm;inl;inc;xsd"
|
||||
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\bitstream.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_public.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\huf.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\huf_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\xxhash.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\mem.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_internal.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_static.h"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
</Files>
|
||||
<Globals>
|
||||
</Globals>
|
||||
</VisualStudioProject>
|
||||
@@ -1,461 +0,0 @@
|
||||
<?xml version="1.0" encoding="Windows-1252"?>
|
||||
<VisualStudioProject
|
||||
ProjectType="Visual C++"
|
||||
Version="9.00"
|
||||
Name="fuzzer"
|
||||
ProjectGUID="{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}"
|
||||
RootNamespace="fuzzer"
|
||||
Keyword="Win32Proj"
|
||||
TargetFrameworkVersion="196613"
|
||||
>
|
||||
<Platforms>
|
||||
<Platform
|
||||
Name="Win32"
|
||||
/>
|
||||
<Platform
|
||||
Name="x64"
|
||||
/>
|
||||
</Platforms>
|
||||
<ToolFiles>
|
||||
</ToolFiles>
|
||||
<Configurations>
|
||||
<Configuration
|
||||
Name="Debug|Win32"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
WarnAsError="true"
|
||||
DebugInformationFormat="4"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
TargetMachine="1"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Release|Win32"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
WholeProgramOptimization="1"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="2"
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
OptimizeReferences="2"
|
||||
EnableCOMDATFolding="2"
|
||||
TargetMachine="1"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Debug|x64"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
TargetEnvironment="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
WarnAsError="true"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
TargetMachine="17"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Release|x64"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
WholeProgramOptimization="1"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
TargetEnvironment="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="2"
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
OptimizeReferences="2"
|
||||
EnableCOMDATFolding="2"
|
||||
TargetMachine="17"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
</Configurations>
|
||||
<References>
|
||||
</References>
|
||||
<Files>
|
||||
<Filter
|
||||
Name="Source Files"
|
||||
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\datagen.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\fse_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\fuzzer.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\huf_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\xxhash.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
<Filter
|
||||
Name="Header Files"
|
||||
Filter="h;hpp;hxx;hm;inl;inc;xsd"
|
||||
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\bitstream.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_public.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\huf.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\huf_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\mem.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\xxhash.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_internal.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_static.h"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
</Files>
|
||||
<Globals>
|
||||
</Globals>
|
||||
</VisualStudioProject>
|
||||
@@ -1,56 +0,0 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 10.00
|
||||
# Visual Studio 2008
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstd", "zstd\zstd.vcproj", "{1A2AB08E-5CE7-4C5B-BE55-458157C14051}"
|
||||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzzer", "fuzzer\fuzzer.vcproj", "{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}"
|
||||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\fullbench.vcproj", "{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}"
|
||||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcproj", "{99DE2A79-7298-4004-A0ED-030D7A3796CA}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Win32 = Debug|Win32
|
||||
Debug|x64 = Debug|x64
|
||||
Release|Win32 = Release|Win32
|
||||
Release|x64 = Release|x64
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.ActiveCfg = Debug|Win32
|
||||
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|Win32.Build.0 = Debug|Win32
|
||||
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Debug|x64.Build.0 = Debug|x64
|
||||
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.ActiveCfg = Release|Win32
|
||||
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|Win32.Build.0 = Release|Win32
|
||||
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.ActiveCfg = Release|x64
|
||||
{1A2AB08E-5CE7-4C5B-BE55-458157C14051}.Release|x64.Build.0 = Release|x64
|
||||
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.ActiveCfg = Debug|Win32
|
||||
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|Win32.Build.0 = Debug|Win32
|
||||
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Debug|x64.Build.0 = Debug|x64
|
||||
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.ActiveCfg = Release|Win32
|
||||
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|Win32.Build.0 = Release|Win32
|
||||
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.ActiveCfg = Release|x64
|
||||
{A62E89D2-9DDE-42BA-8F9B-9DA74889A6B0}.Release|x64.Build.0 = Release|x64
|
||||
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.ActiveCfg = Debug|Win32
|
||||
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|Win32.Build.0 = Debug|Win32
|
||||
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Debug|x64.Build.0 = Debug|x64
|
||||
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.ActiveCfg = Release|Win32
|
||||
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|Win32.Build.0 = Release|Win32
|
||||
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.ActiveCfg = Release|x64
|
||||
{CC8F1D1B-BA2F-43E3-A71F-FA415D81AAD3}.Release|x64.Build.0 = Release|x64
|
||||
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.ActiveCfg = Debug|Win32
|
||||
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|Win32.Build.0 = Debug|Win32
|
||||
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Debug|x64.Build.0 = Debug|x64
|
||||
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.ActiveCfg = Release|Win32
|
||||
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|Win32.Build.0 = Release|Win32
|
||||
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.ActiveCfg = Release|x64
|
||||
{99DE2A79-7298-4004-A0ED-030D7A3796CA}.Release|x64.Build.0 = Release|x64
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
@@ -1,545 +0,0 @@
|
||||
<?xml version="1.0" encoding="Windows-1252"?>
|
||||
<VisualStudioProject
|
||||
ProjectType="Visual C++"
|
||||
Version="9.00"
|
||||
Name="zstd"
|
||||
ProjectGUID="{1A2AB08E-5CE7-4C5B-BE55-458157C14051}"
|
||||
RootNamespace="zstd"
|
||||
Keyword="Win32Proj"
|
||||
TargetFrameworkVersion="196613"
|
||||
>
|
||||
<Platforms>
|
||||
<Platform
|
||||
Name="Win32"
|
||||
/>
|
||||
<Platform
|
||||
Name="x64"
|
||||
/>
|
||||
</Platforms>
|
||||
<ToolFiles>
|
||||
</ToolFiles>
|
||||
<Configurations>
|
||||
<Configuration
|
||||
Name="Debug|Win32"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
WarnAsError="true"
|
||||
DebugInformationFormat="4"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies="setargv.obj"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
TargetMachine="1"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Release|Win32"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
WholeProgramOptimization="1"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="2"
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies="setargv.obj"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
OptimizeReferences="2"
|
||||
EnableCOMDATFolding="2"
|
||||
TargetMachine="1"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Debug|x64"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
TargetEnvironment="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
WarnAsError="true"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies="setargv.obj"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
TargetMachine="17"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Release|x64"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="1"
|
||||
CharacterSet="2"
|
||||
WholeProgramOptimization="1"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
TargetEnvironment="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="2"
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies="setargv.obj"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
OptimizeReferences="2"
|
||||
EnableCOMDATFolding="2"
|
||||
TargetMachine="17"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
</Configurations>
|
||||
<References>
|
||||
</References>
|
||||
<Files>
|
||||
<Filter
|
||||
Name="Source Files"
|
||||
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\bench.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\datagen.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\dibio.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\fileio.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\legacy\fileio_legacy.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\fse_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\huf_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\xxhash.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v01.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v02.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v03.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v04.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v05.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v06.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\programs\zstdcli.c"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
<Filter
|
||||
Name="Header Files"
|
||||
Filter="h;hpp;hxx;hm;inl;inc;xsd"
|
||||
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\bitstream.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_public.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\huf.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\huf_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\mem.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\xxhash.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zbuff.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_internal.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_legacy.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v01.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v02.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v03.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v04.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v05.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v06.h"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
</Files>
|
||||
<Globals>
|
||||
</Globals>
|
||||
</VisualStudioProject>
|
||||
@@ -1,495 +0,0 @@
|
||||
<?xml version="1.0" encoding="Windows-1252"?>
|
||||
<VisualStudioProject
|
||||
ProjectType="Visual C++"
|
||||
Version="9.00"
|
||||
Name="zstdlib"
|
||||
ProjectGUID="{99DE2A79-7298-4004-A0ED-030D7A3796CA}"
|
||||
RootNamespace="zstdlib"
|
||||
Keyword="Win32Proj"
|
||||
TargetFrameworkVersion="196613"
|
||||
>
|
||||
<Platforms>
|
||||
<Platform
|
||||
Name="Win32"
|
||||
/>
|
||||
<Platform
|
||||
Name="x64"
|
||||
/>
|
||||
</Platforms>
|
||||
<ToolFiles>
|
||||
</ToolFiles>
|
||||
<Configurations>
|
||||
<Configuration
|
||||
Name="Debug|Win32"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="2"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
WarnAsError="true"
|
||||
DebugInformationFormat="4"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
TargetMachine="1"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Release|Win32"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="2"
|
||||
CharacterSet="2"
|
||||
WholeProgramOptimization="1"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="2"
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
OptimizeReferences="2"
|
||||
EnableCOMDATFolding="2"
|
||||
TargetMachine="1"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Debug|x64"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="2"
|
||||
CharacterSet="2"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
TargetEnvironment="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
WarnAsError="true"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
TargetMachine="17"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
<Configuration
|
||||
Name="Release|x64"
|
||||
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
|
||||
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
|
||||
ConfigurationType="2"
|
||||
CharacterSet="2"
|
||||
WholeProgramOptimization="1"
|
||||
>
|
||||
<Tool
|
||||
Name="VCPreBuildEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCustomBuildTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXMLDataGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCWebServiceProxyGeneratorTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCMIDLTool"
|
||||
TargetEnvironment="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="2"
|
||||
EnableIntrinsicFunctions="true"
|
||||
OmitFramePointers="true"
|
||||
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
|
||||
PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
|
||||
RuntimeLibrary="0"
|
||||
EnableFunctionLevelLinking="true"
|
||||
UsePrecompiledHeader="0"
|
||||
WarningLevel="4"
|
||||
DebugInformationFormat="3"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManagedResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCResourceCompilerTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPreLinkEventTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
OptimizeReferences="2"
|
||||
EnableCOMDATFolding="2"
|
||||
TargetMachine="17"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCALinkTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCManifestTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCXDCMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCBscMakeTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCFxCopTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCAppVerifierTool"
|
||||
/>
|
||||
<Tool
|
||||
Name="VCPostBuildEventTool"
|
||||
/>
|
||||
</Configuration>
|
||||
</Configurations>
|
||||
<References>
|
||||
</References>
|
||||
<Files>
|
||||
<Filter
|
||||
Name="Source Files"
|
||||
Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
|
||||
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\entropy_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\fse_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\huf_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\xxhash.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_common.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
<Filter
|
||||
Name="Header Files"
|
||||
Filter="h;hpp;hxx;hm;inl;inc;xsd"
|
||||
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
|
||||
>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\bitstream.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_private.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\error_public.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\fse_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\huf.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\huf_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\mem.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\xxhash.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zbuff.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_internal.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_legacy.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zstd_opt.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v01.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v02.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v03.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v04.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\legacy\zstd_v05.h"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
<Filter
|
||||
Name="Resource Files"
|
||||
Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav"
|
||||
UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
|
||||
>
|
||||
</Filter>
|
||||
</Files>
|
||||
<Globals>
|
||||
</Globals>
|
||||
</VisualStudioProject>
|
||||
@@ -1,168 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|Win32">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|Win32">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|x64">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
</ItemGroup>
|
||||
<PropertyGroup Label="Globals">
|
||||
<ProjectGuid>{037E781E-81A6-494B-B1B3-438AB1200523}</ProjectGuid>
|
||||
<Keyword>Win32Proj</Keyword>
|
||||
<RootNamespace>datagen</RootNamespace>
|
||||
<OutDir>$(SolutionDir)bin\$(Platform)\$(Configuration)\</OutDir>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<ClCompile>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagencli.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
</ImportGroup>
|
||||
</Project>
|
||||
@@ -1,26 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup>
|
||||
<Filter Include="Header Files">
|
||||
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
|
||||
<Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
|
||||
</Filter>
|
||||
<Filter Include="Source Files">
|
||||
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
|
||||
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
|
||||
</Filter>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\programs\datagen.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\datagencli.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\programs\datagen.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,51 +0,0 @@
|
||||
// Microsoft Visual C++ generated resource script.
|
||||
//
|
||||
|
||||
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||
#define APSTUDIO_READONLY_SYMBOLS
|
||||
#include "verrsrc.h"
|
||||
#undef APSTUDIO_READONLY_SYMBOLS
|
||||
|
||||
|
||||
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
|
||||
LANGUAGE 9, 1
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Version
|
||||
//
|
||||
|
||||
VS_VERSION_INFO VERSIONINFO
|
||||
FILEVERSION ZSTD_LIB_VERSION
|
||||
PRODUCTVERSION ZSTD_LIB_VERSION
|
||||
FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
|
||||
#ifdef _DEBUG
|
||||
FILEFLAGS VS_FF_DEBUG
|
||||
#else
|
||||
FILEFLAGS 0x0L
|
||||
#endif
|
||||
FILEOS VOS_NT_WINDOWS32
|
||||
FILETYPE VFT_DLL
|
||||
FILESUBTYPE VFT2_UNKNOWN
|
||||
BEGIN
|
||||
BLOCK "StringFileInfo"
|
||||
BEGIN
|
||||
BLOCK "040904B0"
|
||||
BEGIN
|
||||
VALUE "CompanyName", "Yann Collet"
|
||||
VALUE "FileDescription", "Fast and efficient compression algorithm"
|
||||
VALUE "FileVersion", ZSTD_VERSION_STRING
|
||||
VALUE "InternalName", "zstdlib.dll"
|
||||
VALUE "LegalCopyright", "Copyright (C) 2013-2015, Yann Collet"
|
||||
VALUE "OriginalFilename", "zstdlib.dll"
|
||||
VALUE "ProductName", "Zstandard"
|
||||
VALUE "ProductVersion", ZSTD_VERSION_STRING
|
||||
END
|
||||
END
|
||||
BLOCK "VarFileInfo"
|
||||
BEGIN
|
||||
VALUE "Translation", 0x0409, 1200
|
||||
END
|
||||
END
|
||||
|
||||
#endif
|
||||
@@ -1,51 +0,0 @@
|
||||
Here are a few command lines for reference :
|
||||
|
||||
### Build with Visual Studio 2013 for msvcr120.dll
|
||||
|
||||
Running the following command will build both the `Release Win32` and `Release x64` versions:
|
||||
```batch
|
||||
build\build.VS2013.cmd
|
||||
```
|
||||
The result of each build will be in the corresponding `build\bin\Release\{ARCH}\` folder.
|
||||
|
||||
If you want to only need one architecture:
|
||||
- Win32: `build\build.generic.cmd VS2013 Win32 Release v120`
|
||||
- x64: `build\build.generic.cmd VS2013 x64 Release v120`
|
||||
|
||||
If you want a Debug build:
|
||||
- Win32: `build\build.generic.cmd VS2013 Win32 Debug v120`
|
||||
- x64: `build\build.generic.cmd VS2013 x64 Debug v120`
|
||||
|
||||
### Build with Visual Studio 2015 for msvcr140.dll
|
||||
|
||||
Running the following command will build both the `Release Win32` and `Release x64` versions:
|
||||
```batch
|
||||
build\build.VS2015.cmd
|
||||
```
|
||||
The result of each build will be in the corresponding `build\bin\Release\{ARCH}\` folder.
|
||||
|
||||
If you want to only need one architecture:
|
||||
- Win32: `build\build.generic.cmd VS2015 Win32 Release v140`
|
||||
- x64: `build\build.generic.cmd VS2015 x64 Release v140`
|
||||
|
||||
If you want a Debug build:
|
||||
- Win32: `build\build.generic.cmd VS2015 Win32 Debug v140`
|
||||
- x64: `build\build.generic.cmd VS2015 x64 Debug v140`
|
||||
|
||||
### Build with Visual Studio 2015 for msvcr120.dll
|
||||
|
||||
You need to invoke `build\build.generic.cmd` with the proper arguments:
|
||||
|
||||
**For Win32**
|
||||
```batch
|
||||
build\build.generic.cmd VS2015 Win32 Release v120
|
||||
```
|
||||
The result of the build will be in the `build\bin\Release\Win32\` folder.
|
||||
|
||||
**For x64**
|
||||
```batch
|
||||
build\build.generic.cmd VS2015 x64 Release v120
|
||||
```
|
||||
The result of the build will be in the `build\bin\Release\x64\` folder.
|
||||
|
||||
If you want Debug builds, replace `Release` with `Debug`.
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2010 Win32 Release v100
|
||||
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2010 x64 Release v100
|
||||
@@ -1,6 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2012 Win32 Release v110
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2012 x64 Release v110
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2013 Win32 Release v120
|
||||
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2013 x64 Release v120
|
||||
@@ -1,7 +0,0 @@
|
||||
@echo off
|
||||
|
||||
rem build 32-bit
|
||||
call "%~p0%build.generic.cmd" VS2015 Win32 Release v140
|
||||
|
||||
rem build 64-bit
|
||||
call "%~p0%build.generic.cmd" VS2015 x64 Release v140
|
||||
@@ -1,52 +0,0 @@
|
||||
@echo off
|
||||
|
||||
IF "%1%" == "" GOTO display_help
|
||||
|
||||
SETLOCAL
|
||||
|
||||
SET msbuild_version=%1
|
||||
|
||||
SET msbuild_platform=%2
|
||||
IF "%msbuild_platform%" == "" SET msbuild_platform=x64
|
||||
|
||||
SET msbuild_configuration=%3
|
||||
IF "%msbuild_configuration%" == "" SET msbuild_configuration=Release
|
||||
|
||||
SET msbuild_toolset=%4
|
||||
|
||||
GOTO build
|
||||
|
||||
:display_help
|
||||
|
||||
echo Syntax: build.generic.cmd msbuild_version msbuild_platform msbuild_configuration msbuild_toolset
|
||||
echo msbuild_version: VS installed version (VS2012, VS2013, VS2015, ...)
|
||||
echo msbuild_platform: Platform (x64 or Win32)
|
||||
echo msbuild_configuration: VS configuration (Release or Debug)
|
||||
echo msbuild_toolset: Platform Toolset (v100, v110, v120, v140)
|
||||
|
||||
EXIT /B 1
|
||||
|
||||
:build
|
||||
|
||||
SET msbuild="%windir%\Microsoft.NET\Framework\v4.0.30319\MSBuild.exe"
|
||||
IF %msbuild_version% == VS2013 SET msbuild="%programfiles(x86)%\MSBuild\12.0\Bin\MSBuild.exe"
|
||||
IF %msbuild_version% == VS2015 SET msbuild="%programfiles(x86)%\MSBuild\14.0\Bin\MSBuild.exe"
|
||||
rem TODO: Visual Studio "15" (vNext) will use MSBuild 15.0 ?
|
||||
|
||||
SET project="%~p0\..\VS2010\zstd.sln"
|
||||
|
||||
SET msbuild_params=/verbosity:minimal /nologo /t:Clean,Build /p:Platform=%msbuild_platform% /p:Configuration=%msbuild_configuration%
|
||||
IF NOT "%msbuild_toolset%" == "" SET msbuild_params=%msbuild_params% /p:PlatformToolset=%msbuild_toolset%
|
||||
|
||||
SET output=%~p0%bin
|
||||
SET output="%output%/%msbuild_configuration%/%msbuild_platform%/"
|
||||
SET msbuild_params=%msbuild_params% /p:OutDir=%output%
|
||||
|
||||
echo ### Building %msbuild_version% project for %msbuild_configuration% %msbuild_platform% (%msbuild_toolset%)...
|
||||
echo ### Build Params: %msbuild_params%
|
||||
|
||||
%msbuild% %project% %msbuild_params%
|
||||
IF ERRORLEVEL 1 EXIT /B 1
|
||||
echo # Success
|
||||
echo # OutDir: %output%
|
||||
echo #
|
||||
@@ -1,6 +0,0 @@
|
||||
# cmake producted
|
||||
CMakeCache.txt
|
||||
CMakeFiles
|
||||
Makefile
|
||||
cmake_install.cmake
|
||||
cmake_uninstall.cmake
|
||||
@@ -1,8 +0,0 @@
|
||||
# produced by make
|
||||
datagen
|
||||
fullbench
|
||||
fuzzer
|
||||
paramgrill
|
||||
zbufftest
|
||||
zstd
|
||||
zstd-frugal
|
||||
@@ -1,4 +0,0 @@
|
||||
# Tmp test directory
|
||||
zstdtest
|
||||
speedTest
|
||||
versionsTest
|
||||
@@ -1,41 +0,0 @@
|
||||
# ##########################################################################
|
||||
# Zstd tests - Makefile
|
||||
# Based on LZ4 version test, by Takayuki Matsuoka - 2015-2016
|
||||
#
|
||||
# GPL v2 License
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
#
|
||||
# You can contact the author at :
|
||||
# - ZSTD homepage : http://www.zstd.net/
|
||||
# ##########################################################################
|
||||
# versionstest : Compatibility test between zstd versions stored on Github (v0.1+)
|
||||
# ##########################################################################
|
||||
|
||||
PYTHON?= python3
|
||||
TESTDIR := zstdtest
|
||||
|
||||
.PHONY: default all clean versionsTest
|
||||
|
||||
default: all
|
||||
|
||||
all: versionsTest
|
||||
|
||||
versionsTest:
|
||||
$(PYTHON) test-zstd-versions.py
|
||||
|
||||
clean:
|
||||
@rm -fR $(TESTDIR)
|
||||
@echo Cleaning completed
|
||||
@@ -1,55 +0,0 @@
|
||||
scripts for automated testing of zstd
|
||||
================================
|
||||
|
||||
#### test-zstd-versions.py - script for testing zstd interoperability between versions
|
||||
|
||||
This script creates `versionsTest` directory to which zstd repository is cloned.
|
||||
Then all taged (released) versions of zstd are compiled.
|
||||
In the following step interoperability between zstd versions is checked.
|
||||
|
||||
|
||||
#### test-zstd-speed.py - script for testing zstd speed difference between commits
|
||||
|
||||
This script creates `speedTest` directory to which zstd repository is cloned.
|
||||
Then it compiles all branches of zstd and performs a speed benchmark for a given list of files (the `testFileNames` parameter).
|
||||
After `sleepTime` (an optional parameter, default 300 seconds) seconds the script checks repository for new commits.
|
||||
If a new commit is found it is compiled and a speed benchmark for this commit is performed.
|
||||
The results of the speed benchmark are compared to the previous results.
|
||||
If compression or decompression speed for one of zstd levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted.
|
||||
If second results are also lower than `lowerLimit` the warning e-mail is send to recipients from the list (the `emails` parameter).
|
||||
|
||||
Additional remarks:
|
||||
- To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel
|
||||
- Using the script with virtual machines can lead to large variations of speed results
|
||||
- The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75)
|
||||
- The script sends e-mails using `mutt`; if `mutt` is not available it sends e-mails without attachments using `mail`; if both are not available it only prints a warning
|
||||
|
||||
|
||||
The example usage with two test files, one e-mail address, and with an additional message:
|
||||
```
|
||||
./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60
|
||||
```
|
||||
|
||||
To run the script in background please use:
|
||||
```
|
||||
nohup ./test-zstd-speed.py testFileNames emails &
|
||||
```
|
||||
|
||||
The full list of parameters:
|
||||
```
|
||||
positional arguments:
|
||||
testFileNames file names list for speed benchmark
|
||||
emails list of e-mail addresses to send warnings
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--message MESSAGE attach an additional message to e-mail
|
||||
--lowerLimit LOWERLIMIT
|
||||
send email if speed is lower than given limit
|
||||
--maxLoadAvg MAXLOADAVG
|
||||
maximum load average to start testing
|
||||
--lastCLevel LASTCLEVEL
|
||||
last compression level for testing
|
||||
--sleepTime SLEEPTIME
|
||||
frequency of repository checking in seconds
|
||||
```
|
||||
@@ -1,249 +0,0 @@
|
||||
#! /usr/bin/env python
|
||||
# execute(), fetch(), notify() are based on https://github.com/getlantern/build-automation/blob/master/build.py
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import string
|
||||
import time
|
||||
import traceback
|
||||
from subprocess import Popen, PIPE
|
||||
|
||||
repo_url = 'https://github.com/Cyan4973/zstd.git'
|
||||
test_dir_name = 'speedTest'
|
||||
|
||||
|
||||
def log(text):
|
||||
print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text
|
||||
|
||||
def execute(command, print_output=False, print_error=True):
|
||||
log("> " + command)
|
||||
popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True, cwd=execute.cwd)
|
||||
itout = iter(popen.stdout.readline, b"")
|
||||
iterr = iter(popen.stderr.readline, b"")
|
||||
stdout_lines = list(itout)
|
||||
if print_output:
|
||||
print ''.join(stdout_lines)
|
||||
stderr_lines = list(iterr)
|
||||
if print_output:
|
||||
print ''.join(stderr_lines)
|
||||
popen.communicate()
|
||||
if popen.returncode is not None and popen.returncode != 0:
|
||||
if not print_output and print_error:
|
||||
print ''.join(stderr_lines)
|
||||
raise RuntimeError(''.join(stderr_lines))
|
||||
return stdout_lines + stderr_lines
|
||||
execute.cwd = None
|
||||
|
||||
|
||||
def does_command_exist(command):
|
||||
try:
|
||||
execute(command, False, False);
|
||||
except Exception as e:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def fetch():
|
||||
execute('git fetch -p')
|
||||
output = execute('git branch -rl')
|
||||
for line in output:
|
||||
if "HEAD" in line:
|
||||
output.remove(line) # remove "origin/HEAD -> origin/dev"
|
||||
branches = map(lambda l: l.strip(), output)
|
||||
return map(lambda b: (b, execute('git show -s --format=%h ' + b)[0].strip()), branches)
|
||||
|
||||
|
||||
def notify(branch, commit, last_commit):
|
||||
text_tmpl = string.Template('Changes since $last_commit:\r\n$commits')
|
||||
branch = branch.split('/')[1]
|
||||
fmt = '--format="%h: (%an) %s, %ar"'
|
||||
if last_commit is None:
|
||||
commits = execute('git log -n 10 %s %s' % (fmt, commit))
|
||||
else:
|
||||
commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit))
|
||||
|
||||
text = text_tmpl.substitute({'last_commit': last_commit, 'commits': ''.join(commits)})
|
||||
print str("commits for %s: %s" % (commit, text))
|
||||
|
||||
|
||||
def compile(branch, commit, dry_run):
|
||||
local_branch = string.split(branch, '/')[1]
|
||||
version = local_branch.rpartition('-')[2]
|
||||
version = version + '_' + commit
|
||||
execute('git checkout -- . && git checkout ' + branch)
|
||||
if not dry_run:
|
||||
execute('VERSION=' + version + '; make clean zstdprogram')
|
||||
|
||||
|
||||
def get_last_commit(resultsFileName):
|
||||
if not os.path.isfile(resultsFileName):
|
||||
return None, None, None
|
||||
commit = None
|
||||
cspeed = []
|
||||
dspeed = []
|
||||
with open(resultsFileName,'r') as f:
|
||||
for line in f:
|
||||
words = line.split()
|
||||
if len(words) == 2: # branch + commit
|
||||
commit = words[1];
|
||||
cspeed = []
|
||||
dspeed = []
|
||||
if (len(words) == 8):
|
||||
cspeed.append(float(words[3]))
|
||||
dspeed.append(float(words[5]))
|
||||
#if commit != None:
|
||||
# print "commit=%s cspeed=%s dspeed=%s" % (commit, cspeed, dspeed)
|
||||
return commit, cspeed, dspeed
|
||||
|
||||
|
||||
def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFilePath, fileName, last_cspeed, last_dspeed, lower_limit, maxLoadAvg, message):
|
||||
sleepTime = 30
|
||||
while os.getloadavg()[0] > maxLoadAvg:
|
||||
log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds" % (os.getloadavg()[0], maxLoadAvg, sleepTime))
|
||||
time.sleep(sleepTime)
|
||||
start_load = str(os.getloadavg())
|
||||
result = execute('programs/zstd -qi5b1e' + str(lastCLevel) + ' ' + testFilePath)
|
||||
end_load = str(os.getloadavg())
|
||||
linesExpected = lastCLevel + 2;
|
||||
if len(result) != linesExpected:
|
||||
log("ERROR: number of result lines=%d is different that expected %d" % (len(result), linesExpected))
|
||||
return ""
|
||||
with open(resultsFileName, "a") as myfile:
|
||||
myfile.write(branch + " " + commit + "\n")
|
||||
myfile.writelines(result)
|
||||
myfile.close()
|
||||
if (last_cspeed == None):
|
||||
return ""
|
||||
commit, cspeed, dspeed = get_last_commit(resultsFileName)
|
||||
text = ""
|
||||
for i in range(0, min(len(cspeed), len(last_cspeed))):
|
||||
if (cspeed[i]/last_cspeed[i] < lower_limit):
|
||||
text += "WARNING: File=%s level=%d cspeed=%s last=%s diff=%s\n" % (fileName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i])
|
||||
if (dspeed[i]/last_dspeed[i] < lower_limit):
|
||||
text += "WARNING: File=%s level=%d dspeed=%s last=%s diff=%s\n" % (fileName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i])
|
||||
if text:
|
||||
text = message + ("\nmaxLoadAvg=%s load average at start=%s end=%s\n" % (maxLoadAvg, start_load, end_load)) + text
|
||||
return text
|
||||
|
||||
|
||||
def send_email(branch, commit, last_commit, emails, text, results_files, logFileName, lower_limit, have_mutt, have_mail):
|
||||
with open(logFileName, "w") as myfile:
|
||||
myfile.writelines(text)
|
||||
myfile.close()
|
||||
if have_mutt:
|
||||
execute("mutt -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " -a " + results_files + " < " + logFileName)
|
||||
elif have_mail:
|
||||
execute("mail -s \"[ZSTD_speedTest] Warning for branch=" + branch + " commit=" + commit + " last_commit=" + last_commit + " speed<" + str(lower_limit) + "\" " + emails + " < " + logFileName)
|
||||
else:
|
||||
log("e-mail cannot be sent (mail and mutt not found)")
|
||||
|
||||
|
||||
def check_branches(args, test_path, testFilePaths, have_mutt, have_mail):
|
||||
for branch, commit in fetch():
|
||||
try:
|
||||
commitFileName = test_path + "/commit_" + branch.replace("/", "_")
|
||||
if os.path.isfile(commitFileName):
|
||||
last_commit = file(commitFileName, 'r').read()
|
||||
else:
|
||||
last_commit = None
|
||||
file(commitFileName, 'w').write(commit)
|
||||
|
||||
if commit == last_commit:
|
||||
log("skipping branch %s: head %s already processed" % (branch, commit))
|
||||
else:
|
||||
log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
|
||||
compile(branch, commit, args.dry_run)
|
||||
|
||||
logFileName = test_path + "/log_" + branch.replace("/", "_")
|
||||
text_to_send = []
|
||||
results_files = ""
|
||||
for filePath in testFilePaths:
|
||||
fileName = filePath.rpartition('/')[2]
|
||||
resultsFileName = test_path + "/results_" + branch.replace("/", "_") + "_" + fileName
|
||||
last_commit, cspeed, dspeed = get_last_commit(resultsFileName)
|
||||
|
||||
if not args.dry_run:
|
||||
text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
|
||||
if text:
|
||||
text = benchmark_and_compare(branch, commit, resultsFileName, args.lastCLevel, filePath, fileName, cspeed, dspeed, args.lowerLimit, args.maxLoadAvg, args.message)
|
||||
if text:
|
||||
text_to_send.append(text)
|
||||
results_files += resultsFileName + " "
|
||||
if text_to_send:
|
||||
send_email(branch, commit, last_commit, args.emails, text_to_send, results_files, logFileName, args.lowerLimit, have_mutt, have_mail)
|
||||
notify(branch, commit, last_commit)
|
||||
except Exception as e:
|
||||
stack = traceback.format_exc()
|
||||
log("ERROR: build %s, error %s" % (branch, str(e)) )
|
||||
print stack
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('testFileNames', help='file names list for speed benchmark')
|
||||
parser.add_argument('emails', help='list of e-mail addresses to send warnings')
|
||||
parser.add_argument('--message', help='attach an additional message to e-mail', default="")
|
||||
parser.add_argument('--lowerLimit', type=float, help='send email if speed is lower than given limit', default=0.98)
|
||||
parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
|
||||
parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
|
||||
parser.add_argument('--sleepTime', type=int, help='frequency of repository checking in seconds', default=300)
|
||||
parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
# check if test files are accessible
|
||||
testFileNames = args.testFileNames.split()
|
||||
testFilePaths = []
|
||||
for fileName in testFileNames:
|
||||
if os.path.isfile(fileName):
|
||||
testFilePaths.append(os.path.abspath(fileName))
|
||||
else:
|
||||
raise RuntimeError("File not found: " + fileName)
|
||||
|
||||
test_path = os.getcwd() + '/' + test_dir_name # /path/to/zstd/tests/speedTest
|
||||
clone_path = test_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd
|
||||
|
||||
# check availability of e-mail senders
|
||||
have_mutt = does_command_exist("mutt --help");
|
||||
have_mail = does_command_exist("mail -V");
|
||||
if not have_mutt and not have_mail:
|
||||
log("WARNING: e-mail senders mail and mutt not found")
|
||||
|
||||
# clone ZSTD repo if needed
|
||||
if not os.path.isdir(test_path):
|
||||
os.mkdir(test_path)
|
||||
if not os.path.isdir(clone_path):
|
||||
execute.cwd = test_path
|
||||
execute('git clone ' + repo_url)
|
||||
if not os.path.isdir(clone_path):
|
||||
raise RuntimeError("ZSTD clone not found: " + clone_path)
|
||||
execute.cwd = clone_path
|
||||
|
||||
print "PARAMETERS:\ntest_path=%s" % test_path
|
||||
print "clone_path=%s" % clone_path
|
||||
print "testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths)
|
||||
print "message=%s" % args.message
|
||||
print "emails=%s" % args.emails
|
||||
print "maxLoadAvg=%s" % args.maxLoadAvg
|
||||
print "lowerLimit=%s" % args.lowerLimit
|
||||
print "lastCLevel=%s" % args.lastCLevel
|
||||
print "sleepTime=%s" % args.sleepTime
|
||||
print "dry_run=%s" % args.dry_run
|
||||
print "have_mutt=%s have_mail=%s" % (have_mutt, have_mail)
|
||||
|
||||
while True:
|
||||
pid = str(os.getpid())
|
||||
pidfile = "./speedTest.pid"
|
||||
if os.path.isfile(pidfile):
|
||||
log("%s already exists, exiting" % pidfile)
|
||||
else:
|
||||
file(pidfile, 'w').write(pid)
|
||||
try:
|
||||
loadavg = os.getloadavg()[0]
|
||||
if (loadavg <= args.maxLoadAvg):
|
||||
check_branches(args, test_path, testFilePaths, have_mutt, have_mail)
|
||||
else:
|
||||
log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
|
||||
finally:
|
||||
os.unlink(pidfile)
|
||||
log("sleep for %s seconds" % args.sleepTime)
|
||||
time.sleep(args.sleepTime)
|
||||
@@ -1,173 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test zstd interoperability between versions"""
|
||||
# Based on LZ4 version test script, by Takayuki Matsuoka
|
||||
|
||||
import filecmp
|
||||
import glob
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
repo_url = 'https://github.com/Cyan4973/zstd.git'
|
||||
tmp_dir_name = 'tests/versionsTest'
|
||||
make_cmd = 'make'
|
||||
git_cmd = 'git'
|
||||
test_dat_src = 'README.md'
|
||||
test_dat = 'test_dat'
|
||||
head = 'vdevel'
|
||||
|
||||
|
||||
def proc(cmd_args, pipe=True, dummy=False):
|
||||
if dummy:
|
||||
return
|
||||
if pipe:
|
||||
subproc = subprocess.Popen(cmd_args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
else:
|
||||
subproc = subprocess.Popen(cmd_args)
|
||||
return subproc.communicate()
|
||||
|
||||
|
||||
def make(args, pipe=True):
|
||||
return proc([make_cmd] + args, pipe)
|
||||
|
||||
|
||||
def git(args, pipe=True):
|
||||
return proc([git_cmd] + args, pipe)
|
||||
|
||||
|
||||
def get_git_tags():
|
||||
stdout, stderr = git(['tag', '-l', 'v[0-9].[0-9].[0-9]'])
|
||||
tags = stdout.decode('utf-8').split()
|
||||
return tags
|
||||
|
||||
|
||||
def compress_sample(tag, sample):
|
||||
try:
|
||||
from subprocess import DEVNULL # py3k
|
||||
except ImportError:
|
||||
DEVNULL = open(os.devnull, 'wb')
|
||||
if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0:
|
||||
os.rename(sample + '.zst', sample + '_01_64_' + tag + '.zst')
|
||||
if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0:
|
||||
os.rename(sample + '.zst', sample + '_05_64_' + tag + '.zst')
|
||||
if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0:
|
||||
os.rename(sample + '.zst', sample + '_09_64_' + tag + '.zst')
|
||||
if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
|
||||
os.rename(sample + '.zst', sample + '_15_64_' + tag + '.zst')
|
||||
if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
|
||||
os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst')
|
||||
# zstdFiles = glob.glob("*.zst*")
|
||||
# print(zstdFiles)
|
||||
print(tag + " : compression completed")
|
||||
|
||||
|
||||
# http://stackoverflow.com/a/19711609/2132223
|
||||
def sha1_of_file(filepath):
|
||||
with open(filepath, 'rb') as f:
|
||||
return hashlib.sha1(f.read()).hexdigest()
|
||||
|
||||
|
||||
def remove_duplicates():
|
||||
list_of_zst = sorted(glob.glob('*.zst'))
|
||||
for i, ref_zst in enumerate(list_of_zst):
|
||||
if not os.path.isfile(ref_zst):
|
||||
continue
|
||||
for j in range(i + 1, len(list_of_zst)):
|
||||
compared_zst = list_of_zst[j]
|
||||
if not os.path.isfile(compared_zst):
|
||||
continue
|
||||
if filecmp.cmp(ref_zst, compared_zst):
|
||||
os.remove(compared_zst)
|
||||
print('duplicated : {} == {}'.format(ref_zst, compared_zst))
|
||||
|
||||
|
||||
def decompress_zst(tag):
|
||||
dec_error = 0
|
||||
list_zst = sorted(glob.glob('*.zst'))
|
||||
try:
|
||||
from subprocess import DEVNULL # py3k
|
||||
except ImportError:
|
||||
DEVNULL = open(os.devnull, 'wb')
|
||||
for file_zst in list_zst:
|
||||
print(file_zst, end=' ')
|
||||
print(tag, end=' ')
|
||||
file_dec = file_zst + '_d64_' + tag + '.dec'
|
||||
if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL) == 0:
|
||||
if not filecmp.cmp(file_dec, test_dat):
|
||||
print('ERR !! ')
|
||||
dec_error = 1
|
||||
else:
|
||||
print('OK ')
|
||||
else:
|
||||
print('command does not work')
|
||||
return dec_error
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
error_code = 0
|
||||
base_dir = os.getcwd() + '/..' # /path/to/zstd
|
||||
tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest
|
||||
clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd
|
||||
programs_dir = base_dir + '/programs' # /path/to/zstd/programs
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
|
||||
# since Travis clones limited depth, we should clone full repository
|
||||
if not os.path.isdir(clone_dir):
|
||||
git(['clone', repo_url, clone_dir])
|
||||
|
||||
shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat)
|
||||
|
||||
# Retrieve all release tags
|
||||
print('Retrieve all release tags :')
|
||||
os.chdir(clone_dir)
|
||||
tags = get_git_tags() + [head]
|
||||
print(tags)
|
||||
|
||||
# Build all release zstd
|
||||
for tag in tags:
|
||||
os.chdir(base_dir)
|
||||
dst_zstd = '{}/zstd.{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/zstd.<TAG>
|
||||
if not os.path.isfile(dst_zstd) or tag == head:
|
||||
if tag != head:
|
||||
r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/<TAG>
|
||||
os.makedirs(r_dir, exist_ok=True)
|
||||
os.chdir(clone_dir)
|
||||
git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
|
||||
os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest/<TAG>/programs
|
||||
make(['clean', 'zstd'], False)
|
||||
else:
|
||||
os.chdir(programs_dir)
|
||||
make(['zstd'], False)
|
||||
shutil.copy2('zstd', dst_zstd)
|
||||
|
||||
# remove any remaining *.zst and *.dec from previous test
|
||||
os.chdir(tmp_dir)
|
||||
for compressed in glob.glob("*.zst"):
|
||||
os.remove(compressed)
|
||||
for dec in glob.glob("*.dec"):
|
||||
os.remove(dec)
|
||||
|
||||
print('Compress test.dat by all released zstd')
|
||||
|
||||
error_code = 0
|
||||
for tag in tags:
|
||||
print(tag)
|
||||
compress_sample(tag, test_dat)
|
||||
remove_duplicates()
|
||||
if tag >= 'v0.5.1':
|
||||
error_code += decompress_zst(tag)
|
||||
|
||||
print('')
|
||||
print('Enumerate different compressed files')
|
||||
zstds = sorted(glob.glob('*.zst'))
|
||||
for zstd in zstds:
|
||||
print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd))
|
||||
|
||||
if error_code != 0:
|
||||
print('==== ERROR !!! =====')
|
||||
|
||||
sys.exit(error_code)
|
||||
+14
-12
@@ -29,21 +29,25 @@
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
@@ -65,24 +69,22 @@
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath)</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath)</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath)</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath)</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
@@ -125,8 +127,7 @@
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@@ -144,9 +145,8 @@
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@@ -158,7 +158,6 @@
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
@@ -172,11 +171,14 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\common\fse.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
+12
-6
@@ -47,29 +47,38 @@
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\common\fse.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\fse_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zstd.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\programs\datagen.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\huf.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\huf_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
@@ -79,8 +88,5 @@
|
||||
<ClInclude Include="..\..\..\programs\util.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,4 +1,4 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|Win32">
|
||||
@@ -28,22 +28,26 @@
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
@@ -65,25 +69,23 @@
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath)</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath)</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath)</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath)</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<ClCompile>
|
||||
@@ -125,8 +127,7 @@
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@@ -144,9 +145,8 @@
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@@ -158,30 +158,30 @@
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\fuzzer.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\common\fse.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
+18
-18
@@ -14,6 +14,9 @@
|
||||
<ClCompile Include="..\..\..\programs\fuzzer.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\xxhash.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\datagen.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
@@ -41,17 +44,11 @@
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\programs\xxhash.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\programs\datagen.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
@@ -61,32 +58,35 @@
|
||||
<ClInclude Include="..\..\..\lib\common\fse.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\fse_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\huf.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\huf_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zstd.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\programs\util.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -9,8 +9,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench", "fullbench\full
|
||||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdlib", "zstdlib\zstdlib.vcxproj", "{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}"
|
||||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datagen", "datagen\datagen.vcxproj", "{037E781E-81A6-494B-B1B3-438AB1200523}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Win32 = Debug|Win32
|
||||
@@ -51,14 +49,6 @@ Global
|
||||
{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|Win32.Build.0 = Release|Win32
|
||||
{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.ActiveCfg = Release|x64
|
||||
{8BFD8150-94D5-4BF9-8A50-7BD9929A0850}.Release|x64.Build.0 = Release|x64
|
||||
{037E781E-81A6-494B-B1B3-438AB1200523}.Debug|Win32.ActiveCfg = Debug|Win32
|
||||
{037E781E-81A6-494B-B1B3-438AB1200523}.Debug|Win32.Build.0 = Debug|Win32
|
||||
{037E781E-81A6-494B-B1B3-438AB1200523}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{037E781E-81A6-494B-B1B3-438AB1200523}.Debug|x64.Build.0 = Debug|x64
|
||||
{037E781E-81A6-494B-B1B3-438AB1200523}.Release|Win32.ActiveCfg = Release|Win32
|
||||
{037E781E-81A6-494B-B1B3-438AB1200523}.Release|Win32.Build.0 = Release|Win32
|
||||
{037E781E-81A6-494B-B1B3-438AB1200523}.Release|x64.ActiveCfg = Release|x64
|
||||
{037E781E-81A6-494B-B1B3-438AB1200523}.Release|x64.Build.0 = Release|x64
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
Regular → Executable
+25
-22
@@ -1,4 +1,4 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|Win32">
|
||||
@@ -20,7 +20,6 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
@@ -37,23 +36,29 @@
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v03.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v04.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v05.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v08.c" />
|
||||
<ClCompile Include="..\..\..\programs\bench.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\dibio.c" />
|
||||
<ClCompile Include="..\..\..\programs\fileio.c" />
|
||||
<ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\programs\zstdcli.c" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\fse.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
|
||||
@@ -61,12 +66,14 @@
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v03.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v04.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v05.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v06.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v07.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v08.h" />
|
||||
<ClInclude Include="..\..\..\programs\bench.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
<ClInclude Include="..\..\..\programs\dibio.h" />
|
||||
<ClInclude Include="..\..\..\programs\fileio.h" />
|
||||
<ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
</ItemGroup>
|
||||
<PropertyGroup Label="Globals">
|
||||
@@ -79,23 +86,27 @@
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
@@ -116,29 +127,23 @@
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<LibraryPath>$(LibraryPath)</LibraryPath>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<LibraryPath>$(LibraryPath);</LibraryPath>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<LibraryPath>$(LibraryPath)</LibraryPath>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
<LibraryPath>$(LibraryPath);</LibraryPath>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<ClCompile>
|
||||
@@ -153,7 +158,6 @@
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
@@ -163,13 +167,12 @@
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
@@ -177,12 +180,12 @@
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<PrecompiledHeader>
|
||||
</PrecompiledHeader>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<Optimization>Full</Optimization>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
@@ -190,7 +193,7 @@
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalDependencies>setargv.obj; kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
@@ -211,7 +214,7 @@
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<AdditionalDependencies>setargv.obj;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalDependencies>setargv.obj;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
Regular → Executable
+39
-30
@@ -17,34 +17,34 @@
|
||||
<ClCompile Include="..\..\..\programs\fileio.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\xxhash.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\zstdcli.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\dibio.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\datagen.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v02.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v03.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\datagen.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v04.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v05.c">
|
||||
<ClCompile Include="..\..\..\programs\dibio.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v06.c">
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v05.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c">
|
||||
@@ -83,9 +83,6 @@
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\programs\bench.h">
|
||||
@@ -94,65 +91,77 @@
|
||||
<ClInclude Include="..\..\..\programs\fileio.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\programs\datagen.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h">
|
||||
<ClInclude Include="..\..\..\programs\xxhash.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v01.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v02.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v03.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\programs\datagen.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v04.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v05.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v06.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\programs\dibio.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\zdict_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\fse.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\fse_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\huf.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\huf_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zstd.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v05.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\programs\util.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
Binary file not shown.
Binary file not shown.
@@ -20,7 +20,6 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
|
||||
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
|
||||
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
|
||||
@@ -37,13 +36,17 @@
|
||||
<ClInclude Include="..\..\..\lib\common\error_public.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\mem.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\fse.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\huf.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
|
||||
<ClInclude Include="..\..\..\programs\util.h" />
|
||||
<ClInclude Include="resource.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ResourceCompile Include="zstdlib.rc" />
|
||||
@@ -58,23 +61,27 @@
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
@@ -97,28 +104,28 @@
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<TargetName>zstdlib_x86</TargetName>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<TargetName>zstdlib_x64</TargetName>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<TargetName>zstdlib_x86</TargetName>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<TargetName>zstdlib_x64</TargetName>
|
||||
<IntDir>$(Platform)\$(Configuration)\</IntDir>
|
||||
<IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
|
||||
<IncludePath>$(WindowsSdkDir)\include;$(UniversalCRT_IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
@@ -148,7 +155,7 @@
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
@@ -169,7 +176,7 @@
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
</ClCompile>
|
||||
@@ -192,7 +199,7 @@
|
||||
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<OmitFramePointers>true</OmitFramePointers>
|
||||
+22
-7
@@ -9,6 +9,10 @@
|
||||
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
|
||||
<Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
|
||||
</Filter>
|
||||
<Filter Include="Resource Files">
|
||||
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
|
||||
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
|
||||
</Filter>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="..\..\..\lib\common\zstd_common.c">
|
||||
@@ -47,11 +51,11 @@
|
||||
<ClCompile Include="..\..\..\lib\common\entropy_common.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\common\xxhash.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="resource.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\bitstream.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
@@ -67,29 +71,40 @@
|
||||
<ClInclude Include="..\..\..\lib\common\fse.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\fse_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\huf.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\huf_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zbuff_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zstd.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_internal.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\zstd_static.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\programs\util.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\common\xxhash.h">
|
||||
<Filter>Header Files</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ResourceCompile Include="zstdlib.rc" />
|
||||
<ResourceCompile Include="zstdlib.rc">
|
||||
<Filter>Resource Files</Filter>
|
||||
</ResourceCompile>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -19,9 +19,6 @@
|
||||
|
||||
# Default result files
|
||||
_*
|
||||
example
|
||||
example_zstd
|
||||
foo.gz
|
||||
|
||||
# Misc files
|
||||
*.bat
|
||||
|
||||
+12
-19
@@ -6,25 +6,23 @@
|
||||
|
||||
|
||||
# Paths to static and dynamic zlib and zstd libraries
|
||||
# Use "make ZLIBDIR=path/to/zlib" to select a path to library
|
||||
ifdef ZLIBDIR
|
||||
STATICLIB = $(ZLIBDIR)/libz.a ../lib/libzstd.a
|
||||
IMPLIB = $(ZLIBDIR)/libz.dll.a ../lib/libzstd.a
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
STATICLIB = ../../zlib/libz.a ../lib/libzstd.a
|
||||
IMPLIB = ../../zlib/libz.dll.a ../lib/libzstd.a
|
||||
else
|
||||
STATICLIB = -static -lz ../lib/libzstd.a
|
||||
IMPLIB = -lz ../lib/libzstd.a
|
||||
endif
|
||||
|
||||
ZLIBWRAPPER_PATH = .
|
||||
EXAMPLE_PATH = examples
|
||||
EXAMPLE_PATH = examples/
|
||||
CC = gcc
|
||||
CFLAGS = $(LOC) -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -std=gnu90
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
|
||||
CFLAGS = $(LOC) -I../lib/common -I$(ZLIBWRAPPER_PATH) -O3 -Wall -std=gnu89
|
||||
LDFLAGS = $(LOC)
|
||||
RM = rm -f
|
||||
|
||||
|
||||
all: clean test testzstd
|
||||
all: example example_d
|
||||
|
||||
test: example
|
||||
./example
|
||||
@@ -32,9 +30,6 @@ test: example
|
||||
testdll: example_d
|
||||
./example_d
|
||||
|
||||
testzstd: example_zstd
|
||||
./example_zstd
|
||||
|
||||
.c.o:
|
||||
$(CC) $(CFLAGS) -c -o $@ $<
|
||||
|
||||
@@ -44,18 +39,16 @@ example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
|
||||
example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
|
||||
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(IMPLIB)
|
||||
|
||||
example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o
|
||||
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB)
|
||||
|
||||
$(EXAMPLE_PATH)/example.o: $(EXAMPLE_PATH)/example.c
|
||||
$(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/example.c
|
||||
|
||||
$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
|
||||
$(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
|
||||
|
||||
$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
|
||||
$(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
|
||||
|
||||
clean:
|
||||
-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd
|
||||
@echo Cleaning completed
|
||||
-$(RM) $(ZLIBWRAPPER_PATH)/*.o
|
||||
-$(RM) $(EXAMPLE_PATH)/*.o
|
||||
-$(RM) *.o
|
||||
-$(RM) *.exe
|
||||
-$(RM) foo.gz
|
||||
|
||||
|
||||
@@ -61,15 +61,12 @@ void *myalloc(q, n, m)
|
||||
void *q;
|
||||
unsigned n, m;
|
||||
{
|
||||
void *buf = calloc(n, m);
|
||||
q = Z_NULL;
|
||||
/* printf("myalloc %p n=%d m=%d\n", buf, n, m); */
|
||||
return buf;
|
||||
return calloc(n, m);
|
||||
}
|
||||
|
||||
void myfree(void *q, void *p)
|
||||
{
|
||||
/* printf("myfree %p\n", p); */
|
||||
q = Z_NULL;
|
||||
free(p);
|
||||
}
|
||||
|
||||
+143
-219
@@ -29,35 +29,30 @@
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
*/
|
||||
|
||||
#include <stdarg.h> /* va_list, for z_gzprintf */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <zlib.h>
|
||||
#include "zstd_zlibwrapper.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */
|
||||
#include "zstd.h"
|
||||
#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */
|
||||
#include "zstd_static.h" /* ZSTD_MAGICNUMBER */
|
||||
#include "zstd_internal.h" /* MIN */
|
||||
#include "zbuff.h"
|
||||
#include "zstd_internal.h" /* defaultCustomMem */
|
||||
|
||||
|
||||
#define Z_INFLATE_SYNC 8
|
||||
#define ZWRAP_HEADERSIZE 4
|
||||
#define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */
|
||||
#define ZSTD_FRAMEHEADERSIZE_MIN 5
|
||||
|
||||
#define LOG_WRAPPER(...) // printf(__VA_ARGS__)
|
||||
|
||||
|
||||
#define FINISH_WITH_GZ_ERR(msg) { \
|
||||
(void)msg; \
|
||||
return Z_MEM_ERROR; \
|
||||
}
|
||||
|
||||
#define FINISH_WITH_ERR(strm, message) { \
|
||||
strm->msg = message; \
|
||||
#define FINISH_WITH_ERR(msg) { \
|
||||
fprintf(stderr, "ERROR: %s\n", msg); \
|
||||
return Z_MEM_ERROR; \
|
||||
}
|
||||
|
||||
#define FINISH_WITH_NULL_ERR(msg) { \
|
||||
(void)msg; \
|
||||
fprintf(stderr, "ERROR: %s\n", msg); \
|
||||
return NULL; \
|
||||
}
|
||||
|
||||
@@ -71,29 +66,13 @@ static int g_useZSTD = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */
|
||||
|
||||
void useZSTD(int turn_on) { g_useZSTD = turn_on; }
|
||||
|
||||
int isUsingZSTD(void) { return g_useZSTD; }
|
||||
int isUsingZSTD() { return g_useZSTD; }
|
||||
|
||||
const char * zstdVersion(void) { return ZSTD_VERSION_STRING; }
|
||||
const char * zstdVersion() { return ZSTD_VERSION_STRING; }
|
||||
|
||||
ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); }
|
||||
|
||||
|
||||
static void* ZWRAP_allocFunction(void* opaque, size_t size)
|
||||
{
|
||||
z_streamp strm = (z_streamp) opaque;
|
||||
void* address = strm->zalloc(strm->opaque, 1, size);
|
||||
/* printf("ZWRAP alloc %p, %d \n", address, (int)size); */
|
||||
return address;
|
||||
}
|
||||
|
||||
static void ZWRAP_freeFunction(void* opaque, void* address)
|
||||
{
|
||||
z_streamp strm = (z_streamp) opaque;
|
||||
strm->zfree(strm->opaque, address);
|
||||
/* if (address) printf("ZWRAP free %p \n", address); */
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* *** Compression *** */
|
||||
|
||||
@@ -101,45 +80,28 @@ typedef struct {
|
||||
ZBUFF_CCtx* zbc;
|
||||
size_t bytesLeft;
|
||||
int compressionLevel;
|
||||
ZSTD_customMem customMem;
|
||||
z_stream allocFunc; /* copy of zalloc, zfree, opaque */
|
||||
} ZWRAP_CCtx;
|
||||
|
||||
|
||||
ZWRAP_CCtx* ZWRAP_createCCtx()
|
||||
{
|
||||
ZWRAP_CCtx* zwc = (ZWRAP_CCtx*)malloc(sizeof(ZWRAP_CCtx));
|
||||
if (zwc==NULL) return NULL;
|
||||
memset(zwc, 0, sizeof(*zwc));
|
||||
zwc->zbc = ZBUFF_createCCtx();
|
||||
return zwc;
|
||||
}
|
||||
|
||||
|
||||
size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
|
||||
{
|
||||
if (zwc==NULL) return 0; /* support free on NULL */
|
||||
ZBUFF_freeCCtx(zwc->zbc);
|
||||
zwc->customMem.customFree(zwc->customMem.opaque, zwc);
|
||||
free(zwc);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
|
||||
{
|
||||
ZWRAP_CCtx* zwc;
|
||||
|
||||
if (strm->zalloc && strm->zfree) {
|
||||
zwc = (ZWRAP_CCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_CCtx));
|
||||
if (zwc==NULL) return NULL;
|
||||
memset(zwc, 0, sizeof(ZWRAP_CCtx));
|
||||
memcpy(&zwc->allocFunc, strm, sizeof(z_stream));
|
||||
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc };
|
||||
memcpy(&zwc->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
|
||||
}
|
||||
} else {
|
||||
zwc = (ZWRAP_CCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_CCtx));
|
||||
if (zwc==NULL) return NULL;
|
||||
memset(zwc, 0, sizeof(ZWRAP_CCtx));
|
||||
memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
|
||||
}
|
||||
|
||||
zwc->zbc = ZBUFF_createCCtx_advanced(zwc->customMem);
|
||||
if (zwc->zbc == NULL) { ZWRAP_freeCCtx(zwc); return NULL; }
|
||||
return zwc;
|
||||
}
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level,
|
||||
const char *version, int stream_size))
|
||||
{
|
||||
@@ -151,24 +113,21 @@ ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level,
|
||||
}
|
||||
|
||||
LOG_WRAPPER("- deflateInit level=%d\n", level);
|
||||
zwc = ZWRAP_createCCtx(strm);
|
||||
zwc = ZWRAP_createCCtx();
|
||||
if (zwc == NULL) return Z_MEM_ERROR;
|
||||
|
||||
if (level == Z_DEFAULT_COMPRESSION)
|
||||
level = ZWRAP_DEFAULT_CLEVEL;
|
||||
|
||||
{ size_t const errorCode = ZBUFF_compressInit(zwc->zbc, level);
|
||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
|
||||
|
||||
zwc->compressionLevel = level;
|
||||
strm->state = (struct internal_state*) zwc; /* use state which in not used by user */
|
||||
strm->opaque = (voidpf) zwc;
|
||||
strm->total_in = 0;
|
||||
strm->total_out = 0;
|
||||
return Z_OK;
|
||||
}
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method,
|
||||
ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method,
|
||||
int windowBits, int memLevel,
|
||||
int strategy, const char *version,
|
||||
int stream_size))
|
||||
@@ -187,7 +146,7 @@ ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm,
|
||||
if (!g_useZSTD)
|
||||
return deflateSetDictionary(strm, dictionary, dictLength);
|
||||
|
||||
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
|
||||
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->opaque;
|
||||
LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)strm->data_type);
|
||||
{ size_t const errorCode = ZBUFF_compressInitDictionary(zwc->zbc, dictionary, dictLength, zwc->compressionLevel);
|
||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
|
||||
@@ -207,7 +166,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
|
||||
return res;
|
||||
}
|
||||
|
||||
zwc = (ZWRAP_CCtx*) strm->state;
|
||||
zwc = (ZWRAP_CCtx*) strm->opaque;
|
||||
|
||||
LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
|
||||
if (strm->avail_in > 0) {
|
||||
@@ -224,7 +183,7 @@ ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush))
|
||||
strm->avail_in -= srcSize;
|
||||
}
|
||||
|
||||
if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!");
|
||||
if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR("Z_FULL_FLUSH is not supported!");
|
||||
|
||||
if (flush == Z_FINISH || flush == Z_FULL_FLUSH) {
|
||||
size_t bytesLeft;
|
||||
@@ -256,7 +215,7 @@ ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm))
|
||||
return deflateEnd(strm);
|
||||
}
|
||||
LOG_WRAPPER("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
|
||||
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
|
||||
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->opaque;
|
||||
size_t const errorCode = ZWRAP_freeCCtx(zwc);
|
||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||
}
|
||||
@@ -294,37 +253,25 @@ ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm,
|
||||
|
||||
typedef struct {
|
||||
ZBUFF_DCtx* zbd;
|
||||
char headerBuf[ZWRAP_HEADERSIZE];
|
||||
char headerBuf[ZSTD_FRAMEHEADERSIZE_MIN];
|
||||
int errorCount;
|
||||
int requiredHeaderSize;
|
||||
|
||||
/* zlib params */
|
||||
int stream_size;
|
||||
char *version;
|
||||
int windowBits;
|
||||
ZSTD_customMem customMem;
|
||||
z_stream allocFunc; /* copy of zalloc, zfree, opaque */
|
||||
alloc_func zalloc; /* used to allocate the internal state */
|
||||
free_func zfree; /* used to free the internal state */
|
||||
voidpf opaque; /* private data object passed to zalloc and zfree */
|
||||
} ZWRAP_DCtx;
|
||||
|
||||
|
||||
ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
|
||||
ZWRAP_DCtx* ZWRAP_createDCtx(void)
|
||||
{
|
||||
ZWRAP_DCtx* zwd;
|
||||
|
||||
if (strm->zalloc && strm->zfree) {
|
||||
zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx));
|
||||
if (zwd==NULL) return NULL;
|
||||
memset(zwd, 0, sizeof(ZWRAP_DCtx));
|
||||
memcpy(&zwd->allocFunc, strm, sizeof(z_stream));
|
||||
{ ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc };
|
||||
memcpy(&zwd->customMem, &ZWRAP_customMem, sizeof(ZSTD_customMem));
|
||||
}
|
||||
} else {
|
||||
zwd = (ZWRAP_DCtx*)defaultCustomMem.customAlloc(defaultCustomMem.opaque, sizeof(ZWRAP_DCtx));
|
||||
if (zwd==NULL) return NULL;
|
||||
memset(zwd, 0, sizeof(ZWRAP_DCtx));
|
||||
memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
|
||||
}
|
||||
|
||||
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)malloc(sizeof(ZWRAP_DCtx));
|
||||
if (zwd==NULL) return NULL;
|
||||
memset(zwd, 0, sizeof(*zwd));
|
||||
return zwd;
|
||||
}
|
||||
|
||||
@@ -332,9 +279,9 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
|
||||
size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
|
||||
{
|
||||
if (zwd==NULL) return 0; /* support free on null */
|
||||
ZBUFF_freeDCtx(zwd->zbd);
|
||||
if (zwd->version) zwd->customMem.customFree(zwd->customMem.opaque, zwd->version);
|
||||
zwd->customMem.customFree(zwd->customMem.opaque, zwd);
|
||||
if (zwd->version) free(zwd->version);
|
||||
if (zwd->zbd) ZBUFF_freeDCtx(zwd->zbd);
|
||||
free(zwd);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -342,16 +289,18 @@ size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
|
||||
ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
|
||||
const char *version, int stream_size))
|
||||
{
|
||||
ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm);
|
||||
ZWRAP_DCtx* zwd = ZWRAP_createDCtx();
|
||||
LOG_WRAPPER("- inflateInit\n");
|
||||
if (zwd == NULL) { strm->state = NULL; return Z_MEM_ERROR; }
|
||||
|
||||
zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1);
|
||||
if (zwd->version == NULL) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; }
|
||||
strcpy(zwd->version, version);
|
||||
if (zwd == NULL) return Z_MEM_ERROR;
|
||||
|
||||
zwd->stream_size = stream_size;
|
||||
strm->state = (struct internal_state*) zwd; /* use state which in not used by user */
|
||||
zwd->version = strdup(version);
|
||||
zwd->zalloc = strm->zalloc;
|
||||
zwd->zfree = strm->zfree;
|
||||
zwd->opaque = strm->opaque;
|
||||
zwd->requiredHeaderSize = ZWRAP_HEADERSIZE;
|
||||
|
||||
strm->state = (struct internal_state*) zwd;
|
||||
strm->total_in = 0;
|
||||
strm->total_out = 0;
|
||||
strm->reserved = 1; /* mark as unknown steam */
|
||||
@@ -372,6 +321,8 @@ ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int windowBits,
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
|
||||
const Bytef *dictionary,
|
||||
uInt dictLength))
|
||||
@@ -380,11 +331,9 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
|
||||
return inflateSetDictionary(strm, dictionary, dictLength);
|
||||
|
||||
LOG_WRAPPER("- inflateSetDictionary\n");
|
||||
{ size_t errorCode;
|
||||
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
|
||||
if (strm->state == NULL) return Z_MEM_ERROR;
|
||||
errorCode = ZBUFF_decompressInitDictionary(zwd->zbd, dictionary, dictLength);
|
||||
if (ZSTD_isError(errorCode)) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; }
|
||||
{ ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
|
||||
size_t errorCode = ZBUFF_decompressInitDictionary(zwd->zbd, dictionary, dictLength);
|
||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||
|
||||
if (strm->total_in == ZSTD_frameHeaderSize_min) {
|
||||
size_t dstCapacity = 0;
|
||||
@@ -393,7 +342,6 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
|
||||
LOG_WRAPPER("ZBUFF_decompressContinue3 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity);
|
||||
if (dstCapacity > 0 || ZSTD_isError(errorCode)) {
|
||||
LOG_WRAPPER("ERROR: ZBUFF_decompressContinue %s\n", ZSTD_getErrorName(errorCode));
|
||||
ZWRAP_freeDCtx(zwd); strm->state = NULL;
|
||||
return Z_MEM_ERROR;
|
||||
}
|
||||
}
|
||||
@@ -411,16 +359,15 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
|
||||
if (strm->avail_in > 0) {
|
||||
size_t errorCode, dstCapacity, srcSize;
|
||||
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
|
||||
if (strm->state == NULL) return Z_MEM_ERROR;
|
||||
LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
|
||||
if (strm->total_in < ZWRAP_HEADERSIZE)
|
||||
while (strm->total_in < ZSTD_FRAMEHEADERSIZE_MIN)
|
||||
{
|
||||
srcSize = MIN(strm->avail_in, ZWRAP_HEADERSIZE - strm->total_in);
|
||||
srcSize = MIN(strm->avail_in, zwd->requiredHeaderSize - strm->total_in);
|
||||
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
|
||||
strm->total_in += srcSize;
|
||||
strm->next_in += srcSize;
|
||||
strm->avail_in -= srcSize;
|
||||
if (strm->total_in < ZWRAP_HEADERSIZE) return Z_OK;
|
||||
if (strm->total_in < zwd->requiredHeaderSize) return Z_OK;
|
||||
|
||||
if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
|
||||
z_stream strm2;
|
||||
@@ -434,7 +381,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
|
||||
else
|
||||
errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
|
||||
LOG_WRAPPER("ZLIB inflateInit errorCode=%d\n", (int)errorCode);
|
||||
if (errorCode != Z_OK) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return errorCode; }
|
||||
if (errorCode != Z_OK) return errorCode;
|
||||
|
||||
/* inflate header */
|
||||
strm->next_in = (unsigned char*)zwd->headerBuf;
|
||||
@@ -442,37 +389,37 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
|
||||
strm->avail_out = 0;
|
||||
errorCode = inflate(strm, Z_NO_FLUSH);
|
||||
LOG_WRAPPER("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in);
|
||||
if (errorCode != Z_OK) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return errorCode; }
|
||||
if (strm->avail_in > 0) goto error;
|
||||
if (errorCode != Z_OK) return errorCode;
|
||||
if (strm->avail_in > 0) return Z_MEM_ERROR;
|
||||
|
||||
strm->next_in = strm2.next_in;
|
||||
strm->avail_in = strm2.avail_in;
|
||||
strm->next_out = strm2.next_out;
|
||||
strm->avail_out = strm2.avail_out;
|
||||
|
||||
strm->reserved = 0; /* mark as zlib stream */
|
||||
errorCode = ZWRAP_freeDCtx(zwd);
|
||||
if (ZSTD_isError(errorCode)) goto error;
|
||||
|
||||
if (flush == Z_INFLATE_SYNC) return inflateSync(strm);
|
||||
return inflate(strm, flush);
|
||||
}
|
||||
|
||||
zwd->zbd = ZBUFF_createDCtx_advanced(zwd->customMem);
|
||||
if (zwd->zbd == NULL) goto error;
|
||||
if (zwd->requiredHeaderSize < ZSTD_FRAMEHEADERSIZE_MIN) {
|
||||
zwd->requiredHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
|
||||
continue;
|
||||
}
|
||||
|
||||
errorCode = ZBUFF_decompressInit(zwd->zbd);
|
||||
if (ZSTD_isError(errorCode)) goto error;
|
||||
zwd->zbd = ZBUFF_createDCtx();
|
||||
{ size_t const errorCode = ZBUFF_decompressInit(zwd->zbd);
|
||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
|
||||
|
||||
srcSize = ZWRAP_HEADERSIZE;
|
||||
srcSize = ZSTD_FRAMEHEADERSIZE_MIN;
|
||||
dstCapacity = 0;
|
||||
errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, zwd->headerBuf, &srcSize);
|
||||
LOG_WRAPPER("ZBUFF_decompressContinue1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity);
|
||||
if (ZSTD_isError(errorCode)) {
|
||||
LOG_WRAPPER("ERROR: ZBUFF_decompressContinue %s\n", ZSTD_getErrorName(errorCode));
|
||||
goto error;
|
||||
return Z_MEM_ERROR;
|
||||
}
|
||||
if (strm->avail_in == 0) return Z_OK;
|
||||
break;
|
||||
}
|
||||
|
||||
srcSize = strm->avail_in;
|
||||
@@ -482,7 +429,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
|
||||
if (ZSTD_isError(errorCode)) {
|
||||
LOG_WRAPPER("ERROR: ZBUFF_decompressContinue %s\n", ZSTD_getErrorName(errorCode));
|
||||
zwd->errorCount++;
|
||||
if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error;
|
||||
return (zwd->errorCount<=1) ? Z_NEED_DICT : Z_MEM_ERROR;
|
||||
}
|
||||
strm->next_out += dstCapacity;
|
||||
strm->total_out += dstCapacity;
|
||||
@@ -491,11 +438,6 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
|
||||
strm->next_in += srcSize;
|
||||
strm->avail_in -= srcSize;
|
||||
if (errorCode == 0) return Z_STREAM_END;
|
||||
return Z_OK;
|
||||
error:
|
||||
ZWRAP_freeDCtx(zwd);
|
||||
strm->state = NULL;
|
||||
return Z_MEM_ERROR;
|
||||
}
|
||||
return Z_OK;
|
||||
}
|
||||
@@ -505,12 +447,11 @@ ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm))
|
||||
{
|
||||
int ret = Z_OK;
|
||||
if (!strm->reserved)
|
||||
return inflateEnd(strm);
|
||||
ret = inflateEnd(strm);
|
||||
|
||||
LOG_WRAPPER("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
|
||||
{ ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
|
||||
size_t const errorCode = ZWRAP_freeDCtx(zwd);
|
||||
strm->state = NULL;
|
||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||
}
|
||||
return ret;
|
||||
@@ -531,7 +472,7 @@ ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest,
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return deflateCopy(dest, source);
|
||||
FINISH_WITH_ERR(source, "deflateCopy is not supported!");
|
||||
FINISH_WITH_ERR("deflateCopy is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -539,7 +480,7 @@ ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return deflateReset(strm);
|
||||
FINISH_WITH_ERR(strm, "deflateReset is not supported!");
|
||||
FINISH_WITH_ERR("deflateReset is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -551,20 +492,18 @@ ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm,
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return deflateTune(strm, good_length, max_lazy, nice_length, max_chain);
|
||||
FINISH_WITH_ERR(strm, "deflateTune is not supported!");
|
||||
FINISH_WITH_ERR("deflateTune is not supported!");
|
||||
}
|
||||
|
||||
|
||||
#if ZLIB_VERNUM >= 0x1260
|
||||
ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm,
|
||||
unsigned *pending,
|
||||
int *bits))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return deflatePending(strm, pending, bits);
|
||||
FINISH_WITH_ERR(strm, "deflatePending is not supported!");
|
||||
FINISH_WITH_ERR("deflatePending is not supported!");
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm,
|
||||
@@ -573,7 +512,7 @@ ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm,
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return deflatePrime(strm, bits, value);
|
||||
FINISH_WITH_ERR(strm, "deflatePrime is not supported!");
|
||||
FINISH_WITH_ERR("deflatePrime is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -582,23 +521,22 @@ ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm,
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return deflateSetHeader(strm, head);
|
||||
FINISH_WITH_ERR(strm, "deflateSetHeader is not supported!");
|
||||
FINISH_WITH_ERR("deflateSetHeader is not supported!");
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/* Advanced compression functions */
|
||||
#if ZLIB_VERNUM >= 0x1280
|
||||
ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm,
|
||||
Bytef *dictionary,
|
||||
uInt *dictLength))
|
||||
{
|
||||
if (!strm->reserved)
|
||||
return inflateGetDictionary(strm, dictionary, dictLength);
|
||||
FINISH_WITH_ERR(strm, "inflateGetDictionary is not supported!");
|
||||
FINISH_WITH_ERR("inflateGetDictionary is not supported!");
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest,
|
||||
@@ -606,7 +544,7 @@ ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest,
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return inflateCopy(dest, source);
|
||||
FINISH_WITH_ERR(source, "inflateCopy is not supported!");
|
||||
FINISH_WITH_ERR("inflateCopy is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -614,29 +552,17 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
|
||||
{
|
||||
if (!strm->reserved)
|
||||
return inflateReset(strm);
|
||||
FINISH_WITH_ERR(strm, "inflateReset is not supported!");
|
||||
FINISH_WITH_ERR("inflateReset is not supported!");
|
||||
}
|
||||
|
||||
|
||||
#if ZLIB_VERNUM >= 0x1240
|
||||
ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm,
|
||||
int windowBits))
|
||||
{
|
||||
if (!strm->reserved)
|
||||
return inflateReset2(strm, windowBits);
|
||||
FINISH_WITH_ERR(strm, "inflateReset2 is not supported!");
|
||||
FINISH_WITH_ERR("inflateReset2 is not supported!");
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#if ZLIB_VERNUM >= 0x1240
|
||||
ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm))
|
||||
{
|
||||
if (!strm->reserved)
|
||||
return inflateMark(strm);
|
||||
FINISH_WITH_ERR(strm, "inflateMark is not supported!");
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm,
|
||||
@@ -645,7 +571,15 @@ ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm,
|
||||
{
|
||||
if (!strm->reserved)
|
||||
return inflatePrime(strm, bits, value);
|
||||
FINISH_WITH_ERR(strm, "inflatePrime is not supported!");
|
||||
FINISH_WITH_ERR("inflatePrime is not supported!");
|
||||
}
|
||||
|
||||
|
||||
ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm))
|
||||
{
|
||||
if (!strm->reserved)
|
||||
return inflateMark(strm);
|
||||
FINISH_WITH_ERR("inflateMark is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -654,7 +588,7 @@ ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm,
|
||||
{
|
||||
if (!strm->reserved)
|
||||
return inflateGetHeader(strm, head);
|
||||
FINISH_WITH_ERR(strm, "inflateGetHeader is not supported!");
|
||||
FINISH_WITH_ERR("inflateGetHeader is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -665,7 +599,7 @@ ZEXTERN int ZEXPORT z_inflateBackInit_ OF((z_streamp strm, int windowBits,
|
||||
{
|
||||
if (!strm->reserved)
|
||||
return inflateBackInit_(strm, windowBits, window, version, stream_size);
|
||||
FINISH_WITH_ERR(strm, "inflateBackInit is not supported!");
|
||||
FINISH_WITH_ERR("inflateBackInit is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -675,7 +609,7 @@ ZEXTERN int ZEXPORT z_inflateBack OF((z_streamp strm,
|
||||
{
|
||||
if (!strm->reserved)
|
||||
return inflateBack(strm, in, in_desc, out, out_desc);
|
||||
FINISH_WITH_ERR(strm, "inflateBack is not supported!");
|
||||
FINISH_WITH_ERR("inflateBack is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -683,7 +617,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm))
|
||||
{
|
||||
if (!strm->reserved)
|
||||
return inflateBackEnd(strm);
|
||||
FINISH_WITH_ERR(strm, "inflateBackEnd is not supported!");
|
||||
FINISH_WITH_ERR("inflateBackEnd is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -700,9 +634,9 @@ ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen,
|
||||
if (!g_useZSTD)
|
||||
return compress(dest, destLen, source, sourceLen);
|
||||
|
||||
{ size_t dstCapacity = *destLen;
|
||||
size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, ZWRAP_DEFAULT_CLEVEL);
|
||||
LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity);
|
||||
size_t dstCapacity = *destLen;
|
||||
LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity);
|
||||
{ size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, -1);
|
||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||
*destLen = errorCode;
|
||||
}
|
||||
@@ -717,8 +651,8 @@ ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen,
|
||||
if (!g_useZSTD)
|
||||
return compress2(dest, destLen, source, sourceLen, level);
|
||||
|
||||
{ size_t dstCapacity = *destLen;
|
||||
size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level);
|
||||
size_t dstCapacity = *destLen;
|
||||
{ size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level);
|
||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||
*destLen = errorCode;
|
||||
}
|
||||
@@ -742,8 +676,8 @@ ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen,
|
||||
// if (!g_useZSTD)
|
||||
return uncompress(dest, destLen, source, sourceLen);
|
||||
|
||||
{ size_t dstCapacity = *destLen;
|
||||
size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen);
|
||||
size_t dstCapacity = *destLen;
|
||||
{ size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen);
|
||||
if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
|
||||
*destLen = errorCode;
|
||||
}
|
||||
@@ -769,45 +703,19 @@ ZEXTERN gzFile ZEXPORT z_gzdopen OF((int fd, const char *mode))
|
||||
}
|
||||
|
||||
|
||||
#if ZLIB_VERNUM >= 0x1240
|
||||
ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzbuffer(file, size);
|
||||
FINISH_WITH_GZ_ERR("gzbuffer is not supported!");
|
||||
FINISH_WITH_ERR("gzbuffer is not supported!");
|
||||
}
|
||||
|
||||
|
||||
ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzoffset(file);
|
||||
FINISH_WITH_GZ_ERR("gzoffset is not supported!");
|
||||
}
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzclose_r(file);
|
||||
FINISH_WITH_GZ_ERR("gzclose_r is not supported!");
|
||||
}
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzclose_w(file);
|
||||
FINISH_WITH_GZ_ERR("gzclose_w is not supported!");
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzsetparams(file, level, strategy);
|
||||
FINISH_WITH_GZ_ERR("gzsetparams is not supported!");
|
||||
FINISH_WITH_ERR("gzsetparams is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -815,7 +723,7 @@ ZEXTERN int ZEXPORT z_gzread OF((gzFile file, voidp buf, unsigned len))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzread(file, buf, len);
|
||||
FINISH_WITH_GZ_ERR("gzread is not supported!");
|
||||
FINISH_WITH_ERR("gzread is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -824,15 +732,11 @@ ZEXTERN int ZEXPORT z_gzwrite OF((gzFile file,
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzwrite(file, buf, len);
|
||||
FINISH_WITH_GZ_ERR("gzwrite is not supported!");
|
||||
FINISH_WITH_ERR("gzwrite is not supported!");
|
||||
}
|
||||
|
||||
|
||||
#if ZLIB_VERNUM >= 0x1260
|
||||
ZEXTERN int ZEXPORTVA z_gzprintf Z_ARG((gzFile file, const char *format, ...))
|
||||
#else
|
||||
ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...))
|
||||
#endif
|
||||
{
|
||||
if (!g_useZSTD) {
|
||||
int ret;
|
||||
@@ -846,7 +750,7 @@ ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...))
|
||||
// printf("gzprintf ret=%d\n", ret);
|
||||
return ret;
|
||||
}
|
||||
FINISH_WITH_GZ_ERR("gzprintf is not supported!");
|
||||
FINISH_WITH_ERR("gzprintf is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -854,7 +758,7 @@ ZEXTERN int ZEXPORT z_gzputs OF((gzFile file, const char *s))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzputs(file, s);
|
||||
FINISH_WITH_GZ_ERR("gzputs is not supported!");
|
||||
FINISH_WITH_ERR("gzputs is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -870,19 +774,15 @@ ZEXTERN int ZEXPORT z_gzputc OF((gzFile file, int c))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzputc(file, c);
|
||||
FINISH_WITH_GZ_ERR("gzputc is not supported!");
|
||||
FINISH_WITH_ERR("gzputc is not supported!");
|
||||
}
|
||||
|
||||
|
||||
#if ZLIB_VERNUM == 0x1260
|
||||
ZEXTERN int ZEXPORT z_gzgetc_ OF((gzFile file))
|
||||
#else
|
||||
ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file))
|
||||
#endif
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzgetc(file);
|
||||
FINISH_WITH_GZ_ERR("gzgetc is not supported!");
|
||||
FINISH_WITH_ERR("gzgetc is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -890,7 +790,7 @@ ZEXTERN int ZEXPORT z_gzungetc OF((int c, gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzungetc(c, file);
|
||||
FINISH_WITH_GZ_ERR("gzungetc is not supported!");
|
||||
FINISH_WITH_ERR("gzungetc is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -898,7 +798,7 @@ ZEXTERN int ZEXPORT z_gzflush OF((gzFile file, int flush))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzflush(file, flush);
|
||||
FINISH_WITH_GZ_ERR("gzflush is not supported!");
|
||||
FINISH_WITH_ERR("gzflush is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -906,7 +806,7 @@ ZEXTERN z_off_t ZEXPORT z_gzseek OF((gzFile file, z_off_t offset, int whence))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzseek(file, offset, whence);
|
||||
FINISH_WITH_GZ_ERR("gzseek is not supported!");
|
||||
FINISH_WITH_ERR("gzseek is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -914,7 +814,7 @@ ZEXTERN int ZEXPORT z_gzrewind OF((gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzrewind(file);
|
||||
FINISH_WITH_GZ_ERR("gzrewind is not supported!");
|
||||
FINISH_WITH_ERR("gzrewind is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -922,7 +822,15 @@ ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gztell(file);
|
||||
FINISH_WITH_GZ_ERR("gztell is not supported!");
|
||||
FINISH_WITH_ERR("gztell is not supported!");
|
||||
}
|
||||
|
||||
|
||||
ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzoffset(file);
|
||||
FINISH_WITH_ERR("gzoffset is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -930,7 +838,7 @@ ZEXTERN int ZEXPORT z_gzeof OF((gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzeof(file);
|
||||
FINISH_WITH_GZ_ERR("gzeof is not supported!");
|
||||
FINISH_WITH_ERR("gzeof is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -938,7 +846,7 @@ ZEXTERN int ZEXPORT z_gzdirect OF((gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzdirect(file);
|
||||
FINISH_WITH_GZ_ERR("gzdirect is not supported!");
|
||||
FINISH_WITH_ERR("gzdirect is not supported!");
|
||||
}
|
||||
|
||||
|
||||
@@ -946,7 +854,23 @@ ZEXTERN int ZEXPORT z_gzclose OF((gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzclose(file);
|
||||
FINISH_WITH_GZ_ERR("gzclose is not supported!");
|
||||
FINISH_WITH_ERR("gzclose is not supported!");
|
||||
}
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzclose_r(file);
|
||||
FINISH_WITH_ERR("gzclose_r is not supported!");
|
||||
}
|
||||
|
||||
|
||||
ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file))
|
||||
{
|
||||
if (!g_useZSTD)
|
||||
return gzclose_w(file);
|
||||
FINISH_WITH_ERR("gzclose_w is not supported!");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -40,17 +40,10 @@ extern "C" {
|
||||
#define Z_PREFIX
|
||||
#include <zlib.h>
|
||||
|
||||
#if !defined(z_const)
|
||||
#if ZLIB_VERNUM >= 0x1260
|
||||
#define z_const const
|
||||
#else
|
||||
#define z_const
|
||||
#endif
|
||||
#endif
|
||||
|
||||
void useZSTD(int turn_on);
|
||||
int isUsingZSTD(void);
|
||||
const char * zstdVersion(void);
|
||||
int isUsingZSTD();
|
||||
const char * zstdVersion();
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
Reference in New Issue
Block a user