ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4 from facebook/dev

Browse Source 
merge
This commit is contained in:
Lee
2019-06-11 11:25:26 +08:00
committed by GitHub
parent b31df043a0 c601622aa7
commit bc400ed13b
11 changed files with 185 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+4 -26
View File
@@ -1,33 +1,14 @@
version: 2 version: 2
references:
# Install the dependencies required for tests.
# Add the step "- *install-dependencies" to the beginning of your job to run
# this command.
install-dependencies: &install-dependencies
run:
name: Install dependencies
# TODO: We can split these dependencies up by job to reduce installation
# time.
command: |
sudo dpkg --add-architecture i386
sudo apt-get -y -qq update
sudo apt-get -y install \
gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
libcurl4-openssl-dev
jobs: jobs:
# the first half of the jobs are in this test # the first half of the jobs are in this test
short-tests-0: short-tests-0:
# TODO: Create a small custom docker image with all the dependencies we need # TODO: Create a small custom docker image with all the dependencies we need
# preinstalled to reduce installation time. # preinstalled to reduce installation time.
docker: docker:
- image: circleci/buildpack-deps:bionic - image: fbopensource/zstd-circleci-primary:0.0.1
steps: steps:
- checkout - checkout
- *install-dependencies
- run: - run:
name: Test name: Test
command: | command: |
@@ -41,10 +22,9 @@ jobs:
# the second half of the jobs are in this test # the second half of the jobs are in this test
short-tests-1: short-tests-1:
docker: docker:
- image: circleci/buildpack-deps:bionic - image: fbopensource/zstd-circleci-primary:0.0.1
steps: steps:
- checkout - checkout
- *install-dependencies
- run: - run:
name: Test name: Test
command: | command: |
@@ -61,12 +41,11 @@ jobs:
# tagged release. # tagged release.
publish-github-release: publish-github-release:
docker: docker:
- image: circleci/buildpack-deps:bionic - image: fbopensource/zstd-circleci-primary:0.0.1
environment: environment:
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
steps: steps:
- checkout - checkout
- *install-dependencies
- run: - run:
name: Publish name: Publish
command: | command: |
@@ -86,12 +65,11 @@ jobs:
# This step should only be run in a cron job # This step should only be run in a cron job
regression-test: regression-test:
docker: docker:
- image: circleci/buildpack-deps:bionic - image: fbopensource/zstd-circleci-primary:0.0.1
environment: environment:
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
steps: steps:
- checkout - checkout
- *install-dependencies
# Restore the cached resources. # Restore the cached resources.
- restore_cache: - restore_cache:
# We try our best to bust the cache when the data changes by hashing # We try our best to bust the cache when the data changes by hashing
.circleci/images/primary/Dockerfile
+9
View File
@@ -0,0 +1,9 @@
FROM circleci/buildpack-deps:bionic
RUN sudo dpkg --add-architecture i386
RUN sudo apt-get -y -qq update
RUN sudo apt-get -y install \
gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \
libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \
libcurl4-openssl-dev
build/VS2010/libzstd-dll/libzstd-dll.vcxproj
-2
View File
@@ -209,7 +209,6 @@
<EnablePREfast>false</EnablePREfast> <EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary> <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat> <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AssemblerOutput>All</AssemblerOutput>
</ClCompile> </ClCompile>
<Link> <Link>
<SubSystem>Console</SubSystem> <SubSystem>Console</SubSystem>
@@ -234,7 +233,6 @@
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat> <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<WholeProgramOptimization>true</WholeProgramOptimization> <WholeProgramOptimization>true</WholeProgramOptimization>
<OmitFramePointers>true</OmitFramePointers> <OmitFramePointers>true</OmitFramePointers>
<AssemblerOutput>All</AssemblerOutput>
</ClCompile> </ClCompile>
<Link> <Link>
<SubSystem>Console</SubSystem> <SubSystem>Console</SubSystem>
build/VS2010/libzstd/libzstd.vcxproj
+4 -6
View File
@@ -162,7 +162,7 @@
</PrecompiledHeader> </PrecompiledHeader>
<WarningLevel>Level4</WarningLevel> <WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization> <Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<MinimalRebuild>true</MinimalRebuild> <MinimalRebuild>true</MinimalRebuild>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks> <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary> <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -182,7 +182,7 @@
</PrecompiledHeader> </PrecompiledHeader>
<WarningLevel>Level4</WarningLevel> <WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization> <Optimization>Disabled</Optimization>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError> <TreatWarningAsError>true</TreatWarningAsError>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks> <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary> <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -202,11 +202,10 @@
<Optimization>MaxSpeed</Optimization> <Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking> <FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions> <IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast> <EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary> <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat> <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AssemblerOutput>All</AssemblerOutput>
</ClCompile> </ClCompile>
<Link> <Link>
<SubSystem>Console</SubSystem> <SubSystem>Console</SubSystem>
@@ -224,14 +223,13 @@
<Optimization>MaxSpeed</Optimization> <Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking> <FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions> <IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=5;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError> <TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast> <EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary> <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat> <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<WholeProgramOptimization>true</WholeProgramOptimization> <WholeProgramOptimization>true</WholeProgramOptimization>
<OmitFramePointers>true</OmitFramePointers> <OmitFramePointers>true</OmitFramePointers>
<AssemblerOutput>All</AssemblerOutput>
</ClCompile> </ClCompile>
<Link> <Link>
<SubSystem>Console</SubSystem> <SubSystem>Console</SubSystem>
build/cmake/lib/CMakeLists.txt
+8 -6
View File
@@ -194,10 +194,12 @@ if (ZSTD_BUILD_STATIC)
endif () endif ()
# uninstall target # uninstall target
configure_file( if (NOT TARGET uninstall)
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in" configure_file(
"${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
IMMEDIATE @ONLY) "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
IMMEDIATE @ONLY)
add_custom_target(uninstall add_custom_target(uninstall
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
endif ()
doc/zstd_manual.html
+67 -93
View File
@@ -10,37 +10,27 @@
<ol> <ol>
<li><a href="#Chapter1">Introduction</a></li> <li><a href="#Chapter1">Introduction</a></li>
<li><a href="#Chapter2">Version</a></li> <li><a href="#Chapter2">Version</a></li>
<li><a href="#Chapter3">Default constant</a></li> <li><a href="#Chapter3">Simple API</a></li>
<li><a href="#Chapter4">Constants</a></li> <li><a href="#Chapter4">Explicit context</a></li>
<li><a href="#Chapter5">Simple API</a></li> <li><a href="#Chapter5">Advanced compression API</a></li>
<li><a href="#Chapter6">Explicit context</a></li> <li><a href="#Chapter6">Advanced decompression API</a></li>
<li><a href="#Chapter7">Advanced compression API</a></li> <li><a href="#Chapter7">Streaming</a></li>
<li><a href="#Chapter8">Advanced decompression API</a></li> <li><a href="#Chapter8">Streaming compression - HowTo</a></li>
<li><a href="#Chapter9">Streaming</a></li> <li><a href="#Chapter9">Streaming decompression - HowTo</a></li>
<li><a href="#Chapter10">Streaming compression - HowTo</a></li> <li><a href="#Chapter10">Simple dictionary API</a></li>
<li><a href="#Chapter11">This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and</a></li> <li><a href="#Chapter11">Bulk processing dictionary API</a></li>
<li><a href="#Chapter12">Equivalent to:</a></li> <li><a href="#Chapter12">Dictionary helper functions</a></li>
<li><a href="#Chapter13">Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).</a></li> <li><a href="#Chapter13">Advanced dictionary and prefix API</a></li>
<li><a href="#Chapter14">Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush).</a></li> <li><a href="#Chapter14">experimental API (static linking only)</a></li>
<li><a href="#Chapter15">Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).</a></li> <li><a href="#Chapter15">Frame size functions</a></li>
<li><a href="#Chapter16">Streaming decompression - HowTo</a></li> <li><a href="#Chapter16">Memory management</a></li>
<li><a href="#Chapter17">Simple dictionary API</a></li> <li><a href="#Chapter17">Advanced compression functions</a></li>
<li><a href="#Chapter18">Bulk processing dictionary API</a></li> <li><a href="#Chapter18">Advanced decompression functions</a></li>
<li><a href="#Chapter19">Dictionary helper functions</a></li> <li><a href="#Chapter19">Advanced streaming functions</a></li>
<li><a href="#Chapter20">Advanced dictionary and prefix API</a></li> <li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions</a></li>
<li><a href="#Chapter21">ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li> <li><a href="#Chapter21">Buffer-less streaming compression (synchronous mode)</a></li>
<li><a href="#Chapter22">experimental API (static linking only)</a></li> <li><a href="#Chapter22">Buffer-less streaming decompression (synchronous mode)</a></li>
<li><a href="#Chapter23">Frame size functions</a></li> <li><a href="#Chapter23">Block level API</a></li>
<li><a href="#Chapter24">ZSTD_decompressBound() :</a></li>
<li><a href="#Chapter25">Memory management</a></li>
<li><a href="#Chapter26">Advanced compression functions</a></li>
<li><a href="#Chapter27">Advanced decompression functions</a></li>
<li><a href="#Chapter28">Advanced streaming functions</a></li>
<li><a href="#Chapter29">Buffer-less and synchronous inner streaming functions</a></li>
<li><a href="#Chapter30">Buffer-less streaming compression (synchronous mode)</a></li>
<li><a href="#Chapter31">Buffer-less streaming decompression (synchronous mode)</a></li>
<li><a href="#Chapter32">ZSTD_getFrameHeader() :</a></li>
<li><a href="#Chapter33">Block level API</a></li>
</ol> </ol>
<hr> <hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre> <a name="Chapter1"></a><h2>Introduction</h2><pre>
@@ -78,11 +68,7 @@
<pre><b>unsigned ZSTD_versionNumber(void); </b>/**< to check runtime library version */<b> <pre><b>unsigned ZSTD_versionNumber(void); </b>/**< to check runtime library version */<b>
</b></pre><BR> </b></pre><BR>
<a name="Chapter3"></a><h2>Default constant</h2><pre></pre> <a name="Chapter3"></a><h2>Simple API</h2><pre></pre>
<a name="Chapter4"></a><h2>Constants</h2><pre></pre>
<a name="Chapter5"></a><h2>Simple API</h2><pre></pre>
<pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity, <pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
@@ -152,7 +138,7 @@ const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable strin
int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed */<b> int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed */<b>
int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b> int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
</pre></b><BR> </pre></b><BR>
<a name="Chapter6"></a><h2>Explicit context</h2><pre></pre> <a name="Chapter4"></a><h2>Explicit context</h2><pre></pre>
<h3>Compression context</h3><pre> When compressing many times, <h3>Compression context</h3><pre> When compressing many times,
it is recommended to allocate a context just once, and re-use it for each successive compression operation. it is recommended to allocate a context just once, and re-use it for each successive compression operation.
@@ -189,7 +175,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
</p></pre><BR> </p></pre><BR>
<a name="Chapter7"></a><h2>Advanced compression API</h2><pre></pre> <a name="Chapter5"></a><h2>Advanced compression API</h2><pre></pre>
<pre><b>typedef enum { ZSTD_fast=1, <pre><b>typedef enum { ZSTD_fast=1,
ZSTD_dfast=2, ZSTD_dfast=2,
@@ -424,7 +410,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
</p></pre><BR> </p></pre><BR>
<a name="Chapter8"></a><h2>Advanced decompression API</h2><pre></pre> <a name="Chapter6"></a><h2>Advanced decompression API</h2><pre></pre>
<pre><b>typedef enum { <pre><b>typedef enum {
@@ -472,7 +458,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
</p></pre><BR> </p></pre><BR>
<a name="Chapter9"></a><h2>Streaming</h2><pre></pre> <a name="Chapter7"></a><h2>Streaming</h2><pre></pre>
<pre><b>typedef struct ZSTD_inBuffer_s { <pre><b>typedef struct ZSTD_inBuffer_s {
const void* src; </b>/**< start of input buffer */<b> const void* src; </b>/**< start of input buffer */<b>
@@ -486,7 +472,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
size_t pos; </b>/**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */<b> size_t pos; </b>/**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */<b>
} ZSTD_outBuffer; } ZSTD_outBuffer;
</b></pre><BR> </b></pre><BR>
<a name="Chapter10"></a><h2>Streaming compression - HowTo</h2><pre> <a name="Chapter8"></a><h2>Streaming compression - HowTo</h2><pre>
A ZSTD_CStream object is required to track streaming operation. A ZSTD_CStream object is required to track streaming operation.
Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
ZSTD_CStream objects can be reused multiple times on consecutive compression operations. ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
@@ -592,31 +578,28 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
<pre><b>size_t ZSTD_CStreamInSize(void); </b>/**< recommended size for input buffer */<b> <pre><b>size_t ZSTD_CStreamInSize(void); </b>/**< recommended size for input buffer */<b>
</b></pre><BR> </b></pre><BR>
<pre><b>size_t ZSTD_CStreamOutSize(void); </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */<b> <pre><b>size_t ZSTD_CStreamOutSize(void); </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */<b>
</b></pre><BR> </b></pre><BR>
<a name="Chapter11"></a><h2>This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and</h2><pre> ZSTD_compressStream2(). It is redundant, but is still fully supported. <pre><b>size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
Advanced parameters and dictionary compression can only be used through the </b>/*!<b>
new API. * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
<BR></pre> * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
* the next read size (if non-zero and not an error). ZSTD_compressStream2()
<a name="Chapter12"></a><h2>Equivalent to:</h2><pre> * returns the minimum nb of bytes left to flush (if non-zero and not an error).
*/
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
</b>/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */<b>
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
</b>/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */<b>
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
</b><p>
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
<BR></pre> </p></pre><BR>
<a name="Chapter13"></a><h2>Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).</h2><pre> NOTE: The return value is different. ZSTD_compressStream() returns a hint for <a name="Chapter9"></a><h2>Streaming decompression - HowTo</h2><pre>
the next read size (if non-zero and not an error). ZSTD_compressStream2()
returns the number of bytes left to flush (if non-zero and not an error).
<BR></pre>
<a name="Chapter14"></a><h2>Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush).</h2><pre></pre>
<a name="Chapter15"></a><h2>Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).</h2><pre></pre>
<a name="Chapter16"></a><h2>Streaming decompression - HowTo</h2><pre>
A ZSTD_DStream object is required to track streaming operations. A ZSTD_DStream object is required to track streaming operations.
Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
ZSTD_DStream objects can be re-used multiple times. ZSTD_DStream objects can be re-used multiple times.
@@ -652,7 +635,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
</b></pre><BR> </b></pre><BR>
<pre><b>size_t ZSTD_DStreamOutSize(void); </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b> <pre><b>size_t ZSTD_DStreamOutSize(void); </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b>
</b></pre><BR> </b></pre><BR>
<a name="Chapter17"></a><h2>Simple dictionary API</h2><pre></pre> <a name="Chapter10"></a><h2>Simple dictionary API</h2><pre></pre>
<pre><b>size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, <pre><b>size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
@@ -678,7 +661,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
Note : When `dict == NULL || dictSize < 8` no dictionary is used. Note : When `dict == NULL || dictSize < 8` no dictionary is used.
</p></pre><BR> </p></pre><BR>
<a name="Chapter18"></a><h2>Bulk processing dictionary API</h2><pre></pre> <a name="Chapter11"></a><h2>Bulk processing dictionary API</h2><pre></pre>
<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, <pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
int compressionLevel); int compressionLevel);
@@ -721,7 +704,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
Recommended when same dictionary is used multiple times. Recommended when same dictionary is used multiple times.
</p></pre><BR> </p></pre><BR>
<a name="Chapter19"></a><h2>Dictionary helper functions</h2><pre></pre> <a name="Chapter12"></a><h2>Dictionary helper functions</h2><pre></pre>
<pre><b>unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); <pre><b>unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
</b><p> Provides the dictID stored within dictionary. </b><p> Provides the dictID stored within dictionary.
@@ -747,7 +730,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code.
</p></pre><BR> </p></pre><BR>
<a name="Chapter20"></a><h2>Advanced dictionary and prefix API</h2><pre> <a name="Chapter13"></a><h2>Advanced dictionary and prefix API</h2><pre>
This API allows dictionaries to be used with ZSTD_compress2(), This API allows dictionaries to be used with ZSTD_compress2(),
ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and
only reset with the context is reset with ZSTD_reset_parameters or only reset with the context is reset with ZSTD_reset_parameters or
@@ -865,15 +848,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
Note that object memory usage can evolve (increase or decrease) over time. Note that object memory usage can evolve (increase or decrease) over time.
</p></pre><BR> </p></pre><BR>
<a name="Chapter21"></a><h2>ADVANCED AND EXPERIMENTAL FUNCTIONS</h2><pre> <a name="Chapter14"></a><h2>experimental API (static linking only)</h2><pre>
The definitions in the following section are considered experimental.
They are provided for advanced scenarios.
They should never be used with a dynamic library, as prototypes may change in the future.
Use them only in association with static linking.
<BR></pre>
<a name="Chapter22"></a><h2>experimental API (static linking only)</h2><pre>
The following symbols and constants The following symbols and constants
are not planned to join "stable API" status in the near future. are not planned to join "stable API" status in the near future.
They can still change in future versions. They can still change in future versions.
@@ -971,7 +946,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
ZSTD_lcm_uncompressed = 2, </b>/**< Always emit uncompressed literals. */<b> ZSTD_lcm_uncompressed = 2, </b>/**< Always emit uncompressed literals. */<b>
} ZSTD_literalCompressionMode_e; } ZSTD_literalCompressionMode_e;
</b></pre><BR> </b></pre><BR>
<a name="Chapter23"></a><h2>Frame size functions</h2><pre></pre> <a name="Chapter15"></a><h2>Frame size functions</h2><pre></pre>
<pre><b>unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); <pre><b>unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
</b><p> `src` should point to the start of a series of ZSTD encoded and/or skippable frames </b><p> `src` should point to the start of a series of ZSTD encoded and/or skippable frames
@@ -996,7 +971,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
however it does mean that all frame data must be present and valid. however it does mean that all frame data must be present and valid.
</p></pre><BR> </p></pre><BR>
<a name="Chapter24"></a><h2>ZSTD_decompressBound() :</h2><pre> `src` should point to the start of a series of ZSTD encoded and/or skippable frames <pre><b>unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
</b><p> `src` should point to the start of a series of ZSTD encoded and/or skippable frames
`srcSize` must be the _exact_ size of this series `srcSize` must be the _exact_ size of this series
(i.e. there should be a frame boundary at `src + srcSize`) (i.e. there should be a frame boundary at `src + srcSize`)
@return : - upper-bound for the decompressed size of all data in all successive frames @return : - upper-bound for the decompressed size of all data in all successive frames
@@ -1008,7 +984,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
upper-bound = # blocks * min(128 KB, Window_Size) upper-bound = # blocks * min(128 KB, Window_Size)
<BR></pre> </p></pre><BR>
<pre><b>size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); <pre><b>size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
</b><p> srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. </b><p> srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
@@ -1016,7 +992,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
or an error code (if srcSize is too small) or an error code (if srcSize is too small)
</p></pre><BR> </p></pre><BR>
<a name="Chapter25"></a><h2>Memory management</h2><pre></pre> <a name="Chapter16"></a><h2>Memory management</h2><pre></pre>
<pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel); <pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@@ -1096,7 +1072,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
</p></pre><BR> </p></pre><BR>
<a name="Chapter26"></a><h2>Advanced compression functions</h2><pre></pre> <a name="Chapter17"></a><h2>Advanced compression functions</h2><pre></pre>
<pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); <pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
</b><p> Create a digested dictionary for compression </b><p> Create a digested dictionary for compression
@@ -1241,7 +1217,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
</p></pre><BR> </p></pre><BR>
<a name="Chapter27"></a><h2>Advanced decompression functions</h2><pre></pre> <a name="Chapter18"></a><h2>Advanced decompression functions</h2><pre></pre>
<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size); <pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
</b><p> Tells if the content of `buffer` starts with a valid Frame Identifier. </b><p> Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -1303,7 +1279,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
</p></pre><BR> </p></pre><BR>
<a name="Chapter28"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API. <a name="Chapter19"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API.
Once Advanced API reaches "stable" status, Once Advanced API reaches "stable" status,
redundant functions will be deprecated, and then at some point removed. redundant functions will be deprecated, and then at some point removed.
<BR></pre> <BR></pre>
@@ -1432,14 +1408,14 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
*/ */
size_t ZSTD_resetDStream(ZSTD_DStream* zds); size_t ZSTD_resetDStream(ZSTD_DStream* zds);
</pre></b><BR> </pre></b><BR>
<a name="Chapter29"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> <a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
This is an advanced API, giving full control over buffer management, for users which need direct control over memory. This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
But it's also a complex one, with several restrictions, documented below. But it's also a complex one, with several restrictions, documented below.
Prefer normal streaming API for an easier experience. Prefer normal streaming API for an easier experience.
<BR></pre> <BR></pre>
<a name="Chapter30"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> <a name="Chapter21"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
A ZSTD_CCtx object is required to track streaming operations. A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations. ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -1475,7 +1451,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
</pre></b><BR> </pre></b><BR>
<a name="Chapter31"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> <a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
A ZSTD_DCtx object is required to track streaming operations. A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times. A ZSTD_DCtx object can be re-used multiple times.
@@ -1557,23 +1533,21 @@ typedef struct {
unsigned checksumFlag; unsigned checksumFlag;
} ZSTD_frameHeader; } ZSTD_frameHeader;
</pre></b><BR> </pre></b><BR>
<a name="Chapter32"></a><h2>ZSTD_getFrameHeader() :</h2><pre> decode Frame Header, or requires larger `srcSize`. <pre><b>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input */<b>
</b>/*! ZSTD_getFrameHeader_advanced() :<b>
* same as ZSTD_getFrameHeader(),
* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
</b><p> decode Frame Header, or requires larger `srcSize`.
@return : 0, `zfhPtr` is correctly filled, @return : 0, `zfhPtr` is correctly filled,
>0, `srcSize` is too small, value is wanted `srcSize` amount, >0, `srcSize` is too small, value is wanted `srcSize` amount,
or an error code, which can be tested using ZSTD_isError() or an error code, which can be tested using ZSTD_isError()
<BR></pre>
<pre><b>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input */<b>
</b></pre><BR>
<pre><b>size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
</b><p> same as ZSTD_getFrameHeader(),
with added capability to select a format (like ZSTD_f_zstd1_magicless)
</p></pre><BR> </p></pre><BR>
<pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; <pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
</b></pre><BR> </b></pre><BR>
<a name="Chapter33"></a><h2>Block level API</h2><pre></pre> <a name="Chapter23"></a><h2>Block level API</h2><pre></pre>
<pre><b></b><p> Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). <pre><b></b><p> Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
User will have to take in charge required information to regenerate data, such as compressed and content sizes. User will have to take in charge required information to regenerate data, such as compressed and content sizes.
lib/compress/zstd_opt.c
+5 -3
View File
@@ -506,9 +506,11 @@ static U32 ZSTD_insertBt1(
} } } }
*smallerPtr = *largerPtr = 0; *smallerPtr = *largerPtr = 0;
if (bestLength > 384) return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ { U32 positions = 0;
assert(matchEndIdx > current + 8); if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */
return matchEndIdx - (current + 8); assert(matchEndIdx > current + 8);
return MAX(positions, matchEndIdx - (current + 8));
}
} }
FORCE_INLINE_TEMPLATE FORCE_INLINE_TEMPLATE
lib/zstd.h
+35 -19
View File
@@ -82,16 +82,16 @@ ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library v
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */ ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */
/*************************************** /* *************************************
* Default constant * Default constant
***************************************/ ***************************************/
#ifndef ZSTD_CLEVEL_DEFAULT #ifndef ZSTD_CLEVEL_DEFAULT
# define ZSTD_CLEVEL_DEFAULT 3 # define ZSTD_CLEVEL_DEFAULT 3
#endif #endif
/*************************************** /* *************************************
* Constants * Constants
***************************************/ ***************************************/
/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ /* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
@@ -657,17 +657,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
ZSTD_inBuffer* input, ZSTD_inBuffer* input,
ZSTD_EndDirective endOp); ZSTD_EndDirective endOp);
ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
/******************************************************************************* /* These buffer sizes are softly recommended.
* This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
* ZSTD_compressStream2(). It is redundant, but is still fully supported. * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
* reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
*
* However, note that these recommendations are from the perspective of a C caller program.
* If the streaming interface is invoked from some other language,
* especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
* a major performance rule is to reduce crossing such interface to an absolute minimum.
* It's not rare that performance ends being spent more into the interface, rather than compression itself.
* In which cases, prefer using large buffers, as large as practical,
* for both input and output, to reduce the nb of roundtrips.
*/
ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
/* *****************************************************************************
* This following is a legacy streaming API.
* It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
* It is redundant, but remains fully supported.
* Advanced parameters and dictionary compression can only be used through the * Advanced parameters and dictionary compression can only be used through the
* new API. * new API.
******************************************************************************/ ******************************************************************************/
/** /*!
* Equivalent to: * Equivalent to:
* *
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -675,16 +691,16 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
*/ */
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
/** /*!
* Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
* NOTE: The return value is different. ZSTD_compressStream() returns a hint for * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
* the next read size (if non-zero and not an error). ZSTD_compressStream2() * the next read size (if non-zero and not an error). ZSTD_compressStream2()
* returns the number of bytes left to flush (if non-zero and not an error). * returns the minimum nb of bytes left to flush (if non-zero and not an error).
*/ */
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ /*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ /*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
@@ -969,7 +985,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
#endif /* ZSTD_H_235446 */ #endif /* ZSTD_H_235446 */
/**************************************************************************************** /* **************************************************************************************
* ADVANCED AND EXPERIMENTAL FUNCTIONS * ADVANCED AND EXPERIMENTAL FUNCTIONS
**************************************************************************************** ****************************************************************************************
* The definitions in the following section are considered experimental. * The definitions in the following section are considered experimental.
@@ -1162,7 +1178,7 @@ typedef enum {
* however it does mean that all frame data must be present and valid. */ * however it does mean that all frame data must be present and valid. */
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
/** ZSTD_decompressBound() : /*! ZSTD_decompressBound() :
* `src` should point to the start of a series of ZSTD encoded and/or skippable frames * `src` should point to the start of a series of ZSTD encoded and/or skippable frames
* `srcSize` must be the _exact_ size of this series * `srcSize` must be the _exact_ size of this series
* (i.e. there should be a frame boundary at `src + srcSize`) * (i.e. there should be a frame boundary at `src + srcSize`)
@@ -1843,7 +1859,7 @@ typedef struct {
unsigned checksumFlag; unsigned checksumFlag;
} ZSTD_frameHeader; } ZSTD_frameHeader;
/** ZSTD_getFrameHeader() : /*! ZSTD_getFrameHeader() :
* decode Frame Header, or requires larger `srcSize`. * decode Frame Header, or requires larger `srcSize`.
* @return : 0, `zfhPtr` is correctly filled, * @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount, * >0, `srcSize` is too small, value is wanted `srcSize` amount,
programs/README.md
+2 -2
View File
@@ -37,7 +37,7 @@ There are however other Makefile targets that create different variations of CLI
`.gz` support is automatically enabled when `zlib` library is detected at build time. `.gz` support is automatically enabled when `zlib` library is detected at build time.
It's possible to disable `.gz` support, by setting `HAVE_ZLIB=0`. It's possible to disable `.gz` support, by setting `HAVE_ZLIB=0`.
Example : `make zstd HAVE_ZLIB=0` Example : `make zstd HAVE_ZLIB=0`
It's also possible to force compilation with zlib support, `using HAVE_ZLIB=1`. It's also possible to force compilation with zlib support, using `HAVE_ZLIB=1`.
In which case, linking stage will fail if `zlib` library cannot be found. In which case, linking stage will fail if `zlib` library cannot be found.
This is useful to prevent silent feature disabling. This is useful to prevent silent feature disabling.
@@ -45,7 +45,7 @@ There are however other Makefile targets that create different variations of CLI
This is ordered through commands `--format=xz` and `--format=lzma` respectively. This is ordered through commands `--format=xz` and `--format=lzma` respectively.
Alternatively, symlinks named `xz`, `unxz`, `lzma`, or `unlzma` will mimic intended behavior. Alternatively, symlinks named `xz`, `unxz`, `lzma`, or `unlzma` will mimic intended behavior.
`.xz` and `.lzma` support is automatically enabled when `lzma` library is detected at build time. `.xz` and `.lzma` support is automatically enabled when `lzma` library is detected at build time.
It's possible to disable `.xz` and `.lzma` support, by setting `HAVE_LZMA=0` . It's possible to disable `.xz` and `.lzma` support, by setting `HAVE_LZMA=0`.
Example : `make zstd HAVE_LZMA=0` Example : `make zstd HAVE_LZMA=0`
It's also possible to force compilation with lzma support, using `HAVE_LZMA=1`. It's also possible to force compilation with lzma support, using `HAVE_LZMA=1`.
In which case, linking stage will fail if `lzma` library cannot be found. In which case, linking stage will fail if `lzma` library cannot be found.
programs/fileio.c
+11 -3
View File
@@ -1165,6 +1165,8 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName, const char* dstFileName, const char* srcFileName,
int compressionLevel) int compressionLevel)
{ {
UTIL_time_t const timeStart = UTIL_getTime();
clock_t const cpuStart = clock();
U64 readsize = 0; U64 readsize = 0;
U64 compressedfilesize = 0; U64 compressedfilesize = 0;
U64 const fileSize = UTIL_getFileSize(srcFileName); U64 const fileSize = UTIL_getFileSize(srcFileName);
@@ -1217,6 +1219,15 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs,
(unsigned long long)readsize, (unsigned long long) compressedfilesize, (unsigned long long)readsize, (unsigned long long) compressedfilesize,
dstFileName); dstFileName);
/* Elapsed Time and CPU Load */
{ clock_t const cpuEnd = clock();
double const cpuLoad_s = (double)(cpuEnd - cpuStart) / CLOCKS_PER_SEC;
U64 const timeLength_ns = UTIL_clockSpanNano(timeStart);
double const timeLength_s = (double)timeLength_ns / 1000000000;
double const cpuLoad_pct = (cpuLoad_s / timeLength_s) * 100;
DISPLAYLEVEL(4, "%-20s : Completed in %.2f sec (cpu load : %.0f%%)\n",
srcFileName, timeLength_s, cpuLoad_pct);
}
return 0; return 0;
} }
@@ -1339,15 +1350,12 @@ int FIO_compressFilename(FIO_prefs_t* const prefs,
const char* dictFileName, int compressionLevel, const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams) ZSTD_compressionParameters comprParams)
{ {
clock_t const start = clock();
U64 const fileSize = UTIL_getFileSize(srcFileName); U64 const fileSize = UTIL_getFileSize(srcFileName);
U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize; U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize;
cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, srcSize, comprParams); cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, srcSize, comprParams);
int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
FIO_freeCResources(ress); FIO_freeCResources(ress);
return result; return result;
tests/regression/results.csv
+40 -40
View File
@@ -11,10 +11,10 @@ silesia.tar, level 6, compress
silesia.tar, level 7, compress simple, 4606658 silesia.tar, level 7, compress simple, 4606658
silesia.tar, level 9, compress simple, 4554098 silesia.tar, level 9, compress simple, 4554098
silesia.tar, level 13, compress simple, 4491702 silesia.tar, level 13, compress simple, 4491702
silesia.tar, level 16, compress simple, 4381277 silesia.tar, level 16, compress simple, 4381284
silesia.tar, level 19, compress simple, 4281514 silesia.tar, level 19, compress simple, 4281512
silesia.tar, uncompressed literals, compress simple, 4875008 silesia.tar, uncompressed literals, compress simple, 4875008
silesia.tar, uncompressed literals optimal, compress simple, 4281514 silesia.tar, uncompressed literals optimal, compress simple, 4281512
silesia.tar, huffman literals, compress simple, 6186038 silesia.tar, huffman literals, compress simple, 6186038
silesia, level -5, compress cctx, 6737567 silesia, level -5, compress cctx, 6737567
silesia, level -3, compress cctx, 6444663 silesia, level -3, compress cctx, 6444663
@@ -28,7 +28,7 @@ silesia, level 6, compress
silesia, level 7, compress cctx, 4596234 silesia, level 7, compress cctx, 4596234
silesia, level 9, compress cctx, 4543862 silesia, level 9, compress cctx, 4543862
silesia, level 13, compress cctx, 4482073 silesia, level 13, compress cctx, 4482073
silesia, level 16, compress cctx, 4377391 silesia, level 16, compress cctx, 4377389
silesia, level 19, compress cctx, 4293262 silesia, level 19, compress cctx, 4293262
silesia, long distance mode, compress cctx, 4862377 silesia, long distance mode, compress cctx, 4862377
silesia, multithreaded, compress cctx, 4862377 silesia, multithreaded, compress cctx, 4862377
@@ -92,7 +92,7 @@ silesia, level 6, zstdcli,
silesia, level 7, zstdcli, 4596282 silesia, level 7, zstdcli, 4596282
silesia, level 9, zstdcli, 4543910 silesia, level 9, zstdcli, 4543910
silesia, level 13, zstdcli, 4482121 silesia, level 13, zstdcli, 4482121
silesia, level 16, zstdcli, 4377439 silesia, level 16, zstdcli, 4377437
silesia, level 19, zstdcli, 4293310 silesia, level 19, zstdcli, 4293310
silesia, long distance mode, zstdcli, 4853437 silesia, long distance mode, zstdcli, 4853437
silesia, multithreaded, zstdcli, 4862425 silesia, multithreaded, zstdcli, 4862425
@@ -102,7 +102,7 @@ silesia, small hash log, zstdcli,
silesia, small chain log, zstdcli, 4931141 silesia, small chain log, zstdcli, 4931141
silesia, explicit params, zstdcli, 4815380 silesia, explicit params, zstdcli, 4815380
silesia, uncompressed literals, zstdcli, 5155472 silesia, uncompressed literals, zstdcli, 5155472
silesia, uncompressed literals optimal, zstdcli, 4325475 silesia, uncompressed literals optimal, zstdcli, 4325482
silesia, huffman literals, zstdcli, 5331158 silesia, huffman literals, zstdcli, 5331158
silesia, multithreaded with advanced params, zstdcli, 5155472 silesia, multithreaded with advanced params, zstdcli, 5155472
silesia.tar, level -5, zstdcli, 6738906 silesia.tar, level -5, zstdcli, 6738906
@@ -117,8 +117,8 @@ silesia.tar, level 6, zstdcli,
silesia.tar, level 7, zstdcli, 4608342 silesia.tar, level 7, zstdcli, 4608342
silesia.tar, level 9, zstdcli, 4554700 silesia.tar, level 9, zstdcli, 4554700
silesia.tar, level 13, zstdcli, 4491706 silesia.tar, level 13, zstdcli, 4491706
silesia.tar, level 16, zstdcli, 4381281 silesia.tar, level 16, zstdcli, 4381288
silesia.tar, level 19, zstdcli, 4281518 silesia.tar, level 19, zstdcli, 4281516
silesia.tar, no source size, zstdcli, 4875132 silesia.tar, no source size, zstdcli, 4875132
silesia.tar, long distance mode, zstdcli, 4866975 silesia.tar, long distance mode, zstdcli, 4866975
silesia.tar, multithreaded, zstdcli, 4875136 silesia.tar, multithreaded, zstdcli, 4875136
@@ -128,7 +128,7 @@ silesia.tar, small hash log, zstdcli,
silesia.tar, small chain log, zstdcli, 4943259 silesia.tar, small chain log, zstdcli, 4943259
silesia.tar, explicit params, zstdcli, 4839202 silesia.tar, explicit params, zstdcli, 4839202
silesia.tar, uncompressed literals, zstdcli, 5158134 silesia.tar, uncompressed literals, zstdcli, 5158134
silesia.tar, uncompressed literals optimal, zstdcli, 4321098 silesia.tar, uncompressed literals optimal, zstdcli, 4321099
silesia.tar, huffman literals, zstdcli, 5347560 silesia.tar, huffman literals, zstdcli, 5347560
silesia.tar, multithreaded with advanced params, zstdcli, 5158134 silesia.tar, multithreaded with advanced params, zstdcli, 5158134
github, level -5, zstdcli, 207285 github, level -5, zstdcli, 207285
@@ -182,7 +182,7 @@ silesia, level 6, advanced
silesia, level 7, advanced one pass, 4596234 silesia, level 7, advanced one pass, 4596234
silesia, level 9, advanced one pass, 4543862 silesia, level 9, advanced one pass, 4543862
silesia, level 13, advanced one pass, 4482073 silesia, level 13, advanced one pass, 4482073
silesia, level 16, advanced one pass, 4377391 silesia, level 16, advanced one pass, 4377389
silesia, level 19, advanced one pass, 4293262 silesia, level 19, advanced one pass, 4293262
silesia, no source size, advanced one pass, 4862377 silesia, no source size, advanced one pass, 4862377
silesia, long distance mode, advanced one pass, 4853389 silesia, long distance mode, advanced one pass, 4853389
@@ -193,7 +193,7 @@ silesia, small hash log, advanced
silesia, small chain log, advanced one pass, 4931093 silesia, small chain log, advanced one pass, 4931093
silesia, explicit params, advanced one pass, 4815369 silesia, explicit params, advanced one pass, 4815369
silesia, uncompressed literals, advanced one pass, 5155424 silesia, uncompressed literals, advanced one pass, 5155424
silesia, uncompressed literals optimal, advanced one pass, 4325427 silesia, uncompressed literals optimal, advanced one pass, 4325434
silesia, huffman literals, advanced one pass, 5326210 silesia, huffman literals, advanced one pass, 5326210
silesia, multithreaded with advanced params, advanced one pass, 5155424 silesia, multithreaded with advanced params, advanced one pass, 5155424
silesia.tar, level -5, advanced one pass, 6738558 silesia.tar, level -5, advanced one pass, 6738558
@@ -208,8 +208,8 @@ silesia.tar, level 6, advanced
silesia.tar, level 7, advanced one pass, 4606658 silesia.tar, level 7, advanced one pass, 4606658
silesia.tar, level 9, advanced one pass, 4554098 silesia.tar, level 9, advanced one pass, 4554098
silesia.tar, level 13, advanced one pass, 4491702 silesia.tar, level 13, advanced one pass, 4491702
silesia.tar, level 16, advanced one pass, 4381277 silesia.tar, level 16, advanced one pass, 4381284
silesia.tar, level 19, advanced one pass, 4281514 silesia.tar, level 19, advanced one pass, 4281512
silesia.tar, no source size, advanced one pass, 4875008 silesia.tar, no source size, advanced one pass, 4875008
silesia.tar, long distance mode, advanced one pass, 4861218 silesia.tar, long distance mode, advanced one pass, 4861218
silesia.tar, multithreaded, advanced one pass, 4874631 silesia.tar, multithreaded, advanced one pass, 4874631
@@ -219,7 +219,7 @@ silesia.tar, small hash log, advanced
silesia.tar, small chain log, advanced one pass, 4943255 silesia.tar, small chain log, advanced one pass, 4943255
silesia.tar, explicit params, advanced one pass, 4829974 silesia.tar, explicit params, advanced one pass, 4829974
silesia.tar, uncompressed literals, advanced one pass, 5157992 silesia.tar, uncompressed literals, advanced one pass, 5157992
silesia.tar, uncompressed literals optimal, advanced one pass, 4321094 silesia.tar, uncompressed literals optimal, advanced one pass, 4321095
silesia.tar, huffman literals, advanced one pass, 5347283 silesia.tar, huffman literals, advanced one pass, 5347283
silesia.tar, multithreaded with advanced params, advanced one pass, 5158545 silesia.tar, multithreaded with advanced params, advanced one pass, 5158545
github, level -5, advanced one pass, 205285 github, level -5, advanced one pass, 205285
@@ -274,7 +274,7 @@ silesia, level 6, advanced
silesia, level 7, advanced one pass small out, 4596234 silesia, level 7, advanced one pass small out, 4596234
silesia, level 9, advanced one pass small out, 4543862 silesia, level 9, advanced one pass small out, 4543862
silesia, level 13, advanced one pass small out, 4482073 silesia, level 13, advanced one pass small out, 4482073
silesia, level 16, advanced one pass small out, 4377391 silesia, level 16, advanced one pass small out, 4377389
silesia, level 19, advanced one pass small out, 4293262 silesia, level 19, advanced one pass small out, 4293262
silesia, no source size, advanced one pass small out, 4862377 silesia, no source size, advanced one pass small out, 4862377
silesia, long distance mode, advanced one pass small out, 4853389 silesia, long distance mode, advanced one pass small out, 4853389
@@ -285,7 +285,7 @@ silesia, small hash log, advanced
silesia, small chain log, advanced one pass small out, 4931093 silesia, small chain log, advanced one pass small out, 4931093
silesia, explicit params, advanced one pass small out, 4815369 silesia, explicit params, advanced one pass small out, 4815369
silesia, uncompressed literals, advanced one pass small out, 5155424 silesia, uncompressed literals, advanced one pass small out, 5155424
silesia, uncompressed literals optimal, advanced one pass small out, 4325427 silesia, uncompressed literals optimal, advanced one pass small out, 4325434
silesia, huffman literals, advanced one pass small out, 5326210 silesia, huffman literals, advanced one pass small out, 5326210
silesia, multithreaded with advanced params, advanced one pass small out, 5155424 silesia, multithreaded with advanced params, advanced one pass small out, 5155424
silesia.tar, level -5, advanced one pass small out, 6738558 silesia.tar, level -5, advanced one pass small out, 6738558
@@ -300,8 +300,8 @@ silesia.tar, level 6, advanced
silesia.tar, level 7, advanced one pass small out, 4606658 silesia.tar, level 7, advanced one pass small out, 4606658
silesia.tar, level 9, advanced one pass small out, 4554098 silesia.tar, level 9, advanced one pass small out, 4554098
silesia.tar, level 13, advanced one pass small out, 4491702 silesia.tar, level 13, advanced one pass small out, 4491702
silesia.tar, level 16, advanced one pass small out, 4381277 silesia.tar, level 16, advanced one pass small out, 4381284
silesia.tar, level 19, advanced one pass small out, 4281514 silesia.tar, level 19, advanced one pass small out, 4281512
silesia.tar, no source size, advanced one pass small out, 4875008 silesia.tar, no source size, advanced one pass small out, 4875008
silesia.tar, long distance mode, advanced one pass small out, 4861218 silesia.tar, long distance mode, advanced one pass small out, 4861218
silesia.tar, multithreaded, advanced one pass small out, 4874631 silesia.tar, multithreaded, advanced one pass small out, 4874631
@@ -311,7 +311,7 @@ silesia.tar, small hash log, advanced
silesia.tar, small chain log, advanced one pass small out, 4943255 silesia.tar, small chain log, advanced one pass small out, 4943255
silesia.tar, explicit params, advanced one pass small out, 4829974 silesia.tar, explicit params, advanced one pass small out, 4829974
silesia.tar, uncompressed literals, advanced one pass small out, 5157992 silesia.tar, uncompressed literals, advanced one pass small out, 5157992
silesia.tar, uncompressed literals optimal, advanced one pass small out, 4321094 silesia.tar, uncompressed literals optimal, advanced one pass small out, 4321095
silesia.tar, huffman literals, advanced one pass small out, 5347283 silesia.tar, huffman literals, advanced one pass small out, 5347283
silesia.tar, multithreaded with advanced params, advanced one pass small out, 5158545 silesia.tar, multithreaded with advanced params, advanced one pass small out, 5158545
github, level -5, advanced one pass small out, 205285 github, level -5, advanced one pass small out, 205285
@@ -366,7 +366,7 @@ silesia, level 6, advanced
silesia, level 7, advanced streaming, 4596234 silesia, level 7, advanced streaming, 4596234
silesia, level 9, advanced streaming, 4543862 silesia, level 9, advanced streaming, 4543862
silesia, level 13, advanced streaming, 4482073 silesia, level 13, advanced streaming, 4482073
silesia, level 16, advanced streaming, 4377391 silesia, level 16, advanced streaming, 4377389
silesia, level 19, advanced streaming, 4293262 silesia, level 19, advanced streaming, 4293262
silesia, no source size, advanced streaming, 4862341 silesia, no source size, advanced streaming, 4862341
silesia, long distance mode, advanced streaming, 4853389 silesia, long distance mode, advanced streaming, 4853389
@@ -377,7 +377,7 @@ silesia, small hash log, advanced
silesia, small chain log, advanced streaming, 4931093 silesia, small chain log, advanced streaming, 4931093
silesia, explicit params, advanced streaming, 4815380 silesia, explicit params, advanced streaming, 4815380
silesia, uncompressed literals, advanced streaming, 5155424 silesia, uncompressed literals, advanced streaming, 5155424
silesia, uncompressed literals optimal, advanced streaming, 4325427 silesia, uncompressed literals optimal, advanced streaming, 4325434
silesia, huffman literals, advanced streaming, 5331110 silesia, huffman literals, advanced streaming, 5331110
silesia, multithreaded with advanced params, advanced streaming, 5155424 silesia, multithreaded with advanced params, advanced streaming, 5155424
silesia.tar, level -5, advanced streaming, 6982738 silesia.tar, level -5, advanced streaming, 6982738
@@ -392,8 +392,8 @@ silesia.tar, level 6, advanced
silesia.tar, level 7, advanced streaming, 4606658 silesia.tar, level 7, advanced streaming, 4606658
silesia.tar, level 9, advanced streaming, 4554105 silesia.tar, level 9, advanced streaming, 4554105
silesia.tar, level 13, advanced streaming, 4491703 silesia.tar, level 13, advanced streaming, 4491703
silesia.tar, level 16, advanced streaming, 4381277 silesia.tar, level 16, advanced streaming, 4381284
silesia.tar, level 19, advanced streaming, 4281514 silesia.tar, level 19, advanced streaming, 4281512
silesia.tar, no source size, advanced streaming, 4875006 silesia.tar, no source size, advanced streaming, 4875006
silesia.tar, long distance mode, advanced streaming, 4861218 silesia.tar, long distance mode, advanced streaming, 4861218
silesia.tar, multithreaded, advanced streaming, 4875132 silesia.tar, multithreaded, advanced streaming, 4875132
@@ -403,7 +403,7 @@ silesia.tar, small hash log, advanced
silesia.tar, small chain log, advanced streaming, 4943260 silesia.tar, small chain log, advanced streaming, 4943260
silesia.tar, explicit params, advanced streaming, 4830002 silesia.tar, explicit params, advanced streaming, 4830002
silesia.tar, uncompressed literals, advanced streaming, 5157995 silesia.tar, uncompressed literals, advanced streaming, 5157995
silesia.tar, uncompressed literals optimal, advanced streaming, 4321094 silesia.tar, uncompressed literals optimal, advanced streaming, 4321095
silesia.tar, huffman literals, advanced streaming, 5352306 silesia.tar, huffman literals, advanced streaming, 5352306
silesia.tar, multithreaded with advanced params, advanced streaming, 5158130 silesia.tar, multithreaded with advanced params, advanced streaming, 5158130
github, level -5, advanced streaming, 205285 github, level -5, advanced streaming, 205285
@@ -458,7 +458,7 @@ silesia, level 6, old stre
silesia, level 7, old streaming, 4596234 silesia, level 7, old streaming, 4596234
silesia, level 9, old streaming, 4543862 silesia, level 9, old streaming, 4543862
silesia, level 13, old streaming, 4482073 silesia, level 13, old streaming, 4482073
silesia, level 16, old streaming, 4377391 silesia, level 16, old streaming, 4377389
silesia, level 19, old streaming, 4293262 silesia, level 19, old streaming, 4293262
silesia, no source size, old streaming, 4862341 silesia, no source size, old streaming, 4862341
silesia, long distance mode, old streaming, 12000408 silesia, long distance mode, old streaming, 12000408
@@ -484,8 +484,8 @@ silesia.tar, level 6, old stre
silesia.tar, level 7, old streaming, 4606658 silesia.tar, level 7, old streaming, 4606658
silesia.tar, level 9, old streaming, 4554105 silesia.tar, level 9, old streaming, 4554105
silesia.tar, level 13, old streaming, 4491703 silesia.tar, level 13, old streaming, 4491703
silesia.tar, level 16, old streaming, 4381277 silesia.tar, level 16, old streaming, 4381284
silesia.tar, level 19, old streaming, 4281514 silesia.tar, level 19, old streaming, 4281512
silesia.tar, no source size, old streaming, 4875006 silesia.tar, no source size, old streaming, 4875006
silesia.tar, long distance mode, old streaming, 12022046 silesia.tar, long distance mode, old streaming, 12022046
silesia.tar, multithreaded, old streaming, 12022046 silesia.tar, multithreaded, old streaming, 12022046
@@ -495,7 +495,7 @@ silesia.tar, small hash log, old stre
silesia.tar, small chain log, old streaming, 12022046 silesia.tar, small chain log, old streaming, 12022046
silesia.tar, explicit params, old streaming, 12022046 silesia.tar, explicit params, old streaming, 12022046
silesia.tar, uncompressed literals, old streaming, 4875010 silesia.tar, uncompressed literals, old streaming, 4875010
silesia.tar, uncompressed literals optimal, old streaming, 4281514 silesia.tar, uncompressed literals optimal, old streaming, 4281512
silesia.tar, huffman literals, old streaming, 6190789 silesia.tar, huffman literals, old streaming, 6190789
silesia.tar, multithreaded with advanced params, old streaming, 12022046 silesia.tar, multithreaded with advanced params, old streaming, 12022046
github, level -5, old streaming, 205285 github, level -5, old streaming, 205285
@@ -550,7 +550,7 @@ silesia, level 6, old stre
silesia, level 7, old streaming advanced, 4596234 silesia, level 7, old streaming advanced, 4596234
silesia, level 9, old streaming advanced, 4543862 silesia, level 9, old streaming advanced, 4543862
silesia, level 13, old streaming advanced, 4482073 silesia, level 13, old streaming advanced, 4482073
silesia, level 16, old streaming advanced, 4377391 silesia, level 16, old streaming advanced, 4377389
silesia, level 19, old streaming advanced, 4293262 silesia, level 19, old streaming advanced, 4293262
silesia, no source size, old streaming advanced, 4862341 silesia, no source size, old streaming advanced, 4862341
silesia, long distance mode, old streaming advanced, 12000408 silesia, long distance mode, old streaming advanced, 12000408
@@ -576,8 +576,8 @@ silesia.tar, level 6, old stre
silesia.tar, level 7, old streaming advanced, 4606658 silesia.tar, level 7, old streaming advanced, 4606658
silesia.tar, level 9, old streaming advanced, 4554105 silesia.tar, level 9, old streaming advanced, 4554105
silesia.tar, level 13, old streaming advanced, 4491703 silesia.tar, level 13, old streaming advanced, 4491703
silesia.tar, level 16, old streaming advanced, 4381277 silesia.tar, level 16, old streaming advanced, 4381284
silesia.tar, level 19, old streaming advanced, 4281514 silesia.tar, level 19, old streaming advanced, 4281512
silesia.tar, no source size, old streaming advanced, 4875006 silesia.tar, no source size, old streaming advanced, 4875006
silesia.tar, long distance mode, old streaming advanced, 12022046 silesia.tar, long distance mode, old streaming advanced, 12022046
silesia.tar, multithreaded, old streaming advanced, 12022046 silesia.tar, multithreaded, old streaming advanced, 12022046
@@ -587,7 +587,7 @@ silesia.tar, small hash log, old stre
silesia.tar, small chain log, old streaming advanced, 12022046 silesia.tar, small chain log, old streaming advanced, 12022046
silesia.tar, explicit params, old streaming advanced, 12022046 silesia.tar, explicit params, old streaming advanced, 12022046
silesia.tar, uncompressed literals, old streaming advanced, 4875010 silesia.tar, uncompressed literals, old streaming advanced, 4875010
silesia.tar, uncompressed literals optimal, old streaming advanced, 4281514 silesia.tar, uncompressed literals optimal, old streaming advanced, 4281512
silesia.tar, huffman literals, old streaming advanced, 6190789 silesia.tar, huffman literals, old streaming advanced, 6190789
silesia.tar, multithreaded with advanced params, old streaming advanced, 12022046 silesia.tar, multithreaded with advanced params, old streaming advanced, 12022046
github, level -5, old streaming advanced, 205285 github, level -5, old streaming advanced, 205285
@@ -642,7 +642,7 @@ silesia, level 6, old stre
silesia, level 7, old streaming cdcit, 4596234 silesia, level 7, old streaming cdcit, 4596234
silesia, level 9, old streaming cdcit, 4543862 silesia, level 9, old streaming cdcit, 4543862
silesia, level 13, old streaming cdcit, 4482073 silesia, level 13, old streaming cdcit, 4482073
silesia, level 16, old streaming cdcit, 4377391 silesia, level 16, old streaming cdcit, 4377389
silesia, level 19, old streaming cdcit, 4293262 silesia, level 19, old streaming cdcit, 4293262
silesia, no source size, old streaming cdcit, 4862341 silesia, no source size, old streaming cdcit, 4862341
silesia, long distance mode, old streaming cdcit, 12000408 silesia, long distance mode, old streaming cdcit, 12000408
@@ -668,8 +668,8 @@ silesia.tar, level 6, old stre
silesia.tar, level 7, old streaming cdcit, 4606658 silesia.tar, level 7, old streaming cdcit, 4606658
silesia.tar, level 9, old streaming cdcit, 4554105 silesia.tar, level 9, old streaming cdcit, 4554105
silesia.tar, level 13, old streaming cdcit, 4491703 silesia.tar, level 13, old streaming cdcit, 4491703
silesia.tar, level 16, old streaming cdcit, 4381277 silesia.tar, level 16, old streaming cdcit, 4381284
silesia.tar, level 19, old streaming cdcit, 4281514 silesia.tar, level 19, old streaming cdcit, 4281512
silesia.tar, no source size, old streaming cdcit, 4875006 silesia.tar, no source size, old streaming cdcit, 4875006
silesia.tar, long distance mode, old streaming cdcit, 12022046 silesia.tar, long distance mode, old streaming cdcit, 12022046
silesia.tar, multithreaded, old streaming cdcit, 12022046 silesia.tar, multithreaded, old streaming cdcit, 12022046
@@ -679,7 +679,7 @@ silesia.tar, small hash log, old stre
silesia.tar, small chain log, old streaming cdcit, 12022046 silesia.tar, small chain log, old streaming cdcit, 12022046
silesia.tar, explicit params, old streaming cdcit, 12022046 silesia.tar, explicit params, old streaming cdcit, 12022046
silesia.tar, uncompressed literals, old streaming cdcit, 4875010 silesia.tar, uncompressed literals, old streaming cdcit, 4875010
silesia.tar, uncompressed literals optimal, old streaming cdcit, 4281514 silesia.tar, uncompressed literals optimal, old streaming cdcit, 4281512
silesia.tar, huffman literals, old streaming cdcit, 6190789 silesia.tar, huffman literals, old streaming cdcit, 6190789
silesia.tar, multithreaded with advanced params, old streaming cdcit, 12022046 silesia.tar, multithreaded with advanced params, old streaming cdcit, 12022046
github, level -5, old streaming cdcit, 205285 github, level -5, old streaming cdcit, 205285
@@ -734,7 +734,7 @@ silesia, level 6, old stre
silesia, level 7, old streaming advanced cdict, 4596234 silesia, level 7, old streaming advanced cdict, 4596234
silesia, level 9, old streaming advanced cdict, 4543862 silesia, level 9, old streaming advanced cdict, 4543862
silesia, level 13, old streaming advanced cdict, 4482073 silesia, level 13, old streaming advanced cdict, 4482073
silesia, level 16, old streaming advanced cdict, 4377391 silesia, level 16, old streaming advanced cdict, 4377389
silesia, level 19, old streaming advanced cdict, 4293262 silesia, level 19, old streaming advanced cdict, 4293262
silesia, no source size, old streaming advanced cdict, 4862341 silesia, no source size, old streaming advanced cdict, 4862341
silesia, long distance mode, old streaming advanced cdict, 12000408 silesia, long distance mode, old streaming advanced cdict, 12000408
@@ -760,8 +760,8 @@ silesia.tar, level 6, old stre
silesia.tar, level 7, old streaming advanced cdict, 4606658 silesia.tar, level 7, old streaming advanced cdict, 4606658
silesia.tar, level 9, old streaming advanced cdict, 4554105 silesia.tar, level 9, old streaming advanced cdict, 4554105
silesia.tar, level 13, old streaming advanced cdict, 4491703 silesia.tar, level 13, old streaming advanced cdict, 4491703
silesia.tar, level 16, old streaming advanced cdict, 4381277 silesia.tar, level 16, old streaming advanced cdict, 4381284
silesia.tar, level 19, old streaming advanced cdict, 4281514 silesia.tar, level 19, old streaming advanced cdict, 4281512
silesia.tar, no source size, old streaming advanced cdict, 4875006 silesia.tar, no source size, old streaming advanced cdict, 4875006
silesia.tar, long distance mode, old streaming advanced cdict, 12022046 silesia.tar, long distance mode, old streaming advanced cdict, 12022046
silesia.tar, multithreaded, old streaming advanced cdict, 12022046 silesia.tar, multithreaded, old streaming advanced cdict, 12022046
@@ -771,7 +771,7 @@ silesia.tar, small hash log, old stre
silesia.tar, small chain log, old streaming advanced cdict, 12022046 silesia.tar, small chain log, old streaming advanced cdict, 12022046
silesia.tar, explicit params, old streaming advanced cdict, 12022046 silesia.tar, explicit params, old streaming advanced cdict, 12022046
silesia.tar, uncompressed literals, old streaming advanced cdict, 4875010 silesia.tar, uncompressed literals, old streaming advanced cdict, 4875010
silesia.tar, uncompressed literals optimal, old streaming advanced cdict, 4281514 silesia.tar, uncompressed literals optimal, old streaming advanced cdict, 4281512
silesia.tar, huffman literals, old streaming advanced cdict, 6190789 silesia.tar, huffman literals, old streaming advanced cdict, 6190789
silesia.tar, multithreaded with advanced params, old streaming advanced cdict, 12022046 silesia.tar, multithreaded with advanced params, old streaming advanced cdict, 12022046
github, level -5, old streaming advanced cdict, 205285 github, level -5, old streaming advanced cdict, 205285
1 Data Config Method Total compressed size
11 silesia.tar level 7 compress simple 4606658
12 silesia.tar level 9 compress simple 4554098
13 silesia.tar level 13 compress simple 4491702
14 silesia.tar level 16 compress simple 4381277 4381284
15 silesia.tar level 19 compress simple 4281514 4281512
16 silesia.tar uncompressed literals compress simple 4875008
17 silesia.tar uncompressed literals optimal compress simple 4281514 4281512
18 silesia.tar huffman literals compress simple 6186038
19 silesia level -5 compress cctx 6737567
20 silesia level -3 compress cctx 6444663
28 silesia level 7 compress cctx 4596234
29 silesia level 9 compress cctx 4543862
30 silesia level 13 compress cctx 4482073
31 silesia level 16 compress cctx 4377391 4377389
32 silesia level 19 compress cctx 4293262
33 silesia long distance mode compress cctx 4862377
34 silesia multithreaded compress cctx 4862377
92 silesia level 7 zstdcli 4596282
93 silesia level 9 zstdcli 4543910
94 silesia level 13 zstdcli 4482121
95 silesia level 16 zstdcli 4377439 4377437
96 silesia level 19 zstdcli 4293310
97 silesia long distance mode zstdcli 4853437
98 silesia multithreaded zstdcli 4862425
102 silesia small chain log zstdcli 4931141
103 silesia explicit params zstdcli 4815380
104 silesia uncompressed literals zstdcli 5155472
105 silesia uncompressed literals optimal zstdcli 4325475 4325482
106 silesia huffman literals zstdcli 5331158
107 silesia multithreaded with advanced params zstdcli 5155472
108 silesia.tar level -5 zstdcli 6738906
117 silesia.tar level 7 zstdcli 4608342
118 silesia.tar level 9 zstdcli 4554700
119 silesia.tar level 13 zstdcli 4491706
120 silesia.tar level 16 zstdcli 4381281 4381288
121 silesia.tar level 19 zstdcli 4281518 4281516
122 silesia.tar no source size zstdcli 4875132
123 silesia.tar long distance mode zstdcli 4866975
124 silesia.tar multithreaded zstdcli 4875136
128 silesia.tar small chain log zstdcli 4943259
129 silesia.tar explicit params zstdcli 4839202
130 silesia.tar uncompressed literals zstdcli 5158134
131 silesia.tar uncompressed literals optimal zstdcli 4321098 4321099
132 silesia.tar huffman literals zstdcli 5347560
133 silesia.tar multithreaded with advanced params zstdcli 5158134
134 github level -5 zstdcli 207285
182 silesia level 7 advanced one pass 4596234
183 silesia level 9 advanced one pass 4543862
184 silesia level 13 advanced one pass 4482073
185 silesia level 16 advanced one pass 4377391 4377389
186 silesia level 19 advanced one pass 4293262
187 silesia no source size advanced one pass 4862377
188 silesia long distance mode advanced one pass 4853389
193 silesia small chain log advanced one pass 4931093
194 silesia explicit params advanced one pass 4815369
195 silesia uncompressed literals advanced one pass 5155424
196 silesia uncompressed literals optimal advanced one pass 4325427 4325434
197 silesia huffman literals advanced one pass 5326210
198 silesia multithreaded with advanced params advanced one pass 5155424
199 silesia.tar level -5 advanced one pass 6738558
208 silesia.tar level 7 advanced one pass 4606658
209 silesia.tar level 9 advanced one pass 4554098
210 silesia.tar level 13 advanced one pass 4491702
211 silesia.tar level 16 advanced one pass 4381277 4381284
212 silesia.tar level 19 advanced one pass 4281514 4281512
213 silesia.tar no source size advanced one pass 4875008
214 silesia.tar long distance mode advanced one pass 4861218
215 silesia.tar multithreaded advanced one pass 4874631
219 silesia.tar small chain log advanced one pass 4943255
220 silesia.tar explicit params advanced one pass 4829974
221 silesia.tar uncompressed literals advanced one pass 5157992
222 silesia.tar uncompressed literals optimal advanced one pass 4321094 4321095
223 silesia.tar huffman literals advanced one pass 5347283
224 silesia.tar multithreaded with advanced params advanced one pass 5158545
225 github level -5 advanced one pass 205285
274 silesia level 7 advanced one pass small out 4596234
275 silesia level 9 advanced one pass small out 4543862
276 silesia level 13 advanced one pass small out 4482073
277 silesia level 16 advanced one pass small out 4377391 4377389
278 silesia level 19 advanced one pass small out 4293262
279 silesia no source size advanced one pass small out 4862377
280 silesia long distance mode advanced one pass small out 4853389
285 silesia small chain log advanced one pass small out 4931093
286 silesia explicit params advanced one pass small out 4815369
287 silesia uncompressed literals advanced one pass small out 5155424
288 silesia uncompressed literals optimal advanced one pass small out 4325427 4325434
289 silesia huffman literals advanced one pass small out 5326210
290 silesia multithreaded with advanced params advanced one pass small out 5155424
291 silesia.tar level -5 advanced one pass small out 6738558
300 silesia.tar level 7 advanced one pass small out 4606658
301 silesia.tar level 9 advanced one pass small out 4554098
302 silesia.tar level 13 advanced one pass small out 4491702
303 silesia.tar level 16 advanced one pass small out 4381277 4381284
304 silesia.tar level 19 advanced one pass small out 4281514 4281512
305 silesia.tar no source size advanced one pass small out 4875008
306 silesia.tar long distance mode advanced one pass small out 4861218
307 silesia.tar multithreaded advanced one pass small out 4874631
311 silesia.tar small chain log advanced one pass small out 4943255
312 silesia.tar explicit params advanced one pass small out 4829974
313 silesia.tar uncompressed literals advanced one pass small out 5157992
314 silesia.tar uncompressed literals optimal advanced one pass small out 4321094 4321095
315 silesia.tar huffman literals advanced one pass small out 5347283
316 silesia.tar multithreaded with advanced params advanced one pass small out 5158545
317 github level -5 advanced one pass small out 205285
366 silesia level 7 advanced streaming 4596234
367 silesia level 9 advanced streaming 4543862
368 silesia level 13 advanced streaming 4482073
369 silesia level 16 advanced streaming 4377391 4377389
370 silesia level 19 advanced streaming 4293262
371 silesia no source size advanced streaming 4862341
372 silesia long distance mode advanced streaming 4853389
377 silesia small chain log advanced streaming 4931093
378 silesia explicit params advanced streaming 4815380
379 silesia uncompressed literals advanced streaming 5155424
380 silesia uncompressed literals optimal advanced streaming 4325427 4325434
381 silesia huffman literals advanced streaming 5331110
382 silesia multithreaded with advanced params advanced streaming 5155424
383 silesia.tar level -5 advanced streaming 6982738
392 silesia.tar level 7 advanced streaming 4606658
393 silesia.tar level 9 advanced streaming 4554105
394 silesia.tar level 13 advanced streaming 4491703
395 silesia.tar level 16 advanced streaming 4381277 4381284
396 silesia.tar level 19 advanced streaming 4281514 4281512
397 silesia.tar no source size advanced streaming 4875006
398 silesia.tar long distance mode advanced streaming 4861218
399 silesia.tar multithreaded advanced streaming 4875132
403 silesia.tar small chain log advanced streaming 4943260
404 silesia.tar explicit params advanced streaming 4830002
405 silesia.tar uncompressed literals advanced streaming 5157995
406 silesia.tar uncompressed literals optimal advanced streaming 4321094 4321095
407 silesia.tar huffman literals advanced streaming 5352306
408 silesia.tar multithreaded with advanced params advanced streaming 5158130
409 github level -5 advanced streaming 205285
458 silesia level 7 old streaming 4596234
459 silesia level 9 old streaming 4543862
460 silesia level 13 old streaming 4482073
461 silesia level 16 old streaming 4377391 4377389
462 silesia level 19 old streaming 4293262
463 silesia no source size old streaming 4862341
464 silesia long distance mode old streaming 12000408
484 silesia.tar level 7 old streaming 4606658
485 silesia.tar level 9 old streaming 4554105
486 silesia.tar level 13 old streaming 4491703
487 silesia.tar level 16 old streaming 4381277 4381284
488 silesia.tar level 19 old streaming 4281514 4281512
489 silesia.tar no source size old streaming 4875006
490 silesia.tar long distance mode old streaming 12022046
491 silesia.tar multithreaded old streaming 12022046
495 silesia.tar small chain log old streaming 12022046
496 silesia.tar explicit params old streaming 12022046
497 silesia.tar uncompressed literals old streaming 4875010
498 silesia.tar uncompressed literals optimal old streaming 4281514 4281512
499 silesia.tar huffman literals old streaming 6190789
500 silesia.tar multithreaded with advanced params old streaming 12022046
501 github level -5 old streaming 205285
550 silesia level 7 old streaming advanced 4596234
551 silesia level 9 old streaming advanced 4543862
552 silesia level 13 old streaming advanced 4482073
553 silesia level 16 old streaming advanced 4377391 4377389
554 silesia level 19 old streaming advanced 4293262
555 silesia no source size old streaming advanced 4862341
556 silesia long distance mode old streaming advanced 12000408
576 silesia.tar level 7 old streaming advanced 4606658
577 silesia.tar level 9 old streaming advanced 4554105
578 silesia.tar level 13 old streaming advanced 4491703
579 silesia.tar level 16 old streaming advanced 4381277 4381284
580 silesia.tar level 19 old streaming advanced 4281514 4281512
581 silesia.tar no source size old streaming advanced 4875006
582 silesia.tar long distance mode old streaming advanced 12022046
583 silesia.tar multithreaded old streaming advanced 12022046
587 silesia.tar small chain log old streaming advanced 12022046
588 silesia.tar explicit params old streaming advanced 12022046
589 silesia.tar uncompressed literals old streaming advanced 4875010
590 silesia.tar uncompressed literals optimal old streaming advanced 4281514 4281512
591 silesia.tar huffman literals old streaming advanced 6190789
592 silesia.tar multithreaded with advanced params old streaming advanced 12022046
593 github level -5 old streaming advanced 205285
642 silesia level 7 old streaming cdcit 4596234
643 silesia level 9 old streaming cdcit 4543862
644 silesia level 13 old streaming cdcit 4482073
645 silesia level 16 old streaming cdcit 4377391 4377389
646 silesia level 19 old streaming cdcit 4293262
647 silesia no source size old streaming cdcit 4862341
648 silesia long distance mode old streaming cdcit 12000408
668 silesia.tar level 7 old streaming cdcit 4606658
669 silesia.tar level 9 old streaming cdcit 4554105
670 silesia.tar level 13 old streaming cdcit 4491703
671 silesia.tar level 16 old streaming cdcit 4381277 4381284
672 silesia.tar level 19 old streaming cdcit 4281514 4281512
673 silesia.tar no source size old streaming cdcit 4875006
674 silesia.tar long distance mode old streaming cdcit 12022046
675 silesia.tar multithreaded old streaming cdcit 12022046
679 silesia.tar small chain log old streaming cdcit 12022046
680 silesia.tar explicit params old streaming cdcit 12022046
681 silesia.tar uncompressed literals old streaming cdcit 4875010
682 silesia.tar uncompressed literals optimal old streaming cdcit 4281514 4281512
683 silesia.tar huffman literals old streaming cdcit 6190789
684 silesia.tar multithreaded with advanced params old streaming cdcit 12022046
685 github level -5 old streaming cdcit 205285
734 silesia level 7 old streaming advanced cdict 4596234
735 silesia level 9 old streaming advanced cdict 4543862
736 silesia level 13 old streaming advanced cdict 4482073
737 silesia level 16 old streaming advanced cdict 4377391 4377389
738 silesia level 19 old streaming advanced cdict 4293262
739 silesia no source size old streaming advanced cdict 4862341
740 silesia long distance mode old streaming advanced cdict 12000408
760 silesia.tar level 7 old streaming advanced cdict 4606658
761 silesia.tar level 9 old streaming advanced cdict 4554105
762 silesia.tar level 13 old streaming advanced cdict 4491703
763 silesia.tar level 16 old streaming advanced cdict 4381277 4381284
764 silesia.tar level 19 old streaming advanced cdict 4281514 4281512
765 silesia.tar no source size old streaming advanced cdict 4875006
766 silesia.tar long distance mode old streaming advanced cdict 12022046
767 silesia.tar multithreaded old streaming advanced cdict 12022046
771 silesia.tar small chain log old streaming advanced cdict 12022046
772 silesia.tar explicit params old streaming advanced cdict 12022046
773 silesia.tar uncompressed literals old streaming advanced cdict 4875010
774 silesia.tar uncompressed literals optimal old streaming advanced cdict 4281514 4281512
775 silesia.tar huffman literals old streaming advanced cdict 6190789
776 silesia.tar multithreaded with advanced params old streaming advanced cdict 12022046
777 github level -5 old streaming advanced cdict 205285