ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added advanced compression functions

Browse Source
This commit is contained in:
Yann Collet
2015-10-21 09:07:25 +01:00
parent 1c00dc3e24
commit 7010c27a13
4 changed files with 93 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/huff0.c
+1 -1
View File
@@ -1685,7 +1685,7 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS
if (dstSize == 0) return ERROR(dstSize_tooSmall); if (dstSize == 0) return ERROR(dstSize_tooSmall);
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
if (cSrcSize == 1) { memset(dst, ((const BYTE*)cSrc)[0], dstSize); return dstSize; } /* RLE */ if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
/* decoder timing evaluation */ /* decoder timing evaluation */
Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
lib/zstd.c
+65 -52
View File
@@ -30,25 +30,39 @@
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/ */
/**************************************************************** /* ***************************************************************
* Tuning parameters * Tuning parameters
****************************************************************/ *****************************************************************/
/**MEMORY_USAGE : /*!
* MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio * Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect */ * Reduced memory usage can improve speed, due to cache effect
*/
#define ZSTD_MEMORY_USAGE 17 #define ZSTD_MEMORY_USAGE 17
/*!
* HEAPMODE :
* Select how default compression functions will allocate memory for their hash table,
* in memory stack (0, fastest), or in memory heap (1, requires malloc())
* Note that compression context is fairly large, as a consequence heap memory is recommended.
*/
#ifndef ZSTD_HEAPMODE
# define ZSTD_HEAPMODE 1
#endif /* ZSTD_HEAPMODE */
/*!
* LEGACY_SUPPORT :
* decompressor can decode older formats (starting from Zstd 0.1+)
*/
#ifndef ZSTD_LEGACY_SUPPORT #ifndef ZSTD_LEGACY_SUPPORT
/**LEGACY_SUPPORT :
* decompressor can decode older formats (starting from Zstd 0.1+) */
# define ZSTD_LEGACY_SUPPORT 1 # define ZSTD_LEGACY_SUPPORT 1
#endif // ZSTD_LEGACY_SUPPORT #endif
/******************************************************** /* *******************************************************
* Includes * Includes
********************************************************/ *********************************************************/
#include <stdlib.h> /* calloc */ #include <stdlib.h> /* calloc */
#include <string.h> /* memcpy, memmove */ #include <string.h> /* memcpy, memmove */
#include <stdio.h> /* debug : printf */ #include <stdio.h> /* debug : printf */
@@ -59,12 +73,12 @@
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_v01.h" # include "zstd_v01.h"
#endif // defined #endif
/******************************************************** /* *******************************************************
* Compiler specifics * Compiler specifics
********************************************************/ *********************************************************/
#ifdef __AVX2__ #ifdef __AVX2__
# include <immintrin.h> /* AVX2 intrinsics */ # include <immintrin.h> /* AVX2 intrinsics */
#endif #endif
@@ -194,7 +208,7 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr)
} }
typedef struct ZSTD_Cctx_s struct ZSTD_Cctx_s
{ {
const BYTE* base; const BYTE* base;
U32 current; U32 current;
@@ -206,13 +220,12 @@ typedef struct ZSTD_Cctx_s
U32 hashTable[HASH_TABLESIZE]; U32 hashTable[HASH_TABLESIZE];
#endif #endif
BYTE buffer[WORKPLACESIZE]; BYTE buffer[WORKPLACESIZE];
} cctxi_t; };
ZSTD_Cctx* ZSTD_createCCtx(void) void ZSTD_resetCCtx(ZSTD_Cctx* ctx)
{ {
ZSTD_Cctx* ctx = (ZSTD_Cctx*) malloc( sizeof(ZSTD_Cctx) ); ctx->base = NULL;
if (ctx==NULL) return NULL;
ctx->seqStore.buffer = ctx->buffer; ctx->seqStore.buffer = ctx->buffer;
ctx->seqStore.offsetStart = (U32*) (ctx->seqStore.buffer); ctx->seqStore.offsetStart = (U32*) (ctx->seqStore.buffer);
ctx->seqStore.offCodeStart = (BYTE*) (ctx->seqStore.offsetStart + (BLOCKSIZE>>2)); ctx->seqStore.offCodeStart = (BYTE*) (ctx->seqStore.offsetStart + (BLOCKSIZE>>2));
@@ -220,13 +233,15 @@ ZSTD_Cctx* ZSTD_createCCtx(void)
ctx->seqStore.litLengthStart = ctx->seqStore.litStart + BLOCKSIZE; ctx->seqStore.litLengthStart = ctx->seqStore.litStart + BLOCKSIZE;
ctx->seqStore.matchLengthStart = ctx->seqStore.litLengthStart + (BLOCKSIZE>>2); ctx->seqStore.matchLengthStart = ctx->seqStore.litLengthStart + (BLOCKSIZE>>2);
ctx->seqStore.dumpsStart = ctx->seqStore.matchLengthStart + (BLOCKSIZE>>2); ctx->seqStore.dumpsStart = ctx->seqStore.matchLengthStart + (BLOCKSIZE>>2);
return ctx; memset(ctx->hashTable, 0, HASH_TABLESIZE*4);
} }
void ZSTD_resetCCtx(ZSTD_Cctx* ctx) ZSTD_Cctx* ZSTD_createCCtx(void)
{ {
ctx->base = NULL; ZSTD_Cctx* ctx = (ZSTD_Cctx*) malloc( sizeof(ZSTD_Cctx) );
memset(ctx->hashTable, 0, HASH_TABLESIZE*4); if (ctx==NULL) return NULL;
ZSTD_resetCCtx(ctx);
return ctx;
} }
size_t ZSTD_freeCCtx(ZSTD_Cctx* ctx) size_t ZSTD_freeCCtx(ZSTD_Cctx* ctx)
@@ -720,9 +735,8 @@ static int ZSTD_checkMatch(const BYTE* match, const BYTE* ip)
} }
static size_t ZSTD_compressBlock(void* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) static size_t ZSTD_compressBlock(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{ {
cctxi_t* ctx = (cctxi_t*) cctx;
U32* HashTable = (U32*)(ctx->hashTable); U32* HashTable = (U32*)(ctx->hashTable);
seqStore_t* seqStorePtr = &(ctx->seqStore); seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const base = ctx->base; const BYTE* const base = ctx->base;
@@ -745,7 +759,6 @@ static size_t ZSTD_compressBlock(void* cctx, void* dst, size_t maxDstSize, const
const BYTE* match = (const BYTE*) ZSTD_updateMatch(HashTable, ip, base); const BYTE* match = (const BYTE*) ZSTD_updateMatch(HashTable, ip, base);
if (!ZSTD_checkMatch(match,ip)) { ip += ((ip-anchor) >> g_searchStrength) + 1; continue; } if (!ZSTD_checkMatch(match,ip)) { ip += ((ip-anchor) >> g_searchStrength) + 1; continue; }
/* catch up */ /* catch up */
while ((ip>anchor) && (match>base) && (ip[-1] == match[-1])) { ip--; match--; } while ((ip>anchor) && (match>base) && (ip[-1] == match[-1])) { ip--; match--; }
@@ -796,9 +809,8 @@ size_t ZSTD_compressBegin(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize)
} }
static void ZSTD_scaleDownCtx(void* cctx, const U32 limit) static void ZSTD_scaleDownCtx(ZSTD_Cctx* ctx, const U32 limit)
{ {
cctxi_t* ctx = (cctxi_t*) cctx;
int i; int i;
#if defined(__AVX2__) #if defined(__AVX2__)
@@ -825,14 +837,13 @@ static void ZSTD_scaleDownCtx(void* cctx, const U32 limit)
} }
static void ZSTD_limitCtx(void* cctx, const U32 limit) static void ZSTD_limitCtx(ZSTD_Cctx* ctx, const U32 limit)
{ {
cctxi_t* ctx = (cctxi_t*) cctx;
int i; int i;
if (limit > g_maxLimit) if (limit > g_maxLimit)
{ {
ZSTD_scaleDownCtx(cctx, limit); ZSTD_scaleDownCtx(ctx, limit);
ctx->base += limit; ctx->base += limit;
ctx->current -= limit; ctx->current -= limit;
ctx->nextUpdate -= limit; ctx->nextUpdate -= limit;
@@ -865,9 +876,8 @@ static void ZSTD_limitCtx(void* cctx, const U32 limit)
} }
size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) size_t ZSTD_compressContinue(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{ {
cctxi_t* ctx = (cctxi_t*) cctx;
const BYTE* const istart = (const BYTE* const)src; const BYTE* const istart = (const BYTE* const)src;
const BYTE* ip = istart; const BYTE* ip = istart;
BYTE* const ostart = (BYTE* const)dst; BYTE* const ostart = (BYTE* const)dst;
@@ -950,33 +960,28 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize)
} }
static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{ {
BYTE* const ostart = (BYTE* const)dst; BYTE* const ostart = (BYTE* const)dst;
BYTE* op = ostart; BYTE* op = ostart;
size_t oSize;
/* Header */ /* Header */
{ oSize = ZSTD_compressBegin(ctx, dst, maxDstSize);
size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize); if(ZSTD_isError(oSize)) return oSize;
if(ZSTD_isError(headerSize)) return headerSize; op += oSize;
op += headerSize; maxDstSize -= oSize;
maxDstSize -= headerSize;
}
/* Compression */ /* Compression */
{ oSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize);
size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize); if (ZSTD_isError(oSize)) return oSize;
if (ZSTD_isError(cSize)) return cSize; op += oSize;
op += cSize; maxDstSize -= oSize;
maxDstSize -= cSize;
}
/* Close frame */ /* Close frame */
{ oSize = ZSTD_compressEnd(ctx, op, maxDstSize);
size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize); if(ZSTD_isError(oSize)) return oSize;
if(ZSTD_isError(endSize)) return endSize; op += oSize;
op += endSize;
}
return (op - ostart); return (op - ostart);
} }
@@ -984,13 +989,22 @@ static size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, co
size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize) size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{ {
ZSTD_Cctx* ctx;
size_t r; size_t r;
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
ZSTD_Cctx* ctx;
ctx = ZSTD_createCCtx(); ctx = ZSTD_createCCtx();
if (ctx==NULL) return ERROR(GENERIC); if (ctx==NULL) return ERROR(GENERIC);
# else
ZSTD_Cctx ctxBody;
ZSTD_Cctx* const ctx = &ctxBody;
# endif
r = ZSTD_compressCCtx(ctx, dst, maxDstSize, src, srcSize); r = ZSTD_compressCCtx(ctx, dst, maxDstSize, src, srcSize);
#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1)
ZSTD_freeCCtx(ctx); ZSTD_freeCCtx(ctx);
#endif
return r; return r;
} }
@@ -1364,7 +1378,6 @@ static size_t ZSTD_execSequence(BYTE* op,
return oMatchEnd - ostart; return oMatchEnd - ostart;
} }
static size_t ZSTD_decompressSequences( static size_t ZSTD_decompressSequences(
void* ctx, void* ctx,
void* dst, size_t maxDstSize, void* dst, size_t maxDstSize,
lib/zstd.h
+27 -13
View File
@@ -36,15 +36,15 @@
extern "C" { extern "C" {
#endif #endif
/************************************** /* *************************************
* Includes * Includes
**************************************/ ***************************************/
#include <stddef.h> /* size_t */ #include <stddef.h> /* size_t */
/************************************** /* *************************************
* Version * Version
**************************************/ ***************************************/
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ #define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
#define ZSTD_VERSION_MINOR 2 /* for new (non-breaking) interface capabilities */ #define ZSTD_VERSION_MINOR 2 /* for new (non-breaking) interface capabilities */
#define ZSTD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ #define ZSTD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
@@ -52,16 +52,16 @@ extern "C" {
unsigned ZSTD_versionNumber (void); unsigned ZSTD_versionNumber (void);
/************************************** /* *************************************
* Simple functions * Simple functions
**************************************/ ***************************************/
size_t ZSTD_compress( void* dst, size_t maxDstSize, size_t ZSTD_compress( void* dst, size_t maxDstSize,
const void* src, size_t srcSize); const void* src, size_t srcSize);
size_t ZSTD_decompress( void* dst, size_t maxOriginalSize, size_t ZSTD_decompress( void* dst, size_t maxOriginalSize,
const void* src, size_t compressedSize); const void* src, size_t compressedSize);
/* /**
ZSTD_compress() : ZSTD_compress() :
Compresses 'srcSize' bytes from buffer 'src' into buffer 'dst', of maximum size 'dstSize'. Compresses 'srcSize' bytes from buffer 'src' into buffer 'dst', of maximum size 'dstSize'.
Destination buffer must be already allocated. Destination buffer must be already allocated.
@@ -73,19 +73,33 @@ ZSTD_decompress() :
compressedSize : is the exact source size compressedSize : is the exact source size
maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated.
It must be equal or larger than originalSize, otherwise decompression will fail. It must be equal or larger than originalSize, otherwise decompression will fail.
return : the number of bytes decompressed into destination buffer (originalSize <= maxOriginalSize) return : the number of bytes decompressed into destination buffer (<= maxOriginalSize)
or an errorCode if it fails (which can be tested using ZSTD_isError()) or an errorCode if it fails (which can be tested using ZSTD_isError())
*/ */
/************************************** /* *************************************
* Tool functions * Tool functions
**************************************/ ***************************************/
size_t ZSTD_compressBound(size_t srcSize); /* maximum compressed size (worst case scenario) */ size_t ZSTD_compressBound(size_t srcSize); /** maximum compressed size (worst case scenario) */
/* Error Management */ /* Error Management */
unsigned ZSTD_isError(size_t code); /* tells if a return value is an error code */ unsigned ZSTD_isError(size_t code); /** tells if a return value is an error code */
const char* ZSTD_getErrorName(size_t code); /* provides error code string */ const char* ZSTD_getErrorName(size_t code); /** provides error code string */
/* *************************************
* Advanced functions
***************************************/
typedef struct ZSTD_Cctx_s ZSTD_Cctx; /* incomplete type */
ZSTD_Cctx* ZSTD_createCCtx(void);
size_t ZSTD_freeCCtx(ZSTD_Cctx* cctx);
/**
ZSTD_compressCCtx() :
Same as ZSTD_compress(), but requires a ZSTD_Cctx working space already allocated
*/
//size_t ZSTD_compressCCtx(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
#if defined (__cplusplus) #if defined (__cplusplus)
lib/zstd_static.h
-4
View File
@@ -50,10 +50,6 @@ extern "C" {
/************************************** /**************************************
* Streaming functions * Streaming functions
**************************************/ **************************************/
typedef struct ZSTD_Cctx_s ZSTD_Cctx;
ZSTD_Cctx* ZSTD_createCCtx(void);
size_t ZSTD_freeCCtx(ZSTD_Cctx* cctx);
size_t ZSTD_compressBegin(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize); size_t ZSTD_compressBegin(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize);
size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize); size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize);