ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #198 from inikep/dev070

Browse Source 
Dev070
This commit is contained in:
Yann Collet
2016-06-03 21:37:49 +02:00
parent fe48775868 de2c92f6e0
commit ab7b6f1ece
22 changed files with 421 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/common/zstd_common.c
+18
View File
@@ -33,6 +33,7 @@
/*-*************************************
* Dependencies
***************************************/
#include <stdlib.h> /* malloc */
#include "error_private.h"
#include "zstd_static.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode */
#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
@@ -70,3 +71,20 @@ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(c
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
void* ZSTD_defaultAllocFunction(void* opaque, size_t size)
{
void* address = malloc(size);
(void)opaque;
/* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
return address;
}
void ZSTD_defaultFreeFunction(void* opaque, void* address)
{
(void)opaque;
/* if (address) printf("free %p opaque=%p \n", address, opaque); */
free(address);
}
lib/common/zstd_internal.h
+5
View File
@@ -255,4 +255,9 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq);
int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
/* custom memory allocation functions */
void* ZSTD_defaultAllocFunction(void* opaque, size_t size);
void ZSTD_defaultFreeFunction(void* opaque, void* address);
static ZSTD_customMem const defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };
#endif /* ZSTD_CCOMMON_H_MODULE */
lib/common/zstd_static.h
+13 -3
View File
@@ -97,9 +97,10 @@ typedef struct {
ZSTD_frameParameters fParams;
} ZSTD_parameters;
typedef void* (*ZSTD_allocFunction) (size_t size);
typedef void (*ZSTD_freeFunction) (void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; } ZSTD_customMem;
/* custom memory allocation functions */
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
/*-*************************************
@@ -246,6 +247,15 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
Context can then be reset to start a new decompression.
Skippable frames allow the integration of user-defined data into a flow of concatenated frames.
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following:
a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
c) Frame Content - any content (User Data) of length equal to Frame Size
For skippable frames ZSTD_decompressContinue() always returns 0.
For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
It also returns Frame Size as fparamsPtr->frameContentSize.
*/
lib/compress/huf_compress.c
-1
View File
@@ -59,7 +59,6 @@
/* **************************************************************
* Includes
****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
#include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */
#include "huf_static.h"
lib/compress/zbuff_compress.c
+13 -22
View File
@@ -95,14 +95,12 @@ struct ZBUFF_CCtx_s {
size_t outBuffContentSize;
size_t outBuffFlushedSize;
ZBUFF_cStage stage;
ZSTD_allocFunction customAlloc;
ZSTD_freeFunction customFree;
ZSTD_customMem customMem;
}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
ZBUFF_CCtx* ZBUFF_createCCtx(void)
{
ZSTD_customMem customMem = { NULL, NULL };
return ZBUFF_createCCtx_advanced(customMem);
return ZBUFF_createCCtx_advanced(defaultCustomMem);
}
ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
@@ -110,24 +108,17 @@ ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
ZBUFF_CCtx* zbc;
if (!customMem.customAlloc && !customMem.customFree)
{
zbc = (ZBUFF_CCtx*)calloc(1, sizeof(ZBUFF_CCtx));
if (zbc==NULL) return NULL;
zbc->customAlloc = malloc;
zbc->customFree = free;
zbc->zc = ZSTD_createCCtx();
return zbc;
}
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zbc = (ZBUFF_CCtx*)customMem.customAlloc(sizeof(ZBUFF_CCtx));
zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx));
if (zbc==NULL) return NULL;
memset(zbc, 0, sizeof(ZBUFF_CCtx));
zbc->customAlloc = customMem.customAlloc;
zbc->customFree = customMem.customFree;
memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem));
zbc->zc = ZSTD_createCCtx_advanced(customMem);
if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; }
return zbc;
}
@@ -135,9 +126,9 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
{
if (zbc==NULL) return 0; /* support free on NULL */
ZSTD_freeCCtx(zbc->zc);
zbc->customFree(zbc->inBuff);
zbc->customFree(zbc->outBuff);
zbc->customFree(zbc);
if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff);
if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff);
zbc->customMem.customFree(zbc->customMem.opaque, zbc);
return 0;
}
@@ -152,16 +143,16 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
if (zbc->inBuffSize < neededInBuffSize) {
zbc->inBuffSize = neededInBuffSize;
zbc->customFree(zbc->inBuff); /* should not be necessary */
zbc->inBuff = (char*)zbc->customAlloc(neededInBuffSize);
zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */
zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
}
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2);
}
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
zbc->customFree(zbc->outBuff); /* should not be necessary */
zbc->outBuff = (char*)zbc->customAlloc(zbc->outBuffSize);
zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */
zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize);
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
}
lib/compress/zstd_compress.c
+13 -26
View File
@@ -51,7 +51,6 @@
/*-*************************************
* Dependencies
***************************************/
#include <stdlib.h> /* malloc */
#include <string.h> /* memset */
#include "mem.h"
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
@@ -107,8 +106,7 @@ struct ZSTD_CCtx_s
size_t workSpaceSize;
size_t blockSize;
XXH64_state_t xxhState;
ZSTD_allocFunction customAlloc;
ZSTD_freeFunction customFree;
ZSTD_customMem customMem;
seqStore_t seqStore; /* sequences storage ptrs */
U32* hashTable;
@@ -123,8 +121,7 @@ struct ZSTD_CCtx_s
ZSTD_CCtx* ZSTD_createCCtx(void)
{
ZSTD_customMem customMem = { NULL, NULL };
return ZSTD_createCCtx_advanced(customMem);
return ZSTD_createCCtx_advanced(defaultCustomMem);
}
ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
@@ -132,31 +129,23 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
ZSTD_CCtx* ctx;
if (!customMem.customAlloc && !customMem.customFree)
{
ctx = (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx));
if (!ctx) return NULL;
ctx->customAlloc = malloc;
ctx->customFree = free;
return ctx;
}
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
ctx = (ZSTD_CCtx*) customMem.customAlloc(sizeof(ZSTD_CCtx));
ctx = (ZSTD_CCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CCtx));
if (!ctx) return NULL;
memset(ctx, 0, sizeof(ZSTD_CCtx));
ctx->customAlloc = customMem.customAlloc;
ctx->customFree = customMem.customFree;
memcpy(&ctx->customMem, &customMem, sizeof(ZSTD_customMem));
return ctx;
}
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
{
cctx->customFree(cctx->workSpace);
cctx->customFree(cctx);
if (cctx==NULL) return 0; /* support free on NULL */
if (cctx->workSpace) cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace);
cctx->customMem.customFree(cctx->customMem.opaque, cctx);
return 0; /* reserved as a potential error code in the future */
}
@@ -264,8 +253,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
+ ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
if (zc->workSpaceSize < neededSpace) {
zc->customFree(zc->workSpace);
zc->workSpace = zc->customAlloc(neededSpace);
zc->customMem.customFree(zc->customMem.opaque, zc->workSpace);
zc->workSpace = zc->customMem.customAlloc(zc->customMem.opaque, neededSpace);
if (zc->workSpace == NULL) return ERROR(memory_allocation);
zc->workSpaceSize = neededSpace;
} }
@@ -324,8 +313,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
if (srcCCtx->stage!=1) return ERROR(stage_wrong);
dstCCtx->hashLog3 = srcCCtx->hashLog3; /* must be before ZSTD_resetCCtx_advanced */
dstCCtx->customAlloc = srcCCtx->customAlloc;
dstCCtx->customFree = srcCCtx->customFree;
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, 0);
dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */
@@ -2487,10 +2475,9 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
size_t result;
ZSTD_CCtx ctxBody;
memset(&ctxBody, 0, sizeof(ctxBody));
ctxBody.customAlloc = malloc;
ctxBody.customFree = free;
memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
ctxBody.customFree(ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */
ctxBody.customMem.customFree(ctxBody.customMem.opaque, ctxBody.workSpace); /* can't free ctxBody, since it's on stack; just free heap content */
return result;
}
lib/decompress/huf_decompress.c
-1
View File
@@ -59,7 +59,6 @@
/* **************************************************************
* Includes
****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
#include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */
#include "huf_static.h"
lib/decompress/zbuff_decompress.c
+15 -23
View File
@@ -82,15 +82,13 @@ struct ZBUFF_DCtx_s {
size_t blockSize;
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t lhSize;
ZSTD_allocFunction customAlloc;
ZSTD_freeFunction customFree;
ZSTD_customMem customMem;
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
ZBUFF_DCtx* ZBUFF_createDCtx(void)
{
ZSTD_customMem customMem = { NULL, NULL };
return ZBUFF_createDCtx_advanced(customMem);
return ZBUFF_createDCtx_advanced(defaultCustomMem);
}
ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
@@ -98,25 +96,17 @@ ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
ZBUFF_DCtx* zbd;
if (!customMem.customAlloc && !customMem.customFree)
{
zbd = (ZBUFF_DCtx*)calloc(1, sizeof(ZBUFF_DCtx));
if (zbd==NULL) return NULL;
zbd->customAlloc = malloc;
zbd->customFree = free;
zbd->zd = ZSTD_createDCtx();
zbd->stage = ZBUFFds_init;
return zbd;
}
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zbd = (ZBUFF_DCtx*)customMem.customAlloc(sizeof(ZBUFF_DCtx));
zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx));
if (zbd==NULL) return NULL;
memset(zbd, 0, sizeof(ZBUFF_DCtx));
zbd->customAlloc = customMem.customAlloc;
zbd->customFree = customMem.customFree;
memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem));
zbd->zd = ZSTD_createDCtx_advanced(customMem);
if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; }
zbd->stage = ZBUFFds_init;
return zbd;
}
@@ -125,9 +115,9 @@ size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
{
if (zbd==NULL) return 0; /* support free on null */
ZSTD_freeDCtx(zbd->zd);
zbd->customFree(zbd->inBuff);
zbd->customFree(zbd->outBuff);
zbd->customFree(zbd);
if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
zbd->customMem.customFree(zbd->customMem.opaque, zbd);
return 0;
}
@@ -192,20 +182,22 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
if (ZSTD_isError(h2Result)) return h2Result;
} }
if (zbd->fParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) zbd->fParams.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for buffer allocation */
/* Frame header instruct buffer sizes */
{ size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX);
zbd->blockSize = blockSize;
if (zbd->inBuffSize < blockSize) {
zbd->customFree(zbd->inBuff);
zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
zbd->inBuffSize = blockSize;
zbd->inBuff = (char*)zbd->customAlloc(blockSize);
zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
}
{ size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize;
if (zbd->outBuffSize < neededOutSize) {
zbd->customFree(zbd->outBuff);
zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
zbd->outBuffSize = neededOutSize;
zbd->outBuff = (char*)zbd->customAlloc(neededOutSize);
zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize);
if (zbd->outBuff == NULL) return ERROR(memory_allocation);
} } }
zbd->stage = ZBUFFds_read;
lib/decompress/zstd_decompress.c
+11 -21
View File
@@ -53,7 +53,6 @@
/*-*******************************************************
* Dependencies
*********************************************************/
#include <stdlib.h> /* calloc */
#include <string.h> /* memcpy, memmove */
#include <stdio.h> /* debug only : printf */
#include "mem.h" /* low level memory routines */
@@ -120,8 +119,7 @@ struct ZSTD_DCtx_s
size_t headerSize;
ZSTD_frameParams fParams;
XXH64_state_t xxhState;
ZSTD_allocFunction customAlloc;
ZSTD_freeFunction customFree;
ZSTD_customMem customMem;
blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
ZSTD_dStage stage;
U32 dictID;
@@ -153,38 +151,29 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
{
ZSTD_DCtx* dctx;
if (!customMem.customAlloc && !customMem.customFree) {
dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx));
if (!dctx) return NULL;
dctx->customAlloc = malloc;
dctx->customFree = free;
ZSTD_decompressBegin(dctx);
return dctx;
}
if (!customMem.customAlloc && !customMem.customFree)
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx));
dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx));
if (!dctx) return NULL;
dctx->customAlloc = customMem.customAlloc;
dctx->customFree = customMem.customFree;
memcpy(&dctx->customMem, &customMem, sizeof(ZSTD_customMem));
ZSTD_decompressBegin(dctx);
return dctx;
}
ZSTD_DCtx* ZSTD_createDCtx(void)
{
ZSTD_customMem const customMem = { NULL, NULL };
return ZSTD_createDCtx_advanced(customMem);
return ZSTD_createDCtx_advanced(defaultCustomMem);
}
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
{
dctx->customFree(dctx);
if (dctx==NULL) return 0; /* support free on NULL */
dctx->customMem.customFree(dctx->customMem.opaque, dctx);
return 0; /* reserved as a potential error code in the future */
}
@@ -333,8 +322,9 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
fparamsPtr->frameContentSize = 0;
fparamsPtr->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4);
fparamsPtr->windowLog = 0; /* windowLog==0 means a frame is skippable */
return 0;
}
return ERROR(prefix_unknown);