added capability to generate magic-less frames
decoder not implemented yet
This commit is contained in:
@@ -1690,14 +1690,18 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
|||||||
U32 const fcsCode = params.fParams.contentSizeFlag ?
|
U32 const fcsCode = params.fParams.contentSizeFlag ?
|
||||||
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */
|
(pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */
|
||||||
BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
|
BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
|
||||||
size_t pos;
|
size_t pos=0;
|
||||||
|
|
||||||
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
|
if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
|
||||||
DEBUGLOG(5, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
|
DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
|
||||||
!params.fParams.noDictIDFlag, dictID, dictIDSizeCode);
|
!params.fParams.noDictIDFlag, dictID, dictIDSizeCode);
|
||||||
|
|
||||||
|
if (params.format == ZSTD_f_zstd1) {
|
||||||
|
DEBUGLOG(4, "writing zstd magic number");
|
||||||
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
|
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
|
||||||
op[4] = frameHeaderDecriptionByte; pos=5;
|
pos = 4;
|
||||||
|
}
|
||||||
|
op[pos++] = frameHeaderDecriptionByte;
|
||||||
if (!singleSegment) op[pos++] = windowLogByte;
|
if (!singleSegment) op[pos++] = windowLogByte;
|
||||||
switch(dictIDSizeCode)
|
switch(dictIDSizeCode)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -110,6 +110,7 @@ struct ZSTD_DCtx_s
|
|||||||
XXH64_state_t xxhState;
|
XXH64_state_t xxhState;
|
||||||
size_t headerSize;
|
size_t headerSize;
|
||||||
U32 dictID;
|
U32 dictID;
|
||||||
|
ZSTD_format_e format;
|
||||||
const BYTE* litPtr;
|
const BYTE* litPtr;
|
||||||
ZSTD_customMem customMem;
|
ZSTD_customMem customMem;
|
||||||
size_t litSize;
|
size_t litSize;
|
||||||
@@ -264,32 +265,57 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/** ZSTD_frameHeaderSize_internal() :
|
||||||
|
* srcSize must be large enough to reach header size fields.
|
||||||
|
* note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless
|
||||||
|
* @return : size of the Frame Header
|
||||||
|
* or an error code, which can be tested with ZSTD_isError() */
|
||||||
|
static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format)
|
||||||
|
{
|
||||||
|
size_t const minInputSize = (format==ZSTD_f_zstd1_magicless) ?
|
||||||
|
ZSTD_frameHeaderSize_prefix - 4 /* magic number size */ :
|
||||||
|
ZSTD_frameHeaderSize_prefix;
|
||||||
|
ZSTD_STATIC_ASSERT((unsigned)ZSTD_f_zstd1 < (unsigned)ZSTD_f_zstd1_magicless);
|
||||||
|
assert((unsigned)format <= ZSTD_f_zstd1_magicless); /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
|
||||||
|
if (srcSize < minInputSize) return ERROR(srcSize_wrong);
|
||||||
|
|
||||||
|
{ BYTE const fhd = ((const BYTE*)src)[minInputSize-1];
|
||||||
|
U32 const dictID= fhd & 3;
|
||||||
|
U32 const singleSegment = (fhd >> 5) & 1;
|
||||||
|
U32 const fcsId = fhd >> 6;
|
||||||
|
return minInputSize + !singleSegment
|
||||||
|
+ ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
|
||||||
|
+ (singleSegment && !fcsId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** ZSTD_frameHeaderSize() :
|
/** ZSTD_frameHeaderSize() :
|
||||||
* srcSize must be >= ZSTD_frameHeaderSize_prefix.
|
* srcSize must be >= ZSTD_frameHeaderSize_prefix.
|
||||||
* @return : size of the Frame Header */
|
* @return : size of the Frame Header */
|
||||||
size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
|
size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong);
|
return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1);
|
||||||
{ BYTE const fhd = ((const BYTE*)src)[4];
|
|
||||||
U32 const dictID= fhd & 3;
|
|
||||||
U32 const singleSegment = (fhd >> 5) & 1;
|
|
||||||
U32 const fcsId = fhd >> 6;
|
|
||||||
return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
|
|
||||||
+ (singleSegment && !fcsId);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/** ZSTD_getFrameHeader() :
|
/** ZSTD_getFrameHeader_internal() :
|
||||||
* decode Frame Header, or require larger `srcSize`.
|
* decode Frame Header, or require larger `srcSize`.
|
||||||
|
* note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless
|
||||||
* @return : 0, `zfhPtr` is correctly filled,
|
* @return : 0, `zfhPtr` is correctly filled,
|
||||||
* >0, `srcSize` is too small, result is expected `srcSize`,
|
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
|
||||||
* or an error code, which can be tested using ZSTD_isError() */
|
* or an error code, which can be tested using ZSTD_isError() */
|
||||||
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
|
static size_t ZSTD_getFrameHeader_internal(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
|
||||||
{
|
{
|
||||||
const BYTE* ip = (const BYTE*)src;
|
const BYTE* ip = (const BYTE*)src;
|
||||||
if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix;
|
size_t const minInputSize = (format==ZSTD_f_zstd1_magicless) ?
|
||||||
|
ZSTD_frameHeaderSize_prefix - 4 /* magic number size */ :
|
||||||
|
ZSTD_frameHeaderSize_prefix;
|
||||||
|
|
||||||
|
ZSTD_STATIC_ASSERT((unsigned)ZSTD_f_zstd1 < (unsigned)ZSTD_f_zstd1_magicless);
|
||||||
|
assert((unsigned)format <= ZSTD_f_zstd1_magicless); /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
|
||||||
|
if (srcSize < minInputSize) return minInputSize;
|
||||||
|
|
||||||
|
if (format != ZSTD_f_zstd1_magicless)
|
||||||
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
|
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
|
||||||
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||||
/* skippable frame */
|
/* skippable frame */
|
||||||
@@ -304,13 +330,13 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* ensure there is enough `srcSize` to fully read/decode frame header */
|
/* ensure there is enough `srcSize` to fully read/decode frame header */
|
||||||
{ size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize);
|
{ size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format);
|
||||||
if (srcSize < fhsize) return fhsize;
|
if (srcSize < fhsize) return fhsize;
|
||||||
zfhPtr->headerSize = (U32)fhsize;
|
zfhPtr->headerSize = (U32)fhsize;
|
||||||
}
|
}
|
||||||
|
|
||||||
{ BYTE const fhdByte = ip[4];
|
{ BYTE const fhdByte = ip[minInputSize-1];
|
||||||
size_t pos = 5;
|
size_t pos = minInputSize;
|
||||||
U32 const dictIDSizeCode = fhdByte&3;
|
U32 const dictIDSizeCode = fhdByte&3;
|
||||||
U32 const checksumFlag = (fhdByte>>2)&1;
|
U32 const checksumFlag = (fhdByte>>2)&1;
|
||||||
U32 const singleSegment = (fhdByte>>5)&1;
|
U32 const singleSegment = (fhdByte>>5)&1;
|
||||||
@@ -357,6 +383,18 @@ size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t src
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** ZSTD_getFrameHeader() :
|
||||||
|
* decode Frame Header, or require larger `srcSize`.
|
||||||
|
* note : this function does not consume input, it only reads it.
|
||||||
|
* @return : 0, `zfhPtr` is correctly filled,
|
||||||
|
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
|
||||||
|
* or an error code, which can be tested using ZSTD_isError() */
|
||||||
|
size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
|
||||||
|
{
|
||||||
|
return ZSTD_getFrameHeader_internal(zfhPtr, src, srcSize, ZSTD_f_zstd1);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/** ZSTD_getFrameContentSize() :
|
/** ZSTD_getFrameContentSize() :
|
||||||
* compatible with legacy mode
|
* compatible with legacy mode
|
||||||
* @return : decompressed size of the single frame pointed to be `src` if known, otherwise
|
* @return : decompressed size of the single frame pointed to be `src` if known, otherwise
|
||||||
@@ -390,7 +428,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
|||||||
unsigned long long totalDstSize = 0;
|
unsigned long long totalDstSize = 0;
|
||||||
|
|
||||||
while (srcSize >= ZSTD_frameHeaderSize_prefix) {
|
while (srcSize >= ZSTD_frameHeaderSize_prefix) {
|
||||||
const U32 magicNumber = MEM_readLE32(src);
|
U32 const magicNumber = MEM_readLE32(src);
|
||||||
|
|
||||||
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||||
size_t skippableSize;
|
size_t skippableSize;
|
||||||
@@ -422,11 +460,9 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
|||||||
src = (const BYTE *)src + frameSrcSize;
|
src = (const BYTE *)src + frameSrcSize;
|
||||||
srcSize -= frameSrcSize;
|
srcSize -= frameSrcSize;
|
||||||
}
|
}
|
||||||
}
|
} /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
|
||||||
|
|
||||||
if (srcSize) {
|
if (srcSize) return ZSTD_CONTENTSIZE_ERROR;
|
||||||
return ZSTD_CONTENTSIZE_ERROR;
|
|
||||||
}
|
|
||||||
|
|
||||||
return totalDstSize;
|
return totalDstSize;
|
||||||
}
|
}
|
||||||
@@ -442,7 +478,8 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
|
|||||||
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
|
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
|
unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
|
||||||
return ret >= ZSTD_CONTENTSIZE_ERROR ? 0 : ret;
|
ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN);
|
||||||
|
return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -499,7 +536,7 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/*! ZSTD_decodeLiteralsBlock() :
|
/*! ZSTD_decodeLiteralsBlock() :
|
||||||
@return : nb of bytes read from src (< srcSize ) */
|
* @return : nb of bytes read from src (< srcSize ) */
|
||||||
size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
||||||
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
||||||
{
|
{
|
||||||
@@ -700,8 +737,8 @@ static const FSE_decode_t4 OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {
|
|||||||
}; /* OF_defaultDTable */
|
}; /* OF_defaultDTable */
|
||||||
|
|
||||||
/*! ZSTD_buildSeqTable() :
|
/*! ZSTD_buildSeqTable() :
|
||||||
@return : nb bytes read from src,
|
* @return : nb bytes read from src,
|
||||||
or an error code if it fails, testable with ZSTD_isError()
|
* or an error code if it fails, testable with ZSTD_isError()
|
||||||
*/
|
*/
|
||||||
static size_t ZSTD_buildSeqTable(FSE_DTable* DTableSpace, const FSE_DTable** DTablePtr,
|
static size_t ZSTD_buildSeqTable(FSE_DTable* DTableSpace, const FSE_DTable** DTablePtr,
|
||||||
symbolEncodingType_e type, U32 max, U32 maxLog,
|
symbolEncodingType_e type, U32 max, U32 maxLog,
|
||||||
@@ -1424,7 +1461,7 @@ ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, siz
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
|
static size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length)
|
||||||
{
|
{
|
||||||
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
|
if (length > dstCapacity) return ERROR(dstSize_tooSmall);
|
||||||
memset(dst, byte, length);
|
memset(dst, byte, length);
|
||||||
@@ -1607,6 +1644,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
magicNumber = MEM_readLE32(src);
|
magicNumber = MEM_readLE32(src);
|
||||||
|
DEBUGLOG(4, "reading magic number %08X (expecting %08X)",
|
||||||
|
(U32)magicNumber, (U32)ZSTD_MAGICNUMBER);
|
||||||
if (magicNumber != ZSTD_MAGICNUMBER) {
|
if (magicNumber != ZSTD_MAGICNUMBER) {
|
||||||
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
|
||||||
size_t skippableSize;
|
size_t skippableSize;
|
||||||
@@ -1716,7 +1755,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
|
|||||||
{
|
{
|
||||||
DEBUGLOG(5, "ZSTD_decompressContinue");
|
DEBUGLOG(5, "ZSTD_decompressContinue");
|
||||||
/* Sanity check */
|
/* Sanity check */
|
||||||
if (srcSize != dctx->expected) return ERROR(srcSize_wrong); /* unauthorized */
|
if (srcSize != dctx->expected) return ERROR(srcSize_wrong); /* not allowed */
|
||||||
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
|
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
|
||||||
|
|
||||||
switch (dctx->stage)
|
switch (dctx->stage)
|
||||||
@@ -2244,14 +2283,38 @@ size_t ZSTD_resetDStream(ZSTD_DStream* zds)
|
|||||||
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
|
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
|
||||||
ZSTD_DStreamParameter_e paramType, unsigned paramValue)
|
ZSTD_DStreamParameter_e paramType, unsigned paramValue)
|
||||||
{
|
{
|
||||||
|
ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init);
|
||||||
|
if ((unsigned)zds->streamStage > (unsigned)zdss_loadHeader)
|
||||||
|
return ERROR(stage_wrong);
|
||||||
switch(paramType)
|
switch(paramType)
|
||||||
{
|
{
|
||||||
default : return ERROR(parameter_unsupported);
|
default : return ERROR(parameter_unsupported);
|
||||||
case DStream_p_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
|
case DStream_p_maxWindowSize :
|
||||||
|
DEBUGLOG(4, "setting maxWindowSize = %u KB", paramValue >> 10);
|
||||||
|
zds->maxWindowSize = paramValue ? paramValue : (U32)(-1);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
|
||||||
|
{
|
||||||
|
ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init);
|
||||||
|
if ((unsigned)dctx->streamStage > (unsigned)zdss_loadHeader)
|
||||||
|
return ERROR(stage_wrong);
|
||||||
|
dctx->maxWindowSize = maxWindowSize;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
|
||||||
|
{
|
||||||
|
ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init);
|
||||||
|
if ((unsigned)dctx->streamStage > (unsigned)zdss_loadHeader)
|
||||||
|
return ERROR(stage_wrong);
|
||||||
|
dctx->format = format;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
|
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
|
||||||
{
|
{
|
||||||
@@ -2276,7 +2339,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize)
|
|||||||
return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
|
return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
|
size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable */
|
U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable */
|
||||||
ZSTD_frameHeader zfh;
|
ZSTD_frameHeader zfh;
|
||||||
@@ -2389,7 +2452,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* control buffer memory usage */
|
/* control buffer memory usage */
|
||||||
DEBUGLOG(4, "Control max buffer memory usage");
|
DEBUGLOG(4, "Control max buffer memory usage (max %u KB)",
|
||||||
|
(U32)(zds->maxWindowSize >> 10));
|
||||||
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
|
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
|
||||||
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
|
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
|
||||||
|
|
||||||
|
|||||||
+20
-8
@@ -1247,9 +1247,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
|
|||||||
* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
|
* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select
|
||||||
* how dictionary content will be interpreted and loaded.
|
* how dictionary content will be interpreted and loaded.
|
||||||
*/
|
*/
|
||||||
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */
|
||||||
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
|
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); /* not implemented */
|
||||||
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode);
|
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode); /* not implemented */
|
||||||
|
|
||||||
|
|
||||||
/*! ZSTD_DCtx_refDDict() :
|
/*! ZSTD_DCtx_refDDict() :
|
||||||
@@ -1261,7 +1261,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void
|
|||||||
* Special : adding a NULL DDict means "return to no-dictionary mode".
|
* Special : adding a NULL DDict means "return to no-dictionary mode".
|
||||||
* Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
|
* Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
|
||||||
*/
|
*/
|
||||||
ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); /* not implemented */
|
||||||
|
|
||||||
|
|
||||||
/*! ZSTD_DCtx_refPrefix() :
|
/*! ZSTD_DCtx_refPrefix() :
|
||||||
@@ -1273,10 +1273,10 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
|
|||||||
* Note 2 : Prefix buffer is referenced. It must outlive compression job.
|
* Note 2 : Prefix buffer is referenced. It must outlive compression job.
|
||||||
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
|
* Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
|
||||||
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
|
* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode.
|
||||||
* Note 4 : Referencing a raw content prefix costs almost nothing cpu and memory wise.
|
* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
|
||||||
*/
|
*/
|
||||||
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize);
|
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize); /* not implemented */
|
||||||
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode);
|
ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode); /* not implemented */
|
||||||
|
|
||||||
|
|
||||||
/*! ZSTD_DCtx_setMaxWindowSize() :
|
/*! ZSTD_DCtx_setMaxWindowSize() :
|
||||||
@@ -1295,9 +1295,21 @@ ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowS
|
|||||||
* such ZSTD_f_zstd1_magicless for example.
|
* such ZSTD_f_zstd1_magicless for example.
|
||||||
* @return : 0, or an error code (which can be tested using ZSTD_isError()).
|
* @return : 0, or an error code (which can be tested using ZSTD_isError()).
|
||||||
*/
|
*/
|
||||||
ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
|
ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); /* implemented, but not functional */
|
||||||
|
|
||||||
|
|
||||||
|
/* How to decompress ?
|
||||||
|
*
|
||||||
|
* currently, use ZSTD_decompressStream().
|
||||||
|
* We could also create a ZSTD_decompress_generic(),
|
||||||
|
* for an API experience similar to the compression one.
|
||||||
|
* It would effectively works exactly the same as ZSTD_decompressStream().
|
||||||
|
*
|
||||||
|
* Also : to re-init a decoding context, use ZSTD_initDStream().
|
||||||
|
* Here also, for a similar API logic, we could create ZSTD_DCtx_reset().
|
||||||
|
* It would behave the same.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
/** === Block level API === **/
|
/** === Block level API === **/
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -40,7 +40,7 @@ CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
|||||||
-DZSTD_NEWAPI \
|
-DZSTD_NEWAPI \
|
||||||
-DXXH_NAMESPACE=ZSTD_ # because xxhash.o already compiled with this macro from library
|
-DXXH_NAMESPACE=ZSTD_ # because xxhash.o already compiled with this macro from library
|
||||||
CFLAGS ?= -O3
|
CFLAGS ?= -O3
|
||||||
DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||||
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
|
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
|
||||||
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
|
||||||
|
|||||||
+1
-1
@@ -1036,7 +1036,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
|
|||||||
/* Allocation */
|
/* Allocation */
|
||||||
ress.dctx = ZSTD_createDStream();
|
ress.dctx = ZSTD_createDStream();
|
||||||
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
|
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZSTD_DStream");
|
||||||
ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit);
|
CHECK( ZSTD_setDStreamParameter(ress.dctx, DStream_p_maxWindowSize, g_memLimit) );
|
||||||
ress.srcBufferSize = ZSTD_DStreamInSize();
|
ress.srcBufferSize = ZSTD_DStreamInSize();
|
||||||
ress.srcBuffer = malloc(ress.srcBufferSize);
|
ress.srcBuffer = malloc(ress.srcBufferSize);
|
||||||
ress.dstBufferSize = ZSTD_DStreamOutSize();
|
ress.dstBufferSize = ZSTD_DStreamOutSize();
|
||||||
|
|||||||
+38
-1
@@ -918,6 +918,43 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
ZSTD_freeCCtx(cctx);
|
ZSTD_freeCCtx(cctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* custom formats tests */
|
||||||
|
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||||
|
static const size_t inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
|
||||||
|
|
||||||
|
/* basic block compression */
|
||||||
|
DISPLAYLEVEL(4, "test%3i : magic-less format test : ", testNb++);
|
||||||
|
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_p_format, ZSTD_f_zstd1_magicless) );
|
||||||
|
{ ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
|
||||||
|
ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
|
||||||
|
size_t const result = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||||
|
if (result != 0) goto _output_error;
|
||||||
|
if (in.pos != in.size) goto _output_error;
|
||||||
|
cSize = out.pos;
|
||||||
|
}
|
||||||
|
DISPLAYLEVEL(4, "OK (compress : %u -> %u bytes)\n", (U32)inputSize, (U32)cSize);
|
||||||
|
|
||||||
|
DISPLAYLEVEL(4, "test%3i : decompress normally (should fail) : ", testNb++);
|
||||||
|
{ size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
|
||||||
|
if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error;
|
||||||
|
DISPLAYLEVEL(4, "OK : %s \n", ZSTD_getErrorName(decodeResult));
|
||||||
|
}
|
||||||
|
|
||||||
|
DISPLAYLEVEL(4, "test%3i : decompress with magic-less instruction : ", testNb++);
|
||||||
|
CHECK( ZSTD_initDStream(dctx) );
|
||||||
|
CHECK( ZSTD_DCtx_setFormat(dctx, ZSTD_f_zstd1_magicless) );
|
||||||
|
{ ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
|
||||||
|
ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
|
||||||
|
size_t const result = ZSTD_decompressStream(dctx, &out, &in);
|
||||||
|
if (result != 0) goto _output_error;
|
||||||
|
if (in.pos != in.size) goto _output_error;
|
||||||
|
if (out.pos != inputSize) goto _output_error;
|
||||||
|
DISPLAYLEVEL(4, "OK : regenerated %u bytes \n", (U32)out.pos);
|
||||||
|
}
|
||||||
|
|
||||||
|
ZSTD_freeCCtx(cctx);
|
||||||
|
}
|
||||||
|
|
||||||
/* block API tests */
|
/* block API tests */
|
||||||
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||||
static const size_t dictSize = 65 KB;
|
static const size_t dictSize = 65 KB;
|
||||||
@@ -961,8 +998,8 @@ static int basicUnitTests(U32 seed, double compressibility)
|
|||||||
DISPLAYLEVEL(4, "OK \n");
|
DISPLAYLEVEL(4, "OK \n");
|
||||||
|
|
||||||
ZSTD_freeCCtx(cctx);
|
ZSTD_freeCCtx(cctx);
|
||||||
ZSTD_freeDCtx(dctx);
|
|
||||||
}
|
}
|
||||||
|
ZSTD_freeDCtx(dctx);
|
||||||
|
|
||||||
/* long rle test */
|
/* long rle test */
|
||||||
{ size_t sampleSize = 0;
|
{ size_t sampleSize = 0;
|
||||||
|
|||||||
+5
-1
@@ -45,7 +45,6 @@ then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
isWindows=false
|
isWindows=false
|
||||||
ECHO="echo -e"
|
|
||||||
INTOVOID="/dev/null"
|
INTOVOID="/dev/null"
|
||||||
case "$OS" in
|
case "$OS" in
|
||||||
Windows*)
|
Windows*)
|
||||||
@@ -66,6 +65,11 @@ case "$UNAME" in
|
|||||||
SunOS) DIFF="gdiff" ;;
|
SunOS) DIFF="gdiff" ;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
ECHO="echo -e"
|
||||||
|
case "$UNAME" in
|
||||||
|
Darwin) ECHO="echo" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
$ECHO "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'"
|
$ECHO "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'"
|
||||||
|
|
||||||
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
|
[ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
|
||||||
|
|||||||
Reference in New Issue
Block a user