ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #26 from Cyan4973/streamAPI2

Browse Source 
Stream api2
This commit is contained in:
Yann Collet
2015-02-26 16:02:58 +01:00
parent 786872db65 ad68a0e5f7
commit 48310ded03
5 changed files with 183 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/fse.c
+103 -12
View File
@@ -87,6 +87,8 @@
#include "fse_static.h"
#ifndef MEM_ACCESS_MODULE
#define MEM_ACCESS_MODULE
/****************************************************************
* Basic Types
*****************************************************************/
@@ -109,6 +111,7 @@ typedef unsigned long long U64;
typedef signed long long S64;
#endif
#endif /* MEM_ACCESS_MODULE */
/****************************************************************
* Memory I/O
@@ -540,7 +543,6 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
if ((FSE_highbit32((U32)(srcSize - 1)) - 2) < tableLog) tableLog = FSE_highbit32((U32)(srcSize - 1)) - 2; /* Accuracy can be reduced */
if ((FSE_highbit32(maxSymbolValue+1)+1) > tableLog) tableLog = FSE_highbit32(maxSymbolValue+1)+1; /* Need a minimum to safely represent all symbol values */
//if ((FSE_highbit32(maxSymbolValue)+2) > tableLog) tableLog = FSE_highbit32(maxSymbolValue)+2; /* Need a minimum to safely represent all symbol values */
if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
return tableLog;
@@ -561,6 +563,8 @@ int FSE_compareRankT(const void* r1, const void* r2)
return 2 * (R1->count < R2->count) - 1;
}
#if 0
static size_t FSE_adjustNormSlow(short* norm, int pointsToRemove, const unsigned* count, U32 maxSymbolValue)
{
rank_t rank[FSE_MAX_SYMBOL_VALUE+2];
@@ -602,6 +606,100 @@ static size_t FSE_adjustNormSlow(short* norm, int pointsToRemove, const unsigned
return 0;
}
#else
/* Secondary normalization method.
To be used when primary method fails. */
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
{
U32 s;
U32 distributed = 0;
U32 ToDistribute;
/* Init */
U32 lowThreshold = (U32)(total >> tableLog);
U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
for (s=0; s<=maxSymbolValue; s++)
{
if (count[s] == 0)
{
norm[s]=0;
continue;
}
if (count[s] <= lowThreshold)
{
norm[s] = -1;
distributed++;
total -= count[s];
continue;
}
if (count[s] <= lowOne)
{
norm[s] = 1;
distributed++;
total -= count[s];
continue;
}
norm[s]=-2;
}
ToDistribute = (1 << tableLog) - distributed;
if ((total / ToDistribute) > lowOne)
{
/* risk of rounding to zero */
lowOne = (U32)((total * 3) / (ToDistribute * 2));
for (s=0; s<=maxSymbolValue; s++)
{
if ((norm[s] == -2) && (count[s] <= lowOne))
{
norm[s] = 1;
distributed++;
total -= count[s];
continue;
}
}
ToDistribute = (1 << tableLog) - distributed;
}
if (distributed == maxSymbolValue+1)
{
/* all values are pretty poor;
probably incompressible data (should have already been detected);
find max, then give all remaining points to max */
U32 maxV = 0, maxC =0;
for (s=0; s<=maxSymbolValue; s++)
if (count[s] > maxC) maxV=s, maxC=count[s];
norm[maxV] += ToDistribute;
return 0;
}
{
U64 const vStepLog = 62 - tableLog;
U64 const mid = (1ULL << (vStepLog-1)) - 1;
U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
U64 tmpTotal = mid;
for (s=0; s<=maxSymbolValue; s++)
{
if (norm[s]==-2)
{
U64 end = tmpTotal + (count[s] * rStep);
U32 sStart = (U32)(tmpTotal >> vStepLog);
U32 sEnd = (U32)(end >> vStepLog);
U32 weight = sEnd - sStart;
if (weight < 1)
return (size_t)-FSE_ERROR_GENERIC;
norm[s] = weight;
tmpTotal = end;
}
}
}
return 0;
}
#endif
size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
const unsigned* count, size_t total,
@@ -656,10 +754,9 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
}
if (-stillToDistribute >= (normalizedCounter[largest] >> 1))
{
/* corner case, need to converge towards normalization with caution */
size_t errorCode = FSE_adjustNormSlow(normalizedCounter, -stillToDistribute, count, maxSymbolValue);
/* corner case, need another normalization method */
size_t errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
if (FSE_isError(errorCode)) return errorCode;
//FSE_adjustNormSlow(normalizedCounter, -stillToDistribute, count, maxSymbolValue);
}
else normalizedCounter[largest] += (short)stillToDistribute;
}
@@ -869,12 +966,6 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
}
static size_t FSE_compressRLE (BYTE *out, BYTE symbol)
{
*out=symbol;
return 1;
}
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
@@ -901,8 +992,8 @@ size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
/* Scan input and build symbol stats */
errorCode = FSE_count (count, ip, srcSize, &maxSymbolValue);
if (FSE_isError(errorCode)) return errorCode;
if (errorCode == srcSize) return FSE_compressRLE (ostart, *istart);
if (errorCode < ((srcSize * 7) >> 10)) return 0; /* Heuristic : not compressible enough */
if (errorCode == srcSize) return 1;
if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
errorCode = FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue);
lib/zstd.c
+56 -52
View File
@@ -68,9 +68,6 @@
#include <stdio.h> /* debug : printf */
#include "zstd_static.h"
#if defined(__clang__) || defined(__GNUC__)
# ifdef __clang__
# pragma clang diagnostic ignored "-Wtypedef-redefinition"
# endif
# include "fse.c" /* due to GCC/Clang inlining limitations, including *.c runs noticeably faster */
#else
# include "fse_static.h"
@@ -99,6 +96,8 @@
#endif
#ifndef MEM_ACCESS_MODULE
#define MEM_ACCESS_MODULE
/********************************************************
* Basic Types
*********************************************************/
@@ -119,6 +118,8 @@ typedef signed int S32;
typedef unsigned long long U64;
#endif
#endif /* MEM_ACCESS_MODULE */
/********************************************************
* Constants
@@ -129,9 +130,6 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51C; /* Initial (limited) frame for
#define HASH_TABLESIZE (1 << HASH_LOG)
#define HASH_MASK (HASH_TABLESIZE - 1)
#define MAXD_LOG 16
#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
#define KNUTH 2654435761
#define BIT7 128
@@ -141,10 +139,10 @@ static const U32 ZSTD_magicNumber = 0xFD2FB51C; /* Initial (limited) frame for
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<20)
#define GB *(1U<<30)
#define BLOCKSIZE (128 KB) /* define, for static allocation */
static const U32 g_maxDistance = 512 KB;
static const U32 g_maxDistance = 4 * BLOCKSIZE;
static const U32 g_maxLimit = 1 GB;
static const U32 g_searchStrength = 8;
@@ -616,25 +614,27 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
size_t errorCode;
const size_t minGain = ZSTD_minGain(srcSize);
// early out
/* early out */
if (dstSize < FSE_compressBound(srcSize)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
// Scan input and build symbol stats
/* Scan input and build symbol stats */
errorCode = FSE_count (count, ip, srcSize, &maxSymbolValue);
if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC;
if (errorCode == srcSize) return 1;
if (errorCode < ((srcSize * 7) >> 10)) return 0;
//if (errorCode < ((srcSize * 7) >> 10)) return 0;
//if (errorCode < (srcSize >> 7)) return 0;
if (errorCode < (srcSize >> 6)) return 0; /* heuristic : probably not compressible enough */
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
errorCode = (int)FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue);
if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC;
// Write table description header
/* Write table description header */
errorCode = FSE_writeHeader (op, FSE_MAX_HEADERSIZE, norm, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC;
op += errorCode;
// Compress
/* Compress */
errorCode = FSE_buildCTable (&CTable, norm, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC;
errorCode = ZSTD_compressLiterals_usingCTable(op, oend - op, ip, srcSize, &CTable);
@@ -1708,24 +1708,24 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
}
/******************************
/*******************************
* Streaming Decompression API
******************************/
*******************************/
typedef struct
{
U32 ctx[FSE_DTABLE_SIZE_U32(LLFSELog) + FSE_DTABLE_SIZE_U32(OffFSELog) + FSE_DTABLE_SIZE_U32(MLFSELog)];
size_t expected;
blockType_t bType;
U32 started;
U32 phase;
} dctx_t;
ZSTD_dctx_t ZSTD_createDCtx(void)
{
dctx_t* dctx = (dctx_t*)malloc(sizeof(dctx_t));
dctx->expected = 4 + ZSTD_blockHeaderSize; // Frame Header + Block Header
dctx->started = 0;
dctx->expected = ZSTD_frameHeaderSize;
dctx->phase = 0;
return (ZSTD_dctx_t)dctx;
}
@@ -1736,7 +1736,7 @@ size_t ZSTD_freeDCtx(ZSTD_dctx_t dctx)
}
size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx)
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_dctx_t dctx)
{
return ((dctx_t*)dctx)->expected;
}
@@ -1744,63 +1744,67 @@ size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx)
size_t ZSTD_decompressContinue(ZSTD_dctx_t dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
dctx_t* ctx = (dctx_t*)dctx;
size_t cSize = srcSize - ZSTD_blockHeaderSize;
size_t rSize;
// Sanity check
/* Sanity check */
if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_wrongSrcSize;
// Decompress
if (!ctx->started)
/* Decompress : frame header */
if (ctx->phase == 0)
{
// Just check correct magic header
/* Check frame magic header */
U32 magicNumber = ZSTD_readBE32(src);
if (magicNumber != ZSTD_magicNumber) return (size_t)-ZSTD_ERROR_wrongMagicNumber;
rSize = 0;
ctx->phase = 1;
ctx->expected = ZSTD_blockHeaderSize;
return 0;
}
else
/* Decompress : block header */
if (ctx->phase == 1)
{
blockProperties_t bp;
size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
if (ZSTD_isError(blockSize)) return blockSize;
if (bp.blockType == bt_end)
{
ctx->expected = 0;
ctx->phase = 0;
}
else
{
ctx->expected = blockSize;
ctx->bType = bp.blockType;
ctx->phase = 2;
}
return 0;
}
/* Decompress : block content */
{
size_t rSize;
switch(ctx->bType)
{
case bt_compressed:
rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, cSize);
rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize);
break;
case bt_raw :
rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, cSize);
rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, srcSize);
break;
case bt_rle :
return (size_t)-ZSTD_ERROR_GENERIC; /* not yet handled */
break;
case bt_end :
case bt_end : /* should never happen (filtered at phase 1) */
rSize = 0;
break;
default:
return (size_t)-ZSTD_ERROR_GENERIC;
}
ctx->phase = 1;
ctx->expected = ZSTD_blockHeaderSize;
return rSize;
}
// Prepare next block
{
const BYTE* header = (const BYTE*)src;
blockProperties_t bp;
size_t blockSize;
header += cSize;
blockSize = ZSTD_getcBlockSize(header, ZSTD_blockHeaderSize, &bp);
if (ZSTD_isError(blockSize)) return blockSize;
if (bp.blockType == bt_end)
{
ctx->expected = 0;
ctx->started = 0;
}
else
{
ctx->expected = blockSize + ZSTD_blockHeaderSize;
ctx->bType = bp.blockType;
ctx->started = 1;
}
}
return rSize;
}
lib/zstd.h
+1 -1
View File
@@ -47,7 +47,7 @@ extern "C" {
**************************************/
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
#define ZSTD_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */
#define ZSTD_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
unsigned ZSTD_versionNumber (void);
lib/zstd_static.h
+9 -2
View File
@@ -57,9 +57,16 @@ typedef void* ZSTD_dctx_t;
ZSTD_dctx_t ZSTD_createDCtx(void);
size_t ZSTD_freeDCtx(ZSTD_dctx_t dctx);
size_t ZSTD_getNextcBlockSize(ZSTD_dctx_t dctx);
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_dctx_t dctx);
size_t ZSTD_decompressContinue(ZSTD_dctx_t dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
/*
Use above functions alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as input to ZSTD_decompressContinue().
This value is expected to be provided, precisely, as 'srcSize'.
Otherwise, compression will fail (result is an error code, which can be tested using ZSTD_isError() )
ZSTD_decompressContinue() result is the number of bytes regenerated within 'dst'.
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
*/
/**************************************
* Error management
programs/fileio.c
+14 -11
View File
@@ -327,10 +327,10 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
/* Init */
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
dctx = ZSTD_createDCtx();
toRead = ZSTD_getNextcBlockSize(dctx);
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
/* check header */
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
sizeCheck = fread(header, (size_t)1, toRead, finput);
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
@@ -348,7 +348,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
/* Main decompression Loop */
toRead = ZSTD_getNextcBlockSize(dctx);
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
while (toRead)
{
size_t readSize, decodedSize;
@@ -361,16 +361,19 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
/* Decode block */
decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize);
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file");
filesize += decodedSize;
if (decodedSize) /* not a header */
{
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(35, "Write error : unable to write data block to destination file");
filesize += decodedSize;
op += decodedSize;
if (op==oend) op = outBuff;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) );
}
/* prepare for next Block */
op += decodedSize;
if (op==oend) op = outBuff;
toRead = ZSTD_getNextcBlockSize(dctx);
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) );
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
}
DISPLAYLEVEL(2, "\r%79s\r", "");