ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added ZWRAP_DCtx.decompState

Browse Source
This commit is contained in:
inikep
2016-09-20 16:22:28 +02:00
parent 554b3b935c
commit 86fc8e0003
3 changed files with 42 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
zlibWrapper/Makefile
+1 -1
View File
@@ -37,7 +37,7 @@ testzstd: example_zstd
testfitblk: fitblk testfitblk: fitblk
./fitblk 10240 <../zstd_compression_format.md ./fitblk 10240 <../zstd_compression_format.md
#./fitblk 40960 <../zstd_compression_format.md ./fitblk 40960 <../zstd_compression_format.md
.c.o: .c.o:
$(CC) $(CFLAGS) -c -o $@ $< $(CC) $(CFLAGS) -c -o $@ $<
zlibWrapper/examples/fitblk.c
+14 -12
View File
@@ -57,6 +57,7 @@
//#include "zlib.h" //#include "zlib.h"
#include "zstd_zlibwrapper.h" #include "zstd_zlibwrapper.h"
#define LOG_FITBLK(...) /*printf(__VA_ARGS__)*/
#define local static #define local static
/* print nastygram and leave */ /* print nastygram and leave */
@@ -79,14 +80,14 @@ local int partcompress(FILE *in, z_streamp def)
flush = Z_SYNC_FLUSH; flush = Z_SYNC_FLUSH;
do { do {
def->avail_in = fread(raw, 1, RAWLEN, in); def->avail_in = fread(raw, 1, RAWLEN, in);
printf("partcompress def->avail_in=%d\n", def->avail_in);
if (ferror(in)) if (ferror(in))
return Z_ERRNO; return Z_ERRNO;
def->next_in = raw; def->next_in = raw;
if (feof(in)) if (feof(in))
flush = Z_FINISH; flush = Z_FINISH;
LOG_FITBLK("partcompress1 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
ret = deflate(def, flush); ret = deflate(def, flush);
printf("partcompress def->avail_out=%d\n", def->avail_out); LOG_FITBLK("partcompress2 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
assert(ret != Z_STREAM_ERROR); assert(ret != Z_STREAM_ERROR);
} while (def->avail_out != 0 && flush == Z_SYNC_FLUSH); } while (def->avail_out != 0 && flush == Z_SYNC_FLUSH);
return ret; return ret;
@@ -105,10 +106,10 @@ local int recompress(z_streamp inf, z_streamp def)
do { do {
/* decompress */ /* decompress */
inf->avail_out = RAWLEN; inf->avail_out = RAWLEN;
printf("recompress inf->avail_out=%d\n", inf->avail_out);
inf->next_out = raw; inf->next_out = raw;
LOG_FITBLK("recompress1inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out);
ret = inflate(inf, Z_NO_FLUSH); ret = inflate(inf, Z_NO_FLUSH);
LOG_FITBLK("recompress2inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out);
assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR && assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR &&
ret != Z_NEED_DICT); ret != Z_NEED_DICT);
if (ret == Z_MEM_ERROR) if (ret == Z_MEM_ERROR)
@@ -119,9 +120,10 @@ local int recompress(z_streamp inf, z_streamp def)
def->next_in = raw; def->next_in = raw;
if (inf->avail_out != 0) if (inf->avail_out != 0)
flush = Z_FINISH; flush = Z_FINISH;
LOG_FITBLK("recompress1deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
ret = deflate(def, flush); ret = deflate(def, flush);
LOG_FITBLK("recompress2deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
assert(ret != Z_STREAM_ERROR); assert(ret != Z_STREAM_ERROR);
printf("recompress def->avail_out=%d ret=%d\n", def->avail_out, ret);
} while (ret != Z_STREAM_END && def->avail_out != 0); } while (ret != Z_STREAM_END && def->avail_out != 0);
return ret; return ret;
} }
@@ -149,8 +151,7 @@ int main(int argc, char **argv)
quit("need positive size of 8 or greater"); quit("need positive size of 8 or greater");
size = (unsigned)ret; size = (unsigned)ret;
printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n", printf("zlib version %s\n", ZLIB_VERSION);
ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags());
if (isUsingZSTD()) printf("zstd version %s\n", zstdVersion()); if (isUsingZSTD()) printf("zstd version %s\n", zstdVersion());
/* allocate memory for buffers and compression engine */ /* allocate memory for buffers and compression engine */
@@ -165,11 +166,12 @@ int main(int argc, char **argv)
/* compress from stdin until output full, or no more input */ /* compress from stdin until output full, or no more input */
def.avail_out = size + EXCESS; def.avail_out = size + EXCESS;
def.next_out = blk; def.next_out = blk;
LOG_FITBLK("partcompress1 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out);
ret = partcompress(stdin, &def); ret = partcompress(stdin, &def);
LOG_FITBLK("partcompress2 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out);
if (ret == Z_ERRNO) if (ret == Z_ERRNO)
quit("error reading input"); quit("error reading input");
printf("partcompress def.total_out=%d ret=%d\n", (int)def.total_out, ret);
/* if it all fit, then size was undersubscribed -- done! */ /* if it all fit, then size was undersubscribed -- done! */
if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { if (ret == Z_STREAM_END && def.avail_out >= EXCESS) {
/* write block to stdout */ /* write block to stdout */
@@ -205,9 +207,9 @@ printf("partcompress def.total_out=%d ret=%d\n", (int)def.total_out, ret);
inf.next_in = blk; inf.next_in = blk;
def.avail_out = size + EXCESS; def.avail_out = size + EXCESS;
def.next_out = tmp; def.next_out = tmp;
printf("recompress1 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
ret = recompress(&inf, &def); ret = recompress(&inf, &def);
printf("recompress1 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
if (ret == Z_MEM_ERROR) if (ret == Z_MEM_ERROR)
quit("out of memory"); quit("out of memory");
@@ -222,9 +224,9 @@ printf("recompress1 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail
inf.next_in = tmp; inf.next_in = tmp;
def.avail_out = size; def.avail_out = size;
def.next_out = blk; def.next_out = blk;
printf("recompress2 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
ret = recompress(&inf, &def); ret = recompress(&inf, &def);
printf("recompress2 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
if (ret == Z_MEM_ERROR) if (ret == Z_MEM_ERROR)
quit("out of memory"); quit("out of memory");
assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */ assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */
zlibWrapper/zstd_zlibwrapper.c
+27 -12
View File
@@ -23,7 +23,7 @@
#define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ #define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */
#define LOG_WRAPPERC(...) /*printf(__VA_ARGS__)*/ #define LOG_WRAPPERC(...) /*printf(__VA_ARGS__)*/
#define LOG_WRAPPERD(...) /*printf(__VA_ARGS__)*/ #define LOG_WRAPPERD(...) /*printf(__VA_ARGS__)*/
#define FINISH_WITH_GZ_ERR(msg) { \ #define FINISH_WITH_GZ_ERR(msg) { \
@@ -323,6 +323,9 @@ typedef struct {
ZSTD_DStream* zbd; ZSTD_DStream* zbd;
char headerBuf[16]; /* should be equal or bigger than ZSTD_frameHeaderSize_min */ char headerBuf[16]; /* should be equal or bigger than ZSTD_frameHeaderSize_min */
int errorCount; int errorCount;
int decompState;
ZSTD_inBuffer inBuffer;
ZSTD_outBuffer outBuffer;
/* zlib params */ /* zlib params */
int stream_size; int stream_size;
@@ -330,11 +333,16 @@ typedef struct {
int windowBits; int windowBits;
ZSTD_customMem customMem; ZSTD_customMem customMem;
z_stream allocFunc; /* copy of zalloc, zfree, opaque */ z_stream allocFunc; /* copy of zalloc, zfree, opaque */
ZSTD_inBuffer inBuffer;
ZSTD_outBuffer outBuffer;
} ZWRAP_DCtx; } ZWRAP_DCtx;
void ZWRAP_initDCtx(ZWRAP_DCtx* zwd)
{
zwd->errorCount = zwd->decompState = 0;
zwd->outBuffer.pos = 0;
zwd->outBuffer.size = 0;
}
ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
{ {
ZWRAP_DCtx* zwd; ZWRAP_DCtx* zwd;
@@ -353,9 +361,8 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
memset(zwd, 0, sizeof(ZWRAP_DCtx)); memset(zwd, 0, sizeof(ZWRAP_DCtx));
memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
} }
zwd->outBuffer.pos = 0;
zwd->outBuffer.size = 0;
ZWRAP_initDCtx(zwd);
return zwd; return zwd;
} }
@@ -422,6 +429,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
if (zwd == NULL) return Z_STREAM_ERROR; if (zwd == NULL) return Z_STREAM_ERROR;
{ size_t const errorCode = ZSTD_resetDStream(zwd->zbd); { size_t const errorCode = ZSTD_resetDStream(zwd->zbd);
if (ZSTD_isError(errorCode)) return ZWRAPD_finish_with_error(zwd, strm, 0); } if (ZSTD_isError(errorCode)) return ZWRAPD_finish_with_error(zwd, strm, 0); }
ZWRAP_initDCtx(zwd);
} }
strm->total_in = 0; strm->total_in = 0;
@@ -470,7 +478,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
if (!strm->reserved) { if (!strm->reserved) {
LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
res = inflate(strm, flush); res = inflate(strm, flush);
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
return res; return res;
} }
@@ -479,6 +487,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
if (strm->state == NULL) return Z_STREAM_ERROR; if (strm->state == NULL) return Z_STREAM_ERROR;
LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END;
// if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZLIB_HEADERSIZE)) // if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZLIB_HEADERSIZE))
if (strm->total_in < ZLIB_HEADERSIZE) if (strm->total_in < ZLIB_HEADERSIZE)
{ {
@@ -524,7 +535,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
if (flush == Z_INFLATE_SYNC) res = inflateSync(strm); if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
else res = inflate(strm, flush); else res = inflate(strm, flush);
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
return res; return res;
} }
} }
@@ -558,7 +569,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
if (ZSTD_isError(errorCode)) { if (ZSTD_isError(errorCode)) {
LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode));
goto error; goto error;
} }
// LOG_WRAPPERD("1srcSize=%d inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)srcSize, (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); // LOG_WRAPPERD("1srcSize=%d inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)srcSize, (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos);
@@ -573,26 +584,30 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
zwd->outBuffer.size = strm->avail_out; zwd->outBuffer.size = strm->avail_out;
zwd->outBuffer.pos = 0; zwd->outBuffer.pos = 0;
errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
// LOG_WRAPPERD("2 inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos);
LOG_WRAPPERD("inflate ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)strm->avail_in, (int)strm->avail_out); LOG_WRAPPERD("inflate ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)strm->avail_in, (int)strm->avail_out);
if (ZSTD_isError(errorCode)) { if (ZSTD_isError(errorCode)) {
LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode));
zwd->errorCount++; zwd->errorCount++;
LOG_WRAPPERD("ERROR: ZSTD_decompressStream2 %s zwd->errorCount=%d\n", ZSTD_getErrorName(errorCode), zwd->errorCount);
if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error; if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error;
} }
LOG_WRAPPERD("inflate inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n", (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size);
strm->next_out += zwd->outBuffer.pos; strm->next_out += zwd->outBuffer.pos;
strm->total_out += zwd->outBuffer.pos; strm->total_out += zwd->outBuffer.pos;
strm->avail_out -= zwd->outBuffer.pos; strm->avail_out -= zwd->outBuffer.pos;
strm->total_in += zwd->inBuffer.pos - inPos; strm->total_in += zwd->inBuffer.pos - inPos;
strm->next_in += zwd->inBuffer.pos - inPos; strm->next_in += zwd->inBuffer.pos - inPos;
strm->avail_in -= zwd->inBuffer.pos - inPos; strm->avail_in -= zwd->inBuffer.pos - inPos;
if (errorCode == 0) { LOG_WRAPPERD("inflate Z_STREAM_END1 strm->total_in=%d strm->avail_out=%d strm->total_out=%d\n", (int)strm->total_in, (int)strm->avail_out, (int)strm->total_out); return Z_STREAM_END; } if (errorCode == 0) {
LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
zwd->decompState = Z_STREAM_END;
return Z_STREAM_END;
}
goto finish; goto finish;
error: error:
return ZWRAPD_finish_with_error(zwd, strm, 0); return ZWRAPD_finish_with_error(zwd, strm, 0);
} }
finish: finish:
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK);
return Z_OK; return Z_OK;
} }