ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated streaming API

Browse Source
This commit is contained in:
Yann Collet
2016-08-17 01:39:22 +02:00
parent 94ca85d01b
commit 53e17fbd5e
5 changed files with 128 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+18 -25
View File
@@ -2960,16 +2960,15 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
}
}
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input)
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{
size_t sizeRead = input->size;
size_t sizeWritten = output->size;
size_t const result = ZSTD_compressStream_generic(zcs, output->ptr, &sizeWritten, input->ptr, &sizeRead, zsf_gather);
input->ptr = (const char*)(input->ptr) + sizeRead;
input->size -= sizeRead;
output->ptr = (char*)(output->ptr) + sizeWritten;
output->size -= sizeWritten;
output->nbBytesWritten += sizeWritten;
size_t sizeRead = input->size - input->pos;
size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs,
(char*)(output->dst) + output->pos, &sizeWritten,
(const char*)(input->src) + input->pos, &sizeRead, zsf_gather);
input->pos += sizeRead;
output->pos += sizeWritten;
return result;
}
@@ -2978,36 +2977,32 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor
/*! ZSTD_flushStream() :
* @return : amount of data remaining to flush */
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_wCursor* output)
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{
size_t srcSize = 0;
size_t sizeWritten = output->size;
size_t const result = ZSTD_compressStream_generic(zcs, output->ptr, &sizeWritten, &srcSize, &srcSize, zsf_flush); /* use a valid src address instead of NULL */
output->ptr = (char*)(output->ptr) + sizeWritten;
output->size -= sizeWritten;
output->nbBytesWritten += sizeWritten;
size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs, output->dst + output->pos, &sizeWritten, &srcSize, &srcSize, zsf_flush); /* use a valid src address instead of NULL */
output->pos += sizeWritten;
if (ZSTD_isError(result)) return result;
return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */
}
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_wCursor* output)
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
{
BYTE* const ostart = (BYTE*)(output->ptr);
BYTE* const oend = ostart + output->size;
BYTE* const ostart = (BYTE*)(output->dst) + output->pos;
BYTE* const oend = (BYTE*)(output->dst) + output->size;
BYTE* op = ostart;
if (zcs->stage != zcss_final) {
/* flush whatever remains */
size_t srcSize = 0;
size_t sizeWritten = output->size;
size_t sizeWritten = output->size - output->pos;
size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */
size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
op += sizeWritten;
if (remainingToFlush) {
output->ptr = op;
output->size -= sizeWritten;
output->nbBytesWritten += sizeWritten;
output->pos += sizeWritten;
return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4);
}
/* create epilogue */
@@ -3021,9 +3016,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_wCursor* output)
size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
op += flushed;
zcs->outBuffFlushedSize += flushed;
output->ptr = op;
output->size = oend-op;
output->nbBytesWritten += op-ostart;
output->pos += op-ostart;
if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */
return toFlush - flushed;
}
lib/decompress/zstd_decompress.c
+8 -12
View File
@@ -1393,13 +1393,13 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src,
}
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input)
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
{
const char* const istart = (const char*)(input->ptr);
const char* const iend = istart + input->size;
const char* const istart = (const char*)(input->src) + input->pos;
const char* const iend = (const char*)(input->src) + input->size;
const char* ip = istart;
char* const ostart = (char*)(output->ptr);
char* const oend = ostart + output->size;
char* const ostart = (char*)(output->dst) + output->pos;
char* const oend = (char*)(output->dst) + output->size;
char* op = ostart;
U32 someMoreWork = 1;
@@ -1417,8 +1417,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCurs
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
zds->lhSize += iend-ip;
input->ptr = iend;
input->size = 0;
input->pos = input->size;
return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
}
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
@@ -1522,11 +1521,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCurs
} }
/* result */
input->ptr = ip;
input->size = (size_t)(iend-ip);
output->ptr = op;
output->size = (size_t) (oend-op);
output->nbBytesWritten += (size_t)(op-ostart);
input->pos += (size_t)(ip-istart);
output->pos += (size_t)(op-ostart);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);
lib/zstd.h
+19 -18
View File
@@ -324,16 +324,17 @@ ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx);
* Streaming
********************************************************************/
typedef struct ZSTD_readCursor_s {
const void* ptr; /**< position of cursor - update to new position */
size_t size; /**< remaining buffer size to read - update preserves end of buffer */
} ZSTD_rCursor;
typedef struct ZSTD_inBuffer_s {
const void* src; /**< start of input buffer */
size_t size; /**< size of input buffer */
size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
} ZSTD_inBuffer;
typedef struct ZSTD_writeCursor_s {
void* ptr; /**< position of cursor - update to new position */
size_t size; /**< remaining buffer size to write - update preserves end of buffer */
size_t nbBytesWritten; /**< already written bytes - update adds bytes newly written (accumulator) */
} ZSTD_wCursor;
typedef struct ZSTD_outBuffer_s {
void* dst; /**< start of output buffer */
size_t size; /**< size of output buffer */
size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
} ZSTD_outBuffer;
/*====== compression ======*/
@@ -350,14 +351,14 @@ typedef struct ZSTD_writeCursor_s {
* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
*
* Use ZSTD_compressStream() repetitively to consume input stream.
* The function will automatically advance cursors.
* Note that it may not consume the entire input, in which case `input->size > 0`,
* The function will automatically update both `pos`.
* Note that it may not consume the entire input, in which case `pos < size`,
* and it's up to the caller to present again remaining data.
* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
* or an error code, which can be tested using ZSTD_isError().
*
* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
* Cursor will be updated.
* `output->pos` will be updated.
* Note some content might still be left within internal buffer if `output->size` is too small.
* @return : nb of bytes still present within internal buffer (0 if it's empty)
* or an error code, which can be tested using ZSTD_isError().
@@ -380,9 +381,9 @@ size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_wCursor* output, ZSTD_rCursor* input);
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_wCursor* output);
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_wCursor* output);
size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
/* advanced */
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
@@ -404,8 +405,8 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dic
* or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
*
* Use ZSTD_decompressStream() repetitively to consume your input.
* The function will update cursors.
* Note that it may not consume the entire input (input->size > 0),
* The function will update both `pos`.
* Note that it may not consume the entire input (pos < size),
* in which case it's up to the caller to present remaining input again.
* @return : 0 when a frame is completely decoded and fully flushed,
* 1 when there is still some data left within internal buffer to flush,
@@ -422,7 +423,7 @@ size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */
size_t ZSTD_initDStream(ZSTD_DStream* zds);
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_wCursor* output, ZSTD_rCursor* input);
size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
/* advanced */
ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);