diff --git a/contrib/seekable_format/tests/Makefile b/contrib/seekable_format/tests/Makefile index b00657f80..15eadb40e 100644 --- a/contrib/seekable_format/tests/Makefile +++ b/contrib/seekable_format/tests/Makefile @@ -16,13 +16,13 @@ ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME) CPPFLAGS += -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common CFLAGS ?= -O3 -CFLAGS += -g +CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wconversion \ + -Wformat=2 -Wstrict-aliasing=1 SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB) .PHONY: default clean test - -default: seekable_tests +default: test test: seekable_tests ./seekable_tests @@ -30,9 +30,9 @@ test: seekable_tests $(ZSTDLIB): $(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME) -seekable_tests : seekable_tests.c $(SEEKABLE_OBJS) +seekable_tests : $(SEEKABLE_OBJS) clean: - @rm -f core *.o tmp* result* *.zst \ + @$(RM) core *.o tmp* result* *.zst \ seekable_tests @echo Cleaning completed diff --git a/contrib/seekable_format/tests/seekable_tests.c b/contrib/seekable_format/tests/seekable_tests.c index 2a159a319..1d8871c26 100644 --- a/contrib/seekable_format/tests/seekable_tests.c +++ b/contrib/seekable_format/tests/seekable_tests.c @@ -1,6 +1,8 @@ #include #include +#include // malloc #include +#include #include "zstd_seekable.h" @@ -8,7 +10,83 @@ int main(int argc, const char** argv) { unsigned testNb = 1; + (void)argc; (void)argv; printf("Beginning zstd seekable format tests...\n"); + + printf("Test %u - simple round trip: ", testNb++); + { size_t const inSize = 4000; + void* const inBuffer = malloc(inSize); + assert(inBuffer != NULL); + + size_t const seekCapacity = 5000; + void* const seekBuffer = malloc(seekCapacity); + assert(seekBuffer != NULL); + size_t seekSize; + + size_t const outCapacity = inSize; + void* const outBuffer = malloc(outCapacity); + assert(outBuffer != NULL); + + ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream(); + assert(zscs != NULL); + + { size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, (unsigned)inSize /* maxFrameSize */); + assert(!ZSTD_isError(initStatus)); + } + + { ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity }; + ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize }; + + size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb); + assert(!ZSTD_isError(cStatus)); + assert(inb.pos == inb.size); + + size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb); + assert(!ZSTD_isError(endStatus)); + seekSize = outb.pos; + } + + ZSTD_seekable* const stream = ZSTD_seekable_create(); + assert(stream != NULL); + { size_t const initStatus = ZSTD_seekable_initBuff(stream, seekBuffer, seekSize); + assert(!ZSTD_isError(initStatus)); } + + { size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, outCapacity, 0); + assert(decStatus == inSize); } + + /* unit test ZSTD_seekTable functions */ + ZSTD_seekTable* const zst = ZSTD_seekTable_create_fromSeekable(stream); + assert(zst != NULL); + + unsigned const nbFrames = ZSTD_seekTable_getNumFrames(zst); + assert(nbFrames > 0); + + unsigned long long const frame0Offset = ZSTD_seekTable_getFrameCompressedOffset(zst, 0); + assert(frame0Offset == 0); + + unsigned long long const content0Offset = ZSTD_seekTable_getFrameDecompressedOffset(zst, 0); + assert(content0Offset == 0); + + size_t const cSize = ZSTD_seekTable_getFrameCompressedSize(zst, 0); + assert(!ZSTD_isError(cSize)); + assert(cSize <= seekCapacity); + + size_t const origSize = ZSTD_seekTable_getFrameDecompressedSize(zst, 0); + assert(origSize == inSize); + + unsigned const fo1idx = ZSTD_seekTable_offsetToFrameIndex(zst, 1); + assert(fo1idx == 0); + + free(inBuffer); + free(seekBuffer); + free(outBuffer); + ZSTD_seekable_freeCStream(zscs); + ZSTD_seekTable_free(zst); + ZSTD_seekable_free(stream); + } + printf("Success!\n"); + + printf("Test %u - check that seekable decompress does not hang: ", testNb++); { /* Github issue #2335 */ const size_t compressed_size = 17; @@ -34,20 +112,21 @@ int main(int argc, const char** argv) const size_t uncompressed_size = 32; uint8_t uncompressed_data[32]; - ZSTD_seekable* stream = ZSTD_seekable_create(); - size_t status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); - if (ZSTD_isError(status)) { - ZSTD_seekable_free(stream); - goto _test_error; - } + ZSTD_seekable* const stream = ZSTD_seekable_create(); + assert(stream != NULL); + { size_t const status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); + if (ZSTD_isError(status)) { + ZSTD_seekable_free(stream); + goto _test_error; + } } - const size_t offset = 2; /* Should return an error, but not hang */ - status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); - if (!ZSTD_isError(status)) { - ZSTD_seekable_free(stream); - goto _test_error; - } + { const size_t offset = 2; + size_t const status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); + if (!ZSTD_isError(status)) { + ZSTD_seekable_free(stream); + goto _test_error; + } } ZSTD_seekable_free(stream); } @@ -57,34 +136,34 @@ int main(int argc, const char** argv) { /* Github issue #FIXME */ const size_t compressed_size = 27; const uint8_t compressed_data[27] = { - '\x28', - '\xb5', - '\x2f', - '\xfd', - '\x00', - '\x32', - '\x91', - '\x00', - '\x00', - '\x00', - '\x5e', - '\x2a', - '\x4d', - '\x18', - '\x09', - '\x00', - '\x00', - '\x00', - '\x00', - '\x00', - '\x00', - '\x00', - '\x00', - '\xb1', - '\xea', - '\x92', - '\x8f', - }; + (uint8_t)'\x28', + (uint8_t)'\xb5', + (uint8_t)'\x2f', + (uint8_t)'\xfd', + (uint8_t)'\x00', + (uint8_t)'\x32', + (uint8_t)'\x91', + (uint8_t)'\x00', + (uint8_t)'\x00', + (uint8_t)'\x00', + (uint8_t)'\x5e', + (uint8_t)'\x2a', + (uint8_t)'\x4d', + (uint8_t)'\x18', + (uint8_t)'\x09', + (uint8_t)'\x00', + (uint8_t)'\x00', + (uint8_t)'\x00', + (uint8_t)'\x00', + (uint8_t)'\x00', + (uint8_t)'\x00', + (uint8_t)'\x00', + (uint8_t)'\x00', + (uint8_t)'\xb1', + (uint8_t)'\xea', + (uint8_t)'\x92', + (uint8_t)'\x8f', + }; const size_t uncompressed_size = 400; uint8_t uncompressed_data[400]; diff --git a/contrib/seekable_format/zstd_seekable.h b/contrib/seekable_format/zstd_seekable.h index 7ffd1ba0a..d2807cfbd 100644 --- a/contrib/seekable_format/zstd_seekable.h +++ b/contrib/seekable_format/zstd_seekable.h @@ -29,6 +29,7 @@ extern "C" { typedef struct ZSTD_seekable_CStream_s ZSTD_seekable_CStream; typedef struct ZSTD_seekable_s ZSTD_seekable; +typedef struct ZSTD_seekTable_s ZSTD_seekTable; /*-**************************************************************************** * Seekable compression - HowTo @@ -107,6 +108,7 @@ ZSTDLIB_API size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl); ZSTDLIB_API size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl, unsigned compressedSize, unsigned decompressedSize, unsigned checksum); ZSTDLIB_API size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output); + /*-**************************************************************************** * Seekable decompression - HowTo * A ZSTD_seekable object is required to tracking the seekTable. @@ -161,13 +163,42 @@ ZSTDLIB_API size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t ZSTDLIB_API size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned frameIndex); #define ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE (0ULL-2) -/*===== Seek Table access functions =====*/ -ZSTDLIB_API unsigned ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs); -ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex); -ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex); -ZSTDLIB_API size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, unsigned frameIndex); -ZSTDLIB_API size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, unsigned frameIndex); -ZSTDLIB_API unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long offset); +/*===== Seekable seek table access functions =====*/ +ZSTDLIB_API unsigned ZSTD_seekable_getNumFrames(const ZSTD_seekable* zs); +ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameCompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex); +ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameDecompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex); +ZSTDLIB_API size_t ZSTD_seekable_getFrameCompressedSize(const ZSTD_seekable* zs, unsigned frameIndex); +ZSTDLIB_API size_t ZSTD_seekable_getFrameDecompressedSize(const ZSTD_seekable* zs, unsigned frameIndex); +ZSTDLIB_API unsigned ZSTD_seekable_offsetToFrameIndex(const ZSTD_seekable* zs, unsigned long long offset); + + +/*-**************************************************************************** +* Direct exploitation of the seekTable +* +* Memory constrained use cases that manage multiple archives +* benefit from retaining multiple archive seek tables +* without retaining a ZSTD_seekable instance for each. +* +* Below API allow the above-mentioned use cases +* to initialize a ZSTD_seekable, extract its (smaller) ZSTD_seekTable, +* then throw the ZSTD_seekable away to save memory. +* +* Standard ZSTD operations can then be used +* to decompress frames based on seek table offsets. +******************************************************************************/ + +/*===== Independent seek table management =====*/ +ZSTDLIB_API ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs); +ZSTDLIB_API size_t ZSTD_seekTable_free(ZSTD_seekTable* st); + +/*===== Direct seek table access functions =====*/ +ZSTDLIB_API unsigned ZSTD_seekTable_getNumFrames(const ZSTD_seekTable* st); +ZSTDLIB_API unsigned long long ZSTD_seekTable_getFrameCompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex); +ZSTDLIB_API unsigned long long ZSTD_seekTable_getFrameDecompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex); +ZSTDLIB_API size_t ZSTD_seekTable_getFrameCompressedSize(const ZSTD_seekTable* st, unsigned frameIndex); +ZSTDLIB_API size_t ZSTD_seekTable_getFrameDecompressedSize(const ZSTD_seekTable* st, unsigned frameIndex); +ZSTDLIB_API unsigned ZSTD_seekTable_offsetToFrameIndex(const ZSTD_seekTable* st, unsigned long long offset); + /*===== Seekable advanced I/O API =====*/ typedef int(ZSTD_seekable_read)(void* opaque, void* buffer, size_t n); diff --git a/contrib/seekable_format/zstdseek_compress.c b/contrib/seekable_format/zstdseek_compress.c index 3db291887..d92917a62 100644 --- a/contrib/seekable_format/zstdseek_compress.c +++ b/contrib/seekable_format/zstdseek_compress.c @@ -19,6 +19,7 @@ #include "zstd.h" #include "zstd_errors.h" #include "mem.h" + #include "zstd_seekable.h" #define CHECK_Z(f) { size_t const ret = (f); if (ret != 0) return ret; } @@ -63,19 +64,18 @@ struct ZSTD_seekable_CStream_s { int writingSeekTable; }; -size_t ZSTD_seekable_frameLog_allocVec(ZSTD_frameLog* fl) +static size_t ZSTD_seekable_frameLog_allocVec(ZSTD_frameLog* fl) { /* allocate some initial space */ size_t const FRAMELOG_STARTING_CAPACITY = 16; fl->entries = (framelogEntry_t*)malloc( sizeof(framelogEntry_t) * FRAMELOG_STARTING_CAPACITY); if (fl->entries == NULL) return ERROR(memory_allocation); - fl->capacity = FRAMELOG_STARTING_CAPACITY; - + fl->capacity = (U32)FRAMELOG_STARTING_CAPACITY; return 0; } -size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl) +static size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl) { if (fl != NULL) free(fl->entries); return 0; @@ -83,7 +83,7 @@ size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl) ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag) { - ZSTD_frameLog* fl = malloc(sizeof(ZSTD_frameLog)); + ZSTD_frameLog* const fl = malloc(sizeof(ZSTD_frameLog)); if (fl == NULL) return NULL; if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl))) { @@ -106,10 +106,9 @@ size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl) return 0; } -ZSTD_seekable_CStream* ZSTD_seekable_createCStream() +ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void) { - ZSTD_seekable_CStream* zcs = malloc(sizeof(ZSTD_seekable_CStream)); - + ZSTD_seekable_CStream* const zcs = malloc(sizeof(ZSTD_seekable_CStream)); if (zcs == NULL) return NULL; memset(zcs, 0, sizeof(*zcs)); @@ -134,7 +133,6 @@ size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs) ZSTD_freeCStream(zcs->cstream); ZSTD_seekable_frameLog_freeVec(&zcs->framelog); free(zcs); - return 0; } @@ -152,9 +150,8 @@ size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, return ERROR(frameParameter_unsupported); } - zcs->maxFrameSize = maxFrameSize - ? maxFrameSize - : ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE; + zcs->maxFrameSize = maxFrameSize ? + maxFrameSize : ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE; zcs->framelog.checksumFlag = checksumFlag; if (zcs->framelog.checksumFlag) { @@ -204,7 +201,7 @@ size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output /* end the frame */ size_t ret = ZSTD_endStream(zcs->cstream, output); - zcs->frameCSize += output->pos - prevOutPos; + zcs->frameCSize += (U32)(output->pos - prevOutPos); /* need to flush before doing the rest */ if (ret) return ret; @@ -224,8 +221,7 @@ size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output zcs->frameDSize = 0; ZSTD_CCtx_reset(zcs->cstream, ZSTD_reset_session_only); - if (zcs->framelog.checksumFlag) - XXH64_reset(&zcs->xxhState, 0); + if (zcs->framelog.checksumFlag) XXH64_reset(&zcs->xxhState, 0); return 0; } @@ -248,8 +244,8 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* XXH64_update(&zcs->xxhState, inBase, inTmp.pos); } - zcs->frameCSize += output->pos - prevOutPos; - zcs->frameDSize += inTmp.pos; + zcs->frameCSize += (U32)(output->pos - prevOutPos); + zcs->frameDSize += (U32)inTmp.pos; input->pos += inTmp.pos; @@ -290,7 +286,7 @@ static inline size_t ZSTD_stwrite32(ZSTD_frameLog* fl, memcpy((BYTE*)output->dst + output->pos, tmp + (fl->seekTablePos - offset), lenWrite); output->pos += lenWrite; - fl->seekTablePos += lenWrite; + fl->seekTablePos += (U32)lenWrite; if (lenWrite < 4) return ZSTD_seekable_seekTableSize(fl) - fl->seekTablePos; } @@ -339,8 +335,7 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output) if (output->size - output->pos < 1) return seekTableLen - fl->seekTablePos; if (fl->seekTablePos < seekTableLen - 4) { - BYTE sfd = 0; - sfd |= (fl->checksumFlag) << 7; + BYTE const sfd = (BYTE)((fl->checksumFlag) << 7); ((BYTE*)output->dst)[output->pos] = sfd; output->pos++; diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c index ed0880ef8..748e76a3f 100644 --- a/contrib/seekable_format/zstdseek_decompress.c +++ b/contrib/seekable_format/zstdseek_decompress.c @@ -107,7 +107,8 @@ typedef struct { static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n) { - buffWrapper_t* buff = (buffWrapper_t*) opaque; + buffWrapper_t* const buff = (buffWrapper_t*)opaque; + assert(buff != NULL); if (buff->pos + n > buff->size) return -1; memcpy(buffer, (const BYTE*)buff->ptr + buff->pos, n); buff->pos += n; @@ -118,15 +119,17 @@ static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin) { buffWrapper_t* const buff = (buffWrapper_t*) opaque; unsigned long long newOffset; + assert(buff != NULL); switch (origin) { case SEEK_SET: - newOffset = offset; + assert(offset >= 0); + newOffset = (unsigned long long)offset; break; case SEEK_CUR: - newOffset = (unsigned long long)buff->pos + offset; + newOffset = (unsigned long long)((long long)buff->pos + offset); break; case SEEK_END: - newOffset = (unsigned long long)buff->size + offset; + newOffset = (unsigned long long)((long long)buff->size + offset); break; default: assert(0); /* not possible */ @@ -144,18 +147,18 @@ typedef struct { U32 checksum; } seekEntry_t; -typedef struct { +struct ZSTD_seekTable_s { seekEntry_t* entries; size_t tableLen; int checksumFlag; -} seekTable_t; +}; #define SEEKABLE_BUFF_SIZE ZSTD_BLOCKSIZE_MAX struct ZSTD_seekable_s { ZSTD_DStream* dstream; - seekTable_t seekTable; + ZSTD_seekTable seekTable; ZSTD_seekable_customFile src; U64 decompressedOffset; @@ -173,8 +176,7 @@ struct ZSTD_seekable_s { ZSTD_seekable* ZSTD_seekable_create(void) { - ZSTD_seekable* zs = malloc(sizeof(ZSTD_seekable)); - + ZSTD_seekable* const zs = malloc(sizeof(ZSTD_seekable)); if (zs == NULL) return NULL; /* also initializes stage to zsds_init */ @@ -195,7 +197,35 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs) ZSTD_freeDStream(zs->dstream); free(zs->seekTable.entries); free(zs); + return 0; +} +ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs) +{ + ZSTD_seekTable* const st = malloc(sizeof(ZSTD_seekTable)); + if (st==NULL) return NULL; + + st->checksumFlag = zs->seekTable.checksumFlag; + st->tableLen = zs->seekTable.tableLen; + + /* Allocate an extra entry at the end to match logic of initial allocation */ + size_t const entriesSize = sizeof(seekEntry_t) * (zs->seekTable.tableLen + 1); + seekEntry_t* const entries = (seekEntry_t*)malloc(entriesSize); + if (entries==NULL) { + free(st); + return NULL; + } + + memcpy(entries, zs->seekTable.entries, entriesSize); + st->entries = entries; + return st; +} + +size_t ZSTD_seekTable_free(ZSTD_seekTable* st) +{ + if (st == NULL) return 0; /* support free on null */ + free(st->entries); + free(st); return 0; } @@ -203,19 +233,24 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs) * Performs a binary search to find the last frame with a decompressed offset * <= pos * @return : the frame's index */ -unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos) +unsigned ZSTD_seekable_offsetToFrameIndex(const ZSTD_seekable* zs, unsigned long long pos) +{ + return ZSTD_seekTable_offsetToFrameIndex(&zs->seekTable, pos); +} + +unsigned ZSTD_seekTable_offsetToFrameIndex(const ZSTD_seekTable* st, unsigned long long pos) { U32 lo = 0; - U32 hi = (U32)zs->seekTable.tableLen; - assert(zs->seekTable.tableLen <= UINT_MAX); + U32 hi = (U32)st->tableLen; + assert(st->tableLen <= UINT_MAX); - if (pos >= zs->seekTable.entries[zs->seekTable.tableLen].dOffset) { - return (U32)zs->seekTable.tableLen; + if (pos >= st->entries[st->tableLen].dOffset) { + return (unsigned)st->tableLen; } while (lo + 1 < hi) { U32 const mid = lo + ((hi - lo) >> 1); - if (zs->seekTable.entries[mid].dOffset <= pos) { + if (st->entries[mid].dOffset <= pos) { lo = mid; } else { hi = mid; @@ -224,36 +259,61 @@ unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long return lo; } -unsigned ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs) +unsigned ZSTD_seekable_getNumFrames(const ZSTD_seekable* zs) { - assert(zs->seekTable.tableLen <= UINT_MAX); - return (unsigned)zs->seekTable.tableLen; + return ZSTD_seekTable_getNumFrames(&zs->seekTable); } -unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex) +unsigned ZSTD_seekTable_getNumFrames(const ZSTD_seekTable* st) { - if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE; - return zs->seekTable.entries[frameIndex].cOffset; + assert(st->tableLen <= UINT_MAX); + return (unsigned)st->tableLen; } -unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex) +unsigned long long ZSTD_seekable_getFrameCompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex) { - if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE; - return zs->seekTable.entries[frameIndex].dOffset; + return ZSTD_seekTable_getFrameCompressedOffset(&zs->seekTable, frameIndex); } -size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, unsigned frameIndex) +unsigned long long ZSTD_seekTable_getFrameCompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex) { - if (frameIndex >= zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge); - return zs->seekTable.entries[frameIndex + 1].cOffset - - zs->seekTable.entries[frameIndex].cOffset; + if (frameIndex >= st->tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE; + return st->entries[frameIndex].cOffset; } -size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, unsigned frameIndex) +unsigned long long ZSTD_seekable_getFrameDecompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex) { - if (frameIndex > zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge); - return zs->seekTable.entries[frameIndex + 1].dOffset - - zs->seekTable.entries[frameIndex].dOffset; + return ZSTD_seekTable_getFrameDecompressedOffset(&zs->seekTable, frameIndex); +} + +unsigned long long ZSTD_seekTable_getFrameDecompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex) +{ + if (frameIndex >= st->tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE; + return st->entries[frameIndex].dOffset; +} + +size_t ZSTD_seekable_getFrameCompressedSize(const ZSTD_seekable* zs, unsigned frameIndex) +{ + return ZSTD_seekTable_getFrameCompressedSize(&zs->seekTable, frameIndex); +} + +size_t ZSTD_seekTable_getFrameCompressedSize(const ZSTD_seekTable* st, unsigned frameIndex) +{ + if (frameIndex >= st->tableLen) return ERROR(frameIndex_tooLarge); + return st->entries[frameIndex + 1].cOffset - + st->entries[frameIndex].cOffset; +} + +size_t ZSTD_seekable_getFrameDecompressedSize(const ZSTD_seekable* zs, unsigned frameIndex) +{ + return ZSTD_seekTable_getFrameDecompressedSize(&zs->seekTable, frameIndex); +} + +size_t ZSTD_seekTable_getFrameDecompressedSize(const ZSTD_seekTable* st, unsigned frameIndex) +{ + if (frameIndex > st->tableLen) return ERROR(frameIndex_tooLarge); + return st->entries[frameIndex + 1].dOffset - + st->entries[frameIndex].dOffset; } static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs) @@ -274,8 +334,7 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs) /* check reserved bits */ if ((checksumFlag >> 2) & 0x1f) { return ERROR(corruption_detected); - } - } + } } { U32 const numFrames = MEM_readLE32(zs->inBuff); U32 const sizePerEntry = 8 + (checksumFlag?4:0); @@ -283,12 +342,9 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs) U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_SKIPPABLEHEADERSIZE; U32 remaining = frameSize - ZSTD_seekTableFooterSize; /* don't need to re-read footer */ - { - U32 const toRead = MIN(remaining, SEEKABLE_BUFF_SIZE); - + { U32 const toRead = MIN(remaining, SEEKABLE_BUFF_SIZE); CHECK_IO(src.seek(src.opaque, -(S64)frameSize, SEEK_END)); CHECK_IO(src.read(src.opaque, zs->inBuff, toRead)); - remaining -= toRead; } @@ -301,19 +357,15 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs) { /* Allocate an extra entry at the end so that we can do size * computations on the last element without special case */ - seekEntry_t* entries = (seekEntry_t*)malloc(sizeof(seekEntry_t) * (numFrames + 1)); + seekEntry_t* const entries = (seekEntry_t*)malloc(sizeof(seekEntry_t) * (numFrames + 1)); U32 idx = 0; U32 pos = 8; - U64 cOffset = 0; U64 dOffset = 0; - if (!entries) { - free(entries); - return ERROR(memory_allocation); - } + if (entries == NULL) return ERROR(memory_allocation); /* compute cumulative positions */ for (; idx < numFrames; idx++) { @@ -390,8 +442,9 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign zs->decompressedOffset = zs->seekTable.entries[targetFrame].dOffset; zs->curFrame = targetFrame; + assert(zs->seekTable.entries[targetFrame].cOffset < LLONG_MAX); CHECK_IO(zs->src.seek(zs->src.opaque, - zs->seekTable.entries[targetFrame].cOffset, + (long long)zs->seekTable.entries[targetFrame].cOffset, SEEK_SET)); zs->in = (ZSTD_inBuffer){zs->inBuff, 0, 0}; XXH64_reset(&zs->xxhState, 0); @@ -460,7 +513,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign zs->in.size = toRead; zs->in.pos = 0; } - } + } /* while (zs->decompressedOffset < offset + len) */ } while (zs->decompressedOffset != offset + len); return len; @@ -472,8 +525,7 @@ size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSiz return ERROR(frameIndex_tooLarge); } - { - size_t const decompressedSize = + { size_t const decompressedSize = zs->seekTable.entries[frameIndex + 1].dOffset - zs->seekTable.entries[frameIndex].dOffset; if (dstSize < decompressedSize) { diff --git a/lib/common/mem.h b/lib/common/mem.h index 9813bfc42..961f89849 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -308,7 +308,7 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) MEM_STATIC U32 MEM_readLE24(const void* memPtr) { - return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); } MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)