ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2518 from facebook/seekTable

Browse Source 
New direct seekTable access methods
This commit is contained in:
Yann Collet
2021-03-04 15:29:23 -08:00
committed by GitHub
parent 0388054aab 2fa4c8c405
commit c4d54ab9bf
6 changed files with 278 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/seekable_format/tests/Makefile
+5 -5
View File
@@ -16,13 +16,13 @@ ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME)
CPPFLAGS += -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common CPPFLAGS += -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common
CFLAGS ?= -O3 CFLAGS ?= -O3
CFLAGS += -g CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wconversion \
-Wformat=2 -Wstrict-aliasing=1
SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB) SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB)
.PHONY: default clean test .PHONY: default clean test
default: test
default: seekable_tests
test: seekable_tests test: seekable_tests
./seekable_tests ./seekable_tests
@@ -30,9 +30,9 @@ test: seekable_tests
$(ZSTDLIB): $(ZSTDLIB):
$(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME) $(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
seekable_tests : seekable_tests.c $(SEEKABLE_OBJS) seekable_tests : $(SEEKABLE_OBJS)
clean: clean:
@rm -f core *.o tmp* result* *.zst \ @$(RM) core *.o tmp* result* *.zst \
seekable_tests seekable_tests
@echo Cleaning completed @echo Cleaning completed
contrib/seekable_format/tests/seekable_tests.c
+119 -40
View File
@@ -1,6 +1,8 @@
#include <stddef.h> #include <stddef.h>
#include <stdint.h> #include <stdint.h>
#include <stdlib.h> // malloc
#include <stdio.h> #include <stdio.h>
#include <assert.h>
#include "zstd_seekable.h" #include "zstd_seekable.h"
@@ -8,7 +10,83 @@
int main(int argc, const char** argv) int main(int argc, const char** argv)
{ {
unsigned testNb = 1; unsigned testNb = 1;
(void)argc; (void)argv;
printf("Beginning zstd seekable format tests...\n"); printf("Beginning zstd seekable format tests...\n");
printf("Test %u - simple round trip: ", testNb++);
{ size_t const inSize = 4000;
void* const inBuffer = malloc(inSize);
assert(inBuffer != NULL);
size_t const seekCapacity = 5000;
void* const seekBuffer = malloc(seekCapacity);
assert(seekBuffer != NULL);
size_t seekSize;
size_t const outCapacity = inSize;
void* const outBuffer = malloc(outCapacity);
assert(outBuffer != NULL);
ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream();
assert(zscs != NULL);
{ size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, (unsigned)inSize /* maxFrameSize */);
assert(!ZSTD_isError(initStatus));
}
{ ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity };
ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize };
size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb);
assert(!ZSTD_isError(cStatus));
assert(inb.pos == inb.size);
size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb);
assert(!ZSTD_isError(endStatus));
seekSize = outb.pos;
}
ZSTD_seekable* const stream = ZSTD_seekable_create();
assert(stream != NULL);
{ size_t const initStatus = ZSTD_seekable_initBuff(stream, seekBuffer, seekSize);
assert(!ZSTD_isError(initStatus)); }
{ size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, outCapacity, 0);
assert(decStatus == inSize); }
/* unit test ZSTD_seekTable functions */
ZSTD_seekTable* const zst = ZSTD_seekTable_create_fromSeekable(stream);
assert(zst != NULL);
unsigned const nbFrames = ZSTD_seekTable_getNumFrames(zst);
assert(nbFrames > 0);
unsigned long long const frame0Offset = ZSTD_seekTable_getFrameCompressedOffset(zst, 0);
assert(frame0Offset == 0);
unsigned long long const content0Offset = ZSTD_seekTable_getFrameDecompressedOffset(zst, 0);
assert(content0Offset == 0);
size_t const cSize = ZSTD_seekTable_getFrameCompressedSize(zst, 0);
assert(!ZSTD_isError(cSize));
assert(cSize <= seekCapacity);
size_t const origSize = ZSTD_seekTable_getFrameDecompressedSize(zst, 0);
assert(origSize == inSize);
unsigned const fo1idx = ZSTD_seekTable_offsetToFrameIndex(zst, 1);
assert(fo1idx == 0);
free(inBuffer);
free(seekBuffer);
free(outBuffer);
ZSTD_seekable_freeCStream(zscs);
ZSTD_seekTable_free(zst);
ZSTD_seekable_free(stream);
}
printf("Success!\n");
printf("Test %u - check that seekable decompress does not hang: ", testNb++); printf("Test %u - check that seekable decompress does not hang: ", testNb++);
{ /* Github issue #2335 */ { /* Github issue #2335 */
const size_t compressed_size = 17; const size_t compressed_size = 17;
@@ -34,20 +112,21 @@ int main(int argc, const char** argv)
const size_t uncompressed_size = 32; const size_t uncompressed_size = 32;
uint8_t uncompressed_data[32]; uint8_t uncompressed_data[32];
ZSTD_seekable* stream = ZSTD_seekable_create(); ZSTD_seekable* const stream = ZSTD_seekable_create();
size_t status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); assert(stream != NULL);
if (ZSTD_isError(status)) { { size_t const status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size);
ZSTD_seekable_free(stream); if (ZSTD_isError(status)) {
goto _test_error; ZSTD_seekable_free(stream);
} goto _test_error;
} }
const size_t offset = 2;
/* Should return an error, but not hang */ /* Should return an error, but not hang */
status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); { const size_t offset = 2;
if (!ZSTD_isError(status)) { size_t const status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset);
ZSTD_seekable_free(stream); if (!ZSTD_isError(status)) {
goto _test_error; ZSTD_seekable_free(stream);
} goto _test_error;
} }
ZSTD_seekable_free(stream); ZSTD_seekable_free(stream);
} }
@@ -57,34 +136,34 @@ int main(int argc, const char** argv)
{ /* Github issue #FIXME */ { /* Github issue #FIXME */
const size_t compressed_size = 27; const size_t compressed_size = 27;
const uint8_t compressed_data[27] = { const uint8_t compressed_data[27] = {
'\x28', (uint8_t)'\x28',
'\xb5', (uint8_t)'\xb5',
'\x2f', (uint8_t)'\x2f',
'\xfd', (uint8_t)'\xfd',
'\x00', (uint8_t)'\x00',
'\x32', (uint8_t)'\x32',
'\x91', (uint8_t)'\x91',
'\x00', (uint8_t)'\x00',
'\x00', (uint8_t)'\x00',
'\x00', (uint8_t)'\x00',
'\x5e', (uint8_t)'\x5e',
'\x2a', (uint8_t)'\x2a',
'\x4d', (uint8_t)'\x4d',
'\x18', (uint8_t)'\x18',
'\x09', (uint8_t)'\x09',
'\x00', (uint8_t)'\x00',
'\x00', (uint8_t)'\x00',
'\x00', (uint8_t)'\x00',
'\x00', (uint8_t)'\x00',
'\x00', (uint8_t)'\x00',
'\x00', (uint8_t)'\x00',
'\x00', (uint8_t)'\x00',
'\x00', (uint8_t)'\x00',
'\xb1', (uint8_t)'\xb1',
'\xea', (uint8_t)'\xea',
'\x92', (uint8_t)'\x92',
'\x8f', (uint8_t)'\x8f',
}; };
const size_t uncompressed_size = 400; const size_t uncompressed_size = 400;
uint8_t uncompressed_data[400]; uint8_t uncompressed_data[400];
contrib/seekable_format/zstd_seekable.h
+38 -7
View File
@@ -29,6 +29,7 @@ extern "C" {
typedef struct ZSTD_seekable_CStream_s ZSTD_seekable_CStream; typedef struct ZSTD_seekable_CStream_s ZSTD_seekable_CStream;
typedef struct ZSTD_seekable_s ZSTD_seekable; typedef struct ZSTD_seekable_s ZSTD_seekable;
typedef struct ZSTD_seekTable_s ZSTD_seekTable;
/*-**************************************************************************** /*-****************************************************************************
* Seekable compression - HowTo * Seekable compression - HowTo
@@ -107,6 +108,7 @@ ZSTDLIB_API size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl);
ZSTDLIB_API size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl, unsigned compressedSize, unsigned decompressedSize, unsigned checksum); ZSTDLIB_API size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl, unsigned compressedSize, unsigned decompressedSize, unsigned checksum);
ZSTDLIB_API size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output); ZSTDLIB_API size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output);
/*-**************************************************************************** /*-****************************************************************************
* Seekable decompression - HowTo * Seekable decompression - HowTo
* A ZSTD_seekable object is required to tracking the seekTable. * A ZSTD_seekable object is required to tracking the seekTable.
@@ -161,13 +163,42 @@ ZSTDLIB_API size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t
ZSTDLIB_API size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned frameIndex); ZSTDLIB_API size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned frameIndex);
#define ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE (0ULL-2) #define ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE (0ULL-2)
/*===== Seek Table access functions =====*/ /*===== Seekable seek table access functions =====*/
ZSTDLIB_API unsigned ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs); ZSTDLIB_API unsigned ZSTD_seekable_getNumFrames(const ZSTD_seekable* zs);
ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex); ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameCompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex);
ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex); ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameDecompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex);
ZSTDLIB_API size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, unsigned frameIndex); ZSTDLIB_API size_t ZSTD_seekable_getFrameCompressedSize(const ZSTD_seekable* zs, unsigned frameIndex);
ZSTDLIB_API size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, unsigned frameIndex); ZSTDLIB_API size_t ZSTD_seekable_getFrameDecompressedSize(const ZSTD_seekable* zs, unsigned frameIndex);
ZSTDLIB_API unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long offset); ZSTDLIB_API unsigned ZSTD_seekable_offsetToFrameIndex(const ZSTD_seekable* zs, unsigned long long offset);
/*-****************************************************************************
* Direct exploitation of the seekTable
*
* Memory constrained use cases that manage multiple archives
* benefit from retaining multiple archive seek tables
* without retaining a ZSTD_seekable instance for each.
*
* Below API allow the above-mentioned use cases
* to initialize a ZSTD_seekable, extract its (smaller) ZSTD_seekTable,
* then throw the ZSTD_seekable away to save memory.
*
* Standard ZSTD operations can then be used
* to decompress frames based on seek table offsets.
******************************************************************************/
/*===== Independent seek table management =====*/
ZSTDLIB_API ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs);
ZSTDLIB_API size_t ZSTD_seekTable_free(ZSTD_seekTable* st);
/*===== Direct seek table access functions =====*/
ZSTDLIB_API unsigned ZSTD_seekTable_getNumFrames(const ZSTD_seekTable* st);
ZSTDLIB_API unsigned long long ZSTD_seekTable_getFrameCompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex);
ZSTDLIB_API unsigned long long ZSTD_seekTable_getFrameDecompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex);
ZSTDLIB_API size_t ZSTD_seekTable_getFrameCompressedSize(const ZSTD_seekTable* st, unsigned frameIndex);
ZSTDLIB_API size_t ZSTD_seekTable_getFrameDecompressedSize(const ZSTD_seekTable* st, unsigned frameIndex);
ZSTDLIB_API unsigned ZSTD_seekTable_offsetToFrameIndex(const ZSTD_seekTable* st, unsigned long long offset);
/*===== Seekable advanced I/O API =====*/ /*===== Seekable advanced I/O API =====*/
typedef int(ZSTD_seekable_read)(void* opaque, void* buffer, size_t n); typedef int(ZSTD_seekable_read)(void* opaque, void* buffer, size_t n);
contrib/seekable_format/zstdseek_compress.c
+15 -20
View File
@@ -19,6 +19,7 @@
#include "zstd.h" #include "zstd.h"
#include "zstd_errors.h" #include "zstd_errors.h"
#include "mem.h" #include "mem.h"
#include "zstd_seekable.h" #include "zstd_seekable.h"
#define CHECK_Z(f) { size_t const ret = (f); if (ret != 0) return ret; } #define CHECK_Z(f) { size_t const ret = (f); if (ret != 0) return ret; }
@@ -63,19 +64,18 @@ struct ZSTD_seekable_CStream_s {
int writingSeekTable; int writingSeekTable;
}; };
size_t ZSTD_seekable_frameLog_allocVec(ZSTD_frameLog* fl) static size_t ZSTD_seekable_frameLog_allocVec(ZSTD_frameLog* fl)
{ {
/* allocate some initial space */ /* allocate some initial space */
size_t const FRAMELOG_STARTING_CAPACITY = 16; size_t const FRAMELOG_STARTING_CAPACITY = 16;
fl->entries = (framelogEntry_t*)malloc( fl->entries = (framelogEntry_t*)malloc(
sizeof(framelogEntry_t) * FRAMELOG_STARTING_CAPACITY); sizeof(framelogEntry_t) * FRAMELOG_STARTING_CAPACITY);
if (fl->entries == NULL) return ERROR(memory_allocation); if (fl->entries == NULL) return ERROR(memory_allocation);
fl->capacity = FRAMELOG_STARTING_CAPACITY; fl->capacity = (U32)FRAMELOG_STARTING_CAPACITY;
return 0; return 0;
} }
size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl) static size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl)
{ {
if (fl != NULL) free(fl->entries); if (fl != NULL) free(fl->entries);
return 0; return 0;
@@ -83,7 +83,7 @@ size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog* fl)
ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag) ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag)
{ {
ZSTD_frameLog* fl = malloc(sizeof(ZSTD_frameLog)); ZSTD_frameLog* const fl = malloc(sizeof(ZSTD_frameLog));
if (fl == NULL) return NULL; if (fl == NULL) return NULL;
if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl))) { if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl))) {
@@ -106,10 +106,9 @@ size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl)
return 0; return 0;
} }
ZSTD_seekable_CStream* ZSTD_seekable_createCStream() ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void)
{ {
ZSTD_seekable_CStream* zcs = malloc(sizeof(ZSTD_seekable_CStream)); ZSTD_seekable_CStream* const zcs = malloc(sizeof(ZSTD_seekable_CStream));
if (zcs == NULL) return NULL; if (zcs == NULL) return NULL;
memset(zcs, 0, sizeof(*zcs)); memset(zcs, 0, sizeof(*zcs));
@@ -134,7 +133,6 @@ size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs)
ZSTD_freeCStream(zcs->cstream); ZSTD_freeCStream(zcs->cstream);
ZSTD_seekable_frameLog_freeVec(&zcs->framelog); ZSTD_seekable_frameLog_freeVec(&zcs->framelog);
free(zcs); free(zcs);
return 0; return 0;
} }
@@ -152,9 +150,8 @@ size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
return ERROR(frameParameter_unsupported); return ERROR(frameParameter_unsupported);
} }
zcs->maxFrameSize = maxFrameSize zcs->maxFrameSize = maxFrameSize ?
? maxFrameSize maxFrameSize : ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE;
: ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE;
zcs->framelog.checksumFlag = checksumFlag; zcs->framelog.checksumFlag = checksumFlag;
if (zcs->framelog.checksumFlag) { if (zcs->framelog.checksumFlag) {
@@ -204,7 +201,7 @@ size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output
/* end the frame */ /* end the frame */
size_t ret = ZSTD_endStream(zcs->cstream, output); size_t ret = ZSTD_endStream(zcs->cstream, output);
zcs->frameCSize += output->pos - prevOutPos; zcs->frameCSize += (U32)(output->pos - prevOutPos);
/* need to flush before doing the rest */ /* need to flush before doing the rest */
if (ret) return ret; if (ret) return ret;
@@ -224,8 +221,7 @@ size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output
zcs->frameDSize = 0; zcs->frameDSize = 0;
ZSTD_CCtx_reset(zcs->cstream, ZSTD_reset_session_only); ZSTD_CCtx_reset(zcs->cstream, ZSTD_reset_session_only);
if (zcs->framelog.checksumFlag) if (zcs->framelog.checksumFlag) XXH64_reset(&zcs->xxhState, 0);
XXH64_reset(&zcs->xxhState, 0);
return 0; return 0;
} }
@@ -248,8 +244,8 @@ size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer*
XXH64_update(&zcs->xxhState, inBase, inTmp.pos); XXH64_update(&zcs->xxhState, inBase, inTmp.pos);
} }
zcs->frameCSize += output->pos - prevOutPos; zcs->frameCSize += (U32)(output->pos - prevOutPos);
zcs->frameDSize += inTmp.pos; zcs->frameDSize += (U32)inTmp.pos;
input->pos += inTmp.pos; input->pos += inTmp.pos;
@@ -290,7 +286,7 @@ static inline size_t ZSTD_stwrite32(ZSTD_frameLog* fl,
memcpy((BYTE*)output->dst + output->pos, memcpy((BYTE*)output->dst + output->pos,
tmp + (fl->seekTablePos - offset), lenWrite); tmp + (fl->seekTablePos - offset), lenWrite);
output->pos += lenWrite; output->pos += lenWrite;
fl->seekTablePos += lenWrite; fl->seekTablePos += (U32)lenWrite;
if (lenWrite < 4) return ZSTD_seekable_seekTableSize(fl) - fl->seekTablePos; if (lenWrite < 4) return ZSTD_seekable_seekTableSize(fl) - fl->seekTablePos;
} }
@@ -339,8 +335,7 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
if (output->size - output->pos < 1) return seekTableLen - fl->seekTablePos; if (output->size - output->pos < 1) return seekTableLen - fl->seekTablePos;
if (fl->seekTablePos < seekTableLen - 4) { if (fl->seekTablePos < seekTableLen - 4) {
BYTE sfd = 0; BYTE const sfd = (BYTE)((fl->checksumFlag) << 7);
sfd |= (fl->checksumFlag) << 7;
((BYTE*)output->dst)[output->pos] = sfd; ((BYTE*)output->dst)[output->pos] = sfd;
output->pos++; output->pos++;
contrib/seekable_format/zstdseek_decompress.c
+100 -48
View File
@@ -107,7 +107,8 @@ typedef struct {
static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n) static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n)
{ {
buffWrapper_t* buff = (buffWrapper_t*) opaque; buffWrapper_t* const buff = (buffWrapper_t*)opaque;
assert(buff != NULL);
if (buff->pos + n > buff->size) return -1; if (buff->pos + n > buff->size) return -1;
memcpy(buffer, (const BYTE*)buff->ptr + buff->pos, n); memcpy(buffer, (const BYTE*)buff->ptr + buff->pos, n);
buff->pos += n; buff->pos += n;
@@ -118,15 +119,17 @@ static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin)
{ {
buffWrapper_t* const buff = (buffWrapper_t*) opaque; buffWrapper_t* const buff = (buffWrapper_t*) opaque;
unsigned long long newOffset; unsigned long long newOffset;
assert(buff != NULL);
switch (origin) { switch (origin) {
case SEEK_SET: case SEEK_SET:
newOffset = offset; assert(offset >= 0);
newOffset = (unsigned long long)offset;
break; break;
case SEEK_CUR: case SEEK_CUR:
newOffset = (unsigned long long)buff->pos + offset; newOffset = (unsigned long long)((long long)buff->pos + offset);
break; break;
case SEEK_END: case SEEK_END:
newOffset = (unsigned long long)buff->size + offset; newOffset = (unsigned long long)((long long)buff->size + offset);
break; break;
default: default:
assert(0); /* not possible */ assert(0); /* not possible */
@@ -144,18 +147,18 @@ typedef struct {
U32 checksum; U32 checksum;
} seekEntry_t; } seekEntry_t;
typedef struct { struct ZSTD_seekTable_s {
seekEntry_t* entries; seekEntry_t* entries;
size_t tableLen; size_t tableLen;
int checksumFlag; int checksumFlag;
} seekTable_t; };
#define SEEKABLE_BUFF_SIZE ZSTD_BLOCKSIZE_MAX #define SEEKABLE_BUFF_SIZE ZSTD_BLOCKSIZE_MAX
struct ZSTD_seekable_s { struct ZSTD_seekable_s {
ZSTD_DStream* dstream; ZSTD_DStream* dstream;
seekTable_t seekTable; ZSTD_seekTable seekTable;
ZSTD_seekable_customFile src; ZSTD_seekable_customFile src;
U64 decompressedOffset; U64 decompressedOffset;
@@ -173,8 +176,7 @@ struct ZSTD_seekable_s {
ZSTD_seekable* ZSTD_seekable_create(void) ZSTD_seekable* ZSTD_seekable_create(void)
{ {
ZSTD_seekable* zs = malloc(sizeof(ZSTD_seekable)); ZSTD_seekable* const zs = malloc(sizeof(ZSTD_seekable));
if (zs == NULL) return NULL; if (zs == NULL) return NULL;
/* also initializes stage to zsds_init */ /* also initializes stage to zsds_init */
@@ -195,7 +197,35 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
ZSTD_freeDStream(zs->dstream); ZSTD_freeDStream(zs->dstream);
free(zs->seekTable.entries); free(zs->seekTable.entries);
free(zs); free(zs);
return 0;
}
ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs)
{
ZSTD_seekTable* const st = malloc(sizeof(ZSTD_seekTable));
if (st==NULL) return NULL;
st->checksumFlag = zs->seekTable.checksumFlag;
st->tableLen = zs->seekTable.tableLen;
/* Allocate an extra entry at the end to match logic of initial allocation */
size_t const entriesSize = sizeof(seekEntry_t) * (zs->seekTable.tableLen + 1);
seekEntry_t* const entries = (seekEntry_t*)malloc(entriesSize);
if (entries==NULL) {
free(st);
return NULL;
}
memcpy(entries, zs->seekTable.entries, entriesSize);
st->entries = entries;
return st;
}
size_t ZSTD_seekTable_free(ZSTD_seekTable* st)
{
if (st == NULL) return 0; /* support free on null */
free(st->entries);
free(st);
return 0; return 0;
} }
@@ -203,19 +233,24 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
* Performs a binary search to find the last frame with a decompressed offset * Performs a binary search to find the last frame with a decompressed offset
* <= pos * <= pos
* @return : the frame's index */ * @return : the frame's index */
unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos) unsigned ZSTD_seekable_offsetToFrameIndex(const ZSTD_seekable* zs, unsigned long long pos)
{
return ZSTD_seekTable_offsetToFrameIndex(&zs->seekTable, pos);
}
unsigned ZSTD_seekTable_offsetToFrameIndex(const ZSTD_seekTable* st, unsigned long long pos)
{ {
U32 lo = 0; U32 lo = 0;
U32 hi = (U32)zs->seekTable.tableLen; U32 hi = (U32)st->tableLen;
assert(zs->seekTable.tableLen <= UINT_MAX); assert(st->tableLen <= UINT_MAX);
if (pos >= zs->seekTable.entries[zs->seekTable.tableLen].dOffset) { if (pos >= st->entries[st->tableLen].dOffset) {
return (U32)zs->seekTable.tableLen; return (unsigned)st->tableLen;
} }
while (lo + 1 < hi) { while (lo + 1 < hi) {
U32 const mid = lo + ((hi - lo) >> 1); U32 const mid = lo + ((hi - lo) >> 1);
if (zs->seekTable.entries[mid].dOffset <= pos) { if (st->entries[mid].dOffset <= pos) {
lo = mid; lo = mid;
} else { } else {
hi = mid; hi = mid;
@@ -224,36 +259,61 @@ unsigned ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long
return lo; return lo;
} }
unsigned ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs) unsigned ZSTD_seekable_getNumFrames(const ZSTD_seekable* zs)
{ {
assert(zs->seekTable.tableLen <= UINT_MAX); return ZSTD_seekTable_getNumFrames(&zs->seekTable);
return (unsigned)zs->seekTable.tableLen;
} }
unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex) unsigned ZSTD_seekTable_getNumFrames(const ZSTD_seekTable* st)
{ {
if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE; assert(st->tableLen <= UINT_MAX);
return zs->seekTable.entries[frameIndex].cOffset; return (unsigned)st->tableLen;
} }
unsigned long long ZSTD_seekable_getFrameDecompressedOffset(ZSTD_seekable* const zs, unsigned frameIndex) unsigned long long ZSTD_seekable_getFrameCompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex)
{ {
if (frameIndex >= zs->seekTable.tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE; return ZSTD_seekTable_getFrameCompressedOffset(&zs->seekTable, frameIndex);
return zs->seekTable.entries[frameIndex].dOffset;
} }
size_t ZSTD_seekable_getFrameCompressedSize(ZSTD_seekable* const zs, unsigned frameIndex) unsigned long long ZSTD_seekTable_getFrameCompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex)
{ {
if (frameIndex >= zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge); if (frameIndex >= st->tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
return zs->seekTable.entries[frameIndex + 1].cOffset - return st->entries[frameIndex].cOffset;
zs->seekTable.entries[frameIndex].cOffset;
} }
size_t ZSTD_seekable_getFrameDecompressedSize(ZSTD_seekable* const zs, unsigned frameIndex) unsigned long long ZSTD_seekable_getFrameDecompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex)
{ {
if (frameIndex > zs->seekTable.tableLen) return ERROR(frameIndex_tooLarge); return ZSTD_seekTable_getFrameDecompressedOffset(&zs->seekTable, frameIndex);
return zs->seekTable.entries[frameIndex + 1].dOffset - }
zs->seekTable.entries[frameIndex].dOffset;
unsigned long long ZSTD_seekTable_getFrameDecompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex)
{
if (frameIndex >= st->tableLen) return ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE;
return st->entries[frameIndex].dOffset;
}
size_t ZSTD_seekable_getFrameCompressedSize(const ZSTD_seekable* zs, unsigned frameIndex)
{
return ZSTD_seekTable_getFrameCompressedSize(&zs->seekTable, frameIndex);
}
size_t ZSTD_seekTable_getFrameCompressedSize(const ZSTD_seekTable* st, unsigned frameIndex)
{
if (frameIndex >= st->tableLen) return ERROR(frameIndex_tooLarge);
return st->entries[frameIndex + 1].cOffset -
st->entries[frameIndex].cOffset;
}
size_t ZSTD_seekable_getFrameDecompressedSize(const ZSTD_seekable* zs, unsigned frameIndex)
{
return ZSTD_seekTable_getFrameDecompressedSize(&zs->seekTable, frameIndex);
}
size_t ZSTD_seekTable_getFrameDecompressedSize(const ZSTD_seekTable* st, unsigned frameIndex)
{
if (frameIndex > st->tableLen) return ERROR(frameIndex_tooLarge);
return st->entries[frameIndex + 1].dOffset -
st->entries[frameIndex].dOffset;
} }
static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs) static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
@@ -274,8 +334,7 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
/* check reserved bits */ /* check reserved bits */
if ((checksumFlag >> 2) & 0x1f) { if ((checksumFlag >> 2) & 0x1f) {
return ERROR(corruption_detected); return ERROR(corruption_detected);
} } }
}
{ U32 const numFrames = MEM_readLE32(zs->inBuff); { U32 const numFrames = MEM_readLE32(zs->inBuff);
U32 const sizePerEntry = 8 + (checksumFlag?4:0); U32 const sizePerEntry = 8 + (checksumFlag?4:0);
@@ -283,12 +342,9 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_SKIPPABLEHEADERSIZE; U32 const frameSize = tableSize + ZSTD_seekTableFooterSize + ZSTD_SKIPPABLEHEADERSIZE;
U32 remaining = frameSize - ZSTD_seekTableFooterSize; /* don't need to re-read footer */ U32 remaining = frameSize - ZSTD_seekTableFooterSize; /* don't need to re-read footer */
{ { U32 const toRead = MIN(remaining, SEEKABLE_BUFF_SIZE);
U32 const toRead = MIN(remaining, SEEKABLE_BUFF_SIZE);
CHECK_IO(src.seek(src.opaque, -(S64)frameSize, SEEK_END)); CHECK_IO(src.seek(src.opaque, -(S64)frameSize, SEEK_END));
CHECK_IO(src.read(src.opaque, zs->inBuff, toRead)); CHECK_IO(src.read(src.opaque, zs->inBuff, toRead));
remaining -= toRead; remaining -= toRead;
} }
@@ -301,19 +357,15 @@ static size_t ZSTD_seekable_loadSeekTable(ZSTD_seekable* zs)
{ /* Allocate an extra entry at the end so that we can do size { /* Allocate an extra entry at the end so that we can do size
* computations on the last element without special case */ * computations on the last element without special case */
seekEntry_t* entries = (seekEntry_t*)malloc(sizeof(seekEntry_t) * (numFrames + 1)); seekEntry_t* const entries = (seekEntry_t*)malloc(sizeof(seekEntry_t) * (numFrames + 1));
U32 idx = 0; U32 idx = 0;
U32 pos = 8; U32 pos = 8;
U64 cOffset = 0; U64 cOffset = 0;
U64 dOffset = 0; U64 dOffset = 0;
if (!entries) { if (entries == NULL) return ERROR(memory_allocation);
free(entries);
return ERROR(memory_allocation);
}
/* compute cumulative positions */ /* compute cumulative positions */
for (; idx < numFrames; idx++) { for (; idx < numFrames; idx++) {
@@ -390,8 +442,9 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
zs->decompressedOffset = zs->seekTable.entries[targetFrame].dOffset; zs->decompressedOffset = zs->seekTable.entries[targetFrame].dOffset;
zs->curFrame = targetFrame; zs->curFrame = targetFrame;
assert(zs->seekTable.entries[targetFrame].cOffset < LLONG_MAX);
CHECK_IO(zs->src.seek(zs->src.opaque, CHECK_IO(zs->src.seek(zs->src.opaque,
zs->seekTable.entries[targetFrame].cOffset, (long long)zs->seekTable.entries[targetFrame].cOffset,
SEEK_SET)); SEEK_SET));
zs->in = (ZSTD_inBuffer){zs->inBuff, 0, 0}; zs->in = (ZSTD_inBuffer){zs->inBuff, 0, 0};
XXH64_reset(&zs->xxhState, 0); XXH64_reset(&zs->xxhState, 0);
@@ -460,7 +513,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
zs->in.size = toRead; zs->in.size = toRead;
zs->in.pos = 0; zs->in.pos = 0;
} }
} } /* while (zs->decompressedOffset < offset + len) */
} while (zs->decompressedOffset != offset + len); } while (zs->decompressedOffset != offset + len);
return len; return len;
@@ -472,8 +525,7 @@ size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSiz
return ERROR(frameIndex_tooLarge); return ERROR(frameIndex_tooLarge);
} }
{ { size_t const decompressedSize =
size_t const decompressedSize =
zs->seekTable.entries[frameIndex + 1].dOffset - zs->seekTable.entries[frameIndex + 1].dOffset -
zs->seekTable.entries[frameIndex].dOffset; zs->seekTable.entries[frameIndex].dOffset;
if (dstSize < decompressedSize) { if (dstSize < decompressedSize) {
lib/common/mem.h
+1 -1
View File
@@ -308,7 +308,7 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
MEM_STATIC U32 MEM_readLE24(const void* memPtr) MEM_STATIC U32 MEM_readLE24(const void* memPtr)
{ {
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16);
} }
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)