diff --git a/examples/.gitignore b/examples/.gitignore index 280feb36e..d682cae38 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1,6 +1,7 @@ #build simple_compression simple_decompression +multiple_simple_compression dictionary_compression dictionary_decompression streaming_compression diff --git a/examples/Makefile b/examples/Makefile index 96af41b54..25a0a62c2 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -17,6 +17,7 @@ LIB = ../lib/libzstd.a default: all all: simple_compression simple_decompression \ + multiple_simple_compression\ dictionary_compression dictionary_decompression \ streaming_compression streaming_decompression \ multiple_streaming_compression streaming_memory_usage @@ -24,33 +25,37 @@ all: simple_compression simple_decompression \ $(LIB) : $(MAKE) -C ../lib libzstd.a -simple_compression : simple_compression.c $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +simple_compression : simple_compression.c utils.h $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ -simple_decompression : simple_decompression.c $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +simple_decompression : simple_decompression.c utils.h $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ -dictionary_compression : dictionary_compression.c $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +multiple_simple_compression : multiple_simple_compression.c utils.h $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ -dictionary_decompression : dictionary_decompression.c $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +dictionary_compression : dictionary_compression.c utils.h $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ -streaming_compression : streaming_compression.c $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +dictionary_decompression : dictionary_decompression.c utils.h $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ -multiple_streaming_compression : multiple_streaming_compression.c $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +streaming_compression : streaming_compression.c utils.h $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ -streaming_decompression : streaming_decompression.c $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +multiple_streaming_compression : multiple_streaming_compression.c utils.h $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ + +streaming_decompression : streaming_decompression.c utils.h $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ streaming_memory_usage : streaming_memory_usage.c $(LIB) - $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + $(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@ clean: @rm -f core *.o tmp* result* *.zst \ simple_compression simple_decompression \ + multiple_simple_compression \ dictionary_compression dictionary_decompression \ streaming_compression streaming_decompression \ multiple_streaming_compression streaming_memory_usage @@ -62,6 +67,7 @@ test: all @echo -- Simple compression tests ./simple_compression tmp ./simple_decompression tmp.zst + ./multiple_simple_compression *.c ./streaming_decompression tmp.zst > /dev/null @echo -- Streaming memory usage ./streaming_memory_usage diff --git a/examples/README.md b/examples/README.md index eba50c999..0bff7ac19 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,8 +11,14 @@ Zstandard library : usage examples Result remains in memory. Introduces usage of : `ZSTD_decompress()` +- [Multiple simple compression](multiple_simple_compression.c) : + Compress multiple files (in simple mode) in a single command line. + Demonstrates memory preservation technique that + minimizes malloc()/free() calls by re-using existing resources. + Introduces usage of : `ZSTD_compressCCtx()` + - [Streaming memory usage](streaming_memory_usage.c) : - Provides amount of memory used by streaming context + Provides amount of memory used by streaming context. Introduces usage of : `ZSTD_sizeof_CStream()` - [Streaming compression](streaming_compression.c) : @@ -20,7 +26,7 @@ Zstandard library : usage examples Introduces usage of : `ZSTD_compressStream()` - [Multiple Streaming compression](multiple_streaming_compression.c) : - Compress multiple files in a single command line. + Compress multiple files (in streaming mode) in a single command line. Introduces memory usage preservation technique, reducing impact of malloc()/free() and memset() by re-using existing resources. diff --git a/examples/dictionary_compression.c b/examples/dictionary_compression.c index 511b35676..3c4a5bd1e 100644 --- a/examples/dictionary_compression.c +++ b/examples/dictionary_compression.c @@ -21,7 +21,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel) { size_t dictSize; printf("loading dictionary %s \n", dictFileName); - void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize); + void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize); ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel); if (!cdict) { fprintf(stderr, "ZSTD_createCDict error \n"); @@ -35,7 +35,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel) static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict) { size_t fSize; - void* const fBuff = loadFile_orDie(fname, &fSize); + void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize); size_t const cBuffSize = ZSTD_compressBound(fSize); void* const cBuff = malloc_orDie(cBuffSize); diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index 69f56d56b..243e22236 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -25,7 +25,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName) { size_t dictSize; printf("loading dictionary %s \n", dictFileName); - void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize); + void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize); ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize); if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); } free(dictBuffer); @@ -35,7 +35,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName) static void decompress(const char* fname, const ZSTD_DDict* ddict) { size_t cSize; - void* const cBuff = loadFile_orDie(fname, &cSize); + void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize); unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); if (rSize==ZSTD_CONTENTSIZE_ERROR) { fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); diff --git a/examples/multiple_simple_compression.c b/examples/multiple_simple_compression.c new file mode 100644 index 000000000..65c775bfa --- /dev/null +++ b/examples/multiple_simple_compression.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include // malloc, free, exit +#include // fprintf, perror, fopen, etc. +#include // strlen, strcat, memset, strerror +#include // errno +#include // stat +#include // presumes zstd library is installed +#include "utils.h" + +typedef struct { + void* fBuffer; + void* cBuffer; + size_t fBufferSize; + size_t cBufferSize; + ZSTD_CCtx* cctx; +} resources; + +/* + * allocate memory for buffers big enough to compress all files + * as well as memory for output file name (ofn) + */ +static resources createResources_orDie(int argc, const char** argv, char **ofn, int* ofnBufferLen) +{ + size_t maxFilenameLength=0; + size_t maxFileSize = 0; + + int argNb; + for (argNb = 1; argNb < argc; argNb++) { + const char* const filename = argv[argNb]; + size_t const filenameLength = strlen(filename); + size_t const fileSize = fsize_orDie(filename); + + if (filenameLength > maxFilenameLength) maxFilenameLength = filenameLength; + if (fileSize > maxFileSize) maxFileSize = fileSize; + } + + resources ress; + ress.fBufferSize = maxFileSize; + ress.cBufferSize = ZSTD_compressBound(maxFileSize); + + *ofnBufferLen = maxFilenameLength + 5; + *ofn = (char*)malloc_orDie(*ofnBufferLen); + ress.fBuffer = malloc_orDie(ress.fBufferSize); + ress.cBuffer = malloc_orDie(ress.cBufferSize); + ress.cctx = ZSTD_createCCtx(); + if (ress.cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); } + return ress; +} + +static void freeResources(resources ress, char *outFilename) +{ + free(ress.fBuffer); + free(ress.cBuffer); + ZSTD_freeCCtx(ress.cctx); /* never fails */ + free(outFilename); +} + +/* compress with pre-allocated context (ZSTD_CCtx) and input/output buffers*/ +static void compressFile_orDie(resources ress, const char* fname, const char* oname) +{ + size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize); + + size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1); + if (ZSTD_isError(cSize)) { + fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize)); + exit(8); + } + + saveFile_orDie(oname, ress.cBuffer, cSize); + + /* success */ + // printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname); +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc<2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE(s)\n", exeName); + return 1; + } + + /* memory allocation for outFilename and resources */ + char* outFilename; + int outFilenameBufferLen; + resources const ress = createResources_orDie(argc, argv, &outFilename, &outFilenameBufferLen); + + /* compress files with shared context, input and output buffers */ + int argNb; + for (argNb = 1; argNb < argc; argNb++) { + const char* const inFilename = argv[argNb]; + int inFilenameLen = strlen(inFilename); + assert(inFilenameLen + 5 <= outFilenameBufferLen); + memcpy(outFilename, inFilename, inFilenameLen); + memcpy(outFilename+inFilenameLen, ".zst", 5); + compressFile_orDie(ress, inFilename, outFilename); + } + + /* free momery */ + freeResources(ress,outFilename); + + printf("compressed %i files \n", argc-1); + + return 0; +} diff --git a/examples/simple_compression.c b/examples/simple_compression.c index 0193dd40f..829cbd7d8 100644 --- a/examples/simple_compression.c +++ b/examples/simple_compression.c @@ -19,7 +19,7 @@ static void compress_orDie(const char* fname, const char* oname) { size_t fSize; - void* const fBuff = loadFile_orDie(fname, &fSize); + void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize); size_t const cBuffSize = ZSTD_compressBound(fSize); void* const cBuff = malloc_orDie(cBuffSize); diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c index ee055dd67..e7949376c 100644 --- a/examples/simple_decompression.c +++ b/examples/simple_decompression.c @@ -20,7 +20,7 @@ static void decompress(const char* fname) { size_t cSize; - void* const cBuff = loadFile_orDie(fname, &cSize); + void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize); unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); if (rSize==ZSTD_CONTENTSIZE_ERROR) { fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); diff --git a/examples/utils.h b/examples/utils.h index 55a329e23..6d1360457 100644 --- a/examples/utils.h +++ b/examples/utils.h @@ -9,7 +9,7 @@ */ /* - * This header file has common utility functions used in examples. + * This header file has common utility functions used in examples. */ #ifndef UTILS_H #define UTILS_H @@ -18,10 +18,11 @@ #include // fprintf, perror, fopen, etc. #include // strlen, strcat, memset, strerror #include // errno +#include // assert #include // stat /* - * Define the returned error code from utility functions. + * Define the returned error code from utility functions. */ typedef enum { ERROR_fsize = 1, @@ -35,21 +36,34 @@ typedef enum { ERROR_largeFile = 9, } UTILS_ErrorCode; -/*! fsize_orDie() : +/*! fsize_orDie() : * Get the size of a given file path. - * + * * @return The size of a given file path. */ -static off_t fsize_orDie(const char *filename) +static size_t fsize_orDie(const char *filename) { struct stat st; - if (stat(filename, &st) == 0) return st.st_size; - /* error */ - perror(filename); - exit(ERROR_fsize); + if (stat(filename, &st) != 0) { + /* error */ + perror(filename); + exit(ERROR_fsize); + } + + off_t const fileSize = st.st_size; + size_t const size = (size_t)fileSize; + /* 1. fileSize should be non-negative, + * 2. if off_t -> size_t type conversion results in discrepancy, + * the file size is too large for type size_t. + */ + if ((fileSize < 0) || (fileSize != (off_t)size)) { + fprintf(stderr, "%s : filesize too large \n", filename); + exit(ERROR_largeFile); + } + return size; } -/*! fopen_orDie() : +/*! fopen_orDie() : * Open a file using given file path and open option. * * @return If successful this function will return a FILE pointer to an @@ -64,7 +78,7 @@ static FILE* fopen_orDie(const char *filename, const char *instruction) exit(ERROR_fopen); } -/*! fclose_orDie() : +/*! fclose_orDie() : * Close an opened file using given FILE pointer. */ static void fclose_orDie(FILE* file) @@ -75,11 +89,11 @@ static void fclose_orDie(FILE* file) exit(ERROR_fclose); } -/*! fread_orDie() : - * +/*! fread_orDie() : + * * Read sizeToRead bytes from a given file, storing them at the * location given by buffer. - * + * * @return The number of bytes read. */ static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) @@ -93,12 +107,12 @@ static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) } /*! fwrite_orDie() : - * + * * Write sizeToWrite bytes to a file pointed to by file, obtaining * them from a location given by buffer. * * Note: This function will send an error to stderr and exit if it - * cannot write data to the given file pointer. + * cannot write data to the given file pointer. * * @return The number of bytes written. */ @@ -113,7 +127,7 @@ static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) /*! malloc_orDie() : * Allocate memory. - * + * * @return If successful this function returns a pointer to allo- * cated memory. If there is an error, this function will send that * error to stderr and exit. @@ -128,39 +142,51 @@ static void* malloc_orDie(size_t size) } /*! loadFile_orDie() : - * Read size bytes from a file. - * + * load file into buffer (memory). + * * Note: This function will send an error to stderr and exit if it * cannot read data from the given file path. - * - * @return If successful this function will return a pointer to read - * data otherwise it will printout an error to stderr and exit. + * + * @return If successful this function will load file into buffer and + * return file size, otherwise it will printout an error to stderr and exit. */ -static void* loadFile_orDie(const char* fileName, size_t* size) +static size_t loadFile_orDie(const char* fileName, void* buffer, int bufferSize) { - off_t const fileSize = fsize_orDie(fileName); - size_t const buffSize = (size_t)fileSize; - if ((off_t)buffSize < fileSize) { /* narrowcast overflow */ - fprintf(stderr, "%s : filesize too large \n", fileName); - exit(ERROR_largeFile); - } + size_t const fileSize = fsize_orDie(fileName); + assert(fileSize <= bufferSize); + FILE* const inFile = fopen_orDie(fileName, "rb"); - void* const buffer = malloc_orDie(buffSize); - size_t const readSize = fread(buffer, 1, buffSize, inFile); - if (readSize != (size_t)buffSize) { + size_t const readSize = fread(buffer, 1, fileSize, inFile); + if (readSize != (size_t)fileSize) { fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); exit(ERROR_fread); } fclose(inFile); /* can't fail, read only */ - *size = buffSize; + return fileSize; +} + +/*! mallocAndLoadFile_orDie() : + * allocate memory buffer and then load file into it. + * + * Note: This function will send an error to stderr and exit if memory allocation + * fails or it cannot read data from the given file path. + * + * @return If successful this function will return buffer and bufferSize(=fileSize), + * otherwise it will printout an error to stderr and exit. + */ +static void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize) { + size_t const fileSize = fsize_orDie(fileName); + *bufferSize = fileSize; + void* const buffer = malloc_orDie(*bufferSize); + loadFile_orDie(fileName, buffer, *bufferSize); return buffer; } /*! saveFile_orDie() : - * + * * Save buffSize bytes to a given file path, obtaining them from a location pointed * to by buff. - * + * * Note: This function will send an error to stderr and exit if it * cannot write to a given file. */