diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c index 6ce8c8809..dfee45743 100644 --- a/lib/dictBuilder/fastcover.c +++ b/lib/dictBuilder/fastcover.c @@ -245,39 +245,41 @@ static int FASTCOVER_checkParameters(ZDICT_cover_params_t parameters, /** * Clean up a context initialized with `FASTCOVER_ctx_init()`. */ -static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) { - if (!ctx) { - return; - } +static void +FASTCOVER_ctx_destroy(FASTCOVER_ctx_t* ctx) +{ + if (!ctx) return; - free(ctx->freqs); - ctx->freqs = NULL; + free(ctx->freqs); + ctx->freqs = NULL; - free(ctx->offsets); - ctx->offsets = NULL; + free(ctx->offsets); + ctx->offsets = NULL; } /** * Calculate for frequency of hash value of each dmer in ctx->samples */ -static void FASTCOVER_computeFrequency(U32 *freqs, FASTCOVER_ctx_t *ctx){ - const unsigned f = ctx->f; - const unsigned d = ctx->d; - const unsigned skip = ctx->accelParams.skip; - const unsigned readLength = MAX(d, 8); - size_t start; /* start of current dmer */ - size_t i; - for (i = 0; i < ctx->nbTrainSamples; i++) { - size_t currSampleStart = ctx->offsets[i]; - size_t currSampleEnd = ctx->offsets[i+1]; - start = currSampleStart; - while (start + readLength <= currSampleEnd) { - const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, d); - freqs[dmerIndex]++; - start = start + skip + 1; +static void +FASTCOVER_computeFrequency(U32* freqs, const FASTCOVER_ctx_t* ctx) +{ + const unsigned f = ctx->f; + const unsigned d = ctx->d; + const unsigned skip = ctx->accelParams.skip; + const unsigned readLength = MAX(d, 8); + size_t i; + assert(ctx->nbTrainSamples >= 5); + assert(ctx->nbTrainSamples <= ctx->nbSamples); + for (i = 0; i < ctx->nbTrainSamples; i++) { + size_t start = ctx->offsets[i]; /* start of current dmer */ + size_t const currSampleEnd = ctx->offsets[i+1]; + while (start + readLength <= currSampleEnd) { + const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, d); + freqs[dmerIndex]++; + start = start + skip + 1; + } } - } } @@ -288,84 +290,100 @@ static void FASTCOVER_computeFrequency(U32 *freqs, FASTCOVER_ctx_t *ctx){ * Returns 1 on success or zero on error. * The context must be destroyed with `FASTCOVER_ctx_destroy()`. */ -static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer, - const size_t *samplesSizes, unsigned nbSamples, - unsigned d, double splitPoint, unsigned f, - FASTCOVER_accel_t accelParams) { - const BYTE *const samples = (const BYTE *)samplesBuffer; - const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); - /* Split samples into testing and training sets */ - const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples; - const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples; - const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize; - const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize; - /* Checks */ - if (totalSamplesSize < MAX(d, sizeof(U64)) || - totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) { - DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", - (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); - return 0; - } - /* Check if there are at least 5 training samples */ - if (nbTrainSamples < 5) { - DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid\n", nbTrainSamples); - return 0; - } - /* Check if there's testing sample */ - if (nbTestSamples < 1) { - DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.\n", nbTestSamples); - return 0; - } - /* Zero the context */ - memset(ctx, 0, sizeof(*ctx)); - DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples, - (U32)trainingSamplesSize); - DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples, - (U32)testSamplesSize); +static int +FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx, + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + unsigned d, double splitPoint, unsigned f, + FASTCOVER_accel_t accelParams) +{ + const BYTE* const samples = (const BYTE*)samplesBuffer; + const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); + /* Split samples into testing and training sets */ + const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples; + const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples; + const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize; + const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize; - ctx->samples = samples; - ctx->samplesSizes = samplesSizes; - ctx->nbSamples = nbSamples; - ctx->nbTrainSamples = nbTrainSamples; - ctx->nbTestSamples = nbTestSamples; - ctx->nbDmers = trainingSamplesSize - MAX(d, sizeof(U64)) + 1; - ctx->d = d; - ctx->f = f; - ctx->accelParams = accelParams; - - /* The offsets of each file */ - ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t)); - if (!ctx->offsets) { - DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n"); - FASTCOVER_ctx_destroy(ctx); - return 0; - } - - /* Fill offsets from the samplesSizes */ - { - U32 i; - ctx->offsets[0] = 0; - for (i = 1; i <= nbSamples; ++i) { - ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1]; + /* Checks */ + if (totalSamplesSize < MAX(d, sizeof(U64)) || + totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) { + DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", + (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); + return 0; } - } - /* Initialize frequency array of size 2^f */ - ctx->freqs = (U32 *)calloc(((U64)1 << f), sizeof(U32)); + /* Check if there are at least 5 training samples */ + if (nbTrainSamples < 5) { + DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid\n", nbTrainSamples); + return 0; + } - DISPLAYLEVEL(2, "Computing frequencies\n"); - FASTCOVER_computeFrequency(ctx->freqs, ctx); + /* Check if there's testing sample */ + if (nbTestSamples < 1) { + DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.\n", nbTestSamples); + return 0; + } - return 1; + /* Zero the context */ + memset(ctx, 0, sizeof(*ctx)); + DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples, + (U32)trainingSamplesSize); + DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples, + (U32)testSamplesSize); + + ctx->samples = samples; + ctx->samplesSizes = samplesSizes; + ctx->nbSamples = nbSamples; + ctx->nbTrainSamples = nbTrainSamples; + ctx->nbTestSamples = nbTestSamples; + ctx->nbDmers = trainingSamplesSize - MAX(d, sizeof(U64)) + 1; + ctx->d = d; + ctx->f = f; + ctx->accelParams = accelParams; + + /* The offsets of each file */ + ctx->offsets = (size_t*)calloc((nbSamples + 1), sizeof(size_t)); + if (ctx->offsets == NULL) { + DISPLAYLEVEL(1, "Failed to allocate scratch buffers \n"); + FASTCOVER_ctx_destroy(ctx); + return 0; + } + + /* Fill offsets from the samplesSizes */ + { U32 i; + ctx->offsets[0] = 0; + assert(nbSamples >= 5); + for (i = 1; i <= nbSamples; ++i) { + ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1]; + } + } + + /* Initialize frequency array of size 2^f */ + ctx->freqs = (U32*)calloc(((U64)1 << f), sizeof(U32)); + if (ctx->freqs == NULL) { + DISPLAYLEVEL(1, "Failed to allocate frequency table \n"); + FASTCOVER_ctx_destroy(ctx); + return 0; + } + + DISPLAYLEVEL(2, "Computing frequencies\n"); + FASTCOVER_computeFrequency(ctx->freqs, ctx); + + return 1; } /** * Given the prepared context build the dictionary. */ -static size_t FASTCOVER_buildDictionary(const FASTCOVER_ctx_t *ctx, U32 *freqs, - void *dictBuffer, size_t dictBufferCapacity, - ZDICT_cover_params_t parameters, U16* segmentFreqs){ +static size_t +FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx, + U32* freqs, + void* dictBuffer, size_t dictBufferCapacity, + ZDICT_cover_params_t parameters, + U16* segmentFreqs) +{ BYTE *const dict = (BYTE *)dictBuffer; size_t tail = dictBufferCapacity; /* Divide the data up into epochs of equal size. @@ -416,10 +434,10 @@ static size_t FASTCOVER_buildDictionary(const FASTCOVER_ctx_t *ctx, U32 *freqs, * Parameters for FASTCOVER_tryParameters(). */ typedef struct FASTCOVER_tryParameters_data_s { - const FASTCOVER_ctx_t *ctx; - COVER_best_t *best; - size_t dictBufferCapacity; - ZDICT_cover_params_t parameters; + const FASTCOVER_ctx_t* ctx; + COVER_best_t* best; + size_t dictBufferCapacity; + ZDICT_cover_params_t parameters; } FASTCOVER_tryParameters_data_t; @@ -428,7 +446,8 @@ typedef struct FASTCOVER_tryParameters_data_s { * This function is thread safe if zstd is compiled with multithreaded support. * It takes its parameters as an *OWNING* opaque pointer to support threading. */ -static void FASTCOVER_tryParameters(void *opaque) { +static void FASTCOVER_tryParameters(void *opaque) +{ /* Save parameters as local variables */ FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque; const FASTCOVER_ctx_t *const ctx = data->ctx; @@ -447,8 +466,7 @@ static void FASTCOVER_tryParameters(void *opaque) { /* Copy the frequencies because we need to modify them */ memcpy(freqs, ctx->freqs, ((U64)1 << ctx->f) * sizeof(U32)); /* Build the dictionary */ - { - const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, dictBufferCapacity, + { const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, dictBufferCapacity, parameters, segmentFreqs); const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100); dictBufferCapacity = ZDICT_finalizeDictionary( @@ -474,9 +492,10 @@ _cleanup: } - -static void FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverParams, - ZDICT_cover_params_t *coverParams) { +static void +FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverParams, + ZDICT_cover_params_t* coverParams) +{ coverParams->k = fastCoverParams.k; coverParams->d = fastCoverParams.d; coverParams->steps = fastCoverParams.steps; @@ -486,9 +505,11 @@ static void FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverPar } -static void FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams, - ZDICT_fastCover_params_t *fastCoverParams, - unsigned f, unsigned accel) { +static void +FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams, + ZDICT_fastCover_params_t* fastCoverParams, + unsigned f, unsigned accel) +{ fastCoverParams->k = coverParams.k; fastCoverParams->d = coverParams.d; fastCoverParams->steps = coverParams.steps; @@ -500,9 +521,12 @@ static void FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams, } -ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover( - void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, - const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters) { +ZDICTLIB_API size_t +ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t parameters) +{ BYTE* const dict = (BYTE*)dictBuffer; FASTCOVER_ctx_t ctx; ZDICT_cover_params_t coverParams; @@ -562,10 +586,13 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover( } -ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( - void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, - const size_t *samplesSizes, unsigned nbSamples, - ZDICT_fastCover_params_t *parameters) { +ZDICTLIB_API size_t +ZDICT_optimizeTrainFromBuffer_fastCover( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t* parameters) +{ ZDICT_cover_params_t coverParams; FASTCOVER_accel_t accelParams; /* constants */ diff --git a/programs/fileio.c b/programs/fileio.c index 1b04f2ea0..b66426b1d 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -20,10 +20,12 @@ # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ #endif -#if defined(__linux__) || (defined(__APPLE__) && defined(__MACH__)) +#if !defined(BACKTRACES_ENABLE) && \ + (defined(__linux__) || (defined(__APPLE__) && defined(__MACH__)) ) # define BACKTRACES_ENABLE 1 #endif + /*-************************************* * Includes ***************************************/ @@ -32,6 +34,7 @@ #include /* fprintf, fopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ #include /* strcmp, strlen */ +#include #include /* errno */ #include #ifdef BACKTRACES_ENABLE @@ -43,10 +46,8 @@ # include #endif -#include "debug.h" -#include "mem.h" +#include "mem.h" /* U32, U64 */ #include "fileio.h" -#include "util.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zstd.h" @@ -113,7 +114,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; #define EXM_THROW(error, ...) \ { \ DISPLAYLEVEL(1, "zstd: "); \ - DEBUGLOG(1, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ DISPLAYLEVEL(1, "error %i : ", error); \ DISPLAYLEVEL(1, __VA_ARGS__); \ DISPLAYLEVEL(1, " \n"); \ @@ -123,7 +124,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; #define CHECK_V(v, f) \ v = f; \ if (ZSTD_isError(v)) { \ - DEBUGLOG(1, "%s \n", #f); \ + DISPLAYLEVEL(5, "%s \n", #f); \ EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \ } #define CHECK(f) { size_t err; CHECK_V(err, f); } @@ -1066,14 +1067,80 @@ FIO_compressFilename_internal(cRess_t ress, } +/*! FIO_compressFilename_dstFile() : + * open dstFileName, or pass-through if ress.dstFile != NULL, + * then start compression with FIO_compressFilename_internal(). + * Manages source removal (--rm) and file permissions transfer. + * note : ress.srcFile must be != NULL, + * so reach this function through FIO_compressFilename_srcFile(). + * @return : 0 : compression completed correctly, + * 1 : pb + */ +static int FIO_compressFilename_dstFile(cRess_t ress, + const char* dstFileName, + const char* srcFileName, + int compressionLevel) +{ + int closeDstFile = 0; + int result; + stat_t statbuf; + int transfer_permissions = 0; + + assert(ress.srcFile != NULL); + + if (ress.dstFile == NULL) { + closeDstFile = 1; + DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName); + ress.dstFile = FIO_openDstFile(dstFileName); + if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ + /* Must only be added after FIO_openDstFile() succeeds. + * Otherwise we may delete the destination file if it already exists, + * and the user presses Ctrl-C when asked if they wish to overwrite. + */ + addHandler(dstFileName); + + if ( strcmp (srcFileName, stdinmark) + && UTIL_getFileStat(srcFileName, &statbuf)) + transfer_permissions = 1; + } + + result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel); + + if (closeDstFile) { + FILE* const dstFile = ress.dstFile; + ress.dstFile = NULL; + + clearHandler(); + + if (fclose(dstFile)) { /* error closing dstFile */ + DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); + result=1; + } + if ( (result != 0) /* operation failure */ + && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null */ + && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ + ) { + FIO_remove(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ + } else if ( strcmp(dstFileName, stdoutmark) + && strcmp(dstFileName, nulmark) + && transfer_permissions) { + UTIL_setFileStat(dstFileName, &statbuf); + } + } + + return result; +} + + /*! FIO_compressFilename_srcFile() : - * note : ress.destFile already opened * @return : 0 : compression completed correctly, * 1 : missing or pb opening srcFileName */ -static int FIO_compressFilename_srcFile(cRess_t ress, - const char* dstFileName, const char* srcFileName, - int compressionLevel) +static int +FIO_compressFilename_srcFile(cRess_t ress, + const char* dstFileName, + const char* srcFileName, + int compressionLevel) { int result; @@ -1084,12 +1151,16 @@ static int FIO_compressFilename_srcFile(cRess_t ress, } ress.srcFile = FIO_openSrcFile(srcFileName); - if (!ress.srcFile) return 1; /* srcFile could not be opened */ + if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ - result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel); + result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); fclose(ress.srcFile); - if (g_removeSrcFile /* --rm */ && !result && strcmp(srcFileName, stdinmark)) { + ress.srcFile = NULL; + if ( g_removeSrcFile /* --rm */ + && result == 0 /* success */ + && strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */ + ) { /* We must clear the handler, since after this point calling it would * delete both the source and destination files. */ @@ -1101,50 +1172,6 @@ static int FIO_compressFilename_srcFile(cRess_t ress, } -/*! FIO_compressFilename_dstFile() : - * @return : 0 : compression completed correctly, - * 1 : pb - */ -static int FIO_compressFilename_dstFile(cRess_t ress, - const char* dstFileName, - const char* srcFileName, - int compressionLevel) -{ - int result; - stat_t statbuf; - int stat_result = 0; - - DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName); - ress.dstFile = FIO_openDstFile(dstFileName); - if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ - /* Must ony be added after FIO_openDstFile() succeeds. - * Otherwise we may delete the destination file if at already exists, and - * the user presses Ctrl-C when asked if they wish to overwrite. - */ - addHandler(dstFileName); - - if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) - stat_result = 1; - result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); - clearHandler(); - - if (fclose(ress.dstFile)) { /* error closing dstFile */ - DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); - result=1; - } - if ( (result != 0) /* operation failure */ - && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null */ - && strcmp(dstFileName, stdoutmark) ) /* special case : don't remove() stdout */ - FIO_remove(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ - else if ( strcmp(dstFileName, stdoutmark) - && strcmp(dstFileName, nulmark) - && stat_result) - UTIL_setFileStat(dstFileName, &statbuf); - - return result; -} - - int FIO_compressFilename(const char* dstFileName, const char* srcFileName, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) @@ -1154,7 +1181,7 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize; cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); - int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); + int const result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); @@ -1164,57 +1191,76 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, } +/* FIO_determineCompressedName() : + * create a destination filename for compressed srcFileName. + * @return a pointer to it. + * This function never returns an error (it may abort() in case of pb) + */ +static const char* +FIO_determineCompressedName(const char* srcFileName, const char* suffix) +{ + static size_t dfnbCapacity = 0; + static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ + + size_t const sfnSize = strlen(srcFileName); + size_t const suffixSize = strlen(suffix); + + if (dfnbCapacity <= sfnSize+suffixSize+1) { /* resize name buffer */ + free(dstFileNameBuffer); + dfnbCapacity = sfnSize + suffixSize + 30; + dstFileNameBuffer = (char*)malloc(dfnbCapacity); + if (!dstFileNameBuffer) { + EXM_THROW(30, "zstd: %s", strerror(errno)); + } } + assert(dstFileNameBuffer != NULL); + strncpy(dstFileNameBuffer, srcFileName, sfnSize+1 /* Include null */); + strncat(dstFileNameBuffer, suffix, suffixSize); + + return dstFileNameBuffer; +} + + +/* FIO_compressMultipleFilenames() : + * compress nbFiles files + * into one destination (outFileName) + * or into one file each (outFileName == NULL, but suffix != NULL). + */ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, const char* outFileName, const char* suffix, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { - int missed_files = 0; - size_t dfnSize = FNSPACE; - char* dstFileName = (char*)malloc(FNSPACE); - size_t const suffixSize = suffix ? strlen(suffix) : 0; + int error = 0; U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]); U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize; U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : firstSrcSize ; cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); /* init */ - if (dstFileName==NULL) - EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); - if (outFileName == NULL && suffix == NULL) - EXM_THROW(28, "FIO_compressMultipleFilenames : dst unknown"); /* should never happen */ + assert(outFileName != NULL || suffix != NULL); - /* loop on each file */ - if (outFileName != NULL) { - unsigned u; + if (outFileName != NULL) { /* output into a single destination (stdout typically) */ ress.dstFile = FIO_openDstFile(outFileName); - if (ress.dstFile==NULL) { /* could not open outFileName */ - missed_files = nbFiles; + if (ress.dstFile == NULL) { /* could not open outFileName */ + error = 1; } else { + unsigned u; for (u=0; ucompressedSize = UTIL_getFileSize(inFileName); +typedef enum { info_success=0, info_frame_error=1, info_not_zstd=2, info_file_error=3 } InfoError; +#define ERROR_IF(c,n,...) { \ + if (c) { \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + return n; \ + } \ +} + +static InfoError +FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile) +{ /* begin analyzing frame */ for ( ; ; ) { BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; @@ -2013,130 +2093,111 @@ static int getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName){ && (numBytesRead == 0) && (info->compressedSize > 0) && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) { - break; - } - else if (feof(srcFile)) { - DISPLAY("Error: reached end of file with incomplete frame\n"); - detectError = 2; - break; - } - else { - DISPLAY("Error: did not reach end of file but ran out of frames\n"); - detectError = 1; - break; + break; /* correct end of file => success */ } + ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame"); + ERROR_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames"); } { U32 const magicNumber = MEM_readLE32(headerBuffer); /* Zstandard frame */ if (magicNumber == ZSTD_MAGICNUMBER) { ZSTD_frameHeader header; U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); - if (frameContentSize == ZSTD_CONTENTSIZE_ERROR || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN) { + if ( frameContentSize == ZSTD_CONTENTSIZE_ERROR + || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ) { info->decompUnavailable = 1; } else { info->decompressedSize += frameContentSize; } - if (ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0) { - DISPLAY("Error: could not decode frame header\n"); - detectError = 1; - break; - } + ERROR_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0, + info_frame_error, "Error: could not decode frame header"); info->windowSize = header.windowSize; /* move to the end of the frame header */ { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); - if (ZSTD_isError(headerSize)) { - DISPLAY("Error: could not determine frame header size\n"); - detectError = 1; - break; - } - { int const ret = fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not move to end of frame header\n"); - detectError = 1; - break; - } } } + ERROR_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size"); + ERROR_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0, + info_frame_error, "Error: could not move to end of frame header"); + } - /* skip the rest of the blocks in the frame */ + /* skip all blocks in the frame */ { int lastBlock = 0; do { BYTE blockHeaderBuffer[3]; - size_t const readBytes = fread(blockHeaderBuffer, 1, 3, srcFile); - if (readBytes != 3) { - DISPLAY("There was a problem reading the block header\n"); - detectError = 1; - break; - } + ERROR_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3, + info_frame_error, "Error while reading block header"); { U32 const blockHeader = MEM_readLE24(blockHeaderBuffer); U32 const blockTypeID = (blockHeader >> 1) & 3; U32 const isRLE = (blockTypeID == 1); U32 const isWrongBlock = (blockTypeID == 3); long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3); - if (isWrongBlock) { - DISPLAY("Error: unsupported block type \n"); - detectError = 1; - break; - } + ERROR_IF(isWrongBlock, info_frame_error, "Error: unsupported block type"); lastBlock = blockHeader & 1; - { int const ret = fseek(srcFile, blockSize, SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not skip to end of block\n"); - detectError = 1; - break; - } } } + ERROR_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0, + info_frame_error, "Error: could not skip to end of block"); + } } while (lastBlock != 1); - - if (detectError) break; } /* check if checksum is used */ { BYTE const frameHeaderDescriptor = headerBuffer[4]; int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; if (contentChecksumFlag) { - int const ret = fseek(srcFile, 4, SEEK_CUR); info->usesCheck = 1; - if (ret != 0) { - DISPLAY("Error: could not skip past checksum\n"); - detectError = 1; - break; - } } } + ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0, + info_frame_error, "Error: could not skip past checksum"); + } } info->numActualFrames++; } /* Skippable frame */ else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { U32 const frameSize = MEM_readLE32(headerBuffer + 4); long const seek = (long)(8 + frameSize - numBytesRead); - int const ret = LONG_SEEK(srcFile, seek, SEEK_CUR); - if (ret != 0) { - DISPLAY("Error: could not find end of skippable frame\n"); - detectError = 1; - break; - } + ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0, + info_frame_error, "Error: could not find end of skippable frame"); info->numSkippableFrames++; } /* unknown content */ else { - detectError = 2; - break; + return info_not_zstd; } - } - } /* end analyzing frame */ - fclose(srcFile); - info->nbFiles = 1; - return detectError; + } /* magic number analysis */ + } /* end analyzing frames */ + return info_success; } -static int getFileInfo(fileInfo_t* info, const char* srcFileName) + +static InfoError +getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName) { - int const isAFile = UTIL_isRegularFile(srcFileName); - if (!isAFile) { - DISPLAY("Error : %s is not a file", srcFileName); - return 3; - } + InfoError status; + FILE* const srcFile = FIO_openSrcFile(inFileName); + ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName); + + info->compressedSize = UTIL_getFileSize(inFileName); + status = FIO_analyzeFrames(info, srcFile); + + fclose(srcFile); + info->nbFiles = 1; + return status; +} + + +/** getFileInfo() : + * Reads information from file, stores in *info + * @return : InfoError status + */ +static InfoError +getFileInfo(fileInfo_t* info, const char* srcFileName) +{ + ERROR_IF(!UTIL_isRegularFile(srcFileName), + info_file_error, "Error : %s is not a file", srcFileName); return getFileInfo_fileConfirmed(info, srcFileName); } -static void displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel){ +static void +displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel) +{ unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB); const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB"; double const windowSizeUnit = (double)info->windowSize / unit; @@ -2194,46 +2255,50 @@ static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2) return total; } -static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){ +static int +FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel) +{ fileInfo_t info; memset(&info, 0, sizeof(info)); - { int const error = getFileInfo(&info, inFileName); - if (error == 1) { + { InfoError const error = getFileInfo(&info, inFileName); + if (error == info_frame_error) { /* display error, but provide output */ - DISPLAY("An error occurred while getting file info \n"); + DISPLAYLEVEL(1, "Error while parsing %s \n", inFileName); } - else if (error == 2) { + else if (error == info_not_zstd) { DISPLAYOUT("File %s not compressed by zstd \n", inFileName); if (displayLevel > 2) DISPLAYOUT("\n"); return 1; } - else if (error == 3) { + else if (error == info_file_error) { /* error occurred while opening the file */ if (displayLevel > 2) DISPLAYOUT("\n"); return 1; } displayInfo(inFileName, &info, displayLevel); *total = FIO_addFInfo(*total, info); + assert(error>=0 || error<=1); return error; } } -int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel){ - unsigned u; - for (u=0; u 1 && displayLevel <= 2) { /* display total */ unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB); const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB"; diff --git a/tests/playTests.sh b/tests/playTests.sh index c7e066b84..b86a0dc40 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -180,6 +180,8 @@ chmod 400 tmpro.zst $ZSTD -q tmpro && die "should have refused to overwrite read-only file" $ZSTD -q -f tmpro rm -f tmpro tmpro.zst + + $ECHO "test : file removal" $ZSTD -f --rm tmp test ! -f tmp # tmp should no longer be present @@ -196,9 +198,19 @@ $ECHO a | $ZSTD --rm > $INTOVOID # --rm should remain silent rm tmp $ZSTD -f tmp && die "tmp not present : should have failed" test ! -f tmp.zst # tmp.zst should not be created +$ECHO "test : -d -f do not delete destination when source is not present" +touch tmp # create destination file +$ZSTD -d -f tmp.zst && die "attempt to decompress a non existing file" +test -f tmp # destination file should still be present +$ECHO "test : -f do not delete destination when source is not present" +rm tmp # erase source file +touch tmp.zst # create destination file +$ZSTD -f tmp && die "attempt to compress a non existing file" +test -f tmp.zst # destination file should still be present +rm tmp* + $ECHO "test : compress multiple files" -rm tmp* $ECHO hello > tmp1 $ECHO world > tmp2 $ZSTD tmp1 tmp2 -o "$INTOVOID" @@ -817,6 +829,7 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory= $ECHO "\n===> adaptive mode " roundTripTest -g270000000 " --adapt" roundTripTest -g27000000 " --adapt=min=1,max=4" +$ECHO "===> test: --adapt must fail on incoherent bounds " ./datagen > tmp $ZSTD -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" @@ -827,6 +840,7 @@ if [ "$1" != "--test-large-data" ]; then fi +############################################################################# $ECHO "\n===> large files tests "