From c2d909e3965e3b6eb5d342847b20ba78cbee5f8b Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 8 Jun 2017 17:06:30 -0700 Subject: [PATCH 01/73] added code for generating dictionary/test files randomly. Still need to make sure dictionary ID matches --- tests/decodecorpus.c | 104 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 96 insertions(+), 8 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index f7b3c854f..8f2e652cd 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1121,13 +1121,12 @@ static void initFrame(frame_t* fr) } /* Return the final seed */ -static U32 generateFrame(U32 seed, frame_t* fr) +static U32 generateFrame(U32 seed, frame_t* fr, int genDict, size_t dictSize) { /* generate a complete frame */ DISPLAYLEVEL(1, "frame seed: %u\n", seed); - initFrame(fr); - + writeFrameHeader(&seed, fr); writeBlocks(&seed, fr); writeChecksum(fr); @@ -1135,6 +1134,7 @@ static U32 generateFrame(U32 seed, frame_t* fr) return seed; } + /*-******************************************************* * Test Mode *********************************************************/ @@ -1215,7 +1215,7 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS else DISPLAYUPDATE("\r%u ", fnum); - seed = generateFrame(seed, &fr); + seed = generateFrame(seed, &fr, 0, 0); { size_t const r = testDecodeSimple(&fr); if (ZSTD_isError(r)) { @@ -1250,7 +1250,7 @@ static int generateFile(U32 seed, const char* const path, DISPLAY("seed: %u\n", seed); - generateFrame(seed, &fr); + generateFrame(seed, &fr, 0, 0); outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path); if (origPath) { @@ -1272,7 +1272,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - seed = generateFrame(seed, &fr); + seed = generateFrame(seed, &fr, 0, 0); if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); @@ -1294,6 +1294,81 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, return 0; } +static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const path, + const char* const origPath, const size_t dictSize) +{ + const size_t minDictSize = 8; + char outPath[MAX_PATH]; + U32 dictID; + BYTE* dictStart; + unsigned fnum; + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + if(snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { + DISPLAY("Error: path too long\n"); + return 1; + } + + /* Generate the dictionary randomly first */ + if(dictSize < minDictSize){ + DISPLAY("Error: dictionary size (%zu) is too small\n", dictSize); + } + else{ + /* variable declaration */ + dictStart = malloc(dictSize); + size_t pos = 0; + dictID = RAND(&seed) + 1; + + /* write dictionary magic number */ + MEM_writeLE32(dictStart + pos, ZSTD_DICT_MAGIC); + pos += 4; + + /* write random dictionary ID */ + MEM_writeLE32(dictStart + pos, dictID); + pos += 4; + + /* randomly generate the rest of the dictionary */ + RAND_buffer(&seed, dictStart + pos, dictSize-8); + outputBuffer(dictStart, dictSize, outPath); + } + + /* generate random compressed/decompressed files */ + for (fnum = 0; fnum < numFiles; fnum++) { + frame_t fr; + size_t returnValue; + BYTE* decompressedPtr = malloc(MAX_DECOMPRESSED_SIZE); + + DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); + + seed = generateFrame(seed, &fr, 1, dictSize); + + if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { + DISPLAY("Error: path too long\n"); + return 1; + } + outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath); + + if (origPath) { + if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) { + DISPLAY("Error: path too long\n"); + return 1; + } + outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); + } + + /* if asked, supply the decompressed version */ + + returnValue = ZSTD_decompress_usingDict(dctx, decompressedPtr, MAX_DECOMPRESSED_SIZE, + fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, + dictStart,dictSize); + + } + + + /* write uncompressed versions of files */ + DISPLAY("This is origPath: %s\nAnd this is numFiles: %d\n", origPath, numFiles); + return 0; +} + /*_******************************************************* * Command line @@ -1350,6 +1425,8 @@ int main(int argc, char** argv) int testMode = 0; const char* path = NULL; const char* origPath = NULL; + int genDict = 0; + unsigned dictSize = (10 << 10); /* 10 kB default */ int argNb; @@ -1410,6 +1487,10 @@ int main(int argc, char** argv) argument++; if (strcmp(argument, "content-size") == 0) { opts.contentSize = 1; + } else if(strcmp(argument, "train-dict") == 0){ + argument += 11; + dictSize = readInt(&argument); + genDict = 1; } else { advancedUsage(argv[0]); return 1; @@ -1441,9 +1522,16 @@ int main(int argc, char** argv) return 1; } - if (numFiles == 0) { + if (numFiles == 0 && genDict == 0) { return generateFile(seed, path, origPath); - } else { + } else if (genDict == 0){ return generateCorpus(seed, numFiles, path, origPath); + } else if (numFiles == 0){ + /* should generate a single file with a dictionary */ + return generateCorpusWithDict(seed, 1, path, origPath, dictSize); + } else{ + /* should generate multiple files with a dictionary */ + return generateCorpusWithDict(seed, numFiles, path, origPath, dictSize); } + } From 233ee5334ee0e2dc549530e500a20c8cec1a4586 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 8 Jun 2017 17:11:33 -0700 Subject: [PATCH 02/73] set the lower bits of frame header descriptor if dictionary is needed --- tests/decodecorpus.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 8f2e652cd..502ea3339 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -240,7 +240,7 @@ struct { } opts; /* advanced options on generation */ /* Generate and write a random frame header */ -static void writeFrameHeader(U32* seed, frame_t* frame) +static void writeFrameHeader(U32* seed, frame_t* frame, int genDict) { BYTE* const op = frame->data; size_t pos = 0; @@ -308,6 +308,8 @@ static void writeFrameHeader(U32* seed, frame_t* frame) { BYTE const frameHeaderDescriptor = (BYTE) ((fcsCode << 6) | (singleSegment << 5) | (1 << 2)); + if(genDict) + frameHeaderDescriptor += 3; /* set lower bits for dictionary ID */ op[pos++] = frameHeaderDescriptor; } @@ -1126,8 +1128,8 @@ static U32 generateFrame(U32 seed, frame_t* fr, int genDict, size_t dictSize) /* generate a complete frame */ DISPLAYLEVEL(1, "frame seed: %u\n", seed); initFrame(fr); - - writeFrameHeader(&seed, fr); + + writeFrameHeader(&seed, fr, genDict); writeBlocks(&seed, fr); writeChecksum(fr); From 125ed5996881bd0502c87de865d46d6627493585 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 8 Jun 2017 17:27:15 -0700 Subject: [PATCH 03/73] made sure dictionary ID was being written in the frame header of each file --- tests/decodecorpus.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 502ea3339..bdcfd9e23 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -240,7 +240,7 @@ struct { } opts; /* advanced options on generation */ /* Generate and write a random frame header */ -static void writeFrameHeader(U32* seed, frame_t* frame, int genDict) +static void writeFrameHeader(U32* seed, frame_t* frame, int genDict, size_t dictSize) { BYTE* const op = frame->data; size_t pos = 0; @@ -306,17 +306,18 @@ static void writeFrameHeader(U32* seed, frame_t* frame, int genDict) pos += 4; { + int dictBits = genDict ? 3 : 0; BYTE const frameHeaderDescriptor = - (BYTE) ((fcsCode << 6) | (singleSegment << 5) | (1 << 2)); - if(genDict) - frameHeaderDescriptor += 3; /* set lower bits for dictionary ID */ + (BYTE) ((fcsCode << 6) | (singleSegment << 5) | (1 << 2) | dictBits); op[pos++] = frameHeaderDescriptor; } if (!singleSegment) { op[pos++] = windowByte; } - + if(genDict) { + MEM_writeLE32(op + pos, (U32) dictSize); + } if (contentSizeFlag) { switch (fcsCode) { default: /* Impossible */ @@ -1129,7 +1130,7 @@ static U32 generateFrame(U32 seed, frame_t* fr, int genDict, size_t dictSize) DISPLAYLEVEL(1, "frame seed: %u\n", seed); initFrame(fr); - writeFrameHeader(&seed, fr, genDict); + writeFrameHeader(&seed, fr, genDict, dictSize); writeBlocks(&seed, fr); writeChecksum(fr); From 11c3987baff48b03fe4ea10a7a15bd7dd7887914 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 9 Jun 2017 09:48:56 -0700 Subject: [PATCH 04/73] added code to extend the offset when a dictionary is detected --- tests/decodecorpus.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index bdcfd9e23..632acabb8 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -608,7 +608,7 @@ static inline void initSeqStore(seqStore_t *seqStore) { /* Randomly generate sequence commands */ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, - size_t contentSize, size_t literalsSize) + size_t contentSize, size_t literalsSize, int genDict, size_t dictSize) { /* The total length of all the matches */ size_t const remainingMatch = contentSize - literalsSize; @@ -665,6 +665,12 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, MIN(frame->header.windowSize, (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) + 1; + if(genDict && (RAND(seed) & 1)) { + /* need to occasionally generate offsets that go past the start */ + /* we still need to be within the windowSize however */ + const U32 lenPastStart = RAND(seed) % dictSize; + offset = MIN(frame->header.windowSize, offset+lenPastStart); + } offsetCode = offset + ZSTD_REP_MOVE; repIndex = 2; } else { @@ -934,7 +940,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, } static size_t writeSequencesBlock(U32* seed, frame_t* frame, size_t contentSize, - size_t literalsSize) + size_t literalsSize, int genDict, size_t dictSize) { seqStore_t seqStore; size_t numSequences; @@ -943,14 +949,14 @@ static size_t writeSequencesBlock(U32* seed, frame_t* frame, size_t contentSize, initSeqStore(&seqStore); /* randomly generate sequences */ - numSequences = generateSequences(seed, frame, &seqStore, contentSize, literalsSize); + numSequences = generateSequences(seed, frame, &seqStore, contentSize, literalsSize, genDict, dictSize); /* write them out to the frame data */ CHECKERR(writeSequences(seed, frame, &seqStore, numSequences)); return numSequences; } -static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize) +static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize, int genDict, size_t dictSize) { BYTE* const blockStart = (BYTE*)frame->data; size_t literalsSize; @@ -962,7 +968,7 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize DISPLAYLEVEL(4, " literals size: %u\n", (U32)literalsSize); - nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize); + nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize, genDict, dictSize); DISPLAYLEVEL(4, " number of sequences: %u\n", (U32)nbSeq); @@ -970,7 +976,7 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize } static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, - int lastBlock) + int lastBlock, int genDict, size_t dictSize) { int const blockTypeDesc = RAND(seed) % 8; size_t blockSize; @@ -1010,7 +1016,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, frame->oldStats = frame->stats; frame->data = op; - compressedSize = writeCompressedBlock(seed, frame, contentSize); + compressedSize = writeCompressedBlock(seed, frame, contentSize, genDict, dictSize); if (compressedSize > contentSize) { blockType = 0; memcpy(op, frame->src, contentSize); @@ -1036,7 +1042,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, frame->data = op; } -static void writeBlocks(U32* seed, frame_t* frame) +static void writeBlocks(U32* seed, frame_t* frame, int genDict, size_t dictSize) { size_t contentLeft = frame->header.contentSize; size_t const maxBlockSize = MIN(MAX_BLOCK_SIZE, frame->header.windowSize); @@ -1059,7 +1065,7 @@ static void writeBlocks(U32* seed, frame_t* frame) } } - writeBlock(seed, frame, blockContentSize, lastBlock); + writeBlock(seed, frame, blockContentSize, lastBlock, genDict, dictSize); contentLeft -= blockContentSize; if (lastBlock) break; @@ -1131,7 +1137,7 @@ static U32 generateFrame(U32 seed, frame_t* fr, int genDict, size_t dictSize) initFrame(fr); writeFrameHeader(&seed, fr, genDict, dictSize); - writeBlocks(&seed, fr); + writeBlocks(&seed, fr, genDict, dictSize); writeChecksum(fr); return seed; From f35f252e36524c14c53fbebb6c20a9e3cc73ed43 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 13 Jun 2017 11:54:43 -0700 Subject: [PATCH 05/73] added code to generate dictionary using finalizeDictionary --- tests/decodecorpus.c | 104 ++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 42 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 632acabb8..357d831e6 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -18,6 +18,7 @@ #include "zstd.h" #include "zstd_internal.h" #include "mem.h" +#include "zdict.h" // Direct access to internal compression functions is required #include "zstd_compress.c" @@ -316,7 +317,8 @@ static void writeFrameHeader(U32* seed, frame_t* frame, int genDict, size_t dict op[pos++] = windowByte; } if(genDict) { - MEM_writeLE32(op + pos, (U32) dictSize); + MEM_writeLE32(op + pos, (U32) dictID); + pos += 4; } if (contentSizeFlag) { switch (fcsCode) { @@ -608,7 +610,7 @@ static inline void initSeqStore(seqStore_t *seqStore) { /* Randomly generate sequence commands */ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, - size_t contentSize, size_t literalsSize, int genDict, size_t dictSize) + size_t contentSize, size_t literalsSize, int genDict, size_t dictSize, BYTE* dictContent) { /* The total length of all the matches */ size_t const remainingMatch = contentSize - literalsSize; @@ -686,11 +688,17 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, repIndex = MIN(2, offsetCode + 1); } } - } while (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart) || offset == 0); + } while (((!genDict) && (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) || offset == 0); { size_t j; for (j = 0; j < matchLen; j++) { - *srcPtr = *(srcPtr-offset); + if(srcPtr-offset < frame->srcStart){ + /* copy from dictionary instead of literals */ + *srcPtr = *(dictContent + dictSize - (offset-(srcPtr-frame->srcStart))); + } + else{ + *srcPtr = *(srcPtr-offset); + } srcPtr++; } } @@ -940,7 +948,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, } static size_t writeSequencesBlock(U32* seed, frame_t* frame, size_t contentSize, - size_t literalsSize, int genDict, size_t dictSize) + size_t literalsSize, int genDict, size_t dictSize, BYTE* dictContent) { seqStore_t seqStore; size_t numSequences; @@ -949,14 +957,14 @@ static size_t writeSequencesBlock(U32* seed, frame_t* frame, size_t contentSize, initSeqStore(&seqStore); /* randomly generate sequences */ - numSequences = generateSequences(seed, frame, &seqStore, contentSize, literalsSize, genDict, dictSize); + numSequences = generateSequences(seed, frame, &seqStore, contentSize, literalsSize, genDict, dictSize, dictContent); /* write them out to the frame data */ CHECKERR(writeSequences(seed, frame, &seqStore, numSequences)); return numSequences; } -static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize, int genDict, size_t dictSize) +static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize, int genDict, size_t dictSize, BYTE* dictContent) { BYTE* const blockStart = (BYTE*)frame->data; size_t literalsSize; @@ -968,7 +976,7 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize DISPLAYLEVEL(4, " literals size: %u\n", (U32)literalsSize); - nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize, genDict, dictSize); + nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize, genDict, dictSize, dictContent); DISPLAYLEVEL(4, " number of sequences: %u\n", (U32)nbSeq); @@ -976,7 +984,7 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize } static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, - int lastBlock, int genDict, size_t dictSize) + int lastBlock, int genDict, size_t dictSize, BYTE* dictContent) { int const blockTypeDesc = RAND(seed) % 8; size_t blockSize; @@ -1016,7 +1024,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, frame->oldStats = frame->stats; frame->data = op; - compressedSize = writeCompressedBlock(seed, frame, contentSize, genDict, dictSize); + compressedSize = writeCompressedBlock(seed, frame, contentSize, genDict, dictSize, dictContent); if (compressedSize > contentSize) { blockType = 0; memcpy(op, frame->src, contentSize); @@ -1042,7 +1050,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, frame->data = op; } -static void writeBlocks(U32* seed, frame_t* frame, int genDict, size_t dictSize) +static void writeBlocks(U32* seed, frame_t* frame, int genDict, size_t dictSize, BYTE* dictContent) { size_t contentLeft = frame->header.contentSize; size_t const maxBlockSize = MIN(MAX_BLOCK_SIZE, frame->header.windowSize); @@ -1065,7 +1073,7 @@ static void writeBlocks(U32* seed, frame_t* frame, int genDict, size_t dictSize) } } - writeBlock(seed, frame, blockContentSize, lastBlock, genDict, dictSize); + writeBlock(seed, frame, blockContentSize, lastBlock, genDict, dictSize, dictContent); contentLeft -= blockContentSize; if (lastBlock) break; @@ -1130,14 +1138,14 @@ static void initFrame(frame_t* fr) } /* Return the final seed */ -static U32 generateFrame(U32 seed, frame_t* fr, int genDict, size_t dictSize) +static U32 generateFrame(U32 seed, frame_t* fr, int genDict, size_t dictSize, BYTE* dictContent) { /* generate a complete frame */ DISPLAYLEVEL(1, "frame seed: %u\n", seed); initFrame(fr); writeFrameHeader(&seed, fr, genDict, dictSize); - writeBlocks(&seed, fr, genDict, dictSize); + writeBlocks(&seed, fr, genDict, dictSize, dictContent); writeChecksum(fr); return seed; @@ -1224,7 +1232,7 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS else DISPLAYUPDATE("\r%u ", fnum); - seed = generateFrame(seed, &fr, 0, 0); + seed = generateFrame(seed, &fr, 0, 0, NULL); { size_t const r = testDecodeSimple(&fr); if (ZSTD_isError(r)) { @@ -1259,7 +1267,7 @@ static int generateFile(U32 seed, const char* const path, DISPLAY("seed: %u\n", seed); - generateFrame(seed, &fr, 0, 0); + generateFrame(seed, &fr, 0, 0, NULL); outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path); if (origPath) { @@ -1281,7 +1289,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - seed = generateFrame(seed, &fr, 0, 0); + seed = generateFrame(seed, &fr, 0, 0, NULL); if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); @@ -1308,9 +1316,11 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const { const size_t minDictSize = 8; char outPath[MAX_PATH]; + BYTE* dictContent; + BYTE* fullDict; U32 dictID; - BYTE* dictStart; unsigned fnum; + BYTE* decompressedPtr; ZSTD_DCtx* dctx = ZSTD_createDCtx(); if(snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); @@ -1318,37 +1328,50 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } /* Generate the dictionary randomly first */ - if(dictSize < minDictSize){ - DISPLAY("Error: dictionary size (%zu) is too small\n", dictSize); - } - else{ - /* variable declaration */ - dictStart = malloc(dictSize); - size_t pos = 0; - dictID = RAND(&seed) + 1; + dictContent = malloc(dictSize-400); + dictID = RAND(&seed); + fullDict = malloc(dictSize); + RAND_buffer(&seed, dictContent, dictSize-40); + { + /* create random samples */ + unsigned numSamples = RAND(&seed); + unsigned i = 0; + size_t* sampleSizes = malloc(numSamples*sizeof(size_t)); + size_t* curr = sampleSizes; + size_t totalSize = 0; + while(i < numSamples){ + *curr = RAND(&seed) % (4 << 20); + totalSize += *curr; + curr++; + } + ZDICT_params_t zdictParams; + BYTE* samples = malloc(totalSize); + RAND_buffer(&seed, samples, totalSize); - /* write dictionary magic number */ - MEM_writeLE32(dictStart + pos, ZSTD_DICT_MAGIC); - pos += 4; + /* set dictionary params */ + memset(&zdictParams, 0, sizeof(zdictParams)); + zdictParams.notificationLevel = 1; + zdictParams.dictID = dictID; + zdictParams.compressionLevel = 5; - /* write random dictionary ID */ - MEM_writeLE32(dictStart + pos, dictID); - pos += 4; - - /* randomly generate the rest of the dictionary */ - RAND_buffer(&seed, dictStart + pos, dictSize-8); - outputBuffer(dictStart, dictSize, outPath); + /* finalize dictionary with random samples */ + ZDICT_finalizeDictionary(fullDict, dictSize, + dictContent, dictSize-400, + samples, sampleSizes, numSamples, + zdictParams); } + + decompressedPtr = malloc(MAX_DECOMPRESSED_SIZE); /* generate random compressed/decompressed files */ for (fnum = 0; fnum < numFiles; fnum++) { frame_t fr; size_t returnValue; - BYTE* decompressedPtr = malloc(MAX_DECOMPRESSED_SIZE); + DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - seed = generateFrame(seed, &fr, 1, dictSize); + seed = generateFrame(seed, &fr, 1, dictSize, dictContent); if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); @@ -1368,13 +1391,10 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const returnValue = ZSTD_decompress_usingDict(dctx, decompressedPtr, MAX_DECOMPRESSED_SIZE, fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, - dictStart,dictSize); + fullDict, dictSize); } - - /* write uncompressed versions of files */ - DISPLAY("This is origPath: %s\nAnd this is numFiles: %d\n", origPath, numFiles); return 0; } From 8dd621f788f7ba147ea8e3a727517d9975c8193f Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 13 Jun 2017 14:19:35 -0700 Subject: [PATCH 06/73] changed makefile to include zdict functions, wrote out dictionary to path --- tests/Makefile | 2 +- tests/decodecorpus.c | 42 ++++++++++++++++++++++++------------------ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index c275c081f..debe83898 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -180,7 +180,7 @@ legacy : CPPFLAGS+= -I$(ZSTDDIR)/legacy legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c $(CC) $(FLAGS) $^ -o $@$(EXT) -decodecorpus : $(filter-out $(ZSTDDIR)/compress/zstd_compress.c, $(wildcard $(ZSTD_FILES))) decodecorpus.c +decodecorpus : $(filter-out $(ZSTDDIR)/compress/zstd_compress.c, $(wildcard $(ZSTD_FILES))) $(ZDICT_FILES) decodecorpus.c $(CC) $(FLAGS) $^ -o $@$(EXT) -lm symbols : symbols.c diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 357d831e6..32b3659d1 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -18,6 +18,7 @@ #include "zstd.h" #include "zstd_internal.h" #include "mem.h" +#define ZDICT_STATIC_LINKING_ONLY #include "zdict.h" // Direct access to internal compression functions is required @@ -241,7 +242,7 @@ struct { } opts; /* advanced options on generation */ /* Generate and write a random frame header */ -static void writeFrameHeader(U32* seed, frame_t* frame, int genDict, size_t dictSize) +static void writeFrameHeader(U32* seed, frame_t* frame, int genDict, U32 dictID) { BYTE* const op = frame->data; size_t pos = 0; @@ -692,9 +693,9 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, { size_t j; for (j = 0; j < matchLen; j++) { - if(srcPtr-offset < frame->srcStart){ + if((void*)(srcPtr-offset) < (void*)frame->srcStart){ /* copy from dictionary instead of literals */ - *srcPtr = *(dictContent + dictSize - (offset-(srcPtr-frame->srcStart))); + *srcPtr = *(dictContent + dictSize - (offset-(srcPtr-(BYTE*)frame->srcStart))); } else{ *srcPtr = *(srcPtr-offset); @@ -1138,13 +1139,13 @@ static void initFrame(frame_t* fr) } /* Return the final seed */ -static U32 generateFrame(U32 seed, frame_t* fr, int genDict, size_t dictSize, BYTE* dictContent) +static U32 generateFrame(U32 seed, frame_t* fr, int genDict, size_t dictSize, BYTE* dictContent, U32 dictID) { /* generate a complete frame */ DISPLAYLEVEL(1, "frame seed: %u\n", seed); initFrame(fr); - writeFrameHeader(&seed, fr, genDict, dictSize); + writeFrameHeader(&seed, fr, genDict, dictID); writeBlocks(&seed, fr, genDict, dictSize, dictContent); writeChecksum(fr); @@ -1232,7 +1233,7 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS else DISPLAYUPDATE("\r%u ", fnum); - seed = generateFrame(seed, &fr, 0, 0, NULL); + seed = generateFrame(seed, &fr, 0, 0, NULL, 0); { size_t const r = testDecodeSimple(&fr); if (ZSTD_isError(r)) { @@ -1267,7 +1268,7 @@ static int generateFile(U32 seed, const char* const path, DISPLAY("seed: %u\n", seed); - generateFrame(seed, &fr, 0, 0, NULL); + generateFrame(seed, &fr, 0, 0, NULL, 0); outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path); if (origPath) { @@ -1289,7 +1290,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - seed = generateFrame(seed, &fr, 0, 0, NULL); + seed = generateFrame(seed, &fr, 0, 0, NULL, 0); if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); @@ -1314,7 +1315,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const path, const char* const origPath, const size_t dictSize) { - const size_t minDictSize = 8; + DISPLAY("in generateCorpusWithDict()\n"); char outPath[MAX_PATH]; BYTE* dictContent; BYTE* fullDict; @@ -1326,7 +1327,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const DISPLAY("Error: path too long\n"); return 1; } - + DISPLAY("generating the dictionary randomly\n"); /* Generate the dictionary randomly first */ dictContent = malloc(dictSize-400); dictID = RAND(&seed); @@ -1334,12 +1335,13 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const RAND_buffer(&seed, dictContent, dictSize-40); { /* create random samples */ - unsigned numSamples = RAND(&seed); + unsigned numSamples = RAND(&seed) % 50; + DISPLAY("num samples: %u\n", numSamples); unsigned i = 0; size_t* sampleSizes = malloc(numSamples*sizeof(size_t)); size_t* curr = sampleSizes; size_t totalSize = 0; - while(i < numSamples){ + while(i++ < numSamples){ *curr = RAND(&seed) % (4 << 20); totalSize += *curr; curr++; @@ -1349,19 +1351,23 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const RAND_buffer(&seed, samples, totalSize); /* set dictionary params */ - memset(&zdictParams, 0, sizeof(zdictParams)); - zdictParams.notificationLevel = 1; + memset(&zdictParams, 0, sizeof(ZDICT_params_t)); zdictParams.dictID = dictID; - zdictParams.compressionLevel = 5; /* finalize dictionary with random samples */ ZDICT_finalizeDictionary(fullDict, dictSize, dictContent, dictSize-400, samples, sampleSizes, numSamples, zdictParams); + /* write out dictionary */ + if(snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH){ + DISPLAY("Error: dictionary path too long\n"); + return 1; + } + outputBuffer(fullDict, dictSize, outPath); } - + DISPLAY("generating compressed files\n"); decompressedPtr = malloc(MAX_DECOMPRESSED_SIZE); /* generate random compressed/decompressed files */ for (fnum = 0; fnum < numFiles; fnum++) { @@ -1371,7 +1377,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - seed = generateFrame(seed, &fr, 1, dictSize, dictContent); + seed = generateFrame(seed, &fr, 1, dictSize, dictContent, dictID); if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); @@ -1394,7 +1400,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const fullDict, dictSize); } - + DISPLAY("end of function\n"); return 0; } From 07cfc975bb1ea9070b934ead4ce264b1eeef2ab4 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 13 Jun 2017 17:27:02 -0700 Subject: [PATCH 07/73] debugging error with finalizeDictionary() not writing dictionary properly --- tests/decodecorpus.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 32b3659d1..8e49e65b8 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1331,18 +1331,19 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const /* Generate the dictionary randomly first */ dictContent = malloc(dictSize-400); dictID = RAND(&seed); + DISPLAY("this is the dictID that is being stored: %u\n", dictID); fullDict = malloc(dictSize); RAND_buffer(&seed, dictContent, dictSize-40); { + size_t dictWriteSize = 0; /* create random samples */ unsigned numSamples = RAND(&seed) % 50; - DISPLAY("num samples: %u\n", numSamples); unsigned i = 0; size_t* sampleSizes = malloc(numSamples*sizeof(size_t)); size_t* curr = sampleSizes; size_t totalSize = 0; while(i++ < numSamples){ - *curr = RAND(&seed) % (4 << 20); + *curr = RAND(&seed) % (4 << 15); totalSize += *curr; curr++; } @@ -1353,12 +1354,15 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const /* set dictionary params */ memset(&zdictParams, 0, sizeof(ZDICT_params_t)); zdictParams.dictID = dictID; - /* finalize dictionary with random samples */ - ZDICT_finalizeDictionary(fullDict, dictSize, + dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, dictContent, dictSize-400, samples, sampleSizes, numSamples, zdictParams); + DISPLAY("total size: %zu %zu\n", totalSize, dictSize); + if(dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)){ + DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); + } /* write out dictionary */ if(snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH){ DISPLAY("Error: dictionary path too long\n"); From aa95bc0a5f86c8746390c5363ba5b0aff798f487 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 14 Jun 2017 10:54:47 -0700 Subject: [PATCH 08/73] code refactoring and adding some print statements for debugging --- tests/decodecorpus.c | 55 +++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 8e49e65b8..5a897cc4e 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1315,7 +1315,6 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const path, const char* const origPath, const size_t dictSize) { - DISPLAY("in generateCorpusWithDict()\n"); char outPath[MAX_PATH]; BYTE* dictContent; BYTE* fullDict; @@ -1327,42 +1326,51 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const DISPLAY("Error: path too long\n"); return 1; } - DISPLAY("generating the dictionary randomly\n"); + /* Generate the dictionary randomly first */ dictContent = malloc(dictSize-400); dictID = RAND(&seed); - DISPLAY("this is the dictID that is being stored: %u\n", dictID); fullDict = malloc(dictSize); RAND_buffer(&seed, dictContent, dictSize-40); { size_t dictWriteSize = 0; + /* create random samples */ unsigned numSamples = RAND(&seed) % 50; + BYTE* samples; unsigned i = 0; size_t* sampleSizes = malloc(numSamples*sizeof(size_t)); - size_t* curr = sampleSizes; - size_t totalSize = 0; - while(i++ < numSamples){ - *curr = RAND(&seed) % (4 << 15); - totalSize += *curr; - curr++; + { + size_t* curr = sampleSizes; + size_t totalSize = 0; + while(i++ < numSamples){ + *curr = RAND(&seed) % (4 << 15); + totalSize += *curr; + curr++; + } + samples = malloc(totalSize); + RAND_buffer(&seed, samples, totalSize); + } + + { + /* set dictionary params */ + ZDICT_params_t zdictParams; + memset(&zdictParams, 0, sizeof(zdictParams)); + zdictParams.dictID = dictID; + zdictParams.notificationLevel = 1; + + /* finalize dictionary with random samples */ + dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, + dictContent, dictSize-400, + samples, sampleSizes, numSamples, + zdictParams); } - ZDICT_params_t zdictParams; - BYTE* samples = malloc(totalSize); - RAND_buffer(&seed, samples, totalSize); - /* set dictionary params */ - memset(&zdictParams, 0, sizeof(ZDICT_params_t)); - zdictParams.dictID = dictID; - /* finalize dictionary with random samples */ - dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, - dictContent, dictSize-400, - samples, sampleSizes, numSamples, - zdictParams); - DISPLAY("total size: %zu %zu\n", totalSize, dictSize); if(dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)){ DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); + return 1; } + /* write out dictionary */ if(snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH){ DISPLAY("Error: dictionary path too long\n"); @@ -1373,6 +1381,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const DISPLAY("generating compressed files\n"); decompressedPtr = malloc(MAX_DECOMPRESSED_SIZE); + /* generate random compressed/decompressed files */ for (fnum = 0; fnum < numFiles; fnum++) { frame_t fr; @@ -1403,6 +1412,10 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, fullDict, dictSize); + if(ZSTD_isError(returnValue)){ + DISPLAY("Error: %s", ZSTD_getErrorName(returnValue)); + } + } DISPLAY("end of function\n"); return 0; From 3b0e24c06be1f7ac25f1410831baa848d0f012d3 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 14 Jun 2017 12:04:22 -0700 Subject: [PATCH 09/73] added error message to catch case where size not given --- tests/decodecorpus.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 5a897cc4e..9bd6506e2 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1326,12 +1326,15 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const DISPLAY("Error: path too long\n"); return 1; } - + if(dictSize < 400){ + DISPLAY("Error: either no size given or given dictionary size is too small\n"); + return 1; + } /* Generate the dictionary randomly first */ dictContent = malloc(dictSize-400); dictID = RAND(&seed); fullDict = malloc(dictSize); - RAND_buffer(&seed, dictContent, dictSize-40); + RAND_buffer(&seed, dictContent, dictSize-400); { size_t dictWriteSize = 0; From 664ed05ff6af2190c3c65e80458343484e12bbec Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 14 Jun 2017 16:42:51 -0700 Subject: [PATCH 10/73] changed randomly generated samples to randomly selected from the dictionary content --- tests/decodecorpus.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 9bd6506e2..19cf6127b 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1352,7 +1352,22 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const curr++; } samples = malloc(totalSize); - RAND_buffer(&seed, samples, totalSize); + + /* reset pointer and counter */ + curr = sampleSizes; + i = 0; + + { + /* take substring from dictionary content */ + size_t pos = 0; + const BYTE* endDict = dictContent + dictSize - 400; + while(i++ < numSamples){ + size_t currSize = *(curr++); + BYTE* startSubstring = endDict - currSize; + memcpy(samples + pos, (void*)startSubstring, currSize); + pos += currSize; + } + } } { From d93207a79f1cebffa2ab9405bf823168d73dc241 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 14 Jun 2017 17:23:56 -0700 Subject: [PATCH 11/73] changed dictionary from using fixed amount of bytes for the header / entropy tables --- tests/decodecorpus.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 19cf6127b..ba1a9160a 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1316,25 +1316,30 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const const char* const origPath, const size_t dictSize) { char outPath[MAX_PATH]; - BYTE* dictContent; BYTE* fullDict; U32 dictID; unsigned fnum; BYTE* decompressedPtr; + BYTE* dictContent; + const size_t headerSize = dictSize/4; + const size_t dictContentSize = dictSize - dictSize/4; ZSTD_DCtx* dctx = ZSTD_createDCtx(); if(snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); return 1; } - if(dictSize < 400){ - DISPLAY("Error: either no size given or given dictionary size is too small\n"); - return 1; + { + /* use 3/4 of dictionary for content, save rest for header/entropy tables */ + if(dictContentSize < 128 || dictSize < 256){ + DISPLAY("Error: dictionary size is too small\n"); + return 1; + } } /* Generate the dictionary randomly first */ - dictContent = malloc(dictSize-400); dictID = RAND(&seed); fullDict = malloc(dictSize); - RAND_buffer(&seed, dictContent, dictSize-400); + dictContent = fullDict + headerSize; + RAND_buffer(&seed, (void*)dictContent, dictContentSize); { size_t dictWriteSize = 0; @@ -1347,7 +1352,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const size_t* curr = sampleSizes; size_t totalSize = 0; while(i++ < numSamples){ - *curr = RAND(&seed) % (4 << 15); + *curr = RAND(&seed) % dictContentSize; totalSize += *curr; curr++; } @@ -1360,7 +1365,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const { /* take substring from dictionary content */ size_t pos = 0; - const BYTE* endDict = dictContent + dictSize - 400; + BYTE* endDict = dictContent + dictContentSize; while(i++ < numSamples){ size_t currSize = *(curr++); BYTE* startSubstring = endDict - currSize; @@ -1379,7 +1384,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const /* finalize dictionary with random samples */ dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, - dictContent, dictSize-400, + dictContent, dictContentSize, samples, sampleSizes, numSamples, zdictParams); } From 03a208e27cbfe69bd58d27205c16eaf643dd22e9 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 14 Jun 2017 18:06:16 -0700 Subject: [PATCH 12/73] changed samples to geometric distribution so that it is huffman compressible --- tests/decodecorpus.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index ba1a9160a..b1f4d701a 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1344,17 +1344,22 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const size_t dictWriteSize = 0; /* create random samples */ - unsigned numSamples = RAND(&seed) % 50; + unsigned numSamples = 0; BYTE* samples; unsigned i = 0; size_t* sampleSizes = malloc(numSamples*sizeof(size_t)); { size_t* curr = sampleSizes; size_t totalSize = 0; - while(i++ < numSamples){ - *curr = RAND(&seed) % dictContentSize; - totalSize += *curr; - curr++; + while(numSamples < 100){ + unsigned numReps = ROUND(RAND_exp(&seed, 10)); + size_t randSize = RAND(&seed) % dictContentSize; + while(numReps-- > 0){ + *curr = randSize; + totalSize += *curr; + curr++; + } + numSamples += numReps; } samples = malloc(totalSize); @@ -1374,7 +1379,6 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } } } - { /* set dictionary params */ ZDICT_params_t zdictParams; From 52934fbfc87bef37f7e95c380c3118bcb383b2a2 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 09:47:29 -0700 Subject: [PATCH 13/73] added print statements for debugging, fixed first segfault --- tests/decodecorpus.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index b1f4d701a..ec826b01e 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1354,25 +1354,27 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const while(numSamples < 100){ unsigned numReps = ROUND(RAND_exp(&seed, 10)); size_t randSize = RAND(&seed) % dictContentSize; + numSamples += numReps; while(numReps-- > 0){ *curr = randSize; totalSize += *curr; curr++; } - numSamples += numReps; } samples = malloc(totalSize); /* reset pointer and counter */ curr = sampleSizes; i = 0; - + DISPLAY("total size: %zu\n", totalSize); { /* take substring from dictionary content */ size_t pos = 0; BYTE* endDict = dictContent + dictContentSize; while(i++ < numSamples){ size_t currSize = *(curr++); + DISPLAY("current size: %zu\n", currSize); + DISPLAY("dictionary content size: %zu\n", dictContentSize); BYTE* startSubstring = endDict - currSize; memcpy(samples + pos, (void*)startSubstring, currSize); pos += currSize; From 623b8fce85a49c8d4352e240925469a2d9f88944 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 11:37:24 -0700 Subject: [PATCH 14/73] add print statements for debugging and change malloc with updated size --- tests/decodecorpus.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index ec826b01e..96799f8bc 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1347,47 +1347,47 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const unsigned numSamples = 0; BYTE* samples; unsigned i = 0; - size_t* sampleSizes = malloc(numSamples*sizeof(size_t)); + size_t* sampleSizes = malloc(255*sizeof(size_t)); { size_t* curr = sampleSizes; size_t totalSize = 0; - while(numSamples < 100){ - unsigned numReps = ROUND(RAND_exp(&seed, 10)); + unsigned numReps = 1; + while(numSamples < 200){ size_t randSize = RAND(&seed) % dictContentSize; + unsigned counter = numReps; numSamples += numReps; - while(numReps-- > 0){ + while(counter-- > 0){ *curr = randSize; totalSize += *curr; curr++; } + numReps *= 2; } samples = malloc(totalSize); /* reset pointer and counter */ curr = sampleSizes; i = 0; - DISPLAY("total size: %zu\n", totalSize); { /* take substring from dictionary content */ size_t pos = 0; BYTE* endDict = dictContent + dictContentSize; while(i++ < numSamples){ size_t currSize = *(curr++); - DISPLAY("current size: %zu\n", currSize); - DISPLAY("dictionary content size: %zu\n", dictContentSize); BYTE* startSubstring = endDict - currSize; memcpy(samples + pos, (void*)startSubstring, currSize); pos += currSize; } } } + DISPLAY("==================done with generation====================\n"); { /* set dictionary params */ ZDICT_params_t zdictParams; memset(&zdictParams, 0, sizeof(zdictParams)); zdictParams.dictID = dictID; zdictParams.notificationLevel = 1; - + DISPLAY("===================zdict params================\n"); /* finalize dictionary with random samples */ dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, dictContent, dictContentSize, @@ -1399,7 +1399,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); return 1; } - + DISPLAY("=================done with finalize=================\n"); /* write out dictionary */ if(snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH){ DISPLAY("Error: dictionary path too long\n"); @@ -1442,7 +1442,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const fullDict, dictSize); if(ZSTD_isError(returnValue)){ - DISPLAY("Error: %s", ZSTD_getErrorName(returnValue)); + DISPLAY("Error: %s\n", ZSTD_getErrorName(returnValue)); } } From 2f30433cd797a64f6c81b1c2f090214a4607f9bb Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 15 Jun 2017 18:06:32 -0700 Subject: [PATCH 15/73] fixing general style --- tests/decodecorpus.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 96799f8bc..9565d33d7 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1324,13 +1324,13 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const const size_t headerSize = dictSize/4; const size_t dictContentSize = dictSize - dictSize/4; ZSTD_DCtx* dctx = ZSTD_createDCtx(); - if(snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { + if (snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); return 1; } { /* use 3/4 of dictionary for content, save rest for header/entropy tables */ - if(dictContentSize < 128 || dictSize < 256){ + if (dictContentSize < 128 || dictSize < 256) { DISPLAY("Error: dictionary size is too small\n"); return 1; } @@ -1352,7 +1352,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const size_t* curr = sampleSizes; size_t totalSize = 0; unsigned numReps = 1; - while(numSamples < 200){ + while (numSamples < 200) { size_t randSize = RAND(&seed) % dictContentSize; unsigned counter = numReps; numSamples += numReps; @@ -1568,7 +1568,7 @@ int main(int argc, char** argv) argument++; if (strcmp(argument, "content-size") == 0) { opts.contentSize = 1; - } else if(strcmp(argument, "train-dict") == 0){ + } else if (strcmp(argument, "train-dict") == 0) { argument += 11; dictSize = readInt(&argument); genDict = 1; @@ -1607,11 +1607,9 @@ int main(int argc, char** argv) return generateFile(seed, path, origPath); } else if (genDict == 0){ return generateCorpus(seed, numFiles, path, origPath); - } else if (numFiles == 0){ - /* should generate a single file with a dictionary */ - return generateCorpusWithDict(seed, 1, path, origPath, dictSize); - } else{ - /* should generate multiple files with a dictionary */ + } else { + /* should generate files with a dictionary */ + numFiles = (numFiles == 0) ? 1 : numFiles; return generateCorpusWithDict(seed, numFiles, path, origPath, dictSize); } From 09ac51b56be02deea1782aa9e0fda908feb3956b Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 09:24:38 -0700 Subject: [PATCH 16/73] small style changes --- tests/decodecorpus.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 9565d33d7..4f8e89a2b 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1342,7 +1342,6 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const RAND_buffer(&seed, (void*)dictContent, dictContentSize); { size_t dictWriteSize = 0; - /* create random samples */ unsigned numSamples = 0; BYTE* samples; @@ -1361,10 +1360,11 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const totalSize += *curr; curr++; } + DISPLAY("random size: %zu\n", randSize); numReps *= 2; } samples = malloc(totalSize); - + DISPLAY("%zu\n", totalSize); /* reset pointer and counter */ curr = sampleSizes; i = 0; @@ -1372,7 +1372,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const /* take substring from dictionary content */ size_t pos = 0; BYTE* endDict = dictContent + dictContentSize; - while(i++ < numSamples){ + while (i++ < numSamples) { size_t currSize = *(curr++); BYTE* startSubstring = endDict - currSize; memcpy(samples + pos, (void*)startSubstring, currSize); @@ -1441,7 +1441,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, fullDict, dictSize); - if(ZSTD_isError(returnValue)){ + if (ZSTD_isError(returnValue)) { DISPLAY("Error: %s\n", ZSTD_getErrorName(returnValue)); } From 33467add60a9ef1c3c0e0e937b03e78159c25f1c Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 11:27:50 -0700 Subject: [PATCH 17/73] made changes so that HUF_writeCTable no longer fails --- tests/decodecorpus.c | 50 +++++++++++++------------------------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 4f8e89a2b..00eaa9dab 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1342,42 +1342,22 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const RAND_buffer(&seed, (void*)dictContent, dictContentSize); { size_t dictWriteSize = 0; - /* create random samples */ - unsigned numSamples = 0; - BYTE* samples; - unsigned i = 0; - size_t* sampleSizes = malloc(255*sizeof(size_t)); + + /* create samples */ + unsigned numSamples = 4; + BYTE* samples = malloc(5000*sizeof(BYTE)); + size_t* sampleSizes = malloc(numSamples*sizeof(size_t)); { - size_t* curr = sampleSizes; - size_t totalSize = 0; - unsigned numReps = 1; - while (numSamples < 200) { - size_t randSize = RAND(&seed) % dictContentSize; - unsigned counter = numReps; - numSamples += numReps; - while(counter-- > 0){ - *curr = randSize; - totalSize += *curr; - curr++; - } - DISPLAY("random size: %zu\n", randSize); - numReps *= 2; - } - samples = malloc(totalSize); - DISPLAY("%zu\n", totalSize); - /* reset pointer and counter */ - curr = sampleSizes; - i = 0; - { - /* take substring from dictionary content */ - size_t pos = 0; - BYTE* endDict = dictContent + dictContentSize; - while (i++ < numSamples) { - size_t currSize = *(curr++); - BYTE* startSubstring = endDict - currSize; - memcpy(samples + pos, (void*)startSubstring, currSize); - pos += currSize; + unsigned i = 1; + size_t currSize = 1; + BYTE* curr = samples; + while (i <= 4) { + *(sampleSizes + i - 1) = currSize; + for (int j = 0; j < currSize; j++) { + *(curr++) = (BYTE)i; } + i++; + currSize *= 16; } } DISPLAY("==================done with generation====================\n"); @@ -1386,7 +1366,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const ZDICT_params_t zdictParams; memset(&zdictParams, 0, sizeof(zdictParams)); zdictParams.dictID = dictID; - zdictParams.notificationLevel = 1; + zdictParams.notificationLevel = 5; DISPLAY("===================zdict params================\n"); /* finalize dictionary with random samples */ dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, From da4ddc1f76f7c89cf0715de7d4543bab72b261dc Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 11:40:45 -0700 Subject: [PATCH 18/73] fixed decompression so that it uses compressed version --- tests/decodecorpus.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 00eaa9dab..8c94154df 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1353,7 +1353,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const BYTE* curr = samples; while (i <= 4) { *(sampleSizes + i - 1) = currSize; - for (int j = 0; j < currSize; j++) { + for (size_t j = 0; j < currSize; j++) { *(curr++) = (BYTE)i; } i++; @@ -1416,9 +1416,9 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } /* if asked, supply the decompressed version */ - + DISPLAY("Attempting to decompress using the dictionary\n"); returnValue = ZSTD_decompress_usingDict(dctx, decompressedPtr, MAX_DECOMPRESSED_SIZE, - fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, + fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, fullDict, dictSize); if (ZSTD_isError(returnValue)) { From 8f4fa43fe8f9e5c5c1264c215d9a8ba35abcf429 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 13:21:50 -0700 Subject: [PATCH 19/73] got files to decompress --- tests/decodecorpus.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 8c94154df..98c84bcb4 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -671,7 +671,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, if(genDict && (RAND(seed) & 1)) { /* need to occasionally generate offsets that go past the start */ /* we still need to be within the windowSize however */ - const U32 lenPastStart = RAND(seed) % dictSize; + U32 const lenPastStart = RAND(seed) % dictSize; offset = MIN(frame->header.windowSize, offset+lenPastStart); } offsetCode = offset + ZSTD_REP_MOVE; @@ -693,7 +693,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, { size_t j; for (j = 0; j < matchLen; j++) { - if((void*)(srcPtr-offset) < (void*)frame->srcStart){ + if ((void*)(srcPtr - offset) < (void*)frame->srcStart) { /* copy from dictionary instead of literals */ *srcPtr = *(dictContent + dictSize - (offset-(srcPtr-(BYTE*)frame->srcStart))); } @@ -1330,7 +1330,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } { /* use 3/4 of dictionary for content, save rest for header/entropy tables */ - if (dictContentSize < 128 || dictSize < 256) { + if (dictContentSize < ZDICT_CONTENTSIZE_MIN || dictSize < ZDICT_DICTSIZE_MIN) { DISPLAY("Error: dictionary size is too small\n"); return 1; } @@ -1399,7 +1399,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - seed = generateFrame(seed, &fr, 1, dictSize, dictContent, dictID); + seed = generateFrame(seed, &fr, 1, dictContentSize, dictContent, dictID); if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); @@ -1421,6 +1421,14 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, fullDict, dictSize); + { + size_t checkDiff = (BYTE*)fr.src - (BYTE*)fr.srcStart; + for (size_t i = 0; i < checkDiff; i++) { + if (*((BYTE*)(fr.srcStart + i)) != *((BYTE*)(decompressedPtr + i))) { + DISPLAY("i: %zu, fr: %u, decomp: %u\n", i, *((BYTE*)(fr.srcStart + i)), *((BYTE*)(decompressedPtr + i))); + } + } + } if (ZSTD_isError(returnValue)) { DISPLAY("Error: %s\n", ZSTD_getErrorName(returnValue)); } From 1fc1a35dc472eb26bf6c28fe098994b8ec2539dc Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 14:02:27 -0700 Subject: [PATCH 20/73] added calls to free to match calls to malloc, made some stylistic changes, added init function for new struct --- tests/decodecorpus.c | 49 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 98c84bcb4..610df7cb4 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -233,6 +233,12 @@ typedef struct { cblockStats_t oldStats; /* so they can be rolled back if uncompressible */ } frame_t; +typedef struct { + int useDict; + U32 dictID; + size_t dictSize; + BYTE* dictContent; +} dictOptions; /*-******************************************************* * Generator Functions *********************************************************/ @@ -1312,6 +1318,16 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, return 0; } +static dictOptions initDictOptions(int useDict, U32 dictID, size_t dictSize, BYTE* dictContent){ + /* allocate space statically */ + dictOptions dictOp; + memset((void*)(&dictOp), 0, sizeof(dictOp)); + dictOp.useDict = useDict; + dictOp.dictID = dictID; + dictOp.dictSize = dictSize; + dictOp.dictContent = dictContent; + return dictOp; +} static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const path, const char* const origPath, const size_t dictSize) { @@ -1338,6 +1354,10 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const /* Generate the dictionary randomly first */ dictID = RAND(&seed); fullDict = malloc(dictSize); + if (fullDict == NULL) { + DISPLAY("Error: could not allocate space for full dictionary.\n"); + return 1; + } dictContent = fullDict + headerSize; RAND_buffer(&seed, (void*)dictContent, dictContentSize); { @@ -1347,6 +1367,11 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const unsigned numSamples = 4; BYTE* samples = malloc(5000*sizeof(BYTE)); size_t* sampleSizes = malloc(numSamples*sizeof(size_t)); + if (samples == NULL || sampleSizes == NULL) { + DISPLAY("Error: could not generate samples for the dictionary.\n"); + free(fullDict); + return 1; + } { unsigned i = 1; size_t currSize = 1; @@ -1360,36 +1385,40 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const currSize *= 16; } } - DISPLAY("==================done with generation====================\n"); { /* set dictionary params */ ZDICT_params_t zdictParams; memset(&zdictParams, 0, sizeof(zdictParams)); zdictParams.dictID = dictID; - zdictParams.notificationLevel = 5; - DISPLAY("===================zdict params================\n"); + zdictParams.notificationLevel = 1; /* finalize dictionary with random samples */ dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, dictContent, dictContentSize, samples, sampleSizes, numSamples, zdictParams); } - + free(samples); + free(sampleSizes); if(dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)){ DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); + free(fullDict); return 1; } - DISPLAY("=================done with finalize=================\n"); /* write out dictionary */ if(snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH){ DISPLAY("Error: dictionary path too long\n"); + free(fullDict); return 1; } outputBuffer(fullDict, dictSize, outPath); } - DISPLAY("generating compressed files\n"); decompressedPtr = malloc(MAX_DECOMPRESSED_SIZE); + if (decompressedPtr == NULL) { + DISPLAY("Error: could not allocate memory for decompressed pointer\n"); + free(fullDict); + return 1; + } /* generate random compressed/decompressed files */ for (fnum = 0; fnum < numFiles; fnum++) { @@ -1403,6 +1432,8 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); + free(fullDict); + free(decompressedPtr); return 1; } outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath); @@ -1410,13 +1441,14 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const if (origPath) { if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); + free(fullDict); + free(decompressedPtr); return 1; } outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); } /* if asked, supply the decompressed version */ - DISPLAY("Attempting to decompress using the dictionary\n"); returnValue = ZSTD_decompress_usingDict(dctx, decompressedPtr, MAX_DECOMPRESSED_SIZE, fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, fullDict, dictSize); @@ -1434,7 +1466,8 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } } - DISPLAY("end of function\n"); + free(decompressedPtr); + free(fullDict); return 0; } From 34f4e5cab24c202226a523bc7a88f1c05234c61d Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 14:07:56 -0700 Subject: [PATCH 21/73] changed if statement styling --- tests/decodecorpus.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 610df7cb4..ffaad4a17 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -674,7 +674,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, MIN(frame->header.windowSize, (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) + 1; - if(genDict && (RAND(seed) & 1)) { + if (genDict && (RAND(seed) & 1)) { /* need to occasionally generate offsets that go past the start */ /* we still need to be within the windowSize however */ U32 const lenPastStart = RAND(seed) % dictSize; @@ -703,7 +703,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, /* copy from dictionary instead of literals */ *srcPtr = *(dictContent + dictSize - (offset-(srcPtr-(BYTE*)frame->srcStart))); } - else{ + else { *srcPtr = *(srcPtr-offset); } srcPtr++; @@ -1399,13 +1399,13 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } free(samples); free(sampleSizes); - if(dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)){ + if (dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)) { DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); free(fullDict); return 1; } /* write out dictionary */ - if(snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH){ + if (snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { DISPLAY("Error: dictionary path too long\n"); free(fullDict); return 1; From 73ebb646d8b294a3126f4a141966fb178f6c1bbe Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 14:20:35 -0700 Subject: [PATCH 22/73] small stylistic changes --- tests/decodecorpus.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index ffaad4a17..86ec0e985 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1333,8 +1333,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const { char outPath[MAX_PATH]; BYTE* fullDict; - U32 dictID; - unsigned fnum; + U32 const dictID = RAND(&seed); BYTE* decompressedPtr; BYTE* dictContent; const size_t headerSize = dictSize/4; @@ -1352,7 +1351,6 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } } /* Generate the dictionary randomly first */ - dictID = RAND(&seed); fullDict = malloc(dictSize); if (fullDict == NULL) { DISPLAY("Error: could not allocate space for full dictionary.\n"); @@ -1421,7 +1419,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } /* generate random compressed/decompressed files */ - for (fnum = 0; fnum < numFiles; fnum++) { + for (unsigned fnum = 0; fnum < numFiles; fnum++) { frame_t fr; size_t returnValue; From 2a52d72712c3705006c7e182f2ffacd10e3bae24 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 14:21:35 -0700 Subject: [PATCH 23/73] added comment to anotate decompressed error code --- tests/decodecorpus.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 86ec0e985..b26ebc214 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1452,6 +1452,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const fullDict, dictSize); { + /* print differences if any */ size_t checkDiff = (BYTE*)fr.src - (BYTE*)fr.srcStart; for (size_t i = 0; i < checkDiff; i++) { if (*((BYTE*)(fr.srcStart + i)) != *((BYTE*)(decompressedPtr + i))) { From 1ee4f6b36c1bb2290b470815226fce0a5023f0da Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 15:55:01 -0700 Subject: [PATCH 24/73] cleaning up code --- tests/decodecorpus.c | 210 +++++++++++++++++++++++-------------------- 1 file changed, 114 insertions(+), 96 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index b26ebc214..10c8a153d 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1263,6 +1263,69 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS return 0; } +/*_******************************************************* +* Dictionary Helper Functions +*********************************************************/ +/* returns 0 if successful, otherwise returns 1 upon error */ +static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ + const size_t headerSize = dictSize/4; + const size_t dictContentSize = dictSize - dictSize/4; + BYTE* const dictContent = fullDict + headerSize; + + /* use 3/4 of dictionary for content, save rest for header/entropy tables */ + if (dictContentSize < ZDICT_CONTENTSIZE_MIN || dictSize < ZDICT_DICTSIZE_MIN) { + DISPLAY("Error: dictionary size is too small\n"); + return 1; + } + + /* fill in dictionary content */ + RAND_buffer(&seed, (void*)dictContent, dictContentSize); + + /* allocate space for samples */ + { + size_t dictWriteSize = 0; + unsigned const numSamples = 4; + BYTE* const samples = malloc(5000*sizeof(BYTE)); + size_t* const sampleSizes = malloc(numSamples*sizeof(size_t)); + if (samples == NULL || sampleSizes == NULL) { + DISPLAY("Error: could not generate samples for the dictionary.\n"); + return 1; + } + + /* generate samples */ + unsigned i = 1; + size_t currSize = 1; + BYTE* curr = samples; + while (i <= 4) { + *(sampleSizes + i - 1) = currSize; + for (size_t j = 0; j < currSize; j++) { + *(curr++) = (BYTE)i; + } + i++; + currSize *= 16; + } + + /* set dictionary params */ + ZDICT_params_t zdictParams; + memset(&zdictParams, 0, sizeof(zdictParams)); + zdictParams.dictID = dictID; + zdictParams.notificationLevel = 1; + + /* finalize dictionary with random samples */ + dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, + dictContent, dictContentSize, + samples, sampleSizes, numSamples, + zdictParams); + free(samples); + free(sampleSizes); + if (dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)) { + DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); + return 1; + } + } + return 0; +} + /*-******************************************************* * File I/O *********************************************************/ @@ -1328,146 +1391,101 @@ static dictOptions initDictOptions(int useDict, U32 dictID, size_t dictSize, BYT dictOp.dictContent = dictContent; return dictOp; } + static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const path, const char* const origPath, const size_t dictSize) { char outPath[MAX_PATH]; BYTE* fullDict; U32 const dictID = RAND(&seed); - BYTE* decompressedPtr; - BYTE* dictContent; - const size_t headerSize = dictSize/4; - const size_t dictContentSize = dictSize - dictSize/4; - ZSTD_DCtx* dctx = ZSTD_createDCtx(); + int errorDetected = 0; + if (snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); return 1; } - { - /* use 3/4 of dictionary for content, save rest for header/entropy tables */ - if (dictContentSize < ZDICT_CONTENTSIZE_MIN || dictSize < ZDICT_DICTSIZE_MIN) { - DISPLAY("Error: dictionary size is too small\n"); - return 1; - } - } - /* Generate the dictionary randomly first */ + + /* allocate space for the dictionary */ fullDict = malloc(dictSize); if (fullDict == NULL) { DISPLAY("Error: could not allocate space for full dictionary.\n"); return 1; } - dictContent = fullDict + headerSize; - RAND_buffer(&seed, (void*)dictContent, dictContentSize); + + /* randomly generate the dictionary */ { - size_t dictWriteSize = 0; - - /* create samples */ - unsigned numSamples = 4; - BYTE* samples = malloc(5000*sizeof(BYTE)); - size_t* sampleSizes = malloc(numSamples*sizeof(size_t)); - if (samples == NULL || sampleSizes == NULL) { - DISPLAY("Error: could not generate samples for the dictionary.\n"); - free(fullDict); - return 1; + int ret = genRandomDict(dictID, seed, dictSize, fullDict); + if (ret != 0) { + errorDetected = ret; + goto dictCleanup; } - { - unsigned i = 1; - size_t currSize = 1; - BYTE* curr = samples; - while (i <= 4) { - *(sampleSizes + i - 1) = currSize; - for (size_t j = 0; j < currSize; j++) { - *(curr++) = (BYTE)i; - } - i++; - currSize *= 16; - } - } - { - /* set dictionary params */ - ZDICT_params_t zdictParams; - memset(&zdictParams, 0, sizeof(zdictParams)); - zdictParams.dictID = dictID; - zdictParams.notificationLevel = 1; - /* finalize dictionary with random samples */ - dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, - dictContent, dictContentSize, - samples, sampleSizes, numSamples, - zdictParams); - } - free(samples); - free(sampleSizes); - if (dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)) { - DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); - free(fullDict); - return 1; - } - /* write out dictionary */ - if (snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { - DISPLAY("Error: dictionary path too long\n"); - free(fullDict); - return 1; - } - outputBuffer(fullDict, dictSize, outPath); } - decompressedPtr = malloc(MAX_DECOMPRESSED_SIZE); - if (decompressedPtr == NULL) { - DISPLAY("Error: could not allocate memory for decompressed pointer\n"); - free(fullDict); - return 1; + /* write out dictionary */ + if (snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { + DISPLAY("Error: dictionary path too long\n"); + errorDetected = 1; + goto dictCleanup; } + outputBuffer(fullDict, dictSize, outPath); /* generate random compressed/decompressed files */ for (unsigned fnum = 0; fnum < numFiles; fnum++) { frame_t fr; - size_t returnValue; - - DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - - seed = generateFrame(seed, &fr, 1, dictContentSize, dictContent, dictID); - + { + size_t dictContentSize = dictSize-dictSize/4; + BYTE* const dictContent = fullDict+dictSize/4; + seed = generateFrame(seed, &fr, 1, dictContentSize, dictContent, dictID); + } if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); - free(fullDict); - free(decompressedPtr); - return 1; + errorDetected = 1; + goto dictCleanup; } outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath); if (origPath) { if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); - free(fullDict); - free(decompressedPtr); - return 1; + errorDetected = 1; + goto dictCleanup; } outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); } - /* if asked, supply the decompressed version */ - returnValue = ZSTD_decompress_usingDict(dctx, decompressedPtr, MAX_DECOMPRESSED_SIZE, - fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, - fullDict, dictSize); - + /* check the output to make sure that decompressed versions match official zstd */ { + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + BYTE* const decompressedPtr = malloc(MAX_DECOMPRESSED_SIZE); + if (decompressedPtr == NULL) { + DISPLAY("Error: could not allocate memory for decompressed pointer\n"); + errorDetected = 1; + goto dictCleanup; + } + size_t const returnValue = ZSTD_decompress_usingDict(dctx, decompressedPtr, MAX_DECOMPRESSED_SIZE, + fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, + fullDict, dictSize); + if (ZSTD_isError(returnValue)) { + DISPLAY("Error: %s\n", ZSTD_getErrorName(returnValue)); + } + /* print differences if any */ - size_t checkDiff = (BYTE*)fr.src - (BYTE*)fr.srcStart; - for (size_t i = 0; i < checkDiff; i++) { - if (*((BYTE*)(fr.srcStart + i)) != *((BYTE*)(decompressedPtr + i))) { - DISPLAY("i: %zu, fr: %u, decomp: %u\n", i, *((BYTE*)(fr.srcStart + i)), *((BYTE*)(decompressedPtr + i))); + { + size_t checkDiff = (BYTE*)fr.src - (BYTE*)fr.srcStart; + for (size_t i = 0; i < checkDiff; i++) { + if (*((BYTE*)(fr.srcStart + i)) != *((BYTE*)(decompressedPtr + i))) { + DISPLAY("i: %zu, fr: %u, decomp: %u\n", i, *((BYTE*)(fr.srcStart + i)), *((BYTE*)(decompressedPtr + i))); + } } } + free(decompressedPtr); } - if (ZSTD_isError(returnValue)) { - DISPLAY("Error: %s\n", ZSTD_getErrorName(returnValue)); - } - } - free(decompressedPtr); + +dictCleanup: free(fullDict); - return 0; + return errorDetected; } From b91b810c64f3ccc244b0a5f6e2a067f88ee2c6d8 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 16:05:05 -0700 Subject: [PATCH 25/73] made some small changes to dictInfo struct and initDictInfo --- tests/decodecorpus.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 10c8a153d..dba277128 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -237,8 +237,9 @@ typedef struct { int useDict; U32 dictID; size_t dictSize; + BYTE* fullDict; BYTE* dictContent; -} dictOptions; +} dictInfo; /*-******************************************************* * Generator Functions *********************************************************/ @@ -1326,6 +1327,17 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ return 0; } +static dictInfo initDictInfo(int useDict, size_t dictSize, BYTE* fullDict, U32 seed){ + /* allocate space statically */ + dictInfo dictOp; + memset((void*)(&dictOp), 0, sizeof(dictOp)); + dictOp.useDict = useDict; + dictOp.dictSize = dictSize; + dictOp.fullDict = fullDict; + dictOp.dictContent = fullDict + dictSize/4; + if (useDict) dictOp.dictID = RAND(&seed); + return dictOp; +} /*-******************************************************* * File I/O *********************************************************/ @@ -1381,17 +1393,6 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, return 0; } -static dictOptions initDictOptions(int useDict, U32 dictID, size_t dictSize, BYTE* dictContent){ - /* allocate space statically */ - dictOptions dictOp; - memset((void*)(&dictOp), 0, sizeof(dictOp)); - dictOp.useDict = useDict; - dictOp.dictID = dictID; - dictOp.dictSize = dictSize; - dictOp.dictContent = dictContent; - return dictOp; -} - static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const path, const char* const origPath, const size_t dictSize) { From 7ea025cc7df569b95770d23a273d2802ec3e8c7d Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 16:23:00 -0700 Subject: [PATCH 26/73] added in struct to keep functions more orderly --- tests/decodecorpus.c | 216 ++++++++++++++++++++++--------------------- 1 file changed, 112 insertions(+), 104 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index dba277128..76949a6e7 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -236,8 +236,7 @@ typedef struct { typedef struct { int useDict; U32 dictID; - size_t dictSize; - BYTE* fullDict; + size_t dictContentSize; BYTE* dictContent; } dictInfo; /*-******************************************************* @@ -249,7 +248,7 @@ struct { } opts; /* advanced options on generation */ /* Generate and write a random frame header */ -static void writeFrameHeader(U32* seed, frame_t* frame, int genDict, U32 dictID) +static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info) { BYTE* const op = frame->data; size_t pos = 0; @@ -315,7 +314,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame, int genDict, U32 dictID) pos += 4; { - int dictBits = genDict ? 3 : 0; + int dictBits = info.useDict ? 3 : 0; BYTE const frameHeaderDescriptor = (BYTE) ((fcsCode << 6) | (singleSegment << 5) | (1 << 2) | dictBits); op[pos++] = frameHeaderDescriptor; @@ -324,8 +323,8 @@ static void writeFrameHeader(U32* seed, frame_t* frame, int genDict, U32 dictID) if (!singleSegment) { op[pos++] = windowByte; } - if(genDict) { - MEM_writeLE32(op + pos, (U32) dictID); + if(info.useDict) { + MEM_writeLE32(op + pos, (U32) info.dictID); pos += 4; } if (contentSizeFlag) { @@ -618,7 +617,7 @@ static inline void initSeqStore(seqStore_t *seqStore) { /* Randomly generate sequence commands */ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, - size_t contentSize, size_t literalsSize, int genDict, size_t dictSize, BYTE* dictContent) + size_t contentSize, size_t literalsSize, dictInfo info) { /* The total length of all the matches */ size_t const remainingMatch = contentSize - literalsSize; @@ -675,10 +674,10 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, MIN(frame->header.windowSize, (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) + 1; - if (genDict && (RAND(seed) & 1)) { + if (info.useDict && (RAND(seed) & 1)) { /* need to occasionally generate offsets that go past the start */ /* we still need to be within the windowSize however */ - U32 const lenPastStart = RAND(seed) % dictSize; + U32 const lenPastStart = RAND(seed) % info.dictContentSize; offset = MIN(frame->header.windowSize, offset+lenPastStart); } offsetCode = offset + ZSTD_REP_MOVE; @@ -696,13 +695,13 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, repIndex = MIN(2, offsetCode + 1); } } - } while (((!genDict) && (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) || offset == 0); + } while (((!info.useDict) && (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) || offset == 0); { size_t j; for (j = 0; j < matchLen; j++) { if ((void*)(srcPtr - offset) < (void*)frame->srcStart) { /* copy from dictionary instead of literals */ - *srcPtr = *(dictContent + dictSize - (offset-(srcPtr-(BYTE*)frame->srcStart))); + *srcPtr = *(info.dictContent + info.dictContentSize - (offset-(srcPtr-(BYTE*)frame->srcStart))); } else { *srcPtr = *(srcPtr-offset); @@ -956,7 +955,7 @@ static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, } static size_t writeSequencesBlock(U32* seed, frame_t* frame, size_t contentSize, - size_t literalsSize, int genDict, size_t dictSize, BYTE* dictContent) + size_t literalsSize, dictInfo info) { seqStore_t seqStore; size_t numSequences; @@ -965,14 +964,14 @@ static size_t writeSequencesBlock(U32* seed, frame_t* frame, size_t contentSize, initSeqStore(&seqStore); /* randomly generate sequences */ - numSequences = generateSequences(seed, frame, &seqStore, contentSize, literalsSize, genDict, dictSize, dictContent); + numSequences = generateSequences(seed, frame, &seqStore, contentSize, literalsSize, info); /* write them out to the frame data */ CHECKERR(writeSequences(seed, frame, &seqStore, numSequences)); return numSequences; } -static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize, int genDict, size_t dictSize, BYTE* dictContent) +static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize, dictInfo info) { BYTE* const blockStart = (BYTE*)frame->data; size_t literalsSize; @@ -984,7 +983,7 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize DISPLAYLEVEL(4, " literals size: %u\n", (U32)literalsSize); - nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize, genDict, dictSize, dictContent); + nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize, info); DISPLAYLEVEL(4, " number of sequences: %u\n", (U32)nbSeq); @@ -992,7 +991,7 @@ static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize } static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, - int lastBlock, int genDict, size_t dictSize, BYTE* dictContent) + int lastBlock, dictInfo info) { int const blockTypeDesc = RAND(seed) % 8; size_t blockSize; @@ -1032,7 +1031,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, frame->oldStats = frame->stats; frame->data = op; - compressedSize = writeCompressedBlock(seed, frame, contentSize, genDict, dictSize, dictContent); + compressedSize = writeCompressedBlock(seed, frame, contentSize, info); if (compressedSize > contentSize) { blockType = 0; memcpy(op, frame->src, contentSize); @@ -1058,7 +1057,7 @@ static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, frame->data = op; } -static void writeBlocks(U32* seed, frame_t* frame, int genDict, size_t dictSize, BYTE* dictContent) +static void writeBlocks(U32* seed, frame_t* frame, dictInfo info) { size_t contentLeft = frame->header.contentSize; size_t const maxBlockSize = MIN(MAX_BLOCK_SIZE, frame->header.windowSize); @@ -1081,7 +1080,7 @@ static void writeBlocks(U32* seed, frame_t* frame, int genDict, size_t dictSize, } } - writeBlock(seed, frame, blockContentSize, lastBlock, genDict, dictSize, dictContent); + writeBlock(seed, frame, blockContentSize, lastBlock, info); contentLeft -= blockContentSize; if (lastBlock) break; @@ -1146,19 +1145,92 @@ static void initFrame(frame_t* fr) } /* Return the final seed */ -static U32 generateFrame(U32 seed, frame_t* fr, int genDict, size_t dictSize, BYTE* dictContent, U32 dictID) +static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info) { /* generate a complete frame */ DISPLAYLEVEL(1, "frame seed: %u\n", seed); initFrame(fr); - writeFrameHeader(&seed, fr, genDict, dictID); - writeBlocks(&seed, fr, genDict, dictSize, dictContent); + writeFrameHeader(&seed, fr, info); + writeBlocks(&seed, fr, info); writeChecksum(fr); return seed; } +/*_******************************************************* +* Dictionary Helper Functions +*********************************************************/ +/* returns 0 if successful, otherwise returns 1 upon error */ +static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ + const size_t headerSize = dictSize/4; + const size_t dictContentSize = dictSize - dictSize/4; + BYTE* const dictContent = fullDict + headerSize; + + /* use 3/4 of dictionary for content, save rest for header/entropy tables */ + if (dictContentSize < ZDICT_CONTENTSIZE_MIN || dictSize < ZDICT_DICTSIZE_MIN) { + DISPLAY("Error: dictionary size is too small\n"); + return 1; + } + + /* fill in dictionary content */ + RAND_buffer(&seed, (void*)dictContent, dictContentSize); + + /* allocate space for samples */ + { + size_t dictWriteSize = 0; + unsigned const numSamples = 4; + BYTE* const samples = malloc(5000*sizeof(BYTE)); + size_t* const sampleSizes = malloc(numSamples*sizeof(size_t)); + if (samples == NULL || sampleSizes == NULL) { + DISPLAY("Error: could not generate samples for the dictionary.\n"); + return 1; + } + + /* generate samples */ + unsigned i = 1; + size_t currSize = 1; + BYTE* curr = samples; + while (i <= 4) { + *(sampleSizes + i - 1) = currSize; + for (size_t j = 0; j < currSize; j++) { + *(curr++) = (BYTE)i; + } + i++; + currSize *= 16; + } + + /* set dictionary params */ + ZDICT_params_t zdictParams; + memset(&zdictParams, 0, sizeof(zdictParams)); + zdictParams.dictID = dictID; + zdictParams.notificationLevel = 1; + + /* finalize dictionary with random samples */ + dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, + dictContent, dictContentSize, + samples, sampleSizes, numSamples, + zdictParams); + free(samples); + free(sampleSizes); + if (dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)) { + DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); + return 1; + } + } + return 0; +} + +static dictInfo initDictInfo(int useDict, size_t dictContentSize, BYTE* dictContent, U32 dictID){ + /* allocate space statically */ + dictInfo dictOp; + memset((void*)(&dictOp), 0, sizeof(dictOp)); + dictOp.useDict = useDict; + dictOp.dictContentSize = dictContentSize; + dictOp.dictContent = dictContent; + dictOp.dictID = dictID; + return dictOp; +} /*-******************************************************* * Test Mode @@ -1240,7 +1312,10 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS else DISPLAYUPDATE("\r%u ", fnum); - seed = generateFrame(seed, &fr, 0, 0, NULL, 0); + { + dictInfo const info = initDictInfo(0, 0, NULL, 0); + generateFrame(seed, &fr, info); + } { size_t const r = testDecodeSimple(&fr); if (ZSTD_isError(r)) { @@ -1264,80 +1339,6 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS return 0; } -/*_******************************************************* -* Dictionary Helper Functions -*********************************************************/ -/* returns 0 if successful, otherwise returns 1 upon error */ -static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ - const size_t headerSize = dictSize/4; - const size_t dictContentSize = dictSize - dictSize/4; - BYTE* const dictContent = fullDict + headerSize; - - /* use 3/4 of dictionary for content, save rest for header/entropy tables */ - if (dictContentSize < ZDICT_CONTENTSIZE_MIN || dictSize < ZDICT_DICTSIZE_MIN) { - DISPLAY("Error: dictionary size is too small\n"); - return 1; - } - - /* fill in dictionary content */ - RAND_buffer(&seed, (void*)dictContent, dictContentSize); - - /* allocate space for samples */ - { - size_t dictWriteSize = 0; - unsigned const numSamples = 4; - BYTE* const samples = malloc(5000*sizeof(BYTE)); - size_t* const sampleSizes = malloc(numSamples*sizeof(size_t)); - if (samples == NULL || sampleSizes == NULL) { - DISPLAY("Error: could not generate samples for the dictionary.\n"); - return 1; - } - - /* generate samples */ - unsigned i = 1; - size_t currSize = 1; - BYTE* curr = samples; - while (i <= 4) { - *(sampleSizes + i - 1) = currSize; - for (size_t j = 0; j < currSize; j++) { - *(curr++) = (BYTE)i; - } - i++; - currSize *= 16; - } - - /* set dictionary params */ - ZDICT_params_t zdictParams; - memset(&zdictParams, 0, sizeof(zdictParams)); - zdictParams.dictID = dictID; - zdictParams.notificationLevel = 1; - - /* finalize dictionary with random samples */ - dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, - dictContent, dictContentSize, - samples, sampleSizes, numSamples, - zdictParams); - free(samples); - free(sampleSizes); - if (dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)) { - DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); - return 1; - } - } - return 0; -} - -static dictInfo initDictInfo(int useDict, size_t dictSize, BYTE* fullDict, U32 seed){ - /* allocate space statically */ - dictInfo dictOp; - memset((void*)(&dictOp), 0, sizeof(dictOp)); - dictOp.useDict = useDict; - dictOp.dictSize = dictSize; - dictOp.fullDict = fullDict; - dictOp.dictContent = fullDict + dictSize/4; - if (useDict) dictOp.dictID = RAND(&seed); - return dictOp; -} /*-******************************************************* * File I/O *********************************************************/ @@ -1349,7 +1350,10 @@ static int generateFile(U32 seed, const char* const path, DISPLAY("seed: %u\n", seed); - generateFrame(seed, &fr, 0, 0, NULL, 0); + { + dictInfo const info = initDictInfo(0, 0, NULL, 0); + generateFrame(seed, &fr, info); + } outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path); if (origPath) { @@ -1371,7 +1375,10 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - seed = generateFrame(seed, &fr, 0, 0, NULL, 0); + { + dictInfo const info = initDictInfo(0, 0, NULL, 0); + generateFrame(seed, &fr, info); + } if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); @@ -1437,7 +1444,8 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const { size_t dictContentSize = dictSize-dictSize/4; BYTE* const dictContent = fullDict+dictSize/4; - seed = generateFrame(seed, &fr, 1, dictContentSize, dictContent, dictID); + dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); + seed = generateFrame(seed, &fr, info); } if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); @@ -1545,7 +1553,7 @@ int main(int argc, char** argv) int testMode = 0; const char* path = NULL; const char* origPath = NULL; - int genDict = 0; + int useDict = 0; unsigned dictSize = (10 << 10); /* 10 kB default */ int argNb; @@ -1610,7 +1618,7 @@ int main(int argc, char** argv) } else if (strcmp(argument, "train-dict") == 0) { argument += 11; dictSize = readInt(&argument); - genDict = 1; + useDict = 1; } else { advancedUsage(argv[0]); return 1; @@ -1642,9 +1650,9 @@ int main(int argc, char** argv) return 1; } - if (numFiles == 0 && genDict == 0) { + if (numFiles == 0 && useDict == 0) { return generateFile(seed, path, origPath); - } else if (genDict == 0){ + } else if (useDict == 0){ return generateCorpus(seed, numFiles, path, origPath); } else { /* should generate files with a dictionary */ From f4f928595bbe0a976fee0ca7e2f0ddf3d10e7374 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 16:27:05 -0700 Subject: [PATCH 27/73] changed scope to get rid of mixed code/variables warning --- tests/decodecorpus.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 76949a6e7..c4264c9d2 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1188,16 +1188,18 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ } /* generate samples */ - unsigned i = 1; - size_t currSize = 1; - BYTE* curr = samples; - while (i <= 4) { - *(sampleSizes + i - 1) = currSize; - for (size_t j = 0; j < currSize; j++) { - *(curr++) = (BYTE)i; + { + unsigned i = 1; + size_t currSize = 1; + BYTE* curr = samples; + while (i <= 4) { + *(sampleSizes + i - 1) = currSize; + for (size_t j = 0; j < currSize; j++) { + *(curr++) = (BYTE)i; + } + i++; + currSize *= 16; } - i++; - currSize *= 16; } /* set dictionary params */ From 365e265a8eb9ea6efd912a907f03e63f55a38f1a Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 16:28:40 -0700 Subject: [PATCH 28/73] changed scope so that warning disappears --- tests/decodecorpus.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index c4264c9d2..bd6d762e8 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1203,16 +1203,18 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ } /* set dictionary params */ - ZDICT_params_t zdictParams; - memset(&zdictParams, 0, sizeof(zdictParams)); - zdictParams.dictID = dictID; - zdictParams.notificationLevel = 1; + { + ZDICT_params_t zdictParams; + memset(&zdictParams, 0, sizeof(zdictParams)); + zdictParams.dictID = dictID; + zdictParams.notificationLevel = 1; - /* finalize dictionary with random samples */ - dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, - dictContent, dictContentSize, - samples, sampleSizes, numSamples, - zdictParams); + /* finalize dictionary with random samples */ + dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, + dictContent, dictContentSize, + samples, sampleSizes, numSamples, + zdictParams); + } free(samples); free(sampleSizes); if (dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)) { From 791352ad0e4f2e4aabefa224becda064191588af Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 16:32:38 -0700 Subject: [PATCH 29/73] scope change --- tests/decodecorpus.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index bd6d762e8..4c6e4b27f 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1476,11 +1476,13 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const errorDetected = 1; goto dictCleanup; } - size_t const returnValue = ZSTD_decompress_usingDict(dctx, decompressedPtr, MAX_DECOMPRESSED_SIZE, - fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, - fullDict, dictSize); - if (ZSTD_isError(returnValue)) { - DISPLAY("Error: %s\n", ZSTD_getErrorName(returnValue)); + { + size_t const returnValue = ZSTD_decompress_usingDict(dctx, decompressedPtr, MAX_DECOMPRESSED_SIZE, + fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, + fullDict, dictSize); + if (ZSTD_isError(returnValue)) { + DISPLAY("Error: %s\n", ZSTD_getErrorName(returnValue)); + } } /* print differences if any */ From 147ef05f12b1b1f81039e548533e8723ec37eabe Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 16:38:51 -0700 Subject: [PATCH 30/73] for loop declaration not allowed --- tests/decodecorpus.c | 85 +++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 4c6e4b27f..42360d2c4 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1442,59 +1442,62 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const outputBuffer(fullDict, dictSize, outPath); /* generate random compressed/decompressed files */ - for (unsigned fnum = 0; fnum < numFiles; fnum++) { - frame_t fr; - DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); - { - size_t dictContentSize = dictSize-dictSize/4; - BYTE* const dictContent = fullDict+dictSize/4; - dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); - seed = generateFrame(seed, &fr, info); - } - if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { - DISPLAY("Error: path too long\n"); - errorDetected = 1; - goto dictCleanup; - } - outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath); - - if (origPath) { - if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) { + { + unsigned fnum; + for (fnum = 0; fnum < numFiles; fnum++) { + frame_t fr; + DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); + { + size_t dictContentSize = dictSize-dictSize/4; + BYTE* const dictContent = fullDict+dictSize/4; + dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); + seed = generateFrame(seed, &fr, info); + } + if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); errorDetected = 1; goto dictCleanup; } - outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); - } + outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath); - /* check the output to make sure that decompressed versions match official zstd */ - { - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - BYTE* const decompressedPtr = malloc(MAX_DECOMPRESSED_SIZE); - if (decompressedPtr == NULL) { - DISPLAY("Error: could not allocate memory for decompressed pointer\n"); - errorDetected = 1; - goto dictCleanup; - } - { - size_t const returnValue = ZSTD_decompress_usingDict(dctx, decompressedPtr, MAX_DECOMPRESSED_SIZE, - fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, - fullDict, dictSize); - if (ZSTD_isError(returnValue)) { - DISPLAY("Error: %s\n", ZSTD_getErrorName(returnValue)); + if (origPath) { + if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) { + DISPLAY("Error: path too long\n"); + errorDetected = 1; + goto dictCleanup; } + outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); } - /* print differences if any */ + /* check the output to make sure that decompressed versions match official zstd */ { - size_t checkDiff = (BYTE*)fr.src - (BYTE*)fr.srcStart; - for (size_t i = 0; i < checkDiff; i++) { - if (*((BYTE*)(fr.srcStart + i)) != *((BYTE*)(decompressedPtr + i))) { - DISPLAY("i: %zu, fr: %u, decomp: %u\n", i, *((BYTE*)(fr.srcStart + i)), *((BYTE*)(decompressedPtr + i))); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + BYTE* const decompressedPtr = malloc(MAX_DECOMPRESSED_SIZE); + if (decompressedPtr == NULL) { + DISPLAY("Error: could not allocate memory for decompressed pointer\n"); + errorDetected = 1; + goto dictCleanup; + } + { + size_t const returnValue = ZSTD_decompress_usingDict(dctx, decompressedPtr, MAX_DECOMPRESSED_SIZE, + fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, + fullDict, dictSize); + if (ZSTD_isError(returnValue)) { + DISPLAY("Error: %s\n", ZSTD_getErrorName(returnValue)); } } + + /* print differences if any */ + { + size_t checkDiff = (BYTE*)fr.src - (BYTE*)fr.srcStart; + for (size_t i = 0; i < checkDiff; i++) { + if (*((BYTE*)(fr.srcStart + i)) != *((BYTE*)(decompressedPtr + i))) { + DISPLAY("i: %zu, fr: %u, decomp: %u\n", i, *((BYTE*)(fr.srcStart + i)), *((BYTE*)(decompressedPtr + i))); + } + } + } + free(decompressedPtr); } - free(decompressedPtr); } } From 7a263909897bef6aa6ab5c2c0617e7f3574596fc Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 16:39:57 -0700 Subject: [PATCH 31/73] changing for loop declaration --- tests/decodecorpus.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 42360d2c4..08c2cc0a5 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1490,9 +1490,12 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const /* print differences if any */ { size_t checkDiff = (BYTE*)fr.src - (BYTE*)fr.srcStart; - for (size_t i = 0; i < checkDiff; i++) { - if (*((BYTE*)(fr.srcStart + i)) != *((BYTE*)(decompressedPtr + i))) { - DISPLAY("i: %zu, fr: %u, decomp: %u\n", i, *((BYTE*)(fr.srcStart + i)), *((BYTE*)(decompressedPtr + i))); + { + size_t i; + for (i = 0; i < checkDiff; i++) { + if (*((BYTE*)(fr.srcStart + i)) != *((BYTE*)(decompressedPtr + i))) { + DISPLAY("i: %zu, fr: %u, decomp: %u\n", i, *((BYTE*)(fr.srcStart + i)), *((BYTE*)(decompressedPtr + i))); + } } } } From 23098b53e81d3ad9a44c0cc54b14aebafefbf5fa Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 17:13:00 -0700 Subject: [PATCH 32/73] wrote a test for dictionary corpus --- tests/decodecorpus.c | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 08c2cc0a5..d2bf565c9 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1297,6 +1297,54 @@ cleanup: return ret; } +static size_t testDecodeWithDict(U32 seed, size_t dictSize) +{ + U32 const dictID = RAND(&seed); + int errorDetected = 0; + BYTE* const fullDict = malloc(dictSize); + if (fullDict == NULL) { + return ERROR(GENERIC); + } + + { + int ret = genRandomDict(dictID, seed, dictSize, fullDict); + if (ret != 0) { + errorDetected = ERROR(GENERIC); + goto dictTestCleanup; + } + } + + frame_t* fr; + { + size_t dictContentSize = dictSize-dictSize/4; + BYTE* const dictContent = fullDict+dictSize/4; + dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); + seed = generateFrame(seed, &fr, info); + } + + { + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + { + size_t const returnValue = ZSTD_decompress_usingDict(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, + fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, + fullDict, dictSize); + if (ZSTD_isError(returnValue)) { + errorDetected = ZSTD_getErrorName(returnValue); + goto dictTestCleanup + } + } + + if (memcmp(DECOMPRESSED_BUFFER, fr->srcStart, (BYTE*)fr->src - (BYTE*)fr->srcStart) != 0) { + errorDetected = ERROR(corruption_detected); + goto dictTestCleanup; + } + } + +dictTestCleanup: + free(fullDict); + return errorDetected; +} + static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS) { unsigned fnum; From bdec7b8ce7b131779d768f60776f6eb0dee8f54d Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 17:18:43 -0700 Subject: [PATCH 33/73] fixed for loop declaration, fixed test case --- tests/decodecorpus.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index d2bf565c9..14d3f2358 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1194,8 +1194,11 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ BYTE* curr = samples; while (i <= 4) { *(sampleSizes + i - 1) = currSize; - for (size_t j = 0; j < currSize; j++) { - *(curr++) = (BYTE)i; + { + size_t j; + for (j = 0; j < currSize; j++) { + *(curr++) = (BYTE)i; + } } i++; currSize *= 16; @@ -1300,7 +1303,7 @@ cleanup: static size_t testDecodeWithDict(U32 seed, size_t dictSize) { U32 const dictID = RAND(&seed); - int errorDetected = 0; + size_t errorDetected = 0; BYTE* const fullDict = malloc(dictSize); if (fullDict == NULL) { return ERROR(GENERIC); @@ -1314,7 +1317,7 @@ static size_t testDecodeWithDict(U32 seed, size_t dictSize) } } - frame_t* fr; + frame_t fr; { size_t dictContentSize = dictSize-dictSize/4; BYTE* const dictContent = fullDict+dictSize/4; @@ -1329,12 +1332,12 @@ static size_t testDecodeWithDict(U32 seed, size_t dictSize) fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, fullDict, dictSize); if (ZSTD_isError(returnValue)) { - errorDetected = ZSTD_getErrorName(returnValue); - goto dictTestCleanup + errorDetected = returnValue; + goto dictTestCleanup; } } - if (memcmp(DECOMPRESSED_BUFFER, fr->srcStart, (BYTE*)fr->src - (BYTE*)fr->srcStart) != 0) { + if (memcmp(DECOMPRESSED_BUFFER, fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart) != 0) { errorDetected = ERROR(corruption_detected); goto dictTestCleanup; } From fdbb07283a0ca801ba37eb0328e3dcb6ce045f93 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 17:29:15 -0700 Subject: [PATCH 34/73] added test to runTestMode --- tests/decodecorpus.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 14d3f2358..8e2cefb4b 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1386,6 +1386,14 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS return 1; } } + { + size_t const dictSize = RAND(&seed); + size_t const r = testDecodeWithDict(seed, dictSize); + if (ZSTD_isError(r)) { + DISPLAY("Error in dictionary mode on test seed %u: %s\n", seed+fnum, ZSTD_getErrorName(r)); + return 1; + } + } } DISPLAY("\r%u tests completed: ", fnum); From ea2af1ffc429981214124572524790c82126493a Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 17:34:50 -0700 Subject: [PATCH 35/73] changed dictionary to stay reasonable size during test --- tests/decodecorpus.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 8e2cefb4b..4b42c6d7c 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1387,7 +1387,8 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS } } { - size_t const dictSize = RAND(&seed); + /* don't create a dictionary that is too big */ + size_t const dictSize = RAND(&seed) % (10 << 20); size_t const r = testDecodeWithDict(seed, dictSize); if (ZSTD_isError(r)) { DISPLAY("Error in dictionary mode on test seed %u: %s\n", seed+fnum, ZSTD_getErrorName(r)); From 7d2b5613b4f067e36fd435e06ea4a476b736dc51 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 17:37:14 -0700 Subject: [PATCH 36/73] adhere to min dict size --- tests/decodecorpus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 4b42c6d7c..f5cfd6288 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1388,7 +1388,7 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS } { /* don't create a dictionary that is too big */ - size_t const dictSize = RAND(&seed) % (10 << 20); + size_t const dictSize = RAND(&seed) % (10 << 20) + ZDICT_DICTSIZE_MIN; size_t const r = testDecodeWithDict(seed, dictSize); if (ZSTD_isError(r)) { DISPLAY("Error in dictionary mode on test seed %u: %s\n", seed+fnum, ZSTD_getErrorName(r)); From 3a6856edcaaa8ce88b280fed06a91bc6789cab4b Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 17:43:16 -0700 Subject: [PATCH 37/73] changed scope to avoid errors --- tests/decodecorpus.c | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index f5cfd6288..7d91a1c70 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1316,30 +1316,31 @@ static size_t testDecodeWithDict(U32 seed, size_t dictSize) goto dictTestCleanup; } } - - frame_t fr; { - size_t dictContentSize = dictSize-dictSize/4; - BYTE* const dictContent = fullDict+dictSize/4; - dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); - seed = generateFrame(seed, &fr, info); - } - - { - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + frame_t fr; { - size_t const returnValue = ZSTD_decompress_usingDict(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, - fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, - fullDict, dictSize); - if (ZSTD_isError(returnValue)) { - errorDetected = returnValue; - goto dictTestCleanup; - } + size_t dictContentSize = dictSize-dictSize/4; + BYTE* const dictContent = fullDict+dictSize/4; + dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); + seed = generateFrame(seed, &fr, info); } - if (memcmp(DECOMPRESSED_BUFFER, fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart) != 0) { - errorDetected = ERROR(corruption_detected); - goto dictTestCleanup; + { + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + { + size_t const returnValue = ZSTD_decompress_usingDict(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, + fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, + fullDict, dictSize); + if (ZSTD_isError(returnValue)) { + errorDetected = returnValue; + goto dictTestCleanup; + } + } + + if (memcmp(DECOMPRESSED_BUFFER, fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart) != 0) { + errorDetected = ERROR(corruption_detected); + goto dictTestCleanup; + } } } From 4a4f9ba2623e75e433a50fa2a6302848436c36db Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 19 Jun 2017 17:46:21 -0700 Subject: [PATCH 38/73] const value that doesn't change --- tests/decodecorpus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 7d91a1c70..74252d562 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1319,7 +1319,7 @@ static size_t testDecodeWithDict(U32 seed, size_t dictSize) { frame_t fr; { - size_t dictContentSize = dictSize-dictSize/4; + size_t const dictContentSize = dictSize-dictSize/4; BYTE* const dictContent = fullDict+dictSize/4; dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); seed = generateFrame(seed, &fr, info); From 6f17de21323a1974711b3ee7cb18f4a1b3b0e8c2 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 09:50:28 -0700 Subject: [PATCH 39/73] changed computation for readability --- tests/decodecorpus.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 74252d562..fd983136f 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -323,7 +323,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info) if (!singleSegment) { op[pos++] = windowByte; } - if(info.useDict) { + if (info.useDict) { MEM_writeLE32(op + pos, (U32) info.dictID); pos += 4; } @@ -697,11 +697,14 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, } } while (((!info.useDict) && (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) || offset == 0); - { size_t j; + { + size_t j; + BYTE* const dictEnd = info.dictContent + info.dictContentSize; for (j = 0; j < matchLen; j++) { if ((void*)(srcPtr - offset) < (void*)frame->srcStart) { /* copy from dictionary instead of literals */ - *srcPtr = *(info.dictContent + info.dictContentSize - (offset-(srcPtr-(BYTE*)frame->srcStart))); + size_t dictOffset = offset - (srcPtr - (BYTE*)frame->srcStart); + *srcPtr = *(dictEnd - dictOffset); } else { *srcPtr = *(srcPtr-offset); From b8237c6c66fe5da4a345c41623549d3ecd8dc323 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 10:12:13 -0700 Subject: [PATCH 40/73] refactoring for readability --- tests/decodecorpus.c | 90 ++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index fd983136f..5577b5948 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1166,61 +1166,60 @@ static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info) *********************************************************/ /* returns 0 if successful, otherwise returns 1 upon error */ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ - const size_t headerSize = dictSize/4; - const size_t dictContentSize = dictSize - dictSize/4; - BYTE* const dictContent = fullDict + headerSize; - - /* use 3/4 of dictionary for content, save rest for header/entropy tables */ - if (dictContentSize < ZDICT_CONTENTSIZE_MIN || dictSize < ZDICT_DICTSIZE_MIN) { - DISPLAY("Error: dictionary size is too small\n"); + /* allocate space for samples */ + unsigned const numSamples = 4; + BYTE* const samples = malloc(5000*sizeof(BYTE)); + size_t* const sampleSizes = malloc(numSamples*sizeof(size_t)); + if (samples == NULL || sampleSizes == NULL) { + DISPLAY("Error: could not allocate space for samples.\n"); return 1; } - /* fill in dictionary content */ - RAND_buffer(&seed, (void*)dictContent, dictContentSize); - - /* allocate space for samples */ + /* generate samples */ { + unsigned i = 1; + size_t currSize = 1; + BYTE* curr = samples; + while (i <= 4) { + *(sampleSizes + i - 1) = currSize; + { + size_t j; + for (j = 0; j < currSize; j++) { + *(curr++) = (BYTE)i; + } + } + i++; + currSize *= 16; + } + } + + + { + /* create variables */ size_t dictWriteSize = 0; - unsigned const numSamples = 4; - BYTE* const samples = malloc(5000*sizeof(BYTE)); - size_t* const sampleSizes = malloc(numSamples*sizeof(size_t)); - if (samples == NULL || sampleSizes == NULL) { - DISPLAY("Error: could not generate samples for the dictionary.\n"); + ZDICT_params_t zdictParams; + size_t const headerSize = dictSize/4; + size_t const dictContentSize = dictSize - dictSize/4; + BYTE* const dictContent = fullDict + headerSize; + if (dictContentSize < ZDICT_CONTENTSIZE_MIN || dictSize < ZDICT_DICTSIZE_MIN) { + DISPLAY("Error: dictionary size is too small\n"); return 1; } - /* generate samples */ - { - unsigned i = 1; - size_t currSize = 1; - BYTE* curr = samples; - while (i <= 4) { - *(sampleSizes + i - 1) = currSize; - { - size_t j; - for (j = 0; j < currSize; j++) { - *(curr++) = (BYTE)i; - } - } - i++; - currSize *= 16; - } - } + /* init dictionary params */ + memset(&zdictParams, 0, sizeof(zdictParams)); + zdictParams.dictID = dictID; + zdictParams.notificationLevel = 1; - /* set dictionary params */ - { - ZDICT_params_t zdictParams; - memset(&zdictParams, 0, sizeof(zdictParams)); - zdictParams.dictID = dictID; - zdictParams.notificationLevel = 1; + /* fill in dictionary content */ + RAND_buffer(&seed, (void*)dictContent, dictContentSize); + + /* finalize dictionary with random samples */ + dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, + dictContent, dictContentSize, + samples, sampleSizes, numSamples, + zdictParams); - /* finalize dictionary with random samples */ - dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, - dictContent, dictContentSize, - samples, sampleSizes, numSamples, - zdictParams); - } free(samples); free(sampleSizes); if (dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)) { @@ -1228,6 +1227,7 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ return 1; } } + return 0; } From 551fb7ca056e8b8325ff3768fcb46e885a39c1ea Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 10:14:03 -0700 Subject: [PATCH 41/73] changed error condition --- tests/decodecorpus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 5577b5948..c0770fefc 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1222,7 +1222,7 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ free(samples); free(sampleSizes); - if (dictWriteSize != dictSize && ZDICT_isError(dictWriteSize)) { + if (ZDICT_isError(dictWriteSize)) { DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); return 1; } From ff5589c091baf46f987f56ce726bd761cabbabd7 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 10:17:38 -0700 Subject: [PATCH 42/73] added comments for clarity --- tests/decodecorpus.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index c0770fefc..074ac5931 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1305,6 +1305,7 @@ cleanup: static size_t testDecodeWithDict(U32 seed, size_t dictSize) { + /* create variables */ U32 const dictID = RAND(&seed); size_t errorDetected = 0; BYTE* const fullDict = malloc(dictSize); @@ -1312,6 +1313,7 @@ static size_t testDecodeWithDict(U32 seed, size_t dictSize) return ERROR(GENERIC); } + /* generate random dictionary */ { int ret = genRandomDict(dictID, seed, dictSize, fullDict); if (ret != 0) { @@ -1319,8 +1321,12 @@ static size_t testDecodeWithDict(U32 seed, size_t dictSize) goto dictTestCleanup; } } + + { frame_t fr; + + /* generate frame */ { size_t const dictContentSize = dictSize-dictSize/4; BYTE* const dictContent = fullDict+dictSize/4; @@ -1328,6 +1334,7 @@ static size_t testDecodeWithDict(U32 seed, size_t dictSize) seed = generateFrame(seed, &fr, info); } + /* manually decompress and check difference */ { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); { From 15102069a1b6af495b15c9f35cd7261d2f78fd71 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 10:25:16 -0700 Subject: [PATCH 43/73] removed testing code from generation code --- tests/decodecorpus.c | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 074ac5931..8c048f478 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1519,7 +1519,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const frame_t fr; DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); { - size_t dictContentSize = dictSize-dictSize/4; + size_t const dictContentSize = dictSize-dictSize/4; BYTE* const dictContent = fullDict+dictSize/4; dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); seed = generateFrame(seed, &fr, info); @@ -1539,39 +1539,6 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); } - - /* check the output to make sure that decompressed versions match official zstd */ - { - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - BYTE* const decompressedPtr = malloc(MAX_DECOMPRESSED_SIZE); - if (decompressedPtr == NULL) { - DISPLAY("Error: could not allocate memory for decompressed pointer\n"); - errorDetected = 1; - goto dictCleanup; - } - { - size_t const returnValue = ZSTD_decompress_usingDict(dctx, decompressedPtr, MAX_DECOMPRESSED_SIZE, - fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, - fullDict, dictSize); - if (ZSTD_isError(returnValue)) { - DISPLAY("Error: %s\n", ZSTD_getErrorName(returnValue)); - } - } - - /* print differences if any */ - { - size_t checkDiff = (BYTE*)fr.src - (BYTE*)fr.srcStart; - { - size_t i; - for (i = 0; i < checkDiff; i++) { - if (*((BYTE*)(fr.srcStart + i)) != *((BYTE*)(decompressedPtr + i))) { - DISPLAY("i: %zu, fr: %u, decomp: %u\n", i, *((BYTE*)(fr.srcStart + i)), *((BYTE*)(decompressedPtr + i))); - } - } - } - } - free(decompressedPtr); - } } } From 36e14b196373097b6ad813db6b500f83f43384a9 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 10:34:34 -0700 Subject: [PATCH 44/73] added description to advanced usage, changed to use-dict --- tests/decodecorpus.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 8c048f478..0e6630cd1 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1592,6 +1592,7 @@ static void advancedUsage(const char* programName) DISPLAY( "\n"); DISPLAY( "Advanced arguments :\n"); DISPLAY( " --content-size : always include the content size in the frame header\n"); + DISPLAY( " --use-dict # : include a dictionary used to decompress the corpus\n"); } int main(int argc, char** argv) @@ -1665,8 +1666,8 @@ int main(int argc, char** argv) argument++; if (strcmp(argument, "content-size") == 0) { opts.contentSize = 1; - } else if (strcmp(argument, "train-dict") == 0) { - argument += 11; + } else if (strcmp(argument, "use-dict") == 0) { + argument += 9; dictSize = readInt(&argument); useDict = 1; } else { From 0034129270e6d48d2b02336bb57043cb77366787 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 11:07:38 -0700 Subject: [PATCH 45/73] made changes to accommodate single file use case --- tests/decodecorpus.c | 45 ++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 0e6630cd1..09710c379 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1505,17 +1505,22 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const } /* write out dictionary */ - if (snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { - DISPLAY("Error: dictionary path too long\n"); - errorDetected = 1; - goto dictCleanup; + if (numFiles != 0) { + if (snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { + DISPLAY("Error: dictionary path too long\n"); + errorDetected = 1; + goto dictCleanup; + } + outputBuffer(fullDict, dictSize, outPath); + } + else { + outputBuffer(fullDict, dictSize, "dictionary"); } - outputBuffer(fullDict, dictSize, outPath); /* generate random compressed/decompressed files */ { unsigned fnum; - for (fnum = 0; fnum < numFiles; fnum++) { + for (fnum = 0; fnum < MAX(numFiles, 1); fnum++) { frame_t fr; DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); { @@ -1524,20 +1529,29 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); seed = generateFrame(seed, &fr, info); } - if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { - DISPLAY("Error: path too long\n"); - errorDetected = 1; - goto dictCleanup; - } - outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath); - if (origPath) { - if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) { + if (numFiles != 0) { + if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { DISPLAY("Error: path too long\n"); errorDetected = 1; goto dictCleanup; } - outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); + outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath); + + if (origPath) { + if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) { + DISPLAY("Error: path too long\n"); + errorDetected = 1; + goto dictCleanup; + } + outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); + } + } + else { + outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path); + if (origPath) { + outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, origPath); + } } } } @@ -1707,7 +1721,6 @@ int main(int argc, char** argv) return generateCorpus(seed, numFiles, path, origPath); } else { /* should generate files with a dictionary */ - numFiles = (numFiles == 0) ? 1 : numFiles; return generateCorpusWithDict(seed, numFiles, path, origPath, dictSize); } From 1e0d125e7640b5f71b9facfefe321c8459a8c63d Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 11:15:26 -0700 Subject: [PATCH 46/73] const value --- tests/decodecorpus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 09710c379..0c674014b 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -314,7 +314,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info) pos += 4; { - int dictBits = info.useDict ? 3 : 0; + int const dictBits = info.useDict ? 3 : 0; BYTE const frameHeaderDescriptor = (BYTE) ((fcsCode << 6) | (singleSegment << 5) | (1 << 2) | dictBits); op[pos++] = frameHeaderDescriptor; From 43ffcdde4044b2f95d86dea50aeb7228c089b189 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 11:27:39 -0700 Subject: [PATCH 47/73] const dictOffset --- tests/decodecorpus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 0c674014b..f61d43884 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -703,7 +703,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, for (j = 0; j < matchLen; j++) { if ((void*)(srcPtr - offset) < (void*)frame->srcStart) { /* copy from dictionary instead of literals */ - size_t dictOffset = offset - (srcPtr - (BYTE*)frame->srcStart); + size_t const dictOffset = offset - (srcPtr - (BYTE*)frame->srcStart); *srcPtr = *(dictEnd - dictOffset); } else { From ceb4f65620d936f1a80953a3e9c6a128fd05be32 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 15:31:54 -0700 Subject: [PATCH 48/73] allocate statically so we don't have to deal with freeing --- tests/decodecorpus.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index f61d43884..dd2bc562d 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1168,24 +1168,20 @@ static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info) static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ /* allocate space for samples */ unsigned const numSamples = 4; - BYTE* const samples = malloc(5000*sizeof(BYTE)); - size_t* const sampleSizes = malloc(numSamples*sizeof(size_t)); - if (samples == NULL || sampleSizes == NULL) { - DISPLAY("Error: could not allocate space for samples.\n"); - return 1; - } + BYTE samples[5000]; + size_t sampleSizes[4]; /* generate samples */ { unsigned i = 1; + unsigned j = 0; size_t currSize = 1; - BYTE* curr = samples; while (i <= 4) { - *(sampleSizes + i - 1) = currSize; + sampleSizes[i - 1] = currSize; { - size_t j; - for (j = 0; j < currSize; j++) { - *(curr++) = (BYTE)i; + size_t k; + for (k = 0; k < currSize; k++) { + samples[j++] = (BYTE)i; } } i++; @@ -1220,8 +1216,6 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ samples, sampleSizes, numSamples, zdictParams); - free(samples); - free(sampleSizes); if (ZDICT_isError(dictWriteSize)) { DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); return 1; From 69bc4fab25bec6231f3528f24a969d5e598da7c6 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 15:37:14 -0700 Subject: [PATCH 49/73] free the context object once done with decompression --- tests/decodecorpus.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index dd2bc562d..59b2af78a 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1345,6 +1345,7 @@ static size_t testDecodeWithDict(U32 seed, size_t dictSize) errorDetected = ERROR(corruption_detected); goto dictTestCleanup; } + ZSTD_freeDCtx(dctx); } } From acf0df678cf060c11c11821f330dfabb4130a08a Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Tue, 20 Jun 2017 15:41:43 -0700 Subject: [PATCH 50/73] fixed strange spacing --- tests/decodecorpus.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 59b2af78a..2eed515fe 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1336,8 +1336,8 @@ static size_t testDecodeWithDict(U32 seed, size_t dictSize) fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, fullDict, dictSize); if (ZSTD_isError(returnValue)) { - errorDetected = returnValue; - goto dictTestCleanup; + errorDetected = returnValue; + goto dictTestCleanup; } } From ccae9ec8071adc973f9ec12db5bb4db927080090 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 21 Jun 2017 17:43:21 -0700 Subject: [PATCH 51/73] added comment addressing use of bits in frame header descriptor --- tests/decodecorpus.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 2eed515fe..526ddf15b 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -314,6 +314,13 @@ static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info) pos += 4; { + /* + * fcsCode: 2-bit flag specifying how many bytes used to represent Frame_Content_Size (bits 7-6) + * singleSegment: 1-bit flag describing if data must be regenerated within a single continuous memory segment. (bit 5) + * contentChecksumFlag: 1-bit flag that is set if frame includes checksum at the end -- set to 1 below (bit 2) + * dictBits: 2-bit flag describing how many bytes Dictionary_ID uses -- set to 3 (bits 1-0) + * For more information: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_header + */ int const dictBits = info.useDict ? 3 : 0; BYTE const frameHeaderDescriptor = (BYTE) ((fcsCode << 6) | (singleSegment << 5) | (1 << 2) | dictBits); From 2ceaec1aad1d13f147126c63f1e94519feafbfe4 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 21 Jun 2017 17:46:37 -0700 Subject: [PATCH 52/73] got rid of unnecessary casting to void* --- tests/decodecorpus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 526ddf15b..282f959e6 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1235,7 +1235,7 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ static dictInfo initDictInfo(int useDict, size_t dictContentSize, BYTE* dictContent, U32 dictID){ /* allocate space statically */ dictInfo dictOp; - memset((void*)(&dictOp), 0, sizeof(dictOp)); + memset(&dictOp, 0, sizeof(dictOp)); dictOp.useDict = useDict; dictOp.dictContentSize = dictContentSize; dictOp.dictContent = dictContent; From 04253e21bb0f20bc3985f8a850fd4866b090e9f0 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 21 Jun 2017 17:48:01 -0700 Subject: [PATCH 53/73] added const --- tests/decodecorpus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 282f959e6..1423ca1a2 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1316,7 +1316,7 @@ static size_t testDecodeWithDict(U32 seed, size_t dictSize) /* generate random dictionary */ { - int ret = genRandomDict(dictID, seed, dictSize, fullDict); + int const ret = genRandomDict(dictID, seed, dictSize, fullDict); if (ret != 0) { errorDetected = ERROR(GENERIC); goto dictTestCleanup; From eeff2d0e3c15ea7de33450c6ad9da353e7d313e8 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 21 Jun 2017 17:48:37 -0700 Subject: [PATCH 54/73] added const x2 --- tests/decodecorpus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 1423ca1a2..347bc1b50 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1499,7 +1499,7 @@ static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const /* randomly generate the dictionary */ { - int ret = genRandomDict(dictID, seed, dictSize, fullDict); + int const ret = genRandomDict(dictID, seed, dictSize, fullDict); if (ret != 0) { errorDetected = ret; goto dictCleanup; From 32e36c2d9ac4d08633c2a432340d298251c87ad7 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 21 Jun 2017 18:00:47 -0700 Subject: [PATCH 55/73] removed requirement that offset be less than windowSize --- tests/decodecorpus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 347bc1b50..e2b098672 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -685,7 +685,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, /* need to occasionally generate offsets that go past the start */ /* we still need to be within the windowSize however */ U32 const lenPastStart = RAND(seed) % info.dictContentSize; - offset = MIN(frame->header.windowSize, offset+lenPastStart); + offset = offset+lenPastStart; } offsetCode = offset + ZSTD_REP_MOVE; repIndex = 2; From 52a13abf3fe9c6a1388e8e6fab9823345fa69642 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 21 Jun 2017 18:13:52 -0700 Subject: [PATCH 56/73] removed comment regarding windowSize requirement --- tests/decodecorpus.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index e2b098672..64185a13b 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -683,7 +683,6 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, 1; if (info.useDict && (RAND(seed) & 1)) { /* need to occasionally generate offsets that go past the start */ - /* we still need to be within the windowSize however */ U32 const lenPastStart = RAND(seed) % info.dictContentSize; offset = offset+lenPastStart; } From 0b6eedeace6060ff431752eb8800b88c97c60071 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 21 Jun 2017 18:24:19 -0700 Subject: [PATCH 57/73] malloc samples instead of static allocation --- tests/decodecorpus.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 64185a13b..5608e7de0 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1173,9 +1173,14 @@ static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info) /* returns 0 if successful, otherwise returns 1 upon error */ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ /* allocate space for samples */ + int ret = 0; unsigned const numSamples = 4; - BYTE samples[5000]; size_t sampleSizes[4]; + BYTE* const samples = malloc(5000); + if (samples == NULL) { + DISPLAY("Error: could not allocate space for samples\n"); + return 1; + } /* generate samples */ { @@ -1205,7 +1210,8 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ BYTE* const dictContent = fullDict + headerSize; if (dictContentSize < ZDICT_CONTENTSIZE_MIN || dictSize < ZDICT_DICTSIZE_MIN) { DISPLAY("Error: dictionary size is too small\n"); - return 1; + ret = 1; + goto exitGenRandomDict; } /* init dictionary params */ @@ -1224,11 +1230,13 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ if (ZDICT_isError(dictWriteSize)) { DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); - return 1; + ret = 1; } } - return 0; +exitGenRandomDict: + free(samples); + return ret; } static dictInfo initDictInfo(int useDict, size_t dictContentSize, BYTE* dictContent, U32 dictID){ From 0950b3159ac8e065d5e8bb0e5f612e254d3ecadf Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 21 Jun 2017 18:30:27 -0700 Subject: [PATCH 58/73] more meaningful names for count variables --- tests/decodecorpus.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 5608e7de0..dfda93923 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1184,18 +1184,18 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ /* generate samples */ { - unsigned i = 1; - unsigned j = 0; + unsigned literalValue = 1; + unsigned samplesPos = 0; size_t currSize = 1; - while (i <= 4) { - sampleSizes[i - 1] = currSize; + while (literalValue <= 4) { + sampleSizes[literalValue - 1] = currSize; { size_t k; for (k = 0; k < currSize; k++) { - samples[j++] = (BYTE)i; + samples[samplesPos++] = (BYTE)literalValue; } } - i++; + literalValue++; currSize *= 16; } } From 04094f37e987cc509ee0a35a4cb5c930441d3576 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 21 Jun 2017 18:47:40 -0700 Subject: [PATCH 59/73] fixed offset in this case os that it always goes past src start --- tests/decodecorpus.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index dfda93923..3e20b3ba5 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -683,8 +683,8 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, 1; if (info.useDict && (RAND(seed) & 1)) { /* need to occasionally generate offsets that go past the start */ - U32 const lenPastStart = RAND(seed) % info.dictContentSize; - offset = offset+lenPastStart; + U32 const lenPastStart = RAND(seed) % info.dictContentSize + 1; + offset = ((BYTE*)srcPtr - (BYTE*)frame->srcStart)+lenPastStart; } offsetCode = offset + ZSTD_REP_MOVE; repIndex = 2; From 84cfa07d2d3c859c05fc5aab898f93313f43d9f5 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 22 Jun 2017 10:04:14 -0700 Subject: [PATCH 60/73] changed format of command to --use-dict=# --- tests/decodecorpus.c | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 3e20b3ba5..6447567f9 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1615,7 +1615,40 @@ static void advancedUsage(const char* programName) DISPLAY( "\n"); DISPLAY( "Advanced arguments :\n"); DISPLAY( " --content-size : always include the content size in the frame header\n"); - DISPLAY( " --use-dict # : include a dictionary used to decompress the corpus\n"); + DISPLAY( " --use-dict=# : include a dictionary used to decompress the corpus\n"); +} + +/*! readU32FromChar() : + @return : unsigned integer value read from input in `char` format + allows and interprets K, KB, KiB, M, MB and MiB suffix. + Will also modify `*stringPtr`, advancing it to position where it stopped reading. + Note : function result can overflow if digit string > MAX_UINT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + if ((**stringPtr=='K') || (**stringPtr=='M')) { + result <<= 10; + if (**stringPtr=='M') result <<= 10; + (*stringPtr)++ ; + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + +/** longCommandWArg() : + * check if *stringPtr is the same as longCommand. + * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. + * @return 0 and doesn't modify *stringPtr otherwise. + */ +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; } int main(int argc, char** argv) @@ -1689,9 +1722,8 @@ int main(int argc, char** argv) argument++; if (strcmp(argument, "content-size") == 0) { opts.contentSize = 1; - } else if (strcmp(argument, "use-dict") == 0) { - argument += 9; - dictSize = readInt(&argument); + } else if (longCommandWArg(&argument, "use-dict=")) { + dictSize = readU32FromChar(&argument); useDict = 1; } else { advancedUsage(argv[0]); From 98751f69e77433013e6842f742626dc83f1b5cf5 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 22 Jun 2017 10:23:36 -0700 Subject: [PATCH 61/73] should be updating seed whenever multiple files are generated --- tests/decodecorpus.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 6447567f9..9e4ff810c 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1389,7 +1389,7 @@ static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS { dictInfo const info = initDictInfo(0, 0, NULL, 0); - generateFrame(seed, &fr, info); + seed = generateFrame(seed, &fr, info); } { size_t const r = testDecodeSimple(&fr); @@ -1461,7 +1461,7 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, { dictInfo const info = initDictInfo(0, 0, NULL, 0); - generateFrame(seed, &fr, info); + seed = generateFrame(seed, &fr, info); } if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { From 028fb9d01bc451672a49bb3c4862ae6b393053f9 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 22 Jun 2017 13:53:39 -0700 Subject: [PATCH 62/73] added cli test case in Makefile --- tests/Makefile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/Makefile b/tests/Makefile index debe83898..d551a85ef 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -340,6 +340,24 @@ test-legacy: legacy test-decodecorpus: decodecorpus $(QEMU_SYS) ./decodecorpus -t $(DECODECORPUS_TESTTIME) +test-decodecorpus-cli: decodecorpus + @echo "\n ---- decodecorpus basic cli tests ----" + mkdir tests + ./decodecorpus -n5 -otests -ptests + cd testdir + zstd -d z000000.zst -D dictionary -o tmp0 + zstd -d z000001.zst -D dictionary -o tmp1 + zstd -d z000002.zst -D dictionary -o tmp2 + zstd -d z000003.zst -D dictionary -o tmp3 + zstd -d z000004.zst -D dictionary -o tmp4 + diff z000000 tmp0 + diff z000001 tmp1 + diff z000002 tmp2 + diff z000003 tmp3 + diff z000004 tmp4 + cd .. + rm -rf testdir + test-pool: pool $(QEMU_SYS) ./pool From 829eb29033a97f2d5f83e3bef06455f8b8473b90 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 22 Jun 2017 14:43:44 -0700 Subject: [PATCH 63/73] added cli test for decodecorpus inside tests/Makefile. Also changed calculation of offset --- tests/Makefile | 45 +++++++++++++++++++++++++++++--------------- tests/decodecorpus.c | 10 ++++++++-- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index d551a85ef..4789b1ccf 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -342,21 +342,36 @@ test-decodecorpus: decodecorpus test-decodecorpus-cli: decodecorpus @echo "\n ---- decodecorpus basic cli tests ----" - mkdir tests - ./decodecorpus -n5 -otests -ptests - cd testdir - zstd -d z000000.zst -D dictionary -o tmp0 - zstd -d z000001.zst -D dictionary -o tmp1 - zstd -d z000002.zst -D dictionary -o tmp2 - zstd -d z000003.zst -D dictionary -o tmp3 - zstd -d z000004.zst -D dictionary -o tmp4 - diff z000000 tmp0 - diff z000001 tmp1 - diff z000002 tmp2 - diff z000003 tmp3 - diff z000004 tmp4 - cd .. - rm -rf testdir + @mkdir testdir + ./decodecorpus -n5 -otestdir -ptestdir + @cd testdir \ + zstd -d z000000.zst -o tmp0 \ + zstd -d z000001.zst -o tmp1 \ + zstd -d z000002.zst -o tmp2 \ + zstd -d z000003.zst -o tmp3 \ + zstd -d z000004.zst -o tmp4 \ + diff z000000 tmp0 \ + diff z000001 tmp1 \ + diff z000002 tmp2 \ + diff z000003 tmp3 \ + diff z000004 tmp4 \ + @rm ./* \ + @cd .. + + ./decodecorpus -n5 -otestdir -ptestdir --use-dict=1MB + @cd testdir \ + zstd -d z000000.zst -D dictionary -o tmp0 \ + zstd -d z000001.zst -D dictionary -o tmp1 \ + zstd -d z000002.zst -D dictionary -o tmp2 \ + zstd -d z000003.zst -D dictionary -o tmp3 \ + zstd -d z000004.zst -D dictionary -o tmp4 \ + diff z000000 tmp0 \ + diff z000001 tmp1 \ + diff z000002 tmp2 \ + diff z000003 tmp3 \ + diff z000004 tmp4 \ + @cd .. + @rm -rf testdir test-pool: pool $(QEMU_SYS) ./pool diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 9e4ff810c..81ddc4b69 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -648,7 +648,6 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, } DISPLAYLEVEL(5, " total match lengths: %u\n", (U32)remainingMatch); - for (i = 0; i < numSequences; i++) { /* Generate match and literal lengths by exponential distribution to * ensure nice numbers */ @@ -683,8 +682,15 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, 1; if (info.useDict && (RAND(seed) & 1)) { /* need to occasionally generate offsets that go past the start */ - U32 const lenPastStart = RAND(seed) % info.dictContentSize + 1; + U32 const lenPastStart = (RAND(seed) % info.dictContentSize) + 1; offset = ((BYTE*)srcPtr - (BYTE*)frame->srcStart)+lenPastStart; + if (lenPastStart > frame->header.windowSize) { + matchLen = MIN(matchLen, frame->header.windowSize); + } + if (offset > frame->header.windowSize) { + U32 const matchLenBound = MIN(frame->header.windowSize, lenPastStart); + matchLen = MIN(matchLen, matchLenBound); + } } offsetCode = offset + ZSTD_REP_MOVE; repIndex = 2; From 97a8f89bec49f285f779a4601aff7e27b93bfae1 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 22 Jun 2017 15:13:41 -0700 Subject: [PATCH 64/73] made changes to cli test so it runs properly --- tests/Makefile | 51 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 4789b1ccf..033e3f782 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -344,33 +344,32 @@ test-decodecorpus-cli: decodecorpus @echo "\n ---- decodecorpus basic cli tests ----" @mkdir testdir ./decodecorpus -n5 -otestdir -ptestdir - @cd testdir \ - zstd -d z000000.zst -o tmp0 \ - zstd -d z000001.zst -o tmp1 \ - zstd -d z000002.zst -o tmp2 \ - zstd -d z000003.zst -o tmp3 \ - zstd -d z000004.zst -o tmp4 \ - diff z000000 tmp0 \ - diff z000001 tmp1 \ - diff z000002 tmp2 \ - diff z000003 tmp3 \ - diff z000004 tmp4 \ - @rm ./* \ - @cd .. - + @cd testdir && \ + $(ZSTD) -d z000000.zst -o tmp0 && \ + $(ZSTD) -d z000001.zst -o tmp1 && \ + $(ZSTD) -d z000002.zst -o tmp2 && \ + $(ZSTD) -d z000003.zst -o tmp3 && \ + $(ZSTD) -d z000004.zst -o tmp4 && \ + diff z000000 tmp0 && \ + diff z000001 tmp1 && \ + diff z000002 tmp2 && \ + diff z000003 tmp3 && \ + diff z000004 tmp4 && \ + rm ./* && \ + cd .. ./decodecorpus -n5 -otestdir -ptestdir --use-dict=1MB - @cd testdir \ - zstd -d z000000.zst -D dictionary -o tmp0 \ - zstd -d z000001.zst -D dictionary -o tmp1 \ - zstd -d z000002.zst -D dictionary -o tmp2 \ - zstd -d z000003.zst -D dictionary -o tmp3 \ - zstd -d z000004.zst -D dictionary -o tmp4 \ - diff z000000 tmp0 \ - diff z000001 tmp1 \ - diff z000002 tmp2 \ - diff z000003 tmp3 \ - diff z000004 tmp4 \ - @cd .. + @cd testdir && \ + $(ZSTD) -d z000000.zst -D dictionary -o tmp0 && \ + $(ZSTD) -d z000001.zst -D dictionary -o tmp1 && \ + $(ZSTD) -d z000002.zst -D dictionary -o tmp2 && \ + $(ZSTD) -d z000003.zst -D dictionary -o tmp3 && \ + $(ZSTD) -d z000004.zst -D dictionary -o tmp4 && \ + diff z000000 tmp0 && \ + diff z000001 tmp1 && \ + diff z000002 tmp2 && \ + diff z000003 tmp3 && \ + diff z000004 tmp4 && \ + @cd .. @rm -rf testdir test-pool: pool From 29352395445b2c84c08d91c940662f5471115fb0 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 22 Jun 2017 15:23:59 -0700 Subject: [PATCH 65/73] added description for dictionary cli tests --- tests/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Makefile b/tests/Makefile index 033e3f782..723c2a9d9 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -357,6 +357,7 @@ test-decodecorpus-cli: decodecorpus diff z000004 tmp4 && \ rm ./* && \ cd .. + @echo "\n ---- decodecorpus dictionary cli tests ----" ./decodecorpus -n5 -otestdir -ptestdir --use-dict=1MB @cd testdir && \ $(ZSTD) -d z000000.zst -D dictionary -o tmp0 && \ From b325a2e4dbc009f0caf60920b8e9d2a15917be67 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 22 Jun 2017 15:36:28 -0700 Subject: [PATCH 66/73] changed assignment --- tests/decodecorpus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 81ddc4b69..2517483ff 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1198,7 +1198,7 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ { size_t k; for (k = 0; k < currSize; k++) { - samples[samplesPos++] = (BYTE)literalValue; + *(samples + (samplesPos++)) = (BYTE)literalValue; } } literalValue++; From 6d7f4421201c6cbea7ec717a350b6e4d81c1ff1a Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 22 Jun 2017 15:44:01 -0700 Subject: [PATCH 67/73] changed makefile test --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 723c2a9d9..960708875 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -370,7 +370,7 @@ test-decodecorpus-cli: decodecorpus diff z000002 tmp2 && \ diff z000003 tmp3 && \ diff z000004 tmp4 && \ - @cd .. + cd .. @rm -rf testdir test-pool: pool From 2e8cc6f12a59aad052770ef713fec826803d0afb Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Thu, 22 Jun 2017 15:52:33 -0700 Subject: [PATCH 68/73] added sizeof for clarity --- tests/decodecorpus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 2517483ff..60cdc2969 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -1182,7 +1182,7 @@ static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict){ int ret = 0; unsigned const numSamples = 4; size_t sampleSizes[4]; - BYTE* const samples = malloc(5000); + BYTE* const samples = malloc(5000*sizeof(BYTE)); if (samples == NULL) { DISPLAY("Error: could not allocate space for samples\n"); return 1; From 4219acc60adcf6b0c87687a1cfc84920fae3481a Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 23 Jun 2017 11:22:29 -0700 Subject: [PATCH 69/73] fixed bus error bug --- tests/decodecorpus.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 60cdc2969..f04a02fa7 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -672,7 +672,6 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, memcpy(srcPtr, literals, literalLen); srcPtr += literalLen; - do { if (RAND(seed) & 7) { /* do a normal offset */ @@ -682,14 +681,20 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, 1; if (info.useDict && (RAND(seed) & 1)) { /* need to occasionally generate offsets that go past the start */ - U32 const lenPastStart = (RAND(seed) % info.dictContentSize) + 1; + U32 lenPastStart = (RAND(seed) % info.dictContentSize) + 1; offset = ((BYTE*)srcPtr - (BYTE*)frame->srcStart)+lenPastStart; - if (lenPastStart > frame->header.windowSize) { - matchLen = MIN(matchLen, frame->header.windowSize); - } if (offset > frame->header.windowSize) { - U32 const matchLenBound = MIN(frame->header.windowSize, lenPastStart); - matchLen = MIN(matchLen, matchLenBound); + if (lenPastStart < MIN_SEQ_LEN) { + /* when offset > windowSize, matchLen bound by end of dictionary (lenPastStart) */ + /* this also means that lenPastStart must be greater than MIN_SEQ_LEN */ + /* make sure lenPastStart does not go past dictionary start though */ + lenPastStart = MIN(lenPastStart+MIN_SEQ_LEN, info.dictContentSize); + offset = ((BYTE*)srcPtr - (BYTE*)frame->srcStart) + lenPastStart; + } + { + U32 const matchLenBound = MIN(frame->header.windowSize, lenPastStart); + matchLen = MIN(matchLen, matchLenBound); + } } } offsetCode = offset + ZSTD_REP_MOVE; @@ -713,7 +718,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, size_t j; BYTE* const dictEnd = info.dictContent + info.dictContentSize; for (j = 0; j < matchLen; j++) { - if ((void*)(srcPtr - offset) < (void*)frame->srcStart) { + if (((BYTE*)srcPtr - (BYTE*)frame->srcStart) < offset) { /* copy from dictionary instead of literals */ size_t const dictOffset = offset - (srcPtr - (BYTE*)frame->srcStart); *srcPtr = *(dictEnd - dictOffset); From 8cd134559dc9a3bf63205f240b03b74f1f75d3d5 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 23 Jun 2017 12:00:48 -0700 Subject: [PATCH 70/73] type warnings --- tests/decodecorpus.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index f04a02fa7..7ff7fc4f3 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -682,14 +682,14 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, if (info.useDict && (RAND(seed) & 1)) { /* need to occasionally generate offsets that go past the start */ U32 lenPastStart = (RAND(seed) % info.dictContentSize) + 1; - offset = ((BYTE*)srcPtr - (BYTE*)frame->srcStart)+lenPastStart; + offset = (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart)+lenPastStart; if (offset > frame->header.windowSize) { if (lenPastStart < MIN_SEQ_LEN) { /* when offset > windowSize, matchLen bound by end of dictionary (lenPastStart) */ /* this also means that lenPastStart must be greater than MIN_SEQ_LEN */ /* make sure lenPastStart does not go past dictionary start though */ - lenPastStart = MIN(lenPastStart+MIN_SEQ_LEN, info.dictContentSize); - offset = ((BYTE*)srcPtr - (BYTE*)frame->srcStart) + lenPastStart; + lenPastStart = MIN(lenPastStart+MIN_SEQ_LEN, (U32)info.dictContentSize); + offset = (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart) + lenPastStart; } { U32 const matchLenBound = MIN(frame->header.windowSize, lenPastStart); @@ -718,7 +718,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, size_t j; BYTE* const dictEnd = info.dictContent + info.dictContentSize; for (j = 0; j < matchLen; j++) { - if (((BYTE*)srcPtr - (BYTE*)frame->srcStart) < offset) { + if ((U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart) < offset) { /* copy from dictionary instead of literals */ size_t const dictOffset = offset - (srcPtr - (BYTE*)frame->srcStart); *srcPtr = *(dictEnd - dictOffset); From 20853758162d0823b8e5130467ca809e7c728126 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 23 Jun 2017 13:44:24 -0700 Subject: [PATCH 71/73] fixed bug detected by the API test --- tests/decodecorpus.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index 7ff7fc4f3..eaeb05eac 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -679,8 +679,9 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, MIN(frame->header.windowSize, (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) + 1; - if (info.useDict && (RAND(seed) & 1)) { + if (info.useDict && (RAND(seed) & 1) && i + 1 != numSequences) { /* need to occasionally generate offsets that go past the start */ + /* including i+1 != numSequences because the last sequences has to adhere to predetermined contentSize */ U32 lenPastStart = (RAND(seed) % info.dictContentSize) + 1; offset = (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart)+lenPastStart; if (offset > frame->header.windowSize) { From 3a295a91f8c8645888d7701eeb4c813c8fd663b8 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 23 Jun 2017 15:54:51 -0700 Subject: [PATCH 72/73] added additional condition so large offsets into the dictionary are not generated past windowSize --- tests/decodecorpus.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c index eaeb05eac..9e5e973ca 100644 --- a/tests/decodecorpus.c +++ b/tests/decodecorpus.c @@ -675,11 +675,12 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, do { if (RAND(seed) & 7) { /* do a normal offset */ + U32 const dataDecompressed = (U32)((BYTE*)srcPtr-(BYTE*)frame->srcStart); offset = (RAND(seed) % MIN(frame->header.windowSize, (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) + 1; - if (info.useDict && (RAND(seed) & 1) && i + 1 != numSequences) { + if (info.useDict && (RAND(seed) & 1) && i + 1 != numSequences && dataDecompressed < frame->header.windowSize) { /* need to occasionally generate offsets that go past the start */ /* including i+1 != numSequences because the last sequences has to adhere to predetermined contentSize */ U32 lenPastStart = (RAND(seed) % info.dictContentSize) + 1; From 298fe9f23b25b0ce03e6238f6ec777e48e3c94ca Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 23 Jun 2017 17:10:41 -0700 Subject: [PATCH 73/73] added cli tests to test-all --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 960708875..ebfdc1287 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -272,7 +272,7 @@ endif test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 -test-all: test test32 valgrindTest +test-all: test test32 valgrindTest test-decodecorpus-cli test-zstd: ZSTD = $(PRGDIR)/zstd test-zstd: zstd zstd-playTests