ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

huf log speed optimization: unidirectional scan of logs + break when regressing

Browse Source
This commit is contained in:
Danielle Rozenblit
2022-12-20 12:27:38 -08:00
parent c2638212af
commit 482689b995
21 changed files with 96 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+13 -3
View File
@@ -2749,6 +2749,7 @@ typedef struct {
int numSkippableFrames;
int decompUnavailable;
int usesCheck;
BYTE checksum[4];
U32 nbFiles;
unsigned dictID;
} fileInfo_t;
@@ -2843,8 +2844,8 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2;
if (contentChecksumFlag) {
info->usesCheck = 1;
ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0,
info_frame_error, "Error: could not skip past checksum");
ERROR_IF(fread(info->checksum, 1, 4, srcFile) != 4,
info_frame_error, "Error: could not read checksum");
} }
info->numActualFrames++;
}
@@ -2936,7 +2937,16 @@ displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel)
(unsigned long long)info->decompressedSize);
DISPLAYOUT("Ratio: %.4f\n", ratio);
}
DISPLAYOUT("Check: %s\n", checkString);
if (info->usesCheck && info->numActualFrames == 1) {
DISPLAYOUT("Check: %s %02x%02x%02x%02x\n", checkString,
info->checksum[3], info->checksum[2],
info->checksum[1], info->checksum[0]
);
} else {
DISPLAYOUT("Check: %s\n", checkString);
}
DISPLAYOUT("\n");
}
}
programs/util.c
+6 -3
View File
@@ -569,7 +569,7 @@ UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2)
for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) {
size_t const curLen = strlen(table2->fileNames[idx2]);
memcpy(buf+pos, table2->fileNames[idx2], curLen);
assert(newTableIdx <= newTable->tableSize);
assert(newTableIdx < newTable->tableSize);
newTable->fileNames[newTableIdx] = buf+pos;
pos += curLen+1;
} }
@@ -693,8 +693,11 @@ static int UTIL_prepareFileList(const char *dirName,
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
assert(newListSize >= 0);
*bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
*bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
if (*bufStart != NULL) {
*bufEnd = *bufStart + newListSize;
} else {
free(path); closedir(dir); return 0;
}
}
if (*bufStart + *pos + pathLength < *bufEnd) {
memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */
programs/zstdcli.c
+3 -3
View File
@@ -216,7 +216,6 @@ static void usage_advanced(const char* programName)
DISPLAYOUT("\n");
DISPLAYOUT("Advanced compression options :\n");
DISPLAYOUT(" --ultra enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
DISPLAYOUT(" --long[=#] enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
DISPLAYOUT(" --fast[=#] switch to very fast compression levels (default: %u)\n", 1);
#ifdef ZSTD_GZCOMPRESS
if (exeNameMatch(programName, ZSTD_GZ)) { /* behave like gzip */
@@ -224,9 +223,9 @@ static void usage_advanced(const char* programName)
DISPLAYOUT(" --no-name do not store original filename when compressing\n");
}
#endif
DISPLAYOUT(" --adapt dynamically adapt compression level to I/O conditions\n");
DISPLAYOUT(" --[no-]row-match-finder : force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
DISPLAYOUT(" --long[=#] enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
DISPLAYOUT(" --patch-from=FILE : specify the file to be used as a reference point for zstd's diff engine. \n");
DISPLAYOUT(" --adapt dynamically adapt compression level to I/O conditions\n");
# ifdef ZSTD_MULTITHREAD
DISPLAYOUT(" -T# spawn # compression threads (default: 1, 0==# cores) \n");
DISPLAYOUT(" -B# select size of each job (default: 0==automatic) \n");
@@ -240,6 +239,7 @@ static void usage_advanced(const char* programName)
DISPLAYOUT(" --target-compressed-block-size=# : generate compressed block of approximately targeted size \n");
DISPLAYOUT(" --no-dictID don't write dictID into header (dictionary compression only)\n");
DISPLAYOUT(" --[no-]compress-literals : force (un)compressed literals\n");
DISPLAYOUT(" --[no-]row-match-finder : force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies\n");
DISPLAYOUT(" --format=zstd compress files to the .zst format (default)\n");
#ifdef ZSTD_GZCOMPRESS