Elliot Gorokhovsky
2a402626dd
* First building commit with sample matchfinder * Set up ZSTD_externalMatchCtx struct * move seqBuffer to ZSTD_Sequence* * support non-contiguous dictionary * clean up parens * add clearExternalMatchfinder, handle allocation errors * Add useExternalMatchfinder cParam * validate useExternalMatchfinder cParam * Disable LDM + external matchfinder * Check for static CCtx * Validate mState and mStateDestructor * Improve LDM check to cover both branches * Error API with optional fallback * handle RLE properly for external matchfinder * nit * Move to a CDict-like model for resource ownership * Add hidden useExternalMatchfinder bool to CCtx_params_s * Eliminate malloc, move to cwksp allocation * Handle CCtx reset properly * Ensure seqStore has enough space for external sequences * fix capitalization * Add DEBUGLOG statements * Add compressionLevel param to matchfinder API * fix c99 issues and add a param combination error code * nits * Test external matchfinder API * C90 compat for simpleExternalMatchFinder * Fix some @nocommits and an ASAN bug * nit * nit * nits * forward declare copySequencesToSeqStore functions in zstd_compress_internal.h * nit * nit * nits * Update copyright headers * Fix CMake zstreamtest build * Fix copyright headers (again) * typo * Add externalMatchfinder demo program to make contrib * Reduce memory consumption for small blockSize * ZSTD_postProcessExternalMatchFinderResult nits * test sum(matchlen) + sum(litlen) == srcSize in debug builds * refExternalMatchFinder -> registerExternalMatchFinder * C90 nit * zstreamtest nits * contrib nits * contrib nits * allow block splitter + external matchfinder, refactor * add windowSize param * add contrib/externalMatchfinder/README.md * docs * go back to old RLE heuristic because of the first block issue * fix initializer element is not a constant expression * ref contrib from zstd.h * extremely pedantic compiler warning fix, meson fix, typo fix * Additional docs on API limitations * minor nits * Refactor maxNbSeq calculation into a helper function * Fix copyright
108 lines
2.9 KiB
C
108 lines
2.9 KiB
C
/*
|
|
* Copyright (c) Yann Collet, Meta Platforms, Inc.
|
|
* All rights reserved.
|
|
*
|
|
* This source code is licensed under both the BSD-style license (found in the
|
|
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
* in the COPYING file in the root directory of this source tree).
|
|
* You may select, at your option, one of the above-listed licenses.
|
|
*/
|
|
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <assert.h>
|
|
|
|
#define ZSTD_STATIC_LINKING_ONLY
|
|
#include "zstd.h"
|
|
#include "zstd_errors.h"
|
|
#include "matchfinder.h" // simpleExternalMatchFinder
|
|
|
|
#define CHECK(res) \
|
|
do { \
|
|
if (ZSTD_isError(res)) { \
|
|
printf("ERROR: %s\n", ZSTD_getErrorName(res)); \
|
|
return 1; \
|
|
} \
|
|
} while (0) \
|
|
|
|
int main(int argc, char *argv[]) {
|
|
if (argc != 2) {
|
|
printf("Usage: exampleMatchfinder <file>\n");
|
|
return 1;
|
|
}
|
|
|
|
ZSTD_CCtx* const zc = ZSTD_createCCtx();
|
|
|
|
int simpleExternalMatchState = 0xdeadbeef;
|
|
|
|
// Here is the crucial bit of code!
|
|
ZSTD_registerExternalMatchFinder(
|
|
zc,
|
|
&simpleExternalMatchState,
|
|
simpleExternalMatchFinder
|
|
);
|
|
|
|
{
|
|
size_t const res = ZSTD_CCtx_setParameter(zc, ZSTD_c_enableMatchFinderFallback, 1);
|
|
CHECK(res);
|
|
}
|
|
|
|
FILE *f = fopen(argv[1], "rb");
|
|
assert(f);
|
|
{
|
|
int const ret = fseek(f, 0, SEEK_END);
|
|
assert(ret == 0);
|
|
}
|
|
size_t const srcSize = ftell(f);
|
|
{
|
|
int const ret = fseek(f, 0, SEEK_SET);
|
|
assert(ret == 0);
|
|
}
|
|
|
|
char* const src = malloc(srcSize + 1);
|
|
assert(src);
|
|
{
|
|
size_t const ret = fread(src, srcSize, 1, f);
|
|
assert(ret == 1);
|
|
int const ret2 = fclose(f);
|
|
assert(ret2 == 0);
|
|
}
|
|
|
|
size_t const dstSize = ZSTD_compressBound(srcSize);
|
|
char* const dst = malloc(dstSize);
|
|
assert(dst);
|
|
|
|
size_t const cSize = ZSTD_compress2(zc, dst, dstSize, src, srcSize);
|
|
CHECK(cSize);
|
|
|
|
char* const val = malloc(srcSize);
|
|
assert(val);
|
|
|
|
{
|
|
size_t const res = ZSTD_decompress(val, srcSize, dst, cSize);
|
|
CHECK(res);
|
|
}
|
|
|
|
if (memcmp(src, val, srcSize) == 0) {
|
|
printf("Compression and decompression were successful!\n");
|
|
printf("Original size: %lu\n", srcSize);
|
|
printf("Compressed size: %lu\n", cSize);
|
|
} else {
|
|
printf("ERROR: input and validation buffers don't match!\n");
|
|
for (size_t i = 0; i < srcSize; i++) {
|
|
if (src[i] != val[i]) {
|
|
printf("First bad index: %zu\n", i);
|
|
break;
|
|
}
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
ZSTD_freeCCtx(zc);
|
|
free(src);
|
|
free(dst);
|
|
free(val);
|
|
return 0;
|
|
}
|