ddidderr
da0e2c8380
`make all` exercises substantially more than the default zstd binary. It also builds compression-only and decompression-only archives, older contrib tools that compile program C shims directly, and the single-file amalgamations. The Rust migration had changed symbol ownership without updating every one of those feature and link boundaries. The first failure came from `fileio_asyncio`: it is always compiled, but its codec bindings unconditionally referenced both `zstd_compress` and `zstd_decompress`. A compression-only archive therefore required disabled decoder modules, and the decompression-only configuration had the symmetric problem. Gate the concrete codec imports, callback types, helpers, and exports with their Cargo features. Keep the format probe compilable without the legacy decoder predicate when decompression is disabled. Once that boundary compiled, the remaining `make all` paths exposed related integration gaps. Move the C decompression projection declarations out of the compression preprocessor block, while leaving destination callback types shared. Link the Rust CLI helpers archive into zlibWrapper, pzstd, and largeNbDicts, whose util/time/data-generator C files are now declaration shims rather than implementations. The generated single-file C sources also call Rust-owned symbols now. Build and link an appropriately featured Rust archive in their native smoke tests, and include the pool configuration shim in the decoder case. The full amalgamation combines `zstd_lazy.c` and `zstd_opt.c` into one translation unit, so guard their otherwise translation-unit-local dictionary mode enum against duplicate definition. A Rust-backed amalgamation is no longer a standalone Emscripten input. Remove the obsolete emcc/Docker path and report that limitation explicitly; restoring the WebAssembly smoke test requires a Rust WebAssembly archive and a defined cross-language amalgamation contract. Test Plan: - `cargo check --manifest-path rust/Cargo.toml --no-default-features --features compression` -- passed - `cargo check --manifest-path rust/Cargo.toml --no-default-features --features decompression` -- passed - `sh -n build/single_file_libs/build_decoder_test.sh build/single_file_libs/build_library_test.sh` -- passed - `make all` -- passed, including native single-file and seekable-format tests - `git diff --cached --check` -- passed
257 lines
11 KiB
C
257 lines
11 KiB
C
/*
|
|
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
* All rights reserved.
|
|
*
|
|
* This source code is licensed under both the BSD-style license (found in the
|
|
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
|
* in the COPYING file in the root directory of this source tree).
|
|
* You may select, at your option, one of the above-listed licenses.
|
|
*/
|
|
|
|
/* The lazy matchers are implemented in rust/src/zstd_lazy.rs. Keep only the
|
|
* private match-state projection and the established C entry points here. */
|
|
#include "zstd_compress_internal.h"
|
|
#include "zstd_lazy.h"
|
|
|
|
#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \
|
|
|| !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
|
|
|| !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \
|
|
|| !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR)
|
|
|
|
typedef char ZSTD_rust_lazy_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
|
|
typedef char ZSTD_rust_lazy_seqstore_long_length_pos[
|
|
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
|
|
typedef char ZSTD_rust_lazy_seqstore_layout[
|
|
(sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1];
|
|
typedef char ZSTD_rust_lazy_rep_count[(ZSTD_REP_NUM == 3) ? 1 : -1];
|
|
|
|
/* This must remain in lock-step with `ZSTD_RustLazyState` in Rust. The large
|
|
* and private `ZSTD_MatchState_t` itself deliberately never crosses FFI. */
|
|
typedef struct ZSTD_RustLazyState_s {
|
|
U32* hashTable;
|
|
U32* chainTable;
|
|
BYTE* tagTable;
|
|
U32* hashCache;
|
|
const BYTE* base;
|
|
const BYTE* dictBase;
|
|
const BYTE* nextSrc;
|
|
U32 dictLimit;
|
|
U32 lowLimit;
|
|
U32 loadedDictEnd;
|
|
U32* nextToUpdate;
|
|
int* lazySkipping;
|
|
U32 hashLog;
|
|
U32 chainLog;
|
|
U32 minMatch;
|
|
U32 searchLog;
|
|
U32 windowLog;
|
|
U32 rowHashLog;
|
|
U64 hashSalt;
|
|
U32* hashSaltEntropy;
|
|
const struct ZSTD_RustLazyState_s* dictMatchState;
|
|
} ZSTD_RustLazyState;
|
|
|
|
U32 ZSTD_rust_lazy_insertAndFindFirstIndex(ZSTD_RustLazyState* state,
|
|
const void* ip);
|
|
void ZSTD_rust_lazy_row_update(ZSTD_RustLazyState* state, const void* ip);
|
|
void ZSTD_rust_lazy_loadDedicatedDict(ZSTD_RustLazyState* state,
|
|
const void* ip);
|
|
size_t ZSTD_rust_compressBlock_lazy(
|
|
ZSTD_RustLazyState* state, void* seqStore, U32 rep[ZSTD_REP_NUM],
|
|
const void* src, size_t srcSize, int searchMethod, U32 depth,
|
|
int dictMode);
|
|
|
|
enum {
|
|
ZSTD_rust_search_hashChain = 0,
|
|
ZSTD_rust_search_binaryTree = 1,
|
|
ZSTD_rust_search_rowHash = 2,
|
|
};
|
|
|
|
#ifndef ZSTD_RUST_DICT_MODE_ENUM
|
|
#define ZSTD_RUST_DICT_MODE_ENUM
|
|
enum {
|
|
ZSTD_rust_dict_noDict = 0,
|
|
ZSTD_rust_dict_extDict = 1,
|
|
ZSTD_rust_dict_dictMatchState = 2,
|
|
ZSTD_rust_dict_dedicatedDictSearch = 3,
|
|
};
|
|
#endif
|
|
|
|
static void ZSTD_rustLazyState_init(
|
|
ZSTD_RustLazyState* const out,
|
|
ZSTD_MatchState_t* const ms,
|
|
const ZSTD_RustLazyState* const dms)
|
|
{
|
|
out->hashTable = ms->hashTable;
|
|
out->chainTable = ms->chainTable;
|
|
out->tagTable = ms->tagTable;
|
|
out->hashCache = ms->hashCache;
|
|
out->base = ms->window.base;
|
|
out->dictBase = ms->window.dictBase;
|
|
out->nextSrc = ms->window.nextSrc;
|
|
out->dictLimit = ms->window.dictLimit;
|
|
out->lowLimit = ms->window.lowLimit;
|
|
out->loadedDictEnd = ms->loadedDictEnd;
|
|
out->nextToUpdate = &ms->nextToUpdate;
|
|
out->lazySkipping = &ms->lazySkipping;
|
|
out->hashLog = ms->cParams.hashLog;
|
|
out->chainLog = ms->cParams.chainLog;
|
|
out->minMatch = ms->cParams.minMatch;
|
|
out->searchLog = ms->cParams.searchLog;
|
|
out->windowLog = ms->cParams.windowLog;
|
|
out->rowHashLog = ms->rowHashLog;
|
|
out->hashSalt = ms->hashSalt;
|
|
out->hashSaltEntropy = &ms->hashSaltEntropy;
|
|
out->dictMatchState = dms;
|
|
}
|
|
|
|
/* Dictionary match states are searched but never updated by this module. */
|
|
static void ZSTD_rustLazyState_initDict(
|
|
ZSTD_RustLazyState* const out,
|
|
const ZSTD_MatchState_t* const ms)
|
|
{
|
|
out->hashTable = ms->hashTable;
|
|
out->chainTable = ms->chainTable;
|
|
out->tagTable = ms->tagTable;
|
|
out->hashCache = NULL;
|
|
out->base = ms->window.base;
|
|
out->dictBase = ms->window.dictBase;
|
|
out->nextSrc = ms->window.nextSrc;
|
|
out->dictLimit = ms->window.dictLimit;
|
|
out->lowLimit = ms->window.lowLimit;
|
|
out->loadedDictEnd = ms->loadedDictEnd;
|
|
out->nextToUpdate = NULL;
|
|
out->lazySkipping = NULL;
|
|
out->hashLog = ms->cParams.hashLog;
|
|
out->chainLog = ms->cParams.chainLog;
|
|
out->minMatch = ms->cParams.minMatch;
|
|
out->searchLog = ms->cParams.searchLog;
|
|
out->windowLog = ms->cParams.windowLog;
|
|
out->rowHashLog = ms->rowHashLog;
|
|
out->hashSalt = ms->hashSalt;
|
|
out->hashSaltEntropy = NULL;
|
|
out->dictMatchState = NULL;
|
|
}
|
|
|
|
static size_t ZSTD_rust_compressBlock_lazy_call(
|
|
ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore,
|
|
U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize,
|
|
int const searchMethod, U32 const depth, int const dictMode)
|
|
{
|
|
ZSTD_RustLazyState state;
|
|
ZSTD_RustLazyState dictState;
|
|
const ZSTD_RustLazyState* dms = NULL;
|
|
|
|
if (dictMode == ZSTD_rust_dict_dictMatchState
|
|
|| dictMode == ZSTD_rust_dict_dedicatedDictSearch) {
|
|
const ZSTD_MatchState_t* const rawDms = ms->dictMatchState;
|
|
assert(rawDms != NULL);
|
|
ZSTD_rustLazyState_initDict(&dictState, rawDms);
|
|
dms = &dictState;
|
|
}
|
|
ZSTD_rustLazyState_init(&state, ms, dms);
|
|
return ZSTD_rust_compressBlock_lazy(
|
|
&state, seqStore, rep, src, srcSize, searchMethod, depth, dictMode);
|
|
}
|
|
|
|
U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip)
|
|
{
|
|
ZSTD_RustLazyState state;
|
|
ZSTD_rustLazyState_init(&state, ms, NULL);
|
|
return ZSTD_rust_lazy_insertAndFindFirstIndex(&state, ip);
|
|
}
|
|
|
|
void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* const ip)
|
|
{
|
|
ZSTD_RustLazyState state;
|
|
ZSTD_rustLazyState_init(&state, ms, NULL);
|
|
ZSTD_rust_lazy_row_update(&state, ip);
|
|
}
|
|
|
|
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(
|
|
ZSTD_MatchState_t* ms, const BYTE* const ip)
|
|
{
|
|
ZSTD_RustLazyState state;
|
|
ZSTD_rustLazyState_init(&state, ms, NULL);
|
|
ZSTD_rust_lazy_loadDedicatedDict(&state, ip);
|
|
}
|
|
|
|
#define ZSTD_RUST_LAZY_WRAPPER(name, method, parserDepth, mode) \
|
|
size_t name( \
|
|
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
|
|
const void* src, size_t srcSize) \
|
|
{ \
|
|
return ZSTD_rust_compressBlock_lazy_call( \
|
|
ms, seqStore, rep, src, srcSize, method, parserDepth, mode); \
|
|
}
|
|
|
|
#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy,
|
|
ZSTD_rust_search_hashChain, 0, ZSTD_rust_dict_noDict)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_dictMatchState,
|
|
ZSTD_rust_search_hashChain, 0, ZSTD_rust_dict_dictMatchState)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_dedicatedDictSearch,
|
|
ZSTD_rust_search_hashChain, 0, ZSTD_rust_dict_dedicatedDictSearch)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_row,
|
|
ZSTD_rust_search_rowHash, 0, ZSTD_rust_dict_noDict)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_dictMatchState_row,
|
|
ZSTD_rust_search_rowHash, 0, ZSTD_rust_dict_dictMatchState)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_dedicatedDictSearch_row,
|
|
ZSTD_rust_search_rowHash, 0, ZSTD_rust_dict_dedicatedDictSearch)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_extDict,
|
|
ZSTD_rust_search_hashChain, 0, ZSTD_rust_dict_extDict)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_extDict_row,
|
|
ZSTD_rust_search_rowHash, 0, ZSTD_rust_dict_extDict)
|
|
#endif
|
|
|
|
#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy,
|
|
ZSTD_rust_search_hashChain, 1, ZSTD_rust_dict_noDict)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_dictMatchState,
|
|
ZSTD_rust_search_hashChain, 1, ZSTD_rust_dict_dictMatchState)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_dedicatedDictSearch,
|
|
ZSTD_rust_search_hashChain, 1, ZSTD_rust_dict_dedicatedDictSearch)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_row,
|
|
ZSTD_rust_search_rowHash, 1, ZSTD_rust_dict_noDict)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_dictMatchState_row,
|
|
ZSTD_rust_search_rowHash, 1, ZSTD_rust_dict_dictMatchState)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_dedicatedDictSearch_row,
|
|
ZSTD_rust_search_rowHash, 1, ZSTD_rust_dict_dedicatedDictSearch)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_extDict,
|
|
ZSTD_rust_search_hashChain, 1, ZSTD_rust_dict_extDict)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_extDict_row,
|
|
ZSTD_rust_search_rowHash, 1, ZSTD_rust_dict_extDict)
|
|
#endif
|
|
|
|
#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2,
|
|
ZSTD_rust_search_hashChain, 2, ZSTD_rust_dict_noDict)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_dictMatchState,
|
|
ZSTD_rust_search_hashChain, 2, ZSTD_rust_dict_dictMatchState)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_dedicatedDictSearch,
|
|
ZSTD_rust_search_hashChain, 2, ZSTD_rust_dict_dedicatedDictSearch)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_row,
|
|
ZSTD_rust_search_rowHash, 2, ZSTD_rust_dict_noDict)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_dictMatchState_row,
|
|
ZSTD_rust_search_rowHash, 2, ZSTD_rust_dict_dictMatchState)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_dedicatedDictSearch_row,
|
|
ZSTD_rust_search_rowHash, 2, ZSTD_rust_dict_dedicatedDictSearch)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_extDict,
|
|
ZSTD_rust_search_hashChain, 2, ZSTD_rust_dict_extDict)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_extDict_row,
|
|
ZSTD_rust_search_rowHash, 2, ZSTD_rust_dict_extDict)
|
|
#endif
|
|
|
|
#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_btlazy2,
|
|
ZSTD_rust_search_binaryTree, 2, ZSTD_rust_dict_noDict)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_btlazy2_dictMatchState,
|
|
ZSTD_rust_search_binaryTree, 2, ZSTD_rust_dict_dictMatchState)
|
|
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_btlazy2_extDict,
|
|
ZSTD_rust_search_binaryTree, 2, ZSTD_rust_dict_extDict)
|
|
#endif
|
|
|
|
#undef ZSTD_RUST_LAZY_WRAPPER
|
|
|
|
#endif /* lazy matcher build exclusions */
|