feat(rust): port divsufsort

Move the dictionary builder's suffix-array construction from lib/dictBuilder/divsufsort.c to rust/src/divsufsort.rs, the first dictBuilder module to migrate. It rides on the dict-builder cargo feature dimension introduced by the previous commit. divsufsort() is a self-contained algorithm (two-stage sort of type-B* substrings via sssort, rank refinement via trsort, then induced sorting of the full array), so its context-free signature allows a direct symbol takeover: the Rust #[no_mangle] export provides the existing `divsufsort` symbol and the C file becomes a declaration-only shim that just keeps the header's prototypes in the build. Only divsufsort() moved; divbwt() has no callers anywhere in zstd, so it is now declaration-only, keeping the Rust export surface minimal. The unused openMP parameter is retained for signature compatibility (zstd never defines LIBBSC_OPENMP). The port is a mechanical translation of the exact configuration zstd compiles: ALPHABET_SIZE=256, SS_INSERTIONSORT_THRESHOLD=8, SS_BLOCKSIZE=1024, SS_MISORT_STACKSIZE=16, SS_SMERGE_STACKSIZE=32, TR_STACKSIZE=64. Every C `int*` cursor into the SA buffer becomes an `isize` index into a single `&mut [i32]` slice, preserving the pointer arithmetic (including transient one-before-the-range cursors and the bitwise-complement rank marking) while staying bounds-checked; all value arithmetic keeps C int semantics. The C -1/-2 error results are preserved, with Vec::try_reserve_exact standing in for the bucket-array malloc failure path. Behavior is bit-identical by construction and by measurement (see test plan); runtime on an 11 MB training buffer is within ~5% of the C build end-to-end. Users see no behavioral change: dictionaries trained through ZDICT_trainFromBuffer_legacy() are byte-identical to the C build. The only external difference is that the never-called `divbwt` symbol is no longer defined in the library. Test plan: - cd rust && cargo fmt --check && cargo clippy --all-targets -- -D warnings && cargo test --all-targets && cargo build --release (125 tests pass; new unit tests cover empty/one/two-byte inputs, all-equal bytes, an exact hand-computed "abracadabra" SA, and fixed-seed LCG buffers at 256-, 4-, and 2-symbol alphabets verified against a naive reference sort plus permutation/sorted invariants) - Feature matrix: cargo build --release --no-default-features --features compression,decompression (and decompression-only, compression-only, compression,dict-builder); `divsufsort` is exported only when dict-builder is enabled - make -C tests fuzzer && ./tests/fuzzer -i1 --no-big-tests (includes ZDICT training tests): pass - make -C tests test-rust-lib-smoke: pass - make -C tests test-invalidDictionaries: pass - make -C programs zstd zstd-dictBuilder zstd-small zstd-compress zstd-decompress: build; compress/decompress round-trip verified - Byte-identity vs pristine C build (commit 959e4852): a harness calling ZDICT_trainFromBuffer_legacy() (the only zstd path reaching divsufsort) and divsufsort() directly, linked against both libzstd.a builds, produces byte-identical dictionaries (80,288 B and full 112,640 B capacity) and byte-identical suffix arrays on a 1 MB source set and an 11 MB binary/repetitive set; a differential driver over 148 random and structured buffers (sizes 3..6000, alphabets 1..256, Fibonacci word, sawtooth, 6 KB near-constant) shows zero mismatches. The CLI --train path could not be exercised because the Rust CLI frontend rejects --train in both the pristine and ported builds (a pre-existing migration gap unrelated to this change).
build(rust): add dict-builder cargo feature dimension
2026-07-11 14:21:09 +02:00 · 2026-07-11 09:38:04 +02:00
20 changed files with 3222 additions and 2974 deletions
@@ -34,7 +34,6 @@ install/
 # Build artefacts
 /rust/target/
 /rust/cli/target/
 contrib/linux-kernel/linux/
 projects/
 bin/
@@ -142,6 +142,7 @@ endif()
 set(_zstd_rust_features)
 set(_zstd_rust_compression 0)
 set(_zstd_rust_decompression 0)
 set(_zstd_rust_dictbuilder 0)
 if(ZSTD_BUILD_COMPRESSION)
    list(APPEND _zstd_rust_features compression)
    set(_zstd_rust_compression 1)
@@ -150,6 +151,10 @@ if(ZSTD_BUILD_DECOMPRESSION)
    list(APPEND _zstd_rust_features decompression)
    set(_zstd_rust_decompression 1)
 endif()
 if(ZSTD_BUILD_DICTBUILDER)
    list(APPEND _zstd_rust_features dict-builder)
    set(_zstd_rust_dictbuilder 1)
 endif()
 set(_zstd_rust_huf_mode default)
 if(_zstd_huf_force_x1)
@@ -190,7 +195,7 @@ if(_zstd_rust_features)
 endif()
 set(_zstd_rust_build_config
-    "c${_zstd_rust_compression}-d${_zstd_rust_decompression}-${_zstd_rust_huf_mode}")
+    "c${_zstd_rust_compression}-d${_zstd_rust_decompression}-b${_zstd_rust_dictbuilder}-${_zstd_rust_huf_mode}")
 set(ZSTD_RUST_MANIFEST "${ZSTD_SOURCE_DIR}/rust/Cargo.toml")
 set(ZSTD_RUST_TARGET_DIR
    "${CMAKE_CURRENT_BINARY_DIR}/rust-target/${_zstd_rust_build_config}")
@@ -73,7 +73,9 @@ if rust_huf_force_x1 and rust_huf_force_x2
  error('HUF_FORCE_DECOMPRESS_X1 and HUF_FORCE_DECOMPRESS_X2 are mutually exclusive')
 endif
-rust_features = ['compression', 'decompression']
+# Meson always compiles the dictBuilder sources above, so the Rust archive
 # must always carry the matching dict-builder module set.
 rust_features = ['compression', 'decompression', 'dict-builder']
 rust_huf_mode = 'default'
 rust_huf_c_args = []
 if rust_huf_force_x1
@@ -96,7 +98,7 @@ if rust_target == ''
  endif
 endif
-rust_build_config = 'c1-d1-' + rust_huf_mode
+rust_build_config = 'c1-d1-b1-' + rust_huf_mode
 rust_target_dir = join_paths(meson.current_build_dir(), 'rust-target', rust_build_config)
 is_msvc = cc_id == compiler_msvc or cc_id == 'clang-cl'
 rust_staticlib_name = is_msvc ? 'zstd_rs.lib' : 'libzstd_rs.a'
@@ -84,16 +84,20 @@ endif
 ifneq ($(ZSTD_LIB_DECOMPRESSION),0)
 RUST_CARGO_FEATURES += decompression
 endif
 ifneq ($(ZSTD_LIB_DICTBUILDER),0)
 RUST_CARGO_FEATURES += dict-builder
 endif
 RUST_MODULE_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-b$(ZSTD_LIB_DICTBUILDER)
 RUST_HUF_FEATURE :=
-RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-default
+RUST_BUILD_CONFIG := $(RUST_MODULE_CONFIG)-default
 ifneq ($(RUST_HUF_FORCE_X1),0)
 RUST_HUF_FEATURE := huf-force-decompress-x1
-RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-huf-force-decompress-x1
+RUST_BUILD_CONFIG := $(RUST_MODULE_CONFIG)-huf-force-decompress-x1
 endif
 ifneq ($(RUST_HUF_FORCE_X2),0)
 RUST_HUF_FEATURE := huf-force-decompress-x2
-RUST_BUILD_CONFIG := c$(ZSTD_LIB_COMPRESSION)-d$(ZSTD_LIB_DECOMPRESSION)-huf-force-decompress-x2
+RUST_BUILD_CONFIG := $(RUST_MODULE_CONFIG)-huf-force-decompress-x2
 endif
 ifneq ($(RUST_HUF_FEATURE),)
 ifneq ($(ZSTD_LIB_DECOMPRESSION),0)
@@ -24,1890 +24,9 @@
 * OTHER DEALINGS IN THE SOFTWARE.
 */
-/*- Compiler specifics -*/
+/* divsufsort() is implemented in rust/src/divsufsort.rs, which provides the
-#ifdef __clang__
+ * symbol directly.  This translation unit keeps the header's prototypes in
-#pragma clang diagnostic ignored "-Wshorten-64-to-32"
+ * the build so the dictionary builder continues to compile against the
-#endif
+ * original interface.  divbwt() has no callers in zstd and is declaration-
-
+ * only; it moves to Rust if a user ever appears. */
 #if defined(_MSC_VER)
 #  pragma warning(disable : 4244)
 #  pragma warning(disable : 4127)    /* C4127 : Condition expression is constant */
 #endif
 /*- Dependencies -*/
 #include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include "divsufsort.h"
 /*- Constants -*/
 #if defined(INLINE)
 # undef INLINE
 #endif
 #if !defined(INLINE)
 # define INLINE __inline
 #endif
 #if defined(ALPHABET_SIZE) && (ALPHABET_SIZE < 1)
 # undef ALPHABET_SIZE
 #endif
 #if !defined(ALPHABET_SIZE)
 # define ALPHABET_SIZE (256)
 #endif
 #define BUCKET_A_SIZE (ALPHABET_SIZE)
 #define BUCKET_B_SIZE (ALPHABET_SIZE * ALPHABET_SIZE)
 #if defined(SS_INSERTIONSORT_THRESHOLD)
 # if SS_INSERTIONSORT_THRESHOLD < 1
 #  undef SS_INSERTIONSORT_THRESHOLD
 #  define SS_INSERTIONSORT_THRESHOLD (1)
 # endif
 #else
 # define SS_INSERTIONSORT_THRESHOLD (8)
 #endif
 #if defined(SS_BLOCKSIZE)
 # if SS_BLOCKSIZE < 0
 #  undef SS_BLOCKSIZE
 #  define SS_BLOCKSIZE (0)
 # elif 32768 <= SS_BLOCKSIZE
 #  undef SS_BLOCKSIZE
 #  define SS_BLOCKSIZE (32767)
 # endif
 #else
 # define SS_BLOCKSIZE (1024)
 #endif
 /* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */
 #if SS_BLOCKSIZE == 0
 # define SS_MISORT_STACKSIZE (96)
 #elif SS_BLOCKSIZE <= 4096
 # define SS_MISORT_STACKSIZE (16)
 #else
 # define SS_MISORT_STACKSIZE (24)
 #endif
 #define SS_SMERGE_STACKSIZE (32)
 #define TR_INSERTIONSORT_THRESHOLD (8)
 #define TR_STACKSIZE (64)
 /*- Macros -*/
 #ifndef SWAP
 # define SWAP(_a, _b) do { t = (_a); (_a) = (_b); (_b) = t; } while(0)
 #endif /* SWAP */
 #ifndef MIN
 # define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b))
 #endif /* MIN */
 #ifndef MAX
 # define MAX(_a, _b) (((_a) > (_b)) ? (_a) : (_b))
 #endif /* MAX */
 #define STACK_PUSH(_a, _b, _c, _d)\
  do {\
    assert(ssize < STACK_SIZE);\
    stack[ssize].a = (_a), stack[ssize].b = (_b),\
    stack[ssize].c = (_c), stack[ssize++].d = (_d);\
  } while(0)
 #define STACK_PUSH5(_a, _b, _c, _d, _e)\
  do {\
    assert(ssize < STACK_SIZE);\
    stack[ssize].a = (_a), stack[ssize].b = (_b),\
    stack[ssize].c = (_c), stack[ssize].d = (_d), stack[ssize++].e = (_e);\
  } while(0)
 #define STACK_POP(_a, _b, _c, _d)\
  do {\
    assert(0 <= ssize);\
    if(ssize == 0) { return; }\
    (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\
    (_c) = stack[ssize].c, (_d) = stack[ssize].d;\
  } while(0)
 #define STACK_POP5(_a, _b, _c, _d, _e)\
  do {\
    assert(0 <= ssize);\
    if(ssize == 0) { return; }\
    (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\
    (_c) = stack[ssize].c, (_d) = stack[ssize].d, (_e) = stack[ssize].e;\
  } while(0)
 #define BUCKET_A(_c0) bucket_A[(_c0)]
 #if ALPHABET_SIZE == 256
 #define BUCKET_B(_c0, _c1) (bucket_B[((_c1) << 8) | (_c0)])
 #define BUCKET_BSTAR(_c0, _c1) (bucket_B[((_c0) << 8) | (_c1)])
 #else
 #define BUCKET_B(_c0, _c1) (bucket_B[(_c1) * ALPHABET_SIZE + (_c0)])
 #define BUCKET_BSTAR(_c0, _c1) (bucket_B[(_c0) * ALPHABET_SIZE + (_c1)])
 #endif
 /*- Private Functions -*/
 static const int lg_table[256]= {
 -1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
  5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,
  6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
  6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7
 };
 #if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
 static INLINE
 int
 ss_ilg(int n) {
 #if SS_BLOCKSIZE == 0
  return (n & 0xffff0000) ?
          ((n & 0xff000000) ?
            24 + lg_table[(n >> 24) & 0xff] :
            16 + lg_table[(n >> 16) & 0xff]) :
          ((n & 0x0000ff00) ?
             8 + lg_table[(n >>  8) & 0xff] :
             0 + lg_table[(n >>  0) & 0xff]);
 #elif SS_BLOCKSIZE < 256
  return lg_table[n];
 #else
  return (n & 0xff00) ?
          8 + lg_table[(n >> 8) & 0xff] :
          0 + lg_table[(n >> 0) & 0xff];
 #endif
 }
 #endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */
 #if SS_BLOCKSIZE != 0
 static const int sqq_table[256] = {
  0,  16,  22,  27,  32,  35,  39,  42,  45,  48,  50,  53,  55,  57,  59,  61,
 64,  65,  67,  69,  71,  73,  75,  76,  78,  80,  81,  83,  84,  86,  87,  89,
 90,  91,  93,  94,  96,  97,  98,  99, 101, 102, 103, 104, 106, 107, 108, 109,
 110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126,
 128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142,
 143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155,
 156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168,
 169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180,
 181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191,
 192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201,
 202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211,
 212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221,
 221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230,
 230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238,
 239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247,
 247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255
 };
 static INLINE
 int
 ss_isqrt(int x) {
  int y, e;
  if(x >= (SS_BLOCKSIZE * SS_BLOCKSIZE)) { return SS_BLOCKSIZE; }
  e = (x & 0xffff0000) ?
        ((x & 0xff000000) ?
          24 + lg_table[(x >> 24) & 0xff] :
          16 + lg_table[(x >> 16) & 0xff]) :
        ((x & 0x0000ff00) ?
           8 + lg_table[(x >>  8) & 0xff] :
           0 + lg_table[(x >>  0) & 0xff]);
  if(e >= 16) {
    y = sqq_table[x >> ((e - 6) - (e & 1))] << ((e >> 1) - 7);
    if(e >= 24) { y = (y + 1 + x / y) >> 1; }
    y = (y + 1 + x / y) >> 1;
  } else if(e >= 8) {
    y = (sqq_table[x >> ((e - 6) - (e & 1))] >> (7 - (e >> 1))) + 1;
  } else {
    return sqq_table[x] >> 4;
  }
  return (x < (y * y)) ? y - 1 : y;
 }
 #endif /* SS_BLOCKSIZE != 0 */
 /*---------------------------------------------------------------------------*/
 /* Compares two suffixes. */
 static INLINE
 int
 ss_compare(const unsigned char *T,
           const int *p1, const int *p2,
           int depth) {
  const unsigned char *U1, *U2, *U1n, *U2n;
  for(U1 = T + depth + *p1,
      U2 = T + depth + *p2,
      U1n = T + *(p1 + 1) + 2,
      U2n = T + *(p2 + 1) + 2;
      (U1 < U1n) && (U2 < U2n) && (*U1 == *U2);
      ++U1, ++U2) {
  }
  return U1 < U1n ?
        (U2 < U2n ? *U1 - *U2 : 1) :
        (U2 < U2n ? -1 : 0);
 }
 /*---------------------------------------------------------------------------*/
 #if (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1)
 /* Insertionsort for small size groups */
 static
 void
 ss_insertionsort(const unsigned char *T, const int *PA,
                 int *first, int *last, int depth) {
  int *i, *j;
  int t;
  int r;
  for(i = last - 2; first <= i; --i) {
    for(t = *i, j = i + 1; 0 < (r = ss_compare(T, PA + t, PA + *j, depth));) {
      do { *(j - 1) = *j; } while((++j < last) && (*j < 0));
      if(last <= j) { break; }
    }
    if(r == 0) { *j = ~*j; }
    *(j - 1) = t;
  }
 }
 #endif /* (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) */
 /*---------------------------------------------------------------------------*/
 #if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE)
 static INLINE
 void
 ss_fixdown(const unsigned char *Td, const int *PA,
           int *SA, int i, int size) {
  int j, k;
  int v;
  int c, d, e;
  for(v = SA[i], c = Td[PA[v]]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) {
    d = Td[PA[SA[k = j++]]];
    if(d < (e = Td[PA[SA[j]]])) { k = j; d = e; }
    if(d <= c) { break; }
  }
  SA[i] = v;
 }
 /* Simple top-down heapsort. */
 static
 void
 ss_heapsort(const unsigned char *Td, const int *PA, int *SA, int size) {
  int i, m;
  int t;
  m = size;
  if((size % 2) == 0) {
    m--;
    if(Td[PA[SA[m / 2]]] < Td[PA[SA[m]]]) { SWAP(SA[m], SA[m / 2]); }
  }
  for(i = m / 2 - 1; 0 <= i; --i) { ss_fixdown(Td, PA, SA, i, m); }
  if((size % 2) == 0) { SWAP(SA[0], SA[m]); ss_fixdown(Td, PA, SA, 0, m); }
  for(i = m - 1; 0 < i; --i) {
    t = SA[0], SA[0] = SA[i];
    ss_fixdown(Td, PA, SA, 0, i);
    SA[i] = t;
  }
 }
 /*---------------------------------------------------------------------------*/
 /* Returns the median of three elements. */
 static INLINE
 int *
 ss_median3(const unsigned char *Td, const int *PA,
           int *v1, int *v2, int *v3) {
  int *t;
  if(Td[PA[*v1]] > Td[PA[*v2]]) { SWAP(v1, v2); }
  if(Td[PA[*v2]] > Td[PA[*v3]]) {
    if(Td[PA[*v1]] > Td[PA[*v3]]) { return v1; }
    else { return v3; }
  }
  return v2;
 }
 /* Returns the median of five elements. */
 static INLINE
 int *
 ss_median5(const unsigned char *Td, const int *PA,
           int *v1, int *v2, int *v3, int *v4, int *v5) {
  int *t;
  if(Td[PA[*v2]] > Td[PA[*v3]]) { SWAP(v2, v3); }
  if(Td[PA[*v4]] > Td[PA[*v5]]) { SWAP(v4, v5); }
  if(Td[PA[*v2]] > Td[PA[*v4]]) { SWAP(v2, v4); SWAP(v3, v5); }
  if(Td[PA[*v1]] > Td[PA[*v3]]) { SWAP(v1, v3); }
  if(Td[PA[*v1]] > Td[PA[*v4]]) { SWAP(v1, v4); SWAP(v3, v5); }
  if(Td[PA[*v3]] > Td[PA[*v4]]) { return v4; }
  return v3;
 }
 /* Returns the pivot element. */
 static INLINE
 int *
 ss_pivot(const unsigned char *Td, const int *PA, int *first, int *last) {
  int *middle;
  int t;
  t = last - first;
  middle = first + t / 2;
  if(t <= 512) {
    if(t <= 32) {
      return ss_median3(Td, PA, first, middle, last - 1);
    } else {
      t >>= 2;
      return ss_median5(Td, PA, first, first + t, middle, last - 1 - t, last - 1);
    }
  }
  t >>= 3;
  first  = ss_median3(Td, PA, first, first + t, first + (t << 1));
  middle = ss_median3(Td, PA, middle - t, middle, middle + t);
  last   = ss_median3(Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1);
  return ss_median3(Td, PA, first, middle, last);
 }
 /*---------------------------------------------------------------------------*/
 /* Binary partition for substrings. */
 static INLINE
 int *
 ss_partition(const int *PA,
                    int *first, int *last, int depth) {
  int *a, *b;
  int t;
  for(a = first - 1, b = last;;) {
    for(; (++a < b) && ((PA[*a] + depth) >= (PA[*a + 1] + 1));) { *a = ~*a; }
    for(; (a < --b) && ((PA[*b] + depth) <  (PA[*b + 1] + 1));) { }
    if(b <= a) { break; }
    t = ~*b;
    *b = *a;
    *a = t;
  }
  if(first < a) { *first = ~*first; }
  return a;
 }
 /* Multikey introsort for medium size groups. */
 static
 void
 ss_mintrosort(const unsigned char *T, const int *PA,
              int *first, int *last,
              int depth) {
 #define STACK_SIZE SS_MISORT_STACKSIZE
  struct { int *a, *b, c; int d; } stack[STACK_SIZE];
  const unsigned char *Td;
  int *a, *b, *c, *d, *e, *f;
  int s, t;
  int ssize;
  int limit;
  int v, x = 0;
  for(ssize = 0, limit = ss_ilg(last - first);;) {
    if((last - first) <= SS_INSERTIONSORT_THRESHOLD) {
 #if 1 < SS_INSERTIONSORT_THRESHOLD
      if(1 < (last - first)) { ss_insertionsort(T, PA, first, last, depth); }
 #endif
      STACK_POP(first, last, depth, limit);
      continue;
    }
    Td = T + depth;
    if(limit-- == 0) { ss_heapsort(Td, PA, first, last - first); }
    if(limit < 0) {
      for(a = first + 1, v = Td[PA[*first]]; a < last; ++a) {
        if((x = Td[PA[*a]]) != v) {
          if(1 < (a - first)) { break; }
          v = x;
          first = a;
        }
      }
      if(Td[PA[*first] - 1] < v) {
        first = ss_partition(PA, first, a, depth);
      }
      if((a - first) <= (last - a)) {
        if(1 < (a - first)) {
          STACK_PUSH(a, last, depth, -1);
          last = a, depth += 1, limit = ss_ilg(a - first);
        } else {
          first = a, limit = -1;
        }
      } else {
        if(1 < (last - a)) {
          STACK_PUSH(first, a, depth + 1, ss_ilg(a - first));
          first = a, limit = -1;
        } else {
          last = a, depth += 1, limit = ss_ilg(a - first);
        }
      }
      continue;
    }
    /* choose pivot */
    a = ss_pivot(Td, PA, first, last);
    v = Td[PA[*a]];
    SWAP(*first, *a);
    /* partition */
    for(b = first; (++b < last) && ((x = Td[PA[*b]]) == v);) { }
    if(((a = b) < last) && (x < v)) {
      for(; (++b < last) && ((x = Td[PA[*b]]) <= v);) {
        if(x == v) { SWAP(*b, *a); ++a; }
      }
    }
    for(c = last; (b < --c) && ((x = Td[PA[*c]]) == v);) { }
    if((b < (d = c)) && (x > v)) {
      for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) {
        if(x == v) { SWAP(*c, *d); --d; }
      }
    }
    for(; b < c;) {
      SWAP(*b, *c);
      for(; (++b < c) && ((x = Td[PA[*b]]) <= v);) {
        if(x == v) { SWAP(*b, *a); ++a; }
      }
      for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) {
        if(x == v) { SWAP(*c, *d); --d; }
      }
    }
    if(a <= d) {
      c = b - 1;
      if((s = a - first) > (t = b - a)) { s = t; }
      for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
      if((s = d - c) > (t = last - d - 1)) { s = t; }
      for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
      a = first + (b - a), c = last - (d - c);
      b = (v <= Td[PA[*a] - 1]) ? a : ss_partition(PA, a, c, depth);
      if((a - first) <= (last - c)) {
        if((last - c) <= (c - b)) {
          STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
          STACK_PUSH(c, last, depth, limit);
          last = a;
        } else if((a - first) <= (c - b)) {
          STACK_PUSH(c, last, depth, limit);
          STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
          last = a;
        } else {
          STACK_PUSH(c, last, depth, limit);
          STACK_PUSH(first, a, depth, limit);
          first = b, last = c, depth += 1, limit = ss_ilg(c - b);
        }
      } else {
        if((a - first) <= (c - b)) {
          STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
          STACK_PUSH(first, a, depth, limit);
          first = c;
        } else if((last - c) <= (c - b)) {
          STACK_PUSH(first, a, depth, limit);
          STACK_PUSH(b, c, depth + 1, ss_ilg(c - b));
          first = c;
        } else {
          STACK_PUSH(first, a, depth, limit);
          STACK_PUSH(c, last, depth, limit);
          first = b, last = c, depth += 1, limit = ss_ilg(c - b);
        }
      }
    } else {
      limit += 1;
      if(Td[PA[*first] - 1] < v) {
        first = ss_partition(PA, first, last, depth);
        limit = ss_ilg(last - first);
      }
      depth += 1;
    }
  }
 #undef STACK_SIZE
 }
 #endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */
 /*---------------------------------------------------------------------------*/
 #if SS_BLOCKSIZE != 0
 static INLINE
 void
 ss_blockswap(int *a, int *b, int n) {
  int t;
  for(; 0 < n; --n, ++a, ++b) {
    t = *a, *a = *b, *b = t;
  }
 }
 static INLINE
 void
 ss_rotate(int *first, int *middle, int *last) {
  int *a, *b, t;
  int l, r;
  l = middle - first, r = last - middle;
  for(; (0 < l) && (0 < r);) {
    if(l == r) { ss_blockswap(first, middle, l); break; }
    if(l < r) {
      a = last - 1, b = middle - 1;
      t = *a;
      do {
        *a-- = *b, *b-- = *a;
        if(b < first) {
          *a = t;
          last = a;
          if((r -= l + 1) <= l) { break; }
          a -= 1, b = middle - 1;
          t = *a;
        }
      } while(1);
    } else {
      a = first, b = middle;
      t = *a;
      do {
        *a++ = *b, *b++ = *a;
        if(last <= b) {
          *a = t;
          first = a + 1;
          if((l -= r + 1) <= r) { break; }
          a += 1, b = middle;
          t = *a;
        }
      } while(1);
    }
  }
 }
 /*---------------------------------------------------------------------------*/
 static
 void
 ss_inplacemerge(const unsigned char *T, const int *PA,
                int *first, int *middle, int *last,
                int depth) {
  const int *p;
  int *a, *b;
  int len, half;
  int q, r;
  int x;
  for(;;) {
    if(*(last - 1) < 0) { x = 1; p = PA + ~*(last - 1); }
    else                { x = 0; p = PA +  *(last - 1); }
    for(a = first, len = middle - first, half = len >> 1, r = -1;
        0 < len;
        len = half, half >>= 1) {
      b = a + half;
      q = ss_compare(T, PA + ((0 <= *b) ? *b : ~*b), p, depth);
      if(q < 0) {
        a = b + 1;
        half -= (len & 1) ^ 1;
      } else {
        r = q;
      }
    }
    if(a < middle) {
      if(r == 0) { *a = ~*a; }
      ss_rotate(a, middle, last);
      last -= middle - a;
      middle = a;
      if(first == middle) { break; }
    }
    --last;
    if(x != 0) { while(*--last < 0) { } }
    if(middle == last) { break; }
  }
 }
 /*---------------------------------------------------------------------------*/
 /* Merge-forward with internal buffer. */
 static
 void
 ss_mergeforward(const unsigned char *T, const int *PA,
                int *first, int *middle, int *last,
                int *buf, int depth) {
  int *a, *b, *c, *bufend;
  int t;
  int r;
  bufend = buf + (middle - first) - 1;
  ss_blockswap(buf, first, middle - first);
  for(t = *(a = first), b = buf, c = middle;;) {
    r = ss_compare(T, PA + *b, PA + *c, depth);
    if(r < 0) {
      do {
        *a++ = *b;
        if(bufend <= b) { *bufend = t; return; }
        *b++ = *a;
      } while(*b < 0);
    } else if(r > 0) {
      do {
        *a++ = *c, *c++ = *a;
        if(last <= c) {
          while(b < bufend) { *a++ = *b, *b++ = *a; }
          *a = *b, *b = t;
          return;
        }
      } while(*c < 0);
    } else {
      *c = ~*c;
      do {
        *a++ = *b;
        if(bufend <= b) { *bufend = t; return; }
        *b++ = *a;
      } while(*b < 0);
      do {
        *a++ = *c, *c++ = *a;
        if(last <= c) {
          while(b < bufend) { *a++ = *b, *b++ = *a; }
          *a = *b, *b = t;
          return;
        }
      } while(*c < 0);
    }
  }
 }
 /* Merge-backward with internal buffer. */
 static
 void
 ss_mergebackward(const unsigned char *T, const int *PA,
                 int *first, int *middle, int *last,
                 int *buf, int depth) {
  const int *p1, *p2;
  int *a, *b, *c, *bufend;
  int t;
  int r;
  int x;
  bufend = buf + (last - middle) - 1;
  ss_blockswap(buf, middle, last - middle);
  x = 0;
  if(*bufend < 0)       { p1 = PA + ~*bufend; x |= 1; }
  else                  { p1 = PA +  *bufend; }
  if(*(middle - 1) < 0) { p2 = PA + ~*(middle - 1); x |= 2; }
  else                  { p2 = PA +  *(middle - 1); }
  for(t = *(a = last - 1), b = bufend, c = middle - 1;;) {
    r = ss_compare(T, p1, p2, depth);
    if(0 < r) {
      if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; }
      *a-- = *b;
      if(b <= buf) { *buf = t; break; }
      *b-- = *a;
      if(*b < 0) { p1 = PA + ~*b; x |= 1; }
      else       { p1 = PA +  *b; }
    } else if(r < 0) {
      if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; }
      *a-- = *c, *c-- = *a;
      if(c < first) {
        while(buf < b) { *a-- = *b, *b-- = *a; }
        *a = *b, *b = t;
        break;
      }
      if(*c < 0) { p2 = PA + ~*c; x |= 2; }
      else       { p2 = PA +  *c; }
    } else {
      if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; }
      *a-- = ~*b;
      if(b <= buf) { *buf = t; break; }
      *b-- = *a;
      if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; }
      *a-- = *c, *c-- = *a;
      if(c < first) {
        while(buf < b) { *a-- = *b, *b-- = *a; }
        *a = *b, *b = t;
        break;
      }
      if(*b < 0) { p1 = PA + ~*b; x |= 1; }
      else       { p1 = PA +  *b; }
      if(*c < 0) { p2 = PA + ~*c; x |= 2; }
      else       { p2 = PA +  *c; }
    }
  }
 }
 /* D&C based merge. */
 static
 void
 ss_swapmerge(const unsigned char *T, const int *PA,
             int *first, int *middle, int *last,
             int *buf, int bufsize, int depth) {
 #define STACK_SIZE SS_SMERGE_STACKSIZE
 #define GETIDX(a) ((0 <= (a)) ? (a) : (~(a)))
 #define MERGE_CHECK(a, b, c)\
  do {\
    if(((c) & 1) ||\
       (((c) & 2) && (ss_compare(T, PA + GETIDX(*((a) - 1)), PA + *(a), depth) == 0))) {\
      *(a) = ~*(a);\
    }\
    if(((c) & 4) && ((ss_compare(T, PA + GETIDX(*((b) - 1)), PA + *(b), depth) == 0))) {\
      *(b) = ~*(b);\
    }\
  } while(0)
  struct { int *a, *b, *c; int d; } stack[STACK_SIZE];
  int *l, *r, *lm, *rm;
  int m, len, half;
  int ssize;
  int check, next;
  for(check = 0, ssize = 0;;) {
    if((last - middle) <= bufsize) {
      if((first < middle) && (middle < last)) {
        ss_mergebackward(T, PA, first, middle, last, buf, depth);
      }
      MERGE_CHECK(first, last, check);
      STACK_POP(first, middle, last, check);
      continue;
    }
    if((middle - first) <= bufsize) {
      if(first < middle) {
        ss_mergeforward(T, PA, first, middle, last, buf, depth);
      }
      MERGE_CHECK(first, last, check);
      STACK_POP(first, middle, last, check);
      continue;
    }
    for(m = 0, len = MIN(middle - first, last - middle), half = len >> 1;
        0 < len;
        len = half, half >>= 1) {
      if(ss_compare(T, PA + GETIDX(*(middle + m + half)),
                       PA + GETIDX(*(middle - m - half - 1)), depth) < 0) {
        m += half + 1;
        half -= (len & 1) ^ 1;
      }
    }
    if(0 < m) {
      lm = middle - m, rm = middle + m;
      ss_blockswap(lm, middle, m);
      l = r = middle, next = 0;
      if(rm < last) {
        if(*rm < 0) {
          *rm = ~*rm;
          if(first < lm) { for(; *--l < 0;) { } next |= 4; }
          next |= 1;
        } else if(first < lm) {
          for(; *r < 0; ++r) { }
          next |= 2;
        }
      }
      if((l - first) <= (last - r)) {
        STACK_PUSH(r, rm, last, (next & 3) | (check & 4));
        middle = lm, last = l, check = (check & 3) | (next & 4);
      } else {
        if((next & 2) && (r == middle)) { next ^= 6; }
        STACK_PUSH(first, lm, l, (check & 3) | (next & 4));
        first = r, middle = rm, check = (next & 3) | (check & 4);
      }
    } else {
      if(ss_compare(T, PA + GETIDX(*(middle - 1)), PA + *middle, depth) == 0) {
        *middle = ~*middle;
      }
      MERGE_CHECK(first, last, check);
      STACK_POP(first, middle, last, check);
    }
  }
 #undef STACK_SIZE
 }
 #endif /* SS_BLOCKSIZE != 0 */
 /*---------------------------------------------------------------------------*/
 /* Substring sort */
 static
 void
 sssort(const unsigned char *T, const int *PA,
       int *first, int *last,
       int *buf, int bufsize,
       int depth, int n, int lastsuffix) {
  int *a;
 #if SS_BLOCKSIZE != 0
  int *b, *middle, *curbuf;
  int j, k, curbufsize, limit;
 #endif
  int i;
  if(lastsuffix != 0) { ++first; }
 #if SS_BLOCKSIZE == 0
  ss_mintrosort(T, PA, first, last, depth);
 #else
  if((bufsize < SS_BLOCKSIZE) &&
      (bufsize < (last - first)) &&
      (bufsize < (limit = ss_isqrt(last - first)))) {
    if(SS_BLOCKSIZE < limit) { limit = SS_BLOCKSIZE; }
    buf = middle = last - limit, bufsize = limit;
  } else {
    middle = last, limit = 0;
  }
  for(a = first, i = 0; SS_BLOCKSIZE < (middle - a); a += SS_BLOCKSIZE, ++i) {
 #if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
    ss_mintrosort(T, PA, a, a + SS_BLOCKSIZE, depth);
 #elif 1 < SS_BLOCKSIZE
    ss_insertionsort(T, PA, a, a + SS_BLOCKSIZE, depth);
 #endif
    curbufsize = last - (a + SS_BLOCKSIZE);
    curbuf = a + SS_BLOCKSIZE;
    if(curbufsize <= bufsize) { curbufsize = bufsize, curbuf = buf; }
    for(b = a, k = SS_BLOCKSIZE, j = i; j & 1; b -= k, k <<= 1, j >>= 1) {
      ss_swapmerge(T, PA, b - k, b, b + k, curbuf, curbufsize, depth);
    }
  }
 #if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
  ss_mintrosort(T, PA, a, middle, depth);
 #elif 1 < SS_BLOCKSIZE
  ss_insertionsort(T, PA, a, middle, depth);
 #endif
  for(k = SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) {
    if(i & 1) {
      ss_swapmerge(T, PA, a - k, a, middle, buf, bufsize, depth);
      a -= k;
    }
  }
  if(limit != 0) {
 #if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE
    ss_mintrosort(T, PA, middle, last, depth);
 #elif 1 < SS_BLOCKSIZE
    ss_insertionsort(T, PA, middle, last, depth);
 #endif
    ss_inplacemerge(T, PA, first, middle, last, depth);
  }
 #endif
  if(lastsuffix != 0) {
    /* Insert last type B* suffix. */
    int PAi[2]; PAi[0] = PA[*(first - 1)], PAi[1] = n - 2;
    for(a = first, i = *(first - 1);
        (a < last) && ((*a < 0) || (0 < ss_compare(T, &(PAi[0]), PA + *a, depth)));
        ++a) {
      *(a - 1) = *a;
    }
    *(a - 1) = i;
  }
 }
 /*---------------------------------------------------------------------------*/
 static INLINE
 int
 tr_ilg(int n) {
  return (n & 0xffff0000) ?
          ((n & 0xff000000) ?
            24 + lg_table[(n >> 24) & 0xff] :
            16 + lg_table[(n >> 16) & 0xff]) :
          ((n & 0x0000ff00) ?
             8 + lg_table[(n >>  8) & 0xff] :
             0 + lg_table[(n >>  0) & 0xff]);
 }
 /*---------------------------------------------------------------------------*/
 /* Simple insertionsort for small size groups. */
 static
 void
 tr_insertionsort(const int *ISAd, int *first, int *last) {
  int *a, *b;
  int t, r;
  for(a = first + 1; a < last; ++a) {
    for(t = *a, b = a - 1; 0 > (r = ISAd[t] - ISAd[*b]);) {
      do { *(b + 1) = *b; } while((first <= --b) && (*b < 0));
      if(b < first) { break; }
    }
    if(r == 0) { *b = ~*b; }
    *(b + 1) = t;
  }
 }
 /*---------------------------------------------------------------------------*/
 static INLINE
 void
 tr_fixdown(const int *ISAd, int *SA, int i, int size) {
  int j, k;
  int v;
  int c, d, e;
  for(v = SA[i], c = ISAd[v]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) {
    d = ISAd[SA[k = j++]];
    if(d < (e = ISAd[SA[j]])) { k = j; d = e; }
    if(d <= c) { break; }
  }
  SA[i] = v;
 }
 /* Simple top-down heapsort. */
 static
 void
 tr_heapsort(const int *ISAd, int *SA, int size) {
  int i, m;
  int t;
  m = size;
  if((size % 2) == 0) {
    m--;
    if(ISAd[SA[m / 2]] < ISAd[SA[m]]) { SWAP(SA[m], SA[m / 2]); }
  }
  for(i = m / 2 - 1; 0 <= i; --i) { tr_fixdown(ISAd, SA, i, m); }
  if((size % 2) == 0) { SWAP(SA[0], SA[m]); tr_fixdown(ISAd, SA, 0, m); }
  for(i = m - 1; 0 < i; --i) {
    t = SA[0], SA[0] = SA[i];
    tr_fixdown(ISAd, SA, 0, i);
    SA[i] = t;
  }
 }
 /*---------------------------------------------------------------------------*/
 /* Returns the median of three elements. */
 static INLINE
 int *
 tr_median3(const int *ISAd, int *v1, int *v2, int *v3) {
  int *t;
  if(ISAd[*v1] > ISAd[*v2]) { SWAP(v1, v2); }
  if(ISAd[*v2] > ISAd[*v3]) {
    if(ISAd[*v1] > ISAd[*v3]) { return v1; }
    else { return v3; }
  }
  return v2;
 }
 /* Returns the median of five elements. */
 static INLINE
 int *
 tr_median5(const int *ISAd,
           int *v1, int *v2, int *v3, int *v4, int *v5) {
  int *t;
  if(ISAd[*v2] > ISAd[*v3]) { SWAP(v2, v3); }
  if(ISAd[*v4] > ISAd[*v5]) { SWAP(v4, v5); }
  if(ISAd[*v2] > ISAd[*v4]) { SWAP(v2, v4); SWAP(v3, v5); }
  if(ISAd[*v1] > ISAd[*v3]) { SWAP(v1, v3); }
  if(ISAd[*v1] > ISAd[*v4]) { SWAP(v1, v4); SWAP(v3, v5); }
  if(ISAd[*v3] > ISAd[*v4]) { return v4; }
  return v3;
 }
 /* Returns the pivot element. */
 static INLINE
 int *
 tr_pivot(const int *ISAd, int *first, int *last) {
  int *middle;
  int t;
  t = last - first;
  middle = first + t / 2;
  if(t <= 512) {
    if(t <= 32) {
      return tr_median3(ISAd, first, middle, last - 1);
    } else {
      t >>= 2;
      return tr_median5(ISAd, first, first + t, middle, last - 1 - t, last - 1);
    }
  }
  t >>= 3;
  first  = tr_median3(ISAd, first, first + t, first + (t << 1));
  middle = tr_median3(ISAd, middle - t, middle, middle + t);
  last   = tr_median3(ISAd, last - 1 - (t << 1), last - 1 - t, last - 1);
  return tr_median3(ISAd, first, middle, last);
 }
 /*---------------------------------------------------------------------------*/
 typedef struct _trbudget_t trbudget_t;
 struct _trbudget_t {
  int chance;
  int remain;
  int incval;
  int count;
 };
 static INLINE
 void
 trbudget_init(trbudget_t *budget, int chance, int incval) {
  budget->chance = chance;
  budget->remain = budget->incval = incval;
 }
 static INLINE
 int
 trbudget_check(trbudget_t *budget, int size) {
  if(size <= budget->remain) { budget->remain -= size; return 1; }
  if(budget->chance == 0) { budget->count += size; return 0; }
  budget->remain += budget->incval - size;
  budget->chance -= 1;
  return 1;
 }
 /*---------------------------------------------------------------------------*/
 static INLINE
 void
 tr_partition(const int *ISAd,
             int *first, int *middle, int *last,
             int **pa, int **pb, int v) {
  int *a, *b, *c, *d, *e, *f;
  int t, s;
  int x = 0;
  for(b = middle - 1; (++b < last) && ((x = ISAd[*b]) == v);) { }
  if(((a = b) < last) && (x < v)) {
    for(; (++b < last) && ((x = ISAd[*b]) <= v);) {
      if(x == v) { SWAP(*b, *a); ++a; }
    }
  }
  for(c = last; (b < --c) && ((x = ISAd[*c]) == v);) { }
  if((b < (d = c)) && (x > v)) {
    for(; (b < --c) && ((x = ISAd[*c]) >= v);) {
      if(x == v) { SWAP(*c, *d); --d; }
    }
  }
  for(; b < c;) {
    SWAP(*b, *c);
    for(; (++b < c) && ((x = ISAd[*b]) <= v);) {
      if(x == v) { SWAP(*b, *a); ++a; }
    }
    for(; (b < --c) && ((x = ISAd[*c]) >= v);) {
      if(x == v) { SWAP(*c, *d); --d; }
    }
  }
  if(a <= d) {
    c = b - 1;
    if((s = a - first) > (t = b - a)) { s = t; }
    for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
    if((s = d - c) > (t = last - d - 1)) { s = t; }
    for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); }
    first += (b - a), last -= (d - c);
  }
  *pa = first, *pb = last;
 }
 static
 void
 tr_copy(int *ISA, const int *SA,
        int *first, int *a, int *b, int *last,
        int depth) {
  /* sort suffixes of middle partition
     by using sorted order of suffixes of left and right partition. */
  int *c, *d, *e;
  int s, v;
  v = b - SA - 1;
  for(c = first, d = a - 1; c <= d; ++c) {
    if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
      *++d = s;
      ISA[s] = d - SA;
    }
  }
  for(c = last - 1, e = d + 1, d = b; e < d; --c) {
    if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
      *--d = s;
      ISA[s] = d - SA;
    }
  }
 }
 static
 void
 tr_partialcopy(int *ISA, const int *SA,
               int *first, int *a, int *b, int *last,
               int depth) {
  int *c, *d, *e;
  int s, v;
  int rank, lastrank, newrank = -1;
  v = b - SA - 1;
  lastrank = -1;
  for(c = first, d = a - 1; c <= d; ++c) {
    if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
      *++d = s;
      rank = ISA[s + depth];
      if(lastrank != rank) { lastrank = rank; newrank = d - SA; }
      ISA[s] = newrank;
    }
  }
  lastrank = -1;
  for(e = d; first <= e; --e) {
    rank = ISA[*e];
    if(lastrank != rank) { lastrank = rank; newrank = e - SA; }
    if(newrank != rank) { ISA[*e] = newrank; }
  }
  lastrank = -1;
  for(c = last - 1, e = d + 1, d = b; e < d; --c) {
    if((0 <= (s = *c - depth)) && (ISA[s] == v)) {
      *--d = s;
      rank = ISA[s + depth];
      if(lastrank != rank) { lastrank = rank; newrank = d - SA; }
      ISA[s] = newrank;
    }
  }
 }
 static
 void
 tr_introsort(int *ISA, const int *ISAd,
             int *SA, int *first, int *last,
             trbudget_t *budget) {
 #define STACK_SIZE TR_STACKSIZE
  struct { const int *a; int *b, *c; int d, e; }stack[STACK_SIZE];
  int *a, *b, *c;
  int t;
  int v, x = 0;
  int incr = ISAd - ISA;
  int limit, next;
  int ssize, trlink = -1;
  for(ssize = 0, limit = tr_ilg(last - first);;) {
    if(limit < 0) {
      if(limit == -1) {
        /* tandem repeat partition */
        tr_partition(ISAd - incr, first, first, last, &a, &b, last - SA - 1);
        /* update ranks */
        if(a < last) {
          for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; }
        }
        if(b < last) {
          for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; }
        }
        /* push */
        if(1 < (b - a)) {
          STACK_PUSH5(NULL, a, b, 0, 0);
          STACK_PUSH5(ISAd - incr, first, last, -2, trlink);
          trlink = ssize - 2;
        }
        if((a - first) <= (last - b)) {
          if(1 < (a - first)) {
            STACK_PUSH5(ISAd, b, last, tr_ilg(last - b), trlink);
            last = a, limit = tr_ilg(a - first);
          } else if(1 < (last - b)) {
            first = b, limit = tr_ilg(last - b);
          } else {
            STACK_POP5(ISAd, first, last, limit, trlink);
          }
        } else {
          if(1 < (last - b)) {
            STACK_PUSH5(ISAd, first, a, tr_ilg(a - first), trlink);
            first = b, limit = tr_ilg(last - b);
          } else if(1 < (a - first)) {
            last = a, limit = tr_ilg(a - first);
          } else {
            STACK_POP5(ISAd, first, last, limit, trlink);
          }
        }
      } else if(limit == -2) {
        /* tandem repeat copy */
        a = stack[--ssize].b, b = stack[ssize].c;
        if(stack[ssize].d == 0) {
          tr_copy(ISA, SA, first, a, b, last, ISAd - ISA);
        } else {
          if(0 <= trlink) { stack[trlink].d = -1; }
          tr_partialcopy(ISA, SA, first, a, b, last, ISAd - ISA);
        }
        STACK_POP5(ISAd, first, last, limit, trlink);
      } else {
        /* sorted partition */
        if(0 <= *first) {
          a = first;
          do { ISA[*a] = a - SA; } while((++a < last) && (0 <= *a));
          first = a;
        }
        if(first < last) {
          a = first; do { *a = ~*a; } while(*++a < 0);
          next = (ISA[*a] != ISAd[*a]) ? tr_ilg(a - first + 1) : -1;
          if(++a < last) { for(b = first, v = a - SA - 1; b < a; ++b) { ISA[*b] = v; } }
          /* push */
          if(trbudget_check(budget, a - first)) {
            if((a - first) <= (last - a)) {
              STACK_PUSH5(ISAd, a, last, -3, trlink);
              ISAd += incr, last = a, limit = next;
            } else {
              if(1 < (last - a)) {
                STACK_PUSH5(ISAd + incr, first, a, next, trlink);
                first = a, limit = -3;
              } else {
                ISAd += incr, last = a, limit = next;
              }
            }
          } else {
            if(0 <= trlink) { stack[trlink].d = -1; }
            if(1 < (last - a)) {
              first = a, limit = -3;
            } else {
              STACK_POP5(ISAd, first, last, limit, trlink);
            }
          }
        } else {
          STACK_POP5(ISAd, first, last, limit, trlink);
        }
      }
      continue;
    }
    if((last - first) <= TR_INSERTIONSORT_THRESHOLD) {
      tr_insertionsort(ISAd, first, last);
      limit = -3;
      continue;
    }
    if(limit-- == 0) {
      tr_heapsort(ISAd, first, last - first);
      for(a = last - 1; first < a; a = b) {
        for(x = ISAd[*a], b = a - 1; (first <= b) && (ISAd[*b] == x); --b) { *b = ~*b; }
      }
      limit = -3;
      continue;
    }
    /* choose pivot */
    a = tr_pivot(ISAd, first, last);
    SWAP(*first, *a);
    v = ISAd[*first];
    /* partition */
    tr_partition(ISAd, first, first + 1, last, &a, &b, v);
    if((last - first) != (b - a)) {
      next = (ISA[*a] != v) ? tr_ilg(b - a) : -1;
      /* update ranks */
      for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; }
      if(b < last) { for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } }
      /* push */
      if((1 < (b - a)) && (trbudget_check(budget, b - a))) {
        if((a - first) <= (last - b)) {
          if((last - b) <= (b - a)) {
            if(1 < (a - first)) {
              STACK_PUSH5(ISAd + incr, a, b, next, trlink);
              STACK_PUSH5(ISAd, b, last, limit, trlink);
              last = a;
            } else if(1 < (last - b)) {
              STACK_PUSH5(ISAd + incr, a, b, next, trlink);
              first = b;
            } else {
              ISAd += incr, first = a, last = b, limit = next;
            }
          } else if((a - first) <= (b - a)) {
            if(1 < (a - first)) {
              STACK_PUSH5(ISAd, b, last, limit, trlink);
              STACK_PUSH5(ISAd + incr, a, b, next, trlink);
              last = a;
            } else {
              STACK_PUSH5(ISAd, b, last, limit, trlink);
              ISAd += incr, first = a, last = b, limit = next;
            }
          } else {
            STACK_PUSH5(ISAd, b, last, limit, trlink);
            STACK_PUSH5(ISAd, first, a, limit, trlink);
            ISAd += incr, first = a, last = b, limit = next;
          }
        } else {
          if((a - first) <= (b - a)) {
            if(1 < (last - b)) {
              STACK_PUSH5(ISAd + incr, a, b, next, trlink);
              STACK_PUSH5(ISAd, first, a, limit, trlink);
              first = b;
            } else if(1 < (a - first)) {
              STACK_PUSH5(ISAd + incr, a, b, next, trlink);
              last = a;
            } else {
              ISAd += incr, first = a, last = b, limit = next;
            }
          } else if((last - b) <= (b - a)) {
            if(1 < (last - b)) {
              STACK_PUSH5(ISAd, first, a, limit, trlink);
              STACK_PUSH5(ISAd + incr, a, b, next, trlink);
              first = b;
            } else {
              STACK_PUSH5(ISAd, first, a, limit, trlink);
              ISAd += incr, first = a, last = b, limit = next;
            }
          } else {
            STACK_PUSH5(ISAd, first, a, limit, trlink);
            STACK_PUSH5(ISAd, b, last, limit, trlink);
            ISAd += incr, first = a, last = b, limit = next;
          }
        }
      } else {
        if((1 < (b - a)) && (0 <= trlink)) { stack[trlink].d = -1; }
        if((a - first) <= (last - b)) {
          if(1 < (a - first)) {
            STACK_PUSH5(ISAd, b, last, limit, trlink);
            last = a;
          } else if(1 < (last - b)) {
            first = b;
          } else {
            STACK_POP5(ISAd, first, last, limit, trlink);
          }
        } else {
          if(1 < (last - b)) {
            STACK_PUSH5(ISAd, first, a, limit, trlink);
            first = b;
          } else if(1 < (a - first)) {
            last = a;
          } else {
            STACK_POP5(ISAd, first, last, limit, trlink);
          }
        }
      }
    } else {
      if(trbudget_check(budget, last - first)) {
        limit = tr_ilg(last - first), ISAd += incr;
      } else {
        if(0 <= trlink) { stack[trlink].d = -1; }
        STACK_POP5(ISAd, first, last, limit, trlink);
      }
    }
  }
 #undef STACK_SIZE
 }
 /*---------------------------------------------------------------------------*/
 /* Tandem repeat sort */
 static
 void
 trsort(int *ISA, int *SA, int n, int depth) {
  int *ISAd;
  int *first, *last;
  trbudget_t budget;
  int t, skip, unsorted;
  trbudget_init(&budget, tr_ilg(n) * 2 / 3, n);
 /*  trbudget_init(&budget, tr_ilg(n) * 3 / 4, n); */
  for(ISAd = ISA + depth; -n < *SA; ISAd += ISAd - ISA) {
    first = SA;
    skip = 0;
    unsorted = 0;
    do {
      if((t = *first) < 0) { first -= t; skip += t; }
      else {
        if(skip != 0) { *(first + skip) = skip; skip = 0; }
        last = SA + ISA[t] + 1;
        if(1 < (last - first)) {
          budget.count = 0;
          tr_introsort(ISA, ISAd, SA, first, last, &budget);
          if(budget.count != 0) { unsorted += budget.count; }
          else { skip = first - last; }
        } else if((last - first) == 1) {
          skip = -1;
        }
        first = last;
      }
    } while(first < (SA + n));
    if(skip != 0) { *(first + skip) = skip; }
    if(unsorted == 0) { break; }
  }
 }
 /*---------------------------------------------------------------------------*/
 /* Sorts suffixes of type B*. */
 static
 int
 sort_typeBstar(const unsigned char *T, int *SA,
               int *bucket_A, int *bucket_B,
               int n, int openMP) {
  int *PAb, *ISAb, *buf;
 #ifdef LIBBSC_OPENMP
  int *curbuf;
  int l;
 #endif
  int i, j, k, t, m, bufsize;
  int c0, c1;
 #ifdef LIBBSC_OPENMP
  int d0, d1;
 #endif
  (void)openMP;
  /* Initialize bucket arrays. */
  for(i = 0; i < BUCKET_A_SIZE; ++i) { bucket_A[i] = 0; }
  for(i = 0; i < BUCKET_B_SIZE; ++i) { bucket_B[i] = 0; }
  /* Count the number of occurrences of the first one or two characters of each
     type A, B and B* suffix. Moreover, store the beginning position of all
     type B* suffixes into the array SA. */
  for(i = n - 1, m = n, c0 = T[n - 1]; 0 <= i;) {
    /* type A suffix. */
    do { ++BUCKET_A(c1 = c0); } while((0 <= --i) && ((c0 = T[i]) >= c1));
    if(0 <= i) {
      /* type B* suffix. */
      ++BUCKET_BSTAR(c0, c1);
      SA[--m] = i;
      /* type B suffix. */
      for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) {
        ++BUCKET_B(c0, c1);
      }
    }
  }
  m = n - m;
 /*
 note:
  A type B* suffix is lexicographically smaller than a type B suffix that
  begins with the same first two characters.
 */
  /* Calculate the index of start/end point of each bucket. */
  for(c0 = 0, i = 0, j = 0; c0 < ALPHABET_SIZE; ++c0) {
    t = i + BUCKET_A(c0);
    BUCKET_A(c0) = i + j; /* start point */
    i = t + BUCKET_B(c0, c0);
    for(c1 = c0 + 1; c1 < ALPHABET_SIZE; ++c1) {
      j += BUCKET_BSTAR(c0, c1);
      BUCKET_BSTAR(c0, c1) = j; /* end point */
      i += BUCKET_B(c0, c1);
    }
  }
  if(0 < m) {
    /* Sort the type B* suffixes by their first two characters. */
    PAb = SA + n - m; ISAb = SA + m;
    for(i = m - 2; 0 <= i; --i) {
      t = PAb[i], c0 = T[t], c1 = T[t + 1];
      SA[--BUCKET_BSTAR(c0, c1)] = i;
    }
    t = PAb[m - 1], c0 = T[t], c1 = T[t + 1];
    SA[--BUCKET_BSTAR(c0, c1)] = m - 1;
    /* Sort the type B* substrings using sssort. */
 #ifdef LIBBSC_OPENMP
    if (openMP)
    {
        buf = SA + m;
        c0 = ALPHABET_SIZE - 2, c1 = ALPHABET_SIZE - 1, j = m;
 #pragma omp parallel default(shared) private(bufsize, curbuf, k, l, d0, d1)
        {
          bufsize = (n - (2 * m)) / omp_get_num_threads();
          curbuf = buf + omp_get_thread_num() * bufsize;
          k = 0;
          for(;;) {
            #pragma omp critical(sssort_lock)
            {
              if(0 < (l = j)) {
                d0 = c0, d1 = c1;
                do {
                  k = BUCKET_BSTAR(d0, d1);
                  if(--d1 <= d0) {
                    d1 = ALPHABET_SIZE - 1;
                    if(--d0 < 0) { break; }
                  }
                } while(((l - k) <= 1) && (0 < (l = k)));
                c0 = d0, c1 = d1, j = k;
              }
            }
            if(l == 0) { break; }
            sssort(T, PAb, SA + k, SA + l,
                   curbuf, bufsize, 2, n, *(SA + k) == (m - 1));
          }
        }
    }
    else
    {
        buf = SA + m, bufsize = n - (2 * m);
        for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) {
          for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) {
            i = BUCKET_BSTAR(c0, c1);
            if(1 < (j - i)) {
              sssort(T, PAb, SA + i, SA + j,
                     buf, bufsize, 2, n, *(SA + i) == (m - 1));
            }
          }
        }
    }
 #else
    buf = SA + m, bufsize = n - (2 * m);
    for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) {
      for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) {
        i = BUCKET_BSTAR(c0, c1);
        if(1 < (j - i)) {
          sssort(T, PAb, SA + i, SA + j,
                 buf, bufsize, 2, n, *(SA + i) == (m - 1));
        }
      }
    }
 #endif
    /* Compute ranks of type B* substrings. */
    for(i = m - 1; 0 <= i; --i) {
      if(0 <= SA[i]) {
        j = i;
        do { ISAb[SA[i]] = i; } while((0 <= --i) && (0 <= SA[i]));
        SA[i + 1] = i - j;
        if(i <= 0) { break; }
      }
      j = i;
      do { ISAb[SA[i] = ~SA[i]] = j; } while(SA[--i] < 0);
      ISAb[SA[i]] = j;
    }
    /* Construct the inverse suffix array of type B* suffixes using trsort. */
    trsort(ISAb, SA, m, 1);
    /* Set the sorted order of type B* suffixes. */
    for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) {
      for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { }
      if(0 <= i) {
        t = i;
        for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { }
        SA[ISAb[--j]] = ((t == 0) || (1 < (t - i))) ? t : ~t;
      }
    }
    /* Calculate the index of start/end point of each bucket. */
    BUCKET_B(ALPHABET_SIZE - 1, ALPHABET_SIZE - 1) = n; /* end point */
    for(c0 = ALPHABET_SIZE - 2, k = m - 1; 0 <= c0; --c0) {
      i = BUCKET_A(c0 + 1) - 1;
      for(c1 = ALPHABET_SIZE - 1; c0 < c1; --c1) {
        t = i - BUCKET_B(c0, c1);
        BUCKET_B(c0, c1) = i; /* end point */
        /* Move all type B* suffixes to the correct position. */
        for(i = t, j = BUCKET_BSTAR(c0, c1);
            j <= k;
            --i, --k) { SA[i] = SA[k]; }
      }
      BUCKET_BSTAR(c0, c0 + 1) = i - BUCKET_B(c0, c0) + 1; /* start point */
      BUCKET_B(c0, c0) = i; /* end point */
    }
  }
  return m;
 }
 /* Constructs the suffix array by using the sorted order of type B* suffixes. */
 static
 void
 construct_SA(const unsigned char *T, int *SA,
             int *bucket_A, int *bucket_B,
             int n, int m) {
  int *i, *j, *k;
  int s;
  int c0, c1, c2;
  if(0 < m) {
    /* Construct the sorted order of type B suffixes by using
       the sorted order of type B* suffixes. */
    for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
      /* Scan the suffix array from right to left. */
      for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
          j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
          i <= j;
          --j) {
        if(0 < (s = *j)) {
          assert(T[s] == c1);
          assert(((s + 1) < n) && (T[s] <= T[s + 1]));
          assert(T[s - 1] <= T[s]);
          *j = ~s;
          c0 = T[--s];
          if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
          if(c0 != c2) {
            if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
            k = SA + BUCKET_B(c2 = c0, c1);
          }
          assert(k < j); assert(k != NULL);
          *k-- = s;
        } else {
          assert(((s == 0) && (T[s] == c1)) || (s < 0));
          *j = ~s;
        }
      }
    }
  }
  /* Construct the suffix array by using
     the sorted order of type B suffixes. */
  k = SA + BUCKET_A(c2 = T[n - 1]);
  *k++ = (T[n - 2] < c2) ? ~(n - 1) : (n - 1);
  /* Scan the suffix array from left to right. */
  for(i = SA, j = SA + n; i < j; ++i) {
    if(0 < (s = *i)) {
      assert(T[s - 1] >= T[s]);
      c0 = T[--s];
      if((s == 0) || (T[s - 1] < c0)) { s = ~s; }
      if(c0 != c2) {
        BUCKET_A(c2) = k - SA;
        k = SA + BUCKET_A(c2 = c0);
      }
      assert(i < k);
      *k++ = s;
    } else {
      assert(s < 0);
      *i = ~s;
    }
  }
 }
 /* Constructs the burrows-wheeler transformed string directly
   by using the sorted order of type B* suffixes. */
 static
 int
 construct_BWT(const unsigned char *T, int *SA,
              int *bucket_A, int *bucket_B,
              int n, int m) {
  int *i, *j, *k, *orig;
  int s;
  int c0, c1, c2;
  if(0 < m) {
    /* Construct the sorted order of type B suffixes by using
       the sorted order of type B* suffixes. */
    for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
      /* Scan the suffix array from right to left. */
      for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
          j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
          i <= j;
          --j) {
        if(0 < (s = *j)) {
          assert(T[s] == c1);
          assert(((s + 1) < n) && (T[s] <= T[s + 1]));
          assert(T[s - 1] <= T[s]);
          c0 = T[--s];
          *j = ~((int)c0);
          if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
          if(c0 != c2) {
            if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
            k = SA + BUCKET_B(c2 = c0, c1);
          }
          assert(k < j); assert(k != NULL);
          *k-- = s;
        } else if(s != 0) {
          *j = ~s;
 #ifndef NDEBUG
        } else {
          assert(T[s] == c1);
 #endif
        }
      }
    }
  }
  /* Construct the BWTed string by using
     the sorted order of type B suffixes. */
  k = SA + BUCKET_A(c2 = T[n - 1]);
  *k++ = (T[n - 2] < c2) ? ~((int)T[n - 2]) : (n - 1);
  /* Scan the suffix array from left to right. */
  for(i = SA, j = SA + n, orig = SA; i < j; ++i) {
    if(0 < (s = *i)) {
      assert(T[s - 1] >= T[s]);
      c0 = T[--s];
      *i = c0;
      if((0 < s) && (T[s - 1] < c0)) { s = ~((int)T[s - 1]); }
      if(c0 != c2) {
        BUCKET_A(c2) = k - SA;
        k = SA + BUCKET_A(c2 = c0);
      }
      assert(i < k);
      *k++ = s;
    } else if(s != 0) {
      *i = ~s;
    } else {
      orig = i;
    }
  }
  return orig - SA;
 }
 /* Constructs the burrows-wheeler transformed string directly
   by using the sorted order of type B* suffixes. */
 static
 int
 construct_BWT_indexes(const unsigned char *T, int *SA,
                      int *bucket_A, int *bucket_B,
                      int n, int m,
                      unsigned char * num_indexes, int * indexes) {
  int *i, *j, *k, *orig;
  int s;
  int c0, c1, c2;
  int mod = n / 8;
  {
      mod |= mod >> 1;  mod |= mod >> 2;
      mod |= mod >> 4;  mod |= mod >> 8;
      mod |= mod >> 16; mod >>= 1;
      *num_indexes = (unsigned char)((n - 1) / (mod + 1));
  }
  if(0 < m) {
    /* Construct the sorted order of type B suffixes by using
       the sorted order of type B* suffixes. */
    for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) {
      /* Scan the suffix array from right to left. */
      for(i = SA + BUCKET_BSTAR(c1, c1 + 1),
          j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1;
          i <= j;
          --j) {
        if(0 < (s = *j)) {
          assert(T[s] == c1);
          assert(((s + 1) < n) && (T[s] <= T[s + 1]));
          assert(T[s - 1] <= T[s]);
          if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = j - SA;
          c0 = T[--s];
          *j = ~((int)c0);
          if((0 < s) && (T[s - 1] > c0)) { s = ~s; }
          if(c0 != c2) {
            if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; }
            k = SA + BUCKET_B(c2 = c0, c1);
          }
          assert(k < j); assert(k != NULL);
          *k-- = s;
        } else if(s != 0) {
          *j = ~s;
 #ifndef NDEBUG
        } else {
          assert(T[s] == c1);
 #endif
        }
      }
    }
  }
  /* Construct the BWTed string by using
     the sorted order of type B suffixes. */
  k = SA + BUCKET_A(c2 = T[n - 1]);
  if (T[n - 2] < c2) {
    if (((n - 1) & mod) == 0) indexes[(n - 1) / (mod + 1) - 1] = k - SA;
    *k++ = ~((int)T[n - 2]);
  }
  else {
    *k++ = n - 1;
  }
  /* Scan the suffix array from left to right. */
  for(i = SA, j = SA + n, orig = SA; i < j; ++i) {
    if(0 < (s = *i)) {
      assert(T[s - 1] >= T[s]);
      if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = i - SA;
      c0 = T[--s];
      *i = c0;
      if(c0 != c2) {
        BUCKET_A(c2) = k - SA;
        k = SA + BUCKET_A(c2 = c0);
      }
      assert(i < k);
      if((0 < s) && (T[s - 1] < c0)) {
          if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = k - SA;
          *k++ = ~((int)T[s - 1]);
      } else
        *k++ = s;
    } else if(s != 0) {
      *i = ~s;
    } else {
      orig = i;
    }
  }
  return orig - SA;
 }
 /*---------------------------------------------------------------------------*/
 /*- Function -*/
 int
 divsufsort(const unsigned char *T, int *SA, int n, int openMP) {
  int *bucket_A, *bucket_B;
  int m;
  int err = 0;
  /* Check arguments. */
  if((T == NULL) || (SA == NULL) || (n < 0)) { return -1; }
  else if(n == 0) { return 0; }
  else if(n == 1) { SA[0] = 0; return 0; }
  else if(n == 2) { m = (T[0] < T[1]); SA[m ^ 1] = 0, SA[m] = 1; return 0; }
  bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int));
  bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int));
  /* Suffixsort. */
  if((bucket_A != NULL) && (bucket_B != NULL)) {
    m = sort_typeBstar(T, SA, bucket_A, bucket_B, n, openMP);
    construct_SA(T, SA, bucket_A, bucket_B, n, m);
  } else {
    err = -2;
  }
  free(bucket_B);
  free(bucket_A);
  return err;
 }
 int
 divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP) {
  int *B;
  int *bucket_A, *bucket_B;
  int m, pidx, i;
  /* Check arguments. */
  if((T == NULL) || (U == NULL) || (n < 0)) { return -1; }
  else if(n <= 1) { if(n == 1) { U[0] = T[0]; } return n; }
  if((B = A) == NULL) { B = (int *)malloc((size_t)(n + 1) * sizeof(int)); }
  bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int));
  bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int));
  /* Burrows-Wheeler Transform. */
  if((B != NULL) && (bucket_A != NULL) && (bucket_B != NULL)) {
    m = sort_typeBstar(T, B, bucket_A, bucket_B, n, openMP);
    if (num_indexes == NULL || indexes == NULL) {
        pidx = construct_BWT(T, B, bucket_A, bucket_B, n, m);
    } else {
        pidx = construct_BWT_indexes(T, B, bucket_A, bucket_B, n, m, num_indexes, indexes);
    }
    /* Copy to output string. */
    U[0] = T[n - 1];
    for(i = 0; i < pidx; ++i) { U[i + 1] = (unsigned char)B[i]; }
    for(i += 1; i < n; ++i) { U[i] = (unsigned char)B[i]; }
    pidx += 1;
  } else {
    pidx = -2;
  }
  free(bucket_B);
  free(bucket_A);
  if(A == NULL) { free(B); }
  return pidx;
 }
@@ -32,8 +32,7 @@ RUST_CLI_MANIFEST := $(RUST_CLI_DIR)/Cargo.toml
 RUST_SOURCES := $(RUST_MANIFEST) $(RUST_DIR)/Cargo.lock \
                $(shell find $(RUST_DIR)/src -type f -name '*.rs' -print)
 RUST_CLI_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \
-                    $(RUST_CLI_DIR)/src/lib.rs $(RUST_DIR)/src/zstd_cli.rs \
+                    $(RUST_CLI_DIR)/src/lib.rs $(RUST_DIR)/src/zstd_cli.rs
                    $(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs
 # Keep Rust's HUF implementation in lockstep with libzstd.mk's C selection.
 # Forced modes may arrive as libzstd.mk variables or as direct -D flags in
@@ -72,7 +71,7 @@ RUST_TARGET_32 ?= i686-unknown-linux-gnu
 RUST_STATICLIB_32 := $(RUST_TARGET_DIR)/$(RUST_TARGET_32)/release/libzstd_rs.a
 RUST_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
                    --target-dir $(RUST_TARGET_DIR) --no-default-features
-RUST_CARGO_FLAGS += --features compression,decompression
+RUST_CARGO_FLAGS += --features compression,decompression,dict-builder
 ifneq ($(RUST_HUF_FEATURE),)
 RUST_CARGO_FLAGS += --features $(RUST_HUF_FEATURE)
 endif
@@ -89,7 +88,7 @@ RUST_CLI_STATICLIB := $(RUST_CLI_TARGET_DIR)/release/libzstd_cli_rs.a
 RUST_CLI_STATICLIB_32 := $(RUST_CLI_TARGET_DIR)/$(RUST_TARGET_32)/release/libzstd_cli_rs.a
 RUST_CLI_CARGO_FLAGS := --manifest-path $(RUST_CLI_MANIFEST) --release \
                        --target-dir $(RUST_CLI_TARGET_DIR) \
-                        --no-default-features --features cli,compression,decompression
+                        --no-default-features --features compression,decompression
 $(RUST_CLI_STATICLIB): $(RUST_CLI_SOURCES)
 	$(CARGO) build $(RUST_CLI_CARGO_FLAGS)
@@ -97,7 +96,7 @@ $(RUST_CLI_STATICLIB): $(RUST_CLI_SOURCES)
 $(RUST_CLI_STATICLIB_32): $(RUST_CLI_SOURCES)
 	$(CARGO) build $(RUST_CLI_CARGO_FLAGS) --target $(RUST_TARGET_32)
-RUST_DECOMPRESS_BUILD_CONFIG := lib-c0-d1-$(RUST_BUILD_CONFIG)
+RUST_DECOMPRESS_BUILD_CONFIG := lib-c0-d1-b0-$(RUST_BUILD_CONFIG)
 RUST_DECOMPRESS_TARGET_DIR := $(RUST_DIR)/target/$(RUST_DECOMPRESS_BUILD_CONFIG)
 RUST_DECOMPRESS_STATICLIB := $(RUST_DECOMPRESS_TARGET_DIR)/release/libzstd_rs.a
 RUST_DECOMPRESS_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
@@ -115,12 +114,12 @@ RUST_DECOMPRESS_CLI_TARGET_DIR := $(RUST_DIR)/target/$(RUST_DECOMPRESS_CLI_BUILD
 RUST_DECOMPRESS_CLI_STATICLIB := $(RUST_DECOMPRESS_CLI_TARGET_DIR)/release/libzstd_cli_rs.a
 RUST_DECOMPRESS_CLI_CARGO_FLAGS := --manifest-path $(RUST_CLI_MANIFEST) --release \
                                    --target-dir $(RUST_DECOMPRESS_CLI_TARGET_DIR) \
-                                    --no-default-features --features cli,decompression
+                                    --no-default-features --features decompression
 $(RUST_DECOMPRESS_CLI_STATICLIB): $(RUST_CLI_SOURCES)
 	$(CARGO) build $(RUST_DECOMPRESS_CLI_CARGO_FLAGS)
-RUST_COMPRESS_BUILD_CONFIG := lib-c1-d0-$(RUST_BUILD_CONFIG)
+RUST_COMPRESS_BUILD_CONFIG := lib-c1-d0-b0-$(RUST_BUILD_CONFIG)
 RUST_COMPRESS_TARGET_DIR := $(RUST_DIR)/target/$(RUST_COMPRESS_BUILD_CONFIG)
 RUST_COMPRESS_STATICLIB := $(RUST_COMPRESS_TARGET_DIR)/release/libzstd_rs.a
 RUST_COMPRESS_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
@@ -130,12 +129,23 @@ RUST_COMPRESS_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
 $(RUST_COMPRESS_STATICLIB): $(RUST_SOURCES)
 	$(CARGO) build $(RUST_COMPRESS_CARGO_FLAGS)
 RUST_DICTBUILDER_BUILD_CONFIG := lib-c1-d0-b1-$(RUST_BUILD_CONFIG)
 RUST_DICTBUILDER_TARGET_DIR := $(RUST_DIR)/target/$(RUST_DICTBUILDER_BUILD_CONFIG)
 RUST_DICTBUILDER_STATICLIB := $(RUST_DICTBUILDER_TARGET_DIR)/release/libzstd_rs.a
 RUST_DICTBUILDER_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
                                 --target-dir $(RUST_DICTBUILDER_TARGET_DIR) \
                                 --no-default-features \
                                 --features compression,dict-builder
 $(RUST_DICTBUILDER_STATICLIB): $(RUST_SOURCES)
 	$(CARGO) build $(RUST_DICTBUILDER_CARGO_FLAGS)
 RUST_COMPRESS_CLI_BUILD_CONFIG := cli-c1-d0-$(RUST_BUILD_CONFIG)
 RUST_COMPRESS_CLI_TARGET_DIR := $(RUST_DIR)/target/$(RUST_COMPRESS_CLI_BUILD_CONFIG)
 RUST_COMPRESS_CLI_STATICLIB := $(RUST_COMPRESS_CLI_TARGET_DIR)/release/libzstd_cli_rs.a
 RUST_COMPRESS_CLI_CARGO_FLAGS := --manifest-path $(RUST_CLI_MANIFEST) --release \
                                  --target-dir $(RUST_COMPRESS_CLI_TARGET_DIR) \
-                                  --no-default-features --features cli,compression
+                                  --no-default-features --features compression
 $(RUST_COMPRESS_CLI_STATICLIB): $(RUST_CLI_SOURCES)
 	$(CARGO) build $(RUST_COMPRESS_CLI_CARGO_FLAGS)
@@ -413,7 +423,7 @@ zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c ti
 ## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
 CLEAN += zstd-dictBuilder
-zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c dibio.c $(RUST_COMPRESS_STATICLIB) $(RUST_COMPRESS_CLI_STATICLIB)
+zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c dibio.c $(RUST_DICTBUILDER_STATICLIB) $(RUST_COMPRESS_CLI_STATICLIB)
 	$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
 RUST_DIRECT_LINK_TARGETS := zstd32 zstd-nolegacy zstd-small zstd-frugal \
@@ -8,23 +8,249 @@
 * You may select, at your option, one of the above-listed licenses.
 */
 /* The implementation lives in rust/src/benchfn.rs, built into the Rust CLI
 * static archive. This translation unit stays in the original source lists so
 * build configuration keeps working while the implementation is in Rust. */
-#include <stddef.h>   /* size_t, offsetof */
+
 /* *************************************
 *  Includes
 ***************************************/
 #include <stdlib.h>      /* malloc, free */
 #include <string.h>      /* memset */
 #include <assert.h>      /* assert */
 #include "timefn.h"        /* UTIL_time_t, UTIL_getTime */
 #include "benchfn.h"
-/* BMK_runTime_t and BMK_runOutcome_t are returned by value across the C/Rust
+
- * boundary, and BMK_benchParams_t is passed by value. The Rust #[repr(C)]
+/* *************************************
- * definitions mirror the offsets pinned here. */
+*  Constants
-typedef char BMK_staticAssert_runTimeSumOffset[
+***************************************/
-    (offsetof(BMK_runTime_t, sumOfReturn) == sizeof(double)) ? 1 : -1];
+#define TIMELOOP_MICROSEC     SEC_TO_MICRO      /* 1 second */
-typedef char BMK_staticAssert_outcomeResultOffset[
+#define TIMELOOP_NANOSEC      (1*1000000000ULL) /* 1 second */
-    (offsetof(BMK_runOutcome_t, error_result_never_ever_use_directly)
+
-     == sizeof(BMK_runTime_t)) ? 1 : -1];
+#define KB *(1 <<10)
-typedef char BMK_staticAssert_outcomeTagOffset[
+#define MB *(1 <<20)
-    (offsetof(BMK_runOutcome_t, error_tag_never_ever_use_directly)
+#define GB *(1U<<30)
-     == sizeof(BMK_runTime_t) + sizeof(size_t)) ? 1 : -1];
+
-typedef char BMK_staticAssert_shellAlignment[
+
-    (sizeof(BMK_timedFnState_shell) == BMK_TIMEDFNSTATE_SIZE) ? 1 : -1];
+/* *************************************
 *  Debug errors
 ***************************************/
 #if defined(DEBUG) && (DEBUG >= 1)
 #  include <stdio.h>       /* fprintf */
 #  define DISPLAY(...)       fprintf(stderr, __VA_ARGS__)
 #  define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
 #else
 #  define DEBUGOUTPUT(...)
 #endif
 /* error without displaying */
 #define RETURN_QUIET_ERROR(retValue, ...) {           \
    DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__);    \
    DEBUGOUTPUT("Error : ");                          \
    DEBUGOUTPUT(__VA_ARGS__);                         \
    DEBUGOUTPUT(" \n");                               \
    return retValue;                                  \
 }
 /* Abort execution if a condition is not met */
 #define CONTROL(c) { if (!(c)) { DEBUGOUTPUT("error: %s \n", #c); abort(); } }
 /* *************************************
 *  Benchmarking an arbitrary function
 ***************************************/
 int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome)
 {
    return outcome.error_tag_never_ever_use_directly == 0;
 }
 /* warning : this function will stop program execution if outcome is invalid !
 *           check outcome validity first, using BMK_isValid_runResult() */
 BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome)
 {
    CONTROL(outcome.error_tag_never_ever_use_directly == 0);
    return outcome.internal_never_ever_use_directly;
 }
 size_t BMK_extract_errorResult(BMK_runOutcome_t outcome)
 {
    CONTROL(outcome.error_tag_never_ever_use_directly != 0);
    return outcome.error_result_never_ever_use_directly;
 }
 static BMK_runOutcome_t BMK_runOutcome_error(size_t errorResult)
 {
    BMK_runOutcome_t b;
    memset(&b, 0, sizeof(b));
    b.error_tag_never_ever_use_directly = 1;
    b.error_result_never_ever_use_directly = errorResult;
    return b;
 }
 static BMK_runOutcome_t BMK_setValid_runTime(BMK_runTime_t runTime)
 {
    BMK_runOutcome_t outcome;
    outcome.error_tag_never_ever_use_directly = 0;
    outcome.internal_never_ever_use_directly = runTime;
    return outcome;
 }
 /* initFn will be measured once, benchFn will be measured `nbLoops` times */
 /* initFn is optional, provide NULL if none */
 /* benchFn must return a size_t value that errorFn can interpret */
 /* takes # of blocks and list of size & stuff for each. */
 /* can report result of benchFn for each block into blockResult. */
 /* blockResult is optional, provide NULL if this information is not required */
 /* note : time per loop can be reported as zero if run time < timer resolution */
 BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
                                   unsigned nbLoops)
 {
    nbLoops += !nbLoops;   /* minimum nbLoops is 1 */
    /* init */
    {   size_t i;
        for(i = 0; i < p.blockCount; i++) {
            memset(p.dstBuffers[i], 0xE5, p.dstCapacities[i]);  /* warm up and erase result buffer */
    }   }
    /* benchmark */
    {   size_t dstSize = 0;
        UTIL_time_t const clockStart = UTIL_getTime();
        unsigned loopNb, blockNb;
        if (p.initFn != NULL) p.initFn(p.initPayload);
        for (loopNb = 0; loopNb < nbLoops; loopNb++) {
            for (blockNb = 0; blockNb < p.blockCount; blockNb++) {
                size_t const res = p.benchFn(p.srcBuffers[blockNb], p.srcSizes[blockNb],
                                   p.dstBuffers[blockNb], p.dstCapacities[blockNb],
                                   p.benchPayload);
                if (loopNb == 0) {
                    if (p.blockResults != NULL) p.blockResults[blockNb] = res;
                    if ((p.errorFn != NULL) && (p.errorFn(res))) {
                        RETURN_QUIET_ERROR(BMK_runOutcome_error(res),
                            "Function benchmark failed on block %u (of size %u) with error %i",
                            blockNb, (unsigned)p.srcSizes[blockNb], (int)res);
                    }
                    dstSize += res;
            }   }
        }  /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
        {   PTime const totalTime = UTIL_clockSpanNano(clockStart);
            BMK_runTime_t rt;
            rt.nanoSecPerRun = (double)totalTime / nbLoops;
            rt.sumOfReturn = dstSize;
            return BMK_setValid_runTime(rt);
    }   }
 }
 /* ====  Benchmarking any function, providing intermediate results  ==== */
 struct BMK_timedFnState_s {
    PTime timeSpent_ns;
    PTime timeBudget_ns;
    PTime runBudget_ns;
    BMK_runTime_t fastestRun;
    unsigned nbLoops;
    UTIL_time_t coolTime;
 };  /* typedef'd to BMK_timedFnState_t within bench.h */
 BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms)
 {
    BMK_timedFnState_t* const r = (BMK_timedFnState_t*)malloc(sizeof(*r));
    if (r == NULL) return NULL;   /* malloc() error */
    BMK_resetTimedFnState(r, total_ms, run_ms);
    return r;
 }
 void BMK_freeTimedFnState(BMK_timedFnState_t* state) { free(state); }
 BMK_timedFnState_t*
 BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms)
 {
    typedef char check_size[ 2 * (sizeof(BMK_timedFnState_shell) >= sizeof(struct BMK_timedFnState_s)) - 1];  /* static assert : a compilation failure indicates that BMK_timedFnState_shell is not large enough */
    typedef struct { check_size c; BMK_timedFnState_t tfs; } tfs_align;  /* force tfs to be aligned at its next best position */
    size_t const tfs_alignment = offsetof(tfs_align, tfs); /* provides the minimal alignment restriction for BMK_timedFnState_t */
    BMK_timedFnState_t* const r = (BMK_timedFnState_t*)buffer;
    if (buffer == NULL) return NULL;
    if (size < sizeof(struct BMK_timedFnState_s)) return NULL;
    if ((size_t)buffer % tfs_alignment) return NULL;  /* buffer must be properly aligned */
    BMK_resetTimedFnState(r, total_ms, run_ms);
    return r;
 }
 void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms)
 {
    if (!total_ms) total_ms = 1 ;
    if (!run_ms) run_ms = 1;
    if (run_ms > total_ms) run_ms = total_ms;
    timedFnState->timeSpent_ns = 0;
    timedFnState->timeBudget_ns = (PTime)total_ms * TIMELOOP_NANOSEC / 1000;
    timedFnState->runBudget_ns = (PTime)run_ms * TIMELOOP_NANOSEC / 1000;
    timedFnState->fastestRun.nanoSecPerRun = (double)TIMELOOP_NANOSEC * 2000000000;  /* hopefully large enough : must be larger than any potential measurement */
    timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
    timedFnState->nbLoops = 1;
    timedFnState->coolTime = UTIL_getTime();
 }
 /* Tells if nb of seconds set in timedFnState for all runs is spent.
 * note : this function will return 1 if BMK_benchFunctionTimed() has actually errored. */
 int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState)
 {
    return (timedFnState->timeSpent_ns >= timedFnState->timeBudget_ns);
 }
 #undef MIN
 #define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
 #define MINUSABLETIME  (TIMELOOP_NANOSEC / 2)  /* 0.5 seconds */
 BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
                                  BMK_benchParams_t p)
 {
    PTime const runBudget_ns = cont->runBudget_ns;
    PTime const runTimeMin_ns = runBudget_ns / 2;
    int completed = 0;
    BMK_runTime_t bestRunTime = cont->fastestRun;
    while (!completed) {
        BMK_runOutcome_t const runResult = BMK_benchFunction(p, cont->nbLoops);
        if(!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
            return runResult;
        }
        {   BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
            double const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
            cont->timeSpent_ns += (unsigned long long)loopDuration_ns;
            /* estimate nbLoops for next run to last approximately 1 second */
            if (loopDuration_ns > ((double)runBudget_ns / 50)) {
                double const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
                cont->nbLoops = (unsigned)((double)runBudget_ns / fastestRun_ns) + 1;
            } else {
                /* previous run was too short : blindly increase workload by x multiplier */
                const unsigned multiplier = 10;
                assert(cont->nbLoops < ((unsigned)-1) / multiplier);  /* avoid overflow */
                cont->nbLoops *= multiplier;
            }
            if(loopDuration_ns < (double)runTimeMin_ns) {
                /* don't report results for which benchmark run time was too small : increased risks of rounding errors */
                assert(completed == 0);
                continue;
            } else {
                if(newRunTime.nanoSecPerRun < bestRunTime.nanoSecPerRun) {
                    bestRunTime = newRunTime;
                }
                completed = 1;
            }
        }
    }   /* while (!completed) */
    return BMK_setValid_runTime(bestRunTime);
 }
@@ -8,14 +8,161 @@
 * You may select, at your option, one of the above-listed licenses.
 */
-/* The implementation lives in rust/src/timefn.rs, built into the Rust CLI
+
- * static archive. This translation unit stays in the original source lists so
+/* ===  Dependencies  === */
 * build configuration keeps working while the implementation is in Rust. */
 #include "timefn.h"
 #include "platform.h" /* set _POSIX_C_SOURCE */
 #include <time.h>     /* CLOCK_MONOTONIC, TIME_UTC */
-/* The Rust port mirrors this exact ABI: UTIL_time_t is returned by value and
+/*-****************************************
- * must remain a plain 64-bit nanosecond counter. */
+*  Time functions
-typedef char UTIL_staticAssert_ptimeIs64Bit[(sizeof(PTime) == 8) ? 1 : -1];
+******************************************/
-typedef char UTIL_staticAssert_timeIsPlainCounter[
+
-    (sizeof(UTIL_time_t) == sizeof(PTime)) ? 1 : -1];
+#if defined(_WIN32)   /* Windows */
 #include <windows.h>  /* LARGE_INTEGER */
 #include <stdlib.h>   /* abort */
 #include <stdio.h>    /* perror */
 UTIL_time_t UTIL_getTime(void)
 {
    static LARGE_INTEGER ticksPerSecond;
    static int init = 0;
    if (!init) {
        if (!QueryPerformanceFrequency(&ticksPerSecond)) {
            perror("timefn::QueryPerformanceFrequency");
            abort();
        }
        init = 1;
    }
    {   UTIL_time_t r;
        LARGE_INTEGER x;
        QueryPerformanceCounter(&x);
        r.t = (PTime)(x.QuadPart * 1000000000ULL / ticksPerSecond.QuadPart);
        return r;
    }
 }
 #elif defined(__APPLE__) && defined(__MACH__)
 #include <mach/mach_time.h> /* mach_timebase_info_data_t, mach_timebase_info, mach_absolute_time */
 UTIL_time_t UTIL_getTime(void)
 {
    static mach_timebase_info_data_t rate;
    static int init = 0;
    if (!init) {
        mach_timebase_info(&rate);
        init = 1;
    }
    {   UTIL_time_t r;
        r.t = mach_absolute_time() * (PTime)rate.numer / (PTime)rate.denom;
        return r;
    }
 }
 /* POSIX.1-2001 (optional) */
 #elif defined(CLOCK_MONOTONIC)
 #include <stdlib.h>   /* abort */
 #include <stdio.h>    /* perror */
 UTIL_time_t UTIL_getTime(void)
 {
    /* time must be initialized, othersize it may fail msan test.
     * No good reason, likely a limitation of timespec_get() for some target */
    struct timespec time = { 0, 0 };
    if (clock_gettime(CLOCK_MONOTONIC, &time) != 0) {
        perror("timefn::clock_gettime(CLOCK_MONOTONIC)");
        abort();
    }
    {   UTIL_time_t r;
        r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
        return r;
    }
 }
 /* C11 requires support of timespec_get().
 * However, FreeBSD 11 claims C11 compliance while lacking timespec_get().
 * Double confirm timespec_get() support by checking the definition of TIME_UTC.
 * However, some versions of Android manage to simultaneously define TIME_UTC
 * and lack timespec_get() support... */
 #elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
    && defined(TIME_UTC) && !defined(__ANDROID__)
 #include <stdlib.h>   /* abort */
 #include <stdio.h>    /* perror */
 UTIL_time_t UTIL_getTime(void)
 {
    /* time must be initialized, othersize it may fail msan test.
     * No good reason, likely a limitation of timespec_get() for some target */
    struct timespec time = { 0, 0 };
    if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
        perror("timefn::timespec_get(TIME_UTC)");
        abort();
    }
    {   UTIL_time_t r;
        r.t = (PTime)time.tv_sec * 1000000000ULL + (PTime)time.tv_nsec;
        return r;
    }
 }
 #else   /* relies on standard C90 (note : clock_t produces wrong measurements for multi-threaded workloads) */
 UTIL_time_t UTIL_getTime(void)
 {
    UTIL_time_t r;
    r.t = (PTime)clock() * 1000000000ULL / CLOCKS_PER_SEC;
    return r;
 }
 #define TIME_MT_MEASUREMENTS_NOT_SUPPORTED
 #endif
 /* ==== Common functions, valid for all time API ==== */
 PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
 {
    return clockEnd.t - clockStart.t;
 }
 PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
 {
    return UTIL_getSpanTimeNano(begin, end) / 1000ULL;
 }
 PTime UTIL_clockSpanMicro(UTIL_time_t clockStart )
 {
    UTIL_time_t const clockEnd = UTIL_getTime();
    return UTIL_getSpanTimeMicro(clockStart, clockEnd);
 }
 PTime UTIL_clockSpanNano(UTIL_time_t clockStart )
 {
    UTIL_time_t const clockEnd = UTIL_getTime();
    return UTIL_getSpanTimeNano(clockStart, clockEnd);
 }
 void UTIL_waitForNextTick(void)
 {
    UTIL_time_t const clockStart = UTIL_getTime();
    UTIL_time_t clockEnd;
    do {
        clockEnd = UTIL_getTime();
    } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
 }
 int UTIL_support_MT_measurements(void)
 {
 # if defined(TIME_MT_MEASUREMENTS_NOT_SUPPORTED)
    return 0;
 # else
    return 1;
 # endif
 }
@@ -10,67 +10,16 @@
 /* The CLI parser and control flow live in rust/src/zstd_cli.rs. Keep this
 * translation unit as the stable C entry point used by program launchers. */
 #include <stddef.h>       /* size_t */
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressionParameters */
 #include "../lib/zstd.h"
 #ifndef ZSTD_NOBENCH
 #  include "benchzstd.h"  /* BMK_benchFilesAdvanced, BMK_syntheticTest */
 #endif
 int ZSTD_rust_cli_main(int argCount, const char* const argv[]);
 const char* ZSTD_rust_cli_expected_version(void);
 int ZSTD_rust_cli_bench(const char* const* fileNames, unsigned nbFiles,
                        const char* dictFileName,
                        int startCLevel, int endCLevel,
                        const ZSTD_compressionParameters* compressionParams,
                        int displayLevel, unsigned nbSeconds,
                        size_t blockSize, int nbWorkers);
 const char* ZSTD_rust_cli_expected_version(void)
 {
    return ZSTD_VERSION_STRING;
 }
 /* Benchmark bridge for the Rust CLI. Whether benchmarking exists is a C
 * preprocessor property (ZSTD_NOBENCH), so the decision stays in this shim:
 * the Rust frontend calls in unconditionally, and stripped program variants
 * never reference benchmark symbols.
 * @return the benchmark result code (>= 0), or -1 when unavailable. */
 int ZSTD_rust_cli_bench(const char* const* fileNames, unsigned nbFiles,
                        const char* dictFileName,
                        int startCLevel, int endCLevel,
                        const ZSTD_compressionParameters* compressionParams,
                        int displayLevel, unsigned nbSeconds,
                        size_t blockSize, int nbWorkers)
 {
 #ifndef ZSTD_NOBENCH
    BMK_advancedParams_t advancedParams = BMK_initAdvancedParams();
    int startLevel = startCLevel;
    int endLevel = endCLevel;
    advancedParams.nbSeconds = nbSeconds;
    advancedParams.blockSize = blockSize;
    advancedParams.nbWorkers = nbWorkers;
    if (startLevel > ZSTD_maxCLevel()) startLevel = ZSTD_maxCLevel();
    if (endLevel > ZSTD_maxCLevel()) endLevel = ZSTD_maxCLevel();
    if (endLevel < startLevel) endLevel = startLevel;
    if (nbFiles == 0) {
        /* No input file: benchmark a synthetic sample (lorem generator). */
        return BMK_syntheticTest(-1.0, startLevel, endLevel,
                                 compressionParams, displayLevel,
                                 &advancedParams);
    }
    return BMK_benchFilesAdvanced(fileNames, nbFiles, dictFileName,
                                  startLevel, endLevel,
                                  compressionParams, displayLevel,
                                  &advancedParams);
 #else
    (void)fileNames; (void)nbFiles; (void)dictFileName;
    (void)startCLevel; (void)endCLevel; (void)compressionParams;
    (void)displayLevel; (void)nbSeconds; (void)blockSize; (void)nbWorkers;
    return -1;
 #endif
 }
 int main(int argCount, const char* argv[])
 {
    return ZSTD_rust_cli_main(argCount, argv);
@@ -7,9 +7,10 @@ edition = "2021"
 crate-type = ["staticlib"]
 [features]
-default = ["compression", "decompression"]
+default = ["compression", "decompression", "dict-builder"]
 compression = []
 decompression = []
 dict-builder = []
 huf-force-decompress-x1 = []
 huf-force-decompress-x2 = []
@@ -42,6 +42,10 @@ zstd ABI:
    the dynamic-programming optimal parser itself remains in C for now.
  - `zstd_ldm` implements long-distance-match parameter selection, table
    maintenance, sequence generation, and sequence consumption.
 - Dictionary building
  - `divsufsort` constructs the suffix array that drives the legacy `ZDICT`
    trainer (`ZDICT_trainFromBuffer_legacy`).  The sample analysis and
    dictionary assembly in `zdict.c`, `cover.c`, and `fastcover.c` remain C.
 - Runtime support
  - `threading` provides platform pthread wrappers required by zstd headers.
  - `pool` implements the bounded worker pool used by multithreaded compression.
@@ -59,18 +63,9 @@ zstd ABI:
    so library builds do not acquire program-only dependencies.  The C
    `fileio` backend still owns file opening, safe replacement, sparse writes,
    metadata, and streaming I/O.
  - `timefn` provides the monotonic nanosecond clock behind `UTIL_time_t`,
    and `benchfn` owns the benchmark run/timing loop (`BMK_benchFunction`,
    `BMK_benchTimedFn`) used by the CLI benchmark mode and by C test tools.
    Both live in the `cli/` package, but C test binaries (fullbench, fuzzer,
    zstreamtest, paramgrill, ...) link a helpers-only build of that archive,
    produced without the package's `cli` feature, because the parser layer
    requires the C `fileio` backend that tests do not compile.  Benchmark
    orchestration and reporting (`benchzstd.c`) remain C, reached from the
    Rust parser through the `ZSTD_NOBENCH`-gated bridge in `zstdcli.c`.
-The optimal block matcher, high-level frame compression, dictionary-building,
+The optimal block matcher, high-level frame compression, dictionary-building
-legacy decoding callbacks, benchmark orchestration (`benchzstd`), and the CLI
+except suffix-array construction, legacy decoding callbacks, and the CLI
 file-I/O backend are still C. They must move before the rewrite is complete.
 Keeping that boundary explicit prevents a passing hybrid build from being
 mistaken for the final all-Rust result.
@@ -84,8 +79,9 @@ makefile source list as a small shim so header configuration and platform
 preprocessor behavior stay available during the transition.
 The library, test, and program makefiles select an archive directory for the
-active C configuration: enabled compression/decompression modules, default or
+active C configuration: enabled compression/decompression/dictionary-builder
-forced HUF X1/X2, and the matching Rust target for 32-bit C binaries. The
+modules, default or forced HUF X1/X2, and the matching Rust target for 32-bit
 C binaries. The
 native static archive flattens Rust object members rather than nesting a Rust
 archive, while the native shared library retains all migrated Rust exports.
 When the HUF mode changes, the test and program paths also rebuild cached C
@@ -109,9 +105,8 @@ from `rust/cli` as well:
 ```sh
 cargo clippy --all-targets -- -D warnings
 cargo test --all-targets
-cargo test --no-default-features --features cli,compression --all-targets
+cargo test --no-default-features --features compression --all-targets
-cargo test --no-default-features --features cli,decompression --all-targets
+cargo test --no-default-features --features decompression --all-targets
 cargo test --no-default-features --all-targets
 ```
 Then run original compatibility tests from the repository root, starting with
@@ -2,15 +2,6 @@
 # It is not intended for manual editing.
 version = 4
 [[package]]
 name = "libc"
 version = "0.2.186"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
 [[package]]
 name = "zstd-cli-rs"
 version = "0.1.0"
 dependencies = [
 "libc",
 ]
@@ -7,13 +7,6 @@ edition = "2021"
 crate-type = ["staticlib"]
 [features]
-default = ["cli", "compression", "decompression"]
+default = ["compression", "decompression"]
 # The command-line parser and dispatch layer, which requires the C fileio
 # backend at link time. Program archives enable it; C test binaries link a
 # helpers-only archive (timefn) built without it.
 cli = []
 compression = []
 decompression = []
 [dependencies]
 libc = "0.2"
@@ -1,7 +1,2 @@
 #[path = "../../src/benchfn.rs"]
 mod benchfn;
 #[path = "../../src/timefn.rs"]
 mod timefn;
 #[cfg(feature = "cli")]
 #[path = "../../src/zstd_cli.rs"]
 mod zstd_cli;
@@ -1,575 +0,0 @@
 #![allow(non_camel_case_types)]
 #![allow(non_snake_case)]
 #![allow(clippy::missing_safety_doc)]
 //! Benchmark loop for arbitrary functions over a set of blocks.
 //!
 //! Port of `programs/benchfn.c`.  `BMK_benchFunction` measures one batch of
 //! runs; `BMK_benchTimedFn` repeats batches, growing the loop count until a
 //! run lasts long enough to be reported reliably against `run_ms`, within a
 //! `total_ms` budget tracked by `BMK_timedFnState_t`.
 //!
 //! ABI notes: `BMK_runOutcome_t` and `BMK_runTime_t` are returned by value
 //! across the C boundary and `BMK_benchParams_t` is passed by value, so all
 //! three are `repr(C)` mirrors of the `benchfn.h` layout, pinned by asserts
 //! here and in the C shim.  `BMK_timedFnState_t` is opaque to C, but
 //! `BMK_initStatic_timedFnState` guarantees it fits the 64-byte
 //! `BMK_timedFnState_shell`, and `BMK_createTimedFnState` uses `malloc` so
 //! creation and destruction stay interchangeable with C callers.
 use std::os::raw::{c_int, c_uint, c_void};
 use std::ptr;
 use crate::timefn::{PTime, UTIL_clockSpanNano, UTIL_getTime, UTIL_time_t};
 const TIMELOOP_NANOSEC: PTime = 1_000_000_000;
 /// Valid benchmark result (`BMK_runTime_t` in benchfn.h).
 #[repr(C)]
 #[derive(Clone, Copy, Debug)]
 pub struct BMK_runTime_t {
    /// Time per iteration, over all blocks.
    pub nanoSecPerRun: f64,
    /// Sum of the benchmarked function's return values, first loop only.
    pub sumOfReturn: usize,
 }
 /// Outcome variant of a benchmark run (`BMK_runOutcome_t` in benchfn.h):
 /// either a valid `BMK_runTime_t` or an error result. C callers treat it as
 /// opaque and use the accessor functions below.
 #[repr(C)]
 #[derive(Clone, Copy, Debug)]
 pub struct BMK_runOutcome_t {
    pub internal_never_ever_use_directly: BMK_runTime_t,
    pub error_result_never_ever_use_directly: usize,
    pub error_tag_never_ever_use_directly: c_int,
 }
 // These mirror the static asserts in the programs/benchfn.c shim: the structs
 // cross the ABI by value, so field offsets must match the C header exactly.
 const _: () =
    assert!(std::mem::offset_of!(BMK_runTime_t, sumOfReturn) == std::mem::size_of::<f64>());
 const _: () = assert!(
    std::mem::offset_of!(BMK_runOutcome_t, error_result_never_ever_use_directly)
        == std::mem::size_of::<BMK_runTime_t>()
 );
 const _: () = assert!(
    std::mem::offset_of!(BMK_runOutcome_t, error_tag_never_ever_use_directly)
        == std::mem::size_of::<BMK_runTime_t>() + std::mem::size_of::<usize>()
 );
 /// `size_t (*BMK_benchFn_t)(const void*, size_t, void*, size_t, void*)`
 pub type BMK_benchFn_t = Option<
    unsafe extern "C" fn(
        src: *const c_void,
        srcSize: usize,
        dst: *mut c_void,
        dstCapacity: usize,
        customPayload: *mut c_void,
    ) -> usize,
 >;
 /// `size_t (*BMK_initFn_t)(void*)`
 pub type BMK_initFn_t = Option<unsafe extern "C" fn(initPayload: *mut c_void) -> usize>;
 /// `unsigned (*BMK_errorFn_t)(size_t)`
 pub type BMK_errorFn_t = Option<unsafe extern "C" fn(result: usize) -> c_uint>;
 /// Parameters of `BMK_benchFunction`, passed by value (`BMK_benchParams_t`).
 #[repr(C)]
 #[derive(Clone, Copy)]
 pub struct BMK_benchParams_t {
    pub benchFn: BMK_benchFn_t,
    pub benchPayload: *mut c_void,
    pub initFn: BMK_initFn_t,
    pub initPayload: *mut c_void,
    pub errorFn: BMK_errorFn_t,
    pub blockCount: usize,
    pub srcBuffers: *const *const c_void,
    pub srcSizes: *const usize,
    pub dstBuffers: *const *mut c_void,
    pub dstCapacities: *const usize,
    pub blockResults: *mut usize,
 }
 /// Aborts, like benchfn.c's `CONTROL`, when an accessor is used on the wrong
 /// outcome variant.
 fn control(condition: bool) {
    if !condition {
        std::process::abort();
    }
 }
 fn error_outcome(errorResult: usize) -> BMK_runOutcome_t {
    BMK_runOutcome_t {
        internal_never_ever_use_directly: BMK_runTime_t {
            nanoSecPerRun: 0.0,
            sumOfReturn: 0,
        },
        error_result_never_ever_use_directly: errorResult,
        error_tag_never_ever_use_directly: 1,
    }
 }
 fn valid_outcome(runTime: BMK_runTime_t) -> BMK_runOutcome_t {
    BMK_runOutcome_t {
        internal_never_ever_use_directly: runTime,
        error_result_never_ever_use_directly: 0,
        error_tag_never_ever_use_directly: 0,
    }
 }
 /// Tells if the outcome carries a valid measurement.
 #[no_mangle]
 pub extern "C" fn BMK_isSuccessful_runOutcome(outcome: BMK_runOutcome_t) -> c_int {
    c_int::from(outcome.error_tag_never_ever_use_directly == 0)
 }
 /// Extracts the measurement; aborts if the outcome is an error, so validity
 /// must be checked first with `BMK_isSuccessful_runOutcome`.
 #[no_mangle]
 pub extern "C" fn BMK_extract_runTime(outcome: BMK_runOutcome_t) -> BMK_runTime_t {
    control(outcome.error_tag_never_ever_use_directly == 0);
    outcome.internal_never_ever_use_directly
 }
 /// Extracts the faulty `benchFn` return value; aborts if the outcome is
 /// valid, so failure must be checked first.
 #[no_mangle]
 pub extern "C" fn BMK_extract_errorResult(outcome: BMK_runOutcome_t) -> usize {
    control(outcome.error_tag_never_ever_use_directly != 0);
    outcome.error_result_never_ever_use_directly
 }
 /// Runs `initFn` once, then `benchFn` `nbLoops` times over every block, and
 /// reports the mean time per loop.  On the first loop, per-block results are
 /// stored into `blockResults` (when provided) and checked with `errorFn`
 /// (when provided); the first failing block aborts the measurement and
 /// produces an error outcome carrying the faulty return value.
 #[no_mangle]
 pub unsafe extern "C" fn BMK_benchFunction(
    p: BMK_benchParams_t,
    mut nbLoops: c_uint,
 ) -> BMK_runOutcome_t {
    // Minimum nbLoops is 1.
    nbLoops += c_uint::from(nbLoops == 0);
    // Warm up and erase the result buffers.
    for blockNb in 0..p.blockCount {
        unsafe {
            let dst = *p.dstBuffers.add(blockNb);
            ptr::write_bytes(dst.cast::<u8>(), 0xE5, *p.dstCapacities.add(blockNb));
        }
    }
    let benchFn = p.benchFn.expect("benchFn is mandatory");
    let mut dstSize = 0usize;
    let clockStart = UTIL_getTime();
    if let Some(initFn) = p.initFn {
        unsafe { initFn(p.initPayload) };
    }
    for loopNb in 0..nbLoops {
        for blockNb in 0..p.blockCount {
            let res = unsafe {
                benchFn(
                    *p.srcBuffers.add(blockNb),
                    *p.srcSizes.add(blockNb),
                    *p.dstBuffers.add(blockNb),
                    *p.dstCapacities.add(blockNb),
                    p.benchPayload,
                )
            };
            if loopNb == 0 {
                if !p.blockResults.is_null() {
                    unsafe { *p.blockResults.add(blockNb) = res };
                }
                if let Some(errorFn) = p.errorFn {
                    if unsafe { errorFn(res) } != 0 {
                        return error_outcome(res);
                    }
                }
                dstSize = dstSize.wrapping_add(res);
            }
        }
    }
    let totalTime = UTIL_clockSpanNano(clockStart);
    valid_outcome(BMK_runTime_t {
        nanoSecPerRun: totalTime as f64 / f64::from(nbLoops),
        sumOfReturn: dstSize,
    })
 }
 /// Benchmark session state (`struct BMK_timedFnState_s`), opaque to C.
 #[repr(C)]
 pub struct BMK_timedFnState_t {
    timeSpent_ns: PTime,
    timeBudget_ns: PTime,
    runBudget_ns: PTime,
    fastestRun: BMK_runTime_t,
    nbLoops: c_uint,
    coolTime: UTIL_time_t,
 }
 /// `BMK_TIMEDFNSTATE_SIZE` in benchfn.h: capacity of the caller-provided
 /// `BMK_timedFnState_shell`, which the state must always fit.
 const BMK_TIMEDFNSTATE_SIZE: usize = 64;
 const _: () = assert!(std::mem::size_of::<BMK_timedFnState_t>() <= BMK_TIMEDFNSTATE_SIZE);
 // The shell aligns via a `long long` member; the state must not need more.
 const _: () = assert!(std::mem::align_of::<BMK_timedFnState_t>() <= std::mem::align_of::<u64>());
 /// Allocates and initializes a benchmark session lasting a minimum of
 /// `total_ms`, paced at intervals of approximately `run_ms`.  Uses `malloc`
 /// so ownership stays interchangeable with the original C implementation.
 #[no_mangle]
 pub extern "C" fn BMK_createTimedFnState(
    total_ms: c_uint,
    run_ms: c_uint,
 ) -> *mut BMK_timedFnState_t {
    let state = unsafe { libc::malloc(std::mem::size_of::<BMK_timedFnState_t>()) }
        .cast::<BMK_timedFnState_t>();
    if state.is_null() {
        return ptr::null_mut();
    }
    unsafe { BMK_resetTimedFnState(state, total_ms, run_ms) };
    state
 }
 /// Releases a state obtained from `BMK_createTimedFnState`.
 #[no_mangle]
 pub unsafe extern "C" fn BMK_freeTimedFnState(state: *mut BMK_timedFnState_t) {
    unsafe { libc::free(state.cast()) };
 }
 /// Places the session state into a caller-provided buffer, typically a
 /// `BMK_timedFnState_shell`.  Returns NULL when the buffer is missing, too
 /// small, or misaligned.
 #[no_mangle]
 pub unsafe extern "C" fn BMK_initStatic_timedFnState(
    buffer: *mut c_void,
    size: usize,
    total_ms: c_uint,
    run_ms: c_uint,
 ) -> *mut BMK_timedFnState_t {
    if buffer.is_null() {
        return ptr::null_mut();
    }
    if size < std::mem::size_of::<BMK_timedFnState_t>() {
        return ptr::null_mut();
    }
    if !(buffer as usize).is_multiple_of(std::mem::align_of::<BMK_timedFnState_t>()) {
        return ptr::null_mut();
    }
    let state = buffer.cast::<BMK_timedFnState_t>();
    unsafe { BMK_resetTimedFnState(state, total_ms, run_ms) };
    state
 }
 /// Re-arms a session for a new benchmark of `total_ms`, paced at `run_ms`.
 #[no_mangle]
 pub unsafe extern "C" fn BMK_resetTimedFnState(
    timedFnState: *mut BMK_timedFnState_t,
    total_ms: c_uint,
    run_ms: c_uint,
 ) {
    let total_ms = if total_ms == 0 { 1 } else { total_ms };
    let mut run_ms = if run_ms == 0 { 1 } else { run_ms };
    if run_ms > total_ms {
        run_ms = total_ms;
    }
    let state = BMK_timedFnState_t {
        timeSpent_ns: 0,
        timeBudget_ns: PTime::from(total_ms) * TIMELOOP_NANOSEC / 1000,
        runBudget_ns: PTime::from(run_ms) * TIMELOOP_NANOSEC / 1000,
        fastestRun: BMK_runTime_t {
            // Must be larger than any potential measurement.
            nanoSecPerRun: TIMELOOP_NANOSEC as f64 * 2_000_000_000.0,
            sumOfReturn: usize::MAX,
        },
        nbLoops: 1,
        coolTime: UTIL_getTime(),
    };
    unsafe { timedFnState.write(state) };
 }
 /// Tells if the total time budget of the session is spent.  Also reports 1
 /// after `BMK_benchTimedFn` returned an error.
 #[no_mangle]
 pub unsafe extern "C" fn BMK_isCompleted_TimedFn(timedFnState: *const BMK_timedFnState_t) -> c_int {
    let state = unsafe { &*timedFnState };
    c_int::from(state.timeSpent_ns >= state.timeBudget_ns)
 }
 /// Runs one measurement supposed to last about `run_ms`, automatically
 /// scaling `nbLoops`.  Runs shorter than half the run budget are re-tried
 /// with a larger workload instead of being reported, limiting rounding-error
 /// risks; the best (fastest) qualifying run is returned.
 #[no_mangle]
 pub unsafe extern "C" fn BMK_benchTimedFn(
    cont: *mut BMK_timedFnState_t,
    p: BMK_benchParams_t,
 ) -> BMK_runOutcome_t {
    let cont = unsafe { &mut *cont };
    let runBudget_ns = cont.runBudget_ns;
    let runTimeMin_ns = runBudget_ns / 2;
    let mut bestRunTime = cont.fastestRun;
    loop {
        let runResult = unsafe { BMK_benchFunction(p, cont.nbLoops) };
        if BMK_isSuccessful_runOutcome(runResult) == 0 {
            // Error: move out.
            return runResult;
        }
        let newRunTime = BMK_extract_runTime(runResult);
        let loopDuration_ns = newRunTime.nanoSecPerRun * f64::from(cont.nbLoops);
        cont.timeSpent_ns = cont.timeSpent_ns.wrapping_add(loopDuration_ns as PTime);
        // Estimate nbLoops for the next run to last approximately run_ms.
        if loopDuration_ns > runBudget_ns as f64 / 50.0 {
            let fastestRun_ns = bestRunTime.nanoSecPerRun.min(newRunTime.nanoSecPerRun);
            cont.nbLoops = ((runBudget_ns as f64 / fastestRun_ns) as c_uint).wrapping_add(1);
        } else {
            // Previous run was too short: blindly increase workload by a
            // x10 multiplier.
            const MULTIPLIER: c_uint = 10;
            debug_assert!(cont.nbLoops < c_uint::MAX / MULTIPLIER); // avoid overflow
            cont.nbLoops = cont.nbLoops.wrapping_mul(MULTIPLIER);
        }
        if loopDuration_ns < runTimeMin_ns as f64 {
            // Don't report results when the run time was too small, which
            // increases the risk of rounding errors.
            continue;
        }
        if newRunTime.nanoSecPerRun < bestRunTime.nanoSecPerRun {
            bestRunTime = newRunTime;
        }
        return valid_outcome(bestRunTime);
    }
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    /// Test payload observed through `benchPayload`/`initPayload` pointers.
    #[derive(Default)]
    struct CallLog {
        bench_calls: usize,
        init_calls: usize,
    }
    /// Counts invocations and reports `srcSize`, like a size-preserving codec.
    unsafe extern "C" fn counting_bench_fn(
        _src: *const c_void,
        srcSize: usize,
        _dst: *mut c_void,
        _dstCapacity: usize,
        payload: *mut c_void,
    ) -> usize {
        let log = unsafe { &mut *payload.cast::<CallLog>() };
        log.bench_calls += 1;
        srcSize
    }
    unsafe extern "C" fn counting_init_fn(payload: *mut c_void) -> usize {
        let log = unsafe { &mut *payload.cast::<CallLog>() };
        log.init_calls += 1;
        0
    }
    /// Flags results of 5 bytes and above as errors.
    unsafe extern "C" fn error_on_5(result: usize) -> c_uint {
        c_uint::from(result >= 5)
    }
    struct Fixture {
        srcs: Vec<Vec<u8>>,
        dsts: Vec<Vec<u8>>,
        src_ptrs: Vec<*const c_void>,
        src_sizes: Vec<usize>,
        dst_ptrs: Vec<*mut c_void>,
        dst_capacities: Vec<usize>,
        block_results: Vec<usize>,
        log: CallLog,
    }
    impl Fixture {
        fn new(block_sizes: &[usize]) -> Box<Self> {
            let srcs: Vec<Vec<u8>> = block_sizes.iter().map(|size| vec![0u8; *size]).collect();
            let mut dsts: Vec<Vec<u8>> = block_sizes.iter().map(|size| vec![0u8; *size]).collect();
            let src_ptrs = srcs.iter().map(|src| src.as_ptr().cast()).collect();
            let src_sizes = srcs.iter().map(Vec::len).collect();
            let dst_ptrs = dsts.iter_mut().map(|dst| dst.as_mut_ptr().cast()).collect();
            let dst_capacities = dsts.iter().map(Vec::len).collect();
            let block_results = vec![0usize; block_sizes.len()];
            Box::new(Self {
                srcs,
                dsts,
                src_ptrs,
                src_sizes,
                dst_ptrs,
                dst_capacities,
                block_results,
                log: CallLog::default(),
            })
        }
        fn params(&mut self, errorFn: BMK_errorFn_t) -> BMK_benchParams_t {
            let payload: *mut CallLog = &mut self.log;
            BMK_benchParams_t {
                benchFn: Some(counting_bench_fn),
                benchPayload: payload.cast(),
                initFn: Some(counting_init_fn),
                initPayload: payload.cast(),
                errorFn,
                blockCount: self.srcs.len(),
                srcBuffers: self.src_ptrs.as_ptr(),
                srcSizes: self.src_sizes.as_ptr(),
                dstBuffers: self.dst_ptrs.as_ptr(),
                dstCapacities: self.dst_capacities.as_ptr(),
                blockResults: self.block_results.as_mut_ptr(),
            }
        }
    }
    #[test]
    fn bench_function_accounts_loops_blocks_and_first_loop_results() {
        let mut fixture = Fixture::new(&[3, 8]);
        let params = fixture.params(None);
        let outcome = unsafe { BMK_benchFunction(params, 4) };
        assert_eq!(BMK_isSuccessful_runOutcome(outcome), 1);
        let run_time = BMK_extract_runTime(outcome);
        // benchFn ran nbLoops times over each block; initFn ran once.
        assert_eq!(fixture.log.bench_calls, 4 * 2);
        assert_eq!(fixture.log.init_calls, 1);
        // sumOfReturn and blockResults reflect the first loop only.
        assert_eq!(run_time.sumOfReturn, 3 + 8);
        assert_eq!(fixture.block_results, vec![3, 8]);
        assert!(run_time.nanoSecPerRun >= 0.0);
    }
    #[test]
    fn bench_function_treats_zero_loops_as_one_and_warms_up_buffers() {
        let mut fixture = Fixture::new(&[4]);
        let params = fixture.params(None);
        let outcome = unsafe { BMK_benchFunction(params, 0) };
        assert_eq!(BMK_isSuccessful_runOutcome(outcome), 1);
        assert_eq!(fixture.log.bench_calls, 1);
        // The result buffer was erased with the 0xE5 warm-up pattern.
        assert_eq!(fixture.dsts[0], vec![0xE5; 4]);
    }
    #[test]
    fn bench_function_reports_the_first_failing_block() {
        let mut fixture = Fixture::new(&[3, 5, 7]);
        let params = fixture.params(Some(error_on_5));
        let outcome = unsafe { BMK_benchFunction(params, 10) };
        assert_eq!(BMK_isSuccessful_runOutcome(outcome), 0);
        assert_eq!(BMK_extract_errorResult(outcome), 5);
        // Execution stopped at the failing block, before the third one.
        assert_eq!(fixture.log.bench_calls, 2);
        // blockResults were recorded up to and including the failure.
        assert_eq!(fixture.block_results[..2], [3, 5]);
    }
    #[test]
    fn reset_clamps_budgets_and_rearms_the_loop_counter() {
        let state = BMK_createTimedFnState(0, 7);
        assert!(!state.is_null());
        {
            let state = unsafe { &*state };
            // total_ms 0 becomes 1ms, and run_ms is clamped to total_ms.
            assert_eq!(state.timeBudget_ns, 1_000_000);
            assert_eq!(state.runBudget_ns, 1_000_000);
            assert_eq!(state.nbLoops, 1);
            assert_eq!(state.timeSpent_ns, 0);
            assert_eq!(state.fastestRun.sumOfReturn, usize::MAX);
        }
        assert_eq!(unsafe { BMK_isCompleted_TimedFn(state) }, 0);
        unsafe { BMK_resetTimedFnState(state, 2_000, 500) };
        {
            let state = unsafe { &*state };
            assert_eq!(state.timeBudget_ns, 2_000_000_000);
            assert_eq!(state.runBudget_ns, 500_000_000);
        }
        unsafe { BMK_freeTimedFnState(state) };
    }
    #[test]
    fn static_state_initialization_validates_its_buffer() {
        let mut shell = [0u64; BMK_TIMEDFNSTATE_SIZE / 8];
        let buffer: *mut c_void = shell.as_mut_ptr().cast();
        // A properly sized and aligned buffer is accepted.
        let state = unsafe { BMK_initStatic_timedFnState(buffer, 64, 1_000, 100) };
        assert!(!state.is_null());
        assert_eq!(unsafe { BMK_isCompleted_TimedFn(state) }, 0);
        // NULL, undersized, and misaligned buffers are rejected.
        let too_small = std::mem::size_of::<BMK_timedFnState_t>() - 1;
        unsafe {
            assert!(BMK_initStatic_timedFnState(ptr::null_mut(), 64, 1, 1).is_null());
            assert!(BMK_initStatic_timedFnState(buffer, too_small, 1, 1).is_null());
            assert!(
                BMK_initStatic_timedFnState(buffer.cast::<u8>().add(1).cast(), 63, 1, 1).is_null()
            );
        }
    }
    #[test]
    fn timed_runs_grow_the_workload_and_spend_the_budget() {
        let mut fixture = Fixture::new(&[16]);
        let params = fixture.params(None);
        let state = BMK_createTimedFnState(4, 2);
        assert!(!state.is_null());
        let mut rounds = 0usize;
        while unsafe { BMK_isCompleted_TimedFn(state) } == 0 {
            let outcome = unsafe { BMK_benchTimedFn(state, params) };
            assert_eq!(BMK_isSuccessful_runOutcome(outcome), 1);
            let run_time = BMK_extract_runTime(outcome);
            assert_eq!(run_time.sumOfReturn, 16);
            rounds += 1;
            assert!(rounds < 1_000, "the time budget must eventually be spent");
        }
        {
            let state = unsafe { &*state };
            // A reported run had to last at least runBudget/2, which is only
            // reachable for this trivial function with a grown loop counter.
            assert!(state.nbLoops > 1);
            assert!(state.timeSpent_ns >= state.timeBudget_ns);
        }
        // Every reported outcome came from a run of >= runBudget/2, and the
        // budget accounting matches BMK_isCompleted_TimedFn.
        assert!(rounds >= 1);
        assert!(fixture.log.bench_calls >= rounds);
        unsafe { BMK_freeTimedFnState(state) };
    }
    #[test]
    fn timed_runs_propagate_errors_without_aborting() {
        let mut fixture = Fixture::new(&[9]);
        let params = fixture.params(Some(error_on_5));
        let state = BMK_createTimedFnState(1_000, 100);
        let outcome = unsafe { BMK_benchTimedFn(state, params) };
        assert_eq!(BMK_isSuccessful_runOutcome(outcome), 0);
        assert_eq!(BMK_extract_errorResult(outcome), 9);
        unsafe { BMK_freeTimedFnState(state) };
    }
 }
@@ -0,0 +1,2756 @@
 #![allow(clippy::missing_safety_doc)]
 #![allow(clippy::too_many_arguments)]
 //! Suffix-array construction for the dictionary builder.
 //!
 //! Port of `lib/dictBuilder/divsufsort.c` (libdivsufsort-lite, Copyright (c)
 //! 2003-2008 Yuta Mori, MIT license) in the exact configuration zstd compiles
 //! it with: `ALPHABET_SIZE = 256`, `SS_INSERTIONSORT_THRESHOLD = 8`,
 //! `SS_BLOCKSIZE = 1024`, and no OpenMP.  Only `divsufsort()` is exported;
 //! `divbwt()` has no callers anywhere in zstd and was not ported.
 //!
 //! The C implementation walks raw `int*` cursors through the caller's SA
 //! buffer, including transient one-before-the-range positions.  Every such
 //! cursor is translated to an `isize` index into one `&mut [i32]` slice
 //! covering the whole buffer, so all arithmetic — including the
 //! bitwise-complement rank marking and the C `int` value semantics — matches
 //! the original exactly while staying bounds-checked.
 use std::os::raw::c_int;
 use std::slice;
 const BUCKET_A_SIZE: usize = 256; /* ALPHABET_SIZE */
 const BUCKET_B_SIZE: usize = 256 * 256; /* ALPHABET_SIZE * ALPHABET_SIZE */
 const ALPHABET_SIZE: i32 = 256;
 const SS_INSERTIONSORT_THRESHOLD: isize = 8;
 const SS_BLOCKSIZE: isize = 1024;
 /* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */
 const SS_MISORT_STACKSIZE: usize = 16;
 const SS_SMERGE_STACKSIZE: usize = 32;
 const TR_INSERTIONSORT_THRESHOLD: isize = 8;
 const TR_STACKSIZE: usize = 64;
 #[rustfmt::skip]
 static LG_TABLE: [i32; 256] = [
 -1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,
  5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,
  6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
  6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,
  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
 ];
 #[rustfmt::skip]
 static SQQ_TABLE: [i32; 256] = [
  0,  16,  22,  27,  32,  35,  39,  42,  45,  48,  50,  53,  55,  57,  59,  61,
 64,  65,  67,  69,  71,  73,  75,  76,  78,  80,  81,  83,  84,  86,  87,  89,
 90,  91,  93,  94,  96,  97,  98,  99, 101, 102, 103, 104, 106, 107, 108, 109,
 110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126,
 128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142,
 143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155,
 156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168,
 169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180,
 181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191,
 192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201,
 202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211,
 212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221,
 221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230,
 230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238,
 239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247,
 247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255,
 ];
 /* `ss_ilg` in its `256 <= SS_BLOCKSIZE` configuration. */
 #[inline]
 fn ss_ilg(n: isize) -> i32 {
    let n = n as i32;
    if n & 0xff00 != 0 {
        8 + LG_TABLE[((n >> 8) & 0xff) as usize]
    } else {
        LG_TABLE[(n & 0xff) as usize]
    }
 }
 #[inline]
 fn ss_isqrt(x: isize) -> isize {
    if x >= SS_BLOCKSIZE * SS_BLOCKSIZE {
        return SS_BLOCKSIZE;
    }
    let x = x as i32;
    let e = if (x as u32) & 0xffff_0000 != 0 {
        if (x as u32) & 0xff00_0000 != 0 {
            24 + LG_TABLE[((x >> 24) & 0xff) as usize]
        } else {
            16 + LG_TABLE[((x >> 16) & 0xff) as usize]
        }
    } else if x & 0xff00 != 0 {
        8 + LG_TABLE[((x >> 8) & 0xff) as usize]
    } else {
        LG_TABLE[(x & 0xff) as usize]
    };
    let mut y;
    if e >= 16 {
        y = SQQ_TABLE[(x >> ((e - 6) - (e & 1))) as usize] << ((e >> 1) - 7);
        if e >= 24 {
            y = (y + 1 + x / y) >> 1;
        }
        y = (y + 1 + x / y) >> 1;
    } else if e >= 8 {
        y = (SQQ_TABLE[(x >> ((e - 6) - (e & 1))) as usize] >> (7 - (e >> 1))) + 1;
    } else {
        return (SQQ_TABLE[x as usize] >> 4) as isize;
    }
    (if x < y * y { y - 1 } else { y }) as isize
 }
 /* --------------------------------------------------------------------- */
 /// Compares two suffixes.  `(p10, p11)` and `(p20, p21)` are the `p[0]`/`p[1]`
 /// pairs the C routine reads through its `const int*` arguments; passing the
 /// values directly also serves `sssort()`'s local two-element `PAi` array.
 #[inline]
 fn ss_compare(t: &[u8], p10: i32, p11: i32, p20: i32, p21: i32, depth: i32) -> i32 {
    let mut u1 = (depth + p10) as isize;
    let mut u2 = (depth + p20) as isize;
    let u1n = (p11 + 2) as isize;
    let u2n = (p21 + 2) as isize;
    while u1 < u1n && u2 < u2n && t[u1 as usize] == t[u2 as usize] {
        u1 += 1;
        u2 += 1;
    }
    if u1 < u1n {
        if u2 < u2n {
            t[u1 as usize] as i32 - t[u2 as usize] as i32
        } else {
            1
        }
    } else if u2 < u2n {
        -1
    } else {
        0
    }
 }
 /// `ss_compare(T, p1, p2, depth)` for pointers `p1`/`p2` into the SA buffer.
 #[inline]
 fn ss_compare_pa(t: &[u8], sa: &[i32], p1: isize, p2: isize, depth: i32) -> i32 {
    ss_compare(
        t,
        sa[p1 as usize],
        sa[(p1 + 1) as usize],
        sa[p2 as usize],
        sa[(p2 + 1) as usize],
        depth,
    )
 }
 /* --------------------------------------------------------------------- */
 /* Insertionsort for small size groups */
 fn ss_insertionsort(t: &[u8], sa: &mut [i32], pa: isize, first: isize, last: isize, depth: i32) {
    let mut i = last - 2;
    while first <= i {
        let t0 = sa[i as usize];
        let mut j = i + 1;
        let mut r;
        loop {
            r = ss_compare_pa(t, sa, pa + t0 as isize, pa + sa[j as usize] as isize, depth);
            if r <= 0 {
                break;
            }
            loop {
                sa[(j - 1) as usize] = sa[j as usize];
                j += 1;
                if !(j < last && sa[j as usize] < 0) {
                    break;
                }
            }
            if last <= j {
                break;
            }
        }
        if r == 0 {
            sa[j as usize] = !sa[j as usize];
        }
        sa[(j - 1) as usize] = t0;
        i -= 1;
    }
 }
 /* --------------------------------------------------------------------- */
 /// `Td[PA[SA[p]]]` — the depth-`td` sorting key of the suffix stored at `p`.
 #[inline(always)]
 fn ss_key(t: &[u8], sa: &[i32], td: isize, pa: isize, p: isize) -> i32 {
    t[(td + sa[(pa + sa[p as usize] as isize) as usize] as isize) as usize] as i32
 }
 /// `Td[v]` for an already-loaded SA element `v` (`Td[PA[v]]` in C).
 #[inline(always)]
 fn ss_key_of(t: &[u8], sa: &[i32], td: isize, pa: isize, v: i32) -> i32 {
    t[(td + sa[(pa + v as isize) as usize] as isize) as usize] as i32
 }
 /// `Td[PA[SA[p]] - 1]` — the character preceding the depth-`td` key.
 #[inline(always)]
 fn ss_key_pred(t: &[u8], sa: &[i32], td: isize, pa: isize, p: isize) -> i32 {
    t[(td + sa[(pa + sa[p as usize] as isize) as usize] as isize - 1) as usize] as i32
 }
 fn ss_fixdown(t: &[u8], td: isize, sa: &mut [i32], pa: isize, base: isize, i: isize, size: isize) {
    let mut i = i;
    let v = sa[(base + i) as usize];
    let c = ss_key_of(t, sa, td, pa, v);
    loop {
        let mut j = 2 * i + 1;
        if j >= size {
            break;
        }
        let mut k = j;
        j += 1;
        let mut d = ss_key(t, sa, td, pa, base + k);
        let e = ss_key(t, sa, td, pa, base + j);
        if d < e {
            k = j;
            d = e;
        }
        if d <= c {
            break;
        }
        sa[(base + i) as usize] = sa[(base + k) as usize];
        i = k;
    }
    sa[(base + i) as usize] = v;
 }
 /* Simple top-down heapsort. */
 fn ss_heapsort(t: &[u8], td: isize, sa: &mut [i32], pa: isize, base: isize, size: isize) {
    let mut m = size;
    if size % 2 == 0 {
        m -= 1;
        if ss_key(t, sa, td, pa, base + m / 2) < ss_key(t, sa, td, pa, base + m) {
            sa.swap((base + m) as usize, (base + m / 2) as usize);
        }
    }
    let mut i = m / 2 - 1;
    while 0 <= i {
        ss_fixdown(t, td, sa, pa, base, i, m);
        i -= 1;
    }
    if size % 2 == 0 {
        sa.swap(base as usize, (base + m) as usize);
        ss_fixdown(t, td, sa, pa, base, 0, m);
    }
    let mut i = m - 1;
    while 0 < i {
        let t0 = sa[base as usize];
        sa[base as usize] = sa[(base + i) as usize];
        ss_fixdown(t, td, sa, pa, base, 0, i);
        sa[(base + i) as usize] = t0;
        i -= 1;
    }
 }
 /* --------------------------------------------------------------------- */
 /* Returns the median of three elements. */
 #[inline]
 fn ss_median3(
    t: &[u8],
    sa: &[i32],
    td: isize,
    pa: isize,
    v1: isize,
    v2: isize,
    v3: isize,
 ) -> isize {
    let mut v1 = v1;
    let mut v2 = v2;
    if ss_key(t, sa, td, pa, v1) > ss_key(t, sa, td, pa, v2) {
        std::mem::swap(&mut v1, &mut v2);
    }
    if ss_key(t, sa, td, pa, v2) > ss_key(t, sa, td, pa, v3) {
        if ss_key(t, sa, td, pa, v1) > ss_key(t, sa, td, pa, v3) {
            return v1;
        }
        return v3;
    }
    v2
 }
 /* Returns the median of five elements. */
 #[inline]
 fn ss_median5(
    t: &[u8],
    sa: &[i32],
    td: isize,
    pa: isize,
    v1: isize,
    v2: isize,
    v3: isize,
    v4: isize,
    v5: isize,
 ) -> isize {
    let mut v1 = v1;
    let mut v2 = v2;
    let mut v3 = v3;
    let mut v4 = v4;
    let mut v5 = v5;
    if ss_key(t, sa, td, pa, v2) > ss_key(t, sa, td, pa, v3) {
        std::mem::swap(&mut v2, &mut v3);
    }
    if ss_key(t, sa, td, pa, v4) > ss_key(t, sa, td, pa, v5) {
        std::mem::swap(&mut v4, &mut v5);
    }
    if ss_key(t, sa, td, pa, v2) > ss_key(t, sa, td, pa, v4) {
        std::mem::swap(&mut v2, &mut v4);
        std::mem::swap(&mut v3, &mut v5);
    }
    if ss_key(t, sa, td, pa, v1) > ss_key(t, sa, td, pa, v3) {
        std::mem::swap(&mut v1, &mut v3);
    }
    if ss_key(t, sa, td, pa, v1) > ss_key(t, sa, td, pa, v4) {
        std::mem::swap(&mut v1, &mut v4);
        std::mem::swap(&mut v3, &mut v5);
    }
    if ss_key(t, sa, td, pa, v3) > ss_key(t, sa, td, pa, v4) {
        return v4;
    }
    v3
 }
 /* Returns the pivot element. */
 #[inline]
 fn ss_pivot(t: &[u8], sa: &[i32], td: isize, pa: isize, first: isize, last: isize) -> isize {
    let mut t0 = last - first;
    let middle = first + t0 / 2;
    if t0 <= 512 {
        if t0 <= 32 {
            return ss_median3(t, sa, td, pa, first, middle, last - 1);
        }
        t0 >>= 2;
        return ss_median5(
            t,
            sa,
            td,
            pa,
            first,
            first + t0,
            middle,
            last - 1 - t0,
            last - 1,
        );
    }
    t0 >>= 3;
    let first = ss_median3(t, sa, td, pa, first, first + t0, first + (t0 << 1));
    let middle = ss_median3(t, sa, td, pa, middle - t0, middle, middle + t0);
    let last = ss_median3(t, sa, td, pa, last - 1 - (t0 << 1), last - 1 - t0, last - 1);
    ss_median3(t, sa, td, pa, first, middle, last)
 }
 /* --------------------------------------------------------------------- */
 /* Binary partition for substrings. */
 /* The `>= x + 1` comparison deliberately mirrors the C expression shape. */
 #[allow(clippy::int_plus_one)]
 fn ss_partition(sa: &mut [i32], pa: isize, first: isize, last: isize, depth: i32) -> isize {
    let mut a = first - 1;
    let mut b = last;
    loop {
        loop {
            a += 1;
            if !(a < b) {
                break;
            }
            if !(sa[(pa + sa[a as usize] as isize) as usize] + depth
                >= sa[(pa + sa[a as usize] as isize + 1) as usize] + 1)
            {
                break;
            }
            sa[a as usize] = !sa[a as usize];
        }
        loop {
            b -= 1;
            if !(a < b) {
                break;
            }
            if !(sa[(pa + sa[b as usize] as isize) as usize] + depth
                < sa[(pa + sa[b as usize] as isize + 1) as usize] + 1)
            {
                break;
            }
        }
        if b <= a {
            break;
        }
        let t0 = !sa[b as usize];
        sa[b as usize] = sa[a as usize];
        sa[a as usize] = t0;
    }
    if first < a {
        sa[first as usize] = !sa[first as usize];
    }
    a
 }
 /* Multikey introsort for medium size groups. */
 fn ss_mintrosort(t: &[u8], sa: &mut [i32], pa: isize, first: isize, last: isize, depth: i32) {
    let mut stack = [(0isize, 0isize, 0i32, 0i32); SS_MISORT_STACKSIZE];
    let mut ssize = 0usize;
    let mut first = first;
    let mut last = last;
    let mut depth = depth;
    let mut limit = ss_ilg(last - first);
    let mut x: i32 = 0;
    loop {
        if last - first <= SS_INSERTIONSORT_THRESHOLD {
            if 1 < last - first {
                ss_insertionsort(t, sa, pa, first, last, depth);
            }
            /* STACK_POP */
            if ssize == 0 {
                return;
            }
            ssize -= 1;
            (first, last, depth, limit) = stack[ssize];
            continue;
        }
        let td = depth as isize;
        if limit == 0 {
            ss_heapsort(t, td, sa, pa, first, last - first);
        }
        limit -= 1;
        if limit < 0 {
            let mut a = first + 1;
            let mut v = ss_key(t, sa, td, pa, first);
            while a < last {
                x = ss_key(t, sa, td, pa, a);
                if x != v {
                    if 1 < a - first {
                        break;
                    }
                    v = x;
                    first = a;
                }
                a += 1;
            }
            if ss_key_pred(t, sa, td, pa, first) < v {
                first = ss_partition(sa, pa, first, a, depth);
            }
            if a - first <= last - a {
                if 1 < a - first {
                    stack[ssize] = (a, last, depth, -1);
                    ssize += 1;
                    last = a;
                    depth += 1;
                    limit = ss_ilg(a - first);
                } else {
                    first = a;
                    limit = -1;
                }
            } else if 1 < last - a {
                stack[ssize] = (first, a, depth + 1, ss_ilg(a - first));
                ssize += 1;
                first = a;
                limit = -1;
            } else {
                last = a;
                depth += 1;
                limit = ss_ilg(a - first);
            }
            continue;
        }
        /* choose pivot */
        let mut a = ss_pivot(t, sa, td, pa, first, last);
        let v = ss_key(t, sa, td, pa, a);
        sa.swap(first as usize, a as usize);
        /* partition */
        let mut b = first;
        loop {
            b += 1;
            if !(b < last) {
                break;
            }
            x = ss_key(t, sa, td, pa, b);
            if x != v {
                break;
            }
        }
        a = b;
        if a < last && x < v {
            loop {
                b += 1;
                if !(b < last) {
                    break;
                }
                x = ss_key(t, sa, td, pa, b);
                if !(x <= v) {
                    break;
                }
                if x == v {
                    sa.swap(b as usize, a as usize);
                    a += 1;
                }
            }
        }
        let mut c = last;
        loop {
            c -= 1;
            if !(b < c) {
                break;
            }
            x = ss_key(t, sa, td, pa, c);
            if x != v {
                break;
            }
        }
        let mut d = c;
        if b < d && x > v {
            loop {
                c -= 1;
                if !(b < c) {
                    break;
                }
                x = ss_key(t, sa, td, pa, c);
                if !(x >= v) {
                    break;
                }
                if x == v {
                    sa.swap(c as usize, d as usize);
                    d -= 1;
                }
            }
        }
        while b < c {
            sa.swap(b as usize, c as usize);
            loop {
                b += 1;
                if !(b < c) {
                    break;
                }
                x = ss_key(t, sa, td, pa, b);
                if !(x <= v) {
                    break;
                }
                if x == v {
                    sa.swap(b as usize, a as usize);
                    a += 1;
                }
            }
            loop {
                c -= 1;
                if !(b < c) {
                    break;
                }
                x = ss_key(t, sa, td, pa, c);
                if !(x >= v) {
                    break;
                }
                if x == v {
                    sa.swap(c as usize, d as usize);
                    d -= 1;
                }
            }
        }
        if a <= d {
            c = b - 1;
            let mut s = a - first;
            let t0 = b - a;
            if s > t0 {
                s = t0;
            }
            let mut e = first;
            let mut f = b - s;
            while 0 < s {
                sa.swap(e as usize, f as usize);
                s -= 1;
                e += 1;
                f += 1;
            }
            let mut s = d - c;
            let t0 = last - d - 1;
            if s > t0 {
                s = t0;
            }
            let mut e = b;
            let mut f = last - s;
            while 0 < s {
                sa.swap(e as usize, f as usize);
                s -= 1;
                e += 1;
                f += 1;
            }
            a = first + (b - a);
            c = last - (d - c);
            b = if v <= ss_key_pred(t, sa, td, pa, a) {
                a
            } else {
                ss_partition(sa, pa, a, c, depth)
            };
            if a - first <= last - c {
                if last - c <= c - b {
                    stack[ssize] = (b, c, depth + 1, ss_ilg(c - b));
                    ssize += 1;
                    stack[ssize] = (c, last, depth, limit);
                    ssize += 1;
                    last = a;
                } else if a - first <= c - b {
                    stack[ssize] = (c, last, depth, limit);
                    ssize += 1;
                    stack[ssize] = (b, c, depth + 1, ss_ilg(c - b));
                    ssize += 1;
                    last = a;
                } else {
                    stack[ssize] = (c, last, depth, limit);
                    ssize += 1;
                    stack[ssize] = (first, a, depth, limit);
                    ssize += 1;
                    first = b;
                    last = c;
                    depth += 1;
                    limit = ss_ilg(c - b);
                }
            } else if a - first <= c - b {
                stack[ssize] = (b, c, depth + 1, ss_ilg(c - b));
                ssize += 1;
                stack[ssize] = (first, a, depth, limit);
                ssize += 1;
                first = c;
            } else if last - c <= c - b {
                stack[ssize] = (first, a, depth, limit);
                ssize += 1;
                stack[ssize] = (b, c, depth + 1, ss_ilg(c - b));
                ssize += 1;
                first = c;
            } else {
                stack[ssize] = (first, a, depth, limit);
                ssize += 1;
                stack[ssize] = (c, last, depth, limit);
                ssize += 1;
                first = b;
                last = c;
                depth += 1;
                limit = ss_ilg(c - b);
            }
        } else {
            limit += 1;
            if ss_key_pred(t, sa, td, pa, first) < v {
                first = ss_partition(sa, pa, first, last, depth);
                limit = ss_ilg(last - first);
            }
            depth += 1;
        }
    }
 }
 /* --------------------------------------------------------------------- */
 #[inline]
 fn ss_blockswap(sa: &mut [i32], a: isize, b: isize, n: isize) {
    let mut a = a;
    let mut b = b;
    let mut n = n;
    while 0 < n {
        sa.swap(a as usize, b as usize);
        n -= 1;
        a += 1;
        b += 1;
    }
 }
 #[inline]
 fn ss_rotate(sa: &mut [i32], first: isize, middle: isize, last: isize) {
    let mut first = first;
    let mut last = last;
    let mut l = middle - first;
    let mut r = last - middle;
    while 0 < l && 0 < r {
        if l == r {
            ss_blockswap(sa, first, middle, l);
            break;
        }
        if l < r {
            let mut a = last - 1;
            let mut b = middle - 1;
            let mut t0 = sa[a as usize];
            loop {
                sa[a as usize] = sa[b as usize];
                a -= 1;
                sa[b as usize] = sa[a as usize];
                b -= 1;
                if b < first {
                    sa[a as usize] = t0;
                    last = a;
                    r -= l + 1;
                    if r <= l {
                        break;
                    }
                    a -= 1;
                    b = middle - 1;
                    t0 = sa[a as usize];
                }
            }
        } else {
            let mut a = first;
            let mut b = middle;
            let mut t0 = sa[a as usize];
            loop {
                sa[a as usize] = sa[b as usize];
                a += 1;
                sa[b as usize] = sa[a as usize];
                b += 1;
                if last <= b {
                    sa[a as usize] = t0;
                    first = a + 1;
                    l -= r + 1;
                    if l <= r {
                        break;
                    }
                    a += 1;
                    b = middle;
                    t0 = sa[a as usize];
                }
            }
        }
    }
 }
 /* --------------------------------------------------------------------- */
 fn ss_inplacemerge(
    t: &[u8],
    sa: &mut [i32],
    pa: isize,
    first: isize,
    middle: isize,
    last: isize,
    depth: i32,
 ) {
    let mut middle = middle;
    let mut last = last;
    loop {
        let x: i32;
        let p: isize;
        if sa[(last - 1) as usize] < 0 {
            x = 1;
            p = pa + (!sa[(last - 1) as usize]) as isize;
        } else {
            x = 0;
            p = pa + sa[(last - 1) as usize] as isize;
        }
        let mut a = first;
        let mut len = middle - first;
        let mut half = len >> 1;
        let mut r: i32 = -1;
        while 0 < len {
            let b = a + half;
            let bv = sa[b as usize];
            let q = ss_compare_pa(
                t,
                sa,
                pa + (if 0 <= bv { bv } else { !bv }) as isize,
                p,
                depth,
            );
            if q < 0 {
                a = b + 1;
                half -= (len & 1) ^ 1;
            } else {
                r = q;
            }
            len = half;
            half >>= 1;
        }
        if a < middle {
            if r == 0 {
                sa[a as usize] = !sa[a as usize];
            }
            ss_rotate(sa, a, middle, last);
            last -= middle - a;
            middle = a;
            if first == middle {
                break;
            }
        }
        last -= 1;
        if x != 0 {
            loop {
                last -= 1;
                if !(sa[last as usize] < 0) {
                    break;
                }
            }
        }
        if middle == last {
            break;
        }
    }
 }
 /* --------------------------------------------------------------------- */
 /* Merge-forward with internal buffer. */
 fn ss_mergeforward(
    t: &[u8],
    sa: &mut [i32],
    pa: isize,
    first: isize,
    middle: isize,
    last: isize,
    buf: isize,
    depth: i32,
 ) {
    let bufend = buf + (middle - first) - 1;
    ss_blockswap(sa, buf, first, middle - first);
    let mut a = first;
    let t0 = sa[a as usize];
    let mut b = buf;
    let mut c = middle;
    loop {
        let r = ss_compare_pa(
            t,
            sa,
            pa + sa[b as usize] as isize,
            pa + sa[c as usize] as isize,
            depth,
        );
        if r < 0 {
            loop {
                sa[a as usize] = sa[b as usize];
                a += 1;
                if bufend <= b {
                    sa[bufend as usize] = t0;
                    return;
                }
                sa[b as usize] = sa[a as usize];
                b += 1;
                if !(sa[b as usize] < 0) {
                    break;
                }
            }
        } else if r > 0 {
            loop {
                sa[a as usize] = sa[c as usize];
                a += 1;
                sa[c as usize] = sa[a as usize];
                c += 1;
                if last <= c {
                    while b < bufend {
                        sa[a as usize] = sa[b as usize];
                        a += 1;
                        sa[b as usize] = sa[a as usize];
                        b += 1;
                    }
                    sa[a as usize] = sa[b as usize];
                    sa[b as usize] = t0;
                    return;
                }
                if !(sa[c as usize] < 0) {
                    break;
                }
            }
        } else {
            sa[c as usize] = !sa[c as usize];
            loop {
                sa[a as usize] = sa[b as usize];
                a += 1;
                if bufend <= b {
                    sa[bufend as usize] = t0;
                    return;
                }
                sa[b as usize] = sa[a as usize];
                b += 1;
                if !(sa[b as usize] < 0) {
                    break;
                }
            }
            loop {
                sa[a as usize] = sa[c as usize];
                a += 1;
                sa[c as usize] = sa[a as usize];
                c += 1;
                if last <= c {
                    while b < bufend {
                        sa[a as usize] = sa[b as usize];
                        a += 1;
                        sa[b as usize] = sa[a as usize];
                        b += 1;
                    }
                    sa[a as usize] = sa[b as usize];
                    sa[b as usize] = t0;
                    return;
                }
                if !(sa[c as usize] < 0) {
                    break;
                }
            }
        }
    }
 }
 /* Merge-backward with internal buffer. */
 fn ss_mergebackward(
    t: &[u8],
    sa: &mut [i32],
    pa: isize,
    first: isize,
    middle: isize,
    last: isize,
    buf: isize,
    depth: i32,
 ) {
    let bufend = buf + (last - middle) - 1;
    ss_blockswap(sa, buf, middle, last - middle);
    let mut x = 0i32;
    let mut p1: isize;
    let mut p2: isize;
    if sa[bufend as usize] < 0 {
        p1 = pa + (!sa[bufend as usize]) as isize;
        x |= 1;
    } else {
        p1 = pa + sa[bufend as usize] as isize;
    }
    if sa[(middle - 1) as usize] < 0 {
        p2 = pa + (!sa[(middle - 1) as usize]) as isize;
        x |= 2;
    } else {
        p2 = pa + sa[(middle - 1) as usize] as isize;
    }
    let mut a = last - 1;
    let t0 = sa[a as usize];
    let mut b = bufend;
    let mut c = middle - 1;
    loop {
        let r = ss_compare_pa(t, sa, p1, p2, depth);
        if 0 < r {
            if x & 1 != 0 {
                loop {
                    sa[a as usize] = sa[b as usize];
                    a -= 1;
                    sa[b as usize] = sa[a as usize];
                    b -= 1;
                    if !(sa[b as usize] < 0) {
                        break;
                    }
                }
                x ^= 1;
            }
            sa[a as usize] = sa[b as usize];
            a -= 1;
            if b <= buf {
                sa[buf as usize] = t0;
                break;
            }
            sa[b as usize] = sa[a as usize];
            b -= 1;
            if sa[b as usize] < 0 {
                p1 = pa + (!sa[b as usize]) as isize;
                x |= 1;
            } else {
                p1 = pa + sa[b as usize] as isize;
            }
        } else if r < 0 {
            if x & 2 != 0 {
                loop {
                    sa[a as usize] = sa[c as usize];
                    a -= 1;
                    sa[c as usize] = sa[a as usize];
                    c -= 1;
                    if !(sa[c as usize] < 0) {
                        break;
                    }
                }
                x ^= 2;
            }
            sa[a as usize] = sa[c as usize];
            a -= 1;
            sa[c as usize] = sa[a as usize];
            c -= 1;
            if c < first {
                while buf < b {
                    sa[a as usize] = sa[b as usize];
                    a -= 1;
                    sa[b as usize] = sa[a as usize];
                    b -= 1;
                }
                sa[a as usize] = sa[b as usize];
                sa[b as usize] = t0;
                break;
            }
            if sa[c as usize] < 0 {
                p2 = pa + (!sa[c as usize]) as isize;
                x |= 2;
            } else {
                p2 = pa + sa[c as usize] as isize;
            }
        } else {
            if x & 1 != 0 {
                loop {
                    sa[a as usize] = sa[b as usize];
                    a -= 1;
                    sa[b as usize] = sa[a as usize];
                    b -= 1;
                    if !(sa[b as usize] < 0) {
                        break;
                    }
                }
                x ^= 1;
            }
            sa[a as usize] = !sa[b as usize];
            a -= 1;
            if b <= buf {
                sa[buf as usize] = t0;
                break;
            }
            sa[b as usize] = sa[a as usize];
            b -= 1;
            if x & 2 != 0 {
                loop {
                    sa[a as usize] = sa[c as usize];
                    a -= 1;
                    sa[c as usize] = sa[a as usize];
                    c -= 1;
                    if !(sa[c as usize] < 0) {
                        break;
                    }
                }
                x ^= 2;
            }
            sa[a as usize] = sa[c as usize];
            a -= 1;
            sa[c as usize] = sa[a as usize];
            c -= 1;
            if c < first {
                while buf < b {
                    sa[a as usize] = sa[b as usize];
                    a -= 1;
                    sa[b as usize] = sa[a as usize];
                    b -= 1;
                }
                sa[a as usize] = sa[b as usize];
                sa[b as usize] = t0;
                break;
            }
            if sa[b as usize] < 0 {
                p1 = pa + (!sa[b as usize]) as isize;
                x |= 1;
            } else {
                p1 = pa + sa[b as usize] as isize;
            }
            if sa[c as usize] < 0 {
                p2 = pa + (!sa[c as usize]) as isize;
                x |= 2;
            } else {
                p2 = pa + sa[c as usize] as isize;
            }
        }
    }
 }
 /// `GETIDX` — undoes the "already merged" complement marking.
 #[inline(always)]
 fn getidx(a: i32) -> i32 {
    if 0 <= a {
        a
    } else {
        !a
    }
 }
 /// `MERGE_CHECK` — restores or sets the complement marks after a merge.
 #[inline]
 fn ss_merge_check(t: &[u8], sa: &mut [i32], pa: isize, a: isize, b: isize, c: i32, depth: i32) {
    if (c & 1) != 0
        || ((c & 2) != 0
            && ss_compare_pa(
                t,
                sa,
                pa + getidx(sa[(a - 1) as usize]) as isize,
                pa + sa[a as usize] as isize,
                depth,
            ) == 0)
    {
        sa[a as usize] = !sa[a as usize];
    }
    if (c & 4) != 0
        && ss_compare_pa(
            t,
            sa,
            pa + getidx(sa[(b - 1) as usize]) as isize,
            pa + sa[b as usize] as isize,
            depth,
        ) == 0
    {
        sa[b as usize] = !sa[b as usize];
    }
 }
 /* D&C based merge. */
 fn ss_swapmerge(
    t: &[u8],
    sa: &mut [i32],
    pa: isize,
    first: isize,
    middle: isize,
    last: isize,
    buf: isize,
    bufsize: isize,
    depth: i32,
 ) {
    let mut stack = [(0isize, 0isize, 0isize, 0i32); SS_SMERGE_STACKSIZE];
    let mut ssize = 0usize;
    let mut first = first;
    let mut middle = middle;
    let mut last = last;
    let mut check = 0i32;
    loop {
        if last - middle <= bufsize {
            if first < middle && middle < last {
                ss_mergebackward(t, sa, pa, first, middle, last, buf, depth);
            }
            ss_merge_check(t, sa, pa, first, last, check, depth);
            if ssize == 0 {
                return;
            }
            ssize -= 1;
            (first, middle, last, check) = stack[ssize];
            continue;
        }
        if middle - first <= bufsize {
            if first < middle {
                ss_mergeforward(t, sa, pa, first, middle, last, buf, depth);
            }
            ss_merge_check(t, sa, pa, first, last, check, depth);
            if ssize == 0 {
                return;
            }
            ssize -= 1;
            (first, middle, last, check) = stack[ssize];
            continue;
        }
        let mut m: isize = 0;
        let mut len = std::cmp::min(middle - first, last - middle);
        let mut half = len >> 1;
        while 0 < len {
            if ss_compare_pa(
                t,
                sa,
                pa + getidx(sa[(middle + m + half) as usize]) as isize,
                pa + getidx(sa[(middle - m - half - 1) as usize]) as isize,
                depth,
            ) < 0
            {
                m += half + 1;
                half -= (len & 1) ^ 1;
            }
            len = half;
            half >>= 1;
        }
        if 0 < m {
            let lm = middle - m;
            let rm = middle + m;
            ss_blockswap(sa, lm, middle, m);
            let mut l = middle;
            let mut r = middle;
            let mut next = 0i32;
            if rm < last {
                if sa[rm as usize] < 0 {
                    sa[rm as usize] = !sa[rm as usize];
                    if first < lm {
                        loop {
                            l -= 1;
                            if !(sa[l as usize] < 0) {
                                break;
                            }
                        }
                        next |= 4;
                    }
                    next |= 1;
                } else if first < lm {
                    while sa[r as usize] < 0 {
                        r += 1;
                    }
                    next |= 2;
                }
            }
            if l - first <= last - r {
                stack[ssize] = (r, rm, last, (next & 3) | (check & 4));
                ssize += 1;
                middle = lm;
                last = l;
                check = (check & 3) | (next & 4);
            } else {
                if (next & 2) != 0 && r == middle {
                    next ^= 6;
                }
                stack[ssize] = (first, lm, l, (check & 3) | (next & 4));
                ssize += 1;
                first = r;
                middle = rm;
                check = (next & 3) | (check & 4);
            }
        } else {
            if ss_compare_pa(
                t,
                sa,
                pa + getidx(sa[(middle - 1) as usize]) as isize,
                pa + sa[middle as usize] as isize,
                depth,
            ) == 0
            {
                sa[middle as usize] = !sa[middle as usize];
            }
            ss_merge_check(t, sa, pa, first, last, check, depth);
            if ssize == 0 {
                return;
            }
            ssize -= 1;
            (first, middle, last, check) = stack[ssize];
        }
    }
 }
 /* --------------------------------------------------------------------- */
 /* Substring sort */
 fn sssort(
    t: &[u8],
    sa: &mut [i32],
    pa: isize,
    first: isize,
    last: isize,
    buf: isize,
    bufsize: isize,
    depth: i32,
    n: isize,
    lastsuffix: bool,
 ) {
    let mut first = first;
    let mut buf = buf;
    let mut bufsize = bufsize;
    if lastsuffix {
        first += 1;
    }
    let mut limit: isize = 0;
    let mut middle = last;
    if bufsize < SS_BLOCKSIZE && bufsize < last - first {
        limit = ss_isqrt(last - first);
        if bufsize < limit {
            if SS_BLOCKSIZE < limit {
                limit = SS_BLOCKSIZE;
            }
            middle = last - limit;
            buf = middle;
            bufsize = limit;
        } else {
            limit = 0;
        }
    }
    let mut a = first;
    let mut i: isize = 0;
    while SS_BLOCKSIZE < middle - a {
        ss_mintrosort(t, sa, pa, a, a + SS_BLOCKSIZE, depth);
        let mut curbufsize = last - (a + SS_BLOCKSIZE);
        let mut curbuf = a + SS_BLOCKSIZE;
        if curbufsize <= bufsize {
            curbufsize = bufsize;
            curbuf = buf;
        }
        let mut b = a;
        let mut k = SS_BLOCKSIZE;
        let mut j = i;
        while j & 1 != 0 {
            ss_swapmerge(t, sa, pa, b - k, b, b + k, curbuf, curbufsize, depth);
            b -= k;
            k <<= 1;
            j >>= 1;
        }
        a += SS_BLOCKSIZE;
        i += 1;
    }
    ss_mintrosort(t, sa, pa, a, middle, depth);
    let mut k = SS_BLOCKSIZE;
    while i != 0 {
        if i & 1 != 0 {
            ss_swapmerge(t, sa, pa, a - k, a, middle, buf, bufsize, depth);
            a -= k;
        }
        k <<= 1;
        i >>= 1;
    }
    if limit != 0 {
        ss_mintrosort(t, sa, pa, middle, last, depth);
        ss_inplacemerge(t, sa, pa, first, middle, last, depth);
    }
    if lastsuffix {
        /* Insert last type B* suffix. */
        let pai0 = sa[(pa + sa[(first - 1) as usize] as isize) as usize];
        let pai1 = (n - 2) as i32;
        let i0 = sa[(first - 1) as usize];
        let mut a = first;
        while a < last {
            let av = sa[a as usize];
            if !(av < 0
                || 0 < ss_compare(
                    t,
                    pai0,
                    pai1,
                    sa[(pa + av as isize) as usize],
                    sa[(pa + av as isize + 1) as usize],
                    depth,
                ))
            {
                break;
            }
            sa[(a - 1) as usize] = av;
            a += 1;
        }
        sa[(a - 1) as usize] = i0;
    }
 }
 /* --------------------------------------------------------------------- */
 #[inline]
 fn tr_ilg(n: isize) -> i32 {
    let n = n as i32;
    if (n as u32) & 0xffff_0000 != 0 {
        if (n as u32) & 0xff00_0000 != 0 {
            24 + LG_TABLE[((n >> 24) & 0xff) as usize]
        } else {
            16 + LG_TABLE[((n >> 16) & 0xff) as usize]
        }
    } else if n & 0xff00 != 0 {
        8 + LG_TABLE[((n >> 8) & 0xff) as usize]
    } else {
        LG_TABLE[(n & 0xff) as usize]
    }
 }
 /* --------------------------------------------------------------------- */
 /// `ISAd[SA[p]]` — the depth-offset rank of the suffix stored at `p`.
 #[inline(always)]
 fn tr_key(sa: &[i32], isad: isize, p: isize) -> i32 {
    sa[(isad + sa[p as usize] as isize) as usize]
 }
 /* Simple insertionsort for small size groups. */
 fn tr_insertionsort(sa: &mut [i32], isad: isize, first: isize, last: isize) {
    let mut a = first + 1;
    while a < last {
        let t0 = sa[a as usize];
        let mut b = a - 1;
        let mut r;
        loop {
            r = sa[(isad + t0 as isize) as usize] - tr_key(sa, isad, b);
            if !(0 > r) {
                break;
            }
            loop {
                sa[(b + 1) as usize] = sa[b as usize];
                b -= 1;
                if !(first <= b && sa[b as usize] < 0) {
                    break;
                }
            }
            if b < first {
                break;
            }
        }
        if r == 0 {
            sa[b as usize] = !sa[b as usize];
        }
        sa[(b + 1) as usize] = t0;
        a += 1;
    }
 }
 /* --------------------------------------------------------------------- */
 fn tr_fixdown(sa: &mut [i32], isad: isize, base: isize, i: isize, size: isize) {
    let mut i = i;
    let v = sa[(base + i) as usize];
    let c = sa[(isad + v as isize) as usize];
    loop {
        let mut j = 2 * i + 1;
        if j >= size {
            break;
        }
        let mut k = j;
        j += 1;
        let mut d = tr_key(sa, isad, base + k);
        let e = tr_key(sa, isad, base + j);
        if d < e {
            k = j;
            d = e;
        }
        if d <= c {
            break;
        }
        sa[(base + i) as usize] = sa[(base + k) as usize];
        i = k;
    }
    sa[(base + i) as usize] = v;
 }
 /* Simple top-down heapsort. */
 fn tr_heapsort(sa: &mut [i32], isad: isize, base: isize, size: isize) {
    let mut m = size;
    if size % 2 == 0 {
        m -= 1;
        if tr_key(sa, isad, base + m / 2) < tr_key(sa, isad, base + m) {
            sa.swap((base + m) as usize, (base + m / 2) as usize);
        }
    }
    let mut i = m / 2 - 1;
    while 0 <= i {
        tr_fixdown(sa, isad, base, i, m);
        i -= 1;
    }
    if size % 2 == 0 {
        sa.swap(base as usize, (base + m) as usize);
        tr_fixdown(sa, isad, base, 0, m);
    }
    let mut i = m - 1;
    while 0 < i {
        let t0 = sa[base as usize];
        sa[base as usize] = sa[(base + i) as usize];
        tr_fixdown(sa, isad, base, 0, i);
        sa[(base + i) as usize] = t0;
        i -= 1;
    }
 }
 /* --------------------------------------------------------------------- */
 /* Returns the median of three elements. */
 #[inline]
 fn tr_median3(sa: &[i32], isad: isize, v1: isize, v2: isize, v3: isize) -> isize {
    let mut v1 = v1;
    let mut v2 = v2;
    if tr_key(sa, isad, v1) > tr_key(sa, isad, v2) {
        std::mem::swap(&mut v1, &mut v2);
    }
    if tr_key(sa, isad, v2) > tr_key(sa, isad, v3) {
        if tr_key(sa, isad, v1) > tr_key(sa, isad, v3) {
            return v1;
        }
        return v3;
    }
    v2
 }
 /* Returns the median of five elements. */
 #[inline]
 fn tr_median5(
    sa: &[i32],
    isad: isize,
    v1: isize,
    v2: isize,
    v3: isize,
    v4: isize,
    v5: isize,
 ) -> isize {
    let mut v1 = v1;
    let mut v2 = v2;
    let mut v3 = v3;
    let mut v4 = v4;
    let mut v5 = v5;
    if tr_key(sa, isad, v2) > tr_key(sa, isad, v3) {
        std::mem::swap(&mut v2, &mut v3);
    }
    if tr_key(sa, isad, v4) > tr_key(sa, isad, v5) {
        std::mem::swap(&mut v4, &mut v5);
    }
    if tr_key(sa, isad, v2) > tr_key(sa, isad, v4) {
        std::mem::swap(&mut v2, &mut v4);
        std::mem::swap(&mut v3, &mut v5);
    }
    if tr_key(sa, isad, v1) > tr_key(sa, isad, v3) {
        std::mem::swap(&mut v1, &mut v3);
    }
    if tr_key(sa, isad, v1) > tr_key(sa, isad, v4) {
        std::mem::swap(&mut v1, &mut v4);
        std::mem::swap(&mut v3, &mut v5);
    }
    if tr_key(sa, isad, v3) > tr_key(sa, isad, v4) {
        return v4;
    }
    v3
 }
 /* Returns the pivot element. */
 #[inline]
 fn tr_pivot(sa: &[i32], isad: isize, first: isize, last: isize) -> isize {
    let mut t0 = last - first;
    let middle = first + t0 / 2;
    if t0 <= 512 {
        if t0 <= 32 {
            return tr_median3(sa, isad, first, middle, last - 1);
        }
        t0 >>= 2;
        return tr_median5(sa, isad, first, first + t0, middle, last - 1 - t0, last - 1);
    }
    t0 >>= 3;
    let first = tr_median3(sa, isad, first, first + t0, first + (t0 << 1));
    let middle = tr_median3(sa, isad, middle - t0, middle, middle + t0);
    let last = tr_median3(sa, isad, last - 1 - (t0 << 1), last - 1 - t0, last - 1);
    tr_median3(sa, isad, first, middle, last)
 }
 /* --------------------------------------------------------------------- */
 struct TrBudget {
    chance: i32,
    remain: i32,
    incval: i32,
    count: i32,
 }
 impl TrBudget {
    fn new(chance: i32, incval: i32) -> Self {
        TrBudget {
            chance,
            remain: incval,
            incval,
            count: 0,
        }
    }
    fn check(&mut self, size: isize) -> bool {
        let size = size as i32;
        if size <= self.remain {
            self.remain -= size;
            return true;
        }
        if self.chance == 0 {
            self.count += size;
            return false;
        }
        self.remain += self.incval - size;
        self.chance -= 1;
        true
    }
 }
 /* --------------------------------------------------------------------- */
 fn tr_partition(
    sa: &mut [i32],
    isad: isize,
    first: isize,
    middle: isize,
    last: isize,
    v: i32,
 ) -> (isize, isize) {
    let mut first = first;
    let mut last = last;
    let mut x: i32 = 0;
    let mut b = middle - 1;
    loop {
        b += 1;
        if !(b < last) {
            break;
        }
        x = tr_key(sa, isad, b);
        if x != v {
            break;
        }
    }
    let mut a = b;
    if a < last && x < v {
        loop {
            b += 1;
            if !(b < last) {
                break;
            }
            x = tr_key(sa, isad, b);
            if !(x <= v) {
                break;
            }
            if x == v {
                sa.swap(b as usize, a as usize);
                a += 1;
            }
        }
    }
    let mut c = last;
    loop {
        c -= 1;
        if !(b < c) {
            break;
        }
        x = tr_key(sa, isad, c);
        if x != v {
            break;
        }
    }
    let mut d = c;
    if b < d && x > v {
        loop {
            c -= 1;
            if !(b < c) {
                break;
            }
            x = tr_key(sa, isad, c);
            if !(x >= v) {
                break;
            }
            if x == v {
                sa.swap(c as usize, d as usize);
                d -= 1;
            }
        }
    }
    while b < c {
        sa.swap(b as usize, c as usize);
        loop {
            b += 1;
            if !(b < c) {
                break;
            }
            x = tr_key(sa, isad, b);
            if !(x <= v) {
                break;
            }
            if x == v {
                sa.swap(b as usize, a as usize);
                a += 1;
            }
        }
        loop {
            c -= 1;
            if !(b < c) {
                break;
            }
            x = tr_key(sa, isad, c);
            if !(x >= v) {
                break;
            }
            if x == v {
                sa.swap(c as usize, d as usize);
                d -= 1;
            }
        }
    }
    if a <= d {
        c = b - 1;
        let mut s = a - first;
        let t0 = b - a;
        if s > t0 {
            s = t0;
        }
        let mut e = first;
        let mut f = b - s;
        while 0 < s {
            sa.swap(e as usize, f as usize);
            s -= 1;
            e += 1;
            f += 1;
        }
        let mut s = d - c;
        let t0 = last - d - 1;
        if s > t0 {
            s = t0;
        }
        let mut e = b;
        let mut f = last - s;
        while 0 < s {
            sa.swap(e as usize, f as usize);
            s -= 1;
            e += 1;
            f += 1;
        }
        first += b - a;
        last -= d - c;
    }
    (first, last)
 }
 /* sort suffixes of middle partition by using sorted order of suffixes of
 * left and right partition. */
 fn tr_copy(
    sa: &mut [i32],
    isa: isize,
    first: isize,
    a: isize,
    b: isize,
    last: isize,
    depth: isize,
 ) {
    /* All cursor arithmetic is relative to the slice start, which is the C
     * routine's `SA` pointer, so `x - SA` becomes plain `x`. */
    let v = (b - 1) as i32;
    let mut c = first;
    let mut d = a - 1;
    while c <= d {
        let s = sa[c as usize] - depth as i32;
        if 0 <= s && sa[(isa + s as isize) as usize] == v {
            d += 1;
            sa[d as usize] = s;
            sa[(isa + s as isize) as usize] = d as i32;
        }
        c += 1;
    }
    let mut c = last - 1;
    let e = d + 1;
    let mut d = b;
    while e < d {
        let s = sa[c as usize] - depth as i32;
        if 0 <= s && sa[(isa + s as isize) as usize] == v {
            d -= 1;
            sa[d as usize] = s;
            sa[(isa + s as isize) as usize] = d as i32;
        }
        c -= 1;
    }
 }
 fn tr_partialcopy(
    sa: &mut [i32],
    isa: isize,
    first: isize,
    a: isize,
    b: isize,
    last: isize,
    depth: isize,
 ) {
    let v = (b - 1) as i32;
    let mut newrank: i32 = -1;
    let mut lastrank: i32 = -1;
    let mut c = first;
    let mut d = a - 1;
    while c <= d {
        let s = sa[c as usize] - depth as i32;
        if 0 <= s && sa[(isa + s as isize) as usize] == v {
            d += 1;
            sa[d as usize] = s;
            let rank = sa[(isa + s as isize + depth) as usize];
            if lastrank != rank {
                lastrank = rank;
                newrank = d as i32;
            }
            sa[(isa + s as isize) as usize] = newrank;
        }
        c += 1;
    }
    let mut lastrank: i32 = -1;
    let mut e = d;
    while first <= e {
        let rank = sa[(isa + sa[e as usize] as isize) as usize];
        if lastrank != rank {
            lastrank = rank;
            newrank = e as i32;
        }
        if newrank != rank {
            sa[(isa + sa[e as usize] as isize) as usize] = newrank;
        }
        e -= 1;
    }
    let mut lastrank: i32 = -1;
    let mut c = last - 1;
    let e = d + 1;
    let mut d = b;
    while e < d {
        let s = sa[c as usize] - depth as i32;
        if 0 <= s && sa[(isa + s as isize) as usize] == v {
            d -= 1;
            sa[d as usize] = s;
            let rank = sa[(isa + s as isize + depth) as usize];
            if lastrank != rank {
                lastrank = rank;
                newrank = d as i32;
            }
            sa[(isa + s as isize) as usize] = newrank;
        }
        c -= 1;
    }
 }
 fn tr_introsort(
    sa: &mut [i32],
    isa: isize,
    isad: isize,
    first: isize,
    last: isize,
    budget: &mut TrBudget,
 ) {
    /* Stack frames are (ISAd, first, last, limit, trlink); the tandem-repeat
     * copy frame stores its `(a, b)` pair in the pointer fields with a zero
     * placeholder where C pushes a NULL ISAd. */
    let mut stack = [(0isize, 0isize, 0isize, 0i32, 0i32); TR_STACKSIZE];
    let mut ssize = 0usize;
    let mut trlink: i32 = -1;
    let mut isad = isad;
    let mut first = first;
    let mut last = last;
    let incr = isad - isa;
    let mut limit = tr_ilg(last - first);
    loop {
        if limit < 0 {
            if limit == -1 {
                /* tandem repeat partition */
                let (a, b) = tr_partition(sa, isad - incr, first, first, last, (last - 1) as i32);
                /* update ranks */
                if a < last {
                    let v = (a - 1) as i32;
                    let mut c = first;
                    while c < a {
                        sa[(isa + sa[c as usize] as isize) as usize] = v;
                        c += 1;
                    }
                }
                if b < last {
                    let v = (b - 1) as i32;
                    let mut c = a;
                    while c < b {
                        sa[(isa + sa[c as usize] as isize) as usize] = v;
                        c += 1;
                    }
                }
                /* push */
                if 1 < b - a {
                    stack[ssize] = (0, a, b, 0, 0);
                    ssize += 1;
                    stack[ssize] = (isad - incr, first, last, -2, trlink);
                    ssize += 1;
                    trlink = ssize as i32 - 2;
                }
                if a - first <= last - b {
                    if 1 < a - first {
                        stack[ssize] = (isad, b, last, tr_ilg(last - b), trlink);
                        ssize += 1;
                        last = a;
                        limit = tr_ilg(a - first);
                    } else if 1 < last - b {
                        first = b;
                        limit = tr_ilg(last - b);
                    } else {
                        if ssize == 0 {
                            return;
                        }
                        ssize -= 1;
                        (isad, first, last, limit, trlink) = stack[ssize];
                    }
                } else if 1 < last - b {
                    stack[ssize] = (isad, first, a, tr_ilg(a - first), trlink);
                    ssize += 1;
                    first = b;
                    limit = tr_ilg(last - b);
                } else if 1 < a - first {
                    last = a;
                    limit = tr_ilg(a - first);
                } else {
                    if ssize == 0 {
                        return;
                    }
                    ssize -= 1;
                    (isad, first, last, limit, trlink) = stack[ssize];
                }
            } else if limit == -2 {
                /* tandem repeat copy */
                ssize -= 1;
                let a = stack[ssize].1;
                let b = stack[ssize].2;
                if stack[ssize].3 == 0 {
                    tr_copy(sa, isa, first, a, b, last, isad - isa);
                } else {
                    if 0 <= trlink {
                        stack[trlink as usize].3 = -1;
                    }
                    tr_partialcopy(sa, isa, first, a, b, last, isad - isa);
                }
                if ssize == 0 {
                    return;
                }
                ssize -= 1;
                (isad, first, last, limit, trlink) = stack[ssize];
            } else {
                /* sorted partition */
                if 0 <= sa[first as usize] {
                    let mut a = first;
                    loop {
                        sa[(isa + sa[a as usize] as isize) as usize] = a as i32;
                        a += 1;
                        if !(a < last && 0 <= sa[a as usize]) {
                            break;
                        }
                    }
                    first = a;
                }
                if first < last {
                    let mut a = first;
                    loop {
                        sa[a as usize] = !sa[a as usize];
                        a += 1;
                        if !(sa[a as usize] < 0) {
                            break;
                        }
                    }
                    let next =
                        if sa[(isa + sa[a as usize] as isize) as usize] != tr_key(sa, isad, a) {
                            tr_ilg(a - first + 1)
                        } else {
                            -1
                        };
                    a += 1;
                    if a < last {
                        let v = (a - 1) as i32;
                        let mut b = first;
                        while b < a {
                            sa[(isa + sa[b as usize] as isize) as usize] = v;
                            b += 1;
                        }
                    }
                    /* push */
                    if budget.check(a - first) {
                        if a - first <= last - a {
                            stack[ssize] = (isad, a, last, -3, trlink);
                            ssize += 1;
                            isad += incr;
                            last = a;
                            limit = next;
                        } else if 1 < last - a {
                            stack[ssize] = (isad + incr, first, a, next, trlink);
                            ssize += 1;
                            first = a;
                            limit = -3;
                        } else {
                            isad += incr;
                            last = a;
                            limit = next;
                        }
                    } else {
                        if 0 <= trlink {
                            stack[trlink as usize].3 = -1;
                        }
                        if 1 < last - a {
                            first = a;
                            limit = -3;
                        } else {
                            if ssize == 0 {
                                return;
                            }
                            ssize -= 1;
                            (isad, first, last, limit, trlink) = stack[ssize];
                        }
                    }
                } else {
                    if ssize == 0 {
                        return;
                    }
                    ssize -= 1;
                    (isad, first, last, limit, trlink) = stack[ssize];
                }
            }
            continue;
        }
        if last - first <= TR_INSERTIONSORT_THRESHOLD {
            tr_insertionsort(sa, isad, first, last);
            limit = -3;
            continue;
        }
        /* C decrements `limit` here (`limit-- == 0`); the decrement is
         * observable only on the not-taken path because the taken path
         * overwrites `limit` with -3. */
        if limit == 0 {
            tr_heapsort(sa, isad, first, last - first);
            let mut a = last - 1;
            while first < a {
                let x = tr_key(sa, isad, a);
                let mut b = a - 1;
                while first <= b && tr_key(sa, isad, b) == x {
                    sa[b as usize] = !sa[b as usize];
                    b -= 1;
                }
                a = b;
            }
            limit = -3;
            continue;
        }
        limit -= 1;
        /* choose pivot */
        let a = tr_pivot(sa, isad, first, last);
        sa.swap(first as usize, a as usize);
        let v = tr_key(sa, isad, first);
        /* partition */
        let (a, b) = tr_partition(sa, isad, first, first + 1, last, v);
        if last - first != b - a {
            let next = if sa[(isa + sa[a as usize] as isize) as usize] != v {
                tr_ilg(b - a)
            } else {
                -1
            };
            /* update ranks */
            {
                let vv = (a - 1) as i32;
                let mut c = first;
                while c < a {
                    sa[(isa + sa[c as usize] as isize) as usize] = vv;
                    c += 1;
                }
            }
            if b < last {
                let vv = (b - 1) as i32;
                let mut c = a;
                while c < b {
                    sa[(isa + sa[c as usize] as isize) as usize] = vv;
                    c += 1;
                }
            }
            /* push */
            if 1 < b - a && budget.check(b - a) {
                if a - first <= last - b {
                    if last - b <= b - a {
                        if 1 < a - first {
                            stack[ssize] = (isad + incr, a, b, next, trlink);
                            ssize += 1;
                            stack[ssize] = (isad, b, last, limit, trlink);
                            ssize += 1;
                            last = a;
                        } else if 1 < last - b {
                            stack[ssize] = (isad + incr, a, b, next, trlink);
                            ssize += 1;
                            first = b;
                        } else {
                            isad += incr;
                            first = a;
                            last = b;
                            limit = next;
                        }
                    } else if a - first <= b - a {
                        if 1 < a - first {
                            stack[ssize] = (isad, b, last, limit, trlink);
                            ssize += 1;
                            stack[ssize] = (isad + incr, a, b, next, trlink);
                            ssize += 1;
                            last = a;
                        } else {
                            stack[ssize] = (isad, b, last, limit, trlink);
                            ssize += 1;
                            isad += incr;
                            first = a;
                            last = b;
                            limit = next;
                        }
                    } else {
                        stack[ssize] = (isad, b, last, limit, trlink);
                        ssize += 1;
                        stack[ssize] = (isad, first, a, limit, trlink);
                        ssize += 1;
                        isad += incr;
                        first = a;
                        last = b;
                        limit = next;
                    }
                } else if a - first <= b - a {
                    if 1 < last - b {
                        stack[ssize] = (isad + incr, a, b, next, trlink);
                        ssize += 1;
                        stack[ssize] = (isad, first, a, limit, trlink);
                        ssize += 1;
                        first = b;
                    } else if 1 < a - first {
                        stack[ssize] = (isad + incr, a, b, next, trlink);
                        ssize += 1;
                        last = a;
                    } else {
                        isad += incr;
                        first = a;
                        last = b;
                        limit = next;
                    }
                } else if last - b <= b - a {
                    if 1 < last - b {
                        stack[ssize] = (isad, first, a, limit, trlink);
                        ssize += 1;
                        stack[ssize] = (isad + incr, a, b, next, trlink);
                        ssize += 1;
                        first = b;
                    } else {
                        stack[ssize] = (isad, first, a, limit, trlink);
                        ssize += 1;
                        isad += incr;
                        first = a;
                        last = b;
                        limit = next;
                    }
                } else {
                    stack[ssize] = (isad, first, a, limit, trlink);
                    ssize += 1;
                    stack[ssize] = (isad, b, last, limit, trlink);
                    ssize += 1;
                    isad += incr;
                    first = a;
                    last = b;
                    limit = next;
                }
            } else {
                if 1 < b - a && 0 <= trlink {
                    stack[trlink as usize].3 = -1;
                }
                if a - first <= last - b {
                    if 1 < a - first {
                        stack[ssize] = (isad, b, last, limit, trlink);
                        ssize += 1;
                        last = a;
                    } else if 1 < last - b {
                        first = b;
                    } else {
                        if ssize == 0 {
                            return;
                        }
                        ssize -= 1;
                        (isad, first, last, limit, trlink) = stack[ssize];
                    }
                } else if 1 < last - b {
                    stack[ssize] = (isad, first, a, limit, trlink);
                    ssize += 1;
                    first = b;
                } else if 1 < a - first {
                    last = a;
                } else {
                    if ssize == 0 {
                        return;
                    }
                    ssize -= 1;
                    (isad, first, last, limit, trlink) = stack[ssize];
                }
            }
        } else if budget.check(last - first) {
            limit = tr_ilg(last - first);
            isad += incr;
        } else {
            if 0 <= trlink {
                stack[trlink as usize].3 = -1;
            }
            if ssize == 0 {
                return;
            }
            ssize -= 1;
            (isad, first, last, limit, trlink) = stack[ssize];
        }
    }
 }
 /* --------------------------------------------------------------------- */
 /* Tandem repeat sort */
 fn trsort(sa: &mut [i32], isa: isize, n: isize, depth: isize) {
    let mut budget = TrBudget::new(tr_ilg(n) * 2 / 3, n as i32);
    /* trbudget_init(&budget, tr_ilg(n) * 3 / 4, n); */
    let mut isad = isa + depth;
    while -(n as i32) < sa[0] {
        let mut first: isize = 0;
        let mut skip: isize = 0;
        let mut unsorted: i32 = 0;
        loop {
            let t0 = sa[first as usize];
            if t0 < 0 {
                first -= t0 as isize;
                skip += t0 as isize;
            } else {
                if skip != 0 {
                    sa[(first + skip) as usize] = skip as i32;
                    skip = 0;
                }
                let last = sa[(isa + t0 as isize) as usize] as isize + 1;
                if 1 < last - first {
                    budget.count = 0;
                    tr_introsort(sa, isa, isad, first, last, &mut budget);
                    if budget.count != 0 {
                        unsorted += budget.count;
                    } else {
                        skip = first - last;
                    }
                } else if last - first == 1 {
                    skip = -1;
                }
                first = last;
            }
            if !(first < n) {
                break;
            }
        }
        if skip != 0 {
            sa[(first + skip) as usize] = skip as i32;
        }
        if unsorted == 0 {
            break;
        }
        isad += isad - isa;
    }
 }
 /* --------------------------------------------------------------------- */
 /// `BUCKET_B(c0, c1)` for the 256-symbol alphabet.
 #[inline(always)]
 fn bb(c0: i32, c1: i32) -> usize {
    (((c1 as u32) << 8) | c0 as u32) as usize
 }
 /// `BUCKET_BSTAR(c0, c1)` for the 256-symbol alphabet.
 #[inline(always)]
 fn bstar(c0: i32, c1: i32) -> usize {
    (((c0 as u32) << 8) | c1 as u32) as usize
 }
 /* Sorts suffixes of type B*. */
 fn sort_type_bstar(
    t: &[u8],
    sa: &mut [i32],
    bucket_a: &mut [i32],
    bucket_b: &mut [i32],
    n: isize,
 ) -> isize {
    /* Initialize bucket arrays. */
    for slot in bucket_a.iter_mut() {
        *slot = 0;
    }
    for slot in bucket_b.iter_mut() {
        *slot = 0;
    }
    /* Count the number of occurrences of the first one or two characters of
    each type A, B and B* suffix. Moreover, store the beginning position of
    all type B* suffixes into the array SA. */
    let mut i = n - 1;
    let mut m = n;
    let mut c0 = t[(n - 1) as usize] as i32;
    let mut c1;
    while 0 <= i {
        /* type A suffix. */
        loop {
            c1 = c0;
            bucket_a[c1 as usize] += 1;
            i -= 1;
            if 0 <= i {
                c0 = t[i as usize] as i32;
                if c0 >= c1 {
                    continue;
                }
            }
            break;
        }
        if 0 <= i {
            /* type B* suffix. */
            bucket_b[bstar(c0, c1)] += 1;
            m -= 1;
            sa[m as usize] = i as i32;
            /* type B suffix. */
            i -= 1;
            c1 = c0;
            while 0 <= i {
                c0 = t[i as usize] as i32;
                if !(c0 <= c1) {
                    break;
                }
                bucket_b[bb(c0, c1)] += 1;
                i -= 1;
                c1 = c0;
            }
        }
    }
    let m = n - m;
    /*
    note:
      A type B* suffix is lexicographically smaller than a type B suffix that
      begins with the same first two characters.
    */
    /* Calculate the index of start/end point of each bucket. */
    {
        let mut i: i32 = 0;
        let mut j: i32 = 0;
        for c0 in 0..ALPHABET_SIZE {
            let t0 = i + bucket_a[c0 as usize];
            bucket_a[c0 as usize] = i + j; /* start point */
            i = t0 + bucket_b[bb(c0, c0)];
            for c1 in (c0 + 1)..ALPHABET_SIZE {
                j += bucket_b[bstar(c0, c1)];
                bucket_b[bstar(c0, c1)] = j; /* end point */
                i += bucket_b[bb(c0, c1)];
            }
        }
    }
    if 0 < m {
        /* Sort the type B* suffixes by their first two characters. */
        let pab = n - m;
        let isab = m;
        let mut i = m - 2;
        while 0 <= i {
            let t0 = sa[(pab + i) as usize];
            let c0 = t[t0 as usize] as i32;
            let c1 = t[(t0 + 1) as usize] as i32;
            bucket_b[bstar(c0, c1)] -= 1;
            sa[bucket_b[bstar(c0, c1)] as usize] = i as i32;
            i -= 1;
        }
        {
            let t0 = sa[(pab + m - 1) as usize];
            let c0 = t[t0 as usize] as i32;
            let c1 = t[(t0 + 1) as usize] as i32;
            bucket_b[bstar(c0, c1)] -= 1;
            sa[bucket_b[bstar(c0, c1)] as usize] = (m - 1) as i32;
        }
        /* Sort the type B* substrings using sssort. */
        let buf = m;
        let bufsize = n - 2 * m;
        let mut c0 = ALPHABET_SIZE - 2;
        let mut j = m;
        while 0 < j {
            let mut c1 = ALPHABET_SIZE - 1;
            while c0 < c1 {
                let i = bucket_b[bstar(c0, c1)] as isize;
                if 1 < j - i {
                    sssort(
                        t,
                        sa,
                        pab,
                        i,
                        j,
                        buf,
                        bufsize,
                        2,
                        n,
                        sa[i as usize] == (m - 1) as i32,
                    );
                }
                j = i;
                c1 -= 1;
            }
            c0 -= 1;
        }
        /* Compute ranks of type B* substrings. */
        let mut i = m - 1;
        while 0 <= i {
            if 0 <= sa[i as usize] {
                let j = i;
                loop {
                    sa[(isab + sa[i as usize] as isize) as usize] = i as i32;
                    i -= 1;
                    if !(0 <= i && 0 <= sa[i as usize]) {
                        break;
                    }
                }
                sa[(i + 1) as usize] = (i - j) as i32;
                if i <= 0 {
                    break;
                }
            }
            let j = i;
            loop {
                sa[i as usize] = !sa[i as usize];
                sa[(isab + sa[i as usize] as isize) as usize] = j as i32;
                i -= 1;
                if !(sa[i as usize] < 0) {
                    break;
                }
            }
            sa[(isab + sa[i as usize] as isize) as usize] = j as i32;
            i -= 1;
        }
        /* Construct the inverse suffix array of type B* suffixes using
        trsort. */
        trsort(sa, isab, m, 1);
        /* Set the sorted order of type B* suffixes. */
        let mut i = n - 1;
        let mut j = m;
        let mut c0 = t[(n - 1) as usize] as i32;
        while 0 <= i {
            i -= 1;
            let mut c1 = c0;
            while 0 <= i {
                c0 = t[i as usize] as i32;
                if !(c0 >= c1) {
                    break;
                }
                i -= 1;
                c1 = c0;
            }
            if 0 <= i {
                let t0 = i;
                i -= 1;
                c1 = c0;
                while 0 <= i {
                    c0 = t[i as usize] as i32;
                    if !(c0 <= c1) {
                        break;
                    }
                    i -= 1;
                    c1 = c0;
                }
                j -= 1;
                sa[sa[(isab + j) as usize] as usize] = if t0 == 0 || 1 < t0 - i {
                    t0 as i32
                } else {
                    !(t0 as i32)
                };
            }
        }
        /* Calculate the index of start/end point of each bucket. */
        bucket_b[bb(ALPHABET_SIZE - 1, ALPHABET_SIZE - 1)] = n as i32; /* end point */
        let mut k = m - 1;
        let mut c0 = ALPHABET_SIZE - 2;
        while 0 <= c0 {
            let mut i = bucket_a[(c0 + 1) as usize] as isize - 1;
            let mut c1 = ALPHABET_SIZE - 1;
            while c0 < c1 {
                let t0 = i - bucket_b[bb(c0, c1)] as isize;
                bucket_b[bb(c0, c1)] = i as i32; /* end point */
                /* Move all type B* suffixes to the correct position. */
                i = t0;
                let j = bucket_b[bstar(c0, c1)] as isize;
                while j <= k {
                    sa[i as usize] = sa[k as usize];
                    i -= 1;
                    k -= 1;
                }
                c1 -= 1;
            }
            bucket_b[bstar(c0, c0 + 1)] = (i - bucket_b[bb(c0, c0)] as isize + 1) as i32; /* start point */
            bucket_b[bb(c0, c0)] = i as i32; /* end point */
            c0 -= 1;
        }
    }
    m
 }
 /* Constructs the suffix array by using the sorted order of type B*
 * suffixes. */
 fn construct_sa(
    t: &[u8],
    sa: &mut [i32],
    bucket_a: &mut [i32],
    bucket_b: &mut [i32],
    n: isize,
    m: isize,
 ) {
    if 0 < m {
        /* Construct the sorted order of type B suffixes by using
        the sorted order of type B* suffixes. */
        let mut c1 = ALPHABET_SIZE - 2;
        while 0 <= c1 {
            /* Scan the suffix array from right to left. */
            let i = bucket_b[bstar(c1, c1 + 1)] as isize;
            let mut j = bucket_a[(c1 + 1) as usize] as isize - 1;
            let mut k: isize = 0;
            let mut c2: i32 = -1;
            while i <= j {
                let mut s = sa[j as usize];
                if 0 < s {
                    debug_assert_eq!(t[s as usize] as i32, c1);
                    debug_assert!((s as isize + 1) < n && t[s as usize] <= t[(s + 1) as usize]);
                    debug_assert!(t[(s - 1) as usize] <= t[s as usize]);
                    sa[j as usize] = !s;
                    s -= 1;
                    let c0 = t[s as usize] as i32;
                    if 0 < s && (t[(s - 1) as usize] as i32) > c0 {
                        s = !s;
                    }
                    if c0 != c2 {
                        if 0 <= c2 {
                            bucket_b[bb(c2, c1)] = k as i32;
                        }
                        c2 = c0;
                        k = bucket_b[bb(c2, c1)] as isize;
                    }
                    debug_assert!(k < j);
                    sa[k as usize] = s;
                    k -= 1;
                } else {
                    debug_assert!((s == 0 && t[s as usize] as i32 == c1) || s < 0);
                    sa[j as usize] = !s;
                }
                j -= 1;
            }
            c1 -= 1;
        }
    }
    /* Construct the suffix array by using the sorted order of type B
    suffixes. */
    let mut c2 = t[(n - 1) as usize] as i32;
    let mut k = bucket_a[c2 as usize] as isize;
    sa[k as usize] = if (t[(n - 2) as usize] as i32) < c2 {
        !((n - 1) as i32)
    } else {
        (n - 1) as i32
    };
    k += 1;
    /* Scan the suffix array from left to right. */
    let mut i: isize = 0;
    let j = n;
    while i < j {
        let mut s = sa[i as usize];
        if 0 < s {
            debug_assert!(t[(s - 1) as usize] >= t[s as usize]);
            s -= 1;
            let c0 = t[s as usize] as i32;
            if s == 0 || (t[(s - 1) as usize] as i32) < c0 {
                s = !s;
            }
            if c0 != c2 {
                bucket_a[c2 as usize] = k as i32;
                c2 = c0;
                k = bucket_a[c2 as usize] as isize;
            }
            debug_assert!(i < k);
            sa[k as usize] = s;
            k += 1;
        } else {
            debug_assert!(s < 0);
            sa[i as usize] = !s;
        }
        i += 1;
    }
 }
 /* --------------------------------------------------------------------- */
 /// Rust implementation of the `divsufsort()` entry point used by
 /// `ZDICT_trainFromBuffer_legacy()`.
 ///
 /// Integration removes the C function body, so this direct export provides
 /// the existing library symbol without a wrapper.  The `open_mp` parameter is
 /// accepted for signature compatibility only: zstd never defines
 /// `LIBBSC_OPENMP`, so the C implementation ignored it as well.
 ///
 /// Returns 0 on success, -1 for invalid arguments, and -2 when the bucket
 /// work arrays cannot be allocated, exactly like the C routine.
 #[no_mangle]
 pub unsafe extern "C" fn divsufsort(
    t: *const u8,
    sa: *mut c_int,
    n: c_int,
    open_mp: c_int,
 ) -> c_int {
    let _ = open_mp;
    /* Check arguments. */
    if t.is_null() || sa.is_null() || n < 0 {
        return -1;
    }
    if n == 0 {
        return 0;
    }
    let text = unsafe { slice::from_raw_parts(t, n as usize) };
    let suffix = unsafe { slice::from_raw_parts_mut(sa, n as usize) };
    if n == 1 {
        suffix[0] = 0;
        return 0;
    }
    if n == 2 {
        let m = usize::from(text[0] < text[1]);
        suffix[m ^ 1] = 0;
        suffix[m] = 1;
        return 0;
    }
    let mut bucket_a: Vec<i32> = Vec::new();
    let mut bucket_b: Vec<i32> = Vec::new();
    if bucket_a.try_reserve_exact(BUCKET_A_SIZE).is_err()
        || bucket_b.try_reserve_exact(BUCKET_B_SIZE).is_err()
    {
        /* Match the C implementation's -2 result when malloc fails. */
        return -2;
    }
    bucket_a.resize(BUCKET_A_SIZE, 0);
    bucket_b.resize(BUCKET_B_SIZE, 0);
    /* Suffixsort. */
    let m = sort_type_bstar(text, suffix, &mut bucket_a, &mut bucket_b, n as isize);
    construct_sa(text, suffix, &mut bucket_a, &mut bucket_b, n as isize, m);
    0
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    use std::ptr;
    fn build_sa(text: &[u8]) -> Vec<i32> {
        let mut sa = vec![0i32; text.len()];
        let result = unsafe { divsufsort(text.as_ptr(), sa.as_mut_ptr(), text.len() as c_int, 0) };
        assert_eq!(result, 0);
        sa
    }
    /// Trivial O(n^2 log n) reference: sort the suffix start positions by the
    /// suffixes themselves.
    fn reference_sa(text: &[u8]) -> Vec<i32> {
        let mut sa: Vec<i32> = (0..text.len() as i32).collect();
        sa.sort_by(|&a, &b| text[a as usize..].cmp(&text[b as usize..]));
        sa
    }
    /// Suffix-array invariants: a permutation of `0..n` whose suffixes are in
    /// strictly increasing lexicographic order.
    fn assert_valid_sa(text: &[u8], sa: &[i32]) {
        assert_eq!(sa.len(), text.len());
        let mut seen = vec![false; text.len()];
        for &p in sa {
            let p = usize::try_from(p).expect("suffix index must be non-negative");
            assert!(p < text.len(), "suffix index {p} out of range");
            assert!(!seen[p], "duplicate suffix index {p}");
            seen[p] = true;
        }
        for pair in sa.windows(2) {
            assert!(
                text[pair[0] as usize..] < text[pair[1] as usize..],
                "suffixes {} and {} are not in sorted order",
                pair[0],
                pair[1]
            );
        }
    }
    /// Fixed-seed numerical-recipes LCG, used to generate reproducible
    /// pseudo-random sample buffers.
    fn lcg_bytes(len: usize, seed: u32, alphabet: u32) -> Vec<u8> {
        let mut state = seed;
        (0..len)
            .map(|_| {
                state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
                ((state >> 24) % alphabet) as u8
            })
            .collect()
    }
    #[test]
    fn rejects_invalid_arguments() {
        let text = [0u8; 1];
        let mut sa = [0i32; 1];
        assert_eq!(
            unsafe { divsufsort(ptr::null(), sa.as_mut_ptr(), 1, 0) },
            -1
        );
        assert_eq!(
            unsafe { divsufsort(text.as_ptr(), ptr::null_mut(), 1, 0) },
            -1
        );
        assert_eq!(
            unsafe { divsufsort(text.as_ptr(), sa.as_mut_ptr(), -1, 0) },
            -1
        );
    }
    #[test]
    fn sorts_trivial_inputs() {
        /* empty */
        let text = [0u8; 1];
        let mut sa = [i32::MIN; 1];
        assert_eq!(
            unsafe { divsufsort(text.as_ptr(), sa.as_mut_ptr(), 0, 0) },
            0
        );
        assert_eq!(sa[0], i32::MIN, "n == 0 must not touch the output");
        /* single byte */
        assert_eq!(build_sa(b"z"), [0]);
        /* two bytes: ascending, descending, and equal */
        assert_eq!(build_sa(b"ab"), [0, 1]);
        assert_eq!(build_sa(b"ba"), [1, 0]);
        assert_eq!(build_sa(b"aa"), [1, 0]);
    }
    #[test]
    fn sorts_all_equal_bytes() {
        let text = vec![b'q'; 10_000];
        let sa = build_sa(&text);
        /* For a constant text the shortest suffix sorts first. */
        let expected: Vec<i32> = (0..text.len() as i32).rev().collect();
        assert_eq!(sa, expected);
    }
    #[test]
    fn sorts_abracadabra_exactly() {
        /* Hand-computed: a(10) abra(7) abracadabra(0) acadabra(3) adabra(5)
         * bra(8) bracadabra(1) cadabra(4) dabra(6) ra(9) racadabra(2). */
        assert_eq!(build_sa(b"abracadabra"), [10, 7, 0, 3, 5, 8, 1, 4, 6, 9, 2]);
    }
    #[test]
    fn matches_reference_on_periodic_text() {
        /* Tandem repeats exercise trsort's repeat partitioning. */
        let text: Vec<u8> = b"ab".iter().copied().cycle().take(4096).collect();
        let sa = build_sa(&text);
        assert_valid_sa(&text, &sa);
        assert_eq!(sa, reference_sa(&text));
    }
    #[test]
    fn matches_reference_on_random_bytes() {
        let text = lcg_bytes(8192, 0x0BAD_5EED, 256);
        let sa = build_sa(&text);
        assert_valid_sa(&text, &sa);
        assert_eq!(sa, reference_sa(&text));
    }
    #[test]
    fn matches_reference_on_low_alphabet_text() {
        /* A four-symbol alphabet produces the large first-two-character
         * buckets that reach sssort's block merging and the deeper trsort
         * paths. */
        let text = lcg_bytes(16_384, 0xDEAD_BEEF, 4);
        let sa = build_sa(&text);
        assert_valid_sa(&text, &sa);
        assert_eq!(sa, reference_sa(&text));
    }
 }
@@ -5,6 +5,8 @@ pub mod bitstream;
 pub mod common;
 pub mod cpu;
 pub mod debug;
 #[cfg(feature = "dict-builder")]
 pub mod divsufsort;
 pub mod entropy_common;
 pub mod errors;
 #[cfg(feature = "compression")]
@@ -1,198 +0,0 @@
 #![allow(non_camel_case_types)]
 #![allow(non_snake_case)]
 //! Precise monotonic time measurement for the command-line programs.
 //!
 //! Port of `programs/timefn.c`.  `UTIL_time_t` is a plain nanosecond counter
 //! whose absolute value is meaningless; only spans between two measurements
 //! are valid.  The struct crosses the C ABI by value, so it stays `repr(C)`
 //! with the exact `timefn.h` layout.
 //!
 //! Platform selection mirrors the C preprocessor structure: Windows uses the
 //! performance counter, Apple systems use the Mach absolute clock, and other
 //! POSIX systems use `clock_gettime(CLOCK_MONOTONIC)`.  The C90 `clock()`
 //! fallback is never needed on targets Rust supports, so multi-threaded
 //! measurements are always supported.
 use std::os::raw::c_int;
 /// Precise Time (`PTime` in timefn.h): an unsigned 64-bit nanosecond count.
 pub type PTime = u64;
 /// Nanosecond time counter with the `timefn.h` `UTIL_time_t` layout.
 #[repr(C)]
 #[derive(Clone, Copy, Debug)]
 pub struct UTIL_time_t {
    pub t: PTime,
 }
 const _: () = assert!(std::mem::size_of::<PTime>() == 8);
 const _: () = assert!(std::mem::size_of::<UTIL_time_t>() == std::mem::size_of::<PTime>());
 #[cfg(windows)]
 mod platform {
    use super::PTime;
    use std::sync::OnceLock;
    #[link(name = "kernel32")]
    unsafe extern "system" {
        /// Takes a `LARGE_INTEGER*`; the union is ABI-identical to `i64*`.
        fn QueryPerformanceCounter(count: *mut i64) -> i32;
        fn QueryPerformanceFrequency(frequency: *mut i64) -> i32;
    }
    pub fn monotonic_ns() -> PTime {
        static TICKS_PER_SECOND: OnceLock<i64> = OnceLock::new();
        let ticks_per_second = *TICKS_PER_SECOND.get_or_init(|| {
            let mut frequency = 0i64;
            if unsafe { QueryPerformanceFrequency(&mut frequency) } == 0 {
                eprintln!(
                    "timefn::QueryPerformanceFrequency: {}",
                    std::io::Error::last_os_error()
                );
                std::process::abort();
            }
            frequency
        });
        let mut counter = 0i64;
        unsafe { QueryPerformanceCounter(&mut counter) };
        (counter as PTime).wrapping_mul(1_000_000_000) / ticks_per_second as PTime
    }
 }
 #[cfg(all(unix, target_vendor = "apple"))]
 mod platform {
    use super::PTime;
    use std::sync::OnceLock;
    pub fn monotonic_ns() -> PTime {
        static RATE: OnceLock<(PTime, PTime)> = OnceLock::new();
        let (numer, denom) = *RATE.get_or_init(|| {
            let mut rate = libc::mach_timebase_info { numer: 0, denom: 0 };
            unsafe { libc::mach_timebase_info(&mut rate) };
            (PTime::from(rate.numer), PTime::from(rate.denom))
        });
        unsafe { libc::mach_absolute_time() }.wrapping_mul(numer) / denom
    }
 }
 #[cfg(all(unix, not(target_vendor = "apple")))]
 mod platform {
    use super::PTime;
    pub fn monotonic_ns() -> PTime {
        // Zero-initialized like the C source, which works around timespec_get
        // msan limitations on some targets.
        let mut time: libc::timespec = unsafe { std::mem::zeroed() };
        if unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) } != 0 {
            eprintln!(
                "timefn::clock_gettime(CLOCK_MONOTONIC): {}",
                std::io::Error::last_os_error()
            );
            std::process::abort();
        }
        (time.tv_sec as PTime)
            .wrapping_mul(1_000_000_000)
            .wrapping_add(time.tv_nsec as PTime)
    }
 }
 /// Returns the current value of the platform's monotonic nanosecond clock.
 #[no_mangle]
 pub extern "C" fn UTIL_getTime() -> UTIL_time_t {
    UTIL_time_t {
        t: platform::monotonic_ns(),
    }
 }
 /// Nanoseconds elapsed between two measurements, with C unsigned wrap-around.
 #[no_mangle]
 pub extern "C" fn UTIL_getSpanTimeNano(clockStart: UTIL_time_t, clockEnd: UTIL_time_t) -> PTime {
    clockEnd.t.wrapping_sub(clockStart.t)
 }
 /// Microseconds elapsed between two measurements, truncated like C division.
 #[no_mangle]
 pub extern "C" fn UTIL_getSpanTimeMicro(begin: UTIL_time_t, end: UTIL_time_t) -> PTime {
    UTIL_getSpanTimeNano(begin, end) / 1000
 }
 /// Microseconds elapsed since `clockStart`.
 #[no_mangle]
 pub extern "C" fn UTIL_clockSpanMicro(clockStart: UTIL_time_t) -> PTime {
    UTIL_getSpanTimeMicro(clockStart, UTIL_getTime())
 }
 /// Nanoseconds elapsed since `clockStart`.
 #[no_mangle]
 pub extern "C" fn UTIL_clockSpanNano(clockStart: UTIL_time_t) -> PTime {
    UTIL_getSpanTimeNano(clockStart, UTIL_getTime())
 }
 /// Busy-waits until the clock produces a new tick, improving measurement
 /// accuracy on platforms with a low timer resolution.
 #[no_mangle]
 pub extern "C" fn UTIL_waitForNextTick() {
    let clockStart = UTIL_getTime();
    loop {
        let clockEnd = UTIL_getTime();
        if UTIL_getSpanTimeNano(clockStart, clockEnd) != 0 {
            return;
        }
    }
 }
 /// All clock sources used by the Rust port are valid under multi-threaded
 /// workloads; only the C90 `clock()` fallback of the C source was not.
 #[no_mangle]
 pub extern "C" fn UTIL_support_MT_measurements() -> c_int {
    1
 }
 #[cfg(test)]
 mod tests {
    use super::*;
    #[test]
    fn nanosecond_spans_subtract_with_unsigned_wrap_around() {
        let start = UTIL_time_t { t: 100 };
        let end = UTIL_time_t { t: 350 };
        assert_eq!(UTIL_getSpanTimeNano(start, end), 250);
        assert_eq!(UTIL_getSpanTimeNano(end, start), u64::MAX - 249);
        assert_eq!(UTIL_getSpanTimeNano(start, start), 0);
    }
    #[test]
    fn microsecond_spans_truncate_sub_tick_remainders() {
        let start = UTIL_time_t { t: 0 };
        assert_eq!(UTIL_getSpanTimeMicro(start, UTIL_time_t { t: 999 }), 0);
        assert_eq!(UTIL_getSpanTimeMicro(start, UTIL_time_t { t: 1_000 }), 1);
        assert_eq!(UTIL_getSpanTimeMicro(start, UTIL_time_t { t: 1_999 }), 1);
        assert_eq!(UTIL_getSpanTimeMicro(start, UTIL_time_t { t: 2_000 }), 2);
    }
    #[test]
    fn clock_is_monotonic_across_measurements() {
        let first = UTIL_getTime();
        let second = UTIL_getTime();
        assert!(second.t >= first.t);
        assert!(UTIL_clockSpanNano(first) >= UTIL_getSpanTimeNano(first, second));
    }
    #[test]
    fn waiting_for_the_next_tick_advances_the_clock() {
        let before = UTIL_getTime();
        UTIL_waitForNextTick();
        let after = UTIL_getTime();
        assert!(UTIL_getSpanTimeNano(before, after) > 0);
    }
    #[test]
    fn multi_threaded_measurements_are_supported() {
        assert_eq!(UTIL_support_MT_measurements(), 1);
    }
 }
@@ -10,15 +10,9 @@
 //! writes, dictionary loading, streaming, and metadata preservation remain in
 //! `programs/fileio.c` for this first migration step.
 //!
 //! Benchmark mode (`-b`) parses here and dispatches through the
 //! `ZSTD_rust_cli_bench` bridge in `programs/zstdcli.c`: the run/timing loop
 //! (benchfn, timefn) is Rust, while orchestration and result formatting
 //! (`benchzstd.c`) remain C behind the preprocessor-gated bridge, so builds
 //! with `ZSTD_NOBENCH` never reference benchmark symbols.
 //!
 //! Remaining C-only CLI boundaries are called out in `unsupported()` below:
-//! dictionary training, recursive/file-list expansion, tracing,
+//! benchmark execution, dictionary training, recursive/file-list expansion,
-//! alternate-format selection, and the advanced directory modes.
+//! tracing, alternate-format selection, and the advanced directory modes.
 use std::env;
 use std::ffi::{CStr, CString, OsStr, OsString};
@@ -36,7 +30,6 @@ use std::os::unix::fs::FileTypeExt;
 const DEFAULT_CLEVEL: i32 = 3;
 #[cfg(feature = "compression")]
 const DEFAULT_MAX_CLEVEL: i32 = 19;
 const DEFAULT_BENCH_NB_SECONDS: u32 = 3;
 const DEFAULT_MEM_LIMIT: u32 = 1 << 27;
 const DEFAULT_LONG_WINDOW_LOG: u32 = 27;
 const MAX_FAST_ACCELERATION: i32 = 128 << 10;
@@ -180,22 +173,6 @@ unsafe extern "C" {
        output: *const c_char,
        dict: *const c_char,
    ) -> c_int;
    /// Benchmark bridge implemented by the `programs/zstdcli.c` shim, which
    /// owns the `ZSTD_NOBENCH` preprocessor decision. Returns the benchmark
    /// result (>= 0), or -1 when benchmarking is compiled out.
    fn ZSTD_rust_cli_bench(
        file_names: *const *const c_char,
        nb_files: c_uint,
        dict_file_name: *const c_char,
        start_level: c_int,
        end_level: c_int,
        compression_params: *const ZSTD_compressionParameters,
        display_level: c_int,
        nb_seconds: c_uint,
        block_size: usize,
        nb_workers: c_int,
    ) -> c_int;
 }
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -203,7 +180,6 @@ enum Operation {
    Compress,
    Decompress,
    Test,
    Bench,
 }
 #[derive(Debug)]
@@ -253,8 +229,6 @@ struct Cli {
    row_match_finder: i32,
    exclude_compressed: bool,
    compression_params: ZSTD_compressionParameters,
    bench_end_level: Option<i32>,
    bench_nb_seconds: Option<u32>,
    unsupported_program: Option<String>,
 }
@@ -299,8 +273,6 @@ impl Cli {
            row_match_finder: ZSTD_PS_AUTO,
            exclude_compressed: false,
            compression_params: ZSTD_compressionParameters::default(),
            bench_end_level: None,
            bench_nb_seconds: None,
            unsupported_program: None,
        };
@@ -427,22 +399,9 @@ fn usage(advanced: bool) {
            out,
            "  --adapt[=min=#,max=#], --rsyncable, --[no-]row-match-finder"
        );
        let _ = writeln!(out, "\nBenchmark options:");
        let _ = writeln!(
            out,
-            "  -b#                           Benchmark file(s) at compression level #"
+            "\nNot yet migrated: benchmark, dictionary training, recursive/file-list expansion,"
        );
        let _ = writeln!(
            out,
            "  -e#                           Test all levels from -b# up to # included"
        );
        let _ = writeln!(
            out,
            "  -i#                           Set the minimum evaluation time to # seconds"
        );
        let _ = writeln!(
            out,
            "\nNot yet migrated: dictionary training, recursive/file-list expansion,"
        );
        let _ = writeln!(out, "trace, alternate formats, and output-directory modes.");
    }
@@ -933,33 +892,6 @@ fn parse_short_options(
            'd' => cli.operation = Operation::Decompress,
            'z' => cli.operation = Operation::Compress,
            't' => cli.operation = Operation::Test,
            'b' => cli.operation = Operation::Bench,
            'e' | 'i' => {
                // Benchmark range end (-e#) and duration (-i#): like the C
                // parser, digits attach directly and default to 0.
                let mut digits_end = offset + 1;
                while digits_end < bytes.len() && bytes[digits_end].is_ascii_digit() {
                    digits_end += 1;
                }
                let digits = &value[offset + 1..digits_end];
                if option == 'e' {
                    cli.bench_end_level = Some(if digits.is_empty() {
                        0
                    } else {
                        parse_i32(digits, "benchmark end level")?
                    });
                } else {
                    cli.bench_nb_seconds = Some(if digits.is_empty() {
                        0
                    } else {
                        digits
                            .parse::<u32>()
                            .map_err(|_| format!("invalid benchmark duration: {digits:?}"))?
                    });
                }
                offset = digits_end;
                continue;
            }
            'c' => {
                cli.output = Some(cstring(STDOUT_MARK)?);
                cli.force_stdout = true;
@@ -995,7 +927,7 @@ fn parse_short_options(
                }
                break;
            }
-            'l' | 'p' | 'P' | 'r' | 's' | 'S' => {
+            'b' | 'e' | 'i' | 'l' | 'p' | 'P' | 'r' | 's' | 'S' => {
                unsupported(&format!("-{option}"))?;
            }
            _ => return Err(format!("unknown option -{option}")),
@@ -1286,52 +1218,16 @@ unsafe fn run_decompress(
                dictionary,
            )
        },
-        Operation::Compress | Operation::Bench => {
+        Operation::Compress => unreachable!("compression is dispatched separately"),
            unreachable!("compression and benchmark are dispatched separately")
        }
    }
 }
 /// Runs benchmark mode through the C bridge. Level clamping against
 /// `ZSTD_maxCLevel()` happens on the C side, where the symbol is always
 /// available when benchmarking is compiled in. No input file means a
 /// synthetic-sample benchmark, matching the C CLI.
 fn run_bench(cli: &Cli) -> Result<i32, String> {
    let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
    let dictionary = cli
        .dictionary
        .as_ref()
        .map_or(ptr::null(), |value| value.as_ptr());
    let result = unsafe {
        ZSTD_rust_cli_bench(
            inputs.as_ptr(),
            inputs.len() as c_uint,
            dictionary,
            cli.level,
            cli.bench_end_level.unwrap_or(cli.level),
            &cli.compression_params,
            cli.display_level,
            cli.bench_nb_seconds.unwrap_or(DEFAULT_BENCH_NB_SECONDS),
            cli.block_size.unwrap_or(0),
            // The C CLI benchmarks single-threaded unless -T was given.
            cli.workers.unwrap_or(1),
        )
    };
    if result < 0 {
        return Err("benchmark mode is not available in this build".to_owned());
    }
    Ok(result)
 }
 fn run_cli(mut cli: Cli) -> Result<i32, String> {
    if let Some(program_name) = &cli.unsupported_program {
        return Err(format!(
            "{program_name} compatibility mode is not yet implemented by the Rust CLI frontend"
        ));
    }
    if cli.operation == Operation::Bench {
        return run_bench(&cli);
    }
    let explicit_input_count = cli.inputs.len();
    filter_symlink_inputs(&mut cli);
    if explicit_input_count > 0 && cli.inputs.is_empty() {
@@ -1438,7 +1334,6 @@ fn run_cli(mut cli: Cli) -> Result<i32, String> {
                #[cfg(not(feature = "decompression"))]
                unreachable!("unsupported decompression was rejected above")
            }
            Operation::Bench => unreachable!("benchmark mode was dispatched earlier"),
        }
    };
@@ -1643,41 +1538,4 @@ mod tests {
        assert!(error.contains("not yet implemented"));
    }
    #[test]
    fn bench_mode_parses_level_duration_and_defaults() {
        let cli = parse(&["zstd", "-b1", "-i0", "input"]);
        assert_eq!(cli.operation, Operation::Bench);
        assert_eq!(cli.level, 1);
        assert_eq!(cli.bench_nb_seconds, Some(0));
        assert_eq!(cli.bench_end_level, None);
        assert_eq!(
            cli.inputs
                .iter()
                .map(|input| input.as_bytes())
                .collect::<Vec<_>>(),
            vec![&b"input"[..]]
        );
    }
    #[test]
    fn bench_range_aggregates_within_a_single_argument() {
        let cli = parse(&["zstd", "-b5e6i2", "input"]);
        assert_eq!(cli.operation, Operation::Bench);
        assert_eq!(cli.level, 5);
        assert_eq!(cli.bench_end_level, Some(6));
        assert_eq!(cli.bench_nb_seconds, Some(2));
    }
    #[test]
    fn bench_duration_without_digits_defaults_to_zero() {
        let cli = parse(&["zstd", "-b", "-e", "-i"]);
        assert_eq!(cli.operation, Operation::Bench);
        assert_eq!(cli.level, DEFAULT_CLEVEL);
        assert_eq!(cli.bench_end_level, Some(0));
        assert_eq!(cli.bench_nb_seconds, Some(0));
    }
 }
@@ -79,6 +79,8 @@ RUST_TARGET_DIR := $(RUST_DIR)/target/$(RUST_BUILD_CONFIG)
 RUST_STATICLIB := $(RUST_TARGET_DIR)/release/libzstd_rs.a
 RUST_TARGET_32 ?= i686-unknown-linux-gnu
 RUST_STATICLIB_32 := $(RUST_TARGET_DIR)/$(RUST_TARGET_32)/release/libzstd_rs.a
 # Tests build every library module, so they use the crate's default feature
 # set (compression, decompression, and dict-builder) plus any forced HUF mode.
 RUST_CARGO_FLAGS := --manifest-path $(RUST_MANIFEST) --release \
                    --target-dir $(RUST_TARGET_DIR)
 ifneq ($(RUST_HUF_FEATURE),)
@@ -91,28 +93,6 @@ $(RUST_STATICLIB): $(RUST_SOURCES)
 $(RUST_STATICLIB_32): $(RUST_SOURCES)
 	$(CARGO) build $(RUST_CARGO_FLAGS) --target $(RUST_TARGET_32)
 # Program-only helpers that were C sources shared with the tests (timefn,
 # benchfn) now live in the Rust CLI package. The tests link a helpers-only
 # archive, built without the `cli` feature: the parser/dispatch layer needs
 # the C fileio backend, which test binaries do not provide.
 RUST_CLI_DIR := $(RUST_DIR)/cli
 RUST_CLI_MANIFEST := $(RUST_CLI_DIR)/Cargo.toml
 RUST_CLI_HELPER_SOURCES := $(RUST_CLI_MANIFEST) $(RUST_CLI_DIR)/Cargo.lock \
                           $(RUST_CLI_DIR)/src/lib.rs \
                           $(RUST_DIR)/src/timefn.rs $(RUST_DIR)/src/benchfn.rs
 RUST_CLI_HELPERS_TARGET_DIR := $(RUST_DIR)/target/cli-helpers
 RUST_CLI_HELPERS_STATICLIB := $(RUST_CLI_HELPERS_TARGET_DIR)/release/libzstd_cli_rs.a
 RUST_CLI_HELPERS_STATICLIB_32 := $(RUST_CLI_HELPERS_TARGET_DIR)/$(RUST_TARGET_32)/release/libzstd_cli_rs.a
 RUST_CLI_HELPERS_CARGO_FLAGS := --manifest-path $(RUST_CLI_MANIFEST) --release \
                                --target-dir $(RUST_CLI_HELPERS_TARGET_DIR) \
                                --no-default-features
 $(RUST_CLI_HELPERS_STATICLIB): $(RUST_CLI_HELPER_SOURCES)
 	$(CARGO) build $(RUST_CLI_HELPERS_CARGO_FLAGS)
 $(RUST_CLI_HELPERS_STATICLIB_32): $(RUST_CLI_HELPER_SOURCES)
 	$(CARGO) build $(RUST_CLI_HELPERS_CARGO_FLAGS) --target $(RUST_TARGET_32)
 # These test objects have flat filenames, unlike the configuration-hashed
 # program objects. Track the HUF mode separately so a C object set compiled
 # for one decoder is never relinked with a Rust archive for another decoder.
@@ -272,8 +252,8 @@ fuzzer32 : $(ZSTD_FILES)
 	$(LINK.c) $^ -o $@$(EXT)
 # note : broken : requires symbols unavailable from dynamic library
-fuzzer-dll : $(LIB_SRCDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c $(RUST_CLI_HELPERS_STATICLIB)
+fuzzer-dll : $(LIB_SRCDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
-	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(RUST_CLI_HELPERS_STATICLIB) $(LDFLAGS) -o $@$(EXT)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
 CLEAN += zstreamtest zstreamtest32
 ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c external_matchfinder.c
@@ -304,8 +284,8 @@ zstreamtest_ubsan : $(ZSTREAMFILES)
 # note : broken : requires symbols unavailable from dynamic library
 zstreamtest-dll : $(LIB_SRCDIR)/common/xxhash.c  # xxh symbols not exposed from dll
-zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(RUST_CLI_HELPERS_STATICLIB)
+zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(RUST_CLI_HELPERS_STATICLIB) $(LDFLAGS) -o $@$(EXT)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
 CLEAN += paramgrill
 paramgrill : DEBUGFLAGS =   # turn off debug for speed measurements
@@ -364,17 +344,6 @@ $(RUST_LINK_TARGETS_32): $(RUST_STATICLIB_32)
 $(RUST_LINK_TARGETS) $(RUST_LINK_TARGETS_32): $(RUST_HUF_C_MODE_STAMP)
 # Tests that compile the timefn/benchfn C shims also link the Rust CLI
 # helpers archive, which owns those implementations. The archive is a
 # prerequisite so `$^` places it after every C object referencing its symbols.
 RUST_CLI_LINK_TARGETS := fullbench fullbench-lib fullbench-dll fuzzer \
                         zstreamtest zstreamtest_asan zstreamtest_tsan \
                         zstreamtest_ubsan paramgrill decodecorpus poolTests
 $(RUST_CLI_LINK_TARGETS): $(RUST_CLI_HELPERS_STATICLIB)
 RUST_CLI_LINK_TARGETS_32 := fullbench32 fuzzer32 zstreamtest32
 $(RUST_CLI_LINK_TARGETS_32): $(RUST_CLI_HELPERS_STATICLIB_32)
 .PHONY: versionsTest
 versionsTest: clean
 	$(PYTHON) test-zstd-versions.py