notificationLevel into ZDICT_param_t

lib/zdict.c

@@ -102,7 +102,6 @@ static const size_t g_min_fast_dictContent = 192;
 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 static unsigned g_displayLevel = 0;   /* 0 : no display;   1: errors;   2: default;  4: full information */
-void ZDICT_setNotificationLevel(unsigned l) { g_displayLevel=l; }
 
 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
             if (ZDICT_GetMilliSpan(g_time) > refreshRate)  \
@@ -111,7 +110,7 @@ void ZDICT_setNotificationLevel(unsigned l) { g_displayLevel=l; }
 static const unsigned refreshRate = 300;
 static clock_t g_time = 0;
 
-void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
+static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
 {
     const BYTE* const b = (const BYTE*)ptr;
     size_t u;
@@ -813,6 +812,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
     { unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += sampleSizes[u]; }
     if (!dictList) return ERROR(memory_allocation);
     ZDICT_initDictItem(dictList);
+    g_displayLevel = params.notificationLevel;
     if (selectivity==0) selectivity = g_selectivity_default;
     if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
 

lib/zdict_static.h

@@ -53,6 +53,8 @@ extern "C" {
 typedef struct {
     unsigned selectivityLevel;   /* 0 means default; larger => bigger selection => larger dictionary */
     unsigned compressionLevel;   /* 0 means default; target a specific zstd compression level */
+    unsigned notificationLevel;  /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
+    unsigned reserved[3];        /* space for future parameters */
 } ZDICT_params_t;
 
 
@@ -71,32 +73,6 @@ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacit
                              ZDICT_params_t parameters);
 
 
-/*-*************************************
-*  Helper functions
-***************************************/
-/*! ZDICT_setNotificationLevel() :
-    Set amount of notification to be displayed on the console.
-    default : 0 = no console notification.
-    1 = errors; 2 = notifications; 3 = details; 4 = debug;
-    Note : not thread-safe (uses a global constant)
-*/
-void ZDICT_setNotificationLevel(unsigned l);
-
-
-/*-*************************************
-*  Private functions
-***************************************/
-/*! ZDICT_trainFromBuffer_unsafe() :
-    Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
-    note : `samplesBuffer` must be followed by noisy guard band to avoid out-of-buffer reads.
-    @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
-              or an error code.
-*/
-size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
-                              const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
-                              ZDICT_params_t parameters);
-
-
 #if defined (__cplusplus)
 }
 #endif

programs/dibio.c

@@ -85,7 +85,6 @@ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_
 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 static unsigned g_displayLevel = 0;   /* 0 : no display;   1: errors;   2: default;  4: full information */
-void DiB_setNotificationLevel(unsigned l) { g_displayLevel=l; ZDICT_setNotificationLevel(l); }
 
 
 /*-*************************************
@@ -216,6 +215,18 @@ static void DiB_saveDict(const char* dictFileName,
 }
 
 
+/*! ZDICT_trainFromBuffer_unsafe() :
+    Strictly Internal use only !!
+    Same as ZDICT_trainFromBuffer_advanced(), but does not control `samplesBuffer`.
+    `samplesBuffer` must be followed by noisy guard band to avoid out-of-buffer reads.
+    @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
+              or an error code.
+*/
+size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
+                              const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+                              ZDICT_params_t parameters);
+
+
 int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                        const char** fileNamesTable, unsigned nbFiles,
                        ZDICT_params_t params)
@@ -229,6 +240,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
     int result = 0;
 
     /* init */
+    g_displayLevel = params.notificationLevel;
     benchedSize = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
     if ((unsigned long long)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
     if (benchedSize < totalSizeToLoad)

programs/dibio.h

@@ -49,15 +49,4 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                        ZDICT_params_t parameters);
 
 
-/*-*************************************
-*  Helper functions
-***************************************/
-/*! DiB_setNotificationLevel
-    Set amount of notification to be displayed on the console.
-    default initial value : 0 = no console notification.
-    Note : not thread-safe (use a global constant)
-*/
-void DiB_setNotificationLevel(unsigned l);
-
-
 #endif

programs/fileio.c

@@ -52,7 +52,7 @@
 #define _POSIX_SOURCE 1        /* enable fileno() within <stdio.h> on unix */
 
 
-/* *************************************
+/*-*************************************
 *  Includes
 ***************************************/
 #include <stdio.h>      /* fprintf, fopen, fread, _fileno, stdin, stdout */
@@ -68,12 +68,12 @@
 #include "zbuff_static.h"
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-#  include "zstd_legacy.h"    /* legacy */
+#  include "zstd_legacy.h"    /* ZSTD_isLegacy */
-#  include "fileio_legacy.h"  /* legacy */
+#  include "fileio_legacy.h"  /* FIO_decompressLegacyFrame */
 #endif
 
 
-/* *************************************
+/*-*************************************
 *  OS-specific Includes
 ***************************************/
 #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
@@ -92,7 +92,7 @@
 #endif
 
 
-/* *************************************
+/*-*************************************
 *  Constants
 ***************************************/
 #define KB *(1U<<10)
@@ -112,12 +112,13 @@
 #define BLOCKSIZE      (128 KB)
 #define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
 
-#define FIO_FRAMEHEADERSIZE 5        /* as a define, because needed to allocated table on stack */
+#define FIO_FRAMEHEADERSIZE  5        /* as a define, because needed to allocated table on stack */
-#define FSE_CHECKSUM_SEED        0
+#define FSE_CHECKSUM_SEED    0
 
 #define CACHELINE 64
 
-#define MAX_DICT_SIZE (512 KB)
+#define MAX_DICT_SIZE (1 MB)   /* protection against large input (attack scenario) ; can be changed */
+
 
 /* *************************************
 *  Macros
@@ -138,7 +139,6 @@ static clock_t g_time = 0;
 *  Local Parameters
 ***************************************/
 static U32 g_overwrite = 0;
-
 void FIO_overwriteMode(void) { g_overwrite=1; }
 void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 

programs/zstdcli.c

@@ -133,7 +133,7 @@ static int usage_advanced(const char* programName)
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
 #ifndef ZSTD_NODICT
     DISPLAY( "Dictionary builder :\n");
-    DISPLAY( "--train : Create a dictionary from a set of files \n");
+    DISPLAY( "--train : create a dictionary from a training set of files \n");
     DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName);
     DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
     DISPLAY( " -s#    : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
@@ -366,7 +366,7 @@ int main(int argCount, const char** argv)
         ZDICT_params_t dictParams;
         dictParams.compressionLevel = dictCLevel;
         dictParams.selectivityLevel = dictSelect;
-        DiB_setNotificationLevel(displayLevel);
+        dictParams.notificationLevel = displayLevel;
         DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams);
 #endif
         goto _end;