Merge pull request #4609 from Integral-Tech/memory-g-suffix

Accept G, GB and GiB suffixes for options

programs/zstdcli.c

@@ -347,7 +347,7 @@ static void errorOut(const char* msg)
 
 /*! readU32FromCharChecked() :
  * @return 0 if success, and store the result in *value.
- *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ *  allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
  *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
  * @return 1 if an overflow error occurs */
 static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
@@ -362,15 +362,22 @@ static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
         if (result < last) return 1; /* overflow error */
         (*stringPtr)++ ;
     }
-    if ((**stringPtr=='K') || (**stringPtr=='M')) {
+    if ((**stringPtr=='K') || (**stringPtr=='M') || (**stringPtr=='G')) {
-        unsigned const maxK = ((unsigned)(-1)) >> 10;
+        switch (**stringPtr) {
-        if (result > maxK) return 1; /* overflow error */
+            case 'K':
-        result <<= 10;
+                if (result > (((unsigned)-1) >> 10)) return 1; /* overflow error */
-        if (**stringPtr=='M') {
-            if (result > maxK) return 1; /* overflow error */
                 result <<= 10;
+                break;
+            case 'M':
+                if (result > (((unsigned)-1) >> 20)) return 1; /* overflow error */
+                result <<= 20;
+                break;
+            case 'G':
+                if (result > (((unsigned)-1) >> 30)) return 1; /* overflow error */
+                result <<= 30;
+                break;
         }
-        (*stringPtr)++;  /* skip `K` or `M` */
+        (*stringPtr)++;  /* skip `K`, `M` or `G` */
         if (**stringPtr=='i') (*stringPtr)++;
         if (**stringPtr=='B') (*stringPtr)++;
     }
@@ -380,7 +387,7 @@ static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
 
 /*! readU32FromChar() :
  * @return : unsigned integer value read from input in `char` format.
- *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ *  allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
  *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
  *  Note : function will exit() program if digit sequence overflows */
 static unsigned readU32FromChar(const char** stringPtr) {
@@ -392,7 +399,7 @@ static unsigned readU32FromChar(const char** stringPtr) {
 
 /*! readIntFromChar() :
  * @return : signed integer value read from input in `char` format.
- *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ *  allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
  *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
  *  Note : function will exit() program if digit sequence overflows */
 static int readIntFromChar(const char** stringPtr) {
@@ -409,7 +416,7 @@ static int readIntFromChar(const char** stringPtr) {
 
 /*! readSizeTFromCharChecked() :
  * @return 0 if success, and store the result in *value.
- *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ *  allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
  *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
  * @return 1 if an overflow error occurs */
 static int readSizeTFromCharChecked(const char** stringPtr, size_t* value)
@@ -424,15 +431,22 @@ static int readSizeTFromCharChecked(const char** stringPtr, size_t* value)
         if (result < last) return 1; /* overflow error */
         (*stringPtr)++ ;
     }
-    if ((**stringPtr=='K') || (**stringPtr=='M')) {
+    if ((**stringPtr=='K') || (**stringPtr=='M') || (**stringPtr=='G')) {
-        size_t const maxK = ((size_t)(-1)) >> 10;
+        switch (**stringPtr) {
-        if (result > maxK) return 1; /* overflow error */
+            case 'K':
-        result <<= 10;
+                if (result > (((size_t)-1) >> 10)) return 1; /* overflow error */
-        if (**stringPtr=='M') {
-            if (result > maxK) return 1; /* overflow error */
                 result <<= 10;
+                break;
+            case 'M':
+                if (result > (((size_t)-1) >> 20)) return 1; /* overflow error */
+                result <<= 20;
+                break;
+            case 'G':
+                if (result > (((size_t)-1) >> 30)) return 1; /* overflow error */
+                result <<= 30;
+                break;
         }
-        (*stringPtr)++;  /* skip `K` or `M` */
+        (*stringPtr)++;  /* skip `K`, `M` or `G` */
         if (**stringPtr=='i') (*stringPtr)++;
         if (**stringPtr=='B') (*stringPtr)++;
     }
@@ -442,7 +456,7 @@ static int readSizeTFromCharChecked(const char** stringPtr, size_t* value)
 
 /*! readSizeTFromChar() :
  * @return : size_t value read from input in `char` format.
- *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
+ *  allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
  *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
  *  Note : function will exit() program if digit sequence overflows */
 static size_t readSizeTFromChar(const char** stringPtr) {
@@ -830,7 +844,7 @@ static unsigned init_nbWorkers(unsigned defaultNbWorkers) {
     NEXT_FIELD(__nb);                 \
     _varu32 = readU32FromChar(&__nb); \
     if(*__nb != 0) {                  \
-        errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
+        errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed"); \
     }                                 \
 }
 
@@ -839,7 +853,7 @@ static unsigned init_nbWorkers(unsigned defaultNbWorkers) {
     NEXT_FIELD(__nb);                     \
     _varTsize = readSizeTFromChar(&__nb); \
     if(*__nb != 0) {                      \
-        errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
+        errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed"); \
     }                                     \
 }
 

tests/cli-tests/basic/memlimit.sh

@@ -35,6 +35,30 @@ rm file.zst
 println "+ zstd --memory=1MiB file"
 zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
 rm file.zst
+println "+ zstd --memory=1G file"
+zstd -q --memory=1G file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1GB file"
+zstd -q --memory=1GB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=1GiB file"
+zstd -q --memory=1GiB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=3G file"
+zstd -q --memory=3G file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=3GB file"
+zstd -q --memory=3GB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=3GiB file"
+zstd -q --memory=3GiB file && die "Should allow numeric parameter with expected suffix"
+rm file.zst
+println "+ zstd --memory=4G file"
+zstd --memory=4G file && die "Should not allow out-of-bound numeric parameter"
+println "+ zstd --memory=4GB file"
+zstd --memory=4GB file && die "Should not allow out-of-bound numeric parameter"
+println "+ zstd --memory=4GiB file"
+zstd --memory=4GiB file && die "Should not allow out-of-bound numeric parameter"
 
 rm file
 exit 0

tests/cli-tests/basic/memlimit.sh.stderr.exact

@@ -1,9 +1,9 @@
-error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed 
-error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed 
-error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed 
-error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed 
-error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed 
-error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 
+error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed 
 Should allow numeric parameter without suffix
 Should allow numeric parameter with expected suffix
 Should allow numeric parameter with expected suffix
@@ -11,3 +11,12 @@ Should allow numeric parameter with expected suffix
 Should allow numeric parameter with expected suffix
 Should allow numeric parameter with expected suffix
 Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+Should allow numeric parameter with expected suffix
+error: numeric value overflows 32-bit unsigned int 
+error: numeric value overflows 32-bit unsigned int 
+error: numeric value overflows 32-bit unsigned int 

tests/cli-tests/basic/memlimit.sh.stdout.exact

@@ -11,3 +11,12 @@
 + zstd --memory=1M file
 + zstd --memory=1MB file
 + zstd --memory=1MiB file
++ zstd --memory=1G file
++ zstd --memory=1GB file
++ zstd --memory=1GiB file
++ zstd --memory=3G file
++ zstd --memory=3GB file
++ zstd --memory=3GiB file
++ zstd --memory=4G file
++ zstd --memory=4GB file
++ zstd --memory=4GiB file

tests/playTests.sh

@@ -1075,13 +1075,19 @@ cat tmp | zstd -14 -f --size-hint=11050 | zstd -t  # slightly too high
 cat tmp | zstd -14 -f --size-hint=10950 | zstd -t  # slightly too low
 cat tmp | zstd -14 -f --size-hint=22000 | zstd -t  # considerably too high
 cat tmp | zstd -14 -f --size-hint=5500  | zstd -t  # considerably too low
-println "test : allows and interprets K,KB,KiB,M,MB and MiB suffix"
+println "test : allows and interprets K,KB,KiB,M,MB,MiB,G,GB and GiB suffix"
 cat tmp | zstd -14 -f --size-hint=11K | zstd -t
 cat tmp | zstd -14 -f --size-hint=11KB | zstd -t
 cat tmp | zstd -14 -f --size-hint=11KiB | zstd -t
 cat tmp | zstd -14 -f --size-hint=1M  | zstd -t
 cat tmp | zstd -14 -f --size-hint=1MB  | zstd -t
 cat tmp | zstd -14 -f --size-hint=1MiB  | zstd -t
+cat tmp | zstd -14 -f --size-hint=1G  | zstd -t
+cat tmp | zstd -14 -f --size-hint=1GB  | zstd -t
+cat tmp | zstd -14 -f --size-hint=1GiB  | zstd -t
+cat tmp | zstd -14 -f --size-hint=3G  | zstd -t
+cat tmp | zstd -14 -f --size-hint=3GB  | zstd -t
+cat tmp | zstd -14 -f --size-hint=3GiB  | zstd -t
 
 
 println "\n===>  dictionary tests "
@@ -1684,6 +1690,8 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB"
 roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB"
 roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB"
 roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB"
+roundTripTest -g1M -P50 "1 --single-thread --long=30" " --memory=1GB"
+roundTripTest -g1M -P50 "1 --single-thread --long=30" " --zstd=wlog=29 --memory=1GB"
 
 
 if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -ne "1" ]; then