From d0436b2a454f39979ef506be152c597707d7243c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Oct 2020 19:22:45 -0700 Subject: [PATCH 1/9] automatically detect configuration changes Makefile now automatically detects modifications of compilation flags, and produce object files in directories dedicated to this compilation flags. This makes it possible, for example, to compile libzstd with different DEBUGLEVEL. Object files sharing the same configration will be generated into their dedicated directories. Also : new compilation variables - DEBUGLEVEL : select the debug level (assert & traces) inserted during compilation (default == 0 == release) - HASH : select a hash function to differentiate configuration (default == md5sum) - BUILD_DIR : skip the hash stage, store object files into manually specified directory --- lib/Makefile | 56 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 58e80e473..1911bc64f 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -54,17 +54,18 @@ else CFLAGS += -O3 endif -CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_ +DEBUGLEVEL ?= 0 +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL) ifeq ($(TARGET_SYSTEM),Windows_NT) # MinGW assumed - CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting + CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wpointer-arith \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wredundant-decls -Wmissing-prototypes -Wc++-compat -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) GREP_OPTIONS ?= @@ -170,7 +171,29 @@ ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_SRC:.c=.o) ZSTD_SUBDIR := common compress decompress dictBuilder legacy deprecated vpath %.c $(ZSTD_SUBDIR) -BUILD_DIR ?= obj +UNAME := $(shell uname) +ifeq ($(UNAME), Darwin) + HASH ?= md5 +endif +ifeq ($(UNAME), FreeBSD) + HASH ?= gmd5sum +endif +ifeq ($(UNAME), OpenBSD) + HASH ?= md5 +endif +HASH ?= md5sum + +HAVE_HASH := $(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0) +ifeq ($(HAVE_HASH), 1) + HASH_VALUE := $(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) | $(HASH) | head -c 16) + HASH_DIR := conf_$(HASH_VALUE) +else + $(info warning : could not find hash function to differentiate builds with different flags) + HASH_DIR := 0 +endif + + +BUILD_DIR ?= obj/$(HASH_DIR) ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynlib ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/, $(ZSTD_LOCAL_OBJ)) ZSTD_STATLIB_DIR := $(BUILD_DIR)/statlib @@ -200,24 +223,28 @@ lib-all: all all: lib -libzstd.a: ARFLAGS = rcs -libzstd.a: $(ZSTD_STATLIB_OBJ) +$(ZSTD_STATLIB_DIR)/libzstd.a: ARFLAGS = rcs +$(ZSTD_STATLIB_DIR)/libzstd.a: $(ZSTD_STATLIB_OBJ) @echo compiling static library $(Q)$(AR) $(ARFLAGS) $@ $^ +.PHONY: libzstd.a # must be run every time +libzstd.a: $(ZSTD_STATLIB_DIR)/libzstd.a + $(Q)ln -sf $< $@ + ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) LIBZSTD = dll\libzstd.dll -$(LIBZSTD): $(ZSTD_FILES) +$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): $(ZSTD_FILES) @echo compiling dynamic library $(LIBVER) $(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.dll.a -shared $^ -o $@ else LIBZSTD = libzstd.$(SHARED_EXT_VER) -$(LIBZSTD): CFLAGS += -fPIC -$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden -$(LIBZSTD): $(ZSTD_DYNLIB_OBJ) +$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): CFLAGS += -fPIC +$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden +$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): $(ZSTD_DYNLIB_OBJ) @echo compiling dynamic library $(LIBVER) $(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ @echo creating versioned links @@ -226,6 +253,10 @@ $(LIBZSTD): $(ZSTD_DYNLIB_OBJ) endif +.PHONY: $(LIBZSTD) # must be run every time +$(LIBZSTD): $(ZSTD_DYNLIB_DIR)/$(LIBZSTD) + $(Q)ln -sf $< $@ + .PHONY: libzstd libzstd : $(LIBZSTD) @@ -281,8 +312,7 @@ clean: $(Q)$(RM) -r *.dSYM # macOS-specific $(Q)$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc $(Q)$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt* - $(Q)$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o - $(Q)$(RM) -r $(ZSTD_STATLIB_DIR)/* $(ZSTD_DYNLIB_DIR)/* + $(Q)$(RM) -r obj/* @echo Cleaning library completed #----------------------------------------------------------------------------- From 01ecad2326a0161956bf38b9a64a8a0ce4154560 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Oct 2020 23:13:46 -0700 Subject: [PATCH 2/9] zstd in programs/ can also receive a DEBUGLEVEL value simplify tests : only set DEBUGLEVEL, not the flags directly --- Makefile | 10 +++---- appveyor.yml | 2 +- programs/Makefile | 68 +++++++++++++++++++++++------------------------ 3 files changed, 39 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index 1735ab865..9be0557b6 100644 --- a/Makefile +++ b/Makefile @@ -31,9 +31,9 @@ VOID = /dev/null TARGET_SYSTEM ?= $(OS) ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) -EXT =.exe + EXT =.exe else -EXT = + EXT = endif ## default: Build lib-release and zstd-release @@ -65,7 +65,7 @@ lib lib-release lib-all : .PHONY: zstd zstd-release zstd zstd-release: $(Q)$(MAKE) -C $(PRGDIR) $@ - $(Q)cp $(PRGDIR)/zstd$(EXT) . + $(Q)ln -sf $(PRGDIR)/zstd$(EXT) zstd$(EXT) .PHONY: zstdmt zstdmt: @@ -79,9 +79,9 @@ zlibwrapper: lib ## test: run long-duration tests .PHONY: test DEBUGLEVEL ?= 1 -test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror +test: MOREFLAGS += -g -Werror test: - MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants + DEBUGLEVEL=$(DEBUGLEVEL) MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ ZSTD=../../programs/zstd $(MAKE) -C doc/educational_decoder $@ diff --git a/appveyor.yml b/appveyor.yml index 91568e484..6b5b97676 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -194,7 +194,7 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2" + SCRIPT: "CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2" - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" diff --git a/programs/Makefile b/programs/Makefile index 8f14292f3..f6b6a7f8c 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -36,19 +36,20 @@ ZSTD_VERSION = $(LIBVER) HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) GREP_OPTIONS ?= ifeq ($HAVE_COLORNEVER, 1) -GREP_OPTIONS += --color=never + GREP_OPTIONS += --color=never endif GREP = grep $(GREP_OPTIONS) ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1) -ALIGN_LOOP = -falign-loops=32 + ALIGN_LOOP = -falign-loops=32 else -ALIGN_LOOP = + ALIGN_LOOP = endif -CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ +DEBUGLEVEL ?= 0 +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -DDEBUGLEVEL=$(DEBUGLEVEL) ifeq ($(OS),Windows_NT) # MinGW assumed -CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting + CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif CFLAGS ?= -O3 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ @@ -65,11 +66,7 @@ ZSTDLIB_DECOMPRESS := $(ZSTDDIR)/decompress ZDICT_DIR := $(ZSTDDIR)/dictBuilder ZSTDLEGACY_DIR := $(ZSTDDIR)/legacy -vpath %.c $(ZSTDLIB_COMMON) -vpath %.c $(ZSTDLIB_COMPRESS) -vpath %.c $(ZSTDLIB_DECOMPRESS) -vpath %.c $(ZDICT_DIR) -vpath %.c $(ZSTDLEGACY_DIR) +vpath %.c $(ZSTDLIB_COMMON) $(ZSTDLIB_COMPRESS) $(ZSTDLIB_DECOMPRESS) $(ZDICT_DIR) $(ZSTDLEGACY_DIR) ZSTDLIB_COMMON_C := $(wildcard $(ZSTDLIB_COMMON)/*.c) ZSTDLIB_COMPRESS_C := $(wildcard $(ZSTDLIB_COMPRESS)/*.c) @@ -81,7 +78,7 @@ ZSTD_LEGACY_SUPPORT ?= 5 ZSTDLEGACY_SRC := ifneq ($(ZSTD_LEGACY_SUPPORT), 0) ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) - ZSTDLEGACY_SRC += $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') + ZSTDLEGACY_SRC += $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') endif endif @@ -97,16 +94,16 @@ ZSTD_ALL_SRC := $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -RES64_FILE = windres/zstd64.res -RES32_FILE = windres/zstd32.res + EXT =.exe + RES64_FILE = windres/zstd64.res + RES32_FILE = windres/zstd32.res ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine))) RES_FILE = $(RES64_FILE) else RES_FILE = $(RES32_FILE) endif else -EXT = + EXT = endif VOID = /dev/null @@ -119,54 +116,54 @@ NO_THREAD_MSG := ==> no threads, building without multithreading support HAVE_PTHREAD := $(shell printf '$(NUM_SYMBOL)include \nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c) HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0) ifeq ($(HAVE_THREAD), 1) -THREAD_MSG := ==> building with threading support -THREAD_CPP := -DZSTD_MULTITHREAD -THREAD_LD := -pthread + THREAD_MSG := ==> building with threading support + THREAD_CPP := -DZSTD_MULTITHREAD + THREAD_LD := -pthread else -THREAD_MSG := $(NO_THREAD_MSG) + THREAD_MSG := $(NO_THREAD_MSG) endif # zlib detection NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support HAVE_ZLIB := $(shell printf '$(NUM_SYMBOL)include \nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c) ifeq ($(HAVE_ZLIB), 1) -ZLIB_MSG := ==> building zstd with .gz compression support -ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS -ZLIBLD = -lz + ZLIB_MSG := ==> building zstd with .gz compression support + ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS + ZLIBLD = -lz else -ZLIB_MSG := $(NO_ZLIB_MSG) + ZLIB_MSG := $(NO_ZLIB_MSG) endif # lzma detection NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support HAVE_LZMA := $(shell printf '$(NUM_SYMBOL)include \nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c) ifeq ($(HAVE_LZMA), 1) -LZMA_MSG := ==> building zstd with .xz/.lzma compression support -LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS -LZMALD = -llzma + LZMA_MSG := ==> building zstd with .xz/.lzma compression support + LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS + LZMALD = -llzma else -LZMA_MSG := $(NO_LZMA_MSG) + LZMA_MSG := $(NO_LZMA_MSG) endif # lz4 detection NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support HAVE_LZ4 := $(shell printf '$(NUM_SYMBOL)include \n$(NUM_SYMBOL)include \nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c) ifeq ($(HAVE_LZ4), 1) -LZ4_MSG := ==> building zstd with .lz4 compression support -LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS -LZ4LD = -llz4 + LZ4_MSG := ==> building zstd with .lz4 compression support + LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS + LZ4LD = -llz4 else -LZ4_MSG := $(NO_LZ4_MSG) + LZ4_MSG := $(NO_LZ4_MSG) endif # explicit backtrace enable/disable for Linux & Darwin ifeq ($(BACKTRACE), 0) -DEBUGFLAGS += -DBACKTRACE_ENABLE=0 + DEBUGFLAGS += -DBACKTRACE_ENABLE=0 endif ifeq (,$(filter Windows%, $(OS))) ifeq ($(BACKTRACE), 1) -DEBUGFLAGS += -DBACKTRACE_ENABLE=1 -DEBUGFLAGS_LD += -rdynamic + DEBUGFLAGS += -DBACKTRACE_ENABLE=1 + DEBUGFLAGS_LD += -rdynamic endif endif @@ -333,7 +330,8 @@ COMPILE.c = $(CC) $(DEPFLAGS) $(CFLAGS) $(CPPFLAGS) -c %.o : %.c %.o : %.c $(DEPDIR)/%.d | $(DEPDIR) - $(COMPILE.c) $(OUTPUT_OPTION) $< + @echo $@ + $(Q)$(COMPILE.c) $(OUTPUT_OPTION) $< $(DEPDIR): ; $(Q)$(MKDIR) -p $@ From dd24496951dbce5615b26f2632b4dd9e7e494502 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Oct 2020 23:38:33 -0700 Subject: [PATCH 3/9] programs/zstd also automatically generate object dir per conf same rules as lib/libzstd can also be controlled via HASH and BUILD_DIR --- lib/Makefile | 8 ++--- programs/Makefile | 80 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 1911bc64f..97ce0e67e 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -201,7 +201,7 @@ ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/, $(ZSTD_LOCAL_OBJ)) # macOS linker doesn't support -soname, and use different extension # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html -ifeq ($(shell uname), Darwin) +ifeq ($(UNAME), Darwin) SHARED_EXT = dylib SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) @@ -318,7 +318,7 @@ clean: #----------------------------------------------------------------------------- # make install is validated only for below listed environments #----------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) +ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) all: libzstd.pc @@ -354,13 +354,13 @@ $(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(PREFI endif endif -ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly)) +ifneq (,$(filter $(UNAME),FreeBSD NetBSD DragonFly)) PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig else PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig endif -ifneq (,$(filter $(shell uname),SunOS)) +ifneq (,$(filter $(UNAME),SunOS)) INSTALL ?= ginstall else INSTALL ?= install diff --git a/programs/Makefile b/programs/Makefile index f6b6a7f8c..575931f8c 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -91,6 +91,31 @@ ZSTD_CLI_SRC := $(wildcard *.c) ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o) ZSTD_ALL_SRC := $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC) +ZSTD_ALL_OBJ := $(ZSTD_ALL_SRC:.c=.o) + +UNAME := $(shell uname) +ifeq ($(UNAME), Darwin) + HASH ?= md5 +endif +ifeq ($(UNAME), FreeBSD) + HASH ?= gmd5sum +endif +ifeq ($(UNAME), OpenBSD) + HASH ?= md5 +endif +HASH ?= md5sum + +HAVE_HASH := $(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0) +ifeq ($(HAVE_HASH), 1) + HASH_VALUE := $(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) | $(HASH) | head -c 16) + HASH_DIR := conf_$(HASH_VALUE) +else + $(info warning : could not find hash function to differentiate builds with different flags) + HASH_DIR := 0 +endif + +BUILD_DIR ?= obj/$(HASH_DIR) +ZSTD_OBJ := $(addprefix $(BUILD_DIR)/, $(ZSTD_ALL_OBJ)) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -177,18 +202,23 @@ all: zstd .PHONY: allVariants allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-dictBuilder +$(BUILD_DIR)/zstd : $(ZSTD_OBJ) + @echo "$(THREAD_MSG)" + @echo "$(ZLIB_MSG)" + @echo "$(LZMA_MSG)" + @echo "$(LZ4_MSG)" + $(Q)$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) + +.PHONY: zstd zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP) zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD) zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ifneq (,$(filter Windows%,$(OS))) zstd : $(RES_FILE) endif -zstd : $(ZSTDLIB_LOCAL_OBJ) $(ZSTD_CLI_OBJ) - @echo "$(THREAD_MSG)" - @echo "$(ZLIB_MSG)" - @echo "$(LZMA_MSG)" - @echo "$(LZ4_MSG)" - $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) +zstd : $(BUILD_DIR)/zstd + $(Q)ln -sf $^ $@ + @echo zstd build completed .PHONY: zstd-release zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0 @@ -289,6 +319,7 @@ clean: zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \ zstd-dictBuilder$(EXT) *.gcda default*.profraw default.profdata have_zlib$(EXT) + $(Q)$(RM) -r obj/* @echo Cleaning completed MD2ROFF = ronn @@ -321,35 +352,32 @@ preview-man: clean-man man # Generate .h dependencies automatically -MKDIR = mkdir +DEPFLAGS = -MT $@ -MMD -MP -MF -DEPDIR := .deps -DEPFLAGS = -MT $@ -MMD -MP -MF $(DEPDIR)/$*.d - -COMPILE.c = $(CC) $(DEPFLAGS) $(CFLAGS) $(CPPFLAGS) -c - -%.o : %.c -%.o : %.c $(DEPDIR)/%.d | $(DEPDIR) +$(BUILD_DIR)/%.o : %.c $(BUILD_DIR)/%.d | $(BUILD_DIR) @echo $@ - $(Q)$(COMPILE.c) $(OUTPUT_OPTION) $< + $(Q)$(COMPILE.c) $(DEPFLAGS) $(BUILD_DIR)/$*.d $(OUTPUT_OPTION) $< -$(DEPDIR): ; $(Q)$(MKDIR) -p $@ +MKDIR ?= mkdir +$(BUILD_DIR): ; $(Q)$(MKDIR) -p $@ -DEPFILES := $(ZSTD_ALL_SRC:%.c=$(DEPDIR)/%.d) +DEPFILES := $(ZSTD_OBJ:.o=.d) $(DEPFILES): include $(wildcard $(DEPFILES)) + #----------------------------------------------------------------------------- # make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets #----------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) +UNAME := $(shell uname) +ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku)) HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) EGREP_OPTIONS ?= ifeq ($HAVE_COLORNEVER, 1) -EGREP_OPTIONS += --color=never + EGREP_OPTIONS += --color=never endif EGREP = egrep $(EGREP_OPTIONS) AWK = awk @@ -387,17 +415,17 @@ datarootdir ?= $(PREFIX)/share mandir ?= $(datarootdir)/man man1dir ?= $(mandir)/man1 -ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) -MANDIR ?= $(PREFIX)/man -MAN1DIR ?= $(MANDIR)/man1 +ifneq (,$(filter $(UNAME),OpenBSD FreeBSD NetBSD DragonFly SunOS)) + MANDIR ?= $(PREFIX)/man + MAN1DIR ?= $(MANDIR)/man1 else -MAN1DIR ?= $(man1dir) + MAN1DIR ?= $(man1dir) endif -ifneq (,$(filter $(shell uname),SunOS)) -INSTALL ?= ginstall +ifneq (,$(filter $(UNAME),SunOS)) + INSTALL ?= ginstall else -INSTALL ?= install + INSTALL ?= install endif INSTALL_PROGRAM ?= $(INSTALL) From ca75da8fa3fc834eeb3d9dcfb5c09967976ea981 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Oct 2020 23:51:13 -0700 Subject: [PATCH 4/9] fix test DEBUGLEVEL redefinition --- tests/Makefile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index d347a948a..87252ea80 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -96,7 +96,7 @@ dll: fuzzer-dll zstreamtest-dll PHONY: zstd zstd32 zstd-nolegacy # must be phony, only external makefile knows how to build them, or if they need an update zstd zstd32 zstd-nolegacy: - $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" + $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" DEBUGLEVEL=$(DEBUGLEVEL) gzstd: $(MAKE) -C $(PRGDIR) $@ HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)" @@ -255,7 +255,8 @@ clean: #---------------------------------------------------------------------------------- # valgrind tests are validated only for some posix platforms #---------------------------------------------------------------------------------- -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) +UNAME := $(shell uname) +ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) HOST_OS = POSIX valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1 @@ -275,8 +276,8 @@ valgrindTest: zstd datagen fuzzer fullbench endif -ifneq (,$(filter MINGW% MSYS%,$(shell uname))) -HOST_OS = MSYS +ifneq (,$(filter MINGW% MSYS%,$(UNAME))) + HOST_OS = MSYS endif @@ -286,8 +287,8 @@ endif ifneq (,$(filter $(HOST_OS),MSYS POSIX)) DIFF:=diff -ifneq (,$(filter $(shell uname),SunOS)) -DIFF:=gdiff +ifneq (,$(filter $(UNAME),SunOS)) + DIFF:=gdiff endif .PHONY: list From 494f7169ed7431280cb6c44decf7c73783a51dba Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Oct 2020 00:15:31 -0700 Subject: [PATCH 5/9] fix directory creation for Windows' libzstd --- lib/Makefile | 3 +++ tests/Makefile | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/Makefile b/lib/Makefile index 97ce0e67e..8924bfb43 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -224,6 +224,7 @@ lib-all: all all: lib $(ZSTD_STATLIB_DIR)/libzstd.a: ARFLAGS = rcs +$(ZSTD_STATLIB_DIR)/libzstd.a: | $(ZSTD_STATLIB_DIR) $(ZSTD_STATLIB_DIR)/libzstd.a: $(ZSTD_STATLIB_OBJ) @echo compiling static library $(Q)$(AR) $(ARFLAGS) $@ $^ @@ -235,6 +236,7 @@ libzstd.a: $(ZSTD_STATLIB_DIR)/libzstd.a ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) LIBZSTD = dll\libzstd.dll +$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): | $(ZSTD_DYNLIB_DIR) $(ZSTD_DYNLIB_DIR)/$(LIBZSTD): $(ZSTD_FILES) @echo compiling dynamic library $(LIBVER) $(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.dll.a -shared $^ -o $@ @@ -244,6 +246,7 @@ else LIBZSTD = libzstd.$(SHARED_EXT_VER) $(ZSTD_DYNLIB_DIR)/$(LIBZSTD): CFLAGS += -fPIC $(ZSTD_DYNLIB_DIR)/$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden +$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): | $(ZSTD_DYNLIB_DIR) $(ZSTD_DYNLIB_DIR)/$(LIBZSTD): $(ZSTD_DYNLIB_OBJ) @echo compiling dynamic library $(LIBVER) $(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ diff --git a/tests/Makefile b/tests/Makefile index 87252ea80..ff67ead9c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -99,7 +99,7 @@ zstd zstd32 zstd-nolegacy: $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" DEBUGLEVEL=$(DEBUGLEVEL) gzstd: - $(MAKE) -C $(PRGDIR) $@ HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)" + $(MAKE) -C $(PRGDIR) $@ HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)" DEBUGLEVEL=$(DEBUGLEVEL) .PHONY: libzstd libzstd : From 91a8cb9559342835aa7506d90354e03dfe91f4f0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Oct 2020 00:20:40 -0700 Subject: [PATCH 6/9] fix DEBUGLEVEL redefinition from tests/ --- tests/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index ff67ead9c..c39c5ca0f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -94,7 +94,7 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus # note : broken : requires symbols unavailable from dynamic library dll: fuzzer-dll zstreamtest-dll -PHONY: zstd zstd32 zstd-nolegacy # must be phony, only external makefile knows how to build them, or if they need an update +PHONY: zstd zstd32 zstd-nolegacy # phony: only external makefile knows how to build or update them zstd zstd32 zstd-nolegacy: $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" DEBUGLEVEL=$(DEBUGLEVEL) @@ -103,14 +103,14 @@ gzstd: .PHONY: libzstd libzstd : - $(MAKE) -C $(ZSTDDIR) libzstd + $(MAKE) -C $(ZSTDDIR) libzstd DEBUGLEVEL=$(DEBUGLEVEL) %-dll : libzstd %-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd .PHONY: zstd-staticLib zstd-staticLib : - $(MAKE) -C $(ZSTDDIR) libzstd.a + $(MAKE) -C $(ZSTDDIR) libzstd.a DEBUGLEVEL=$(DEBUGLEVEL) zstdm_%.o : $(ZSTDDIR)/common/%.c $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ From e3867fb735842ce4165f027204a17a2e90e1f597 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Oct 2020 11:28:46 -0700 Subject: [PATCH 7/9] fixed libzstd.dll compilation on mingw and zstd linking --- lib/Makefile | 8 ++++---- programs/Makefile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 8924bfb43..1f10c7ab1 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -236,8 +236,8 @@ libzstd.a: $(ZSTD_STATLIB_DIR)/libzstd.a ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) LIBZSTD = dll\libzstd.dll -$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): | $(ZSTD_DYNLIB_DIR) -$(ZSTD_DYNLIB_DIR)/$(LIBZSTD): $(ZSTD_FILES) +$(LIBZSTD): +$(LIBZSTD): $(ZSTD_FILES) @echo compiling dynamic library $(LIBVER) $(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.dll.a -shared $^ -o $@ @@ -254,12 +254,12 @@ $(ZSTD_DYNLIB_DIR)/$(LIBZSTD): $(ZSTD_DYNLIB_OBJ) $(Q)ln -sf $@ libzstd.$(SHARED_EXT_MAJOR) $(Q)ln -sf $@ libzstd.$(SHARED_EXT) -endif - .PHONY: $(LIBZSTD) # must be run every time $(LIBZSTD): $(ZSTD_DYNLIB_DIR)/$(LIBZSTD) $(Q)ln -sf $< $@ +endif + .PHONY: libzstd libzstd : $(LIBZSTD) diff --git a/programs/Makefile b/programs/Makefile index 575931f8c..70fb2c32d 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -217,7 +217,7 @@ ifneq (,$(filter Windows%,$(OS))) zstd : $(RES_FILE) endif zstd : $(BUILD_DIR)/zstd - $(Q)ln -sf $^ $@ + $(Q)ln -sf $< $@ @echo zstd build completed .PHONY: zstd-release From ce6cd07c3344df80e9830f619f950329bf29a388 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Oct 2020 12:31:23 -0700 Subject: [PATCH 8/9] updated build documentation --- lib/README.md | 20 +++++++++++++++ programs/README.md | 63 ++++++++++++++++++++++++++++------------------ 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/lib/README.md b/lib/README.md index ff62b668a..db9170a75 100644 --- a/lib/README.md +++ b/lib/README.md @@ -168,6 +168,26 @@ file it should be linked with `dll\libzstd.dll`. For example: The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`. +#### Advanced Build options + +The build system requires a hash function in order to +separate object files created with different compilation flags. +By default, it tries to use `md5sum` or equivalent. +The hash function can be manually switched by setting the `HASH` variable. +For example : `make HASH=xxhsum` +The hash function needs to generate at least 64-bit using hexadecimal format. +When no hash function is found, +the Makefile just generates all object files into the same default directory, +irrespective of compilation flags. +This functionality only matters if `libzstd` is compiled multiple times +with different build flags. + +The build directory, where object files are stored +can also be manually controlled using variable `BUILD_DIR`, +for example `make BUILD_DIR=objectDir/v1`. +In which case, the hash function doesn't matter. + + #### Deprecated API Obsolete API on their way out are stored in directory `lib/deprecated`. diff --git a/programs/README.md b/programs/README.md index 53f716bda..cf7f5ba46 100644 --- a/programs/README.md +++ b/programs/README.md @@ -3,7 +3,7 @@ Command Line Interface for Zstandard library Command Line Interface (CLI) can be created using the `make` command without any additional parameters. There are however other Makefile targets that create different variations of CLI: -- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and support for decompression of legacy zstd formats +- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and supports decompression of legacy zstd formats - `zstd_nolegacy` : Same as `zstd` but without support for legacy zstd formats - `zstd-small` : CLI optimized for minimal size; no dictionary builder, no benchmark, and no support for legacy zstd formats - `zstd-compress` : version of CLI which can only compress into zstd format @@ -147,76 +147,91 @@ FILE : a filename Arguments : -# : # compression level (1-19, default: 3) -d : decompression - -D file: use `file` as Dictionary - -o file: result stored into `file` (only if 1 input file) - -f : overwrite output without prompting and (de)compress links + -D DICT: use DICT as Dictionary for compression or decompression + -o file: result stored into `file` (only 1 output file) + -f : overwrite output without prompting, also (de)compress links --rm : remove source file(s) after successful de/compression -k : preserve source file(s) (default) -h/-H : display help/long help and exit Advanced arguments : -V : display Version number and exit + -c : force write to standard output, even if it is the console -v : verbose mode; specify multiple times to increase verbosity -q : suppress warnings; specify twice to suppress errors too - -c : force write to standard output, even if it is the console - -l : print information about zstd compressed files ---exclude-compressed: only compress files that are not previously compressed +--no-progress : do not display the progress counter + -r : operate recursively on directories +--filelist FILE : read list of files to operate upon from FILE +--output-dir-flat DIR : processed files are stored into DIR +--output-dir-mirror DIR : processed files are stored into DIR respecting original directory structure +--[no-]check : during compression, add XXH64 integrity checksum to frame (default: enabled). If specified with -d, decompressor will ignore/validate checksums in compressed frame (default: validate). +-- : All arguments after "--" are treated as files + +Advanced compression arguments : --ultra : enable levels beyond 19, up to 22 (requires more memory) --long[=#]: enable long distance matching with given window log (default: 27) --fast[=#]: switch to very fast compression levels (default: 1) --adapt : dynamically adapt compression level to I/O conditions ---stream-size=# : optimize compression parameters for streaming input of given number of bytes ---size-hint=# optimize compression parameters for streaming input of approximately this size ---target-compressed-block-size=# : make compressed block near targeted size -T# : spawns # compression threads (default: 1, 0==# cores) -B# : select size of each job (default: 0==automatic) +--single-thread : use a single thread for both I/O and compression (result slightly different than -T1) --rsyncable : compress using a rsync-friendly method (-B sets block size) ---no-dictID : don't write dictID into header (dictionary compression) ---[no-]check : integrity check (default: enabled) +--exclude-compressed: only compress files that are not already compressed +--stream-size=# : specify size of streaming input from `stdin` +--size-hint=# optimize compression parameters for streaming input of approximately this size +--target-compressed-block-size=# : generate compressed block of approximately targeted size +--no-dictID : don't write dictID into header (dictionary compression only) --[no-]compress-literals : force (un)compressed literals - -r : operate recursively on directories ---output-dir-flat[=directory]: all resulting files stored into `directory`. --format=zstd : compress files to the .zst format (default) --format=gzip : compress files to the .gz format +--format=xz : compress files to the .xz format +--format=lzma : compress files to the .lzma format +--format=lz4 : compress files to the .lz4 format + +Advanced decompression arguments : + -l : print information about zstd compressed files --test : test compressed file integrity ---[no-]sparse : sparse mode (default: disabled) -M# : Set a memory usage limit for decompression ---no-progress : do not display the progress bar --- : All arguments after "--" are treated as files +--[no-]sparse : sparse mode (default: disabled) Dictionary builder : --train ## : create a dictionary from a training set of files --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]] : use the cover algorithm with optional args --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]] : use the fast cover algorithm with optional args --train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9) - -o file : `file` is dictionary name (default: dictionary) + -o DICT : DICT is dictionary name (default: dictionary) --maxdict=# : limit dictionary to specified size (default: 112640) --dictID=# : force dictionary ID to specified value (default: random) Benchmark arguments : -b# : benchmark file(s), using # compression level (default: 3) - -e# : test all compression levels from -bX to # (default: 1) + -e# : test all compression levels successively from -b# to -e# (default: 1) -i# : minimum evaluation time in seconds (default: 3s) -B# : cut file into independent blocks of size # (default: no block) + -S : output one benchmark result per input file (default: consolidated result) --priority=rt : set process priority to real-time ``` ### Passing parameters through Environment Variables +There is no "generic" way to pass "any kind of parameter" to `zstd` in a pass-through manner. +Using environment variables for this purpose has security implications. +Therefore, this avenue is intentionally restricted and only supports `ZSTD_CLEVEL` and `ZSTD_NBTHREADS`. + `ZSTD_CLEVEL` can be used to modify the default compression level of `zstd` (usually set to `3`) to another value between 1 and 19 (the "normal" range). -`ZSTD_NBTHREADS` can be used to specify number of threads that `zstd` will use during compression, which by default is `1`. + +`ZSTD_NBTHREADS` can be used to specify a number of threads +that `zstd` will use for compression, which by default is `1`. This functionality only exists when `zstd` is compiled with multithread support. +`0` means "use as many threads as detected cpu cores on local system". The max # of threads is capped at: `ZSTDMT_NBWORKERS_MAX==200`. This functionality can be useful when `zstd` CLI is invoked in a way that doesn't allow passing arguments. One such scenario is `tar --zstd`. As `ZSTD_CLEVEL` and `ZSTD_NBTHREADS` only replace the default compression level -and number of threads, respectively, they can both be overridden by corresponding command line arguments: +and number of threads respectively, they can both be overridden by corresponding command line arguments: `-#` for compression level and `-T#` for number of threads. -There is no "generic" way to pass "any kind of parameter" to `zstd` in a pass-through manner. -Using environment variables for this purpose has security implications. -Therefore, this avenue is intentionally restricted and only supports `ZSTD_CLEVEL` and `ZSTD_NBTHREADS`. ### Long distance matching mode The long distance matching mode, enabled with `--long`, is designed to improve From d3f1a9b5bdc37dbad741c87cc58d65d1bb2effa8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 22 Oct 2020 19:53:01 -0700 Subject: [PATCH 9/9] fix partial-build test sometimes, the scope difference is solely determined by the list of source files, not by the flags. --- lib/Makefile | 2 +- tests/libzstd_partial_builds.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 1f10c7ab1..b8295ff03 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -185,7 +185,7 @@ HASH ?= md5sum HAVE_HASH := $(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0) ifeq ($(HAVE_HASH), 1) - HASH_VALUE := $(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) | $(HASH) | head -c 16) + HASH_VALUE := $(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(ZSTD_FILES) | $(HASH) | head -c 16) HASH_DIR := conf_$(HASH_VALUE) else $(info warning : could not find hash function to differentiate builds with different flags) diff --git a/tests/libzstd_partial_builds.sh b/tests/libzstd_partial_builds.sh index b1c1e3b1a..bee2dbda5 100755 --- a/tests/libzstd_partial_builds.sh +++ b/tests/libzstd_partial_builds.sh @@ -22,7 +22,6 @@ mustBeAbsent() { } # default compilation : all features enabled -make clean > /dev/null $ECHO "testing default library compilation" CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog