From 80cee8d3fee5997d819c6cb24bfdae1a13c141a2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 16 Oct 2020 00:01:41 -0700 Subject: [PATCH 1/8] faster rebuild of zstd Building the zstd CLI costs time. Some part of it is incompressible, leading to substantial iteration delay when testing code modifications. That's mainly because all source files from the library must be rebuilt from source every time. The main reason we don't build the CLI from library object files is that we can't just build the object directly in the lib/ directory (which they would by default) since they use different compilation flags. Specifically, the CLI enables multithreading, while the library doesn't (by default). This is solved in this commit, by generating the object files locally. Now, the CLI and the library can employ different sets of flags, without tripping over each other. All library object files are generated directly into programs/ dir. This works because no 2 source files have the same name. Now, modifying a file doesn't require to recompile the entire lib, just the modified files. The recipe is also compatible with `-j` parallel build, leading to large build time reductions on multi-core systems. --- programs/Makefile | 63 +++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 6cb4a57c2..4899c69c3 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -15,7 +15,7 @@ # zstd-decompress : decompressor-only version of zstd # ########################################################################## -ZSTDDIR = ../lib +ZSTDDIR := ../lib # Version numbers LIBVER_SRC := $(ZSTDDIR)/zstd.h @@ -43,41 +43,48 @@ else ALIGN_LOOP = endif -CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_ +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ ifeq ($(OS),Windows_NT) # MinGW assumed -CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif -CFLAGS ?= -O3 +CFLAGS ?= -O3 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wpointer-arith \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wredundant-decls -Wmissing-prototypes -Wc++-compat -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +ZSTDLIB_COMMON := $(ZSTDDIR)/common +ZSTDLIB_COMPRESS := $(ZSTDDIR)/compress +ZSTDLIB_DECOMPRESS := $(ZSTDDIR)/decompress +ZDICT_DIR := $(ZSTDDIR)/dictBuilder +ZSTDLEGACY_DIR := $(ZSTDDIR)/legacy -ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c -ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) -ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c -ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o +VPATH := $(ZSTDLIB_COMMON):$(ZSTDLIB_COMPRESS):$(ZSTDLIB_DECOMPRESS):$(ZDICT_DIR):$(ZSTDLEGACY_DIR) + +ZSTDLIB_COMMON_C := $(wildcard $(ZSTDLIB_COMMON)/*.c) +ZSTDLIB_COMPRESS_C := $(wildcard $(ZSTDLIB_COMPRESS)/*.c) +ZSTDLIB_DECOMPRESS_C := $(wildcard $(ZSTDLIB_DECOMPRESS)/*.c) +ZSTDLIB_CORE_SRC := $(ZSTDLIB_DECOMPRESS_C) $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) +ZDICT_SRC := $(wildcard $(ZDICT_DIR)/*.c) ZSTD_LEGACY_SUPPORT ?= 5 -ZSTDLEGACY_FILES := +ZSTDLEGACY_SRC := ifneq ($(ZSTD_LEGACY_SUPPORT), 0) ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) - ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') + ZSTDLEGACY_SRC += $(notdir $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')) endif -else endif # Sort files in alphabetical order for reproducible builds -ZSTDLIB_FILES := $(sort $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES))) +ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC)) +ZSTDLIB_LOCAL_SRC = $(notdir $(ZSTDLIB_FULL_SRC)) +ZSTDLIB_LOCAL_OBJ = $(ZSTDLIB_LOCAL_SRC:.c=.o) ZSTD_CLI_FILES := $(wildcard *.c) -ZSTD_CLI_OBJ := $(patsubst %.c,%.o,$(ZSTD_CLI_FILES)) +ZSTD_CLI_OBJ := $(ZSTD_CLI_FILES:.c=.o) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -164,15 +171,13 @@ all: zstd .PHONY: allVariants allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-dictBuilder -$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) - zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP) zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD) zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ifneq (,$(filter Windows%,$(OS))) zstd : $(RES_FILE) endif -zstd : $(ZSTDLIB_FILES) $(ZSTD_CLI_OBJ) +zstd : $(ZSTDLIB_LOCAL_OBJ) $(ZSTD_CLI_OBJ) @echo "$(THREAD_MSG)" @echo "$(ZLIB_MSG)" @echo "$(LZMA_MSG)" @@ -190,12 +195,12 @@ zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ifneq (,$(filter Windows%,$(OS))) zstd32 : $(RES32_FILE) endif -zstd32 : $(ZSTDLIB_FILES) $(ZSTD_CLI_FILES) +zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_FILES) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) ## zstd-nolegacy: same scope as zstd, with just support of legacy formats removed zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD) -zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) $(ZSTD_CLI_OBJ) +zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ) $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) zstd-nomt : THREAD_CPP := @@ -222,11 +227,15 @@ zstd-noxz : zstd # It's unclear at this stage if this is a scenario that must be supported .PHONY: zstd-dll zstd-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -zstd-dll : ZSTDLIB_FILES = +zstd-dll : ZSTDLIB_FULL_SRC = zstd-dll : $(ZSTD_CLI_OBJ) $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) +ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o + +$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) + ## zstd-pgo: zstd executable optimized with PGO. zstd-pgo : $(MAKE) clean @@ -243,18 +252,18 @@ zstd-pgo : ## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format. zstd-small: CFLAGS = -Os -s -zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c timefn.c fileio.c +zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT) -zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c timefn.c fileio.c +zstd-decompress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_DECOMPRESS_C) zstdcli.c util.c timefn.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT) -zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c timefn.c fileio.c +zstd-compress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) zstdcli.c util.c timefn.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) ## zstd-dictBuilder: executable supporting dictionary creation and compression (only) zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -zstd-dictBuilder: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(ZDICT_FILES) zstdcli.c util.c timefn.c fileio.c dibio.c +zstd-dictBuilder: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c $(CC) $(FLAGS) $^ -o $@$(EXT) zstdmt: zstd From b6b87c2b77299cdeabc08e52c4fa14fbe468f5f3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 16 Oct 2020 00:18:16 -0700 Subject: [PATCH 2/8] track header dependencies during build modifying a header file will now trigger recompilation of related *.c units --- programs/Makefile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/programs/Makefile b/programs/Makefile index 4899c69c3..a0dc0fbac 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -15,6 +15,9 @@ # zstd-decompress : decompressor-only version of zstd # ########################################################################## +# verbose mode can be triggered by V=1 or VERBOSE=1 +Q = $(if $(filter 1,$(V) $(VERBOSE)),,@) + ZSTDDIR := ../lib # Version numbers @@ -318,6 +321,28 @@ preview-man: clean-man man man ./zstdgrep.1 man ./zstdless.1 + +# Generate .h dependencies automatically + +MKDIR = mkdir + +DEPDIR := .deps +DEPFLAGS = -MT $@ -MMD -MP -MF $(DEPDIR)/$*.d + +COMPILE.c = $(CC) $(DEPFLAGS) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c + +%.o : %.c +%.o : %.c $(DEPDIR)/%.d | $(DEPDIR) + $(COMPILE.c) $(OUTPUT_OPTION) $< + +$(DEPDIR): ; $(Q)$(MKDIR) -p $@ + +DEPFILES := $(ZSTDLIB_LOCAL_SRC:%.c=$(DEPDIR)/%.d) +$(DEPFILES): + +include $(wildcard $(DEPFILES)) + + #----------------------------------------------------------------------------- # make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets #----------------------------------------------------------------------------- From c355d149deec151959ba22f546efefc0fa9eaf40 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 16 Oct 2020 00:21:49 -0700 Subject: [PATCH 3/8] support verbose build output with V=1 --- programs/Makefile | 52 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index a0dc0fbac..348557657 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -287,8 +287,8 @@ endif .PHONY: clean clean: $(MAKE) -C $(ZSTDDIR) clean - @$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda - @$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ + $(Q)$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda + $(Q)$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \ zstd-dictBuilder$(EXT) *.gcda default*.profraw default.profdata have_zlib$(EXT) @@ -362,7 +362,7 @@ AWK = awk ## list: Print all targets and their descriptions (if provided) .PHONY: list list: - @TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \ + $(Q)TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \ | $(AWK) -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \ | $(EGREP) -v -e '^[^[:alnum:]]' | sort); \ { \ @@ -410,34 +410,34 @@ INSTALL_MAN ?= $(INSTALL_DATA) .PHONY: install install: zstd @echo Installing binaries - @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MAN1DIR)/ - @$(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT) - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT) - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT) - @ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT) - @$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless - @$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep + $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MAN1DIR)/ + $(Q)$(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT) + $(Q)ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT) + $(Q)ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT) + $(Q)ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT) + $(Q)$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless + $(Q)$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep @echo Installing man pages - @$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1 - @ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1 - @ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1 - @$(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1 - @$(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1 + $(Q)$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1 + $(Q)ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1 + $(Q)ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1 + $(Q)$(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1 + $(Q)$(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1 @echo zstd installation completed .PHONY: uninstall uninstall: - @$(RM) $(DESTDIR)$(BINDIR)/zstdgrep - @$(RM) $(DESTDIR)$(BINDIR)/zstdless - @$(RM) $(DESTDIR)$(BINDIR)/zstdcat - @$(RM) $(DESTDIR)$(BINDIR)/unzstd - @$(RM) $(DESTDIR)$(BINDIR)/zstdmt - @$(RM) $(DESTDIR)$(BINDIR)/zstd - @$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1 - @$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1 - @$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1 - @$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1 - @$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1 + $(Q)$(RM) $(DESTDIR)$(BINDIR)/zstdgrep + $(Q)$(RM) $(DESTDIR)$(BINDIR)/zstdless + $(Q)$(RM) $(DESTDIR)$(BINDIR)/zstdcat + $(Q)$(RM) $(DESTDIR)$(BINDIR)/unzstd + $(Q)$(RM) $(DESTDIR)$(BINDIR)/zstdmt + $(Q)$(RM) $(DESTDIR)$(BINDIR)/zstd + $(Q)$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1 + $(Q)$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1 + $(Q)$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1 + $(Q)$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1 + $(Q)$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1 @echo zstd programs successfully uninstalled endif From eb1959d926f6afb2aece627b9b42352fbd0d36ec Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 16 Oct 2020 10:19:16 -0700 Subject: [PATCH 4/8] fix .deps list and minor simplications --- programs/Makefile | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 348557657..1170ccbd3 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -77,17 +77,19 @@ ZSTD_LEGACY_SUPPORT ?= 5 ZSTDLEGACY_SRC := ifneq ($(ZSTD_LEGACY_SUPPORT), 0) ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) - ZSTDLEGACY_SRC += $(notdir $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')) + ZSTDLEGACY_SRC += $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') endif endif # Sort files in alphabetical order for reproducible builds ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC)) -ZSTDLIB_LOCAL_SRC = $(notdir $(ZSTDLIB_FULL_SRC)) -ZSTDLIB_LOCAL_OBJ = $(ZSTDLIB_LOCAL_SRC:.c=.o) +ZSTDLIB_LOCAL_SRC := $(notdir $(ZSTDLIB_FULL_SRC)) +ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_SRC:.c=.o) -ZSTD_CLI_FILES := $(wildcard *.c) -ZSTD_CLI_OBJ := $(ZSTD_CLI_FILES:.c=.o) +ZSTD_CLI_SRC := $(wildcard *.c) +ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o) + +ZSTD_ALL_SRC := $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -198,7 +200,7 @@ zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ifneq (,$(filter Windows%,$(OS))) zstd32 : $(RES32_FILE) endif -zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_FILES) +zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_SRC) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) ## zstd-nolegacy: same scope as zstd, with just support of legacy formats removed @@ -235,10 +237,6 @@ zstd-dll : $(ZSTD_CLI_OBJ) $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) -ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o - -$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) - ## zstd-pgo: zstd executable optimized with PGO. zstd-pgo : $(MAKE) clean @@ -249,7 +247,7 @@ zstd-pgo : ./zstd -b $(PROFILE_WITH) ./zstd -b7i2 $(PROFILE_WITH) ./zstd -b5 $(PROFILE_WITH) - $(RM) zstd *.o $(ZSTDDECOMP_O) $(ZSTDDIR)/compress/*.o + $(RM) zstd *.o case $(CC) in *clang*) if ! [ -e default.profdata ]; then llvm-profdata merge -output=default.profdata default*.profraw; fi ;; esac $(MAKE) zstd MOREFLAGS=-fprofile-use @@ -287,7 +285,6 @@ endif .PHONY: clean clean: $(MAKE) -C $(ZSTDDIR) clean - $(Q)$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda $(Q)$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \ @@ -329,7 +326,7 @@ MKDIR = mkdir DEPDIR := .deps DEPFLAGS = -MT $@ -MMD -MP -MF $(DEPDIR)/$*.d -COMPILE.c = $(CC) $(DEPFLAGS) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c +COMPILE.c = $(CC) $(DEPFLAGS) $(CFLAGS) $(CPPFLAGS) -c %.o : %.c %.o : %.c $(DEPDIR)/%.d | $(DEPDIR) @@ -337,7 +334,7 @@ COMPILE.c = $(CC) $(DEPFLAGS) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c $(DEPDIR): ; $(Q)$(MKDIR) -p $@ -DEPFILES := $(ZSTDLIB_LOCAL_SRC:%.c=$(DEPDIR)/%.d) +DEPFILES := $(ZSTD_ALL_SRC:%.c=$(DEPDIR)/%.d) $(DEPFILES): include $(wildcard $(DEPFILES)) From 314c7df170ddc496fdd147a9a45905b7cd685e71 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 16 Oct 2020 13:26:47 -0700 Subject: [PATCH 5/8] minor : change test order to reduce a warning with `clang` on Windows --- lib/dictBuilder/zdict.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index ff2e77faf..b782993f9 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -279,7 +279,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API -# elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__) +# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405) # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message))) # elif (ZDICT_GCC_VERSION >= 301) # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated)) From de985b1c1ac2625597abdaa214dc7841f9f513f2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 16 Oct 2020 14:31:40 -0700 Subject: [PATCH 6/8] fix appveyor test was mixing `*.o` object files from different compilation settings --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 5d77b3103..91568e484 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -285,5 +285,6 @@ - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% - if [%HOST%]==[mingw] ( set "CC=%COMPILER%" && + make clean && make check ) From a9f7e77d440bc2c8694a78f4cdc4403a72b2df85 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 20 Oct 2020 16:32:22 -0700 Subject: [PATCH 7/8] fixed travis tests --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 554aa0d2f..8498e0efd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,13 +77,13 @@ matrix: - name: Minimal Decompressor Macros # ~5mn script: - make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror" - - make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror" + - make clean && make check ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror" - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT" - - make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT" + - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X1 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT" - make clean && make -j all MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG" - - make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG" + - make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG" - make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" - - make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" + - make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS" - name: cmake build and test check # ~6mn script: From 684f8ea970f15a19f82728d596ecfbf509ea686f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 20 Oct 2020 18:16:11 -0700 Subject: [PATCH 8/8] decouple zstd object files from lib/ That was a subtle one : VPATH is affecting search for both %.c source and %.o object files. This meant that, when an object file already exists in lib/, it's used in programs/, even though programs/ is supposed to generate its own %.o object files. With the new vpath directive, this is no longer the case : the search is only activated for %.c source files. Now, local programs/%.o are always generated even if equivalent ones are already created in lib/. It more clearly guarantees that lib/ and programs/ can use different compilation directives without mixing resulting %.o object files. --- programs/Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 1170ccbd3..8f14292f3 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -65,7 +65,11 @@ ZSTDLIB_DECOMPRESS := $(ZSTDDIR)/decompress ZDICT_DIR := $(ZSTDDIR)/dictBuilder ZSTDLEGACY_DIR := $(ZSTDDIR)/legacy -VPATH := $(ZSTDLIB_COMMON):$(ZSTDLIB_COMPRESS):$(ZSTDLIB_DECOMPRESS):$(ZDICT_DIR):$(ZSTDLEGACY_DIR) +vpath %.c $(ZSTDLIB_COMMON) +vpath %.c $(ZSTDLIB_COMPRESS) +vpath %.c $(ZSTDLIB_DECOMPRESS) +vpath %.c $(ZDICT_DIR) +vpath %.c $(ZSTDLEGACY_DIR) ZSTDLIB_COMMON_C := $(wildcard $(ZSTDLIB_COMMON)/*.c) ZSTDLIB_COMPRESS_C := $(wildcard $(ZSTDLIB_COMPRESS)/*.c) @@ -284,7 +288,6 @@ endif .PHONY: clean clean: - $(MAKE) -C $(ZSTDDIR) clean $(Q)$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ zstd-small$(EXT) zstd-frugal$(EXT) zstd-nolegacy$(EXT) zstd4$(EXT) \