From 911dbdbb4b1374fcc4c2c8a6199bb15ecb69b24a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 20 Oct 2020 22:19:57 -0700 Subject: [PATCH 1/5] build libzstd.so from object files %.o object files generated for dynamic library must be different from those generated for static library. Due to this difference, %.o were so far only generated for the static library. The dynamic library was rebuilt from %.c source. This meant that, for every minor change, the entire dynamic library had to be rebuilt. This is fixed in this PR : only the modified %.c source get rebuilt. --- lib/Makefile | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index aba0c99ba..be6fe1d27 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -82,7 +82,7 @@ ZDEPR_FILES := $(sort $(wildcard deprecated/*.c)) ZSTD_FILES := $(ZSTDCOMMON_FILES) ifeq ($(findstring GCC,$(CCVER)),GCC) -decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize +decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize endif # Modules @@ -164,20 +164,29 @@ endif endif CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) -ZSTD_OBJ := $(patsubst %.c,%.o,$(ZSTD_FILES)) +ZSTD_OBJ := $(ZSTD_FILES:.c=.o) + +vpath %.c ./common +vpath %.c ./compress +vpath %.c ./decompress +vpath %.c ./dictBuilder +vpath %.c ./legacy +vpath %.c ./deprecated + +ZSTD_DYNLIB_OBJ := $(notdir $(ZSTD_OBJ)) # macOS linker doesn't support -soname, and use different extension # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html ifeq ($(shell uname), Darwin) - SHARED_EXT = dylib - SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) - SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) - SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) + SHARED_EXT = dylib + SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) + SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) + SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) else - SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR) - SHARED_EXT = so - SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) - SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) + SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR) + SHARED_EXT = so + SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) + SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif @@ -205,8 +214,9 @@ $(LIBZSTD): $(ZSTD_FILES) else LIBZSTD = libzstd.$(SHARED_EXT_VER) -$(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden -$(LIBZSTD): $(ZSTD_FILES) +$(LIBZSTD): CFLAGS += -fPIC +$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden +$(LIBZSTD): $(ZSTD_DYNLIB_OBJ) @echo compiling dynamic library $(LIBVER) $(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ @echo creating versioned links From 3e519be9656c586e2b80441929b1692adec21a9e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Oct 2020 03:22:27 -0700 Subject: [PATCH 2/5] minor cleaning --- lib/Makefile | 83 ++++++++++++++++++++++++++-------------------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index be6fe1d27..dac225218 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -35,28 +35,28 @@ CCVER := $(shell $(CC) --version) # space-optimized defaults. ZSTD_LIB_MINIFY ?= 0 ifneq ($(ZSTD_LIB_MINIFY), 0) - HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0) - ZSTD_LEGACY_SUPPORT ?= 0 - ZSTD_LIB_DEPRECATED ?= 0 - HUF_FORCE_DECOMPRESS_X1 ?= 1 - ZSTD_FORCE_DECOMPRESS_SHORT ?= 1 - ZSTD_NO_INLINE ?= 1 - ZSTD_STRIP_ERROR_STRINGS ?= 1 - ifneq ($(HAVE_CC_OZ), 0) - # Some compilers (clang) support an even more space-optimized setting. - CFLAGS += -Oz - else - CFLAGS += -Os - endif - CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \ - -DDYNAMIC_BMI2=0 -DNDEBUG + HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0) + ZSTD_LEGACY_SUPPORT ?= 0 + ZSTD_LIB_DEPRECATED ?= 0 + HUF_FORCE_DECOMPRESS_X1 ?= 1 + ZSTD_FORCE_DECOMPRESS_SHORT ?= 1 + ZSTD_NO_INLINE ?= 1 + ZSTD_STRIP_ERROR_STRINGS ?= 1 +ifneq ($(HAVE_CC_OZ), 0) + # Some compilers (clang) support an even more space-optimized setting. + CFLAGS += -Oz else - CFLAGS += -O3 + CFLAGS += -Os +endif + CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \ + -DDYNAMIC_BMI2=0 -DNDEBUG +else + CFLAGS += -O3 endif CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_ ifeq ($(TARGET_SYSTEM),Windows_NT) # MinGW assumed -CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting + CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ @@ -69,7 +69,7 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0) GREP_OPTIONS ?= ifeq ($HAVE_COLORNEVER, 1) -GREP_OPTIONS += --color=never + GREP_OPTIONS += --color=never endif GREP = grep $(GREP_OPTIONS) SED_ERE_OPT ?= -E @@ -104,67 +104,68 @@ ZSTD_NO_INLINE ?= 0 ZSTD_STRIP_ERROR_STRINGS ?= 0 ifeq ($(ZSTD_LIB_COMPRESSION), 0) - ZSTD_LIB_DICTBUILDER = 0 - ZSTD_LIB_DEPRECATED = 0 + ZSTD_LIB_DICTBUILDER = 0 + ZSTD_LIB_DEPRECATED = 0 endif ifeq ($(ZSTD_LIB_DECOMPRESSION), 0) - ZSTD_LEGACY_SUPPORT = 0 - ZSTD_LIB_DEPRECATED = 0 + ZSTD_LEGACY_SUPPORT = 0 + ZSTD_LIB_DEPRECATED = 0 endif ifneq ($(ZSTD_LIB_COMPRESSION), 0) - ZSTD_FILES += $(ZSTDCOMP_FILES) + ZSTD_FILES += $(ZSTDCOMP_FILES) endif ifneq ($(ZSTD_LIB_DECOMPRESSION), 0) - ZSTD_FILES += $(ZSTDDECOMP_FILES) + ZSTD_FILES += $(ZSTDDECOMP_FILES) endif ifneq ($(ZSTD_LIB_DEPRECATED), 0) - ZSTD_FILES += $(ZDEPR_FILES) + ZSTD_FILES += $(ZDEPR_FILES) endif ifneq ($(ZSTD_LIB_DICTBUILDER), 0) - ZSTD_FILES += $(ZDICT_FILES) + ZSTD_FILES += $(ZDICT_FILES) endif ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0) - CFLAGS += -DHUF_FORCE_DECOMPRESS_X1 + CFLAGS += -DHUF_FORCE_DECOMPRESS_X1 endif ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0) - CFLAGS += -DHUF_FORCE_DECOMPRESS_X2 + CFLAGS += -DHUF_FORCE_DECOMPRESS_X2 endif ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0) - CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT + CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT endif ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0) - CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG + CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG endif ifneq ($(ZSTD_NO_INLINE), 0) - CFLAGS += -DZSTD_NO_INLINE + CFLAGS += -DZSTD_NO_INLINE endif ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0) - CFLAGS += -DZSTD_STRIP_ERROR_STRINGS + CFLAGS += -DZSTD_STRIP_ERROR_STRINGS endif ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0) - CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API + CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API endif ifneq ($(ZSTD_LEGACY_SUPPORT), 0) ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) - ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') + ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') endif endif CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) -ZSTD_OBJ := $(ZSTD_FILES:.c=.o) +ZSTD_FULL_OBJ := $(ZSTD_FILES:.c=.o) +ZSTD_LOCAL_OBJ := $(notdir $(ZSTD_FULL_OBJ)) vpath %.c ./common vpath %.c ./compress @@ -173,7 +174,7 @@ vpath %.c ./dictBuilder vpath %.c ./legacy vpath %.c ./deprecated -ZSTD_DYNLIB_OBJ := $(notdir $(ZSTD_OBJ)) +ZSTD_DYNLIB_OBJ := $(ZSTD_LOCAL_OBJ) # macOS linker doesn't support -soname, and use different extension # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html @@ -200,7 +201,7 @@ lib-all: all all: lib libzstd.a: ARFLAGS = rcs -libzstd.a: $(ZSTD_OBJ) +libzstd.a: $(ZSTD_FULL_OBJ) @echo compiling static library $(Q)$(AR) $(ARFLAGS) $@ $^ @@ -300,15 +301,15 @@ endif endif ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly)) -PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig + PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig else -PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig + PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig endif ifneq (,$(filter $(shell uname),SunOS)) -INSTALL ?= ginstall + INSTALL ?= ginstall else -INSTALL ?= install + INSTALL ?= install endif INSTALL_PROGRAM ?= $(INSTALL) From e8eb2939feb98c5dd425181247557b9c59649fca Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Oct 2020 03:44:38 -0700 Subject: [PATCH 3/5] store %.o object files into obj/ both static and dynamic libraries have their own object directory --- lib/Makefile | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index dac225218..4f4ae57aa 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -174,7 +174,10 @@ vpath %.c ./dictBuilder vpath %.c ./legacy vpath %.c ./deprecated -ZSTD_DYNLIB_OBJ := $(ZSTD_LOCAL_OBJ) +ZSTD_DYNLIB_DIR := obj/dynlib +ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/, $(ZSTD_LOCAL_OBJ)) +ZSTD_STATLIB_DIR := obj/statlib +ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/, $(ZSTD_LOCAL_OBJ)) # macOS linker doesn't support -soname, and use different extension # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html @@ -201,7 +204,7 @@ lib-all: all all: lib libzstd.a: ARFLAGS = rcs -libzstd.a: $(ZSTD_FULL_OBJ) +libzstd.a: $(ZSTD_STATLIB_OBJ) @echo compiling static library $(Q)$(AR) $(ARFLAGS) $@ $^ @@ -232,6 +235,7 @@ libzstd : $(LIBZSTD) .PHONY: lib lib : libzstd.a libzstd + # note : do not define lib-mt or lib-release as .PHONY # make does not consider implicit pattern rule for .PHONY target @@ -244,6 +248,13 @@ lib : libzstd.a libzstd %-release : % @echo release build completed +$(ZSTD_DYNLIB_DIR)/%.o : %.c | $(ZSTD_DYNLIB_DIR) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ +$(ZSTD_STATLIB_DIR)/%.o : %.c | $(ZSTD_STATLIB_DIR) + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + +MKDIR ?= mkdir +$(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR): ; $(Q)$(MKDIR) -p $@ # Special case : building library in single-thread mode _and_ without zstdmt_compress.c ZSTDMT_FILES = compress/zstdmt_compress.c @@ -259,6 +270,7 @@ clean: $(Q)$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc $(Q)$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt* $(Q)$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o + $(Q)$(RM) -r $(ZSTD_STATLIB_DIR)/*.o $(ZSTD_DYNLIB_DIR)/*.o @echo Cleaning library completed #----------------------------------------------------------------------------- From 2b99bc29bf80eb0e0e7e6cf7d21014bd8f08d62f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Oct 2020 04:01:01 -0700 Subject: [PATCH 4/5] consolidated vpath --- lib/Makefile | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 4f4ae57aa..61f80e6e5 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -10,13 +10,13 @@ Q = $(if $(filter 1,$(V) $(VERBOSE)),,@) -# When cross-compiling from linux to windows, you might -# need to specify this as "Windows." Fedora build fails -# without it. +# When cross-compiling from linux to windows, +# you might need to specify this as "Windows." +# Fedora build fails without it. # -# Note: mingw-w64 build from linux to windows does not -# fail on other tested distros (ubuntu, debian) even -# without manually specifying the TARGET_SYSTEM. +# Note: mingw-w64 build from linux to windows +# does not fail on other tested distros (ubuntu, debian) +# even without manually specifying the TARGET_SYSTEM. TARGET_SYSTEM ?= $(OS) # Version numbers @@ -167,12 +167,7 @@ CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ZSTD_FULL_OBJ := $(ZSTD_FILES:.c=.o) ZSTD_LOCAL_OBJ := $(notdir $(ZSTD_FULL_OBJ)) -vpath %.c ./common -vpath %.c ./compress -vpath %.c ./decompress -vpath %.c ./dictBuilder -vpath %.c ./legacy -vpath %.c ./deprecated +vpath %.c common compress decompress dictBuilder legacy deprecated ZSTD_DYNLIB_DIR := obj/dynlib ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/, $(ZSTD_LOCAL_OBJ)) From 8a453a34c57d2097bf499fb81446e90636851f86 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Oct 2020 17:25:07 -0700 Subject: [PATCH 5/5] automatic %.h header dependency tracking also : BUILD_DIR can be manually specified --- lib/Makefile | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 61f80e6e5..58e80e473 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -164,14 +164,16 @@ endif endif CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) -ZSTD_FULL_OBJ := $(ZSTD_FILES:.c=.o) -ZSTD_LOCAL_OBJ := $(notdir $(ZSTD_FULL_OBJ)) +ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES)) +ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_SRC:.c=.o) -vpath %.c common compress decompress dictBuilder legacy deprecated +ZSTD_SUBDIR := common compress decompress dictBuilder legacy deprecated +vpath %.c $(ZSTD_SUBDIR) -ZSTD_DYNLIB_DIR := obj/dynlib +BUILD_DIR ?= obj +ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynlib ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/, $(ZSTD_LOCAL_OBJ)) -ZSTD_STATLIB_DIR := obj/statlib +ZSTD_STATLIB_DIR := $(BUILD_DIR)/statlib ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/, $(ZSTD_LOCAL_OBJ)) # macOS linker doesn't support -soname, and use different extension @@ -243,13 +245,28 @@ lib : libzstd.a libzstd %-release : % @echo release build completed -$(ZSTD_DYNLIB_DIR)/%.o : %.c | $(ZSTD_DYNLIB_DIR) - $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ -$(ZSTD_STATLIB_DIR)/%.o : %.c | $(ZSTD_STATLIB_DIR) - $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + +# Generate .h dependencies automatically + +DEPFLAGS = -MT $@ -MMD -MP -MF + +$(ZSTD_DYNLIB_DIR)/%.o : %.c $(ZSTD_DYNLIB_DIR)/%.d | $(ZSTD_DYNLIB_DIR) + @echo $@ + $(Q)$(COMPILE.c) $(DEPFLAGS) $(ZSTD_DYNLIB_DIR)/$*.d $(OUTPUT_OPTION) $< + +$(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR) + @echo $@ + $(Q)$(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $< MKDIR ?= mkdir -$(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR): ; $(Q)$(MKDIR) -p $@ +$(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR): + $(Q)$(MKDIR) -p $@ + +DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATLIB_OBJ:.o=.d) +$(DEPFILES): + +include $(wildcard $(DEPFILES)) + # Special case : building library in single-thread mode _and_ without zstdmt_compress.c ZSTDMT_FILES = compress/zstdmt_compress.c @@ -265,7 +282,7 @@ clean: $(Q)$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc $(Q)$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt* $(Q)$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o - $(Q)$(RM) -r $(ZSTD_STATLIB_DIR)/*.o $(ZSTD_DYNLIB_DIR)/*.o + $(Q)$(RM) -r $(ZSTD_STATLIB_DIR)/* $(ZSTD_DYNLIB_DIR)/* @echo Cleaning library completed #----------------------------------------------------------------------------- @@ -295,7 +312,7 @@ ifeq (,$(PCLIBDIR)) # Additional prefix check is required, since the empty string is technically a # valid PCLIBDIR ifeq (,$(shell echo "$(LIBDIR)" | sed -n $(SED_ERE_OPT) -e "\\@^$(EXEC_PREFIX)(/|$$)@ p")) -$(error configured libdir ($(LIBDIR)) is outside of prefix ($(PREFIX)), can't generate pkg-config file) +$(error configured libdir ($(LIBDIR)) is outside of prefix ($(EXEC_PREFIX)), can't generate pkg-config file) endif endif @@ -303,7 +320,7 @@ ifeq (,$(PCINCDIR)) # Additional prefix check is required, since the empty string is technically a # valid PCINCDIR ifeq (,$(shell echo "$(INCLUDEDIR)" | sed -n $(SED_ERE_OPT) -e "\\@^$(PREFIX)(/|$$)@ p")) -$(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(EXEC_PREFIX)), can't generate pkg-config file) +$(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(PREFIX)), can't generate pkg-config file) endif endif