Merge pull request #2368 from facebook/progressive_libzstd

faster rebuild of libzstd

lib/Makefile

@@ -10,13 +10,13 @@
 
 Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
 
-# When cross-compiling from linux to windows, you might
+# When cross-compiling from linux to windows,
-# need to specify this as "Windows." Fedora build fails
+# you might need to specify this as "Windows."
-# without it.
+# Fedora build fails without it.
 #
-# Note: mingw-w64 build from linux to windows does not
+# Note: mingw-w64 build from linux to windows
-# fail on other tested distros (ubuntu, debian) even
+# does not fail on other tested distros (ubuntu, debian)
-# without manually specifying the TARGET_SYSTEM.
+# even without manually specifying the TARGET_SYSTEM.
 TARGET_SYSTEM ?= $(OS)
 
 # Version numbers
@@ -35,28 +35,28 @@ CCVER := $(shell $(CC) --version)
 # space-optimized defaults.
 ZSTD_LIB_MINIFY ?= 0
 ifneq ($(ZSTD_LIB_MINIFY), 0)
-	HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
+  HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
-	ZSTD_LEGACY_SUPPORT ?= 0
+  ZSTD_LEGACY_SUPPORT ?= 0
-	ZSTD_LIB_DEPRECATED ?= 0
+  ZSTD_LIB_DEPRECATED ?= 0
-	HUF_FORCE_DECOMPRESS_X1 ?= 1
+  HUF_FORCE_DECOMPRESS_X1 ?= 1
-	ZSTD_FORCE_DECOMPRESS_SHORT ?= 1
+  ZSTD_FORCE_DECOMPRESS_SHORT ?= 1
-	ZSTD_NO_INLINE ?= 1
+  ZSTD_NO_INLINE ?= 1
-	ZSTD_STRIP_ERROR_STRINGS ?= 1
+  ZSTD_STRIP_ERROR_STRINGS ?= 1
-	ifneq ($(HAVE_CC_OZ), 0)
+ifneq ($(HAVE_CC_OZ), 0)
-		# Some compilers (clang) support an even more space-optimized setting.
+    # Some compilers (clang) support an even more space-optimized setting.
-		CFLAGS += -Oz
+    CFLAGS += -Oz
-	else
-		CFLAGS += -Os
-	endif
-	CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \
-	          -DDYNAMIC_BMI2=0 -DNDEBUG
 else
-	CFLAGS += -O3
+    CFLAGS += -Os
+endif
+  CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \
+            -DDYNAMIC_BMI2=0 -DNDEBUG
+else
+  CFLAGS += -O3
 endif
 
 CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_
 ifeq ($(TARGET_SYSTEM),Windows_NT)   # MinGW assumed
-CPPFLAGS   += -D__USE_MINGW_ANSI_STDIO   # compatibility with %zu formatting
+  CPPFLAGS   += -D__USE_MINGW_ANSI_STDIO   # compatibility with %zu formatting
 endif
 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
@@ -69,7 +69,7 @@ FLAGS    = $(CPPFLAGS) $(CFLAGS)
 HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
 GREP_OPTIONS ?=
 ifeq ($HAVE_COLORNEVER, 1)
-GREP_OPTIONS += --color=never
+  GREP_OPTIONS += --color=never
 endif
 GREP = grep $(GREP_OPTIONS)
 SED_ERE_OPT ?= -E
@@ -82,7 +82,7 @@ ZDEPR_FILES := $(sort $(wildcard deprecated/*.c))
 ZSTD_FILES := $(ZSTDCOMMON_FILES)
 
 ifeq ($(findstring GCC,$(CCVER)),GCC)
-decompress/zstd_decompress_block.o :	CFLAGS+=-fno-tree-vectorize
+decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
 endif
 
 # Modules
@@ -104,80 +104,90 @@ ZSTD_NO_INLINE ?= 0
 ZSTD_STRIP_ERROR_STRINGS ?= 0
 
 ifeq ($(ZSTD_LIB_COMPRESSION), 0)
-	ZSTD_LIB_DICTBUILDER = 0
+  ZSTD_LIB_DICTBUILDER = 0
-	ZSTD_LIB_DEPRECATED = 0
+  ZSTD_LIB_DEPRECATED = 0
 endif
 
 ifeq ($(ZSTD_LIB_DECOMPRESSION), 0)
-	ZSTD_LEGACY_SUPPORT = 0
+  ZSTD_LEGACY_SUPPORT = 0
-	ZSTD_LIB_DEPRECATED = 0
+  ZSTD_LIB_DEPRECATED = 0
 endif
 
 ifneq ($(ZSTD_LIB_COMPRESSION), 0)
-	ZSTD_FILES += $(ZSTDCOMP_FILES)
+  ZSTD_FILES += $(ZSTDCOMP_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DECOMPRESSION), 0)
-	ZSTD_FILES += $(ZSTDDECOMP_FILES)
+  ZSTD_FILES += $(ZSTDDECOMP_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DEPRECATED), 0)
-	ZSTD_FILES += $(ZDEPR_FILES)
+  ZSTD_FILES += $(ZDEPR_FILES)
 endif
 
 ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
-	ZSTD_FILES += $(ZDICT_FILES)
+  ZSTD_FILES += $(ZDICT_FILES)
 endif
 
 ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
-	CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
+  CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
 endif
 
 ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
-	CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
+  CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
 endif
 
 ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0)
-	CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
+  CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
 endif
 
 ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0)
-	CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
+  CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
 endif
 
 ifneq ($(ZSTD_NO_INLINE), 0)
-	CFLAGS += -DZSTD_NO_INLINE
+  CFLAGS += -DZSTD_NO_INLINE
 endif
 
 ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
-	CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
+  CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
 endif
 
 ifneq ($(ZSTD_LEGACY_MULTITHREADED_API), 0)
-	CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
+  CFLAGS += -DZSTD_LEGACY_MULTITHREADED_API
 endif
 
 ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
 ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
-	ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
+  ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
 endif
 endif
 CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
 
-ZSTD_OBJ   := $(patsubst %.c,%.o,$(ZSTD_FILES))
+ZSTD_LOCAL_SRC := $(notdir $(ZSTD_FILES))
+ZSTD_LOCAL_OBJ := $(ZSTD_LOCAL_SRC:.c=.o)
+
+ZSTD_SUBDIR := common compress decompress dictBuilder legacy deprecated
+vpath %.c $(ZSTD_SUBDIR)
+
+BUILD_DIR ?= obj
+ZSTD_DYNLIB_DIR := $(BUILD_DIR)/dynlib
+ZSTD_DYNLIB_OBJ := $(addprefix $(ZSTD_DYNLIB_DIR)/, $(ZSTD_LOCAL_OBJ))
+ZSTD_STATLIB_DIR := $(BUILD_DIR)/statlib
+ZSTD_STATLIB_OBJ := $(addprefix $(ZSTD_STATLIB_DIR)/, $(ZSTD_LOCAL_OBJ))
 
 # macOS linker doesn't support -soname, and use different extension
 # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
 ifeq ($(shell uname), Darwin)
-	SHARED_EXT = dylib
+  SHARED_EXT = dylib
-	SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
+  SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
-	SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
+  SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
-	SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
+  SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
 else
-	SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
+  SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
-	SHARED_EXT = so
+  SHARED_EXT = so
-	SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
+  SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
-	SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
+  SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
 endif
 
 
@@ -191,7 +201,7 @@ lib-all: all
 all: lib
 
 libzstd.a: ARFLAGS = rcs
-libzstd.a: $(ZSTD_OBJ)
+libzstd.a: $(ZSTD_STATLIB_OBJ)
 	@echo compiling static library
 	$(Q)$(AR) $(ARFLAGS) $@ $^
 
@@ -205,8 +215,9 @@ $(LIBZSTD): $(ZSTD_FILES)
 else
 
 LIBZSTD = libzstd.$(SHARED_EXT_VER)
-$(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden
+$(LIBZSTD): CFLAGS += -fPIC
-$(LIBZSTD): $(ZSTD_FILES)
+$(LIBZSTD): LDFLAGS += -shared -fvisibility=hidden
+$(LIBZSTD): $(ZSTD_DYNLIB_OBJ)
 	@echo compiling dynamic library $(LIBVER)
 	$(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
@@ -221,6 +232,7 @@ libzstd : $(LIBZSTD)
 .PHONY: lib
 lib : libzstd.a libzstd
 
+
 # note : do not define lib-mt or lib-release as .PHONY
 # make does not consider implicit pattern rule for .PHONY target
 
@@ -234,6 +246,28 @@ lib : libzstd.a libzstd
 	@echo release build completed
 
 
+# Generate .h dependencies automatically
+
+DEPFLAGS = -MT $@ -MMD -MP -MF
+
+$(ZSTD_DYNLIB_DIR)/%.o : %.c $(ZSTD_DYNLIB_DIR)/%.d | $(ZSTD_DYNLIB_DIR)
+	@echo $@
+	$(Q)$(COMPILE.c) $(DEPFLAGS) $(ZSTD_DYNLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
+
+$(ZSTD_STATLIB_DIR)/%.o : %.c $(ZSTD_STATLIB_DIR)/%.d | $(ZSTD_STATLIB_DIR)
+	@echo $@
+	$(Q)$(COMPILE.c) $(DEPFLAGS) $(ZSTD_STATLIB_DIR)/$*.d $(OUTPUT_OPTION) $<
+
+MKDIR ?= mkdir
+$(ZSTD_DYNLIB_DIR) $(ZSTD_STATLIB_DIR):
+	$(Q)$(MKDIR) -p $@
+
+DEPFILES := $(ZSTD_DYNLIB_OBJ:.o=.d) $(ZSTD_STATLIB_OBJ:.o=.d)
+$(DEPFILES):
+
+include $(wildcard $(DEPFILES))
+
+
 # Special case : building library in single-thread mode _and_ without zstdmt_compress.c
 ZSTDMT_FILES = compress/zstdmt_compress.c
 ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES))
@@ -248,6 +282,7 @@ clean:
 	$(Q)$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
 	$(Q)$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
 	$(Q)$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
+	$(Q)$(RM) -r $(ZSTD_STATLIB_DIR)/* $(ZSTD_DYNLIB_DIR)/*
 	@echo Cleaning library completed
 
 #-----------------------------------------------------------------------------
@@ -277,7 +312,7 @@ ifeq (,$(PCLIBDIR))
 # Additional prefix check is required, since the empty string is technically a
 # valid PCLIBDIR
 ifeq (,$(shell echo "$(LIBDIR)" | sed -n $(SED_ERE_OPT) -e "\\@^$(EXEC_PREFIX)(/|$$)@ p"))
-$(error configured libdir ($(LIBDIR)) is outside of prefix ($(PREFIX)), can't generate pkg-config file)
+$(error configured libdir ($(LIBDIR)) is outside of prefix ($(EXEC_PREFIX)), can't generate pkg-config file)
 endif
 endif
 
@@ -285,20 +320,20 @@ ifeq (,$(PCINCDIR))
 # Additional prefix check is required, since the empty string is technically a
 # valid PCINCDIR
 ifeq (,$(shell echo "$(INCLUDEDIR)" | sed -n $(SED_ERE_OPT) -e "\\@^$(PREFIX)(/|$$)@ p"))
-$(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(EXEC_PREFIX)), can't generate pkg-config file)
+$(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(PREFIX)), can't generate pkg-config file)
 endif
 endif
 
 ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly))
-PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
+  PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
 else
-PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
+  PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
 endif
 
 ifneq (,$(filter $(shell uname),SunOS))
-INSTALL ?= ginstall
+  INSTALL ?= ginstall
 else
-INSTALL ?= install
+  INSTALL ?= install
 endif
 
 INSTALL_PROGRAM ?= $(INSTALL)