chore(webfuse): debian build support (#22)

* adds debian builder image * adds arm32v7 debian based builder image * sets the default build target to amd64-ubuntu-builder if MARCH is unset * adds distro filter and set default to ubuntu for travis disables debian builds for now * fixes wrapper for non container hosts * fixes distro selection and parallel mflags * changes parallelization option * fixes parallel flags and variable names * enhances makefile adds checksum for fetch dependencies discovers necessary commands fixes error when script get_container_id not found * fixes copy artifacts to image * fixes typo * fixes another typo * removes bash dependency - simplifies scripts to support posix shell - adds REGISTERY_PREFIX to support usage of local docker registry - renames wrapper template - adds discovery for tty and interactive for wrappers * adds docker-compose wrapper * adds ability to start wrapper in namespace of another container * renames compose wrapper script * adds support to retrieve source epoch date for svn * fixes handling for positional arguments * removes docker connection options * removes dependency to readlink * removes duplicate options
2026-03-02 04:09:18 +00:00 · 2019-04-06 08:09:21 +02:00
parent b1527d5581
commit 500353115c
11 changed files with 498 additions and 109 deletions
--- a/206
+++ b/206
@@ -1,29 +1,33 @@
-PARALLELMFLAGS ?= -j$(shell nproc)
-MAKEFLAGS += $(PARALLELMFLAGS) --no-builtin-rules
+MAKEFLAGS += $(_PARALLELMFLAGS) --no-builtin-rules

 .PHONY: default
 default: all

 # Overridable defaults

-export SOURCE_DATE_EPOCH ?= $(shell $(PROJECT_ROOT)/build/get_source_date_epoch.sh)
-export BUILDTIME ?= $(shell date -u -d '@$(SOURCE_DATE_EPOCH)' --rfc-3339 ns 2>/dev/null | sed -e 's/ /T/')
-export DOCKER ?= docker
-
+DOCKER ?= docker
 DOCKER_BUILDKIT ?= 

+PARALLELMFLAGS ?= 
+NPROC ?= $(shell echo '$(PARALLELMFLAGS)' | sed -n -e 's@.*-j.*\([0-9]\+\)@\1@p')
+
 VERBOSE ?= 
 BUILDVERBOSE ?= 
-BUILDTARGET ?= 
+
+$(MARCH)BUILDTARGET ?= amd64-ubuntu-builder
 BUILDTYPE ?= Debug
 MARCH ?= $(call march,$(BUILDTARGET))
+DISTRO ?=

 PROJECT_NAME ?= webfuse
 PROJECT_ROOT ?= .
-VERSION ?= $(shell cat $(PROJECT_ROOT)/VERSION)
 OUT ?= $(PROJECT_ROOT)/.build

-UID ?= $(shell id -u)
+VERSION ?= $(shell cat $(PROJECT_ROOT)/VERSION)
+VERSION := $(VERSION)
+
+USERID ?= $(shell id -u)
+USERID := $(USERID)

 CONTAINER_USER ?= user
 CONTAINER_GROUP ?= user
@@ -32,44 +36,65 @@ CONTAINER_CGROUP_PARENT ?=
 HOST_CONTAINER ?= $(shell $(PROJECT_ROOT)/build/get_container_id.sh)
 HOST_CONTAINER := $(HOST_CONTAINER)

+UBUNTU_CODENAME ?= bionic
+DEBIAN_CODENAME ?= testing-slim
+
+SOURCE_DATE_EPOCH ?= $(shell $(PROJECT_ROOT)/build/get_source_date_epoch.sh)
+SOURCE_DATE_EPOCH := $(SOURCE_DATE_EPOCH)
+export SOURCE_DATE_EPOCH
+
+BUILDTIME ?= $(shell date -u -d '@$(SOURCE_DATE_EPOCH)' --rfc-3339 ns 2>/dev/null | sed -e 's/ /T/')
+BUILDTIME := $(BUILDTIME)
+export BUILDTIME
+
 $(HOST_CONTAINER)PORTABLE_WORSPACE ?= 
 CONTAINER_PROJECT_ROOT ?= /workspace/src
 CONTAINER_OUT ?= /workspace/out
 $(PORTABLE_WORSPACE)CONTAINER_PROJECT_ROOT = $(abspath $(PROJECT_ROOT))
 $(PORTABLE_WORSPACE)CONTAINER_OUT = $(abspath $(OUT))

-UBUNTU_CODENAME ?= bionic
+filter_out_command = $(filter $1,$(foreach CMD,$1,$(shell command -v $(CMD) 2>&1 1>/dev/null || echo $(CMD))))

-# Dependencies
+DISABLE_MD5SUM ?= $(call filter_out_command,md5sum)
+DISABLE_MD5SUM := $(DISABLE_MD5SUM)

-DUMB_INIT_VERISON ?= 1.2.2
-DOCKER_BUILDARGS += DUMB_INIT_VERISON=$(DUMB_INIT_VERISON)
-FETCH_TARGETS += $(OUT)/dumb-init-$(DUMB_INIT_VERISON).tar.gz
-$(OUT)/dumb-init-$(DUMB_INIT_VERISON).tar.gz: URL := https://github.com/Yelp/dumb-init/archive/v${DUMB_INIT_VERISON}.tar.gz
+# Project dependencies
+
+DUMB_INIT_VERSION ?= 1.2.2
+DOCKER_BUILDARGS += DUMB_INIT_VERSION=$(DUMB_INIT_VERSION)
+FETCH_TARGETS += $(OUT)/dumb-init-$(DUMB_INIT_VERSION).tar.gz
+$(OUT)/dumb-init-$(DUMB_INIT_VERSION).tar.gz: URL := https://github.com/Yelp/dumb-init/archive/v${DUMB_INIT_VERSION}.tar.gz
+$(DISABLE_MD5SUM)$(OUT)/dumb-init-$(DUMB_INIT_VERSION).tar.gz: MD5 := 6166084b05772cdcf615a762c6f3b32e

 GTEST_VERSION ?= 1.8.1
 DOCKER_BUILDARGS += GTEST_VERSION=$(GTEST_VERSION)
 FETCH_TARGETS += $(OUT)/googletest-release-$(GTEST_VERSION).tar.gz
 $(OUT)/googletest-release-$(GTEST_VERSION).tar.gz: URL := https://github.com/google/googletest/archive/release-$(GTEST_VERSION).tar.gz
+$(DISABLE_MD5SUM)$(OUT)/googletest-release-$(GTEST_VERSION).tar.gz: MD5 := 2e6fbeb6a91310a16efe181886c59596

 FUSE_VERSION ?= 3.1.1
 DOCKER_BUILDARGS += FUSE_VERSION=$(FUSE_VERSION)
 FETCH_TARGETS += $(OUT)/libfuse-fuse-$(FUSE_VERSION).tar.gz
 $(OUT)/libfuse-fuse-$(FUSE_VERSION).tar.gz: URL := https://github.com/libfuse/libfuse/archive/fuse-$(FUSE_VERSION).tar.gz
+$(DISABLE_MD5SUM)$(OUT)/libfuse-fuse-$(FUSE_VERSION).tar.gz: MD5 := 097f194856938afdd98bea1a5c046edd

 WEBSOCKETS_VERSION ?= 3.1.0
 DOCKER_BUILDARGS += WEBSOCKETS_VERSION=$(WEBSOCKETS_VERSION)
 FETCH_TARGETS += $(OUT)/libwebsockets-$(WEBSOCKETS_VERSION).tar.gz
 $(OUT)/libwebsockets-$(WEBSOCKETS_VERSION).tar.gz: URL := https://github.com/warmcat/libwebsockets/archive/v$(WEBSOCKETS_VERSION).tar.gz
+$(DISABLE_MD5SUM)$(OUT)/libwebsockets-$(WEBSOCKETS_VERSION).tar.gz: MD5 := 325359a25d5f6d22725ff5d086db1c76

 JANSSON_VERSION ?= 2.12
 DOCKER_BUILDARGS += JANSSON_VERSION=$(JANSSON_VERSION)
 FETCH_TARGETS += $(OUT)/jansson-$(JANSSON_VERSION).tar.gz
 $(OUT)/jansson-$(JANSSON_VERSION).tar.gz: URL := https://github.com/akheron/jansson/archive/v$(JANSSON_VERSION).tar.gz
+$(DISABLE_MD5SUM)$(OUT)/jansson-$(JANSSON_VERSION).tar.gz: MD5 := c4b106528d5ffb521178565de1ba950d

 QEMU_VERSION ?= v3.1.0-2
+DOCKER_BUILDARGS += QEMU_VERSION_=$(QEMU_VERSION)
 FETCH_TARGETS += $(OUT)/docker/qemu-arm-static-$(QEMU_VERSION)
 $(OUT)/docker/qemu-arm-static-$(QEMU_VERSION): URL := https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-arm-static
+$(DISABLE_MD5SUM)$(OUT)/docker/qemu-arm-static-$(QEMU_VERSION): MD5 := 8ebd24e63fdfa07c557d45373bd831b1

 # Architecture-specific rule target configuration

@@ -78,17 +103,25 @@ march = $(shell echo '$1' | sed -n -e 's@\([^-]*\)-.*@\1@p')
 TARGETS += $(BUILDTARGET)
 MARCH := $(MARCH)

-MARCH_AMD64 := $(filter-out amd64,$(MARCH))
-$(MARCH_AMD64)MARCHS += amd64
-$(MARCH_AMD64)TARGETS += amd64-ubuntu-builder
+DISABLE_AMD64 := $(filter-out amd64,$(MARCH))
+$(DISABLE_AMD64)MARCHS += amd64
+
+$(DISABLE_AMD64)$(BUILDTARGET)TARGETS += amd64-ubuntu-builder
 $(OUT)/amd64-ubuntu-builder/rules.mk: TARGET := amd64-ubuntu-builder

-MARCH_ARM32V7 := $(filter-out arm32v7,$(MARCH))
-$(MARCH_ARM32V7)MARCHS += arm32v7
-$(MARCH_ARM32V7)TARGETS += arm32v7-ubuntu-builder
+$(DISABLE_AMD64)$(BUILDTARGET)TARGETS += amd64-debian-builder
+$(OUT)/amd64-debian-builder/rules.mk: TARGET := amd64-debian-builder
+
+DISABLE_ARM32V7 := $(filter-out arm32v7,$(MARCH))
+$(DISABLE_ARM32V7)MARCHS += arm32v7
+
+$(DISABLE_ARM32V7)$(BUILDTARGET)TARGETS += arm32v7-ubuntu-builder
 $(OUT)/arm32v7-ubuntu-builder/rules.mk: TARGET := arm32v7-ubuntu-builder

-$(MARCH_AMD64)MEMCHECK_TARGETS += $(addprefix memcheck-,$(TARGETS))
+$(DISABLE_ARM32V7)$(BUILDTARGET)TARGETS += arm32v7-debian-builder
+$(OUT)/arm32v7-debian-builder/rules.mk: TARGET := arm32v7-debian-builder
+
+$(DISABLE_AMD64)MEMCHECK_TARGETS += $(addprefix memcheck-,$(TARGETS))

 ARM_TARGETS = $(filter arm%,$(TARGETS))
 $(addprefix $(OUT)/docker/,$(ARM_TARGETS)): $(OUT)/docker/qemu-arm-static-$(QEMU_VERSION)
@@ -96,6 +129,9 @@ $(addprefix $(OUT)/docker/,$(ARM_TARGETS)): $(OUT)/docker/qemu-arm-static-$(QEMU
 UBUNTU_TARGETS = $(filter $(addsuffix -ubuntu%,$(MARCHS)),$(TARGETS))
 $(addprefix $(OUT)/docker/,$(UBUNTU_TARGETS)): CODENAME := $(UBUNTU_CODENAME)

+DEBIAN_TARGETS = $(filter $(addsuffix -debian%,$(MARCHS)),$(TARGETS))
+$(addprefix $(OUT)/docker/,$(DEBIAN_TARGETS)): CODENAME := $(DEBIAN_CODENAME)
+
 # Common rule target configuration

 CMAKEFLAGS += '-GNinja'
@@ -104,6 +140,8 @@ CMAKEFLAGS += '-DCMAKE_BUILD_TYPE=$(BUILDTYPE)'
 BUILDSILENT := $(if $(BUILDVERBOSE),,1)
 $(BUILDSILENT)NINJAFLAGS += -v

+CURLFLAGS += -s
+
 DOCKER_RUNFLAGS += --device /dev/fuse
 DOCKER_RUNFLAGS += --cap-add SYS_ADMIN
 DOCKER_RUNFLAGS += --security-opt apparmor:unconfined
@@ -111,20 +149,18 @@ DOCKER_RUNFLAGS += --security-opt apparmor:unconfined
 DOCKER_RUNFLAGS += --cap-add SYS_PTRACE
 DOCKER_RUNFLAGS += --security-opt seccomp=unconfined

-DOCKER_RUNFLAGS += --interactive
-DOCKER_RUNFLAGS += --rm
 DOCKER_RUNFLAGS += --user $(CONTAINER_USER):$(CONTAINER_GROUP)
 DOCKER_RUNFLAGS += --env SOURCE_DATE_EPOCH
 DOCKER_RUNFLAGS += --env BUILDTIME
 DOCKER_RUNFLAGS += --env NINJA_STATUS

 DOCKER_BUILDARGS += CODENAME=$(CODENAME)
-DOCKER_BUILDARGS += PARALLELMFLAGS=$(PARALLELMFLAGS)
-DOCKER_BUILDARGS += USERID=$(UID)
+DOCKER_BUILDARGS += PARALLELMFLAGS=$(_PARALLELMFLAGS)
+DOCKER_BUILDARGS += USERID=$(USERID)
 DOCKER_BUILDARGS += PROJECT_ROOT=$(CONTAINER_PROJECT_ROOT)
 DOCKER_BUILDARGS += OUT=$(CONTAINER_OUT)
+DOCKER_BUILDARGS += REGISTRY_PREFIX=$(REGISTRY_PREFIX)

-DOCKER_BUILDFLAGS += --rm
 DOCKER_BUILDFLAGS += $(addprefix --build-arg ,$(DOCKER_BUILDARGS))

 OUT_TARGETS += $(addprefix $(OUT)/,$(TARGETS))
@@ -142,20 +178,47 @@ EXTRACT_TARGETS += $(patsubst $(OUT)/%.tar.gz,$(OUT)/src/%,$(FETCH_TARGETS))
 DISCOVER_CC_TARGETS += $(addprefix discover-cc-,$(firstword $(TARGETS)))
 RULE_TARGETS += $(addsuffix /rules.mk,$(OUT_TARGETS))

-TARGETS := $(sort $(TARGETS))
+uc = $(shell echo '$1' | sed -e 's/.*/\U&/g')

-# Macros
+DISTRO_PREFIX = $(addsuffix _,$(call uc,$(DISTRO)))
+
+MARCHS := $(sort $(MARCHS))
+TARGETS := $(sort $($(DISTRO_PREFIX)TARGETS))
+
+TASKS := $(words $(if $(TARGETS),$(TARGETS),_))
+
+DISBALE_OSYNC ?= $(filter 1,$(TASKS))
+$(DISBALE_OSYNC)GNUMAKEFLAGS += --output-sync
+
+_NPROC = $(NPROC)
+ifeq ($(strip $(_NPROC)),)
+_NPROC = $(shell nproc)
+endif
+_PARALLELMFLAGS := $(addprefix -j,$(shell echo "$$(($(_NPROC)/$(TASKS)))"))
+
+# Common macros

 echo_if_silent = VERBOSE=1
 $(VERBOSE)echo_if_silent = echo $1
 $(VERBOSE)SILENT := @

+curl = $(call echo_if_silent,curl -fSL $(CURLFLAGS) -o $1 $2) \
+  && curl -fSL $(CURLFLAGS) -o $1 $2 \
+  && { \
+       if [ -n "$3" ]; then \
+         echo "$3\t$1" > $1.md5; \
+         md5sum -c $1.md5; \
+       else \
+         echo 'warning:$1: no md5 skipping verification' 1>&2; \
+       fi; \
+     }
+
 $(HOST_CONTAINER)image_run_volumes += --volume '$(realpath $(PROJECT_ROOT)):$(CONTAINER_PROJECT_ROOT):cached'
 $(HOST_CONTAINER)image_run_volumes += --volume '$(realpath $(OUT)/$1):$(CONTAINER_OUT)/$1:delegated'
 image_run_volumes += $(addprefix --volumes-from ,$2)

 image_name = $(REGISTRY_PREFIX)$(subst -,/,$1)/$(PROJECT_NAME):$(VERSION)
-image_run = $(DOCKER) run $(DOCKER_RUNFLAGS) \
+image_run = $(DOCKER) run --rm --interactive $(DOCKER_RUNFLAGS) \
  $(call image_run_volumes,$1,$(HOST_CONTAINER)) \
  $(addprefix --cgroup-parent ,$(CONTAINER_CGROUP_PARENT)) \
  --workdir '$(CONTAINER_OUT)/$1/$(BUILDTYPE)' \
@@ -168,7 +231,7 @@ image_rule = \
    $$(SILENT)$$(call image,$1)
 image = \
     $(call echo_if_silent,TARGET=$1 docker build $(call image_name,$1) $(OUT)) \
-  && $(DOCKER) build $(DOCKER_BUILDFLAGS) --iidfile $@ --file $< --tag $(call image_name,$1) $(OUT)
+  && $(DOCKER) build --rm $(DOCKER_BUILDFLAGS) --iidfile $@ --file $< --tag $(call image_name,$1) $(OUT)

 configure_rule = \
  $$(OUT)/$1/$$(BUILDTYPE)/CMakeCache.txt: $$(PROJECT_ROOT)/CMakeLists.txt $$(OUT)/docker/$1 | $$(OUT)/$1/$$(BUILDTYPE)/gdbserver; \
@@ -180,7 +243,7 @@ configure = \
 build_rule = \
  build-$1: $$(OUT)/$1/$$(BUILDTYPE)/CMakeCache.txt; \
    $$(SILENT)$$(call build,$1)
-build = $(call run,$1,ninja $(PARALLELMFLAGS) $(NINJAFLAGS) $(GOALS))
+build = $(call run,$1,ninja $(_PARALLELMFLAGS) $(NINJAFLAGS) $(GOALS))

 check_rule = \
  check-$1: build-$1;
@@ -212,7 +275,7 @@ discover_cc_rule = \
 discover_cc = cat $<

 wrapper_rule = \
-  $$(OUT)/$1/$$(BUILDTYPE)/gdbserver: $$(PROJECT_ROOT)/build/run_image.template $$(OUT)/docker/$1; \
+  $$(OUT)/$1/$$(BUILDTYPE)/gdbserver: $$(PROJECT_ROOT)/build/run_image.sh.template $$(OUT)/docker/$1; \
    $$(SILENT)$$(call wrapper,$1)
 wrapper = \
     $(call echo_if_silent,generating $@) \
@@ -225,35 +288,52 @@ wrapper = \
       $< > $@ \
  && chmod +x $@

+# Makefile dependencies
+
+MAKEDEPS += $(DOCKER)
+MAKEDEPS += id
+MAKEDEPS += cat
+MAKEDEPS += cp
+MAKEDEPS += rm
+MAKEDEPS += mkdir
+MAKEDEPS += sed
+MAKEDEPS += chmod
+MAKEDEPS += test
+MAKEDEPS += touch
+MAKEDEPS += curl
+MAKEDEPS += tar
+MAKEDEPS += gunzip
+
+MISSING_MAKEDEPS += $(call filter_out_command,$(MAKEDEPS))
+
 # Rules

 ifneq ($(MAKECMDGOALS),distclean)
 -include $(RULE_TARGETS)
 endif

-$(RULE_TARGETS): $(PROJECT_ROOT)/Makefile | $(OUT_DIRS)
+$(RULE_TARGETS): $(PROJECT_ROOT)/Makefile | $(MISSING_MAKEDEPS) $(OUT_DIRS)
 	$(SILENT) \
 	{ \
-		echo; \
-		echo '$(call image_rule,$(TARGET))'; \
-		echo; \
-		echo '$(call configure_rule,$(TARGET))'; \
-		echo; \
-		echo '$(call build_rule,$(TARGET))'; \
-		echo; \
-		echo '$(call check_rule,$(TARGET))'; \
-		echo; \
-		echo '$(call memcheck_rule,$(TARGET))'; \
-		echo; \
-		echo '$(call run_rule,$(TARGET))'; \
-		echo; \
-		echo '$(call clean_rule,$(TARGET))'; \
-		echo; \
-		echo '$(call discover_cc_settings_rule,$(TARGET))'; \
-		echo; \
-		echo '$(call discover_cc_rule,$(TARGET))'; \
-		echo; \
-		echo '$(call wrapper_rule,$(TARGET))'; \
+	  echo '$(call image_rule,$(TARGET))'; \
+	  echo; \
+	  echo '$(call configure_rule,$(TARGET))'; \
+	  echo; \
+	  echo '$(call build_rule,$(TARGET))'; \
+	  echo; \
+	  echo '$(call check_rule,$(TARGET))'; \
+	  echo; \
+	  echo '$(call memcheck_rule,$(TARGET))'; \
+	  echo; \
+	  echo '$(call run_rule,$(TARGET))'; \
+	  echo; \
+	  echo '$(call clean_rule,$(TARGET))'; \
+	  echo; \
+	  echo '$(call discover_cc_settings_rule,$(TARGET))'; \
+	  echo; \
+	  echo '$(call discover_cc_rule,$(TARGET))'; \
+	  echo; \
+	  echo '$(call wrapper_rule,$(TARGET))'; \
 	} > $@

 .PHONY: all build-%
@@ -290,17 +370,14 @@ debug-print-%:

 $(CHECK_TARGETS): GOALS := test

-$(OUT)/docker/qemu-arm-static-$(QEMU_VERSION):
-	$(SILENT) \
-	     $(call echo_if_silent,curl -fsSL -o $@ $(URL)) \
-	  && curl -fsSL -o $@ $(URL) \
-	  && chmod +x $@ 
-
 $(OUT)/docker/% : $(PROJECT_ROOT)/build/% | $(OUT_DIRS)
 	cp $< $@

-$(OUT)/%.tar.gz: | $(OUT_DIRS)
-	curl -fsSL -o $@ $(URL)
+$(OUT)/docker/qemu-arm-static-$(QEMU_VERSION): $(PROJECT_ROOT)/Makefile
+	$(SILENT)$(call curl,$@,$(URL),$(MD5)) && chmod +x $@ 
+
+$(OUT)/%.tar.gz: $(PROJECT_ROOT)/Makefile | $(OUT_DIRS)
+	$(SILENT)$(call curl,$@,$(URL),$(MD5))

 $(OUT)/src/%: $(OUT)/%.tar.gz | $(OUT_DIRS)
 	$(SILENT) \
@@ -311,3 +388,8 @@ $(OUT)/src/%: $(OUT)/%.tar.gz | $(OUT_DIRS)
 $(OUT_DIRS):
 	$(SILENT)mkdir -p $@

+$(MISSING_MAKEDEPS):
+	$(error 'required commands $(MISSING_MAKEDEPS) not found; install appropriate packages e.g. docker-ce, busybox and curl')
+
+.DELETE_ON_ERROR: $(FETCH_TARGETS)
+