moves download of artifacts to bootstrapping makefile

.gitignore

@@ -1,4 +1,3 @@
 .vscode
 .build
 .deps
-docker/qemu-*

.travis.yml

@@ -10,7 +10,7 @@ before_install:
 - sudo apt-get install -y qemu-user-static
 env:
   global:
-    - MAKEFLAGS="-j 2"
+    - PARALLELMFLAGS="-j2"
   matrix:
     - MARCH=amd64
     - MARCH=arm32v7

Makefile

@@ -1,27 +1,32 @@
-MAKEFLAGS += --no-builtin-rules
-
 .PHONY: default
 default: all
 
-## Overridable defaults
+# Overridable defaults
 
 $(VERBOSE)SILENT ?= @
 PROJECT_ROOT ?= .
 OUT ?= $(PROJECT_ROOT)/.build
 VERSION ?= $(shell cat $(PROJECT_ROOT)/VERSION)
-NPROC ?= $(shell nproc)
+PARALLELMFLAGS ?= -j$(shell nproc)
 UID ?= $(shell id -u)
 DOCKER ?= docker
 DOCKER_RUNUSER ?= $(UID)
 DOCKER_RUNGROUP ?= $(shell id -g)
 
 UBUNTU_CODENAME ?= bionic
+
 QEMU_VERSION ?= v3.1.0-2
+GTEST_VERSION ?= 1.8.1
+FUSE_VERSION ?= 3.1.1
+WEBSOCKETS_VERSION ?= 3.1.0
+JANSSON_VERSION ?= 2.12
+
+# Target configuration
 
 MARCH_AMD64 := $(filter-out amd64,$(MARCH))
 MARCH_ARM32V7 := $(filter-out arm32v7,$(MARCH))
 
-## Defaults
+MAKEFLAGS += $(PARALLELMFLAGS) --no-builtin-rules
 
 $(MARCH_AMD64)TARGETS += wsfs-builder-amd64-ubuntu
 $(MARCH_ARM32V7)TARGETS += wsfs-builder-arm32v7-ubuntu
@@ -35,41 +40,42 @@ DOCKER_RUNFLAGS += --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmo
 DOCKER_RUNFLAGS += --env SOURCE_DATE_EPOCH
 
 DOCKER_BUILDARGS += CODENAME=$(CODENAME)
-DOCKER_BUILDARGS += NPROC=$(NPROC)
+DOCKER_BUILDARGS += PARALLELMFLAGS=$(PARALLELMFLAGS)
 DOCKER_BUILDARGS += USERID=$(UID)
+DOCKER_BUILDARGS += SRC=$(realpath $(PROJECT_ROOT))
+DOCKER_BUILDARGS += OUT=$(realpath $(OUT))
 
 DOCKER_BUILDFLAGS += $(addprefix --build-arg ,$(DOCKER_BUILDARGS))
 
 OUT_DIRS += $(OUT)
 OUT_DIRS += $(OUT)/docker
+OUT_DIRS += $(OUT)/src
 OUT_DIRS += $(addprefix $(OUT)/,$(TARGETS))
 
-PROJECT_RESOURCES = $(call glob_files,$(PROJECT_ROOT)/**)
-
 ALL_TARGETS += $(addprefix compile-,$(TARGETS)) 
 CHECK_TARGETS += $(addprefix check-,$(TARGETS))
 CONFIGURE_TARGETS += $(addsuffix /CMakeCache.txt,$(addprefix $(OUT)/,$(TARGETS)))
 
-## Helper macros
+FETCH_TARGETS += $(OUT)/docker/qemu-arm-static-$(QEMU_VERSION)
+FETCH_TARGETS += $(OUT)/googletest-release-$(GTEST_VERSION).tar.gz
+FETCH_TARGETS += $(OUT)/libfuse-fuse-$(FUSE_VERSION).tar.gz
+FETCH_TARGETS += $(OUT)/libwebsockets-$(WEBSOCKETS_VERSION).tar.gz
+FETCH_TARGETS += $(OUT)/jansson-$(JANSSON_VERSION).tar.gz
 
-# $(call glob,include_pattern...,exclude_regex...,flags...)
+EXTRACT_TARGETS += $(patsubst $(OUT)/%.tar.gz,$(OUT)/src/%,$(FETCH_TARGETS))
-glob = $(shell bash -c 'shopt -s globstar nullglob && find $1 -maxdepth 0 $3 | grep -v $(addprefix -e, $2)')
-# $(call glob_files,include_pattern...,exclude_regex...)
-glob_files = $(call glob,$1,^$(OUT) $2,-type f)
-
-## Force eager evaluation
 
 DOCKER_RUNGROUP := $(DOCKER_RUNGROUP)
 VERSION := $(VERSION)
 PROJECT_ROOT := $(PROJECT_ROOT)
 OUT := $(OUT)
 
-## Special targets
+# Rules
 
-.PRECIOUS: $(OUT)/docker/%
+$(OUT)/docker/qemu-arm-static-$(QEMU_VERSION): URL := https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-arm-static
-.DELETE_ON_ERROR: $(OUT)/%/CMakeCache.txt $(OUT)/docker/%
+$(OUT)/googletest-release-$(GTEST_VERSION).tar.gz: URL := https://github.com/google/googletest/archive/release-$(GTEST_VERSION).tar.gz
-
+$(OUT)/libfuse-fuse-$(FUSE_VERSION).tar.gz: URL := https://github.com/libfuse/libfuse/archive/fuse-$(FUSE_VERSION).tar.gz
-## Targets
+$(OUT)/libwebsockets-$(WEBSOCKETS_VERSION).tar.gz: URL := https://github.com/warmcat/libwebsockets/archive/v$(WEBSOCKETS_VERSION).tar.gz
+$(OUT)/jansson-$(JANSSON_VERSION).tar.gz: URL := https://github.com/akheron/jansson/archive/v$(JANSSON_VERSION).tar.gz
 
 .PHONY: all
 all: $(ALL_TARGETS)
@@ -81,20 +87,50 @@ check: $(CHECK_TARGETS)
 clean: $(CLEAN_TARGETS)
 	$(SILENT)-rm -rf $(OUT_DIRS)
 
-.PHONY: configure
+.PHONY: get-deps
-configure: $(CONFIGURE_TARGETS) 
+get-deps: $(EXTRACT_TARGETS)
+
+.PHONY: debug-print-%
+debug-print-%:
+	@printf '%s\n' '$*:' $($*)
+
+.PHONY: check-%
+check-%: compile-%;
+
+.PHONY: compile-%
+compile-%: $(OUT)/%/CMakeCache.txt
+	$(SILENT)$(DOCKER) run $(DOCKER_RUNFLAGS) \
+	  --volume '$(realpath $(PROJECT_ROOT)):/tmp' \
+	  --volume '$(realpath $(dir $<)):/tmp/$(notdir $(OUT))' \
+	  --workdir '/tmp/$(notdir $(OUT))' \
+	  $*:$(VERSION) \
+	  ninja $(PARALLELMFLAGS) $(GLOAS)
+
+$(CHECK_TARGETS): GLOAS := test
+$(CHECK_TARGETS): DOCKER_RUNUSER := user
+
+$(OUT)/docker/qemu-arm-static-$(QEMU_VERSION):
+	$(SILENT) \
+	     curl -fsSL -o $@ $(URL) \
+	  && chmod +x $@ 
+
+$(OUT)/docker/wsfs-builder-arm32v7-ubuntu: $(OUT)/docker/qemu-arm-static-$(QEMU_VERSION)
+
+$(OUT)/% : $(PROJECT_ROOT)/% | $(OUT_DIRS)
+	cp $< $@
 
 %-ubuntu: CODENAME := $(UBUNTU_CODENAME)
 
-$(PROJECT_ROOT)/docker/qemu-arm-static-$(QEMU_VERSION):
+$(OUT)/docker/%: $(OUT)/docker/%.dockerfile $(EXTRACT_TARGETS) $(PROJECT_ROOT)/Makefile | $(OUT_DIRS)
+	$(SILENT)$(DOCKER) build --rm $(DOCKER_BUILDFLAGS) --iidfile $@ --file $< --tag $*:$(VERSION) $(OUT)
+
+$(OUT)/%.tar.gz: | $(OUT_DIRS)
+	curl -fsSL -o $@ $(URL)
+
+$(OUT)/src/%: $(OUT)/%.tar.gz | $(OUT_DIRS)
 	$(SILENT) \
-	     curl -fSL -o $@ https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-arm-static \
+	     tar -C $(dir $@) -xf $< \
-	  && chmod +x $@ 
+	  && touch $@
-
-$(OUT)/docker/wsfs-builder-arm32v7-ubuntu: $(PROJECT_ROOT)/docker/qemu-arm-static-$(QEMU_VERSION)
-
-$(OUT)/docker/%: $(PROJECT_ROOT)/docker/%.dockerfile $(PROJECT_RESOURCES) | $(OUT_DIRS)
-	$(SILENT)$(DOCKER) build --rm $(DOCKER_BUILDFLAGS) --iidfile $@ --file $< --tag $*:$(VERSION) $(dir $<)
 
 $(OUT)/%/CMakeCache.txt: $(PROJECT_ROOT)/CMakeLists.txt $(OUT)/docker/% | $(OUT_DIRS)
 	$(SILENT)$(DOCKER) run $(DOCKER_RUNFLAGS) \
@@ -104,22 +140,9 @@ $(OUT)/%/CMakeCache.txt: $(PROJECT_ROOT)/CMakeLists.txt $(OUT)/docker/% | $(OUT_
 	  $*:$(VERSION) \
 	  cmake -GNinja $(CMAKEFLAGS) .. && touch $@
 
-$(CHECK_TARGETS): GLOAS := test
-$(CHECK_TARGETS): DOCKER_RUNUSER := user
-
-check-%: compile-%;
-
-compile-%: $(OUT)/%/CMakeCache.txt
-	$(SILENT)$(DOCKER) run $(DOCKER_RUNFLAGS) \
-	  --volume '$(realpath $(PROJECT_ROOT)):/tmp' \
-	  --volume '$(realpath $(dir $<)):/tmp/$(notdir $(OUT))' \
-	  --workdir '/tmp/$(notdir $(OUT))' \
-	  $*:$(VERSION) \
-	  ninja -j$(NPROC) $(GLOAS)
-
 $(OUT_DIRS):
 	$(SILENT)mkdir -p $@
 
-debug-print-%:
+.PRECIOUS: $(OUT)/docker/% $(OUT)/%/CMakeCache.txt
-	@printf '%s\n' '$*:' $($*)
+.DELETE_ON_ERROR: $(OUT)/%/CMakeCache.txt
 

docker/wsfs-builder-amd64-ubuntu.dockerfile

@@ -1,33 +1,27 @@
 ARG CODENAME=bionic
 
-FROM ubuntu:$CODENAME
+FROM ubuntu:$CODENAME as builder
 
 RUN set -x \
   && apt update \
   && apt upgrade -y \
   && apt install --yes --no-install-recommends \
-       openssl \
-       ca-certificates \
-       curl \
        build-essential \
        cmake \
        ninja-build \
        pkg-config \
   && rm -rf /var/lib/apt/lists/*
 
-ARG NPROC=1
+COPY src /usr/local/src
-ARG GTEST_VERSION=1.8.1
+
+ARG PARALLELMFLAGS=-j2
 
 RUN set -x \
-  && curl -fSL https://github.com/google/googletest/archive/release-$GTEST_VERSION.tar.gz -o /tmp/gtest-$GTEST_VERSION.tar.gz \
+  && mkdir -p /tmp/out \
-  && tar -C /tmp -xf /tmp/gtest-$GTEST_VERSION.tar.gz \
+  && cd /tmp/out \
-  && (   cd /tmp/googletest-release-$GTEST_VERSION \
+  && cmake /usr/local/src/googletest-release-* \
-      && cmake . \
+  && make $PARALLELMFLAGS install \
-      && make -j$NPROC install) \
+  && rm -rf /tmp/out
-  && rm /tmp/gtest-$GTEST_VERSION.tar.gz \
-  && rm -rf /tmp/googletest-release-$GTEST_VERSION
-
-ARG FUSE_VERSION=3.1.1
 
 RUN set -x \
   && apt update \
@@ -35,41 +29,33 @@ RUN set -x \
        libtool \
        automake \
        gettext \
-  && curl -fSL https://github.com/libfuse/libfuse/archive/fuse-$FUSE_VERSION.tar.gz -o /tmp/fuse-$FUSE_VERSION.tar.gz \
+  && cd /usr/local/src/libfuse-fuse-* \
-  && tar -C /tmp -xf /tmp/fuse-$FUSE_VERSION.tar.gz \
+  && ./makeconf.sh \
-  && (   cd /tmp/libfuse-fuse-$FUSE_VERSION \
+  && mkdir -p /tmp/out \
-      && ./makeconf.sh \
+  && cd /tmp/out \
-      && ./configure \
+  && /usr/local/src/libfuse-fuse-*/configure \
-      && make -j$NPROC install) \
+  && make $PARALLELMFLAGS install \
-  && rm /tmp/fuse-$FUSE_VERSION.tar.gz \
+  && rm -rf /tmp/out \
-  && rm -rf /tmp/libfuse-fuse-$FUSE_VERSION \
   && rm -rf /var/lib/apt/lists/*
 
-ARG WEBSOCKETS_VERSION=3.1.0
-
 RUN set -x \
   && apt update \
   && apt install --yes --no-install-recommends \
+       openssl \
        libssl-dev \
-  && curl -fSL https://github.com/warmcat/libwebsockets/archive/v$WEBSOCKETS_VERSION.tar.gz -o /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
+  && mkdir -p /tmp/out \
-  && tar -C /tmp -xf /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
+  && cd /tmp/out \
-  && (   cd /tmp/libwebsockets-$WEBSOCKETS_VERSION \
+  && cmake /usr/local/src/libwebsockets-* \
-      && cmake . \
+  && make $PARALLELMFLAGS install \
-      && make -j$NPROC install) \
+  && rm -rf /tmp/out \
-  && rm /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
-  && rm -rf /tmp/libwebsockets-$WEBSOCKETS_VERSION \
   && rm -rf /var/lib/apt/lists/*
 
-ARG JANSSON_VERSION=2.12
-
 RUN set -x \
-  && curl -fSL https://github.com/akheron/jansson/archive/v$JANSSON_VERSION.tar.gz -o /tmp/libjansson-$JANSSON_VERSION.tar.gz \
+  && mkdir -p /tmp/out \
-  && tar -C /tmp -xf /tmp/libjansson-$JANSSON_VERSION.tar.gz \
+  && cd /tmp/out \
-  && (   cd /tmp/jansson-$JANSSON_VERSION \
+  && cmake -DJANSSON_BUILD_DOCS=OFF /usr/local/src/jansson-* \
-      && cmake -DJANSSON_BUILD_DOCS=OFF . \
+  && make $PARALLELMFLAGS install \
-      && make -j$NPROC install) \
+  && rm -rf /tmp/out
-  && rm /tmp/libjansson-$JANSSON_VERSION.tar.gz \
-  && rm -rf /tmp/jansson-$JANSSON_VERSION
 
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
 

docker/wsfs-builder-arm32v7-ubuntu.dockerfile

@@ -1,35 +1,29 @@
 ARG CODENAME=bionic
 
-FROM arm32v7/ubuntu:$CODENAME
+FROM arm32v7/ubuntu:$CODENAME as builder
 
-COPY qemu-arm-static-* /usr/bin/qemu-arm-static
+COPY docker/qemu-arm-static-* /usr/bin/qemu-arm-static
 
 RUN set -x \
   && apt update \
   && apt upgrade -y \
   && apt install --yes --no-install-recommends \
-       openssl \
-       ca-certificates \
-       curl \
        build-essential \
        cmake \
        ninja-build \
        pkg-config \
   && rm -rf /var/lib/apt/lists/*
 
-ARG NPROC=1
+COPY src /usr/local/src
-ARG GTEST_VERSION=1.8.1
+
+ARG PARALLELMFLAGS=-j2
 
 RUN set -x \
-  && curl -fSL https://github.com/google/googletest/archive/release-$GTEST_VERSION.tar.gz -o /tmp/gtest-$GTEST_VERSION.tar.gz \
+  && mkdir -p /tmp/out \
-  && tar -C /tmp -xf /tmp/gtest-$GTEST_VERSION.tar.gz \
+  && cd /tmp/out \
-  && (   cd /tmp/googletest-release-$GTEST_VERSION \
+  && cmake /usr/local/src/googletest-release-* \
-      && cmake . \
+  && make $PARALLELMFLAGS install \
-      && make -j$NPROC install) \
+  && rm -rf /tmp/out
-  && rm /tmp/gtest-$GTEST_VERSION.tar.gz \
-  && rm -rf /tmp/googletest-release-$GTEST_VERSION
-
-ARG FUSE_VERSION=3.1.1
 
 RUN set -x \
   && apt update \
@@ -37,41 +31,33 @@ RUN set -x \
        libtool \
        automake \
        gettext \
-  && curl -fSL https://github.com/libfuse/libfuse/archive/fuse-$FUSE_VERSION.tar.gz -o /tmp/fuse-$FUSE_VERSION.tar.gz \
+  && cd /usr/local/src/libfuse-fuse-* \
-  && tar -C /tmp -xf /tmp/fuse-$FUSE_VERSION.tar.gz \
+  && ./makeconf.sh \
-  && (   cd /tmp/libfuse-fuse-$FUSE_VERSION \
+  && mkdir -p /tmp/out \
-      && ./makeconf.sh \
+  && cd /tmp/out \
-      && ./configure \
+  && /usr/local/src/libfuse-fuse-*/configure \
-      && make -j$NPROC install) \
+  && make $PARALLELMFLAGS install \
-  && rm /tmp/fuse-$FUSE_VERSION.tar.gz \
+  && rm -rf /tmp/out \
-  && rm -rf /tmp/libfuse-fuse-$FUSE_VERSION \
   && rm -rf /var/lib/apt/lists/*
 
-ARG WEBSOCKETS_VERSION=3.1.0
-
 RUN set -x \
   && apt update \
   && apt install --yes --no-install-recommends \
+       openssl \
        libssl-dev \
-  && curl -fSL https://github.com/warmcat/libwebsockets/archive/v$WEBSOCKETS_VERSION.tar.gz -o /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
+  && mkdir -p /tmp/out \
-  && tar -C /tmp -xf /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
+  && cd /tmp/out \
-  && (   cd /tmp/libwebsockets-$WEBSOCKETS_VERSION \
+  && cmake /usr/local/src/libwebsockets-* \
-      && cmake . \
+  && make $PARALLELMFLAGS install \
-      && make -j$NPROC install) \
+  && rm -rf /tmp/out \
-  && rm /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
-  && rm -rf /tmp/libwebsockets-$WEBSOCKETS_VERSION \
   && rm -rf /var/lib/apt/lists/*
 
-ARG JANSSON_VERSION=2.12
-
 RUN set -x \
-  && curl -fSL https://github.com/akheron/jansson/archive/v$JANSSON_VERSION.tar.gz -o /tmp/libjansson-$JANSSON_VERSION.tar.gz \
+  && mkdir -p /tmp/out \
-  && tar -C /tmp -xf /tmp/libjansson-$JANSSON_VERSION.tar.gz \
+  && cd /tmp/out \
-  && (   cd /tmp/jansson-$JANSSON_VERSION \
+  && cmake -DJANSSON_BUILD_DOCS=OFF /usr/local/src/jansson-* \
-      && cmake -DJANSSON_BUILD_DOCS=OFF . \
+  && make $PARALLELMFLAGS install \
-      && make -j$NPROC install) \
+  && rm -rf /tmp/out
-  && rm /tmp/libjansson-$JANSSON_VERSION.tar.gz \
-  && rm -rf /tmp/jansson-$JANSSON_VERSION
 
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
 
@@ -80,4 +66,3 @@ ARG USERID=1000
 RUN useradd -u $USERID -ms /bin/bash user
 
 
-