moves download of artifacts to bootstrapping makefile

5 years ago · 740b247250
parent 38a2240b65
commit 740b247250
5 changed files with 120 additions and 127 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,4 +1,3 @@
 .vscode
 .build
 .deps
-docker/qemu-*
--- a/.travis.yml
+++ b/.travis.yml
@ -10,7 +10,7 @@ before_install:
 - sudo apt-get install -y qemu-user-static
 env:
  global:
-    - MAKEFLAGS="-j 2"
+    - PARALLELMFLAGS="-j2"
  matrix:
    - MARCH=amd64
    - MARCH=arm32v7
--- a/105
+++ b/105
@ -1,27 +1,32 @@
-MAKEFLAGS += --no-builtin-rules
-
 .PHONY: default
 default: all

-## Overridable defaults
+# Overridable defaults

 $(VERBOSE)SILENT ?= @
 PROJECT_ROOT ?= .
 OUT ?= $(PROJECT_ROOT)/.build
 VERSION ?= $(shell cat $(PROJECT_ROOT)/VERSION)
-NPROC ?= $(shell nproc)
+PARALLELMFLAGS ?= -j$(shell nproc)
 UID ?= $(shell id -u)
 DOCKER ?= docker
 DOCKER_RUNUSER ?= $(UID)
 DOCKER_RUNGROUP ?= $(shell id -g)

 UBUNTU_CODENAME ?= bionic
+
 QEMU_VERSION ?= v3.1.0-2
+GTEST_VERSION ?= 1.8.1
+FUSE_VERSION ?= 3.1.1
+WEBSOCKETS_VERSION ?= 3.1.0
+JANSSON_VERSION ?= 2.12
+
+# Target configuration

 MARCH_AMD64 := $(filter-out amd64,$(MARCH))
 MARCH_ARM32V7 := $(filter-out arm32v7,$(MARCH))

-## Defaults
+MAKEFLAGS += $(PARALLELMFLAGS) --no-builtin-rules

 $(MARCH_AMD64)TARGETS += wsfs-builder-amd64-ubuntu
 $(MARCH_ARM32V7)TARGETS += wsfs-builder-arm32v7-ubuntu
@ -35,41 +40,42 @@ DOCKER_RUNFLAGS += --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmo
 DOCKER_RUNFLAGS += --env SOURCE_DATE_EPOCH

 DOCKER_BUILDARGS += CODENAME=$(CODENAME)
-DOCKER_BUILDARGS += NPROC=$(NPROC)
+DOCKER_BUILDARGS += PARALLELMFLAGS=$(PARALLELMFLAGS)
 DOCKER_BUILDARGS += USERID=$(UID)
+DOCKER_BUILDARGS += SRC=$(realpath $(PROJECT_ROOT))
+DOCKER_BUILDARGS += OUT=$(realpath $(OUT))

 DOCKER_BUILDFLAGS += $(addprefix --build-arg ,$(DOCKER_BUILDARGS))

 OUT_DIRS += $(OUT)
 OUT_DIRS += $(OUT)/docker
+OUT_DIRS += $(OUT)/src
 OUT_DIRS += $(addprefix $(OUT)/,$(TARGETS))

-PROJECT_RESOURCES = $(call glob_files,$(PROJECT_ROOT)/**)
-
 ALL_TARGETS += $(addprefix compile-,$(TARGETS)) 
 CHECK_TARGETS += $(addprefix check-,$(TARGETS))
 CONFIGURE_TARGETS += $(addsuffix /CMakeCache.txt,$(addprefix $(OUT)/,$(TARGETS)))

-## Helper macros
-
-# $(call glob,include_pattern...,exclude_regex...,flags...)
-glob = $(shell bash -c 'shopt -s globstar nullglob && find $1 -maxdepth 0 $3 | grep -v $(addprefix -e, $2)')
-# $(call glob_files,include_pattern...,exclude_regex...)
-glob_files = $(call glob,$1,^$(OUT) $2,-type f)
+FETCH_TARGETS += $(OUT)/docker/qemu-arm-static-$(QEMU_VERSION)
+FETCH_TARGETS += $(OUT)/googletest-release-$(GTEST_VERSION).tar.gz
+FETCH_TARGETS += $(OUT)/libfuse-fuse-$(FUSE_VERSION).tar.gz
+FETCH_TARGETS += $(OUT)/libwebsockets-$(WEBSOCKETS_VERSION).tar.gz
+FETCH_TARGETS += $(OUT)/jansson-$(JANSSON_VERSION).tar.gz

-## Force eager evaluation
+EXTRACT_TARGETS += $(patsubst $(OUT)/%.tar.gz,$(OUT)/src/%,$(FETCH_TARGETS))

 DOCKER_RUNGROUP := $(DOCKER_RUNGROUP)
 VERSION := $(VERSION)
 PROJECT_ROOT := $(PROJECT_ROOT)
 OUT := $(OUT)

-## Special targets
+# Rules

-.PRECIOUS: $(OUT)/docker/%
-.DELETE_ON_ERROR: $(OUT)/%/CMakeCache.txt $(OUT)/docker/%
-
-## Targets
+$(OUT)/docker/qemu-arm-static-$(QEMU_VERSION): URL := https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-arm-static
+$(OUT)/googletest-release-$(GTEST_VERSION).tar.gz: URL := https://github.com/google/googletest/archive/release-$(GTEST_VERSION).tar.gz
+$(OUT)/libfuse-fuse-$(FUSE_VERSION).tar.gz: URL := https://github.com/libfuse/libfuse/archive/fuse-$(FUSE_VERSION).tar.gz
+$(OUT)/libwebsockets-$(WEBSOCKETS_VERSION).tar.gz: URL := https://github.com/warmcat/libwebsockets/archive/v$(WEBSOCKETS_VERSION).tar.gz
+$(OUT)/jansson-$(JANSSON_VERSION).tar.gz: URL := https://github.com/akheron/jansson/archive/v$(JANSSON_VERSION).tar.gz

 .PHONY: all
 all: $(ALL_TARGETS)
@ -81,45 +87,62 @@ check: $(CHECK_TARGETS)
 clean: $(CLEAN_TARGETS)
 	$(SILENT)-rm -rf $(OUT_DIRS)

-.PHONY: configure
-configure: $(CONFIGURE_TARGETS) 
-
-%-ubuntu: CODENAME := $(UBUNTU_CODENAME)
+.PHONY: get-deps
+get-deps: $(EXTRACT_TARGETS)

-$(PROJECT_ROOT)/docker/qemu-arm-static-$(QEMU_VERSION):
-	$(SILENT) \
-	     curl -fSL -o $@ https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/qemu-arm-static \
-	  && chmod +x $@ 
-
-$(OUT)/docker/wsfs-builder-arm32v7-ubuntu: $(PROJECT_ROOT)/docker/qemu-arm-static-$(QEMU_VERSION)
+.PHONY: debug-print-%
+debug-print-%:
+	@printf '%s\n' '$*:' $($*)

-$(OUT)/docker/%: $(PROJECT_ROOT)/docker/%.dockerfile $(PROJECT_RESOURCES) | $(OUT_DIRS)
-	$(SILENT)$(DOCKER) build --rm $(DOCKER_BUILDFLAGS) --iidfile $@ --file $< --tag $*:$(VERSION) $(dir $<)
+.PHONY: check-%
+check-%: compile-%;

-$(OUT)/%/CMakeCache.txt: $(PROJECT_ROOT)/CMakeLists.txt $(OUT)/docker/% | $(OUT_DIRS)
+.PHONY: compile-%
+compile-%: $(OUT)/%/CMakeCache.txt
 	$(SILENT)$(DOCKER) run $(DOCKER_RUNFLAGS) \
 	  --volume '$(realpath $(PROJECT_ROOT)):/tmp' \
-	  --volume '$(realpath $(dir $@)):/tmp/$(notdir $(OUT))' \
+	  --volume '$(realpath $(dir $<)):/tmp/$(notdir $(OUT))' \
 	  --workdir '/tmp/$(notdir $(OUT))' \
 	  $*:$(VERSION) \
-	  cmake -GNinja $(CMAKEFLAGS) .. && touch $@
+	  ninja $(PARALLELMFLAGS) $(GLOAS)

 $(CHECK_TARGETS): GLOAS := test
 $(CHECK_TARGETS): DOCKER_RUNUSER := user

-check-%: compile-%;
+$(OUT)/docker/qemu-arm-static-$(QEMU_VERSION):
+	$(SILENT) \
+	     curl -fsSL -o $@ $(URL) \
+	  && chmod +x $@ 

-compile-%: $(OUT)/%/CMakeCache.txt
+$(OUT)/docker/wsfs-builder-arm32v7-ubuntu: $(OUT)/docker/qemu-arm-static-$(QEMU_VERSION)
+
+$(OUT)/% : $(PROJECT_ROOT)/% | $(OUT_DIRS)
+	cp $< $@
+
+%-ubuntu: CODENAME := $(UBUNTU_CODENAME)
+
+$(OUT)/docker/%: $(OUT)/docker/%.dockerfile $(EXTRACT_TARGETS) $(PROJECT_ROOT)/Makefile | $(OUT_DIRS)
+	$(SILENT)$(DOCKER) build --rm $(DOCKER_BUILDFLAGS) --iidfile $@ --file $< --tag $*:$(VERSION) $(OUT)
+
+$(OUT)/%.tar.gz: | $(OUT_DIRS)
+	curl -fsSL -o $@ $(URL)
+
+$(OUT)/src/%: $(OUT)/%.tar.gz | $(OUT_DIRS)
+	$(SILENT) \
+	     tar -C $(dir $@) -xf $< \
+	  && touch $@
+
+$(OUT)/%/CMakeCache.txt: $(PROJECT_ROOT)/CMakeLists.txt $(OUT)/docker/% | $(OUT_DIRS)
 	$(SILENT)$(DOCKER) run $(DOCKER_RUNFLAGS) \
 	  --volume '$(realpath $(PROJECT_ROOT)):/tmp' \
-	  --volume '$(realpath $(dir $<)):/tmp/$(notdir $(OUT))' \
+	  --volume '$(realpath $(dir $@)):/tmp/$(notdir $(OUT))' \
 	  --workdir '/tmp/$(notdir $(OUT))' \
 	  $*:$(VERSION) \
-	  ninja -j$(NPROC) $(GLOAS)
+	  cmake -GNinja $(CMAKEFLAGS) .. && touch $@

 $(OUT_DIRS):
 	$(SILENT)mkdir -p $@

-debug-print-%:
-	@printf '%s\n' '$*:' $($*)
+.PRECIOUS: $(OUT)/docker/% $(OUT)/%/CMakeCache.txt
+.DELETE_ON_ERROR: $(OUT)/%/CMakeCache.txt

--- a/docker/wsfs-builder-amd64-ubuntu.dockerfile
+++ b/docker/wsfs-builder-amd64-ubuntu.dockerfile
@ -1,33 +1,27 @@
 ARG CODENAME=bionic

-FROM ubuntu:$CODENAME
+FROM ubuntu:$CODENAME as builder

 RUN set -x \
  && apt update \
  && apt upgrade -y \
  && apt install --yes --no-install-recommends \
-       openssl \
-       ca-certificates \
-       curl \
       build-essential \
       cmake \
       ninja-build \
       pkg-config \
  && rm -rf /var/lib/apt/lists/*

-ARG NPROC=1
-ARG GTEST_VERSION=1.8.1
+COPY src /usr/local/src

-RUN set -x \
-  && curl -fSL https://github.com/google/googletest/archive/release-$GTEST_VERSION.tar.gz -o /tmp/gtest-$GTEST_VERSION.tar.gz \
-  && tar -C /tmp -xf /tmp/gtest-$GTEST_VERSION.tar.gz \
-  && (   cd /tmp/googletest-release-$GTEST_VERSION \
-      && cmake . \
-      && make -j$NPROC install) \
-  && rm /tmp/gtest-$GTEST_VERSION.tar.gz \
-  && rm -rf /tmp/googletest-release-$GTEST_VERSION
+ARG PARALLELMFLAGS=-j2

-ARG FUSE_VERSION=3.1.1
+RUN set -x \
+  && mkdir -p /tmp/out \
+  && cd /tmp/out \
+  && cmake /usr/local/src/googletest-release-* \
+  && make $PARALLELMFLAGS install \
+  && rm -rf /tmp/out

 RUN set -x \
  && apt update \
@ -35,41 +29,33 @@ RUN set -x \
       libtool \
       automake \
       gettext \
-  && curl -fSL https://github.com/libfuse/libfuse/archive/fuse-$FUSE_VERSION.tar.gz -o /tmp/fuse-$FUSE_VERSION.tar.gz \
-  && tar -C /tmp -xf /tmp/fuse-$FUSE_VERSION.tar.gz \
-  && (   cd /tmp/libfuse-fuse-$FUSE_VERSION \
-      && ./makeconf.sh \
-      && ./configure \
-      && make -j$NPROC install) \
-  && rm /tmp/fuse-$FUSE_VERSION.tar.gz \
-  && rm -rf /tmp/libfuse-fuse-$FUSE_VERSION \
+  && cd /usr/local/src/libfuse-fuse-* \
+  && ./makeconf.sh \
+  && mkdir -p /tmp/out \
+  && cd /tmp/out \
+  && /usr/local/src/libfuse-fuse-*/configure \
+  && make $PARALLELMFLAGS install \
+  && rm -rf /tmp/out \
  && rm -rf /var/lib/apt/lists/*

-ARG WEBSOCKETS_VERSION=3.1.0
-
 RUN set -x \
  && apt update \
  && apt install --yes --no-install-recommends \
+       openssl \
       libssl-dev \
-  && curl -fSL https://github.com/warmcat/libwebsockets/archive/v$WEBSOCKETS_VERSION.tar.gz -o /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
-  && tar -C /tmp -xf /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
-  && (   cd /tmp/libwebsockets-$WEBSOCKETS_VERSION \
-      && cmake . \
-      && make -j$NPROC install) \
-  && rm /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
-  && rm -rf /tmp/libwebsockets-$WEBSOCKETS_VERSION \
+  && mkdir -p /tmp/out \
+  && cd /tmp/out \
+  && cmake /usr/local/src/libwebsockets-* \
+  && make $PARALLELMFLAGS install \
+  && rm -rf /tmp/out \
  && rm -rf /var/lib/apt/lists/*

-ARG JANSSON_VERSION=2.12
-
 RUN set -x \
-  && curl -fSL https://github.com/akheron/jansson/archive/v$JANSSON_VERSION.tar.gz -o /tmp/libjansson-$JANSSON_VERSION.tar.gz \
-  && tar -C /tmp -xf /tmp/libjansson-$JANSSON_VERSION.tar.gz \
-  && (   cd /tmp/jansson-$JANSSON_VERSION \
-      && cmake -DJANSSON_BUILD_DOCS=OFF . \
-      && make -j$NPROC install) \
-  && rm /tmp/libjansson-$JANSSON_VERSION.tar.gz \
-  && rm -rf /tmp/jansson-$JANSSON_VERSION
+  && mkdir -p /tmp/out \
+  && cd /tmp/out \
+  && cmake -DJANSSON_BUILD_DOCS=OFF /usr/local/src/jansson-* \
+  && make $PARALLELMFLAGS install \
+  && rm -rf /tmp/out

 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib

--- a/docker/wsfs-builder-arm32v7-ubuntu.dockerfile
+++ b/docker/wsfs-builder-arm32v7-ubuntu.dockerfile
@ -1,35 +1,29 @@
 ARG CODENAME=bionic

-FROM arm32v7/ubuntu:$CODENAME
+FROM arm32v7/ubuntu:$CODENAME as builder

-COPY qemu-arm-static-* /usr/bin/qemu-arm-static
+COPY docker/qemu-arm-static-* /usr/bin/qemu-arm-static

 RUN set -x \
  && apt update \
  && apt upgrade -y \
  && apt install --yes --no-install-recommends \
-       openssl \
-       ca-certificates \
-       curl \
       build-essential \
       cmake \
       ninja-build \
       pkg-config \
  && rm -rf /var/lib/apt/lists/*

-ARG NPROC=1
-ARG GTEST_VERSION=1.8.1
+COPY src /usr/local/src

-RUN set -x \
-  && curl -fSL https://github.com/google/googletest/archive/release-$GTEST_VERSION.tar.gz -o /tmp/gtest-$GTEST_VERSION.tar.gz \
-  && tar -C /tmp -xf /tmp/gtest-$GTEST_VERSION.tar.gz \
-  && (   cd /tmp/googletest-release-$GTEST_VERSION \
-      && cmake . \
-      && make -j$NPROC install) \
-  && rm /tmp/gtest-$GTEST_VERSION.tar.gz \
-  && rm -rf /tmp/googletest-release-$GTEST_VERSION
+ARG PARALLELMFLAGS=-j2

-ARG FUSE_VERSION=3.1.1
+RUN set -x \
+  && mkdir -p /tmp/out \
+  && cd /tmp/out \
+  && cmake /usr/local/src/googletest-release-* \
+  && make $PARALLELMFLAGS install \
+  && rm -rf /tmp/out

 RUN set -x \
  && apt update \
@ -37,41 +31,33 @@ RUN set -x \
       libtool \
       automake \
       gettext \
-  && curl -fSL https://github.com/libfuse/libfuse/archive/fuse-$FUSE_VERSION.tar.gz -o /tmp/fuse-$FUSE_VERSION.tar.gz \
-  && tar -C /tmp -xf /tmp/fuse-$FUSE_VERSION.tar.gz \
-  && (   cd /tmp/libfuse-fuse-$FUSE_VERSION \
-      && ./makeconf.sh \
-      && ./configure \
-      && make -j$NPROC install) \
-  && rm /tmp/fuse-$FUSE_VERSION.tar.gz \
-  && rm -rf /tmp/libfuse-fuse-$FUSE_VERSION \
+  && cd /usr/local/src/libfuse-fuse-* \
+  && ./makeconf.sh \
+  && mkdir -p /tmp/out \
+  && cd /tmp/out \
+  && /usr/local/src/libfuse-fuse-*/configure \
+  && make $PARALLELMFLAGS install \
+  && rm -rf /tmp/out \
  && rm -rf /var/lib/apt/lists/*

-ARG WEBSOCKETS_VERSION=3.1.0
-
 RUN set -x \
  && apt update \
  && apt install --yes --no-install-recommends \
+       openssl \
       libssl-dev \
-  && curl -fSL https://github.com/warmcat/libwebsockets/archive/v$WEBSOCKETS_VERSION.tar.gz -o /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
-  && tar -C /tmp -xf /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
-  && (   cd /tmp/libwebsockets-$WEBSOCKETS_VERSION \
-      && cmake . \
-      && make -j$NPROC install) \
-  && rm /tmp/libwebsockets-$WEBSOCKETS_VERSION.tar.gz \
-  && rm -rf /tmp/libwebsockets-$WEBSOCKETS_VERSION \
+  && mkdir -p /tmp/out \
+  && cd /tmp/out \
+  && cmake /usr/local/src/libwebsockets-* \
+  && make $PARALLELMFLAGS install \
+  && rm -rf /tmp/out \
  && rm -rf /var/lib/apt/lists/*

-ARG JANSSON_VERSION=2.12
-
 RUN set -x \
-  && curl -fSL https://github.com/akheron/jansson/archive/v$JANSSON_VERSION.tar.gz -o /tmp/libjansson-$JANSSON_VERSION.tar.gz \
-  && tar -C /tmp -xf /tmp/libjansson-$JANSSON_VERSION.tar.gz \
-  && (   cd /tmp/jansson-$JANSSON_VERSION \
-      && cmake -DJANSSON_BUILD_DOCS=OFF . \
-      && make -j$NPROC install) \
-  && rm /tmp/libjansson-$JANSSON_VERSION.tar.gz \
-  && rm -rf /tmp/jansson-$JANSSON_VERSION
+  && mkdir -p /tmp/out \
+  && cd /tmp/out \
+  && cmake -DJANSSON_BUILD_DOCS=OFF /usr/local/src/jansson-* \
+  && make $PARALLELMFLAGS install \
+  && rm -rf /tmp/out

 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib

@ -80,4 +66,3 @@ ARG USERID=1000
 RUN useradd -u $USERID -ms /bin/bash user


-