log.h: null-terminate compile-time filename strings

Fix all hrefs and vue routes to correctly operate against the
<base href> tag. Add a configuration parameter to override the
content of the href attribute, and describe its use.

CMakeLists.txt

@@ -1,5 +1,5 @@
 ###
-### Copyright 2015-2017 Oliver Giles
+### Copyright 2015-2019 Oliver Giles
 ###
 ### This file is part of Laminar
 ###
@@ -22,7 +22,7 @@ cmake_policy(SET CMP0058 NEW)
 
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 
-add_definitions("-std=c++11 -Wall -Wextra -Wno-unused-parameter -Wno-sign-compare")
+add_definitions("-std=c++14 -Wall -Wextra -Wno-unused-parameter -Wno-sign-compare")
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Werror -DDEBUG")
 
 # This macro takes a list of files, gzips them and converts the output into
@@ -43,7 +43,11 @@ macro(generate_compressed_bins BASEDIR)
             DEPENDS ${BASEDIR}/${FILE}
         )
         add_custom_command(OUTPUT ${OUTPUT_FILE}
-            COMMAND ld -r -b binary -o ${OUTPUT_FILE} ${COMPRESSED_FILE}
+            COMMAND ${CMAKE_LINKER} -r -b binary -o ${OUTPUT_FILE} ${COMPRESSED_FILE}
+            COMMAND ${CMAKE_OBJCOPY}
+              --rename-section .data=.rodata.alloc,load,readonly,data,contents
+              --add-section .note.GNU-stack=/dev/null
+              --set-section-flags .note.GNU-stack=contents,readonly ${OUTPUT_FILE}
             DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${COMPRESSED_FILE}
         )
         list(APPEND COMPRESSED_BINS ${OUTPUT_FILE})
@@ -60,6 +64,11 @@ add_custom_command(OUTPUT laminar.capnp.c++ laminar.capnp.h
 generate_compressed_bins(${CMAKE_SOURCE_DIR}/src/resources index.html js/app.js
     favicon.ico favicon-152.png icon.png)
 
+# The code that allows dynamic modifying of index.html requires knowing its original size
+add_custom_command(OUTPUT index_html_size.h
+    COMMAND sh -c '( echo -n "\\#define INDEX_HTML_UNCOMPRESSED_SIZE " && wc -c < "${CMAKE_SOURCE_DIR}/src/resources/index.html" ) > index_html_size.h'
+    DEPENDS src/resources/index.html)
+
 # Download 3rd-party frontend JS libs...
 file(DOWNLOAD https://cdnjs.cloudflare.com/ajax/libs/vue/2.3.4/vue.min.js
         js/vue.min.js EXPECTED_MD5 ae2fca1cfa0e31377819b1b0ffef704c)
@@ -67,22 +76,19 @@ file(DOWNLOAD https://cdnjs.cloudflare.com/ajax/libs/vue-router/2.7.0/vue-router
         js/vue-router.min.js EXPECTED_MD5 5d3e35710dbe02de78c39e3e439b8d4e)
 file(DOWNLOAD https://raw.githubusercontent.com/drudru/ansi_up/v1.3.0/ansi_up.js
         js/ansi_up.js EXPECTED_MD5 158566dc1ff8f2804de972f7e841e2f6)
-file(DOWNLOAD https://cdnjs.cloudflare.com/ajax/libs/Chart.js/1.0.2/Chart.min.js
-        js/Chart.min.js EXPECTED_MD5 0d3004601c1a855a3d203502549528a7)
-file(DOWNLOAD https://raw.githubusercontent.com/tomsouthall/Chart.HorizontalBar.js/v1.04/Chart.HorizontalBar.js
-        js/Chart.HorizontalBar.js EXPECTED_MD5 95070a38e69bc56534e1b2086d985270)
+file(DOWNLOAD https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.7.2/Chart.min.js
+        js/Chart.min.js EXPECTED_MD5 f6c8efa65711e0cbbc99ba72997ecd0e)
 file(DOWNLOAD https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css
         css/bootstrap.min.css EXPECTED_MD5 5d5357cb3704e1f43a1f5bfed2aebf42)
 # ...and compile them
 generate_compressed_bins(${CMAKE_BINARY_DIR} js/vue-router.min.js js/vue.min.js
-    js/ansi_up.js js/Chart.min.js js/Chart.HorizontalBar.js css/bootstrap.min.css)
+    js/ansi_up.js js/Chart.min.js css/bootstrap.min.css)
 # (see resources.cpp where these are fetched)
 
 ## Server
 add_executable(laminard src/database.cpp src/main.cpp src/server.cpp src/laminar.cpp
-    src/conf.cpp src/resources.cpp src/run.cpp laminar.capnp.c++ ${COMPRESSED_BINS})
-# TODO: some alternative to boost::filesystem?
-target_link_libraries(laminard capnp-rpc capnp kj-async kj pthread boost_filesystem boost_system sqlite3 z)
+    src/conf.cpp src/resources.cpp src/run.cpp laminar.capnp.c++ ${COMPRESSED_BINS} index_html_size.h)
+target_link_libraries(laminard capnp-rpc capnp kj-http kj-async kj pthread sqlite3 z)
 
 ## Client
 add_executable(laminarc src/client.cpp laminar.capnp.c++)
@@ -94,10 +100,14 @@ if(BUILD_TESTS)
     find_package(GTest REQUIRED)
     include_directories(${GTEST_INCLUDE_DIRS} src)
     add_executable(laminar-tests src/conf.cpp src/database.cpp src/laminar.cpp src/run.cpp src/server.cpp laminar.capnp.c++ src/resources.cpp ${COMPRESSED_BINS} test/test-conf.cpp test/test-database.cpp test/test-laminar.cpp test/test-run.cpp test/test-server.cpp)
-    target_link_libraries(laminar-tests ${GTEST_BOTH_LIBRARIES} gmock capnp-rpc capnp kj-async kj pthread boost_filesystem boost_system sqlite3 z)
+    target_link_libraries(laminar-tests ${GTEST_BOTH_LIBRARIES} gmock capnp-rpc capnp kj-http kj-async kj pthread sqlite3 z)
 endif()
 
-set(SYSTEMD_UNITDIR /lib/systemd/system CACHE PATH "Path to systemd unit files")
+set(SYSTEMD_UNITDIR lib/systemd/system CACHE PATH "Path to systemd unit files")
+set(BASH_COMPLETIONS_DIR usr/share/bash-completion/completions CACHE PATH "Path to bash completions directory")
+set(ZSH_COMPLETIONS_DIR usr/share/zsh/site-functions CACHE PATH "Path to zsh completions directory")
 install(TARGETS laminard laminarc RUNTIME DESTINATION usr/bin)
-install(FILES laminar.service DESTINATION ${SYSTEMD_UNITDIR})
-install(FILES laminar.conf DESTINATION etc)
+install(FILES etc/laminar.service DESTINATION ${SYSTEMD_UNITDIR})
+install(FILES etc/laminar.conf DESTINATION etc)
+install(FILES etc/laminarc-completion.bash DESTINATION ${BASH_COMPLETIONS_DIR} RENAME laminarc)
+install(FILES etc/laminarc-completion.zsh DESTINATION ${ZSH_COMPLETIONS_DIR} RENAME _laminarc)

README.md

@@ -1,16 +1,16 @@
-# Laminar CI
+# Laminar CI [![status](https://ci.ohwg.net/badge/laminar.svg)](https://ci.ohwg.net/jobs/laminar)
 
-Laminar (http://laminar.ohwg.net) is a lightweight and modular Continuous Integration service for Linux. It is self-hosted and developer-friendly, eschewing a configuration UI in favor of simple version-controllable configuration files and scripts.
+Laminar (https://laminar.ohwg.net) is a lightweight and modular Continuous Integration service for Linux. It is self-hosted and developer-friendly, eschewing a configuration UI in favour of simple version-controllable configuration files and scripts.
 
 Laminar encourages the use of existing GNU/Linux tools such as `bash` and `cron` instead of reinventing them.
 
 Although the status and progress front-end is very user-friendly, administering a Laminar instance requires writing shell scripts and manually editing configuration files. That being said, there is nothing esoteric here and the [guide](http://laminar.ohwg.net/docs.html) should be straightforward for anyone with even very basic Linux server administration experience.
 
-See [the website](http://laminar.ohwg.net) and the [documentation](http://laminar.ohwg.net/docs.html) for more information.
+See [the website](https://laminar.ohwg.net) and the [documentation](https://laminar.ohwg.net/docs.html) for more information.
 
 ## Building from source
 
-First install development packages for `capnproto (git)`, `rapidjson`, `websocketpp`, `sqlite` and `boost-filesystem` from your distribution's repository or other source. Then:
+First install development packages for `capnproto (version 0.7.0 or newer)`, `rapidjson`, `sqlite` and `boost` (for the header-only `multi_index_container` library) from your distribution's repository or other source. Then:
 
 ```bash
 git clone https://github.com/ohwgiles/laminar.git

UserManual.md

@@ -69,6 +69,8 @@ For Apache, see [Apache Reverse Proxy](https://httpd.apache.org/docs/2.4/howto/r
 
 If you use [artefacts](#Archiving-artefacts), note that Laminar is not designed as a file server, and better performance will be achieved by allowing the frontend web server to directly serve the archive directory directly (e.g. using a `Location` directive).
 
+If you use a reverse proxy to host Laminar at a subfolder instead of a subdomain root, the `<base href>` needs to be updated to ensure all links point to their proper targets. This can be done by setting `LAMINAR_BASE_URL` in `/etc/laminar.conf`.
+
 ## Set the page title
 
 Change `LAMINAR_TITLE` in `/etc/laminar.conf` to your preferred page title. For further WebUI customization, consider using a [custom style sheet](#Customizing-the-WebUI).
@@ -97,7 +99,7 @@ Laminar uses your script's exit code to determine whether to mark the run as suc
 Don't forget to mark the script executable:
 
 ```bash
-chmod +x /var/lib/laminar/cfg/hello.run
+chmod +x /var/lib/laminar/cfg/jobs/hello.run
 ```
 
 ---
@@ -128,7 +130,7 @@ To launch and run the `hello` job to completion, execute
 laminarc run hello
 ```
 
-In all cases, a started run means the `/var/lib/laminar/cfg/hello.run` script will be executed, with a working directory of `/var/lib/laminar/run/hello/1` (or current run number)
+In all cases, a started run means the `/var/lib/laminar/cfg/jobs/hello.run` script will be executed, with a working directory of `/var/lib/laminar/run/hello/1` (or current run number)
 
 The result and log output should be visible in the Web UI at http://localhost:8080/jobs/hello/1
 
@@ -142,6 +144,14 @@ laminarc queue test-host test-target
 
 This is against the design principles of Laminar and was deliberately excluded. Laminar's web UI is strictly read-only, making it simple to deploy in mixed-permission or public environments without an authentication layer. Furthermore, Laminar tries to encourage ideal continuous integration, where manual triggering is an anti-pattern. Want to make a release? Push a git tag and implement a post-receive hook. Want to re-run a build due to sporadic failure/flaky tests? Fix the tests locally and push a patch. Experience shows that a manual trigger such as a "Build Now" button is often used as a crutch to avoid doing the correct thing, negatively impacting traceability and quality.
 
+## Listing jobs from the command line
+
+`laminarc` may be used to inspect the server state:
+
+- `laminarc show-jobs`: Lists all files matching `/var/lib/laminar/cfg/jobs/*.run` on the server side.
+- `laminarc show-running`: Lists all currently running jobs and their run numbers.
+- `laminarc show-queued`: Lists all jobs waiting in the queue.
+
 ## Triggering a job at a certain time
 
 This is what `cron` is for. To trigger a build of `hello` every day at 0300, add
@@ -197,6 +207,12 @@ This can be securely and flexibly combined with remote triggering using `ssh`. T
 
 Consider using [webhook](https://github.com/adnanh/webhook) or a similar application to call `laminarc`.
 
+## Viewing job logs
+
+A job's console output can be viewed on the Web UI at http://localhost:8080/jobs/$NAME/$NUMBER.
+
+Additionally, the raw log output may be fetched over a plain HTTP request to http://localhost:8080/log/$NAME/$NUMBER. The response will be chunked, allowing this mechanism to also be used for in-progress jobs. Furthermore, the special endpoint http://localhost:8080/log/$NAME/latest will redirect to the most recent log output. Be aware that the use of this endpoint may be subject to races when new jobs start.
+
 ---
 
 # Job chains
@@ -390,13 +406,52 @@ cmake $WORKSPACE/myproject
 make -j4
 ```
 
-**CAUTION**: By default, laminar permits multiple simultaneous runs of the same job. If a job can **modify** the workspace, this might result in inconsistent builds when the job has multiple simultaneous runs. This is unlikely to be an issue for nightly builds, but for SCM-triggered builds it will be. To solve this, use [nodes](#Nodes-and-Tags) to restrict simultaneous execution of jobs, or [locks](#Locks) to temporarily take exclusive control of a resource.
+Laminar will automatically create the workspace for a job if it doesn't exist when a job is executed. In this case, the `/var/lib/laminar/cfg/jobs/JOBNAME.init` will be executed if it exists. This is an excellent place to prepare the workspace to a state where subsequent builds can rely on its content:
 
-Laminar will automatically create the workspace for a job if it doesn't exist when a job is executed. In this case, the `/var/lib/laminar/cfg/jobs/JOBNAME.init` will be executed if it exists. This is an excellent place to prepare the workspace to a state where subsequent builds can rely on its content.
+```bash
+#!/bin/bash -e
+echo Initializing workspace
+git clone git@example.com:company/project.git .
+```
+
+**CAUTION**: By default, laminar permits multiple simultaneous runs of the same job. If a job can **modify** the workspace, this might result in inconsistent builds when simultaneous runs access the same content. This is unlikely to be an issue for nightly builds, but for SCM-triggered builds it will be. To solve this, use [nodes](#Nodes-and-Tags) to restrict simultaneous execution of jobs, or consider [flock](https://linux.die.net/man/1/flock).
+
+The following example uses [flock](https://linux.die.net/man/1/flock) to efficiently share a git repository workspace between multiple simultaneous builds:
+
+```bash
+#!/bin/bash -xe
+
+# This script expects to be passed the parameter 'rev' which
+# should refer to a specific git commit in its source repository.
+# The commit ids could have been read from a server-side
+# post-commit git hook, where many commits could have been pushed
+# at once, but we want to check them all individually. This means
+# this job can be executed several times (with different values
+# for $rev) simultaneously.
+
+# Locked subshell for modifying the workspace
+(
+  flock 200
+  cd $WORKSPACE
+  # Download all the latest commits
+  git fetch
+  git checkout $rev
+  cd -
+  # Fast copy (hard-link) the source from the specific checkout
+  # to the build dir. This relies on the fact that git unlinks
+  # during checkout, effectively implementing copy-on-write.
+  cp -al $WORKSPACE/src src
+) 200>$WORKSPACE
+
+# run the (much longer) regular build process
+make -C src
+```
 
 ---
 
-# Abort on timeout
+# Aborting running jobs
+
+## After a timeout
 
 To configure a maximum execution time in seconds for a job, add a line to `/var/lib/laminar/cfg/jobs/JOBNAME.conf`:
 
@@ -404,6 +459,10 @@ To configure a maximum execution time in seconds for a job, add a line to `/var/
 TIMEOUT=120
 ```
 
+## Manually
+
+`laminarc abort $JOBNAME $NUMBER`
+
 ---
 
 # Nodes and Tags
@@ -495,45 +554,6 @@ EOF
 
 ---
 
-# Locks
-
-*Locks* are a simple way to control access to shared resources. Any string may be used as a lock name. The command `laminarc lock mylock` locks `mylock`. Subsequent calls to `laminarc lock mylock` will block until `laminarc release mylock` is called a corresponding number of times.
-
-**CAUTION**: Locks are independent of any other job control mechanism in laminar, and will not be released automatically. Making sure calls to lock and release are symmetric is the administrator's responsibility.
-
-An example use builds on the situation described in [Data sharing and Workspaces](#Data-sharing-and-Workspaces), where a large git repository is stored in the workspace. Conisder this run script:
-
-```bash
-#!/bin/bash -x
-
-# This script expects to be passed the parameter `rev` which
-# should refer to a specific git commit in its source repository.
-# The commit ids could have been read from a server-side
-# post-commit git hook, where many commits could have been pushed
-# at once, but we want to check them all individually. This means
-# this job can be executed several times (with different commit ids)
-# at once.
-
-cd $WORKSPACE
-# Acquire a lock for modifying the workspace
-laminarc lock $JOB-workspace
-# Download all the latest commits
-git fetch
-git checkout $rev
-cd -
-# Fast copy (hard-link) the specific checkout to the build dir
-cp -al $WORKSPACE/src src
-# Release the lock to allow other jobs to do the same
-laminarc release $JOB-workspace
-
-# run the (much longer) build process
-set -e
-cmake src
-make
-```
-
----
-
 # Customizing the WebUI
 
 If it exists, the file `/var/lib/laminar/custom/style.css` will be served by laminar and may be used to change the appearance of Laminar's WebUI.
@@ -542,6 +562,18 @@ This directory is also a good place to add any extra assets needed for this cust
 
 ---
 
+# Badges
+
+Laminar will serve a job's current status as a pretty badge at the url `/badge/JOBNAME.svg`. This can be used as a link to your server instance from your Github README.md file or cat blog:
+
+```
+<a href="https://my-example-laminar-server.com/jobs/my-project">
+  <img src="https://my-example-laminar-server.com/badge/my-project.svg">
+</a>
+```
+
+---
+
 # Reference
 
 ## Service configuration file
@@ -595,9 +627,13 @@ Finally, variables supplied on the command-line call to `laminarc queue`, `lamin
 
 - `queue [JOB [PARAMS...]]...` adds one or more jobs to the queue with optional parameters, returning immediately.
 - `start [JOB [PARAMS...]]...` starts one or more jobs with optional parameters, returning when the jobs begin execution.
-- `run [JOB [PARAMS...]]...` triggers one or more jobs with optional parameters and waits for the completion of all jobs. Returns a non-zero error code if any job failed.
+- `run [JOB [PARAMS...]]...` triggers one or more jobs with optional parameters and waits for the completion of all jobs.
 - `set [VARIABLE=VALUE]...` sets one or more variables to be exported in subsequent scripts for the run identified by the `$JOB` and `$RUN` environment variables
-- `lock [NAME]` acquires the [lock](#Locks) named `NAME`
-- `release [NAME]` releases the lock named `NAME`
+- `show-jobs` shows the known jobs on the server (`$LAMINAR_HOME/cfg/jobs/*.run`).
+- `show-running` shows the currently running jobs with their numbers.
+- `show-queued` shows the names of the jobs waiting in the queue.
+- `abort JOB NUMBER` manually aborts a currently running job by name and number.
 
 `laminarc` connects to `laminard` using the address supplied by the `LAMINAR_HOST` environment variable. If it is not set, `laminarc` will first attempt to use `LAMINAR_BIND_RPC`, which will be available if `laminarc` is executed from a script within `laminard`. If neither `LAMINAR_HOST` nor `LAMINAR_BIND_RPC` is set, `laminarc` will assume a default host of `unix-abstract:laminar`.
+
+All commands return zero on success or a non-zero code if the command could not be executed. `laminarc run` will return a non-zero exit status if any executed job failed.

docker-build-centos.sh

@@ -8,37 +8,32 @@ VERSION=$(cd "$SOURCE_DIR" && git describe --tags --abbrev=8 --dirty | tr - .)
 
 DOCKER_TAG=$(docker build -q - <<EOS
 FROM centos:7
-RUN yum -y install epel-release && yum -y install rpm-build cmake3 make gcc gcc-c++ wget sqlite-devel boost-devel zlib-devel
+RUN yum -y install epel-release centos-release-scl && yum-config-manager --enable rhel-server-rhscl-7-rpms && yum -y install rpm-build cmake3 make devtoolset-7-gcc-c++ wget sqlite-devel boost-devel zlib-devel
 EOS
 )
 
 docker run --rm -i -v $SOURCE_DIR:/root/rpmbuild/SOURCES/laminar-$VERSION:ro -v $OUTPUT_DIR:/output $DOCKER_TAG bash -xe <<EOS
+# for new gcc
+export PATH=/opt/rh/devtoolset-7/root/usr/bin:\$PATH
 
 mkdir /build
 cd /build
 
-wget -O capnproto.tar.gz https://github.com/capnproto/capnproto/archive/3079784bfaf3ba05edacfc63d6d494b76a85a3a5.tar.gz
-wget -O websocketpp.tar.gz https://github.com/zaphoyd/websocketpp/archive/0.7.0.tar.gz
+wget -O capnproto.tar.gz https://github.com/capnproto/capnproto/archive/v0.7.0.tar.gz
 wget -O rapidjson.tar.gz https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz
 md5sum -c <<EOF
-c5c04c1892a381e30bd032a6bceef111  capnproto.tar.gz
-5027c20cde76fdaef83a74acfcf98e23  websocketpp.tar.gz
+a9de5f042f4cf05515c2d7dfc7f5df21  capnproto.tar.gz
 badd12c511e081fec6c89c43a7027bce  rapidjson.tar.gz
 EOF
 
 tar xzf capnproto.tar.gz
-tar xzf websocketpp.tar.gz
 tar xzf rapidjson.tar.gz
 
-cd /build/capnproto-3079784bfaf3ba05edacfc63d6d494b76a85a3a5/c++/
+cd /build/capnproto-0.7.0/c++/
 cmake3 -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=off .
 make -j4
 make install
 
-cd /build/websocketpp-0.7.0/
-cmake3 .
-make install
-
 cd /build/rapidjson-1.1.0/
 cmake3 -DRAPIDJSON_BUILD_EXAMPLES=off .
 make install
@@ -51,7 +46,7 @@ Version: $VERSION
 Release: 1
 License: GPL
 BuildRequires: systemd-units
-Requires: sqlite boost-filesystem zlib
+Requires: sqlite zlib
 
 %description
 Lightweight Continuous Integration Service
@@ -71,6 +66,8 @@ make
 %{_bindir}/laminard
 %{_unitdir}/laminar.service
 %config(noreplace) %{_sysconfdir}/laminar.conf
+%{_datarootdir}/bash-completion/completions/laminarc
+%{_datarootdir}/zsh/site-functions/_laminarc
 
 %post
 echo Creating laminar user with home in %{_sharedstatedir}/laminar

docker-build-debian.sh

@@ -8,7 +8,7 @@ VERSION=$(cd "$SOURCE_DIR" && git describe --tags --abbrev=8 --dirty)
 
 DOCKER_TAG=$(docker build -q - <<EOS
 FROM debian:9-slim
-RUN apt-get update && apt-get install -y wget cmake g++ libsqlite3-dev libboost-filesystem1.62-dev zlib1g-dev
+RUN apt-get update && apt-get install -y wget cmake g++ libsqlite3-dev libboost-dev zlib1g-dev
 EOS
 )
 
@@ -17,34 +17,27 @@ docker run --rm -i -v $SOURCE_DIR:/laminar:ro -v $OUTPUT_DIR:/output $DOCKER_TAG
 mkdir /build
 cd /build
 
-wget -O capnproto.tar.gz https://github.com/capnproto/capnproto/archive/3079784bfaf3ba05edacfc63d6d494b76a85a3a5.tar.gz
-wget -O websocketpp.tar.gz https://github.com/zaphoyd/websocketpp/archive/0.7.0.tar.gz
+wget -O capnproto.tar.gz https://github.com/capnproto/capnproto/archive/v0.7.0.tar.gz
 wget -O rapidjson.tar.gz https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz
 md5sum -c <<EOF
-c5c04c1892a381e30bd032a6bceef111  capnproto.tar.gz
-5027c20cde76fdaef83a74acfcf98e23  websocketpp.tar.gz
+a9de5f042f4cf05515c2d7dfc7f5df21  capnproto.tar.gz
 badd12c511e081fec6c89c43a7027bce  rapidjson.tar.gz
 EOF
 
 tar xzf capnproto.tar.gz
-tar xzf websocketpp.tar.gz
 tar xzf rapidjson.tar.gz
 
-cd /build/capnproto-3079784bfaf3ba05edacfc63d6d494b76a85a3a5/c++/
+cd /build/capnproto-0.7.0/c++/
 cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=off .
 make -j4
 make install
 
-cd /build/websocketpp-0.7.0/
-cmake .
-make install
-
 cd /build/rapidjson-1.1.0/
 cmake -DRAPIDJSON_BUILD_EXAMPLES=off .
 make install
 
 cd /build
-cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/ /laminar
+cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/ -DZSH_COMPLETIONS_DIR=/usr/share/zsh/functions/Completion/Unix /laminar
 make -j4
 mkdir laminar
 make DESTDIR=laminar install/strip
@@ -57,7 +50,7 @@ Section:
 Priority: optional
 Architecture: amd64
 Maintainer: Oliver Giles <web ohwg net>
-Depends: libsqlite3-0, libboost-filesystem1.62.0, zlib1g
+Depends: libsqlite3-0, zlib1g
 Description: Lightweight Continuous Integration Service
 EOF
 cat <<EOF > laminar/DEBIAN/postinst

docker-xbuild-debian-armhf.sh

@@ -0,0 +1,87 @@
+#!/bin/bash -e
+
+OUTPUT_DIR=$PWD
+
+SOURCE_DIR=$(readlink -f $(dirname ${BASH_SOURCE[0]}))
+
+VERSION=$(cd "$SOURCE_DIR" && git describe --tags --abbrev=8 --dirty)
+
+DOCKER_TAG=$(docker build -q - <<EOS
+FROM debian:9-slim
+RUN dpkg --add-architecture armhf && apt-get update && apt-get install -y wget cmake crossbuild-essential-armhf libsqlite3-dev:armhf libboost-dev:armhf zlib1g-dev:armhf
+EOS
+)
+
+docker run --rm -i -v $SOURCE_DIR:/laminar:ro -v $OUTPUT_DIR:/output $DOCKER_TAG bash -xe <<EOS
+
+mkdir /build
+cd /build
+
+cat > toolchain.cmake <<EOF
+SET(CMAKE_C_COMPILER arm-linux-gnueabihf-gcc)
+SET(CMAKE_CXX_COMPILER arm-linux-gnueabihf-g++)
+set(CMAKE_LIBRARY_ARCHITECTURE arm-linux-gnueabihf)
+EOF
+
+wget -O capnproto.tar.gz https://github.com/capnproto/capnproto/archive/v0.7.0.tar.gz
+wget -O rapidjson.tar.gz https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz
+md5sum -c <<EOF
+a9de5f042f4cf05515c2d7dfc7f5df21  capnproto.tar.gz
+badd12c511e081fec6c89c43a7027bce  rapidjson.tar.gz
+EOF
+
+tar xzf capnproto.tar.gz
+tar xzf rapidjson.tar.gz
+
+mkdir capnproto-host
+cd capnproto-host
+cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=off /build/capnproto-0.7.0/c++/
+make -j4
+make install
+
+cd /build/capnproto-0.7.0/c++/
+cmake -DCMAKE_TOOLCHAIN_FILE=../../toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/arm-linux-gnueabihf/ .
+make -j4
+make install
+
+cd /build/rapidjson-1.1.0/
+cmake -DRAPIDJSON_BUILD_EXAMPLES=off -DCMAKE_INSTALL_PREFIX=/usr/arm-linux-gnueabihf/ .
+make install
+
+cd /build
+cmake \
+	-DCMAKE_TOOLCHAIN_FILE=toolchain.cmake \
+	-DCMAKE_LINKER=/usr/bin/arm-linux-gnueabihf-ld \
+	-DCMAKE_OBJCOPY=/usr/bin/arm-linux-gnueabihf-objcopy \
+	-DCMAKE_STRIP=/usr/bin/arm-linux-gnueabihf-strip \
+	-DCMAKE_BUILD_TYPE=Release \
+	-DCMAKE_INSTALL_PREFIX=/ \
+	-DZSH_COMPLETIONS_DIR=/usr/share/zsh/functions/Completion/Unix \
+	/laminar
+make -j4
+mkdir laminar
+make DESTDIR=laminar install/strip
+
+mkdir laminar/DEBIAN
+cat <<EOF > laminar/DEBIAN/control
+Package: laminar
+Version: $VERSION
+Section: 
+Priority: optional
+Architecture: armhf
+Maintainer: Oliver Giles <web ohwg net>
+Depends: libsqlite3-0, zlib1g
+Description: Lightweight Continuous Integration Service
+EOF
+cat <<EOF > laminar/DEBIAN/postinst
+#!/bin/bash
+echo Creating laminar user with home in /var/lib/laminar
+useradd -r -d /var/lib/laminar -s /usr/sbin/nologin laminar
+mkdir -p /var/lib/laminar/cfg/{jobs,nodes,scripts}
+chown -R laminar: /var/lib/laminar
+EOF
+chmod +x laminar/DEBIAN/postinst
+
+dpkg-deb --build laminar
+mv laminar.deb /output/laminar-$VERSION-1-armhf.deb
+EOS

etc/laminar.conf

@@ -43,6 +43,16 @@
 ###
 #LAMINAR_KEEP_RUNDIRS=0
 
+
+###
+### LAMINAR_BASE_URL
+###
+### Base url for the frontend. This affects the <base href> tag and needs
+### to be set if Laminar runs behind a reverse-proxy that hosts Laminar
+### within a subfolder (rather than at a subdomain root)
+###
+#LAMINAR_BASE_URL=/
+
 ###
 ### LAMINAR_ARCHIVE_URL
 ###
@@ -51,4 +61,4 @@
 ### uses a very naive and inefficient method. Best to let a real
 ### webserver handle serving those requests.
 ###
-#LAMINAR_ARCHIVE_URL=http://backbone.example.com/ci/archive
+#LAMINAR_ARCHIVE_URL=http://backbone.example.com/ci/archive/

etc/laminar.service

@@ -1,5 +1,6 @@
 [Unit]
 Description=Laminar continuous integration service
+After=network.target
 
 [Service]
 User=laminar

etc/laminarc-completion.bash

@@ -0,0 +1,28 @@
+# Bash completion file for laminarc
+# vim: ft=sh
+
+_laminarc() {
+	local cur prev words cword
+	_init_completion || return
+	if [ "$cword" -gt 1 ]; then
+		case "${words[1]}" in
+			queue|start|run)
+				if [ "$cword" -eq 2 ]; then
+					COMPREPLY+=($(compgen -W "$(laminarc show-jobs)" -- ${cur}))
+				fi
+				;;
+			abort)
+				if [ "$cword" -eq 2 ]; then
+					COMPREPLY+=($(compgen -W "$(laminarc show-running | cut -d : -f 1)" -- ${cur}))
+				elif [ "$cword" -eq 3 ]; then
+					COMPREPLY+=($(compgen -W "$(laminarc show-running | cut -d : -f 2)" -- ${cur}))
+				fi
+				;;
+		esac
+	else
+		local cmds="queue start run set show-jobs show-queued show-running abort"
+		COMPREPLY+=($(compgen -W "${cmds}" -- ${cur}))
+	fi
+}
+
+complete  -F _laminarc laminarc

etc/laminarc-completion.zsh

@@ -0,0 +1,34 @@
+#compdef laminarc
+#autoload
+
+_laminarc() {
+	if (( CURRENT == 2 )); then
+		_values "Operation" \
+			"queue" \
+			"start" \
+			"run" \
+			"set" \
+			"show-jobs" \
+			"show-queued" \
+			"show-running" \
+			"abort"
+	else
+		case "${words[2]}" in
+			queue|start|run)
+				if (( CURRENT == 3 )); then
+					_values "Jobs" $(laminarc show-jobs)
+				fi
+				;;
+			abort)
+				if (( CURRENT == 3 )); then
+					_values "Jobs" $(laminarc show-running | cut -d : -f 1)
+				elif (( CURRENT == 4 )); then
+					_values "Runs" $(laminarc show-running | cut -d : -f 2)
+				fi
+				;;
+		esac
+	fi
+}
+
+_laminarc
+# vim: ft=zsh

src/client.cpp

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015-2017 Oliver Giles
+/// Copyright 2015-2018 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -23,9 +23,10 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <errno.h>
 
-#define EFAILED 55
+#define EXIT_BAD_ARGUMENT     1
+#define EXIT_OPERATION_FAILED 2
+#define EXIT_RUN_FAILED       3
 
 template<typename T>
 static int setParams(int argc, char** argv, T& request) {
@@ -36,20 +37,13 @@ static int setParams(int argc, char** argv, T& request) {
         n++;
     }
 
-    int argsConsumed = n;
-
     char* job = getenv("JOB");
     char* num = getenv("RUN");
     char* reason = getenv("LAMINAR_REASON");
 
-    if(job && num) n+=2;
-    else if(reason) n++;
+    auto params = request.initParams(n + (job&&num?2:0) + (reason?1:0));
 
-    if(n == 0) return argsConsumed;
-
-    auto params = request.initParams(n);
-
-    for(int i = 0; i < argsConsumed; ++i) {
+    for(int i = 0; i < n; ++i) {
         char* name = argv[i];
         char* val = strchr(name, '=');
         *val++ = '\0';
@@ -57,23 +51,32 @@ static int setParams(int argc, char** argv, T& request) {
         params[i].setValue(val);
     }
 
+    int argsConsumed = n;
+
     if(job && num) {
-        params[argsConsumed].setName("=parentJob");
-        params[argsConsumed].setValue(job);
-        params[argsConsumed+1].setName("=parentBuild");
-        params[argsConsumed+1].setValue(num);
-    } else if(reason) {
-        params[argsConsumed].setName("=reason");
-        params[argsConsumed].setValue(reason);
+        params[n].setName("=parentJob");
+        params[n++].setValue(job);
+        params[n].setName("=parentBuild");
+        params[n++].setValue(num);
+    }
+    if(reason) {
+        params[n].setName("=reason");
+        params[n].setValue(reason);
     }
 
     return argsConsumed;
 }
 
+static void printTriggerLink(const char* job, uint run) {
+    // use a private ANSI CSI sequence to mark the JOB:NUM so the
+    // frontend can recognise it and generate a hyperlink.
+    printf("\033[{%s:%d\033\\\n", job, run);
+}
+
 int main(int argc, char** argv) {
     if(argc < 2) {
         fprintf(stderr, "Usage: %s <command> [parameters...]\n", argv[0]);
-        return EINVAL;
+        return EXIT_BAD_ARGUMENT;
     }
 
     int ret = 0;
@@ -87,7 +90,7 @@ int main(int argc, char** argv) {
     if(strcmp(argv[1], "queue") == 0) {
         if(argc < 3) {
             fprintf(stderr, "Usage %s queue <jobName>\n", argv[0]);
-            return EINVAL;
+            return EXIT_BAD_ARGUMENT;
         }
         kj::Vector<capnp::RemotePromise<LaminarCi::QueueResults>> promises;
         int jobNameIndex = 2;
@@ -103,7 +106,7 @@ int main(int argc, char** argv) {
         for(auto& p : promises) {
             if(p.wait(waitScope).getResult() != LaminarCi::MethodResult::SUCCESS) {
                 fprintf(stderr, "Failed to queue job '%s'\n", argv[2]);
-                return ENOENT;
+                return EXIT_OPERATION_FAILED;
             }
         }
     } else if(strcmp(argv[1], "start") == 0 || strcmp(argv[1], "trigger") == 0) {
@@ -111,7 +114,7 @@ int main(int argc, char** argv) {
             fprintf(stderr, "Warning: 'trigger' is deprecated, use 'queue' for the old behavior\n");
         if(argc < 3) {
             fprintf(stderr, "Usage %s queue <jobName>\n", argv[0]);
-            return EINVAL;
+            return EXIT_BAD_ARGUMENT;
         }
         kj::Vector<capnp::RemotePromise<LaminarCi::StartResults>> promises;
         struct: public kj::TaskSet::ErrorHandler {
@@ -127,9 +130,9 @@ int main(int argc, char** argv) {
             ts.add(req.send().then([&ret,argv,jobNameIndex](capnp::Response<LaminarCi::StartResults> resp){
                 if(resp.getResult() != LaminarCi::MethodResult::SUCCESS) {
                     fprintf(stderr, "Failed to start job '%s'\n", argv[2]);
-                    ret = ENOENT;
+                    ret = EXIT_OPERATION_FAILED;
                 }
-                printf("%s:%d\n", argv[jobNameIndex], resp.getBuildNum());
+                printTriggerLink(argv[jobNameIndex], resp.getBuildNum());
             }));
             jobNameIndex += n + 1;
         } while(jobNameIndex < argc);
@@ -137,7 +140,7 @@ int main(int argc, char** argv) {
     } else if(strcmp(argv[1], "run") == 0) {
         if(argc < 3) {
             fprintf(stderr, "Usage %s run <jobName>\n", argv[0]);
-            return EINVAL;
+            return EXIT_BAD_ARGUMENT;
         }
         struct: public kj::TaskSet::ErrorHandler {
             void taskFailed(kj::Exception&&) override {}
@@ -150,9 +153,9 @@ int main(int argc, char** argv) {
             req.setJobName(argv[jobNameIndex]);
             int n = setParams(argc - jobNameIndex - 1, &argv[jobNameIndex + 1], req);
             ts.add(req.send().then([&ret,argv,jobNameIndex](capnp::Response<LaminarCi::RunResults> resp){
-                printf("%s:%d\n", argv[jobNameIndex], resp.getBuildNum());
+                printTriggerLink(argv[jobNameIndex], resp.getBuildNum());
                 if(resp.getResult() != LaminarCi::JobResult::SUCCESS) {
-                    ret = EFAILED;
+                    ret = EXIT_RUN_FAILED;
                 }
             }));
             jobNameIndex += n + 1;
@@ -161,7 +164,7 @@ int main(int argc, char** argv) {
     } else if(strcmp(argv[1], "set") == 0) {
         if(argc < 3) {
             fprintf(stderr, "Usage %s set param=value\n", argv[0]);
-            return EINVAL;
+            return EXIT_BAD_ARGUMENT;
         }
         auto req = laminar.setRequest();
         char* eq = strchr(argv[2], '=');
@@ -171,26 +174,52 @@ int main(int argc, char** argv) {
             char* name = argv[2];
             *eq++ = '\0';
             char* val = eq;
-            req.setJobName(job);
-            req.setBuildNum(atoi(num));
+            req.getRun().setJob(job);
+            req.getRun().setBuildNum(atoi(num));
             req.getParam().setName(name);
             req.getParam().setValue(val);
             req.send().wait(waitScope);
         } else {
             fprintf(stderr, "Missing $JOB or $RUN or param is not in the format key=value\n");
-            return EINVAL;
+            return EXIT_BAD_ARGUMENT;
+        }
+    } else if(strcmp(argv[1], "abort") == 0) {
+        if(argc != 4) {
+            fprintf(stderr, "Usage %s abort <jobName> <jobNumber>\n", argv[0]);
+            return EXIT_BAD_ARGUMENT;
+        }
+        auto req = laminar.abortRequest();
+        req.getRun().setJob(argv[2]);
+        req.getRun().setBuildNum(atoi(argv[3]));
+        if(req.send().wait(waitScope).getResult() != LaminarCi::MethodResult::SUCCESS)
+            ret = EXIT_OPERATION_FAILED;
+    } else if(strcmp(argv[1], "show-jobs") == 0) {
+        if(argc != 2) {
+            fprintf(stderr, "Usage: %s show-jobs\n", argv[0]);
+            return EXIT_BAD_ARGUMENT;
+        }
+        for(auto it : laminar.listKnownRequest().send().wait(waitScope).getResult()) {
+            printf("%s\n", it.cStr());
+        }
+    } else if(strcmp(argv[1], "show-queued") == 0) {
+        if(argc != 2) {
+            fprintf(stderr, "Usage: %s show-queued\n", argv[0]);
+            return EXIT_BAD_ARGUMENT;
+        }
+        for(auto it : laminar.listQueuedRequest().send().wait(waitScope).getResult()) {
+            printf("%s\n", it.cStr());
+        }
+    } else if(strcmp(argv[1], "show-running") == 0) {
+        if(argc != 2) {
+            fprintf(stderr, "Usage: %s show-running\n", argv[0]);
+            return EXIT_BAD_ARGUMENT;
+        }
+        for(auto it : laminar.listRunningRequest().send().wait(waitScope).getResult()) {
+            printf("%s:%d\n", it.getJob().cStr(), it.getBuildNum());
         }
-    } else if(strcmp(argv[1], "lock") == 0) {
-        auto req = laminar.lockRequest();
-        req.setLockName(argv[2]);
-        req.send().wait(waitScope);
-    } else if(strcmp(argv[1], "release") == 0) {
-        auto req = laminar.releaseRequest();
-        req.setLockName(argv[2]);
-        req.send().wait(waitScope);
     } else {
         fprintf(stderr, "Unknown command %s\n", argv[1]);
-        return EINVAL;
+        return EXIT_BAD_ARGUMENT;
     }
 
     return ret;

src/database.cpp

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015-2016 Oliver Giles
+/// Copyright 2015-2018 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -20,9 +20,42 @@
 
 #include <sqlite3.h>
 #include <string.h>
+#include <math.h>
+
+struct StdevCtx {
+    double mean;
+    double M2;
+    int64_t count;
+};
+
+static void stdevStep(sqlite3_context *ctx, int, sqlite3_value **args)
+{
+    StdevCtx* p = static_cast<StdevCtx*>(sqlite3_aggregate_context(ctx, sizeof(StdevCtx)));
+    // Welford's Online Algorithm
+    if(sqlite3_value_numeric_type(args[0]) != SQLITE_NULL) {
+        p->count++;
+        double val = sqlite3_value_double(args[0]);
+        double delta = val - p->mean;
+        p->mean += delta / p->count;
+        p->M2 += delta * (val - p->mean);
+    }
+}
+
+static void stdevFinalize(sqlite3_context *context){
+    StdevCtx* p = static_cast<StdevCtx*>(sqlite3_aggregate_context(context, 0));
+    if(p && p->count > 1)
+        sqlite3_result_double(context, sqrt(p->M2 / (p->count-1)));
+    else
+        sqlite3_result_null(context);
+}
 
 Database::Database(const char *path) {
     sqlite3_open(path, &hdl);
+    int create_func_flags = SQLITE_UTF8;
+#if SQLITE_VERSION_NUMBER >= 3008003
+    create_func_flags |= SQLITE_DETERMINISTIC;
+#endif
+    sqlite3_create_function(hdl, "STDEV", 1, create_func_flags, NULL, NULL, stdevStep, stdevFinalize);
 }
 
 Database::~Database() {
@@ -96,6 +129,10 @@ template<> ulong Database::Statement::fetchColumn(int col) {
     return static_cast<ulong>(sqlite3_column_int64(stmt, col));
 }
 
+template<> double Database::Statement::fetchColumn(int col) {
+    return sqlite3_column_double(stmt, col);
+}
+
 bool Database::Statement::row() {
     return sqlite3_step(stmt) == SQLITE_ROW;
 }

src/database.h

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015-2016 Oliver Giles
+/// Copyright 2015-2018 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -151,5 +151,6 @@ template<> int Database::Statement::fetchColumn(int col);
 template<> uint Database::Statement::fetchColumn(int col);
 template<> long Database::Statement::fetchColumn(int col);
 template<> ulong Database::Statement::fetchColumn(int col);
+template<> double Database::Statement::fetchColumn(int col);
 
 #endif // LAMINAR_DATABASE_H_

src/interface.h

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015 Oliver Giles
+/// Copyright 2015-2019 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -25,8 +25,6 @@
 #include <memory>
 #include <unordered_map>
 
-typedef std::unordered_map<std::string, std::string> ParamMap;
-
 // Simple struct to define which information a frontend client is interested
 // in, both in initial request phase and real-time updates. It corresponds
 // loosely to frontend URLs
@@ -42,7 +40,10 @@ struct MonitorScope {
     MonitorScope(Type type = HOME, std::string job = std::string(), uint num = 0) :
         type(type),
         job(job),
-        num(num)
+        num(num),
+        page(0),
+        field("number"),
+        order_desc(true)
     {}
 
     // whether this scope wants status information about the given job or run
@@ -60,14 +61,20 @@ struct MonitorScope {
     Type type;
     std::string job;
     uint num = 0;
+    // sorting
+    uint page = 0;
+    std::string field;
+    bool order_desc;
 };
 
 // Represents a (websocket) client that wants to be notified about events
 // matching the supplied scope. Pass instances of this to LaminarInterface
 // registerClient and deregisterClient
 struct LaminarClient {
-    virtual ~LaminarClient() =default;
+    virtual ~LaminarClient() noexcept(false) {}
     virtual void sendMessage(std::string payload) = 0;
+    // TODO: redesign
+    virtual void notifyJobFinished() {}
     MonitorScope scope;
 };
 
@@ -79,11 +86,18 @@ struct LaminarWaiter {
     virtual void complete(const Run*) = 0;
 };
 
+// Represents a file mapped in memory. Used to serve artefacts
+struct MappedFile {
+    virtual ~MappedFile() =default;
+    virtual const void* address() = 0;
+    virtual size_t size() = 0;
+};
+
 // The interface connecting the network layer to the application business
 // logic. These methods fulfil the requirements of both the HTTP/Websocket
 // and RPC interfaces.
 struct LaminarInterface {
-    virtual ~LaminarInterface() =default;
+    virtual ~LaminarInterface() {}
 
     // Queues a job, returns immediately. Return value will be nullptr if
     // the supplied name is not a known job.
@@ -106,6 +120,13 @@ struct LaminarInterface {
     // to call LaminarWaiter::complete on invalid data
     virtual void deregisterWaiter(LaminarWaiter* waiter) = 0;
 
+    // Return the latest known number of the named job
+    virtual uint latestRun(std::string job) = 0;
+
+    // Given a job name and number, return existence and (via reference params)
+    // its current log output and whether the job is ongoing
+    virtual bool handleLogRequest(std::string name, uint num, std::string& output, bool& complete) = 0;
+
     // Synchronously send a snapshot of the current status to the given
     // client (as governed by the client's MonitorScope). This is called on
     // initial websocket connect.
@@ -116,22 +137,36 @@ struct LaminarInterface {
     // the environment of subsequent scripts.
     virtual bool setParam(std::string job, uint buildNum, std::string param, std::string value) = 0;
 
+    // Gets the list of jobs currently waiting in the execution queue
+    virtual const std::list<std::shared_ptr<Run>>& listQueuedJobs() = 0;
+
+    // Gets the list of currently executing jobs
+    virtual const RunSet& listRunningJobs() = 0;
+
+    // Gets the list of known jobs - scans cfg/jobs for *.run files
+    virtual std::list<std::string> listKnownJobs() = 0;
+
     // Fetches the content of an artifact given its filename relative to
-    // $LAMINAR_HOME/archive. This shouldn't be used, because the sysadmin
-    // should have configured a real webserver to serve these things.
-    virtual bool getArtefact(std::string path, std::string& result) = 0;
+    // $LAMINAR_HOME/archive. Ideally, this would instead be served by a
+    // proper web server which handles this url.
+    virtual kj::Maybe<kj::Own<const kj::ReadableFile>> getArtefact(std::string path) = 0;
+
+    // Given the name of a job, populate the provided string reference with
+    // SVG content describing the last known state of the job. Returns false
+    // if the job is unknown.
+    virtual bool handleBadgeRequest(std::string job, std::string& badge) = 0;
 
     // Fetches the content of $LAMINAR_HOME/custom/style.css or an empty
-    // string. This shouldn't be used, because the sysadmin should have
-    // configured a real webserver to serve these things.
+    // string. Ideally, this would instead be served by a proper web server
+    // which handles this url.
     virtual std::string getCustomCss() = 0;
 
+    // Aborts a single job
+    virtual bool abort(std::string job, uint buildNum) = 0;
+
     // Abort all running jobs
     virtual void abortAll() = 0;
 
-    // Callback for laminar to reap child processes.
-    virtual void reapChildren() = 0;
-
     // Callback to handle a configuration modification notification
     virtual void notifyConfigChanged() = 0;
 };

src/laminar.capnp

@@ -5,9 +5,16 @@ interface LaminarCi {
     queue @0 (jobName :Text, params :List(JobParam)) -> (result :MethodResult);
     start @1 (jobName :Text, params :List(JobParam)) -> (result :MethodResult, buildNum :UInt32);
     run @2 (jobName :Text, params :List(JobParam)) -> (result :JobResult, buildNum :UInt32);
-    set @3 (jobName :Text, buildNum :UInt32, param :JobParam) -> (result :MethodResult);
-    lock @4 (lockName :Text) -> ();
-    release @5 (lockName :Text) -> ();
+    set @3 (run :Run, param :JobParam) -> (result :MethodResult);
+    listQueued @4 () -> (result :List(Text));
+    listRunning @5 () -> (result :List(Run));
+    listKnown @6 () -> (result :List(Text));
+    abort @7 (run :Run) -> (result :MethodResult);
+
+    struct Run {
+        job @0 :Text;
+        buildNum @1 :UInt32;
+    }
 
     struct JobParam {
         name @0 :Text;

src/laminar.cpp

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015-2018 Oliver Giles
+/// Copyright 2015-2019 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -22,12 +22,13 @@
 #include "log.h"
 
 #include <sys/wait.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 #include <fstream>
 #include <zlib.h>
 
-#include <boost/filesystem.hpp>
-namespace fs = boost::filesystem;
-
 #define COMPRESS_LOG_MIN_SIZE 1024
 
 #include <rapidjson/stringbuffer.h>
@@ -47,6 +48,7 @@ public:
 private:
     rapidjson::StringBuffer buf;
 };
+template<> Json& Json::set(const char* key, double value) { String(key); Double(value); return *this; }
 template<> Json& Json::set(const char* key, const char* value) { String(key); String(value); return *this; }
 template<> Json& Json::set(const char* key, std::string value) { String(key); String(value.c_str()); return *this; }
 
@@ -54,26 +56,37 @@ namespace {
 // Default values when none were supplied in $LAMINAR_CONF_FILE (/etc/laminar.conf)
 constexpr const char* INTADDR_RPC_DEFAULT = "unix-abstract:laminar";
 constexpr const char* INTADDR_HTTP_DEFAULT = "*:8080";
-constexpr const char* ARCHIVE_URL_DEFAULT = "/archive";
+constexpr const char* ARCHIVE_URL_DEFAULT = "/archive/";
 }
 
-// helper for appending to boost::filesystem::path
-fs::path operator+(fs::path p, const char* ext) {
-    std::string leaf = p.leaf().string();
-    leaf += ext;
-    return p.remove_leaf()/leaf;
+// short syntax helpers for kj::Path
+template<typename T>
+inline kj::Path operator/(const kj::Path& p, const T& ext) {
+    return p.append(ext);
+}
+template<typename T>
+inline kj::Path operator/(const std::string& p, const T& ext) {
+    return kj::Path{p}/ext;
 }
 
 typedef std::string str;
 
-Laminar::Laminar() {
-    archiveUrl = ARCHIVE_URL_DEFAULT;
-    if(char* envArchive = getenv("LAMINAR_ARCHIVE_URL"))
-        archiveUrl = envArchive;
-    numKeepRunDirs = 0;
-    homeDir = getenv("LAMINAR_HOME") ?: "/var/lib/laminar";
+Laminar::Laminar(const char *home) :
+    homePath(kj::Path::parse(&home[1])),
+    fsHome(kj::newDiskFilesystem()->getRoot().openSubdir(homePath, kj::WriteMode::MODIFY))
+{
+    LASSERT(home[0] == '/');
 
-    db = new Database((fs::path(homeDir)/"laminar.sqlite").string().c_str());
+    archiveUrl = ARCHIVE_URL_DEFAULT;
+    if(char* envArchive = getenv("LAMINAR_ARCHIVE_URL")) {
+        archiveUrl = envArchive;
+        if(archiveUrl.back() != '/')
+            archiveUrl.append("/");
+    }
+
+    numKeepRunDirs = 0;
+
+    db = new Database((homePath/"laminar.sqlite").toString(true).cStr());
     // Prepare database for first use
     // TODO: error handling
     db->exec("CREATE TABLE IF NOT EXISTS builds("
@@ -113,6 +126,52 @@ void Laminar::deregisterWaiter(LaminarWaiter *waiter) {
     waiters.erase(waiter);
 }
 
+uint Laminar::latestRun(std::string job) {
+    auto it = activeJobs.byJobName().equal_range(job);
+    if(it.first == it.second) {
+        uint result = 0;
+        db->stmt("SELECT MAX(number) FROM builds WHERE name = ?")
+                .bind(job)
+                .fetch<uint>([&](uint x){
+            result = x;
+        });
+        return result;
+    } else {
+        return (*--it.second)->build;
+    }
+}
+
+// TODO: reunify with sendStatus. The difference is that this method is capable of
+// returning "not found" to the caller, and sendStatus isn't
+bool Laminar::handleLogRequest(std::string name, uint num, std::string& output, bool& complete) {
+    if(Run* run = activeRun(name, num)) {
+        output = run->log;
+        complete = false;
+        return true;
+    } else { // it must be finished, fetch it from the database
+        db->stmt("SELECT output, outputLen FROM builds WHERE name = ? AND number = ?")
+                .bind(name, num)
+                .fetch<str,int>([&](str maybeZipped, unsigned long sz) {
+            str log(sz,'\0');
+            if(sz >= COMPRESS_LOG_MIN_SIZE) {
+                int res = ::uncompress((uint8_t*) log.data(), &sz,
+                                       (const uint8_t*) maybeZipped.data(), maybeZipped.size());
+                if(res == Z_OK)
+                    std::swap(output, log);
+                else
+                    LLOG(ERROR, "Failed to uncompress log", res);
+            } else {
+                std::swap(output, maybeZipped);
+            }
+        });
+        if(output.size()) {
+            complete = true;
+            return true;
+        }
+    }
+    return false;
+}
+
 bool Laminar::setParam(std::string job, uint buildNum, std::string param, std::string value) {
     if(Run* run = activeRun(job, buildNum)) {
         run->params[param] = value;
@@ -121,18 +180,37 @@ bool Laminar::setParam(std::string job, uint buildNum, std::string param, std::s
     return false;
 }
 
+const std::list<std::shared_ptr<Run>>& Laminar::listQueuedJobs() {
+    return queuedJobs;
+}
+
+const RunSet& Laminar::listRunningJobs() {
+    return activeJobs;
+}
+
+std::list<std::string> Laminar::listKnownJobs() {
+    std::list<std::string> res;
+    KJ_IF_MAYBE(dir, fsHome->tryOpenSubdir(kj::Path{"cfg","jobs"})) {
+        for(kj::Directory::Entry& entry : (*dir)->listEntries()) {
+            if(entry.name.endsWith(".run")) {
+                res.emplace_back(entry.name.cStr(), entry.name.findLast('.').orDefault(0));
+            }
+        }
+    }
+    return res;
+}
 
 void Laminar::populateArtifacts(Json &j, std::string job, uint num) const {
-    fs::path dir(fs::path(homeDir)/"archive"/job/std::to_string(num));
-    if(fs::is_directory(dir)) {
-        size_t prefixLen = (fs::path(homeDir)/"archive").string().length();
-        size_t scopeLen = dir.string().length();
-        for(fs::recursive_directory_iterator it(dir); it != fs::recursive_directory_iterator(); ++it) {
-            if(!fs::is_regular_file(*it))
+    kj::Path runArchive{job,std::to_string(num)};
+    KJ_IF_MAYBE(dir, fsHome->tryOpenSubdir("archive"/runArchive)) {
+        for(kj::StringPtr file : (*dir)->listNames()) {
+            kj::FsNode::Metadata meta = (*dir)->lstat(kj::Path{file});
+            if(meta.type != kj::FsNode::Type::FILE)
                 continue;
             j.StartObject();
-            j.set("url", archiveUrl + it->path().string().substr(prefixLen));
-            j.set("filename", it->path().string().substr(scopeLen+1));
+            j.set("url", archiveUrl + (runArchive/file).toString().cStr());
+            j.set("filename", file.cStr());
+            j.set("size", meta.size);
             j.EndObject();
         }
     }
@@ -159,6 +237,7 @@ void Laminar::sendStatus(LaminarClient* client) {
                     client->sendMessage(maybeZipped);
                 }
             });
+            client->notifyJobFinished();
         }
         return;
     }
@@ -169,20 +248,22 @@ void Laminar::sendStatus(LaminarClient* client) {
     j.set("time", time(nullptr));
     j.startObject("data");
     if(client->scope.type == MonitorScope::RUN) {
-        db->stmt("SELECT queuedAt,startedAt,completedAt, result, reason FROM builds WHERE name = ? AND number = ?")
+        db->stmt("SELECT queuedAt,startedAt,completedAt,result,reason,parentJob,parentBuild FROM builds WHERE name = ? AND number = ?")
         .bind(client->scope.job, client->scope.num)
-        .fetch<time_t, time_t, time_t, int, std::string>([&](time_t queued, time_t started, time_t completed, int result, std::string reason) {
+        .fetch<time_t, time_t, time_t, int, std::string, std::string, uint>([&](time_t queued, time_t started, time_t completed, int result, std::string reason, std::string parentJob, uint parentBuild) {
             j.set("queued", started-queued);
             j.set("started", started);
             j.set("completed", completed);
             j.set("result", to_string(RunState(result)));
             j.set("reason", reason);
+            j.startObject("upstream").set("name", parentJob).set("num", parentBuild).EndObject(2);
         });
         if(const Run* run = activeRun(client->scope.job, client->scope.num)) {
             j.set("queued", run->startedAt - run->queuedAt);
             j.set("started", run->startedAt);
-            j.set("reason", run->reason());
             j.set("result", to_string(RunState::RUNNING));
+            j.set("reason", run->reason());
+            j.startObject("upstream").set("name", run->parentName).set("num", run->parentBuild).EndObject(2);
             db->stmt("SELECT completedAt - startedAt FROM builds WHERE name = ? ORDER BY completedAt DESC LIMIT 1")
              .bind(run->name)
              .fetch<uint>([&](uint lastRuntime){
@@ -194,9 +275,25 @@ void Laminar::sendStatus(LaminarClient* client) {
         populateArtifacts(j, client->scope.job, client->scope.num);
         j.EndArray();
     } else if(client->scope.type == MonitorScope::JOB) {
+        const uint runsPerPage = 10;
         j.startArray("recent");
-        db->stmt("SELECT number,startedAt,completedAt,result,reason FROM builds WHERE name = ? ORDER BY completedAt DESC LIMIT 25")
-        .bind(client->scope.job)
+        // ORDER BY param cannot be bound
+        std::string order_by;
+        std::string direction = client->scope.order_desc ? "DESC" : "ASC";
+        if(client->scope.field == "number")
+            order_by = "number " + direction;
+        else if(client->scope.field == "result")
+            order_by = "result " + direction + ", number DESC";
+        else if(client->scope.field == "started")
+            order_by = "startedAt " + direction + ", number DESC";
+        else if(client->scope.field == "duration")
+            order_by = "(completedAt-startedAt) " + direction + ", number DESC";
+        else
+            order_by = "number DESC";
+        std::string stmt = "SELECT number,startedAt,completedAt,result,reason FROM builds WHERE name = ? ORDER BY "
+                + order_by + " LIMIT ?,?";
+        db->stmt(stmt.c_str())
+        .bind(client->scope.job, client->scope.page * runsPerPage, runsPerPage)
         .fetch<uint,time_t,time_t,int,str>([&](uint build,time_t started,time_t completed,int result,str reason){
             j.StartObject();
             j.set("number", build)
@@ -207,6 +304,17 @@ void Laminar::sendStatus(LaminarClient* client) {
              .EndObject();
         });
         j.EndArray();
+        db->stmt("SELECT COUNT(*),AVG(completedAt-startedAt) FROM builds WHERE name = ?")
+        .bind(client->scope.job)
+        .fetch<uint,uint>([&](uint nRuns, uint averageRuntime){
+            j.set("averageRuntime", averageRuntime);
+            j.set("pages", (nRuns-1) / runsPerPage + 1);
+            j.startObject("sort");
+            j.set("page", client->scope.page)
+             .set("field", client->scope.field)
+             .set("order", client->scope.order_desc ? "dsc" : "asc")
+             .EndObject();
+        });
         j.startArray("running");
         auto p = activeJobs.byJobName().equal_range(client->scope.job);
         for(auto it = p.first; it != p.second; ++it) {
@@ -244,7 +352,7 @@ void Laminar::sendStatus(LaminarClient* client) {
 
     } else if(client->scope.type == MonitorScope::ALL) {
         j.startArray("jobs");
-        db->stmt("SELECT name,number,startedAt,completedAt,result FROM builds GROUP BY name ORDER BY number DESC")
+        db->stmt("SELECT name,number,startedAt,completedAt,result FROM builds b JOIN (SELECT name n,MAX(number) l FROM builds GROUP BY n) q ON b.name = q.n AND b.number = q.l")
         .fetch<str,uint,time_t,time_t,int>([&](str name,uint number, time_t started, time_t completed, int result){
             j.StartObject();
             j.set("name", name);
@@ -323,7 +431,7 @@ void Laminar::sendStatus(LaminarClient* client) {
         j.startArray("buildsPerDay");
         for(int i = 6; i >= 0; --i) {
             j.StartObject();
-            db->stmt("SELECT result, COUNT(*) FROM builds WHERE completedAt > ? AND completedAt < ? GROUP by result")
+            db->stmt("SELECT result, COUNT(*) FROM builds WHERE completedAt > ? AND completedAt < ? GROUP BY result")
                     .bind(86400*(time(nullptr)/86400 - i), 86400*(time(nullptr)/86400 - (i-1)))
                     .fetch<int,int>([&](int result, int num){
                 j.set(to_string(RunState(result)).c_str(), num);
@@ -332,29 +440,87 @@ void Laminar::sendStatus(LaminarClient* client) {
         }
         j.EndArray();
         j.startObject("buildsPerJob");
-        db->stmt("SELECT name, COUNT(*) FROM builds WHERE completedAt > ? GROUP BY name")
+        db->stmt("SELECT name, COUNT(*) c FROM builds WHERE completedAt > ? GROUP BY name ORDER BY c DESC LIMIT 5")
                 .bind(time(nullptr) - 86400)
                 .fetch<str, int>([&](str job, int count){
             j.set(job.c_str(), count);
         });
         j.EndObject();
         j.startObject("timePerJob");
-        db->stmt("SELECT name, AVG(completedAt-startedAt) FROM builds WHERE completedAt > ? GROUP BY name")
+        db->stmt("SELECT name, AVG(completedAt-startedAt) av FROM builds WHERE completedAt > ? GROUP BY name ORDER BY av DESC LIMIT 8")
                 .bind(time(nullptr) - 7 * 86400)
                 .fetch<str, uint>([&](str job, uint time){
             j.set(job.c_str(), time);
         });
         j.EndObject();
+        j.startArray("resultChanged");
+        db->stmt("SELECT b.name,MAX(b.number) as lastSuccess,lastFailure FROM builds AS b JOIN (SELECT name,MAX(number) AS lastFailure FROM builds WHERE result<>? GROUP BY name) AS t ON t.name=b.name WHERE b.result=? GROUP BY b.name ORDER BY lastSuccess>lastFailure, lastFailure-lastSuccess DESC LIMIT 8")
+                .bind(int(RunState::SUCCESS), int(RunState::SUCCESS))
+                .fetch<str, uint, uint>([&](str job, uint lastSuccess, uint lastFailure){
+            j.StartObject();
+            j.set("name", job)
+             .set("lastSuccess", lastSuccess)
+             .set("lastFailure", lastFailure);
+            j.EndObject();
+        });
+        j.EndArray();
+        j.startArray("lowPassRates");
+        db->stmt("SELECT name,CAST(SUM(result==?) AS FLOAT)/COUNT(*) AS passRate FROM builds GROUP BY name ORDER BY passRate ASC LIMIT 8")
+                .bind(int(RunState::SUCCESS))
+                .fetch<str, double>([&](str job, double passRate){
+            j.StartObject();
+            j.set("name", job).set("passRate", passRate);
+            j.EndObject();
+        });
+        j.EndArray();
+        j.startArray("buildTimeChanges");
+        db->stmt("SELECT name,GROUP_CONCAT(number),GROUP_CONCAT(completedAt-startedAt) FROM builds WHERE number > (SELECT MAX(number)-10 FROM builds b WHERE b.name=builds.name) GROUP BY name ORDER BY (MAX(completedAt-startedAt)-MIN(completedAt-startedAt))-STDEV(completedAt-startedAt) DESC LIMIT 8")
+                .fetch<str,str,str>([&](str name, str numbers, str durations){
+            j.StartObject();
+            j.set("name", name);
+            j.startArray("numbers");
+            j.RawValue(numbers.data(), numbers.length(), rapidjson::Type::kArrayType);
+            j.EndArray();
+            j.startArray("durations");
+            j.RawValue(durations.data(), durations.length(), rapidjson::Type::kArrayType);
+            j.EndArray();
+            j.EndObject();
+        });
+        j.EndArray();
+
+        j.startArray("buildTimeDist");
+        db->stmt("WITH ba AS (SELECT name,AVG(completedAt-startedAt) a FROM builds GROUP BY name) SELECT "
+                 "COUNT(CASE WHEN               a <    30 THEN 1 END),"
+                 "COUNT(CASE WHEN a >=   30 AND a <    60 THEN 1 END),"
+                 "COUNT(CASE WHEN a >=   60 AND a <   300 THEN 1 END),"
+                 "COUNT(CASE WHEN a >=  300 AND a <   600 THEN 1 END),"
+                 "COUNT(CASE WHEN a >=  600 AND a <  1200 THEN 1 END),"
+                 "COUNT(CASE WHEN a >= 1200 AND a <  2400 THEN 1 END),"
+                 "COUNT(CASE WHEN a >= 2400 AND a <  3600 THEN 1 END),"
+                 "COUNT(CASE WHEN a >= 3600               THEN 1 END) FROM ba")
+                .fetch<uint,uint,uint,uint,uint,uint,uint,uint>([&](uint c1, uint c2, uint c3, uint c4, uint c5, uint c6, uint c7, uint c8){
+            j.Int(c1);
+            j.Int(c2);
+            j.Int(c3);
+            j.Int(c4);
+            j.Int(c5);
+            j.Int(c6);
+            j.Int(c7);
+            j.Int(c8);
+        });
+        j.EndArray();
 
     }
     j.EndObject();
     client->sendMessage(j.str());
-
 }
 
-Laminar::~Laminar() {
+Laminar::~Laminar() noexcept try {
     delete db;
     delete srv;
+} catch (std::exception& e) {
+    LLOG(ERROR, e.what());
+    return;
 }
 
 void Laminar::run() {
@@ -362,8 +528,8 @@ void Laminar::run() {
     const char* listen_http = getenv("LAMINAR_BIND_HTTP") ?: INTADDR_HTTP_DEFAULT;
 
     srv = new Server(*this, listen_rpc, listen_http);
-    srv->addWatchPath(fs::path(fs::path(homeDir)/"cfg"/"nodes").string().c_str());
-    srv->addWatchPath(fs::path(fs::path(homeDir)/"cfg"/"jobs").string().c_str());
+    srv->addWatchPath((homePath/"cfg"/"nodes").toString(true).cStr());
+    srv->addWatchPath((homePath/"cfg"/"jobs").toString(true).cStr());
     srv->start();
 }
 
@@ -377,30 +543,28 @@ bool Laminar::loadConfiguration() {
 
     std::set<std::string> knownNodes;
 
-    fs::path nodeCfg = fs::path(homeDir)/"cfg"/"nodes";
-
-    if(fs::is_directory(nodeCfg)) {
-        for(fs::directory_iterator it(nodeCfg); it != fs::directory_iterator(); ++it) {
-            if(!fs::is_regular_file(it->status()) || it->path().extension() != ".conf")
+    KJ_IF_MAYBE(nodeDir, fsHome->tryOpenSubdir(kj::Path{"cfg","nodes"})) {
+        for(kj::Directory::Entry& entry : (*nodeDir)->listEntries()) {
+            if(!entry.name.endsWith(".conf"))
                 continue;
 
-            StringMap conf = parseConfFile(it->path().string().c_str());
+            StringMap conf = parseConfFile((homePath/"cfg"/"nodes"/entry.name).toString(true).cStr());
 
-            std::string nodeName = it->path().stem().string();
+            std::string nodeName(entry.name.cStr(), entry.name.findLast('.').orDefault(0));
             auto existingNode = nodes.find(nodeName);
             std::shared_ptr<Node> node = existingNode == nodes.end() ? nodes.emplace(nodeName, std::shared_ptr<Node>(new Node)).first->second : existingNode->second;
             node->name = nodeName;
             node->numExecutors = conf.get<int>("EXECUTORS", 6);
 
-            std::string tags = conf.get<std::string>("TAGS");
-            if(!tags.empty()) {
-                std::istringstream iss(tags);
-                std::set<std::string> tagList;
+            std::string tagString = conf.get<std::string>("TAGS");
+            std::set<std::string> tagList;
+            if(!tagString.empty()) {
+                std::istringstream iss(tagString);
                 std::string tag;
                 while(std::getline(iss, tag, ','))
                     tagList.insert(tag);
-                node->tags = tagList;
             }
+            std::swap(node->tags, tagList);
 
             knownNodes.insert(nodeName);
         }
@@ -417,19 +581,20 @@ bool Laminar::loadConfiguration() {
 
     // add a default node
     if(nodes.empty()) {
+        LLOG(INFO, "Creating a default node with 6 executors");
         std::shared_ptr<Node> node(new Node);
         node->name = "";
         node->numExecutors = 6;
         nodes.emplace("", node);
     }
 
-    fs::path jobsDir = fs::path(homeDir)/"cfg"/"jobs";
-    if(fs::is_directory(jobsDir)) {
-        for(fs::directory_iterator it(jobsDir); it != fs::directory_iterator(); ++it) {
-            if(!fs::is_regular_file(it->status()) || it->path().extension() != ".conf")
+    KJ_IF_MAYBE(jobsDir, fsHome->tryOpenSubdir(kj::Path{"cfg","jobs"})) {
+        for(kj::Directory::Entry& entry : (*jobsDir)->listEntries()) {
+            if(!entry.name.endsWith(".conf"))
                 continue;
+            StringMap conf = parseConfFile((homePath/"cfg"/"jobs"/entry.name).toString(true).cStr());
 
-            StringMap conf = parseConfFile(it->path().string().c_str());
+            std::string jobName(entry.name.cStr(), entry.name.findLast('.').orDefault(0));
 
             std::string tags = conf.get<std::string>("TAGS");
             if(!tags.empty()) {
@@ -438,7 +603,7 @@ bool Laminar::loadConfiguration() {
                 std::string tag;
                 while(std::getline(iss, tag, ','))
                     tagList.insert(tag);
-                jobTags[it->path().stem().string()] = tagList;
+                jobTags[jobName] = tagList;
             }
 
         }
@@ -448,30 +613,12 @@ bool Laminar::loadConfiguration() {
 }
 
 std::shared_ptr<Run> Laminar::queueJob(std::string name, ParamMap params) {
-    if(!fs::exists(fs::path(homeDir)/"cfg"/"jobs"/name+".run")) {
+    if(!fsHome->exists(kj::Path{"cfg","jobs",name+".run"})) {
         LLOG(ERROR, "Non-existent job", name);
         return nullptr;
     }
 
-    std::shared_ptr<Run> run = std::make_shared<Run>();
-    run->name = name;
-    run->queuedAt = time(nullptr);
-    for(auto it = params.begin(); it != params.end();) {
-        if(it->first[0] == '=') {
-            if(it->first == "=parentJob") {
-                run->parentName = it->second;
-            } else if(it->first == "=parentBuild") {
-                run->parentBuild = atoi(it->second.c_str());
-            } else if(it->first == "=reason") {
-                run->reasonMsg = it->second;
-            } else {
-                LLOG(ERROR, "Unknown internal job parameter", it->first);
-            }
-            it = params.erase(it);
-        } else
-            ++it;
-    }
-    run->params = params;
+    std::shared_ptr<Run> run = std::make_shared<Run>(name, kj::mv(params), homePath.clone());
     queuedJobs.push_back(run);
 
     // notify clients
@@ -490,69 +637,29 @@ std::shared_ptr<Run> Laminar::queueJob(std::string name, ParamMap params) {
     return run;
 }
 
-bool Laminar::stepRun(std::shared_ptr<Run> run) {
-    bool complete = run->step();
-    if(!complete) {
-        srv->addDescriptor(run->fd, [this, run](const char* b,size_t n){
-            handleRunLog(run, std::string(b,n));
-        });
-    }
-    return complete;
-}
-
-void Laminar::handleRunLog(std::shared_ptr<Run> run, std::string s) {
-    run->log += s;
-    for(LaminarClient* c : clients) {
-        if(c->scope.wantsLog(run->name, run->build))
-            c->sendMessage(s);
-    }
-}
-
-// Reaps a zombie and steps the corresponding Run to its next state.
-// Should be called on SIGCHLD
-void Laminar::reapChildren() {
-    int ret = 0;
-    pid_t pid;
-    constexpr int bufsz = 1024;
-    static thread_local char buf[bufsz];
-    while((pid = waitpid(-1, &ret, WNOHANG)) > 0) {
-        LLOG(INFO, "Reaping", pid);
-        auto it = activeJobs.byPid().find(pid);
-        std::shared_ptr<Run> run = *it;
-        // The main event loop might schedule this SIGCHLD handler before the final
-        // output handler (from addDescriptor). In that case, because it keeps a
-        // shared_ptr to the run it would successfully add to the log output buffer,
-        // but by then reapAdvance would have stored the log and ensured no-one cares.
-        // Preempt this case by forcing a final (non-blocking) read here.
-        for(ssize_t n = read(run->fd, buf, bufsz); n > 0; n = read(run->fd, buf, bufsz)) {
-            handleRunLog(run, std::string(buf, static_cast<size_t>(n)));
-        }
-        bool completed = true;
-        activeJobs.byPid().modify(it, [&](std::shared_ptr<Run> run){
-            run->reaped(ret);
-            completed = stepRun(run);
-        });
-        if(completed)
-            run->complete();
-    }
-
-    assignNewJobs();
-}
-
 void Laminar::notifyConfigChanged()
 {
+    LLOG(INFO, "Reloading configuration");
     loadConfiguration();
     // config change may allow stuck jobs to dequeue
     assignNewJobs();
 }
 
+bool Laminar::abort(std::string job, uint buildNum) {
+    if(Run* run = activeRun(job, buildNum)) {
+        run->abort(true);
+        return true;
+    }
+    return false;
+}
+
 void Laminar::abortAll() {
     for(std::shared_ptr<Run> run : activeJobs) {
-        run->abort();
+        run->abort(false);
     }
 }
 
-bool Laminar::nodeCanQueue(const Node& node, const Run& run) const {
+bool Laminar::nodeCanQueue(const Node& node, std::string jobName) const {
     // if a node is too busy, it can't take the job
     if(node.busyExecutors >= node.numExecutors)
         return false;
@@ -561,7 +668,7 @@ bool Laminar::nodeCanQueue(const Node& node, const Run& run) const {
     if(node.tags.size() == 0)
         return true;
 
-    auto it = jobTags.find(run.name);
+    auto it = jobTags.find(jobName);
     // if the job has no tags, it cannot be run on this node
     if(it == jobTags.end())
         return false;
@@ -575,169 +682,110 @@ bool Laminar::nodeCanQueue(const Node& node, const Run& run) const {
     return false;
 }
 
+bool Laminar::tryStartRun(std::shared_ptr<Run> run, int queueIndex) {
+    for(auto& sn : nodes) {
+        std::shared_ptr<Node> node = sn.second;
+
+        if(nodeCanQueue(*node.get(), run->name) && run->configure(buildNums[run->name] + 1, node, *fsHome)) {
+            node->busyExecutors++;
+            // set the last known result if exists
+            db->stmt("SELECT result FROM builds WHERE name = ? ORDER BY completedAt DESC LIMIT 1")
+             .bind(run->name)
+             .fetch<int>([=](int result){
+                run->lastResult = RunState(result);
+            });
+
+            // Actually schedules the Run steps
+            kj::Promise<void> exec = handleRunStep(run.get()).then([=]{
+                runFinished(run.get());
+            });
+            if(run->timeout > 0) {
+                exec = exec.attach(srv->addTimeout(run->timeout, [r=run.get()](){
+                    r->abort(true);
+                }));
+            }
+            srv->addTask(kj::mv(exec));
+            LLOG(INFO, "Started job on node", run->name, run->build, node->name);
+
+            // update next build number
+            buildNums[run->name]++;
+
+            // notify clients
+            Json j;
+            j.set("type", "job_started")
+             .startObject("data")
+             .set("queueIndex", queueIndex)
+             .set("name", run->name)
+             .set("queued", run->startedAt - run->queuedAt)
+             .set("started", run->startedAt)
+             .set("number", run->build)
+             .set("reason", run->reason());
+            db->stmt("SELECT completedAt - startedAt FROM builds WHERE name = ? ORDER BY completedAt DESC LIMIT 1")
+             .bind(run->name)
+             .fetch<uint>([&](uint etc){
+                j.set("etc", time(nullptr) + etc);
+            });
+            j.startArray("tags");
+            for(const str& t: jobTags[run->name]) {
+                j.String(t.c_str());
+            }
+            j.EndArray();
+            j.EndObject();
+            const char* msg = j.str();
+            for(LaminarClient* c : clients) {
+                if(c->scope.wantsStatus(run->name, run->build)
+                    // The run page also should know that another job has started
+                    // (so maybe it can show a previously hidden "next" button).
+                    // Hence this small hack:
+                        || (c->scope.type == MonitorScope::Type::RUN && c->scope.job == run->name))
+                    c->sendMessage(msg);
+            }
+
+            return true;
+        }
+    }
+    return false;
+}
+
 void Laminar::assignNewJobs() {
     auto it = queuedJobs.begin();
     while(it != queuedJobs.end()) {
-        bool assigned = false;
-        for(auto& sn : nodes) {
-            std::shared_ptr<Node> node = sn.second;
-            std::shared_ptr<Run> run = *it;
-            if(nodeCanQueue(*node.get(), *run)) {
-                fs::path cfgDir = fs::path(homeDir)/"cfg";
-                boost::system::error_code err;
-
-                // create a workspace for this job if it doesn't exist
-                fs::path ws = fs::path(homeDir)/"run"/run->name/"workspace";
-                if(!fs::exists(ws)) {
-                    if(!fs::create_directories(ws, err)) {
-                        LLOG(ERROR, "Could not create job workspace", run->name);
-                        break;
-                    }
-                    // prepend the workspace init script
-                    if(fs::exists(cfgDir/"jobs"/run->name+".init"))
-                        run->addScript((cfgDir/"jobs"/run->name+".init").string(), ws.string());
-                }
-
-                uint buildNum = buildNums[run->name] + 1;
-                // create the run directory
-                fs::path rd = fs::path(homeDir)/"run"/run->name/std::to_string(buildNum);
-                bool createWorkdir = true;
-                if(fs::is_directory(rd)) {
-                    LLOG(WARNING, "Working directory already exists, removing", rd.string());
-                    fs::remove_all(rd, err);
-                    if(err) {
-                        LLOG(WARNING, "Failed to remove working directory", err.message());
-                        createWorkdir = false;
-                    }
-                }
-                if(createWorkdir && !fs::create_directory(rd, err)) {
-                    LLOG(ERROR, "Could not create working directory", rd.string());
-                    break;
-                }
-                run->runDir = rd.string();
-
-                // create an archive directory
-                fs::path archive = fs::path(homeDir)/"archive"/run->name/std::to_string(buildNum);
-                if(fs::is_directory(archive)) {
-                    LLOG(WARNING, "Archive directory already exists", archive.string());
-                } else if(!fs::create_directories(archive)) {
-                    LLOG(ERROR, "Could not create archive directory", archive.string());
-                    break;
-                }
-
-                // add scripts
-                // global before-run script
-                if(fs::exists(cfgDir/"before"))
-                    run->addScript((cfgDir/"before").string());
-                // per-node before-run script
-                if(fs::exists(cfgDir/"nodes"/node->name+".before"))
-                    run->addScript((cfgDir/"nodes"/node->name+".before").string());
-                // job before-run script
-                if(fs::exists(cfgDir/"jobs"/run->name+".before"))
-                    run->addScript((cfgDir/"jobs"/run->name+".before").string());
-                // main run script. must exist.
-                run->addScript((cfgDir/"jobs"/run->name+".run").string());
-                // job after-run script
-                if(fs::exists(cfgDir/"jobs"/run->name+".after"))
-                    run->addScript((cfgDir/"jobs"/run->name+".after").string());
-                // per-node after-run script
-                if(fs::exists(cfgDir/"nodes"/node->name+".after"))
-                    run->addScript((cfgDir/"nodes"/node->name+".after").string());
-                // global after-run script
-                if(fs::exists(cfgDir/"after"))
-                    run->addScript((cfgDir/"after").string());
-
-                // add environment files
-                if(fs::exists(cfgDir/"env"))
-                    run->addEnv((cfgDir/"env").string());
-                if(fs::exists(cfgDir/"nodes"/node->name+".env"))
-                    run->addEnv((cfgDir/"nodes"/node->name+".env").string());
-                if(fs::exists(cfgDir/"jobs"/run->name+".env"))
-                    run->addEnv((cfgDir/"jobs"/run->name+".env").string());
-
-                // add job timeout if specified
-                if(fs::exists(cfgDir/"jobs"/run->name+".conf")) {
-                    int timeout = parseConfFile(fs::path(cfgDir/"jobs"/run->name+".conf").string().c_str()).get<int>("TIMEOUT", 0);
-                    if(timeout > 0) {
-                        // A raw pointer to run is used here so as not to have a circular reference.
-                        // The captured raw pointer is safe because if the Run is destroyed the Promise
-                        // will be cancelled and the callback never called.
-                        Run* r = run.get();
-                        r->timeout = srv->addTimeout(timeout, [r](){
-                            r->abort();
-                        });
-                    }
-                }
-
-                // start the job
-                node->busyExecutors++;
-                run->node = node;
-                run->startedAt = time(nullptr);
-                run->laminarHome = homeDir;
-                run->build = buildNum;
-                // set the last known result if exists
-                db->stmt("SELECT result FROM builds WHERE name = ? ORDER BY completedAt DESC LIMIT 1")
-                 .bind(run->name)
-                 .fetch<int>([=](int result){
-                    run->lastResult = RunState(result);
-                });
-                // update next build number
-                buildNums[run->name] = buildNum;
-
-                LLOG(INFO, "Queued job to node", run->name, run->build, node->name);
-
-                // notify clients
-                Json j;
-                j.set("type", "job_started")
-                 .startObject("data")
-                 .set("queueIndex", std::distance(it,queuedJobs.begin()))
-                 .set("name", run->name)
-                 .set("queued", run->startedAt - run->queuedAt)
-                 .set("started", run->startedAt)
-                 .set("number", run->build)
-                 .set("reason", run->reason());
-                db->stmt("SELECT completedAt - startedAt FROM builds WHERE name = ? ORDER BY completedAt DESC LIMIT 1")
-                 .bind(run->name)
-                 .fetch<uint>([&](uint etc){
-                    j.set("etc", time(nullptr) + etc);
-                });
-                j.EndObject();
-                const char* msg = j.str();
-                for(LaminarClient* c : clients) {
-                    if(c->scope.wantsStatus(run->name, run->build)
-                        // The run page also should know that another job has started
-                        // (so maybe it can show a previously hidden "next" button).
-                        // Hence this small hack:
-                            || (c->scope.type == MonitorScope::Type::RUN && c->scope.job == run->name))
-                        c->sendMessage(msg);
-                }
-
-                // notify the rpc client if the start command was used
-                run->started.fulfiller->fulfill();
-
-                // setup run completion handler
-                run->notifyCompletion = [this](Run* r) { runFinished(r); };
-
-                // trigger the first step of the run
-                if(stepRun(run)) {
-                    // should never happen
-                    LLOG(INFO, "No steps for run");
-                    run->complete();
-                }
-
-                assigned = true;
-                break;
-            }
-        }
-        if(assigned) {
+        if(tryStartRun(*it, std::distance(it, queuedJobs.begin()))) {
             activeJobs.insert(*it);
             it = queuedJobs.erase(it);
-        } else
+        } else {
             ++it;
-
+        }
     }
 }
 
+kj::Promise<void> Laminar::handleRunStep(Run* run) {
+    if(run->step()) {
+        // no more steps
+        return kj::READY_NOW;
+    }
+
+    kj::Promise<int> exited = srv->onChildExit(run->current_pid);
+    // promise is fulfilled when the process is reaped. But first we wait for all
+    // output from the pipe (Run::output_fd) to be consumed.
+    return srv->readDescriptor(run->output_fd, [this,run](const char*b,size_t n){
+        // handle log output
+        std::string s(b, n);
+        run->log += s;
+        for(LaminarClient* c : clients) {
+            if(c->scope.wantsLog(run->name, run->build))
+                c->sendMessage(s);
+        }
+    }).then([p = std::move(exited)]() mutable {
+        // wait until the process is reaped
+        return kj::mv(p);
+    }).then([this, run](int status){
+        run->reaped(status);
+        // next step in Run
+        return handleRunStep(run);
+    });
+}
+
 void Laminar::runFinished(Run * r) {
     std::shared_ptr<Node> node = r->node;
 
@@ -775,6 +823,11 @@ void Laminar::runFinished(Run * r) {
             .set("started", r->startedAt)
             .set("result", to_string(r->result))
             .set("reason", r->reason());
+    j.startArray("tags");
+    for(const str& t: jobTags[r->name]) {
+        j.String(t.c_str());
+    }
+    j.EndArray();
     j.startArray("artifacts");
     populateArtifacts(j, r->name, r->build);
     j.EndArray();
@@ -783,6 +836,7 @@ void Laminar::runFinished(Run * r) {
     for(LaminarClient* c : clients) {
         if(c->scope.wantsStatus(r->name, r->build))
             c->sendMessage(msg);
+        c->notifyJobFinished();
     }
 
     // notify the waiters
@@ -790,7 +844,9 @@ void Laminar::runFinished(Run * r) {
         w->complete(r);
     }
 
-    // erase reference to run from activeJobs
+    // erase reference to run from activeJobs. Since runFinished is called in a
+    // lambda whose context contains a shared_ptr<Run>, the run won't be deleted
+    // until the context is destroyed at the end of the lambda execution.
     activeJobs.byRunPtr().erase(r);
 
     // remove old run directories
@@ -804,48 +860,71 @@ void Laminar::runFinished(Run * r) {
     auto it = activeJobs.byJobName().equal_range(r->name);
     uint oldestActive = (it.first == it.second)? buildNums[r->name] : (*it.first)->build - 1;
     for(int i = static_cast<int>(oldestActive - numKeepRunDirs); i > 0; i--) {
-        fs::path d = fs::path(homeDir)/"run"/r->name/std::to_string(i);
+        kj::Path d{"run",r->name,std::to_string(i)};
         // Once the directory does not exist, it's probably not worth checking
         // any further. 99% of the time this loop should only ever have 1 iteration
         // anyway so hence this (admittedly debatable) optimization.
-        if(!fs::exists(d))
+        if(!fsHome->exists(d))
             break;
-        fs::remove_all(d);
+        fsHome->remove(d);
     }
+
+    fsHome->symlink(kj::Path{"archive", r->name, "latest"}, std::to_string(r->build), kj::WriteMode::CREATE|kj::WriteMode::MODIFY);
+
+    // in case we freed up an executor, check the queue
+    assignNewJobs();
 }
 
-// Small helper function to return the full contents of a file given its path.
-// It reads in the whole file into the given string reference.
-// This is a terrible way to serve files (especially large ones). Hopefully
-// no-one uses this function and configures their webservers appropriately.
-static bool slurp(fs::path path, std::string& output) {
-    if(!fs::is_regular_file(path))
+kj::Maybe<kj::Own<const kj::ReadableFile>> Laminar::getArtefact(std::string path) {
+    return fsHome->openFile(kj::Path("archive").append(kj::Path::parse(path)));
+}
+
+bool Laminar::handleBadgeRequest(std::string job, std::string &badge) {
+    RunState rs = RunState::UNKNOWN;
+    db->stmt("SELECT result FROM builds WHERE name = ? ORDER BY number DESC LIMIT 1")
+            .bind(job)
+            .fetch<int>([&](int result){
+        rs = (RunState) result;
+    });
+    if(rs == RunState::UNKNOWN)
         return false;
-    std::ifstream fstr(path.string());
-    fstr.seekg(0, std::ios::end);
-    ssize_t sz = fstr.tellg();
-    if(fstr.good()) {
-        output.resize(static_cast<size_t>(sz));
-        fstr.seekg(0);
-        fstr.read(&output[0], sz);
-        return true;
-    }
-    return false;
+
+    std::string status = to_string(rs);
+    // Empirical approximation of pixel width. Not particularly stable.
+    const int jobNameWidth = job.size() * 7 + 10;
+    const int statusWidth = status.size() * 7 + 10;
+    const char* gradient1 = (rs == RunState::SUCCESS) ? "#2aff4d" : "#ff2a2a";
+    const char* gradient2 = (rs == RunState::SUCCESS) ? "#24b43c" : "#b42424";
+    char* svg = NULL;
+    asprintf(&svg,
+R"x(
+<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="20">
+  <clipPath id="clip">
+    <rect width="%d" height="20" rx="4"/>
+  </clipPath>
+  <linearGradient id="job" x1="0" x2="0" y1="0" y2="1">
+    <stop offset="0" stop-color="#666" />
+    <stop offset="1" stop-color="#333" />
+  </linearGradient>
+  <linearGradient id="status" x1="0" x2="0" y1="0" y2="1">
+    <stop offset="0" stop-color="%s" />
+    <stop offset="1" stop-color="%s" />
+  </linearGradient>
+  <g clip-path="url(#clip)" font-family="DejaVu Sans,Verdana,sans-serif" font-size="12" text-anchor="middle">
+    <rect width="%d" height="20" fill="url(#job)"/>
+    <text x="%d" y="14" fill="#fff">%s</text>
+    <rect x="%d" width="%d" height="20" fill="url(#status)"/>
+    <text x="%d" y="14" fill="#000">%s</text>
+  </g>
+</svg>)x", jobNameWidth+statusWidth, jobNameWidth+statusWidth, gradient1, gradient2, jobNameWidth, jobNameWidth/2+1, job.data(), jobNameWidth, statusWidth, jobNameWidth+statusWidth/2, status.data());
+    badge = svg;
+    return true;
 }
 
-bool Laminar::getArtefact(std::string path, std::string& result) {
-    if(archiveUrl != ARCHIVE_URL_DEFAULT) {
-        // we shouldn't have got here. Probably an invalid link.
-        return false;
+std::string Laminar::getCustomCss() {
+    KJ_IF_MAYBE(cssFile, fsHome->tryOpenFile(kj::Path{"custom","style.css"})) {
+        return (*cssFile)->readAllText().cStr();
+    } else {
+        return std::string();
     }
-    fs::path file(fs::path(homeDir)/"archive"/path);
-    return slurp(file, result);
-}
-
-std::string Laminar::getCustomCss()
-{
-    fs::path file(fs::path(homeDir)/"custom"/"style.css");
-    std::string result;
-    slurp(file, result);
-    return result;
 }

src/laminar.h

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015-2017 Oliver Giles
+/// Copyright 2015-2019 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -25,6 +25,7 @@
 #include "database.h"
 
 #include <unordered_map>
+#include <kj/filesystem.h>
 
 // Node name to node object map
 typedef std::unordered_map<std::string, std::shared_ptr<Node>> NodeMap;
@@ -38,8 +39,8 @@ class Json;
 // the LaminarClient objects (see interface.h)
 class Laminar final : public LaminarInterface {
 public:
-    Laminar();
-    ~Laminar() override;
+    Laminar(const char* homePath);
+    ~Laminar() noexcept override;
 
     // Runs the application forever
     void run();
@@ -52,28 +53,34 @@ public:
     void deregisterClient(LaminarClient* client) override;
     void registerWaiter(LaminarWaiter* waiter) override;
     void deregisterWaiter(LaminarWaiter* waiter) override;
+    uint latestRun(std::string job) override;
+    bool handleLogRequest(std::string name, uint num, std::string& output, bool& complete) override;
 
     void sendStatus(LaminarClient* client) override;
     bool setParam(std::string job, uint buildNum, std::string param, std::string value) override;
-    bool getArtefact(std::string path, std::string& result) override;
+    const std::list<std::shared_ptr<Run>>& listQueuedJobs() override;
+    const RunSet& listRunningJobs() override;
+    std::list<std::string> listKnownJobs() override;
+    kj::Maybe<kj::Own<const kj::ReadableFile>> getArtefact(std::string path) override;
+    bool handleBadgeRequest(std::string job, std::string& badge) override;
     std::string getCustomCss() override;
+    bool abort(std::string job, uint buildNum) override;
     void abortAll() override;
-    void reapChildren() override;
     void notifyConfigChanged() override;
 
 private:
     bool loadConfiguration();
     void assignNewJobs();
-    bool stepRun(std::shared_ptr<Run> run);
-    void handleRunLog(std::shared_ptr<Run> run, std::string log);
+    bool tryStartRun(std::shared_ptr<Run> run, int queueIndex);
+    kj::Promise<void> handleRunStep(Run *run);
     void runFinished(Run*);
-    bool nodeCanQueue(const Node&, const Run&) const;
+    bool nodeCanQueue(const Node&, std::string jobName) const;
     // expects that Json has started an array
     void populateArtifacts(Json& out, std::string job, uint num) const;
 
-    Run* activeRun(std::string name, uint num) {
-        auto it = activeJobs.byRun().find(boost::make_tuple(name, num));
-        return it == activeJobs.byRun().end() ? nullptr : it->get();
+    Run* activeRun(const std::string name, uint num) {
+        auto it = activeJobs.byNameNumber().find(boost::make_tuple(name, num));
+        return it == activeJobs.byNameNumber().end() ? nullptr : it->get();
     }
 
     std::list<std::shared_ptr<Run>> queuedJobs;
@@ -86,7 +93,8 @@ private:
     Database* db;
     Server* srv;
     NodeMap nodes;
-    std::string homeDir;
+    kj::Path homePath;
+    kj::Own<const kj::Directory> fsHome;
     std::set<LaminarClient*> clients;
     std::set<LaminarWaiter*> waiters;
     uint numKeepRunDirs;

src/log.h

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015 Oliver Giles
+/// Copyright 2015-2019 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -20,18 +20,51 @@
 #define LAMINAR_LOG_H_
 
 #include <kj/debug.h>
+#include <utility>
 
-// Simple override to prevent full paths to source files from
-// appearing in log messages. Assumes / is the path separator.
-// @see kj/debug.h
+// Prevent full file paths from appearing in log messages. With compiler
+// optimization, this compile-time method should completely prevent the
+// paths from being encoded into the binary at all. Assumes / is the
+// path separator.
+namespace _ {
+	constexpr const char* static_basename_impl(const char* b, const char* t) {
+		return *t == '\0' ? b : static_basename_impl(*t == '/' ? t+1 : b, t+1);
+	}
+	constexpr const char* static_basename(const char* p) {
+		return static_basename_impl(p, p); 
+	}
+	constexpr int static_strlen(const char* s) {
+		return *s == '\0' ? 0 : static_strlen(s + 1) + 1;
+	}
+	template<int N, int...I>
+	static constexpr decltype(auto) static_alloc_str_impl(const char* str, std::integer_sequence<int, I...>) { 
+		return (struct {char buf[N+1];}) {str[I]..., 0};
+	}
+	template<int N>
+	static constexpr decltype(auto) static_alloc_str(const char* str) {
+		return static_alloc_str_impl<N>(str, std::make_integer_sequence<int, N>());
+	}
+}
+#define __FILE_BASE__ (_::static_alloc_str<_::static_strlen(_::static_basename(__FILE__))>(_::static_basename(__FILE__)).buf)
 
-#define _LBASENAME(path) strrchr(path, '/') ? strrchr(path, '/') + 1 : path
 
+// Provide alternative implementations to those from kj/debug.h which
+// use __FILE__ directly and thus cause the full path to be encoded in
+// the final binary
 #define LLOG(severity, ...) \
   if (!::kj::_::Debug::shouldLog(::kj::_::Debug::Severity::severity)) {} else \
-    ::kj::_::Debug::log(_LBASENAME(__FILE__), __LINE__, ::kj::_::Debug::Severity::severity, \
-                        #__VA_ARGS__, __VA_ARGS__)
+    ::kj::_::Debug::log(__FILE_BASE__, __LINE__, \
+      ::kj::_::Debug::Severity::severity, #__VA_ARGS__, __VA_ARGS__)
 
+#define LASSERT(cond, ...) \
+  if (KJ_LIKELY(cond)) {} else \
+    for (::kj::_::Debug::Fault f(__FILE_BASE__, __LINE__, \
+      ::kj::Exception::Type::FAILED, #cond, #__VA_ARGS__, ##__VA_ARGS__);; f.fatal())
+
+#define LSYSCALL(call, ...) \
+  if (auto _kjSyscallResult = ::kj::_::Debug::syscall([&](){return (call);}, false)) {} else \
+    for (::kj::_::Debug::Fault f(__FILE_BASE__, __LINE__, \
+      _kjSyscallResult.getErrorNumber(), #call, #__VA_ARGS__, ##__VA_ARGS__);; f.fatal())
 
 #endif // LAMINAR_LOG_H_
 

src/main.cpp

@@ -18,8 +18,9 @@
 ///
 #include "laminar.h"
 #include "log.h"
-
 #include <signal.h>
+#include <kj/async-unix.h>
+#include <kj/filesystem.h>
 
 static Laminar* laminar;
 
@@ -34,10 +35,12 @@ int main(int argc, char** argv) {
         }
     }
 
-    laminar = new Laminar;
+    laminar = new Laminar(getenv("LAMINAR_HOME") ?: "/var/lib/laminar");
+    kj::UnixEventPort::captureChildExit();
 
     signal(SIGINT, &laminar_quit);
     signal(SIGTERM, &laminar_quit);
+
     laminar->run();
 
     delete laminar;

src/resources.cpp

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015-2017 Oliver Giles
+/// Copyright 2015-2019 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -17,7 +17,10 @@
 /// along with Laminar.  If not, see <http://www.gnu.org/licenses/>
 ///
 #include "resources.h"
+#include "log.h"
+#include "index_html_size.h"
 #include <string.h>
+#include <zlib.h>
 
 #define INIT_RESOURCE(route, name, content_type) \
     extern const char _binary_##name##_z_start[];\
@@ -30,6 +33,8 @@
 #define CONTENT_TYPE_JS   "application/javascript; charset=utf-8"
 #define CONTENT_TYPE_CSS  "text/css; charset=utf-8"
 
+#define GZIP_FORMAT 16
+
 Resources::Resources()
 {
     INIT_RESOURCE("/", index_html, CONTENT_TYPE_HTML);
@@ -37,14 +42,52 @@ Resources::Resources()
     INIT_RESOURCE("/favicon-152.png", favicon_152_png, CONTENT_TYPE_PNG);
     INIT_RESOURCE("/icon.png", icon_png, CONTENT_TYPE_PNG);
     INIT_RESOURCE("/js/app.js", js_app_js, CONTENT_TYPE_JS);
-    INIT_RESOURCE("/js/Chart.HorizontalBar.js", js_Chart_HorizontalBar_js, CONTENT_TYPE_JS);
     INIT_RESOURCE("/js/ansi_up.js", js_ansi_up_js, CONTENT_TYPE_JS);
     INIT_RESOURCE("/js/vue.min.js", js_vue_min_js, CONTENT_TYPE_JS);
     INIT_RESOURCE("/js/vue-router.min.js", js_vue_router_min_js, CONTENT_TYPE_JS);
     INIT_RESOURCE("/js/ansi_up.js", js_ansi_up_js, CONTENT_TYPE_JS);
     INIT_RESOURCE("/js/Chart.min.js", js_Chart_min_js, CONTENT_TYPE_JS);
-    INIT_RESOURCE("/js/Chart.HorizontalBar.js", js_Chart_HorizontalBar_js, CONTENT_TYPE_JS);
     INIT_RESOURCE("/css/bootstrap.min.css", css_bootstrap_min_css, CONTENT_TYPE_CSS);
+
+    if(const char* baseUrl = getenv("LAMINAR_BASE_URL")) {
+        // The administrator needs to customize the <base href>. Unfortunately this seems
+        // to be the only thing that needs to be customizable but cannot be done via dynamic
+        // DOM manipulation without heavy compromises. So replace the static char array with
+        // a modified buffer accordingly.
+        z_stream strm;
+        memset(&strm, 0, sizeof(z_stream));
+        std::string tmp;
+        tmp.resize(INDEX_HTML_UNCOMPRESSED_SIZE);
+        // inflate
+        inflateInit2(&strm, MAX_WBITS|GZIP_FORMAT);
+        strm.next_in = (unsigned char*) _binary_index_html_z_start;
+        strm.avail_in = _binary_index_html_z_end - _binary_index_html_z_start;
+        strm.next_out = (unsigned char*) tmp.data();
+        strm.avail_out = INDEX_HTML_UNCOMPRESSED_SIZE;
+        if(inflate(&strm, Z_FINISH) != Z_STREAM_END) {
+            LLOG(FATAL, "Failed to uncompress index_html");
+        }
+        // replace
+        // There's no validation on the replacement string, so you can completely mangle
+        // the html if you like. This isn't really an issue because if you can modify laminar's
+        // environment you already have elevated permissions
+        if(auto it = tmp.find("base href=\"/"))
+            tmp.replace(it+11, 1, baseUrl);
+        // deflate
+        index_html.resize(tmp.size());
+        deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS|GZIP_FORMAT, 8, Z_DEFAULT_STRATEGY);
+        strm.next_in = (unsigned char*) tmp.data();
+        strm.avail_in = tmp.size();
+        strm.next_out = (unsigned char*) index_html.data();
+        strm.avail_out = tmp.size();
+        if(deflate(&strm, Z_FINISH) != Z_STREAM_END) {
+            LLOG(FATAL, "Failed to compress index.html");
+        }
+        index_html.resize(strm.total_out);
+        // update resource map
+        resources["/"].start = index_html.data();
+        resources["/"].end = index_html.data() + index_html.size();
+    }
 }
 
 inline bool beginsWith(std::string haystack, const char* needle) {

src/resources.h

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015-2017 Oliver Giles
+/// Copyright 2015-2019 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -40,7 +40,8 @@ private:
         const char* end;
         const char* content_type;
     };
-    std::unordered_map<std::string, const Resource> resources;
+    std::unordered_map<std::string, Resource> resources;
+    std::string index_html;
 };
 
 #endif // LAMINAR_RESOURCES_H_

src/resources/index.html

@@ -6,16 +6,16 @@
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <meta name="apple-mobile-web-app-capable" content="yes" />
- <link rel="apple-touch-icon-precomposed" href="/favicon-152.png">
+ <link rel="apple-touch-icon-precomposed" href="favicon-152.png">
+ <link rel="icon" href="favicon.ico">
  <title>Laminar</title>
- <script src="/js/vue.min.js"></script>
- <script src="/js/vue-router.min.js"></script>
- <script src="/js/ansi_up.js"></script>
- <script src="/js/Chart.min.js"></script>
- <script src="/js/Chart.HorizontalBar.js"></script>
- <link href="/css/bootstrap.min.css" rel="stylesheet">
- <link href="/custom/style.css" rel="stylesheet">
- <script src="/js/app.js" defer></script>
+ <script src="js/vue.min.js"></script>
+ <script src="js/vue-router.min.js"></script>
+ <script src="js/ansi_up.js"></script>
+ <script src="js/Chart.min.js"></script>
+ <link href="css/bootstrap.min.css" rel="stylesheet">
+ <link href="custom/style.css" rel="stylesheet">
+ <script src="js/app.js" defer></script>
  <style>
   body, html { height: 100%; }
   .navbar { margin-bottom: 0; border-radius: 0; }
@@ -48,6 +48,7 @@
     right: 10px;
     padding: 20px;
   }
+  /* status icons */
   span.status {
     display: inline-block;
     width: 1em;
@@ -64,6 +65,43 @@
   @keyframes spin {
     to { transform: rotate(360deg); }
   }
+  /* chart overlay */
+  li.chart-overlay {
+    position: absolute;
+    display: inline-block;
+    background: white;
+    border: 1px solid lightgray;
+    padding: 3px;
+  }
+  /* sort indicators */
+  a.sort {
+    position: relative;
+    margin-left: 7px;
+  }
+  a.sort:before, a.sort:after {
+    border: 4px solid transparent;
+    content: "";
+    position: absolute;
+    display: block;
+    height: 0;
+    width: 0;
+    right: 0;
+    top: 50%;
+  }
+  a.sort:before {
+    border-bottom-color: #ccc;
+    margin-top: -9px;
+  }
+  a.sort:after {
+    border-top-color: #ccc;
+    margin-top: 1px;
+  }
+  a.sort.dsc:after { border-top-color: #000; }
+  a.sort.asc:before { border-bottom-color: #000; }
+  a.sort:hover { text-decoration: none; cursor:pointer; }
+  a.sort:not(.asc):hover:before { border-bottom-color: #777;  }
+  a.sort:not(.dsc):hover:after { border-top-color: #777;  }
+
  </style>
 </head>
 <body>
@@ -91,7 +129,7 @@
    <div class="col-sm-7 col-md-8 col-lg-9"><div class="row">
     <div class="col-md-6">
      <div class="panel panel-default">
-      <div class="panel-heading">Total builds per day this week</div>
+      <div class="panel-heading">Total runs per day this week</div>
       <div class="panel-body">
        <canvas id="chartBpd"></canvas>
       </div>
@@ -99,15 +137,15 @@
     </div>
     <div class="col-md-6">
      <div class="panel panel-default">
-      <div class="panel-heading">Builds per job in the last 24 hours</div>
+      <div class="panel-heading">Most runs per job in the last 24 hours</div>
       <div class="panel-body" id="chartStatus">
        <canvas id="chartBpj"></canvas>
       </div>
      </div>
-    </div>
+    </div></div><div class="row">
     <div class="col-md-6">
      <div class="panel panel-default">
-      <div class="panel-heading">Average build time per job this week</div>
+      <div class="panel-heading">Longest average run time per job this week</div>
       <div class="panel-body">
        <canvas id="chartTpj"></canvas>
       </div>
@@ -120,6 +158,42 @@
        <canvas id="chartUtil"></canvas>
       </div>
      </div>
+    </div></div><div class="row">
+    <div class="col-md-6">
+     <div class="panel panel-default">
+      <div class="panel-heading">Regressions and recoveries</div>
+      <div class="panel-body"><div style="position:relative">
+       <canvas id="chartResultChanges"></canvas>
+       <ul v-for="job in resultChanged">
+         <li v-if="job.lastFailure>job.lastSuccess" :id="'rcd_'+job.name" class="chart-overlay"><router-link :to="'/jobs/'+job.name">{{job.name}}</router-link>: <span v-html="runIcon('failed')"></span> <router-link :to="'/jobs/'+job.name+'/'+job.lastFailure">#{{job.lastFailure}}</router-link> since <span v-html="runIcon('success')"></span> <router-link :to="'/jobs/'+job.name+'/'+job.lastSuccess">#{{job.lastSuccess}}</router-link></li>
+         <li v-if="job.lastFailure<job.lastSuccess" :id="'rcd_'+job.name" class="chart-overlay"><router-link :to="'/jobs/'+job.name">{{job.name}}</router-link>: <span v-html="runIcon('success')"></span> <router-link :to="'/jobs/'+job.name+'/'+job.lastSuccess">#{{job.lastSuccess}}</router-link> since <span v-html="runIcon('failed')"></span> <router-link :to="'/jobs/'+job.name+'/'+job.lastFailure">#{{job.lastFailure}}</router-link></li>
+       </ul>
+      </div></div>
+     </div>
+    </div>
+    <div class="col-md-6">
+     <div class="panel panel-default">
+      <div class="panel-heading">Low pass rates</div>
+      <div class="panel-body">
+       <canvas id="chartPassRates"></canvas>
+      </div>
+     </div>
+    </div></div><div class="row">
+    <div class="col-md-6">
+     <div class="panel panel-default">
+      <div class="panel-heading">Run time changes</div>
+      <div class="panel-body">
+       <canvas id="chartBuildTimeChanges"></canvas>
+      </div>
+     </div>
+    </div>
+    <div class="col-md-6">
+     <div class="panel panel-default">
+      <div class="panel-heading">Average run time distribution</div>
+      <div class="panel-body">
+       <canvas id="chartBuildTimeDist"></canvas>
+      </div>
+     </div>
     </div>
    </div></div>
   </div></div> 
@@ -137,7 +211,7 @@
      <li v-for="t in tags" :class="{'active':t==tag}"><a href v-on:click.prevent="tag = t">{{t}}</a></li>
     </ul>
     <table class="table table-striped" id="joblist">
-     <tr   v-for="job in filteredJobs">
+     <tr   v-for="job in filteredJobs()">
       <td><router-link :to="'/jobs/'+job.name">{{job.name}}</router-link></td>
       <td class="text-center"><span v-html="runIcon(job.result)"></span> <router-link :to="'/jobs/'+job.name+'/'+job.number">#{{job.number}}</router-link></td>
       <td class="text-center">{{formatDate(job.started)}}</td>
@@ -171,24 +245,37 @@
     </div>
    </div>
    <div class="row"><div class="col-xs-12">
-    <table class="table table-striped"><thead>
-     <tr><th>Run</th><th class="text-center">Started</th><th class="text-center">Duration</th><th class="text-center hidden-xs">Reason</th></tr></thead>
+    <table class="table table-striped">
+     <thead><tr>
+      <th><a class="sort" :class="(sort.field=='result'?sort.order:'')" v-on:click="do_sort('result')">&nbsp;</a></th>
+      <th>Run <a class="sort" :class="(sort.field=='number'?sort.order:'')" v-on:click="do_sort('number')">&nbsp;</a></th>
+      <th class="text-center">Started <a class="sort" :class="(sort.field=='started'?sort.order:'')" v-on:click="do_sort('started')">&nbsp;</a></th>
+      <th class="text-center">Duration <a class="sort" :class="(sort.field=='duration'?sort.order:'')" v-on:click="do_sort('duration')">&nbsp;</a></th>
+      <th class="text-center hidden-xs">Reason <a class="sort" :class="(sort.field=='reason'?sort.order:'')" v-on:click="do_sort('reason')">&nbsp;</a></th>
+     </tr></thead>
      <tr v-show="nQueued">
-      <td colspan="4"><i>{{nQueued}} run(s) queued</i></td>
+      <td colspan="5"><i>{{nQueued}} run(s) queued</i></td>
      </tr>
      <tr v-for="job in jobsRunning" track-by="$index">
-      <td><span v-html="runIcon(job.result)"></span> <router-link :to="'/jobs/'+$route.params.name+'/'+job.number">#{{job.number}}</router-link></td>
+      <td style="width:1px"><span v-html="runIcon(job.result)"></span></td>
+      <td><router-link :to="'/jobs/'+$route.params.name+'/'+job.number">#{{job.number}}</router-link></td>
       <td class="text-center">{{formatDate(job.started)}}</td>
       <td class="text-center">{{formatDuration(job.started, job.completed)}}</td>
       <td class="text-center hidden-xs">{{job.reason}}</td>
      </tr>
      <tr v-for="job in jobsRecent" track-by="$index">
-      <td><span v-html="runIcon(job.result)"></span> <router-link :to="'/jobs/'+$route.params.name+'/'+job.number">#{{job.number}}</router-link></td>
+      <td style="width:1px"><span v-html="runIcon(job.result)"></span></td>
+      <td><router-link :to="'/jobs/'+$route.params.name+'/'+job.number">#{{job.number}}</router-link></td>
       <td class="text-center">{{formatDate(job.started)}}</td>
       <td class="text-center">{{formatDuration(job.started, job.completed)}}</td>
       <td class="text-center hidden-xs">{{job.reason}}</td>
      </tr>
     </table>
+    <ul class="pagination pull-right">
+      <li><button class="btn btn-default" v-on:click="page_prev" :disabled="sort.page==0">&laquo;</button></li>
+      <li>Page {{sort.page+1}} of {{pages}}</li>
+      <li><button class="btn btn-default" v-on:click="page_next" :disabled="sort.page==pages-1">&raquo;</button></li>
+    </ul>
    </div></div>
   </div>
  </div></template>
@@ -208,6 +295,7 @@
      <div style="clear:both;"></div>
      <dl class="dl-horizontal">
       <dt>Reason</dt><dd>{{job.reason}}</dd>
+      <dt v-show="job.upstream.num > 0">Upstream</dt><dd v-show="job.upstream.num > 0"><router-link :to="'/jobs/'+job.upstream.name">{{job.upstream.name}}</router-link> <router-link :to="'/jobs/'+job.upstream.name+'/'+job.upstream.num">#{{job.upstream.num}}</router-link></li></dd>
       <dt>Queued for</dt><dd>{{job.queued}}s</dd>
       <dt>Started</dt><dd>{{formatDate(job.started)}}</dd>
       <dt v-show="runComplete(job)">Completed</dt><dd v-show="job.completed">{{formatDate(job.completed)}}</dd>
@@ -222,7 +310,7 @@
       <div class="panel-heading">Artifacts</div>
       <div class="panel-body">
        <ul class="list-unstyled" style="margin-bottom: 0">
-        <li v-for="art in job.artifacts"><a :href="art.url" target="_self">{{art.filename}}</a></li>
+        <li v-for="art in job.artifacts"><a :href="art.url" target="_self">{{art.filename}}</a> [{{ art.size | iecFileSize }}]</li>
        </ul>
       </div>
      </div>
@@ -239,7 +327,7 @@
  <div id="app">
   <nav class="navbar navbar-inverse">
    <div>
-    <router-link to="/" class="navbar-brand"><img src="/icon.png">{{title}}</router-link>
+    <router-link to="/" class="navbar-brand"><img src="icon.png">{{title}}</router-link>
     <router-link to="/jobs" class="btn navbar-btn pull-right">Jobs</router-link>
    </div>
   </nav>

src/resources/js/app.js

@@ -2,20 +2,44 @@
  * frontend application for Laminar Continuous Integration
  * https://laminar.ohwg.net
  */
-const wsp = function(path) {
-  return new WebSocket((location.protocol === 'https:'?'wss://':'ws://')
-                          + location.host + path);
+
+String.prototype.hashCode = function() {
+  for(var r=0, i=0; i<this.length; i++)
+    r=(r<<5)-r+this.charCodeAt(i),r&=r;
+  return r;
+};
+
+Vue.filter('iecFileSize', function(bytes) {
+  var exp = Math.floor(Math.log(bytes) / Math.log(1024));
+  return (bytes / Math.pow(1024, exp)).toFixed(1) + ' ' +
+    ['B', 'KiB', 'MiB', 'GiB', 'TiB'][exp];
+});
+
+const timeScale = function(max){
+  return max > 3600
+  ? { scale:function(v){return Math.round(v/360)/10}, label:'Hours' }
+  : max > 60
+  ? { scale:function(v){return Math.round(v/60)/10}, label:'Minutes' }
+  : { scale:function(v){return v;}, label:'Seconds' };
 }
+
 const WebsocketHandler = function() {
   function setupWebsocket(path, next) {
-    var ws = wsp(path);
+    let ws = new WebSocket(document.head.baseURI.replace(/^http/,'ws') + path.substr(1));
     ws.onmessage = function(msg) {
       msg = JSON.parse(msg.data);
       // "status" is the first message the websocket always delivers.
       // Use this to confirm the navigation. The component is not
       // created until next() is called, so creating a reference
-      // for other message types must be deferred
-      if (msg.type === 'status') {
+      // for other message types must be deferred. There are some extra
+      // subtle checks here. If this websocket already has a component,
+      // then this is not the first time the status message has been
+      // received. If the frontend requests an update, the status message
+      // should not be handled here, but treated the same as any other
+      // message. An exception is if the connection has been lost - in
+      // that case we should treat this as a "first-time" status message.
+      // this.comp.ws is used as a proxy for this.
+      if (msg.type === 'status' && (!this.comp || !this.comp.ws)) {
         next(comp => {
           // Set up bidirectional reference
           // 1. needed to reference the component for other msg types
@@ -46,11 +70,17 @@ const WebsocketHandler = function() {
       // and a re-connection isn't meaningful
       if(!ev.wasClean && 'comp' in this) {
         this.comp.$root.connected = false;
+        // remove the reference to the websocket from the component.
+        // This not only cleans up an unneeded reference but ensures a
+        // status message on reconnection is treated as "first-time"
+        delete this.comp.ws;
         this.reconnectTimeout = setTimeout(()=>{
           var newWs = setupWebsocket(path, (fn) => { fn(this.comp); });
-          // pass on the current component for the cases where the
-          // connection fails (onclose is called again) before the
-          // status message can reassign the current component
+          // the next() callback won't happen if the server is still
+          // unreachable. Save the reference to the last component
+          // here so we can recover if/when it does return. This means
+          // passing this.comp in the next() callback above is redundant
+          // but necessary to keep the same implementation.
           newWs.comp = this.comp;
         }, 2000);
       }
@@ -151,14 +181,15 @@ const ProgressUpdater = {
 const Home = function() {
   var state = {
     jobsQueued: [],
-    jobsRecent: []
+    jobsRecent: [],
+    resultChanged: []
   };
 
   var chtUtilization, chtBuildsPerDay, chtBuildsPerJob, chtTimePerJob;
 
   var updateUtilization = function(busy) {
-    chtUtilization.segments[0].value += busy ? 1 : -1;
-    chtUtilization.segments[1].value -= busy ? 1 : -1;
+    chtUtilization.data.datasets[0].data[0] += busy ? 1 : -1;
+    chtUtilization.data.datasets[0].data[1] -= busy ? 1 : -1;
     chtUtilization.update();
   }
 
@@ -173,72 +204,185 @@ const Home = function() {
         state.jobsQueued = msg.queued;
         state.jobsRunning = msg.running;
         state.jobsRecent = msg.recent;
+        state.resultChanged = msg.resultChanged;
         this.$forceUpdate();
 
         // setup charts
-        chtUtilization = new Chart(document.getElementById("chartUtil").getContext("2d")).Pie(
-          [{
-              value: msg.executorsBusy,
-              color: "tan",
-              label: "Busy"
-            },
-            {
-              value: msg.executorsTotal - msg.executorsBusy,
-              color: "darkseagreen",
-              label: "Idle"
-            }
-          ], {
-            animationEasing: 'easeInOutQuad'
+        chtUtilization = new Chart(document.getElementById("chartUtil"), {
+          type: 'pie',
+          data: {
+            labels: ["Busy", "Idle"],
+            datasets: [{
+              data: [ msg.executorsBusy, msg.executorsTotal - msg.executorsBusy ],
+              backgroundColor: ["darkgoldenrod", "forestgreen"]
+            }]
+          }
+        });
+        var buildsPerDayDates = function(){
+          res = [];
+          var now = new Date();
+          for (var i = 6; i >= 0; --i) {
+            var then = new Date(now.getTime() - i * 86400000);
+            res.push({
+              short: ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"][then.getDay()],
+              long: then.toLocaleDateString()}
+            );
+          }
+          return res;
+        }();
+        chtBuildsPerDay = new Chart(document.getElementById("chartBpd"), {
+          type: 'line',
+          data: {
+            labels: buildsPerDayDates.map((e)=>{ return e.short; }),
+            datasets: [{
+              label: 'Successful Builds',
+              backgroundColor: "rgba(34,139,34,0.65)", //forestgreen at 0.65
+              borderColor: "forestgreen",
+              data: msg.buildsPerDay.map((e)=>{ return e.success || 0; })
+            }, {
+              label: 'Failed Builds',
+              backgroundColor: "rgba(178,34,34,0.65)", //firebrick at 0.65
+              borderColor: "firebrick",
+              data: msg.buildsPerDay.map((e)=>{ return e.failed || 0; })
+            }]
+          },
+          options:{
+            tooltips:{callbacks:{title: function(tip, data) {
+              return buildsPerDayDates[tip[0].index].long;
+            }}},
+            scales:{yAxes:[{ticks:{userCallback: (label, index, labels)=>{
+              if(Number.isInteger(label))
+                return label;
+            }}}]}
+          }
+        });
+        chtBuildsPerJob = new Chart(document.getElementById("chartBpj"), {
+          type: 'horizontalBar',
+          data: {
+            labels: Object.keys(msg.buildsPerJob),
+            datasets: [{
+              label: 'Runs in last 24 hours',
+              backgroundColor: "steelblue",
+              data: Object.keys(msg.buildsPerJob).map((e)=>{ return msg.buildsPerJob[e]; })
+            }]
+          },
+          options:{
+            scales:{xAxes:[{ticks:{userCallback: (label, index, labels)=>{
+              if(Number.isInteger(label))
+                return label;
+            }}}]}
+          }
+        });
+        var tpjScale = timeScale(Math.max(Object.values(msg.timePerJob)));
+        chtTimePerJob = new Chart(document.getElementById("chartTpj"), {
+          type: 'horizontalBar',
+          data: {
+            labels: Object.keys(msg.timePerJob),
+            datasets: [{
+              label: 'Mean run time this week',
+              backgroundColor: "steelblue",
+              data: Object.keys(msg.timePerJob).map((e)=>{ return msg.timePerJob[e]; })
+            }]
+          },
+          options:{
+            scales:{xAxes:[{
+              ticks:{userCallback: tpjScale.scale},
+              scaleLabel: {
+                display: true,
+                labelString: tpjScale.label
+              }
+            }]},
+            tooltips:{callbacks:{label:(tip, data)=>{
+              return data.datasets[tip.datasetIndex].label + ': ' + tip.xLabel + ' ' + tpjScale.label.toLowerCase();
+            }}}
+          }
+        });
+        var chtResultChanges = new Chart(document.getElementById("chartResultChanges"), {
+          type: 'horizontalBar',
+          data: {
+            labels: msg.resultChanged.map((e)=>{ return e.name; }),
+            datasets: [{
+              //label: '% Passed',
+              backgroundColor: msg.resultChanged.map((e)=>{return e.lastFailure > e.lastSuccess ? 'firebrick' : 'forestgreen';}),
+              data: msg.resultChanged.map((e)=>{ return e.lastSuccess - e.lastFailure; }),
+              itemid: msg.resultChanged.map((e)=> { return 'rcd_' + e.name; })
+            }]
+          },
+          options:{
+            scales:{
+              xAxes:[{ticks:{display: false}}],
+              yAxes:[{ticks:{display: false}}]
+            },
+            tooltips:{
+              enabled:false
+            }
+          }
+        });
+        var chtPassRates = new Chart(document.getElementById("chartPassRates"), {
+          type: 'horizontalBar',
+          data: {
+            labels: msg.lowPassRates.map((e)=>{ return e.name }),
+            datasets: [{
+              stack: 'passrate',
+              label: '% Passed',
+              backgroundColor: "forestgreen",
+              data: msg.lowPassRates.map((e)=>{ return e.passRate*100; })
+            },{
+              stack:'passrate',
+              label: '% Failed',
+              backgroundColor: "firebrick",
+              data: msg.lowPassRates.map((e)=>{ return (1-e.passRate)*100; })
+            }],
+          },
+          options:{
+            scales:{xAxes:[{ticks:{callback:(val,idx,values)=>{
+              return val + '%';
+            }}}]},
+            tooltips:{
+              enabled:false
+            }
+          }
+        });
+        var btcScale = timeScale(Math.max(msg.buildTimeChanges.map((e)=>{return Math.max(e.durations)})));
+        var chtBuildTimeChanges = new Chart(document.getElementById("chartBuildTimeChanges"), {
+          type: 'line',
+          data: {
+            labels: [...Array(10).keys()],
+            datasets: msg.buildTimeChanges.map((e)=>{return {
+              label: e.name,
+              data: e.durations,
+              borderColor: 'hsl('+(e.name.hashCode() % 360)+', 61%, 34%)',
+              backgroundColor: 'transparent'
+            }})
+          },
+          options:{
+            legend:{display:true},
+            scales:{
+              xAxes:[{ticks:{display: false}}],
+              yAxes:[{
+                ticks:{userCallback: btcScale.scale},
+                scaleLabel: {
+                  display: true,
+                  labelString: btcScale.label
+                }
+              }]
+            },
+            tooltips:{
+              enabled:false
+            }
+          }
+        });
+        var chtBuildTimeDist = new Chart(document.getElementById("chartBuildTimeDist"), {
+          type: 'line',
+          data: {
+            labels: ['<30s','30s-1m','1m-5m','5m-10m','10m-20m','20m-40m','40m-60m','>60m'],
+            datasets: [{
+              label: 'Number jobs with average build time in range',
+              data: msg.buildTimeDist,
+              backgroundColor: "steelblue",
+            }]
           }
-        );
-        chtBuildsPerDay = new Chart(document.getElementById("chartBpd").getContext("2d")).Line({
-          labels: function() {
-            res = [];
-            var now = new Date();
-            for (var i = 6; i >= 0; --i) {
-              var then = new Date(now.getTime() - i * 86400000);
-              res.push(["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"][then.getDay()]);
-            }
-            return res;
-          }(),
-          datasets: [{
-            label: "Successful Builds",
-            fillColor: "rgba(143,188,143,0.65)", //darkseagreen at 0.65
-            strokeColor: "forestgreen",
-            data: msg.buildsPerDay.map(function(e) {
-              return e.success || 0;
-            })
-          }, {
-            label: "Failed Bulids",
-            fillColor: "rgba(233,150,122,0.65)", //darksalmon at 0.65
-            strokeColor: "crimson",
-            data: msg.buildsPerDay.map(function(e) {
-              return e.failed || 0;
-            })
-          }]
-        }, {
-          showTooltips: false
         });
-        chtBuildsPerJob = new Chart(document.getElementById("chartBpj").getContext("2d")).HorizontalBar({
-          labels: Object.keys(msg.buildsPerJob),
-          datasets: [{
-            fillColor: "lightsteelblue",
-            data: Object.keys(msg.buildsPerJob).map(function(e) {
-              return msg.buildsPerJob[e];
-            })
-          }]
-        }, {});
-        chtTimePerJob = new Chart(document.getElementById("chartTpj").getContext("2d")).HorizontalBar({
-          labels: Object.keys(msg.timePerJob),
-          datasets: [{
-            fillColor: "lightsteelblue",
-            data: Object.keys(msg.timePerJob).map(function(e) {
-              return msg.timePerJob[e];
-            })
-          }]
-        }, {});
-
-
       },
       job_queued: function(data) {
         state.jobsQueued.splice(0, 0, data);
@@ -252,9 +396,9 @@ const Home = function() {
       },
       job_completed: function(data) {
         if (data.result === "success")
-          chtBuildsPerDay.datasets[0].points[6].value++;
+          chtBuildsPerDay.data.datasets[0].data[6]++;
         else
-          chtBuildsPerDay.datasets[1].points[6].value++;
+          chtBuildsPerDay.data.datasets[1].data[6]++;
         chtBuildsPerDay.update();
 
         for (var i = 0; i < state.jobsRunning.length; ++i) {
@@ -267,9 +411,9 @@ const Home = function() {
           }
         }
         updateUtilization(false);
-        for (var j = 0; j < chtBuildsPerJob.datasets[0].bars.length; ++j) {
-          if (chtBuildsPerJob.datasets[0].bars[j].label == job.name) {
-            chtBuildsPerJob.datasets[0].bars[j].value++;
+        for (var j = 0; j < chtBuildsPerJob.data.datasets[0].data.length; ++j) {
+          if (chtBuildsPerJob.data.labels[j] == job.name) {
+            chtBuildsPerJob.data.datasets[0].data[j]++;
             chtBuildsPerJob.update();
             break;
           }
@@ -290,24 +434,6 @@ const Jobs = function() {
     template: '#jobs',
     mixins: [WebsocketHandler, Utils, ProgressUpdater],
     data: function() { return state; },
-    computed: {
-      filteredJobs() {
-        var ret = this.jobs;
-        var tag = this.tag;
-        if (tag) {
-          ret = ret.filter(function(job) {
-            return job.tags.indexOf(tag) >= 0;
-          });
-        }
-        var search = this.search;
-        if (search) {
-          ret = ret.filter(function(job) {
-            return job.name.indexOf(search) > -1;
-          });
-        }
-        return ret;
-      }
-    },
     methods: {
       status: function(msg) {
         state.jobs = msg.jobs;
@@ -333,6 +459,7 @@ const Jobs = function() {
       },
       job_started: function(data) {
         var updAt = null;
+        // jobsRunning must be maintained for ProgressUpdater
         for (var i in state.jobsRunning) {
           if (state.jobsRunning[i].name === data.name) {
             updAt = i;
@@ -375,7 +502,23 @@ const Jobs = function() {
             break;
           }
         }
-      }
+      },
+      filteredJobs: function() {
+        var ret = state.jobs;
+        var tag = state.tag;
+        if (tag) {
+          ret = ret.filter(function(job) {
+            return job.tags.indexOf(tag) >= 0;
+          });
+        }
+        var search = this.search;
+        if (search) {
+          ret = ret.filter(function(job) {
+            return job.name.indexOf(search) > -1;
+          });
+        }
+        return ret;
+      },
     }
   };
 }();
@@ -387,7 +530,10 @@ var Job = function() {
     lastSuccess: null,
     lastFailed: null,
     nQueued: 0,
+    pages: 0,
+    sort: {}
   };
+  var chtBt = null;
   return Vue.extend({
     template: '#job',
     mixins: [WebsocketHandler, Utils, ProgressUpdater],
@@ -401,32 +547,70 @@ var Job = function() {
         state.lastSuccess = msg.lastSuccess;
         state.lastFailed = msg.lastFailed;
         state.nQueued = msg.nQueued;
+        state.pages = msg.pages;
+        state.sort = msg.sort;
 
-        var chtBt = new Chart(document.getElementById("chartBt").getContext("2d")).Bar({
-          labels: msg.recent.map(function(e) {
-            return '#' + e.number;
-          }).reverse(),
-          datasets: [{
-            fillColor: "darkseagreen",
-            strokeColor: "forestgreen",
-            data: msg.recent.map(function(e) {
-              return e.completed - e.started;
-            }).reverse()
-          }]
-        }, {
-          barValueSpacing: 1,
-          barStrokeWidth: 1,
-          barDatasetSpacing: 0
+        // "status" comes again if we change page/sorting. Delete the
+        // old chart and recreate it to prevent flickering of old data
+        if(chtBt)
+          chtBt.destroy();
+        var btScale = timeScale(Math.max(msg.recent.map(v=>{v.completed-v.started})));
+        chtBt = new Chart(document.getElementById("chartBt"), {
+          type: 'bar',
+          data: {
+            labels: msg.recent.map(function(e) {
+              return '#' + e.number;
+            }).reverse(),
+            datasets: [{
+              label: 'Average',
+              type: 'line',
+              data: [{x:0,y:msg.averageRuntime},{x:1,y:msg.averageRuntime}],
+              borderColor: 'steelblue',
+              backgroundColor: 'transparent',
+              xAxisID: 'avg',
+              pointRadius: 0,
+              pointHitRadius: 0,
+              pointHoverRadius: 0,
+            },{
+              label: 'Build time',
+              backgroundColor: (new Array(msg.recent.length)).fill('darkseagreen'),
+              borderColor: (new Array(msg.recent.length)).fill('forestgreen'),
+              data: msg.recent.map(function(e) {
+                return e.completed - e.started;
+              }).reverse()
+            }]
+          },
+          options: {
+            scales:{
+              xAxes:[{},{
+                id: 'avg',
+                type: 'linear',
+                ticks: {
+                  display: false
+                },
+                gridLines: {
+                  display: false,
+                  drawBorder: false
+                }
+              }],
+              yAxes:[{
+                ticks:{userCallback: btScale.scale},
+                scaleLabel:{display: true, labelString: btScale.label}
+              }]
+            },
+            tooltips:{callbacks:{label:(tip, data)=>{
+              return data.datasets[tip.datasetIndex].label + ': ' + tip.yLabel + ' ' + btScale.label.toLowerCase();
+            }}}
+          }
         });
 
         for (var i = 0, n = msg.recent.length; i < n; ++i) {
           if (msg.recent[i].result != "success") {
-            chtBt.datasets[0].bars[n - i - 1].fillColor = "darksalmon";
-            chtBt.datasets[0].bars[n - i - 1].strokeColor = "crimson";
+            chtBt.data.datasets[0].backgroundColor[n - i - 1] = "darksalmon";
+            chtBt.data.datasets[0].borderColor[n - i - 1] = "crimson";
           }
         }
         chtBt.update();
-
       },
       job_queued: function() {
         state.nQueued++;
@@ -447,28 +631,62 @@ var Job = function() {
             break;
           }
         }
+      },
+      page_next: function() {
+        state.sort.page++;
+        this.ws.send(JSON.stringify(state.sort));
+      },
+      page_prev: function() {
+        state.sort.page--;
+        this.ws.send(JSON.stringify(state.sort));
+      },
+      do_sort: function(field) {
+        if(state.sort.field == field) {
+          state.sort.order = state.sort.order == 'asc' ? 'dsc' : 'asc';
+        } else {
+          state.sort.order = 'dsc';
+          state.sort.field = field;
+        }
+        this.ws.send(JSON.stringify(state.sort));
       }
     }
   });
 }();
 
 const Run = function() {
+  const utf8decoder = new TextDecoder('utf-8');
   var state = {
-    job: { artifacts: [] },
+    job: { artifacts: [], upstream: {} },
     latestNum: null,
     log: '',
     autoscroll: false
   };
   var firstLog = false;
-  var logHandler = function(vm, d) {
-    state.log += ansi_up.ansi_to_html(d.replace(/</g,'&lt;').replace(/>/g,'&gt;'));
-    vm.$forceUpdate();
-    if (!firstLog) {
-      firstLog = true;
-    } else if (state.autoscroll) {
-      window.scrollTo(0, document.body.scrollHeight);
-    }
-  };
+  const logFetcher = (vm, name, num) => {
+    const abort = new AbortController();
+    fetch('log/'+name+'/'+num, {signal:abort.signal}).then(res => {
+      // ATOW pipeThrough not supported in Firefox
+      //const reader = res.body.pipeThrough(new TextDecoderStream).getReader();
+      const reader = res.body.getReader();
+      let total = 0;
+      return function pump() {
+        return reader.read().then(({done, value}) => {
+          value = utf8decoder.decode(value);
+          if (done)
+            return;
+          state.log += ansi_up.ansi_to_html(value.replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/\033\[\{([^:]+):(\d+)\033\\/g, (m,$1,$2)=>{return '<a href="jobs/'+$1+'" onclick="return vroute(this);">'+$1+'</a>:<a href="jobs/'+$1+'/'+$2+'" onclick="return vroute(this);">#'+$2+'</a>';}));
+          vm.$forceUpdate();
+          if (!firstLog) {
+            firstLog = true;
+          } else if (state.autoscroll) {
+            window.scrollTo(0, document.body.scrollHeight);
+          }
+          return pump();
+        });
+      }();
+    }).catch(e => {});
+    return abort;
+  }
 
   return {
     template: '#run',
@@ -478,6 +696,14 @@ const Run = function() {
     },
     methods: {
       status: function(data) {
+        // Check for the /latest endpoint. An intuitive check might be
+        //  if(this.$route.params.number == 'latest'), but unfortunately
+        // after calling $router.replace, we re-enter status() before
+        // $route.params is updated. Instead, assume that if there is
+        // no 'started' field, we should redirect to the latest number
+        if(!('started' in data) && 'latestNum' in data)
+          return this.$router.replace('/jobs/' + this.$route.params.name + '/' + data.latestNum);
+
         state.jobsRunning = [];
         state.job = data;
         state.latestNum = data.latestNum;
@@ -488,7 +714,7 @@ const Run = function() {
         this.$forceUpdate();
       },
       job_completed: function(data) {
-        state.job = data;
+        state.job = Object.assign(state.job, data);
         state.jobsRunning = [];
         this.$forceUpdate();
       },
@@ -499,29 +725,44 @@ const Run = function() {
     beforeRouteEnter(to, from, next) {
       next(vm => {
         state.log = '';
-        vm.logws = wsp(to.path + '/log');
-        vm.logws.onmessage = function(msg) {
-          logHandler(vm, msg.data);
-        }
+        vm.logstream = logFetcher(vm, to.params.name, to.params.number);
       });
     },
     beforeRouteUpdate(to, from, next) {
       var vm = this;
-      vm.logws.close();
+      vm.logstream.abort();
       state.log = '';
-      vm.logws = wsp(to.path + '/log');
-      vm.logws.onmessage = function(msg) {
-        logHandler(vm, msg.data);
-      }
+      vm.logstream = logFetcher(vm, to.params.name, to.params.number);
       next();
     },
     beforeRouteLeave(to, from, next) {
-      this.logws.close();
+      this.logstream.abort();
       next();
     }
   };
 }();
 
+// For all charts, set miniumum Y to 0
+Chart.scaleService.updateScaleDefaults('linear', {
+    ticks: { suggestedMin: 0 }
+});
+// Don't display legend by default
+Chart.defaults.global.legend.display = false;
+// Plugin to move a DOM item on top of a chart element
+Chart.plugins.register({
+  afterDatasetsDraw: (chart) => {
+    chart.data.datasets.forEach((dataset, i) => {
+      var meta = chart.getDatasetMeta(i);
+      if(dataset.itemid)
+        meta.data.forEach((e,j) => {
+          var pos = e.getCenterPoint();
+          var node = document.getElementById(dataset.itemid[j]);
+          node.style.top = (pos.y - node.clientHeight/2) + 'px';
+        });
+    });
+  }
+});
+
 new Vue({
   el: '#app',
   data: {
@@ -554,6 +795,7 @@ new Vue({
   },
   router: new VueRouter({
     mode: 'history',
+    base: document.head.baseURI.substr(location.origin.length),
     routes: [
       { path: '/',                   component: Home },
       { path: '/jobs',               component: Jobs },

src/run.cpp

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015-2017 Oliver Giles
+/// Copyright 2015-2018 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -25,8 +25,11 @@
 #include <unistd.h>
 #include <signal.h>
 
-#include <boost/filesystem.hpp>
-namespace fs = boost::filesystem;
+// short syntax helper for kj::Path
+template<typename T>
+inline kj::Path operator/(const kj::Path& p, const T& ext) {
+    return p.append(ext);
+}
 
 std::string to_string(const RunState& rs) {
     switch(rs) {
@@ -41,105 +44,211 @@ std::string to_string(const RunState& rs) {
 }
 
 
-Run::Run() {
-    result = RunState::SUCCESS;
-    lastResult = RunState::UNKNOWN;
+Run::Run(std::string name, ParamMap pm, kj::Path&& rootPath) :
+    result(RunState::SUCCESS),
+    lastResult(RunState::UNKNOWN),
+    name(name),
+    params(kj::mv(pm)),
+    queuedAt(time(nullptr)),
+    rootPath(kj::mv(rootPath)),
+    started(kj::newPromiseAndFulfiller<void>())
+{
+    for(auto it = params.begin(); it != params.end();) {
+        if(it->first[0] == '=') {
+            if(it->first == "=parentJob") {
+                parentName = it->second;
+            } else if(it->first == "=parentBuild") {
+                parentBuild = atoi(it->second.c_str());
+            } else if(it->first == "=reason") {
+                reasonMsg = it->second;
+            } else {
+                LLOG(ERROR, "Unknown internal job parameter", it->first);
+            }
+            it = params.erase(it);
+        } else
+            ++it;
+    }
 }
 
 Run::~Run() {
     LLOG(INFO, "Run destroyed");
 }
 
-std::string Run::reason() const {
-    if(!parentName.empty()) {
-        return std::string("Triggered by upstream ") + parentName + " #" + std::to_string(parentBuild);
+bool Run::configure(uint buildNum, std::shared_ptr<Node> nd, const kj::Directory& fsHome)
+{
+    kj::Path cfgDir{"cfg"};
+
+    // create the run directory
+    kj::Path rd{"run",name,std::to_string(buildNum)};
+    bool createWorkdir = true;
+    KJ_IF_MAYBE(ls, fsHome.tryLstat(rd)) {
+        LASSERT(ls->type == kj::FsNode::Type::DIRECTORY);
+        LLOG(WARNING, "Working directory already exists, removing", rd.toString());
+        if(fsHome.tryRemove(rd) == false) {
+            LLOG(WARNING, "Failed to remove working directory");
+            createWorkdir = false;
+        }
     }
+    if(createWorkdir && fsHome.tryOpenSubdir(rd, kj::WriteMode::CREATE|kj::WriteMode::CREATE_PARENT) == nullptr) {
+        LLOG(ERROR, "Could not create working directory", rd.toString());
+        return false;
+    }
+
+    // create an archive directory
+    kj::Path archive = kj::Path{"archive",name,std::to_string(buildNum)};
+    if(fsHome.exists(archive)) {
+        LLOG(WARNING, "Archive directory already exists", archive.toString());
+    } else if(fsHome.tryOpenSubdir(archive, kj::WriteMode::CREATE|kj::WriteMode::CREATE_PARENT) == nullptr) {
+        LLOG(ERROR, "Could not create archive directory", archive.toString());
+        return false;
+    }
+
+    // create a workspace for this job if it doesn't exist
+    kj::Path ws{"run",name,"workspace"};
+    if(!fsHome.exists(ws)) {
+        fsHome.openSubdir(ws, kj::WriteMode::CREATE|kj::WriteMode::CREATE_PARENT);
+        // prepend the workspace init script
+        if(fsHome.exists(cfgDir/"jobs"/(name+".init")))
+            addScript(cfgDir/"jobs"/(name+".init"), kj::mv(ws));
+    }
+
+    // add scripts
+    // global before-run script
+    if(fsHome.exists(cfgDir/"before"))
+        addScript(cfgDir/"before", rd.clone());
+    // per-node before-run script
+    if(fsHome.exists(cfgDir/"nodes"/(nd->name+".before")))
+        addScript(cfgDir/"nodes"/(nd->name+".before"), rd.clone());
+    // job before-run script
+    if(fsHome.exists(cfgDir/"jobs"/(name+".before")))
+        addScript(cfgDir/"jobs"/(name+".before"), rd.clone());
+    // main run script. must exist.
+    addScript(cfgDir/"jobs"/(name+".run"), rd.clone());
+    // job after-run script
+    if(fsHome.exists(cfgDir/"jobs"/(name+".after")))
+        addScript(cfgDir/"jobs"/(name+".after"), rd.clone(), true);
+    // per-node after-run script
+    if(fsHome.exists(cfgDir/"nodes"/(nd->name+".after")))
+        addScript(cfgDir/"nodes"/(nd->name+".after"), rd.clone(), true);
+    // global after-run script
+    if(fsHome.exists(cfgDir/"after"))
+        addScript(cfgDir/"after", rd.clone(), true);
+
+    // add environment files
+    if(fsHome.exists(cfgDir/"env"))
+        addEnv(cfgDir/"env");
+    if(fsHome.exists(cfgDir/"nodes"/(nd->name+".env")))
+        addEnv(cfgDir/"nodes"/(nd->name+".env"));
+    if(fsHome.exists(cfgDir/"jobs"/(name+".env")))
+        addEnv(cfgDir/"jobs"/(name+".env"));
+
+    // add job timeout if specified
+    if(fsHome.exists(cfgDir/"jobs"/(name+".conf"))) {
+        timeout = parseConfFile((rootPath/cfgDir/"jobs"/(name+".conf")).toString(true).cStr()).get<int>("TIMEOUT", 0);
+    }
+
+    // All good, we've "started"
+    startedAt = time(nullptr);
+    build = buildNum;
+    node = nd;
+
+    // notifies the rpc client if the start command was used
+    started.fulfiller->fulfill();
+
+    return true;
+}
+
+std::string Run::reason() const {
     return reasonMsg;
 }
 
 bool Run::step() {
-    if(scripts.size()) {
-        currentScript = scripts.front();
-        scripts.pop();
+    if(!scripts.size())
+        return true;
 
-        int pfd[2];
-        pipe(pfd);
-        pid_t pid = fork();
-        if(pid == 0) { // child
-            // reset signal mask (SIGCHLD blocked in Laminar::start)
-            sigset_t mask;
-            sigemptyset(&mask);
-            sigaddset(&mask, SIGCHLD);
-            sigprocmask(SIG_UNBLOCK, &mask, nullptr);
+    Script currentScript = kj::mv(scripts.front());
+    scripts.pop();
 
-            // set pgid == pid for easy killing on abort
-            setpgid(0, 0);
+    int pfd[2];
+    pipe(pfd);
+    pid_t pid = fork();
+    if(pid == 0) { // child
+        // reset signal mask (SIGCHLD blocked in Laminar::start)
+        sigset_t mask;
+        sigemptyset(&mask);
+        sigaddset(&mask, SIGCHLD);
+        sigprocmask(SIG_UNBLOCK, &mask, nullptr);
 
-            close(pfd[0]);
-            dup2(pfd[1], 1);
-            dup2(pfd[1], 2);
-            close(pfd[1]);
-            std::string buildNum = std::to_string(build);
+        // set pgid == pid for easy killing on abort
+        setpgid(0, 0);
 
-            std::string PATH = (fs::path(laminarHome)/"cfg"/"scripts").string() + ":";
-            if(const char* p = getenv("PATH")) {
-                PATH.append(p);
-            }
+        close(pfd[0]);
+        dup2(pfd[1], 1);
+        dup2(pfd[1], 2);
+        close(pfd[1]);
+        std::string buildNum = std::to_string(build);
 
-            chdir(currentScript.cwd.c_str());
-
-            // conf file env vars
-            for(std::string file : env) {
-                StringMap vars = parseConfFile(file.c_str());
-                for(auto& it : vars) {
-                    setenv(it.first.c_str(), it.second.c_str(), true);
-                }
-            }
-            // parameterized vars
-            for(auto& pair : params) {
-                setenv(pair.first.c_str(), pair.second.c_str(), false);
-            }
-
-            setenv("PATH", PATH.c_str(), true);
-            setenv("RUN", buildNum.c_str(), true);
-            setenv("JOB", name.c_str(), true);
-            if(!node->name.empty())
-                setenv("NODE", node->name.c_str(), true);
-            setenv("RESULT", to_string(result).c_str(), true);
-            setenv("LAST_RESULT", to_string(lastResult).c_str(), true);
-            setenv("WORKSPACE", (fs::path(laminarHome)/"run"/name/"workspace").string().c_str(), true);
-            setenv("ARCHIVE", (fs::path(laminarHome)/"archive"/name/buildNum.c_str()).string().c_str(), true);
-
-            fprintf(stderr, "[laminar] Executing %s\n", currentScript.path.c_str());
-            execl(currentScript.path.c_str(), currentScript.path.c_str(), NULL);
-            // cannot use LLOG because stdout/stderr are captured
-            fprintf(stderr, "[laminar] Failed to execute %s\n", currentScript.path.c_str());
-            _exit(1);
+        std::string PATH = (rootPath/"cfg"/"scripts").toString(true).cStr();
+        if(const char* p = getenv("PATH")) {
+            PATH.append(":");
+            PATH.append(p);
         }
 
-        LLOG(INFO, "Forked", currentScript.path, currentScript.cwd, pid);
-        close(pfd[1]);
-        fd = pfd[0];
-        this->pid = pid;
+        LSYSCALL(chdir((rootPath/currentScript.cwd).toString(true).cStr()));
 
-        return false;
-    } else {
-        return true;
+        // conf file env vars
+        for(kj::Path& file : env) {
+            StringMap vars = parseConfFile((rootPath/file).toString(true).cStr());
+            for(auto& it : vars) {
+                setenv(it.first.c_str(), it.second.c_str(), true);
+            }
+        }
+        // parameterized vars
+        for(auto& pair : params) {
+            setenv(pair.first.c_str(), pair.second.c_str(), false);
+        }
+
+        setenv("PATH", PATH.c_str(), true);
+        setenv("RUN", buildNum.c_str(), true);
+        setenv("JOB", name.c_str(), true);
+        if(!node->name.empty())
+            setenv("NODE", node->name.c_str(), true);
+        setenv("RESULT", to_string(result).c_str(), true);
+        setenv("LAST_RESULT", to_string(lastResult).c_str(), true);
+        setenv("WORKSPACE", (rootPath/"run"/name/"workspace").toString(true).cStr(), true);
+        setenv("ARCHIVE", (rootPath/"archive"/name/buildNum).toString(true).cStr(), true);
+
+        fprintf(stderr, "[laminar] Executing %s\n", currentScript.path.toString().cStr());
+        kj::String execPath = (rootPath/currentScript.path).toString(true);
+        execl(execPath.cStr(), execPath.cStr(), NULL);
+        // cannot use LLOG because stdout/stderr are captured
+        fprintf(stderr, "[laminar] Failed to execute %s\n", currentScript.path.toString().cStr());
+        _exit(1);
     }
+
+    LLOG(INFO, "Forked", currentScript.path, currentScript.cwd, pid);
+    close(pfd[1]);
+
+    current_pid = pid;
+    output_fd = pfd[0];
+    return false;
 }
 
-void Run::addScript(std::string scriptPath, std::string scriptWorkingDir) {
-    scripts.push({scriptPath, scriptWorkingDir});
+void Run::addScript(kj::Path scriptPath, kj::Path scriptWorkingDir, bool runOnAbort) {
+    scripts.push({kj::mv(scriptPath), kj::mv(scriptWorkingDir), runOnAbort});
 }
 
-void Run::addEnv(std::string path) {
-    env.push_back(path);
+void Run::addEnv(kj::Path path) {
+    env.push_back(kj::mv(path));
 }
 
-void Run::abort() {
-    // clear all pending scripts
-    std::queue<Script>().swap(scripts);
-    kill(-pid, SIGTERM);
+void Run::abort(bool respectRunOnAbort) {
+    while(scripts.size() && (!respectRunOnAbort || !scripts.front().runOnAbort))
+        scripts.pop();
+    // if the Maybe is empty, wait() was already called on this process
+    KJ_IF_MAYBE(p, current_pid) {
+        kill(-*p, SIGTERM);
+    }
 }
 
 void Run::reaped(int status) {
@@ -153,7 +262,3 @@ void Run::reaped(int status) {
         result = RunState::FAILED;
     // otherwise preserve earlier status
 }
-
-void Run::complete() {
-    notifyCompletion(this);
-}

src/run.h

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015-2017 Oliver Giles
+/// Copyright 2015-2018 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -27,6 +27,7 @@
 #include <unordered_map>
 #include <memory>
 #include <kj/async.h>
+#include <kj/filesystem.h>
 
 enum class RunState {
     UNKNOWN,
@@ -41,34 +42,27 @@ std::string to_string(const RunState& rs);
 
 class Node;
 
-// Represents an execution of a job. Not much more than POD
+typedef std::unordered_map<std::string, std::string> ParamMap;
+
+// Represents an execution of a job.
 class Run {
 public:
-    Run();
+    Run(std::string name, ParamMap params, kj::Path&& rootPath);
     ~Run();
 
     // copying this class would be asking for trouble...
     Run(const Run&) = delete;
     Run& operator=(const Run&) = delete;
 
+    // Call this to "start" the run with a specific number and node
+    bool configure(uint buildNum, std::shared_ptr<Node> node, const kj::Directory &fsHome);
+
     // executes the next script (if any), returning true if there is nothing
-    // more to be done - in this case the caller should call complete()
+    // more to be done.
     bool step();
 
-    // call this when all scripts are done to get the notifyCompletion callback
-    void complete();
-
-    // adds a script to the queue of scripts to be executed by this run
-    void addScript(std::string scriptPath, std::string scriptWorkingDir);
-
-    // adds a script to the queue using the runDir as the scripts CWD
-    void addScript(std::string script) { addScript(script, runDir); }
-
-    // adds an environment file that will be sourced before this run
-    void addEnv(std::string path);
-
     // aborts this run
-    void abort();
+    void abort(bool respectRunOnAbort);
 
     // called when a process owned by this run has been reaped. The status
     // may be used to set the run's job status
@@ -76,36 +70,41 @@ public:
 
     std::string reason() const;
 
-    std::function<void(Run*)> notifyCompletion;
+    kj::Promise<void>&& whenStarted() { return kj::mv(started.promise); }
+
     std::shared_ptr<Node> node;
     RunState result;
     RunState lastResult;
-    std::string laminarHome;
     std::string name;
-    std::string runDir;
     std::string parentName;
     int parentBuild = 0;
-    std::string reasonMsg;
     uint build = 0;
     std::string log;
-    pid_t pid;
-    int fd;
+    kj::Maybe<pid_t> current_pid;
+    int output_fd;
     std::unordered_map<std::string, std::string> params;
-    kj::Promise<void> timeout = kj::NEVER_DONE;
-    kj::PromiseFulfillerPair<void> started = kj::newPromiseAndFulfiller<void>();
+    int timeout = 0;
 
     time_t queuedAt;
     time_t startedAt;
 private:
+    // adds a script to the queue of scripts to be executed by this run
+    void addScript(kj::Path scriptPath, kj::Path scriptWorkingDir, bool runOnAbort = false);
+
+    // adds an environment file that will be sourced before this run
+    void addEnv(kj::Path path);
 
     struct Script {
-        std::string path;
-        std::string cwd;
+        kj::Path path;
+        kj::Path cwd;
+        bool runOnAbort;
     };
 
+    kj::Path rootPath;
     std::queue<Script> scripts;
-    Script currentScript;
-    std::list<std::string> env;
+    std::list<kj::Path> env;
+    std::string reasonMsg;
+    kj::PromiseFulfillerPair<void> started;
 };
 
 
@@ -132,8 +131,6 @@ struct _run_same {
 
 // A single Run can be fetched by...
 struct _run_index : bmi::indexed_by<
-        // their current running pid
-        bmi::hashed_unique<bmi::member<Run, pid_t, &Run::pid>>,
         bmi::hashed_unique<bmi::composite_key<
             std::shared_ptr<Run>,
         // a combination of their job name and build number
@@ -153,20 +150,17 @@ struct RunSet: public boost::multi_index_container<
     std::shared_ptr<Run>,
     _run_index
 > {
-    typename bmi::nth_index<RunSet, 0>::type& byPid() { return get<0>(); }
-    typename bmi::nth_index<RunSet, 0>::type const& byPid() const { return get<0>(); }
+    typename bmi::nth_index<RunSet, 0>::type& byNameNumber() { return get<0>(); }
+    typename bmi::nth_index<RunSet, 0>::type const& byNameNumber() const { return get<0>(); }
 
-    typename bmi::nth_index<RunSet, 1>::type& byRun() { return get<1>(); }
-    typename bmi::nth_index<RunSet, 1>::type const& byRun() const { return get<1>(); }
+    typename bmi::nth_index<RunSet, 1>::type& byRunPtr() { return get<1>(); }
+    typename bmi::nth_index<RunSet, 1>::type const& byRunPtr() const { return get<1>(); }
 
-    typename bmi::nth_index<RunSet, 2>::type& byRunPtr() { return get<2>(); }
-    typename bmi::nth_index<RunSet, 2>::type const& byRunPtr() const { return get<2>(); }
+    typename bmi::nth_index<RunSet, 2>::type& byStartedAt() { return get<2>(); }
+    typename bmi::nth_index<RunSet, 2>::type const& byStartedAt() const { return get<2>(); }
 
-    typename bmi::nth_index<RunSet, 3>::type& byStartedAt() { return get<3>(); }
-    typename bmi::nth_index<RunSet, 3>::type const& byStartedAt() const { return get<3>(); }
-
-    typename bmi::nth_index<RunSet, 4>::type& byJobName() { return get<4>(); }
-    typename bmi::nth_index<RunSet, 4>::type const& byJobName() const { return get<4>(); }
+    typename bmi::nth_index<RunSet, 3>::type& byJobName() { return get<3>(); }
+    typename bmi::nth_index<RunSet, 3>::type const& byJobName() const { return get<3>(); }
 };
 
 #endif // LAMINAR_RUN_H_

src/server.cpp

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015-2018 Oliver Giles
+/// Copyright 2015-2019 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -26,39 +26,20 @@
 #include <capnp/rpc-twoparty.h>
 #include <capnp/rpc.capnp.h>
 #include <kj/async-io.h>
+#include <kj/async-unix.h>
 #include <kj/threadlocal.h>
 
-#include <websocketpp/config/core.hpp>
-#include <websocketpp/server.hpp>
-
 #include <sys/eventfd.h>
 #include <sys/inotify.h>
 #include <sys/signal.h>
 #include <sys/signalfd.h>
 
+#include <rapidjson/document.h>
+
 // Size of buffer used to read from file descriptors. Should be
 // a multiple of sizeof(struct signalfd_siginfo) == 128
 #define PROC_IO_BUFSIZE 4096
 
-// Configuration struct for the websocketpp template library.
-struct wsconfig : public websocketpp::config::core {
-//    static const websocketpp::log::level elog_level =
-//        websocketpp::log::elevel::info;
-
-//    static const websocketpp::log::level alog_level =
-//        websocketpp::log::alevel::access_core |
-//        websocketpp::log::alevel::message_payload ;
-
-    static const websocketpp::log::level elog_level =
-        websocketpp::log::elevel::none;
-
-    static const websocketpp::log::level alog_level =
-        websocketpp::log::alevel::none;
-
-    typedef struct { LaminarClient* lc; } connection_base;
-};
-typedef websocketpp::server<wsconfig> websocket;
-
 namespace {
 
 // Used for returning run state to RPC clients
@@ -94,11 +75,7 @@ public:
     kj::Promise<void> queue(QueueContext context) override {
         std::string jobName = context.getParams().getJobName();
         LLOG(INFO, "RPC queue", jobName);
-        ParamMap params;
-        for(auto p : context.getParams().getParams()) {
-            params[p.getName().cStr()] = p.getValue().cStr();
-        }
-        LaminarCi::MethodResult result = laminar.queueJob(jobName, params)
+        LaminarCi::MethodResult result = laminar.queueJob(jobName, params(context.getParams().getParams()))
                 ? LaminarCi::MethodResult::SUCCESS
                 : LaminarCi::MethodResult::FAILED;
         context.getResults().setResult(result);
@@ -109,13 +86,9 @@ public:
     kj::Promise<void> start(StartContext context) override {
         std::string jobName = context.getParams().getJobName();
         LLOG(INFO, "RPC start", jobName);
-        ParamMap params;
-        for(auto p : context.getParams().getParams()) {
-            params[p.getName().cStr()] = p.getValue().cStr();
-        }
-        std::shared_ptr<Run> run = laminar.queueJob(jobName, params);
+        std::shared_ptr<Run> run = laminar.queueJob(jobName, params(context.getParams().getParams()));
         if(Run* r = run.get()) {
-            return r->started.promise.then([context,r]() mutable {
+            return r->whenStarted().then([context,r]() mutable {
                 context.getResults().setResult(LaminarCi::MethodResult::SUCCESS);
                 context.getResults().setBuildNum(r->build);
             });
@@ -129,11 +102,7 @@ public:
     kj::Promise<void> run(RunContext context) override {
         std::string jobName = context.getParams().getJobName();
         LLOG(INFO, "RPC run", jobName);
-        ParamMap params;
-        for(auto p : context.getParams().getParams()) {
-            params[p.getName().cStr()] = p.getValue().cStr();
-        }
-        std::shared_ptr<Run> run = laminar.queueJob(jobName, params);
+        std::shared_ptr<Run> run = laminar.queueJob(jobName, params(context.getParams().getParams()));
         if(const Run* r = run.get()) {
             runWaiters[r].emplace_back(kj::newPromiseAndFulfiller<RunState>());
             return runWaiters[r].back().promise.then([context,run](RunState state) mutable {
@@ -148,8 +117,8 @@ public:
 
     // Set a parameter on a running build
     kj::Promise<void> set(SetContext context) override {
-        std::string jobName = context.getParams().getJobName();
-        uint buildNum = context.getParams().getBuildNum();
+        std::string jobName = context.getParams().getRun().getJob();
+        uint buildNum = context.getParams().getRun().getBuildNum();
         LLOG(INFO, "RPC set", jobName, buildNum);
 
         LaminarCi::MethodResult result = laminar.setParam(jobName, buildNum,
@@ -160,32 +129,62 @@ public:
         return kj::READY_NOW;
     }
 
-    // Take a named lock
-    kj::Promise<void> lock(LockContext context) override {
-        std::string lockName = context.getParams().getLockName();
-        LLOG(INFO, "RPC lock", lockName);
-        auto& lockList = locks[lockName];
-        lockList.emplace_back(kj::newPromiseAndFulfiller<void>());
-        if(lockList.size() == 1)
-            lockList.front().fulfiller->fulfill();
-        return std::move(lockList.back().promise);
-    }
-
-    // Release a named lock
-    kj::Promise<void> release(ReleaseContext context) override {
-        std::string lockName = context.getParams().getLockName();
-        LLOG(INFO, "RPC release", lockName);
-        auto& lockList = locks[lockName];
-        if(lockList.size() == 0) {
-            LLOG(INFO, "Attempt to release unheld lock", lockName);
-            return kj::READY_NOW;
+    // List jobs in queue
+    kj::Promise<void> listQueued(ListQueuedContext context) override {
+        const std::list<std::shared_ptr<Run>>& queue = laminar.listQueuedJobs();
+        auto res = context.getResults().initResult(queue.size());
+        int i = 0;
+        for(auto it : queue) {
+            res.set(i++, it->name);
         }
-        lockList.erase(lockList.begin());
-        if(lockList.size() > 0)
-            lockList.front().fulfiller->fulfill();
         return kj::READY_NOW;
     }
+
+    // List running jobs
+    kj::Promise<void> listRunning(ListRunningContext context) override {
+        const RunSet& active = laminar.listRunningJobs();
+        auto res = context.getResults().initResult(active.size());
+        int i = 0;
+        for(auto it : active) {
+            res[i].setJob(it->name);
+            res[i].setBuildNum(it->build);
+            i++;
+        }
+        return kj::READY_NOW;
+    }
+
+    // List known jobs
+    kj::Promise<void> listKnown(ListKnownContext context) override {
+        std::list<std::string> known = laminar.listKnownJobs();
+        auto res = context.getResults().initResult(known.size());
+        int i = 0;
+        for(auto it : known) {
+            res.set(i++, it);
+        }
+        return kj::READY_NOW;
+    }
+
+    kj::Promise<void> abort(AbortContext context) override {
+        std::string jobName = context.getParams().getRun().getJob();
+        uint buildNum = context.getParams().getRun().getBuildNum();
+        LLOG(INFO, "RPC abort", jobName, buildNum);
+        LaminarCi::MethodResult result = laminar.abort(jobName, buildNum)
+                ? LaminarCi::MethodResult::SUCCESS
+                : LaminarCi::MethodResult::FAILED;
+        context.getResults().setResult(result);
+        return kj::READY_NOW;
+    }
+
 private:
+    // Helper to convert an RPC parameter list to a hash map
+    ParamMap params(const capnp::List<LaminarCi::JobParam>::Reader& paramReader) {
+        ParamMap res;
+        for(auto p : paramReader) {
+            res[p.getName().cStr()] = p.getValue().cStr();
+        }
+        return res;
+    }
+
     // Implements LaminarWaiter::complete
     void complete(const Run* r) override {
         for(kj::PromiseFulfillerPair<RunState>& w : runWaiters[r])
@@ -194,122 +193,275 @@ private:
     }
 private:
     LaminarInterface& laminar;
-    std::unordered_map<std::string, std::list<kj::PromiseFulfillerPair<void>>> locks;
     std::unordered_map<const Run*, std::list<kj::PromiseFulfillerPair<RunState>>> runWaiters;
 };
 
-
-// This is the implementation of the HTTP/Websocket interface. It exposes
+// This is the implementation of the HTTP/Websocket interface. It creates
 // websocket connections as LaminarClients and registers them with the
 // LaminarInterface so that status messages will be delivered to the client.
 // On opening a websocket connection, it delivers a status snapshot message
 // (see LaminarInterface::sendStatus)
-class Server::HttpImpl {
+class HttpImpl : public kj::HttpService {
 public:
-    HttpImpl(LaminarInterface& l) :
-        laminar(l)
-    {
-        // debug logging
-        // wss.set_access_channels(websocketpp::log::alevel::all);
-        // wss.set_error_channels(websocketpp::log::elevel::all);
-
-        // TODO: This could be used in the future to trigger actions on the
-        // server in response to a web client request. Currently not supported.
-        // wss.set_message_handler([](std::weak_ptr<void> s, websocket::message_ptr msg){
-        //     msg->get_payload();
-        // });
-
-        // Handle plain HTTP requests by delivering the binary resource
-        wss.set_http_handler([this](websocketpp::connection_hdl hdl){
-            websocket::connection_ptr c = wss.get_con_from_hdl(hdl);
-            const char* start, *end, *content_type;
-            std::string resource = c->get_resource();
-            if(resource.compare(0, strlen("/archive/"), "/archive/") == 0) {
-                std::string file(resource.substr(strlen("/archive/")));
-                std::string content;
-                if(laminar.getArtefact(file, content)) {
-                    c->set_status(websocketpp::http::status_code::ok);
-                    c->append_header("Content-Transfer-Encoding", "binary");
-                    c->set_body(content);
-                } else {
-                    c->set_status(websocketpp::http::status_code::not_found);
-                }
-            } else if(resource.compare("/custom/style.css") == 0) {
-                c->set_status(websocketpp::http::status_code::ok);
-                c->append_header("Content-Type", "text/css; charset=utf-8");
-                c->append_header("Content-Transfer-Encoding", "binary");
-                c->set_body(laminar.getCustomCss());
-            } else if(resources.handleRequest(resource, &start, &end, &content_type)) {
-                c->set_status(websocketpp::http::status_code::ok);
-                c->append_header("Content-Type", content_type);
-                c->append_header("Content-Encoding", "gzip");
-                c->append_header("Content-Transfer-Encoding", "binary");
-                std::string response(start,end);
-                c->set_body(response);
-            } else {
-                // 404
-                c->set_status(websocketpp::http::status_code::not_found);
-            }
-        });
-
-        // Handle new websocket connection. Parse the URL to determine
-        // the client's scope of interest, register the client for update
-        // messages, and call sendStatus.
-        wss.set_open_handler([this](websocketpp::connection_hdl hdl){
-            websocket::connection_ptr c = wss.get_con_from_hdl(hdl);
-            std::string res = c->get_resource();
-            if(res.substr(0, 5) == "/jobs") {
-                if(res.length() == 5) {
-                    c->lc->scope.type = MonitorScope::ALL;
-                } else {
-                    res = res.substr(5);
-                    size_t split = res.find('/',1);
-                    std::string job = res.substr(1,split-1);
-                    if(!job.empty()) {
-                        c->lc->scope.job = job;
-                        c->lc->scope.type = MonitorScope::JOB;
-                    }
-                    if(split != std::string::npos) {
-                        size_t split2 = res.find('/', split+1);
-                        std::string run = res.substr(split+1, split2-split);
-                        if(!run.empty()) {
-                            c->lc->scope.num = static_cast<uint>(atoi(run.c_str()));
-                            c->lc->scope.type = MonitorScope::RUN;
-                        }
-                        if(split2 != std::string::npos && res.compare(split2, 4, "/log") == 0) {
-                            c->lc->scope.type = MonitorScope::LOG;
-                        }
-                    }
-                }
-            }
-            // registerClient can happen after a successful websocket handshake.
-            // However, the connection might not be closed gracefully, so the
-            // corresponding deregister operation happens in the connection
-            // destructor rather than the close handler
-            laminar.registerClient(c->lc);
-            laminar.sendStatus(c->lc);
-        });
-    }
-
-    // Return a new connection object linked with the context defined below.
-    // This is a bit untidy, it would be better to make them a single object,
-    // but I didn't yet figure it out
-    websocket::connection_ptr newConnection(LaminarClient* lc) {
-        websocket::connection_ptr c = wss.get_connection();
-        c->lc = lc;
-        return c;
-    }
-
-    void connectionDestroyed(LaminarClient* lc) {
-        // This will be called for all connections, not just websockets, so
-        // the laminar instance should silently ignore unknown clients
-        laminar.deregisterClient(lc);
-    }
+    HttpImpl(LaminarInterface& laminar, kj::HttpHeaderTable&tbl) :
+        laminar(laminar),
+        responseHeaders(tbl)
+    {}
+    virtual ~HttpImpl() {}
 
 private:
-    Resources resources;
+    class HttpChunkedClient : public LaminarClient {
+    public:
+        HttpChunkedClient(LaminarInterface& laminar) :
+            laminar(laminar)
+        {}
+        ~HttpChunkedClient() override {
+            laminar.deregisterClient(this);
+        }
+        void sendMessage(std::string payload) override {
+            chunks.push_back(kj::mv(payload));
+            fulfiller->fulfill();
+        }
+        void notifyJobFinished() override {
+            done = true;
+            fulfiller->fulfill();
+        }
+        LaminarInterface& laminar;
+        std::list<std::string> chunks;
+        // cannot use chunks.empty() because multiple fulfill()s
+        // could be coalesced
+        bool done = false;
+        kj::Own<kj::PromiseFulfiller<void>> fulfiller;
+    };
+
+    // Implements LaminarClient and holds the Websocket connection object.
+    // Automatically destructed when the promise created in request() resolves
+    // or is cancelled
+    class WebsocketClient : public LaminarClient {
+    public:
+        WebsocketClient(LaminarInterface& laminar, kj::Own<kj::WebSocket>&& ws) :
+            laminar(laminar),
+            ws(kj::mv(ws))
+        {}
+        ~WebsocketClient() override {
+            laminar.deregisterClient(this);
+        }
+        virtual void sendMessage(std::string payload) override {
+            messages.emplace_back(kj::mv(payload));
+            // sendMessage might be called several times before the event loop
+            // gets a chance to act on the fulfiller. So store the payload here
+            // where it can be fetched later and don't pass the payload with the
+            // fulfiller because subsequent calls to fulfill() are ignored.
+            fulfiller->fulfill();
+        }
+        LaminarInterface& laminar;
+        kj::Own<kj::WebSocket> ws;
+        std::list<std::string> messages;
+        kj::Own<kj::PromiseFulfiller<void>> fulfiller;
+    };
+
+    kj::Promise<void> websocketRead(WebsocketClient& lc)
+    {
+        return lc.ws->receive().then([&lc,this](kj::WebSocket::Message&& message) {
+            KJ_SWITCH_ONEOF(message) {
+                KJ_CASE_ONEOF(str, kj::String) {
+                    rapidjson::Document d;
+                    d.ParseInsitu(const_cast<char*>(str.cStr()));
+                    if(d.HasMember("page") && d["page"].IsInt() && d.HasMember("field") && d["field"].IsString() && d.HasMember("order") && d["order"].IsString()) {
+                        lc.scope.page = d["page"].GetInt();
+                        lc.scope.field = d["field"].GetString();
+                        lc.scope.order_desc = strcmp(d["order"].GetString(), "dsc") == 0;
+                        laminar.sendStatus(&lc);
+                        return websocketRead(lc);
+                    }
+                }
+                KJ_CASE_ONEOF(close, kj::WebSocket::Close) {
+                    // clean socket shutdown
+                    return lc.ws->close(close.code, close.reason);
+                }
+                KJ_CASE_ONEOF_DEFAULT {}
+            }
+            // unhandled/unknown message
+            return lc.ws->disconnect();
+        }, [](kj::Exception&& e){
+            // server logs suggest early catching here avoids fatal exception later
+            // TODO: reproduce in unit test
+            LLOG(WARNING, e.getDescription());
+            return kj::READY_NOW;
+        });
+    }
+
+    kj::Promise<void> websocketWrite(WebsocketClient& lc)
+    {
+        auto paf = kj::newPromiseAndFulfiller<void>();
+        lc.fulfiller = kj::mv(paf.fulfiller);
+        return paf.promise.then([this,&lc]{
+            kj::Promise<void> p = kj::READY_NOW;
+            std::list<std::string> messages = kj::mv(lc.messages);
+            for(std::string& s : messages) {
+                p = p.then([&s,&lc]{
+                    kj::String str = kj::str(s);
+                    return lc.ws->send(str).attach(kj::mv(str));
+                });
+            }
+            return p.attach(kj::mv(messages)).then([this,&lc]{
+                return websocketWrite(lc);
+            });
+        });
+    }
+
+    kj::Promise<void> websocketUpgraded(WebsocketClient& lc, std::string resource) {
+        // convert the requested URL to a MonitorScope
+        if(resource.substr(0, 5) == "/jobs") {
+            if(resource.length() == 5) {
+                lc.scope.type = MonitorScope::ALL;
+            } else {
+                resource = resource.substr(5);
+                size_t split = resource.find('/',1);
+                std::string job = resource.substr(1,split-1);
+                if(!job.empty()) {
+                    lc.scope.job = job;
+                    lc.scope.type = MonitorScope::JOB;
+                }
+                if(split != std::string::npos) {
+                    size_t split2 = resource.find('/', split+1);
+                    std::string run = resource.substr(split+1, split2-split);
+                    if(!run.empty()) {
+                        lc.scope.num = static_cast<uint>(atoi(run.c_str()));
+                        lc.scope.type = MonitorScope::RUN;
+                    }
+                    if(split2 != std::string::npos && resource.compare(split2, 4, "/log") == 0) {
+                        lc.scope.type = MonitorScope::LOG;
+                    }
+                }
+            }
+        }
+        laminar.registerClient(&lc);
+        kj::Promise<void> connection = websocketRead(lc).exclusiveJoin(websocketWrite(lc));
+        // registerClient can happen after a successful websocket handshake.
+        // However, the connection might not be closed gracefully, so the
+        // corresponding deregister operation happens in the WebsocketClient
+        // destructor rather than the close handler or a then() clause
+        laminar.sendStatus(&lc);
+        return connection;
+    }
+
+    // Parses the url of the form /log/NAME/NUMBER, filling in the passed
+    // references and returning true if successful. /log/NAME/latest is
+    // also allowed, in which case the num reference is set to 0
+    bool parseLogEndpoint(kj::StringPtr url, std::string& name, uint& num) {
+        if(url.startsWith("/log/")) {
+            kj::StringPtr path = url.slice(5);
+            KJ_IF_MAYBE(sep, path.findFirst('/')) {
+                name = path.slice(0, *sep).begin();
+                kj::StringPtr tail = path.slice(*sep+1);
+                num = static_cast<uint>(atoi(tail.begin()));
+                name.erase(*sep);
+                if(tail == "latest")
+                    num = laminar.latestRun(name);
+                if(num > 0)
+                    return true;
+            }
+        }
+        return false;
+    }
+
+    kj::Promise<void> writeLogChunk(HttpChunkedClient* client, kj::AsyncOutputStream* stream) {
+        auto paf = kj::newPromiseAndFulfiller<void>();
+        client->fulfiller = kj::mv(paf.fulfiller);
+        return paf.promise.then([=]{
+            kj::Promise<void> p = kj::READY_NOW;
+            std::list<std::string> chunks = kj::mv(client->chunks);
+            for(std::string& s : chunks) {
+                p = p.then([=,&s]{
+                    return stream->write(s.data(), s.size());
+                });
+            }
+            return p.attach(kj::mv(chunks)).then([=]{
+                return client->done ? kj::Promise<void>(kj::READY_NOW) : writeLogChunk(client, stream);
+            });
+        });
+    }
+
+    virtual kj::Promise<void> request(kj::HttpMethod method, kj::StringPtr url, const kj::HttpHeaders& headers,
+            kj::AsyncInputStream& requestBody, Response& response) override
+    {
+        if(headers.isWebSocket()) {
+            responseHeaders.clear();
+            kj::Own<WebsocketClient> lc = kj::heap<WebsocketClient>(laminar, response.acceptWebSocket(responseHeaders));
+            return websocketUpgraded(*lc, url.cStr()).attach(kj::mv(lc));
+        } else {
+            // handle regular HTTP request
+            const char* start, *end, *content_type;
+            std::string badge;
+            // for log requests
+            std::string name;
+            uint num;
+            responseHeaders.clear();
+            // Clients usually expect that http servers will ignore unknown query parameters,
+            // and expect to use this feature to work around browser limitations like there
+            // being no way to programatically force a resource to be reloaded from the server
+            // (without "Cache-Control: no-store", which is overkill). See issue #89.
+            // Since we currently don't handle any query parameters at all, the easiest way
+            // to achieve this is unconditionally remove all query parameters from the request.
+            // This will need to be redone if we ever accept query parameters, which probably
+            // will happen as part of issue #90.
+            KJ_IF_MAYBE(queryIdx, url.findFirst('?')) {
+                const_cast<char*>(url.begin())[*queryIdx] = '\0';
+                url = url.begin();
+            }
+            if(url.startsWith("/archive/")) {
+                KJ_IF_MAYBE(file, laminar.getArtefact(url.slice(strlen("/archive/")))) {
+                    auto array = (*file)->mmap(0, (*file)->stat().size);
+                    responseHeaders.add("Content-Transfer-Encoding", "binary");
+                    auto stream = response.send(200, "OK", responseHeaders, array.size());
+                    return stream->write(array.begin(), array.size()).attach(kj::mv(array)).attach(kj::mv(file)).attach(kj::mv(stream));
+                }
+            } else if(parseLogEndpoint(url, name, num)) {
+                kj::Own<HttpChunkedClient> cc = kj::heap<HttpChunkedClient>(laminar);
+                cc->scope.job = name;
+                cc->scope.num = num;
+                bool complete;
+                std::string output;
+                cc->scope.type = MonitorScope::LOG;
+                if(laminar.handleLogRequest(name, num, output, complete)) {
+                    responseHeaders.set(kj::HttpHeaderId::CONTENT_TYPE, "text/plain; charset=utf-8");
+                    responseHeaders.add("Content-Transfer-Encoding", "binary");
+                    // Disables nginx reverse-proxy's buffering. Necessary for dynamic log output.
+                    responseHeaders.add("X-Accel-Buffering", "no");
+                    auto stream = response.send(200, "OK", responseHeaders, nullptr);
+                    laminar.registerClient(cc.get());
+                    return stream->write(output.data(), output.size()).then([=,s=stream.get(),c=cc.get()]{
+                        if(complete)
+                            return kj::Promise<void>(kj::READY_NOW);
+                        return writeLogChunk(c, s);
+                    }).attach(kj::mv(output)).attach(kj::mv(stream)).attach(kj::mv(cc));
+                }
+            } else if(url == "/custom/style.css") {
+                responseHeaders.set(kj::HttpHeaderId::CONTENT_TYPE, "text/css; charset=utf-8");
+                responseHeaders.add("Content-Transfer-Encoding", "binary");
+                std::string css = laminar.getCustomCss();
+                auto stream = response.send(200, "OK", responseHeaders, css.size());
+                return stream->write(css.data(), css.size()).attach(kj::mv(css)).attach(kj::mv(stream));
+            } else if(resources.handleRequest(url.cStr(), &start, &end, &content_type)) {
+                responseHeaders.set(kj::HttpHeaderId::CONTENT_TYPE, content_type);
+                responseHeaders.add("Content-Encoding", "gzip");
+                responseHeaders.add("Content-Transfer-Encoding", "binary");
+                auto stream = response.send(200, "OK", responseHeaders, end-start);
+                return stream->write(start, end-start).attach(kj::mv(stream));
+            } else if(url.startsWith("/badge/") && url.endsWith(".svg") && laminar.handleBadgeRequest(std::string(url.begin()+7, url.size()-11), badge)) {
+                responseHeaders.set(kj::HttpHeaderId::CONTENT_TYPE, "image/svg+xml");
+                responseHeaders.add("Cache-Control", "no-cache");
+                auto stream = response.send(200, "OK", responseHeaders, badge.size());
+                return stream->write(badge.data(), badge.size()).attach(kj::mv(badge)).attach(kj::mv(stream));
+            }
+            return response.sendError(404, "Not Found", responseHeaders);
+        }
+    }
+
     LaminarInterface& laminar;
-    websocket wss;
+    Resources resources;
+    kj::HttpHeaders responseHeaders;
 };
 
 // Context for an RPC connection
@@ -327,77 +479,14 @@ struct RpcConnection {
     capnp::RpcSystem<capnp::rpc::twoparty::VatId> rpcSystem;
 };
 
-// Context for a WebsocketConnection (implements LaminarClient)
-// This object maps read and write handlers between the websocketpp library
-// and the corresponding kj async methods
-struct Server::WebsocketConnection : public LaminarClient {
-    WebsocketConnection(kj::Own<kj::AsyncIoStream>&& stream, Server::HttpImpl& http) :
-        http(http),
-        stream(kj::mv(stream)),
-        cn(http.newConnection(this)),
-        writePaf(kj::newPromiseAndFulfiller<void>())
-    {
-        cn->set_write_handler([this](websocketpp::connection_hdl, const char* s, size_t sz) {
-            outputBuffer.append(std::string(s, sz));
-            writePaf.fulfiller->fulfill();
-            return std::error_code();
-        });
-        cn->start();
-    }
-
-    virtual ~WebsocketConnection() noexcept(true) override {
-        // Removes the connection from the list of registered clients. Must be
-        // here rather than in the websocket closing handshake because connections
-        // might be unexpectedly/aggressively closed and any references must be
-        // removed.
-        http.connectionDestroyed(this);
-    }
-
-    kj::Promise<void> pend() {
-        return stream->tryRead(ibuf, 1, sizeof(ibuf)).then([this](size_t sz){
-            cn->read_some(ibuf, sz);
-            if(sz == 0 || cn->get_state() == websocketpp::session::state::closed) {
-                return kj::Promise<void>(kj::READY_NOW);
-            }
-            return pend();
-        });
-    }
-
-    kj::Promise<void> writeTask() {
-        return writePaf.promise.then([this]() {
-            std::string payload;
-            // clear the outputBuffer for more context, and take a chunk
-            // to send now
-            payload.swap(outputBuffer);
-            writePaf = kj::newPromiseAndFulfiller<void>();
-            KJ_ASSERT(!payload.empty());
-            return stream->write(payload.data(), payload.size()).then([this](){
-                if(cn->get_state() == websocketpp::session::state::closed) {
-                    return kj::Promise<void>(kj::READY_NOW);
-                }
-                return writeTask();
-            }).attach(kj::mv(payload));
-        });
-    }
-
-    void sendMessage(std::string payload) override {
-        cn->send(payload, websocketpp::frame::opcode::text);
-    }
-
-    HttpImpl& http;
-    kj::Own<kj::AsyncIoStream> stream;
-    websocket::connection_ptr cn;
-    std::string outputBuffer;
-    kj::PromiseFulfillerPair<void> writePaf;
-    char ibuf[131072];
-};
-
 Server::Server(LaminarInterface& li, kj::StringPtr rpcBindAddress,
                kj::StringPtr httpBindAddress) :
     rpcInterface(kj::heap<RpcImpl>(li)),
     laminarInterface(li),
-    httpInterface(kj::heap<HttpImpl>(li)),
     ioContext(kj::setupAsyncIo()),
+    headerTable(),
+    httpService(kj::heap<HttpImpl>(li, headerTable)),
+    httpServer(kj::heap<kj::HttpServer>(ioContext.provider->getTimer(), headerTable, *httpService)),
     listeners(kj::heap<kj::TaskSet>(*this)),
     childTasks(*this),
     httpConnections(*this),
@@ -418,33 +507,16 @@ Server::Server(LaminarInterface& li, kj::StringPtr rpcBindAddress,
               .then([this](kj::Own<kj::NetworkAddress>&& addr) {
         // TODO: a better way? Currently used only for testing
         httpReady.fulfiller->fulfill();
-        return acceptHttpClient(addr->listen());
+        kj::Own<kj::ConnectionReceiver> listener = addr->listen();
+        return httpServer->listenHttp(*listener).attach(kj::mv(listener));
     }));
 
-    // handle SIGCHLD
-    {
-        sigset_t mask;
-        sigemptyset(&mask);
-        sigaddset(&mask, SIGCHLD);
-        sigprocmask(SIG_BLOCK, &mask, nullptr);
-        int sigchld = signalfd(-1, &mask, SFD_NONBLOCK|SFD_CLOEXEC);
-        auto event = ioContext.lowLevelProvider->wrapInputFd(sigchld, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP);
-        auto buffer = kj::heapArrayBuilder<char>(PROC_IO_BUFSIZE);
-        reapWatch = handleFdRead(event, buffer.asPtr().begin(), [this](const char* buf, size_t){
-            const struct signalfd_siginfo* siginfo = reinterpret_cast<const struct signalfd_siginfo*>(buf);
-            KJ_ASSERT(siginfo->ssi_signo == SIGCHLD);
-            laminarInterface.reapChildren();
-        }).attach(std::move(event)).attach(std::move(buffer));
-    }
-
     // handle watched paths
     {
         inotify_fd = inotify_init1(IN_NONBLOCK | IN_CLOEXEC);
-        auto event = ioContext.lowLevelProvider->wrapInputFd(inotify_fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP);
-        auto buffer = kj::heapArrayBuilder<char>(PROC_IO_BUFSIZE);
-        pathWatch = handleFdRead(event, buffer.asPtr().begin(), [this](const char*, size_t){
+        pathWatch = readDescriptor(inotify_fd, [this](const char*, size_t){
             laminarInterface.notifyConfigChanged();
-        }).attach(std::move(event)).attach(std::move(buffer));
+        });
     }
 }
 
@@ -473,7 +545,7 @@ void Server::start() {
     childTasks.onEmpty().wait(ioContext.waitScope);
     // 4. run the loop once more to send any pending output to websocket clients
     ioContext.waitScope.poll();
-    // 5. return: websockets will be destructed
+    // 5. return: websockets will be destructed when class is deleted
 }
 
 void Server::stop() {
@@ -482,10 +554,14 @@ void Server::stop() {
     eventfd_write(efd_quit, 1);
 }
 
-void Server::addDescriptor(int fd, std::function<void(const char*,size_t)> cb) {
+kj::Promise<void> Server::readDescriptor(int fd, std::function<void(const char*,size_t)> cb) {
     auto event = this->ioContext.lowLevelProvider->wrapInputFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP);
     auto buffer = kj::heapArrayBuilder<char>(PROC_IO_BUFSIZE);
-    childTasks.add(handleFdRead(event, buffer.asPtr().begin(), cb).attach(std::move(event)).attach(std::move(buffer)));
+    return handleFdRead(event, buffer.asPtr().begin(), cb).attach(std::move(event)).attach(std::move(buffer));
+}
+
+void Server::addTask(kj::Promise<void>&& task) {
+    childTasks.add(kj::mv(task));
 }
 
 kj::Promise<void> Server::addTimeout(int seconds, std::function<void ()> cb) {
@@ -494,19 +570,12 @@ kj::Promise<void> Server::addTimeout(int seconds, std::function<void ()> cb) {
     }).eagerlyEvaluate(nullptr);
 }
 
-void Server::addWatchPath(const char* dpath) {
-    inotify_add_watch(inotify_fd, dpath, IN_ONLYDIR | IN_CLOSE_WRITE | IN_CREATE | IN_DELETE);
+kj::Promise<int> Server::onChildExit(kj::Maybe<pid_t> &pid) {
+    return ioContext.unixEventPort.onChildExit(pid);
 }
 
-kj::Promise<void> Server::acceptHttpClient(kj::Own<kj::ConnectionReceiver>&& listener) {
-    kj::ConnectionReceiver& cr = *listener.get();
-    return cr.accept().then(kj::mvCapture(kj::mv(listener),
-        [this](kj::Own<kj::ConnectionReceiver>&& listener, kj::Own<kj::AsyncIoStream>&& connection) {
-            auto conn = kj::heap<WebsocketConnection>(kj::mv(connection), *httpInterface);
-            // delete the connection when either the read or write task completes
-            httpConnections.add(conn->pend().exclusiveJoin(conn->writeTask()).attach(kj::mv(conn)));
-            return acceptHttpClient(kj::mv(listener));
-        }));
+void Server::addWatchPath(const char* dpath) {
+    inotify_add_watch(inotify_fd, dpath, IN_ONLYDIR | IN_CLOSE_WRITE | IN_CREATE | IN_DELETE);
 }
 
 kj::Promise<void> Server::acceptRpcClient(kj::Own<kj::ConnectionReceiver>&& listener) {
@@ -533,8 +602,8 @@ kj::Promise<void> Server::handleFdRead(kj::AsyncInputStream* stream, char* buffe
 }
 
 void Server::taskFailed(kj::Exception &&exception) {
-    // An unexpected http connection close can cause an exception, so don't re-throw.
-    // TODO: consider re-throwing selected exceptions
-    LLOG(INFO, exception);
     //kj::throwFatalException(kj::mv(exception));
+    // prettier
+    fprintf(stderr, "fatal: %s\n", exception.getDescription().cStr());
+    exit(EXIT_FAILURE);
 }

src/server.h

@@ -20,6 +20,7 @@
 #define LAMINAR_SERVER_H_
 
 #include <kj/async-io.h>
+#include <kj/compat/http.h>
 #include <capnp/message.h>
 #include <capnp/capability.h>
 #include <functional>
@@ -41,30 +42,31 @@ public:
 
     // add a file descriptor to be monitored for output. The callback will be
     // invoked with the read data
-    void addDescriptor(int fd, std::function<void(const char*,size_t)> cb);
+    kj::Promise<void> readDescriptor(int fd, std::function<void(const char*,size_t)> cb);
 
+    void addTask(kj::Promise<void> &&task);
     // add a one-shot timer callback
     kj::Promise<void> addTimeout(int seconds, std::function<void()> cb);
 
+    // get a promise which resolves when a child process exits
+    kj::Promise<int> onChildExit(kj::Maybe<pid_t>& pid);
     // add a path to be watched for changes
     void addWatchPath(const char* dpath);
 
 private:
-    kj::Promise<void> acceptHttpClient(kj::Own<kj::ConnectionReceiver>&& listener);
     kj::Promise<void> acceptRpcClient(kj::Own<kj::ConnectionReceiver>&& listener);
     kj::Promise<void> handleFdRead(kj::AsyncInputStream* stream, char* buffer, std::function<void(const char*,size_t)> cb);
 
     void taskFailed(kj::Exception&& exception) override;
 
 private:
-    struct WebsocketConnection;
-    class HttpImpl;
-
     int efd_quit;
     capnp::Capability::Client rpcInterface;
     LaminarInterface& laminarInterface;
-    kj::Own<HttpImpl> httpInterface;
     kj::AsyncIoContext ioContext;
+    kj::HttpHeaderTable headerTable;
+    kj::Own<kj::HttpService> httpService;
+    kj::Own<kj::HttpServer> httpServer;
     kj::Own<kj::TaskSet> listeners;
     kj::TaskSet childTasks;
     kj::TaskSet httpConnections;

test/tempdir.h

@@ -0,0 +1,45 @@
+///
+/// Copyright 2018 Oliver Giles
+///
+/// This file is part of Laminar
+///
+/// Laminar is free software: you can redistribute it and/or modify
+/// it under the terms of the GNU General Public License as published by
+/// the Free Software Foundation, either version 3 of the License, or
+/// (at your option) any later version.
+///
+/// Laminar is distributed in the hope that it will be useful,
+/// but WITHOUT ANY WARRANTY; without even the implied warranty of
+/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+/// GNU General Public License for more details.
+///
+/// You should have received a copy of the GNU General Public License
+/// along with Laminar.  If not, see <http://www.gnu.org/licenses/>
+///
+#ifndef LAMINAR_TEMPDIR_H_
+#define LAMINAR_TEMPDIR_H_
+
+#include <kj/filesystem.h>
+#include <stdlib.h>
+
+class TempDir {
+public:
+    TempDir() :
+        path(mkdtemp()),
+        fs(kj::newDiskFilesystem()->getRoot().openSubdir(path, kj::WriteMode::CREATE|kj::WriteMode::MODIFY))
+    {
+    }
+    ~TempDir() noexcept {
+        kj::newDiskFilesystem()->getRoot().remove(path);
+    }
+    kj::Path path;
+    kj::Own<const kj::Directory> fs;
+private:
+    static kj::Path mkdtemp() {
+        char dir[] = "/tmp/laminar-test-XXXXXX";
+        ::mkdtemp(dir);
+        return kj::Path::parse(&dir[1]);
+    }
+};
+
+#endif // LAMINAR_TEMPDIR_H_

test/test-database.cpp

@@ -73,3 +73,11 @@ TEST_F(DatabaseTest, MultiRow) {
     });
     EXPECT_EQ(10, i);
 }
+
+TEST_F(DatabaseTest, StdevFunc) {
+    double res = 0;
+    db.stmt("with a (x) as (values (7),(3),(45),(23)) select stdev(x) from a").fetch<double>([&](double r){
+        res = r;
+    });
+    EXPECT_FLOAT_EQ(19.0700463205171, res);
+}

test/test-laminar.cpp

@@ -27,8 +27,12 @@ public:
     std::string payload;
 };
 
-class LaminarTest : public ::testing::Test {
+class LaminarTest : public testing::Test {
 protected:
+    LaminarTest() :
+        testing::Test(),
+        laminar("/tmp")
+    {}
     Laminar laminar;
 };
 

test/test-run.cpp

@@ -21,27 +21,35 @@
 #include "log.h"
 #include "node.h"
 #include "conf.h"
+#include "tempdir.h"
 
-class RunTest : public ::testing::Test {
+class RunTest : public testing::Test {
 protected:
-    virtual ~RunTest() noexcept override {}
-
-    void SetUp() override {
-        run.node = node;
+    RunTest() :
+        testing::Test(),
+        node(std::make_shared<Node>()),
+        tmp(),
+        run("foo", ParamMap{}, tmp.path.clone())
+    {
     }
+
+    ~RunTest() noexcept {}
+
     void wait() {
         int state = -1;
-        waitpid(run.pid, &state, 0);
+        waitpid(run.current_pid.orDefault(0), &state, 0);
         run.reaped(state);
     }
+
     void runAll() {
         while(!run.step())
             wait();
     }
+
     std::string readAllOutput() {
         std::string res;
         char tmp[64];
-        for(ssize_t n = read(run.fd, tmp, 64); n > 0; n = read(run.fd, tmp, 64))
+        for(ssize_t n = read(run.output_fd, tmp, 64); n > 0; n = read(run.output_fd, tmp, 64))
             res += std::string(tmp, n);
         // strip the first "[laminar] executing.. line
         return strchr(res.c_str(), '\n') + 1;
@@ -56,67 +64,70 @@ protected:
         return map;
     }
 
+    std::shared_ptr<Node> node;
+    TempDir tmp;
     class Run run;
-    std::shared_ptr<Node> node = std::shared_ptr<Node>(new Node);
+
+    void setRunLink(const char* path) {
+        tmp.fs->symlink(kj::Path{"cfg","jobs",run.name+".run"}, path, kj::WriteMode::CREATE|kj::WriteMode::CREATE_PARENT|kj::WriteMode::EXECUTABLE);
+    }
 };
 
 TEST_F(RunTest, WorkingDirectory) {
-    run.addScript("/bin/pwd", "/home");
+    setRunLink("/bin/pwd");
+    run.configure(1, node, *tmp.fs);
     runAll();
-    EXPECT_EQ("/home\n", readAllOutput());
+    std::string cwd{tmp.path.append(kj::Path{"run","foo","1"}).toString(true).cStr()};
+    EXPECT_EQ(cwd + "\n", readAllOutput());
 }
 
 TEST_F(RunTest, SuccessStatus) {
-    run.addScript("/bin/true");
+    setRunLink("/bin/true");
+    run.configure(1, node, *tmp.fs);
     runAll();
     EXPECT_EQ(RunState::SUCCESS, run.result);
 }
 
 TEST_F(RunTest, FailedStatus) {
-    run.addScript("/bin/false");
+    setRunLink("/bin/false");
+    run.configure(1, node, *tmp.fs);
     runAll();
     EXPECT_EQ(RunState::FAILED, run.result);
 }
 
 TEST_F(RunTest, Environment) {
-    run.name = "foo";
-    run.build = 1234;
-    run.laminarHome = "/tmp";
-    run.addScript("/usr/bin/env");
+    setRunLink("/usr/bin/env");
+    run.configure(1234, node, *tmp.fs);
     runAll();
+
+    std::string ws{tmp.path.append(kj::Path{"run","foo","workspace"}).toString(true).cStr()};
+    std::string archive{tmp.path.append(kj::Path{"archive","foo","1234"}).toString(true).cStr()};
+
     StringMap map = parseFromString(readAllOutput());
     EXPECT_EQ("1234", map["RUN"]);
     EXPECT_EQ("foo", map["JOB"]);
     EXPECT_EQ("success", map["RESULT"]);
     EXPECT_EQ("unknown", map["LAST_RESULT"]);
-    EXPECT_EQ("/tmp/run/foo/workspace", map["WORKSPACE"]);
-    EXPECT_EQ("/tmp/archive/foo/1234", map["ARCHIVE"]);
+    EXPECT_EQ(ws, map["WORKSPACE"]);
+    EXPECT_EQ(archive, map["ARCHIVE"]);
 }
 
 TEST_F(RunTest, ParamsToEnv) {
+    setRunLink("/usr/bin/env");
     run.params["foo"] = "bar";
-    run.addScript("/usr/bin/env");
+    run.configure(1, node, *tmp.fs);
     runAll();
     StringMap map = parseFromString(readAllOutput());
     EXPECT_EQ("bar", map["foo"]);
 }
 
 TEST_F(RunTest, Abort) {
-    run.addScript("/usr/bin/yes");
+    setRunLink("/usr/bin/yes");
+    run.configure(1, node, *tmp.fs);
     run.step();
     usleep(200); // TODO fix
-    run.abort();
+    run.abort(false);
     wait();
     EXPECT_EQ(RunState::ABORTED, run.result);
 }
 
-TEST_F(RunTest, AbortAfterFailed) {
-    run.addScript("/bin/false");
-    runAll();
-    run.addScript("/usr/bin/yes");
-    run.step();
-    usleep(200); // TODO fix
-    run.abort();
-    wait();
-    EXPECT_EQ(RunState::FAILED, run.result);
-}

test/test-server.cpp

@@ -18,32 +18,18 @@
 ///
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
-#include <boost/filesystem.hpp>
 #include <thread>
 #include <sys/socket.h>
 #include "server.h"
 #include "log.h"
 #include "interface.h"
 #include "laminar.capnp.h"
-
-namespace fs = boost::filesystem;
-
-class TempDir : public fs::path {
-public:
-    TempDir(const char* tmpl) {
-        char* t = strdup(tmpl);
-        mkdtemp(t);
-        *static_cast<fs::path*>(this) = t;
-        free(t);
-    }
-    ~TempDir() {
-        fs::remove_all(*this);
-    }
-};
+#include "tempdir.h"
 
 class MockLaminar : public LaminarInterface {
 public:
     LaminarClient* client = nullptr;
+    ~MockLaminar() {}
     virtual void registerClient(LaminarClient* c) override {
         ASSERT_EQ(nullptr, client);
         client = c;
@@ -55,28 +41,34 @@ public:
         client = nullptr;
     }
 
+    // MOCK_METHOD does not seem to work with return values whose destructors have noexcept(false)
+    kj::Maybe<kj::Own<const kj::ReadableFile>> getArtefact(std::string path) override { return nullptr; }
+
     MOCK_METHOD2(queueJob, std::shared_ptr<Run>(std::string name, ParamMap params));
     MOCK_METHOD1(registerWaiter, void(LaminarWaiter* waiter));
     MOCK_METHOD1(deregisterWaiter, void(LaminarWaiter* waiter));
+    MOCK_METHOD1(latestRun, uint(std::string));
+    MOCK_METHOD4(handleLogRequest, bool(std::string, uint, std::string&, bool&));
+
     MOCK_METHOD1(sendStatus, void(LaminarClient* client));
     MOCK_METHOD4(setParam, bool(std::string job, uint buildNum, std::string param, std::string value));
-    MOCK_METHOD2(getArtefact, bool(std::string path, std::string& result));
+    MOCK_METHOD0(listQueuedJobs, const std::list<std::shared_ptr<Run>>&());
+    MOCK_METHOD0(listRunningJobs, const RunSet&());
+    MOCK_METHOD0(listKnownJobs, std::list<std::string>());
+
     MOCK_METHOD0(getCustomCss, std::string());
+    MOCK_METHOD2(handleBadgeRequest, bool(std::string, std::string&));
+    MOCK_METHOD2(abort, bool(std::string, uint));
     MOCK_METHOD0(abortAll, void());
-    MOCK_METHOD0(reapChildren, void());
     MOCK_METHOD0(notifyConfigChanged, void());
 };
 
 class ServerTest : public ::testing::Test {
 protected:
-    ServerTest() :
-        tempDir("/tmp/laminar-test-XXXXXX")
-    {
-    }
     void SetUp() override {
         EXPECT_CALL(mockLaminar, registerWaiter(testing::_));
         EXPECT_CALL(mockLaminar, deregisterWaiter(testing::_));
-        server = new Server(mockLaminar, "unix:"+fs::path(tempDir/"rpc.sock").string(), "127.0.0.1:8080");
+        server = new Server(mockLaminar, "unix:"+std::string(tempDir.path.append("rpc.sock").toString(true).cStr()), "127.0.0.1:8080");
     }
     void TearDown() override {
         delete server;