Replace boost/filesystem with kj/filesystem

Lose the boost dependency since recent versions of capnproto's kj
also provide a nice filesystem library. Take the opportunity to
refactor the Run object to become more than POD and to encapsulate
some of the functionality that was done in the Laminar class

Part of #49 refactor

CMakeLists.txt

@@ -1,5 +1,5 @@
 ###
-### Copyright 2015-2017 Oliver Giles
+### Copyright 2015-2018 Oliver Giles
 ###
 ### This file is part of Laminar
 ###
@@ -83,8 +83,7 @@ generate_compressed_bins(${CMAKE_BINARY_DIR} js/vue-router.min.js js/vue.min.js
 ## Server
 add_executable(laminard src/database.cpp src/main.cpp src/server.cpp src/laminar.cpp
     src/conf.cpp src/resources.cpp src/run.cpp laminar.capnp.c++ ${COMPRESSED_BINS})
-# TODO: some alternative to boost::filesystem?
-target_link_libraries(laminard capnp-rpc capnp kj-http kj-async kj pthread boost_filesystem boost_system sqlite3 z)
+target_link_libraries(laminard capnp-rpc capnp kj-http kj-async kj pthread sqlite3 z)
 
 ## Client
 add_executable(laminarc src/client.cpp laminar.capnp.c++)
@@ -96,7 +95,7 @@ if(BUILD_TESTS)
     find_package(GTest REQUIRED)
     include_directories(${GTEST_INCLUDE_DIRS} src)
     add_executable(laminar-tests src/conf.cpp src/database.cpp src/laminar.cpp src/run.cpp src/server.cpp laminar.capnp.c++ src/resources.cpp ${COMPRESSED_BINS} test/test-conf.cpp test/test-database.cpp test/test-laminar.cpp test/test-run.cpp test/test-server.cpp)
-    target_link_libraries(laminar-tests ${GTEST_BOTH_LIBRARIES} gmock capnp-rpc capnp kj-http kj-async kj pthread boost_filesystem boost_system sqlite3 z)
+    target_link_libraries(laminar-tests ${GTEST_BOTH_LIBRARIES} gmock capnp-rpc capnp kj-http kj-async kj pthread sqlite3 z)
 endif()
 
 set(SYSTEMD_UNITDIR /lib/systemd/system CACHE PATH "Path to systemd unit files")

docker-build-centos.sh

@@ -46,7 +46,7 @@ Version: $VERSION
 Release: 1
 License: GPL
 BuildRequires: systemd-units
-Requires: sqlite boost-filesystem zlib
+Requires: sqlite zlib
 
 %description
 Lightweight Continuous Integration Service

docker-build-debian.sh

@@ -8,7 +8,7 @@ VERSION=$(cd "$SOURCE_DIR" && git describe --tags --abbrev=8 --dirty)
 
 DOCKER_TAG=$(docker build -q - <<EOS
 FROM debian:9-slim
-RUN apt-get update && apt-get install -y wget cmake g++ libsqlite3-dev libboost-filesystem1.62-dev zlib1g-dev
+RUN apt-get update && apt-get install -y wget cmake g++ libsqlite3-dev libboost-dev zlib1g-dev
 EOS
 )
 
@@ -50,7 +50,7 @@ Section:
 Priority: optional
 Architecture: amd64
 Maintainer: Oliver Giles <web ohwg net>
-Depends: libsqlite3-0, libboost-filesystem1.62.0, zlib1g
+Depends: libsqlite3-0, zlib1g
 Description: Lightweight Continuous Integration Service
 EOF
 cat <<EOF > laminar/DEBIAN/postinst

src/interface.h

@@ -25,8 +25,6 @@
 #include <memory>
 #include <unordered_map>
 
-typedef std::unordered_map<std::string, std::string> ParamMap;
-
 // Simple struct to define which information a frontend client is interested
 // in, both in initial request phase and real-time updates. It corresponds
 // loosely to frontend URLs
@@ -133,7 +131,7 @@ struct LaminarInterface {
     // Fetches the content of an artifact given its filename relative to
     // $LAMINAR_HOME/archive. Ideally, this would instead be served by a
     // proper web server which handles this url.
-    virtual kj::Own<MappedFile> getArtefact(std::string path) = 0;
+    virtual kj::Maybe<kj::Own<const kj::ReadableFile>> getArtefact(std::string path) = 0;
 
     // Given the name of a job, populate the provided string reference with
     // SVG content describing the last known state of the job. Returns false

src/laminar.cpp

@@ -29,9 +29,6 @@
 #include <fstream>
 #include <zlib.h>
 
-#include <boost/filesystem.hpp>
-namespace fs = boost::filesystem;
-
 #define COMPRESS_LOG_MIN_SIZE 1024
 
 #include <rapidjson/stringbuffer.h>
@@ -62,23 +59,30 @@ constexpr const char* INTADDR_HTTP_DEFAULT = "*:8080";
 constexpr const char* ARCHIVE_URL_DEFAULT = "/archive";
 }
 
-// helper for appending to boost::filesystem::path
-fs::path operator+(fs::path p, const char* ext) {
-    std::string leaf = p.leaf().string();
-    leaf += ext;
-    return p.remove_leaf()/leaf;
+// short syntax helpers for kj::Path
+template<typename T>
+inline kj::Path operator/(const kj::Path& p, const T& ext) {
+    return p.append(ext);
+}
+template<typename T>
+inline kj::Path operator/(const std::string& p, const T& ext) {
+    return kj::Path{p}/ext;
 }
 
 typedef std::string str;
 
-Laminar::Laminar() {
+Laminar::Laminar(const char *home) :
+    homePath(kj::Path::parse(&home[1])),
+    fsHome(kj::newDiskFilesystem()->getRoot().openSubdir(homePath, kj::WriteMode::MODIFY))
+{
+    KJ_ASSERT(home[0] == '/');
+
     archiveUrl = ARCHIVE_URL_DEFAULT;
     if(char* envArchive = getenv("LAMINAR_ARCHIVE_URL"))
         archiveUrl = envArchive;
     numKeepRunDirs = 0;
-    homeDir = getenv("LAMINAR_HOME") ?: "/var/lib/laminar";
 
-    db = new Database((fs::path(homeDir)/"laminar.sqlite").string().c_str());
+    db = new Database((homePath/"laminar.sqlite").toString(true).cStr());
     // Prepare database for first use
     // TODO: error handling
     db->exec("CREATE TABLE IF NOT EXISTS builds("
@@ -128,17 +132,16 @@ bool Laminar::setParam(std::string job, uint buildNum, std::string param, std::s
 
 
 void Laminar::populateArtifacts(Json &j, std::string job, uint num) const {
-    fs::path dir(fs::path(homeDir)/"archive"/job/std::to_string(num));
-    if(fs::is_directory(dir)) {
-        size_t prefixLen = (fs::path(homeDir)/"archive").string().length();
-        size_t scopeLen = dir.string().length();
-        for(fs::recursive_directory_iterator it(dir); it != fs::recursive_directory_iterator(); ++it) {
-            if(!fs::is_regular_file(*it))
+    kj::Path runArchive{job,std::to_string(num)};
+    KJ_IF_MAYBE(dir, fsHome->tryOpenSubdir("archive"/runArchive)) {
+        for(kj::StringPtr file : (*dir)->listNames()) {
+            kj::FsNode::Metadata meta = (*dir)->lstat(kj::Path{file});
+            if(meta.type != kj::FsNode::Type::FILE)
                 continue;
             j.StartObject();
-            j.set("url", archiveUrl + it->path().string().substr(prefixLen));
-            j.set("filename", it->path().string().substr(scopeLen+1));
-            j.set("size", fs::file_size(it->path()));
+            j.set("url", archiveUrl + (runArchive/file).toString().cStr());
+            j.set("filename", file.cStr());
+            j.set("size", meta.size);
             j.EndObject();
         }
     }
@@ -440,9 +443,12 @@ void Laminar::sendStatus(LaminarClient* client) {
     client->sendMessage(j.str());
 }
 
-Laminar::~Laminar() {
+Laminar::~Laminar() noexcept try {
     delete db;
     delete srv;
+} catch (std::exception& e) {
+    LLOG(ERROR, e.what());
+    return;
 }
 
 void Laminar::run() {
@@ -450,8 +456,8 @@ void Laminar::run() {
     const char* listen_http = getenv("LAMINAR_BIND_HTTP") ?: INTADDR_HTTP_DEFAULT;
 
     srv = new Server(*this, listen_rpc, listen_http);
-    srv->addWatchPath(fs::path(fs::path(homeDir)/"cfg"/"nodes").string().c_str());
-    srv->addWatchPath(fs::path(fs::path(homeDir)/"cfg"/"jobs").string().c_str());
+    srv->addWatchPath((homePath/"cfg"/"nodes").toString().cStr());
+    srv->addWatchPath((homePath/"cfg"/"jobs").toString().cStr());
     srv->start();
 }
 
@@ -465,16 +471,14 @@ bool Laminar::loadConfiguration() {
 
     std::set<std::string> knownNodes;
 
-    fs::path nodeCfg = fs::path(homeDir)/"cfg"/"nodes";
-
-    if(fs::is_directory(nodeCfg)) {
-        for(fs::directory_iterator it(nodeCfg); it != fs::directory_iterator(); ++it) {
-            if(!fs::is_regular_file(it->status()) || it->path().extension() != ".conf")
+    KJ_IF_MAYBE(nodeDir, fsHome->tryOpenSubdir(kj::Path{"cfg","nodes"})) {
+        for(kj::Directory::Entry& entry : (*nodeDir)->listEntries()) {
+            if(entry.type != kj::FsNode::Type::FILE || !entry.name.endsWith(".conf"))
                 continue;
 
-            StringMap conf = parseConfFile(it->path().string().c_str());
+            StringMap conf = parseConfFile((homePath/entry.name).toString().cStr());
 
-            std::string nodeName = it->path().stem().string();
+            std::string nodeName(entry.name.cStr(), entry.name.findLast('.').orDefault(0)-1);
             auto existingNode = nodes.find(nodeName);
             std::shared_ptr<Node> node = existingNode == nodes.end() ? nodes.emplace(nodeName, std::shared_ptr<Node>(new Node)).first->second : existingNode->second;
             node->name = nodeName;
@@ -511,13 +515,13 @@ bool Laminar::loadConfiguration() {
         nodes.emplace("", node);
     }
 
-    fs::path jobsDir = fs::path(homeDir)/"cfg"/"jobs";
-    if(fs::is_directory(jobsDir)) {
-        for(fs::directory_iterator it(jobsDir); it != fs::directory_iterator(); ++it) {
-            if(!fs::is_regular_file(it->status()) || it->path().extension() != ".conf")
+    KJ_IF_MAYBE(jobsDir, fsHome->tryOpenSubdir(kj::Path{"cfg","jobs"})) {
+        for(kj::Directory::Entry& entry : (*jobsDir)->listEntries()) {
+            if(entry.type != kj::FsNode::Type::FILE || !entry.name.endsWith(".conf"))
                 continue;
+            StringMap conf = parseConfFile((homePath/entry.name).toString().cStr());
 
-            StringMap conf = parseConfFile(it->path().string().c_str());
+            std::string jobName(entry.name.cStr(), entry.name.findLast('.').orDefault(0)-1);
 
             std::string tags = conf.get<std::string>("TAGS");
             if(!tags.empty()) {
@@ -526,7 +530,7 @@ bool Laminar::loadConfiguration() {
                 std::string tag;
                 while(std::getline(iss, tag, ','))
                     tagList.insert(tag);
-                jobTags[it->path().stem().string()] = tagList;
+                jobTags[jobName] = tagList;
             }
 
         }
@@ -536,30 +540,12 @@ bool Laminar::loadConfiguration() {
 }
 
 std::shared_ptr<Run> Laminar::queueJob(std::string name, ParamMap params) {
-    if(!fs::exists(fs::path(homeDir)/"cfg"/"jobs"/name+".run")) {
+    if(!fsHome->exists(kj::Path{"cfg","jobs",name+".run"})) {
         LLOG(ERROR, "Non-existent job", name);
         return nullptr;
     }
 
-    std::shared_ptr<Run> run = std::make_shared<Run>();
-    run->name = name;
-    run->queuedAt = time(nullptr);
-    for(auto it = params.begin(); it != params.end();) {
-        if(it->first[0] == '=') {
-            if(it->first == "=parentJob") {
-                run->parentName = it->second;
-            } else if(it->first == "=parentBuild") {
-                run->parentBuild = atoi(it->second.c_str());
-            } else if(it->first == "=reason") {
-                run->reasonMsg = it->second;
-            } else {
-                LLOG(ERROR, "Unknown internal job parameter", it->first);
-            }
-            it = params.erase(it);
-        } else
-            ++it;
-    }
-    run->params = params;
+    std::shared_ptr<Run> run = std::make_shared<Run>(name, kj::mv(params), homePath.clone());
     queuedJobs.push_back(run);
 
     // notify clients
@@ -591,7 +577,7 @@ void Laminar::abortAll() {
     }
 }
 
-bool Laminar::nodeCanQueue(const Node& node, const Run& run) const {
+bool Laminar::nodeCanQueue(const Node& node, std::string jobName) const {
     // if a node is too busy, it can't take the job
     if(node.busyExecutors >= node.numExecutors)
         return false;
@@ -600,7 +586,7 @@ bool Laminar::nodeCanQueue(const Node& node, const Run& run) const {
     if(node.tags.size() == 0)
         return true;
 
-    auto it = jobTags.find(run.name);
+    auto it = jobTags.find(jobName);
     // if the job has no tags, it cannot be run on this node
     if(it == jobTags.end())
         return false;
@@ -617,109 +603,30 @@ bool Laminar::nodeCanQueue(const Node& node, const Run& run) const {
 bool Laminar::tryStartRun(std::shared_ptr<Run> run, int queueIndex) {
     for(auto& sn : nodes) {
         std::shared_ptr<Node> node = sn.second;
-        if(nodeCanQueue(*node.get(), *run)) {
-            fs::path cfgDir = fs::path(homeDir)/"cfg";
-            boost::system::error_code err;
-
-            // create a workspace for this job if it doesn't exist
-            fs::path ws = fs::path(homeDir)/"run"/run->name/"workspace";
-            if(!fs::exists(ws)) {
-                if(!fs::create_directories(ws, err)) {
-                    LLOG(ERROR, "Could not create job workspace", run->name);
-                    break;
-                }
-                // prepend the workspace init script
-                if(fs::exists(cfgDir/"jobs"/run->name+".init"))
-                    run->addScript((cfgDir/"jobs"/run->name+".init").string(), ws.string());
-            }
-
-            uint buildNum = buildNums[run->name] + 1;
-            // create the run directory
-            fs::path rd = fs::path(homeDir)/"run"/run->name/std::to_string(buildNum);
-            bool createWorkdir = true;
-            if(fs::is_directory(rd)) {
-                LLOG(WARNING, "Working directory already exists, removing", rd.string());
-                fs::remove_all(rd, err);
-                if(err) {
-                    LLOG(WARNING, "Failed to remove working directory", err.message());
-                    createWorkdir = false;
-                }
-            }
-            if(createWorkdir && !fs::create_directory(rd, err)) {
-                LLOG(ERROR, "Could not create working directory", rd.string());
-                break;
-            }
-            run->runDir = rd.string();
-
-            // create an archive directory
-            fs::path archive = fs::path(homeDir)/"archive"/run->name/std::to_string(buildNum);
-            if(fs::is_directory(archive)) {
-                LLOG(WARNING, "Archive directory already exists", archive.string());
-            } else if(!fs::create_directories(archive)) {
-                LLOG(ERROR, "Could not create archive directory", archive.string());
-                break;
-            }
 
-            // add scripts
-            // global before-run script
-            if(fs::exists(cfgDir/"before"))
-                run->addScript((cfgDir/"before").string());
-            // per-node before-run script
-            if(fs::exists(cfgDir/"nodes"/node->name+".before"))
-                run->addScript((cfgDir/"nodes"/node->name+".before").string());
-            // job before-run script
-            if(fs::exists(cfgDir/"jobs"/run->name+".before"))
-                run->addScript((cfgDir/"jobs"/run->name+".before").string());
-            // main run script. must exist.
-            run->addScript((cfgDir/"jobs"/run->name+".run").string());
-            // job after-run script
-            if(fs::exists(cfgDir/"jobs"/run->name+".after"))
-                run->addScript((cfgDir/"jobs"/run->name+".after").string(), true);
-            // per-node after-run script
-            if(fs::exists(cfgDir/"nodes"/node->name+".after"))
-                run->addScript((cfgDir/"nodes"/node->name+".after").string(), true);
-            // global after-run script
-            if(fs::exists(cfgDir/"after"))
-                run->addScript((cfgDir/"after").string(), true);
-
-            // add environment files
-            if(fs::exists(cfgDir/"env"))
-                run->addEnv((cfgDir/"env").string());
-            if(fs::exists(cfgDir/"nodes"/node->name+".env"))
-                run->addEnv((cfgDir/"nodes"/node->name+".env").string());
-            if(fs::exists(cfgDir/"jobs"/run->name+".env"))
-                run->addEnv((cfgDir/"jobs"/run->name+".env").string());
-
-            // add job timeout if specified
-            if(fs::exists(cfgDir/"jobs"/run->name+".conf")) {
-                int timeout = parseConfFile(fs::path(cfgDir/"jobs"/run->name+".conf").string().c_str()).get<int>("TIMEOUT", 0);
-                if(timeout > 0) {
-                    // A raw pointer to run is used here so as not to have a circular reference.
-                    // The captured raw pointer is safe because if the Run is destroyed the Promise
-                    // will be cancelled and the callback never called.
-                    Run* r = run.get();
-                    r->timeout = srv->addTimeout(timeout, [r](){
-                        r->abort(true);
-                    });
-                }
-            }
-
-            // start the job
+        if(nodeCanQueue(*node.get(), run->name) && run->configure(buildNums[run->name] + 1, node, *fsHome)) {
             node->busyExecutors++;
-            run->node = node;
-            run->startedAt = time(nullptr);
-            run->laminarHome = homeDir;
-            run->build = buildNum;
             // set the last known result if exists
             db->stmt("SELECT result FROM builds WHERE name = ? ORDER BY completedAt DESC LIMIT 1")
              .bind(run->name)
              .fetch<int>([=](int result){
                 run->lastResult = RunState(result);
             });
-            // update next build number
-            buildNums[run->name] = buildNum;
 
-            LLOG(INFO, "Queued job to node", run->name, run->build, node->name);
+            // Actually schedules the Run steps
+            kj::Promise<void> exec = handleRunStep(run.get()).then([this,r=run.get()]{
+                runFinished(r);
+            });
+            if(run->timeout > 0) {
+                exec = exec.attach(srv->addTimeout(run->timeout, [r=run.get()](){
+                    r->abort(true);
+                }));
+            }
+            srv->addTask(kj::mv(exec));
+            LLOG(INFO, "Started job on node", run->name, run->build, node->name);
+
+            // update next build number
+            buildNums[run->name]++;
 
             // notify clients
             Json j;
@@ -752,14 +659,6 @@ bool Laminar::tryStartRun(std::shared_ptr<Run> run, int queueIndex) {
                     c->sendMessage(msg);
             }
 
-            // notify the rpc client if the start command was used
-            run->started.fulfiller->fulfill();
-
-            // this actually spawns the first step
-            srv->addTask(handleRunStep(run.get()).then([this,run]{
-                runFinished(run.get());
-            }));
-
             return true;
         }
     }
@@ -878,50 +777,21 @@ void Laminar::runFinished(Run * r) {
     auto it = activeJobs.byJobName().equal_range(r->name);
     uint oldestActive = (it.first == it.second)? buildNums[r->name] : (*it.first)->build - 1;
     for(int i = static_cast<int>(oldestActive - numKeepRunDirs); i > 0; i--) {
-        fs::path d = fs::path(homeDir)/"run"/r->name/std::to_string(i);
+        kj::Path d{"run",r->name,std::to_string(i)};
         // Once the directory does not exist, it's probably not worth checking
         // any further. 99% of the time this loop should only ever have 1 iteration
         // anyway so hence this (admittedly debatable) optimization.
-        if(!fs::exists(d))
+        if(!fsHome->exists(d))
             break;
-        fs::remove_all(d);
+        fsHome->remove(d);
     }
 
     // in case we freed up an executor, check the queue
     assignNewJobs();
 }
 
-class MappedFileImpl : public MappedFile {
-public:
-    MappedFileImpl(const char* path) :
-        fd(open(path, O_RDONLY)),
-        sz(0),
-        ptr(nullptr)
-    {
-        if(fd == -1) return;
-        struct stat st;
-        if(fstat(fd, &st) != 0) return;
-        sz = st.st_size;
-        ptr = mmap(nullptr, sz, PROT_READ, MAP_SHARED, fd, 0);
-        if(ptr == MAP_FAILED)
-            ptr = nullptr;
-    }
-    ~MappedFileImpl() override {
-        if(ptr)
-            munmap(ptr, sz);
-        if(fd != -1)
-            close(fd);
-    }
-    virtual const void* address() override { return ptr; }
-    virtual size_t size() override { return sz; }
-private:
-    int fd;
-    size_t sz;
-    void* ptr;
-};
-
-kj::Own<MappedFile> Laminar::getArtefact(std::string path) {
-    return kj::heap<MappedFileImpl>(fs::path(fs::path(homeDir)/"archive"/path).c_str());
+kj::Maybe<kj::Own<const kj::ReadableFile>> Laminar::getArtefact(std::string path) {
+    return fsHome->openFile(kj::Path("archive").append(kj::Path::parse(path)));
 }
 
 bool Laminar::handleBadgeRequest(std::string job, std::string &badge) {
@@ -967,9 +837,9 @@ R"x(
 }
 
 std::string Laminar::getCustomCss() {
-    MappedFileImpl cssFile(fs::path(fs::path(homeDir)/"custom"/"style.css").c_str());
-    if(cssFile.address()) {
-        return std::string(static_cast<const char*>(cssFile.address()), cssFile.size());
+    KJ_IF_MAYBE(cssFile, fsHome->tryOpenFile(kj::Path{"custom","style.css"})) {
+        return (*cssFile)->readAllText().cStr();
+    } else {
+        return std::string();
     }
-    return std::string();
 }

src/laminar.h

@@ -25,6 +25,7 @@
 #include "database.h"
 
 #include <unordered_map>
+#include <kj/filesystem.h>
 
 // Node name to node object map
 typedef std::unordered_map<std::string, std::shared_ptr<Node>> NodeMap;
@@ -38,8 +39,8 @@ class Json;
 // the LaminarClient objects (see interface.h)
 class Laminar final : public LaminarInterface {
 public:
-    Laminar();
-    ~Laminar() override;
+    Laminar(const char* homePath);
+    ~Laminar() noexcept override;
 
     // Runs the application forever
     void run();
@@ -55,7 +56,7 @@ public:
 
     void sendStatus(LaminarClient* client) override;
     bool setParam(std::string job, uint buildNum, std::string param, std::string value) override;
-    kj::Own<MappedFile> getArtefact(std::string path) override;
+    kj::Maybe<kj::Own<const kj::ReadableFile>> getArtefact(std::string path) override;
     bool handleBadgeRequest(std::string job, std::string& badge) override;
     std::string getCustomCss() override;
     void abortAll() override;
@@ -67,7 +68,7 @@ private:
     bool tryStartRun(std::shared_ptr<Run> run, int queueIndex);
     kj::Promise<void> handleRunStep(Run *run);
     void runFinished(Run*);
-    bool nodeCanQueue(const Node&, const Run&) const;
+    bool nodeCanQueue(const Node&, std::string jobName) const;
     // expects that Json has started an array
     void populateArtifacts(Json& out, std::string job, uint num) const;
 
@@ -86,7 +87,8 @@ private:
     Database* db;
     Server* srv;
     NodeMap nodes;
-    std::string homeDir;
+    kj::Path homePath;
+    kj::Own<const kj::Directory> fsHome;
     std::set<LaminarClient*> clients;
     std::set<LaminarWaiter*> waiters;
     uint numKeepRunDirs;

src/main.cpp

@@ -20,6 +20,7 @@
 #include "log.h"
 #include <signal.h>
 #include <kj/async-unix.h>
+#include <kj/filesystem.h>
 
 static Laminar* laminar;
 
@@ -34,8 +35,9 @@ int main(int argc, char** argv) {
         }
     }
 
-    laminar = new Laminar;
+    laminar = new Laminar(getenv("LAMINAR_HOME") ?: "/var/lib/laminar");
     kj::UnixEventPort::captureChildExit();
+
     signal(SIGINT, &laminar_quit);
     signal(SIGTERM, &laminar_quit);
 

src/run.cpp

@@ -25,8 +25,11 @@
 #include <unistd.h>
 #include <signal.h>
 
-#include <boost/filesystem.hpp>
-namespace fs = boost::filesystem;
+// short syntax helper for kj::Path
+template<typename T>
+inline kj::Path operator/(const kj::Path& p, const T& ext) {
+    return p.append(ext);
+}
 
 std::string to_string(const RunState& rs) {
     switch(rs) {
@@ -41,16 +44,120 @@ std::string to_string(const RunState& rs) {
 }
 
 
-Run::Run() :
+Run::Run(std::string name, ParamMap pm, kj::Path&& rootPath) :
     result(RunState::SUCCESS),
-    lastResult(RunState::UNKNOWN)
+    lastResult(RunState::UNKNOWN),
+    name(name),
+    params(kj::mv(pm)),
+    queuedAt(time(nullptr)),
+    rootPath(kj::mv(rootPath)),
+    started(kj::newPromiseAndFulfiller<void>())
 {
+    for(auto it = params.begin(); it != params.end();) {
+        if(it->first[0] == '=') {
+            if(it->first == "=parentJob") {
+                parentName = it->second;
+            } else if(it->first == "=parentBuild") {
+                parentBuild = atoi(it->second.c_str());
+            } else if(it->first == "=reason") {
+                reasonMsg = it->second;
+            } else {
+                LLOG(ERROR, "Unknown internal job parameter", it->first);
+            }
+            it = params.erase(it);
+        } else
+            ++it;
+    }
 }
 
 Run::~Run() {
     LLOG(INFO, "Run destroyed");
 }
 
+bool Run::configure(uint buildNum, std::shared_ptr<Node> nd, const kj::Directory& fsHome)
+{
+    kj::Path cfgDir{"cfg"};
+
+    // create the run directory
+    kj::Path rd{"run",name,std::to_string(buildNum)};
+    bool createWorkdir = true;
+    KJ_IF_MAYBE(ls, fsHome.tryLstat(rd)) {
+        KJ_ASSERT(ls->type == kj::FsNode::Type::DIRECTORY);
+        LLOG(WARNING, "Working directory already exists, removing", rd.toString());
+        if(fsHome.tryRemove(rd) == false) {
+            LLOG(WARNING, "Failed to remove working directory");
+            createWorkdir = false;
+        }
+    }
+    if(createWorkdir && fsHome.tryOpenSubdir(rd, kj::WriteMode::CREATE|kj::WriteMode::CREATE_PARENT) == nullptr) {
+        LLOG(ERROR, "Could not create working directory", rd.toString());
+        return false;
+    }
+
+    // create an archive directory
+    kj::Path archive = kj::Path{"archive",name,std::to_string(buildNum)};
+    if(fsHome.exists(archive)) {
+        LLOG(WARNING, "Archive directory already exists", archive.toString());
+    } else if(fsHome.tryOpenSubdir(archive, kj::WriteMode::CREATE|kj::WriteMode::CREATE_PARENT) == nullptr) {
+        LLOG(ERROR, "Could not create archive directory", archive.toString());
+        return false;
+    }
+
+    // create a workspace for this job if it doesn't exist
+    kj::Path ws{"run",name,"workspace"};
+    if(!fsHome.exists(ws)) {
+        fsHome.openSubdir(ws, kj::WriteMode::CREATE|kj::WriteMode::CREATE_PARENT);
+        // prepend the workspace init script
+        if(fsHome.exists(cfgDir/"jobs"/(name+".init")))
+            addScript(cfgDir/"jobs"/(name+".init"), kj::mv(ws));
+    }
+
+    // add scripts
+    // global before-run script
+    if(fsHome.exists(cfgDir/"before"))
+        addScript(cfgDir/"before", rd.clone());
+    // per-node before-run script
+    if(fsHome.exists(cfgDir/"nodes"/(nd->name+".before")))
+        addScript(cfgDir/"nodes"/(nd->name+".before"), rd.clone());
+    // job before-run script
+    if(fsHome.exists(cfgDir/"jobs"/(name+".before")))
+        addScript(cfgDir/"jobs"/(name+".before"), rd.clone());
+    // main run script. must exist.
+    addScript(cfgDir/"jobs"/(name+".run"), rd.clone());
+    // job after-run script
+    if(fsHome.exists(cfgDir/"jobs"/(name+".after")))
+        addScript(cfgDir/"jobs"/(name+".after"), rd.clone(), true);
+    // per-node after-run script
+    if(fsHome.exists(cfgDir/"nodes"/(nd->name+".after")))
+        addScript(cfgDir/"nodes"/(nd->name+".after"), rd.clone(), true);
+    // global after-run script
+    if(fsHome.exists(cfgDir/"after"))
+        addScript(cfgDir/"after", rd.clone(), true);
+
+    // add environment files
+    if(fsHome.exists(cfgDir/"env"))
+        addEnv(cfgDir/"env");
+    if(fsHome.exists(cfgDir/"nodes"/(nd->name+".env")))
+        addEnv(cfgDir/"nodes"/(nd->name+".env"));
+    if(fsHome.exists(cfgDir/"jobs"/(name+".env")))
+        addEnv(cfgDir/"jobs"/(name+".env"));
+
+    // add job timeout if specified
+    if(fsHome.exists(cfgDir/"jobs"/(name+".conf"))) {
+        timeout = parseConfFile((rootPath/cfgDir/"jobs"/(name+".conf")).toString(true).cStr()).get<int>("TIMEOUT", 0);
+    }
+
+    // All good, we've "started"
+    startedAt = time(nullptr);
+    build = buildNum;
+    node = nd;
+
+    // notifies the rpc client if the start command was used
+    started.fulfiller->fulfill();
+
+    return true;
+}
+
 std::string Run::reason() const {
     if(!parentName.empty()) {
         return std::string("Triggered by upstream ") + parentName + " #" + std::to_string(parentBuild);
@@ -62,7 +169,7 @@ bool Run::step() {
     if(!scripts.size())
         return true;
 
-    currentScript = scripts.front();
+    Script currentScript = kj::mv(scripts.front());
     scripts.pop();
 
     int pfd[2];
@@ -84,16 +191,17 @@ bool Run::step() {
         close(pfd[1]);
         std::string buildNum = std::to_string(build);
 
-        std::string PATH = (fs::path(laminarHome)/"cfg"/"scripts").string() + ":";
+        std::string PATH = (rootPath/"cfg"/"scripts").toString(true).cStr();
         if(const char* p = getenv("PATH")) {
+            PATH.append(":");
             PATH.append(p);
         }
 
-        chdir(currentScript.cwd.c_str());
+        KJ_SYSCALL(chdir((rootPath/currentScript.cwd).toString(true).cStr()));
 
         // conf file env vars
-        for(std::string file : env) {
-            StringMap vars = parseConfFile(file.c_str());
+        for(kj::Path& file : env) {
+            StringMap vars = parseConfFile((rootPath/file).toString().cStr());
             for(auto& it : vars) {
                 setenv(it.first.c_str(), it.second.c_str(), true);
             }
@@ -110,13 +218,14 @@ bool Run::step() {
             setenv("NODE", node->name.c_str(), true);
         setenv("RESULT", to_string(result).c_str(), true);
         setenv("LAST_RESULT", to_string(lastResult).c_str(), true);
-        setenv("WORKSPACE", (fs::path(laminarHome)/"run"/name/"workspace").string().c_str(), true);
-        setenv("ARCHIVE", (fs::path(laminarHome)/"archive"/name/buildNum.c_str()).string().c_str(), true);
+        setenv("WORKSPACE", (rootPath/"run"/name/"workspace").toString(true).cStr(), true);
+        setenv("ARCHIVE", (rootPath/"archive"/name/buildNum).toString(true).cStr(), true);
 
-        fprintf(stderr, "[laminar] Executing %s\n", currentScript.path.c_str());
-        execl(currentScript.path.c_str(), currentScript.path.c_str(), NULL);
+        fprintf(stderr, "[laminar] Executing %s\n", currentScript.path.toString().cStr());
+        kj::String execPath = (rootPath/currentScript.path).toString(true);
+        execl(execPath.cStr(), execPath.cStr(), NULL);
         // cannot use LLOG because stdout/stderr are captured
-        fprintf(stderr, "[laminar] Failed to execute %s\n", currentScript.path.c_str());
+        fprintf(stderr, "[laminar] Failed to execute %s\n", currentScript.path.toString().cStr());
         _exit(1);
     }
 
@@ -128,12 +237,12 @@ bool Run::step() {
     return false;
 }
 
-void Run::addScript(std::string scriptPath, std::string scriptWorkingDir, bool runOnAbort) {
-    scripts.push({scriptPath, scriptWorkingDir, runOnAbort});
+void Run::addScript(kj::Path scriptPath, kj::Path scriptWorkingDir, bool runOnAbort) {
+    scripts.push({kj::mv(scriptPath), kj::mv(scriptWorkingDir), runOnAbort});
 }
 
-void Run::addEnv(std::string path) {
-    env.push_back(path);
+void Run::addEnv(kj::Path path) {
+    env.push_back(kj::mv(path));
 }
 
 void Run::abort(bool respectRunOnAbort) {

src/run.h

@@ -27,6 +27,7 @@
 #include <unordered_map>
 #include <memory>
 #include <kj/async.h>
+#include <kj/filesystem.h>
 
 enum class RunState {
     UNKNOWN,
@@ -41,29 +42,25 @@ std::string to_string(const RunState& rs);
 
 class Node;
 
-// Represents an execution of a job. Not much more than POD
+typedef std::unordered_map<std::string, std::string> ParamMap;
+
+// Represents an execution of a job.
 class Run {
 public:
-    Run();
+    Run(std::string name, ParamMap params, kj::Path&& rootPath);
     ~Run();
 
     // copying this class would be asking for trouble...
     Run(const Run&) = delete;
     Run& operator=(const Run&) = delete;
 
+    // Call this to "start" the run with a specific number and node
+    bool configure(uint buildNum, std::shared_ptr<Node> node, const kj::Directory &fsHome);
+
     // executes the next script (if any), returning true if there is nothing
     // more to be done.
     bool step();
 
-    // adds a script to the queue of scripts to be executed by this run
-    void addScript(std::string scriptPath, std::string scriptWorkingDir, bool runOnAbort = false);
-
-    // adds a script to the queue using the runDir as the scripts CWD
-    void addScript(std::string script, bool runOnAbort = false) { addScript(script, runDir, runOnAbort); }
-
-    // adds an environment file that will be sourced before this run
-    void addEnv(std::string path);
-
     // aborts this run
     void abort(bool respectRunOnAbort);
 
@@ -73,35 +70,41 @@ public:
 
     std::string reason() const;
 
+    kj::Promise<void>&& whenStarted() { return kj::mv(started.promise); }
+
     std::shared_ptr<Node> node;
     RunState result;
     RunState lastResult;
-    std::string laminarHome;
     std::string name;
-    std::string runDir;
     std::string parentName;
     int parentBuild = 0;
-    std::string reasonMsg;
     uint build = 0;
     std::string log;
     kj::Maybe<pid_t> current_pid;
     int output_fd;
     std::unordered_map<std::string, std::string> params;
-    kj::Promise<void> timeout = kj::NEVER_DONE;
-    kj::PromiseFulfillerPair<void> started = kj::newPromiseAndFulfiller<void>();
+    int timeout;
 
     time_t queuedAt;
     time_t startedAt;
 private:
+    // adds a script to the queue of scripts to be executed by this run
+    void addScript(kj::Path scriptPath, kj::Path scriptWorkingDir, bool runOnAbort = false);
+
+    // adds an environment file that will be sourced before this run
+    void addEnv(kj::Path path);
+
     struct Script {
-        std::string path;
-        std::string cwd;
+        kj::Path path;
+        kj::Path cwd;
         bool runOnAbort;
     };
 
+    kj::Path rootPath;
     std::queue<Script> scripts;
-    Script currentScript;
-    std::list<std::string> env;
+    std::list<kj::Path> env;
+    std::string reasonMsg;
+    kj::PromiseFulfillerPair<void> started;
 };
 
 

src/server.cpp

@@ -96,7 +96,7 @@ public:
         }
         std::shared_ptr<Run> run = laminar.queueJob(jobName, params);
         if(Run* r = run.get()) {
-            return r->started.promise.then([context,r]() mutable {
+            return r->whenStarted().then([context,r]() mutable {
                 context.getResults().setResult(LaminarCi::MethodResult::SUCCESS);
                 context.getResults().setBuildNum(r->build);
             });
@@ -314,11 +314,11 @@ private:
             std::string badge;
             responseHeaders.clear();
             if(resource.compare(0, strlen("/archive/"), "/archive/") == 0) {
-                kj::Own<MappedFile> file = laminar.getArtefact(resource.substr(strlen("/archive/")));
-                if(file->address() != nullptr) {
+                KJ_IF_MAYBE(file, laminar.getArtefact(resource.substr(strlen("/archive/")))) {
+                    auto array = (*file)->mmap(0, (*file)->stat().size);
                     responseHeaders.add("Content-Transfer-Encoding", "binary");
-                    auto stream = response.send(200, "OK", responseHeaders, file->size());
-                    return stream->write(file->address(), file->size()).attach(kj::mv(file)).attach(kj::mv(stream));
+                    auto stream = response.send(200, "OK", responseHeaders, array.size());
+                    return stream->write(array.begin(), array.size()).attach(kj::mv(array)).attach(kj::mv(file)).attach(kj::mv(stream));
                 }
             } else if(resource.compare("/custom/style.css") == 0) {
                 responseHeaders.set(kj::HttpHeaderId::CONTENT_TYPE, "text/css; charset=utf-8");

test/tempdir.h

@@ -0,0 +1,45 @@
+///
+/// Copyright 2018 Oliver Giles
+///
+/// This file is part of Laminar
+///
+/// Laminar is free software: you can redistribute it and/or modify
+/// it under the terms of the GNU General Public License as published by
+/// the Free Software Foundation, either version 3 of the License, or
+/// (at your option) any later version.
+///
+/// Laminar is distributed in the hope that it will be useful,
+/// but WITHOUT ANY WARRANTY; without even the implied warranty of
+/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+/// GNU General Public License for more details.
+///
+/// You should have received a copy of the GNU General Public License
+/// along with Laminar.  If not, see <http://www.gnu.org/licenses/>
+///
+#ifndef LAMINAR_TEMPDIR_H_
+#define LAMINAR_TEMPDIR_H_
+
+#include <kj/filesystem.h>
+#include <stdlib.h>
+
+class TempDir {
+public:
+    TempDir() :
+        path(mkdtemp()),
+        fs(kj::newDiskFilesystem()->getRoot().openSubdir(path, kj::WriteMode::CREATE|kj::WriteMode::MODIFY))
+    {
+    }
+    ~TempDir() noexcept {
+        kj::newDiskFilesystem()->getRoot().remove(path);
+    }
+    kj::Path path;
+    kj::Own<const kj::Directory> fs;
+private:
+    static kj::Path mkdtemp() {
+        char dir[] = "/tmp/laminar-test-XXXXXX";
+        ::mkdtemp(dir);
+        return kj::Path::parse(&dir[1]);
+    }
+};
+
+#endif // LAMINAR_TEMPDIR_H_

test/test-laminar.cpp

@@ -27,8 +27,12 @@ public:
     std::string payload;
 };
 
-class LaminarTest : public ::testing::Test {
+class LaminarTest : public testing::Test {
 protected:
+    LaminarTest() :
+        testing::Test(),
+        laminar("/tmp")
+    {}
     Laminar laminar;
 };
 

test/test-run.cpp

@@ -21,23 +21,31 @@
 #include "log.h"
 #include "node.h"
 #include "conf.h"
+#include "tempdir.h"
 
-class RunTest : public ::testing::Test {
+class RunTest : public testing::Test {
 protected:
-    virtual ~RunTest() noexcept override {}
-
-    void SetUp() override {
-        run.node = node;
+    RunTest() :
+        testing::Test(),
+        node(std::make_shared<Node>()),
+        tmp(),
+        run("foo", ParamMap{}, tmp.path.clone())
+    {
     }
+
+    ~RunTest() noexcept {}
+
     void wait() {
         int state = -1;
         waitpid(run.current_pid.orDefault(0), &state, 0);
         run.reaped(state);
     }
+
     void runAll() {
         while(!run.step())
             wait();
     }
+
     std::string readAllOutput() {
         std::string res;
         char tmp[64];
@@ -56,53 +64,66 @@ protected:
         return map;
     }
 
+    std::shared_ptr<Node> node;
+    TempDir tmp;
     class Run run;
-    std::shared_ptr<Node> node = std::shared_ptr<Node>(new Node);
+
+    void setRunLink(const char* path) {
+        tmp.fs->symlink(kj::Path{"cfg","jobs",run.name+".run"}, path, kj::WriteMode::CREATE|kj::WriteMode::CREATE_PARENT|kj::WriteMode::EXECUTABLE);
+    }
 };
 
 TEST_F(RunTest, WorkingDirectory) {
-    run.addScript("/bin/pwd", "/home");
+    setRunLink("/bin/pwd");
+    run.configure(1, node, *tmp.fs);
     runAll();
-    EXPECT_EQ("/home\n", readAllOutput());
+    std::string cwd{tmp.path.append(kj::Path{"run","foo","1"}).toString(true).cStr()};
+    EXPECT_EQ(cwd + "\n", readAllOutput());
 }
 
 TEST_F(RunTest, SuccessStatus) {
-    run.addScript("/bin/true");
+    setRunLink("/bin/true");
+    run.configure(1, node, *tmp.fs);
     runAll();
     EXPECT_EQ(RunState::SUCCESS, run.result);
 }
 
 TEST_F(RunTest, FailedStatus) {
-    run.addScript("/bin/false");
+    setRunLink("/bin/false");
+    run.configure(1, node, *tmp.fs);
     runAll();
     EXPECT_EQ(RunState::FAILED, run.result);
 }
 
 TEST_F(RunTest, Environment) {
-    run.name = "foo";
-    run.build = 1234;
-    run.laminarHome = "/tmp";
-    run.addScript("/usr/bin/env");
+    setRunLink("/usr/bin/env");
+    run.configure(1234, node, *tmp.fs);
     runAll();
+
+    std::string ws{tmp.path.append(kj::Path{"run","foo","workspace"}).toString(true).cStr()};
+    std::string archive{tmp.path.append(kj::Path{"archive","foo","1234"}).toString(true).cStr()};
+
     StringMap map = parseFromString(readAllOutput());
     EXPECT_EQ("1234", map["RUN"]);
     EXPECT_EQ("foo", map["JOB"]);
     EXPECT_EQ("success", map["RESULT"]);
     EXPECT_EQ("unknown", map["LAST_RESULT"]);
-    EXPECT_EQ("/tmp/run/foo/workspace", map["WORKSPACE"]);
-    EXPECT_EQ("/tmp/archive/foo/1234", map["ARCHIVE"]);
+    EXPECT_EQ(ws, map["WORKSPACE"]);
+    EXPECT_EQ(archive, map["ARCHIVE"]);
 }
 
 TEST_F(RunTest, ParamsToEnv) {
+    setRunLink("/usr/bin/env");
     run.params["foo"] = "bar";
-    run.addScript("/usr/bin/env");
+    run.configure(1, node, *tmp.fs);
     runAll();
     StringMap map = parseFromString(readAllOutput());
     EXPECT_EQ("bar", map["foo"]);
 }
 
 TEST_F(RunTest, Abort) {
-    run.addScript("/usr/bin/yes");
+    setRunLink("/usr/bin/yes");
+    run.configure(1, node, *tmp.fs);
     run.step();
     usleep(200); // TODO fix
     run.abort(false);
@@ -110,13 +131,3 @@ TEST_F(RunTest, Abort) {
     EXPECT_EQ(RunState::ABORTED, run.result);
 }
 
-TEST_F(RunTest, AbortAfterFailed) {
-    run.addScript("/bin/false");
-    runAll();
-    run.addScript("/usr/bin/yes");
-    run.step();
-    usleep(200); // TODO fix
-    run.abort(false);
-    wait();
-    EXPECT_EQ(RunState::FAILED, run.result);
-}

test/test-server.cpp

@@ -18,28 +18,13 @@
 ///
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
-#include <boost/filesystem.hpp>
 #include <thread>
 #include <sys/socket.h>
 #include "server.h"
 #include "log.h"
 #include "interface.h"
 #include "laminar.capnp.h"
-
-namespace fs = boost::filesystem;
-
-class TempDir : public fs::path {
-public:
-    TempDir(const char* tmpl) {
-        char* t = strdup(tmpl);
-        mkdtemp(t);
-        *static_cast<fs::path*>(this) = t;
-        free(t);
-    }
-    ~TempDir() {
-        fs::remove_all(*this);
-    }
-};
+#include "tempdir.h"
 
 class MockLaminar : public LaminarInterface {
 public:
@@ -57,7 +42,7 @@ public:
     }
 
     // MOCK_METHOD does not seem to work with return values whose destructors have noexcept(false)
-    kj::Own<MappedFile> getArtefact(std::string path) override { return kj::Own<MappedFile>(nullptr, kj::NullDisposer()); }
+    kj::Maybe<kj::Own<const kj::ReadableFile>> getArtefact(std::string path) override { return nullptr; }
 
     MOCK_METHOD2(queueJob, std::shared_ptr<Run>(std::string name, ParamMap params));
     MOCK_METHOD1(registerWaiter, void(LaminarWaiter* waiter));
@@ -72,14 +57,10 @@ public:
 
 class ServerTest : public ::testing::Test {
 protected:
-    ServerTest() :
-        tempDir("/tmp/laminar-test-XXXXXX")
-    {
-    }
     void SetUp() override {
         EXPECT_CALL(mockLaminar, registerWaiter(testing::_));
         EXPECT_CALL(mockLaminar, deregisterWaiter(testing::_));
-        server = new Server(mockLaminar, "unix:"+fs::path(tempDir/"rpc.sock").string(), "127.0.0.1:8080");
+        server = new Server(mockLaminar, "unix:"+std::string(tempDir.path.append("rpc.sock").toString(true).cStr()), "127.0.0.1:8080");
     }
     void TearDown() override {
         delete server;