diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a9f46c..6906fe8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ ### -### Copyright 2015-2017 Oliver Giles +### Copyright 2015-2018 Oliver Giles ### ### This file is part of Laminar ### @@ -83,8 +83,7 @@ generate_compressed_bins(${CMAKE_BINARY_DIR} js/vue-router.min.js js/vue.min.js ## Server add_executable(laminard src/database.cpp src/main.cpp src/server.cpp src/laminar.cpp src/conf.cpp src/resources.cpp src/run.cpp laminar.capnp.c++ ${COMPRESSED_BINS}) -# TODO: some alternative to boost::filesystem? -target_link_libraries(laminard capnp-rpc capnp kj-http kj-async kj pthread boost_filesystem boost_system sqlite3 z) +target_link_libraries(laminard capnp-rpc capnp kj-http kj-async kj pthread sqlite3 z) ## Client add_executable(laminarc src/client.cpp laminar.capnp.c++) @@ -96,7 +95,7 @@ if(BUILD_TESTS) find_package(GTest REQUIRED) include_directories(${GTEST_INCLUDE_DIRS} src) add_executable(laminar-tests src/conf.cpp src/database.cpp src/laminar.cpp src/run.cpp src/server.cpp laminar.capnp.c++ src/resources.cpp ${COMPRESSED_BINS} test/test-conf.cpp test/test-database.cpp test/test-laminar.cpp test/test-run.cpp test/test-server.cpp) - target_link_libraries(laminar-tests ${GTEST_BOTH_LIBRARIES} gmock capnp-rpc capnp kj-http kj-async kj pthread boost_filesystem boost_system sqlite3 z) + target_link_libraries(laminar-tests ${GTEST_BOTH_LIBRARIES} gmock capnp-rpc capnp kj-http kj-async kj pthread sqlite3 z) endif() set(SYSTEMD_UNITDIR /lib/systemd/system CACHE PATH "Path to systemd unit files") diff --git a/docker-build-centos.sh b/docker-build-centos.sh index 92bcd10..7fe391c 100755 --- a/docker-build-centos.sh +++ b/docker-build-centos.sh @@ -46,7 +46,7 @@ Version: $VERSION Release: 1 License: GPL BuildRequires: systemd-units -Requires: sqlite boost-filesystem zlib +Requires: sqlite zlib %description Lightweight Continuous Integration Service diff --git a/docker-build-debian.sh b/docker-build-debian.sh index b241c01..8a641c6 100755 --- a/docker-build-debian.sh +++ b/docker-build-debian.sh @@ -8,7 +8,7 @@ VERSION=$(cd "$SOURCE_DIR" && git describe --tags --abbrev=8 --dirty) DOCKER_TAG=$(docker build -q - < -Depends: libsqlite3-0, libboost-filesystem1.62.0, zlib1g +Depends: libsqlite3-0, zlib1g Description: Lightweight Continuous Integration Service EOF cat < laminar/DEBIAN/postinst diff --git a/src/interface.h b/src/interface.h index a8c596f..6bd840e 100644 --- a/src/interface.h +++ b/src/interface.h @@ -25,8 +25,6 @@ #include #include -typedef std::unordered_map ParamMap; - // Simple struct to define which information a frontend client is interested // in, both in initial request phase and real-time updates. It corresponds // loosely to frontend URLs @@ -133,7 +131,7 @@ struct LaminarInterface { // Fetches the content of an artifact given its filename relative to // $LAMINAR_HOME/archive. Ideally, this would instead be served by a // proper web server which handles this url. - virtual kj::Own getArtefact(std::string path) = 0; + virtual kj::Maybe> getArtefact(std::string path) = 0; // Given the name of a job, populate the provided string reference with // SVG content describing the last known state of the job. Returns false diff --git a/src/laminar.cpp b/src/laminar.cpp index 7e29f0b..4007904 100644 --- a/src/laminar.cpp +++ b/src/laminar.cpp @@ -29,9 +29,6 @@ #include #include -#include -namespace fs = boost::filesystem; - #define COMPRESS_LOG_MIN_SIZE 1024 #include @@ -62,23 +59,30 @@ constexpr const char* INTADDR_HTTP_DEFAULT = "*:8080"; constexpr const char* ARCHIVE_URL_DEFAULT = "/archive"; } -// helper for appending to boost::filesystem::path -fs::path operator+(fs::path p, const char* ext) { - std::string leaf = p.leaf().string(); - leaf += ext; - return p.remove_leaf()/leaf; +// short syntax helpers for kj::Path +template +inline kj::Path operator/(const kj::Path& p, const T& ext) { + return p.append(ext); +} +template +inline kj::Path operator/(const std::string& p, const T& ext) { + return kj::Path{p}/ext; } typedef std::string str; -Laminar::Laminar() { +Laminar::Laminar(const char *home) : + homePath(kj::Path::parse(&home[1])), + fsHome(kj::newDiskFilesystem()->getRoot().openSubdir(homePath, kj::WriteMode::MODIFY)) +{ + KJ_ASSERT(home[0] == '/'); + archiveUrl = ARCHIVE_URL_DEFAULT; if(char* envArchive = getenv("LAMINAR_ARCHIVE_URL")) archiveUrl = envArchive; numKeepRunDirs = 0; - homeDir = getenv("LAMINAR_HOME") ?: "/var/lib/laminar"; - db = new Database((fs::path(homeDir)/"laminar.sqlite").string().c_str()); + db = new Database((homePath/"laminar.sqlite").toString(true).cStr()); // Prepare database for first use // TODO: error handling db->exec("CREATE TABLE IF NOT EXISTS builds(" @@ -128,17 +132,16 @@ bool Laminar::setParam(std::string job, uint buildNum, std::string param, std::s void Laminar::populateArtifacts(Json &j, std::string job, uint num) const { - fs::path dir(fs::path(homeDir)/"archive"/job/std::to_string(num)); - if(fs::is_directory(dir)) { - size_t prefixLen = (fs::path(homeDir)/"archive").string().length(); - size_t scopeLen = dir.string().length(); - for(fs::recursive_directory_iterator it(dir); it != fs::recursive_directory_iterator(); ++it) { - if(!fs::is_regular_file(*it)) + kj::Path runArchive{job,std::to_string(num)}; + KJ_IF_MAYBE(dir, fsHome->tryOpenSubdir("archive"/runArchive)) { + for(kj::StringPtr file : (*dir)->listNames()) { + kj::FsNode::Metadata meta = (*dir)->lstat(kj::Path{file}); + if(meta.type != kj::FsNode::Type::FILE) continue; j.StartObject(); - j.set("url", archiveUrl + it->path().string().substr(prefixLen)); - j.set("filename", it->path().string().substr(scopeLen+1)); - j.set("size", fs::file_size(it->path())); + j.set("url", archiveUrl + (runArchive/file).toString().cStr()); + j.set("filename", file.cStr()); + j.set("size", meta.size); j.EndObject(); } } @@ -440,9 +443,12 @@ void Laminar::sendStatus(LaminarClient* client) { client->sendMessage(j.str()); } -Laminar::~Laminar() { +Laminar::~Laminar() noexcept try { delete db; delete srv; +} catch (std::exception& e) { + LLOG(ERROR, e.what()); + return; } void Laminar::run() { @@ -450,8 +456,8 @@ void Laminar::run() { const char* listen_http = getenv("LAMINAR_BIND_HTTP") ?: INTADDR_HTTP_DEFAULT; srv = new Server(*this, listen_rpc, listen_http); - srv->addWatchPath(fs::path(fs::path(homeDir)/"cfg"/"nodes").string().c_str()); - srv->addWatchPath(fs::path(fs::path(homeDir)/"cfg"/"jobs").string().c_str()); + srv->addWatchPath((homePath/"cfg"/"nodes").toString().cStr()); + srv->addWatchPath((homePath/"cfg"/"jobs").toString().cStr()); srv->start(); } @@ -465,16 +471,14 @@ bool Laminar::loadConfiguration() { std::set knownNodes; - fs::path nodeCfg = fs::path(homeDir)/"cfg"/"nodes"; - - if(fs::is_directory(nodeCfg)) { - for(fs::directory_iterator it(nodeCfg); it != fs::directory_iterator(); ++it) { - if(!fs::is_regular_file(it->status()) || it->path().extension() != ".conf") + KJ_IF_MAYBE(nodeDir, fsHome->tryOpenSubdir(kj::Path{"cfg","nodes"})) { + for(kj::Directory::Entry& entry : (*nodeDir)->listEntries()) { + if(entry.type != kj::FsNode::Type::FILE || !entry.name.endsWith(".conf")) continue; - StringMap conf = parseConfFile(it->path().string().c_str()); + StringMap conf = parseConfFile((homePath/entry.name).toString().cStr()); - std::string nodeName = it->path().stem().string(); + std::string nodeName(entry.name.cStr(), entry.name.findLast('.').orDefault(0)-1); auto existingNode = nodes.find(nodeName); std::shared_ptr node = existingNode == nodes.end() ? nodes.emplace(nodeName, std::shared_ptr(new Node)).first->second : existingNode->second; node->name = nodeName; @@ -511,13 +515,13 @@ bool Laminar::loadConfiguration() { nodes.emplace("", node); } - fs::path jobsDir = fs::path(homeDir)/"cfg"/"jobs"; - if(fs::is_directory(jobsDir)) { - for(fs::directory_iterator it(jobsDir); it != fs::directory_iterator(); ++it) { - if(!fs::is_regular_file(it->status()) || it->path().extension() != ".conf") + KJ_IF_MAYBE(jobsDir, fsHome->tryOpenSubdir(kj::Path{"cfg","jobs"})) { + for(kj::Directory::Entry& entry : (*jobsDir)->listEntries()) { + if(entry.type != kj::FsNode::Type::FILE || !entry.name.endsWith(".conf")) continue; + StringMap conf = parseConfFile((homePath/entry.name).toString().cStr()); - StringMap conf = parseConfFile(it->path().string().c_str()); + std::string jobName(entry.name.cStr(), entry.name.findLast('.').orDefault(0)-1); std::string tags = conf.get("TAGS"); if(!tags.empty()) { @@ -526,7 +530,7 @@ bool Laminar::loadConfiguration() { std::string tag; while(std::getline(iss, tag, ',')) tagList.insert(tag); - jobTags[it->path().stem().string()] = tagList; + jobTags[jobName] = tagList; } } @@ -536,30 +540,12 @@ bool Laminar::loadConfiguration() { } std::shared_ptr Laminar::queueJob(std::string name, ParamMap params) { - if(!fs::exists(fs::path(homeDir)/"cfg"/"jobs"/name+".run")) { + if(!fsHome->exists(kj::Path{"cfg","jobs",name+".run"})) { LLOG(ERROR, "Non-existent job", name); return nullptr; } - std::shared_ptr run = std::make_shared(); - run->name = name; - run->queuedAt = time(nullptr); - for(auto it = params.begin(); it != params.end();) { - if(it->first[0] == '=') { - if(it->first == "=parentJob") { - run->parentName = it->second; - } else if(it->first == "=parentBuild") { - run->parentBuild = atoi(it->second.c_str()); - } else if(it->first == "=reason") { - run->reasonMsg = it->second; - } else { - LLOG(ERROR, "Unknown internal job parameter", it->first); - } - it = params.erase(it); - } else - ++it; - } - run->params = params; + std::shared_ptr run = std::make_shared(name, kj::mv(params), homePath.clone()); queuedJobs.push_back(run); // notify clients @@ -591,7 +577,7 @@ void Laminar::abortAll() { } } -bool Laminar::nodeCanQueue(const Node& node, const Run& run) const { +bool Laminar::nodeCanQueue(const Node& node, std::string jobName) const { // if a node is too busy, it can't take the job if(node.busyExecutors >= node.numExecutors) return false; @@ -600,7 +586,7 @@ bool Laminar::nodeCanQueue(const Node& node, const Run& run) const { if(node.tags.size() == 0) return true; - auto it = jobTags.find(run.name); + auto it = jobTags.find(jobName); // if the job has no tags, it cannot be run on this node if(it == jobTags.end()) return false; @@ -617,109 +603,30 @@ bool Laminar::nodeCanQueue(const Node& node, const Run& run) const { bool Laminar::tryStartRun(std::shared_ptr run, int queueIndex) { for(auto& sn : nodes) { std::shared_ptr node = sn.second; - if(nodeCanQueue(*node.get(), *run)) { - fs::path cfgDir = fs::path(homeDir)/"cfg"; - boost::system::error_code err; - - // create a workspace for this job if it doesn't exist - fs::path ws = fs::path(homeDir)/"run"/run->name/"workspace"; - if(!fs::exists(ws)) { - if(!fs::create_directories(ws, err)) { - LLOG(ERROR, "Could not create job workspace", run->name); - break; - } - // prepend the workspace init script - if(fs::exists(cfgDir/"jobs"/run->name+".init")) - run->addScript((cfgDir/"jobs"/run->name+".init").string(), ws.string()); - } - - uint buildNum = buildNums[run->name] + 1; - // create the run directory - fs::path rd = fs::path(homeDir)/"run"/run->name/std::to_string(buildNum); - bool createWorkdir = true; - if(fs::is_directory(rd)) { - LLOG(WARNING, "Working directory already exists, removing", rd.string()); - fs::remove_all(rd, err); - if(err) { - LLOG(WARNING, "Failed to remove working directory", err.message()); - createWorkdir = false; - } - } - if(createWorkdir && !fs::create_directory(rd, err)) { - LLOG(ERROR, "Could not create working directory", rd.string()); - break; - } - run->runDir = rd.string(); - - // create an archive directory - fs::path archive = fs::path(homeDir)/"archive"/run->name/std::to_string(buildNum); - if(fs::is_directory(archive)) { - LLOG(WARNING, "Archive directory already exists", archive.string()); - } else if(!fs::create_directories(archive)) { - LLOG(ERROR, "Could not create archive directory", archive.string()); - break; - } - // add scripts - // global before-run script - if(fs::exists(cfgDir/"before")) - run->addScript((cfgDir/"before").string()); - // per-node before-run script - if(fs::exists(cfgDir/"nodes"/node->name+".before")) - run->addScript((cfgDir/"nodes"/node->name+".before").string()); - // job before-run script - if(fs::exists(cfgDir/"jobs"/run->name+".before")) - run->addScript((cfgDir/"jobs"/run->name+".before").string()); - // main run script. must exist. - run->addScript((cfgDir/"jobs"/run->name+".run").string()); - // job after-run script - if(fs::exists(cfgDir/"jobs"/run->name+".after")) - run->addScript((cfgDir/"jobs"/run->name+".after").string(), true); - // per-node after-run script - if(fs::exists(cfgDir/"nodes"/node->name+".after")) - run->addScript((cfgDir/"nodes"/node->name+".after").string(), true); - // global after-run script - if(fs::exists(cfgDir/"after")) - run->addScript((cfgDir/"after").string(), true); - - // add environment files - if(fs::exists(cfgDir/"env")) - run->addEnv((cfgDir/"env").string()); - if(fs::exists(cfgDir/"nodes"/node->name+".env")) - run->addEnv((cfgDir/"nodes"/node->name+".env").string()); - if(fs::exists(cfgDir/"jobs"/run->name+".env")) - run->addEnv((cfgDir/"jobs"/run->name+".env").string()); - - // add job timeout if specified - if(fs::exists(cfgDir/"jobs"/run->name+".conf")) { - int timeout = parseConfFile(fs::path(cfgDir/"jobs"/run->name+".conf").string().c_str()).get("TIMEOUT", 0); - if(timeout > 0) { - // A raw pointer to run is used here so as not to have a circular reference. - // The captured raw pointer is safe because if the Run is destroyed the Promise - // will be cancelled and the callback never called. - Run* r = run.get(); - r->timeout = srv->addTimeout(timeout, [r](){ - r->abort(true); - }); - } - } - - // start the job + if(nodeCanQueue(*node.get(), run->name) && run->configure(buildNums[run->name] + 1, node, *fsHome)) { node->busyExecutors++; - run->node = node; - run->startedAt = time(nullptr); - run->laminarHome = homeDir; - run->build = buildNum; // set the last known result if exists db->stmt("SELECT result FROM builds WHERE name = ? ORDER BY completedAt DESC LIMIT 1") .bind(run->name) .fetch([=](int result){ run->lastResult = RunState(result); }); - // update next build number - buildNums[run->name] = buildNum; - LLOG(INFO, "Queued job to node", run->name, run->build, node->name); + // Actually schedules the Run steps + kj::Promise exec = handleRunStep(run.get()).then([this,r=run.get()]{ + runFinished(r); + }); + if(run->timeout > 0) { + exec = exec.attach(srv->addTimeout(run->timeout, [r=run.get()](){ + r->abort(true); + })); + } + srv->addTask(kj::mv(exec)); + LLOG(INFO, "Started job on node", run->name, run->build, node->name); + + // update next build number + buildNums[run->name]++; // notify clients Json j; @@ -752,14 +659,6 @@ bool Laminar::tryStartRun(std::shared_ptr run, int queueIndex) { c->sendMessage(msg); } - // notify the rpc client if the start command was used - run->started.fulfiller->fulfill(); - - // this actually spawns the first step - srv->addTask(handleRunStep(run.get()).then([this,run]{ - runFinished(run.get()); - })); - return true; } } @@ -878,50 +777,21 @@ void Laminar::runFinished(Run * r) { auto it = activeJobs.byJobName().equal_range(r->name); uint oldestActive = (it.first == it.second)? buildNums[r->name] : (*it.first)->build - 1; for(int i = static_cast(oldestActive - numKeepRunDirs); i > 0; i--) { - fs::path d = fs::path(homeDir)/"run"/r->name/std::to_string(i); + kj::Path d{"run",r->name,std::to_string(i)}; // Once the directory does not exist, it's probably not worth checking // any further. 99% of the time this loop should only ever have 1 iteration // anyway so hence this (admittedly debatable) optimization. - if(!fs::exists(d)) + if(!fsHome->exists(d)) break; - fs::remove_all(d); + fsHome->remove(d); } // in case we freed up an executor, check the queue assignNewJobs(); } -class MappedFileImpl : public MappedFile { -public: - MappedFileImpl(const char* path) : - fd(open(path, O_RDONLY)), - sz(0), - ptr(nullptr) - { - if(fd == -1) return; - struct stat st; - if(fstat(fd, &st) != 0) return; - sz = st.st_size; - ptr = mmap(nullptr, sz, PROT_READ, MAP_SHARED, fd, 0); - if(ptr == MAP_FAILED) - ptr = nullptr; - } - ~MappedFileImpl() override { - if(ptr) - munmap(ptr, sz); - if(fd != -1) - close(fd); - } - virtual const void* address() override { return ptr; } - virtual size_t size() override { return sz; } -private: - int fd; - size_t sz; - void* ptr; -}; - -kj::Own Laminar::getArtefact(std::string path) { - return kj::heap(fs::path(fs::path(homeDir)/"archive"/path).c_str()); +kj::Maybe> Laminar::getArtefact(std::string path) { + return fsHome->openFile(kj::Path("archive").append(kj::Path::parse(path))); } bool Laminar::handleBadgeRequest(std::string job, std::string &badge) { @@ -967,9 +837,9 @@ R"x( } std::string Laminar::getCustomCss() { - MappedFileImpl cssFile(fs::path(fs::path(homeDir)/"custom"/"style.css").c_str()); - if(cssFile.address()) { - return std::string(static_cast(cssFile.address()), cssFile.size()); + KJ_IF_MAYBE(cssFile, fsHome->tryOpenFile(kj::Path{"custom","style.css"})) { + return (*cssFile)->readAllText().cStr(); + } else { + return std::string(); } - return std::string(); } diff --git a/src/laminar.h b/src/laminar.h index 07042bf..2e2abc3 100644 --- a/src/laminar.h +++ b/src/laminar.h @@ -25,6 +25,7 @@ #include "database.h" #include +#include // Node name to node object map typedef std::unordered_map> NodeMap; @@ -38,8 +39,8 @@ class Json; // the LaminarClient objects (see interface.h) class Laminar final : public LaminarInterface { public: - Laminar(); - ~Laminar() override; + Laminar(const char* homePath); + ~Laminar() noexcept override; // Runs the application forever void run(); @@ -55,7 +56,7 @@ public: void sendStatus(LaminarClient* client) override; bool setParam(std::string job, uint buildNum, std::string param, std::string value) override; - kj::Own getArtefact(std::string path) override; + kj::Maybe> getArtefact(std::string path) override; bool handleBadgeRequest(std::string job, std::string& badge) override; std::string getCustomCss() override; void abortAll() override; @@ -67,7 +68,7 @@ private: bool tryStartRun(std::shared_ptr run, int queueIndex); kj::Promise handleRunStep(Run *run); void runFinished(Run*); - bool nodeCanQueue(const Node&, const Run&) const; + bool nodeCanQueue(const Node&, std::string jobName) const; // expects that Json has started an array void populateArtifacts(Json& out, std::string job, uint num) const; @@ -86,7 +87,8 @@ private: Database* db; Server* srv; NodeMap nodes; - std::string homeDir; + kj::Path homePath; + kj::Own fsHome; std::set clients; std::set waiters; uint numKeepRunDirs; diff --git a/src/main.cpp b/src/main.cpp index a76b51f..ead42c9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -20,6 +20,7 @@ #include "log.h" #include #include +#include static Laminar* laminar; @@ -34,8 +35,9 @@ int main(int argc, char** argv) { } } - laminar = new Laminar; + laminar = new Laminar(getenv("LAMINAR_HOME") ?: "/var/lib/laminar"); kj::UnixEventPort::captureChildExit(); + signal(SIGINT, &laminar_quit); signal(SIGTERM, &laminar_quit); diff --git a/src/run.cpp b/src/run.cpp index 9f84a2f..d1ad469 100644 --- a/src/run.cpp +++ b/src/run.cpp @@ -25,8 +25,11 @@ #include #include -#include -namespace fs = boost::filesystem; +// short syntax helper for kj::Path +template +inline kj::Path operator/(const kj::Path& p, const T& ext) { + return p.append(ext); +} std::string to_string(const RunState& rs) { switch(rs) { @@ -41,16 +44,120 @@ std::string to_string(const RunState& rs) { } -Run::Run() : +Run::Run(std::string name, ParamMap pm, kj::Path&& rootPath) : result(RunState::SUCCESS), - lastResult(RunState::UNKNOWN) + lastResult(RunState::UNKNOWN), + name(name), + params(kj::mv(pm)), + queuedAt(time(nullptr)), + rootPath(kj::mv(rootPath)), + started(kj::newPromiseAndFulfiller()) { + for(auto it = params.begin(); it != params.end();) { + if(it->first[0] == '=') { + if(it->first == "=parentJob") { + parentName = it->second; + } else if(it->first == "=parentBuild") { + parentBuild = atoi(it->second.c_str()); + } else if(it->first == "=reason") { + reasonMsg = it->second; + } else { + LLOG(ERROR, "Unknown internal job parameter", it->first); + } + it = params.erase(it); + } else + ++it; + } } Run::~Run() { LLOG(INFO, "Run destroyed"); } +bool Run::configure(uint buildNum, std::shared_ptr nd, const kj::Directory& fsHome) +{ + kj::Path cfgDir{"cfg"}; + + // create the run directory + kj::Path rd{"run",name,std::to_string(buildNum)}; + bool createWorkdir = true; + KJ_IF_MAYBE(ls, fsHome.tryLstat(rd)) { + KJ_ASSERT(ls->type == kj::FsNode::Type::DIRECTORY); + LLOG(WARNING, "Working directory already exists, removing", rd.toString()); + if(fsHome.tryRemove(rd) == false) { + LLOG(WARNING, "Failed to remove working directory"); + createWorkdir = false; + } + } + if(createWorkdir && fsHome.tryOpenSubdir(rd, kj::WriteMode::CREATE|kj::WriteMode::CREATE_PARENT) == nullptr) { + LLOG(ERROR, "Could not create working directory", rd.toString()); + return false; + } + + // create an archive directory + kj::Path archive = kj::Path{"archive",name,std::to_string(buildNum)}; + if(fsHome.exists(archive)) { + LLOG(WARNING, "Archive directory already exists", archive.toString()); + } else if(fsHome.tryOpenSubdir(archive, kj::WriteMode::CREATE|kj::WriteMode::CREATE_PARENT) == nullptr) { + LLOG(ERROR, "Could not create archive directory", archive.toString()); + return false; + } + + // create a workspace for this job if it doesn't exist + kj::Path ws{"run",name,"workspace"}; + if(!fsHome.exists(ws)) { + fsHome.openSubdir(ws, kj::WriteMode::CREATE|kj::WriteMode::CREATE_PARENT); + // prepend the workspace init script + if(fsHome.exists(cfgDir/"jobs"/(name+".init"))) + addScript(cfgDir/"jobs"/(name+".init"), kj::mv(ws)); + } + + // add scripts + // global before-run script + if(fsHome.exists(cfgDir/"before")) + addScript(cfgDir/"before", rd.clone()); + // per-node before-run script + if(fsHome.exists(cfgDir/"nodes"/(nd->name+".before"))) + addScript(cfgDir/"nodes"/(nd->name+".before"), rd.clone()); + // job before-run script + if(fsHome.exists(cfgDir/"jobs"/(name+".before"))) + addScript(cfgDir/"jobs"/(name+".before"), rd.clone()); + // main run script. must exist. + addScript(cfgDir/"jobs"/(name+".run"), rd.clone()); + // job after-run script + if(fsHome.exists(cfgDir/"jobs"/(name+".after"))) + addScript(cfgDir/"jobs"/(name+".after"), rd.clone(), true); + // per-node after-run script + if(fsHome.exists(cfgDir/"nodes"/(nd->name+".after"))) + addScript(cfgDir/"nodes"/(nd->name+".after"), rd.clone(), true); + // global after-run script + if(fsHome.exists(cfgDir/"after")) + addScript(cfgDir/"after", rd.clone(), true); + + // add environment files + if(fsHome.exists(cfgDir/"env")) + addEnv(cfgDir/"env"); + if(fsHome.exists(cfgDir/"nodes"/(nd->name+".env"))) + addEnv(cfgDir/"nodes"/(nd->name+".env")); + if(fsHome.exists(cfgDir/"jobs"/(name+".env"))) + addEnv(cfgDir/"jobs"/(name+".env")); + + // add job timeout if specified + if(fsHome.exists(cfgDir/"jobs"/(name+".conf"))) { + timeout = parseConfFile((rootPath/cfgDir/"jobs"/(name+".conf")).toString(true).cStr()).get("TIMEOUT", 0); + } + + // All good, we've "started" + startedAt = time(nullptr); + build = buildNum; + node = nd; + + // notifies the rpc client if the start command was used + started.fulfiller->fulfill(); + + return true; +} + std::string Run::reason() const { if(!parentName.empty()) { return std::string("Triggered by upstream ") + parentName + " #" + std::to_string(parentBuild); @@ -62,7 +169,7 @@ bool Run::step() { if(!scripts.size()) return true; - currentScript = scripts.front(); + Script currentScript = kj::mv(scripts.front()); scripts.pop(); int pfd[2]; @@ -84,16 +191,17 @@ bool Run::step() { close(pfd[1]); std::string buildNum = std::to_string(build); - std::string PATH = (fs::path(laminarHome)/"cfg"/"scripts").string() + ":"; + std::string PATH = (rootPath/"cfg"/"scripts").toString(true).cStr(); if(const char* p = getenv("PATH")) { + PATH.append(":"); PATH.append(p); } - chdir(currentScript.cwd.c_str()); + KJ_SYSCALL(chdir((rootPath/currentScript.cwd).toString(true).cStr())); // conf file env vars - for(std::string file : env) { - StringMap vars = parseConfFile(file.c_str()); + for(kj::Path& file : env) { + StringMap vars = parseConfFile((rootPath/file).toString().cStr()); for(auto& it : vars) { setenv(it.first.c_str(), it.second.c_str(), true); } @@ -110,13 +218,14 @@ bool Run::step() { setenv("NODE", node->name.c_str(), true); setenv("RESULT", to_string(result).c_str(), true); setenv("LAST_RESULT", to_string(lastResult).c_str(), true); - setenv("WORKSPACE", (fs::path(laminarHome)/"run"/name/"workspace").string().c_str(), true); - setenv("ARCHIVE", (fs::path(laminarHome)/"archive"/name/buildNum.c_str()).string().c_str(), true); + setenv("WORKSPACE", (rootPath/"run"/name/"workspace").toString(true).cStr(), true); + setenv("ARCHIVE", (rootPath/"archive"/name/buildNum).toString(true).cStr(), true); - fprintf(stderr, "[laminar] Executing %s\n", currentScript.path.c_str()); - execl(currentScript.path.c_str(), currentScript.path.c_str(), NULL); + fprintf(stderr, "[laminar] Executing %s\n", currentScript.path.toString().cStr()); + kj::String execPath = (rootPath/currentScript.path).toString(true); + execl(execPath.cStr(), execPath.cStr(), NULL); // cannot use LLOG because stdout/stderr are captured - fprintf(stderr, "[laminar] Failed to execute %s\n", currentScript.path.c_str()); + fprintf(stderr, "[laminar] Failed to execute %s\n", currentScript.path.toString().cStr()); _exit(1); } @@ -128,12 +237,12 @@ bool Run::step() { return false; } -void Run::addScript(std::string scriptPath, std::string scriptWorkingDir, bool runOnAbort) { - scripts.push({scriptPath, scriptWorkingDir, runOnAbort}); +void Run::addScript(kj::Path scriptPath, kj::Path scriptWorkingDir, bool runOnAbort) { + scripts.push({kj::mv(scriptPath), kj::mv(scriptWorkingDir), runOnAbort}); } -void Run::addEnv(std::string path) { - env.push_back(path); +void Run::addEnv(kj::Path path) { + env.push_back(kj::mv(path)); } void Run::abort(bool respectRunOnAbort) { diff --git a/src/run.h b/src/run.h index 9d38074..8ca6b43 100644 --- a/src/run.h +++ b/src/run.h @@ -27,6 +27,7 @@ #include #include #include +#include enum class RunState { UNKNOWN, @@ -41,29 +42,25 @@ std::string to_string(const RunState& rs); class Node; -// Represents an execution of a job. Not much more than POD +typedef std::unordered_map ParamMap; + +// Represents an execution of a job. class Run { public: - Run(); + Run(std::string name, ParamMap params, kj::Path&& rootPath); ~Run(); // copying this class would be asking for trouble... Run(const Run&) = delete; Run& operator=(const Run&) = delete; + // Call this to "start" the run with a specific number and node + bool configure(uint buildNum, std::shared_ptr node, const kj::Directory &fsHome); + // executes the next script (if any), returning true if there is nothing // more to be done. bool step(); - // adds a script to the queue of scripts to be executed by this run - void addScript(std::string scriptPath, std::string scriptWorkingDir, bool runOnAbort = false); - - // adds a script to the queue using the runDir as the scripts CWD - void addScript(std::string script, bool runOnAbort = false) { addScript(script, runDir, runOnAbort); } - - // adds an environment file that will be sourced before this run - void addEnv(std::string path); - // aborts this run void abort(bool respectRunOnAbort); @@ -73,35 +70,41 @@ public: std::string reason() const; + kj::Promise&& whenStarted() { return kj::mv(started.promise); } + std::shared_ptr node; RunState result; RunState lastResult; - std::string laminarHome; std::string name; - std::string runDir; std::string parentName; int parentBuild = 0; - std::string reasonMsg; uint build = 0; std::string log; kj::Maybe current_pid; int output_fd; std::unordered_map params; - kj::Promise timeout = kj::NEVER_DONE; - kj::PromiseFulfillerPair started = kj::newPromiseAndFulfiller(); + int timeout; time_t queuedAt; time_t startedAt; private: + // adds a script to the queue of scripts to be executed by this run + void addScript(kj::Path scriptPath, kj::Path scriptWorkingDir, bool runOnAbort = false); + + // adds an environment file that will be sourced before this run + void addEnv(kj::Path path); + struct Script { - std::string path; - std::string cwd; + kj::Path path; + kj::Path cwd; bool runOnAbort; }; + kj::Path rootPath; std::queue