Archives/ohwgiles_laminar
1
0
Fork 0
mirror of https://github.com/ohwgiles/laminar.git synced 2026-03-02 03:40:21 +00:00
Code Issues Projects Releases Wiki Activity

job leader process

Browse Source 
Implement a separate process, the "leader", which runs all the
scripts for a job run, instead of directly from the main laminard
process. This makes for a cleaner process tree view, where the
owning job for a given script is clear; also the leader process
acts as a subreaper to clean up any wayward descendent processes.

Resolves #78.
This commit is contained in:
Oliver Giles
2019-12-21 15:29:37 +02:00
parent 304ef797b8
commit 3fde38c6b8
16 changed files with 600 additions and 409 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
test/laminar-fixture.h
View File

@@ -26,6 +26,7 @@
#include "tempdir.h"
#include "laminar.h"
#include "server.h"
#include "conf.h"
class LaminarFixture : public ::testing::Test {
public:
@@ -38,7 +39,7 @@ public:
settings.bind_rpc = bind_rpc.c_str();
settings.bind_http = bind_http.c_str();
settings.archive_url = "/test-archive/";
server = new Server(ioContext);
server = new Server(*ioContext);
laminar = new Laminar(*server, settings);
}
~LaminarFixture() noexcept(true) {
@@ -47,7 +48,7 @@ public:
}
kj::Own<EventSource> eventSource(const char* path) {
return kj::heap<EventSource>(ioContext, bind_http.c_str(), path);
return kj::heap<EventSource>(*ioContext, bind_http.c_str(), path);
}
void defineJob(const char* name, const char* scriptContent) {
@@ -57,9 +58,61 @@ public:
}
}
struct RunExec {
LaminarCi::JobResult result;
kj::String log;
};
RunExec runJob(const char* name, kj::Maybe<StringMap> params = nullptr) {
auto req = client().runRequest();
req.setJobName(name);
KJ_IF_MAYBE(p, params) {
auto params = req.initParams(p->size());
int i = 0;
for(auto kv : *p) {
params[i].setName(kv.first);
params[i].setValue(kv.second);
i++;
}
}
auto res = req.send().wait(ioContext->waitScope);
std::string path = std::string{"/log/"} + name + "/" + std::to_string(res.getBuildNum());
kj::HttpHeaderTable headerTable;
kj::String log = kj::newHttpClient(ioContext->lowLevelProvider->getTimer(), headerTable,
*ioContext->provider->getNetwork().parseAddress(bind_http.c_str()).wait(ioContext->waitScope))
->request(kj::HttpMethod::GET, path, kj::HttpHeaders(headerTable)).response.wait(ioContext->waitScope).body
->readAllText().wait(ioContext->waitScope);
return { res.getResult(), kj::mv(log) };
}
kj::String stripLaminarLogLines(const kj::String& str) {
auto out = kj::heapString(str.size());
char *o = out.begin();
for(const char *p = str.cStr(), *e = p + str.size(); p < e;) {
const char *nl = strchrnul(p, '\n');
if(!kj::StringPtr{p}.startsWith("[laminar]")) {
memcpy(o, p, nl - p + 1);
o += nl - p + 1;
}
p = nl + 1;
}
*o = '\0';
return out;
}
StringMap parseFromString(kj::StringPtr content) {
char tmp[16] = "/tmp/lt.XXXXXX";
int fd = mkstemp(tmp);
write(fd, content.begin(), content.size());
close(fd);
StringMap map = parseConfFile(tmp);
unlink(tmp);
return map;
}
LaminarCi::Client client() {
if(!rpc) {
auto stream = ioContext.provider->getNetwork().parseAddress(bind_rpc).wait(ioContext.waitScope)->connect().wait(ioContext.waitScope);
auto stream = ioContext->provider->getNetwork().parseAddress(bind_rpc).wait(ioContext->waitScope)->connect().wait(ioContext->waitScope);
auto net = kj::heap<capnp::TwoPartyVatNetwork>(*stream, capnp::rpc::twoparty::Side::CLIENT);
rpc = kj::heap<capnp::RpcSystem<capnp::rpc::twoparty::VatId>>(*net, nullptr).attach(kj::mv(net), kj::mv(stream));
}
@@ -76,7 +129,7 @@ public:
Settings settings;
Server* server;
Laminar* laminar;
static kj::AsyncIoContext ioContext;
static kj::AsyncIoContext* ioContext;
};
#endif // LAMINAR_FIXTURE_H_

80
test/laminar-functional.cpp
View File

@@ -18,13 +18,14 @@
///
#include <kj/async-unix.h>
#include "laminar-fixture.h"
#include "conf.h"
// TODO: consider handling this differently
kj::AsyncIoContext LaminarFixture::ioContext = kj::setupAsyncIo();
kj::AsyncIoContext* LaminarFixture::ioContext;
TEST_F(LaminarFixture, EmptyStatusMessageStructure) {
auto es = eventSource("/");
ioContext.waitScope.poll();
ioContext->waitScope.poll();
ASSERT_EQ(1, es->messages().size());
auto json = es->messages().front().GetObject();
@@ -51,12 +52,7 @@ TEST_F(LaminarFixture, JobNotifyHomePage) {
defineJob("foo", "true");
auto es = eventSource("/");
auto req = client().runRequest();
req.setJobName("foo");
ASSERT_EQ(LaminarCi::JobResult::SUCCESS, req.send().wait(ioContext.waitScope).getResult());
// wait for job completed
ioContext.waitScope.poll();
runJob("foo");
ASSERT_EQ(4, es->messages().size());
@@ -84,13 +80,8 @@ TEST_F(LaminarFixture, OnlyRelevantNotifications) {
auto es1Run = eventSource("/jobs/job1/1");
auto es2Run = eventSource("/jobs/job2/1");
auto req1 = client().runRequest();
req1.setJobName("job1");
ASSERT_EQ(LaminarCi::JobResult::SUCCESS, req1.send().wait(ioContext.waitScope).getResult());
auto req2 = client().runRequest();
req2.setJobName("job2");
ASSERT_EQ(LaminarCi::JobResult::SUCCESS, req2.send().wait(ioContext.waitScope).getResult());
ioContext.waitScope.poll();
runJob("job1");
runJob("job2");
EXPECT_EQ(7, esHome->messages().size());
EXPECT_EQ(7, esJobs->messages().size());
@@ -101,3 +92,62 @@ TEST_F(LaminarFixture, OnlyRelevantNotifications) {
EXPECT_EQ(4, es1Run->messages().size());
EXPECT_EQ(4, es2Run->messages().size());
}
TEST_F(LaminarFixture, FailedStatus) {
defineJob("job1", "false");
auto run = runJob("job1");
ASSERT_EQ(LaminarCi::JobResult::FAILED, run.result);
}
TEST_F(LaminarFixture, WorkingDirectory) {
defineJob("job1", "pwd");
auto run = runJob("job1");
ASSERT_EQ(LaminarCi::JobResult::SUCCESS, run.result);
std::string cwd{tmp.path.append(kj::Path{"run","job1","1"}).toString(true).cStr()};
EXPECT_EQ(cwd + "\n", stripLaminarLogLines(run.log).cStr());
}
TEST_F(LaminarFixture, Environment) {
defineJob("foo", "env");
auto run = runJob("foo");
std::string ws{tmp.path.append(kj::Path{"run","foo","workspace"}).toString(true).cStr()};
std::string archive{tmp.path.append(kj::Path{"archive","foo","1"}).toString(true).cStr()};
StringMap map = parseFromString(run.log);
EXPECT_EQ("1", map["RUN"]);
EXPECT_EQ("foo", map["JOB"]);
EXPECT_EQ("success", map["RESULT"]);
EXPECT_EQ("unknown", map["LAST_RESULT"]);
EXPECT_EQ(ws, map["WORKSPACE"]);
EXPECT_EQ(archive, map["ARCHIVE"]);
}
TEST_F(LaminarFixture, ParamsToEnv) {
defineJob("foo", "env");
StringMap params;
params["foo"] = "bar";
auto run = runJob("foo", params);
StringMap map = parseFromString(run.log);
EXPECT_EQ("bar", map["foo"]);
}
TEST_F(LaminarFixture, Abort) {
defineJob("job1", "yes");
auto req = client().runRequest();
req.setJobName("job1");
auto res = req.send();
// There isn't a nice way of knowing when the leader process is ready to
// handle SIGTERM. Just wait until it prints something to the log
ioContext->waitScope.poll();
kj::HttpHeaderTable headerTable;
char _;
kj::newHttpClient(ioContext->lowLevelProvider->getTimer(), headerTable,
*ioContext->provider->getNetwork().parseAddress(bind_http.c_str()).wait(ioContext->waitScope))
->request(kj::HttpMethod::GET, "/log/job1/1", kj::HttpHeaders(headerTable)).response.wait(ioContext->waitScope).body
->tryRead(&_, 1, 1).wait(ioContext->waitScope);
// now it should be ready to abort
ASSERT_TRUE(laminar->abort("job1", 1));
EXPECT_EQ(LaminarCi::JobResult::ABORTED, res.wait(ioContext->waitScope).getResult());
}

14
test/main.cpp
View File

@@ -18,10 +18,22 @@
///
#include <kj/async-unix.h>
#include <gtest/gtest.h>
#include <kj/debug.h>
// gtest main supplied in order to call captureChildExit
#include "laminar-fixture.h"
#include "leader.h"
// gtest main supplied in order to call captureChildExit and handle process leader
int main(int argc, char **argv) {
if(argv[0][0] == '{')
return leader_main();
// TODO: consider handling this differently
auto ioContext = kj::setupAsyncIo();
LaminarFixture::ioContext = &ioContext;
kj::UnixEventPort::captureChildExit();
//kj::_::Debug::setLogLevel(kj::_::Debug::Severity::INFO);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();

136
test/unit-run.cpp
View File

@@ -1,136 +0,0 @@
///
/// Copyright 2018 Oliver Giles
///
/// This file is part of Laminar
///
/// Laminar is free software: you can redistribute it and/or modify
/// it under the terms of the GNU General Public License as published by
/// the Free Software Foundation, either version 3 of the License, or
/// (at your option) any later version.
///
/// Laminar is distributed in the hope that it will be useful,
/// but WITHOUT ANY WARRANTY; without even the implied warranty of
/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
/// GNU General Public License for more details.
///
/// You should have received a copy of the GNU General Public License
/// along with Laminar. If not, see <http://www.gnu.org/licenses/>
///
#include <gtest/gtest.h>
#include "run.h"
#include "log.h"
#include "context.h"
#include "conf.h"
#include "tempdir.h"
class RunTest : public testing::Test {
protected:
RunTest() :
testing::Test(),
context(std::make_shared<Context>()),
tmp(),
run("foo", ParamMap{}, tmp.path.clone())
{
}
~RunTest() noexcept {}
void wait() {
int state = -1;
waitpid(run.current_pid.orDefault(0), &state, 0);
run.reaped(state);
}
void runAll() {
while(!run.step())
wait();
}
std::string readAllOutput() {
std::string res;
char tmp[64];
for(ssize_t n = read(run.output_fd, tmp, 64); n > 0; n = read(run.output_fd, tmp, 64))
res += std::string(tmp, n);
// strip the first "[laminar] executing.. line
return strchr(res.c_str(), '\n') + 1;
}
StringMap parseFromString(std::string content) {
char tmp[16] = "/tmp/lt.XXXXXX";
int fd = mkstemp(tmp);
write(fd, content.data(), content.size());
close(fd);
StringMap map = parseConfFile(tmp);
unlink(tmp);
return map;
}
std::shared_ptr<Context> context;
TempDir tmp;
class Run run;
void setRunLink(const char * path) {
KJ_IF_MAYBE(f, tmp.fs->tryOpenFile(kj::Path{"cfg", "jobs", run.name + ".run"},
kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT | kj::WriteMode::EXECUTABLE)) {
(f->get())->writeAll(std::string("#!/bin/sh\nexec ") + path + "\n");
}
}
};
TEST_F(RunTest, WorkingDirectory) {
setRunLink("pwd");
run.configure(1, context, *tmp.fs);
runAll();
std::string cwd{tmp.path.append(kj::Path{"run","foo","1"}).toString(true).cStr()};
EXPECT_EQ(cwd + "\n", readAllOutput());
}
TEST_F(RunTest, SuccessStatus) {
setRunLink("true");
run.configure(1, context, *tmp.fs);
runAll();
EXPECT_EQ(RunState::SUCCESS, run.result);
}
TEST_F(RunTest, FailedStatus) {
setRunLink("false");
run.configure(1, context, *tmp.fs);
runAll();
EXPECT_EQ(RunState::FAILED, run.result);
}
TEST_F(RunTest, Environment) {
setRunLink("env");
run.configure(1234, context, *tmp.fs);
runAll();
std::string ws{tmp.path.append(kj::Path{"run","foo","workspace"}).toString(true).cStr()};
std::string archive{tmp.path.append(kj::Path{"archive","foo","1234"}).toString(true).cStr()};
StringMap map = parseFromString(readAllOutput());
EXPECT_EQ("1234", map["RUN"]);
EXPECT_EQ("foo", map["JOB"]);
EXPECT_EQ("success", map["RESULT"]);
EXPECT_EQ("unknown", map["LAST_RESULT"]);
EXPECT_EQ(ws, map["WORKSPACE"]);
EXPECT_EQ(archive, map["ARCHIVE"]);
}
TEST_F(RunTest, ParamsToEnv) {
setRunLink("env");
run.params["foo"] = "bar";
run.configure(1, context, *tmp.fs);
runAll();
StringMap map = parseFromString(readAllOutput());
EXPECT_EQ("bar", map["foo"]);
}
TEST_F(RunTest, Abort) {
setRunLink("yes");
run.configure(1, context, *tmp.fs);
run.step();
usleep(200); // TODO fix
run.abort(false);
wait();
EXPECT_EQ(RunState::ABORTED, run.result);
}