replace websockets with sse and refactor

Large refactor that more closely aligns the codebase to the kj async
style, more clearly exposes an interface for functional testing and
removes cruft. There is a slight increase in coupling between the
Laminar and Http/Rpc classes, but this was always an issue, just until
now more obscured by the arbitrary pure virtual LaminarInterface class
(which has been removed in this change) and the previous lumping
together of all the async stuff in the Server class (which is now
more spread around the code according to function).

This change replaces the use of Websockets with Server Side Events
(SSE). They are simpler and more suitable for the publish-style messages
used by Laminar, and typically require less configuration of the
reverse proxy HTTP server.

Use of gmock is also removed, which eases testing in certain envs.

Resolves #90.

test/eventsource.h

@@ -0,0 +1,71 @@
+///
+/// Copyright 2019 Oliver Giles
+///
+/// This file is part of Laminar
+///
+/// Laminar is free software: you can redistribute it and/or modify
+/// it under the terms of the GNU General Public License as published by
+/// the Free Software Foundation, either version 3 of the License, or
+/// (at your option) any later version.
+///
+/// Laminar is distributed in the hope that it will be useful,
+/// but WITHOUT ANY WARRANTY; without even the implied warranty of
+/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+/// GNU General Public License for more details.
+///
+/// You should have received a copy of the GNU General Public License
+/// along with Laminar.  If not, see <http://www.gnu.org/licenses/>
+///
+#ifndef LAMINAR_EVENTSOURCE_H_
+#define LAMINAR_EVENTSOURCE_H_
+
+#include <kj/async-io.h>
+#include <kj/compat/http.h>
+#include <rapidjson/document.h>
+#include <vector>
+
+class EventSource {
+public:
+    EventSource(kj::AsyncIoContext& ctx, const char* httpConnectAddr, const char* path) :
+        networkAddress(ctx.provider->getNetwork().parseAddress(httpConnectAddr).wait(ctx.waitScope)),
+        httpClient(kj::newHttpClient(ctx.lowLevelProvider->getTimer(), headerTable, *networkAddress)),
+        headerTable(),
+        headers(headerTable),
+        buffer(kj::heapArrayBuilder<char>(BUFFER_SIZE))
+    {
+        headers.add("Accept", "text/event-stream");
+        auto resp = httpClient->request(kj::HttpMethod::GET, path, headers).response.wait(ctx.waitScope);
+        promise = waitForMessages(resp.body.get(), 0).attach(kj::mv(resp));
+    }
+
+    const std::vector<rapidjson::Document>& messages() {
+        return receivedMessages;
+    }
+
+private:
+    kj::Own<kj::NetworkAddress> networkAddress;
+    kj::Own<kj::HttpClient> httpClient;
+    kj::HttpHeaderTable headerTable;
+    kj::HttpHeaders headers;
+    kj::ArrayBuilder<char> buffer;
+    kj::Maybe<kj::Promise<void>> promise;
+    std::vector<rapidjson::Document> receivedMessages;
+
+    kj::Promise<void> waitForMessages(kj::AsyncInputStream* stream, ulong offset) {
+        return stream->read(buffer.asPtr().begin() + offset, 1, BUFFER_SIZE).then([=](size_t s) {
+            ulong end = offset + s;
+            buffer.asPtr().begin()[end] = '\0';
+            if(strcmp(&buffer.asPtr().begin()[end - 2], "\n\n") == 0) {
+                rapidjson::Document d;
+                d.Parse(buffer.begin() + strlen("data: "));
+                receivedMessages.emplace_back(kj::mv(d));
+                end = 0;
+            }
+            return waitForMessages(stream, end);
+        });
+    }
+
+    static const int BUFFER_SIZE = 1024;
+};
+
+#endif // LAMINAR_EVENTSOURCE_H_

test/laminar-fixture.h

@@ -0,0 +1,82 @@
+///
+/// Copyright 2019 Oliver Giles
+///
+/// This file is part of Laminar
+///
+/// Laminar is free software: you can redistribute it and/or modify
+/// it under the terms of the GNU General Public License as published by
+/// the Free Software Foundation, either version 3 of the License, or
+/// (at your option) any later version.
+///
+/// Laminar is distributed in the hope that it will be useful,
+/// but WITHOUT ANY WARRANTY; without even the implied warranty of
+/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+/// GNU General Public License for more details.
+///
+/// You should have received a copy of the GNU General Public License
+/// along with Laminar.  If not, see <http://www.gnu.org/licenses/>
+///
+#ifndef LAMINAR_FIXTURE_H_
+#define LAMINAR_FIXTURE_H_
+
+#include <capnp/rpc-twoparty.h>
+#include <gtest/gtest.h>
+#include "laminar.capnp.h"
+#include "eventsource.h"
+#include "tempdir.h"
+#include "laminar.h"
+#include "server.h"
+
+class LaminarFixture : public ::testing::Test {
+public:
+    LaminarFixture() {
+        bind_rpc = std::string("unix:/") + tmp.path.toString(true).cStr() + "/rpc.sock";
+        bind_http = std::string("unix:/") + tmp.path.toString(true).cStr() + "/http.sock";
+        home = tmp.path.toString(true).cStr();
+        tmp.fs->openSubdir(kj::Path{"cfg", "jobs"}, kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT);
+        settings.home = home.c_str();
+        settings.bind_rpc = bind_rpc.c_str();
+        settings.bind_http = bind_http.c_str();
+        settings.archive_url = "/test-archive/";
+        server = new Server(ioContext);
+        laminar = new Laminar(*server, settings);
+    }
+    ~LaminarFixture() noexcept(true) {
+        delete server;
+        delete laminar;
+    }
+
+    kj::Own<EventSource> eventSource(const char* path) {
+        return kj::heap<EventSource>(ioContext, bind_http.c_str(), path);
+    }
+
+    void defineJob(const char* name, const char* scriptContent) {
+        KJ_IF_MAYBE(f, tmp.fs->tryOpenFile(kj::Path{"cfg", "jobs", std::string(name) + ".run"},
+                kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT | kj::WriteMode::EXECUTABLE)) {
+            (*f)->writeAll(std::string("#!/bin/sh\n") + scriptContent + "\n");
+        }
+    }
+
+    LaminarCi::Client client() {
+        if(!rpc) {
+            auto stream = ioContext.provider->getNetwork().parseAddress(bind_rpc).wait(ioContext.waitScope)->connect().wait(ioContext.waitScope);
+            auto net = kj::heap<capnp::TwoPartyVatNetwork>(*stream, capnp::rpc::twoparty::Side::CLIENT);
+            rpc = kj::heap<capnp::RpcSystem<capnp::rpc::twoparty::VatId>>(*net, nullptr).attach(kj::mv(net), kj::mv(stream));
+        }
+        static capnp::word scratch[4];
+        memset(scratch, 0, sizeof(scratch));
+        auto hostId = capnp::MallocMessageBuilder(scratch).getRoot<capnp::rpc::twoparty::VatId>();
+        hostId.setSide(capnp::rpc::twoparty::Side::SERVER);
+        return rpc->bootstrap(hostId).castAs<LaminarCi>();
+    }
+
+    kj::Own<capnp::RpcSystem<capnp::rpc::twoparty::VatId>> rpc;
+    TempDir tmp;
+    std::string home, bind_rpc, bind_http;
+    Settings settings;
+    Server* server;
+    Laminar* laminar;
+    static kj::AsyncIoContext ioContext;
+};
+
+#endif // LAMINAR_FIXTURE_H_

test/laminar-functional.cpp

@@ -0,0 +1,75 @@
+///
+/// Copyright 2019 Oliver Giles
+///
+/// This file is part of Laminar
+///
+/// Laminar is free software: you can redistribute it and/or modify
+/// it under the terms of the GNU General Public License as published by
+/// the Free Software Foundation, either version 3 of the License, or
+/// (at your option) any later version.
+///
+/// Laminar is distributed in the hope that it will be useful,
+/// but WITHOUT ANY WARRANTY; without even the implied warranty of
+/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+/// GNU General Public License for more details.
+///
+/// You should have received a copy of the GNU General Public License
+/// along with Laminar.  If not, see <http://www.gnu.org/licenses/>
+///
+#include <kj/async-unix.h>
+#include "laminar-fixture.h"
+
+// TODO: consider handling this differently
+kj::AsyncIoContext LaminarFixture::ioContext = kj::setupAsyncIo();
+
+TEST_F(LaminarFixture, EmptyStatusMessageStructure) {
+    auto es = eventSource("/");
+    ioContext.waitScope.poll();
+    ASSERT_EQ(1, es->messages().size());
+
+    auto json = es->messages().front().GetObject();
+    EXPECT_STREQ("status", json["type"].GetString());
+    EXPECT_STREQ("Laminar", json["title"].GetString());
+    EXPECT_LT(time(nullptr) - json["time"].GetInt64(), 1);
+
+    auto data = json["data"].GetObject();
+    EXPECT_TRUE(data.HasMember("recent"));
+    EXPECT_TRUE(data.HasMember("running"));
+    EXPECT_TRUE(data.HasMember("queued"));
+    EXPECT_TRUE(data.HasMember("executorsTotal"));
+    EXPECT_TRUE(data.HasMember("executorsBusy"));
+    EXPECT_TRUE(data.HasMember("buildsPerDay"));
+    EXPECT_TRUE(data.HasMember("buildsPerJob"));
+    EXPECT_TRUE(data.HasMember("timePerJob"));
+    EXPECT_TRUE(data.HasMember("resultChanged"));
+    EXPECT_TRUE(data.HasMember("lowPassRates"));
+    EXPECT_TRUE(data.HasMember("buildTimeChanges"));
+    EXPECT_TRUE(data.HasMember("buildTimeDist"));
+}
+
+TEST_F(LaminarFixture, JobNotifyHomePage) {
+    defineJob("foo", "true");
+    auto es = eventSource("/");
+
+    auto req = client().runRequest();
+    req.setJobName("foo");
+    ASSERT_EQ(LaminarCi::JobResult::SUCCESS, req.send().wait(ioContext.waitScope).getResult());
+
+    // wait for job completed
+    ioContext.waitScope.poll();
+
+    ASSERT_EQ(4, es->messages().size());
+
+    auto job_queued = es->messages().at(1).GetObject();
+    EXPECT_STREQ("job_queued", job_queued["type"].GetString());
+    EXPECT_STREQ("foo", job_queued["data"]["name"].GetString());
+
+    auto job_started = es->messages().at(2).GetObject();
+    EXPECT_STREQ("job_started", job_started["type"].GetString());
+    EXPECT_STREQ("foo", job_started["data"]["name"].GetString());
+
+    auto job_completed = es->messages().at(3).GetObject();
+    EXPECT_STREQ("job_completed", job_completed["type"].GetString());
+    EXPECT_STREQ("foo", job_completed["data"]["name"].GetString());
+}
+

test/main.cpp

@@ -0,0 +1,28 @@
+///
+/// Copyright 2019 Oliver Giles
+///
+/// This file is part of Laminar
+///
+/// Laminar is free software: you can redistribute it and/or modify
+/// it under the terms of the GNU General Public License as published by
+/// the Free Software Foundation, either version 3 of the License, or
+/// (at your option) any later version.
+///
+/// Laminar is distributed in the hope that it will be useful,
+/// but WITHOUT ANY WARRANTY; without even the implied warranty of
+/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+/// GNU General Public License for more details.
+///
+/// You should have received a copy of the GNU General Public License
+/// along with Laminar.  If not, see <http://www.gnu.org/licenses/>
+///
+#include <kj/async-unix.h>
+#include <gtest/gtest.h>
+
+// gtest main supplied in order to call captureChildExit
+int main(int argc, char **argv) {
+    kj::UnixEventPort::captureChildExit();
+
+    ::testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+}

test/test-laminar.cpp

@@ -1,47 +0,0 @@
-///
-/// Copyright 2018 Oliver Giles
-///
-/// This file is part of Laminar
-///
-/// Laminar is free software: you can redistribute it and/or modify
-/// it under the terms of the GNU General Public License as published by
-/// the Free Software Foundation, either version 3 of the License, or
-/// (at your option) any later version.
-///
-/// Laminar is distributed in the hope that it will be useful,
-/// but WITHOUT ANY WARRANTY; without even the implied warranty of
-/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-/// GNU General Public License for more details.
-///
-/// You should have received a copy of the GNU General Public License
-/// along with Laminar.  If not, see <http://www.gnu.org/licenses/>
-///
-#include <gtest/gtest.h>
-#include <rapidjson/stringbuffer.h>
-#include <rapidjson/document.h>
-#include "laminar.h"
-
-class TestLaminarClient : public LaminarClient {
-public:
-    virtual void sendMessage(std::string p) { payload = p; }
-    std::string payload;
-};
-
-class LaminarTest : public testing::Test {
-protected:
-    LaminarTest() :
-        testing::Test(),
-        laminar("/tmp")
-    {}
-    Laminar laminar;
-};
-
-TEST_F(LaminarTest, StatusMessageContainsTime) {
-    TestLaminarClient testClient;
-    laminar.sendStatus(&testClient);
-    rapidjson::Document d;
-    d.Parse(testClient.payload.c_str());
-    ASSERT_TRUE(d.IsObject());
-    ASSERT_TRUE(d.HasMember("time"));
-    EXPECT_GE(1, d["time"].GetInt() - time(nullptr));
-}

test/test-server.cpp

@@ -1,138 +0,0 @@
-///
-/// Copyright 2018 Oliver Giles
-///
-/// This file is part of Laminar
-///
-/// Laminar is free software: you can redistribute it and/or modify
-/// it under the terms of the GNU General Public License as published by
-/// the Free Software Foundation, either version 3 of the License, or
-/// (at your option) any later version.
-///
-/// Laminar is distributed in the hope that it will be useful,
-/// but WITHOUT ANY WARRANTY; without even the implied warranty of
-/// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-/// GNU General Public License for more details.
-///
-/// You should have received a copy of the GNU General Public License
-/// along with Laminar.  If not, see <http://www.gnu.org/licenses/>
-///
-#include <gtest/gtest.h>
-#include <gmock/gmock.h>
-#include <thread>
-#include <sys/socket.h>
-#include "server.h"
-#include "log.h"
-#include "interface.h"
-#include "laminar.capnp.h"
-#include "tempdir.h"
-#include "rpc.h"
-
-class MockLaminar : public LaminarInterface {
-public:
-    LaminarClient* client = nullptr;
-    ~MockLaminar() {}
-    virtual void registerClient(LaminarClient* c) override {
-        ASSERT_EQ(nullptr, client);
-        client = c;
-        EXPECT_CALL(*this, sendStatus(client)).Times(testing::Exactly(1));
-    }
-
-    virtual void deregisterClient(LaminarClient* c) override {
-        ASSERT_EQ(client, c);
-        client = nullptr;
-    }
-
-    // MOCK_METHOD does not seem to work with return values whose destructors have noexcept(false)
-    kj::Maybe<kj::Own<const kj::ReadableFile>> getArtefact(std::string path) override { return nullptr; }
-
-    MOCK_METHOD2(queueJob, std::shared_ptr<Run>(std::string name, ParamMap params));
-    MOCK_METHOD1(registerWaiter, void(LaminarWaiter* waiter));
-    MOCK_METHOD1(deregisterWaiter, void(LaminarWaiter* waiter));
-    MOCK_METHOD1(latestRun, uint(std::string));
-    MOCK_METHOD4(handleLogRequest, bool(std::string, uint, std::string&, bool&));
-
-    MOCK_METHOD1(sendStatus, void(LaminarClient* client));
-    MOCK_METHOD4(setParam, bool(std::string job, uint buildNum, std::string param, std::string value));
-    MOCK_METHOD0(listQueuedJobs, const std::list<std::shared_ptr<Run>>&());
-    MOCK_METHOD0(listRunningJobs, const RunSet&());
-    MOCK_METHOD0(listKnownJobs, std::list<std::string>());
-
-    MOCK_METHOD0(getCustomCss, std::string());
-    MOCK_METHOD2(handleBadgeRequest, bool(std::string, std::string&));
-    MOCK_METHOD2(abort, bool(std::string, uint));
-    MOCK_METHOD0(abortAll, void());
-    MOCK_METHOD0(notifyConfigChanged, void());
-};
-
-class ServerTest : public ::testing::Test {
-protected:
-    void SetUp() override {
-        EXPECT_CALL(mockLaminar, registerWaiter(testing::_));
-        EXPECT_CALL(mockLaminar, deregisterWaiter(testing::_));
-        server = new Server(mockLaminar, "unix:"+std::string(tempDir.path.append("rpc.sock").toString(true).cStr()), "127.0.0.1:8080");
-    }
-    void TearDown() override {
-        delete server;
-    }
-
-    LaminarCi::Client client() const {
-        return server->rpc->rpcInterface.castAs<LaminarCi>();
-    }
-    kj::WaitScope& ws() const {
-        return server->ioContext.waitScope;
-    }
-    void waitForHttpReady() {
-        server->httpReady.promise.wait(server->ioContext.waitScope);
-    }
-
-    kj::Network& network() { return server->ioContext.provider->getNetwork(); }
-    TempDir tempDir;
-    MockLaminar mockLaminar;
-    Server* server;
-};
-
-TEST_F(ServerTest, RpcQueue) {
-    auto req = client().queueRequest();
-    req.setJobName("foo");
-    EXPECT_CALL(mockLaminar, queueJob("foo", ParamMap())).Times(testing::Exactly(1));
-    req.send().wait(ws());
-}
-
-// Tests that agressively closed websockets are properly removed
-// and will not be attempted to be contacted again
-TEST_F(ServerTest, HttpWebsocketRST) {
-    waitForHttpReady();
-
-    // TODO: generalize
-    constexpr const char* WS =
-        "GET / HTTP/1.1\r\n"
-        "Host: localhost:8080\r\n"
-        "Connection: Upgrade\r\n"
-        "Upgrade: websocket\r\n"
-        "Sec-WebSocket-Key: GTFmrUCM9N6B32LdDE3Rzw==\r\n"
-        "Sec-WebSocket-Version: 13\r\n\r\n";
-
-    static char buffer[256];
-    network().parseAddress("localhost:8080").then([this](kj::Own<kj::NetworkAddress>&& addr){
-        return addr->connect().attach(kj::mv(addr)).then([this](kj::Own<kj::AsyncIoStream>&& stream){
-            kj::AsyncIoStream* s = stream.get();
-            return s->write(WS, strlen(WS)).then([this,s](){
-                // Read the websocket header response, ensure the client has been registered
-                return s->tryRead(buffer, 64, 256).then([this,s](size_t sz){
-                    EXPECT_LE(64, sz);
-                    EXPECT_NE(nullptr, mockLaminar.client);
-                    // agressively abort the connection
-                    struct linger so_linger;
-                    so_linger.l_onoff = 1;
-                    so_linger.l_linger = 0;
-                    s->setsockopt(SOL_SOCKET, SO_LINGER, &so_linger, sizeof(so_linger));
-                    return kj::Promise<void>(kj::READY_NOW);
-                });
-            }).attach(kj::mv(stream));
-        });
-    }).wait(ws());
-    ws().poll();
-    // Expect that the client has been cleared. If it has not, Laminar could
-    // try to write to the closed file descriptor, causing an exception
-    EXPECT_EQ(nullptr, mockLaminar.client);
-}