diff --git a/CMakeLists.txt b/CMakeLists.txt index f70743e..15e4eeb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,7 +84,7 @@ generate_compressed_bins(${CMAKE_BINARY_DIR} js/angular.min.js js/angular-route. add_executable(laminard src/database.cpp src/main.cpp src/server.cpp src/laminar.cpp src/conf.cpp src/resources.cpp src/run.cpp laminar.capnp.c++ ${COMPRESSED_BINS}) # TODO: some alternative to boost::filesystem? -target_link_libraries(laminard capnp-rpc capnp kj-async kj pthread boost_filesystem boost_system sqlite3) +target_link_libraries(laminard capnp-rpc capnp kj-async kj pthread boost_filesystem boost_system sqlite3 z) ## Client add_executable(laminarc src/client.cpp laminar.capnp.c++) diff --git a/src/database.cpp b/src/database.cpp index 053e3db..e1865cb 100644 --- a/src/database.cpp +++ b/src/database.cpp @@ -19,6 +19,7 @@ #include "database.h" #include +#include Database::Database(const char *path) { sqlite3_open(path, &hdl); @@ -47,15 +48,17 @@ void Database::Statement::bindValue(int i, int e) { void Database::Statement::bindValue(int i, const char* e) { sqlite3_bind_text(stmt, i, e, -1, NULL); - } void Database::Statement::bindValue(int i, std::string e) { - sqlite3_bind_text(stmt, i, e.c_str(), e.length(), NULL); + sqlite3_bind_blob(stmt, i, e.data(), e.size(), NULL); } template<> std::string Database::Statement::fetchColumn(int col) { - return (char*)sqlite3_column_text(stmt, col); + int sz = sqlite3_column_bytes(stmt, col); + std::string res(sz, '\0'); + memcpy(&res[0], sqlite3_column_blob(stmt, col), sz); + return res; } template<> const char* Database::Statement::fetchColumn(int col) { diff --git a/src/laminar.cpp b/src/laminar.cpp index c0629db..3848cdc 100644 --- a/src/laminar.cpp +++ b/src/laminar.cpp @@ -23,6 +23,7 @@ #include #include +#include #include namespace fs = boost::filesystem; @@ -71,7 +72,7 @@ Laminar::Laminar() { db->exec("CREATE TABLE IF NOT EXISTS builds(" "name TEXT, number INT UNSIGNED, node TEXT, queuedAt INT, " "startedAt INT, completedAt INT, result INT, output TEXT, " - "parentJob TEXT, parentBuild INT, reason TEXT," + "outputLen INT, parentJob TEXT, parentBuild INT, reason TEXT, " "PRIMARY KEY (name, number))"); db->exec("CREATE INDEX IF NOT EXISTS idx_completion_time ON builds(" "completedAt DESC)"); @@ -128,10 +129,16 @@ void Laminar::sendStatus(LaminarClient* client) { if(const Run* run = activeRun(client->scope.job, client->scope.num)) { client->sendMessage(run->log.c_str()); } else { // it must be finished, fetch it from the database - db->stmt("SELECT output FROM builds WHERE name = ? AND number = ?") + db->stmt("SELECT output, outputLen FROM builds WHERE name = ? AND number = ?") .bind(client->scope.job, client->scope.num) - .fetch([=](const char* log) { - client->sendMessage(log); + .fetch([=](str zipped, unsigned long sz) { + std::string log(sz+1,'\0'); + int res = ::uncompress((unsigned char*)&log[0], &sz, + (unsigned char*)zipped.data(), zipped.size()); + if(res == Z_OK) + client->sendMessage(log); + else + LLOG(ERROR, "Failed to uncompress log", res); }); } return; @@ -625,7 +632,7 @@ void Laminar::assignNewJobs() { } // setup run completion handler - run->notifyCompletion = [this](const Run* r) { runFinished(r); }; + run->notifyCompletion = [this](Run* r) { runFinished(r); }; // trigger the first step of the run if(stepRun(run)) { @@ -647,15 +654,24 @@ void Laminar::assignNewJobs() { } } -void Laminar::runFinished(const Run * r) { +void Laminar::runFinished(Run * r) { Node* node = r->node; node->busyExecutors--; LLOG(INFO, "Run completed", r->name, to_string(r->result)); time_t completedAt = time(0); - db->stmt("INSERT INTO builds VALUES(?,?,?,?,?,?,?,?,?,?,?)") + + // compress log + std::string zipped(r->log.size(), '\0'); + size_t logsize = r->log.length(); + size_t zippedSize = zipped.size(); + ::compress((unsigned char*)&zipped[0], &zippedSize, + (unsigned char*)&r->log[0], logsize); + zipped.resize(zippedSize); + + db->stmt("INSERT INTO builds VALUES(?,?,?,?,?,?,?,?,?,?,?,?)") .bind(r->name, r->build, node->name, r->queuedAt, r->startedAt, completedAt, int(r->result), - r->log, r->parentName, r->parentBuild, r->reason()) + zipped, logsize, r->parentName, r->parentBuild, r->reason()) .exec(); // notify clients diff --git a/src/laminar.h b/src/laminar.h index 925ab94..9546766 100644 --- a/src/laminar.h +++ b/src/laminar.h @@ -61,7 +61,7 @@ private: void reapAdvance(); void assignNewJobs(); bool stepRun(std::shared_ptr run); - void runFinished(const Run*); + void runFinished(Run*); bool nodeCanQueue(const Node&, const Run&) const; // expects that Json has started an array void populateArtifacts(Json& out, std::string job, int num) const; diff --git a/src/run.h b/src/run.h index b3cd302..4f61d29 100644 --- a/src/run.h +++ b/src/run.h @@ -68,7 +68,7 @@ public: std::string reason() const; - std::function notifyCompletion; + std::function notifyCompletion; Node* node; RunState result; RunState lastResult;