Merge a91ebc610c into 03c5aca663

CMakeLists.txt

@@ -75,6 +75,13 @@ endif()
 set_source_files_properties(src/version.cpp PROPERTIES COMPILE_DEFINITIONS
 	LAMINAR_VERSION=${LAMINAR_VERSION})
 
+# Set default LAMINAR_HOME based on install prefix
+if(NOT LAMINAR_DEFAULT_HOME)
+    set(LAMINAR_DEFAULT_HOME "${CMAKE_INSTALL_PREFIX}/var")
+endif()
+set_source_files_properties(src/main.cpp PROPERTIES COMPILE_DEFINITIONS
+	LAMINAR_DEFAULT_HOME="${LAMINAR_DEFAULT_HOME}")
+
 # This macro takes a list of files, gzips them and converts the output into
 # object files so they can be linked directly into the application.
 # ld generates symbols based on the string argument given to its executable,

src/client.cpp

@@ -101,6 +101,7 @@ static void usage(std::ostream& out) {
     out << "  show-jobs             lists all known jobs.\n";
     out << "  show-queued           lists currently queued jobs.\n";
     out << "  show-running          lists currently running jobs.\n";
+    out << "  output-log JOB [RUN]  outputs the log for the specified job and run (defaults to latest).\n";
     out << "JOB_LIST is of the form:\n";
     out << "  [JOB_NAME [PARAMETER_LIST...]]...\n";
     out << "PARAMETER_LIST is of the form:\n";
@@ -244,6 +245,20 @@ int main(int argc, char** argv) {
         for(auto it : running.getResult()) {
             printf("%s:%d\n", it.getJob().cStr(), it.getBuildNum());
         }
+    } else if(strcmp(argv[1], "output-log") == 0) {
+        if(argc < 3 || argc > 4) {
+            fprintf(stderr, "Usage: %s output-log JOB_NAME [RUN_NUMBER]\n", argv[0]);
+            return EXIT_BAD_ARGUMENT;
+        }
+        auto req = laminar.getLogRequest();
+        req.getRun().setJob(argv[2]);
+        // If run number not provided, use 0 to indicate latest
+        uint buildNum = argc == 4 ? atoi(argv[3]) : 0;
+        req.getRun().setBuildNum(buildNum);
+        auto resp = req.send().wait(waitScope);
+        // Print header showing job and actual run number
+        fprintf(stderr, "=== Log for %s #%u ===\n", argv[2], resp.getBuildNum());
+        printf("%s", resp.getOutput().cStr());
     } else {
         fprintf(stderr, "Unknown command %s\n", argv[1]);
         return EXIT_BAD_ARGUMENT;

src/laminar.capnp

@@ -9,6 +9,7 @@ interface LaminarCi {
     listRunning @4 () -> (result :List(Run));
     listKnown @5 () -> (result :List(Text));
     abort @6 (run :Run) -> (result :MethodResult);
+    getLog @7 (run :Run) -> (output :Text, complete :Bool, buildNum :UInt32);
 
     struct Run {
         job @0 :Text;

src/laminar.cpp

@@ -91,6 +91,7 @@ Laminar::Laminar(Server &server, Settings settings) :
         archiveUrl.append("/");
 
     numKeepRunDirs = 0;
+    numOnDiskLogs = 0;
 
     db = new Database((homePath/"laminar.sqlite").toString(true).cStr());
     // Prepare database for first use
@@ -517,6 +518,9 @@ bool Laminar::loadConfiguration() {
     if(const char* ndirs = getenv("LAMINAR_KEEP_RUNDIRS"))
         numKeepRunDirs = static_cast<uint>(atoi(ndirs));
 
+    if(const char* nlogs = getenv("LAMINAR_ON_DISK_LOGS"))
+        numOnDiskLogs = static_cast<uint>(atoi(nlogs));
+
     std::set<std::string> knownContexts;
 
     KJ_IF_MAYBE(contextsDir, fsHome->tryOpenSubdir(kj::Path{"cfg","contexts"})) {
@@ -772,6 +776,18 @@ void Laminar::handleRunFinished(Run * r) {
      .bind(completedAt, int(r->result), maybeZipped, logsize, r->name, r->build)
      .exec();
 
+    // write uncompressed log to archive directory if enabled
+    if(numOnDiskLogs > 0) {
+        kj::Path archivePath{"archive", r->name, std::to_string(r->build)};
+        try {
+            fsHome->openSubdir(archivePath, kj::WriteMode::CREATE | kj::WriteMode::MODIFY)
+                   ->openFile(kj::Path{"log"}, kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT)
+                   ->writeAll(r->log);
+        } catch(kj::Exception& e) {
+            LLOG(ERROR, "Failed to write log to archive", archivePath.toString(), e.getDescription());
+        }
+    }
+
     // notify clients
     Json j;
     j.set("type", "job_completed")
@@ -821,6 +837,20 @@ void Laminar::handleRunFinished(Run * r) {
         }
     }
 
+    // remove old on-disk logs (independent of run directories)
+    if(numOnDiskLogs > 0) {
+        for(int i = static_cast<int>(oldestActive - numOnDiskLogs); i > 0; i--) {
+            kj::Path logFile{"archive", r->name, std::to_string(i), "log"};
+            if(!fsHome->exists(logFile))
+                break;
+            try {
+                fsHome->remove(logFile);
+            } catch(kj::Exception& e) {
+                LLOG(ERROR, "Could not remove on-disk log", logFile.toString(), e.getDescription());
+            }
+        }
+    }
+
     fsHome->symlink(kj::Path{"archive", r->name, "latest"}, std::to_string(r->build), kj::WriteMode::CREATE|kj::WriteMode::MODIFY);
 
     // in case we freed up an executor, check the queue

src/laminar.h

@@ -128,6 +128,7 @@ private:
     kj::Path homePath;
     kj::Own<const kj::Directory> fsHome;
     uint numKeepRunDirs;
+    uint numOnDiskLogs;
     std::string archiveUrl;
 
     kj::Own<Http> http;

src/main.cpp

@@ -88,7 +88,10 @@ int main(int argc, char** argv) {
 
     Settings settings;
     // Default values when none were supplied in $LAMINAR_CONF_FILE (/etc/laminar.conf)
-    settings.home = getenv("LAMINAR_HOME") ?: "/var/lib/laminar";
+#ifndef LAMINAR_DEFAULT_HOME
+#define LAMINAR_DEFAULT_HOME "/var/lib/laminar"
+#endif
+    settings.home = getenv("LAMINAR_HOME") ?: LAMINAR_DEFAULT_HOME;
     settings.bind_rpc = getenv("LAMINAR_BIND_RPC") ?: INTADDR_RPC_DEFAULT;
     settings.bind_http = getenv("LAMINAR_BIND_HTTP") ?: INTADDR_HTTP_DEFAULT;
     settings.archive_url = getenv("LAMINAR_ARCHIVE_URL") ?: ARCHIVE_URL_DEFAULT;

src/rpc.cpp

@@ -143,6 +143,35 @@ public:
         return kj::READY_NOW;
     }
 
+    kj::Promise<void> getLog(GetLogContext context) override {
+        std::string jobName = context.getParams().getRun().getJob();
+        uint buildNum = context.getParams().getRun().getBuildNum();
+        // If buildNum is 0, get the latest run number
+        if(buildNum == 0) {
+            buildNum = laminar.latestRun(jobName);
+            if(buildNum == 0) {
+                // No runs found for this job
+                context.getResults().setOutput("");
+                context.getResults().setComplete(true);
+                context.getResults().setBuildNum(0);
+                return kj::READY_NOW;
+            }
+        }
+        LLOG(INFO, "RPC getLog", jobName, buildNum);
+        std::string output;
+        bool complete;
+        if(laminar.handleLogRequest(jobName, buildNum, output, complete)) {
+            context.getResults().setOutput(output);
+            context.getResults().setComplete(complete);
+            context.getResults().setBuildNum(buildNum);
+        } else {
+            context.getResults().setOutput("");
+            context.getResults().setComplete(true);
+            context.getResults().setBuildNum(buildNum);
+        }
+        return kj::READY_NOW;
+    }
+
 private:
     // Helper to convert an RPC parameter list to a hash map
     ParamMap params(const capnp::List<LaminarCi::JobParam>::Reader& paramReader) {