allow contexts to specify accepted jobs

using a glob expression, the same way jobs can specify
contexts. This allows more flexibility in situations where
there are many jobs sharing limited contexts because it
may obviate the need to create a conf file for each job.

resolves #124

UserManual.md

@@ -504,13 +504,23 @@ EXECUTORS=1
 
 ## Associating a job with a context
 
-When trying to start a job, laminar will wait until the job can be matched to a context which has at least one free executor. You can define which contexts the job will associate with by setting, for example,
+When trying to start a job, laminar will wait until the job can be matched to a context which has at least one free executor. There are two ways to associate jobs and contexts. You can specify a comma-separated list of patterns `JOBS` in the context configuration file `/var/lib/laminar/cfg/contexts/CONTEXT.conf`:
+
+```
+JOBS=amd64-target-*,usage-monitor
+```
+
+This approach is often preferred when you have many jobs that need to share limited resources.
+
+Alternatively, you can set
 
 ```
 CONTEXTS=my-env-*,special_context
 ```
 
-in `/var/lib/laminar/cfg/jobs/JOB.conf`. For each of the patterns in the comma-separated list `CONTEXTS`, Laminar will iterate over the known contexts and associate the run with the first context with free executors. Patterns are [glob expressions](http://man7.org/linux/man-pages/man7/glob.7.html).
+in `/var/lib/laminar/cfg/jobs/JOB.conf`. This approach is often preferred when you have a small number of jobs that require exclusive access to an environment and you can supply alternative environments (e.g. target devices), because new contexts can be added without modifying the job configuration.
+
+In both cases, Laminar will iterate over the known contexts and associate the run with the first matching context with free executors. Patterns are [glob expressions](http://man7.org/linux/man-pages/man7/glob.7.html).
 
 If `CONTEXTS` is empty or absent (or if `JOB.conf` doesn't exist), laminar will behave as if `CONTEXTS=default` were defined.
 

src/context.h

@@ -1,5 +1,5 @@
 ///
-/// Copyright 2015-2019 Oliver Giles
+/// Copyright 2015-2020 Oliver Giles
 ///
 /// This file is part of Laminar
 ///
@@ -19,16 +19,10 @@
 #ifndef LAMINAR_CONTEXT_H_
 #define LAMINAR_CONTEXT_H_
 
-#include <fnmatch.h>
 #include <string>
 #include <set>
 class Run;
 
-// FNM_EXTMATCH isn't supported under musl
-#if !defined(FNM_EXTMATCH)
-#define FNM_EXTMATCH 0
-#endif
-
 // Represents a context within which a Run will be executed. Allows applying
 // a certain environment to a set of Jobs, or setting a limit on the number
 // of parallel Runs
@@ -39,17 +33,7 @@ public:
     std::string name;
     int numExecutors;
     int busyExecutors = 0;
-
-    bool canQueue(std::set<std::string>& patterns) {
-        if(busyExecutors >= numExecutors)
-            return false;
-
-        for(std::string pattern : patterns) {
-            if(fnmatch(pattern.c_str(), name.c_str(), FNM_EXTMATCH) == 0)
-                return true;
-        }
-        return false;
-    }
+    std::set<std::string> jobPatterns;
 };
 
 

src/laminar.cpp

@@ -28,6 +28,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
+#include <fnmatch.h>
 #include <fstream>
 #include <zlib.h>
 
@@ -36,6 +37,11 @@
 #include <rapidjson/stringbuffer.h>
 #include <rapidjson/writer.h>
 
+// FNM_EXTMATCH isn't supported under musl
+#if !defined(FNM_EXTMATCH)
+#define FNM_EXTMATCH 0
+#endif
+
 // rapidjson::Writer with a StringBuffer is used a lot in Laminar for
 // preparing JSON messages to send to HTTP clients. A small wrapper
 // class here reduces verbosity later for this common use case.
@@ -490,6 +496,16 @@ bool Laminar::loadConfiguration() {
             context->name = name;
             context->numExecutors = conf.get<int>("EXECUTORS", 6);
 
+            std::string jobPtns = conf.get<std::string>("JOBS");
+            std::set<std::string> jobPtnsList;
+            if(!jobPtns.empty()) {
+                std::istringstream iss(jobPtns);
+                std::string job;
+                while(std::getline(iss, job, ','))
+                    jobPtnsList.insert(job);
+            }
+            context->jobPatterns.swap(jobPtnsList);
+
             knownContexts.insert(name);
         }
     }
@@ -522,14 +538,20 @@ bool Laminar::loadConfiguration() {
 
             std::string ctxPtns = conf.get<std::string>("CONTEXTS");
 
+            std::set<std::string> ctxPtnList;
             if(!ctxPtns.empty()) {
                 std::istringstream iss(ctxPtns);
-                std::set<std::string> ctxPtnList;
                 std::string ctx;
                 while(std::getline(iss, ctx, ','))
                     ctxPtnList.insert(ctx);
-                jobContexts[jobName].swap(ctxPtnList);
             }
+            // Must be present both here and in queueJob because otherwise if a context
+            // were created while a job is already queued, the default context would be
+            // dropped when the set of contexts is updated here.
+            if(ctxPtnList.empty())
+                ctxPtnList.insert("default");
+            jobContexts[jobName].swap(ctxPtnList);
+
             std::string desc = conf.get<std::string>("DESCRIPTION");
             if(!desc.empty()) {
                 jobDescriptions[jobName] = desc;
@@ -552,7 +574,7 @@ std::shared_ptr<Run> Laminar::queueJob(std::string name, ParamMap params) {
         return nullptr;
     }
 
-    // If the job has no contexts (maybe there is no .conf file at all), add the default context
+    // jobContexts[name] can be empty if there is no .conf file at all
     if(jobContexts[name].empty())
         jobContexts.at(name).insert("default");
 
@@ -588,11 +610,30 @@ void Laminar::abortAll() {
     }
 }
 
+bool Laminar::canQueue(const Context& ctx, const Run& run) const {
+    if(ctx.busyExecutors >= ctx.numExecutors)
+        return false;
+
+    // match may be jobs as defined by the context...
+    for(std::string p : ctx.jobPatterns) {
+        if(fnmatch(p.c_str(), run.name.c_str(), FNM_EXTMATCH) == 0)
+            return true;
+    }
+
+    // ...or context as defined by the job.
+    for(std::string p : jobContexts.at(run.name)) {
+        if(fnmatch(p.c_str(), ctx.name.c_str(), FNM_EXTMATCH) == 0)
+            return true;
+    }
+
+    return false;
+}
+
 bool Laminar::tryStartRun(std::shared_ptr<Run> run, int queueIndex) {
     for(auto& sc : contexts) {
         std::shared_ptr<Context> ctx = sc.second;
 
-        if(ctx->canQueue(jobContexts.at(run->name))) {
+        if(canQueue(*ctx, *run)) {
             RunState lastResult = RunState::UNKNOWN;
 
             // set the last known result if exists. Runs which haven't started yet should

src/laminar.h

@@ -105,6 +105,7 @@ private:
     bool loadConfiguration();
     void loadCustomizations();
     void assignNewJobs();
+    bool canQueue(const Context& ctx, const Run& run) const;
     bool tryStartRun(std::shared_ptr<Run> run, int queueIndex);
     void handleRunFinished(Run*);
     // expects that Json has started an array