From 4fb95fcb4f2ca6ba362044f3d865b99247e6522b Mon Sep 17 00:00:00 2001 From: Oliver Giles Date: Fri, 13 Nov 2020 13:47:02 +1300 Subject: [PATCH] allow contexts to specify accepted jobs using a glob expression, the same way jobs can specify contexts. This allows more flexibility in situations where there are many jobs sharing limited contexts because it may obviate the need to create a conf file for each job. resolves #124 --- UserManual.md | 14 ++++++++++++-- src/context.h | 20 ++------------------ src/laminar.cpp | 49 +++++++++++++++++++++++++++++++++++++++++++++---- src/laminar.h | 1 + 4 files changed, 60 insertions(+), 24 deletions(-) diff --git a/UserManual.md b/UserManual.md index 6708d19..b2db7f0 100644 --- a/UserManual.md +++ b/UserManual.md @@ -504,13 +504,23 @@ EXECUTORS=1 ## Associating a job with a context -When trying to start a job, laminar will wait until the job can be matched to a context which has at least one free executor. You can define which contexts the job will associate with by setting, for example, +When trying to start a job, laminar will wait until the job can be matched to a context which has at least one free executor. There are two ways to associate jobs and contexts. You can specify a comma-separated list of patterns `JOBS` in the context configuration file `/var/lib/laminar/cfg/contexts/CONTEXT.conf`: + +``` +JOBS=amd64-target-*,usage-monitor +``` + +This approach is often preferred when you have many jobs that need to share limited resources. + +Alternatively, you can set ``` CONTEXTS=my-env-*,special_context ``` -in `/var/lib/laminar/cfg/jobs/JOB.conf`. For each of the patterns in the comma-separated list `CONTEXTS`, Laminar will iterate over the known contexts and associate the run with the first context with free executors. Patterns are [glob expressions](http://man7.org/linux/man-pages/man7/glob.7.html). +in `/var/lib/laminar/cfg/jobs/JOB.conf`. This approach is often preferred when you have a small number of jobs that require exclusive access to an environment and you can supply alternative environments (e.g. target devices), because new contexts can be added without modifying the job configuration. + +In both cases, Laminar will iterate over the known contexts and associate the run with the first matching context with free executors. Patterns are [glob expressions](http://man7.org/linux/man-pages/man7/glob.7.html). If `CONTEXTS` is empty or absent (or if `JOB.conf` doesn't exist), laminar will behave as if `CONTEXTS=default` were defined. diff --git a/src/context.h b/src/context.h index b11058c..7ae1b0b 100644 --- a/src/context.h +++ b/src/context.h @@ -1,5 +1,5 @@ /// -/// Copyright 2015-2019 Oliver Giles +/// Copyright 2015-2020 Oliver Giles /// /// This file is part of Laminar /// @@ -19,16 +19,10 @@ #ifndef LAMINAR_CONTEXT_H_ #define LAMINAR_CONTEXT_H_ -#include #include #include class Run; -// FNM_EXTMATCH isn't supported under musl -#if !defined(FNM_EXTMATCH) -#define FNM_EXTMATCH 0 -#endif - // Represents a context within which a Run will be executed. Allows applying // a certain environment to a set of Jobs, or setting a limit on the number // of parallel Runs @@ -39,17 +33,7 @@ public: std::string name; int numExecutors; int busyExecutors = 0; - - bool canQueue(std::set& patterns) { - if(busyExecutors >= numExecutors) - return false; - - for(std::string pattern : patterns) { - if(fnmatch(pattern.c_str(), name.c_str(), FNM_EXTMATCH) == 0) - return true; - } - return false; - } + std::set jobPatterns; }; diff --git a/src/laminar.cpp b/src/laminar.cpp index 70c7d40..9700c56 100644 --- a/src/laminar.cpp +++ b/src/laminar.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,11 @@ #include #include +// FNM_EXTMATCH isn't supported under musl +#if !defined(FNM_EXTMATCH) +#define FNM_EXTMATCH 0 +#endif + // rapidjson::Writer with a StringBuffer is used a lot in Laminar for // preparing JSON messages to send to HTTP clients. A small wrapper // class here reduces verbosity later for this common use case. @@ -490,6 +496,16 @@ bool Laminar::loadConfiguration() { context->name = name; context->numExecutors = conf.get("EXECUTORS", 6); + std::string jobPtns = conf.get("JOBS"); + std::set jobPtnsList; + if(!jobPtns.empty()) { + std::istringstream iss(jobPtns); + std::string job; + while(std::getline(iss, job, ',')) + jobPtnsList.insert(job); + } + context->jobPatterns.swap(jobPtnsList); + knownContexts.insert(name); } } @@ -522,14 +538,20 @@ bool Laminar::loadConfiguration() { std::string ctxPtns = conf.get("CONTEXTS"); + std::set ctxPtnList; if(!ctxPtns.empty()) { std::istringstream iss(ctxPtns); - std::set ctxPtnList; std::string ctx; while(std::getline(iss, ctx, ',')) ctxPtnList.insert(ctx); - jobContexts[jobName].swap(ctxPtnList); } + // Must be present both here and in queueJob because otherwise if a context + // were created while a job is already queued, the default context would be + // dropped when the set of contexts is updated here. + if(ctxPtnList.empty()) + ctxPtnList.insert("default"); + jobContexts[jobName].swap(ctxPtnList); + std::string desc = conf.get("DESCRIPTION"); if(!desc.empty()) { jobDescriptions[jobName] = desc; @@ -552,7 +574,7 @@ std::shared_ptr Laminar::queueJob(std::string name, ParamMap params) { return nullptr; } - // If the job has no contexts (maybe there is no .conf file at all), add the default context + // jobContexts[name] can be empty if there is no .conf file at all if(jobContexts[name].empty()) jobContexts.at(name).insert("default"); @@ -588,11 +610,30 @@ void Laminar::abortAll() { } } +bool Laminar::canQueue(const Context& ctx, const Run& run) const { + if(ctx.busyExecutors >= ctx.numExecutors) + return false; + + // match may be jobs as defined by the context... + for(std::string p : ctx.jobPatterns) { + if(fnmatch(p.c_str(), run.name.c_str(), FNM_EXTMATCH) == 0) + return true; + } + + // ...or context as defined by the job. + for(std::string p : jobContexts.at(run.name)) { + if(fnmatch(p.c_str(), ctx.name.c_str(), FNM_EXTMATCH) == 0) + return true; + } + + return false; +} + bool Laminar::tryStartRun(std::shared_ptr run, int queueIndex) { for(auto& sc : contexts) { std::shared_ptr ctx = sc.second; - if(ctx->canQueue(jobContexts.at(run->name))) { + if(canQueue(*ctx, *run)) { RunState lastResult = RunState::UNKNOWN; // set the last known result if exists. Runs which haven't started yet should diff --git a/src/laminar.h b/src/laminar.h index 2b85df3..fa9f4ea 100644 --- a/src/laminar.h +++ b/src/laminar.h @@ -105,6 +105,7 @@ private: bool loadConfiguration(); void loadCustomizations(); void assignNewJobs(); + bool canQueue(const Context& ctx, const Run& run) const; bool tryStartRun(std::shared_ptr run, int queueIndex); void handleRunFinished(Run*); // expects that Json has started an array