From 18012a8d7af38f33fe618c5c4885b34778fc1b32 Mon Sep 17 00:00:00 2001 From: Oliver Giles Date: Fri, 28 Sep 2018 12:42:46 +0300 Subject: [PATCH] resolves #63: remove locks Add an example to the User Manual for using flock instead --- UserManual.md | 80 +++++++++++++++++++++++------------------------ src/client.cpp | 8 ----- src/laminar.capnp | 2 -- src/server.cpp | 26 --------------- 4 files changed, 39 insertions(+), 77 deletions(-) diff --git a/UserManual.md b/UserManual.md index 0011b25..b72e46e 100644 --- a/UserManual.md +++ b/UserManual.md @@ -390,9 +390,46 @@ cmake $WORKSPACE/myproject make -j4 ``` -**CAUTION**: By default, laminar permits multiple simultaneous runs of the same job. If a job can **modify** the workspace, this might result in inconsistent builds when the job has multiple simultaneous runs. This is unlikely to be an issue for nightly builds, but for SCM-triggered builds it will be. To solve this, use [nodes](#Nodes-and-Tags) to restrict simultaneous execution of jobs, or [locks](#Locks) to temporarily take exclusive control of a resource. +Laminar will automatically create the workspace for a job if it doesn't exist when a job is executed. In this case, the `/var/lib/laminar/cfg/jobs/JOBNAME.init` will be executed if it exists. This is an excellent place to prepare the workspace to a state where subsequent builds can rely on its content: -Laminar will automatically create the workspace for a job if it doesn't exist when a job is executed. In this case, the `/var/lib/laminar/cfg/jobs/JOBNAME.init` will be executed if it exists. This is an excellent place to prepare the workspace to a state where subsequent builds can rely on its content. +```bash +#!/bin/bash -e +echo Initializing workspace +git clone git@example.com:company/project.git . +``` + +**CAUTION**: By default, laminar permits multiple simultaneous runs of the same job. If a job can **modify** the workspace, this might result in inconsistent builds when simultaneous runs access the same content. This is unlikely to be an issue for nightly builds, but for SCM-triggered builds it will be. To solve this, use [nodes](#Nodes-and-Tags) to restrict simultaneous execution of jobs, or consider [flock](https://linux.die.net/man/1/flock). + +The following example uses [flock](https://linux.die.net/man/1/flock) to efficiently share a git repository workspace between multiple simultaneous builds: + +```bash +#!/bin/bash -xe + +# This script expects to be passed the parameter 'rev' which +# should refer to a specific git commit in its source repository. +# The commit ids could have been read from a server-side +# post-commit git hook, where many commits could have been pushed +# at once, but we want to check them all individually. This means +# this job can be executed several times (with different values +# for $rev) simultaneously. + +# Locked subshell for modifying the workspace +( + flock 200 + cd $WORKSPACE + # Download all the latest commits + git fetch + git checkout $rev + cd - + # Fast copy (hard-link) the source from the specific checkout + # to the build dir. This relies on the fact that git unlinks + # during checkout, effectively implementing copy-on-write. + cp -al $WORKSPACE/src src +) 200>$WORKSPACE + +# run the (much longer) regular build process +make -C src +``` --- @@ -495,45 +532,6 @@ EOF --- -# Locks - -*Locks* are a simple way to control access to shared resources. Any string may be used as a lock name. The command `laminarc lock mylock` locks `mylock`. Subsequent calls to `laminarc lock mylock` will block until `laminarc release mylock` is called a corresponding number of times. - -**CAUTION**: Locks are independent of any other job control mechanism in laminar, and will not be released automatically. Making sure calls to lock and release are symmetric is the administrator's responsibility. - -An example use builds on the situation described in [Data sharing and Workspaces](#Data-sharing-and-Workspaces), where a large git repository is stored in the workspace. Conisder this run script: - -```bash -#!/bin/bash -x - -# This script expects to be passed the parameter `rev` which -# should refer to a specific git commit in its source repository. -# The commit ids could have been read from a server-side -# post-commit git hook, where many commits could have been pushed -# at once, but we want to check them all individually. This means -# this job can be executed several times (with different commit ids) -# at once. - -cd $WORKSPACE -# Acquire a lock for modifying the workspace -laminarc lock $JOB-workspace -# Download all the latest commits -git fetch -git checkout $rev -cd - -# Fast copy (hard-link) the specific checkout to the build dir -cp -al $WORKSPACE/src src -# Release the lock to allow other jobs to do the same -laminarc release $JOB-workspace - -# run the (much longer) build process -set -e -cmake src -make -``` - ---- - # Customizing the WebUI If it exists, the file `/var/lib/laminar/custom/style.css` will be served by laminar and may be used to change the appearance of Laminar's WebUI. diff --git a/src/client.cpp b/src/client.cpp index abbb18e..4ba28b4 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -180,14 +180,6 @@ int main(int argc, char** argv) { fprintf(stderr, "Missing $JOB or $RUN or param is not in the format key=value\n"); return EINVAL; } - } else if(strcmp(argv[1], "lock") == 0) { - auto req = laminar.lockRequest(); - req.setLockName(argv[2]); - req.send().wait(waitScope); - } else if(strcmp(argv[1], "release") == 0) { - auto req = laminar.releaseRequest(); - req.setLockName(argv[2]); - req.send().wait(waitScope); } else { fprintf(stderr, "Unknown command %s\n", argv[1]); return EINVAL; diff --git a/src/laminar.capnp b/src/laminar.capnp index a5701f4..7f5d0e3 100644 --- a/src/laminar.capnp +++ b/src/laminar.capnp @@ -6,8 +6,6 @@ interface LaminarCi { start @1 (jobName :Text, params :List(JobParam)) -> (result :MethodResult, buildNum :UInt32); run @2 (jobName :Text, params :List(JobParam)) -> (result :JobResult, buildNum :UInt32); set @3 (jobName :Text, buildNum :UInt32, param :JobParam) -> (result :MethodResult); - lock @4 (lockName :Text) -> (); - release @5 (lockName :Text) -> (); struct JobParam { name @0 :Text; diff --git a/src/server.cpp b/src/server.cpp index 4002655..0719ca3 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -141,31 +141,6 @@ public: return kj::READY_NOW; } - // Take a named lock - kj::Promise lock(LockContext context) override { - std::string lockName = context.getParams().getLockName(); - LLOG(INFO, "RPC lock", lockName); - auto& lockList = locks[lockName]; - lockList.emplace_back(kj::newPromiseAndFulfiller()); - if(lockList.size() == 1) - lockList.front().fulfiller->fulfill(); - return std::move(lockList.back().promise); - } - - // Release a named lock - kj::Promise release(ReleaseContext context) override { - std::string lockName = context.getParams().getLockName(); - LLOG(INFO, "RPC release", lockName); - auto& lockList = locks[lockName]; - if(lockList.size() == 0) { - LLOG(INFO, "Attempt to release unheld lock", lockName); - return kj::READY_NOW; - } - lockList.erase(lockList.begin()); - if(lockList.size() > 0) - lockList.front().fulfiller->fulfill(); - return kj::READY_NOW; - } private: // Implements LaminarWaiter::complete void complete(const Run* r) override { @@ -175,7 +150,6 @@ private: } private: LaminarInterface& laminar; - std::unordered_map>> locks; std::unordered_map>> runWaiters; };