From 010af57ed4f2ce226729f001a62ee97d1017bbd2 Mon Sep 17 00:00:00 2001 From: Oliver Giles Date: Sat, 8 Sep 2018 18:16:23 +0300 Subject: [PATCH] resolves #45: new graphs - regressions and recoveries: list of jobs whose run status changed, ordered first by currently failing jobs, secondly by count of jobs since the status change, descending for currently failing jobs and ascending for currently passing jobs - low pass rates: list of the jobs with the worst pass rates calculated over all time - run time changes: jobs with the largest changes in build time. This is calculated as the difference between the range and the standard deviation over the past 10 runs. - average run time distribution: shows the number of jobs in the system divided into buckets based on their average runtime --- src/database.cpp | 35 +++++++++++++- src/database.h | 3 +- src/laminar.cpp | 61 ++++++++++++++++++++++- src/resources/index.html | 47 +++++++++++++++++- src/resources/js/app.js | 102 ++++++++++++++++++++++++++++++++++++--- test/test-database.cpp | 8 +++ 6 files changed, 243 insertions(+), 13 deletions(-) diff --git a/src/database.cpp b/src/database.cpp index b26ed4d..24a4c2d 100644 --- a/src/database.cpp +++ b/src/database.cpp @@ -1,5 +1,5 @@ /// -/// Copyright 2015-2016 Oliver Giles +/// Copyright 2015-2018 Oliver Giles /// /// This file is part of Laminar /// @@ -20,9 +20,38 @@ #include #include +#include + +struct StdevCtx { + double mean; + double M2; + int64_t count; +}; + +static void stdevStep(sqlite3_context *ctx, int, sqlite3_value **args) +{ + StdevCtx* p = static_cast(sqlite3_aggregate_context(ctx, sizeof(StdevCtx))); + // Welford's Online Algorithm + if(sqlite3_value_numeric_type(args[0]) != SQLITE_NULL) { + p->count++; + double val = sqlite3_value_double(args[0]); + double delta = val - p->mean; + p->mean += delta / p->count; + p->M2 += delta * (val - p->mean); + } +} + +static void stdevFinalize(sqlite3_context *context){ + StdevCtx* p = static_cast(sqlite3_aggregate_context(context, 0)); + if(p && p->count > 1) + sqlite3_result_double(context, sqrt(p->M2 / (p->count-1))); + else + sqlite3_result_null(context); +} Database::Database(const char *path) { sqlite3_open(path, &hdl); + sqlite3_create_function(hdl, "STDEV", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, NULL, NULL, stdevStep, stdevFinalize); } Database::~Database() { @@ -96,6 +125,10 @@ template<> ulong Database::Statement::fetchColumn(int col) { return static_cast(sqlite3_column_int64(stmt, col)); } +template<> double Database::Statement::fetchColumn(int col) { + return sqlite3_column_double(stmt, col); +} + bool Database::Statement::row() { return sqlite3_step(stmt) == SQLITE_ROW; } diff --git a/src/database.h b/src/database.h index c7dbc80..87ec48d 100644 --- a/src/database.h +++ b/src/database.h @@ -1,5 +1,5 @@ /// -/// Copyright 2015-2016 Oliver Giles +/// Copyright 2015-2018 Oliver Giles /// /// This file is part of Laminar /// @@ -151,5 +151,6 @@ template<> int Database::Statement::fetchColumn(int col); template<> uint Database::Statement::fetchColumn(int col); template<> long Database::Statement::fetchColumn(int col); template<> ulong Database::Statement::fetchColumn(int col); +template<> double Database::Statement::fetchColumn(int col); #endif // LAMINAR_DATABASE_H_ diff --git a/src/laminar.cpp b/src/laminar.cpp index 37f1dbe..f7fd9da 100644 --- a/src/laminar.cpp +++ b/src/laminar.cpp @@ -51,6 +51,7 @@ public: private: rapidjson::StringBuffer buf; }; +template<> Json& Json::set(const char* key, double value) { String(key); Double(value); return *this; } template<> Json& Json::set(const char* key, const char* value) { String(key); String(value); return *this; } template<> Json& Json::set(const char* key, std::string value) { String(key); String(value.c_str()); return *this; } @@ -354,7 +355,7 @@ void Laminar::sendStatus(LaminarClient* client) { j.startArray("buildsPerDay"); for(int i = 6; i >= 0; --i) { j.StartObject(); - db->stmt("SELECT result, COUNT(*) FROM builds WHERE completedAt > ? AND completedAt < ? GROUP by result") + db->stmt("SELECT result, COUNT(*) FROM builds WHERE completedAt > ? AND completedAt < ? GROUP BY result") .bind(86400*(time(nullptr)/86400 - i), 86400*(time(nullptr)/86400 - (i-1))) .fetch([&](int result, int num){ j.set(to_string(RunState(result)).c_str(), num); @@ -370,12 +371,68 @@ void Laminar::sendStatus(LaminarClient* client) { }); j.EndObject(); j.startObject("timePerJob"); - db->stmt("SELECT name, AVG(completedAt-startedAt) av FROM builds WHERE completedAt > ? GROUP BY name ORDER BY av DESC LIMIT 5") + db->stmt("SELECT name, AVG(completedAt-startedAt) av FROM builds WHERE completedAt > ? GROUP BY name ORDER BY av DESC LIMIT 8") .bind(time(nullptr) - 7 * 86400) .fetch([&](str job, uint time){ j.set(job.c_str(), time); }); j.EndObject(); + j.startArray("resultChanged"); + db->stmt("SELECT b.name,MAX(b.number) as lastSuccess,lastFailure FROM builds AS b JOIN (SELECT name,MAX(number) AS lastFailure FROM builds WHERE result<>? GROUP BY name) AS t ON t.name=b.name WHERE b.result=? GROUP BY b.name ORDER BY lastSuccess>lastFailure, lastFailure-lastSuccess DESC LIMIT 8") + .bind(int(RunState::SUCCESS), int(RunState::SUCCESS)) + .fetch([&](str job, uint lastSuccess, uint lastFailure){ + j.StartObject(); + j.set("name", job) + .set("lastSuccess", lastSuccess) + .set("lastFailure", lastFailure); + j.EndObject(); + }); + j.EndArray(); + j.startArray("lowPassRates"); + db->stmt("SELECT name,CAST(SUM(result==?) AS FLOAT)/COUNT(*) AS passRate FROM builds GROUP BY name ORDER BY passRate ASC LIMIT 8") + .bind(int(RunState::SUCCESS)) + .fetch([&](str job, double passRate){ + j.StartObject(); + j.set("name", job).set("passRate", passRate); + j.EndObject(); + }); + j.EndArray(); + j.startArray("buildTimeChanges"); + db->stmt("SELECT name,GROUP_CONCAT(number),GROUP_CONCAT(completedAt-startedAt) FROM builds WHERE number > (SELECT MAX(number)-10 FROM builds b WHERE b.name=builds.name) GROUP BY name ORDER BY (MAX(completedAt-startedAt)-MIN(completedAt-startedAt))-STDEV(completedAt-startedAt) DESC LIMIT 8") + .fetch([&](str name, str numbers, str durations){ + j.StartObject(); + j.set("name", name); + j.startArray("numbers"); + j.RawValue(numbers.data(), numbers.length(), rapidjson::Type::kArrayType); + j.EndArray(); + j.startArray("durations"); + j.RawValue(durations.data(), durations.length(), rapidjson::Type::kArrayType); + j.EndArray(); + j.EndObject(); + }); + j.EndArray(); + + j.startArray("buildTimeDist"); + db->stmt("WITH ba AS (SELECT name,AVG(completedAt-startedAt) a FROM builds GROUP BY name) SELECT " + "COUNT(CASE WHEN a < 30 THEN 1 END)," + "COUNT(CASE WHEN a >= 30 AND a < 60 THEN 1 END)," + "COUNT(CASE WHEN a >= 60 AND a < 300 THEN 1 END)," + "COUNT(CASE WHEN a >= 300 AND a < 600 THEN 1 END)," + "COUNT(CASE WHEN a >= 600 AND a < 1200 THEN 1 END)," + "COUNT(CASE WHEN a >= 1200 AND a < 2400 THEN 1 END)," + "COUNT(CASE WHEN a >= 2400 AND a < 3600 THEN 1 END)," + "COUNT(CASE WHEN a >= 3600 THEN 1 END) FROM ba") + .fetch([&](uint c1, uint c2, uint c3, uint c4, uint c5, uint c6, uint c7, uint c8){ + j.Int(c1); + j.Int(c2); + j.Int(c3); + j.Int(c4); + j.Int(c5); + j.Int(c6); + j.Int(c7); + j.Int(c8); + }); + j.EndArray(); } j.EndObject(); diff --git a/src/resources/index.html b/src/resources/index.html index 3581c40..f24a0ac 100644 --- a/src/resources/index.html +++ b/src/resources/index.html @@ -47,6 +47,7 @@ right: 10px; padding: 20px; } + /* status icons */ span.status { display: inline-block; width: 1em; @@ -63,6 +64,14 @@ @keyframes spin { to { transform: rotate(360deg); } } + /* chart overlay */ + li.chart-overlay { + position: absolute; + display: inline-block; + background: white; + border: 1px solid lightgray; + padding: 3px; + } /* sort indicators */ a.sort { position: relative; @@ -132,7 +141,7 @@ - +
Longest average run time per job this week
@@ -148,6 +157,42 @@
+
+
+
+
Regressions and recoveries
+
+ +
    +
  • {{job.name}}: #{{job.lastFailure}} since #{{job.lastSuccess}}
    • +
  • {{job.name}}: #{{job.lastSuccess}} since #{{job.lastFailure}}
    • +
+
+
+
+
+
+
Low pass rates
+
+ +
+
+
+
+
+
Run time changes
+
+ +
+
+
+
+
+
Average run time distribution
+
+ +
+
diff --git a/src/resources/js/app.js b/src/resources/js/app.js index b4bc696..08af265 100644 --- a/src/resources/js/app.js +++ b/src/resources/js/app.js @@ -3,6 +3,12 @@ * https://laminar.ohwg.net */ +String.prototype.hashCode = function() { + for(var r=0, i=0; i{ return e.success || 0; }) }, { label: 'Failed Builds', - backgroundColor: "rgba(233,150,122,0.65)", //darkseagreen at 0.65 - borderColor: "crimson", + backgroundColor: "rgba(178,34,34,0.65)", //firebrick at 0.65 + borderColor: "firebrick", data: msg.buildsPerDay.map((e)=>{ return e.failed || 0; }) }] } @@ -237,7 +245,7 @@ const Home = function() { labels: Object.keys(msg.buildsPerJob), datasets: [{ label: 'Most runs per job in last 24hrs', - backgroundColor: "lightsteelblue", + backgroundColor: "steelblue", data: Object.keys(msg.buildsPerJob).map((e)=>{ return msg.buildsPerJob[e]; }) }] } @@ -248,11 +256,73 @@ const Home = function() { labels: Object.keys(msg.timePerJob), datasets: [{ label: 'Longest average runtime this week', - backgroundColor: "lightsteelblue", + backgroundColor: "steelblue", data: Object.keys(msg.timePerJob).map((e)=>{ return msg.timePerJob[e]; }) }] } }); + var chtResultChanges = new Chart(document.getElementById("chartResultChanges"), { + type: 'horizontalBar', + data: { + labels: msg.resultChanged.map((e)=>{ return e.name; }), + datasets: [{ + //label: '% Passed', + backgroundColor: msg.resultChanged.map((e)=>{return e.lastFailure > e.lastSuccess ? 'firebrick' : 'forestgreen';}), + data: msg.resultChanged.map((e)=>{ return e.lastSuccess - e.lastFailure; }), + itemid: msg.resultChanged.map((e)=> { return 'rcd_' + e.name; }) + }] + }, + options:{ + scales:{ + xAxes:[{ticks:{display: false}}], + yAxes:[{ticks:{display: false}}] + }, + tooltips:{ + enabled:false + } + } + }); + var chtPassRates = new Chart(document.getElementById("chartPassRates"), { + type: 'horizontalBar', + data: { + labels: msg.lowPassRates.map((e)=>{ return e.name }), + datasets: [{ + stack: 'passrate', + label: '% Passed', + backgroundColor: "forestgreen", + data: msg.lowPassRates.map((e)=>{ return e.passRate*100; }) + },{ + stack:'passrate', + label: '% Failed', + backgroundColor: "firebrick", + data: msg.lowPassRates.map((e)=>{ return (1-e.passRate)*100; }) + }], + } + }); + var chtBuildTimeChanges = new Chart(document.getElementById("chartBuildTimeChanges"), { + type: 'line', + data: { + labels: [...Array(10).keys()], + datasets: msg.buildTimeChanges.map((e)=>{return { + label: e.name, + data: e.durations, + borderColor: 'hsl('+(e.name.hashCode() % 360)+', 61%, 34%)', + backgroundColor: 'transparent' + }}) + }, + options:{legend:{display:true}} + }); + var chtBuildTimeDist = new Chart(document.getElementById("chartBuildTimeDist"), { + type: 'line', + data: { + labels: ['<30s','30s-1m','1m-5m','5m-10m','10m-20m','20m-40m','40m-60m','>60m'], + datasets: [{ + label: 'Number jobs with average build time in range', + data: msg.buildTimeDist, + backgroundColor: "steelblue", + }] + } + }); }, job_queued: function(data) { state.jobsQueued.splice(0, 0, data); @@ -562,7 +632,23 @@ const Run = function() { // For all charts, set miniumum Y to 0 Chart.scaleService.updateScaleDefaults('linear', { - ticks: { min: 0 } + ticks: { suggestedMin: 0 } +}); +// Don't display legend by default +Chart.defaults.global.legend.display = false; +// Plugin to move a DOM item on top of a chart element +Chart.plugins.register({ + afterDatasetsDraw: (chart) => { + chart.data.datasets.forEach((dataset, i) => { + var meta = chart.getDatasetMeta(i); + if(dataset.itemid) + meta.data.forEach((e,j) => { + var pos = e.getCenterPoint(); + var node = document.getElementById(dataset.itemid[j]); + node.style.top = (pos.y - node.clientHeight/2) + 'px'; + }); + }); + } }); new Vue({ diff --git a/test/test-database.cpp b/test/test-database.cpp index f74709c..4cb5f5a 100644 --- a/test/test-database.cpp +++ b/test/test-database.cpp @@ -73,3 +73,11 @@ TEST_F(DatabaseTest, MultiRow) { }); EXPECT_EQ(10, i); } + +TEST_F(DatabaseTest, StdevFunc) { + double res = 0; + db.stmt("with a (x) as (values (7),(3),(45),(23)) select stdev(x) from a").fetch([&](double r){ + res = r; + }); + EXPECT_FLOAT_EQ(19.0700463205171, res); +}