diff --git a/README.md b/README.md
index d2e0a9e4..95fb1803 100644
--- a/README.md
+++ b/README.md
@@ -332,8 +332,8 @@ ASSISTANT_MODEL | optional. If set, this string is passed along in calls to
ASSISTANT_LONGER_CONTEXT_MODEL | optional. If set, requests that fail because of a context length limitation will be retried with this model set.
OPENAI_API_KEY | optional. Synonym for ASSISTANT_API_KEY that assumes an OpenAI endpoint is being used. Sign up for an account on OpenAI and then generate a secret key [here](https://platform.openai.com/account/api-keys).
-At the time of writing, the AI Assistant is known to function against OpenAI chat completion endpoints for gpt-3.5-turbo and gpt-4.
-It can also function against the chat completion endpoint provided by llama-cpp-python.
+At the time of writing, the AI Assistant is known to function against OpenAI chat completion endpoints (those ending in `/v1/chat/completions`).
+It is also known to function against the chat completion endpoint provided by llama-cpp-python and by [LM Studio](https://lmstudio.ai/). For useful results, the LLM should be on par with GPT 3.5 or above.
#### Sandbox related variables:
diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts
index e529ddb6..4ce03fcb 100644
--- a/app/server/lib/FlexServer.ts
+++ b/app/server/lib/FlexServer.ts
@@ -42,6 +42,7 @@ import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap';
import {Hosts, RequestWithOrg} from 'app/server/lib/extractOrg';
import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth";
+import {GristBullMQJobs, GristJobs} from 'app/server/lib/GristJobs';
import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer,
RequestWithGrist} from 'app/server/lib/GristServer';
import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions';
@@ -186,6 +187,7 @@ export class FlexServer implements GristServer {
private _isReady: boolean = false;
private _updateManager: UpdateManager;
private _sandboxInfo: SandboxInfo;
+ private _jobs?: GristJobs;
constructor(public port: number, public name: string = 'flexServer',
public readonly options: FlexServerOptions = {}) {
@@ -339,6 +341,14 @@ export class FlexServer implements GristServer {
return this.server ? (this.server.address() as AddressInfo).port : this.port;
}
+ /**
+ * Get interface to job queues.
+ */
+ public getJobs(): GristJobs {
+ const jobs = this._jobs || new GristBullMQJobs();
+ return jobs;
+ }
+
/**
* Get a url to an org that should be accessible by all signed-in users. For now, this
* returns the base URL of the personal org (typically docs[-s]).
@@ -943,6 +953,7 @@ export class FlexServer implements GristServer {
if (this.server) { this.server.close(); }
if (this.httpsServer) { this.httpsServer.close(); }
if (this.housekeeper) { await this.housekeeper.stop(); }
+ if (this._jobs) { await this._jobs.stop(); }
await this._shutdown();
if (this._accessTokens) { await this._accessTokens.close(); }
// Do this after _shutdown, since DocWorkerMap is used during shutdown.
diff --git a/app/server/lib/GristJobs.ts b/app/server/lib/GristJobs.ts
new file mode 100644
index 00000000..7ebca916
--- /dev/null
+++ b/app/server/lib/GristJobs.ts
@@ -0,0 +1,339 @@
+import { makeId } from 'app/server/lib/idUtils';
+import log from 'app/server/lib/log';
+import { Queue, Worker } from 'bullmq';
+import IORedis from 'ioredis';
+
+/**
+ *
+ * Support for queues.
+ *
+ * We use BullMQ for queuing, since it seems currently the best the
+ * node ecosystem has to offer. BullMQ relies on Redis. Since queuing
+ * is so handy, but we'd like most of Grist to be usable without Redis,
+ * we make some effort to support queuing without BullMQ. This
+ * may not be sustainable, we'll see.
+ *
+ * Important: if you put a job in a queue, it can outlast your process.
+ * That has implications for testing and deployment, so be careful.
+ *
+ * Long running jobs may be a challenge. BullMQ cancelation
+ * relies on non-open source features:
+ * https://docs.bullmq.io/bullmq-pro/observables/cancelation
+ *
+ */
+export interface GristJobs {
+ /**
+ * All workers and jobs are scoped to individual named queues,
+ * with the real interfaces operating at that level.
+ */
+ queue(queueName?: string): GristQueueScope;
+
+ /**
+ * Shut everything down that we're responsible for.
+ * Set obliterate flag to destroy jobs even if they are
+ * stored externally (useful for testing).
+ */
+ stop(options?: {
+ obliterate?: boolean,
+ }): Promise;
+}
+
+/**
+ * For a given queue, we can add jobs, or methods to process jobs,
+ */
+export interface GristQueueScope {
+ /**
+ * Add a job.
+ */
+ add(name: string, data: any, options?: JobAddOptions): Promise;
+
+
+ /**
+ * Add a job handler for all jobs regardless of name.
+ * Handlers given by handleName take priority, but no
+ * job handling will happen until handleDefault has been
+ * called.
+ */
+ handleDefault(defaultCallback: JobHandler): void;
+
+ /**
+ * Add a job handler for jobs with a specific name.
+ * Handler will only be effective once handleAll is called
+ * to specify what happens to jobs not matching expected
+ * names.
+ */
+ handleName(name: string,
+ callback: (job: GristJob) => Promise): void;
+
+ /**
+ * Shut everything down that we're responsible for.
+ * Set obliterate flag to destroy jobs even if they are
+ * stored externally (useful for testing).
+ */
+ stop(options?: {
+ obliterate?: boolean,
+ }): Promise;
+}
+
+/**
+ * The type of a function for handling jobs on a queue.
+ */
+export type JobHandler = (job: GristJob) => Promise;
+
+/**
+ * The name used for a queue if no specific name is given.
+ */
+export const DEFAULT_QUEUE_NAME = 'default';
+
+/**
+ * BullMQ jobs are a string name, and then a data object.
+ */
+interface GristJob {
+ name: string;
+ data: any;
+}
+
+/**
+ * Options when adding a job. BullMQ has many more.
+ */
+interface JobAddOptions {
+ delay?: number;
+ jobId?: string;
+ repeat?: {
+ every: number;
+ }
+}
+
+/**
+ * Implementation for job functionality across the application.
+ * Will use BullMQ, with an in-memory fallback if Redis is
+ * unavailable.
+ */
+export class GristBullMQJobs implements GristJobs {
+ private _connection?: IORedis;
+ private _checkedForConnection: boolean = false;
+ private _queues = new Map();
+
+ /**
+ * Get BullMQ-compatible options for the queue.
+ */
+ public getQueueOptions() {
+ // Following BullMQ, queue options contain the connection
+ // to redis, if any.
+ if (!this._checkedForConnection) {
+ this._connect();
+ this._checkedForConnection = true;
+ }
+ if (!this._connection) {
+ return {};
+ }
+ return {
+ connection: this._connection,
+ maxRetriesPerRequest: null,
+ };
+ }
+
+ /**
+ * Get an interface scoped to a particular queue by name.
+ */
+ public queue(queueName: string = DEFAULT_QUEUE_NAME): GristQueueScope {
+ if (!this._queues.get(queueName)) {
+ this._queues.set(
+ queueName,
+ new GristBullMQQueueScope(queueName, this),
+ );
+ }
+ return this._queues.get(queueName)!;
+ }
+
+ public async stop(options: {
+ obliterate?: boolean,
+ } = {}) {
+ for (const q of this._queues.values()) {
+ await q.stop(options);
+ }
+ this._queues.clear();
+ this._connection?.disconnect();
+ }
+
+ /**
+ * Connect to Redis if available.
+ */
+ private _connect() {
+ // Connect to Redis for use with BullMQ, if REDIS_URL is set.
+ const urlTxt = process.env.REDIS_URL || process.env.TEST_REDIS_URL;
+ if (!urlTxt) {
+ this._connection = undefined;
+ log.warn('Using in-memory queues, Redis is unavailable');
+ return;
+ }
+ const url = new URL(urlTxt);
+ const conn = new IORedis({
+ host: url.hostname,
+ port: url.port ? parseInt(url.port, 10) : undefined,
+ db: (url.pathname.charAt(0) === '/') ?
+ parseInt(url.pathname.substring(1), 10) : undefined,
+ maxRetriesPerRequest: null,
+ });
+ this._connection = conn;
+ log.info('Storing queues externally in Redis');
+ }
+}
+
+/**
+ * Work with a particular named queue.
+ */
+export class GristBullMQQueueScope implements GristQueueScope {
+ private _queue: Queue|GristWorker|undefined;
+ private _worker: Worker|GristWorker|undefined;
+ private _namedProcessors: Record = {};
+
+ public constructor(public readonly queueName: string,
+ private _owner: GristBullMQJobs) {}
+
+ public handleDefault(defaultCallback: JobHandler) {
+ // The default callback passes any recognized named jobs to
+ // processors added with handleName(), then, if there is no
+ // specific processor, calls the defaultCallback.
+ const callback = async (job: GristJob) => {
+ const processor = this._namedProcessors[job.name] || defaultCallback;
+ return processor(job);
+ };
+ const options = this._owner.getQueueOptions();
+ if (!options.connection) {
+ // If Redis isn't available, we go our own way, not
+ // using BullMQ.
+ const worker = new GristWorker(this.queueName, callback);
+ this._worker = worker;
+ return worker;
+ }
+ const worker = new Worker(this.queueName, callback, options);
+ this._worker = worker;
+ return worker;
+ }
+
+ public handleName(name: string,
+ callback: (job: GristJob) => Promise) {
+ this._namedProcessors[name] = callback;
+ }
+
+ public async stop(options: {
+ obliterate?: boolean,
+ } = {}) {
+ await this._worker?.close();
+ if (options.obliterate) {
+ await this._queue?.obliterate();
+ }
+ }
+
+ public async add(name: string, data: any, options?: JobAddOptions) {
+ await this._getQueue().add(name, data, {
+ ...options,
+ // These settings are quite arbitrary, and should be
+ // revised when it matters, or made controllable.
+ removeOnComplete: {
+ age: 3600, // keep up to 1 hour
+ count: 1000, // keep up to 1000 jobs
+ },
+ removeOnFail: {
+ age: 24 * 3600, // keep up to 24 hours
+ },
+ });
+ }
+
+ private _getQueue(): Queue|GristWorker {
+ if (this._queue) { return this._queue; }
+ const queue = this._pickQueueImplementation();
+ this._queue = queue;
+ return queue;
+ }
+
+ private _pickQueueImplementation() {
+ const name = this.queueName;
+ const queueOptions = this._owner.getQueueOptions();
+ // If we have Redis, get a proper BullMQ interface.
+ // Otherwise, make do.
+ if (queueOptions.connection) {
+ return new Queue(name, queueOptions);
+ }
+ // If in memory, we hand a job directly to the single worker for their
+ // queue. This is very crude.
+ const worker = this._worker;
+ if (!worker) {
+ throw new Error(`no handler yet for ${this.queueName}`);
+ }
+ // We only access workers directly when working in-memory, to
+ // hand jobs directly to them.
+ if (isBullMQWorker(worker)) {
+ // Not expected! Somehow we have a BullMQ worker.
+ throw new Error(`wrong kind of worker for ${this.queueName}`);
+ }
+ return worker;
+ }
+}
+
+/**
+ * If running in memory without Redis, all jobs need to be
+ * created and served by the the same process. This class
+ * pretends to be a BullMQ worker, but accepts jobs directly
+ * without any intermediate queue. This could be elaborated
+ * in future if needed.
+ */
+class GristWorker {
+ private _jobs: Map = new Map();
+
+ public constructor(public queueName: string,
+ private _callback: (job: GristJob) => Promise) {
+ }
+
+ public async close() {
+ for (const job of this._jobs.keys()) {
+ // Key deletion is safe with the keys() iterator.
+ this._clearJob(job);
+ }
+ }
+
+ public async add(name: string, data: any, options?: JobAddOptions) {
+ if (options?.delay) {
+ if (options.repeat) {
+ // Unexpected combination.
+ throw new Error('cannot delay and repeat');
+ }
+ const jobId = options.jobId || makeId();
+ this._clearJob(jobId);
+ this._jobs.set(jobId, setTimeout(() => this._callback({name, data}),
+ options.delay));
+ return;
+ }
+ if (options?.repeat) {
+ const jobId = options.jobId || makeId();
+ this._clearJob(jobId);
+ this._jobs.set(jobId, setInterval(() => this._callback({name, data}),
+ options.repeat.every));
+ return;
+ }
+ await this._callback({name, data});
+ }
+
+ public async obliterate() {
+ await this.close();
+ }
+
+ private _clearJob(id: string) {
+ const job = this._jobs.get(id);
+ if (!job) { return; }
+ // We don't know if the job is a once-off or repeating,
+ // so we call both clearInterval and clearTimeout, which
+ // apparently works.
+ clearInterval(job);
+ clearTimeout(job);
+ this._jobs.delete(id);
+ }
+}
+
+/**
+ * Check if a worker is a real BullMQ worker, or just pretend.
+ */
+function isBullMQWorker(worker: Worker|GristWorker): worker is Worker {
+ return 'isNextJob' in worker;
+}
diff --git a/app/server/lib/GristServer.ts b/app/server/lib/GristServer.ts
index 23feb8c0..602e7760 100644
--- a/app/server/lib/GristServer.ts
+++ b/app/server/lib/GristServer.ts
@@ -15,6 +15,7 @@ import { RequestWithLogin } from 'app/server/lib/Authorizer';
import { Comm } from 'app/server/lib/Comm';
import { create } from 'app/server/lib/create';
import { Hosts } from 'app/server/lib/extractOrg';
+import { GristJobs } from 'app/server/lib/GristJobs';
import { ICreate } from 'app/server/lib/ICreate';
import { IDocStorageManager } from 'app/server/lib/IDocStorageManager';
import { INotifier } from 'app/server/lib/INotifier';
@@ -71,6 +72,7 @@ export interface GristServer {
getBootKey(): string|undefined;
getSandboxInfo(): SandboxInfo|undefined;
getInfo(key: string): any;
+ getJobs(): GristJobs;
}
export interface GristLoginSystem {
@@ -164,7 +166,8 @@ export function createDummyGristServer(): GristServer {
getBundledWidgets() { return []; },
getBootKey() { return undefined; },
getSandboxInfo() { return undefined; },
- getInfo(key: string) { return undefined; }
+ getInfo(key: string) { return undefined; },
+ getJobs(): GristJobs { throw new Error('no job system'); },
};
}
diff --git a/package.json b/package.json
index 9d552213..9e92bc91 100644
--- a/package.json
+++ b/package.json
@@ -115,7 +115,7 @@
"@googleapis/oauth2": "0.2.0",
"@gristlabs/connect-sqlite3": "0.9.11-grist.5",
"@gristlabs/express-session": "1.17.0",
- "@gristlabs/grist-widget": "^0.0.4",
+ "@gristlabs/grist-widget": "^0.0.5",
"@gristlabs/moment-guess": "1.2.4-grist.1",
"@gristlabs/pidusage": "2.0.17",
"@gristlabs/sqlite3": "5.1.4-grist.8",
@@ -128,6 +128,7 @@
"bootstrap": "3.4.1",
"bootstrap-datepicker": "1.9.0",
"bowser": "2.7.0",
+ "bullmq": "5.8.7",
"collect-js-deps": "^0.1.1",
"color-convert": "2.0.1",
"commander": "9.3.0",
diff --git a/static/locales/en.client.json b/static/locales/en.client.json
index 889fbf18..c291d404 100644
--- a/static/locales/en.client.json
+++ b/static/locales/en.client.json
@@ -277,7 +277,11 @@
"Workspace not found": "Workspace not found",
"You are on the {{siteName}} site. You also have access to the following sites:": "You are on the {{siteName}} site. You also have access to the following sites:",
"You are on your personal site. You also have access to the following sites:": "You are on your personal site. You also have access to the following sites:",
- "You may delete a workspace forever once it has no documents in it.": "You may delete a workspace forever once it has no documents in it."
+ "You may delete a workspace forever once it has no documents in it.": "You may delete a workspace forever once it has no documents in it.",
+ "Any documents created in this site will appear here.": "Any documents created in this site will appear here.",
+ "Create my first document": "Create my first document",
+ "You have read-only access to this site. Currently there are no documents.": "You have read-only access to this site. Currently there are no documents.",
+ "personal site": "personal site"
},
"DocPageModel": {
"Add Empty Table": "Add Empty Table",
@@ -531,7 +535,8 @@
"Sign in": "Sign in",
"To use Grist, please either sign up or sign in.": "To use Grist, please either sign up or sign in.",
"Visit our {{link}} to learn more about Grist.": "Visit our {{link}} to learn more about Grist.",
- "Learn more in our {{helpCenterLink}}.": "Learn more in our {{helpCenterLink}}."
+ "Learn more in our {{helpCenterLink}}.": "Learn more in our {{helpCenterLink}}.",
+ "Only show documents": "Only show documents"
},
"HomeLeftPane": {
"Access Details": "Access Details",
@@ -793,7 +798,8 @@
"API Console": "API Console"
},
"TopBar": {
- "Manage Team": "Manage Team"
+ "Manage Team": "Manage Team",
+ "Manage team": "Manage team"
},
"TriggerFormulas": {
"Any field": "Any field",
@@ -1204,7 +1210,10 @@
"These rules are applied after all column rules have been processed, if applicable.": "These rules are applied after all column rules have been processed, if applicable.",
"Example: {{example}}": "Example: {{example}}",
"Filter displayed dropdown values with a condition.": "Filter displayed dropdown values with a condition.",
- "Community widgets are created and maintained by Grist community members.": "Community widgets are created and maintained by Grist community members."
+ "Community widgets are created and maintained by Grist community members.": "Community widgets are created and maintained by Grist community members.",
+ "Creates a reverse column in target table that can be edited from either end.": "Creates a reverse column in target table that can be edited from either end.",
+ "This limitation occurs when one end of a two-way reference is configured as a single Reference.": "This limitation occurs when one end of a two-way reference is configured as a single Reference.",
+ "To allow multiple assignments, change the type of the Reference column to Reference List.": "To allow multiple assignments, change the type of the Reference column to Reference List."
},
"DescriptionConfig": {
"DESCRIPTION": "DESCRIPTION"
@@ -1731,5 +1740,46 @@
},
"The toggle is **off**": "The toggle is **off**",
"The toggle is **on**": "The toggle is **on**"
+ },
+ "HomeIntroCards": {
+ "3 minute video tour": "3 minute video tour",
+ "Blank document": "Blank document",
+ "Find solutions and explore more resources {{helpCenterLink}}": "Find solutions and explore more resources {{helpCenterLink}}",
+ "Finish our basics tutorial": "Finish our basics tutorial",
+ "Help center": "Help center",
+ "Import file": "Import file",
+ "Learn more {{webinarsLinks}}": "Learn more {{webinarsLinks}}",
+ "Start a new document": "Start a new document",
+ "Templates": "Templates",
+ "Tutorial": "Tutorial",
+ "Webinars": "Webinars"
+ },
+ "ReverseReferenceConfig": {
+ "Add two-way reference": "Add two-way reference",
+ "Column": "Column",
+ "Delete": "Delete",
+ "Delete column {{column}} in table {{table}}?": "Delete column {{column}} in table {{table}}?",
+ "It is the reverse of the reference column {{column}} in table {{table}}.": "It is the reverse of the reference column {{column}} in table {{table}}.",
+ "Table": "Table",
+ "Two-way Reference": "Two-way Reference"
+ },
+ "SupportGristButton": {
+ "Admin Panel": "Admin Panel",
+ "Close": "Close",
+ "Help Center": "Help Center",
+ "Opt in to Telemetry": "Opt in to Telemetry",
+ "Opted In": "Opted In",
+ "Support Grist": "Support Grist",
+ "Thank you! Your trust and support is greatly appreciated. Opt out any time from the {{link}} in the user menu.": "Thank you! Your trust and support is greatly appreciated. Opt out any time from the {{link}} in the user menu."
+ },
+ "buildReassignModal": {
+ "Cancel": "Cancel",
+ "Each {{targetTable}} record may only be assigned to a single {{sourceTable}} record.": "Each {{targetTable}} record may only be assigned to a single {{sourceTable}} record.",
+ "Reassign": "Reassign",
+ "Reassign to new {{sourceTable}} records.": "Reassign to new {{sourceTable}} records.",
+ "Reassign to {{sourceTable}} record {{sourceName}}.": "Reassign to {{sourceTable}} record {{sourceName}}.",
+ "Record already assigned_one": "Record already assigned",
+ "Record already assigned_other": "Record already assigned",
+ "{{targetTable}} record {{targetName}} is already assigned to {{sourceTable}} record {{oldSourceName}}.": "{{targetTable}} record {{targetName}} is already assigned to {{sourceTable}} record {{oldSourceName}}."
}
}
diff --git a/test/server/lib/GristJobs.ts b/test/server/lib/GristJobs.ts
new file mode 100644
index 00000000..73c1b569
--- /dev/null
+++ b/test/server/lib/GristJobs.ts
@@ -0,0 +1,150 @@
+import { delay } from 'app/common/delay';
+import { GristBullMQJobs, GristJobs } from 'app/server/lib/GristJobs';
+import { assert } from 'chai';
+
+describe('GristJobs', function() {
+ this.timeout(20000);
+
+ // Clean up any jobs left over from previous round of tests,
+ // if external queues are in use (Redis).
+ beforeEach(async function() {
+ const jobs = new GristBullMQJobs();
+ const q = jobs.queue();
+ await q.stop({obliterate: true});
+ });
+
+ it('can run immediate jobs', async function() {
+ const jobs: GristJobs = new GristBullMQJobs();
+ const q = jobs.queue();
+ try {
+ let ct = 0;
+ let defaultCt = 0;
+ q.handleName('add', async (job) => {
+ ct += job.data.delta;
+ });
+ q.handleDefault(async (job) => {
+ defaultCt++;
+ });
+ await q.add('add', {delta: 2});
+ await waitToPass(async () => {
+ assert.equal(ct, 2);
+ assert.equal(defaultCt, 0);
+ });
+ await q.add('add', {delta: 3});
+ await waitToPass(async () => {
+ assert.equal(ct, 5);
+ assert.equal(defaultCt, 0);
+ });
+ await q.add('badd', {delta: 4});
+ await waitToPass(async () => {
+ assert.equal(ct, 5);
+ assert.equal(defaultCt, 1);
+ });
+ } finally {
+ await jobs.stop({obliterate: true});
+ }
+ });
+
+ it('can run delayed jobs', async function() {
+ const jobs: GristJobs = new GristBullMQJobs();
+ const q = jobs.queue();
+ try {
+ let ct = 0;
+ let defaultCt = 0;
+ q.handleName('add', async (job) => {
+ ct += job.data.delta;
+ });
+ q.handleDefault(async () => {
+ defaultCt++;
+ });
+ await q.add('add', {delta: 2}, {delay: 500});
+ assert.equal(ct, 0);
+ assert.equal(defaultCt, 0);
+ // We need to wait long enough to see the effect.
+ await delay(100);
+ assert.equal(ct, 0);
+ assert.equal(defaultCt, 0);
+ await delay(900);
+ assert.equal(ct, 2);
+ assert.equal(defaultCt, 0);
+ } finally {
+ await jobs.stop({obliterate: true});
+ }
+ });
+
+ it('can run repeated jobs', async function() {
+ const jobs: GristJobs = new GristBullMQJobs();
+ const q = jobs.queue();
+ try {
+ let ct = 0;
+ let defaultCt = 0;
+ q.handleName('add', async (job) => {
+ ct += job.data.delta;
+ });
+ q.handleDefault(async () => {
+ defaultCt++;
+ });
+ await q.add('add', {delta: 2}, {repeat: {every: 250}});
+ await q.add('badd', {delta: 2}, {repeat: {every: 100}});
+ assert.equal(ct, 0);
+ assert.equal(defaultCt, 0);
+ await delay(1000);
+ // allow for a lot of slop on CI
+ assert.isAtLeast(ct, 8 - 4);
+ assert.isAtMost(ct, 8 + 4);
+ assert.isAtLeast(defaultCt, 10 - 3);
+ assert.isAtMost(defaultCt, 10 + 3);
+ } finally {
+ await jobs.stop({obliterate: true});
+ }
+ });
+
+ it('can pick up jobs again', async function() {
+ // this test is only appropriate if we have an external queue.
+ if (!process.env.REDIS_URL) { this.skip(); }
+ const jobs1: GristJobs = new GristBullMQJobs();
+ const q = jobs1.queue();
+ try {
+ let ct = 0;
+ q.handleName('add', async (job) => {
+ ct += job.data.delta;
+ });
+ q.handleDefault(async () => {});
+ await q.add('add', {delta: 1}, {delay: 250});
+ await q.add('add', {delta: 1}, {delay: 1000});
+ await delay(500);
+ assert.equal(ct, 1);
+ await jobs1.stop();
+ const jobs2: GristJobs = new GristBullMQJobs();
+ const q2 = jobs2.queue();
+ try {
+ q2.handleName('add', async (job) => {
+ ct += job.data.delta * 2;
+ });
+ q2.handleDefault(async () => {});
+ await delay(1000);
+ assert.equal(ct, 3);
+ } finally {
+ await jobs2.stop({obliterate: true});
+ }
+ } finally {
+ await jobs1.stop({obliterate: true});
+ }
+ });
+});
+
+async function waitToPass(fn: () => Promise,
+ maxWaitMs: number = 2000) {
+ const start = Date.now();
+ while (Date.now() - start < maxWaitMs) {
+ try {
+ await fn();
+ return true;
+ } catch (e) {
+ // continue after a small delay.
+ await delay(10);
+ }
+ }
+ await fn();
+ return true;
+}
diff --git a/yarn.lock b/yarn.lock
index 69e799b0..3465da77 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -383,10 +383,10 @@
safe-buffer "5.2.0"
uid-safe "~2.1.5"
-"@gristlabs/grist-widget@^0.0.4":
- version "0.0.4"
- resolved "https://registry.yarnpkg.com/@gristlabs/grist-widget/-/grist-widget-0.0.4.tgz#df50d988bcdf8fc26a876cf23b82e258bbdb0ccc"
- integrity sha512-Q0k+GuudU2+0JkuvVkB9UZzqeUKJH8PsaO9ZfxKuqL9/ssIXUd080msB+PJLXB0TU9BkpzPSl7+kLqXTBSnA5g==
+"@gristlabs/grist-widget@^0.0.5":
+ version "0.0.5"
+ resolved "https://registry.yarnpkg.com/@gristlabs/grist-widget/-/grist-widget-0.0.5.tgz#b56b91ad0ee12020fa83993bb11e70d41c11a77b"
+ integrity sha512-0JhCFLjcbNqKsDyipQSWJhj5VglzixHXuqfAvHB858vmbRyG9s5G2x1lCZj2MGD+PCrIXA/qwNmJx/Ei7b6e8Q==
"@gristlabs/moment-guess@1.2.4-grist.1":
version "1.2.4-grist.1"
@@ -435,6 +435,11 @@
resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45"
integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==
+"@ioredis/commands@^1.1.1":
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11"
+ integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==
+
"@jridgewell/gen-mapping@^0.1.0":
version "0.1.1"
resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz#e5d2e450306a9491e3bd77e323e38d7aff315996"
@@ -535,6 +540,36 @@
semver "^7.3.5"
tar "^6.1.11"
+"@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3":
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz#9edec61b22c3082018a79f6d1c30289ddf3d9d11"
+ integrity sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==
+
+"@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3":
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz#33677a275204898ad8acbf62734fc4dc0b6a4855"
+ integrity sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==
+
+"@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3":
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz#19edf7cdc2e7063ee328403c1d895a86dd28f4bb"
+ integrity sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==
+
+"@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3":
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz#94fb0543ba2e28766c3fc439cabbe0440ae70159"
+ integrity sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==
+
+"@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3":
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz#4a0609ab5fe44d07c9c60a11e4484d3c38bbd6e3"
+ integrity sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==
+
+"@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3":
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz#0aa5502d547b57abfc4ac492de68e2006e417242"
+ integrity sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==
+
"@nodelib/fs.scandir@2.1.5":
version "2.1.5"
resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -2220,6 +2255,19 @@ builtin-status-codes@^3.0.0:
resolved "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz"
integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=
+bullmq@^5.8.7:
+ version "5.8.7"
+ resolved "https://registry.yarnpkg.com/bullmq/-/bullmq-5.8.7.tgz#d5da6215377fe29494d74ad307f195f7408b9e2e"
+ integrity sha512-IdAgB9WvJHRAcZtamRLj6fbjMyuIogEa1cjOTWM1pkVoHUOpO34q6FzNMX1R8VOeUhkvkOkWcxI5ENgFLh+TVA==
+ dependencies:
+ cron-parser "^4.6.0"
+ ioredis "^5.4.1"
+ msgpackr "^1.10.1"
+ node-abort-controller "^3.1.1"
+ semver "^7.5.4"
+ tslib "^2.0.0"
+ uuid "^9.0.0"
+
bytes@3.1.2:
version "3.1.2"
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5"
@@ -2559,6 +2607,11 @@ clone@^2.1.2:
resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f"
integrity sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==
+cluster-key-slot@^1.1.0:
+ version "1.1.2"
+ resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac"
+ integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==
+
collect-js-deps@^0.1.1:
version "0.1.1"
resolved "https://registry.npmjs.org/collect-js-deps/-/collect-js-deps-0.1.1.tgz"
@@ -2848,6 +2901,13 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7:
safe-buffer "^5.0.1"
sha.js "^2.4.8"
+cron-parser@^4.6.0:
+ version "4.9.0"
+ resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.9.0.tgz#0340694af3e46a0894978c6f52a6dbb5c0f11ad5"
+ integrity sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==
+ dependencies:
+ luxon "^3.2.1"
+
cross-spawn@^7.0.2, cross-spawn@^7.0.3:
version "7.0.3"
resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
@@ -3071,6 +3131,11 @@ denque@^1.5.0:
resolved "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz"
integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==
+denque@^2.1.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1"
+ integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==
+
depd@2.0.0, depd@~2.0.0:
version "2.0.0"
resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz"
@@ -3109,6 +3174,11 @@ detect-libc@^2.0.0:
resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.1.tgz#e1897aa88fa6ad197862937fbc0441ef352ee0cd"
integrity sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==
+detect-libc@^2.0.1:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.3.tgz#f0cd503b40f9939b894697d19ad50895e30cf700"
+ integrity sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==
+
detective@^4.0.0:
version "4.7.1"
resolved "https://registry.npmjs.org/detective/-/detective-4.7.1.tgz"
@@ -4827,6 +4897,21 @@ interpret@^2.2.0:
resolved "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz"
integrity sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==
+ioredis@^5.4.1:
+ version "5.4.1"
+ resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.4.1.tgz#1c56b70b759f01465913887375ed809134296f40"
+ integrity sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==
+ dependencies:
+ "@ioredis/commands" "^1.1.1"
+ cluster-key-slot "^1.1.0"
+ debug "^4.3.4"
+ denque "^2.1.0"
+ lodash.defaults "^4.2.0"
+ lodash.isarguments "^3.1.0"
+ redis-errors "^1.2.0"
+ redis-parser "^3.0.0"
+ standard-as-callback "^2.1.0"
+
ip@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.1.tgz#e8f3595d33a3ea66490204234b77636965307105"
@@ -5411,6 +5496,11 @@ lodash.includes@^4.3.0:
resolved "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz"
integrity sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==
+lodash.isarguments@^3.1.0:
+ version "3.1.0"
+ resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a"
+ integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==
+
lodash.isboolean@^3.0.3:
version "3.0.3"
resolved "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz"
@@ -5530,6 +5620,11 @@ lru-cache@^6.0.0:
dependencies:
yallist "^4.0.0"
+luxon@^3.2.1:
+ version "3.4.4"
+ resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.4.4.tgz#cf20dc27dc532ba41a169c43fdcc0063601577af"
+ integrity sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==
+
make-dir@^3.0.0, make-dir@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f"
@@ -5948,6 +6043,27 @@ ms@2.1.3, ms@^2.0.0, ms@^2.1.3:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
+msgpackr-extract@^3.0.2:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz#e9d87023de39ce714872f9e9504e3c1996d61012"
+ integrity sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==
+ dependencies:
+ node-gyp-build-optional-packages "5.2.2"
+ optionalDependencies:
+ "@msgpackr-extract/msgpackr-extract-darwin-arm64" "3.0.3"
+ "@msgpackr-extract/msgpackr-extract-darwin-x64" "3.0.3"
+ "@msgpackr-extract/msgpackr-extract-linux-arm" "3.0.3"
+ "@msgpackr-extract/msgpackr-extract-linux-arm64" "3.0.3"
+ "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.3"
+ "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.3"
+
+msgpackr@^1.10.1:
+ version "1.10.2"
+ resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.2.tgz#a73de4767f76659e8c69cf9c80fdfce83937a44a"
+ integrity sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==
+ optionalDependencies:
+ msgpackr-extract "^3.0.2"
+
multiparty@4.2.2:
version "4.2.2"
resolved "https://registry.npmjs.org/multiparty/-/multiparty-4.2.2.tgz"
@@ -6019,6 +6135,11 @@ node-abort-controller@3.0.1:
resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e"
integrity sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw==
+node-abort-controller@^3.1.1:
+ version "3.1.1"
+ resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548"
+ integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==
+
node-addon-api@^3.0.0:
version "3.2.1"
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
@@ -6055,6 +6176,13 @@ node-forge@^0.10.0:
resolved "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz"
integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==
+node-gyp-build-optional-packages@5.2.2:
+ version "5.2.2"
+ resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz#522f50c2d53134d7f3a76cd7255de4ab6c96a3a4"
+ integrity sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==
+ dependencies:
+ detect-libc "^2.0.1"
+
node-gyp-build@^4.2.2:
version "4.6.0"
resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.6.0.tgz#0c52e4cbf54bbd28b709820ef7b6a3c2d6209055"
@@ -7522,6 +7650,11 @@ stackback@0.0.2:
resolved "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz"
integrity sha1-Gsig2Ug4SNFpXkGLbQMaPDzmjjs=
+standard-as-callback@^2.1.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45"
+ integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==
+
statuses@2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63"