(core) updates from grist-core

README.md

@@ -332,8 +332,8 @@ ASSISTANT_MODEL     | optional. If set, this string is passed along in calls to
 ASSISTANT_LONGER_CONTEXT_MODEL     | optional. If set, requests that fail because of a context length limitation will be retried with this model set.
 OPENAI_API_KEY      | optional. Synonym for ASSISTANT_API_KEY that assumes an OpenAI endpoint is being used. Sign up for an account on OpenAI and then generate a secret key [here](https://platform.openai.com/account/api-keys).
 
-At the time of writing, the AI Assistant is known to function against OpenAI chat completion endpoints for gpt-3.5-turbo and gpt-4.
+At the time of writing, the AI Assistant is known to function against OpenAI chat completion endpoints (those ending in `/v1/chat/completions`).
-It can also function against the chat completion endpoint provided by <a href="https://github.com/abetlen/llama-cpp-python">llama-cpp-python</a>.
+It is also known to function against the chat completion endpoint provided by <a href="https://github.com/abetlen/llama-cpp-python">llama-cpp-python</a> and by [LM Studio](https://lmstudio.ai/). For useful results, the LLM should be on par with GPT 3.5 or above.
 
 #### Sandbox related variables:
 

app/server/lib/FlexServer.ts

@@ -42,6 +42,7 @@ import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap';
 import {Hosts, RequestWithOrg} from 'app/server/lib/extractOrg';
 import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth";
+import {GristBullMQJobs, GristJobs} from 'app/server/lib/GristJobs';
 import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer,
   RequestWithGrist} from 'app/server/lib/GristServer';
 import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions';
@@ -186,6 +187,7 @@ export class FlexServer implements GristServer {
   private _isReady: boolean = false;
   private _updateManager: UpdateManager;
   private _sandboxInfo: SandboxInfo;
+  private _jobs?: GristJobs;
 
   constructor(public port: number, public name: string = 'flexServer',
               public readonly options: FlexServerOptions = {}) {
@@ -339,6 +341,14 @@ export class FlexServer implements GristServer {
     return this.server ? (this.server.address() as AddressInfo).port : this.port;
   }
 
+  /**
+   * Get interface to job queues.
+   */
+  public getJobs(): GristJobs {
+    const jobs = this._jobs || new GristBullMQJobs();
+    return jobs;
+  }
+
   /**
    * Get a url to an org that should be accessible by all signed-in users. For now, this
    * returns the base URL of the personal org (typically docs[-s]).
@@ -943,6 +953,7 @@ export class FlexServer implements GristServer {
     if (this.server)      { this.server.close(); }
     if (this.httpsServer) { this.httpsServer.close(); }
     if (this.housekeeper) { await this.housekeeper.stop(); }
+    if (this._jobs)       { await this._jobs.stop(); }
     await this._shutdown();
     if (this._accessTokens) { await this._accessTokens.close(); }
     // Do this after _shutdown, since DocWorkerMap is used during shutdown.

app/server/lib/GristJobs.ts

@@ -0,0 +1,339 @@
+import { makeId } from 'app/server/lib/idUtils';
+import log from 'app/server/lib/log';
+import { Queue, Worker } from 'bullmq';
+import IORedis from 'ioredis';
+
+/**
+ *
+ * Support for queues.
+ *
+ * We use BullMQ for queuing, since it seems currently the best the
+ * node ecosystem has to offer. BullMQ relies on Redis. Since queuing
+ * is so handy, but we'd like most of Grist to be usable without Redis,
+ * we make some effort to support queuing without BullMQ. This
+ * may not be sustainable, we'll see.
+ *
+ * Important: if you put a job in a queue, it can outlast your process.
+ * That has implications for testing and deployment, so be careful.
+ *
+ * Long running jobs may be a challenge. BullMQ cancelation
+ * relies on non-open source features:
+ *  https://docs.bullmq.io/bullmq-pro/observables/cancelation
+ *
+ */
+export interface GristJobs {
+  /**
+   * All workers and jobs are scoped to individual named queues,
+   * with the real interfaces operating at that level.
+   */
+  queue(queueName?: string): GristQueueScope;
+
+  /**
+   * Shut everything down that we're responsible for.
+   * Set obliterate flag to destroy jobs even if they are
+   * stored externally (useful for testing).
+   */
+  stop(options?: {
+    obliterate?: boolean,
+  }): Promise<void>;
+}
+
+/**
+ * For a given queue, we can add jobs, or methods to process jobs,
+ */
+export interface GristQueueScope {
+  /**
+   * Add a job.
+   */
+  add(name: string, data: any, options?: JobAddOptions): Promise<void>;
+
+
+  /**
+   * Add a job handler for all jobs regardless of name.
+   * Handlers given by handleName take priority, but no
+   * job handling will happen until handleDefault has been
+   * called.
+   */
+  handleDefault(defaultCallback: JobHandler): void;
+
+  /**
+   * Add a job handler for jobs with a specific name.
+   * Handler will only be effective once handleAll is called
+   * to specify what happens to jobs not matching expected
+   * names.
+   */
+  handleName(name: string,
+             callback: (job: GristJob) => Promise<any>): void;
+
+  /**
+   * Shut everything down that we're responsible for.
+   * Set obliterate flag to destroy jobs even if they are
+   * stored externally (useful for testing).
+   */
+  stop(options?: {
+    obliterate?: boolean,
+  }): Promise<void>;
+}
+
+/**
+ * The type of a function for handling jobs on a queue.
+ */
+export type JobHandler = (job: GristJob) => Promise<any>;
+
+/**
+ * The name used for a queue if no specific name is given.
+ */
+export const DEFAULT_QUEUE_NAME = 'default';
+
+/**
+ * BullMQ jobs are a string name, and then a data object.
+ */
+interface GristJob {
+  name: string;
+  data: any;
+}
+
+/**
+ * Options when adding a job. BullMQ has many more.
+ */
+interface JobAddOptions {
+  delay?: number;
+  jobId?: string;
+  repeat?: {
+    every: number;
+  }
+}
+
+/**
+ * Implementation for job functionality across the application.
+ * Will use BullMQ, with an in-memory fallback if Redis is
+ * unavailable.
+ */
+export class GristBullMQJobs implements GristJobs {
+  private _connection?: IORedis;
+  private _checkedForConnection: boolean = false;
+  private _queues = new Map<string, GristQueueScope>();
+
+  /**
+   * Get BullMQ-compatible options for the queue.
+   */
+  public getQueueOptions() {
+    // Following BullMQ, queue options contain the connection
+    // to redis, if any.
+    if (!this._checkedForConnection) {
+      this._connect();
+      this._checkedForConnection = true;
+    }
+    if (!this._connection) {
+      return {};
+    }
+    return {
+      connection: this._connection,
+      maxRetriesPerRequest: null,
+    };
+  }
+
+  /**
+   * Get an interface scoped to a particular queue by name.
+   */
+  public queue(queueName: string = DEFAULT_QUEUE_NAME): GristQueueScope {
+    if (!this._queues.get(queueName)) {
+      this._queues.set(
+        queueName,
+        new GristBullMQQueueScope(queueName, this),
+      );
+    }
+    return this._queues.get(queueName)!;
+  }
+
+  public async stop(options: {
+    obliterate?: boolean,
+  } = {}) {
+    for (const q of this._queues.values()) {
+      await q.stop(options);
+    }
+    this._queues.clear();
+    this._connection?.disconnect();
+  }
+
+  /**
+   * Connect to Redis if available.
+   */
+  private _connect() {
+    // Connect to Redis for use with BullMQ, if REDIS_URL is set.
+    const urlTxt = process.env.REDIS_URL || process.env.TEST_REDIS_URL;
+    if (!urlTxt) {
+      this._connection = undefined;
+      log.warn('Using in-memory queues, Redis is unavailable');
+      return;
+    }
+    const url = new URL(urlTxt);
+    const conn = new IORedis({
+      host: url.hostname,
+      port: url.port ? parseInt(url.port, 10) : undefined,
+      db: (url.pathname.charAt(0) === '/') ?
+          parseInt(url.pathname.substring(1), 10) : undefined,
+      maxRetriesPerRequest: null,
+    });
+    this._connection = conn;
+    log.info('Storing queues externally in Redis');
+  }
+}
+
+/**
+ * Work with a particular named queue.
+ */
+export class GristBullMQQueueScope implements GristQueueScope {
+  private _queue: Queue|GristWorker|undefined;
+  private _worker: Worker|GristWorker|undefined;
+  private _namedProcessors: Record<string, JobHandler> = {};
+
+  public constructor(public readonly queueName: string,
+                     private _owner: GristBullMQJobs) {}
+
+  public handleDefault(defaultCallback: JobHandler) {
+    // The default callback passes any recognized named jobs to
+    // processors added with handleName(), then, if there is no
+    // specific processor, calls the defaultCallback.
+    const callback = async (job: GristJob) => {
+      const processor = this._namedProcessors[job.name] || defaultCallback;
+      return processor(job);
+    };
+    const options = this._owner.getQueueOptions();
+    if (!options.connection) {
+      // If Redis isn't available, we go our own way, not
+      // using BullMQ.
+      const worker = new GristWorker(this.queueName, callback);
+      this._worker = worker;
+      return worker;
+    }
+    const worker = new Worker(this.queueName, callback, options);
+    this._worker = worker;
+    return worker;
+  }
+
+  public handleName(name: string,
+                    callback: (job: GristJob) => Promise<any>) {
+    this._namedProcessors[name] = callback;
+  }
+
+  public async stop(options: {
+    obliterate?: boolean,
+  } = {}) {
+    await this._worker?.close();
+    if (options.obliterate) {
+      await this._queue?.obliterate();
+    }
+  }
+
+  public async add(name: string, data: any, options?: JobAddOptions) {
+    await this._getQueue().add(name, data, {
+      ...options,
+      // These settings are quite arbitrary, and should be
+      // revised when it matters, or made controllable.
+      removeOnComplete: {
+        age: 3600, // keep up to 1 hour
+        count: 1000, // keep up to 1000 jobs
+      },
+      removeOnFail: {
+        age: 24 * 3600, // keep up to 24 hours
+      },
+    });
+  }
+
+  private _getQueue(): Queue|GristWorker {
+    if (this._queue) { return this._queue; }
+    const queue = this._pickQueueImplementation();
+    this._queue = queue;
+    return queue;
+  }
+
+  private _pickQueueImplementation() {
+    const name = this.queueName;
+    const queueOptions = this._owner.getQueueOptions();
+    // If we have Redis, get a proper BullMQ interface.
+    // Otherwise, make do.
+    if (queueOptions.connection) {
+      return new Queue(name, queueOptions);
+    }
+    // If in memory, we hand a job directly to the single worker for their
+    // queue. This is very crude.
+    const worker = this._worker;
+    if (!worker) {
+      throw new Error(`no handler yet for ${this.queueName}`);
+    }
+    // We only access workers directly when working in-memory, to
+    // hand jobs directly to them.
+    if (isBullMQWorker(worker)) {
+      // Not expected! Somehow we have a BullMQ worker.
+      throw new Error(`wrong kind of worker for ${this.queueName}`);
+    }
+    return worker;
+  }
+}
+
+/**
+ * If running in memory without Redis, all jobs need to be
+ * created and served by the the same process. This class
+ * pretends to be a BullMQ worker, but accepts jobs directly
+ * without any intermediate queue. This could be elaborated
+ * in future if needed.
+ */
+class GristWorker {
+  private _jobs: Map<string, NodeJS.Timeout> = new Map();
+
+  public constructor(public queueName: string,
+                     private _callback: (job: GristJob) => Promise<void>) {
+  }
+
+  public async close() {
+    for (const job of this._jobs.keys()) {
+      // Key deletion is safe with the keys() iterator.
+      this._clearJob(job);
+    }
+  }
+
+  public async add(name: string, data: any, options?: JobAddOptions) {
+    if (options?.delay) {
+      if (options.repeat) {
+        // Unexpected combination.
+        throw new Error('cannot delay and repeat');
+      }
+      const jobId = options.jobId || makeId();
+      this._clearJob(jobId);
+      this._jobs.set(jobId, setTimeout(() => this._callback({name, data}),
+                                       options.delay));
+      return;
+    }
+    if (options?.repeat) {
+      const jobId = options.jobId || makeId();
+      this._clearJob(jobId);
+      this._jobs.set(jobId, setInterval(() => this._callback({name, data}),
+                                        options.repeat.every));
+      return;
+    }
+    await this._callback({name, data});
+  }
+
+  public async obliterate() {
+    await this.close();
+  }
+
+  private _clearJob(id: string) {
+    const job = this._jobs.get(id);
+    if (!job) { return; }
+    // We don't know if the job is a once-off or repeating,
+    // so we call both clearInterval and clearTimeout, which
+    // apparently works.
+    clearInterval(job);
+    clearTimeout(job);
+    this._jobs.delete(id);
+  }
+}
+
+/**
+ * Check if a worker is a real BullMQ worker, or just pretend.
+ */
+function isBullMQWorker(worker: Worker|GristWorker): worker is Worker {
+  return 'isNextJob' in worker;
+}

app/server/lib/GristServer.ts

@@ -15,6 +15,7 @@ import { RequestWithLogin } from 'app/server/lib/Authorizer';
 import { Comm } from 'app/server/lib/Comm';
 import { create } from 'app/server/lib/create';
 import { Hosts } from 'app/server/lib/extractOrg';
+import { GristJobs } from 'app/server/lib/GristJobs';
 import { ICreate } from 'app/server/lib/ICreate';
 import { IDocStorageManager } from 'app/server/lib/IDocStorageManager';
 import { INotifier } from 'app/server/lib/INotifier';
@@ -71,6 +72,7 @@ export interface GristServer {
   getBootKey(): string|undefined;
   getSandboxInfo(): SandboxInfo|undefined;
   getInfo(key: string): any;
+  getJobs(): GristJobs;
 }
 
 export interface GristLoginSystem {
@@ -164,7 +166,8 @@ export function createDummyGristServer(): GristServer {
     getBundledWidgets() { return []; },
     getBootKey() { return undefined; },
     getSandboxInfo() { return undefined; },
-    getInfo(key: string) { return undefined; }
+    getInfo(key: string) { return undefined; },
+    getJobs(): GristJobs { throw new Error('no job system'); },
   };
 }
 

package.json

@@ -115,7 +115,7 @@
     "@googleapis/oauth2": "0.2.0",
     "@gristlabs/connect-sqlite3": "0.9.11-grist.5",
     "@gristlabs/express-session": "1.17.0",
-    "@gristlabs/grist-widget": "^0.0.4",
+    "@gristlabs/grist-widget": "^0.0.5",
     "@gristlabs/moment-guess": "1.2.4-grist.1",
     "@gristlabs/pidusage": "2.0.17",
     "@gristlabs/sqlite3": "5.1.4-grist.8",
@@ -128,6 +128,7 @@
     "bootstrap": "3.4.1",
     "bootstrap-datepicker": "1.9.0",
     "bowser": "2.7.0",
+    "bullmq": "5.8.7",
     "collect-js-deps": "^0.1.1",
     "color-convert": "2.0.1",
     "commander": "9.3.0",

static/locales/en.client.json

@@ -277,7 +277,11 @@
         "Workspace not found": "Workspace not found",
         "You are on the {{siteName}} site. You also have access to the following sites:": "You are on the {{siteName}} site. You also have access to the following sites:",
         "You are on your personal site. You also have access to the following sites:": "You are on your personal site. You also have access to the following sites:",
-        "You may delete a workspace forever once it has no documents in it.": "You may delete a workspace forever once it has no documents in it."
+        "You may delete a workspace forever once it has no documents in it.": "You may delete a workspace forever once it has no documents in it.",
+        "Any documents created in this site will appear here.": "Any documents created in this site will appear here.",
+        "Create my first document": "Create my first document",
+        "You have read-only access to this site. Currently there are no documents.": "You have read-only access to this site. Currently there are no documents.",
+        "personal site": "personal site"
     },
     "DocPageModel": {
         "Add Empty Table": "Add Empty Table",
@@ -531,7 +535,8 @@
         "Sign in": "Sign in",
         "To use Grist, please either sign up or sign in.": "To use Grist, please either sign up or sign in.",
         "Visit our {{link}} to learn more about Grist.": "Visit our {{link}} to learn more about Grist.",
-        "Learn more in our {{helpCenterLink}}.": "Learn more in our {{helpCenterLink}}."
+        "Learn more in our {{helpCenterLink}}.": "Learn more in our {{helpCenterLink}}.",
+        "Only show documents": "Only show documents"
     },
     "HomeLeftPane": {
         "Access Details": "Access Details",
@@ -793,7 +798,8 @@
         "API Console": "API Console"
     },
     "TopBar": {
-        "Manage Team": "Manage Team"
+        "Manage Team": "Manage Team",
+        "Manage team": "Manage team"
     },
     "TriggerFormulas": {
         "Any field": "Any field",
@@ -1204,7 +1210,10 @@
         "These rules are applied after all column rules have been processed, if applicable.": "These rules are applied after all column rules have been processed, if applicable.",
         "Example: {{example}}": "Example: {{example}}",
         "Filter displayed dropdown values with a condition.": "Filter displayed dropdown values with a condition.",
-        "Community widgets are created and maintained by Grist community members.": "Community widgets are created and maintained by Grist community members."
+        "Community widgets are created and maintained by Grist community members.": "Community widgets are created and maintained by Grist community members.",
+        "Creates a reverse column in target table that can be edited from either end.": "Creates a reverse column in target table that can be edited from either end.",
+        "This limitation occurs when one end of a two-way reference is configured as a single Reference.": "This limitation occurs when one end of a two-way reference is configured as a single Reference.",
+        "To allow multiple assignments, change the type of the Reference column to Reference List.": "To allow multiple assignments, change the type of the Reference column to Reference List."
     },
     "DescriptionConfig": {
         "DESCRIPTION": "DESCRIPTION"
@@ -1731,5 +1740,46 @@
         },
         "The toggle is **off**": "The toggle is **off**",
         "The toggle is **on**": "The toggle is **on**"
+    },
+    "HomeIntroCards": {
+        "3 minute video tour": "3 minute video tour",
+        "Blank document": "Blank document",
+        "Find solutions and explore more resources {{helpCenterLink}}": "Find solutions and explore more resources {{helpCenterLink}}",
+        "Finish our basics tutorial": "Finish our basics tutorial",
+        "Help center": "Help center",
+        "Import file": "Import file",
+        "Learn more {{webinarsLinks}}": "Learn more {{webinarsLinks}}",
+        "Start a new document": "Start a new document",
+        "Templates": "Templates",
+        "Tutorial": "Tutorial",
+        "Webinars": "Webinars"
+    },
+    "ReverseReferenceConfig": {
+        "Add two-way reference": "Add two-way reference",
+        "Column": "Column",
+        "Delete": "Delete",
+        "Delete column {{column}} in table {{table}}?": "Delete column {{column}} in table {{table}}?",
+        "It is the reverse of the reference column {{column}} in table {{table}}.": "It is the reverse of the reference column {{column}} in table {{table}}.",
+        "Table": "Table",
+        "Two-way Reference": "Two-way Reference"
+    },
+    "SupportGristButton": {
+        "Admin Panel": "Admin Panel",
+        "Close": "Close",
+        "Help Center": "Help Center",
+        "Opt in to Telemetry": "Opt in to Telemetry",
+        "Opted In": "Opted In",
+        "Support Grist": "Support Grist",
+        "Thank you! Your trust and support is greatly appreciated. Opt out any time from the {{link}} in the user menu.": "Thank you! Your trust and support is greatly appreciated. Opt out any time from the {{link}} in the user menu."
+    },
+    "buildReassignModal": {
+        "Cancel": "Cancel",
+        "Each {{targetTable}} record may only be assigned to a single {{sourceTable}} record.": "Each {{targetTable}} record may only be assigned to a single {{sourceTable}} record.",
+        "Reassign": "Reassign",
+        "Reassign to new {{sourceTable}} records.": "Reassign to new {{sourceTable}} records.",
+        "Reassign to {{sourceTable}} record {{sourceName}}.": "Reassign to {{sourceTable}} record {{sourceName}}.",
+        "Record already assigned_one": "Record already assigned",
+        "Record already assigned_other": "Record already assigned",
+        "{{targetTable}} record {{targetName}} is already assigned to {{sourceTable}} record          {{oldSourceName}}.": "{{targetTable}} record {{targetName}} is already assigned to {{sourceTable}} record          {{oldSourceName}}."
     }
 }

test/server/lib/GristJobs.ts

@@ -0,0 +1,150 @@
+import { delay } from 'app/common/delay';
+import { GristBullMQJobs, GristJobs } from 'app/server/lib/GristJobs';
+import { assert } from 'chai';
+
+describe('GristJobs', function() {
+  this.timeout(20000);
+
+  // Clean up any jobs left over from previous round of tests,
+  // if external queues are in use (Redis).
+  beforeEach(async function() {
+    const jobs = new GristBullMQJobs();
+    const q = jobs.queue();
+    await q.stop({obliterate: true});
+  });
+
+  it('can run immediate jobs', async function() {
+    const jobs: GristJobs = new GristBullMQJobs();
+    const q = jobs.queue();
+    try {
+      let ct = 0;
+      let defaultCt = 0;
+      q.handleName('add', async (job) => {
+        ct += job.data.delta;
+      });
+      q.handleDefault(async (job) => {
+        defaultCt++;
+      });
+      await q.add('add', {delta: 2});
+      await waitToPass(async () => {
+        assert.equal(ct, 2);
+        assert.equal(defaultCt, 0);
+      });
+      await q.add('add', {delta: 3});
+      await waitToPass(async () => {
+        assert.equal(ct, 5);
+        assert.equal(defaultCt, 0);
+      });
+      await q.add('badd', {delta: 4});
+      await waitToPass(async () => {
+        assert.equal(ct, 5);
+        assert.equal(defaultCt, 1);
+      });
+    } finally {
+      await jobs.stop({obliterate: true});
+    }
+  });
+
+  it('can run delayed jobs', async function() {
+    const jobs: GristJobs = new GristBullMQJobs();
+    const q = jobs.queue();
+    try {
+      let ct = 0;
+      let defaultCt = 0;
+      q.handleName('add', async (job) => {
+        ct += job.data.delta;
+      });
+      q.handleDefault(async () => {
+        defaultCt++;
+      });
+      await q.add('add', {delta: 2}, {delay: 500});
+      assert.equal(ct, 0);
+      assert.equal(defaultCt, 0);
+      // We need to wait long enough to see the effect.
+      await delay(100);
+      assert.equal(ct, 0);
+      assert.equal(defaultCt, 0);
+      await delay(900);
+      assert.equal(ct, 2);
+      assert.equal(defaultCt, 0);
+    } finally {
+      await jobs.stop({obliterate: true});
+    }
+  });
+
+  it('can run repeated jobs', async function() {
+    const jobs: GristJobs = new GristBullMQJobs();
+    const q = jobs.queue();
+    try {
+      let ct = 0;
+      let defaultCt = 0;
+      q.handleName('add', async (job) => {
+        ct += job.data.delta;
+      });
+      q.handleDefault(async () => {
+        defaultCt++;
+      });
+      await q.add('add', {delta: 2}, {repeat: {every: 250}});
+      await q.add('badd', {delta: 2}, {repeat: {every: 100}});
+      assert.equal(ct, 0);
+      assert.equal(defaultCt, 0);
+      await delay(1000);
+      // allow for a lot of slop on CI
+      assert.isAtLeast(ct, 8 - 4);
+      assert.isAtMost(ct, 8 + 4);
+      assert.isAtLeast(defaultCt, 10 - 3);
+      assert.isAtMost(defaultCt, 10 + 3);
+    } finally {
+      await jobs.stop({obliterate: true});
+    }
+  });
+
+  it('can pick up jobs again', async function() {
+    // this test is only appropriate if we have an external queue.
+    if (!process.env.REDIS_URL) { this.skip(); }
+    const jobs1: GristJobs = new GristBullMQJobs();
+    const q = jobs1.queue();
+    try {
+      let ct = 0;
+      q.handleName('add', async (job) => {
+        ct += job.data.delta;
+      });
+      q.handleDefault(async () => {});
+      await q.add('add', {delta: 1}, {delay: 250});
+      await q.add('add', {delta: 1}, {delay: 1000});
+      await delay(500);
+      assert.equal(ct, 1);
+      await jobs1.stop();
+      const jobs2: GristJobs = new GristBullMQJobs();
+      const q2 = jobs2.queue();
+      try {
+        q2.handleName('add', async (job) => {
+          ct += job.data.delta * 2;
+        });
+        q2.handleDefault(async () => {});
+        await delay(1000);
+        assert.equal(ct, 3);
+      } finally {
+        await jobs2.stop({obliterate: true});
+      }
+    } finally {
+      await jobs1.stop({obliterate: true});
+    }
+  });
+});
+
+async function waitToPass(fn: () => Promise<void>,
+                          maxWaitMs: number = 2000) {
+  const start = Date.now();
+  while (Date.now() - start < maxWaitMs) {
+    try {
+      await fn();
+      return true;
+    } catch (e) {
+      // continue after a small delay.
+      await delay(10);
+    }
+  }
+  await fn();
+  return true;
+}

yarn.lock

@@ -383,10 +383,10 @@
     safe-buffer "5.2.0"
     uid-safe "~2.1.5"
 
-"@gristlabs/grist-widget@^0.0.4":
+"@gristlabs/grist-widget@^0.0.5":
-  version "0.0.4"
+  version "0.0.5"
-  resolved "https://registry.yarnpkg.com/@gristlabs/grist-widget/-/grist-widget-0.0.4.tgz#df50d988bcdf8fc26a876cf23b82e258bbdb0ccc"
+  resolved "https://registry.yarnpkg.com/@gristlabs/grist-widget/-/grist-widget-0.0.5.tgz#b56b91ad0ee12020fa83993bb11e70d41c11a77b"
-  integrity sha512-Q0k+GuudU2+0JkuvVkB9UZzqeUKJH8PsaO9ZfxKuqL9/ssIXUd080msB+PJLXB0TU9BkpzPSl7+kLqXTBSnA5g==
+  integrity sha512-0JhCFLjcbNqKsDyipQSWJhj5VglzixHXuqfAvHB858vmbRyG9s5G2x1lCZj2MGD+PCrIXA/qwNmJx/Ei7b6e8Q==
 
 "@gristlabs/moment-guess@1.2.4-grist.1":
   version "1.2.4-grist.1"
@@ -435,6 +435,11 @@
   resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45"
   integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==
 
+"@ioredis/commands@^1.1.1":
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11"
+  integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==
+
 "@jridgewell/gen-mapping@^0.1.0":
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz#e5d2e450306a9491e3bd77e323e38d7aff315996"
@@ -535,6 +540,36 @@
     semver "^7.3.5"
     tar "^6.1.11"
 
+"@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3":
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz#9edec61b22c3082018a79f6d1c30289ddf3d9d11"
+  integrity sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==
+
+"@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3":
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz#33677a275204898ad8acbf62734fc4dc0b6a4855"
+  integrity sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==
+
+"@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3":
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz#19edf7cdc2e7063ee328403c1d895a86dd28f4bb"
+  integrity sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==
+
+"@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3":
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz#94fb0543ba2e28766c3fc439cabbe0440ae70159"
+  integrity sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==
+
+"@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3":
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz#4a0609ab5fe44d07c9c60a11e4484d3c38bbd6e3"
+  integrity sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==
+
+"@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3":
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz#0aa5502d547b57abfc4ac492de68e2006e417242"
+  integrity sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==
+
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -2220,6 +2255,19 @@ builtin-status-codes@^3.0.0:
   resolved "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz"
   integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=
 
+bullmq@^5.8.7:
+  version "5.8.7"
+  resolved "https://registry.yarnpkg.com/bullmq/-/bullmq-5.8.7.tgz#d5da6215377fe29494d74ad307f195f7408b9e2e"
+  integrity sha512-IdAgB9WvJHRAcZtamRLj6fbjMyuIogEa1cjOTWM1pkVoHUOpO34q6FzNMX1R8VOeUhkvkOkWcxI5ENgFLh+TVA==
+  dependencies:
+    cron-parser "^4.6.0"
+    ioredis "^5.4.1"
+    msgpackr "^1.10.1"
+    node-abort-controller "^3.1.1"
+    semver "^7.5.4"
+    tslib "^2.0.0"
+    uuid "^9.0.0"
+
 bytes@3.1.2:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5"
@@ -2559,6 +2607,11 @@ clone@^2.1.2:
   resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f"
   integrity sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==
 
+cluster-key-slot@^1.1.0:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac"
+  integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==
+
 collect-js-deps@^0.1.1:
   version "0.1.1"
   resolved "https://registry.npmjs.org/collect-js-deps/-/collect-js-deps-0.1.1.tgz"
@@ -2848,6 +2901,13 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7:
     safe-buffer "^5.0.1"
     sha.js "^2.4.8"
 
+cron-parser@^4.6.0:
+  version "4.9.0"
+  resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.9.0.tgz#0340694af3e46a0894978c6f52a6dbb5c0f11ad5"
+  integrity sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==
+  dependencies:
+    luxon "^3.2.1"
+
 cross-spawn@^7.0.2, cross-spawn@^7.0.3:
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
@@ -3071,6 +3131,11 @@ denque@^1.5.0:
   resolved "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz"
   integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==
 
+denque@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1"
+  integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==
+
 depd@2.0.0, depd@~2.0.0:
   version "2.0.0"
   resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz"
@@ -3109,6 +3174,11 @@ detect-libc@^2.0.0:
   resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.1.tgz#e1897aa88fa6ad197862937fbc0441ef352ee0cd"
   integrity sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==
 
+detect-libc@^2.0.1:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.3.tgz#f0cd503b40f9939b894697d19ad50895e30cf700"
+  integrity sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==
+
 detective@^4.0.0:
   version "4.7.1"
   resolved "https://registry.npmjs.org/detective/-/detective-4.7.1.tgz"
@@ -4827,6 +4897,21 @@ interpret@^2.2.0:
   resolved "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz"
   integrity sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==
 
+ioredis@^5.4.1:
+  version "5.4.1"
+  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.4.1.tgz#1c56b70b759f01465913887375ed809134296f40"
+  integrity sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==
+  dependencies:
+    "@ioredis/commands" "^1.1.1"
+    cluster-key-slot "^1.1.0"
+    debug "^4.3.4"
+    denque "^2.1.0"
+    lodash.defaults "^4.2.0"
+    lodash.isarguments "^3.1.0"
+    redis-errors "^1.2.0"
+    redis-parser "^3.0.0"
+    standard-as-callback "^2.1.0"
+
 ip@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.1.tgz#e8f3595d33a3ea66490204234b77636965307105"
@@ -5411,6 +5496,11 @@ lodash.includes@^4.3.0:
   resolved "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz"
   integrity sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==
 
+lodash.isarguments@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a"
+  integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==
+
 lodash.isboolean@^3.0.3:
   version "3.0.3"
   resolved "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz"
@@ -5530,6 +5620,11 @@ lru-cache@^6.0.0:
   dependencies:
     yallist "^4.0.0"
 
+luxon@^3.2.1:
+  version "3.4.4"
+  resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.4.4.tgz#cf20dc27dc532ba41a169c43fdcc0063601577af"
+  integrity sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==
+
 make-dir@^3.0.0, make-dir@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f"
@@ -5948,6 +6043,27 @@ ms@2.1.3, ms@^2.0.0, ms@^2.1.3:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
 
+msgpackr-extract@^3.0.2:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz#e9d87023de39ce714872f9e9504e3c1996d61012"
+  integrity sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==
+  dependencies:
+    node-gyp-build-optional-packages "5.2.2"
+  optionalDependencies:
+    "@msgpackr-extract/msgpackr-extract-darwin-arm64" "3.0.3"
+    "@msgpackr-extract/msgpackr-extract-darwin-x64" "3.0.3"
+    "@msgpackr-extract/msgpackr-extract-linux-arm" "3.0.3"
+    "@msgpackr-extract/msgpackr-extract-linux-arm64" "3.0.3"
+    "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.3"
+    "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.3"
+
+msgpackr@^1.10.1:
+  version "1.10.2"
+  resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.2.tgz#a73de4767f76659e8c69cf9c80fdfce83937a44a"
+  integrity sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==
+  optionalDependencies:
+    msgpackr-extract "^3.0.2"
+
 multiparty@4.2.2:
   version "4.2.2"
   resolved "https://registry.npmjs.org/multiparty/-/multiparty-4.2.2.tgz"
@@ -6019,6 +6135,11 @@ node-abort-controller@3.0.1:
   resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e"
   integrity sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw==
 
+node-abort-controller@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548"
+  integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==
+
 node-addon-api@^3.0.0:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
@@ -6055,6 +6176,13 @@ node-forge@^0.10.0:
   resolved "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz"
   integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==
 
+node-gyp-build-optional-packages@5.2.2:
+  version "5.2.2"
+  resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz#522f50c2d53134d7f3a76cd7255de4ab6c96a3a4"
+  integrity sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==
+  dependencies:
+    detect-libc "^2.0.1"
+
 node-gyp-build@^4.2.2:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.6.0.tgz#0c52e4cbf54bbd28b709820ef7b6a3c2d6209055"
@@ -7522,6 +7650,11 @@ stackback@0.0.2:
   resolved "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz"
   integrity sha1-Gsig2Ug4SNFpXkGLbQMaPDzmjjs=
 
+standard-as-callback@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45"
+  integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==
+
 statuses@2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63"