diff --git a/README.md b/README.md index d2e0a9e4..95fb1803 100644 --- a/README.md +++ b/README.md @@ -332,8 +332,8 @@ ASSISTANT_MODEL | optional. If set, this string is passed along in calls to ASSISTANT_LONGER_CONTEXT_MODEL | optional. If set, requests that fail because of a context length limitation will be retried with this model set. OPENAI_API_KEY | optional. Synonym for ASSISTANT_API_KEY that assumes an OpenAI endpoint is being used. Sign up for an account on OpenAI and then generate a secret key [here](https://platform.openai.com/account/api-keys). -At the time of writing, the AI Assistant is known to function against OpenAI chat completion endpoints for gpt-3.5-turbo and gpt-4. -It can also function against the chat completion endpoint provided by llama-cpp-python. +At the time of writing, the AI Assistant is known to function against OpenAI chat completion endpoints (those ending in `/v1/chat/completions`). +It is also known to function against the chat completion endpoint provided by llama-cpp-python and by [LM Studio](https://lmstudio.ai/). For useful results, the LLM should be on par with GPT 3.5 or above. #### Sandbox related variables: diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts index e529ddb6..4ce03fcb 100644 --- a/app/server/lib/FlexServer.ts +++ b/app/server/lib/FlexServer.ts @@ -42,6 +42,7 @@ import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap'; import {Hosts, RequestWithOrg} from 'app/server/lib/extractOrg'; import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth"; +import {GristBullMQJobs, GristJobs} from 'app/server/lib/GristJobs'; import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer, RequestWithGrist} from 'app/server/lib/GristServer'; import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions'; @@ -186,6 +187,7 @@ export class FlexServer implements GristServer { private _isReady: boolean = false; private _updateManager: UpdateManager; private _sandboxInfo: SandboxInfo; + private _jobs?: GristJobs; constructor(public port: number, public name: string = 'flexServer', public readonly options: FlexServerOptions = {}) { @@ -339,6 +341,14 @@ export class FlexServer implements GristServer { return this.server ? (this.server.address() as AddressInfo).port : this.port; } + /** + * Get interface to job queues. + */ + public getJobs(): GristJobs { + const jobs = this._jobs || new GristBullMQJobs(); + return jobs; + } + /** * Get a url to an org that should be accessible by all signed-in users. For now, this * returns the base URL of the personal org (typically docs[-s]). @@ -943,6 +953,7 @@ export class FlexServer implements GristServer { if (this.server) { this.server.close(); } if (this.httpsServer) { this.httpsServer.close(); } if (this.housekeeper) { await this.housekeeper.stop(); } + if (this._jobs) { await this._jobs.stop(); } await this._shutdown(); if (this._accessTokens) { await this._accessTokens.close(); } // Do this after _shutdown, since DocWorkerMap is used during shutdown. diff --git a/app/server/lib/GristJobs.ts b/app/server/lib/GristJobs.ts new file mode 100644 index 00000000..7ebca916 --- /dev/null +++ b/app/server/lib/GristJobs.ts @@ -0,0 +1,339 @@ +import { makeId } from 'app/server/lib/idUtils'; +import log from 'app/server/lib/log'; +import { Queue, Worker } from 'bullmq'; +import IORedis from 'ioredis'; + +/** + * + * Support for queues. + * + * We use BullMQ for queuing, since it seems currently the best the + * node ecosystem has to offer. BullMQ relies on Redis. Since queuing + * is so handy, but we'd like most of Grist to be usable without Redis, + * we make some effort to support queuing without BullMQ. This + * may not be sustainable, we'll see. + * + * Important: if you put a job in a queue, it can outlast your process. + * That has implications for testing and deployment, so be careful. + * + * Long running jobs may be a challenge. BullMQ cancelation + * relies on non-open source features: + * https://docs.bullmq.io/bullmq-pro/observables/cancelation + * + */ +export interface GristJobs { + /** + * All workers and jobs are scoped to individual named queues, + * with the real interfaces operating at that level. + */ + queue(queueName?: string): GristQueueScope; + + /** + * Shut everything down that we're responsible for. + * Set obliterate flag to destroy jobs even if they are + * stored externally (useful for testing). + */ + stop(options?: { + obliterate?: boolean, + }): Promise; +} + +/** + * For a given queue, we can add jobs, or methods to process jobs, + */ +export interface GristQueueScope { + /** + * Add a job. + */ + add(name: string, data: any, options?: JobAddOptions): Promise; + + + /** + * Add a job handler for all jobs regardless of name. + * Handlers given by handleName take priority, but no + * job handling will happen until handleDefault has been + * called. + */ + handleDefault(defaultCallback: JobHandler): void; + + /** + * Add a job handler for jobs with a specific name. + * Handler will only be effective once handleAll is called + * to specify what happens to jobs not matching expected + * names. + */ + handleName(name: string, + callback: (job: GristJob) => Promise): void; + + /** + * Shut everything down that we're responsible for. + * Set obliterate flag to destroy jobs even if they are + * stored externally (useful for testing). + */ + stop(options?: { + obliterate?: boolean, + }): Promise; +} + +/** + * The type of a function for handling jobs on a queue. + */ +export type JobHandler = (job: GristJob) => Promise; + +/** + * The name used for a queue if no specific name is given. + */ +export const DEFAULT_QUEUE_NAME = 'default'; + +/** + * BullMQ jobs are a string name, and then a data object. + */ +interface GristJob { + name: string; + data: any; +} + +/** + * Options when adding a job. BullMQ has many more. + */ +interface JobAddOptions { + delay?: number; + jobId?: string; + repeat?: { + every: number; + } +} + +/** + * Implementation for job functionality across the application. + * Will use BullMQ, with an in-memory fallback if Redis is + * unavailable. + */ +export class GristBullMQJobs implements GristJobs { + private _connection?: IORedis; + private _checkedForConnection: boolean = false; + private _queues = new Map(); + + /** + * Get BullMQ-compatible options for the queue. + */ + public getQueueOptions() { + // Following BullMQ, queue options contain the connection + // to redis, if any. + if (!this._checkedForConnection) { + this._connect(); + this._checkedForConnection = true; + } + if (!this._connection) { + return {}; + } + return { + connection: this._connection, + maxRetriesPerRequest: null, + }; + } + + /** + * Get an interface scoped to a particular queue by name. + */ + public queue(queueName: string = DEFAULT_QUEUE_NAME): GristQueueScope { + if (!this._queues.get(queueName)) { + this._queues.set( + queueName, + new GristBullMQQueueScope(queueName, this), + ); + } + return this._queues.get(queueName)!; + } + + public async stop(options: { + obliterate?: boolean, + } = {}) { + for (const q of this._queues.values()) { + await q.stop(options); + } + this._queues.clear(); + this._connection?.disconnect(); + } + + /** + * Connect to Redis if available. + */ + private _connect() { + // Connect to Redis for use with BullMQ, if REDIS_URL is set. + const urlTxt = process.env.REDIS_URL || process.env.TEST_REDIS_URL; + if (!urlTxt) { + this._connection = undefined; + log.warn('Using in-memory queues, Redis is unavailable'); + return; + } + const url = new URL(urlTxt); + const conn = new IORedis({ + host: url.hostname, + port: url.port ? parseInt(url.port, 10) : undefined, + db: (url.pathname.charAt(0) === '/') ? + parseInt(url.pathname.substring(1), 10) : undefined, + maxRetriesPerRequest: null, + }); + this._connection = conn; + log.info('Storing queues externally in Redis'); + } +} + +/** + * Work with a particular named queue. + */ +export class GristBullMQQueueScope implements GristQueueScope { + private _queue: Queue|GristWorker|undefined; + private _worker: Worker|GristWorker|undefined; + private _namedProcessors: Record = {}; + + public constructor(public readonly queueName: string, + private _owner: GristBullMQJobs) {} + + public handleDefault(defaultCallback: JobHandler) { + // The default callback passes any recognized named jobs to + // processors added with handleName(), then, if there is no + // specific processor, calls the defaultCallback. + const callback = async (job: GristJob) => { + const processor = this._namedProcessors[job.name] || defaultCallback; + return processor(job); + }; + const options = this._owner.getQueueOptions(); + if (!options.connection) { + // If Redis isn't available, we go our own way, not + // using BullMQ. + const worker = new GristWorker(this.queueName, callback); + this._worker = worker; + return worker; + } + const worker = new Worker(this.queueName, callback, options); + this._worker = worker; + return worker; + } + + public handleName(name: string, + callback: (job: GristJob) => Promise) { + this._namedProcessors[name] = callback; + } + + public async stop(options: { + obliterate?: boolean, + } = {}) { + await this._worker?.close(); + if (options.obliterate) { + await this._queue?.obliterate(); + } + } + + public async add(name: string, data: any, options?: JobAddOptions) { + await this._getQueue().add(name, data, { + ...options, + // These settings are quite arbitrary, and should be + // revised when it matters, or made controllable. + removeOnComplete: { + age: 3600, // keep up to 1 hour + count: 1000, // keep up to 1000 jobs + }, + removeOnFail: { + age: 24 * 3600, // keep up to 24 hours + }, + }); + } + + private _getQueue(): Queue|GristWorker { + if (this._queue) { return this._queue; } + const queue = this._pickQueueImplementation(); + this._queue = queue; + return queue; + } + + private _pickQueueImplementation() { + const name = this.queueName; + const queueOptions = this._owner.getQueueOptions(); + // If we have Redis, get a proper BullMQ interface. + // Otherwise, make do. + if (queueOptions.connection) { + return new Queue(name, queueOptions); + } + // If in memory, we hand a job directly to the single worker for their + // queue. This is very crude. + const worker = this._worker; + if (!worker) { + throw new Error(`no handler yet for ${this.queueName}`); + } + // We only access workers directly when working in-memory, to + // hand jobs directly to them. + if (isBullMQWorker(worker)) { + // Not expected! Somehow we have a BullMQ worker. + throw new Error(`wrong kind of worker for ${this.queueName}`); + } + return worker; + } +} + +/** + * If running in memory without Redis, all jobs need to be + * created and served by the the same process. This class + * pretends to be a BullMQ worker, but accepts jobs directly + * without any intermediate queue. This could be elaborated + * in future if needed. + */ +class GristWorker { + private _jobs: Map = new Map(); + + public constructor(public queueName: string, + private _callback: (job: GristJob) => Promise) { + } + + public async close() { + for (const job of this._jobs.keys()) { + // Key deletion is safe with the keys() iterator. + this._clearJob(job); + } + } + + public async add(name: string, data: any, options?: JobAddOptions) { + if (options?.delay) { + if (options.repeat) { + // Unexpected combination. + throw new Error('cannot delay and repeat'); + } + const jobId = options.jobId || makeId(); + this._clearJob(jobId); + this._jobs.set(jobId, setTimeout(() => this._callback({name, data}), + options.delay)); + return; + } + if (options?.repeat) { + const jobId = options.jobId || makeId(); + this._clearJob(jobId); + this._jobs.set(jobId, setInterval(() => this._callback({name, data}), + options.repeat.every)); + return; + } + await this._callback({name, data}); + } + + public async obliterate() { + await this.close(); + } + + private _clearJob(id: string) { + const job = this._jobs.get(id); + if (!job) { return; } + // We don't know if the job is a once-off or repeating, + // so we call both clearInterval and clearTimeout, which + // apparently works. + clearInterval(job); + clearTimeout(job); + this._jobs.delete(id); + } +} + +/** + * Check if a worker is a real BullMQ worker, or just pretend. + */ +function isBullMQWorker(worker: Worker|GristWorker): worker is Worker { + return 'isNextJob' in worker; +} diff --git a/app/server/lib/GristServer.ts b/app/server/lib/GristServer.ts index 23feb8c0..602e7760 100644 --- a/app/server/lib/GristServer.ts +++ b/app/server/lib/GristServer.ts @@ -15,6 +15,7 @@ import { RequestWithLogin } from 'app/server/lib/Authorizer'; import { Comm } from 'app/server/lib/Comm'; import { create } from 'app/server/lib/create'; import { Hosts } from 'app/server/lib/extractOrg'; +import { GristJobs } from 'app/server/lib/GristJobs'; import { ICreate } from 'app/server/lib/ICreate'; import { IDocStorageManager } from 'app/server/lib/IDocStorageManager'; import { INotifier } from 'app/server/lib/INotifier'; @@ -71,6 +72,7 @@ export interface GristServer { getBootKey(): string|undefined; getSandboxInfo(): SandboxInfo|undefined; getInfo(key: string): any; + getJobs(): GristJobs; } export interface GristLoginSystem { @@ -164,7 +166,8 @@ export function createDummyGristServer(): GristServer { getBundledWidgets() { return []; }, getBootKey() { return undefined; }, getSandboxInfo() { return undefined; }, - getInfo(key: string) { return undefined; } + getInfo(key: string) { return undefined; }, + getJobs(): GristJobs { throw new Error('no job system'); }, }; } diff --git a/package.json b/package.json index 9d552213..9e92bc91 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,7 @@ "@googleapis/oauth2": "0.2.0", "@gristlabs/connect-sqlite3": "0.9.11-grist.5", "@gristlabs/express-session": "1.17.0", - "@gristlabs/grist-widget": "^0.0.4", + "@gristlabs/grist-widget": "^0.0.5", "@gristlabs/moment-guess": "1.2.4-grist.1", "@gristlabs/pidusage": "2.0.17", "@gristlabs/sqlite3": "5.1.4-grist.8", @@ -128,6 +128,7 @@ "bootstrap": "3.4.1", "bootstrap-datepicker": "1.9.0", "bowser": "2.7.0", + "bullmq": "5.8.7", "collect-js-deps": "^0.1.1", "color-convert": "2.0.1", "commander": "9.3.0", diff --git a/static/locales/en.client.json b/static/locales/en.client.json index 889fbf18..c291d404 100644 --- a/static/locales/en.client.json +++ b/static/locales/en.client.json @@ -277,7 +277,11 @@ "Workspace not found": "Workspace not found", "You are on the {{siteName}} site. You also have access to the following sites:": "You are on the {{siteName}} site. You also have access to the following sites:", "You are on your personal site. You also have access to the following sites:": "You are on your personal site. You also have access to the following sites:", - "You may delete a workspace forever once it has no documents in it.": "You may delete a workspace forever once it has no documents in it." + "You may delete a workspace forever once it has no documents in it.": "You may delete a workspace forever once it has no documents in it.", + "Any documents created in this site will appear here.": "Any documents created in this site will appear here.", + "Create my first document": "Create my first document", + "You have read-only access to this site. Currently there are no documents.": "You have read-only access to this site. Currently there are no documents.", + "personal site": "personal site" }, "DocPageModel": { "Add Empty Table": "Add Empty Table", @@ -531,7 +535,8 @@ "Sign in": "Sign in", "To use Grist, please either sign up or sign in.": "To use Grist, please either sign up or sign in.", "Visit our {{link}} to learn more about Grist.": "Visit our {{link}} to learn more about Grist.", - "Learn more in our {{helpCenterLink}}.": "Learn more in our {{helpCenterLink}}." + "Learn more in our {{helpCenterLink}}.": "Learn more in our {{helpCenterLink}}.", + "Only show documents": "Only show documents" }, "HomeLeftPane": { "Access Details": "Access Details", @@ -793,7 +798,8 @@ "API Console": "API Console" }, "TopBar": { - "Manage Team": "Manage Team" + "Manage Team": "Manage Team", + "Manage team": "Manage team" }, "TriggerFormulas": { "Any field": "Any field", @@ -1204,7 +1210,10 @@ "These rules are applied after all column rules have been processed, if applicable.": "These rules are applied after all column rules have been processed, if applicable.", "Example: {{example}}": "Example: {{example}}", "Filter displayed dropdown values with a condition.": "Filter displayed dropdown values with a condition.", - "Community widgets are created and maintained by Grist community members.": "Community widgets are created and maintained by Grist community members." + "Community widgets are created and maintained by Grist community members.": "Community widgets are created and maintained by Grist community members.", + "Creates a reverse column in target table that can be edited from either end.": "Creates a reverse column in target table that can be edited from either end.", + "This limitation occurs when one end of a two-way reference is configured as a single Reference.": "This limitation occurs when one end of a two-way reference is configured as a single Reference.", + "To allow multiple assignments, change the type of the Reference column to Reference List.": "To allow multiple assignments, change the type of the Reference column to Reference List." }, "DescriptionConfig": { "DESCRIPTION": "DESCRIPTION" @@ -1731,5 +1740,46 @@ }, "The toggle is **off**": "The toggle is **off**", "The toggle is **on**": "The toggle is **on**" + }, + "HomeIntroCards": { + "3 minute video tour": "3 minute video tour", + "Blank document": "Blank document", + "Find solutions and explore more resources {{helpCenterLink}}": "Find solutions and explore more resources {{helpCenterLink}}", + "Finish our basics tutorial": "Finish our basics tutorial", + "Help center": "Help center", + "Import file": "Import file", + "Learn more {{webinarsLinks}}": "Learn more {{webinarsLinks}}", + "Start a new document": "Start a new document", + "Templates": "Templates", + "Tutorial": "Tutorial", + "Webinars": "Webinars" + }, + "ReverseReferenceConfig": { + "Add two-way reference": "Add two-way reference", + "Column": "Column", + "Delete": "Delete", + "Delete column {{column}} in table {{table}}?": "Delete column {{column}} in table {{table}}?", + "It is the reverse of the reference column {{column}} in table {{table}}.": "It is the reverse of the reference column {{column}} in table {{table}}.", + "Table": "Table", + "Two-way Reference": "Two-way Reference" + }, + "SupportGristButton": { + "Admin Panel": "Admin Panel", + "Close": "Close", + "Help Center": "Help Center", + "Opt in to Telemetry": "Opt in to Telemetry", + "Opted In": "Opted In", + "Support Grist": "Support Grist", + "Thank you! Your trust and support is greatly appreciated. Opt out any time from the {{link}} in the user menu.": "Thank you! Your trust and support is greatly appreciated. Opt out any time from the {{link}} in the user menu." + }, + "buildReassignModal": { + "Cancel": "Cancel", + "Each {{targetTable}} record may only be assigned to a single {{sourceTable}} record.": "Each {{targetTable}} record may only be assigned to a single {{sourceTable}} record.", + "Reassign": "Reassign", + "Reassign to new {{sourceTable}} records.": "Reassign to new {{sourceTable}} records.", + "Reassign to {{sourceTable}} record {{sourceName}}.": "Reassign to {{sourceTable}} record {{sourceName}}.", + "Record already assigned_one": "Record already assigned", + "Record already assigned_other": "Record already assigned", + "{{targetTable}} record {{targetName}} is already assigned to {{sourceTable}} record {{oldSourceName}}.": "{{targetTable}} record {{targetName}} is already assigned to {{sourceTable}} record {{oldSourceName}}." } } diff --git a/test/server/lib/GristJobs.ts b/test/server/lib/GristJobs.ts new file mode 100644 index 00000000..73c1b569 --- /dev/null +++ b/test/server/lib/GristJobs.ts @@ -0,0 +1,150 @@ +import { delay } from 'app/common/delay'; +import { GristBullMQJobs, GristJobs } from 'app/server/lib/GristJobs'; +import { assert } from 'chai'; + +describe('GristJobs', function() { + this.timeout(20000); + + // Clean up any jobs left over from previous round of tests, + // if external queues are in use (Redis). + beforeEach(async function() { + const jobs = new GristBullMQJobs(); + const q = jobs.queue(); + await q.stop({obliterate: true}); + }); + + it('can run immediate jobs', async function() { + const jobs: GristJobs = new GristBullMQJobs(); + const q = jobs.queue(); + try { + let ct = 0; + let defaultCt = 0; + q.handleName('add', async (job) => { + ct += job.data.delta; + }); + q.handleDefault(async (job) => { + defaultCt++; + }); + await q.add('add', {delta: 2}); + await waitToPass(async () => { + assert.equal(ct, 2); + assert.equal(defaultCt, 0); + }); + await q.add('add', {delta: 3}); + await waitToPass(async () => { + assert.equal(ct, 5); + assert.equal(defaultCt, 0); + }); + await q.add('badd', {delta: 4}); + await waitToPass(async () => { + assert.equal(ct, 5); + assert.equal(defaultCt, 1); + }); + } finally { + await jobs.stop({obliterate: true}); + } + }); + + it('can run delayed jobs', async function() { + const jobs: GristJobs = new GristBullMQJobs(); + const q = jobs.queue(); + try { + let ct = 0; + let defaultCt = 0; + q.handleName('add', async (job) => { + ct += job.data.delta; + }); + q.handleDefault(async () => { + defaultCt++; + }); + await q.add('add', {delta: 2}, {delay: 500}); + assert.equal(ct, 0); + assert.equal(defaultCt, 0); + // We need to wait long enough to see the effect. + await delay(100); + assert.equal(ct, 0); + assert.equal(defaultCt, 0); + await delay(900); + assert.equal(ct, 2); + assert.equal(defaultCt, 0); + } finally { + await jobs.stop({obliterate: true}); + } + }); + + it('can run repeated jobs', async function() { + const jobs: GristJobs = new GristBullMQJobs(); + const q = jobs.queue(); + try { + let ct = 0; + let defaultCt = 0; + q.handleName('add', async (job) => { + ct += job.data.delta; + }); + q.handleDefault(async () => { + defaultCt++; + }); + await q.add('add', {delta: 2}, {repeat: {every: 250}}); + await q.add('badd', {delta: 2}, {repeat: {every: 100}}); + assert.equal(ct, 0); + assert.equal(defaultCt, 0); + await delay(1000); + // allow for a lot of slop on CI + assert.isAtLeast(ct, 8 - 4); + assert.isAtMost(ct, 8 + 4); + assert.isAtLeast(defaultCt, 10 - 3); + assert.isAtMost(defaultCt, 10 + 3); + } finally { + await jobs.stop({obliterate: true}); + } + }); + + it('can pick up jobs again', async function() { + // this test is only appropriate if we have an external queue. + if (!process.env.REDIS_URL) { this.skip(); } + const jobs1: GristJobs = new GristBullMQJobs(); + const q = jobs1.queue(); + try { + let ct = 0; + q.handleName('add', async (job) => { + ct += job.data.delta; + }); + q.handleDefault(async () => {}); + await q.add('add', {delta: 1}, {delay: 250}); + await q.add('add', {delta: 1}, {delay: 1000}); + await delay(500); + assert.equal(ct, 1); + await jobs1.stop(); + const jobs2: GristJobs = new GristBullMQJobs(); + const q2 = jobs2.queue(); + try { + q2.handleName('add', async (job) => { + ct += job.data.delta * 2; + }); + q2.handleDefault(async () => {}); + await delay(1000); + assert.equal(ct, 3); + } finally { + await jobs2.stop({obliterate: true}); + } + } finally { + await jobs1.stop({obliterate: true}); + } + }); +}); + +async function waitToPass(fn: () => Promise, + maxWaitMs: number = 2000) { + const start = Date.now(); + while (Date.now() - start < maxWaitMs) { + try { + await fn(); + return true; + } catch (e) { + // continue after a small delay. + await delay(10); + } + } + await fn(); + return true; +} diff --git a/yarn.lock b/yarn.lock index 69e799b0..3465da77 100644 --- a/yarn.lock +++ b/yarn.lock @@ -383,10 +383,10 @@ safe-buffer "5.2.0" uid-safe "~2.1.5" -"@gristlabs/grist-widget@^0.0.4": - version "0.0.4" - resolved "https://registry.yarnpkg.com/@gristlabs/grist-widget/-/grist-widget-0.0.4.tgz#df50d988bcdf8fc26a876cf23b82e258bbdb0ccc" - integrity sha512-Q0k+GuudU2+0JkuvVkB9UZzqeUKJH8PsaO9ZfxKuqL9/ssIXUd080msB+PJLXB0TU9BkpzPSl7+kLqXTBSnA5g== +"@gristlabs/grist-widget@^0.0.5": + version "0.0.5" + resolved "https://registry.yarnpkg.com/@gristlabs/grist-widget/-/grist-widget-0.0.5.tgz#b56b91ad0ee12020fa83993bb11e70d41c11a77b" + integrity sha512-0JhCFLjcbNqKsDyipQSWJhj5VglzixHXuqfAvHB858vmbRyG9s5G2x1lCZj2MGD+PCrIXA/qwNmJx/Ei7b6e8Q== "@gristlabs/moment-guess@1.2.4-grist.1": version "1.2.4-grist.1" @@ -435,6 +435,11 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== +"@ioredis/commands@^1.1.1": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" + integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg== + "@jridgewell/gen-mapping@^0.1.0": version "0.1.1" resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz#e5d2e450306a9491e3bd77e323e38d7aff315996" @@ -535,6 +540,36 @@ semver "^7.3.5" tar "^6.1.11" +"@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz#9edec61b22c3082018a79f6d1c30289ddf3d9d11" + integrity sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw== + +"@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz#33677a275204898ad8acbf62734fc4dc0b6a4855" + integrity sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw== + +"@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz#19edf7cdc2e7063ee328403c1d895a86dd28f4bb" + integrity sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg== + +"@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz#94fb0543ba2e28766c3fc439cabbe0440ae70159" + integrity sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw== + +"@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz#4a0609ab5fe44d07c9c60a11e4484d3c38bbd6e3" + integrity sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg== + +"@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz#0aa5502d547b57abfc4ac492de68e2006e417242" + integrity sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" @@ -2220,6 +2255,19 @@ builtin-status-codes@^3.0.0: resolved "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz" integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug= +bullmq@^5.8.7: + version "5.8.7" + resolved "https://registry.yarnpkg.com/bullmq/-/bullmq-5.8.7.tgz#d5da6215377fe29494d74ad307f195f7408b9e2e" + integrity sha512-IdAgB9WvJHRAcZtamRLj6fbjMyuIogEa1cjOTWM1pkVoHUOpO34q6FzNMX1R8VOeUhkvkOkWcxI5ENgFLh+TVA== + dependencies: + cron-parser "^4.6.0" + ioredis "^5.4.1" + msgpackr "^1.10.1" + node-abort-controller "^3.1.1" + semver "^7.5.4" + tslib "^2.0.0" + uuid "^9.0.0" + bytes@3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" @@ -2559,6 +2607,11 @@ clone@^2.1.2: resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" integrity sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w== +cluster-key-slot@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + collect-js-deps@^0.1.1: version "0.1.1" resolved "https://registry.npmjs.org/collect-js-deps/-/collect-js-deps-0.1.1.tgz" @@ -2848,6 +2901,13 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7: safe-buffer "^5.0.1" sha.js "^2.4.8" +cron-parser@^4.6.0: + version "4.9.0" + resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.9.0.tgz#0340694af3e46a0894978c6f52a6dbb5c0f11ad5" + integrity sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q== + dependencies: + luxon "^3.2.1" + cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" @@ -3071,6 +3131,11 @@ denque@^1.5.0: resolved "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz" integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@2.0.0, depd@~2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz" @@ -3109,6 +3174,11 @@ detect-libc@^2.0.0: resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.1.tgz#e1897aa88fa6ad197862937fbc0441ef352ee0cd" integrity sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w== +detect-libc@^2.0.1: + version "2.0.3" + resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.3.tgz#f0cd503b40f9939b894697d19ad50895e30cf700" + integrity sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw== + detective@^4.0.0: version "4.7.1" resolved "https://registry.npmjs.org/detective/-/detective-4.7.1.tgz" @@ -4827,6 +4897,21 @@ interpret@^2.2.0: resolved "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz" integrity sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw== +ioredis@^5.4.1: + version "5.4.1" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.4.1.tgz#1c56b70b759f01465913887375ed809134296f40" + integrity sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ip@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.1.tgz#e8f3595d33a3ea66490204234b77636965307105" @@ -5411,6 +5496,11 @@ lodash.includes@^4.3.0: resolved "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz" integrity sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w== +lodash.isarguments@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" + integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== + lodash.isboolean@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz" @@ -5530,6 +5620,11 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" +luxon@^3.2.1: + version "3.4.4" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.4.4.tgz#cf20dc27dc532ba41a169c43fdcc0063601577af" + integrity sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA== + make-dir@^3.0.0, make-dir@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" @@ -5948,6 +6043,27 @@ ms@2.1.3, ms@^2.0.0, ms@^2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== +msgpackr-extract@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz#e9d87023de39ce714872f9e9504e3c1996d61012" + integrity sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA== + dependencies: + node-gyp-build-optional-packages "5.2.2" + optionalDependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-darwin-x64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-arm" "3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-arm64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.3" + "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.3" + +msgpackr@^1.10.1: + version "1.10.2" + resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.2.tgz#a73de4767f76659e8c69cf9c80fdfce83937a44a" + integrity sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA== + optionalDependencies: + msgpackr-extract "^3.0.2" + multiparty@4.2.2: version "4.2.2" resolved "https://registry.npmjs.org/multiparty/-/multiparty-4.2.2.tgz" @@ -6019,6 +6135,11 @@ node-abort-controller@3.0.1: resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e" integrity sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw== +node-abort-controller@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" + integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== + node-addon-api@^3.0.0: version "3.2.1" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161" @@ -6055,6 +6176,13 @@ node-forge@^0.10.0: resolved "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz" integrity sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA== +node-gyp-build-optional-packages@5.2.2: + version "5.2.2" + resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz#522f50c2d53134d7f3a76cd7255de4ab6c96a3a4" + integrity sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw== + dependencies: + detect-libc "^2.0.1" + node-gyp-build@^4.2.2: version "4.6.0" resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.6.0.tgz#0c52e4cbf54bbd28b709820ef7b6a3c2d6209055" @@ -7522,6 +7650,11 @@ stackback@0.0.2: resolved "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz" integrity sha1-Gsig2Ug4SNFpXkGLbQMaPDzmjjs= +standard-as-callback@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45" + integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== + statuses@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63"