(core) add a yarn run cli tool, and add a sqlite gristify option

Summary:
This adds rudimentary support for opening certain SQLite files in Grist.

If you have a file such as `landing.db` in Grist, you can convert it to Grist format by doing (either in monorepo or grist-core):
```
yarn run cli -h
yarn run cli sqlite -h
yarn run cli sqlite gristify landing.db
```

The file is now openable by Grist. To actually do so with the regular Grist server, you'll need to either import it, or convert some doc you don't care about in the `samples/` directory to be a soft link to it (and then force a reload).

This implementation is a rudimentary experiment. Here are some awkwardnesses:
 * Only tables that happen to have a column called `id`, and where the column happens to be an integer, can be opened directly with Grist as it is today. That could be generalized, but it looked more than a Gristathon's worth of work, so I instead used SQLite views.
 * Grist will handle tables that start with an uncapitalized letter a bit erratically. You can successfully add columns, for example, but removing them will cause sadness - Grist will rename the table in a confused way.
 * I didn't attempt to deal with column names with spaces etc (though views could deal with those).
 * I haven't tried to do any fancy type mapping.
 * Columns with constraints can make adding new rows impossible in Grist, since Grist requires that a row can be added with just a single cell set.

Test Plan: added small test

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3502

app/gen-server/migration/1536634251710-Initial.ts

@@ -1,3 +1,4 @@
+import * as sqlUtils from "app/gen-server/sqlUtils";
 import {MigrationInterface, QueryRunner, Table} from "typeorm";
 
 
@@ -6,9 +7,9 @@ export class Initial1536634251710 implements MigrationInterface {
     // TypeORM doesn't currently help with types of created tables:
     //   https://github.com/typeorm/typeorm/issues/305
     // so we need to do a little smoothing over postgres and sqlite.
-    const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
+    const dbType = queryRunner.connection.driver.options.type;
-    const datetime = sqlite ? "datetime" : "timestamp with time zone";
+    const datetime = sqlUtils.datetime(dbType);
-    const now = "now()";
+    const now = sqlUtils.now(dbType);
 
     await queryRunner.createTable(new Table({
       name: "users",

app/gen-server/migration/1652273656610-Activations.ts

@@ -1,12 +1,13 @@
+import * as sqlUtils from "app/gen-server/sqlUtils";
 import {MigrationInterface, QueryRunner, Table} from 'typeorm';
 
 export class Activations1652273656610 implements MigrationInterface {
 
   public async up(queryRunner: QueryRunner): Promise<any> {
     // created_at and updated_at code is based on *-Initial.ts
-    const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
+    const dbType = queryRunner.connection.driver.options.type;
-    const datetime = sqlite ? "datetime" : "timestamp with time zone";
+    const datetime = sqlUtils.datetime(dbType);
-    const now = "now()";
+    const now = sqlUtils.now(dbType);
     await queryRunner.createTable(new Table({
       name: 'activations',
       columns: [

app/server/companion.ts

@@ -0,0 +1,239 @@
+import { synchronizeProducts } from 'app/gen-server/entity/Product';
+import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
+import { applyPatch } from 'app/gen-server/lib/TypeORMPatches';
+import { getMigrations, getOrCreateConnection, undoLastMigration, updateDb } from 'app/server/lib/dbUtils';
+import { getDatabaseUrl } from 'app/server/lib/serverUtils';
+import { Gristifier } from 'app/server/utils/gristify';
+import { pruneActionHistory } from 'app/server/utils/pruneActionHistory';
+import * as commander from 'commander';
+import { Connection, getConnectionOptions } from 'typeorm';
+
+/**
+ * Main entrypoint for a cli toolbox for configuring aspects of Grist
+ * and Grist documents.
+ */
+async function main() {
+  // Tweak TypeORM support of SQLite a little bit to support transactions.
+  applyPatch();
+  const program = getProgram();
+  await program.parseAsync(process.argv);
+}
+
+if (require.main === module) {
+  main().then(() => process.exit(0)).catch(e => {
+    // tslint:disable-next-line:no-console
+    console.error(e);
+    process.exit(1);
+  });
+}
+
+/**
+ * Get the Grist companion client program as a commander object.
+ * To actually run it, call parseAsync(argv), optionally after
+ * adding any other commands that may be available.
+ */
+export function getProgram(): commander.Command {
+  const program = commander.program;
+  program
+    .name('grist-toolbox')    // haven't really settled on a name yet.
+                              // want to reserve "grist" for electron app?
+    .description('a toolbox of handy Grist-related utilities');
+
+  addDbCommand(program, {nested: true});
+  addHistoryCommand(program, {nested: true});
+  addSiteCommand(program, {nested: true});
+  addSqliteCommand(program);
+  return program;
+}
+
+// Add commands related to document history:
+//   history prune <docId> [N]
+export function addHistoryCommand(program: commander.Command, options: CommandOptions) {
+  const sub = section(program, {
+    sectionName: 'history',
+    sectionDescription: 'fiddle with history of a Grist document',
+    ...options,
+  });
+  sub('prune <docId>')
+    .description('remove all but last N actions from doc')
+    .argument('[N]', 'number of actions to keep', parseIntForCommander, 1)
+    .action(pruneActionHistory);
+}
+
+// Add commands related to sites:
+//   site create <domain> <owner-email>
+export function addSiteCommand(program: commander.Command,
+                               options: CommandOptions) {
+  const sub = section(program, {
+    sectionName: 'site',
+    sectionDescription: 'set up sites',
+    ...options
+  });
+  sub('create <domain> <owner-email>')
+    .description('create a site')
+    .action(async (domain, email) => {
+      console.log("create a site");
+      const profile = {email, name: email};
+      const db = await getHomeDBManager();
+      const user = await db.getUserByLogin(email, {profile});
+      if (!user) {
+        // This should not happen.
+        throw new Error('failed to create user');
+      }
+      await db.addOrg(user, {
+        name: domain,
+        domain,
+      }, {
+        setUserAsOwner: false,
+        useNewPlan: true,
+        planType: 'teamFree'
+      });
+    });
+}
+
+// Add commands related to home/landing database:
+//   db migrate
+//   db revert
+//   db check
+//   db url
+export function addDbCommand(program: commander.Command,
+                             options: CommandOptions,
+                             reuseConnection?: Connection) {
+  function withConnection(op: (connection: Connection) => Promise<number>) {
+    return async () => {
+      if (!process.env.TYPEORM_LOGGING) {
+        process.env.TYPEORM_LOGGING = 'true';
+      }
+      const connection = reuseConnection || await getOrCreateConnection();
+      const exitCode = await op(connection);
+      if (exitCode !== 0) {
+        program.error('db command failed', {exitCode});
+      }
+    };
+  }
+  const sub = section(program, {
+    sectionName: 'db',
+    sectionDescription: 'maintain the database of users, sites, workspaces, and docs',
+    ...options,
+  });
+
+  sub('migrate')
+    .description('run all pending migrations on database')
+    .action(withConnection(async (connection) => {
+      await updateDb(connection);
+      return 0;
+    }));
+
+  sub('revert')
+    .description('revert last migration on database')
+    .action(withConnection(async (connection) => {
+      await undoLastMigration(connection);
+      return 0;
+    }));
+
+  sub('check')
+    .description('check that there are no pending migrations on database')
+    .action(withConnection(dbCheck));
+
+  sub('url')
+    .description('construct a url for the database (for psql, catsql etc)')
+    .action(withConnection(async () => {
+      console.log(getDatabaseUrl(await getConnectionOptions(), true));
+      return 0;
+    }));
+}
+
+// Add command related to sqlite:
+//   sqlite gristify <sqlite-file>
+//   sqlite clean <sqlite-file>
+export function addSqliteCommand(program: commander.Command) {
+  const sub = program.command('sqlite')
+    .description('commands for accessing sqlite files');
+
+  sub.command('gristify <sqlite-file>')
+    .description('add grist metadata to an sqlite file')
+    .option('--add-sort', 'add a manualSort column, important for adding/removing rows')
+    .action((filename, options) => new Gristifier(filename).gristify(options));
+
+  sub.command('clean <sqlite-file>')
+    .description('remove grist metadata from an sqlite file')
+    .action(filename => new Gristifier(filename).degristify());
+}
+
+// Report the status of the database. Migrations appied, migrations pending,
+// product information applied, product changes pending.
+export async function dbCheck(connection: Connection) {
+  const migrations = await getMigrations(connection);
+  const changingProducts = await synchronizeProducts(connection, false);
+  // eslint-disable-next-line @typescript-eslint/no-shadow
+  const log = process.env.TYPEORM_LOGGING === 'true' ? console.log : (...args: any[]) => null;
+  const options = await getConnectionOptions();
+  log("database url:", getDatabaseUrl(options, false));
+  log("migration files:", options.migrations);
+  log("migration directory:", (options.cli && options.cli.migrationsDir) || 'unspecified');
+  log("migrations applied to db:", migrations.migrationsInDb);
+  log("migrations listed in code:", migrations.migrationsInCode);
+  let exitCode: number = 0;
+  if (migrations.pendingMigrations.length) {
+    log(`Migration(s) need to be applied: ${migrations.pendingMigrations}`);
+    exitCode = 1;
+  } else {
+    log("No migrations need to be applied");
+  }
+  log("");
+  if (changingProducts.length) {
+    log("Products need updating:", changingProducts);
+    log(`   (to revert a product change, run an older version of the code)`);
+    log(`   (db:revert will not undo product changes)`);
+    exitCode = 1;
+  } else {
+    log(`Products unchanged`);
+  }
+  return exitCode;
+}
+
+// Get an interface to the home db.
+export async function getHomeDBManager() {
+  const dbManager = new HomeDBManager();
+  await dbManager.connect();
+  await dbManager.initializeSpecialIds();
+  return dbManager;
+}
+
+// Get a function for adding a command to a section of related commands.
+// There is a "nested" option that uses commander's nested command feature.
+// Older cli code may use an older unnested style.
+function section(program: commander.Command, options: {
+  sectionName: string,
+  sectionDescription: string,
+  nested: boolean
+}) {
+  // If unnested, we'll return a function that adds commands directly to the
+  // program (section description is ignored in this case). If nested, we make
+  // a command to represent the section, and return a function that adds to that.
+  const sub = options.nested ?
+    program.command(options.sectionName).description(options.sectionDescription) :
+    program;
+  return (name: string) => {
+    if (options.nested) {
+      return sub.command(name);
+    } else {
+      return sub.command(`${options.sectionName}:${name}`);
+    }
+  };
+}
+
+// Options for command style.
+export interface CommandOptions {
+  nested: boolean,
+  sectionName?: string,
+}
+
+// This is based on the recommended way to parse integers for commander.
+export function parseIntForCommander(value: string, prev: number) {
+  const pvalue = parseInt(value, 10);
+  if (isNaN(pvalue)) {
+    throw new Error('Not a number.');
+  }
+  return pvalue;
+}

app/server/lib/ActiveDoc.ts

@@ -33,6 +33,7 @@ import {
   BulkUpdateRecord,
   CellValue,
   DocAction,
+  getTableId,
   TableDataAction,
   TableRecordValue,
   toTableDataAction,
@@ -170,7 +171,7 @@ export class ActiveDoc extends EventEmitter {
   }
 
   public readonly docStorage: DocStorage;
-  public readonly docPluginManager: DocPluginManager;
+  public readonly docPluginManager: DocPluginManager|null;
   public readonly docClients: DocClients;               // Only exposed for Sharing.ts
   public docData: DocData|null = null;
 
@@ -204,6 +205,7 @@ export class ActiveDoc extends EventEmitter {
   private _product?: Product;
   private _gracePeriodStart: Date|null = null;
   private _isForkOrSnapshot: boolean = false;
+  private _onlyAllowMetaDataActionsOnDb: boolean = false;
 
   // Client watching for 'product changed' event published by Billing to update usage
   private _redisSubscriber?: RedisClient;
@@ -276,8 +278,9 @@ export class ActiveDoc extends EventEmitter {
     this._triggers = new DocTriggers(this);
     this._requests = new DocRequests(this);
     this._actionHistory = new ActionHistoryImpl(this.docStorage);
-    this.docPluginManager = new DocPluginManager(docManager.pluginManager.getPlugins(),
+    this.docPluginManager = docManager.pluginManager ?
-      docManager.pluginManager.appRoot!, this, this._docManager.gristServer);
+      new DocPluginManager(docManager.pluginManager.getPlugins(),
+                           docManager.pluginManager.appRoot!, this, this._docManager.gristServer) : null;
     this._tableMetadataLoader = new TableMetadataLoader({
       decodeBuffer: this.docStorage.decodeMarshalledData.bind(this.docStorage),
       fetchTable: this.docStorage.fetchTable.bind(this.docStorage),
@@ -529,7 +532,7 @@ export class ActiveDoc extends EventEmitter {
         }
         await Promise.all([
           this.docStorage.shutdown(),
-          this.docPluginManager.shutdown(),
+          this.docPluginManager?.shutdown(),
           dataEngine?.shutdown()
         ]);
         // The this.waitForInitialization promise may not yet have resolved, but
@@ -576,7 +579,7 @@ export class ActiveDoc extends EventEmitter {
     await this._initDoc(docSession);
     await this._tableMetadataLoader.clean();
     // Makes sure docPluginManager is ready in case new doc is used to import new data
-    await this.docPluginManager.ready;
+    await this.docPluginManager?.ready;
     this._fullyLoaded = true;
     return this;
   }
@@ -585,10 +588,13 @@ export class ActiveDoc extends EventEmitter {
    * Create a new blank document (no "Table1"), used as a stub when importing.
    */
   @ActiveDoc.keepDocOpen
-  public async createEmptyDoc(docSession: OptDocSession): Promise<ActiveDoc> {
+  public async createEmptyDoc(docSession: OptDocSession,
-    await this.loadDoc(docSession, {forceNew: true, skipInitialTable: true});
+                              options?: { useExisting?: boolean }): Promise<ActiveDoc> {
+    await this.loadDoc(docSession, {forceNew: true,
+                                    skipInitialTable: true,
+                                    ...options});
     // Makes sure docPluginManager is ready in case new doc is used to import new data
-    await this.docPluginManager.ready;
+    await this.docPluginManager?.ready;
     this._fullyLoaded = true;
     return this;
   }
@@ -603,13 +609,18 @@ export class ActiveDoc extends EventEmitter {
   public async loadDoc(docSession: OptDocSession, options?: {
     forceNew?: boolean,          // If set, document will be created.
     skipInitialTable?: boolean,  // If set, and document is new, "Table1" will not be added.
+    useExisting?: boolean,       // If set, document can be created as an overlay on
+                                 // an existing sqlite file.
   }): Promise<ActiveDoc> {
     const startTime = Date.now();
     this._log.debug(docSession, "loadDoc");
     try {
       const isNew: boolean = options?.forceNew || await this._docManager.storageManager.prepareLocalDoc(this.docName);
       if (isNew) {
-        await this._createDocFile(docSession, {skipInitialTable: options?.skipInitialTable});
+        await this._createDocFile(docSession, {
+          skipInitialTable: options?.skipInitialTable,
+          useExisting: options?.useExisting,
+        });
       } else {
         await this.docStorage.openFile({
           beforeMigration: async (currentVersion, newVersion) => {
@@ -1253,6 +1264,7 @@ export class ActiveDoc extends EventEmitter {
     if (await this._granularAccess.hasNuancedAccess(docSession)) {
       throw new Error('cannot confirm access to plugin');
     }
+    if (!this.docPluginManager) { throw new Error('no plugin manager available'); }
     const pluginRpc = this.docPluginManager.plugins[pluginId].rpc;
     switch (msg.mtype) {
       case MsgType.RpcCall: return pluginRpc.forwardCall(msg);
@@ -1266,6 +1278,7 @@ export class ActiveDoc extends EventEmitter {
    */
   public async reloadPlugins(docSession: DocSession) {
     // refresh the list plugins found on the system
+    if (!this._docManager.pluginManager || !this.docPluginManager) { return; }
     await this._docManager.pluginManager.reloadPlugins();
     const plugins = this._docManager.pluginManager.getPlugins();
     // reload found plugins
@@ -1433,6 +1446,7 @@ export class ActiveDoc extends EventEmitter {
   }
 
   public getGristDocAPI(): GristDocAPI {
+    if (!this.docPluginManager) { throw new Error('no plugin manager available'); }
     return this.docPluginManager.gristDocAPI;
   }
 
@@ -1573,6 +1587,27 @@ export class ActiveDoc extends EventEmitter {
     return this._docManager.makeAccessId(userId);
   }
 
+  /**
+   * Apply actions that have already occurred in the data engine to the
+   * database also.
+   */
+  public async applyStoredActionsToDocStorage(docActions: DocAction[]): Promise<void> {
+    // When "gristifying" an sqlite database, we may take create tables and
+    // columns in the data engine that already exist in the sqlite database.
+    // During that process, _onlyAllowMetaDataActionsOnDb will be turned on,
+    // and we silently swallow any non-metadata actions.
+    if (this._onlyAllowMetaDataActionsOnDb) {
+      docActions = docActions.filter(a => getTableId(a).startsWith('_grist'));
+    }
+    await this.docStorage.applyStoredActions(docActions);
+  }
+
+  // Set a flag that controls whether user data can be changed in the database,
+  // or only grist-managed tables (those whose names start with _grist)
+  public onlyAllowMetaDataActionsOnDb(flag: boolean) {
+    this._onlyAllowMetaDataActionsOnDb = flag;
+  }
+
   /**
    * Called by Sharing manager when working on modifying the document.
    * Called when DocActions have been produced from UserActions, but
@@ -1720,10 +1755,12 @@ export class ActiveDoc extends EventEmitter {
   @ActiveDoc.keepDocOpen
   private async _createDocFile(docSession: OptDocSession, options?: {
     skipInitialTable?: boolean,  // If set, "Table1" will not be added.
+    useExisting?: boolean,       // If set, an existing sqlite db is permitted.
+                                 // Useful for "gristifying" an existing db.
   }): Promise<void> {
     this._log.debug(docSession, "createDoc");
     await this._docManager.storageManager.prepareToCreateDoc(this.docName);
-    await this.docStorage.createFile();
+    await this.docStorage.createFile(options);
     const sql = options?.skipInitialTable ? GRIST_DOC_SQL : GRIST_DOC_WITH_TABLE1_SQL;
     await this.docStorage.exec(sql);
     const timezone = docSession.browserSettings?.timezone ?? DEFAULT_TIMEZONE;

app/server/lib/ActiveDocImport.ts

@@ -242,6 +242,7 @@ export class ActiveDocImport {
 
     // The upload must be within the plugin-accessible directory. Once moved, subsequent calls to
     // moveUpload() will return without having to do anything.
+    if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); }
     await moveUpload(upload, this._activeDoc.docPluginManager.tmpDir());
 
     const importResult: ImportResult = {options: parseOptions, tables: []};
@@ -287,6 +288,7 @@ export class ActiveDocImport {
     const {originalFilename, parseOptions, mergeOptionsMap, isHidden, uploadFileIndex,
            transformRuleMap} = importOptions;
     log.info("ActiveDoc._importFileAsNewTable(%s, %s)", tmpPath, originalFilename);
+    if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); }
     const optionsAndData: ParseFileResult =
       await this._activeDoc.docPluginManager.parseFile(tmpPath, originalFilename, parseOptions);
     const options = optionsAndData.parseOptions;

app/server/lib/DocClients.ts

@@ -95,7 +95,7 @@ export class DocClients {
     }
     if (type === "docUserAction" && messageData.docActions) {
       for (const action of messageData.docActions) {
-        this.activeDoc.docPluginManager.receiveAction(action);
+        this.activeDoc.docPluginManager?.receiveAction(action);
       }
     }
   }

app/server/lib/DocManager.ts

@@ -54,7 +54,7 @@ export class DocManager extends EventEmitter {
 
   constructor(
     public readonly storageManager: IDocStorageManager,
-    public readonly pluginManager: PluginManager,
+    public readonly pluginManager: PluginManager|null,
     private _homeDbManager: HomeDBManager|null,
     public gristServer: GristServer
   ) {
@@ -610,7 +610,7 @@ export class DocManager extends EventEmitter {
       return docUtils.createExclusive(this.storageManager.getPath(name));
     });
     log.debug('DocManager._createNewDoc picked name', docName);
-    await this.pluginManager.pluginsLoaded;
+    await this.pluginManager?.pluginsLoaded;
     return docName;
   }
 }

app/server/lib/DocStorage.ts

@@ -30,6 +30,7 @@ import uuidv4 from "uuid/v4";
 import {OnDemandStorage} from './OnDemandActions';
 import {ISQLiteDB, MigrationHooks, OpenMode, quoteIdent, ResultRow, SchemaInfo, SQLiteDB} from './SQLiteDB';
 import chunk = require('lodash/chunk');
+import cloneDeep = require('lodash/cloneDeep');
 import groupBy = require('lodash/groupBy');
 
 
@@ -666,10 +667,17 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
    * After a database is created it should be initialized by applying the InitNewDoc action
    * or by executing the initialDocSql.
    */
-  public createFile(): Promise<void> {
+  public createFile(options?: {
+    useExisting?: boolean,  // If set, it is ok if an sqlite file already exists
+                            // where we would store the Grist document. Its content
+                            // will not be touched. Useful when "gristifying" an
+                            // existing SQLite DB.
+  }): Promise<void> {
     // It turns out to be important to return a bluebird promise, a lot of code outside
     // of DocStorage ultimately depends on this.
-    return bluebird.Promise.resolve(this._openFile(OpenMode.CREATE_EXCL, {}))
+    return bluebird.Promise.resolve(this._openFile(
+      options?.useExisting ? OpenMode.OPEN_EXISTING : OpenMode.CREATE_EXCL,
+      {}))
       .then(() => this._initDB());
     // Note that we don't call _updateMetadata() as there are no metadata tables yet anyway.
   }
@@ -927,26 +935,50 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
   public async applyStoredActions(docActions: DocAction[]): Promise<void> {
     debuglog('DocStorage.applyStoredActions');
 
-    await bluebird.Promise.each(docActions, (action: DocAction) => {
+    docActions = this._compressStoredActions(docActions);
-      const actionType = action[0];
+    for (const action of docActions) {
-      const f = (this as any)["_process_" + actionType];
+      try {
-      if (!_.isFunction(f)) {
+        await this.applyStoredAction(action);
-        log.error("Unknown action: " + actionType);
+      } catch (e) {
-      } else {
+        // If the table doesn't have a manualSort column, we'll try
-        return f.apply(this, action.slice(1))
+        // again without setting manualSort. This should never happen
-          .then(() => {
+        // for regular Grist documents, but could happen for a
-            const tableId = action[1]; // The first argument is always tableId;
+        // "gristified" Sqlite database where we are choosing to
-            if (DocStorage._isMetadataTable(tableId) && actionType !== 'AddTable') {
+        // leave the user tables untouched. The manualSort column doesn't
-              // We only need to update the metadata for actions that change
+        // make much sense outside the context of spreadsheets.
-              // the metadata. We don't update on AddTable actions
+        // TODO: it could be useful to make Grist more inherently aware of
-              // because the additional of a table gives no additional data
+        // and tolerant of tables without manual sorting.
-              // and if we tried to update when only _grist_Tables was added
+        if (String(e).match(/no column named manualSort/)) {
-              // without _grist_Tables_column, we would get an error
+          const modifiedAction = this._considerWithoutManualSort(action);
-              return this._updateMetadata();
+          if (modifiedAction) {
-            }
+            await this.applyStoredAction(modifiedAction);
-          });
+            return;
+          }
+        }
+        throw e;
       }
-    });
+    }
+  }
+
+  // Apply a single stored action, dispatching to an appropriate
+  // _process_<ActionType> handler.
+  public async applyStoredAction(action: DocAction): Promise<void> {
+    const actionType = action[0];
+    const f = (this as any)["_process_" + actionType];
+    if (!_.isFunction(f)) {
+      log.error("Unknown action: " + actionType);
+    } else {
+      await f.apply(this, action.slice(1));
+      const tableId = action[1]; // The first argument is always tableId;
+      if (DocStorage._isMetadataTable(tableId) && actionType !== 'AddTable') {
+        // We only need to update the metadata for actions that change
+        // the metadata. We don't update on AddTable actions
+        // because the additional of a table gives no additional data
+        // and if we tried to update when only _grist_Tables was added
+        // without _grist_Tables_column, we would get an error
+        await this._updateMetadata();
+      }
+    }
   }
 
   /**
@@ -1169,6 +1201,8 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
     // Note that SQLite does not support easily dropping columns. To drop a column from a table, we
     // need to follow the instructions at https://sqlite.org/lang_altertable.html Since we don't use
     // indexes or triggers, we skip a few steps.
+    // TODO: SQLite has since added support for ALTER TABLE DROP COLUMN, should
+    // use that to be more efficient and less disruptive.
 
     // This returns rows with (at least) {name, type, dflt_value}.
     return this.all(`PRAGMA table_info(${quote(tableId)})`)
@@ -1713,6 +1747,42 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
       `${joinClauses} ${whereClause} ${limitClause}`;
     return sql;
   }
+
+  // If we are being asked to add a record and then update several of its
+  // columns, compact that into a single action. For fully Grist-managed
+  // documents, this makes no difference. But if the underlying SQLite DB
+  // has extra constraints on columns, it can make a difference.
+  // TODO: consider dealing with other scenarios, especially a BulkAddRecord.
+  private _compressStoredActions(docActions: DocAction[]): DocAction[] {
+    if (docActions.length > 1) {
+      const first = docActions[0];
+      if (first[0] === 'AddRecord' &&
+        docActions.slice(1).every(
+          // Check other actions are UpdateRecords for the same table and row.
+          a => a[0] === 'UpdateRecord' && a[1] === first[1] && a[2] === first[2]
+        )) {
+        const merged = cloneDeep(first);
+        for (const a2 of docActions.slice(1)) {
+          Object.assign(merged[3], a2[3]);
+        }
+        docActions = [merged];
+      }
+    }
+    return docActions;
+  }
+
+  // If an action can have manualSort removed, go ahead and do it (after cloning),
+  // otherwise return null.
+  private _considerWithoutManualSort(act: DocAction): DocAction|null {
+    if (act[0] === 'AddRecord' || act[0] === 'UpdateRecord' ||
+      act[0] === 'BulkAddRecord' || act[0] === 'BulkUpdateRecord' &&
+      'manualSort' in act[3]) {
+      act = cloneDeep(act);
+      delete act[3].manualSort;
+      return act;
+    }
+    return null;
+  }
 }
 
 interface RebuildResult {

app/server/lib/GristServer.ts

@@ -6,6 +6,7 @@ import { Workspace } from 'app/gen-server/entity/Workspace';
 import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
 import { RequestWithLogin } from 'app/server/lib/Authorizer';
 import { Comm } from 'app/server/lib/Comm';
+import { create } from 'app/server/lib/create';
 import { Hosts } from 'app/server/lib/extractOrg';
 import { ICreate } from 'app/server/lib/ICreate';
 import { IDocStorageManager } from 'app/server/lib/IDocStorageManager';
@@ -88,3 +89,33 @@ export interface DocTemplate {
   page: string,
   tag: string,
 }
+
+/**
+ * A very minimal GristServer object that throws an error if its bluff is
+ * called.
+ */
+export function createDummyGristServer(): GristServer {
+  return {
+    create,
+    settings: {},
+    getHost() { return 'localhost:4242'; },
+    getHomeUrl() { return 'http://localhost:4242'; },
+    getHomeUrlByDocId() { return Promise.resolve('http://localhost:4242'); },
+    getMergedOrgUrl() { return 'http://localhost:4242'; },
+    getOwnUrl() { return 'http://localhost:4242'; },
+    getPermitStore() { throw new Error('no permit store'); },
+    getExternalPermitStore() { throw new Error('no external permit store'); },
+    getGristConfig() { return { homeUrl: '', timestampMs: 0 }; },
+    getOrgUrl() { return Promise.resolve(''); },
+    getResourceUrl() { return Promise.resolve(''); },
+    getSessions() { throw new Error('no sessions'); },
+    getComm() { throw new Error('no comms'); },
+    getHosts() { throw new Error('no hosts'); },
+    getHomeDBManager() { throw new Error('no db'); },
+    getStorageManager() { throw new Error('no storage manager'); },
+    getNotifier() { throw new Error('no notifier'); },
+    getDocTemplate() { throw new Error('no doc template'); },
+    getTag() { return 'tag'; },
+    sendAppPage() { return Promise.resolve(); },
+  };
+}

app/server/lib/IDocStorageManager.ts

@@ -38,3 +38,33 @@ export interface IDocStorageManager {
   removeSnapshots(docName: string, snapshotIds: string[]): Promise<void>;
   replace(docName: string, options: DocReplacementOptions): Promise<void>;
 }
+
+/**
+ * A very minimal implementation of IDocStorageManager that is just
+ * enough to allow an ActiveDoc to open and get to work.
+ */
+export class TrivialDocStorageManager implements IDocStorageManager {
+  public getPath(docName: string): string { return docName; }
+  public getSampleDocPath() { return null; }
+  public async getCanonicalDocName(altDocName: string) { return altDocName; }
+  public async prepareLocalDoc() { return false; }
+  public async prepareToCreateDoc() { }
+  public async prepareFork(): Promise<never> { throw new Error('no'); }
+  public async listDocs() { return []; }
+  public async deleteDoc(): Promise<never> { throw new Error('no'); }
+  public async renameDoc(): Promise<never> { throw new Error('no'); }
+  public async makeBackup(): Promise<never> { throw new Error('no'); }
+  public async showItemInFolder(): Promise<never> { throw new Error('no'); }
+  public async closeStorage() {}
+  public async closeDocument() {}
+  public markAsChanged() {}
+  public scheduleUsageUpdate() {}
+  public testReopenStorage() {}
+  public async addToStorage(): Promise<never> { throw new Error('no'); }
+  public prepareToCloseStorage() {}
+  public async getCopy(): Promise<never> { throw new Error('no'); }
+  public async flushDoc() {}
+  public async getSnapshots(): Promise<never> { throw new Error('no'); }
+  public async removeSnapshots(): Promise<never> { throw new Error('no'); }
+  public async replace(): Promise<never> { throw new Error('no'); }
+}

app/server/lib/SQLiteDB.ts

@@ -166,7 +166,7 @@ export class SQLiteDB implements ISQLiteDB {
 
     // It's possible that userVersion is 0 for a non-empty DB if it was created without this
     // module. In that case, we apply migrations starting with the first one.
-    if (userVersion === 0 && (await isEmpty(db))) {
+    if (userVersion === 0 && (await isGristEmpty(db))) {
       await db._initNewDB(schemaInfo);
     } else if (mode === OpenMode.CREATE_EXCL) {
       await db.close();
@@ -537,15 +537,15 @@ export class SQLiteDB implements ISQLiteDB {
 // dummy DB, and we use it to do sanity checking, in particular after migrations. To avoid
 // creating dummy DBs multiple times, the result is cached, keyed by the "create" function itself.
 const dbMetadataCache: Map<DBFunc, DBMetadata> = new Map();
-interface DBMetadata {
+export interface DBMetadata {
   [tableName: string]: {
     [colName: string]: string;      // Maps column name to SQLite type, e.g. "TEXT".
   };
 }
 
-// Helper to see if a database is empty.
+// Helper to see if a database is empty of grist metadata tables.
-async function isEmpty(db: SQLiteDB): Promise<boolean> {
+async function isGristEmpty(db: SQLiteDB): Promise<boolean> {
-  return (await db.get("SELECT count(*) as count FROM sqlite_master"))!.count === 0;
+  return (await db.get("SELECT count(*) as count FROM sqlite_master WHERE name LIKE '_grist%'"))!.count === 0;
 }
 
 /**

app/server/lib/Sharing.ts

@@ -281,7 +281,7 @@ export class Sharing {
       // Apply the action to the database, and record in the action log.
       if (!trivial) {
         await this._activeDoc.docStorage.execTransaction(async () => {
-          await this._activeDoc.docStorage.applyStoredActions(getEnvContent(ownActionBundle.stored));
+          await this._activeDoc.applyStoredActionsToDocStorage(getEnvContent(ownActionBundle.stored));
           if (this.isShared() && branch === Branch.Local) {
             // this call will compute an actionHash for localActionBundle
             await this._actionHistory.recordNextLocalUnsent(localActionBundle);

app/server/utils/gristify.ts

@@ -0,0 +1,256 @@
+import { ColInfoWithId } from 'app/common/DocActions';
+import { ActiveDoc } from 'app/server/lib/ActiveDoc';
+import { DocManager } from 'app/server/lib/DocManager';
+import { makeExceptionalDocSession, OptDocSession } from 'app/server/lib/DocSession';
+import { createDummyGristServer } from 'app/server/lib/GristServer';
+import { TrivialDocStorageManager } from 'app/server/lib/IDocStorageManager';
+import { DBMetadata, quoteIdent, SQLiteDB } from 'app/server/lib/SQLiteDB';
+
+/**
+ * A utility class for modifying a SQLite file to be viewed/edited with Grist.
+ */
+export class Gristifier {
+  public constructor(private _filename: string) {
+  }
+
+  /**
+   * Add Grist metadata tables to a SQLite file. After this action,
+   * the file can be opened as a Grist document, with partial functionality.
+   * Level of functionality will depend on the nature of the tables in the
+   * SQLite file.
+   *
+   * The `user_version` slot of SQLite will be modified by this operation,
+   * losing whatever was in it previously.
+   *
+   * A "manualSort" column may be added to tables by specifying `addSort`,
+   * to support a notion of order that exists in spreadsheets.
+   *
+   * Grist is very finicky about primary keys, and tables that don't match
+   * its expectations cannot be viewed or edited directly at the moment.
+   * Instead, views are added supporting selects, updates, inserts, and
+   * deletes. Structure changes (e.g. adding/removing columns) are not
+   * supported unfortunately.
+   *
+   * This is very much an experiment, with plenty of limits and
+   * sharp edges. In general it isn't possible to treat an arbitrary
+   * SQLite file as a Grist document, but in particular cases it can
+   * work and be very useful.
+   */
+  public async gristify(options: {addSort?: boolean}) {
+    // Remove any existing Grist material from the file.
+    await this.degristify();
+
+    // Enumerate user tables and columns.
+    const inventory = await this._getUserTables();
+
+    // Grist keeps a schema number in the SQLite "user_version" slot,
+    // so we need to zap it. This is the one destructive operation
+    // involved in gristification.
+    // TODO: consider moving schema information somewhere more neutral.
+    await this._zapUserVersion();
+
+    // Open the file as an empty Grist document, creating Grist metadata
+    // tables.
+    const docManager = new DocManager(
+      new TrivialDocStorageManager(), null, null, createDummyGristServer()
+    );
+    const activeDoc = new ActiveDoc(docManager, this._filename);
+    const docSession = makeExceptionalDocSession('system');
+    await activeDoc.createEmptyDoc(docSession, {useExisting: true});
+    await activeDoc.waitForInitialization();
+
+    // Now "create" user tables and columns with Grist. The creation
+    // will be fictitious since the tables and columns already exist -
+    // they just don't have metadata describing them to Grist.
+    const outcomes: TableOutcome[] = [];
+    for (const [tableId, table] of Object.entries(inventory)) {
+      const columnDefs = this._collectColumnDefinitions(table);
+      if (!('id' in columnDefs)) {
+        // Can't handle this table in Grist directly at the moment, but
+        // we can do something via a view.
+        await this._createView(docSession, activeDoc, tableId, Object.keys(table), columnDefs);
+        outcomes.push({tableId, viewed: true, reason: 'id complications'});
+      } else {
+        await this._registerTable(docSession, activeDoc, tableId, columnDefs);
+        if (options.addSort) {
+          await this._addManualSort(activeDoc, tableId);
+          outcomes.push({tableId, addManualSort: true});
+        } else {
+          outcomes.push({tableId});
+        }
+      }
+    }
+    await activeDoc.shutdown();
+
+    // Give a final readout of what happened for every table, since the
+    // conversion process is quite noisy.
+    for (const outcome of outcomes) {
+      console.log(JSON.stringify(outcome));
+    }
+  }
+
+  /**
+   * Remove all Grist metadata tables. Warning: attachments are considered metadata.
+   */
+  public async degristify() {
+    const db = await SQLiteDB.openDBRaw(this._filename);
+    const tables = await db.all(
+      `SELECT name FROM sqlite_master WHERE type='table' ` +
+        `  AND name LIKE '_grist%'`
+    );
+    for (const table of tables) {
+      console.log(`Removing ${table.name}`);
+      await db.exec(`DROP TABLE ${quoteIdent(table.name)}`);
+    }
+    const views = await db.all(
+      `SELECT name FROM sqlite_master WHERE type='view' ` +
+        `  AND name LIKE 'GristView%'`
+    );
+    for (const view of views) {
+      console.log(`Removing ${view.name}`);
+      await db.exec(`DROP VIEW ${quoteIdent(view.name)}`);
+    }
+    await db.close();
+  }
+
+  /**
+   * Make definitions for the table's columns. This is very crude, it handles
+   * integers and leaves everything else as "Any".
+   */
+  private _collectColumnDefinitions(table: DBMetadata[string]) {
+    const defs: Record<string, ColInfoWithId> = {};
+    for (const [colId, info] of Object.entries(table)) {
+      if (colId.startsWith('manualSort')) { continue; }
+      const type = info.toLowerCase();
+      const c: ColInfoWithId = {
+        id: colId,
+        type: 'Any',
+        isFormula: false,
+        formula: '',
+      };
+      // see https://www.sqlite.org/datatype3.html#determination_of_column_affinity
+      if (type.includes('int')) {
+        c.type = 'Int';
+      }
+      if (colId === 'id') {
+        if (c.type !== 'Int') {
+          // Grist can only support integer id columns.
+          // For now, just rename this column out of the way to id2, and use
+          // a view to map SQLite's built-in ROWID to the id column.
+          // TODO: could collide with a column called "id2".
+          c.id = 'id2';
+        }
+      }
+      defs[c.id] = c;
+    }
+    return defs;
+  }
+
+  /**
+   * Support tables that don't have an integer column called "id" through views.
+   * It would be better to enhance Grist to support a wider variety of scenarios,
+   * but this is helpful for now.
+   */
+  private async _createView(docSession: OptDocSession, activeDoc: ActiveDoc, tableId: string,
+                            cols: string[], columnDefs: Record<string, ColInfoWithId>) {
+    const newName = `GristView_${tableId}`;
+    function quote(name: string) {
+      return quoteIdent(name === 'id' ? 'id2' : name);
+    }
+    function quoteForSelect(name: string) {
+      if (name === 'id') { return 'id as id2'; }
+      return quoteIdent(name);
+    }
+
+    // View table tableId via a view GristView_tableId, with id and manualSort supplied
+    // from ROWID. SQLite tables may not have a ROWID, but this is relatively rare.
+    await activeDoc.docStorage.exec(`CREATE VIEW ${quoteIdent(newName)} AS SELECT ` +
+      ['ROWID AS id', 'ROWID AS manualSort', ...cols.map(quoteForSelect)].join(', ') +
+      ` FROM ${quoteIdent(tableId)}`);
+
+    // Make an INSTEAD OF UPDATE trigger, so that if someone tries to update the view,
+    // we instead update the underlying table. Updates of manualSort or id are just ignored.
+    // The trigger is a little awkward to write since we need to compare OLD and NEW
+    // to see what changed - updating unchanged material could needlessly run afoul of
+    // constraints.
+    const updateTrigger = `CREATE TRIGGER ${quoteIdent('trigger_update_' + newName)} ` +
+      `INSTEAD OF UPDATE ON ${quoteIdent(newName)} BEGIN ` +
+      cols.map(col =>
+        `UPDATE ${quoteIdent(tableId)} SET ` +
+        `${quoteIdent(col)} = NEW.${quote(col)} ` +
+        ` WHERE OLD.${quote(col)} <> NEW.${quote(col)} ` +
+        ` AND ${quoteIdent(tableId)}.ROWID = NEW.ROWID`
+              ).join('; ') +
+      `; END`;
+    await activeDoc.docStorage.exec(updateTrigger);
+
+    // Make an INSTEAD OF INSERT trigger.
+    const insertTrigger = `create trigger ${quoteIdent('trigger_insert_' + newName)} ` +
+      `INSTEAD OF INSERT ON ${quoteIdent(newName)} BEGIN ` +
+      `INSERT INTO ${quoteIdent(tableId)}` +
+      '(' + cols.map(quoteIdent).join(',') + ') VALUES(' +
+      cols.map(col => `NEW.${quote(col)}`).join(', ') +
+      `); END`;
+    await activeDoc.docStorage.exec(insertTrigger);
+
+    // Make an INSTEAD OF DELETE trigger.
+    const deleteTrigger = `create trigger ${quoteIdent('trigger_delete_' + newName)} ` +
+      `INSTEAD OF DELETE ON ${quoteIdent(newName)} BEGIN ` +
+      `DELETE FROM ${quoteIdent(tableId)} WHERE ${quoteIdent(tableId)}.ROWID = OLD.ROWID` +
+      `; END`;
+    await activeDoc.docStorage.exec(deleteTrigger);
+
+    const result = await this._registerTable(docSession, activeDoc, newName, columnDefs);
+
+    // Now, tweak the Grist metadata to make the table name the expected one
+    // (the table id as far as Grist is concerned must remain that of the view)
+    const id = result.retValues[0].id;
+    await activeDoc.docStorage.run('update _grist_Views_section set title = ? ' +
+      'where id in (select rawViewSectionRef from _grist_Tables where id = ?)',
+                                   [tableId, id]);
+    await activeDoc.docStorage.run('update _grist_Views set name = ? ' +
+      'where id in (select primaryViewId from _grist_Tables where id = ?)',
+                                   [tableId, id]);
+  }
+
+  private async _getUserTables(): Promise<DBMetadata> {
+    // Enumerate existing tables and columns.
+    const db = await SQLiteDB.openDBRaw(this._filename);
+    const inventory = await db.collectMetadata();
+    await db.close();
+    // We are not interested in the special "sqlite_sequence" table.
+    delete inventory.sqlite_sequence;
+    return inventory;
+  }
+
+  private async _zapUserVersion(): Promise<void> {
+    const db = await SQLiteDB.openDBRaw(this._filename);
+    await db.exec(`PRAGMA user_version = 0`);
+    await db.close();
+  }
+
+  private async _addManualSort(activeDoc: ActiveDoc, tableId: string) {
+    const db = activeDoc.docStorage;
+    await db.exec(`ALTER TABLE ${quoteIdent(tableId)} ADD COLUMN manualSort INTEGER`).catch(e => null);
+    await db.exec(`UPDATE ${quoteIdent(tableId)} SET manualSort = id`);
+  }
+
+  private async _registerTable(docSession: OptDocSession, activeDoc: ActiveDoc,
+                               tableId: string, args: Record<string, ColInfoWithId>) {
+    delete args.id;
+    activeDoc.onlyAllowMetaDataActionsOnDb(true);
+    const result = await activeDoc.applyUserActions(docSession, [
+      ['AddTable', tableId, Object.values(args)],
+    ]);
+    activeDoc.onlyAllowMetaDataActionsOnDb(false);
+    return result;
+  }
+}
+
+interface TableOutcome {
+  tableId: string;
+  skipped?: boolean;
+  viewed?: boolean;
+  addManualSort?: boolean;
+  reason?: string;
+}

app/server/utils/pruneActionHistory.ts

@@ -0,0 +1,63 @@
+import * as gutil from 'app/common/gutil';
+import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl';
+import {DocStorage} from 'app/server/lib/DocStorage';
+import {DocStorageManager} from 'app/server/lib/DocStorageManager';
+import * as docUtils from 'app/server/lib/docUtils';
+import log from 'app/server/lib/log';
+
+/**
+ * A utility script for cleaning up the action log.
+ *
+ * @param {String} docPath - The path to the document from the current directory including
+ *  the document name.
+ * @param {Int} keepN - The number of recent actions to keep. Must be at least 1. Defaults to 1
+ *  if not provided.
+ */
+export async function pruneActionHistory(docPath: string, keepN: number) {
+  if (!docPath || !gutil.endsWith(docPath, '.grist')) {
+    throw new Error('Invalid document: Document should be a valid .grist file');
+  }
+
+  const storageManager = new DocStorageManager(".", ".");
+  const docStorage = new DocStorage(storageManager, docPath);
+  const backupPath = gutil.removeSuffix(docPath, '.grist') + "-backup.grist";
+
+  // If the backup already exists, abort. Otherwise, create a backup copy and continue.
+  const exists = await docUtils.pathExists(backupPath);
+  if (exists) { throw new Error('Backup file already exists, aborting pruneActionHistory'); }
+  await docUtils.copyFile(docPath, backupPath);
+  await docStorage.openFile();
+  try {
+    const history = new ActionHistoryImpl(docStorage);
+    await history.initialize();
+    await history.deleteActions(keepN);
+  } finally {
+    await docStorage.shutdown();
+  }
+}
+
+/**
+ * Variant that accepts and parses command line arguments.
+ */
+export async function pruneActionHistoryFromConsole(argv: string[]): Promise<number> {
+  if (argv.length === 0) {
+    log.error("Please supply document name, and optionally the number of actions to preserve (default=1)");
+    return 1;
+  }
+  const docPath = argv[0];
+  const keepN = parseInt(argv[1], 10) || 1;
+  try {
+    await pruneActionHistory(docPath, keepN);
+  } catch (e) {
+    log.error(e);
+    return 1;
+  }
+  return 0;
+}
+
+if (require.main === module) {
+  pruneActionHistoryFromConsole(process.argv.slice(2)).catch((e) => {
+    log.error("pruneActionHistory failed: %s", e);
+    process.exit(1);
+  });
+}

package.json

@@ -15,7 +15,8 @@
     "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha -g ${GREP_TEST:-''} _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js",
     "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/server/**/*.js _build/test/gen-server/**/*.js",
     "test:smoke": "NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/nbrowser/Smoke.js",
-    "test:docker": "./test/test_under_docker.sh"
+    "test:docker": "./test/test_under_docker.sh",
+    "cli": "NODE_PATH=_build:_build/stubs:_build/ext node _build/app/server/companion.js"
   },
   "keywords": [
     "grist",
@@ -96,6 +97,7 @@
     "bowser": "2.7.0",
     "brace": "0.11.1",
     "collect-js-deps": "^0.1.1",
+    "commander": "9.3.0",
     "components-jqueryui": "1.12.1",
     "connect-redis": "3.4.0",
     "cookie": "0.5.0",

test/server/docTools.ts

@@ -1,11 +1,10 @@
 import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {DummyAuthorizer} from 'app/server/lib/Authorizer';
-import {create} from 'app/server/lib/create';
 import {DocManager} from 'app/server/lib/DocManager';
 import {DocSession, makeExceptionalDocSession} from 'app/server/lib/DocSession';
 import {DocStorageManager} from 'app/server/lib/DocStorageManager';
-import {GristServer} from 'app/server/lib/GristServer';
+import {createDummyGristServer, GristServer} from 'app/server/lib/GristServer';
 import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
 import {getAppRoot} from 'app/server/lib/places';
 import {PluginManager} from 'app/server/lib/PluginManager';
@@ -151,32 +150,6 @@ export async function createDocManager(
   });
 }
 
-export function createDummyGristServer(): GristServer {
-  return {
-    create,
-    settings: {},
-    getHost() { return 'localhost:4242'; },
-    getHomeUrl() { return 'http://localhost:4242'; },
-    getHomeUrlByDocId() { return Promise.resolve('http://localhost:4242'); },
-    getMergedOrgUrl() { return 'http://localhost:4242'; },
-    getOwnUrl() { return 'http://localhost:4242'; },
-    getPermitStore() { throw new Error('no permit store'); },
-    getExternalPermitStore() { throw new Error('no external permit store'); },
-    getGristConfig() { return { homeUrl: '', timestampMs: 0 }; },
-    getOrgUrl() { return Promise.resolve(''); },
-    getResourceUrl() { return Promise.resolve(''); },
-    getSessions() { throw new Error('no sessions'); },
-    getComm() { throw new Error('no comms'); },
-    getHosts() { throw new Error('no hosts'); },
-    getHomeDBManager() { throw new Error('no db'); },
-    getStorageManager() { throw new Error('no storage manager'); },
-    getNotifier() { throw new Error('no notifier'); },
-    getDocTemplate() { throw new Error('no doc template'); },
-    getTag() { return 'tag'; },
-    sendAppPage() { return Promise.resolve(); },
-  };
-}
-
 export async function createTmpDir(): Promise<string> {
   const tmpRootDir = process.env.TESTDIR || tmpdir();
   await fse.mkdirs(tmpRootDir);

yarn.lock

@@ -1776,6 +1776,11 @@ commander@2.15.1:
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.15.1.tgz#df46e867d0fc2aec66a34662b406a9ccafff5b0f"
   integrity sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==
 
+commander@9.3.0:
+  version "9.3.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-9.3.0.tgz#f619114a5a2d2054e0d9ff1b31d5ccf89255e26b"
+  integrity sha512-hv95iU5uXPbK83mjrJKuZyFM/LBAoCV/XhVGkS5Je6tl7sxr6A0ITMw5WoRV46/UaJ46Nllm3Xt7IaJhXTIkzw==
+
 commander@^2.11.0, commander@^2.20.0:
   version "2.20.3"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"