(core) add a yarn run cli tool, and add a sqlite gristify option

Summary:
This adds rudimentary support for opening certain SQLite files in Grist.

If you have a file such as `landing.db` in Grist, you can convert it to Grist format by doing (either in monorepo or grist-core):
```
yarn run cli -h
yarn run cli sqlite -h
yarn run cli sqlite gristify landing.db
```

The file is now openable by Grist. To actually do so with the regular Grist server, you'll need to either import it, or convert some doc you don't care about in the `samples/` directory to be a soft link to it (and then force a reload).

This implementation is a rudimentary experiment. Here are some awkwardnesses:
 * Only tables that happen to have a column called `id`, and where the column happens to be an integer, can be opened directly with Grist as it is today. That could be generalized, but it looked more than a Gristathon's worth of work, so I instead used SQLite views.
 * Grist will handle tables that start with an uncapitalized letter a bit erratically. You can successfully add columns, for example, but removing them will cause sadness - Grist will rename the table in a confused way.
 * I didn't attempt to deal with column names with spaces etc (though views could deal with those).
 * I haven't tried to do any fancy type mapping.
 * Columns with constraints can make adding new rows impossible in Grist, since Grist requires that a row can be added with just a single cell set.

Test Plan: added small test

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3502

app/server/lib/ActiveDoc.ts

@@ -33,6 +33,7 @@ import {
   BulkUpdateRecord,
   CellValue,
   DocAction,
+  getTableId,
   TableDataAction,
   TableRecordValue,
   toTableDataAction,
@@ -170,7 +171,7 @@ export class ActiveDoc extends EventEmitter {
   }
 
   public readonly docStorage: DocStorage;
-  public readonly docPluginManager: DocPluginManager;
+  public readonly docPluginManager: DocPluginManager|null;
   public readonly docClients: DocClients;               // Only exposed for Sharing.ts
   public docData: DocData|null = null;
 
@@ -204,6 +205,7 @@ export class ActiveDoc extends EventEmitter {
   private _product?: Product;
   private _gracePeriodStart: Date|null = null;
   private _isForkOrSnapshot: boolean = false;
+  private _onlyAllowMetaDataActionsOnDb: boolean = false;
 
   // Client watching for 'product changed' event published by Billing to update usage
   private _redisSubscriber?: RedisClient;
@@ -276,8 +278,9 @@ export class ActiveDoc extends EventEmitter {
     this._triggers = new DocTriggers(this);
     this._requests = new DocRequests(this);
     this._actionHistory = new ActionHistoryImpl(this.docStorage);
-    this.docPluginManager = new DocPluginManager(docManager.pluginManager.getPlugins(),
-      docManager.pluginManager.appRoot!, this, this._docManager.gristServer);
+    this.docPluginManager = docManager.pluginManager ?
+      new DocPluginManager(docManager.pluginManager.getPlugins(),
+                           docManager.pluginManager.appRoot!, this, this._docManager.gristServer) : null;
     this._tableMetadataLoader = new TableMetadataLoader({
       decodeBuffer: this.docStorage.decodeMarshalledData.bind(this.docStorage),
       fetchTable: this.docStorage.fetchTable.bind(this.docStorage),
@@ -529,7 +532,7 @@ export class ActiveDoc extends EventEmitter {
         }
         await Promise.all([
           this.docStorage.shutdown(),
-          this.docPluginManager.shutdown(),
+          this.docPluginManager?.shutdown(),
           dataEngine?.shutdown()
         ]);
         // The this.waitForInitialization promise may not yet have resolved, but
@@ -576,7 +579,7 @@ export class ActiveDoc extends EventEmitter {
     await this._initDoc(docSession);
     await this._tableMetadataLoader.clean();
     // Makes sure docPluginManager is ready in case new doc is used to import new data
-    await this.docPluginManager.ready;
+    await this.docPluginManager?.ready;
     this._fullyLoaded = true;
     return this;
   }
@@ -585,10 +588,13 @@ export class ActiveDoc extends EventEmitter {
    * Create a new blank document (no "Table1"), used as a stub when importing.
    */
   @ActiveDoc.keepDocOpen
-  public async createEmptyDoc(docSession: OptDocSession): Promise<ActiveDoc> {
-    await this.loadDoc(docSession, {forceNew: true, skipInitialTable: true});
+  public async createEmptyDoc(docSession: OptDocSession,
+                              options?: { useExisting?: boolean }): Promise<ActiveDoc> {
+    await this.loadDoc(docSession, {forceNew: true,
+                                    skipInitialTable: true,
+                                    ...options});
     // Makes sure docPluginManager is ready in case new doc is used to import new data
-    await this.docPluginManager.ready;
+    await this.docPluginManager?.ready;
     this._fullyLoaded = true;
     return this;
   }
@@ -603,13 +609,18 @@ export class ActiveDoc extends EventEmitter {
   public async loadDoc(docSession: OptDocSession, options?: {
     forceNew?: boolean,          // If set, document will be created.
     skipInitialTable?: boolean,  // If set, and document is new, "Table1" will not be added.
+    useExisting?: boolean,       // If set, document can be created as an overlay on
+                                 // an existing sqlite file.
   }): Promise<ActiveDoc> {
     const startTime = Date.now();
     this._log.debug(docSession, "loadDoc");
     try {
       const isNew: boolean = options?.forceNew || await this._docManager.storageManager.prepareLocalDoc(this.docName);
       if (isNew) {
-        await this._createDocFile(docSession, {skipInitialTable: options?.skipInitialTable});
+        await this._createDocFile(docSession, {
+          skipInitialTable: options?.skipInitialTable,
+          useExisting: options?.useExisting,
+        });
       } else {
         await this.docStorage.openFile({
           beforeMigration: async (currentVersion, newVersion) => {
@@ -1253,6 +1264,7 @@ export class ActiveDoc extends EventEmitter {
     if (await this._granularAccess.hasNuancedAccess(docSession)) {
       throw new Error('cannot confirm access to plugin');
     }
+    if (!this.docPluginManager) { throw new Error('no plugin manager available'); }
     const pluginRpc = this.docPluginManager.plugins[pluginId].rpc;
     switch (msg.mtype) {
       case MsgType.RpcCall: return pluginRpc.forwardCall(msg);
@@ -1266,6 +1278,7 @@ export class ActiveDoc extends EventEmitter {
    */
   public async reloadPlugins(docSession: DocSession) {
     // refresh the list plugins found on the system
+    if (!this._docManager.pluginManager || !this.docPluginManager) { return; }
     await this._docManager.pluginManager.reloadPlugins();
     const plugins = this._docManager.pluginManager.getPlugins();
     // reload found plugins
@@ -1433,6 +1446,7 @@ export class ActiveDoc extends EventEmitter {
   }
 
   public getGristDocAPI(): GristDocAPI {
+    if (!this.docPluginManager) { throw new Error('no plugin manager available'); }
     return this.docPluginManager.gristDocAPI;
   }
 
@@ -1573,6 +1587,27 @@ export class ActiveDoc extends EventEmitter {
     return this._docManager.makeAccessId(userId);
   }
 
+  /**
+   * Apply actions that have already occurred in the data engine to the
+   * database also.
+   */
+  public async applyStoredActionsToDocStorage(docActions: DocAction[]): Promise<void> {
+    // When "gristifying" an sqlite database, we may take create tables and
+    // columns in the data engine that already exist in the sqlite database.
+    // During that process, _onlyAllowMetaDataActionsOnDb will be turned on,
+    // and we silently swallow any non-metadata actions.
+    if (this._onlyAllowMetaDataActionsOnDb) {
+      docActions = docActions.filter(a => getTableId(a).startsWith('_grist'));
+    }
+    await this.docStorage.applyStoredActions(docActions);
+  }
+
+  // Set a flag that controls whether user data can be changed in the database,
+  // or only grist-managed tables (those whose names start with _grist)
+  public onlyAllowMetaDataActionsOnDb(flag: boolean) {
+    this._onlyAllowMetaDataActionsOnDb = flag;
+  }
+
   /**
    * Called by Sharing manager when working on modifying the document.
    * Called when DocActions have been produced from UserActions, but
@@ -1720,10 +1755,12 @@ export class ActiveDoc extends EventEmitter {
   @ActiveDoc.keepDocOpen
   private async _createDocFile(docSession: OptDocSession, options?: {
     skipInitialTable?: boolean,  // If set, "Table1" will not be added.
+    useExisting?: boolean,       // If set, an existing sqlite db is permitted.
+                                 // Useful for "gristifying" an existing db.
   }): Promise<void> {
     this._log.debug(docSession, "createDoc");
     await this._docManager.storageManager.prepareToCreateDoc(this.docName);
-    await this.docStorage.createFile();
+    await this.docStorage.createFile(options);
     const sql = options?.skipInitialTable ? GRIST_DOC_SQL : GRIST_DOC_WITH_TABLE1_SQL;
     await this.docStorage.exec(sql);
     const timezone = docSession.browserSettings?.timezone ?? DEFAULT_TIMEZONE;

app/server/lib/ActiveDocImport.ts

@@ -242,6 +242,7 @@ export class ActiveDocImport {
 
     // The upload must be within the plugin-accessible directory. Once moved, subsequent calls to
     // moveUpload() will return without having to do anything.
+    if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); }
     await moveUpload(upload, this._activeDoc.docPluginManager.tmpDir());
 
     const importResult: ImportResult = {options: parseOptions, tables: []};
@@ -287,6 +288,7 @@ export class ActiveDocImport {
     const {originalFilename, parseOptions, mergeOptionsMap, isHidden, uploadFileIndex,
            transformRuleMap} = importOptions;
     log.info("ActiveDoc._importFileAsNewTable(%s, %s)", tmpPath, originalFilename);
+    if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); }
     const optionsAndData: ParseFileResult =
       await this._activeDoc.docPluginManager.parseFile(tmpPath, originalFilename, parseOptions);
     const options = optionsAndData.parseOptions;

app/server/lib/DocClients.ts

@@ -95,7 +95,7 @@ export class DocClients {
     }
     if (type === "docUserAction" && messageData.docActions) {
       for (const action of messageData.docActions) {
-        this.activeDoc.docPluginManager.receiveAction(action);
+        this.activeDoc.docPluginManager?.receiveAction(action);
       }
     }
   }

app/server/lib/DocManager.ts

@@ -54,7 +54,7 @@ export class DocManager extends EventEmitter {
 
   constructor(
     public readonly storageManager: IDocStorageManager,
-    public readonly pluginManager: PluginManager,
+    public readonly pluginManager: PluginManager|null,
     private _homeDbManager: HomeDBManager|null,
     public gristServer: GristServer
   ) {
@@ -610,7 +610,7 @@ export class DocManager extends EventEmitter {
       return docUtils.createExclusive(this.storageManager.getPath(name));
     });
     log.debug('DocManager._createNewDoc picked name', docName);
-    await this.pluginManager.pluginsLoaded;
+    await this.pluginManager?.pluginsLoaded;
     return docName;
   }
 }

app/server/lib/DocStorage.ts

@@ -30,6 +30,7 @@ import uuidv4 from "uuid/v4";
 import {OnDemandStorage} from './OnDemandActions';
 import {ISQLiteDB, MigrationHooks, OpenMode, quoteIdent, ResultRow, SchemaInfo, SQLiteDB} from './SQLiteDB';
 import chunk = require('lodash/chunk');
+import cloneDeep = require('lodash/cloneDeep');
 import groupBy = require('lodash/groupBy');
 
 
@@ -666,10 +667,17 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
    * After a database is created it should be initialized by applying the InitNewDoc action
    * or by executing the initialDocSql.
    */
-  public createFile(): Promise<void> {
+  public createFile(options?: {
+    useExisting?: boolean,  // If set, it is ok if an sqlite file already exists
+                            // where we would store the Grist document. Its content
+                            // will not be touched. Useful when "gristifying" an
+                            // existing SQLite DB.
+  }): Promise<void> {
     // It turns out to be important to return a bluebird promise, a lot of code outside
     // of DocStorage ultimately depends on this.
-    return bluebird.Promise.resolve(this._openFile(OpenMode.CREATE_EXCL, {}))
+    return bluebird.Promise.resolve(this._openFile(
+      options?.useExisting ? OpenMode.OPEN_EXISTING : OpenMode.CREATE_EXCL,
+      {}))
       .then(() => this._initDB());
     // Note that we don't call _updateMetadata() as there are no metadata tables yet anyway.
   }
@@ -927,26 +935,50 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
   public async applyStoredActions(docActions: DocAction[]): Promise<void> {
     debuglog('DocStorage.applyStoredActions');
 
-    await bluebird.Promise.each(docActions, (action: DocAction) => {
-      const actionType = action[0];
-      const f = (this as any)["_process_" + actionType];
-      if (!_.isFunction(f)) {
-        log.error("Unknown action: " + actionType);
-      } else {
-        return f.apply(this, action.slice(1))
-          .then(() => {
-            const tableId = action[1]; // The first argument is always tableId;
-            if (DocStorage._isMetadataTable(tableId) && actionType !== 'AddTable') {
-              // We only need to update the metadata for actions that change
-              // the metadata. We don't update on AddTable actions
-              // because the additional of a table gives no additional data
-              // and if we tried to update when only _grist_Tables was added
-              // without _grist_Tables_column, we would get an error
-              return this._updateMetadata();
-            }
-          });
+    docActions = this._compressStoredActions(docActions);
+    for (const action of docActions) {
+      try {
+        await this.applyStoredAction(action);
+      } catch (e) {
+        // If the table doesn't have a manualSort column, we'll try
+        // again without setting manualSort. This should never happen
+        // for regular Grist documents, but could happen for a
+        // "gristified" Sqlite database where we are choosing to
+        // leave the user tables untouched. The manualSort column doesn't
+        // make much sense outside the context of spreadsheets.
+        // TODO: it could be useful to make Grist more inherently aware of
+        // and tolerant of tables without manual sorting.
+        if (String(e).match(/no column named manualSort/)) {
+          const modifiedAction = this._considerWithoutManualSort(action);
+          if (modifiedAction) {
+            await this.applyStoredAction(modifiedAction);
+            return;
+          }
+        }
+        throw e;
       }
-    });
+    }
+  }
+
+  // Apply a single stored action, dispatching to an appropriate
+  // _process_<ActionType> handler.
+  public async applyStoredAction(action: DocAction): Promise<void> {
+    const actionType = action[0];
+    const f = (this as any)["_process_" + actionType];
+    if (!_.isFunction(f)) {
+      log.error("Unknown action: " + actionType);
+    } else {
+      await f.apply(this, action.slice(1));
+      const tableId = action[1]; // The first argument is always tableId;
+      if (DocStorage._isMetadataTable(tableId) && actionType !== 'AddTable') {
+        // We only need to update the metadata for actions that change
+        // the metadata. We don't update on AddTable actions
+        // because the additional of a table gives no additional data
+        // and if we tried to update when only _grist_Tables was added
+        // without _grist_Tables_column, we would get an error
+        await this._updateMetadata();
+      }
+    }
   }
 
   /**
@@ -1169,6 +1201,8 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
     // Note that SQLite does not support easily dropping columns. To drop a column from a table, we
     // need to follow the instructions at https://sqlite.org/lang_altertable.html Since we don't use
     // indexes or triggers, we skip a few steps.
+    // TODO: SQLite has since added support for ALTER TABLE DROP COLUMN, should
+    // use that to be more efficient and less disruptive.
 
     // This returns rows with (at least) {name, type, dflt_value}.
     return this.all(`PRAGMA table_info(${quote(tableId)})`)
@@ -1713,6 +1747,42 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
       `${joinClauses} ${whereClause} ${limitClause}`;
     return sql;
   }
+
+  // If we are being asked to add a record and then update several of its
+  // columns, compact that into a single action. For fully Grist-managed
+  // documents, this makes no difference. But if the underlying SQLite DB
+  // has extra constraints on columns, it can make a difference.
+  // TODO: consider dealing with other scenarios, especially a BulkAddRecord.
+  private _compressStoredActions(docActions: DocAction[]): DocAction[] {
+    if (docActions.length > 1) {
+      const first = docActions[0];
+      if (first[0] === 'AddRecord' &&
+        docActions.slice(1).every(
+          // Check other actions are UpdateRecords for the same table and row.
+          a => a[0] === 'UpdateRecord' && a[1] === first[1] && a[2] === first[2]
+        )) {
+        const merged = cloneDeep(first);
+        for (const a2 of docActions.slice(1)) {
+          Object.assign(merged[3], a2[3]);
+        }
+        docActions = [merged];
+      }
+    }
+    return docActions;
+  }
+
+  // If an action can have manualSort removed, go ahead and do it (after cloning),
+  // otherwise return null.
+  private _considerWithoutManualSort(act: DocAction): DocAction|null {
+    if (act[0] === 'AddRecord' || act[0] === 'UpdateRecord' ||
+      act[0] === 'BulkAddRecord' || act[0] === 'BulkUpdateRecord' &&
+      'manualSort' in act[3]) {
+      act = cloneDeep(act);
+      delete act[3].manualSort;
+      return act;
+    }
+    return null;
+  }
 }
 
 interface RebuildResult {

app/server/lib/GristServer.ts

@@ -6,6 +6,7 @@ import { Workspace } from 'app/gen-server/entity/Workspace';
 import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
 import { RequestWithLogin } from 'app/server/lib/Authorizer';
 import { Comm } from 'app/server/lib/Comm';
+import { create } from 'app/server/lib/create';
 import { Hosts } from 'app/server/lib/extractOrg';
 import { ICreate } from 'app/server/lib/ICreate';
 import { IDocStorageManager } from 'app/server/lib/IDocStorageManager';
@@ -88,3 +89,33 @@ export interface DocTemplate {
   page: string,
   tag: string,
 }
+
+/**
+ * A very minimal GristServer object that throws an error if its bluff is
+ * called.
+ */
+export function createDummyGristServer(): GristServer {
+  return {
+    create,
+    settings: {},
+    getHost() { return 'localhost:4242'; },
+    getHomeUrl() { return 'http://localhost:4242'; },
+    getHomeUrlByDocId() { return Promise.resolve('http://localhost:4242'); },
+    getMergedOrgUrl() { return 'http://localhost:4242'; },
+    getOwnUrl() { return 'http://localhost:4242'; },
+    getPermitStore() { throw new Error('no permit store'); },
+    getExternalPermitStore() { throw new Error('no external permit store'); },
+    getGristConfig() { return { homeUrl: '', timestampMs: 0 }; },
+    getOrgUrl() { return Promise.resolve(''); },
+    getResourceUrl() { return Promise.resolve(''); },
+    getSessions() { throw new Error('no sessions'); },
+    getComm() { throw new Error('no comms'); },
+    getHosts() { throw new Error('no hosts'); },
+    getHomeDBManager() { throw new Error('no db'); },
+    getStorageManager() { throw new Error('no storage manager'); },
+    getNotifier() { throw new Error('no notifier'); },
+    getDocTemplate() { throw new Error('no doc template'); },
+    getTag() { return 'tag'; },
+    sendAppPage() { return Promise.resolve(); },
+  };
+}

app/server/lib/IDocStorageManager.ts

@@ -38,3 +38,33 @@ export interface IDocStorageManager {
   removeSnapshots(docName: string, snapshotIds: string[]): Promise<void>;
   replace(docName: string, options: DocReplacementOptions): Promise<void>;
 }
+
+/**
+ * A very minimal implementation of IDocStorageManager that is just
+ * enough to allow an ActiveDoc to open and get to work.
+ */
+export class TrivialDocStorageManager implements IDocStorageManager {
+  public getPath(docName: string): string { return docName; }
+  public getSampleDocPath() { return null; }
+  public async getCanonicalDocName(altDocName: string) { return altDocName; }
+  public async prepareLocalDoc() { return false; }
+  public async prepareToCreateDoc() { }
+  public async prepareFork(): Promise<never> { throw new Error('no'); }
+  public async listDocs() { return []; }
+  public async deleteDoc(): Promise<never> { throw new Error('no'); }
+  public async renameDoc(): Promise<never> { throw new Error('no'); }
+  public async makeBackup(): Promise<never> { throw new Error('no'); }
+  public async showItemInFolder(): Promise<never> { throw new Error('no'); }
+  public async closeStorage() {}
+  public async closeDocument() {}
+  public markAsChanged() {}
+  public scheduleUsageUpdate() {}
+  public testReopenStorage() {}
+  public async addToStorage(): Promise<never> { throw new Error('no'); }
+  public prepareToCloseStorage() {}
+  public async getCopy(): Promise<never> { throw new Error('no'); }
+  public async flushDoc() {}
+  public async getSnapshots(): Promise<never> { throw new Error('no'); }
+  public async removeSnapshots(): Promise<never> { throw new Error('no'); }
+  public async replace(): Promise<never> { throw new Error('no'); }
+}

app/server/lib/SQLiteDB.ts

@@ -166,7 +166,7 @@ export class SQLiteDB implements ISQLiteDB {
 
     // It's possible that userVersion is 0 for a non-empty DB if it was created without this
     // module. In that case, we apply migrations starting with the first one.
-    if (userVersion === 0 && (await isEmpty(db))) {
+    if (userVersion === 0 && (await isGristEmpty(db))) {
       await db._initNewDB(schemaInfo);
     } else if (mode === OpenMode.CREATE_EXCL) {
       await db.close();
@@ -537,15 +537,15 @@ export class SQLiteDB implements ISQLiteDB {
 // dummy DB, and we use it to do sanity checking, in particular after migrations. To avoid
 // creating dummy DBs multiple times, the result is cached, keyed by the "create" function itself.
 const dbMetadataCache: Map<DBFunc, DBMetadata> = new Map();
-interface DBMetadata {
+export interface DBMetadata {
   [tableName: string]: {
     [colName: string]: string;      // Maps column name to SQLite type, e.g. "TEXT".
   };
 }
 
-// Helper to see if a database is empty.
-async function isEmpty(db: SQLiteDB): Promise<boolean> {
-  return (await db.get("SELECT count(*) as count FROM sqlite_master"))!.count === 0;
+// Helper to see if a database is empty of grist metadata tables.
+async function isGristEmpty(db: SQLiteDB): Promise<boolean> {
+  return (await db.get("SELECT count(*) as count FROM sqlite_master WHERE name LIKE '_grist%'"))!.count === 0;
 }
 
 /**

app/server/lib/Sharing.ts

@@ -281,7 +281,7 @@ export class Sharing {
       // Apply the action to the database, and record in the action log.
       if (!trivial) {
         await this._activeDoc.docStorage.execTransaction(async () => {
-          await this._activeDoc.docStorage.applyStoredActions(getEnvContent(ownActionBundle.stored));
+          await this._activeDoc.applyStoredActionsToDocStorage(getEnvContent(ownActionBundle.stored));
           if (this.isShared() && branch === Branch.Local) {
             // this call will compute an actionHash for localActionBundle
             await this._actionHistory.recordNextLocalUnsent(localActionBundle);