(core) Perform migrations of Grist schema using only metadata tables when possible.

Summary:
Loading all user data to run a migration is risky (creates more than usual
memory pressure), and almost never needed (only one migration requires it).

This diff attempts to run migrations using only metadata (_grist_* tables),
but retries if the sandbox tells it that all data is needed.

The intent is for new migrations to avoid needing all data.

Test Plan: Added a somewhat contrived unittest.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2659

app/server/lib/ActiveDoc.ts

@@ -27,7 +27,7 @@ import {toTableDataAction} from 'app/common/DocActions';
 import {DocData} from 'app/common/DocData';
 import {DocSnapshots} from 'app/common/DocSnapshot';
 import {EncActionBundleFromHub} from 'app/common/EncActionBundle';
-import {byteString} from 'app/common/gutil';
+import {byteString, countIf} from 'app/common/gutil';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import * as marshal from 'app/common/marshal';
 import {Peer} from 'app/common/sharing';
@@ -1125,13 +1125,44 @@ export class ActiveDoc extends EventEmitter {
    * collaborators.
    */
   private async _migrate(docSession: OptDocSession): Promise<void> {
-    // TODO fetchAllTables() creates more memory pressure than usual since full data is present in
-    // memory at once both in node and in Python. This is liable to cause crashes. This doesn't
-    // even skip onDemand tables. We should try to migrate using metadata only. Data migrations
-    // would need to be done table-by-table, and differently still for on-demand tables.
-    const allTables = await this.docStorage.fetchAllTables();
-    const docActions: DocAction[] = await this._dataEngine.pyCall('create_migrations', allTables);
-    this.logInfo(docSession, "_migrate: applying %d migration actions", docActions.length);
+    const tableNames = await this.docStorage.getAllTableNames();
+
+    // Fetch only metadata tables first, and try to migrate with only those.
+    const tableData: {[key: string]: Buffer|null} = {};
+    for (const tableName of tableNames) {
+      if (tableName.startsWith('_grist_')) {
+        tableData[tableName] = await this.docStorage.fetchTable(tableName);
+      }
+    }
+
+    let docActions: DocAction[];
+    try {
+      // The last argument tells create_migrations() that only metadata is included.
+      docActions = await this._dataEngine.pyCall('create_migrations', tableData, true);
+    } catch (e) {
+      if (!/need all tables/.test(e.message)) {
+        throw e;
+      }
+      // If the migration failed because it needs all tables (i.e. involves changes to data), then
+      // fetch them all. TODO: This is used for some older migrations, and is relied on by tests.
+      // If a new migration needs this flag, more work is needed. The current approach creates
+      // more memory pressure than usual since full data is present in memory at once both in node
+      // and in Python; and it doesn't skip onDemand tables. This is liable to cause crashes.
+      this.logWarn(docSession, "_migrate: retrying with all tables");
+      for (const tableName of tableNames) {
+        if (!tableData[tableName] && !tableName.startsWith('_gristsys_')) {
+          tableData[tableName] = await this.docStorage.fetchTable(tableName);
+        }
+      }
+      docActions = await this._dataEngine.pyCall('create_migrations', tableData);
+    }
+
+    const processedTables = Object.keys(tableData);
+    const numSchema = countIf(processedTables, t => t.startsWith("_grist_"));
+    const numUser = countIf(processedTables, t => !t.startsWith("_grist_"));
+    this.logInfo(docSession, "_migrate: applying %d migration actions (processed %s schema, %s user tables)",
+      docActions.length, numSchema, numUser);
+
     docActions.forEach((action, i) => this.logInfo(docSession, "_migrate: docAction %s: %s", i, shortDesc(action)));
     await this.docStorage.execTransaction(() => this.docStorage.applyStoredActions(docActions));
   }

app/server/lib/DocStorage.ts

@@ -714,22 +714,11 @@ export class DocStorage implements ISQLiteDB {
   }
 
   /**
-   * Fetches all tables from the database, and returns a dictionary mapping table names to encoded
-   * TableData objects (marshalled dicts mapping column ids to arrays of values).
+   * Fetches and returns the names of all tables in the database (including _gristsys_ tables).
    */
-  public fetchAllTables(): Promise<{[key: string]: Buffer|null}> {
-    const tables: {[key: string]: Buffer|null} = {};
-    return bluebird.Promise.each(
-      this.all("SELECT name FROM sqlite_master WHERE type='table'"),
-      (row: ResultRow) => {
-        if (!row.name.startsWith('_gristsys_')) {
-          return this.fetchTable(row.name)
-            .then(data => {
-              tables[row.name] = data;
-            });
-        }
-      })
-      .return(tables);
+  public async getAllTableNames(): Promise<string[]> {
+    const rows = await this.all("SELECT name FROM sqlite_master WHERE type='table'");
+    return rows.map(row => row.name);
   }
 
   /**