(core) start applying defenses for untrusted document uploads

Summary:
This applies some mitigations suggested by SQLite authors when
opening untrusted SQLite databases, as we do when Grist docs
are uploaded by the user.  See:
  https://www.sqlite.org/security.html#untrusted_sqlite_database_files

Steps implemented in this diff are:
  * Setting `trusted_schema` to off
  * Running a SQLite-level integrity check on uploads

Other steps will require updates to our node-sqlite3 fork, since they
are not available via the node-sqlite3 api (one more reason to migrate
to better-sqlite3).

I haven't yet managed to create a file that triggers an integrity
check failure without also being detected as corruption by sqlite
at a more basic level, so that is a TODO for testing.

Test Plan:
existing tests pass; need to come up with exploits to
actually test the defences and have not yet

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2909

app/server/declarations.d.ts

@@ -100,3 +100,7 @@ declare module '@gristlabs/pidusage' {
 }
 
 declare module "csv";
+
+declare module 'winston/lib/winston/common' {
+  export function serialize(meta: any): string;
+}

app/server/lib/ActiveDoc.ts

@@ -64,7 +64,6 @@ import {DocClients} from './DocClients';
 import {DocPluginManager} from './DocPluginManager';
 import {
   DocSession,
-  getDocSessionAccess,
   getDocSessionUser,
   getDocSessionUserId,
   makeExceptionalDocSession,
@@ -74,6 +73,7 @@ import {DocStorage} from './DocStorage';
 import {expandQuery} from './ExpandedQuery';
 import {GranularAccess, GranularAccessForBundle} from './GranularAccess';
 import {OnDemandActions} from './OnDemandActions';
+import {getLogMetaFromDocSession} from './serverUtils';
 import {findOrAddAllEnvelope, Sharing} from './Sharing';
 import cloneDeep = require('lodash/cloneDeep');
 import flatten = require('lodash/flatten');
@@ -195,15 +195,10 @@ export class ActiveDoc extends EventEmitter {
 
   // Constructs metadata for logging, given a Client or an OptDocSession.
   public getLogMeta(docSession: OptDocSession, docMethod?: string): log.ILogMeta {
-    const client = docSession.client;
-    const access = getDocSessionAccess(docSession);
-    const user = getDocSessionUser(docSession);
     return {
+      ...getLogMetaFromDocSession(docSession),
       docId: this._docName,
-      access,
       ...(docMethod ? {docMethod} : {}),
-      ...(user ? {userId: user.id, email: user.email} : {}),
-      ...(client ? client.getLogMeta() : {}),   // Client if present will repeat and add to user info.
     };
   }
 

app/server/lib/DocManager.ts

@@ -23,6 +23,7 @@ import * as docUtils from 'app/server/lib/docUtils';
 import {GristServer} from 'app/server/lib/GristServer';
 import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
 import {makeForkIds, makeId} from 'app/server/lib/idUtils';
+import {checkAllegedGristDoc} from 'app/server/lib/serverUtils';
 import * as log from 'app/server/lib/log';
 import {ActiveDoc} from './ActiveDoc';
 import {PluginManager} from './PluginManager';
@@ -476,7 +477,9 @@ export class DocManager extends EventEmitter {
         // security vulnerability. See https://phab.getgrist.com/T457.
         const docName = await this._createNewDoc(id);
         const docPath: string = this.storageManager.getPath(docName);
-        await docUtils.copyFile(uploadInfo.files[0].absPath, docPath);
+        const srcDocPath = uploadInfo.files[0].absPath;
+        await checkAllegedGristDoc(docSession, srcDocPath);
+        await docUtils.copyFile(srcDocPath, docPath);
         await this.storageManager.addToStorage(docName);
         return {title: basename, id: docName};
       } else {

app/server/lib/DocStorage.ts

@@ -677,7 +677,8 @@ export class DocStorage implements ISQLiteDB {
       // "PRAGMA journal_mode        = WAL;" +
       // "PRAGMA auto_vacuum         = 0;" +
       // "PRAGMA synchronous         = NORMAL"
-      "PRAGMA synchronous         = OFF;"
+      "PRAGMA synchronous         = OFF;" +
+      "PRAGMA trusted_schema      = OFF;"  // mitigation suggested by https://www.sqlite.org/security.html#untrusted_sqlite_database_files
     );
   }
 

app/server/lib/serverUtils.ts

@@ -1,8 +1,13 @@
 import * as bluebird from 'bluebird';
-import {ChildProcess} from 'child_process';
+import { ChildProcess } from 'child_process';
 import * as net from 'net';
 import * as path from 'path';
-import {ConnectionOptions} from 'typeorm';
+import { ConnectionOptions } from 'typeorm';
+import * as uuidv4 from 'uuid/v4';
+
+import * as log from 'app/server/lib/log';
+import { OpenMode, SQLiteDB } from 'app/server/lib/SQLiteDB';
+import { getDocSessionAccess, getDocSessionUser, OptDocSession } from './DocSession';
 
 /**
  * Promisify a node-style callback function. E.g.
@@ -114,3 +119,33 @@ export function getDatabaseUrl(options: ConnectionOptions, includeCredentials: b
     return `${options.type}://?`;
   }
 }
+
+/**
+ * Collect checks to be applied to incoming documents that are alleged to be
+ * Grist documents. For now, the only check is a sqlite-level integrity check,
+ * as suggested by https://www.sqlite.org/security.html#untrusted_sqlite_database_files
+ */
+export async function checkAllegedGristDoc(docSession: OptDocSession, fname: string) {
+  const db = await SQLiteDB.openDBRaw(fname, OpenMode.OPEN_READONLY);
+  const integrityCheckResults = await db.all('PRAGMA integrity_check');
+  if (integrityCheckResults.length !== 1 || integrityCheckResults[0].integrity_check !== 'ok') {
+    const uuid = uuidv4();
+    log.info('Integrity check failure on import', {uuid, integrityCheckResults,
+                                                   ...getLogMetaFromDocSession(docSession)});
+    throw new Error(`Document failed integrity checks - is it corrupted? Event ID: ${uuid}`);
+  }
+}
+
+/**
+ * Extract access, userId, email, and client (if applicable) from session, for logging purposes.
+ */
+export function getLogMetaFromDocSession(docSession: OptDocSession) {
+  const client = docSession.client;
+  const access = getDocSessionAccess(docSession);
+  const user = getDocSessionUser(docSession);
+  return {
+    access,
+    ...(user ? {userId: user.id, email: user.email} : {}),
+    ...(client ? client.getLogMeta() : {}),   // Client if present will repeat and add to user info.
+  };
+}