(core) move home server into core

Summary: This moves enough server material into core to run a home server.  The data engine is not yet incorporated (though in manual testing it works when ported).

Test Plan: existing tests pass

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2552

stubs/app/server/declarations.d.ts

@@ -0,0 +1,71 @@
+// Copy official sqlite3 types to apply to @gristlabs/sqlite3.
+declare module "@gristlabs/sqlite3" {
+  export * from 'sqlite3';
+
+  // Add minimal typings for sqlite backup api.
+  // TODO: remove this once the type definitions are updated upstream.
+  import {Database} from 'sqlite3';
+  export class Backup {
+    public readonly remaining: number;
+    public readonly pageCount: number;
+    public readonly idle: boolean;
+    public readonly completed: boolean;
+    public readonly failed: boolean;
+    public step(pages: number, callback?: (err: Error|null) => void): void;
+  }
+  export class DatabaseWithBackup extends Database {
+    public backup(filename: string, callback?: (err: Error|null) => void): Backup;
+    public backup(filename: string, destDbName: 'main', srcDbName: 'main',
+                  filenameIsDest: boolean, callback?: (err: Error|null) => void): Backup;
+  }
+}
+
+
+// Add declarations of the promisified methods of redis.
+// This is not exhaustive, there are a *lot* of methods.
+
+declare module "redis" {
+  function createClient(url?: string): RedisClient;
+
+  class RedisClient {
+    public eval(args: any[], callback?: (err: Error | null, res: any) => void): any;
+
+    public delAsync(key: string): Promise<'OK'>;
+    public flushdbAsync(): Promise<void>;
+    public getAsync(key: string): Promise<string|null>;
+    public hdelAsync(key: string, field: string): Promise<number>;
+    public hgetallAsync(key: string): Promise<{[field: string]: any}|null>;
+    public hkeysAsync(key: string): Promise<string[]|null>;
+    public hmsetAsync(key: string, val: {[field: string]: any}): Promise<'OK'>;
+    public hsetAsync(key: string, field: string, val: string): Promise<1|0>;
+    public keysAsync(pattern: string): Promise<string[]>;
+    public multi(): Multi;
+    public quitAsync(): Promise<void>;
+    public saddAsync(key: string, val: string): Promise<'OK'>;
+    public selectAsync(db: number): Promise<void>;
+    public setAsync(key: string, val: string): Promise<'OK'>;
+    public setexAsync(key: string, ttl: number, val: string): Promise<'OK'>;
+    public sismemberAsync(key: string, val: string): Promise<0|1>;
+    public smembersAsync(key: string): Promise<string[]>;
+    public srandmemberAsync(key: string): Promise<string|null>;
+    public sremAsync(key: string, val: string): Promise<'OK'>;
+    public ttlAsync(key: string): Promise<number|null>;
+    public unwatchAsync(): Promise<'OK'>;
+    public watchAsync(key: string): Promise<void>;
+  }
+
+  class Multi {
+    public del(key: string): Multi;
+    public execAsync(): Promise<any[]|null>;
+    public get(key: string): Multi;
+    public hgetall(key: string): Multi;
+    public hmset(key: string, val: {[field: string]: any}): Multi;
+    public hset(key: string, field: string, val: string): Multi;
+    public sadd(key: string, val: string): Multi;
+    public set(key: string, val: string): Multi;
+    public setex(key: string, ttl: number, val: string): Multi;
+    public smembers(key: string): Multi;
+    public srandmember(key: string): Multi;
+    public srem(key: string, val: string): Multi;
+  }
+}

stubs/app/server/lib/LoginSession.ts

@@ -0,0 +1,25 @@
+import {UserProfile} from 'app/common/LoginSessionAPI';
+import {Client} from 'app/server/lib/Client';
+import {ILoginSession} from 'app/server/lib/ILoginSession';
+
+export class LoginSession implements ILoginSession {
+  public clients: Set<Client> = new Set();
+  public async getEmail() {
+    return 'anon@getgrist.com';
+  }
+  public async clearSession(): Promise<void> {
+    // do nothing
+  }
+  public async testSetProfile(profile: UserProfile|null): Promise<void> {
+    // do nothing
+  }
+  public async updateTokenForTesting(idToken: string): Promise<void> {
+    // do nothing
+  }
+  public async getCurrentTokenForTesting(): Promise<string|null> {
+    return null;
+  }
+  public async useTestToken(idToken: string): Promise<void> {
+    // do nothing
+  }
+}

stubs/app/server/lib/StandaloneExtras.ts

@@ -0,0 +1,4 @@
+export function addStandaloneMethods(...args: any[]) {
+  console.log("Not adding any standalone methods");
+  return {} as any;
+}

stubs/app/server/lib/create.ts

@@ -0,0 +1,65 @@
+import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
+import { ActiveDoc } from 'app/server/lib/ActiveDoc';
+import { DocManager } from 'app/server/lib/DocManager';
+import { ICreate } from 'app/server/lib/ICreate';
+import { LoginSession } from 'app/server/lib/LoginSession';
+import { NSandbox } from 'app/server/lib/NSandbox';
+
+export const create: ICreate = {
+  LoginSession() {
+    return new LoginSession();
+  },
+  Billing(dbManager: HomeDBManager) {
+    return {
+      addEndpoints(app: any) { /* do nothing */ },
+      addEventHandlers() { /* do nothing */ },
+      addWebhooks(app: any) { /* do nothing */ }
+    };
+  },
+  Notifier(dbManager: HomeDBManager, homeUrl: string) {
+    return {
+      get testPending() { return false; }
+    };
+  },
+  Shell() {
+    return {
+      moveItemToTrash()  { throw new Error('moveToTrash unavailable'); },
+      showItemInFolder() { throw new Error('showItemInFolder unavailable'); }
+    };
+  },
+  ExternalStorage() { return undefined; },
+  ActiveDoc(docManager, docName) { return new ActiveDoc(docManager, docName); },
+  DocManager(storageManager, pluginManager, homeDBManager, gristServer) {
+    return new DocManager(storageManager, pluginManager, homeDBManager, gristServer);
+  },
+  NSandbox(options) {
+    const args = [options.entryPoint || 'grist/main.pyc'];
+    if (!options.entryPoint && options.comment) {
+      // Note that docName isn't used by main.py, but it makes it possible to tell in `ps` output
+      // which sandbox process is for which document.
+      args.push(options.comment);
+    }
+    const selLdrArgs: string[] = [];
+    if (options.sandboxMount) {
+      selLdrArgs.push(
+        // TODO: Only modules that we share with plugins should be mounted. They could be gathered in
+        // a "$APPROOT/sandbox/plugin" folder, only which get mounted.
+        '-E', 'PYTHONPATH=grist:thirdparty',
+        '-m', `${options.sandboxMount}:/sandbox:ro`);
+    }
+    if (options.importMount) {
+      selLdrArgs.push('-m', `${options.importMount}:/importdir:ro`);
+    }
+    return new NSandbox({
+      args,
+      logCalls: options.logCalls,
+      logMeta: options.logMeta,
+      logTimes: options.logTimes,
+      selLdrArgs,
+    });
+  },
+  sessionSecret() {
+    return process.env.GRIST_SESSION_SECRET ||
+      'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh';
+  }
+};

stubs/app/server/lib/logins.ts

@@ -0,0 +1,12 @@
+import {GristLoginMiddleware} from 'app/server/lib/GristServer';
+
+export async function getLoginMiddleware(): Promise<GristLoginMiddleware> {
+  return {
+    async getLoginRedirectUrl(target: URL)  { throw new Error('logins not implemented'); },
+    async getLogoutRedirectUrl(target: URL) { throw new Error('logins not implemented'); },
+    async getSignUpRedirectUrl(target: URL) { throw new Error('logins not implemented'); },
+    addEndpoints(...args: any[]) {
+      return "no-logins";
+    }
+  };
+}

stubs/app/server/server.ts

@@ -0,0 +1,25 @@
+import {updateDb} from 'app/server/lib/dbUtils';
+import {main as mergedServerMain} from 'app/server/mergedServerMain';
+import * as fse from 'fs-extra';
+
+const G = {
+  port: parseInt(process.env.PORT!, 10) || 8484,
+};
+
+export async function main() {
+  // Use a distinct cookie.
+  if (!process.env.GRIST_SESSION_COOKIE) {
+    process.env.GRIST_SESSION_COOKIE = 'grist_core';
+  }
+  // This is where documents are placed, for historic reasons.
+  await fse.mkdirp('samples');
+  // Make a blank db if needed.
+  await updateDb();
+  // Launch single-port, self-contained version of Grist.
+  // You probably want to have GRIST_DEFAULT_EMAIL set since there's no login system yet.
+  await mergedServerMain(G.port, ["home", "docs", "static"]);
+}
+
+if (require.main === module) {
+  main().catch((err) => console.error(err));
+}

stubs/app/server/tmp.d.ts

@@ -0,0 +1,7 @@
+// Add declarations of the promisifies methods of tmp.
+import {Options, SimpleOptions} from "tmp";
+declare module "tmp" {
+  function dirAsync(config?: Options): Promise<string>;
+  function fileAsync(config?: Options): Promise<string>;
+  function tmpNameAsync(config?: SimpleOptions): Promise<string>;
+}