(core) move home server into core

Summary: This moves enough server material into core to run a home server.  The data engine is not yet incorporated (though in manual testing it works when ported).

Test Plan: existing tests pass

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2552

app/gen-server/entity/AclRule.ts

@@ -0,0 +1,58 @@
+import {BaseEntity, ChildEntity, Column, Entity, JoinColumn, ManyToOne, OneToOne,
+        PrimaryGeneratedColumn, RelationId, TableInheritance} from "typeorm";
+
+import {Document} from "./Document";
+import {Group} from "./Group";
+import {Organization} from "./Organization";
+import {Workspace} from "./Workspace";
+
+@Entity('acl_rules')
+@TableInheritance({ column: { type: "int", name: "type" } })
+export class AclRule extends BaseEntity {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public permissions: number;
+
+  @OneToOne(type => Group, group => group.aclRule)
+  @JoinColumn({name: "group_id"})
+  public group: Group;
+}
+
+
+@ChildEntity()
+export class AclRuleWs extends AclRule {
+
+  @ManyToOne(type => Workspace, workspace => workspace.aclRules)
+  @JoinColumn({name: "workspace_id"})
+  public workspace: Workspace;
+
+  @RelationId((aclRule: AclRuleWs) => aclRule.workspace)
+  public workspaceId: number;
+}
+
+
+@ChildEntity()
+export class AclRuleOrg extends AclRule {
+
+  @ManyToOne(type => Organization, organization => organization.aclRules)
+  @JoinColumn({name: "org_id"})
+  public organization: Organization;
+
+  @RelationId((aclRule: AclRuleOrg) => aclRule.organization)
+  public orgId: number;
+}
+
+
+@ChildEntity()
+export class AclRuleDoc extends AclRule {
+
+  @ManyToOne(type => Document, document => document.aclRules)
+  @JoinColumn({name: "doc_id"})
+  public document: Document;
+
+  @RelationId((aclRule: AclRuleDoc) => aclRule.document)
+  public docId: number;
+}

app/gen-server/entity/Alias.ts

@@ -0,0 +1,27 @@
+import {BaseEntity, Column, CreateDateColumn, Entity, JoinColumn, ManyToOne,
+        PrimaryColumn} from 'typeorm';
+import {Document} from './Document';
+import {Organization} from './Organization';
+
+@Entity({name: 'aliases'})
+export class Alias extends BaseEntity {
+  @PrimaryColumn({name: 'org_id'})
+  public orgId: number;
+
+  @PrimaryColumn({name: 'url_id'})
+  public urlId: string;
+
+  @Column({name: 'doc_id'})
+  public docId: string;
+
+  @ManyToOne(type => Document)
+  @JoinColumn({name: 'doc_id'})
+  public doc: Document;
+
+  @ManyToOne(type => Organization)
+  @JoinColumn({name: 'org_id'})
+  public org: Organization;
+
+  @CreateDateColumn({name: 'created_at'})
+  public createdAt: Date;
+}

app/gen-server/entity/BillingAccount.ts

@@ -0,0 +1,65 @@
+import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn} from 'typeorm';
+import {BillingAccountManager} from 'app/gen-server/entity/BillingAccountManager';
+import {Organization} from 'app/gen-server/entity/Organization';
+import {Product} from 'app/gen-server/entity/Product';
+import {nativeValues} from 'app/gen-server/lib/values';
+
+// This type is for billing account status information.  Intended for stuff
+// like "free trial running out in N days".
+interface BillingAccountStatus {
+  stripeStatus?: string;
+  currentPeriodEnd?: Date;
+  message?: string;
+}
+
+/**
+ * This relates organizations to products.  It holds any stripe information
+ * needed to be able to update and pay for the product that applies to the
+ * organization.  It has a list of managers detailing which users have the
+ * right to view and edit these settings.
+ */
+@Entity({name: 'billing_accounts'})
+export class BillingAccount extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @ManyToOne(type => Product)
+  @JoinColumn({name: 'product_id'})
+  public product: Product;
+
+  @Column()
+  public individual: boolean;
+
+  // A flag for when all is well with the user's subscription.
+  // Probably shouldn't use this to drive whether service is provided or not.
+  // Strip recommends updating an end-of-service datetime every time payment
+  // is received, adding on a grace period of some days.
+  @Column({name: 'in_good_standing', default: nativeValues.trueValue})
+  public inGoodStanding: boolean;
+
+  @Column({type: nativeValues.jsonEntityType, nullable: true})
+  public status: BillingAccountStatus;
+
+  @Column({name: 'stripe_customer_id', type: String, nullable: true})
+  public stripeCustomerId: string | null;
+
+  @Column({name: 'stripe_subscription_id', type: String, nullable: true})
+  public stripeSubscriptionId: string | null;
+
+  @Column({name: 'stripe_plan_id', type: String, nullable: true})
+  public stripePlanId: string | null;
+
+  @OneToMany(type => BillingAccountManager, manager => manager.billingAccount)
+  public managers: BillingAccountManager[];
+
+  @OneToMany(type => Organization, org => org.billingAccount)
+  public orgs: Organization[];
+
+  // A calculated column that is true if it looks like there is a paid plan.
+  @Column({name: 'paid', type: 'boolean', insert: false, select: false})
+  public paid?: boolean;
+
+  // A calculated column summarizing whether active user is a manager of the billing account.
+  // (No @Column needed since calculation is done in javascript not sql)
+  public isManager?: boolean;
+}

app/gen-server/entity/BillingAccountManager.ts

@@ -0,0 +1,26 @@
+import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, PrimaryGeneratedColumn} from 'typeorm';
+import {BillingAccount} from 'app/gen-server/entity/BillingAccount';
+import {User} from 'app/gen-server/entity/User';
+
+/**
+ * A list of users with the right to modify a giving billing account.
+ */
+@Entity({name: 'billing_account_managers'})
+export class BillingAccountManager extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column({name: 'billing_account_id'})
+  public billingAccountId: number;
+
+  @ManyToOne(type => BillingAccount, { onDelete: 'CASCADE' })
+  @JoinColumn({name: 'billing_account_id'})
+  public billingAccount: BillingAccount;
+
+  @Column({name: 'user_id'})
+  public userId: number;
+
+  @ManyToOne(type => User, { onDelete: 'CASCADE' })
+  @JoinColumn({name: 'user_id'})
+  public user: User;
+}

app/gen-server/entity/Document.ts

@@ -0,0 +1,69 @@
+import {ApiError} from 'app/common/ApiError';
+import {Role} from 'app/common/roles';
+import {DocumentProperties, documentPropertyKeys, NEW_DOCUMENT_CODE} from "app/common/UserAPI";
+import {nativeValues} from 'app/gen-server/lib/values';
+import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryColumn} from "typeorm";
+import {AclRuleDoc} from "./AclRule";
+import {Alias} from "./Alias";
+import {Resource} from "./Resource";
+import {Workspace} from "./Workspace";
+
+// Acceptable ids for use in document urls.
+const urlIdRegex = /^[-a-z0-9]+$/i;
+
+function isValidUrlId(urlId: string) {
+  if (urlId === NEW_DOCUMENT_CODE) { return false; }
+  return urlIdRegex.exec(urlId);
+}
+
+@Entity({name: 'docs'})
+export class Document extends Resource {
+
+  @PrimaryColumn()
+  public id: string;
+
+  @ManyToOne(type => Workspace)
+  @JoinColumn({name: 'workspace_id'})
+  public workspace: Workspace;
+
+  @OneToMany(type => AclRuleDoc, aclRule => aclRule.document)
+  public aclRules: AclRuleDoc[];
+
+  // Indicates whether the doc is pinned to the org it lives in.
+  @Column({name: 'is_pinned', default: false})
+  public isPinned: boolean;
+
+  // Property that may be returned when the doc is fetched to indicate the access the
+  // fetching user has on the doc, i.e. 'owners', 'editors', 'viewers'
+  public access: Role|null;
+
+  // a computed column with permissions.
+  // {insert: false} makes sure typeorm doesn't try to put values into such
+  // a column when creating documents.
+  @Column({name: 'permissions', type: 'text', select: false, insert: false, update: false})
+  public permissions?: any;
+
+  @Column({name: 'url_id', type: 'text', nullable: true})
+  public urlId: string|null;
+
+  @Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true})
+  public removedAt: Date|null;
+
+  @OneToMany(type => Alias, alias => alias.doc)
+  public aliases: Alias[];
+
+  public checkProperties(props: any): props is Partial<DocumentProperties> {
+    return super.checkProperties(props, documentPropertyKeys);
+  }
+
+  public updateFromProperties(props: Partial<DocumentProperties>) {
+    super.updateFromProperties(props);
+    if (props.isPinned !== undefined) { this.isPinned = props.isPinned; }
+    if (props.urlId !== undefined) {
+      if (props.urlId !== null && !isValidUrlId(props.urlId)) {
+        throw new ApiError('invalid urlId', 400);
+      }
+      this.urlId = props.urlId;
+    }
+  }
+}

app/gen-server/entity/Group.ts

@@ -0,0 +1,33 @@
+import {BaseEntity, Column, Entity, JoinTable, ManyToMany, OneToOne, PrimaryGeneratedColumn} from "typeorm";
+
+import {AclRule} from "./AclRule";
+import {User} from "./User";
+
+@Entity({name: 'groups'})
+export class Group extends BaseEntity {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public name: string;
+
+  @ManyToMany(type => User)
+  @JoinTable({
+    name: 'group_users',
+    joinColumn: {name: 'group_id'},
+    inverseJoinColumn: {name: 'user_id'}
+  })
+  public memberUsers: User[];
+
+  @ManyToMany(type => Group)
+  @JoinTable({
+    name: 'group_groups',
+    joinColumn: {name: 'group_id'},
+    inverseJoinColumn: {name: 'subgroup_id'}
+  })
+  public memberGroups: Group[];
+
+  @OneToOne(type => AclRule, aclRule => aclRule.group)
+  public aclRule: AclRule;
+}

app/gen-server/entity/Login.ts

@@ -0,0 +1,25 @@
+import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, PrimaryColumn} from "typeorm";
+
+import {User} from "./User";
+
+@Entity({name: 'logins'})
+export class Login extends BaseEntity {
+
+  @PrimaryColumn()
+  public id: number;
+
+  // This is the normalized email address we use for equality and indexing.
+  @Column()
+  public email: string;
+
+  // This is how the user's email address should be displayed.
+  @Column({name: 'display_email'})
+  public displayEmail: string;
+
+  @Column({name: 'user_id'})
+  public userId: number;
+
+  @ManyToOne(type => User)
+  @JoinColumn({name: 'user_id'})
+  public user: User;
+}

app/gen-server/entity/Organization.ts

@@ -0,0 +1,79 @@
+import {Column, Entity, JoinColumn, ManyToOne, OneToMany, OneToOne,
+        PrimaryGeneratedColumn, RelationId} from "typeorm";
+import {Role} from "app/common/roles";
+import {OrganizationProperties, organizationPropertyKeys} from "app/common/UserAPI";
+import {AclRuleOrg} from "./AclRule";
+import {BillingAccount} from "./BillingAccount";
+import {Resource} from "./Resource";
+import {User} from "./User";
+import {Workspace} from "./Workspace";
+
+// Information about how an organization may be accessed.
+export interface AccessOption {
+  id: number;     // a user id
+  email: string;  // a user email
+  name: string;   // a user name
+  perms: number;  // permissions the user would have on organization
+}
+
+export interface AccessOptionWithRole extends AccessOption {
+  access: Role;   // summary of permissions
+}
+
+@Entity({name: 'orgs'})
+export class Organization extends Resource {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column({
+    nullable: true
+  })
+  public domain: string;
+
+  @OneToOne(type => User, user => user.personalOrg)
+  @JoinColumn({name: 'owner_id'})
+  public owner: User;
+
+  @RelationId((org: Organization) => org.owner)
+  public ownerId: number;
+
+  @OneToMany(type => Workspace, workspace => workspace.org)
+  public workspaces: Workspace[];
+
+  @OneToMany(type => AclRuleOrg, aclRule => aclRule.organization)
+  public aclRules: AclRuleOrg[];
+
+  @Column({name: 'billing_account_id'})
+  public billingAccountId: number;
+
+  @ManyToOne(type => BillingAccount)
+  @JoinColumn({name: 'billing_account_id'})
+  public billingAccount: BillingAccount;
+
+  // Property that may be returned when the org is fetched to indicate the access the
+  // fetching user has on the org, i.e. 'owners', 'editors', 'viewers'
+  public access: string;
+
+  // Property that may be used internally to track multiple ways an org can be accessed
+  public accessOptions?: AccessOptionWithRole[];
+
+  // a computed column with permissions.
+  // {insert: false} makes sure typeorm doesn't try to put values into such
+  // a column when creating organizations.
+  @Column({name: 'permissions', type: 'text', select: false, insert: false})
+  public permissions?: any;
+
+  // For custom domains, this is the preferred host associated with this org/team.
+  @Column({name: 'host', type: 'text', nullable: true})
+  public host: string|null;
+
+  public checkProperties(props: any): props is Partial<OrganizationProperties> {
+    return super.checkProperties(props, organizationPropertyKeys);
+  }
+
+  public updateFromProperties(props: Partial<OrganizationProperties>) {
+    super.updateFromProperties(props);
+    if (props.domain) { this.domain = props.domain; }
+  }
+}

app/gen-server/entity/Product.ts

@@ -0,0 +1,176 @@
+import {Features} from 'app/common/Features';
+import {nativeValues} from 'app/gen-server/lib/values';
+import * as assert from 'assert';
+import {BaseEntity, Column, Connection, Entity, PrimaryGeneratedColumn} from 'typeorm';
+
+/**
+ * A summary of features used in 'starter' plans.
+ */
+export const starterFeatures: Features = {
+  workspaces: true,
+  // no vanity domain
+  maxDocsPerOrg: 10,
+  maxSharesPerDoc: 2,
+  maxWorkspacesPerOrg: 1
+};
+
+/**
+ * A summary of features used in 'team' plans.
+ */
+export const teamFeatures: Features = {
+  workspaces: true,
+  vanityDomain: true,
+  maxSharesPerWorkspace: 0,   // all workspace shares need to be org members.
+  maxSharesPerDoc: 2
+};
+
+/**
+ * A summary of features used in unrestricted grandfathered accounts, and also
+ * in some test settings.
+ */
+export const grandfatherFeatures: Features = {
+  workspaces: true,
+  vanityDomain: true,
+};
+
+export const suspendedFeatures: Features = {
+  workspaces: true,
+  vanityDomain: true,
+  readOnlyDocs: true,
+  // clamp down on new docs/workspaces/shares
+  maxDocsPerOrg: 0,
+  maxSharesPerDoc: 0,
+  maxWorkspacesPerOrg: 0,
+};
+
+/**
+ * Basic fields needed for products supported by Grist.
+ */
+export interface IProduct {
+  name: string;
+  features: Features;
+}
+
+/**
+ *
+ * Products are a bundle of enabled features.  Most products in
+ * Grist correspond to products in stripe.  The correspondence is
+ * established by a gristProduct metadata field on stripe plans.
+ *
+ * In addition, there are the following products in Grist that don't
+ * exist in stripe:
+ *   - The product named 'Free'.  This is a product used for organizations
+ *     created prior to the billing system being set up.
+ *   - The product named 'stub'.  This is product assigned to new
+ *     organizations that should not be usable until a paid plan
+ *     is set up for them.
+ *
+ * TODO: change capitalization of name of grandfather product.
+ *
+ */
+const PRODUCTS: IProduct[] = [
+  // This is a product for grandfathered accounts/orgs.
+  {
+    name: 'Free',
+    features: grandfatherFeatures,
+  },
+
+  // This is a product for newly created accounts/orgs.
+  {
+    name: 'stub',
+    features: {},
+  },
+
+  // These are products set up in stripe.
+  {
+    name: 'starter',
+    features: starterFeatures,
+  },
+  {
+    name: 'professional',  // deprecated, can be removed once no longer referred to in stripe.
+    features: teamFeatures,
+  },
+  {
+    name: 'team',
+    features: teamFeatures,
+  },
+
+  // This is a product for a team site that is no longer in good standing, but isn't yet
+  // to be removed / deactivated entirely.
+  {
+    name: 'suspended',
+    features: suspendedFeatures,
+  },
+];
+
+/**
+ * Get names of products for different situations.
+ */
+export function getDefaultProductNames() {
+  return {
+    personal: 'starter',  // Personal site start off on a functional plan.
+    teamInitial: 'stub',  // Team site starts off on a limited plan, requiring subscription.
+    team: 'team',         // Functional team site
+  };
+}
+
+/**
+ * A Grist product.  Corresponds to a set of enabled features and a choice of limits.
+ */
+@Entity({name: 'products'})
+export class Product extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public name: string;
+
+  @Column({type: nativeValues.jsonEntityType})
+  public features: Features;
+}
+
+/**
+ * Make sure the products defined for the current stripe setup are
+ * in the database and up to date.  Other products in the database
+ * are untouched.
+ *
+ * If `apply` is set, the products are changed in the db, otherwise
+ * the are left unchanged.  A summary of affected products is returned.
+ */
+export async function synchronizeProducts(connection: Connection, apply: boolean): Promise<string[]> {
+  try {
+    await connection.query('select name, features, stripe_product_id from products limit 1');
+  } catch (e) {
+    // No usable products table, do not try to synchronize.
+    return [];
+  }
+  const changingProducts: string[] = [];
+  await connection.transaction(async transaction => {
+    const desiredProducts = new Map(PRODUCTS.map(p => [p.name, p]));
+    const existingProducts = new Map((await transaction.find(Product))
+                                     .map(p => [p.name, p]));
+    for (const product of desiredProducts.values()) {
+      if (existingProducts.has(product.name)) {
+        const p = existingProducts.get(product.name)!;
+        try {
+          assert.deepStrictEqual(p.features, product.features);
+        } catch (e) {
+          if (apply) {
+            p.features = product.features;
+            await transaction.save(p);
+          }
+          changingProducts.push(p.name);
+        }
+      } else {
+        if (apply) {
+          const p = new Product();
+          p.name = product.name;
+          p.features = product.features;
+          await transaction.save(p);
+        }
+        changingProducts.push(product.name);
+      }
+    }
+  });
+  return changingProducts;
+}

app/gen-server/entity/Resource.ts

@@ -0,0 +1,46 @@
+import {BaseEntity, Column} from "typeorm";
+import {ApiError} from 'app/common/ApiError';
+import {CommonProperties} from "app/common/UserAPI";
+
+export class Resource extends BaseEntity {
+  @Column()
+  public name: string;
+
+  @Column({name: 'created_at', default: () => "CURRENT_TIMESTAMP"})
+  public createdAt: Date;
+
+  @Column({name: 'updated_at', default: () => "CURRENT_TIMESTAMP"})
+  public updatedAt: Date;
+
+  // a computed column which, when present, means the entity should be filtered out
+  // of results.
+  @Column({name: 'filtered_out', type: 'boolean', select: false, insert: false})
+  public filteredOut?: boolean;
+
+  public updateFromProperties(props: Partial<CommonProperties>) {
+    if (props.createdAt) { this.createdAt = _propertyToDate(props.createdAt); }
+    if (props.updatedAt) {
+      this.updatedAt = _propertyToDate(props.updatedAt);
+    } else {
+      this.updatedAt = new Date();
+    }
+    if (props.name) { this.name = props.name; }
+  }
+
+  protected checkProperties(props: any, keys: string[]): props is Partial<CommonProperties> {
+    for (const key of Object.keys(props)) {
+      if (!keys.includes(key)) {
+        throw new ApiError(`unrecognized property ${key}`, 400);
+      }
+    }
+    return true;
+  }
+}
+
+// Ensure iso-string-or-date value is converted to a date.
+function _propertyToDate(d: string|Date): Date {
+  if (typeof(d) === 'string') {
+    return new Date(d);
+  }
+  return d;
+}

app/gen-server/entity/User.ts

@@ -0,0 +1,54 @@
+import {BaseEntity, Column, Entity, JoinTable, ManyToMany, OneToMany, OneToOne,
+        PrimaryGeneratedColumn} from "typeorm";
+
+import {Group} from "./Group";
+import {Login} from "./Login";
+import {Organization} from "./Organization";
+
+@Entity({name: 'users'})
+export class User extends BaseEntity {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public name: string;
+
+  @Column({name: 'api_key', type: String, nullable: true})
+  // Found how to make a type nullable in this discussion: https://github.com/typeorm/typeorm/issues/2567
+  // todo: adds constraint for api_key not to equal ''
+  public apiKey: string | null;
+
+  @Column({name: 'picture', type: String, nullable: true})
+  public picture: string | null;
+
+  @Column({name: 'first_login_at', type: Date, nullable: true})
+  public firstLoginAt: Date | null;
+
+  @OneToOne(type => Organization, organization => organization.owner)
+  public personalOrg: Organization;
+
+  @OneToMany(type => Login, login => login.user)
+  public logins: Login[];
+
+  @ManyToMany(type => Group)
+  @JoinTable({
+    name: 'group_users',
+    joinColumn: {name: 'user_id'},
+    inverseJoinColumn: {name: 'group_id'}
+  })
+  public groups: Group[];
+
+  @Column({name: 'is_first_time_user', default: false})
+  public isFirstTimeUser: boolean;
+
+  /**
+   * Get user's email.  Returns undefined if logins has not been joined, or no login
+   * is available
+   */
+  public get loginEmail(): string|undefined {
+    const login = this.logins && this.logins[0];
+    if (!login) { return undefined; }
+    return login.email;
+  }
+}

app/gen-server/entity/Workspace.ts

@@ -0,0 +1,49 @@
+import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn} from "typeorm";
+import {WorkspaceProperties, workspacePropertyKeys} from "app/common/UserAPI";
+import {nativeValues} from 'app/gen-server/lib/values';
+import {AclRuleWs} from "./AclRule";
+import {Document} from "./Document";
+import {Organization} from "./Organization";
+import {Resource} from "./Resource";
+
+@Entity({name: 'workspaces'})
+export class Workspace extends Resource {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @ManyToOne(type => Organization)
+  @JoinColumn({name: 'org_id'})
+  public org: Organization;
+
+  @OneToMany(type => Document, document => document.workspace)
+  public docs: Document[];
+
+  @OneToMany(type => AclRuleWs, aclRule => aclRule.workspace)
+  public aclRules: AclRuleWs[];
+
+  // Property that may be returned when the workspace is fetched to indicate the access the
+  // fetching user has on the workspace, i.e. 'owners', 'editors', 'viewers'
+  public access: string;
+
+  // A computed column that is true if the workspace is a support workspace.
+  @Column({name: 'support', type: 'boolean', insert: false, select: false})
+  public isSupportWorkspace?: boolean;
+
+  // a computed column with permissions.
+  // {insert: false} makes sure typeorm doesn't try to put values into such
+  // a column when creating workspaces.
+  @Column({name: 'permissions', type: 'text', select: false, insert: false})
+  public permissions?: any;
+
+  @Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true})
+  public removedAt: Date|null;
+
+  public checkProperties(props: any): props is Partial<WorkspaceProperties> {
+    return super.checkProperties(props, workspacePropertyKeys);
+  }
+
+  public updateFromProperties(props: Partial<WorkspaceProperties>) {
+    super.updateFromProperties(props);
+  }
+}