(core) Persist forks in home db

Summary:
Adds information about forks to the home db. This will be used
later by the UI to list forks of documents.

Test Plan: Browser and server tests.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3772

app/common/ActiveDocAPI.ts

@@ -169,6 +169,7 @@ export interface QueryResult extends TableFetchResult {
  * docId of XXXXX~FORKID[~USERID] and a urlId of UUUUU~FORKID[~USERID].
  */
 export interface ForkResult {
+  forkId: string;
   docId: string;
   urlId: string;
 }

app/common/UserAPI.ts

@@ -1,5 +1,5 @@
 import {ActionSummary} from 'app/common/ActionSummary';
-import {ApplyUAResult, PermissionDataWithExtraUsers, QueryFilters} from 'app/common/ActiveDocAPI';
+import {ApplyUAResult, ForkResult, PermissionDataWithExtraUsers, QueryFilters} from 'app/common/ActiveDocAPI';
 import {BaseAPI, IOptions} from 'app/common/BaseAPI';
 import {BillingAPI, BillingAPIImpl} from 'app/common/BillingAPI';
 import {BrowserSettings} from 'app/common/BrowserSettings';
@@ -108,6 +108,8 @@ export interface Workspace extends WorkspaceProperties {
   isSupportWorkspace?: boolean;
 }
 
+export type DocumentType = 'tutorial';
+
 // Non-core options for a document.
 // "Non-core" means bundled into a single options column in the database.
 // TODO: consider smoothing over this distinction in the API.
@@ -120,6 +122,8 @@ export interface DocumentOptions {
 export interface DocumentProperties extends CommonProperties {
   isPinned: boolean;
   urlId: string|null;
+  trunkId: string|null;
+  type: DocumentType|null;
   options: DocumentOptions|null;
 }
 
@@ -130,6 +134,13 @@ export interface Document extends DocumentProperties {
   workspace: Workspace;
   access: roles.Role;
   trunkAccess?: roles.Role|null;
+  forks?: Fork[];
+}
+
+export interface Fork {
+  id: string;
+  trunkId: string;
+  updatedAt: string;  // ISO date string
 }
 
 // Non-core options for a user.
@@ -385,6 +396,7 @@ export interface DocAPI {
   updateRows(tableId: string, changes: TableColValues): Promise<number[]>;
   addRows(tableId: string, additions: BulkColValues): Promise<number[]>;
   removeRows(tableId: string, removals: number[]): Promise<number[]>;
+  fork(): Promise<ForkResult>;
   replace(source: DocReplacementOptions): Promise<void>;
   // Get list of document versions (specify raw to bypass caching, which should only make
   // a difference if snapshots have "leaked")
@@ -837,6 +849,12 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
     });
   }
 
+  public async fork(): Promise<ForkResult> {
+    return this.requestJson(`${this._url}/fork`, {
+      method: 'POST'
+    });
+  }
+
   public async replace(source: DocReplacementOptions): Promise<void> {
     return this.requestJson(`${this._url}/replace`, {
       body: JSON.stringify(source),

app/gen-server/entity/Document.ts

@@ -1,7 +1,8 @@
 import {ApiError} from 'app/common/ApiError';
 import {DocumentUsage} from 'app/common/DocUsage';
 import {Role} from 'app/common/roles';
-import {DocumentOptions, DocumentProperties, documentPropertyKeys, NEW_DOCUMENT_CODE} from "app/common/UserAPI";
+import {DocumentOptions, DocumentProperties, documentPropertyKeys,
+        DocumentType, NEW_DOCUMENT_CODE} from "app/common/UserAPI";
 import {nativeValues} from 'app/gen-server/lib/values';
 import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryColumn} from "typeorm";
 import {AclRuleDoc} from "./AclRule";
@@ -69,6 +70,25 @@ export class Document extends Resource {
   @Column({name: 'usage', type: nativeValues.jsonEntityType, nullable: true})
   public usage: DocumentUsage | null;
 
+  @Column({name: 'created_by', type: 'integer', nullable: true})
+  public createdBy: number|null;
+
+  @Column({name: 'trunk_id', type: 'text', nullable: true})
+  public trunkId: string|null;
+
+  // Property set for forks, containing the URL ID of the trunk.
+  public trunkUrlId?: string|null;
+
+  @ManyToOne(_type => Document, document => document.forks)
+  @JoinColumn({name: 'trunk_id'})
+  public trunk: Document|null;
+
+  @OneToMany(_type => Document, document => document.trunk)
+  public forks: Document[];
+
+  @Column({name: 'type', type: 'text', nullable: true})
+  public type: DocumentType|null;
+
   public checkProperties(props: any): props is Partial<DocumentProperties> {
     return super.checkProperties(props, documentPropertyKeys);
   }
@@ -82,6 +102,7 @@ export class Document extends Resource {
       }
       this.urlId = props.urlId;
     }
+    if (props.type !== undefined) { this.type = props.type; }
     if (props.options !== undefined) {
       // Options are merged over the existing state - unless options
       // object is set to "null", in which case the state is wiped

app/gen-server/lib/HomeDBManager.ts

@@ -1213,6 +1213,9 @@ export class HomeDBManager extends EventEmitter {
       (doc.workspace as any).owner = doc.workspace.org.owner;
     }
     if (forkId || snapshotId) {
+      doc.trunkId = doc.id;
+      doc.trunkUrlId = doc.urlId;
+
       // Fix up our reply to be correct for the fork, rather than the trunk.
       // The "id" and "urlId" fields need updating.
       doc.id = buildUrlId({trunkId: doc.id, forkId, forkUserId, snapshotId});
@@ -1294,6 +1297,20 @@ export class HomeDBManager extends EventEmitter {
     return this._single(await this._verifyAclPermissions(qb));
   }
 
+  /**
+   * Gets a list of all forks whose trunk is `docId`.
+   *
+   * NOTE: This is not a part of the API. It should only be called by the DocApi when
+   * deleting a document.
+   */
+  public async getDocForks(docId: string): Promise<Document[]> {
+    return this._connection.createQueryBuilder()
+      .select('forks')
+      .from(Document, 'forks')
+      .where('forks.trunk_id = :docId', {docId})
+      .getMany();
+  }
+
   /**
    *
    * Adds an org with the given name. Returns a query result with the id of the added org.
@@ -1778,6 +1795,7 @@ export class HomeDBManager extends EventEmitter {
         doc.aliases = [];
       }
       doc.workspace = workspace;
+      doc.createdBy = scope.userId;
       // Create the special initial permission groups for the new workspace.
       const groupMap = this._createGroups(workspace, scope.userId);
       doc.aclRules = this.defaultCommonGroups.map(_grpDesc => {
@@ -1872,7 +1890,7 @@ export class HomeDBManager extends EventEmitter {
 
       const queryResult = await verifyIsPermitted(docQuery);
       if (queryResult.status !== 200) {
-        // If the query for the workspace failed, return the failure result.
+        // If the query for the doc failed, return the failure result.
         return queryResult;
       }
       // Update the name and save.
@@ -1945,6 +1963,30 @@ export class HomeDBManager extends EventEmitter {
     return this._setDocumentRemovedAt(scope, null);
   }
 
+  /**
+   * Like `deleteDocument`, but for deleting a fork.
+   *
+   * NOTE: This is not a part of the API. It should only be called by the DocApi when
+   * deleting a fork.
+   */
+  public async deleteFork(scope: DocScope): Promise<QueryResult<number>> {
+    return await this._connection.transaction(async manager => {
+      const forkQuery = this._doc(scope, {
+        manager,
+        allowSpecialPermit: true
+      });
+      const result = await forkQuery.getRawAndEntities();
+      if (result.entities.length === 0) {
+        return {
+          status: 404,
+          errMessage: 'fork not found'
+        };
+      }
+      await manager.remove(result.entities[0]);
+      return {status: 200};
+    });
+  }
+
   // Fetches and provides a callback with the billingAccount so it may be updated within
   // a transaction. The billingAccount is saved after any changes applied in the callback.
   // Will throw an error if the user does not have access to the org's billingAccount.
@@ -2540,6 +2582,31 @@ export class HomeDBManager extends EventEmitter {
     });
   }
 
+  /**
+   * Creates a fork of `doc`, using the specified `forkId`.
+   *
+   * NOTE: This is not a part of the API. It should only be called by the ActiveDoc when
+   * a new fork is initiated.
+   */
+  public async forkDoc(
+    userId: number,
+    doc: Document,
+    forkId: string,
+  ): Promise<QueryResult<string>> {
+    return await this._connection.transaction(async manager => {
+      const fork = new Document();
+      fork.id = forkId;
+      fork.name = doc.name;
+      fork.createdBy = userId;
+      fork.trunkId = doc.trunkId || doc.id;
+      const result = await manager.save([fork]);
+      return {
+        status: 200,
+        data: result[0].id,
+      };
+    });
+  }
+
   /**
    * Updates the updatedAt and usage values for several docs. Takes a map where each entry maps
    * a docId to a metadata object containing the updatedAt and/or usage values. This is not a part
@@ -2803,6 +2870,9 @@ export class HomeDBManager extends EventEmitter {
     let query = this.org(scope, org, options)
       .leftJoinAndSelect('orgs.workspaces', 'workspaces')
       .leftJoinAndSelect('workspaces.docs', 'docs', this._onDoc(scope))
+      .leftJoin('docs.forks', 'forks', this._onFork())
+      .addSelect(['forks.id', 'forks.trunkId', 'forks.createdBy', 'forks.updatedAt'])
+      .setParameter('anonId', this.getAnonymousUserId())
       .leftJoin('orgs.billingAccount', 'account')
       .leftJoin('account.product', 'product')
       .addSelect('product.features')
@@ -3421,6 +3491,13 @@ export class HomeDBManager extends EventEmitter {
     }
   }
 
+  /**
+   * Like _onDoc, but for joining forks.
+   */
+  private _onFork() {
+    return 'forks.created_by = :userId AND forks.created_by <> :anonId';
+  }
+
   /**
    * Construct a QueryBuilder for a select query on a specific workspace given by
    * wsId. Provides options for running in a transaction and adding permission info.

app/gen-server/lib/Housekeeper.ts

@@ -1,4 +1,5 @@
 import { ApiError } from 'app/common/ApiError';
+import { buildUrlId } from 'app/common/gristUrls';
 import { Document } from 'app/gen-server/entity/Document';
 import { Workspace } from 'app/gen-server/entity/Workspace';
 import { HomeDBManager, Scope } from 'app/gen-server/lib/HomeDBManager';
@@ -119,6 +120,29 @@ export class Housekeeper {
       };
       await this._dbManager.deleteWorkspace(scope, workspace.id);
     }
+
+    // Delete old forks
+    const forks = await this._getForksToDelete();
+    for (const fork of forks) {
+      const docId = buildUrlId({trunkId: fork.trunkId!, forkId: fork.id, forkUserId: fork.createdBy!});
+      const permitKey = await this._permitStore.setPermit({docId});
+      try {
+        const result = await fetch(
+          await this._server.getHomeUrlByDocId(docId, `/api/docs/${docId}`),
+          {
+            method: 'DELETE',
+            headers: {
+              Permit: permitKey,
+            },
+          }
+        );
+        if (result.status !== 200) {
+          log.error(`failed to delete fork ${docId}: error status ${result.status}`);
+        }
+      } finally {
+        await this._permitStore.removePermit(permitKey);
+      }
+    }
   }
 
   public addEndpoints(app: express.Application) {
@@ -202,7 +226,7 @@ export class Housekeeper {
   }
 
   private async _getWorkspacesToDelete() {
-    const docs = await this._dbManager.connection.createQueryBuilder()
+    const workspaces = await this._dbManager.connection.createQueryBuilder()
       .select('workspaces')
       .from(Workspace, 'workspaces')
       .leftJoin('workspaces.docs', 'docs')
@@ -212,7 +236,17 @@ export class Housekeeper {
       // wait for workspace to be empty
       .andWhere('docs.id IS NULL')
       .getMany();
-    return docs;
+    return workspaces;
+  }
+
+  private async _getForksToDelete() {
+    const forks = await this._dbManager.connection.createQueryBuilder()
+      .select('forks')
+      .from(Document, 'forks')
+      .where('forks.trunk_id IS NOT NULL')
+      .andWhere(`forks.updated_at <= ${this._getThreshold()}`)
+      .getMany();
+    return forks;
   }
 
   /**

app/gen-server/migration/1673051005072-Forks.ts

@@ -0,0 +1,42 @@
+import {MigrationInterface, QueryRunner, TableColumn, TableForeignKey} from "typeorm";
+
+export class Forks1673051005072 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.addColumns("docs", [
+      new TableColumn({
+        name: "created_by",
+        type: "integer",
+        isNullable: true,
+      }),
+      new TableColumn({
+        name: "trunk_id",
+        type: "text",
+        isNullable: true,
+      }),
+      new TableColumn({
+        name: "type",
+        type: "text",
+        isNullable: true,
+      }),
+    ]);
+
+    await queryRunner.createForeignKeys("docs", [
+      new TableForeignKey({
+        columnNames: ["created_by"],
+        referencedTableName: "users",
+        referencedColumnNames: ["id"],
+        onDelete: "CASCADE",
+      }),
+      new TableForeignKey({
+        columnNames: ["trunk_id"],
+        referencedTableName: "docs",
+        referencedColumnNames: ["id"],
+        onDelete: "CASCADE",
+      }),
+    ]);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.dropColumns("docs", ["created_by", "trunk_id", "type"]);
+  }
+}

app/server/lib/ActiveDoc.ts

@@ -78,6 +78,7 @@ import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/Us
 import {convertFromColumn} from 'app/common/ValueConverter';
 import {guessColInfoWithDocData} from 'app/common/ValueGuesser';
 import {parseUserAction} from 'app/common/ValueParser';
+import {Document} from 'app/gen-server/entity/Document';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
 import {AccessTokenOptions, AccessTokenResult, GristDocAPI} from 'app/plugin/GristAPI';
 import {compileAclFormula} from 'app/server/lib/ACLFormula';
@@ -1375,10 +1376,16 @@ export class ActiveDoc extends EventEmitter {
   }
 
   /**
-   * Fork the current document.  In fact, all that requires is calculating a good
-   * ID for the fork.  TODO: reconcile the two ways there are now of preparing a fork.
+   * Fork the current document.
+   *
+   * TODO: reconcile the two ways there are now of preparing a fork.
    */
   public async fork(docSession: OptDocSession): Promise<ForkResult> {
+    const dbManager = this.getHomeDbManager();
+    if (!dbManager) {
+      throw new Error('HomeDbManager not available');
+    }
+
     const user = getDocSessionUser(docSession);
     // For now, fork only if user can read everything (or is owner).
     // TODO: allow forks with partial content.
@@ -1387,9 +1394,19 @@ export class ActiveDoc extends EventEmitter {
     }
     const userId = user.id;
     const isAnonymous = this._docManager.isAnonymous(userId);
+
     // Get fresh document metadata (the cached metadata doesn't include the urlId).
-    const doc = await docSession.authorizer?.getDoc();
-    if (!doc) { throw new Error('document id not known'); }
+    let doc: Document | undefined;
+    if (docSession.authorizer) {
+      doc = await docSession.authorizer.getDoc();
+    } else if (docSession.req) {
+      doc = await this.getHomeDbManager()?.getDoc(docSession.req);
+    }
+    if (!doc) { throw new Error('Document not found'); }
+
+    // Don't allow creating forks of forks (for now).
+    if (doc.trunkId) { throw new ApiError("Cannot fork a document that's already a fork", 400); }
+
     const trunkDocId = doc.id;
     const trunkUrlId = doc.urlId || doc.id;
     await this.flushDoc();  // Make sure fork won't be too out of date.
@@ -1415,6 +1432,8 @@ export class ActiveDoc extends EventEmitter {
       if (resp.status !== 200) {
         throw new ApiError(resp.statusText, resp.status);
       }
+
+      await dbManager.forkDoc(userId, doc, forkIds.forkId);
     } finally {
       await permitStore.removePermit(permitKey);
     }

app/server/lib/DocApi.ts

@@ -3,13 +3,13 @@ import {ApiError} from 'app/common/ApiError';
 import {BrowserSettings} from "app/common/BrowserSettings";
 import {BulkColValues, ColValues, fromTableDataAction, TableColValues, TableRecordValue} from 'app/common/DocActions';
 import {isRaisedException} from "app/common/gristTypes";
-import {parseUrlId} from "app/common/gristUrls";
+import {buildUrlId, parseUrlId} from "app/common/gristUrls";
 import {isAffirmative} from "app/common/gutil";
 import {SortFunc} from 'app/common/SortFunc';
 import {Sort} from 'app/common/SortSpec';
 import {MetaRowRecord} from 'app/common/TableData';
 import {DocReplacementOptions, DocState, DocStateComparison, DocStates, NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
-import {HomeDBManager, makeDocAuthResult} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager, makeDocAuthResult, QueryResult} from 'app/gen-server/lib/HomeDBManager';
 import * as Types from "app/plugin/DocApiTypes";
 import DocApiTypesTI from "app/plugin/DocApiTypes-ti";
 import GristDataTI from 'app/plugin/GristData-ti';
@@ -1181,12 +1181,26 @@ export class DocWorkerApi {
     const scope = getDocScope(req);
     const docId = getDocId(req);
     if (permanent) {
-      // Soft delete the doc first, to de-list the document.
-      await this._dbManager.softDeleteDocument(scope);
-      // Delete document content from storage.
-      await this._docManager.deleteDoc(null, docId, true);
+      const {forkId} = parseUrlId(docId);
+      if (!forkId) {
+        // Soft delete the doc first, to de-list the document.
+        await this._dbManager.softDeleteDocument(scope);
+      }
+      // Delete document content from storage. Include forks if doc is a trunk.
+      const forks = forkId ? [] : await this._dbManager.getDocForks(docId);
+      const docsToDelete = [
+        docId,
+        ...forks.map((fork) =>
+          buildUrlId({forkId: fork.id, forkUserId: fork.createdBy!, trunkId: docId})),
+      ];
+      await Promise.all(docsToDelete.map(docName => this._docManager.deleteDoc(null, docName, true)));
       // Permanently delete from database.
-      const query = await this._dbManager.deleteDocument(scope);
+      let query: QueryResult<number>;
+      if (forkId) {
+        query = await this._dbManager.deleteFork({...scope, urlId: forkId});
+      } else {
+        query = await this._dbManager.deleteDocument(scope);
+      }
       this._dbManager.checkQueryResult(query);
       await sendReply(req, res, query);
     } else {

app/server/lib/HostedStorageManager.ts

@@ -556,9 +556,14 @@ export class HostedStorageManager implements IDocStorageManager {
    * This is called when a document was edited by the user.
    */
   private _markAsEdited(docName: string, timestamp: string): void {
-    if (parseUrlId(docName).snapshotId || !this._metadataManager) { return; }
+    if (!this._metadataManager) { return; }
+
+    const {forkId, snapshotId} = parseUrlId(docName);
+    if (snapshotId) { return; }
+
     // Schedule a metadata update for the modified doc.
-    this._metadataManager.scheduleUpdate(docName, {updatedAt: timestamp});
+    const docId = forkId || docName;
+    this._metadataManager.scheduleUpdate(docId, {updatedAt: timestamp});
   }
 
   /**

app/server/lib/idUtils.ts

@@ -30,6 +30,7 @@ export function makeForkIds(options: { userId: number|null, isAnonymous: boolean
   const docId = parseUrlId(options.trunkDocId).trunkId;
   const urlId = parseUrlId(options.trunkUrlId).trunkId;
   return {
+    forkId,
     docId: buildUrlId({trunkId: docId, forkId, forkUserId}),
     urlId: buildUrlId({trunkId: urlId, forkId, forkUserId}),
   };

app/server/lib/requestUtils.ts

@@ -22,7 +22,7 @@ export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ?
 const INTERNAL_FIELDS = new Set([
   'apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId', 'gracePeriodStart', 'stripeCustomerId',
   'stripeSubscriptionId', 'stripePlanId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin',
-  'authSubject', 'usage'
+  'authSubject', 'usage', 'createdBy'
 ]);
 
 /**