(core) add SELF_HYPERLINK() function for generating links to the current document

Summary:
 * Adds a `SELF_HYPERLINK()` python function, with optional keyword arguments to set a label, the page, and link parameters.
 * Adds a `UUID()` python function, since using python's uuid.uuidv4 hits a problem accessing /dev/urandom in the sandbox.  UUID makes no particular quality claims since it doesn't use an audited implementation.  A difficult to guess code is convenient for some use cases that `SELF_HYPERLINK()` enables.

The canonical URL for a document is mutable, but older versions generally forward.  So for implementation simplicity the document url is passed it on sandbox creation and remains fixed throughout the lifetime of the sandbox.  This could and should be improved in future.

The URL is passed into the sandbox as a `DOC_URL` environment variable.

The code for creating the URL is factored out of `Notifier.ts`. Since the url is a function of the organization as well as the document, some rejiggering is needed to make that information available to DocManager.

On document imports, the new document is registered in the database slightly earlier now, in order to keep the procedure for constructing the URL in different starting conditions more homogeneous.

Test Plan: updated test

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2759

app/gen-server/lib/HomeDBManager.ts

@@ -155,6 +155,7 @@ export interface DocAuthResult {
   removed: boolean|null;      // Set if the doc is soft-deleted. Users may still have access
                               // to removed documents for some purposes. Null on error.
   error?: ApiError;
+  cachedDoc?: Document;       // For cases where stale info is ok.
 }
 
 // Represent a DocAuthKey as a string.  The format is "<urlId>:<org> <userId>".
@@ -932,7 +933,8 @@ export class HomeDBManager extends EventEmitter {
       if (!forkId) { throw new ApiError('invalid document identifier', 400); }
       // We imagine current user owning trunk if there is no embedded userId, or
       // the embedded userId matches the current user.
-      const access = (forkUserId === undefined || forkUserId === userId) ? 'owners' : null;
+      const access = (forkUserId === undefined || forkUserId === userId) ? 'owners' :
+        (userId === this.getPreviewerUserId() ? 'viewers' : null);
       if (!access) { throw new ApiError("access denied", 403); }
       doc = {
         name: 'Untitled',
@@ -3854,7 +3856,7 @@ export async function makeDocAuthResult(docPromise: Promise<Document>): Promise<
   try {
     const doc = await docPromise;
     const removed = Boolean(doc.removedAt || doc.workspace.removedAt);
-    return {docId: doc.id, access: doc.access, removed};
+    return {docId: doc.id, access: doc.access, removed, cachedDoc: doc};
   } catch (error) {
     return {docId: null, access: null, removed: null, error};
   }

app/server/lib/ActiveDoc.ts

@@ -129,9 +129,12 @@ export class ActiveDoc extends EventEmitter {
   private _inactivityTimer = new InactivityTimer(() => this.shutdown(), Deps.ACTIVEDOC_TIMEOUT * 1000);
   private _recoveryMode: boolean = false;
 
-  constructor(docManager: DocManager, docName: string, wantRecoveryMode?: boolean) {
+  constructor(docManager: DocManager, docName: string, options?: {
+    safeMode?: boolean,
+    docUrl?: string
+  }) {
     super();
-    if (wantRecoveryMode) { this._recoveryMode = true; }
+    if (options?.safeMode) { this._recoveryMode = true; }
     this._docManager = docManager;
     this._docName = docName;
     this.docStorage = new DocStorage(docManager.storageManager, docName);
@@ -148,6 +151,7 @@ export class ActiveDoc extends EventEmitter {
       logCalls: false,
       logTimes: true,
       logMeta: {docId: docName},
+      docUrl: options?.docUrl,
     });
 
     this._activeDocImport = new ActiveDocImport(this);

app/server/lib/DocManager.ts

@@ -16,7 +16,7 @@ import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {assertAccess, Authorizer, DocAuthorizer, DummyAuthorizer,
         isSingleUserMode} from 'app/server/lib/Authorizer';
 import {Client} from 'app/server/lib/Client';
-import {makeExceptionalDocSession, makeOptDocSession, OptDocSession} from 'app/server/lib/DocSession';
+import {getDocSessionCachedDoc, makeExceptionalDocSession, makeOptDocSession, OptDocSession} from 'app/server/lib/DocSession';
 import * as docUtils from 'app/server/lib/docUtils';
 import {GristServer} from 'app/server/lib/GristServer';
 import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
@@ -181,26 +181,26 @@ export class DocManager extends EventEmitter {
 
     const accessId = this.makeAccessId(userId);
     const docSession = makeExceptionalDocSession('nascent', {browserSettings});
-    const result = await this._doImportDoc(docSession,
+    const register = async (docId: string, docTitle: string) => {
-                                           globalUploadSet.getUploadInfo(uploadId, accessId), {
+      if (!workspaceId || !this._homeDbManager) { return; }
-                                             naming: workspaceId ? 'saved' : 'unsaved',
-                                             userId,
-                                           });
-    if (workspaceId) {
       const queryResult = await this._homeDbManager.addDocument({userId}, workspaceId,
-                                                                {name: result.title}, result.id);
+                                                                {name: docTitle}, docId);
       if (queryResult.status !== 200) {
         // TODO The ready-to-add document is not yet in storageManager, but is in the filesystem. It
         // should get cleaned up in case of error here.
         throw new ApiError(queryResult.errMessage || 'unable to add imported document', queryResult.status);
       }
-    }
+    };
+    return this._doImportDoc(docSession,
+                             globalUploadSet.getUploadInfo(uploadId, accessId), {
+                               naming: workspaceId ? 'saved' : 'unsaved',
+                               register,
+                               userId,
+                             });
 
     // The imported document is associated with the worker that did the import.
     // We could break that association (see /api/docs/:docId/assign for how) if
     // we start using dedicated import workers.
-
-    return result;
   }
 
   /**
@@ -369,7 +369,8 @@ export class DocManager extends EventEmitter {
   public async createNewEmptyDoc(docSession: OptDocSession, basenameHint: string): Promise<ActiveDoc> {
     const docName = await this._createNewDoc(basenameHint);
     return mapSetOrClear(this._activeDocs, docName,
-                         this.gristServer.create.ActiveDoc(this, docName).createDoc(docSession));
+                         this._createActiveDoc(docSession, docName)
+                         .then(newDoc => newDoc.createDoc(docSession)));
   }
 
   /**
@@ -389,12 +390,15 @@ export class DocManager extends EventEmitter {
         }
       }
       if (!this._activeDocs.has(docName)) {
-        const newDoc = this.gristServer.create.ActiveDoc(this, docName, wantRecoveryMode);
+        return mapSetOrClear(this._activeDocs, docName,
+                             this._createActiveDoc(docSession, docName, wantRecoveryMode)
+                             .then(newDoc => {
                                // Propagate backupMade events from newly opened activeDocs (consolidate all to DocMan)
                                newDoc.on('backupMade', (bakPath: string) => {
                                  this.emit('backupMade', bakPath);
                                });
-        return mapSetOrClear(this._activeDocs, docName, newDoc.loadDoc(docSession));
+                               return newDoc.loadDoc(docSession);
+                             }));
       }
       const activeDoc = await this._activeDocs.get(docName)!;
       if (!activeDoc.muted) { return activeDoc; }
@@ -425,7 +429,28 @@ export class DocManager extends EventEmitter {
                                       encBundles: EncActionBundleFromHub[]): Promise<ActiveDoc> {
     const docName = await this._createNewDoc(basenameHint);
     return mapSetOrClear(this._activeDocs, docName,
-      this.gristServer.create.ActiveDoc(this, docName).downloadSharedDoc(docId, instanceId, encBundles));
+                         this._createActiveDoc({client: null}, docName)
+                         .then(newDoc => newDoc.downloadSharedDoc(docId, instanceId, encBundles)));
+  }
+
+  private async _createActiveDoc(docSession: OptDocSession, docName: string, safeMode?: boolean) {
+    // Get URL for document for use with SELF_HYPERLINK().
+    const cachedDoc = getDocSessionCachedDoc(docSession);
+    let docUrl: string|undefined;
+    try {
+      if (cachedDoc) {
+        docUrl = await this.gristServer.getResourceUrl(cachedDoc);
+      } else {
+        docUrl = await this.gristServer.getDocUrl(docName);
+      }
+    } catch (e) {
+      // If there is no home url, we cannot construct links.  Accept this, for the benefit
+      // of legacy tests.
+      if (!String(e).match(/need APP_HOME_URL/)) {
+        throw e;
+      }
+    }
+    return this.gristServer.create.ActiveDoc(this, docName, {docUrl, safeMode});
   }
 
   /**
@@ -435,6 +460,7 @@ export class DocManager extends EventEmitter {
   private async _doImportDoc(docSession: OptDocSession, uploadInfo: UploadInfo,
                              options: {
                                naming: 'classic'|'saved'|'unsaved',
+                               register?: (docId: string, docTitle: string) => Promise<void>,
                                userId?: number,
                              }): Promise<DocCreationInfo> {
     try {
@@ -466,6 +492,7 @@ export class DocManager extends EventEmitter {
         default:
           throw new Error('naming mode not recognized');
       }
+      await options.register?.(id, basename);
       if (ext === '.grist') {
         // If the import is a grist file, copy it to the docs directory.
         // TODO: We should be skeptical of the upload file to close a possible

app/server/lib/DocSession.ts

@@ -1,6 +1,7 @@
 import {BrowserSettings} from 'app/common/BrowserSettings';
 import {Role} from 'app/common/roles';
 import {FullUser} from 'app/common/UserAPI';
+import {Document} from 'app/gen-server/entity/Document';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {Authorizer, getUser, getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {Client} from 'app/server/lib/Client';
@@ -148,3 +149,10 @@ export function getDocSessionAccess(docSession: OptDocSession): Role {
   }
   throw new Error('getDocSessionAccess could not find access information in DocSession');
 }
+
+/**
+ * Get cached information about the document, if available.  May be stale.
+ */
+export function getDocSessionCachedDoc(docSession: OptDocSession): Document|undefined {
+  return (docSession.req as RequestWithLogin)?.docAuth?.cachedDoc;
+}

app/server/lib/FlexServer.ts

@@ -1,13 +1,17 @@
 import {BillingTask} from 'app/common/BillingAPI';
 import {delay} from 'app/common/delay';
 import {DocCreationInfo} from 'app/common/DocListAPI';
-import {isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
+import {encodeUrl, getSlugIfNeeded, GristLoadConfig, IGristUrlState, isOrgInPathOnly,
+        parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
 import {getOrgUrlInfo} from 'app/common/gristUrls';
 import {UserProfile} from 'app/common/LoginSessionAPI';
 import {tbind} from 'app/common/tbind';
 import {UserConfig} from 'app/common/UserConfig';
 import * as version from 'app/common/version';
 import {ApiServer} from 'app/gen-server/ApiServer';
+import {Document} from "app/gen-server/entity/Document";
+import {Organization} from "app/gen-server/entity/Organization";
+import {Workspace} from 'app/gen-server/entity/Workspace';
 import {DocApiForwarder} from 'app/gen-server/lib/DocApiForwarder';
 import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
@@ -1102,8 +1106,48 @@ export class FlexServer implements GristServer {
     // and all that is needed is a refactor to pass that info along.  But there is also the
     // case of notification(s) from stripe.  May need to associate a preferred base domain
     // with org/user and persist that?
-    const gristConfig = makeGristConfig(this.getDefaultHomeUrl(), {}, this._defaultBaseDomain);
+    this.notifier = this.create.Notifier(this.dbManager, this);
-    this.notifier = this.create.Notifier(this.dbManager, gristConfig);
+  }
+
+  public getGristConfig(): GristLoadConfig {
+    return makeGristConfig(this.getDefaultHomeUrl(), {}, this._defaultBaseDomain);
+  }
+
+  /**
+   * Get a url for a document.  The id provided should be a genuine docId, since we query
+   * the db for document details without including organization disambiguation.
+   */
+  public async getDocUrl(docId: string): Promise<string> {
+    if (!this.dbManager) { throw new Error('database missing'); }
+    const doc = await this.dbManager.getDoc({
+      userId: this.dbManager.getPreviewerUserId(),
+      urlId: docId,
+      showAll: true
+    });
+    return this.getResourceUrl(doc);
+  }
+
+  /**
+   * Get a url for an organization, workspace, or document.
+   */
+  public async getResourceUrl(resource: Organization|Workspace|Document): Promise<string> {
+    if (!this.dbManager) { throw new Error('database missing'); }
+    const gristConfig = this.getGristConfig();
+    const state: IGristUrlState = {};
+    let org: Organization;
+    if (resource instanceof Organization) {
+      org = resource;
+    } else if (resource instanceof Workspace) {
+      org = resource.org;
+      state.ws = resource.id;
+    } else {
+      org = resource.workspace.org;
+      state.doc = resource.id;
+      state.slug = getSlugIfNeeded(resource);
+    }
+    state.org = this.dbManager.normalizeOrgDomain(org.id, org.domain, org.ownerId);
+    if (!gristConfig.homeUrl) { throw new Error('Computing a resource URL requires a home URL'); }
+    return encodeUrl(gristConfig, state, new URL(gristConfig.homeUrl));
   }
 
   public addUsage() {

app/server/lib/GranularAccess.ts

@@ -938,7 +938,7 @@ export class GranularAccess implements GranularAccessForBundle {
     user.Name = fullUser?.name || null;
     // If viewed from a websocket, collect any link parameters included.
     // TODO: could also get this from rest api access, just via a different route.
-    user.Link = linkParameters;
+    user.LinkKey = linkParameters;
     // Include origin info if accessed via the rest api.
     // TODO: could also get this for websocket access, just via a different route.
     user.Origin = docSession.req?.get('origin') || null;

app/server/lib/GristServer.ts

@@ -1,3 +1,7 @@
+import { GristLoadConfig } from 'app/common/gristUrls';
+import { Document } from 'app/gen-server/entity/Document';
+import { Organization } from 'app/gen-server/entity/Organization';
+import { Workspace } from 'app/gen-server/entity/Workspace';
 import { SessionUserObj } from 'app/server/lib/BrowserSession';
 import * as Comm from 'app/server/lib/Comm';
 import { Hosts } from 'app/server/lib/extractOrg';
@@ -15,6 +19,9 @@ export interface GristServer {
   getHost(): string;
   getHomeUrl(req: express.Request, relPath?: string): string;
   getHomeUrlByDocId(docId: string, relPath?: string): Promise<string>;
+  getDocUrl(docId: string): Promise<string>;
+  getResourceUrl(resource: Organization|Workspace|Document): Promise<string>;
+  getGristConfig(): GristLoadConfig;
   getPermitStore(): IPermitStore;
 }
 

app/server/lib/ICreate.ts

@@ -1,4 +1,3 @@
-import { GristLoadConfig } from 'app/common/gristUrls';
 import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
 import { ActiveDoc } from 'app/server/lib/ActiveDoc';
 import { ScopedSession } from 'app/server/lib/BrowserSession';
@@ -19,7 +18,7 @@ export interface ICreate {
   LoginSession(comm: Comm, sid: string, domain: string, scopeSession: ScopedSession,
                instanceManager: IInstanceManager|null): ILoginSession;
   Billing(dbManager: HomeDBManager): IBilling;
-  Notifier(dbManager: HomeDBManager, gristConfig: GristLoadConfig): INotifier;
+  Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
   Shell(): IShell|undefined;
 
   // Create a space to store files externally, for storing either:
@@ -29,7 +28,7 @@ export interface ICreate {
   // should not interfere with each other.
   ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage|undefined;
 
-  ActiveDoc(docManager: DocManager, docName: string, safeMode?: boolean): ActiveDoc;
+  ActiveDoc(docManager: DocManager, docName: string, options: ICreateActiveDocOptions): ActiveDoc;
   DocManager(storageManager: IDocStorageManager, pluginManager: PluginManager,
              homeDbManager: HomeDBManager|null, gristServer: GristServer): DocManager;
   NSandbox(options: ISandboxCreationOptions): ISandbox;
@@ -38,3 +37,8 @@ export interface ICreate {
   // Get configuration information to show at start-up.
   configurationOptions(): {[key: string]: any};
 }
+
+export interface ICreateActiveDocOptions {
+  safeMode?: boolean;
+  docUrl?: string;
+}

app/server/lib/ISandbox.ts

@@ -15,6 +15,8 @@ export interface ISandboxCreationOptions {
   entryPoint?: string;   // main script to call - leave undefined for default
   sandboxMount?: string; // if defined, make this path available read-only as "/sandbox"
   importMount?: string;  // if defined, make this path available read-only as "/importdir"
+
+  docUrl?: string;      // to support SELF_HYPERLINK.
 }
 
 export interface ISandbox {

app/server/lib/NSandbox.ts

@@ -17,6 +17,7 @@ type SandboxMethod = (...args: any[]) => any;
 export interface ISandboxCommand {
   process: string;
   libraryPath: string;
+  docUrl: string;
 }
 
 export interface ISandboxOptions {
@@ -60,7 +61,7 @@ export class NSandbox implements ISandbox {
 
     if (command) {
       return spawn(command.process, pythonArgs,
-                   {env: {PYTHONPATH: command.libraryPath},
+                   {env: {PYTHONPATH: command.libraryPath, DOC_URL: command.docUrl},
                     cwd: path.join(process.cwd(), 'sandbox'), ...spawnOptions});
     }
 
@@ -346,6 +347,10 @@ export class NSandboxCreator implements ISandboxCreator {
     if (options.importMount) {
       selLdrArgs.push('-m', `${options.importMount}:/importdir:ro`);
     }
+    const docUrl = (options.docUrl || '').replace(/[^-a-zA-Z0-9_:/?&.]/, '');
+    if (this._flavor === 'pynbox') {
+      selLdrArgs.push('-E', `DOC_URL=${docUrl}`);
+    }
     return new NSandbox({
       args,
       logCalls: options.logCalls,
@@ -355,7 +360,8 @@ export class NSandboxCreator implements ISandboxCreator {
       ...(this._flavor === 'pynbox' ? {} : {
         command: {
           process: pythonVersion,
-          libraryPath
+          libraryPath,
+          docUrl,
         }
       })
     });

sandbox/grist/functions/lookup.py

@@ -1,4 +1,8 @@
 # pylint: disable=redefined-builtin, line-too-long
+from collections import OrderedDict
+import os
+from urllib import urlencode
+import urlparse
 from unimplemented import unimplemented
 
 @unimplemented
@@ -71,6 +75,57 @@ def ROWS(range):
   """Returns the number of rows in a specified array or range."""
   raise NotImplementedError()
 
+def SELF_HYPERLINK(label=None, page=None, **kwargs):
+  """
+  Creates a link to the current document.  All parameters are optional.
+
+  The returned string is in URL format, optionally preceded by a label and a space
+  (the format expected for Grist Text columns with the HyperLink option enabled).
+
+  A numeric page number can be supplied, which will create a link to the
+  specified page.  To find the numeric page number you need, visit a page
+  and examine its URL for a `/p/NN` part.
+
+  Any number of arguments of the form `LinkKey_NAME` may be provided, to set
+  `user.LinkKey.NAME` values that will be available in access rules.  For example,
+  if a rule allows users to view rows when `user.LinkKey.Code == rec.Code`,
+  we might want to create links with `SELF_HYPERLINK(LinkKey_Code=$Code)`.
+
+  >>> SELF_HYPERLINK()
+  'https://docs.getgrist.com/sbaltsirg/Example'
+  >>> SELF_HYPERLINK(label='doc')
+  'doc https://docs.getgrist.com/sbaltsirg/Example'
+  >>> SELF_HYPERLINK(page=2)
+  'https://docs.getgrist.com/sbaltsirg/Example/p/2'
+  >>> SELF_HYPERLINK(LinkKey_Code='X1234')
+  'https://docs.getgrist.com/sbaltsirg/Example?Code_=X1234'
+  >>> SELF_HYPERLINK(label='order', page=3, LinkKey_Code='X1234', LinkKey_Name='Bi Ngo')
+  'order https://docs.getgrist.com/sbaltsirg/Example/p/3?Code_=X1234&Name_=Bi+Ngo'
+  >>> SELF_HYPERLINK(Linky_Link='Link')
+  Traceback (most recent call last):
+  ...
+  TypeError: unexpected keyword argument 'Linky_Link' (not of form LinkKey_NAME)
+  """
+  txt = os.environ.get('DOC_URL')
+  if not txt:
+    return None
+  if page:
+    txt += "/p/{}".format(page)
+  if kwargs:
+    parts = list(urlparse.urlparse(txt))
+    query = OrderedDict(urlparse.parse_qsl(parts[4]))
+    for [key, value] in kwargs.iteritems():
+      key_parts = key.split('LinkKey_')
+      if len(key_parts) == 2 and key_parts[0] == '':
+        query[key_parts[1] + '_'] = value
+      else:
+        raise TypeError("unexpected keyword argument '{}' (not of form LinkKey_NAME)".format(key))
+    parts[4] = urlencode(query)
+    txt = urlparse.urlunparse(parts)
+  if label:
+    txt = "{} {}".format(label, txt)
+  return txt
+
 def VLOOKUP(table, **field_value_pairs):
   """
   Vertical lookup. Searches the given table for a record matching the given `field=value`

sandbox/grist/functions/math.py

@@ -5,6 +5,7 @@ import itertools
 import math as _math
 import operator
 import random
+import uuid
 
 from functions.info import ISNUMBER, ISLOGICAL
 from functions.unimplemented import unimplemented
@@ -833,3 +834,7 @@ def TRUNC(value, places=0):
   """
   # TRUNC seems indistinguishable from ROUNDDOWN.
   return ROUNDDOWN(value, places)
+
+def UUID():
+  """Generate a random UUID-formatted string identifier."""
+  return str(uuid.UUID(bytes=[chr(random.randrange(0, 256)) for _ in xrange(0, 16)], version=4))

sandbox/grist/test_functions.py

@@ -1,4 +1,5 @@
 import doctest
+import os
 import functions
 import moment
 
@@ -18,6 +19,8 @@ def date_tearDown(doc_test):
 # This works with the unittest module to turn all the doctests in the functions' doc-comments into
 # unittest test cases.
 def load_tests(loader, tests, ignore):
+  # Set DOC_URL for SELF_HYPERLINK()
+  os.environ['DOC_URL'] = 'https://docs.getgrist.com/sbaltsirg/Example'
   tests.addTests(doctest.DocTestSuite(functions.date, setUp = date_setUp, tearDown = date_tearDown))
   tests.addTests(doctest.DocTestSuite(functions.info, setUp = date_setUp, tearDown = date_tearDown))
   tests.addTests(doctest.DocTestSuite(functions.logical))
@@ -26,4 +29,5 @@ def load_tests(loader, tests, ignore):
   tests.addTests(doctest.DocTestSuite(functions.text))
   tests.addTests(doctest.DocTestSuite(functions.schedule,
                                       setUp = date_setUp, tearDown = date_tearDown))
+  tests.addTests(doctest.DocTestSuite(functions.lookup))
   return tests

stubs/app/server/lib/create.ts

@@ -31,7 +31,7 @@ export const create: ICreate = {
     };
   },
   ExternalStorage() { return undefined; },
-  ActiveDoc(docManager, docName, wantSafeMode) { return new ActiveDoc(docManager, docName, wantSafeMode); },
+  ActiveDoc(docManager, docName, options) { return new ActiveDoc(docManager, docName, options); },
   DocManager(storageManager, pluginManager, homeDBManager, gristServer) {
     return new DocManager(storageManager, pluginManager, homeDBManager, gristServer);
   },