(core) back-end support for tables that are accessible only by owners

Summary:
This makes it possible to serve a table or tables only to owners.

 * The _grist_ACLResources table is abused (temporarily) such that rows of the form `{colId: '~o', tableId}` are interpreted as meaning that `tableId` is private to owners.
 * Many websocket and api endpoints are updated to preserve the privacy of these tables.
 * In a document where some tables are private, a lot of capabilities are turned off for non-owners to avoid leaking info indirectly.
 * The client is tweaked minimally, to show '-' where a page with some private material would otherwise go.

No attempt is made to protect data from private tables pulled into non-private tables via formulas.

There are some known leaks remaining:
 * Changes to the schema of private tables are still broadcast to all clients (fixable).
 * Non-owner may be able to access snapshots or make forks or use other corners of API (fixable).
 * Changing name of table makes it public, since tableId in ACLResource is not updated (fixable).

Security will require some work, the attack surface is large.

Test Plan: added tests

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2604

app/server/lib/ActiveDoc.ts

@@ -32,7 +32,7 @@ import {InactivityTimer} from 'app/common/InactivityTimer';
 import * as marshal from 'app/common/marshal';
 import {Peer} from 'app/common/sharing';
 import {UploadResult} from 'app/common/uploads';
-import {DocReplacementOptions} from 'app/common/UserAPI';
+import {DocReplacementOptions, DocState} from 'app/common/UserAPI';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
 import {GristDocAPI} from 'app/plugin/GristAPI';
 import {Authorizer} from 'app/server/lib/Authorizer';
@@ -55,6 +55,7 @@ import {DocSession, getDocSessionAccess, getDocSessionUserId, makeExceptionalDoc
         OptDocSession} from './DocSession';
 import {DocStorage} from './DocStorage';
 import {expandQuery} from './ExpandedQuery';
+import {GranularAccess} from './GranularAccess';
 import {OnDemandActions} from './OnDemandActions';
 import {findOrAddAllEnvelope, Sharing} from './Sharing';
 
@@ -104,6 +105,7 @@ export class ActiveDoc extends EventEmitter {
   private readonly _dataEngine: ISandbox;
   private _activeDocImport: ActiveDocImport;
   private _onDemandActions: OnDemandActions;
+  private _granularAccess: GranularAccess;
   private _muted: boolean = false;  // If set, changes to this document should not propagate
                                     // to outside world
   private _initializationPromise: Promise<boolean>|null = null;
@@ -176,7 +178,9 @@ export class ActiveDoc extends EventEmitter {
     return this._actionHistory.getRecentActions(maxActions);
   }
 
-  public getRecentStates(maxStates?: number) {
+  public async getRecentStates(docSession: OptDocSession, maxStates?: number): Promise<DocState[]> {
+    // Doc states currently don't include user content, so it seems ok to let all
+    // viewers have access to them.
     return this._actionHistory.getRecentStates(maxStates);
   }
 
@@ -187,7 +191,8 @@ export class ActiveDoc extends EventEmitter {
    */
   public async getRecentActions(docSession: OptDocSession, summarize: boolean): Promise<ActionGroup[]> {
     const actions = await this._actionHistory.getRecentActions(MAX_RECENT_ACTIONS);
-    return actions.map(act => asActionGroup(this._actionHistory, act, {client: docSession.client, summarize}));
+    const groups = actions.map(act => asActionGroup(this._actionHistory, act, {client: docSession.client, summarize}));
+    return groups.filter(actionGroup => this._granularAccess.allowActionGroup(docSession, actionGroup));
   }
 
   /** expose action history for tests */
@@ -368,6 +373,7 @@ export class ActiveDoc extends EventEmitter {
     this._onDemandActions = new OnDemandActions(this.docStorage, this.docData);
 
     await this._actionHistory.initialize();
+    this._granularAccess = new GranularAccess(this.docData);
     this._sharing = new Sharing(this, this._actionHistory);
 
     await this.openSharedDoc(docSession);
@@ -478,6 +484,10 @@ export class ActiveDoc extends EventEmitter {
    * @returns {Promise<Buffer>} Promise for the data of this attachment; rejected on error.
    */
   public async getAttachmentData(docSession: OptDocSession, fileIdent: string): Promise<Buffer> {
+    // We don't know for sure whether the attachment is available via a table the user
+    // has access to, but at least they are presenting a SHA1 checksum of the file content,
+    // and they have at least view access to the document to get to this point.  So we go ahead
+    // and serve the attachment.
     const data = await this.docStorage.getFileData(fileIdent);
     if (!data) { throw new ApiError("Invalid attachment identifier", 404); }
     this.logInfo(docSession, "getAttachment: %s -> %s bytes", fileIdent, data.length);
@@ -497,8 +507,8 @@ export class ActiveDoc extends EventEmitter {
       if (!tableId.startsWith('_grist_')) { continue; }
       tables[tableId] = tableData.getTableDataAction();
     }
-    return tables;
+    return this._granularAccess.filterMetaTables(docSession, tables);
-}
+  }
 
   /**
    * Makes sure document is completely initialized.  May throw if doc is broken.
@@ -512,6 +522,12 @@ export class ActiveDoc extends EventEmitter {
     return true;
   }
 
+  // Check if user has rights to download this doc.
+  public canDownload(docSession: OptDocSession) {
+    return this._granularAccess.hasViewAccess(docSession) &&
+      !this._granularAccess.hasNuancedAccess(docSession);
+  }
+
   /**
    * Fetches a particular table from the data engine to return to the client.
    * @param {String} tableId: The string identifier of the table.
@@ -536,6 +552,11 @@ export class ActiveDoc extends EventEmitter {
                           waitForFormulas: boolean = false): Promise<TableDataAction> {
     this._inactivityTimer.ping();     // The doc is in active use; ping it to stay open longer.
 
+    // If user does not have rights to access what this query is asking for, fail.
+    if (!this._granularAccess.hasQueryAccess(docSession, query)) {
+      throw new Error('not authorized to read table');
+    }
+
     // Some tests read _grist_ tables via the api.  The _fetchQueryFromDB method
     // currently cannot read those tables, so we load them from the data engine
     // when ready.
@@ -612,6 +633,9 @@ export class ActiveDoc extends EventEmitter {
    */
   public async findColFromValues(docSession: DocSession, values: any[], n: number,
                                  optTableId?: string): Promise<number[]> {
+    // This could leak information about private tables, so if there are any nuanced
+    // permissions in force and the user does not have full access, do nothing.
+    if (this._granularAccess.hasNuancedAccess(docSession)) { return []; }
     this.logInfo(docSession, "findColFromValues(%s, %s, %s)", docSession, values, n);
     await this.waitForInitialization();
     return this._dataEngine.pyCall('find_col_from_values', values, n, optTableId);
@@ -626,6 +650,7 @@ export class ActiveDoc extends EventEmitter {
    */
   public async getFormulaError(docSession: DocSession, tableId: string, colId: string,
                                rowId: number): Promise<CellValue> {
+    if (!this._granularAccess.hasTableAccess(docSession, tableId)) { return null; }
     this.logInfo(docSession, "getFormulaError(%s, %s, %s, %s)",
       docSession, tableId, colId, rowId);
     await this.waitForInitialization();
@@ -649,6 +674,7 @@ export class ActiveDoc extends EventEmitter {
     // there'll be a deadlock.
     await this.waitForInitialization();
     const newOptions = {linkId: docSession.linkId, ...options};
+    // Granular access control implemented in _applyUserActions.
     const result: ApplyUAResult = await this._applyUserActions(docSession, actions, newOptions);
     docSession.linkId = docSession.shouldBundleActions ? result.actionNum : 0;
     return result;
@@ -686,6 +712,9 @@ export class ActiveDoc extends EventEmitter {
     } else {
       actions = flatten(actionBundles.map(a => a!.userActions));
     }
+    // Granular access control implemented ultimately in _applyUserActions.
+    // It could be that error cases and timing etc leak some info prior to this
+    // point.
     return this.applyUserActions(docSession, actions, options);
   }
 
@@ -698,6 +727,7 @@ export class ActiveDoc extends EventEmitter {
     localActionBundle.stored.forEach(da => docData.receiveAction(da[1]));
     localActionBundle.calc.forEach(da => docData.receiveAction(da[1]));
     const docActions = getEnvContent(localActionBundle.stored);
+    this._granularAccess.update();
     if (docActions.some(docAction => this._onDemandActions.isSchemaAction(docAction))) {
       const indexes = this._onDemandActions.getDesiredIndexes();
       await this.docStorage.updateIndexes(indexes);
@@ -752,6 +782,8 @@ export class ActiveDoc extends EventEmitter {
   }
 
   public async autocomplete(docSession: DocSession, txt: string, tableId: string): Promise<string[]> {
+    // Autocompletion can leak names of tables and columns.
+    if (this._granularAccess.hasNuancedAccess(docSession)) { return []; }
     await this.waitForInitialization();
     return this._dataEngine.pyCall('autocomplete', txt, tableId);
   }
@@ -761,6 +793,9 @@ export class ActiveDoc extends EventEmitter {
   }
 
   public forwardPluginRpc(docSession: DocSession, pluginId: string, msg: IMessage): Promise<any> {
+    if (this._granularAccess.hasNuancedAccess(docSession)) {
+      throw new Error('cannot confirm access to plugin');
+    }
     const pluginRpc = this.docPluginManager.plugins[pluginId].rpc;
     switch (msg.mtype) {
       case MsgType.RpcCall: return pluginRpc.forwardCall(msg);
@@ -793,6 +828,9 @@ export class ActiveDoc extends EventEmitter {
    * ID for the fork.  TODO: reconcile the two ways there are now of preparing a fork.
    */
   public async fork(docSession: DocSession): Promise<ForkResult> {
+    if (this._granularAccess.hasNuancedAccess(docSession)) {
+      throw new Error('cannot confirm authority to copy document');
+    }
     const userId = docSession.client.getCachedUserId();
     const isAnonymous = docSession.client.isAnonymous();
     // Get fresh document metadata (the cached metadata doesn't include the urlId).
@@ -865,6 +903,7 @@ export class ActiveDoc extends EventEmitter {
   }
 
   public async getSnapshots(): Promise<DocSnapshots> {
+    // Assume any viewer can access this list.
     return this._docManager.storageManager.getSnapshots(this.docName);
   }
 
@@ -889,7 +928,8 @@ export class ActiveDoc extends EventEmitter {
     actionGroup: ActionGroup,
     docActions: DocAction[]
   }) {
-    await this.docClients.broadcastDocMessage(client, 'docUserAction', message);
+    await this.docClients.broadcastDocMessage(client, 'docUserAction', message,
+                                              (docSession) => this._filterDocUpdate(docSession, message));
   }
 
   /**
@@ -959,6 +999,11 @@ export class ActiveDoc extends EventEmitter {
    */
   protected async _applyUserActions(docSession: OptDocSession, actions: UserAction[],
                                     options: ApplyUAOptions = {}): Promise<ApplyUAResult> {
+
+    if (!this._granularAccess.canApplyUserActions(docSession, actions)) {
+      throw new Error('cannot perform a requested action');
+    }
+
     const client = docSession.client;
     this.logDebug(docSession, "_applyUserActions(%s, %s)", client, shortDesc(actions));
     this._inactivityTimer.ping();     // The doc is in active use; ping it to stay open longer.
@@ -1129,6 +1174,23 @@ export class ActiveDoc extends EventEmitter {
   private _log(level: string, docSession: OptDocSession, msg: string, ...args: any[]) {
     log.origLog(level, `ActiveDoc ` + msg, ...args, this.getLogMeta(docSession));
   }
+
+  /**
+   * This filters a message being broadcast to all clients to be appropriate for one
+   * particular client, if that client may need some material filtered out.
+   */
+  private _filterDocUpdate(docSession: OptDocSession, message: {
+    actionGroup: ActionGroup,
+    docActions: DocAction[]
+  }) {
+    if (!this._granularAccess.hasNuancedAccess(docSession)) { return message; }
+    const result = {
+      actionGroup: this._granularAccess.filterActionGroup(docSession, message.actionGroup),
+      docActions: this._granularAccess.filterOutgoingDocActions(docSession, message.docActions),
+    };
+    if (result.docActions.length === 0) { return null; }
+    return result;
+  }
 }
 
 // Helper to initialize a sandbox action bundle with no values.

app/server/lib/AppEndpoint.ts

@@ -7,8 +7,7 @@ import * as express from 'express';
 import fetch, {RequestInit, Response as FetchResponse} from 'node-fetch';
 
 import {ApiError} from 'app/common/ApiError';
-import {getSlugIfNeeded, isOrgInPathOnly,
+import {getSlugIfNeeded, parseSubdomainStrictly} from 'app/common/gristUrls';
-        parseSubdomainStrictly} from 'app/common/gristUrls';
 import {removeTrailingSlash} from 'app/common/gutil';
 import {Document as APIDocument} from 'app/common/UserAPI';
 import {Document} from "app/gen-server/entity/Document";
@@ -19,7 +18,7 @@ import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {expressWrap} from 'app/server/lib/expressWrap';
 import {getAssignmentId} from 'app/server/lib/idUtils';
 import * as log from 'app/server/lib/log';
-import {adaptServerUrl, pruneAPIResult, trustOrigin} from 'app/server/lib/requestUtils';
+import {adaptServerUrl, addOrgToPathIfNeeded, pruneAPIResult, trustOrigin} from 'app/server/lib/requestUtils';
 import {ISendAppPageOptions} from 'app/server/lib/sendAppPage';
 
 export interface AttachOptions {
@@ -199,16 +198,13 @@ export function attachAppEndpoint(options: AttachOptions): void {
       const preferredUrlId = doc.urlId || doc.id;
       if (urlId !== preferredUrlId || slugMismatch) {
         // Prepare to redirect to canonical url for document.
-        // Preserve org in url path if necessary.
-        const prefix = isOrgInPathOnly(req.hostname) ? `/o/${mreq.org}` : '';
         // Preserve any query parameters or fragments.
         const queryOrFragmentCheck = req.originalUrl.match(/([#?].*)/);
         const queryOrFragment = (queryOrFragmentCheck && queryOrFragmentCheck[1]) || '';
-        if (slug) {
+        const target = slug ?
-          res.redirect(`${prefix}/${preferredUrlId}/${slug}${req.params.remainder}${queryOrFragment}`);
+          `/${preferredUrlId}/${slug}${req.params.remainder}${queryOrFragment}` :
-        } else {
+          `/doc/${preferredUrlId}${req.params.remainder}${queryOrFragment}`;
-          res.redirect(`${prefix}/doc/${preferredUrlId}${req.params.remainder}${queryOrFragment}`);
+        res.redirect(addOrgToPathIfNeeded(req, target));
-        }
         return;
       }
 

app/server/lib/DocApi.ts

@@ -11,7 +11,7 @@ import { ActiveDoc } from "app/server/lib/ActiveDoc";
 import { assertAccess, getOrSetDocAuth, getTransitiveHeaders, getUserId, isAnonymousUser,
          RequestWithLogin } from 'app/server/lib/Authorizer';
 import { DocManager } from "app/server/lib/DocManager";
-import { makeExceptionalDocSession } from "app/server/lib/DocSession";
+import { docSessionFromRequest, makeExceptionalDocSession, OptDocSession } from "app/server/lib/DocSession";
 import { DocWorker } from "app/server/lib/DocWorker";
 import { expressWrap } from 'app/server/lib/expressWrap';
 import { GristServer } from 'app/server/lib/GristServer';
@@ -91,7 +91,7 @@ export class DocWorkerApi {
 
     // Apply user actions to a document.
     this._app.post('/api/docs/:docId/apply', canEdit, withDoc(async (activeDoc, req, res) => {
-      res.json(await activeDoc.applyUserActions({ client: null, req }, req.body));
+      res.json(await activeDoc.applyUserActions(docSessionFromRequest(req), req.body));
     }));
 
     // Get the specified table.
@@ -102,7 +102,7 @@ export class DocWorkerApi {
       }
       const tableId = req.params.tableId;
       const tableData = await handleSandboxError(tableId, [], activeDoc.fetchQuery(
-        {client: null, req}, {tableId, filters}, true));
+        docSessionFromRequest(req), {tableId, filters}, true));
       // Apply sort/limit parameters, if set.  TODO: move sorting/limiting into data engine
       // and sql.
       const params = getQueryParameters(req);
@@ -113,7 +113,7 @@ export class DocWorkerApi {
     // Returns the list of rowIds for the rows created in the _grist_Attachments table.
     this._app.post('/api/docs/:docId/attachments', canEdit, withDoc(async (activeDoc, req, res) => {
       const uploadResult = await handleUpload(req, res);
-      res.json(await activeDoc.addAttachments({client: null, req}, uploadResult.uploadId));
+      res.json(await activeDoc.addAttachments(docSessionFromRequest(req), uploadResult.uploadId));
     }));
 
     // Returns the metadata for a given attachment ID (i.e. a rowId in _grist_Attachments table).
@@ -131,7 +131,7 @@ export class DocWorkerApi {
       const ext = path.extname(fileIdent);
       const origName = attRecord.fileName as string;
       const fileName = ext ? path.basename(origName, path.extname(origName)) + ext : origName;
-      const fileData = await activeDoc.getAttachmentData({client: null, req}, fileIdent);
+      const fileData = await activeDoc.getAttachmentData(docSessionFromRequest(req), fileIdent);
       res.status(200)
         .type(ext)
         // Construct a content-disposition header of the form 'attachment; filename="NAME"'
@@ -150,7 +150,8 @@ export class DocWorkerApi {
       const count = columnValues[colNames[0]].length;
       // then, let's create [null, ...]
       const rowIds = arrayRepeat(count, null);
-      const sandboxRes = await handleSandboxError(tableId, colNames, activeDoc.applyUserActions({client: null, req},
+      const sandboxRes = await handleSandboxError(tableId, colNames, activeDoc.applyUserActions(
+        docSessionFromRequest(req),
         [['BulkAddRecord', tableId, rowIds, columnValues]]));
       res.json(sandboxRes.retValues[0]);
     }));
@@ -158,7 +159,8 @@ export class DocWorkerApi {
     this._app.post('/api/docs/:docId/tables/:tableId/data/delete', canEdit, withDoc(async (activeDoc, req, res) => {
       const tableId = req.params.tableId;
       const rowIds = req.body;
-      const sandboxRes = await handleSandboxError(tableId, [], activeDoc.applyUserActions({client: null, req},
+      const sandboxRes = await handleSandboxError(tableId, [], activeDoc.applyUserActions(
+        docSessionFromRequest(req),
         [['BulkRemoveRecord', tableId, rowIds]]));
       res.json(sandboxRes.retValues[0]);
     }));
@@ -167,20 +169,33 @@ export class DocWorkerApi {
     // TODO: look at download behavior if ActiveDoc is shutdown during call (cannot
     // use withDoc wrapper)
     this._app.get('/api/docs/:docId/download', canView, throttled(async (req, res) => {
-      try {
+      // We want to be have a way download broken docs that ActiveDoc may not be able
-        // We carefully avoid creating an ActiveDoc for the document being downloaded,
+      // to load.  So, if the user owns the document, we unconditionally let them
-        // in case it is broken in some way.  It is convenient to be able to download
+      // download.
-        // broken files for diagnosis/recovery.
+      if (await this._isOwner(req)) {
-        return await this._docWorker.downloadDoc(req, res, this._docManager.storageManager);
+        try {
-      } catch (e) {
+          // We carefully avoid creating an ActiveDoc for the document being downloaded,
-        if (e.message && e.message.match(/does not exist yet/)) {
+          // in case it is broken in some way.  It is convenient to be able to download
-          // The document has never been seen on file system / s3.  It may be new, so
+          // broken files for diagnosis/recovery.
-          // we try again after having created an ActiveDoc for the document.
+          return await this._docWorker.downloadDoc(req, res, this._docManager.storageManager);
-          await this._getActiveDoc(req);
+        } catch (e) {
-          return this._docWorker.downloadDoc(req, res, this._docManager.storageManager);
+          if (e.message && e.message.match(/does not exist yet/)) {
-        } else {
+            // The document has never been seen on file system / s3.  It may be new, so
-          throw e;
+            // we try again after having created an ActiveDoc for the document.
+            await this._getActiveDoc(req);
+            return this._docWorker.downloadDoc(req, res, this._docManager.storageManager);
+          } else {
+            throw e;
+          }
         }
+      } else {
+        // If the user is not an owner, we load the document as an ActiveDoc, and then
+        // check if the user has download permissions.
+        const activeDoc = await this._getActiveDoc(req);
+        if (!activeDoc.canDownload(docSessionFromRequest(req))) {
+          throw new Error('not authorized to download this document');
+        }
+        return this._docWorker.downloadDoc(req, res, this._docManager.storageManager);
       }
     }));
 
@@ -193,7 +208,8 @@ export class DocWorkerApi {
       const rowIds = columnValues.id;
       // sandbox expects no id column
       delete columnValues.id;
-      await handleSandboxError(tableId, colNames, activeDoc.applyUserActions({client: null, req},
+      await handleSandboxError(tableId, colNames, activeDoc.applyUserActions(
+        docSessionFromRequest(req),
         [['BulkUpdateRecord', tableId, rowIds, columnValues]]));
       res.json(null);
     }));
@@ -260,11 +276,13 @@ export class DocWorkerApi {
     }));
 
     this._app.get('/api/docs/:docId/states', canView, withDoc(async (activeDoc, req, res) => {
-      res.json(await this._getStates(activeDoc));
+      const docSession = docSessionFromRequest(req);
+      res.json(await this._getStates(docSession, activeDoc));
     }));
 
     this._app.get('/api/docs/:docId/compare/:docId2', canView, withDoc(async (activeDoc, req, res) => {
-      const {states} = await this._getStates(activeDoc);
+      const docSession = docSessionFromRequest(req);
+      const {states} = await this._getStates(docSession, activeDoc);
       const ref = await fetch(this._grist.getHomeUrl(req, `/api/docs/${req.params.docId2}/states`), {
         headers: {
           ...getTransitiveHeaders(req),
@@ -359,7 +377,7 @@ export class DocWorkerApi {
   }
 
   private _getActiveDoc(req: RequestWithLogin): Promise<ActiveDoc> {
-    return this._docManager.fetchDoc({ client: null, req }, getDocId(req));
+    return this._docManager.fetchDoc(docSessionFromRequest(req), getDocId(req));
   }
 
   private _getActiveDocIfAvailable(req: RequestWithLogin): Promise<ActiveDoc>|undefined {
@@ -375,6 +393,15 @@ export class DocWorkerApi {
     next();
   }
 
+  /**
+   * Check if user is an owner of the document.
+   */
+  private async _isOwner(req: Request) {
+    const scope = getDocScope(req);
+    const docAuth = await getOrSetDocAuth(req as RequestWithLogin, this._dbManager, scope.urlId);
+    return docAuth.access === 'owners';
+  }
+
   // Helper to generate a 503 if the ActiveDoc has been muted.
   private _checkForMute(activeDoc: ActiveDoc|undefined) {
     if (activeDoc && activeDoc.muted) {
@@ -403,8 +430,8 @@ export class DocWorkerApi {
     };
   }
 
-  private async _getStates(activeDoc: ActiveDoc): Promise<DocStates> {
+  private async _getStates(docSession: OptDocSession, activeDoc: ActiveDoc): Promise<DocStates> {
-    const states = await activeDoc.getRecentStates();
+    const states = await activeDoc.getRecentStates(docSession);
     return {
       states,
     };

app/server/lib/DocClients.ts

@@ -8,7 +8,7 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {Authorizer} from 'app/server/lib/Authorizer';
 import {Client} from 'app/server/lib/Client';
 import {sendDocMessage} from 'app/server/lib/Comm';
-import {DocSession} from 'app/server/lib/DocSession';
+import {DocSession, OptDocSession} from 'app/server/lib/DocSession';
 import * as log from 'app/server/lib/log';
 
 export class DocClients {
@@ -73,14 +73,26 @@ export class DocClients {
    * @param {Object} client: Originating client used to set the `fromSelf` flag in the message.
    * @param {String} type: The type of the message, e.g. 'docUserAction'.
    * @param {Object} messageData: The data for this type of message.
+   * @param {Object} filterMessage: Optional callback to filter message per client.
    */
-  public async broadcastDocMessage(client: Client|null, type: string, messageData: any): Promise<void> {
+  public async broadcastDocMessage(client: Client|null, type: string, messageData: any,
+                                   filterMessage?: (docSession: OptDocSession,
+                                                    messageData: any) => any): Promise<void> {
     await Promise.all(this._docSessions.map(async curr => {
       const fromSelf = (curr.client === client);
       try {
         // Make sure user still has view access.
         await curr.authorizer.assertAccess('viewers');
-        sendDocMessage(curr.client, curr.fd, type, messageData, fromSelf);
+        if (!filterMessage) {
+          sendDocMessage(curr.client, curr.fd, type, messageData, fromSelf);
+        } else {
+          const filteredMessageData = filterMessage(curr, messageData);
+          if (filteredMessageData) {
+            sendDocMessage(curr.client, curr.fd, type, filteredMessageData, fromSelf);
+          } else {
+            this.activeDoc.logDebug(curr, 'skip broadcastDocMessage because it is not allowed for this client');
+          }
+        }
       } catch (e) {
         if (e.code === 'AUTH_NO_VIEW') {
           // Skip sending data to this user, they have no view access.

app/server/lib/DocSession.ts

@@ -39,6 +39,14 @@ export function makeExceptionalDocSession(mode: 'nascent'|'plugin'|'system',
   return docSession;
 }
 
+/**
+ * Create an OptDocSession from a request.  Request should have user and doc access
+ * middleware.
+ */
+export function docSessionFromRequest(req: RequestWithLogin): OptDocSession {
+  return {client: null, req};
+}
+
 /**
  * DocSession objects maintain information for a single session<->doc instance.
  */

app/server/lib/FlexServer.ts

@@ -38,8 +38,8 @@ import {getLoginMiddleware} from 'app/server/lib/logins';
 import {getAppPathTo, getAppRoot, getUnpackedAppRoot} from 'app/server/lib/places';
 import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint';
 import {PluginManager} from 'app/server/lib/PluginManager';
-import {adaptServerUrl, addPermit, getScope, optStringParam, RequestWithGristInfo, stringParam, TEST_HTTPS_OFFSET,
+import {adaptServerUrl, addOrgToPathIfNeeded, addPermit, getScope, optStringParam, RequestWithGristInfo, stringParam,
-        trustOrigin} from 'app/server/lib/requestUtils';
+        TEST_HTTPS_OFFSET, trustOrigin} from 'app/server/lib/requestUtils';
 import {ISendAppPageOptions, makeSendAppPage} from 'app/server/lib/sendAppPage';
 import * as ServerMetrics from 'app/server/lib/ServerMetrics';
 import {getDatabaseUrl} from 'app/server/lib/serverUtils';
@@ -1082,9 +1082,12 @@ export class FlexServer implements GristServer {
   private addSupportPaths(docAccessMiddleware: express.RequestHandler[]) {
     if (!this._docWorker) { throw new Error("need DocWorker"); }
 
-    // TODO: Need a way to translate docName from query to filesystem path. Use OpenDocManager.
     this.app.get('/download', ...docAccessMiddleware, expressWrap(async (req, res) => {
-      return this._docWorker.downloadDoc(req, res, this._storageManager);
+      // Forward this endpoint to regular API.  This endpoint is now deprecated.
+      const docId = String(req.query.doc);
+      let url = await this.getHomeUrlByDocId(docId, addOrgToPathIfNeeded(req, `/api/docs/${docId}/download`));
+      if (req.query.template === '1') { url += '?template=1'; }
+      return res.redirect(url);
     }));
 
     const basicMiddleware = [this._userIdMiddleware, this.tagChecker.requireTag];
@@ -1094,7 +1097,9 @@ export class FlexServer implements GristServer {
     // This doesn't check for doc access permissions because the request isn't tied to a document.
     addUploadRoute(this, this.app, ...basicMiddleware);
 
-    this.app.get('/gen_csv', ...docAccessMiddleware, (req, res) => this._docWorker.getCSV(req, res));
+    this.app.get('/gen_csv', ...docAccessMiddleware, expressWrap(async (req, res) => {
+      return this._docWorker.getCSV(req, res);
+    }));
 
     this.app.get('/attachment', ...docAccessMiddleware,
       expressWrap(async (req, res) => this._docWorker.getAttachment(req, res)));

app/server/lib/GranularAccess.ts

@@ -0,0 +1,290 @@
+import { ActionGroup } from 'app/common/ActionGroup';
+import { createEmptyActionSummary } from 'app/common/ActionSummary';
+import { Query } from 'app/common/ActiveDocAPI';
+import { BulkColValues, DocAction, TableDataAction, UserAction } from 'app/common/DocActions';
+import { DocData } from 'app/common/DocData';
+import { canView } from 'app/common/roles';
+import { TableData } from 'app/common/TableData';
+import { getDocSessionAccess, OptDocSession } from 'app/server/lib/DocSession';
+
+// Actions that may be allowed for a user with nuanced access to a document, depending
+// on what table they refer to.
+const ACTION_WITH_TABLE_ID = new Set(['AddRecord', 'BulkAddRecord', 'UpdateRecord', 'BulkUpdateRecord',
+                                      'RemoveRecord', 'BulkRemoveRecord',
+                                      'ReplaceTableData', 'TableData',
+                                    ]);
+
+// Actions that won't be allowed (yet) for a user with nuanced access to a document.
+// A few may be innocuous, but generally I've put them in this list if there are problems
+// tracking down what table the refer to, or they could allow creation/modification of a
+// formula.
+const SPECIAL_ACTIONS = new Set(['InitNewDoc',
+                                 'EvalCode',
+                                 'SetDisplayFormula',
+                                 'CreateViewSection',
+                                 'UpdateSummaryViewSection',
+                                 'DetachSummaryViewSection',
+                                 'GenImporterView',
+                                 'TransformAndFinishImport',
+                                 'AddColumn', 'RemoveColumn', 'RenameColumn', 'ModifyColumn',
+                                 'AddTable', 'RemoveTable', 'RenameTable',
+                                 'AddView',
+                                 'CopyFromColumn',
+                                 'AddHiddenColumn',
+                                 'RemoveViewSection'
+                                ]);
+
+// Odd-ball actions marked as deprecated or which seem unlikely to be used.
+const SURPRISING_ACTIONS = new Set(['AddUser',
+                                    'RemoveUser',
+                                    'AddInstance',
+                                    'RemoveInstance',
+                                    'RemoveView',
+                                    'AddViewSection',
+                                   ]);
+
+// Actions we'll allow unconditionally for now.
+const OK_ACTIONS = new Set(['Calculate', 'AddEmptyTable']);
+
+/**
+ * Manage granular access to a document.  This allows nuances other than the coarse
+ * owners/editors/viewers distinctions.
+ *
+ * Currently the only supported nuance is to mark certain tables as accessible by
+ * owners only.  To do so, in the _grist_ACLResources table, add a row like the
+ * one already there, but with "~o" as the colIds, and the desired tableId set.
+ * This is just a placeholder for a future representation.
+ */
+export class GranularAccess {
+  private _resources: TableData;
+  private _ownerOnlyTableIds = new Set<string>();
+
+  public constructor(private _docData: DocData) {
+    this.update();
+  }
+
+  /**
+   * Update granular access from DocData.
+   */
+  public update() {
+    this._resources = this._docData.getTable('_grist_ACLResources')!;
+    this._ownerOnlyTableIds.clear();
+    for (const res of this._resources.getRecords()) {
+      if (res.tableId && String(res.colIds).startsWith('~o')) {
+        this._ownerOnlyTableIds.add(String(res.tableId));
+      }
+    }
+  }
+
+  /**
+   * Check whether user can carry out query.
+   */
+  public hasQueryAccess(docSession: OptDocSession, query: Query) {
+    return this.hasTableAccess(docSession, query.tableId);
+  }
+
+  /**
+   * Check whether user has access to table.
+   */
+  public hasTableAccess(docSession: OptDocSession, tableId: string) {
+    return !this._ownerOnlyTableIds.has(tableId) || this.hasFullAccess(docSession);
+  }
+
+  /**
+   * Filter DocActions to be sent to a client.
+   */
+  public filterOutgoingDocActions(docSession: OptDocSession, docActions: DocAction[]): DocAction[] {
+    return docActions.filter(action => this.canApplyUserAction(docSession, action, 'out'));
+  }
+
+  /**
+   * Filter an ActionGroup to be sent to a client.
+   */
+  public filterActionGroup(docSession: OptDocSession, actionGroup: ActionGroup): ActionGroup {
+    if (!this.allowActionGroup(docSession, actionGroup)) { return actionGroup; }
+    // For now, if there's any nuance at all, suppress the summary and description.
+    // TODO: create an empty action summary, to be sure not to leak anything important.
+    const result: ActionGroup = { ...actionGroup };
+    result.actionSummary = createEmptyActionSummary();
+    result.desc = '';
+    return result;
+  }
+
+  /**
+   * Check whether an ActionGroup can be sent to the client.  TODO: in future, we'll want
+   * to filter acceptible parts of ActionGroup, rather than denying entirely.
+   */
+  public allowActionGroup(docSession: OptDocSession, actionGroup: ActionGroup): boolean {
+    return !this.hasNuancedAccess(docSession);
+  }
+
+  /**
+   * Check if user can apply a list of actions.
+   */
+  public canApplyUserActions(docSession: OptDocSession, actions: UserAction[]): boolean {
+    return actions.every(action => this.canApplyUserAction(docSession, action));
+  }
+
+  /**
+   * Check if user can apply a given action.
+   * When the direction is 'in', we are checking if it is ok for user to apply the
+   * action on the document.  When the direction is 'out', we are checking if it
+   * is ok to send the action to the user's client.
+   */
+  public canApplyUserAction(docSession: OptDocSession, a: UserAction|DocAction,
+                            direction: 'in' | 'out' = 'in'): boolean {
+    const name = a[0] as string;
+    if (OK_ACTIONS.has(name)) { return true; }
+    if (SPECIAL_ACTIONS.has(name)) {
+      // When broadcasting to client, allow renames etc for now.
+      // This is a bit weak, since it leaks changes to private table schemas.
+      // TODO: tighten up.
+      if (direction === 'out') { return true; }
+      return !this.hasNuancedAccess(docSession);
+    }
+    if (SURPRISING_ACTIONS.has(name)) {
+      return this.hasFullAccess(docSession);
+    }
+    const isTableAction = ACTION_WITH_TABLE_ID.has(name);
+    if (a[0] === 'ApplyUndoActions') {
+      return this.canApplyUserActions(docSession, a[1] as UserAction[]);
+    } else if (a[0] === 'ApplyDocActions') {
+      return this.canApplyUserActions(docSession, a[1] as UserAction[]);
+    } else if (isTableAction) {
+      const tableId = a[1] as string;
+      // Allow _grist_ table info to be broadcast to client unconditionally.
+      // This is a bit weak, since it leaks changes to private table schemas.
+      // TODO: tighten up.
+      if (tableId.startsWith('_grist_') && direction === 'in') {
+        return !this.hasNuancedAccess(docSession);
+      }
+      return this.hasTableAccess(docSession, tableId);
+    }
+    return false;
+  }
+
+  /**
+   * Check whether access is simple, or there are granular nuances that need to be
+   * worked through.  Currently if there are no owner-only tables, then everyone's
+   * access is simple and without nuance.
+   */
+  public hasNuancedAccess(docSession: OptDocSession): boolean {
+    if (this._ownerOnlyTableIds.size === 0) { return false; }
+    return !this.hasFullAccess(docSession);
+  }
+
+  /**
+   * Check whether user has owner-level access to the document.
+   */
+  public hasFullAccess(docSession: OptDocSession): boolean {
+    const access = getDocSessionAccess(docSession);
+    return access === 'owners';
+  }
+
+  /**
+   * Check for view access to the document.  For most code paths, a request or message
+   * won't even be considered if there isn't view access, but there's no harm in double
+   * checking.
+   */
+  public hasViewAccess(docSession: OptDocSession): boolean {
+    const access = getDocSessionAccess(docSession);
+    return canView(access);
+  }
+
+  /**
+   *
+   * If the user does not have access to the full document, we need to filter out
+   * parts of the document metadata.  For simplicity, we overwrite rather than
+   * filter for now, so that the overall structure remains consistent.  We overwrite:
+   *
+   *   - names, textual ids, formulas, and other textual options
+   *   - foreign keys linking columns/views/sections back to a forbidden table
+   *
+   * On the client, a page with a blank name will be marked gracefully as unavailable.
+   *
+   * Some information leaks, for example the existence of private tables and how
+   * many columns they had, and something of the relationships between them. Long term,
+   * it could be better to zap rows entirely, and do the work of cleaning up any cross
+   * references to them.
+   *
+   */
+  public filterMetaTables(docSession: OptDocSession,
+                          tables: {[key: string]: TableDataAction}): {[key: string]: TableDataAction} {
+    // If there are no nuances, return immediately.
+    if (!this.hasNuancedAccess(docSession)) { return tables; }
+    // If we are going to modify metadata, make a copy.
+    tables = JSON.parse(JSON.stringify(tables));
+    // Collect a list of all tables (by tableRef) to which the user has no access.
+    const censoredTables: Set<number> = new Set();
+    for (const tableId of this._ownerOnlyTableIds) {
+      if (this.hasTableAccess(docSession, tableId)) { continue; }
+      const tableRef = this._docData.getTable('_grist_Tables')?.findRow('tableId', tableId);
+      if (tableRef) { censoredTables.add(tableRef); }
+    }
+    // Collect a list of all sections and views containing a table to which the user has no access.
+    const censoredSections: Set<number> = new Set();
+    const censoredViews: Set<number> = new Set();
+    for (const section of this._docData.getTable('_grist_Views_section')?.getRecords() || []) {
+      if (!censoredTables.has(section.tableRef as number)) { continue; }
+      if (section.parentId) { censoredViews.add(section.parentId as number); }
+      censoredSections.add(section.id);
+    }
+    // Collect a list of all columns from tables to which the user has no access.
+    const censoredColumns: Set<number> = new Set();
+    for (const column of this._docData.getTable('_grist_Tables_column')?.getRecords() || []) {
+      if (!censoredTables.has(column.parentId as number)) { continue; }
+      censoredColumns.add(column.id);
+    }
+    // Collect a list of all fields from sections to which the user has no access.
+    const censoredFields: Set<number> = new Set();
+    for (const field of this._docData.getTable('_grist_Views_section_field')?.getRecords() || []) {
+      if (!censoredSections.has(field.parentId as number)) { continue; }
+      censoredFields.add(field.id);
+    }
+    // Clear the tableId for any tables the user does not have access to.  This is just
+    // to keep the name of the table private, in case its name itself is sensitive.
+    // TODO: tableId may appear elsewhere, such as in _grist_ACLResources - user with
+    // nuanced rights probably should not receive that table.
+    this._censor(tables._grist_Tables, censoredTables, (idx, cols) => {
+      cols.tableId[idx] = '';
+    });
+    // Clear the name of private views, in case the name itself is sensitive.
+    this._censor(tables._grist_Views, censoredViews, (idx, cols) => {
+      cols.name[idx] = '';
+    });
+    // Clear the title of private sections, and break the connection with the private
+    // table as extra grit in the way of snooping.
+    this._censor(tables._grist_Views_section, censoredSections, (idx, cols) => {
+      cols.title[idx] = '';
+      cols.tableRef[idx] = 0;
+    });
+    // Clear text metadata from private columns, and break the connection with the
+    // private table.
+    this._censor(tables._grist_Tables_column, censoredColumns, (idx, cols) => {
+      cols.label[idx] = cols.colId[idx] = '';
+      cols.widgetOptions[idx] = cols.formula[idx] = '';
+      cols.type[idx] = 'Any';
+      cols.parentId[idx] = 0;
+    });
+    // Clear text metadata from private fields, and break the connection with the
+    // private table.
+    this._censor(tables._grist_Views_section_field, censoredFields, (idx, cols) => {
+      cols.widgetOptions[idx] = cols.filter[idx] = '';
+      cols.parentId[idx] = 0;
+    });
+    return tables;
+  }
+
+  /**
+   * Modify the given TableDataAction in place by calling the supplied operation with
+   * the indexes of any ids supplied and the columns in that TableDataAction.
+   */
+  public _censor(table: TableDataAction, ids: Set<number>,
+                 op: (idx: number, cols: BulkColValues) => unknown) {
+    const availableIds = table[2];
+    const cols = table[3];
+    for (let idx = 0; idx < availableIds.length; idx++) {
+      if (ids.has(availableIds[idx])) { op(idx, cols); }
+    }
+  }
+}

app/server/lib/requestUtils.ts

@@ -1,5 +1,5 @@
 import {ApiError} from 'app/common/ApiError';
-import {DEFAULT_HOME_SUBDOMAIN, parseSubdomain} from 'app/common/gristUrls';
+import {DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain} from 'app/common/gristUrls';
 import * as gutil from 'app/common/gutil';
 import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/HomeDBManager';
 import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
@@ -47,6 +47,13 @@ export function adaptServerUrl(url: URL, req: RequestWithOrg): void {
   }
 }
 
+/**
+ * If org is not encoded in domain, prefix it to path - otherwise leave path unchanged.
+ */
+export function addOrgToPathIfNeeded(req: RequestWithOrg, path: string): string {
+  return (isOrgInPathOnly(req.hostname) && req.org) ? `/o/${req.org}${path}` : path;
+}
+
 /**
  * Returns true for requests from permitted origins.  For such requests, an
  * "Access-Control-Allow-Origin" header is added to the response.  Vary: Origin

app/server/serverMethods.js

@@ -1,6 +1,7 @@
 const gutil = require('app/common/gutil');
 const {SortFunc} = require('app/common/SortFunc');
 const ValueFormatter = require('app/common/ValueFormatter');
+const {docSessionFromRequest} = require('app/server/lib/DocSession');
 const Promise = require('bluebird');
 const contentDisposition = require('content-disposition');
 const csv = require('csv');
@@ -88,7 +89,7 @@ function makeCSV(activeDoc, viewSectionId, sortOrder, req) {
       return directionalColRef > 0 ? effectiveColRef : -effectiveColRef;
     });
 
-    return [activeDoc.fetchTable({client: null, req}, table.tableId, true), tableColumns, viewColumns];
+    return [activeDoc.fetchTable(docSessionFromRequest(req), table.tableId, true), tableColumns, viewColumns];
   }).spread((data, tableColumns, viewColumns) => {
     const rowIds = data[2];
     const dataByColId = data[3];

sandbox/grist/acl.py

@@ -119,6 +119,9 @@ class ResourceMap(object):
       table_id = resource.tableId or None
       if not resource.colIds:
         self._default_resources[table_id] = resource
+      elif resource.colIds.startswith('~'):
+        # Rows with colIds that start with '~' are for trial purposes - ignore.
+        pass
       else:
         col_id_set = set(resource.colIds.split(','))
         self._col_resources.setdefault(table_id, []).append((resource, col_id_set))