Merge branch 'main' of github.com:gristlabs/grist-core into alex/_importParsedFileAsNewTable

2026-03-02 04:09:24 +00:00 · 2023-07-11 13:11:05 +02:00
parent 25f7a89442 958ea096f3
commit 6fbe3db677
131 changed files with 4296 additions and 983 deletions
--- a/app/server/companion.ts
+++ b/app/server/companion.ts
@@ -6,7 +6,7 @@ import { applyPatch } from 'app/gen-server/lib/TypeORMPatches';
 import { getMigrations, getOrCreateConnection, getTypeORMSettings,
         undoLastMigration, updateDb } from 'app/server/lib/dbUtils';
 import { getDatabaseUrl } from 'app/server/lib/serverUtils';
-import { getTelemetryLevel } from 'app/server/lib/Telemetry';
+import { getTelemetryPrefs } from 'app/server/lib/Telemetry';
 import { Gristifier } from 'app/server/utils/gristify';
 import { pruneActionHistory } from 'app/server/utils/pruneActionHistory';
 import * as commander from 'commander';
@@ -81,12 +81,14 @@ export function addSettingsCommand(program: commander.Command,
    .action(showTelemetry);
 }

-function showTelemetry(options: {
+async function showTelemetry(options: {
  json?: boolean,
  all?: boolean,
 }) {
  const contracts = TelemetryContracts;
-  const levelName = getTelemetryLevel();
+  const db = await getHomeDBManager();
+  const prefs = await getTelemetryPrefs(db);
+  const levelName = prefs.telemetryLevel.value;
  const level = Level[levelName];
  if (options.json) {
    console.log(JSON.stringify({
--- a/app/server/lib/ActionHistory.ts
+++ b/app/server/lib/ActionHistory.ts
@@ -16,9 +16,7 @@
 import {LocalActionBundle} from 'app/common/ActionBundle';
 import {ActionGroup, MinimalActionGroup} from 'app/common/ActionGroup';
 import {createEmptyActionSummary} from 'app/common/ActionSummary';
-import {getSelectionDesc, UserAction} from 'app/common/DocActions';
 import {DocState} from 'app/common/UserAPI';
-import toPairs = require('lodash/toPairs');
 import {summarizeAction} from 'app/common/ActionSummarizer';

 export interface ActionGroupOptions {
@@ -163,81 +161,6 @@ export abstract class ActionHistory {
 }


-/**
- * Old helper to display the actionGroup in a human-readable way.  Being maintained
- * to avoid having to change too much at once.
- */
-export function humanDescription(actions: UserAction[]): string {
-  const action = actions[0];
-  if (!action) { return ""; }
-  let output = '';
-  // Common names for various action parameters
-  const name    = action[0];
-  const table   = action[1];
-  const rows    = action[2];
-  const colId   = action[2];
-  const columns: any = action[3];  // TODO - better typing - but code may evaporate
-  switch (name) {
-  case 'UpdateRecord':
-  case 'BulkUpdateRecord':
-  case 'AddRecord':
-  case 'BulkAddRecord':
-    output = name + ' ' + getSelectionDesc(action, columns);
-    break;
-  case 'ApplyUndoActions':
-    // Currently cannot display information about what action was undone, as the action comes
-    // with the description of the "undo" message, which might be very different
-    // Also, cannot currently properly log redos as they are not distinguished from others in any way
-    // TODO: make an ApplyRedoActions type for redoing actions
-    output = 'Undo Previous Action';
-    break;
-  case 'InitNewDoc':
-    output = 'Initialized new Document';
-    break;
-  case 'AddColumn':
-    output = 'Added column ' + colId + ' to ' + table;
-    break;
-  case 'RemoveColumn':
-    output = 'Removed column ' + colId + ' from ' + table;
-    break;
-  case 'RemoveRecord':
-  case 'BulkRemoveRecord':
-    output = 'Removed record(s) ' + rows + ' from ' + table;
-    break;
-  case 'EvalCode':
-    output = 'Evaluated Code ' + action[1];
-    break;
-  case 'AddTable':
-    output = 'Added table ' + table;
-    break;
-  case 'RemoveTable':
-    output = 'Removed table ' + table;
-    break;
-  case 'ModifyColumn':
-    // TODO: The Action Log currently only logs user actions,
-    // But ModifyColumn/Rename Column are almost always triggered from the client
-    // through a meta-table UpdateRecord.
-    // so, this is a case where making use of explicit sandbox engine 'looged' actions
-    // may be useful
-    output = 'Modify column ' + colId + ", ";
-    for (const [col, val] of toPairs(columns)) {
-      output += col + ": " + val + ", ";
-    }
-    output += ' in table ' + table;
-    break;
-  case 'RenameColumn': {
-    const newColId = action[3];
-    output = 'Renamed Column ' + colId + ' to ' + newColId + ' in ' + table;
-    break;
-  }
-  default:
-    output = name + ' [No Description]';
-  }
-  // A period for good grammar
-  output += '.';
-  return output;
-}
-
 /**
 * Convert an ActionBundle into an ActionGroup.  ActionGroups are the representation of
 * actions on the client.
@@ -260,7 +183,9 @@ export function asActionGroup(history: ActionHistory,
  return {
    actionNum: act.actionNum,
    actionHash: act.actionHash || "",
-    desc: info.desc || humanDescription(act.userActions),
+    // Desc is a human-readable description of the user action set in a few places by client-side
+    // code, but is mostly (or maybe completely) unused.
+    desc: info.desc,
    actionSummary: summarize ? summarizeAction(act) : createEmptyActionSummary(),
    fromSelf,
    linkId: info.linkId,
--- a/app/server/lib/ActiveDoc.ts
+++ b/app/server/lib/ActiveDoc.ts
@@ -14,7 +14,6 @@ import {
 } from 'app/common/ActionBundle';
 import {ActionGroup, MinimalActionGroup} from 'app/common/ActionGroup';
 import {ActionSummary} from "app/common/ActionSummary";
-import {AssistanceRequest, AssistanceResponse} from "app/common/AssistancePrompts";
 import {
  AclResources,
  AclTableDescription,
@@ -68,14 +67,12 @@ import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccess
 import {isHiddenCol} from 'app/common/gristTypes';
 import {commonUrls, parseUrlId} from 'app/common/gristUrls';
 import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil';
-import {hashId} from 'app/common/hashingUtils';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import {Interval} from 'app/common/Interval';
 import * as roles from 'app/common/roles';
 import {schema, SCHEMA_VERSION} from 'app/common/schema';
-import {MetaRowRecord, SingleCell} from 'app/common/TableData';
+import {MetaRowRecord, SingleCell, UIRowId} from 'app/common/TableData';
 import {TelemetryEvent, TelemetryMetadataByLevel} from 'app/common/Telemetry';
-import {UIRowId} from 'app/common/UIRowId';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {Document as APIDocument, DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
 import {convertFromColumn} from 'app/common/ValueConverter';
@@ -86,7 +83,7 @@ import {Document} from 'app/gen-server/entity/Document';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
 import {AccessTokenOptions, AccessTokenResult, GristDocAPI} from 'app/plugin/GristAPI';
 import {compileAclFormula} from 'app/server/lib/ACLFormula';
-import {AssistanceDoc, AssistanceSchemaPromptV1Context, sendForCompletion} from 'app/server/lib/Assistance';
+import {AssistanceSchemaPromptV1Context} from 'app/server/lib/Assistance';
 import {Authorizer} from 'app/server/lib/Authorizer';
 import {checksumFile} from 'app/server/lib/checksumFile';
 import {Client} from 'app/server/lib/Client';
@@ -186,7 +183,7 @@ interface UpdateUsageOptions {
 * either .loadDoc() or .createEmptyDoc() is called.
 * @param {String} docName - The document's filename, without the '.grist' extension.
 */
-export class ActiveDoc extends EventEmitter implements AssistanceDoc {
+export class ActiveDoc extends EventEmitter {
  /**
   * Decorator for ActiveDoc methods that prevents shutdown while the method is running, i.e.
   * until the returned promise is resolved.
@@ -1266,18 +1263,14 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
    return this._pyCall('autocomplete', txt, tableId, columnId, rowId, user.toJSON());
  }

-  public async getAssistance(docSession: DocSession, request: AssistanceRequest): Promise<AssistanceResponse> {
-    return this.getAssistanceWithOptions(docSession, request);
-  }
-
-  public async getAssistanceWithOptions(docSession: DocSession,
-                                        request: AssistanceRequest): Promise<AssistanceResponse> {
+  // Callback to generate a prompt containing schema info for assistance.
+  public async assistanceSchemaPromptV1(
+    docSession: OptDocSession, options: AssistanceSchemaPromptV1Context): Promise<string> {
    // Making a prompt leaks names of tables and columns etc.
    if (!await this._granularAccess.canScanData(docSession)) {
      throw new Error("Permission denied");
    }
-    await this.waitForInitialization();
-    return sendForCompletion(this, request);
+    return await this._pyCall('get_formula_prompt', options.tableId, options.colId, options.docString);
  }

  // Callback to make a data-engine formula tweak for assistance.
@@ -1285,11 +1278,6 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
    return this._pyCall('convert_formula_completion', txt);
  }

-  // Callback to generate a prompt containing schema info for assistance.
-  public assistanceSchemaPromptV1(options: AssistanceSchemaPromptV1Context): Promise<string> {
-    return this._pyCall('get_formula_prompt', options.tableId, options.colId, options.docString);
-  }
-
  public fetchURL(docSession: DocSession, url: string, options?: FetchUrlOptions): Promise<UploadResult> {
    return fetchURL(url, this.makeAccessId(docSession.authorizer.getUserId()), options);
  }
@@ -1396,9 +1384,9 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
      const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
      this.logTelemetryEvent(docSession, 'documentForked', {
        limited: {
-          forkIdDigest: hashId(forkIds.forkId),
-          forkDocIdDigest: hashId(forkIds.docId),
-          trunkIdDigest: doc.trunkId ? hashId(doc.trunkId) : undefined,
+          forkIdDigest: forkIds.forkId,
+          forkDocIdDigest: forkIds.docId,
+          trunkIdDigest: doc.trunkId,
          isTemplate,
          lastActivity: doc.updatedAt,
        },
@@ -2540,7 +2528,7 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
      altSessionId ? {altSessionId} : {},
      {
        limited: {
-          docIdDigest: hashId(this._docName),
+          docIdDigest: this._docName,
        },
        full: {
          siteId: this._doc?.workspace.org.id,
--- a/app/server/lib/AppEndpoint.ts
+++ b/app/server/lib/AppEndpoint.ts
@@ -9,7 +9,6 @@ import fetch, {Response as FetchResponse, RequestInit} from 'node-fetch';
 import {ApiError} from 'app/common/ApiError';
 import {getSlugIfNeeded, parseSubdomainStrictly, parseUrlId} from 'app/common/gristUrls';
 import {removeTrailingSlash} from 'app/common/gutil';
-import {hashId} from 'app/common/hashingUtils';
 import {LocalPlugin} from "app/common/plugin";
 import {TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME} from 'app/common/Telemetry';
 import {Document as APIDocument} from 'app/common/UserAPI';
@@ -304,13 +303,13 @@ export function attachAppEndpoint(options: AttachOptions): void {
    }

    const isPublic = ((doc as unknown) as APIDocument).public ?? false;
-    const isSnapshot = parseUrlId(urlId).snapshotId;
+    const isSnapshot = Boolean(parseUrlId(urlId).snapshotId);
    // TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
    const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
    if (isPublic || isTemplate) {
      gristServer.getTelemetry().logEvent('documentOpened', {
        limited: {
-          docIdDigest: hashId(docId),
+          docIdDigest: docId,
          access: doc.access,
          isPublic,
          isSnapshot,
--- a/app/server/lib/Assistance.ts
+++ b/app/server/lib/Assistance.ts
@@ -5,6 +5,7 @@
 import {AssistanceRequest, AssistanceResponse} from 'app/common/AssistancePrompts';
 import {delay} from 'app/common/delay';
 import {DocAction} from 'app/common/DocActions';
+import {OptDocSession} from 'app/server/lib/DocSession';
 import log from 'app/server/lib/log';
 import fetch from 'node-fetch';

@@ -15,7 +16,7 @@ export const DEPS = { fetch };
 * by interfacing with an external LLM endpoint.
 */
 export interface Assistant {
-  apply(doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse>;
+  apply(session: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse>;
 }

 /**
@@ -30,8 +31,7 @@ export interface AssistanceDoc {
   * Marked "V1" to suggest that it is a particular prompt and it would
   * be great to try variants.
   */
-  assistanceSchemaPromptV1(options: AssistanceSchemaPromptV1Context): Promise<string>;
-
+  assistanceSchemaPromptV1(session: OptDocSession, options: AssistanceSchemaPromptV1Context): Promise<string>;
  /**
   * Some tweaks to a formula after it has been generated.
   */
@@ -46,7 +46,7 @@ export interface AssistanceSchemaPromptV1Context {

 /**
 * A flavor of assistant for use with the OpenAI API.
- * Tested primarily with text-davinci-002 and gpt-3.5-turbo.
+ * Tested primarily with gpt-3.5-turbo.
 */
 export class OpenAIAssistant implements Assistant {
  private _apiKey: string;
@@ -60,37 +60,43 @@ export class OpenAIAssistant implements Assistant {
      throw new Error('OPENAI_API_KEY not set');
    }
    this._apiKey = apiKey;
-    this._model = process.env.COMPLETION_MODEL || "text-davinci-002";
+    this._model = process.env.COMPLETION_MODEL || "gpt-3.5-turbo-0613";
    this._chatMode = this._model.includes('turbo');
+    if (!this._chatMode) {
+      throw new Error('Only turbo models are currently supported');
+    }
    this._endpoint = `https://api.openai.com/v1/${this._chatMode ? 'chat/' : ''}completions`;
  }

-  public async apply(doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse> {
+  public async apply(
+    optSession: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse> {
    const messages = request.state?.messages || [];
    const chatMode = this._chatMode;
    if (chatMode) {
      if (messages.length === 0) {
        messages.push({
          role: 'system',
-          content: 'The user gives you one or more Python classes, ' +
-            'with one last method that needs completing. Write the ' +
-            'method body as a single code block, ' +
-            'including the docstring the user gave. ' +
-            'Just give the Python code as a markdown block, ' +
-            'do not give any introduction, that will just be ' +
-            'awkward for the user when copying and pasting. ' +
-            'You are working with Grist, an environment very like ' +
-            'regular Python except `rec` (like record) is used ' +
-            'instead of `self`. ' +
-            'Include at least one `return` statement or the method ' +
-            'will fail, disappointing the user. ' +
-            'Your answer should be the body of a single method, ' +
-            'not a class, and should not include `dataclass` or ' +
-            '`class` since the user is counting on you to provide ' +
-            'a single method. Thanks!'
+          content: 'You are a helpful assistant for a user of software called Grist. ' +
+            'Below are one or more Python classes. ' +
+            'The last method needs completing. ' +
+            "The user will probably give a description of what they want the method (a 'formula') to return. " +
+            'If so, your response should include the method body as Python code in a markdown block. ' +
+            'Do not include the class or method signature, just the method body. ' +
+            'If your code starts with `class`, `@dataclass`, or `def` it will fail. Only give the method body. ' +
+            'You can import modules inside the method body if needed. ' +
+            'You cannot define additional functions or methods. ' +
+            'The method should be a pure function that performs some computation and returns a result. ' +
+            'It CANNOT perform any side effects such as adding/removing/modifying rows/columns/cells/tables/etc. ' +
+            'It CANNOT interact with files/databases/networks/etc. ' +
+            'It CANNOT display images/charts/graphs/maps/etc. ' +
+            'If the user asks for these things, tell them that you cannot help. ' +
+            'The method uses `rec` instead of `self` as the first parameter.\n\n' +
+            '```python\n' +
+            await makeSchemaPromptV1(optSession, doc, request) +
+            '\n```',
        });
        messages.push({
-          role: 'user', content: await makeSchemaPromptV1(doc, request),
+          role: 'user', content: request.text,
        });
      } else {
        if (request.regenerate) {
@@ -105,7 +111,7 @@ export class OpenAIAssistant implements Assistant {
    } else {
      messages.length = 0;
      messages.push({
-        role: 'user', content: await makeSchemaPromptV1(doc, request),
+        role: 'user', content: await makeSchemaPromptV1(optSession, doc, request),
      });
    }

@@ -173,11 +179,12 @@ export class HuggingFaceAssistant implements Assistant {

  }

-  public async apply(doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse> {
+  public async apply(
+    optSession: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse> {
    if (request.state) {
      throw new Error("HuggingFaceAssistant does not support state");
    }
-    const prompt = await makeSchemaPromptV1(doc, request);
+    const prompt = await makeSchemaPromptV1(optSession, doc, request);
    const response = await DEPS.fetch(
      this._completionUrl,
      {
@@ -215,7 +222,10 @@ export class HuggingFaceAssistant implements Assistant {
 * Test assistant that mimics ChatGPT and just returns the input.
 */
 export class EchoAssistant implements Assistant {
-  public async apply(doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse> {
+  public async apply(sess: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse> {
+    if (request.text === "ERROR") {
+      throw new Error(`ERROR`);
+    }
    const messages = request.state?.messages || [];
    if (messages.length === 0) {
      messages.push({
@@ -250,25 +260,28 @@ export class EchoAssistant implements Assistant {
 /**
 * Instantiate an assistant, based on environment variables.
 */
-function getAssistant() {
+export function getAssistant() {
  if (process.env.OPENAI_API_KEY === 'test') {
    return new EchoAssistant();
  }
  if (process.env.OPENAI_API_KEY) {
    return new OpenAIAssistant();
  }
-  if (process.env.HUGGINGFACE_API_KEY) {
-    return new HuggingFaceAssistant();
-  }
-  throw new Error('Please set OPENAI_API_KEY or HUGGINGFACE_API_KEY');
+  // Maintaining this is too much of a burden for now.
+  // if (process.env.HUGGINGFACE_API_KEY) {
+  //   return new HuggingFaceAssistant();
+  // }
+  throw new Error('Please set OPENAI_API_KEY');
 }

 /**
 * Service a request for assistance, with a little retry logic
 * since these endpoints can be a bit flakey.
 */
-export async function sendForCompletion(doc: AssistanceDoc,
-                                        request: AssistanceRequest): Promise<AssistanceResponse> {
+export async function sendForCompletion(
+  optSession: OptDocSession,
+  doc: AssistanceDoc,
+  request: AssistanceRequest): Promise<AssistanceResponse> {
  const assistant = getAssistant();

  let retries: number = 0;
@@ -276,7 +289,7 @@ export async function sendForCompletion(doc: AssistanceDoc,
  let response: AssistanceResponse|null = null;
  while(retries++ < 3) {
    try {
-      response = await assistant.apply(doc, request);
+      response = await assistant.apply(optSession, doc, request);
      break;
    } catch(e) {
      log.error(`Completion error: ${e}`);
@@ -289,11 +302,11 @@ export async function sendForCompletion(doc: AssistanceDoc,
  return response;
 }

-async function makeSchemaPromptV1(doc: AssistanceDoc, request: AssistanceRequest) {
+async function makeSchemaPromptV1(session: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest) {
  if (request.context.type !== 'formula') {
    throw new Error('makeSchemaPromptV1 only works for formulas');
  }
-  return doc.assistanceSchemaPromptV1({
+  return doc.assistanceSchemaPromptV1(session, {
    tableId: request.context.tableId,
    colId: request.context.colId,
    docString: request.text,
--- a/app/server/lib/DocApi.ts
+++ b/app/server/lib/DocApi.ts
@@ -1,5 +1,5 @@
 import {createEmptyActionSummary} from "app/common/ActionSummary";
-import {ApiError} from 'app/common/ApiError';
+import {ApiError, LimitType} from 'app/common/ApiError';
 import {BrowserSettings} from "app/common/BrowserSettings";
 import {
  BulkColValues,
@@ -11,7 +11,6 @@ import {
 import {isRaisedException} from "app/common/gristTypes";
 import {buildUrlId, parseUrlId} from "app/common/gristUrls";
 import {isAffirmative} from "app/common/gutil";
-import {hashId} from "app/common/hashingUtils";
 import {SchemaTypes} from "app/common/schema";
 import {SortFunc} from 'app/common/SortFunc';
 import {Sort} from 'app/common/SortSpec';
@@ -69,6 +68,7 @@ import {
 } from 'app/server/lib/requestUtils';
 import {ServerColumnGetters} from 'app/server/lib/ServerColumnGetters';
 import {localeFromRequest} from "app/server/lib/ServerLocale";
+import {sendForCompletion} from 'app/server/lib/Assistance';
 import {isUrlAllowed, WebhookAction, WebHookSecret} from "app/server/lib/Triggers";
 import {handleOptionalUpload, handleUpload} from "app/server/lib/uploads";
 import * as assert from 'assert';
@@ -162,6 +162,8 @@ export class DocWorkerApi {
    const canEditMaybeRemoved = expressWrap(this._assertAccess.bind(this, 'editors', true));
    // converts google code to access token and adds it to request object
    const decodeGoogleToken = expressWrap(googleAuthTokenMiddleware.bind(null));
+    // check that limit can be increased by 1
+    const checkLimit = (type: LimitType) => expressWrap(this._checkLimit.bind(this, type));

    // Middleware to limit number of outstanding requests per document.  Will also
    // handle errors like expressWrap would.
@@ -918,8 +920,8 @@ export class DocWorkerApi {
          const {forkId} = parseUrlId(scope.urlId);
          activeDoc.logTelemetryEvent(docSession, 'tutorialRestarted', {
            full: {
-              tutorialForkIdDigest: forkId ? hashId(forkId) : undefined,
-              tutorialTrunkIdDigest: hashId(tutorialTrunkId),
+              tutorialForkIdDigest: forkId,
+              tutorialTrunkIdDigest: tutorialTrunkId,
            },
          });
        }
@@ -1053,6 +1055,20 @@ export class DocWorkerApi {

    this._app.get('/api/docs/:docId/send-to-drive', canView, decodeGoogleToken, withDoc(exportToDrive));

+    /**
+     * Send a request to the formula assistant to get completions for a formula. Increases the
+     * usage of the formula assistant for the billing account in case of success.
+     */
+    this._app.post('/api/docs/:docId/assistant', canView, checkLimit('assistant'),
+      withDoc(async (activeDoc, req, res) => {
+        const docSession = docSessionFromRequest(req);
+        const request = req.body;
+        const result = await sendForCompletion(docSession, activeDoc, request);
+        await this._increaseLimit('assistant', req);
+        res.json(result);
+      })
+    );
+
    // Create a document.  When an upload is included, it is imported as the initial
    // state of the document.  Otherwise a fresh empty document is created.
    // A "timezone" option can be supplied.
@@ -1235,6 +1251,21 @@ export class DocWorkerApi {
    return false;
  }

+  /**
+   * Creates a middleware that checks the current usage of a limit and rejects the request if it is exceeded.
+   */
+  private async _checkLimit(limit: LimitType, req: Request, res: Response, next: NextFunction) {
+    await this._dbManager.increaseUsage(getDocScope(req), limit, {dryRun: true, delta: 1});
+    next();
+  }
+
+  /**
+   * Increases the current usage of a limit by 1.
+   */
+  private async _increaseLimit(limit: LimitType, req: Request) {
+    await this._dbManager.increaseUsage(getDocScope(req), limit, {delta: 1});
+  }
+
  private async _assertAccess(role: 'viewers'|'editors'|'owners'|null, allowRemoved: boolean,
                              req: Request, res: Response, next: NextFunction) {
    const scope = getDocScope(req);
--- a/app/server/lib/DocStorage.ts
+++ b/app/server/lib/DocStorage.ts
@@ -378,7 +378,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
          const colListSql = newCols.map(c => `${quoteIdent(c.colId)}=?`).join(', ');
          const types = newCols.map(c => c.type);
          const sqlParams = DocStorage._encodeColumnsToRows(types, newCols.map(c => [PENDING_VALUE]));
-          await db.run(`UPDATE ${quoteIdent(tableId)} SET ${colListSql}`, sqlParams[0]);
+          await db.run(`UPDATE ${quoteIdent(tableId)} SET ${colListSql}`, ...sqlParams[0]);
        }
      },

@@ -1093,7 +1093,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
  public _process_RemoveRecord(tableId: string, rowId: string): Promise<RunResult> {
    const sql = "DELETE FROM " + quoteIdent(tableId) + " WHERE id=?";
    debuglog("RemoveRecord SQL: " + sql, [rowId]);
-    return this.run(sql, [rowId]);
+    return this.run(sql, rowId);
  }


@@ -1130,7 +1130,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
      const stmt = await this.prepare(preSql + chunkParams + postSql);
      for (const index of _.range(0, numChunks * chunkSize, chunkSize)) {
        debuglog("DocStorage.BulkRemoveRecord: chunk delete " + index + "-" + (index + chunkSize - 1));
-        await stmt.run(rowIds.slice(index, index + chunkSize));
+        await stmt.run(...rowIds.slice(index, index + chunkSize));
      }
      await stmt.finalize();
    }
@@ -1139,7 +1139,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
      debuglog("DocStorage.BulkRemoveRecord: leftover delete " + (numChunks * chunkSize) + "-" + (rowIds.length - 1));
      const leftoverParams = _.range(numLeftovers).map(q).join(',');
      await this.run(preSql + leftoverParams + postSql,
-                     rowIds.slice(numChunks * chunkSize, rowIds.length));
+                     ...rowIds.slice(numChunks * chunkSize, rowIds.length));
    }
  }

--- a/app/server/lib/DocWorker.ts
+++ b/app/server/lib/DocWorker.ts
@@ -110,7 +110,6 @@ export class DocWorker {
      applyUserActionsById:     activeDocMethod.bind(null, 'editors', 'applyUserActionsById'),
      findColFromValues:        activeDocMethod.bind(null, 'viewers', 'findColFromValues'),
      getFormulaError:          activeDocMethod.bind(null, 'viewers', 'getFormulaError'),
-      getAssistance:            activeDocMethod.bind(null, 'editors', 'getAssistance'),
      importFiles:              activeDocMethod.bind(null, 'editors', 'importFiles'),
      finishImportFiles:        activeDocMethod.bind(null, 'editors', 'finishImportFiles'),
      cancelImportFiles:        activeDocMethod.bind(null, 'editors', 'cancelImportFiles'),
--- a/app/server/lib/FlexServer.ts
+++ b/app/server/lib/FlexServer.ts
@@ -5,6 +5,7 @@ import {encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
        GristLoadConfig, IGristUrlState, isOrgInPathOnly, parseSubdomain,
        sanitizePathTail} from 'app/common/gristUrls';
 import {getOrgUrlInfo} from 'app/common/gristUrls';
+import {InstallProperties} from 'app/common/InstallAPI';
 import {UserProfile} from 'app/common/LoginSessionAPI';
 import {tbind} from 'app/common/tbind';
 import * as version from 'app/common/version';
@@ -50,14 +51,15 @@ import {getAppPathTo, getAppRoot, getUnpackedAppRoot} from 'app/server/lib/place
 import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint';
 import {PluginManager} from 'app/server/lib/PluginManager';
 import * as ProcessMonitor from 'app/server/lib/ProcessMonitor';
-import {adaptServerUrl, getOrgUrl, getOriginUrl, getScope, optStringParam,
-        RequestWithGristInfo, stringParam, TEST_HTTPS_OFFSET, trustOrigin} from 'app/server/lib/requestUtils';
+import {adaptServerUrl, getOrgUrl, getOriginUrl, getScope, isDefaultUser, optStringParam,
+        RequestWithGristInfo, sendOkReply, stringParam, TEST_HTTPS_OFFSET,
+        trustOrigin} from 'app/server/lib/requestUtils';
 import {ISendAppPageOptions, makeGristConfig, makeMessagePage, makeSendAppPage} from 'app/server/lib/sendAppPage';
 import {getDatabaseUrl, listenPromise} from 'app/server/lib/serverUtils';
 import {Sessions} from 'app/server/lib/Sessions';
 import * as shutdown from 'app/server/lib/shutdown';
 import {TagChecker} from 'app/server/lib/TagChecker';
-import {ITelemetry} from 'app/server/lib/Telemetry';
+import {getTelemetryPrefs, ITelemetry} from 'app/server/lib/Telemetry';
 import {startTestingHooks} from 'app/server/lib/TestingHooks';
 import {getTestLoginSystem} from 'app/server/lib/TestLogin';
 import {addUploadRoute} from 'app/server/lib/uploads';
@@ -706,11 +708,17 @@ export class FlexServer implements GristServer {
    });
  }

-  public addTelemetry() {
+  public async addTelemetry() {
    if (this._check('telemetry', 'homedb', 'json', 'api-mw')) { return; }

    this._telemetry = this.create.Telemetry(this._dbManager, this);
    this._telemetry.addEndpoints(this.app);
+    this._telemetry.addPages(this.app, [
+      this._redirectToHostMiddleware,
+      this._userIdMiddleware,
+      this._redirectToLoginWithoutExceptionsMiddleware,
+    ]);
+    await this._telemetry.start();

    // Start up a monitor for memory and cpu usage.
    this._processMonitorStop = ProcessMonitor.start(this._telemetry);
@@ -1124,7 +1132,11 @@ export class FlexServer implements GristServer {
    await this.loadConfig();
    this.addComm();

+    // Temporary duplication of external storage configuration.
+    // This may break https://github.com/gristlabs/grist-core/pull/546,
+    // but will revive other uses of external storage. TODO: reconcile.
    await this.create.configure?.();
+
    if (!isSingleUserMode()) {
      const externalStorage = appSettings.section('externalStorage');
      const haveExternalStorage = Object.values(externalStorage.nested)
@@ -1135,6 +1147,7 @@ export class FlexServer implements GristServer {
        this._disableExternalStorage = true;
        externalStorage.flag('active').set(false);
      }
+      await this.create.configure?.();
      const workers = this._docWorkerMap;
      const docWorkerId = await this._addSelfAsWorker(workers);

@@ -1198,7 +1211,7 @@ export class FlexServer implements GristServer {
    ];

    this.app.get('/account', ...middleware, expressWrap(async (req, resp) => {
-      return this._sendAppPage(req, resp, {path: 'account.html', status: 200, config: {}});
+      return this._sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}});
    }));
  }

@@ -1460,6 +1473,43 @@ export class FlexServer implements GristServer {
    addGoogleAuthEndpoint(this.app, messagePage);
  }

+  public addInstallEndpoints() {
+    if (this._check('install')) { return; }
+
+    const isManager = expressWrap(
+      (req: express.Request, _res: express.Response, next: express.NextFunction) => {
+        if (!isDefaultUser(req)) { throw new ApiError('Access denied', 403); }
+
+        next();
+      }
+    );
+
+    this.app.get('/api/install/prefs', expressWrap(async (_req, resp) => {
+      const activation = await this._activations.current();
+
+      return sendOkReply(null, resp, {
+        telemetry: await getTelemetryPrefs(this._dbManager, activation),
+      });
+    }));
+
+    this.app.patch('/api/install/prefs', isManager, expressWrap(async (req, resp) => {
+      const props = {prefs: req.body};
+      const activation = await this._activations.current();
+      activation.checkProperties(props);
+      activation.updateFromProperties(props);
+      await activation.save();
+
+      if ((props as Partial<InstallProperties>).prefs?.telemetry) {
+        // Make sure the Telemetry singleton picks up the changes to telemetry preferences.
+        // TODO: if there are multiple home server instances, notify them all of changes to
+        // preferences (via Redis Pub/Sub).
+        await this._telemetry.fetchTelemetryPrefs();
+      }
+
+      return resp.status(200).send();
+    }));
+  }
+
  // Get the HTML template sent for document pages.
  public async getDocTemplate(): Promise<DocTemplate> {
    const page = await fse.readFile(path.join(getAppPathTo(this.appRoot, 'static'),
--- a/app/server/lib/GristServer.ts
+++ b/app/server/lib/GristServer.ts
@@ -138,8 +138,11 @@ export function createDummyGristServer(): GristServer {

 export function createDummyTelemetry(): ITelemetry {
  return {
-    logEvent() { return Promise.resolve(); },
    addEndpoints() { /* do nothing */ },
-    getTelemetryLevel() { return 'off'; },
+    addPages() { /* do nothing */ },
+    start() { return Promise.resolve(); },
+    logEvent() { return Promise.resolve(); },
+    getTelemetryConfig() { return undefined; },
+    fetchTelemetryPrefs() { return Promise.resolve(); },
  };
 }
--- a/app/server/lib/ICreate.ts
+++ b/app/server/lib/ICreate.ts
@@ -50,6 +50,7 @@ export interface ICreateActiveDocOptions {
 export interface ICreateStorageOptions {
  name: string;
  check(): boolean;
+  checkBackend?(): Promise<void>;
  create(purpose: 'doc'|'meta', extraPrefix: string): ExternalStorage|undefined;
 }

@@ -119,7 +120,10 @@ export function makeSimpleCreator(opts: {
    },
    async configure() {
      for (const s of storage || []) {
-        if (s.check()) { break; }
+        if (s.check()) {
+          await s.checkBackend?.();
+          break;
+        }
      }
    },
    ...(opts.shell && {
--- a/app/server/lib/MinIOExternalStorage.ts
+++ b/app/server/lib/MinIOExternalStorage.ts
@@ -107,6 +107,11 @@ export class MinIOExternalStorage implements ExternalStorage {
    }
  }

+  public async hasVersioning(): Promise<Boolean> {
+    const versioning = await this._s3.getBucketVersioning(this.bucket);
+    return versioning && versioning.Status === 'Enabled';
+  }
+
  public async versions(key: string, options?: { includeDeleteMarkers?: boolean }) {
    const results: minio.BucketItem[] = [];
    await new Promise((resolve, reject) => {
--- a/app/server/lib/Telemetry.ts
+++ b/app/server/lib/Telemetry.ts
@@ -1,8 +1,10 @@
 import {ApiError} from 'app/common/ApiError';
+import {TelemetryConfig} from 'app/common/gristUrls';
+import {assertIsDefined} from 'app/common/gutil';
 import {
  buildTelemetryEventChecker,
-  filterMetadata,
-  removeNullishKeys,
+  Level,
+  TelemetryContracts,
  TelemetryEvent,
  TelemetryEventChecker,
  TelemetryEvents,
@@ -11,18 +13,28 @@ import {
  TelemetryMetadata,
  TelemetryMetadataByLevel,
 } from 'app/common/Telemetry';
-import {HomeDBManager, HomeDBTelemetryEvents} from 'app/gen-server/lib/HomeDBManager';
+import {TelemetryPrefsWithSources} from 'app/common/InstallAPI';
+import {Activation} from 'app/gen-server/entity/Activation';
+import {Activations} from 'app/gen-server/lib/Activations';
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {RequestWithLogin} from 'app/server/lib/Authorizer';
+import {expressWrap} from 'app/server/lib/expressWrap';
 import {GristServer} from 'app/server/lib/GristServer';
+import {hashId} from 'app/server/lib/hashingUtils';
 import {LogMethods} from 'app/server/lib/LogMethods';
 import {stringParam} from 'app/server/lib/requestUtils';
 import * as express from 'express';
+import fetch from 'node-fetch';
 import merge = require('lodash/merge');
+import pickBy = require('lodash/pickBy');

 export interface ITelemetry {
+  start(): Promise<void>;
  logEvent(name: TelemetryEvent, metadata?: TelemetryMetadataByLevel): Promise<void>;
  addEndpoints(app: express.Express): void;
-  getTelemetryLevel(): TelemetryLevel;
+  addPages(app: express.Express, middleware: express.RequestHandler[]): void;
+  getTelemetryConfig(): TelemetryConfig | undefined;
+  fetchTelemetryPrefs(): Promise<void>;
 }

 const MAX_PENDING_FORWARD_EVENT_REQUESTS = 25;
@@ -31,26 +43,30 @@ const MAX_PENDING_FORWARD_EVENT_REQUESTS = 25;
 * Manages telemetry for Grist.
 */
 export class Telemetry implements ITelemetry {
-  private _telemetryLevel: TelemetryLevel;
-  private _deploymentType = this._gristServer.getDeploymentType();
-  private _shouldForwardTelemetryEvents = this._deploymentType !== 'saas';
-  private _forwardTelemetryEventsUrl = process.env.GRIST_TELEMETRY_URL ||
-  'https://telemetry.getgrist.com/api/telemetry';
+  private _activation: Activation | undefined;
+  private readonly _deploymentType = this._gristServer.getDeploymentType();
+
+  private _telemetryPrefs: TelemetryPrefsWithSources | undefined;
+
+  private readonly _shouldForwardTelemetryEvents = this._deploymentType !== 'saas';
+  private readonly _forwardTelemetryEventsUrl = process.env.GRIST_TELEMETRY_URL ||
+    'https://telemetry.getgrist.com/api/telemetry';
  private _numPendingForwardEventRequests = 0;

-  private _installationId: string | undefined;

-  private _logger = new LogMethods('Telemetry ', () => ({}));
-  private _telemetryLogger = new LogMethods('Telemetry ', () => ({
+  private readonly _logger = new LogMethods('Telemetry ', () => ({}));
+  private readonly _telemetryLogger = new LogMethods('Telemetry ', () => ({
    eventType: 'telemetry',
  }));

-  private _checkEvent: TelemetryEventChecker | undefined;
+  private _checkTelemetryEvent: TelemetryEventChecker | undefined;

  constructor(private _dbManager: HomeDBManager, private _gristServer: GristServer) {
-    this._initialize().catch((e) => {
-      this._logger.error(undefined, 'failed to initialize', e);
-    });
+
+  }
+
+  public async start() {
+    await this.fetchTelemetryPrefs();
  }

  /**
@@ -96,19 +112,29 @@ export class Telemetry implements ITelemetry {
    event: TelemetryEvent,
    metadata?: TelemetryMetadataByLevel
  ) {
-    if (this._telemetryLevel === 'off') { return; }
+    if (!this._checkTelemetryEvent) {
+      this._logger.error(undefined, 'logEvent called but telemetry event checker is undefined');
+      return;
+    }

-    metadata = filterMetadata(metadata, this._telemetryLevel);
+    const prefs = this._telemetryPrefs;
+    if (!prefs) {
+      this._logger.error(undefined, 'logEvent called but telemetry preferences are undefined');
+      return;
+    }
+
+    const {telemetryLevel} = prefs;
+    if (TelemetryContracts[event] && TelemetryContracts[event].minimumTelemetryLevel > Level[telemetryLevel.value]) {
+      return;
+    }
+
+    metadata = filterMetadata(metadata, telemetryLevel.value);
    this._checkTelemetryEvent(event, metadata);

    if (this._shouldForwardTelemetryEvents) {
      await this._forwardEvent(event, metadata);
    } else {
-      this._telemetryLogger.rawLog('info', null, event, {
-        eventName: event,
-        eventSource: `grist-${this._deploymentType}`,
-        ...metadata,
-      });
+      this._logEvent(event, metadata);
    }
  }

@@ -125,7 +151,7 @@ export class Telemetry implements ITelemetry {
     * source. Otherwise, the event will only be logged after passing various
     * checks.
     */
-    app.post('/api/telemetry', async (req, resp) => {
+    app.post('/api/telemetry', expressWrap(async (req, resp) => {
      const mreq = req as RequestWithLogin;
      const event = stringParam(req.body.event, 'event', TelemetryEvents.values);
      if ('eventSource' in req.body.metadata) {
@@ -135,11 +161,12 @@ export class Telemetry implements ITelemetry {
        });
      } else {
        try {
+          this._assertTelemetryIsReady();
          await this.logEvent(event as TelemetryEvent, merge(
            {
              limited: {
                eventSource: `grist-${this._deploymentType}`,
-                ...(this._deploymentType !== 'saas' ? {installationId: this._installationId} : {}),
+                ...(this._deploymentType !== 'saas' ? {installationId: this._activation!.id} : {}),
              },
              full: {
                userId: mreq.userId,
@@ -154,38 +181,51 @@ export class Telemetry implements ITelemetry {
        }
      }
      return resp.status(200).send();
+    }));
+  }
+
+  public addPages(app: express.Application, middleware: express.RequestHandler[]) {
+    if (this._deploymentType === 'core') {
+      app.get('/support-grist', ...middleware, expressWrap(async (req, resp) => {
+        return this._gristServer.sendAppPage(req, resp,
+          {path: 'app.html', status: 200, config: {}});
+      }));
+    }
+  }
+
+  public getTelemetryConfig(): TelemetryConfig | undefined {
+    const prefs = this._telemetryPrefs;
+    if (!prefs) {
+      this._logger.error(undefined, 'getTelemetryConfig called but telemetry preferences are undefined');
+      return undefined;
+    }
+
+    return {
+      telemetryLevel: prefs.telemetryLevel.value,
+    };
+  }
+
+  public async fetchTelemetryPrefs() {
+    this._activation = await this._gristServer.getActivations().current();
+    await this._fetchTelemetryPrefs();
+  }
+
+  private async _fetchTelemetryPrefs() {
+    this._telemetryPrefs = await getTelemetryPrefs(this._dbManager, this._activation);
+    this._checkTelemetryEvent = buildTelemetryEventChecker(this._telemetryPrefs.telemetryLevel.value);
+  }
+
+  private _logEvent(
+    event: TelemetryEvent,
+    metadata?: TelemetryMetadata
+  ) {
+    this._telemetryLogger.rawLog('info', null, event, {
+      eventName: event,
+      eventSource: `grist-${this._deploymentType}`,
+      ...metadata,
    });
  }

-  public getTelemetryLevel() {
-    return this._telemetryLevel;
-  }
-
-  private async _initialize() {
-    this._telemetryLevel = getTelemetryLevel();
-    if (process.env.GRIST_TELEMETRY_LEVEL !== undefined) {
-      this._checkTelemetryEvent = buildTelemetryEventChecker(this._telemetryLevel);
-    }
-
-    const {id} = await this._gristServer.getActivations().current();
-    this._installationId = id;
-
-    for (const event of HomeDBTelemetryEvents.values) {
-      this._dbManager.on(event, async (metadata) => {
-        this.logEvent(event, metadata).catch(e =>
-          this._logger.error(undefined, `failed to log telemetry event ${event}`, e));
-      });
-    }
-  }
-
-  private _checkTelemetryEvent(event: TelemetryEvent, metadata?: TelemetryMetadata) {
-    if (!this._checkEvent) {
-      throw new Error('Telemetry._checkEvent is undefined');
-    }
-
-    this._checkEvent(event, metadata);
-  }
-
  private async _forwardEvent(
    event: TelemetryEvent,
    metadata?: TelemetryMetadata
@@ -198,7 +238,7 @@ export class Telemetry implements ITelemetry {

    try {
      this._numPendingForwardEventRequests += 1;
-      await this._postJsonPayload(JSON.stringify({event, metadata}));
+      await this._doForwardEvent(JSON.stringify({event, metadata}));
    } catch (e) {
      this._logger.error(undefined, `failed to forward telemetry event ${event}`, e);
    } finally {
@@ -206,7 +246,7 @@ export class Telemetry implements ITelemetry {
    }
  }

-  private async _postJsonPayload(payload: string) {
+  private async _doForwardEvent(payload: string) {
    await fetch(this._forwardTelemetryEventsUrl, {
      method: 'POST',
      headers: {
@@ -215,12 +255,90 @@ export class Telemetry implements ITelemetry {
      body: payload,
    });
  }
-}

-export function getTelemetryLevel(): TelemetryLevel {
-  if (process.env.GRIST_TELEMETRY_LEVEL !== undefined) {
-    return TelemetryLevels.check(process.env.GRIST_TELEMETRY_LEVEL);
-  } else {
-    return 'off';
+  private _assertTelemetryIsReady() {
+    try {
+      assertIsDefined('activation', this._activation);
+    } catch (e) {
+      this._logger.error(null, 'activation is undefined', e);
+      throw new ApiError('Telemetry is not ready', 500);
+    }
  }
 }
+
+export async function getTelemetryPrefs(
+  db: HomeDBManager,
+  activation?: Activation
+): Promise<TelemetryPrefsWithSources> {
+  const GRIST_TELEMETRY_LEVEL = process.env.GRIST_TELEMETRY_LEVEL;
+  if (GRIST_TELEMETRY_LEVEL !== undefined) {
+    const value = TelemetryLevels.check(GRIST_TELEMETRY_LEVEL);
+    return {
+      telemetryLevel: {
+        value,
+        source: 'environment-variable',
+      },
+    };
+  }
+
+  const {prefs} = activation ?? await new Activations(db).current();
+  return {
+    telemetryLevel: {
+      value: prefs?.telemetry?.telemetryLevel ?? 'off',
+      source: 'preferences',
+    }
+  };
+}
+
+/**
+ * Returns a new, filtered metadata object, or undefined if `metadata` is undefined.
+ *
+ * Filtering currently:
+ *  - removes keys in groups that exceed `telemetryLevel`
+ *  - removes keys with values of null or undefined
+ *  - hashes the values of keys suffixed with "Digest" (e.g. doc ids, fork ids)
+ *  - flattens the entire metadata object (i.e. removes the nesting of keys under
+ *    "limited" or "full")
+ */
+export function filterMetadata(
+  metadata: TelemetryMetadataByLevel | undefined,
+  telemetryLevel: TelemetryLevel
+): TelemetryMetadata | undefined {
+  if (!metadata) { return; }
+
+  let filteredMetadata: TelemetryMetadata = {};
+  for (const level of ['limited', 'full'] as const) {
+    if (Level[telemetryLevel] < Level[level]) { break; }
+
+    filteredMetadata = {...filteredMetadata, ...metadata[level]};
+  }
+
+  filteredMetadata = removeNullishKeys(filteredMetadata);
+  filteredMetadata = hashDigestKeys(filteredMetadata);
+
+  return filteredMetadata;
+}
+
+/**
+ * Returns a copy of `object` with all null and undefined keys removed.
+ */
+export function removeNullishKeys(object: Record<string, any>) {
+  return pickBy(object, value => value !== null && value !== undefined);
+}
+
+/**
+ * Returns a copy of `metadata`, replacing the values of all keys suffixed
+ * with "Digest" with the result of hashing the value. The hash is prefixed with
+ * the first 4 characters of the original value, to assist with troubleshooting.
+ */
+export function hashDigestKeys(metadata: TelemetryMetadata): TelemetryMetadata {
+  const filteredMetadata: TelemetryMetadata = {};
+  Object.entries(metadata).forEach(([key, value]) => {
+    if (key.endsWith('Digest') && typeof value === 'string') {
+      filteredMetadata[key] = hashId(value);
+    } else {
+      filteredMetadata[key] = value;
+    }
+  });
+  return filteredMetadata;
+}
--- a/app/server/lib/configureMinIOExternalStorage.ts
+++ b/app/server/lib/configureMinIOExternalStorage.ts
@@ -60,3 +60,16 @@ export function checkMinIOExternalStorage() {
    region
  };
 }
+
+export async function checkMinIOBucket() {
+  const options = checkMinIOExternalStorage();
+  if (!options) {
+    throw new Error('Configuration check failed for MinIO backend storage.');
+  }
+
+  const externalStorage = new MinIOExternalStorage(options.bucket, options);
+  if (!await externalStorage.hasVersioning()) {
+    await externalStorage.close();
+    throw new Error(`FATAL: the MinIO bucket "${options.bucket}" does not have versioning enabled`);
+  }
+}
--- a/app/server/lib/hashingUtils.ts
+++ b/app/server/lib/hashingUtils.ts
@@ -0,0 +1,12 @@
+import {createHash} from 'crypto';
+
+/**
+ * Returns a hash of `id` prefixed with the first 4 characters of `id`. The first 4
+ * characters are included to assist with troubleshooting.
+ *
+ * Useful for situations where potentially sensitive identifiers are logged, such as
+ * doc ids of docs that have public link sharing enabled.
+ */
+export function hashId(id: string): string {
+  return `${id.slice(0, 4)}:${createHash('sha256').update(id.slice(4)).digest('base64')}`;
+}
--- a/app/server/lib/requestUtils.ts
+++ b/app/server/lib/requestUtils.ts
@@ -2,7 +2,7 @@ import {ApiError} from 'app/common/ApiError';
 import {DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
 import * as gutil from 'app/common/gutil';
 import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/HomeDBManager';
-import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
+import {getUser, getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {RequestWithOrg} from 'app/server/lib/extractOrg';
 import {RequestWithGrist} from 'app/server/lib/GristServer';
 import log from 'app/server/lib/log';
@@ -352,3 +352,9 @@ export function addAbortHandler(req: Request, res: Writable, op: () => void) {
    }
  });
 }
+
+export function isDefaultUser(req: Request) {
+  const defaultEmail = process.env.GRIST_DEFAULT_EMAIL;
+  const {loginEmail} = getUser(req);
+  return defaultEmail && defaultEmail === loginEmail;
+}
--- a/app/server/lib/sendAppPage.ts
+++ b/app/server/lib/sendAppPage.ts
@@ -76,7 +76,7 @@ export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig
    featureFormulaAssistant: isAffirmative(process.env.GRIST_FORMULA_ASSISTANT),
    supportEmail: SUPPORT_EMAIL,
    userLocale: (req as RequestWithLogin | undefined)?.user?.options?.locale,
-    telemetry: server ? getTelemetryConfig(server) : undefined,
+    telemetry: server?.getTelemetry().getTelemetryConfig(),
    deploymentType: server?.getDeploymentType(),
    ...extra,
  };
@@ -163,13 +163,6 @@ function getFeatures(): IFeature[] {
  return Features.checkAll(difference(enabledFeatures, disabledFeatures));
 }

-function getTelemetryConfig(server: GristServer) {
-  const telemetry = server.getTelemetry();
-  return {
-    telemetryLevel: telemetry.getTelemetryLevel(),
-  };
-}
-
 function configuredPageTitleSuffix() {
  const result = process.env.GRIST_PAGE_TITLE_SUFFIX;
  return result === "_blank" ? "" : result;
--- a/app/server/mergedServerMain.ts
+++ b/app/server/mergedServerMain.ts
@@ -106,44 +106,50 @@ export async function main(port: number, serverTypes: ServerType[],
  server.addApiMiddleware();
  await server.addBillingMiddleware();

-  await server.start();
+  try {
+    await server.start();

-  if (includeHome) {
-    server.addUsage();
-    if (!includeDocs) {
-      server.addDocApiForwarder();
+    if (includeHome) {
+      server.addUsage();
+      if (!includeDocs) {
+        server.addDocApiForwarder();
+      }
+      server.addJsonSupport();
+      await server.addLandingPages();
+      // todo: add support for home api to standalone app
+      server.addHomeApi();
+      server.addBillingApi();
+      server.addNotifier();
+      await server.addTelemetry();
+      await server.addHousekeeper();
+      await server.addLoginRoutes();
+      server.addAccountPage();
+      server.addBillingPages();
+      server.addWelcomePaths();
+      server.addLogEndpoint();
+      server.addGoogleAuthEndpoint();
+      server.addInstallEndpoints();
    }
-    server.addJsonSupport();
-    await server.addLandingPages();
-    // todo: add support for home api to standalone app
-    server.addHomeApi();
-    server.addBillingApi();
-    server.addNotifier();
-    server.addTelemetry();
-    await server.addHousekeeper();
-    await server.addLoginRoutes();
-    server.addAccountPage();
-    server.addBillingPages();
-    server.addWelcomePaths();
-    server.addLogEndpoint();
-    server.addGoogleAuthEndpoint();
+
+    if (includeDocs) {
+      server.addJsonSupport();
+      await server.addTelemetry();
+      await server.addDoc();
+    }
+
+    if (includeHome) {
+      server.addClientSecrets();
+    }
+
+    server.finalize();
+
+    server.checkOptionCombinations();
+    server.summary();
+    return server;
+  } catch(e) {
+    await server.close();
+    throw e;
  }
-
-  if (includeDocs) {
-    server.addJsonSupport();
-    server.addTelemetry();
-    await server.addDoc();
-  }
-
-  if (includeHome) {
-    server.addClientSecrets();
-  }
-
-  server.finalize();
-
-  server.checkOptionCombinations();
-  server.summary();
-  return server;
 }