(core) Ask the user some questions after they sign up and set their name.

Summary:
- Add a /welcome/info endpoint, to serve a page after /welcome/user
- Add a new forms module to factor out the styles that feel more natural for a web form.
- Simplify form submission using JSON with a BaseAPI helper.
- The POST submission to /welcome/info gets added to a Grist doc, using a
  specialPermit grant to gain access. A failure (e.g. missing doc) is logged
  but does not affect the user.

Test Plan: Added a test case.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2640

app/client/lib/uploads.ts

@@ -10,6 +10,7 @@
 import {DocComm} from 'app/client/components/DocComm';
 import {UserError} from 'app/client/models/errors';
 import {FileDialogOptions, openFilePicker} from 'app/client/ui/FileDialog';
+import {BaseAPI} from 'app/common/BaseAPI';
 import {GristLoadConfig} from 'app/common/gristUrls';
 import {byteString, safeJsonParse} from 'app/common/gutil';
 import {UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
@@ -172,32 +173,24 @@ export async function fetchURL(
   return res!;
 }
 
-// Submit a form using XHR.  Send inputs as JSON, and interpret any reply as JSON.
-export async function submitForm(form: HTMLFormElement): Promise<any> {
-  return new Promise((resolve, reject) => {
-    const xhr = new XMLHttpRequest();
+/**
+ * Convert a form to a JSON-stringifiable object, ignoring any File fields.
+ */
+export function formDataToObj(formElem: HTMLFormElement): {[key: string]: string} {
+  // Use FormData to collect values (rather than e.g. finding <input> elements) to ensure we get
+  // values from all form items correctly (e.g. checkboxes and textareas).
+  const formData = new FormData(formElem);
   const data: {[key: string]: string} = {};
-    for (const element of [...form.getElementsByTagName('input')]) {
-      data[element.name] = element.value;
+  for (const [name, value] of formData.entries()) {
+    if (typeof value === 'string') {
+      data[name] = value;
     }
-    xhr.open('post', form.action, true);
-    xhr.setRequestHeader('Content-Type', 'application/json');
-    xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest');
-    xhr.withCredentials = true;
-    xhr.send(JSON.stringify(data));
-    xhr.addEventListener('error', (e: ProgressEvent) => {
-      console.warn("Form error", e);    // tslint:disable-line:no-console
-      reject(new Error('Form error, please try again'));
-    });
-    xhr.addEventListener('load', () => {
-      if (xhr.status !== 200) {
-        // tslint:disable-next-line:no-console
-        console.warn("Form failed", xhr.status, xhr.responseText);
-        const err = safeJsonParse(xhr.responseText, null);
-        reject(new UserError('Form failed: ' + (err && err.error || xhr.status)));
-      } else {
-        resolve(safeJsonParse(xhr.responseText, null));
   }
-    });
-  });
+  return data;
+}
+
+// Submit a form using BaseAPI.  Send inputs as JSON, and interpret any reply as JSON.
+export async function submitForm(form: HTMLFormElement): Promise<any> {
+  const data = formDataToObj(form);
+  return BaseAPI.requestJson(form.action, {method: 'POST', body: JSON.stringify(data)});
 }

app/client/ui/WelcomePage.ts

@@ -6,11 +6,28 @@ import { urlState } from "app/client/models/gristUrlState";
 import { AccountWidget } from "app/client/ui/AccountWidget";
 import { appHeader } from 'app/client/ui/AppHeader';
 import * as BillingPageCss from "app/client/ui/BillingPageCss";
+import * as forms from "app/client/ui/forms";
 import { pagePanels } from "app/client/ui/PagePanels";
-import { bigPrimaryButton, bigPrimaryButtonLink, cssButton } from "app/client/ui2018/buttons";
+import { bigBasicButton, bigPrimaryButton, bigPrimaryButtonLink, cssButton } from "app/client/ui2018/buttons";
 import { colors, testId, vars } from "app/client/ui2018/cssVars";
 import { getOrgName, Organization } from "app/common/UserAPI";
 
+async function _submitForm(form: HTMLFormElement) {
+  const result = await submitForm(form);
+  const redirectUrl = result.redirectUrl;
+  if (!redirectUrl) {
+    throw new Error('form failed to redirect');
+  }
+  window.location.assign(redirectUrl);
+}
+
+function handleSubmit(): (elem: HTMLFormElement) => void {
+  return dom.on('submit', async (e, form) => {
+    e.preventDefault();
+    _submitForm(form).catch(reportError);
+  });
+}
+
 export class WelcomePage extends Disposable {
 
   private _currentUserName = this._appModel.currentUser && this._appModel.currentUser.name || '';
@@ -42,6 +59,7 @@ export class WelcomePage extends Disposable {
 
       domComputed(urlState().state, (state) => (
         state.welcome === 'user' ? dom.create(this._buildNameForm.bind(this)) :
+        state.welcome === 'info' ? dom.create(this._buildInfoForm.bind(this)) :
         state.welcome === 'teams' ? dom.create(this._buildOrgPicker.bind(this)) :
         null
       )),
@@ -60,16 +78,12 @@ export class WelcomePage extends Disposable {
     return form = dom(
       'form',
       { method: "post" },
-      dom.on('submit', (e) => {
-        e.preventDefault();
-        this._submitForm(form).catch(reportError);
-        return false;
-      }),
+      handleSubmit(),
       cssLabel('Your full name, as you\'d like it displayed to your collaborators.'),
       inputEl = cssInput(
         value, { onInput: true, },
         { name: "username" },
-        dom.onKeyDown({Enter: () => isNameValid.get() && this._submitForm(form).catch(reportError)}),
+        dom.onKeyDown({Enter: () => isNameValid.get() && _submitForm(form).catch(reportError)}),
       ),
       dom.maybe((use) => use(value) && !use(isNameValid), buildNameWarningsDom),
       cssButtonGroup(
@@ -82,14 +96,47 @@ export class WelcomePage extends Disposable {
     );
   }
 
-  private async _submitForm(form: HTMLFormElement) {
-    const result = await submitForm(form);
-    const redirectUrl = result.redirectUrl;
-    if (!redirectUrl) {
-      throw new Error('form failed to redirect');
-    }
-    window.location.assign(redirectUrl);
-    return false;
+  /**
+   * Builds a form to ask the new user a few questions.
+   */
+  private _buildInfoForm(owner: MultiHolder) {
+    const allFilled = Observable.create(owner, false);
+    return forms.form({method: "post"},
+      handleSubmit(),
+      (elem) => { setTimeout(() => elem.focus(), 0); },
+      forms.text('Please help us serve you better by answering a few questions.'),
+      forms.question(
+        forms.text('Where do you plan to use Grist?'),
+        forms.checkboxItem([{name: 'use_work'}], 'Work'),
+        forms.checkboxItem([{name: 'use_personal'}], 'Personal'),
+      ),
+      forms.question(
+        forms.text('What brings you to Grist?'),
+        forms.checkboxItem([{name: 'reason_problem'}], 'Solve a particular problem or need'),
+        forms.checkboxItem([{name: 'reason_tool'}], 'Find a better tool than the one I am using'),
+        forms.checkboxItem([{name: 'reason_curious'}], 'Just curious about a new product'),
+        forms.checkboxOther([{name: 'reason_other'}], {name: 'other_reason', placeholder: 'Other...'}),
+      ),
+      forms.question(
+        forms.text('What kind of industry do you work in?'),
+        forms.textBox({name: 'industry', placeholder: 'Your answer'}),
+      ),
+      forms.question(
+        forms.text('What is your role?'),
+        forms.textBox({name: 'role', placeholder: 'Your answer'}),
+      ),
+      dom.on('change', (e, form) => {
+        allFilled.set(forms.isFormFilled(form, ['use_*', 'reason_*', 'industry', 'role']));
+      }),
+      cssButtonGroup(
+        cssButtonGroup.cls('-right'),
+        bigBasicButton('Continue',
+          cssButton.cls('-primary', allFilled),
+          {tabIndex: '0'},
+          testId('continue-button')),
+      ),
+      testId('info-form'),
+    );
   }
 
   private async _fetchOrgs() {
@@ -174,6 +221,9 @@ const cssParagraph = styled('p', textStyle);
 const cssButtonGroup = styled('div', `
   margin-top: 24px;
   display: flex;
+  &-right {
+    justify-content: flex-end;
+  }
 `);
 
 const cssWarning = styled('div', `

app/client/ui/forms.ts

@@ -0,0 +1,132 @@
+/**
+ * Collection of styled elements to put together basic forms. Intended usage is:
+ *
+ *   return forms.form({method: 'POST',
+ *     forms.question(
+ *       forms.text('What color is the sky right now?'),
+ *       forms.checkboxItem([{name: 'sky-blue'}], 'Blue'),
+ *       forms.checkboxItem([{name: 'sky-orange'}], 'Orange'),
+ *       forms.checkboxOther([], {name: 'sky-other', placeholder: 'Other...'}),
+ *     ),
+ *     forms.question(
+ *       forms.text('What is the meaning of life, universe, and everything?'),
+ *       forms.textBox({name: 'meaning', placeholder: 'Your answer'}),
+ *     ),
+ *   );
+ */
+import {cssCheckboxSquare, cssLabel} from 'app/client/ui2018/checkbox';
+import {dom, DomElementArg, styled} from 'grainjs';
+
+export {
+  form,
+  cssQuestion as question,
+  cssText as text,
+  textBox,
+};
+
+
+/**
+ * Create a checkbox accompanied by a label. The first argument should be the (possibly empty)
+ * array of arguments to the checkbox; the rest goes into the label. E.g.
+ *    checkboxItem([{name: 'ok'}], 'Check to approve');
+ */
+export function checkboxItem(checkboxArgs: DomElementArg[], ...labelArgs: DomElementArg[]): HTMLElement {
+  return cssCheckboxLabel(
+    cssCheckbox({type: 'checkbox'}, ...checkboxArgs),
+    ...labelArgs);
+}
+
+/**
+ * Create a checkbox accompanied by a textbox, for a choice of "Other". The checkbox gets checked
+ * automatically when something is typed into the textbox.
+ *    checkboxOther([{name: 'choice-other'}], {name: 'other-text', placeholder: '...'});
+ */
+export function checkboxOther(checkboxArgs: DomElementArg[], ...textboxArgs: DomElementArg[]): HTMLElement {
+  let checkbox: HTMLInputElement;
+  return cssCheckboxLabel(
+    checkbox = cssCheckbox({type: 'checkbox'}, ...checkboxArgs),
+    cssTextBox(...textboxArgs,
+      dom.on('input', (e, elem) => { checkbox.checked = Boolean(elem.value); }),
+    ),
+  );
+}
+
+/**
+ * Returns whether the form is fully filled, i.e. has a value for each of the provided names of
+ * form elements. If a name ends with "*", it is treated as a prefix, and any element matching it
+ * would satisfy this key (e.g. use "foo_*" to accept any checkbox named "foo_<something>").
+ */
+export function isFormFilled(formElem: HTMLFormElement, names: string[]): boolean {
+  const formData = new FormData(formElem);
+  return names.every(name => hasValue(formData, name));
+}
+
+/**
+ * Returns true of the form includes a non-empty value for the given name. If the second argument
+ * ends with "-", it is treated as a prefix, and the function returns true if the form includes
+ * any value for a key that starts with that prefix.
+ */
+export function hasValue(formData: FormData, nameOrPrefix: string): boolean {
+  if (nameOrPrefix.endsWith('*')) {
+    const prefix = nameOrPrefix.slice(0, -1);
+    return [...formData.keys()].filter(k => k.startsWith(prefix)).some(k => formData.get(k));
+  } else {
+    return Boolean(formData.get(nameOrPrefix));
+  }
+}
+
+const cssForm = styled('form', `
+  margin-bottom: 32px;
+  font-size: 14px;
+  &:focus {
+    outline: none;
+  }
+  & input:focus, & button:focus {
+    outline: none;
+    box-shadow: 0 0 1px 2px lightblue;
+  }
+`);
+
+const cssQuestion = styled('div', `
+  margin: 32px 0;
+  padding-left: 24px;
+  & > :first-child {
+    margin-left: -24px;
+  }
+`);
+
+const cssText = styled('div', `
+  margin: 16px 0;
+  font-size: 15px;
+`);
+
+const cssCheckboxLabel = styled(cssLabel, `
+  font-size: 14px;
+  font-weight: normal;
+  display: flex;
+  align-items: center;
+  margin: 12px 0;
+  user-select: unset;
+`);
+
+const cssCheckbox = styled(cssCheckboxSquare, `
+  position: relative;
+  margin-right: 12px !important;
+  border-radius: var(--radius);
+`);
+
+const cssTextBox = styled('input', `
+  flex: auto;
+  width: 100%;
+  font-size: inherit;
+  padding: 4px 8px;
+  border: 1px solid #D9D9D9;
+  border-radius: 3px;
+
+  &-invalid {
+    color: red;
+  }
+`);
+
+const form = cssForm.bind(null, {tabIndex: '-1'});
+const textBox = cssTextBox.bind(null, {type: 'text'});

app/common/BaseAPI.ts

@@ -37,6 +37,12 @@ export class BaseAPI {
     };
   }
 
+  // Make a JSON request to the given URL, and read the esponse as JSON. Handles errors, and
+  // counts pending requests in the same way as BaseAPI methods do.
+  public static requestJson(url: string, init: RequestInit = {}): Promise<unknown> {
+    return new BaseAPI().requestJson(url, init);
+  }
+
   private static _numPendingRequests: number = 0;
 
   protected fetch: typeof fetch;

app/common/gristUrls.ts

@@ -14,7 +14,7 @@ export type IDocPage = number | 'new' | 'code';
 export const HomePage = StringUnion('all', 'workspace', 'trash');
 export type IHomePage = typeof HomePage.type;
 
-export const WelcomePage = StringUnion('user', 'teams');
+export const WelcomePage = StringUnion('user', 'info', 'teams');
 export type WelcomePage = typeof WelcomePage.type;
 
 // Overall UI style.  "full" is normal, "light" is a single page focused, panels hidden experience.

app/server/lib/FlexServer.ts

@@ -40,7 +40,7 @@ import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint'
 import {PluginManager} from 'app/server/lib/PluginManager';
 import {adaptServerUrl, addOrgToPathIfNeeded, addPermit, getScope, optStringParam, RequestWithGristInfo, stringParam,
         TEST_HTTPS_OFFSET, trustOrigin} from 'app/server/lib/requestUtils';
-import {ISendAppPageOptions, makeSendAppPage, makeGristConfig} from 'app/server/lib/sendAppPage';
+import {ISendAppPageOptions, makeGristConfig, makeSendAppPage} from 'app/server/lib/sendAppPage';
 import {getDatabaseUrl} from 'app/server/lib/serverUtils';
 import {Sessions} from 'app/server/lib/Sessions';
 import * as shutdown from 'app/server/lib/shutdown';
@@ -53,8 +53,10 @@ import * as express from 'express';
 import * as fse from 'fs-extra';
 import * as http from 'http';
 import * as https from 'https';
+import mapValues = require('lodash/mapValues');
 import * as morganLogger from 'morgan';
 import {AddressInfo} from 'net';
+import fetch from 'node-fetch';
 import * as path from 'path';
 import * as serveStatic from "serve-static";
 
@@ -64,6 +66,9 @@ const HEALTH_CHECK_LOG_SHOW_FIRST_N = 10;
 // And we show every Nth health check:
 const HEALTH_CHECK_LOG_SHOW_EVERY_N = 100;
 
+// DocID of Grist doc to collect the Welcome questionnaire responses.
+const DOC_ID_NEW_USER_INFO = process.env.DOC_ID_NEW_USER_INFO || 'GristNewUserInfo';
+
 export interface FlexServerOptions {
   dataDir?: string;
 
@@ -947,31 +952,76 @@ export class FlexServer implements GristServer {
       this._redirectToLoginWithoutExceptionsMiddleware,
     ];
 
-    this.app.get('/welcome/user', ...middleware, expressWrap(async (req, resp, next) => {
+    this.app.get('/welcome/:page', ...middleware, expressWrap(async (req, resp, next) => {
       return this._sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}, googleTagManager: true});
     }));
 
-    this.app.post('/welcome/user', ...middleware, expressWrap(async (req, resp, next) => {
+    this.app.post('/welcome/:page', ...middleware, expressWrap(async (req, resp, next) => {
       const mreq = req as RequestWithLogin;
       const userId = getUserId(req);
       const domain = mreq.org;
-      const result = await this.dbManager.getMergedOrgs(userId, userId, domain || null);
-      const orgs = (result.status === 200) ? result.data : null;
+      let redirectPath: string = '/';
 
+      if (req.params.page === 'user') {
         const name: string|undefined = req.body && req.body.username || undefined;
         await this.dbManager.updateUser(userId, {name, isFirstTimeUser: false});
+        redirectPath = '/welcome/info';
+
+      } else if (req.params.page === 'info') {
+        const urlId = DOC_ID_NEW_USER_INFO;
+        let body: string|undefined;
+        let permitKey: string|undefined;
+        try {
+          // Take an extra step to translate the special urlId to a docId. This is helpful to
+          // allow the same urlId to be used in production and in test. We need the docId for the
+          // specialPermit below, which we need to be able to write to this doc.
+          //
+          // TODO With proper forms support, we could give an origin-based permission to submit a
+          // form to this doc, and do it from the client directly.
+          const previewerUserId = this.dbManager.getPreviewerUserId();
+          const docAuth = await this.dbManager.getDocAuthCached({urlId, userId: previewerUserId});
+          const docId = docAuth.docId;
+          if (!docId) {
+            throw new Error(`Can't resolve ${urlId}: ${docAuth.error}`);
+          }
+
+          const user = getUser(req);
+          const row = {...req.body, UserID: userId, Name: user.name, Email: user.loginEmail};
+          body = JSON.stringify(mapValues(row, value => [value]));
+
+          permitKey = await this._docWorkerMap.setPermit({docId});
+          const res = await fetch(await this.getHomeUrlByDocId(docId, `/api/docs/${docId}/tables/Responses/data`), {
+            method: 'POST',
+            headers: {'Permit': permitKey, 'Content-Type': 'application/json'},
+            body,
+          });
+          if (res.status !== 200) {
+            throw new Error(`API call failed with ${res.status}`);
+          }
+        } catch (e) {
+          // If we failed to record, at least log the data, so we could potentially recover it.
+          log.rawWarn(`Failed to record new user info: ${e.message}`, {newUserQuestions: body});
+        } finally {
+          if (permitKey) {
+            await this._docWorkerMap.removePermit(permitKey);
+          }
+        }
 
         // redirect to teams page if users has access to more than one org. Otherwise redirect to
         // personal org.
-      const pathname = orgs && orgs.length > 1 ? '/welcome/teams' : '/';
-      const mergedOrgDomain = this.dbManager.mergedOrgDomain();
-      const redirectUrl = this._getOrgRedirectUrl(mreq, mergedOrgDomain, pathname);
-      resp.json({redirectUrl});
-    }));
+        const result = await this.dbManager.getMergedOrgs(userId, userId, domain || null);
+        const orgs = (result.status === 200) ? result.data : null;
+        if (orgs && orgs.length > 1) {
+          redirectPath = '/welcome/teams';
+        }
+      }
 
-    this.app.get('/welcome/teams', ...middleware, expressWrap(async (req, resp, next) => {
-      return this._sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}});
-    }));
+      const mergedOrgDomain = this.dbManager.mergedOrgDomain();
+      const redirectUrl = this._getOrgRedirectUrl(mreq, mergedOrgDomain, redirectPath);
+      resp.json({redirectUrl});
+    }),
+    // Add a final error handler that reports errors as JSON.
+    jsonErrorHandler);
   }
 
   public finalize() {