(core) Exposing custom widgets on the UI

Summary:
Exposing custom widgets as a dropdown menu in custom section configuration panel.

Adding new environmental variable GRIST_WIDGET_LIST_URL that points to a
json file with an array of available widgets. When not present, custom widget menu is
hidden, exposing only Custom URL option.

Available widget list can be fetched from:
https://github.com/gristlabs/grist-widget/releases/download/latest/manifest.json

Test Plan: New tests, and updated old ones.

Reviewers: paulfitz, dsagal

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3127

app/server/lib/FlexServer.ts

@@ -54,6 +54,7 @@ import * as shutdown from 'app/server/lib/shutdown';
 import {TagChecker} from 'app/server/lib/TagChecker';
 import {startTestingHooks} from 'app/server/lib/TestingHooks';
 import {addUploadRoute} from 'app/server/lib/uploads';
+import {buildWidgetRepository, IWidgetRepository} from 'app/server/lib/WidgetRepository';
 import axios from 'axios';
 import * as bodyParser from 'body-parser';
 import * as express from 'express';
@@ -121,6 +122,7 @@ export class FlexServer implements GristServer {
   private _sessionStore: SessionStore;
   private _storageManager: IDocStorageManager;
   private _docWorkerMap: IDocWorkerMap;
+  private _widgetRepository: IWidgetRepository;
   private _internalPermitStore: IPermitStore;  // store for permits that stay within our servers
   private _externalPermitStore: IPermitStore;  // store for permits that pass through outside servers
   private _disabled: boolean = false;
@@ -271,6 +273,11 @@ export class FlexServer implements GristServer {
     return this._storageManager;
   }
 
+  public getWidgetRepository(): IWidgetRepository {
+    if (!this._widgetRepository) { throw new Error('no widget repository available'); }
+    return this._widgetRepository;
+  }
+
   public addLogging() {
     if (this._check('logging')) { return; }
     if (process.env.GRIST_LOG_SKIP_HTTP) { return; }
@@ -524,7 +531,7 @@ export class FlexServer implements GristServer {
 
     // ApiServer's constructor adds endpoints to the app.
     // tslint:disable-next-line:no-unused-expression
-    new ApiServer(this.app, this._dbManager);
+    new ApiServer(this.app, this._dbManager, this._widgetRepository = buildWidgetRepository());
   }
 
   public addBillingApi() {

app/server/lib/ITestingHooks-ti.ts

@@ -19,6 +19,7 @@ export const ITestingHooks = t.iface([], {
   "getDocClientCounts": t.func(t.array(t.tuple("string", "number"))),
   "setActiveDocTimeout": t.func("number", t.param("seconds", "number")),
   "setDiscourseConnectVar": t.func(t.union("string", "null"), t.param("varName", "string"), t.param("value", t.union("string", "null"))),
+  "setWidgetRepositoryUrl": t.func("void", t.param("url", "string")),
 });
 
 const exportedTypeSuite: t.ITypeSuite = {

app/server/lib/ITestingHooks.ts

@@ -15,4 +15,5 @@ export interface ITestingHooks {
   getDocClientCounts(): Promise<Array<[string, number]>>;
   setActiveDocTimeout(seconds: number): Promise<number>;
   setDiscourseConnectVar(varName: string, value: string|null): Promise<string|null>;
+  setWidgetRepositoryUrl(url: string): Promise<void>;
 }

app/server/lib/TestingHooks.ts

@@ -12,6 +12,7 @@ import {FlexServer} from './FlexServer';
 import {ITestingHooks} from './ITestingHooks';
 import ITestingHooksTI from './ITestingHooks-ti';
 import {connect, fromCallback} from './serverUtils';
+import {WidgetRepositoryImpl} from 'app/server/lib/WidgetRepository';
 
 const tiCheckers = t.createCheckers(ITestingHooksTI, {UserProfile: t.name("object")});
 
@@ -194,4 +195,12 @@ export class TestingHooks implements ITestingHooks {
     }
     return prev;
   }
+
+  public async setWidgetRepositoryUrl(url: string): Promise<void> {
+    const repo = this._server.getWidgetRepository() as WidgetRepositoryImpl;
+    if (!(repo instanceof WidgetRepositoryImpl)) {
+      throw new Error("Unsupported widget repository");
+    }
+    repo.testOverrideUrl(url);
+  }
 }

app/server/lib/WidgetRepository.ts

@@ -0,0 +1,92 @@
+import {ICustomWidget} from 'app/common/CustomWidget';
+import * as log from 'app/server/lib/log';
+import fetch from 'node-fetch';
+import {ApiError} from 'app/common/ApiError';
+import * as LRUCache from 'lru-cache';
+
+/**
+ * Widget Repository returns list of available Custom Widgets.
+ */
+export interface IWidgetRepository {
+  getWidgets(): Promise<ICustomWidget[]>;
+}
+
+// Static url for StaticWidgetRepository
+const STATIC_URL = process.env.GRIST_WIDGET_LIST_URL;
+
+/**
+ * Default repository that gets list of available widgets from a static URL.
+ */
+export class WidgetRepositoryImpl implements IWidgetRepository {
+  constructor(protected _staticUrl = STATIC_URL) {}
+
+  /**
+   * Method exposed for testing, overrides widget url.
+   */
+  public testOverrideUrl(url: string) {
+    this._staticUrl = url;
+  }
+
+  public async getWidgets(): Promise<ICustomWidget[]> {
+    if (!this._staticUrl) {
+      log.warn(
+        'WidgetRepository: Widget repository is not configured.' + !STATIC_URL
+          ? ' Missing GRIST_WIDGET_LIST_URL environmental variable.'
+          : ''
+      );
+      return [];
+    }
+    try {
+      const response = await fetch(this._staticUrl);
+      if (!response.ok) {
+        if (response.status === 404) {
+          throw new ApiError('WidgetRepository: Remote widget list not found', 404);
+        } else {
+          const body = await response.text().catch(() => '');
+          throw new ApiError(
+            `WidgetRepository: Remote server returned an error: ${body || response.statusText}`, response.status
+          );
+        }
+      }
+      const widgets = await response.json().catch(() => null);
+      if (!widgets || !Array.isArray(widgets)) {
+        throw new ApiError('WidgetRepository: Error reading widget list', 500);
+      }
+      return widgets;
+    } catch (err) {
+      if (!(err instanceof ApiError)) {
+        throw new ApiError(String(err), 500);
+      }
+      throw err;
+    }
+  }
+}
+
+/**
+ * Version of WidgetRepository that caches successful result for 2 minutes.
+ */
+class CachedWidgetRepository extends WidgetRepositoryImpl {
+  private _cache = new LRUCache<1, ICustomWidget[]>({maxAge : 1000 * 60 /* minute */ * 2});
+  public async getWidgets() {
+    if (this._cache.has(1)) {
+      log.debug("WidgetRepository: Widget list taken from the cache.");
+      return this._cache.get(1)!;
+    }
+    const list = await super.getWidgets();
+    // Cache only if there are some widgets.
+    if (list.length) { this._cache.set(1, list); }
+    return list;
+  }
+
+  public testOverrideUrl(url: string) {
+    super.testOverrideUrl(url);
+    this._cache.reset();
+  }
+}
+
+/**
+ * Returns widget repository implementation.
+ */
+export function buildWidgetRepository() {
+  return new CachedWidgetRepository();
+}

app/server/lib/sendAppPage.ts

@@ -45,6 +45,7 @@ export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadCo
     maxUploadSizeImport: (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || undefined,
     maxUploadSizeAttachment: (Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024) || undefined,
     timestampMs: Date.now(),
+    enableWidgetRepository: Boolean(process.env.GRIST_WIDGET_LIST_URL),
     ...extra,
   };
 }