From 5ef889adddbcaf96158566b729fb27e81b223c3d Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Tue, 21 Jul 2020 09:20:51 -0400 Subject: [PATCH] (core) move home server into core Summary: This moves enough server material into core to run a home server. The data engine is not yet incorporated (though in manual testing it works when ported). Test Plan: existing tests pass Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2552 --- app/common/ActionBundle.ts | 74 + app/common/ActionDispatcher.ts | 71 + app/common/ActionGroup.ts | 19 + app/common/ActionRouter.ts | 49 + app/common/ActionSummary.ts | 235 ++ app/common/ActiveDocAPI.ts | 226 + app/common/ApiError.ts | 40 + app/common/AsyncCreate.ts | 147 + app/common/AsyncFlow.ts | 83 + app/common/BaseAPI.ts | 124 + app/common/BasketClientAPI.ts | 11 + app/common/BigInt.ts | 72 + app/common/BillingAPI.ts | 212 + app/common/BinaryIndexedTree.js | 258 ++ app/common/BrowserSettings.ts | 7 + app/common/ColumnGetters.ts | 26 + app/common/DisposableWithEvents.ts | 28 + app/common/DocActions.ts | 148 + app/common/DocData.ts | 121 + app/common/DocListAPI.ts | 83 + app/common/EncActionBundle.ts | 73 + app/common/ErrorWithCode.ts | 23 + app/common/Features.ts | 45 + app/common/Formula.ts | 72 + app/common/GristServerAPI.ts | 84 + app/common/InactivityTimer.ts | 119 + app/common/KeyedOps.ts | 222 + app/common/LoginSessionAPI.ts | 27 + app/common/LoginState.ts | 31 + app/common/MemBuffer.js | 294 ++ app/common/MetricCollector.js | 101 + app/common/NumberFormat.ts | 76 + app/common/PluginInstance.ts | 213 + app/common/RefCountMap.ts | 123 + app/common/SortFunc.ts | 88 + app/common/StringUnion.ts | 38 + app/common/TableData.ts | 427 ++ app/common/TabularDiff.ts | 31 + app/common/TestState.ts | 4 + app/common/UserAPI.ts | 667 +++ app/common/UserConfig.ts | 31 + app/common/ValueFormatter.ts | 144 + app/common/arrayToString.ts | 28 + app/common/declarations.d.ts | 5 + app/common/delay.ts | 7 + app/common/emails.ts | 43 + app/common/gristTypes.ts | 242 ++ app/common/gristUrls.ts | 584 ++- app/common/gutil.ts | 782 ++++ app/common/marshal.ts | 502 +++ app/common/metricConfig.js | 252 ++ app/common/metricTools.js | 261 ++ app/common/orgNameUtils.ts | 56 + app/common/parseDate.ts | 107 + app/common/plugin.ts | 77 + app/common/resetOrg.ts | 44 + app/common/roles.ts | 85 + app/common/schema.ts | 336 ++ app/common/sharing.ts | 43 + app/common/tbind.ts | 15 + app/common/timeFormat.ts | 56 + app/common/tpromisified.ts | 25 + app/common/tsconfig.json | 3 + app/common/tsvFormat.ts | 58 + app/common/uploads.ts | 44 + app/common/urlUtils.ts | 88 + app/common/version.ts | 3 + app/gen-server/ApiServer.ts | 475 +++ app/gen-server/entity/AclRule.ts | 58 + app/gen-server/entity/Alias.ts | 27 + app/gen-server/entity/BillingAccount.ts | 65 + .../entity/BillingAccountManager.ts | 26 + app/gen-server/entity/Document.ts | 69 + app/gen-server/entity/Group.ts | 33 + app/gen-server/entity/Login.ts | 25 + app/gen-server/entity/Organization.ts | 79 + app/gen-server/entity/Product.ts | 176 + app/gen-server/entity/Resource.ts | 46 + app/gen-server/entity/User.ts | 54 + app/gen-server/entity/Workspace.ts | 49 + app/gen-server/lib/DocApiForwarder.ts | 103 + app/gen-server/lib/DocWorkerMap.ts | 440 ++ app/gen-server/lib/HomeDBManager.ts | 3706 +++++++++++++++++ app/gen-server/lib/Permissions.ts | 21 + app/gen-server/lib/TypeORMPatches.ts | 196 + app/gen-server/lib/Usage.ts | 62 + app/gen-server/lib/scrubUserFromOrg.ts | 209 + app/gen-server/lib/values.ts | 36 + .../migration/1536634251710-Initial.ts | 304 ++ .../migration/1539031763952-Login.ts | 39 + .../migration/1549313797109-PinDocs.ts | 17 + .../migration/1549381727494-UserPicture.ts | 16 + .../1551805156919-LoginDisplayEmail.ts | 16 + .../1552416614755-LoginDisplayEmailNonNull.ts | 32 + .../migration/1553016106336-Indexes.ts | 66 + .../migration/1556726945436-Billing.ts | 225 + .../1557157922339-OrgDomainUnique.ts | 27 + .../migration/1561589211752-Aliases.ts | 67 + .../migration/1568238234987-TeamMembers.ts | 56 + .../migration/1569593726320-FirstLogin.ts | 18 + .../migration/1569946508569-FirstTimeUser.ts | 18 + .../migration/1573569442552-CustomerIndex.ts | 15 + .../migration/1579559983067-ExtraIndexes.ts | 45 + .../migration/1591755411755-OrgHost.ts | 17 + .../migration/1592261300044-DocRemovedAt.ts | 26 + app/gen-server/sqlUtils.ts | 90 + app/plugin/CustomSectionAPI-ti.ts | 11 + app/plugin/CustomSectionAPI.ts | 10 + app/plugin/FileParserAPI-ti.ts | 41 + app/plugin/FileParserAPI.ts | 52 + app/plugin/GristAPI-ti.ts | 34 + app/plugin/GristAPI.ts | 99 + app/plugin/GristTable-ti.ts | 24 + app/plugin/GristTable.ts | 38 + app/plugin/ImportSourceAPI-ti.ts | 44 + app/plugin/ImportSourceAPI.ts | 53 + app/plugin/InternalImportSourceAPI-ti.ts | 11 + app/plugin/InternalImportSourceAPI.ts | 24 + app/plugin/PluginManifest-ti.ts | 60 + app/plugin/PluginManifest.ts | 219 + app/plugin/RenderOptions-ti.ts | 14 + app/plugin/RenderOptions.ts | 11 + app/plugin/StorageAPI-ti.ts | 15 + app/plugin/StorageAPI.ts | 8 + app/plugin/TypeCheckers.ts | 50 + app/plugin/grist-plugin-api.ts | 135 + app/plugin/tsconfig.json | 3 + app/server/declarations.d.ts | 102 + app/server/devServerMain.ts | 144 + app/server/lib/ActionHistory.ts | 252 ++ app/server/lib/ActionHistoryImpl.ts | 673 +++ app/server/lib/ActionSummary.ts | 434 ++ app/server/lib/ActiveDoc.ts | 1132 +++++ app/server/lib/ActiveDocImport.ts | 311 ++ app/server/lib/AppEndpoint.ts | 254 ++ app/server/lib/Authorizer.ts | 429 ++ app/server/lib/BrowserSession.ts | 225 + app/server/lib/Client.ts | 362 ++ app/server/lib/Comm.js | 395 ++ app/server/lib/DocApi.ts | 560 +++ app/server/lib/DocClients.ts | 90 + app/server/lib/DocManager.ts | 481 +++ app/server/lib/DocPluginData.ts | 32 + app/server/lib/DocPluginManager.ts | 218 + app/server/lib/DocSession.ts | 49 + app/server/lib/DocSnapshots.ts | 151 + app/server/lib/DocStorage.ts | 1353 ++++++ app/server/lib/DocStorageManager.ts | 352 ++ app/server/lib/DocWorker.ts | 197 + app/server/lib/DocWorkerMap.ts | 62 + app/server/lib/ExpandedQuery.ts | 135 + app/server/lib/ExternalStorage.ts | 320 ++ app/server/lib/FileParserElement.ts | 49 + app/server/lib/FlexServer.ts | 1423 +++++++ app/server/lib/GristServer.ts | 26 + app/server/lib/HostedMetadataManager.ts | 89 + app/server/lib/HostedStorageManager.ts | 713 ++++ app/server/lib/IBilling.ts | 7 + app/server/lib/ICreate.ts | 30 + app/server/lib/IDocStorageManager.ts | 35 + app/server/lib/IInstanceManager.ts | 5 + app/server/lib/ILoginSession.ts | 15 + app/server/lib/INotifier.ts | 4 + app/server/lib/ISandbox.ts | 28 + app/server/lib/IShell.ts | 4 + app/server/lib/ITestingHooks-ti.ts | 34 + app/server/lib/ITestingHooks.ts | 20 + app/server/lib/NSandbox.ts | 354 ++ app/server/lib/OnDemandActions.ts | 206 + app/server/lib/PluginEndpoint.ts | 78 + app/server/lib/PluginManager.ts | 165 + app/server/lib/SQLiteDB.ts | 534 +++ app/server/lib/SafePythonComponent.ts | 73 + app/server/lib/ServerColumnGetters.ts | 41 + app/server/lib/ServerMetrics.js | 205 + app/server/lib/Sessions.ts | 107 + app/server/lib/Sharing.ts | 387 ++ app/server/lib/TagChecker.ts | 74 + app/server/lib/TestingHooks.ts | 205 + app/server/lib/Throttle.ts | 252 ++ app/server/lib/TimeQuery.ts | 164 + app/server/lib/UnsafeNodeComponent.ts | 161 + app/server/lib/WorkCoordinator.ts | 66 + app/server/lib/checksumFile.ts | 21 + app/server/lib/dbUtils.ts | 67 + app/server/lib/docUtils.d.ts | 9 + app/server/lib/docUtils.js | 145 + app/server/lib/expressWrap.ts | 36 + app/server/lib/extractOrg.ts | 166 + app/server/lib/gristSessions.ts | 138 + app/server/lib/guessExt.ts | 51 + app/server/lib/idUtils.ts | 48 + app/server/lib/log.ts | 89 + app/server/lib/manifest.ts | 79 + app/server/lib/places.ts | 46 + app/server/lib/requestUtils.ts | 204 + app/server/lib/sandboxUtil.js | 58 + app/server/lib/sendAppPage.ts | 105 + app/server/lib/serverUtils.ts | 116 + app/server/lib/shortDesc.ts | 60 + app/server/lib/shutdown.js | 116 + app/server/lib/uploads.ts | 425 ++ app/server/mergedServerMain.ts | 171 + app/server/server.ts | 24 - app/server/serverMethods.js | 110 + app/server/tsconfig.json | 5 + app/tsconfig.json | 1 + ormconfig.js | 49 + package.json | 68 +- static/app.html | 74 + stubs/app/server/declarations.d.ts | 71 + stubs/app/server/lib/LoginSession.ts | 25 + stubs/app/server/lib/StandaloneExtras.ts | 4 + stubs/app/server/lib/create.ts | 65 + stubs/app/server/lib/logins.ts | 12 + stubs/app/server/server.ts | 25 + stubs/app/server/tmp.d.ts | 7 + stubs/app/tsconfig.json | 3 +- 218 files changed, 33640 insertions(+), 38 deletions(-) create mode 100644 app/common/ActionBundle.ts create mode 100644 app/common/ActionDispatcher.ts create mode 100644 app/common/ActionGroup.ts create mode 100644 app/common/ActionRouter.ts create mode 100644 app/common/ActionSummary.ts create mode 100644 app/common/ActiveDocAPI.ts create mode 100644 app/common/ApiError.ts create mode 100644 app/common/AsyncCreate.ts create mode 100644 app/common/AsyncFlow.ts create mode 100644 app/common/BaseAPI.ts create mode 100644 app/common/BasketClientAPI.ts create mode 100644 app/common/BigInt.ts create mode 100644 app/common/BillingAPI.ts create mode 100644 app/common/BinaryIndexedTree.js create mode 100644 app/common/BrowserSettings.ts create mode 100644 app/common/ColumnGetters.ts create mode 100644 app/common/DisposableWithEvents.ts create mode 100644 app/common/DocActions.ts create mode 100644 app/common/DocData.ts create mode 100644 app/common/DocListAPI.ts create mode 100644 app/common/EncActionBundle.ts create mode 100644 app/common/ErrorWithCode.ts create mode 100644 app/common/Features.ts create mode 100644 app/common/Formula.ts create mode 100644 app/common/GristServerAPI.ts create mode 100644 app/common/InactivityTimer.ts create mode 100644 app/common/KeyedOps.ts create mode 100644 app/common/LoginSessionAPI.ts create mode 100644 app/common/LoginState.ts create mode 100644 app/common/MemBuffer.js create mode 100644 app/common/MetricCollector.js create mode 100644 app/common/NumberFormat.ts create mode 100644 app/common/PluginInstance.ts create mode 100644 app/common/RefCountMap.ts create mode 100644 app/common/SortFunc.ts create mode 100644 app/common/StringUnion.ts create mode 100644 app/common/TableData.ts create mode 100644 app/common/TabularDiff.ts create mode 100644 app/common/TestState.ts create mode 100644 app/common/UserAPI.ts create mode 100644 app/common/UserConfig.ts create mode 100644 app/common/ValueFormatter.ts create mode 100644 app/common/arrayToString.ts create mode 100644 app/common/declarations.d.ts create mode 100644 app/common/delay.ts create mode 100644 app/common/emails.ts create mode 100644 app/common/gristTypes.ts create mode 100644 app/common/gutil.ts create mode 100644 app/common/marshal.ts create mode 100644 app/common/metricConfig.js create mode 100644 app/common/metricTools.js create mode 100644 app/common/orgNameUtils.ts create mode 100644 app/common/parseDate.ts create mode 100644 app/common/plugin.ts create mode 100644 app/common/resetOrg.ts create mode 100644 app/common/roles.ts create mode 100644 app/common/schema.ts create mode 100644 app/common/sharing.ts create mode 100644 app/common/tbind.ts create mode 100644 app/common/timeFormat.ts create mode 100644 app/common/tpromisified.ts create mode 100644 app/common/tsvFormat.ts create mode 100644 app/common/uploads.ts create mode 100644 app/common/urlUtils.ts create mode 100644 app/common/version.ts create mode 100644 app/gen-server/ApiServer.ts create mode 100644 app/gen-server/entity/AclRule.ts create mode 100644 app/gen-server/entity/Alias.ts create mode 100644 app/gen-server/entity/BillingAccount.ts create mode 100644 app/gen-server/entity/BillingAccountManager.ts create mode 100644 app/gen-server/entity/Document.ts create mode 100644 app/gen-server/entity/Group.ts create mode 100644 app/gen-server/entity/Login.ts create mode 100644 app/gen-server/entity/Organization.ts create mode 100644 app/gen-server/entity/Product.ts create mode 100644 app/gen-server/entity/Resource.ts create mode 100644 app/gen-server/entity/User.ts create mode 100644 app/gen-server/entity/Workspace.ts create mode 100644 app/gen-server/lib/DocApiForwarder.ts create mode 100644 app/gen-server/lib/DocWorkerMap.ts create mode 100644 app/gen-server/lib/HomeDBManager.ts create mode 100644 app/gen-server/lib/Permissions.ts create mode 100644 app/gen-server/lib/TypeORMPatches.ts create mode 100644 app/gen-server/lib/Usage.ts create mode 100644 app/gen-server/lib/scrubUserFromOrg.ts create mode 100644 app/gen-server/lib/values.ts create mode 100644 app/gen-server/migration/1536634251710-Initial.ts create mode 100644 app/gen-server/migration/1539031763952-Login.ts create mode 100644 app/gen-server/migration/1549313797109-PinDocs.ts create mode 100644 app/gen-server/migration/1549381727494-UserPicture.ts create mode 100644 app/gen-server/migration/1551805156919-LoginDisplayEmail.ts create mode 100644 app/gen-server/migration/1552416614755-LoginDisplayEmailNonNull.ts create mode 100644 app/gen-server/migration/1553016106336-Indexes.ts create mode 100644 app/gen-server/migration/1556726945436-Billing.ts create mode 100644 app/gen-server/migration/1557157922339-OrgDomainUnique.ts create mode 100644 app/gen-server/migration/1561589211752-Aliases.ts create mode 100644 app/gen-server/migration/1568238234987-TeamMembers.ts create mode 100644 app/gen-server/migration/1569593726320-FirstLogin.ts create mode 100644 app/gen-server/migration/1569946508569-FirstTimeUser.ts create mode 100644 app/gen-server/migration/1573569442552-CustomerIndex.ts create mode 100644 app/gen-server/migration/1579559983067-ExtraIndexes.ts create mode 100644 app/gen-server/migration/1591755411755-OrgHost.ts create mode 100644 app/gen-server/migration/1592261300044-DocRemovedAt.ts create mode 100644 app/gen-server/sqlUtils.ts create mode 100644 app/plugin/CustomSectionAPI-ti.ts create mode 100644 app/plugin/CustomSectionAPI.ts create mode 100644 app/plugin/FileParserAPI-ti.ts create mode 100644 app/plugin/FileParserAPI.ts create mode 100644 app/plugin/GristAPI-ti.ts create mode 100644 app/plugin/GristAPI.ts create mode 100644 app/plugin/GristTable-ti.ts create mode 100644 app/plugin/GristTable.ts create mode 100644 app/plugin/ImportSourceAPI-ti.ts create mode 100644 app/plugin/ImportSourceAPI.ts create mode 100644 app/plugin/InternalImportSourceAPI-ti.ts create mode 100644 app/plugin/InternalImportSourceAPI.ts create mode 100644 app/plugin/PluginManifest-ti.ts create mode 100644 app/plugin/PluginManifest.ts create mode 100644 app/plugin/RenderOptions-ti.ts create mode 100644 app/plugin/RenderOptions.ts create mode 100644 app/plugin/StorageAPI-ti.ts create mode 100644 app/plugin/StorageAPI.ts create mode 100644 app/plugin/TypeCheckers.ts create mode 100644 app/plugin/grist-plugin-api.ts create mode 100644 app/plugin/tsconfig.json create mode 100644 app/server/declarations.d.ts create mode 100644 app/server/devServerMain.ts create mode 100644 app/server/lib/ActionHistory.ts create mode 100644 app/server/lib/ActionHistoryImpl.ts create mode 100644 app/server/lib/ActionSummary.ts create mode 100644 app/server/lib/ActiveDoc.ts create mode 100644 app/server/lib/ActiveDocImport.ts create mode 100644 app/server/lib/AppEndpoint.ts create mode 100644 app/server/lib/Authorizer.ts create mode 100644 app/server/lib/BrowserSession.ts create mode 100644 app/server/lib/Client.ts create mode 100644 app/server/lib/Comm.js create mode 100644 app/server/lib/DocApi.ts create mode 100644 app/server/lib/DocClients.ts create mode 100644 app/server/lib/DocManager.ts create mode 100644 app/server/lib/DocPluginData.ts create mode 100644 app/server/lib/DocPluginManager.ts create mode 100644 app/server/lib/DocSession.ts create mode 100644 app/server/lib/DocSnapshots.ts create mode 100644 app/server/lib/DocStorage.ts create mode 100644 app/server/lib/DocStorageManager.ts create mode 100644 app/server/lib/DocWorker.ts create mode 100644 app/server/lib/DocWorkerMap.ts create mode 100644 app/server/lib/ExpandedQuery.ts create mode 100644 app/server/lib/ExternalStorage.ts create mode 100644 app/server/lib/FileParserElement.ts create mode 100644 app/server/lib/FlexServer.ts create mode 100644 app/server/lib/GristServer.ts create mode 100644 app/server/lib/HostedMetadataManager.ts create mode 100644 app/server/lib/HostedStorageManager.ts create mode 100644 app/server/lib/IBilling.ts create mode 100644 app/server/lib/ICreate.ts create mode 100644 app/server/lib/IDocStorageManager.ts create mode 100644 app/server/lib/IInstanceManager.ts create mode 100644 app/server/lib/ILoginSession.ts create mode 100644 app/server/lib/INotifier.ts create mode 100644 app/server/lib/ISandbox.ts create mode 100644 app/server/lib/IShell.ts create mode 100644 app/server/lib/ITestingHooks-ti.ts create mode 100644 app/server/lib/ITestingHooks.ts create mode 100644 app/server/lib/NSandbox.ts create mode 100644 app/server/lib/OnDemandActions.ts create mode 100644 app/server/lib/PluginEndpoint.ts create mode 100644 app/server/lib/PluginManager.ts create mode 100644 app/server/lib/SQLiteDB.ts create mode 100644 app/server/lib/SafePythonComponent.ts create mode 100644 app/server/lib/ServerColumnGetters.ts create mode 100644 app/server/lib/ServerMetrics.js create mode 100644 app/server/lib/Sessions.ts create mode 100644 app/server/lib/Sharing.ts create mode 100644 app/server/lib/TagChecker.ts create mode 100644 app/server/lib/TestingHooks.ts create mode 100644 app/server/lib/Throttle.ts create mode 100644 app/server/lib/TimeQuery.ts create mode 100644 app/server/lib/UnsafeNodeComponent.ts create mode 100644 app/server/lib/WorkCoordinator.ts create mode 100644 app/server/lib/checksumFile.ts create mode 100644 app/server/lib/dbUtils.ts create mode 100644 app/server/lib/docUtils.d.ts create mode 100644 app/server/lib/docUtils.js create mode 100644 app/server/lib/expressWrap.ts create mode 100644 app/server/lib/extractOrg.ts create mode 100644 app/server/lib/gristSessions.ts create mode 100644 app/server/lib/guessExt.ts create mode 100644 app/server/lib/idUtils.ts create mode 100644 app/server/lib/log.ts create mode 100644 app/server/lib/manifest.ts create mode 100644 app/server/lib/places.ts create mode 100644 app/server/lib/requestUtils.ts create mode 100644 app/server/lib/sandboxUtil.js create mode 100644 app/server/lib/sendAppPage.ts create mode 100644 app/server/lib/serverUtils.ts create mode 100644 app/server/lib/shortDesc.ts create mode 100644 app/server/lib/shutdown.js create mode 100644 app/server/lib/uploads.ts create mode 100644 app/server/mergedServerMain.ts delete mode 100644 app/server/server.ts create mode 100644 app/server/serverMethods.js create mode 100644 ormconfig.js create mode 100644 static/app.html create mode 100644 stubs/app/server/declarations.d.ts create mode 100644 stubs/app/server/lib/LoginSession.ts create mode 100644 stubs/app/server/lib/StandaloneExtras.ts create mode 100644 stubs/app/server/lib/create.ts create mode 100644 stubs/app/server/lib/logins.ts create mode 100644 stubs/app/server/server.ts create mode 100644 stubs/app/server/tmp.d.ts diff --git a/app/common/ActionBundle.ts b/app/common/ActionBundle.ts new file mode 100644 index 00000000..5a006af6 --- /dev/null +++ b/app/common/ActionBundle.ts @@ -0,0 +1,74 @@ +/** + * Basic definitions of types needed for ActionBundles. + * See also EncActionBundle for how these are packaged for encryption. + */ + +import {DocAction, UserAction} from 'app/common/DocActions'; + +// Metadata about the action. +export interface ActionInfo { + time: number; // Milliseconds since epoch. + user: string; + inst: string; + desc?: string; + otherId: number; + linkId: number; +} + +// Envelope contains information about recipients. In EncActionBundle, it's augmented with +// information about the symmetric key that encrypts this envelope's contents. +export interface Envelope { + recipients: string[]; // sorted array of recipient instanceIds +} + +// EnvContent packages arbitrary content with the index of the envelope to which it belongs. +export type EnvContent = [number, Content]; + +// ActionBundle contains actions arranged into envelopes, i.e. split up by sets of recipients. +// Note that different Envelopes contain different sets of recipients (which may overlap however). +// ActionBundle is what gets encrypted/decrypted and then sent between hub and instance. +export interface ActionBundle { + actionNum: number; + actionHash: string|null; // a checksum of bundle, (not including actionHash and other parts). + parentActionHash: string|null; // a checksum of the parent action bundle, if there is one. + envelopes: Envelope[]; + info: EnvContent; // Should be in the envelope addressed to all peers. + stored: Array>; + calc: Array>; +} + +export function getEnvContent(items: Array>): Content[] { + return items.map((item) => item[1]); +} + +// ====================================================================== +// Types for ActionBundles used locally inside an instance. + +// Local action received from the browser, that is not yet applied. It is usually one UserAction, +// but when multiple actions are sent by the browser in one call, they will form one bundle. +export interface UserActionBundle { + info: ActionInfo; + userActions: UserAction[]; +} + +// ActionBundle as received from the sandbox. It does not have some action metadata, but does have +// undo information and a retValue for each input UserAction. Note that it is satisfied by the +// ActionBundle structure defined in sandbox/grist/action_obj.py. +export interface SandboxActionBundle { + envelopes: Envelope[]; + stored: Array>; + calc: Array>; + undo: Array>; // Inverse actions for all 'stored' actions. + retValues: any[]; // Contains retValue for each of userActions. +} + +// Local action that's been applied. It now has an actionNum, and includes doc actions packaged +// into envelopes, as well as undo, and userActions, which allow rebasing. +export interface LocalActionBundle extends ActionBundle { + userActions: UserAction[]; + + // Inverse actions for all 'stored' actions. These aren't shared and not split by envelope. + // Applying 'undo' is governed by EDIT rather than READ permissions, so we always apply all undo + // actions. (It is the result of applying 'undo' that may be addressed to different recipients). + undo: DocAction[]; +} diff --git a/app/common/ActionDispatcher.ts b/app/common/ActionDispatcher.ts new file mode 100644 index 00000000..603aeaad --- /dev/null +++ b/app/common/ActionDispatcher.ts @@ -0,0 +1,71 @@ +import mapValues = require('lodash/mapValues'); +import {BulkColValues, ColInfo, ColInfoWithId, ColValues, DocAction} from "./DocActions"; + +// TODO this replaces modelUtil's ActionDispatcher and bulkActionExpand. Those should be removed. + +/** + * Helper class which provides a `dispatchAction` method that dispatches DocActions received from + * the server to methods `this.on{ActionType}`, e.g. `this.onUpdateRecord`. + * + * Implementation methods `on*` are called with the action as the first argument, and with + * the action arguments as additional method arguments, for convenience. + * + * Methods for bulk actions may be implemented directly, or will iterate through each record in + * the action, and call the single-record methods for each one. + */ +export abstract class ActionDispatcher { + public dispatchAction(action: DocAction): void { + // In node 6 testing, this switch is 5+ times faster than looking up "on"+action[0]. + const a: any[] = action; + switch (action[0]) { + case "AddRecord": return this.onAddRecord (action, a[1], a[2], a[3]); + case "UpdateRecord": return this.onUpdateRecord (action, a[1], a[2], a[3]); + case "RemoveRecord": return this.onRemoveRecord (action, a[1], a[2]); + case "BulkAddRecord": return this.onBulkAddRecord (action, a[1], a[2], a[3]); + case "BulkUpdateRecord": return this.onBulkUpdateRecord(action, a[1], a[2], a[3]); + case "BulkRemoveRecord": return this.onBulkRemoveRecord(action, a[1], a[2]); + case "ReplaceTableData": return this.onReplaceTableData(action, a[1], a[2], a[3]); + case "AddColumn": return this.onAddColumn (action, a[1], a[2], a[3]); + case "RemoveColumn": return this.onRemoveColumn (action, a[1], a[2]); + case "RenameColumn": return this.onRenameColumn (action, a[1], a[2], a[3]); + case "ModifyColumn": return this.onModifyColumn (action, a[1], a[2], a[3]); + case "AddTable": return this.onAddTable (action, a[1], a[2]); + case "RemoveTable": return this.onRemoveTable (action, a[1]); + case "RenameTable": return this.onRenameTable (action, a[1], a[2]); + default: throw new Error(`Received unknown action ${action[0]}`); + } + } + + protected abstract onAddRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void; + protected abstract onUpdateRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void; + protected abstract onRemoveRecord(action: DocAction, tableId: string, rowId: number): void; + + // If not overridden, these will make multiple calls to single-record action methods. + protected onBulkAddRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void { + for (let i = 0; i < rowIds.length; i++) { + this.onAddRecord(action, tableId, rowIds[i], mapValues(colValues, (values) => values[i])); + } + } + protected onBulkUpdateRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void { + for (let i = 0; i < rowIds.length; i++) { + this.onUpdateRecord(action, tableId, rowIds[i], mapValues(colValues, (values) => values[i])); + } + } + protected onBulkRemoveRecord(action: DocAction, tableId: string, rowIds: number[]) { + for (const r of rowIds) { + this.onRemoveRecord(action, tableId, r); + } + } + + protected abstract onReplaceTableData( + action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void; + + protected abstract onAddColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void; + protected abstract onRemoveColumn(action: DocAction, tableId: string, colId: string): void; + protected abstract onRenameColumn(action: DocAction, tableId: string, oldColId: string, newColId: string): void; + protected abstract onModifyColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void; + + protected abstract onAddTable(action: DocAction, tableId: string, columns: ColInfoWithId[]): void; + protected abstract onRemoveTable(action: DocAction, tableId: string): void; + protected abstract onRenameTable(action: DocAction, oldTableId: string, newTableId: string): void; +} diff --git a/app/common/ActionGroup.ts b/app/common/ActionGroup.ts new file mode 100644 index 00000000..f8977e24 --- /dev/null +++ b/app/common/ActionGroup.ts @@ -0,0 +1,19 @@ +import {ActionSummary} from 'app/common/ActionSummary'; + +/** This is the action representation the client works with. */ +export interface ActionGroup { + actionNum: number; + actionHash: string; + desc?: string; + actionSummary: ActionSummary; + fromSelf: boolean; + linkId: number; + otherId: number; + time: number; + user: string; + rowIdHint: number; // If non-zero, this is a rowId that would be a good place to put + // the cursor after an undo. + primaryAction: string; // The name of the first user action in the ActionGroup. + isUndo: boolean; // True if the first user action is ApplyUndoActions. + internal: boolean; // True if it is inappropriate to log/undo the action. +} diff --git a/app/common/ActionRouter.ts b/app/common/ActionRouter.ts new file mode 100644 index 00000000..d10ce0b7 --- /dev/null +++ b/app/common/ActionRouter.ts @@ -0,0 +1,49 @@ +import { Rpc } from "grain-rpc"; + +/** + * ActionRouter allows to choose what actions to send over rpc. Action are posted as message `{type: + * "docAction", action }` over rpc. + */ +export class ActionRouter { + + private _subscribedTables: Set = new Set(); + + constructor(private _rpc: Rpc) {} + + /** + * Subscribe to send all actions related to a table. Keeps sending actions if table is renamed. + */ + public subscribeTable(tableId: string): Promise { + this._subscribedTables.add(tableId); + return Promise.resolve(); + } + + /** + * Stop sending all message related to a table. + */ + public unsubscribeTable(tableId: string): Promise { + this._subscribedTables.delete(tableId); + return Promise.resolve(); + } + + /** + * Process a action updates subscription set in case of table rename and table remove, and post + * action if it matches a subscriptions. + */ + public process(action: any[]): Promise { + const tableId = action[1]; + if (!this._subscribedTables.has(tableId)) { + return Promise.resolve(); + } + switch (action[0]) { + case "RemoveTable": + this._subscribedTables.delete(tableId); + break; + case "RenameTable": + this._subscribedTables.delete(tableId); + this._subscribedTables.add(action[2]); + break; + } + return this._rpc.postMessage({type: "docAction", action}); + } +} diff --git a/app/common/ActionSummary.ts b/app/common/ActionSummary.ts new file mode 100644 index 00000000..8f4f36d4 --- /dev/null +++ b/app/common/ActionSummary.ts @@ -0,0 +1,235 @@ +import {CellDelta, TabularDiff, TabularDiffs} from 'app/common/TabularDiff'; +import toPairs = require('lodash/toPairs'); + +/** + * An ActionSummary represents the overall effect of changes that took place + * during a period of history. + * - Only net changes are represented. Intermediate changes within the period are + * not represented. Changes that are done and undone within the period are not + * represented. + * - Net addition, removal, and renaming of tables is represented. The names + * of tables, for ActionSummary purposes are their tableIds, the database-safe + * version of their names. + * - Net addition, removal, and renaming of columns is represented. As for tables, + * the names of columns for ActionSummary purposes are their colIds. + * - Net additions and removals of rows are partially represented. The rowIds of added + * and removed rows are represented fully. The *values* of cells in the rows that + * were added or removed are stored in some cases. There is a threshold on the + * number of rows whose values will be cached for each DocAction scanned. + * - Net updates of rows are partially represented. The rowIds of updated rows are + * represented fully, but the *values* of updated cells partially, as for additions/ + * removals. + * - Cell value changes affecting _grist_* tables are always represented in full, + * even if they are bulk changes. + * + * The representation of table name changes and column name changes is the same, + * simply a list of name pairs [before, after]. We represent the addition of a + * a table (or column) as the special name pair [null, initialName], and the + * removal of a table (or column) as the special name pair [finalName, null]. + * + * An ActionSummary contains two fields: + * - tableRenames: a list of table name changes (incuding addition/removal). + * - tableDeltas: a dictionary of changes within a table. + * + * The key of the tableDeltas dictionary is the name of a table at the end of the + * period of history covered by the ActionSummary. + * - For example, if we add a table called N, we use the key N for it. + * - If we rename a table from N1 to N2, we use the key N2 for it. + * - If we add a table called N1, then rename it to N2, we use the key N2 for it. + * If the table was removed during that period, we use its name at the beginning + * of the period, preceded by "-". + * - If we remove a table called N, we use the key -N for it. + * - If we add a table called N, then remove it, there is no net change to represent. + * - If we remove a table called N, then add a new table called N, we use the key -N + * for the first, and the key N for the second. + * + * The changes within a table are represented as a TableDelta, which has the following + * fields: + * - columnRenames: a list of column name changes (incuding addition/removal). + * - columnDeltas: a dictionary of changes within a column. + * - updateRows, removeRows, addRows: lists of affected rows. + * + * The columnRenames/columnDeltas pair work just like tableRenames/tableDeltas, just + * on the scope of columns within a table rather than tables within a document. + * + * The changes within a column are represented as a ColumnDelta, which is a dictionary + * keyed by rowIds. It contains CellDelta values. CellDelta values represent before + * and after values of a particular cell. + * - a CellDelta of [null, [value]] represents a cell that was non-existent coming into + * existence with the given value. + * - a CellDelta of [[value], null] represents an existing cell with the given value that + * is removed. + * - a CellDelta of [[value1], [value2]] represents a change in value of a cell between + * two known values. + * - a CellDelta of ['?', [value2]] represents a change in value of a cell from an + * unknown value to a known value. Unknown values happen when we know a cell was + * implicated in a bulk change but its value didn't happen to be stored. + * - a CellDelta of [[value1], '?'] represents a change in value of a cell from an + * known value to an unknown value. + * The CellDelta itself does not tell you whether the rowId has the same identity before + * and after -- for example it may have been removed and then added. That information + * is available by consulting the removeRows and addRows fields. + * + */ + +/** + * A collection of changes related to a set of tables. + */ +export interface ActionSummary { + tableRenames: LabelDelta[]; /** a list of table renames/additions/removals */ + tableDeltas: {[tableId: string]: TableDelta}; /** changes within an individual table */ +} + +/** + * A collection of changes related to rows and columns of a single table. + */ +export interface TableDelta { + updateRows: number[]; /** rowIds of rows that exist before+after and were changed during */ + removeRows: number[]; /** rowIds of rows that existed before but were removed during */ + addRows: number[]; /** rowIds of rows that were added during, and exist after */ + /** Partial record of cell-level changes - large bulk changes not included. */ + columnDeltas: {[colId: string]: ColumnDelta}; + columnRenames: LabelDelta[]; /** a list of column renames/additions/removals */ +} + +/** + * Pairs of before/after names of tables and columns. Null represents non-existence, + * so the addition and removal of tables/columns can be represented. + */ +export type LabelDelta = [string|null, string|null]; + +/** + * A collection of changes related to cells in a specific column. + */ +export interface ColumnDelta { + [rowId: number]: CellDelta; +} + + +/** Create an ActionSummary for a period with no action */ +export function createEmptyActionSummary(): ActionSummary { + return { tableRenames: [], tableDeltas: {} }; +} + +/** Create a TableDelta for a period with no action */ +export function createEmptyTableDelta(): TableDelta { + return { + updateRows: [], + removeRows: [], + addRows: [], + columnDeltas: {}, + columnRenames: [] + }; +} + + +/** + * Distill a summary further, into tabular form, for ease of rendering. + */ +export function asTabularDiffs(summary: ActionSummary): TabularDiffs { + const allChanges: TabularDiffs = {}; + for (const [tableId, td] of toPairs(summary.tableDeltas)) { + const tableChanges: TabularDiff = allChanges[tableId] = { + header: [], + cells: [], + }; + // swap order to row-dominant for visualization purposes + const perRow: {[row: number]: {[name: string]: any}} = {}; + const activeCols = new Set(); + // iterate through the column-dominant representation grist prefers internally + for (const [col, perCol] of toPairs(td.columnDeltas)) { + activeCols.add(col); + // iterate through the rows for that column, writing out the row-dominant + // results we want for visualization. + for (const row of Object.keys(perCol)) { + if (!perRow[row as any]) { perRow[row as any] = {}; } + perRow[row as any][col] = perCol[row as any]; + } + } + // TODO: recover order of columns; recover row numbers (as opposed to rowIds) + const activeColsWithoutManualSort = [...activeCols].filter(c => c !== 'manualSort'); + tableChanges.header = activeColsWithoutManualSort; + const addedRows = new Set(td.addRows); + const removedRows = new Set(td.removeRows); + const updatedRows = new Set(td.updateRows); + const rowIds = Object.keys(perRow).map(row => parseInt(row, 10)); + const presentRows = new Set(rowIds); + const droppedRows = [...addedRows, ...removedRows, ...updatedRows] + .filter(x => !presentRows.has(x)) + .sort((a, b) => a - b); + + // Now that we have pulled together rows of changes, we will add a summary cell + // to each row to show whether they were caused by row updates, additions or removals. + // We also at this point make sure the cells of the row are output in a consistent + // order with a header. + for (const rowId of rowIds) { + if (droppedRows.length > 0) { + // Bulk additions/removals/updates may result in just some rows being saved. + // We signal this visually with a "..." row. The order of where this should + // go isn't well defined at this point (there's a row number TODO above). + if (rowId > droppedRows[0]) { + tableChanges.cells.push(['...', droppedRows[0], + activeColsWithoutManualSort.map(x => [null, null] as [null, null])]); + while (rowId > droppedRows[0]) { + droppedRows.shift(); + } + } + } + // For each rowId, we need to issue either 1 or 2 rows. We issue 2 rows + // if the rowId is both added and removed - in this scenario, the rows + // before and after are unrelated. In all other cases, the before and + // after values refer to the same row. + const versions: Array<[string, (diff: CellDelta) => CellDelta]> = []; + if (addedRows.has(rowId) && removedRows.has(rowId)) { + versions.push(['-', (diff) => [diff[0], null]]); + versions.push(['+', (diff) => [null, diff[1]]]); + } else { + let code: string = '...'; + if (updatedRows.has(rowId)) { code = '→'; } + if (addedRows.has(rowId)) { code = '+'; } + if (removedRows.has(rowId)) { code = '-'; } + versions.push([code, (diff) => diff]); + } + for (const [code, transform] of versions) { + const acc: CellDelta[] = []; + const perCol = perRow[rowId]; + activeColsWithoutManualSort.forEach(col => { + const diff = perCol ? perCol[col] : null; + if (!diff) { + acc.push([null, null]); + } else { + acc.push(transform(diff)); + } + }); + tableChanges.cells.push([code, rowId, acc]); + } + } + } + return allChanges; +} + +/** + * Return a suitable key for a removed table/column. We cannot use their id directly + * since it could clash with an added table/column of the same name. + */ +export function defunctTableName(id: string): string { + return `-${id}`; +} + +export function rootTableName(id: string): string { + return id.replace('-', ''); +} + +/** + * Returns a list of all tables changed by the summarized action. Changes include + * schema or data changes. Tables are identified by their post-action name. + * Deleted tables are identified by their pre-action name, with "-" prepended. + */ +export function getAffectedTables(summary: ActionSummary): string[] { + return [ + // Tables added, renamed, or removed in this action. + ...summary.tableRenames.map(pair => pair[1] || defunctTableName(pair[0] || "")), + // Tables modified in this action. + ...Object.keys(summary.tableDeltas) + ]; +} diff --git a/app/common/ActiveDocAPI.ts b/app/common/ActiveDocAPI.ts new file mode 100644 index 00000000..46aa352a --- /dev/null +++ b/app/common/ActiveDocAPI.ts @@ -0,0 +1,226 @@ +import {ActionGroup} from 'app/common/ActionGroup'; +import {CellValue, TableDataAction, UserAction} from 'app/common/DocActions'; +import {Peer} from 'app/common/sharing'; +import {UploadResult} from 'app/common/uploads'; +import {ParseOptions} from 'app/plugin/FileParserAPI'; +import {IMessage} from 'grain-rpc'; + +export interface ApplyUAOptions { + desc?: string; // Overrides the description of the action. + otherId?: number; // For undo/redo; the actionNum of the original action to which it applies. + linkId?: number; // For bundled actions, actionNum of the previous action in the bundle. +} + +export interface ApplyUAResult { + actionNum: number; // number of the action that got recorded. + retValues: any[]; // array of return values, one for each of the passed-in user actions. + isModification: boolean; // true if document was modified. +} + +export interface DataSourceTransformed { + // Identifies the upload, which may include multiple files. + uploadId: number; + + // For each file in the upload, the transform rules for that file. + transforms: TransformRuleMap[]; +} + +export interface TransformRuleMap { + [origTableName: string]: TransformRule; +} + +export interface TransformRule { + destTableId: string|null; + destCols: TransformColumn[]; + sourceCols: string[]; +} + +export interface TransformColumn { + label: string; + colId: string|null; + type: string; + formula: string; +} + +export interface ImportResult { + options: ParseOptions; + tables: ImportTableResult[]; +} + +export interface ImportTableResult { + hiddenTableId: string; + uploadFileIndex: number; // Index into upload.files array, for the file reponsible for this table. + origTableName: string; + transformSectionRef: number; + destTableId: string|null; +} + +/** + * Represents a query for Grist data. The tableId is required. An empty set of filters indicates + * the full table. Examples: + * {tableId: "Projects", filters: {}} + * {tableId: "Employees", filters: {Status: ["Active"], Dept: ["Sales", "HR"]}} + */ +export interface Query { + tableId: string; + filters: { + [colId: string]: any[]; + }; + + // Queries to server for onDemand tables will set a limit to avoid bringing down the browser. + limit?: number; +} + +/** + * Response from useQuerySet(). A query returns data AND creates a subscription to receive + * DocActions that affect this data. The querySubId field identifies this subscription, and must + * be used in a disposeQuerySet() call to unsubscribe. + */ +export interface QueryResult { + querySubId: number; // ID of the subscription, to use with disposeQuerySet. + tableData: TableDataAction; +} + +/** + * Result of a fork operation, with newly minted ids. + * For a document with docId XXXXX and urlId UUUUU, the fork will have a + * docId of XXXXX~FORKID[~USERID] and a urlId of UUUUU~FORKID[~USERID]. + */ +export interface ForkResult { + docId: string; + urlId: string; +} + +export interface ActiveDocAPI { + /** + * Closes a document, and unsubscribes from its userAction events. + */ + closeDoc(): Promise; + + /** + * Fetches a particular table from the data engine to return to the client. + */ + fetchTable(tableId: string): Promise; + + /** + * Fetches the generated Python code for this document. (TODO rename this misnomer.) + */ + fetchTableSchema(): Promise; + + /** + * Makes a query (documented elsewhere) and subscribes to it, so that the client receives + * docActions that affect this query's results. The subscription remains functional even when + * tables or columns get renamed. + */ + useQuerySet(query: Query): Promise; + + /** + * Removes the subscription to a Query, identified by QueryResult.querySubId, so that the + * client stops receiving docActions relevant only to that query. + */ + disposeQuerySet(querySubId: number): Promise; + + /** + * Applies an array of user actions to the document. + */ + applyUserActions(actions: UserAction[], options?: ApplyUAOptions): Promise; + + /** + * A variant of applyUserActions where actions are passed in by ids (actionNum, actionHash) + * rather than by value. + */ + applyUserActionsById(actionNums: number[], actionHashes: string[], + undo: boolean, options?: ApplyUAOptions): Promise; + + /** + * Imports files, removes previously created temporary hidden tables and creates the new ones. + */ + importFiles(dataSource: DataSourceTransformed, + parseOptions: ParseOptions, prevTableIds: string[]): Promise; + + /** + * Finishes import files, creates the new tables, and cleans up temporary hidden tables and uploads. + */ + finishImportFiles(dataSource: DataSourceTransformed, + parseOptions: ParseOptions, prevTableIds: string[]): Promise; + + /** + * Cancels import files, cleans up temporary hidden tables and uploads. + */ + cancelImportFiles(dataSource: DataSourceTransformed, prevTableIds: string[]): Promise; + + /** + * Saves attachments from a given upload and creates an entry for them in the database. It + * returns the list of rowIds for the rows created in the _grist_Attachments table. + */ + addAttachments(uploadId: number): Promise; + + /** + * Returns up to n columns in the document, or a specific table, which contain the given values. + * Columns are returned ordered from best to worst based on an estimate for number of matches. + */ + findColFromValues(values: any[], n: number, optTableId?: string): Promise; + + /** + * Returns cell value with an error message (traceback) for one invalid formula cell. + */ + getFormulaError(tableId: string, colId: string, rowId: number): Promise; + + /** + * Fetch content at a url. + */ + fetchURL(url: string): Promise; + + /** + * Find and return a list of auto-complete suggestions that start with `txt`, when editing a + * formula in table `tableId`. + */ + autocomplete(txt: string, tableId: string): Promise; + + /** + * Shares the doc and invites peers. + */ + shareDoc(peers: Peer[]): Promise; + + /** + * Removes the current instance from the doc. + */ + removeInstanceFromDoc(): Promise; + + /** + * Get recent actions in ActionGroup format with summaries included. + */ + getActionSummaries(): Promise; + + /** + * Initiates user actions bandling for undo. + */ + startBundleUserActions(): Promise; + + /** + * Stopes user actions bandling for undo. + */ + stopBundleUserActions(): Promise; + + /** + * Forward a grain-rpc message to a given plugin. + */ + forwardPluginRpc(pluginId: string, msg: IMessage): Promise; + + /** + * Reload documents plugins. + */ + reloadPlugins(): Promise; + + /** + * Immediately close the document and data engine, to be reloaded from scratch, and cause all + * browser clients to reopen it. + */ + reloadDoc(): Promise; + + /** + * Prepare a fork of the document, and return the id(s) of the fork. + * TODO: remove string option here, it is present to ease transition. + */ + fork(): Promise; +} diff --git a/app/common/ApiError.ts b/app/common/ApiError.ts new file mode 100644 index 00000000..0dbfd581 --- /dev/null +++ b/app/common/ApiError.ts @@ -0,0 +1,40 @@ +/** + * A tip for fixing an error. + */ +export interface ApiTip { + action: 'add-members' | 'upgrade' |'ask-for-help'; + message: string; +} + +/** + * Documentation of a limit relevant to an API error. + */ +export interface ApiLimit { + quantity: 'collaborators' | 'docs' | 'workspaces'; // what are we counting + subquantity?: string; // a nuance to what we are counting + maximum: number; // maximum allowed + value: number; // current value of quantity for user + projectedValue: number; // value of quantity expected if request had been allowed +} + +/** + * Structured details about an API error. + */ +export interface ApiErrorDetails { + limit?: ApiLimit; + + // If set, this is the more user-friendly message to show to the user than error.message. + userError?: string; + + // If set, contains suggestions for fixing a problem. + tips?: ApiTip[]; +} + +/** + * An error with an http status code. + */ +export class ApiError extends Error { + constructor(message: string, public status: number, public details?: ApiErrorDetails) { + super(message); + } +} diff --git a/app/common/AsyncCreate.ts b/app/common/AsyncCreate.ts new file mode 100644 index 00000000..b0fc2b15 --- /dev/null +++ b/app/common/AsyncCreate.ts @@ -0,0 +1,147 @@ +/** + * Implements a pattern for creating objects requiring asynchronous construction. The given + * asynchronous createFunc() is called on the .get() call, and the result is cached on success. + * On failure, the result is cleared, so that subsequent calls attempt the creation again. + * + * Usage: + * this._obj = AsyncCreate(asyncCreateFunc); + * obj = await this._obj.get(); // calls asyncCreateFunc + * obj = await this._obj.get(); // uses cached object if asyncCreateFunc succeeded, else calls it again. + * + * Note that multiple calls while createFunc() is running will return the same promise, and will + * succeed or fail together. + */ +export class AsyncCreate { + private _value?: Promise = undefined; + + constructor(private _createFunc: () => Promise) {} + + /** + * Returns createFunc() result, returning the cached promise if createFunc() succeeded, or if + * another call to it is currently pending. + */ + public get(): Promise { + return this._value || (this._value = this._clearOnError(this._createFunc.call(null))); + } + + /** Clears the cached promise, forcing createFunc to be called again on next get(). */ + public clear(): void { + this._value = undefined; + } + + /** Returns a boolean indicating whether the object is created. */ + public isSet(): boolean { + return Boolean(this._value); + } + + /** Returns the value if it's set and successful, or undefined otherwise. */ + public async getIfValid(): Promise { + return this._value ? this._value.catch(() => undefined) : undefined; + } + + // Helper which clears this AsyncCreate if the given promise is rejected. + private _clearOnError(p: Promise): Promise { + p.catch(() => this.clear()); + return p; + } +} + +/** + * Supports a usage similar to AsyncCreate in a Map. Returns map.get(key) if it is set to a + * resolved or pending promise. Otherwise, calls creator(key) to create and return a new promise, + * and sets the key to it. If the new promise is rejected, the key will be removed from the map, + * so that subsequent calls would call creator() again. + * + * As with AsyncCreate, while the promise for a key is pending, multiple calls to that key will + * return the same promise, and will succeed or fail together. + */ +export function mapGetOrSet(map: Map>, key: K, creator: (key: K) => Promise): Promise { + return map.get(key) || mapSetOrClear(map, key, creator(key)); +} + +/** + * Supports a usage similar to AsyncCreate in a Map. Sets the given key in a map to the given + * promise, and removes it later if the promise is rejected. Returns the same promise. + */ +export function mapSetOrClear(map: Map>, key: K, pvalue: Promise): Promise { + pvalue.catch(() => map.delete(key)); + map.set(key, pvalue); + return pvalue; +} + +/** + * A Map implementation that allows for expiration of old values. + */ +export class MapWithTTL extends Map { + private _timeouts = new Map(); + + /** + * Create a map with keys that will be automatically deleted _ttlMs + * milliseconds after they have been last set. Precision of timing + * may vary. + */ + constructor(private _ttlMs: number) { + super(); + } + + /** + * Set a key, with expiration. + */ + public set(key: K, value: V): this { + return this.setWithCustomTTL(key, value, this._ttlMs); + } + + /** + * Set a key, with custom expiration. + */ + public setWithCustomTTL(key: K, value: V, ttlMs: number): this { + const curr = this._timeouts.get(key); + if (curr) { clearTimeout(curr); } + super.set(key, value); + this._timeouts.set(key, setTimeout(this.delete.bind(this, key), ttlMs)); + return this; + } + + /** + * Remove a key. + */ + public delete(key: K): boolean { + const result = super.delete(key); + const timeout = this._timeouts.get(key); + if (timeout) { + clearTimeout(timeout); + this._timeouts.delete(key); + } + return result; + } + + /** + * Forcibly expire everything. + */ + public clear(): void { + for (const timeout of this._timeouts.values()) { + clearTimeout(timeout); + } + this._timeouts.clear(); + super.clear(); + } +} + +/** + * Sometimes it is desirable to cache either fulfilled or rejected + * outcomes. This method wraps a promise so that it never throws. + * The result has an unfreeze method which, when called, is either + * fulfilled or rejected. + */ +export async function freezeError(promise: Promise): Promise> { + try { + const value = await promise; + return { unfreeze: async () => value }; + } catch (error) { + return { unfreeze: async () => { throw error; } }; + } +} + +export interface ErrorOrValue { + unfreeze(): Promise; +} diff --git a/app/common/AsyncFlow.ts b/app/common/AsyncFlow.ts new file mode 100644 index 00000000..8a8e821b --- /dev/null +++ b/app/common/AsyncFlow.ts @@ -0,0 +1,83 @@ +/** + * This module is a helper for asynchronous work. It allows resources acquired asynchronously to + * be conveniently and reliably released. + * + * Usage: + * (1) Implement a function `myFunc(flow: AsyncFlow)`. The `flow` argument provides some helpers: + * + * // Create a disposable, making it owned by the flow. It will be disposed when the flow + * // ends, whether successfully, on error, or by being cancelled. + * const foo = Foo.create(flow, ...); + * + * // As with Disposables in general, schedule a callback to be called when the flow ends. + * flow.onDispose(...); + * + * // Release foo from the flow's ownership, and give its ownership to another object. This way + * // `other` will be responsible for disposing foo, and not flow. + * other.autoDispose(flow.release(foo)) + * + * // Abort the flow (by throwing CancelledError) if cancellation is requested. This should + * // be called after async work, in case the flow shouldn't be continued. + * checkIfCancelled(); + * + * (2) Call `runner = FlowRunner.create(owner, myFunc)`. The flow will start. Once myFunc's + * promise resolves (including on failure), the objects owned by the flow will be disposed. + * + * The runner exposes the promise for when the flow ends as `runner.resultPromise`. + * + * If the runner itself is disposed, the flow will be cancelled, and disposed once it notices + * the cancellation. + * + * To replace one FlowRunner with another, put it in a grainjs Holder. + */ +import {Disposable, IDisposable} from 'grainjs'; + +type DisposeListener = ReturnType; + +export class CancelledError extends Error {} + +export class FlowRunner extends Disposable { + public resultPromise: Promise; + + constructor(func: (flow: AsyncFlow) => Promise) { + super(); + const flow = AsyncFlow.create(null); + async function runFlow() { + try { + return await func(flow); + } finally { + flow.dispose(); + } + } + this.resultPromise = runFlow(); + this.onDispose(flow.cancel, flow); + } +} + +export class AsyncFlow extends Disposable { + private _handles = new Map(); + private _isCancelled = false; + + public autoDispose(obj: T): T { + const lis = this.onDispose(obj.dispose, obj); + this._handles.set(obj, lis); + return obj; + } + + public release(obj: T): T { + const h = this._handles.get(obj); + if (h) { h.dispose(); } + this._handles.delete(obj); + return obj; + } + + public checkIfCancelled() { + if (this._isCancelled) { + throw new CancelledError('cancelled'); + } + } + + public cancel() { + this._isCancelled = true; + } +} diff --git a/app/common/BaseAPI.ts b/app/common/BaseAPI.ts new file mode 100644 index 00000000..18fe4b76 --- /dev/null +++ b/app/common/BaseAPI.ts @@ -0,0 +1,124 @@ +import {ApiError, ApiErrorDetails} from 'app/common/ApiError'; +import axios, {AxiosRequestConfig, AxiosResponse} from 'axios'; +import {tbind} from './tbind'; + +export type ILogger = Pick; + +export interface IOptions { + headers?: Record; + fetch?: typeof fetch; + newFormData?: () => FormData; // constructor for FormData depends on platform. + logger?: ILogger; + extraParameters?: Map; // if set, add query parameters to requests. +} + +/** + * Base setup class for creating a REST API client interface. + */ +export class BaseAPI { + // Count of pending requests. It is relied on by tests. + public static numPendingRequests(): number { return this._numPendingRequests; } + + // Wrap a promise to add to the count of pending requests until the promise is resolved. + public static async countPendingRequest(promise: Promise): Promise { + try { + BaseAPI._numPendingRequests++; + return await promise; + } finally { + BaseAPI._numPendingRequests--; + } + } + + // Define a decorator for methods in BaseAPI or derived classes. + public static countRequest(target: unknown, propertyKey: string, descriptor: PropertyDescriptor) { + const originalMethod = descriptor.value; + descriptor.value = async function(...args: any[]) { + return BaseAPI.countPendingRequest(originalMethod.apply(this, args)); + }; + } + + private static _numPendingRequests: number = 0; + + protected fetch: typeof fetch; + protected newFormData: () => FormData; + private _headers: Record; + private _logger: ILogger; + private _extraParameters?: Map; + + constructor(options: IOptions = {}) { + this.fetch = options.fetch || tbind(window.fetch, window); + this.newFormData = options.newFormData || (() => new FormData()); + this._logger = options.logger || console; + this._headers = { + 'Content-Type': 'application/json', + ...options.headers + }; + this._extraParameters = options.extraParameters; + } + + // Make a modified request, exposed for test convenience. + public async testRequest(url: string, init: RequestInit = {}): Promise { + return this.request(url, init); + } + + // Similar to request, but uses the axios library, and supports progress indicator. + @BaseAPI.countRequest + protected async requestAxios(url: string, config: AxiosRequestConfig): Promise { + // If using with FormData in node, axios needs the headers prepared by FormData. + let headers = config.headers; + if (config.data && typeof config.data.getHeaders === 'function') { + headers = {...config.data.getHeaders(), ...headers}; + } + const resp = await axios.request({ + url, + withCredentials: true, + validateStatus: (status) => true, // This is more like fetch + ...config, + headers, + }); + if (resp.status !== 200) { + throwApiError(url, resp, resp.data); + } + return resp; + } + + @BaseAPI.countRequest + protected async request(input: string, init: RequestInit = {}): Promise { + init = Object.assign({ headers: this._headers, credentials: 'include' }, init); + if (this._extraParameters) { + const url = new URL(input); + for (const [key, val] of this._extraParameters.entries()) { + url.searchParams.set(key, val); + input = url.href; + } + } + const resp = await this.fetch(input, init); + this._logger.log("Fetched", input); + if (resp.status !== 200) { + const body = await resp.json().catch(() => ({})); + throwApiError(input, resp, body); + } + return resp; + } + + /** + * Make a request, and read the response as JSON. This allows counting the request as pending + * until it has been read, which is relied on by tests. + */ + @BaseAPI.countRequest + protected async requestJson(input: string, init: RequestInit = {}): Promise { + return (await this.request(input, init)).json(); + } +} + +function throwApiError(url: string, resp: Response | AxiosResponse, body: any) { + // If the response includes details, include them into the ApiError we construct. Include + // also the error message from the server as details.userError. It's used by the Notifier. + if (!body) { body = {}; } + const details: ApiErrorDetails = body.details && typeof body.details === 'object' ? body.details : {}; + if (body.error) { + details.userError = body.error; + } + throw new ApiError(`Request to ${url} failed with status ${resp.status}: ` + + `${resp.statusText} (${body.error || 'unknown cause'})`, resp.status, details); +} diff --git a/app/common/BasketClientAPI.ts b/app/common/BasketClientAPI.ts new file mode 100644 index 00000000..a1ac2877 --- /dev/null +++ b/app/common/BasketClientAPI.ts @@ -0,0 +1,11 @@ +export interface BasketClientAPI { + /** + * Returns an array of all tableIds in this basket. + */ + getBasketTables(): Promise; + + /** + * Adds, updates or deletes a table's data to/from Grist Basket. + */ + embedTable(tableId: string, action: "add"|"update"|"delete"): Promise; +} diff --git a/app/common/BigInt.ts b/app/common/BigInt.ts new file mode 100644 index 00000000..5d43f765 --- /dev/null +++ b/app/common/BigInt.ts @@ -0,0 +1,72 @@ +/** + * A minimal library to represent arbitrarily large integers. Unlike the many third party + * libraries, which are big, this only implements a representation and conversion to string (such + * as base 10 or base 16), so it's tiny in comparison. + * + * Big integers + * base: number - the base for the digits + * digits: number[] - digits, from least significant to most significant, in [0, base) range. + * sign: number - 1 or -1 + */ +export class BigInt { + constructor( + private _base: number, // Base for the digits + private _digits: number[], // Digits from least to most significant, in [0, base) range. + private _sign: number, // +1 or -1 + ) {} + + public copy() { return new BigInt(this._base, this._digits, this._sign); } + + /** Convert to Number if there is no loss of precision, or string (base 10) otherwise. */ + public toNative(): number|string { + const num = this.toNumber(); + return Number.isSafeInteger(num) ? num : this.toString(10); + } + + /** Convert to Number as best we can. This will lose precision beying 53 bits. */ + public toNumber(): number { + let res = 0; + let baseFactor = 1; + for (const digit of this._digits) { + res += digit * baseFactor; + baseFactor *= this._base; + } + return res * (this._sign < 0 ? -1 : 1); + } + + /** Like Number.toString(). Radix (or base) is an integer between 2 and 36, defaulting to 10. */ + public toString(radix: number = 10): string { + const copy = this.copy(); + const decimals = []; + while (copy._digits.length > 0) { + decimals.push(copy._mod(radix).toString(radix)); + copy._divide(radix); + } + if (decimals.length === 0) { return "0"; } + return (this._sign < 0 ? "-" : "") + decimals.reverse().join(""); + } + + /** Returns the remainder when this number is divided by divisor. */ + private _mod(divisor: number): number { + let res = 0; + let baseFactor = 1; + for (const digit of this._digits) { + res = (res + (digit % divisor) * baseFactor) % divisor; + baseFactor = (baseFactor * this._base) % divisor; + } + return res; + } + + /** Divides this number in-place. */ + private _divide(divisor: number): void { + if (this._digits.length === 0) { return; } + for (let i = this._digits.length - 1; i > 0; i--) { + this._digits[i - 1] += (this._digits[i] % divisor) * this._base; + this._digits[i] = Math.floor(this._digits[i] / divisor); + } + this._digits[0] = Math.floor(this._digits[0] / divisor); + while (this._digits.length > 0 && this._digits[this._digits.length - 1] === 0) { + this._digits.pop(); + } + } +} diff --git a/app/common/BillingAPI.ts b/app/common/BillingAPI.ts new file mode 100644 index 00000000..129be64a --- /dev/null +++ b/app/common/BillingAPI.ts @@ -0,0 +1,212 @@ +import {BaseAPI, IOptions} from 'app/common/BaseAPI'; +import {FullUser} from 'app/common/LoginSessionAPI'; +import {StringUnion} from 'app/common/StringUnion'; +import {addCurrentOrgToPath} from 'app/common/urlUtils'; +import {BillingAccount, ManagerDelta, OrganizationWithoutAccessInfo} from 'app/common/UserAPI'; + +export const BillingSubPage = StringUnion('payment', 'plans'); +export type BillingSubPage = typeof BillingSubPage.type; + +export const BillingPage = StringUnion(...BillingSubPage.values, 'billing'); +export type BillingPage = typeof BillingPage.type; + +export const BillingTask = StringUnion('signUp', 'updatePlan', 'addCard', 'updateCard', 'updateAddress'); +export type BillingTask = typeof BillingTask.type; + +// Note that IBillingPlan includes selected fields from the Stripe plan object along with +// custom metadata fields that are present on plans we store in Stripe. +// For reference: https://stripe.com/docs/api/plans/object +export interface IBillingPlan { + id: string; // the Stripe plan id + nickname: string; + currency: string; // lowercase three-letter ISO currency code + interval: string; // billing frequency - one of day, week, month or year + amount: number; // amount in cents charged at each interval + metadata: { + family?: string; // groups plans for filtering by GRIST_STRIPE_FAMILY env variable + isStandard: boolean; // indicates that the plan should be returned by the API to be offered. + supportAvailable: boolean; + gristProduct: string; // name of grist product that should be used with this plan. + unthrottledApi: boolean; + customSubdomain: boolean; + workspaces: boolean; + maxDocs?: number; // if given, limit of docs that can be created + maxUsersPerDoc?: number; // if given, limit of users each doc can be shared with + }; + trial_period_days: number|null; // Number of days in the trial period, or null if there is none. + product: string; // the Stripe product id. +} + +// Stripe customer address information. Used to maintain the company address. +// For reference: https://stripe.com/docs/api/customers/object#customer_object-address +export interface IBillingAddress { + line1: string; + line2?: string; + city?: string; + state?: string; + postal_code?: string; + country?: string; +} + +export interface IBillingCard { + funding?: 'credit'|'debit'|'prepaid'|'unknown'; + brand?: string; + country?: string; // uppercase two-letter ISO country code + last4?: string; // last 4 digits of the card number + name?: string|null; +} + +export interface IBillingSubscription { + // All standard plan options. + plans: IBillingPlan[]; + // Index in the plans array of the plan currently in effect. + planIndex: number; + // Index in the plans array of the plan to be in effect after the current period end. + // Equal to the planIndex when the plan has not been downgraded or cancelled. + upcomingPlanIndex: number; + // Timestamp in milliseconds indicating when the current plan period ends. + // Null if the account is not signed up with Stripe. + periodEnd: number|null; + // Whether the subscription is in the trial period. + isInTrial: boolean; + // Value in cents remaining for the current subscription. This indicates the amount that + // will be discounted from a subscription upgrade. + valueRemaining: number; + // The payment card, or null if none is attached. + card: IBillingCard|null; + // The company address. + address: IBillingAddress|null; + // The effective tax rate of the customer for the given address. + taxRate: number; + // The current number of users with whom the paid org is shared. + userCount: number; + // The next total in cents that Stripe is going to charge (includes tax and discount). + nextTotal: number; + // Name of the discount if any. + discountName: string|null; + // Last plan we had a subscription for, if any. + lastPlanId: string|null; + // Whether there is a valid plan in effect + isValidPlan: boolean; +} + +export interface IBillingOrgSettings { + name: string; + domain: string; +} + +// Full description of billing account, including nested list of orgs and managers. +export interface FullBillingAccount extends BillingAccount { + orgs: OrganizationWithoutAccessInfo[]; + managers: FullUser[]; +} + +export interface BillingAPI { + isDomainAvailable(domain: string): Promise; + getTaxRate(address: IBillingAddress): Promise; + getPlans(): Promise; + getSubscription(): Promise; + getBillingAccount(): Promise; + // The signUp function takes the tokenId generated when card data is submitted to Stripe. + // See: https://stripe.com/docs/stripe-js/reference#stripe-create-token + signUp(planId: string, tokenId: string, address: IBillingAddress, + settings: IBillingOrgSettings): Promise; + setCard(tokenId: string): Promise; + removeCard(): Promise; + setSubscription(planId: string, tokenId?: string): Promise; + updateAddress(address?: IBillingAddress, settings?: IBillingOrgSettings): Promise; + updateBillingManagers(delta: ManagerDelta): Promise; +} + +export class BillingAPIImpl extends BaseAPI implements BillingAPI { + constructor(private _homeUrl: string, options: IOptions = {}) { + super(options); + } + + public async isDomainAvailable(domain: string): Promise { + const resp = await this.request(`${this._url}/api/billing/domain`, { + method: 'POST', + body: JSON.stringify({ domain }) + }); + return resp.json(); + } + + public async getTaxRate(address: IBillingAddress): Promise { + const resp = await this.request(`${this._url}/api/billing/tax`, { + method: 'POST', + body: JSON.stringify({ address }) + }); + return resp.json(); + } + + public async getPlans(): Promise { + const resp = await this.request(`${this._url}/api/billing/plans`, {method: 'GET'}); + return resp.json(); + } + + // Returns an IBillingSubscription + public async getSubscription(): Promise { + const resp = await this.request(`${this._url}/api/billing/subscription`, {method: 'GET'}); + return resp.json(); + } + + public async getBillingAccount(): Promise { + const resp = await this.request(`${this._url}/api/billing`, {method: 'GET'}); + return resp.json(); + } + + // Returns the new Stripe customerId. + public async signUp( + planId: string, + tokenId: string, + address: IBillingAddress, + settings: IBillingOrgSettings + ): Promise { + const resp = await this.request(`${this._url}/api/billing/signup`, { + method: 'POST', + body: JSON.stringify({ tokenId, planId, address, settings }) + }); + const parsed = await resp.json(); + return parsed.data; + } + + public async setSubscription(planId: string, tokenId?: string): Promise { + await this.request(`${this._url}/api/billing/subscription`, { + method: 'POST', + body: JSON.stringify({ tokenId, planId }) + }); + } + + public async removeSubscription(): Promise { + await this.request(`${this._url}/api/billing/subscription`, {method: 'DELETE'}); + } + + public async setCard(tokenId: string): Promise { + await this.request(`${this._url}/api/billing/card`, { + method: 'POST', + body: JSON.stringify({ tokenId }) + }); + } + + public async removeCard(): Promise { + await this.request(`${this._url}/api/billing/card`, {method: 'DELETE'}); + } + + public async updateAddress(address?: IBillingAddress, settings?: IBillingOrgSettings): Promise { + await this.request(`${this._url}/api/billing/address`, { + method: 'POST', + body: JSON.stringify({ address, settings }) + }); + } + + public async updateBillingManagers(delta: ManagerDelta): Promise { + await this.request(`${this._url}/api/billing/managers`, { + method: 'PATCH', + body: JSON.stringify({delta}) + }); + } + + private get _url(): string { + return addCurrentOrgToPath(this._homeUrl); + } +} diff --git a/app/common/BinaryIndexedTree.js b/app/common/BinaryIndexedTree.js new file mode 100644 index 00000000..b0291a86 --- /dev/null +++ b/app/common/BinaryIndexedTree.js @@ -0,0 +1,258 @@ +/** + * Implements a binary indexed tree, aka Fenwick tree. See + * http://en.wikipedia.org/wiki/Fenwick_tree + */ +function BinaryIndexedTree(optSize) { + this.tree = []; + if (optSize > 0) { + this.tree.length = optSize; + for (var i = 0; i < optSize; i++) { + this.tree[i] = 0; + } + // The last valid index rounded down to the nearest power of 2. + this.mask = mostSignificantOne(this.tree.length - 1); + } +} + +/** + * Returns a number that contains only the least significant one in `num`. + * @param {Number} num - Positive integer. + * @returns {Number} The least significant one in `num`, e.g. for 10110, returns 00010. + */ +function leastSignificantOne(num) { + return num & (-num); +} +BinaryIndexedTree.leastSignificantOne = leastSignificantOne; + + +/** + * Strips the least significant one from `num`. + * @param {Number} num - Positive integer. + * @returns {Number} `num` with the least significant one removed, e.g. for 10110, returns 10100. + */ +function stripLeastSignificantOne(num) { + return num & (num - 1); +} +BinaryIndexedTree.stripLeastSignificantOne = stripLeastSignificantOne; + + +function mostSignificantOne(num) { + if (num === 0) { + return 0; + } + var msb = 1; + while ((num >>>= 1)) { + msb <<= 1; + } + return msb; +} +BinaryIndexedTree.mostSignificantOne = mostSignificantOne; + +/** + * Converts in-place an array of cumulative values to the original values. + * @param {Array[Number]} values - Array of cumulative values, or partial sums. + * @returns {Array[Number]} - same `values` array, with elements replaced by deltas. + * E.g. [1,3,6,10] is converted to [1,2,3,4]. + */ +function cumulToValues(values) { + for (var i = values.length - 1; i >= 1; i--) { + values[i] -= values[i - 1]; + } + return values; +} +BinaryIndexedTree.cumulToValues = cumulToValues; + + +/** + * Converts in-place an array of values to cumulative values, or partial sums. + * @param {Array[Number]} values - Array of numerical values. + * @returns {Array[Number]} - same `values` array, with elements replaced by partial sums. + * E.g. [1,2,3,4] is converted to [1,3,6,10]. + */ +function valuesToCumul(values) { + for (var i = 1; i < values.length; i++) { + values[i] += values[i - 1]; + } + return values; +} +BinaryIndexedTree.valuesToCumul = valuesToCumul; + + +/** + * @returns {Number} length of the tree. + */ +BinaryIndexedTree.prototype.size = function() { + return this.tree.length; +}; + + +/** + * Converts the BinaryIndexedTree to a cumulative array. + * Takes time linear in the size of the array. + * @returns {Array[Number]} - array with each element a partial sum. + */ +BinaryIndexedTree.prototype.toCumulativeArray = function() { + var cumulValues = [this.tree[0]]; + var len = cumulValues.length = this.tree.length; + for (var i = 1; i < len; i++) { + cumulValues[i] = this.tree[i] + cumulValues[stripLeastSignificantOne(i)]; + } + return cumulValues; +}; + + +/** + * Converts the BinaryIndexedTree to an array of individual values. + * Takes time linear in the size of the array. + * @returns {Array[Number]} - array with each element containing the value that was inserted. + */ +BinaryIndexedTree.prototype.toValueArray = function() { + return cumulToValues(this.toCumulativeArray()); +}; + + +/** + * Creates a tree from an array of cumulative values. + * Takes time linear in the size of the array. + * @param {Array[Number]} - array with each element a partial sum. + */ +BinaryIndexedTree.prototype.fillFromCumulative = function(cumulValues) { + var len = this.tree.length = cumulValues.length; + if (len > 0) { + this.tree[0] = cumulValues[0]; + for (var i = 1; i < len; i++) { + this.tree[i] = cumulValues[i] - cumulValues[stripLeastSignificantOne(i)]; + } + // The last valid index rounded down to the nearest power of 2. + this.mask = mostSignificantOne(this.tree.length - 1); + } else { + this.mask = 0; + } +}; + + +/** + * Creates a tree from an array of invididual values. + * Takes time linear in the size of the array. + * @param {Array[Number]} - array with each element containing the value to insert. + */ +BinaryIndexedTree.prototype.fillFromValues = function(values) { + this.fillFromCumulative(valuesToCumul(values.slice())); +}; + + +/** + * Reads the cumulative value at the given index. Takes time O(log(index)). + * @param {Number} index - index in the array. + * @returns {Number} - cumulative values up to and including `index`. + */ +BinaryIndexedTree.prototype.getCumulativeValue = function(index) { + var sum = this.tree[0]; + while (index > 0) { + sum += this.tree[index]; + index = stripLeastSignificantOne(index); + } + return sum; +}; + +/** + * Reads the cumulative value from start(inclusive) to end(exclusive). Takes time O(log(end)). + * @param {Number} start - start index + * @param {Number} end - end index + * @returns {Number} - cumulative values between start(inclusive) and end(exclusive) + */ +BinaryIndexedTree.prototype.getCumulativeValueRange = function(start, end) { + return this.getSumTo(end) - this.getSumTo(start); +}; + +/** + * Returns the sum of values up to the given index. Takes time O(log(index)). + * @param {Number} index - index in the array. + * @returns {Number} - cumulative values up to but not including `index`. + */ +BinaryIndexedTree.prototype.getSumTo = function(index) { + return (index > 0 ? this.getCumulativeValue(index - 1) : 0); +}; + + +/** + * Returns the total of all values in the tree. Takes time O(log(N)). + * @returns {Number} - sum of all values. + */ +BinaryIndexedTree.prototype.getTotal = function() { + return this.getCumulativeValue(this.tree.length - 1); +}; + + +/** + * Reads a single value at the given index. Takes time O(log(index)). + * @param {Number} index - index in the array. + * @returns {Number} - the value that was inserted at `index`. + */ +BinaryIndexedTree.prototype.getValue = function(index) { + var value = this.tree[index]; + if (index > 0) { + var parent = stripLeastSignificantOne(index); + index--; + while (index !== parent) { + value -= this.tree[index]; + index = stripLeastSignificantOne(index); + } + } + return value; +}; + + +/** + * Updates a value at an index. Takes time O(log(table size)). + * @param {Number} index - index in the array. + * @param {Number} delta - value to add to the previous value at `index`. + */ +BinaryIndexedTree.prototype.addValue = function(index, delta) { + if (index === 0) { + this.tree[0] += delta; + } else { + while (index < this.tree.length) { + this.tree[index] += delta; + index += leastSignificantOne(index); + } + } +}; + + +/** + * Sets a value at an index. Takes time O(log(table size)). + * @param {Number} index - index in the array. + * @param {Number} value - new value to set at `index`. + */ +BinaryIndexedTree.prototype.setValue = function(index, value) { + this.addValue(index, value - this.getValue(index)); +}; + + +/** + * Given a cumulative value, finds the first element whose inclusion reaches the value. + * E.g. for values [1,2,3,4] (cumulative [1,3,6,10]), getIndex(3) = 1, getIndex(3.1) = 2. + * @param {Number} cumulValue - cumulative value to exceed. + * @returns {Number} index - the first index such that getCumulativeValue(index) >= cumulValue. + * If cumulValue is too large, return one more than the highest valid index. + */ +BinaryIndexedTree.prototype.getIndex = function(cumulValue) { + if (this.tree.length === 0 || this.tree[0] >= cumulValue) { + return 0; + } + var index = 0; + var mask = this.mask; + var sum = this.tree[0]; + while (mask !== 0) { + var testIndex = index + mask; + if (testIndex < this.tree.length && sum + this.tree[testIndex] < cumulValue) { + index = testIndex; + sum += this.tree[index]; + } + mask >>>= 1; + } + return index + 1; +}; + +module.exports = BinaryIndexedTree; diff --git a/app/common/BrowserSettings.ts b/app/common/BrowserSettings.ts new file mode 100644 index 00000000..cb56edf0 --- /dev/null +++ b/app/common/BrowserSettings.ts @@ -0,0 +1,7 @@ +/** + * Describes the settings that a browser sends to the server. + */ +export interface BrowserSettings { + // The browser's timezone, must be one of `momet.tz.names()`. + timezone?: string; +} diff --git a/app/common/ColumnGetters.ts b/app/common/ColumnGetters.ts new file mode 100644 index 00000000..3a381558 --- /dev/null +++ b/app/common/ColumnGetters.ts @@ -0,0 +1,26 @@ +/** + * + * An interface for accessing the columns of a table by their + * ID in _grist_Tables_column, which is the ID used in sort specifications. + * Implementations of this interface can be supplied to SortFunc to + * sort the rows of a table according to such a specification. + * + */ +export interface ColumnGetters { + + /** + * + * Takes a _grist_Tables_column ID and returns a function that maps + * rowIds to values for that column. Those values should be display + * values if available, drawn from a corresponding display column. + * + */ + getColGetter(colRef: number): ((rowId: number) => any) | null; + + /** + * + * Returns a getter for the manual sort column if it is available. + * + */ + getManualSortGetter(): ((rowId: number) => any) | null; +} diff --git a/app/common/DisposableWithEvents.ts b/app/common/DisposableWithEvents.ts new file mode 100644 index 00000000..287cb818 --- /dev/null +++ b/app/common/DisposableWithEvents.ts @@ -0,0 +1,28 @@ +/** + * A base class which combines grainjs Disposable with mixed-in backbone Events. It includes the + * backbone Events methods, and when disposed, stops backbone listeners started with listenTo(). + */ +import {Events as BackboneEvents, EventsHash} from 'backbone'; +import {Disposable} from 'grainjs'; + +// In Typescript, mixins are awkward. This follows the recommendation here +// https://www.typescriptlang.org/docs/handbook/mixins.html +export class DisposableWithEvents extends Disposable implements BackboneEvents { + public on: (eventName: string|EventsHash, callback?: (...args: any[]) => void, context?: any) => any; + public off: (eventName?: string, callback?: (...args: any[]) => void, context?: any) => any; + public trigger: (eventName: string, ...args: any[]) => any; + public bind: (eventName: string, callback: (...args: any[]) => void, context?: any) => any; + public unbind: (eventName?: string, callback?: (...args: any[]) => void, context?: any) => any; + + public once: (events: string, callback: (...args: any[]) => void, context?: any) => any; + public listenTo: (object: any, events: string, callback: (...args: any[]) => void) => any; + public listenToOnce: (object: any, events: string, callback: (...args: any[]) => void) => any; + public stopListening: (object?: any, events?: string, callback?: (...args: any[]) => void) => any; + + // DisposableWithEvents knows also how to stop any backbone listeners started with listenTo(). + constructor() { + super(); + this.onDispose(this.stopListening, this); + } +} +Object.assign(DisposableWithEvents.prototype, BackboneEvents); diff --git a/app/common/DocActions.ts b/app/common/DocActions.ts new file mode 100644 index 00000000..1b137f4a --- /dev/null +++ b/app/common/DocActions.ts @@ -0,0 +1,148 @@ +/** + * This mirrors action definitions from sandbox/grist/actions.py + */ + +import map = require('lodash/map'); + +export type AddRecord = ['AddRecord', string, number, ColValues]; +export type BulkAddRecord = ['BulkAddRecord', string, number[], BulkColValues]; +export type RemoveRecord = ['RemoveRecord', string, number]; +export type BulkRemoveRecord = ['BulkRemoveRecord', string, number[]]; +export type UpdateRecord = ['UpdateRecord', string, number, ColValues]; +export type BulkUpdateRecord = ['BulkUpdateRecord', string, number[], BulkColValues]; + +export type ReplaceTableData = ['ReplaceTableData', string, number[], BulkColValues]; + +// This is the format in which data comes when we fetch a table from the sandbox. +export type TableDataAction = ['TableData', string, number[], BulkColValues]; + +export type AddColumn = ['AddColumn', string, string, ColInfo]; +export type RemoveColumn = ['RemoveColumn', string, string]; +export type RenameColumn = ['RenameColumn', string, string, string]; +export type ModifyColumn = ['ModifyColumn', string, string, ColInfo]; + +export type AddTable = ['AddTable', string, ColInfoWithId[]]; +export type RemoveTable = ['RemoveTable', string]; +export type RenameTable = ['RenameTable', string, string]; + +export type DocAction = ( + AddRecord | + BulkAddRecord | + RemoveRecord | + BulkRemoveRecord | + UpdateRecord | + BulkUpdateRecord | + ReplaceTableData | + TableDataAction | + AddColumn | + RemoveColumn | + RenameColumn | + ModifyColumn | + AddTable | + RemoveTable | + RenameTable +); + +// type guards for convenience - see: +// https://www.typescriptlang.org/docs/handbook/advanced-types.html#user-defined-type-guards +export function isAddRecord(act: DocAction): act is AddRecord { return act[0] === 'AddRecord'; } +export function isBulkAddRecord(act: DocAction): act is BulkAddRecord { return act[0] === 'BulkAddRecord'; } +export function isRemoveRecord(act: DocAction): act is RemoveRecord { return act[0] === 'RemoveRecord'; } +export function isBulkRemoveRecord(act: DocAction): act is BulkRemoveRecord { return act[0] === 'BulkRemoveRecord'; } +export function isUpdateRecord(act: DocAction): act is UpdateRecord { return act[0] === 'UpdateRecord'; } +export function isBulkUpdateRecord(act: DocAction): act is BulkUpdateRecord { return act[0] === 'BulkUpdateRecord'; } + +export function isReplaceTableData(act: DocAction): act is ReplaceTableData { return act[0] === 'ReplaceTableData'; } + +export function isAddColumn(act: DocAction): act is AddColumn { return act[0] === 'AddColumn'; } +export function isRemoveColumn(act: DocAction): act is RemoveColumn { return act[0] === 'RemoveColumn'; } +export function isRenameColumn(act: DocAction): act is RenameColumn { return act[0] === 'RenameColumn'; } +export function isModifyColumn(act: DocAction): act is ModifyColumn { return act[0] === 'ModifyColumn'; } + +export function isAddTable(act: DocAction): act is AddTable { return act[0] === 'AddTable'; } +export function isRemoveTable(act: DocAction): act is RemoveTable { return act[0] === 'RemoveTable'; } +export function isRenameTable(act: DocAction): act is RenameTable { return act[0] === 'RenameTable'; } + + +const SCHEMA_ACTIONS = new Set(['AddTable', 'RemoveTable', 'RenameTable', 'AddColumn', + 'RemoveColumn', 'RenameColumn', 'ModifyColumn']); + +/** + * Determines whether a given action is a schema action or not. + */ +export function isSchemaAction(action: DocAction): boolean { + return SCHEMA_ACTIONS.has(action[0]); +} + +/** + * Returns the tableId from the action. + */ +export function getTableId(action: DocAction): string { + return action[1]; // It happens to always be in the same position in the action tuple. +} + +// Helper types used in the definitions above. + +export type CellValue = number|string|boolean|null|[string, any?]; +export interface ColValues { [colId: string]: CellValue; } +export interface BulkColValues { [colId: string]: CellValue[]; } +export interface ColInfoMap { [colId: string]: ColInfo; } + +export interface ColInfo { + type: string; + isFormula: boolean; + formula: string; +} + +export interface ColInfoWithId extends ColInfo { + id: string; +} + +export interface RowRecord { + id: number; + [colId: string]: CellValue; +} + +// Multiple records in column-oriented format, i.e. same as BulkColValues but with a mandatory +// 'id' column. This is preferred over TableDataAction in external APIs. +export interface TableColValues { + id: number[]; + [colId: string]: CellValue[]; +} + +// Both UserActions and DocActions are represented as [ActionName, ...actionArgs]. +// TODO I think it's better to represent DocAction as a Buffer containing the marshalled action. + +export type UserAction = Array; + +/** + * Gives a description for an action which involves setting values to a selection. + * @param {Array} action - The (Bulk)AddRecord/(Bulk)UpdateRecord action to describe. + * @param {Boolean} optExcludeVals - Indicates whether the values should be excluded from + * the description. + */ +export function getSelectionDesc(action: UserAction, optExcludeVals: boolean): string { + const table = action[1]; + const rows = action[2]; + const colValues: number[] = action[3] as any; // TODO: better typing - but code may evaporate + const columns = map(colValues, (values, col) => optExcludeVals ? col : `${col}: ${values}`); + const s = typeof rows === 'object' ? 's' : ''; + return `table ${table}, row${s} ${rows}; ${columns.join(", ")}`; +} + +// Convert from TableColValues (used by DocStorage and external APIs) to TableDataAction (used +// mainly by the sandbox). +export function toTableDataAction(tableId: string, colValues: TableColValues): TableDataAction { + const colData = {...colValues}; // Make a copy to avoid changing passed-in arguments. + const rowIds: number[] = colData.id; + delete colData.id; + return ['TableData', tableId, rowIds, colData]; +} + +// Convert from TableDataAction (used mainly by the sandbox) to TableColValues (used by DocStorage +// and external APIs). +export function fromTableDataAction(tableData: TableDataAction): TableColValues { + const rowIds: number[] = tableData[2]; + const colValues: BulkColValues = tableData[3]; + return {id: rowIds, ...colValues}; +} diff --git a/app/common/DocData.ts b/app/common/DocData.ts new file mode 100644 index 00000000..6086e7bf --- /dev/null +++ b/app/common/DocData.ts @@ -0,0 +1,121 @@ +/** + * DocData maintains all underlying data for a Grist document, knows how to load it, + * subscribes to actions which change it, and forwards those actions to individual tables. + * It also provides the interface to apply actions to data. + */ +import {schema} from 'app/common/schema'; +import fromPairs = require('lodash/fromPairs'); +import groupBy = require('lodash/groupBy'); +import {ActionDispatcher} from './ActionDispatcher'; +import {BulkColValues, ColInfo, ColInfoWithId, ColValues, DocAction, + RowRecord, TableDataAction} from './DocActions'; +import {ColTypeMap, TableData} from './TableData'; + +type FetchTableFunc = (tableId: string) => Promise; + +export class DocData extends ActionDispatcher { + private _tables: Map = new Map(); + + constructor(private _fetchTableFunc: FetchTableFunc, metaTableData: {[tableId: string]: TableDataAction}) { + super(); + // Create all meta tables, and populate data we already have. + for (const tableId in schema) { + if (schema.hasOwnProperty(tableId)) { + const colTypes: ColTypeMap = (schema as any)[tableId]; + this._tables.set(tableId, this.createTableData(tableId, metaTableData[tableId], colTypes)); + } + } + + // Build a map from tableRef to [columnRecords] + const colsByTable = groupBy(this._tables.get('_grist_Tables_column')!.getRecords(), 'parentId'); + for (const t of this._tables.get('_grist_Tables')!.getRecords()) { + const tableId = t.tableId as string; + const colRecords: RowRecord[] = colsByTable[t.id] || []; + const colTypes = fromPairs(colRecords.map(c => [c.colId, c.type])); + this._tables.set(tableId, this.createTableData(tableId, null, colTypes)); + } + } + + /** + * Creates a new TableData object. A derived class may override to return an object derived from TableData. + */ + public createTableData(tableId: string, tableData: TableDataAction|null, colTypes: ColTypeMap): TableData { + return new TableData(tableId, tableData, colTypes); + } + + /** + * Returns the TableData object for the requested table. + */ + public getTable(tableId: string): TableData|undefined { + return this._tables.get(tableId); + } + + /** + * Returns an unsorted list of all tableIds in this doc, including both metadata and user tables. + */ + public getTables(): ReadonlyMap { + return this._tables; + } + + /** + * Fetches the data for tableId if needed, and returns a promise that is fulfilled when the data + * is loaded. + */ + public fetchTable(tableId: string, force?: boolean): Promise { + const table = this._tables.get(tableId); + if (!table) { throw new Error(`DocData.fetchTable: unknown table ${tableId}`); } + return (!table.isLoaded || force) ? table.fetchData(this._fetchTableFunc) : Promise.resolve(); + } + + /** + * Handles an action received from the server, by forwarding it to the appropriate TableData + * object. + */ + public receiveAction(action: DocAction): void { + // Look up TableData before processing the action in case we rename or remove it. + const tableId: string = action[1]; + const table = this._tables.get(tableId); + + this.dispatchAction(action); + + // Forward all actions to per-table TableData objects. + if (table) { + table.receiveAction(action); + } + } + + // ---- The following methods implement ActionDispatcher interface ---- + + protected onAddTable(action: DocAction, tableId: string, columns: ColInfoWithId[]): void { + const colTypes = fromPairs(columns.map(c => [c.id, c.type])); + this._tables.set(tableId, this.createTableData(tableId, null, colTypes)); + } + + protected onRemoveTable(action: DocAction, tableId: string): void { + this._tables.delete(tableId); + } + + protected onRenameTable(action: DocAction, oldTableId: string, newTableId: string): void { + const table = this._tables.get(oldTableId); + if (table) { + this._tables.set(newTableId, table); + this._tables.delete(oldTableId); + } + } + + // tslint:disable:no-empty + protected onAddRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {} + protected onUpdateRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {} + protected onRemoveRecord(action: DocAction, tableId: string, rowId: number): void {} + + protected onBulkAddRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {} + protected onBulkUpdateRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {} + protected onBulkRemoveRecord(action: DocAction, tableId: string, rowIds: number[]) {} + + protected onReplaceTableData(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {} + + protected onAddColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void {} + protected onRemoveColumn(action: DocAction, tableId: string, colId: string): void {} + protected onRenameColumn(action: DocAction, tableId: string, oldColId: string, newColId: string): void {} + protected onModifyColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void {} +} diff --git a/app/common/DocListAPI.ts b/app/common/DocListAPI.ts new file mode 100644 index 00000000..3fb0638d --- /dev/null +++ b/app/common/DocListAPI.ts @@ -0,0 +1,83 @@ +import {ActionGroup} from 'app/common/ActionGroup'; +import {TableDataAction} from 'app/common/DocActions'; +import {LocalPlugin} from 'app/common/plugin'; +import {StringUnion} from 'app/common/StringUnion'; + +// Possible flavors of items in a list of documents. +export type DocEntryTag = ''|'sample'|'invite'|'shared'; + +export const OpenDocMode = StringUnion( + 'default', // open doc with user's maximal access level + 'fork', // open doc limited to view access (if user has at least that level of access) + 'view' // as for 'view', but suggest a fork on any attempt to edit +); +export type OpenDocMode = typeof OpenDocMode.type; + +/** + * Represents an entry in the DocList. + */ +export interface DocEntry { + docId?: string; // Set for shared docs and invites + name: string; + mtime?: Date; + size?: number; + tag: DocEntryTag; + senderName?: string; + senderEmail?: string; +} + +export interface DocCreationInfo { + id: string; + title: string; +} + +/** + * This documents the members of the structure returned when a local + * grist document is opened. + */ +export interface OpenLocalDocResult { + docFD: number; + clientId: string; // the docFD is meaningful only in the context of this session + doc: {[tableId: string]: TableDataAction}; + log: ActionGroup[]; + plugins: LocalPlugin[]; +} + +export interface DocListAPI { + /** + * Returns a all known Grist documents and document invites to show in the doc list. + */ + getDocList(): Promise<{docs: DocEntry[], docInvites: DocEntry[]}>; + + /** + * Creates a new document, fetches it, and adds a table to it. Returns its name. + */ + createNewDoc(): Promise; + + /** + * Makes a copy of the given sample doc. Returns the name of the new document. + */ + importSampleDoc(sampleDocName: string): Promise; + + /** + * Processes an upload, containing possibly multiple files, to create a single new document, and + * returns the new document's name. + */ + importDoc(uploadId: number): Promise; + + /** + * Deletes a Grist document. Returns the name of the deleted document. If `deletePermanently` is + * true, the doc is deleted permanently rather than just moved to the trash. + */ + deleteDoc(docName: string, deletePermanently: boolean): Promise; + + /** + * Renames a document. + */ + renameDoc(oldName: string, newName: string): Promise; + + /** + * Opens a document, loads it, subscribes to its userAction events, and returns its metadata. + */ + openDoc(userDocName: string, openMode?: OpenDocMode): Promise; +} diff --git a/app/common/EncActionBundle.ts b/app/common/EncActionBundle.ts new file mode 100644 index 00000000..d8005fe8 --- /dev/null +++ b/app/common/EncActionBundle.ts @@ -0,0 +1,73 @@ +/** + * Types for encrypted ActionBundles that get sent between instances and hub. + */ + +import {ActionInfo, Envelope} from 'app/common/ActionBundle'; +import {DocAction} from 'app/common/DocActions'; + +// Type representing a point in time as milliseconds since Epoch. +export type Timestamp = number; + +// Type representing binary data encoded as a base64 string. +export type Base64String = string; + +// Metadata about a symmetric encryption key. +export interface KeyInfo { + firstActionNum: number; // ActionNum of first action for which this key was used. + firstUsedTime: Timestamp; // Timestamp of first action for which this key was used. +} + +// Encrypted symmetric key with metadata, sent from hub to instance with each envelope. +export interface EncKeyInfo extends KeyInfo { + encryptedKey: Base64String; // Symmetric key encrypted with the recipient's public key. +} + +// Bundle of encryptions of the symmetric key. Note that the hub will store EncKeyBundles for +// lookup, indexed by the combination {recipients: string[], firstActionNum: number}. +export interface EncKeyBundle extends KeyInfo { + encryptedKeys: { + // Map of instanceId to the symmetric key encrypted with that instance's public key. + // A single symmetric key is used for all, and only present here in encrypted form. + [instanceId: string]: Base64String; + }; +} + +// This allows reorganizing ActionBundle by envelope while preserving order information for +// actions. E.g. if ActionBundle contains {stored: [(0,A), (1,B), (2,C), (0,D)], then we'll have: +// - in envelopes 0: {stored: [[0, A], [3, D]]} +// - in envelopes 1: {stored: [[1, B]]} +// - in envelopes 2: {stored: [[2, C]]} +// Then recipients of multiple envelopes can sort actions by index to get their correct order. +export interface DecryptedEnvelopeContent { + info?: ActionInfo; + // number is the index into the bundle-wide array of 'stored' or 'calc' DocActions. + stored: Array<[number, DocAction]>; + calc: Array<[number, DocAction]>; +} + +export type DecryptedEnvelope = Envelope & DecryptedEnvelopeContent; + +// Sent from instance to hub. +export interface EncEnvelopeToHub extends Envelope { + encKeyReused?: number; // If reusing a key, firstActionNum of the key being reused. + encKeyBundle?: EncKeyBundle; // If created a new key, its encryption for all recipients. + content: Base64String; // Marshalled and encrypted DecryptedEnvelopeContent as a base64 string. +} + +// Sent from hub to instance. +export interface EncEnvelopeFromHub extends Envelope { + encKeyInfo: EncKeyInfo; + content: Base64String; // Marshalled and encrypted DecryptedEnvelopeContent as a base64 string. +} + +// EncActionBundle is an encrypted version of ActionBundle. It comes in two varieties, one for +// sending ActionBundle to the hub, and one for receiving from the hub. +export interface EncActionBundle { + actionNum: number; + actionHash: string|null; + parentActionHash: string|null; + envelopes: EncEnvelope[]; +} + +export type EncActionBundleToHub = EncActionBundle; +export type EncActionBundleFromHub = EncActionBundle; diff --git a/app/common/ErrorWithCode.ts b/app/common/ErrorWithCode.ts new file mode 100644 index 00000000..0fecba47 --- /dev/null +++ b/app/common/ErrorWithCode.ts @@ -0,0 +1,23 @@ +import {OpenDocMode} from 'app/common/DocListAPI'; + +interface ErrorDetails { + status?: number; + accessMode?: OpenDocMode; +} + +/** + * + * An error with a human-readable message and a machine-readable code. + * Makes it easier to change the human-readable message without breaking + * error handlers. + * + */ +export class ErrorWithCode extends Error { + public accessMode?: OpenDocMode; + public status?: number; + constructor(public code: string, message: string, details: ErrorDetails = {}) { + super(message); + this.status = details.status; + this.accessMode = details.accessMode; + } +} diff --git a/app/common/Features.ts b/app/common/Features.ts new file mode 100644 index 00000000..a508ea81 --- /dev/null +++ b/app/common/Features.ts @@ -0,0 +1,45 @@ +// A product is essentially a list of flags and limits that we may enforce/support. +export interface Features { + vanityDomain?: boolean; // are user-selected domains allowed (unenforced) (default: true) + + workspaces?: boolean; // are workspaces shown in web interface (default: true) + // (this was intended as something we can turn off to shut down + // web access to content while leaving access to billing) + + /** + * Some optional limits. Since orgs can change plans, limits will typically be checked + * at the point of creation. E.g. adding someone new to a document, or creating a + * new document. If, after an operation, the limit would be exceeded, that operation + * is denied. That means it is possible to exceed limits if the limits were not in + * place when shares/docs were originally being added. The action that would need + * to be taken when infringement is pre-existing is not so obvious. + */ + + maxSharesPerDoc?: number; // Maximum number of users that can be granted access to a + // particular doc. Doesn't count users granted access at + // workspace or organization level. Doesn't count billable + // users if applicable (default: unlimited) + + maxSharesPerDocPerRole?: {[role: string]: number}; // As maxSharesPerDoc, but + // for specific roles. Roles are named as in app/common/roles. + // Applied independently to maxSharesPerDoc. + // (default: unlimited) + maxSharesPerWorkspace?: number; // Maximum number of users that can be granted access to + // a particular workspace. Doesn't count users granted access + // at organizational level, or billable users (default: unlimited) + + maxDocsPerOrg?: number; // Maximum number of documents allowed per org. + // (default: unlimited) + maxWorkspacesPerOrg?: number; // Maximum number of workspaces allowed per org. + // (default: unlimited) + + readOnlyDocs?: boolean; // if set, docs can only be read, not written. +} + +// Check whether it is possible to add members at the org level. There's no flag +// for this right now, it isn't enforced at the API level, it is just a bluff. +// For now, when maxWorkspacesPerOrg is 1, we should assume members can't be added +// to org (even though this is not enforced). +export function canAddOrgMembers(features: Features): boolean { + return features.maxWorkspacesPerOrg !== 1; +} diff --git a/app/common/Formula.ts b/app/common/Formula.ts new file mode 100644 index 00000000..5f2e547d --- /dev/null +++ b/app/common/Formula.ts @@ -0,0 +1,72 @@ +/** + * + * This represents a formula supported under SQL for on-demand tables. This is currently + * a very small subset of the formulas supported by the data engine for regular tables. + * + * The following kinds of formula are supported: + * $refColId.colId [where colId is not itself a formula] + * $colId [where colId is not itself a formula] + * NNN [a non-negative integer] + * TODO: support a broader range of formula, by adding a parser or reusing Python parser. + * An argument for reusing Python parser: wwe already do substantial parsing of the formula code. + * E.g. Python does such amazing things as handle updating the formula when any of the columns + * referred to in Foo.lookup(bar=$baz).blah get updated. + * + */ +export type Formula = LiteralNumberFormula | ColumnFormula | ForeignColumnFormula | FormulaError; + +// A simple copy of another column. E.g. "$Person" +export interface ColumnFormula { + kind: 'column'; + colId: string; +} + +// A copy of a column in another table (via a reference column). E.g. "$Person.FirstName" +export interface ForeignColumnFormula { + kind: 'foreignColumn'; + colId: string; + refColId: string; +} + +export interface LiteralNumberFormula { + kind: 'literalNumber'; + value: number; +} + +// A formula that couldn't be parsed. +export interface FormulaError { + kind: 'error'; + msg: string; +} + +/** + * Convert a string to a parsed formula. Regexes are adequate for the very few + * supported formulas, but once the syntax is at all flexible a proper parser will + * be needed. In principle, it might make sense to support python syntax, for + * compatibility with the data engine, but compatibility in corner cases will be + * fiddly given underlying differences between sqlite and python. + */ +export function parseFormula(txt: string): Formula { + // Formula of form: $x.y + let m = txt.match(/^\$([a-z]\w*)\.([a-z]\w*)$/i); + if (m) { + return {kind: 'foreignColumn', refColId: m[1], colId: m[2]}; + } + + // Formula of form: $x + m = txt.match(/^\$([a-z][a-z_0-9]*)$/i); + if (m) { + return {kind: 'column', colId: m[1]}; + } + + // Formula of form: NNN + m = txt.match(/^[0-9]+$/); + if (m) { + const value = parseInt(txt, 10); + if (isNaN(value)) { return {kind: 'error', msg: 'Cannot parse integer'}; } + return {kind: 'literalNumber', value}; + } + + // Everything else is an error. + return {kind: 'error', msg: 'Formula not supported'}; +} diff --git a/app/common/GristServerAPI.ts b/app/common/GristServerAPI.ts new file mode 100644 index 00000000..42ab05ac --- /dev/null +++ b/app/common/GristServerAPI.ts @@ -0,0 +1,84 @@ +import {BasketClientAPI} from 'app/common/BasketClientAPI'; +import {DocListAPI} from 'app/common/DocListAPI'; +import {LoginSessionAPI} from 'app/common/LoginSessionAPI'; +import {EmailResult, Invite} from 'app/common/sharing'; +import {UserConfig} from 'app/common/UserConfig'; + +export interface GristServerAPI extends + DocListAPI, + LoginSessionAPI, + BasketClientAPI, + ServerMetricsAPI, + UserAPI, + SharingAPI, + MiscAPI {} + + +interface ServerMetricsAPI { + /** + * Registers the list of client metric names. The calls to pushClientMetrics() send metric + * values as an array parallel to this list of names. + */ + registerClientMetrics(clientMetricsList: string[]): Promise; + + /** + * Sends bucketed client metric data to the server. The .values arrays contain one value for + * each of the registered metric names, as a parallel array. + */ + pushClientMetrics(clientBuckets: Array<{startTime: number, values: number[]}>): Promise; +} + +interface UserAPI { + /** + * Gets the Grist configuration from the server. + */ + getConfig(): Promise; + + /** + * Updates the user configuration and saves it to the server. + * @param {Object} config - Configuration object to save. + * @returns {Promise:Object} Configuration object as persisted by the server. You can use it to + * validate the configuration. + */ + updateConfig(config: UserConfig): Promise; + + /** + * Re-load plugins. + */ + reloadPlugins(): Promise; +} + +interface SharingAPI { + /** + * Looks up a user account by email, return an object with basic user profile information. + */ + lookupEmail(email: string): Promise; + + /** + * Fetches and saves invites from the hub which are not already present locally. + * Returns the new invites from the hub only. + */ + getNewInvites(): Promise; + + /** + * Fetches local invites and marks them all as read. + */ + getLocalInvites(): Promise; + + /** + * Marks the stored local invite belonging to the calling instance as ignored. + * Called when the user declines an invite. + */ + ignoreLocalInvite(docId: string): Promise; + + /** + * Downloads a shared doc by creating a new doc and applying the snapshot actions associated + * with the given docId on the sharing hub. Must be called from a logged in account and instance + * invited to download the doc. Returns the actual non-conflicting docName used. + */ + downloadSharedDoc(docId: string, docName: string): Promise; +} + +interface MiscAPI { + showItemInFolder(docName: string): Promise; +} diff --git a/app/common/InactivityTimer.ts b/app/common/InactivityTimer.ts new file mode 100644 index 00000000..c2dea90a --- /dev/null +++ b/app/common/InactivityTimer.ts @@ -0,0 +1,119 @@ +/** + * InactivityTimer allows to set a function that executes after a certain time of + * inactivity. Activities can be of two kinds: synchronous or asynchronous. Asynchronous activities, + * are handle with the `disableUntiFinish` method that takes in a Promise and makes sure that the + * timer does not start before the promise resolves. Synchroneous activities are monitored with the + * `ping` method which resets the timer if called during inactivity. + * + * Timer won't start before any activity happens, but you may simply call ping() after construction + * to start it. After cb is called, timer is disabled but enabled again if there is more activity. + * + * Example usage: InactivityTimer is used internally for implementing the plugins' component + * deactivation after a certain time of inactivity. + * + */ + +// important to explicitly import this, or webpack --watch gets confused. +import {clearTimeout, setTimeout} from "timers"; + +export class InactivityTimer { + + private _timeout?: NodeJS.Timer | null; + private _counter: number = 0; + private _enabled: boolean = true; + + constructor(private _callback: () => void, private _delay: number) {} + + // Returns the delay used by InactivityTimer, in ms. + public getDelay(): number { + return this._delay; + } + + // Sets a different delay to use, in ms. + public setDelay(delayMs: number): void { + this._delay = delayMs; + this.ping(); + } + + /** + * Enable the InactivityTimer and schedule the callback. + */ + public enable(): void { + this._enabled = true; + this.ping(); + } + + /** + * Clears the timeout and prevents the callback from being called until enable() is called. + */ + public disable(): void { + this._enabled = false; + this._clearTimeout(); + } + + /** + * Returns whether the InactivityTimer is enabled. If not, the callback will not be scheduled. + */ + public isEnabled(): boolean { + return this._enabled; + } + + /** + * Whether the callback is currently scheduled, and would trigger if there is no activity and if + * it's not disabled before it triggers. + */ + public isScheduled(): boolean { + return Boolean(this._timeout); + } + + /** + * Resets the timer if called during inactivity. + */ + public ping() { + if (!this._counter && this._enabled) { + this._setTimeout(); + } + } + + /** + * The `disableUntilFinish` method takes in a promise and makes sure the timer won't start before + * it resolves. It returns a promise that resolves to the same object. + */ + public async disableUntilFinish(promise: Promise): Promise { + this._beginActivity(); + try { + return await promise; + } finally { + this._endActivity(); + } + } + + private _beginActivity() { + this._counter++; + this._clearTimeout(); + } + + private _endActivity() { + this._counter = Math.max(this._counter - 1, 0); + this.ping(); + } + + private _clearTimeout() { + if (this._timeout) { + clearTimeout(this._timeout); + this._timeout = null; + } + } + + private _setTimeout() { + this._clearTimeout(); + this._timeout = setTimeout(() => this._onTimeoutTriggered(), this._delay); + } + + private _onTimeoutTriggered() { + this._clearTimeout(); + // _counter is set to 0, even if there's no reason why it should be any thing else. + this._counter = 0; + this._callback(); + } +} diff --git a/app/common/KeyedOps.ts b/app/common/KeyedOps.ts new file mode 100644 index 00000000..f3d1ca6e --- /dev/null +++ b/app/common/KeyedOps.ts @@ -0,0 +1,222 @@ +/** + * A class for scheduling a particular operation on resources + * identified by a key. For operations which should be applied + * some time after an event. + */ +export class KeyedOps { + private _operations = new Map(); // status of operations + private _history = new Map(); // history of operations + // (will accumulate without limit) + private _changed = new Set(); // set when key needs an operation + private _operating = new Set(); // set when operation is in progress for key + + /** + * Provide a function to apply operation, and some optional + * parameters. + * + * - delayBeforeOperationMs: if set, a call to addOperation(key) will have + * a delayed effect. It will schedule (or reschedule) the operation to occur + * after this interval. If the operation is currently in progress, it will + * get rerun after it completes. + * + * - minDelaybetweenOperationsMs: is set, scheduling for operations will have + * additional delays inserted as necessary to keep this minimal delay between + * the start of successive operations. + * + * - retry: if `retry` is set, the operation will be retried + * indefinitely with a rather primitive retry mechanism - + * otherwise no attempt is made to retry failures. + * + * - logError: called when errors occur, with a count of number of failures so + * far. + */ + constructor(private _op: (key: string) => Promise, private _options: { + delayBeforeOperationMs?: number, + minDelayBetweenOperationsMs?: number, + retry?: boolean, + logError?: (key: string, failureCount: number, err: Error) => void + }) { + } + + /** + * Request an operation be done (eventually) on the specified resourse. + */ + public addOperation(key: string) { + this._changed.add(key); + this._schedule(key); + } + + /** + * Check whether any work is scheduled or in progress. + */ + public hasPendingOperations() { + return this._changed.size > 0 || this._operating.size > 0; + } + + /** + * Check whether any work is scheduled or in progress for a specific resource. + */ + public hasPendingOperation(key: string) { + return this._changed.has(key) || this._operating.has(key); + } + + /** + * Take all scheduled operations and re-schedule them for right now. Useful + * when shutting down. Affects retries. Cannot be undone. Returns immediately. + */ + public expediteOperations() { + this._options.delayBeforeOperationMs = 0; + this._options.minDelayBetweenOperationsMs = 0; + for (const op of this._operations.values()) { + if (op.timeout) { + this._schedule(op.key, true); + } + } + } + + /** + * Wait for all operations to complete. This makes most sense to use during + * shutdown - otherwise it might be a very long wait to reach a moment where + * there are no operations. + */ + public async wait(logRepeat?: (count: number) => void) { + let repeats: number = 0; + while (this.hasPendingOperations()) { + if (repeats && logRepeat) { logRepeat(repeats); } + await Promise.all([...this._operating.keys(), ...this._changed.keys()] + .map(key => this.expediteOperationAndWait(key))); + repeats++; + } + } + + /** + * Re-schedules any pending operation on a resource for right now. Returns + * when operations on the resource are complete. Does not affect retries. + */ + public async expediteOperationAndWait(key: string) { + const status = this._getOperationStatus(key); + if (status.promise) { + await status.promise; + return; + } + const callback = new Promise((resolve) => { + status.callbacks.push(resolve); + }); + this._schedule(key, true); + await callback; + } + + /** + * Schedule an operation for a resource. + * If the operation is already in progress, we do nothing. + * If the operation has not yet happened, it is rescheduled. + * If `immediate` is set, the operation is scheduled with no delay. + */ + private _schedule(key: string, immediate: boolean = false) { + const status = this._getOperationStatus(key); + if (status.promise) { return; } + if (status.timeout) { + clearTimeout(status.timeout); + delete status.timeout; + } + let ticks = this._options.delayBeforeOperationMs || 0; + const {lastStart} = this._getOperationHistory(key); + if (lastStart && this._options.minDelayBetweenOperationsMs && !immediate) { + ticks = Math.max(ticks, lastStart + this._options.minDelayBetweenOperationsMs - Date.now()); + } + // Primitive slow-down on retries. + // Will do nothing if neither delayBeforeOperationMs nor minDelayBetweenOperationsMs + // are set. + ticks *= 1 + Math.min(5, status.failures); + status.timeout = setTimeout(() => this._update(key), immediate ? 0 : ticks); + } + + private _getOperationStatus(key: string): OperationStatus { + let status = this._operations.get(key); + if (!status) { + status = { + key, + failures: 0, + callbacks: [] + }; + this._operations.set(key, status); + } + return status; + } + + private _getOperationHistory(key: string): OperationHistory { + let hist = this._history.get(key); + if (!hist) { + hist = {}; + this._history.set(key, hist); + } + return hist; + } + + // Implement the next scheduled operation for a resource. + private _update(key: string) { + const status = this._getOperationStatus(key); + delete status.timeout; + + // We don't have to do anything if there have been no changes. + if (!this._changed.has(key)) { return; } + // We don't have to do anything (yet) if an operation is already in progress. + if (status.promise) { return; } + + // Switch status from changed to operating. + this._changed.delete(key); + this._operating.add(key); + const history = this._getOperationHistory(key); + history.lastStart = Date.now(); + + // Store a promise for the operation. + status.promise = this._op(key).then(() => { + // Successful push! Reset failure count, notify callbacks. + status.failures = 0; + status.callbacks.forEach(callback => callback()); + status.callbacks = []; + }).catch(err => { + // Operation failed. Increment failure count, notify callbacks. + status.failures++; + if (this._options.retry) { + this._changed.add(key); + } + if (this._options.logError) { + this._options.logError(key, status.failures, err); + } + status.callbacks.forEach(callback => callback(err)); + status.callbacks = []; + }).then(() => { + // Clean up and schedule follow-up if necessary. + this._operating.delete(key); + delete status.promise; + if (this._changed.has(key)) { + this._schedule(key); + } else { + // No event information left to track, we can delete our OperationStatus entry. + if (status.failures === 0 && !status.timeout) { + this._operations.delete(key); + } + } + }); + } +} + +/** + * Status of an operation. + */ +interface OperationStatus { + timeout?: NodeJS.Timeout; // a timeout for a scheduled future operation + promise?: Promise; // a promise for an operation that is under way + key: string; // the operation key + failures: number; // consecutive number of times the operation has failed + callbacks: Array<(err?: Error) => void>; // callbacks for notifications when op is done/fails +} + + +/** + * History of an operation. + */ +interface OperationHistory { + lastStart?: number; // last time operation was started, in ms since epoch +} diff --git a/app/common/LoginSessionAPI.ts b/app/common/LoginSessionAPI.ts new file mode 100644 index 00000000..69d682e5 --- /dev/null +++ b/app/common/LoginSessionAPI.ts @@ -0,0 +1,27 @@ +// User profile info for the user. When using Cognito, it is fetched during login. +export interface UserProfile { + email: string; + name: string; + picture?: string|null; // when present, a url to a public image of unspecified dimensions. + anonymous?: boolean; // when present, asserts whether user is anonymous (not authorized). + loginMethod?: 'Google'|'Email + Password'; +} + +// User profile including user id. All information in it should +// have been validated against database. +export interface FullUser extends UserProfile { + id: number; +} + +export interface LoginSessionAPI { + /** + * Logs out by clearing all data in the session store besides the session cookie itself. + * Broadcasts the logged out state to all clients. + */ + logout(): Promise; + + /** + * Replaces the user profile object in the session and broadcasts the new profile to all clients. + */ + updateProfile(profile: UserProfile): Promise; +} diff --git a/app/common/LoginState.ts b/app/common/LoginState.ts new file mode 100644 index 00000000..e842edb2 --- /dev/null +++ b/app/common/LoginState.ts @@ -0,0 +1,31 @@ +import {parseSubdomain} from 'app/common/gristUrls'; + +// This interface is used by the standalone login-connect tool for knowing where to redirect to, +// by Client.ts to construct this info, and by CognitoClient to decide what to do. + +export interface LoginState { + // Locally-running Grist uses localPort, while hosted uses subdomain. Login-connect uses this to + // redirect back to the localhost or to the subdomain. + localPort?: number; + subdomain?: string; + baseDomain?: string; // the domain with the (left-most) subdomain removed, e.g. ".getgrist.com". + // undefined on localhost. + + // Standalone version sets clientId, used later to find the LoginSession. Hosted and dev + // versions rely on the browser cookies instead, specifically on the session cookie. + clientId?: string; + + // Hosted and dev versions set redirectUrl and redirect to it when login or logout completes. + // Standalone version omits redirectUrl, and serves a page which closes the window. + redirectUrl?: string; +} + +/// Allowed localhost addresses. +export const localhostRegex = /^localhost(?::(\d+))?$/i; + +export function getLoginState(reqHost: string): LoginState|null { + const {org, base} = parseSubdomain(reqHost); + const matchPort = localhostRegex.exec(reqHost); + return org ? {subdomain: org, baseDomain: base} : + matchPort ? {localPort: matchPort[1] ? parseInt(matchPort[1], 10) : 80} : null; +} diff --git a/app/common/MemBuffer.js b/app/common/MemBuffer.js new file mode 100644 index 00000000..71367b40 --- /dev/null +++ b/app/common/MemBuffer.js @@ -0,0 +1,294 @@ +const gutil = require('./gutil'); +const {arrayToString, stringToArray} = require('./arrayToString'); + + +/** + * Class for a dynamic memory buffer. You can optionally pass the number of bytes + * to reserve initially. + */ +function MemBuffer(optBytesToReserve) { + this.buffer = new ArrayBuffer(optBytesToReserve || 64); + this.asArray = new Uint8Array(this.buffer); + this.asDataView = new DataView(this.buffer); + this.startPos = 0; + this.endPos = 0; +} + +// These are defined in gutil now because they are used there (and to avoid a circular import), +// but were originally defined in MemBuffer and various code still uses them as MemBuffer members. +MemBuffer.arrayToString = arrayToString; +MemBuffer.stringToArray = stringToArray; + +/** + * Returns the number of bytes in the buffer. + */ +MemBuffer.prototype.size = function() { + return this.endPos - this.startPos; +}; + +/** + * Returns the number of bytes reserved in the buffer for data. This is at least size(). + */ +MemBuffer.prototype.reserved = function() { + return this.buffer.byteLength - this.startPos; +}; + +/** + * Reserves enough space in the buffer to hold a nbytes of data, counting the data already in the + * buffer. + */ +MemBuffer.prototype.reserve = function(nbytes) { + if (this.startPos + nbytes > this.buffer.byteLength) { + var origArray = new Uint8Array(this.buffer, this.startPos, this.size()); + if (nbytes > this.buffer.byteLength) { + // At least double the size of the buffer. + var newBytes = Math.max(nbytes, this.buffer.byteLength * 2); + this.buffer = new ArrayBuffer(newBytes); + this.asArray = new Uint8Array(this.buffer); + this.asDataView = new DataView(this.buffer); + } + // If we did not allocate more space, this line will just move data to the beginning. + this.asArray.set(origArray); + this.endPos = this.size(); + this.startPos = 0; + } +}; + +/** + * Clears the buffer. + */ +MemBuffer.prototype.clear = function() { + this.startPos = this.endPos = 0; + // If the buffer has grown somewhat big, use this chance to free the memory. + if (this.buffer.byteLength >= 256 * 1024) { + this.buffer = new ArrayBuffer(64); + this.asArray = new Uint8Array(this.buffer); + this.asDataView = new DataView(this.buffer); + } +}; + +/** + * Returns a Uint8Array viewing all the data in the buffer. It is the caller's responsibility to + * make a copy if needed to avoid it being affected by subsequent changes to the buffer. + */ +MemBuffer.prototype.asByteArray = function() { + return new Uint8Array(this.buffer, this.startPos, this.size()); +}; + +/** + * Converts all buffer data to string using UTF8 encoding. + * This is mainly for testing. + */ +MemBuffer.prototype.toString = function() { + return arrayToString(this.asByteArray()); +}; + +/* + * (Dmitry 2017/03/20. Some unittests that include timing (e.g. Sandbox.js measuring serializing + * of data using marshal.js) indicated that gutil.arrayCopyForward gets deoptimized. Narrowing it + * down, I found it was because it was used with different argument types (Arrays, Buffers, + * Uint8Arrays). To keep it optimized, we'll use a cloned copy of arrayCopyForward (for copying to + * a Uint8Array) in this module. + */ +let arrayCopyForward = gutil.cloneFunc(gutil.arrayCopyForward); + +/** + * Appends an array of bytes to this MemBuffer. + * @param {Uint8Array|Buffer} bytes: Array of bytes to append. May be a Node Buffer. + */ +MemBuffer.prototype.writeByteArray = function(bytes) { + // Note that the implementation is identical for Uint8Array and a Node Buffer. + this.reserve(this.size() + bytes.length); + arrayCopyForward(this.asArray, this.endPos, bytes, 0, bytes.length); + this.endPos += bytes.length; +}; + +/** + * Encodes the given string in UTF8 and appends to the buffer. + */ +if (typeof TextDecoder !== 'undefined') { + MemBuffer.prototype.writeString = function(string) { + this.writeByteArray(stringToArray(string)); + }; +} else { + // We can write faster without using stringToArray, to avoid allocating new buffers. + // We'll encode data in chunks reusing a single buffer. The buffer is a multiple of chunk size + // to have enough space for multi-byte characters. + var encodeChunkSize = 1024; + var encodeBufferPad = Buffer.alloc(encodeChunkSize * 4); + + MemBuffer.prototype.writeString = function(string) { + // Reserve one byte per character initially (common case), but we'll reserve more below as + // needed. + this.reserve(this.size() + string.length); + for (var i = 0; i < string.length; i += encodeChunkSize) { + var bytesWritten = encodeBufferPad.write(string.slice(i, i + encodeChunkSize)); + this.reserve(this.size() + bytesWritten); + arrayCopyForward(this.asArray, this.endPos, encodeBufferPad, 0, bytesWritten); + this.endPos += bytesWritten; + } + }; +} + + +function makeWriteFunc(typeName, bytes, optLittleEndian) { + var setter = DataView.prototype['set' + typeName]; + return function(value) { + this.reserve(this.size() + bytes); + setter.call(this.asDataView, this.endPos, value, optLittleEndian); + this.endPos += bytes; + }; +} + +/** + * The following methods append a value of the given type to the buffer. + * These are analogous to Node Buffer's write* family of methods. + */ +MemBuffer.prototype.writeInt8 = makeWriteFunc('Int8', 1); +MemBuffer.prototype.writeUint8 = makeWriteFunc('Uint8', 1); +MemBuffer.prototype.writeInt16LE = makeWriteFunc('Int16', 2, true); +MemBuffer.prototype.writeInt16BE = makeWriteFunc('Int16', 2, false); +MemBuffer.prototype.writeUint16LE = makeWriteFunc('Uint16', 2, true); +MemBuffer.prototype.writeUint16BE = makeWriteFunc('Uint16', 2, false); +MemBuffer.prototype.writeInt32LE = makeWriteFunc('Int32', 4, true); +MemBuffer.prototype.writeInt32BE = makeWriteFunc('Int32', 4, false); +MemBuffer.prototype.writeUint32LE = makeWriteFunc('Uint32', 4, true); +MemBuffer.prototype.writeUint32BE = makeWriteFunc('Uint32', 4, false); +MemBuffer.prototype.writeFloat32LE = makeWriteFunc('Float32', 4, true); +MemBuffer.prototype.writeFloat32BE = makeWriteFunc('Float32', 4, false); +MemBuffer.prototype.writeFloat64LE = makeWriteFunc('Float64', 8, true); +MemBuffer.prototype.writeFloat64BE = makeWriteFunc('Float64', 8, false); + +/** + * To consume data from an mbuf, the following pattern is recommended: + * var consumer = mbuf.makeConsumer(); + * try { + * mbuf.readInt8(consumer); + * mbuf.readByteArray(consumer, len); + * ... + * } catch (e) { + * if (e.needMoreData) { + * ... + * } + * } + * mbuf.consume(consumer); + */ +MemBuffer.prototype.makeConsumer = function() { + return new Consumer(this); +}; + +/** + * After some data has been read via a consumer, mbuf.consume(consumer) will clear out the + * consumed data from the buffer. + */ +MemBuffer.prototype.consume = function(consumer) { + this.startPos = consumer.pos; + if (this.size() === 0) { + this.clear(); + consumer.pos = this.startPos; + } +}; + +/** + * Helper class for reading data from the buffer. It keeps track of an offset into the buffer + * without changing anything in the MemBuffer itself. To affect the MemBuffer, + * mbuf.consume(consumer) should be called. + */ +function Consumer(mbuf) { + this.mbuf = mbuf; + this.pos = mbuf.startPos; +} + +/** + * Helper for reading data, used by MemBuffer's read* methods. + */ +Consumer.prototype._consume = function(nbytes) { + var offset = this.pos; + if (this.pos + nbytes > this.mbuf.endPos) { + var err = new RangeError("MemBuffer: read past end"); + err.needMoreData = true; + err.consumedData = this.pos - this.mbuf.startPos; + throw err; + } + this.pos += nbytes; + return offset; +}; + +/** + * Reads length bytes from the buffer using the passed-in consumer, as created by + * mbuf.makeConsumer(). Returns a view on the underlying data. + * @returns {Uint8Array} array of bytes viewing underlying MemBuffer data. + */ +MemBuffer.prototype.readByteArraySlice = function(cons, length) { + return new Uint8Array(this.buffer, cons._consume(length), length); +}; + +/** + * Reads length bytes from the buffer using the passed-in consumer. + * @returns {Uint8Array} array of bytes that's a copy of the underlying data. + */ +MemBuffer.prototype.readByteArray = function(cons, length) { + return new Uint8Array(this.readByteArraySlice(cons, length)); +}; + +/** + * Reads length bytes from the buffer using the passed-in consumer. + * @returns {Buffer} copy of data as a Node Buffer. + */ +MemBuffer.prototype.readBuffer = function(cons, length) { + return Buffer.from(this.readByteArraySlice(cons, length)); +}; + +/** + * Decodes byteLength bytes from the buffer using UTF8 and returns the resulting string. Uses the + * passed-in consumer, as created by mbuf.makeConsumer(). + * @returns {string} + */ +if (typeof TextDecoder !== 'undefined') { + MemBuffer.prototype.readString = function(cons, byteLength) { + return arrayToString(this.readByteArraySlice(cons, byteLength)); + }; +} else { + var decodeBuffer = Buffer.alloc(1024); + MemBuffer.prototype.readString = function(cons, byteLength) { + var offset = cons._consume(byteLength); + if (byteLength <= decodeBuffer.length) { + gutil.arrayCopyForward(decodeBuffer, 0, this.asArray, offset, byteLength); + return decodeBuffer.toString('utf8', 0, byteLength); + } else { + return Buffer.from(new Uint8Array(this.buffer, offset, byteLength)).toString(); + } + }; +} + +function makeReadFunc(typeName, bytes, optLittleEndian) { + var getter = DataView.prototype['get' + typeName]; + return function(cons) { + return getter.call(this.asDataView, cons._consume(bytes), optLittleEndian); + }; +} + +/** + * The following methods read and return a value of the given type from the buffer using the + * passed-in consumer, as created by mbuf.makeConsumer(). E.g. + * var consumer = mbuf.makeConsumer(); + * mbuf.readInt8(consumer); + * mbuf.consume(consumer); + * These are analogous to Node Buffer's read* family of methods. + */ +MemBuffer.prototype.readInt8 = makeReadFunc('Int8', 1); +MemBuffer.prototype.readUint8 = makeReadFunc('Uint8', 1); +MemBuffer.prototype.readInt16LE = makeReadFunc('Int16', 2, true); +MemBuffer.prototype.readUint16LE = makeReadFunc('Uint16', 2, true); +MemBuffer.prototype.readInt16BE = makeReadFunc('Int16', 2, false); +MemBuffer.prototype.readUint16BE = makeReadFunc('Uint16', 2, false); +MemBuffer.prototype.readInt32LE = makeReadFunc('Int32', 4, true); +MemBuffer.prototype.readUint32LE = makeReadFunc('Uint32', 4, true); +MemBuffer.prototype.readInt32BE = makeReadFunc('Int32', 4, false); +MemBuffer.prototype.readUint32BE = makeReadFunc('Uint32', 4, false); +MemBuffer.prototype.readFloat32LE = makeReadFunc('Float32', 4, true); +MemBuffer.prototype.readFloat32BE = makeReadFunc('Float32', 4, false); +MemBuffer.prototype.readFloat64LE = makeReadFunc('Float64', 8, true); +MemBuffer.prototype.readFloat64BE = makeReadFunc('Float64', 8, false); + +module.exports = MemBuffer; diff --git a/app/common/MetricCollector.js b/app/common/MetricCollector.js new file mode 100644 index 00000000..64931dd4 --- /dev/null +++ b/app/common/MetricCollector.js @@ -0,0 +1,101 @@ +const _ = require('underscore'); +const metricConfig = require('./metricConfig'); +const metricTools = require('./metricTools'); +const gutil = require('app/common/gutil'); + +/** + * Base class for metrics collection used by both the server metrics collector, ServerMetrics.js, + * and the client metrics collector, ClientMetrics.js. Should not be instantiated. + * Establishes interval attempts to push metrics to the server on creation. + */ +function MetricsCollector() { + this.startTime = metricTools.getBucketStartTime(Date.now()); + this.readyToExport = []; + // used (as a protected member) by the derived ServerMetrics class. + this._collect = setTimeout(() => this.scheduleBucketPreparation(), metricTools.getDeltaMs(Date.now())); +} + +// Should return a map from metric names (as entered in metricConfig.js) to their metricTools. +MetricsCollector.prototype.getMetrics = function() { + throw new Error("Not implemented"); +}; + +// Should return a promise that is resolved when the metrics have been pushed. +MetricsCollector.prototype.pushMetrics = function() { + throw new Error("Not implemented"); +}; + +// Should return a bucket of metric data, formatted for either the client or server. +MetricsCollector.prototype.createBucket = function(bucketStart) { + throw new Error("Not implemented"); +}; + +// Takes a list of metrics specifications and creates an object mapping metric names to +// a new instance of the metric gathering tool matching that metric's type. +MetricsCollector.prototype.initMetricTools = function(metricsList) { + var metrics = {}; + metricsList.forEach(metricInfo => { + metrics[metricInfo.name] = new metricTools[metricInfo.type](metricInfo.name); + }); + return metrics; +}; + +// Called each push interval. +MetricsCollector.prototype.attemptPush = function() { + this.pushMetrics(this.readyToExport); + this.readyToExport = []; +}; + +// Pushes bucket to the end of the readyToExport queue. Should be called sequentially, since it +// handles deletion of buckets older than the export memory limit. +MetricsCollector.prototype.queueBucket = function(bucket) { + // If readyToExport is at maximum length, delete the oldest element + this.readyToExport.push(bucket); + var length = this.readyToExport.length; + if (length > metricConfig.MAX_PENDING_BUCKETS) { + this.readyToExport.splice(0, length - metricConfig.MAX_PENDING_BUCKETS); + } +}; + +MetricsCollector.prototype.scheduleBucketPreparation = function() { + this.prepareCompletedBuckets(Date.now()); + this._collect = setTimeout(() => this.scheduleBucketPreparation(), metricTools.getDeltaMs(Date.now())); +}; + +/** + * Checks if each bucket since the last update is completed and for each one adds all data and + * pushes it to the export ready array. + */ +MetricsCollector.prototype.prepareCompletedBuckets = function(now) { + var bucketStart = metricTools.getBucketStartTime(now); + while (bucketStart > this.startTime) { + this.queueBucket(this.createBucket(this.startTime)); + this.startTime += metricConfig.BUCKET_SIZE; + } +}; + +/** + * Collects primitive metrics tools into a list. + */ +MetricsCollector.prototype.collectPrimitiveMetrics = function() { + var metricTools = []; + _.forEach(this.getMetrics(), metricTool => { + gutil.arrayExtend(metricTools, metricTool.getPrimitiveMetrics()); + }); + return metricTools; +}; + +/** + * Loops through metric tools for a chosen bucket and performs the provided callback on each. + * Resets each tool after the callback is performed. + * @param {Number} bucketStart - The desired bucket's start time in milliseconds + * @param {Function} callback - The callback to perform on each metric tool. + */ +MetricsCollector.prototype.forEachBucketMetric = function(bucketEnd, callback) { + this.collectPrimitiveMetrics().forEach(tool => { + callback(tool); + tool.reset(bucketEnd); + }); +}; + +module.exports = MetricsCollector; diff --git a/app/common/NumberFormat.ts b/app/common/NumberFormat.ts new file mode 100644 index 00000000..daea6161 --- /dev/null +++ b/app/common/NumberFormat.ts @@ -0,0 +1,76 @@ +/** + * Here are the most relevant formats we want to support. + * -1234.56 Plain + * -1,234.56 Number (with separators) + * 12.34% Percent + * 1.23E3 Scientific + * $(1,234.56) Accounting + * (1,234.56) Financial + * -$1,234.56 Currency + * + * We implement a button-based UI, using one selector button to choose mode: + * none = NumMode undefined (plain number, no thousand separators) + * `$` = NumMode 'currency' + * `,` = NumMode 'decimal' (plain number, with thousand separators) + * `%` = NumMode 'percent' + * `Exp` = NumMode 'scientific' + * A second toggle button is `(-)` for Sign, to use parentheses rather than "-" for negative + * numbers. It is Ignored and disabled when mode is 'scientific'. + */ + +import {clamp} from 'app/common/gutil'; + +// Options for number formatting. +export type NumMode = 'currency' | 'decimal' | 'percent' | 'scientific'; +export type NumSign = 'parens'; + +// TODO: In the future, locale should be a value associated with the document or the user. +const defaultLocale = 'en-US'; + +// TODO: The currency to use for currency formatting could be made configurable. +const defaultCurrency = 'USD'; + +export interface NumberFormatOptions { + numMode?: NumMode; + numSign?: NumSign; + decimals?: number; // aka minimum fraction digits + maxDecimals?: number; +} + +export function buildNumberFormat(options: NumberFormatOptions): Intl.NumberFormat { + const nfOptions: Intl.NumberFormatOptions = parseNumMode(options.numMode); + + // numSign is implemented outside of Intl.NumberFormat since the latter's similar 'currencySign' + // option is not well-supported, and doesn't apply to non-currency formats. + + if (options.decimals !== undefined) { + // Should be at least 0 + nfOptions.minimumFractionDigits = clamp(Number(options.decimals), 0, 20); + } + + // maximumFractionDigits must not be less than the minimum, so we need to know the minimum + // implied by numMode. + const tmp = new Intl.NumberFormat(defaultLocale, nfOptions).resolvedOptions(); + + if (options.maxDecimals !== undefined) { + // Should be at least 0 and at least minimumFractionDigits. + nfOptions.maximumFractionDigits = clamp(Number(options.maxDecimals), tmp.minimumFractionDigits || 0, 20); + } else if (!options.numMode) { + // For the default format, keep max digits at 10 as we had before. + nfOptions.maximumFractionDigits = clamp(10, tmp.minimumFractionDigits || 0, 20); + } + + return new Intl.NumberFormat(defaultLocale, nfOptions); +} + +function parseNumMode(numMode?: NumMode): Intl.NumberFormatOptions { + switch (numMode) { + case 'currency': return {style: 'currency', currency: defaultCurrency}; + case 'decimal': return {useGrouping: true}; + case 'percent': return {style: 'percent'}; + // TODO 'notation' option (and therefore numMode 'scientific') works on recent Firefox and + // Chrome, not on Safari or Node 10. + case 'scientific': return {notation: 'scientific'} as Intl.NumberFormatOptions; + default: return {useGrouping: false}; + } +} diff --git a/app/common/PluginInstance.ts b/app/common/PluginInstance.ts new file mode 100644 index 00000000..78cd5165 --- /dev/null +++ b/app/common/PluginInstance.ts @@ -0,0 +1,213 @@ +import {IForwarderDest, IMessage, IMsgCustom, IMsgRpcCall, IRpcLogger, MsgType, Rpc} from 'grain-rpc'; +import {Checker} from "ts-interface-checker"; + +import {InactivityTimer} from 'app/common/InactivityTimer'; +import {LocalPlugin} from 'app/common/plugin'; +import {BarePlugin} from 'app/plugin/PluginManifest'; + +import {Implementation} from 'app/plugin/PluginManifest'; +import {RenderOptions, RenderTarget} from 'app/plugin/RenderOptions'; + + +export type ComponentKind = "safeBrowser" | "safePython" | "unsafeNode"; + +// Describes a function that appends some html content to `containerElement` given some +// options. Usefull for provided by a plugin. +export type TargetRenderFunc = (containerElement: HTMLElement, options?: RenderOptions) => void; + +/** + * The `BaseComponent` is the base implementation for a plugins' component. It exposes methods + * related to its activation. It provides basic features including the inactivity timer, activated + * state for the component. A custom component must override the `deactivateImplementation`, + * `activeImplementation` and `useRemoteAPI` methods. + */ +export abstract class BaseComponent implements IForwarderDest { + + public inactivityTimer: InactivityTimer; + private _activated: boolean = false; + + constructor(plugin: BarePlugin, private _logger: IRpcLogger) { + const deactivate = plugin.components.deactivate; + const delay = (deactivate && deactivate.inactivitySec) ? deactivate.inactivitySec : 300; + this.inactivityTimer = new InactivityTimer(() => this.deactivate(), delay * 1000); + } + + /** + * Wether the Component component have been activated. + */ + public activated(): boolean { + return this._activated; + } + + /** + * Activates the component. + */ + public async activate(): Promise { + if (this._logger.info) { this._logger.info("Activating plugin component"); } + await this.activateImplementation(); + this._activated = true; + this.inactivityTimer.enable(); + } + + /** + * Force deactivate the component. + */ + public async deactivate(): Promise { + if (this._activated) { + if (this._logger.info) { this._logger.info("Deactivating plugin component"); } + this._activated = false; + // Cancel the timer to ensure we don't have an unnecessary hanging timeout (in tests it will + // prevent node from exiting, but also it's just wasteful). + this.inactivityTimer.disable(); + try { + await this.deactivateImplementation(); + } catch (e) { + // If it fails, we warn and swallow the exception (or it would be an unhandled rejection). + if (this._logger.warn) { this._logger.warn(`Deactivate failed: ${e.message}`); } + } + } + } + + public async forwardCall(c: IMsgRpcCall): Promise { + if (!this._activated) { await this.activate(); } + return await this.inactivityTimer.disableUntilFinish(this.doForwardCall(c)); + } + + public async forwardMessage(msg: IMsgCustom): Promise { + if (!this._activated) { await this.activate(); } + this.inactivityTimer.ping(); + this.doForwardMessage(msg); // tslint:disable-line:no-floating-promises TODO + } + + protected abstract doForwardCall(c: IMsgRpcCall): Promise; + + protected abstract doForwardMessage(msg: IMsgCustom): Promise; + + protected abstract deactivateImplementation(): Promise; + + protected abstract activateImplementation(): Promise; +} + + +/** + * Node Implementation for the PluginElement interface. A PluginInstance take care of activation of + * the the plugins's components (activating, timing and deactivating), and create the api's for each contributions. + * + * Do not try to instanciate yourself, PluginManager does it for you. Instead use the + * PluginManager.getPlugin(id) method that get instances for you. + * + */ +export class PluginInstance { + + public rpc: Rpc; + public safeBrowser?: BaseComponent; + public unsafeNode?: BaseComponent; + public safePython?: BaseComponent; + + private _renderTargets: Map = new Map(); + + private _nextRenderTargetId = 0; + + constructor(public definition: LocalPlugin, rpcLogger: IRpcLogger) { + + const rpc = this.rpc = new Rpc({logger: rpcLogger}); + rpc.setSendMessage((mssg: any) => rpc.receiveMessage(mssg)); + + this._renderTargets.set("fullscreen", renderFullScreen); + } + + /** + * Create an instance for the implementation, this implementation is specific to node environment. + */ + public getStub(implementation: Implementation, checker: Checker): Iface { + const components: any = this.definition.manifest.components; + // the component forwarder was registered under the same relative path that was used to declare + // it in the manifest + const forwardName = components[implementation.component]; + return this.rpc.getStubForward(forwardName, implementation.name, checker); + } + + /** + * Stop and clean up all components of this plugin. + */ + public async shutdown(): Promise { + await Promise.all([ + this.safeBrowser && this.safeBrowser.deactivate(), + this.safePython && this.safePython.deactivate(), + this.unsafeNode && this.unsafeNode.deactivate(), + ]); + } + + /** + * Create a render target and return its identifier. When a plugin calls `render` with `inline` + * mode and this identifier, it will append the safe browser process to `element`. + */ + public addRenderTarget(renderPluginContent: TargetRenderFunc): number { + const id = this._nextRenderTargetId++; + this._renderTargets.set(id, renderPluginContent); + return id; + } + + /** + * Get the function that render an HTML element based on RenderTarget and RenderOptions. + */ + public getRenderTarget(target: RenderTarget, options?: RenderOptions): TargetRenderFunc { + const targetRenderPluginContent = this._renderTargets.get(target); + if (!targetRenderPluginContent) { + throw new Error(`Unknown render target ${target}`); + } + return (containerElement, opts) => targetRenderPluginContent(containerElement, opts || options); + } + + /** + * Removes the render target. + */ + public removeRenderTarget(target: RenderTarget): boolean { + return this._renderTargets.delete(target); + } + +} + +/** + * Renders safe browser plugin in fullscreen. + */ +function renderFullScreen(element: Element) { + element.classList.add("plugin_instance_fullscreen"); + document.body.appendChild(element); +} + +// Basically the union of relevant interfaces of console and server log. +export interface BaseLogger { + log?(message: string, ...args: any[]): void; + debug?(message: string, ...args: any[]): void; + warn?(message: string, ...args: any[]): void; +} + +/** + * Create IRpcLogger which logs to console or server log with the given prefix. Specifically will + * warn using baseLog.warn, and log info using baseLog.debug or baseLog.log, as available. + */ +export function createRpcLogger(baseLog: BaseLogger, prefix: string): IRpcLogger { + const info = baseLog.debug || baseLog.log; + const warn = baseLog.warn; + return { + warn: warn && ((msg: string) => warn("%s %s", prefix, msg)), + info: info && ((msg: string) => info("%s %s", prefix, msg)), + }; +} + +/** + * If msec milliseconds pass without receiving a Ready message, print the given message as a + * warning. + * TODO: I propose making it a method of rpc itself, as rpc.warnIfNotReady(msec, message). Until + * we have that, this implements it via an ugly hack. + */ +export function warnIfNotReady(rpc: Rpc, msec: number, message: string): void { + if (!(rpc as any)._logger.warn) { return; } + const timer = setTimeout(() => (rpc as any)._logger.warn(message), msec); + const origDispatch = (rpc as any)._dispatch; + (rpc as any)._dispatch = (msg: IMessage) => { + if (msg.mtype === MsgType.Ready) { clearTimeout(timer); } + origDispatch.call(rpc, msg); + }; +} diff --git a/app/common/RefCountMap.ts b/app/common/RefCountMap.ts new file mode 100644 index 00000000..614722f1 --- /dev/null +++ b/app/common/RefCountMap.ts @@ -0,0 +1,123 @@ +/** + * RefCountMap maintains a reference-counted key-value map. Its sole method is use(key) which + * increments the counter for the key, and returns a disposable object which exposes the value via + * the get() method, and decrements the counter back on disposal. + * + * The value is constructed on first reference using options.create(key) callback. After the last + * reference is gone, and an optional gracePeriodMs elapsed, the value is cleaned up using + * options.dispose(key, value) callback. + */ +import {IDisposable} from 'grainjs'; + +export interface IRefCountSub extends IDisposable { + get(): Value; + dispose(): void; +} + +export class RefCountMap implements IDisposable { + private _map: Map> = new Map(); + private _createKey: (key: Key) => Value; + private _disposeKey: (key: Key, value: Value) => void; + private _gracePeriodMs: number; + + /** + * Values are created using options.create(key) on first use. They are disposed after last use, + * using options.dispose(key, value). If options.gracePeriodMs is greater than zero, values + * stick around for this long after last use. + */ + constructor(options: { + create: (key: Key) => Value, + dispose: (key: Key, value: Value) => void, + gracePeriodMs: number, + }) { + this._createKey = options.create; + this._disposeKey = options.dispose; + this._gracePeriodMs = options.gracePeriodMs; + } + + /** + * Use a value, constructing it if needed, or only incrementing the reference count if this key + * is already in the map. The returned subscription object has a get() method which returns the + * actual value, and a dispose() method, which must be called to release this subscription (i.e. + * decrement back the reference count). + */ + public use(key: Key): IRefCountSub { + const rcValue = this._useKey(key); + return { + get: () => rcValue.value, + dispose: () => this._releaseKey(rcValue, key), + }; + } + + /** + * Purge a key by immediately removing it from the map. Disposing the remaining IRefCountSub + * values will be no-ops. + */ + public purgeKey(key: Key): void { + // Note that we must be careful that disposing stale IRefCountSub values is a no-op even when + // the same key gets re-added to the map after purgeKey. + this._doDisposeKey(key); + } + + /** + * Disposing clears the map immediately, and calls options.dispose on all values. + */ + public dispose(): void { + // Note that a clear() method like this one would not be OK. If the map were to continue being + // used after clear(), subscriptions created before clear() would wreak havoc when disposed. + for (const [key, r] of this._map) { + r.count = 0; + this._disposeKey.call(null, key, r.value); + } + this._map.clear(); + } + + private _useKey(key: Key): RefCountValue { + const r = this._map.get(key); + if (r) { + r.count += 1; + if (r.disposeTimeout) { + clearTimeout(r.disposeTimeout); + r.disposeTimeout = undefined; + } + return r; + } + const value = this._createKey.call(null, key); + const rcValue = new RefCountValue(value); + this._map.set(key, rcValue); + return rcValue; + } + + private _releaseKey(r: RefCountValue, key: Key): void { + if (r.count > 0) { + r.count -= 1; + if (r.count === 0) { + if (this._gracePeriodMs > 0) { + if (!r.disposeTimeout) { + r.disposeTimeout = setTimeout(() => this._doDisposeKey(key), this._gracePeriodMs); + } + } else { + this._doDisposeKey(key); + } + } + } + } + + private _doDisposeKey(key: Key): void { + const r = this._map.get(key); + if (r) { + this._map.delete(key); + r.count = 0; + this._disposeKey.call(null, key, r.value); + } + } +} + +/** + * This is an implementation detail of the RefCountMap, which represents a single item. + */ +class RefCountValue { + public count: number = 1; + public disposeTimeout?: ReturnType = undefined; + constructor(public value: Value) {} +} diff --git a/app/common/SortFunc.ts b/app/common/SortFunc.ts new file mode 100644 index 00000000..b7d13e6f --- /dev/null +++ b/app/common/SortFunc.ts @@ -0,0 +1,88 @@ +/** + * SortFunc class interprets the sortSpec (as saved in viewSection.sortColRefs), exposing a + * compare(rowId1, rowId2) function that can be used to actually sort rows in a view. + * + * TODO: When an operation (such as a paste) would cause rows to jump in the sort order, this + * class should support freezing of row positions until the user chooses to re-sort. This is not + * currently implemented. + */ +import {ColumnGetters} from 'app/common/ColumnGetters'; +import {nativeCompare} from 'app/common/gutil'; + +/** + * Compare two cell values, paying attention to types and values. Note that native JS comparison + * can't be used for sorting because it isn't transitive across types (e.g. both 1 < "2" and "2" < + * "a" are true, but 1 < "a" is false.). In addition, we handle complex values represented in + * Grist as arrays. + * + * Note that we need to handle different types of values regardless of the column type, + * because e.g. a numerical column may contain text (alttext) or null values. + */ +export function typedCompare(val1: any, val2: any): number { + // TODO: We should use Intl.Collator for string comparisons to handle accented strings. + let type1, array1; + return nativeCompare(type1 = typeof val1, typeof val2) || + // We need to worry about Array comparisons because formulas returing Any may return null or + // object values represented as arrays (e.g. ['D', ...] for dates). Comparing those without + // distinguishing types would break the sort. Also, arrays need a special comparator. + (type1 === 'object' && + (nativeCompare(array1 = val1 instanceof Array, val2 instanceof Array) || + (array1 && _arrayCompare(val1, val2)))) || + nativeCompare(val1, val2); +} + +function _arrayCompare(val1: any[], val2: any[]): number { + for (let i = 0; i < val1.length; i++) { + if (i >= val2.length) { + return 1; + } + const value = nativeCompare(val1[i], val2[i]); + if (value) { + return value; + } + } + return val1.length === val2.length ? 0 : -1; +} + +type ColumnGetter = (rowId: number) => any; + +/** + * getters is an implementation of app.common.ColumnGetters + */ +export class SortFunc { + // updateSpec() or updateGetters() can populate these fields, used by the compare() method. + private _colGetters: ColumnGetter[] = []; // Array of column getters (mapping rowId to column value) + private _ascFlags: number[] = []; // Array of 1 (ascending) or -1 (descending) flags. + + constructor(private _getters: ColumnGetters) {} + + public updateSpec(sortSpec: number[]): void { + // Prepare an array of column getters for each column in sortSpec. + this._colGetters = sortSpec.map(colRef => { + return this._getters.getColGetter(Math.abs(colRef)); + }).filter(getter => getter) as ColumnGetter[]; + + // Collect "ascending" flags as an array of 1 or -1, one for each column. + this._ascFlags = sortSpec.map(colRef => (colRef >= 0 ? 1 : -1)); + + const manualSortGetter = this._getters.getManualSortGetter(); + if (manualSortGetter) { + this._colGetters.push(manualSortGetter); + this._ascFlags.push(1); + } + } + + /** + * Returns 1 or -1 depending on whether rowId1 should be shown before rowId2. + */ + public compare(rowId1: number, rowId2: number): number { + for (let i = 0, len = this._colGetters.length; i < len; i++) { + const getter = this._colGetters[i]; + const value = typedCompare(getter(rowId1), getter(rowId2)); + if (value) { + return value * this._ascFlags[i]; + } + } + return nativeCompare(rowId1, rowId2); + } +} diff --git a/app/common/StringUnion.ts b/app/common/StringUnion.ts new file mode 100644 index 00000000..4a783161 --- /dev/null +++ b/app/common/StringUnion.ts @@ -0,0 +1,38 @@ +/** + * TypeScript will infer a string union type from the literal values passed to + * this function. Without `extends string`, it would instead generalize them + * to the common string type. + * + * Example definition: + * const Race = StringUnion( + * "orc", + * "human", + * "night elf", + * "undead", + * ); + * type Race = typeof Race.type; + * + * For more details, see: + * https://stackoverflow.com/questions/36836011/checking-validity-of-string + * -literal-union-type-at-runtime?answertab=active#tab-top + */ +export const StringUnion = (...values: UnionType[]) => { + Object.freeze(values); + const valueSet: Set = new Set(values); + + const guard = (value: string): value is UnionType => { + return valueSet.has(value); + }; + + const check = (value: string): UnionType => { + if (!guard(value)) { + const actual = JSON.stringify(value); + const expected = values.map(s => JSON.stringify(s)).join(' | '); + throw new TypeError(`Value '${actual}' is not assignable to type '${expected}'.`); + } + return value; + }; + + const unionNamespace = {guard, check, values}; + return Object.freeze(unionNamespace as typeof unionNamespace & {type: UnionType}); +}; diff --git a/app/common/TableData.ts b/app/common/TableData.ts new file mode 100644 index 00000000..b3c8f76a --- /dev/null +++ b/app/common/TableData.ts @@ -0,0 +1,427 @@ +/** + * TableData maintains a single table's data. + */ +import {getDefaultForType} from 'app/common/gristTypes'; +import fromPairs = require('lodash/fromPairs'); +import {ActionDispatcher} from './ActionDispatcher'; +import {BulkColValues, CellValue, ColInfo, ColInfoWithId, ColValues, DocAction, + isSchemaAction, ReplaceTableData, RowRecord, TableDataAction} from './DocActions'; +import {arrayRemove, arraySplice} from './gutil'; + +export interface ColTypeMap { [colId: string]: string; } + +interface ColData { + colId: string; + type: string; + defl: any; + values: CellValue[]; +} + +/** + * TableData class to maintain a single table's data. + * + * In the browser's memory, table data needs a representation that's reasonably compact. We + * represent it as column-wise arrays. (An early hope was to allow use of TypedArrays, but since + * types can be mixed, those are not used.) + */ +export class TableData extends ActionDispatcher { + private _tableId: string; + private _isLoaded: boolean = false; + private _fetchPromise?: Promise; + + // Storage of the underlying data. Each column is an array, all of the same length. Includes + // 'id' column, containing a reference to _rowIdCol. + private _columns: Map = new Map(); + + // Array of all ColData objects, omitting 'id'. + private _colArray: ColData[] = []; + + // The `id` column is direct reference to the 'id' column, and contains row ids. + private _rowIdCol: number[] = []; + + // Maps row id to index in the arrays in _columns. I.e. it's the inverse of _rowIdCol. + private _rowMap: Map = new Map(); + + constructor(tableId: string, tableData: TableDataAction|null, colTypes: ColTypeMap) { + super(); + this._tableId = tableId; + + // Initialize all columns to empty arrays, while nothing is yet loaded. + for (const colId in colTypes) { + if (colTypes.hasOwnProperty(colId)) { + const type = colTypes[colId]; + const defl = getDefaultForType(type); + const colData: ColData = { colId, type, defl, values: [] }; + this._columns.set(colId, colData); + this._colArray.push(colData); + } + } + this._columns.set('id', {colId: 'id', type: 'Id', defl: 0, values: this._rowIdCol}); + + if (tableData) { + this.loadData(tableData); + } + // TODO: We should probably unload big sets of data when no longer needed. This can be left for + // when we support loading only parts of a table. + } + + /** + * Fetch data (as long as a fetch is not in progress), and load it in memory when done. + * Returns a promise that's resolved when data finishes loading, and isLoaded becomes true. + */ + public fetchData(fetchFunc: (tableId: string) => Promise): Promise { + if (!this._fetchPromise) { + this._fetchPromise = fetchFunc(this._tableId).then(data => { + this._fetchPromise = undefined; + this.loadData(data); + }); + } + return this._fetchPromise; + } + + /** + * Populates the data for this table. Returns the array of old rowIds that were loaded before. + */ + public loadData(tableData: TableDataAction|ReplaceTableData): number[] { + const rowIds: number[] = tableData[2]; + const colValues: BulkColValues = tableData[3]; + const oldRowIds: number[] = this._rowIdCol.slice(0); + + reassignArray(this._rowIdCol, rowIds); + for (const colData of this._colArray) { + const values = colValues[colData.colId]; + // If colId is missing from tableData, use an array of default values. Note that reusing + // default value like this is only OK because all default values we use are primitive. + reassignArray(colData.values, values || this._rowIdCol.map(() => colData.defl)); + } + + this._rowMap.clear(); + for (let i = 0; i < rowIds.length; i++) { + this._rowMap.set(rowIds[i], i); + } + + this._isLoaded = true; + return oldRowIds; + } + + // Used by QuerySet to load new rows for onDemand tables. + public loadPartial(data: TableDataAction): void { + // Add the new rows, reusing BulkAddData code. + const rowIds: number[] = data[2]; + this.onBulkAddRecord(data, data[1], rowIds, data[3]); + + // Mark the table as loaded. + this._isLoaded = true; + } + + // Used by QuerySet to remove unused rows for onDemand tables when a QuerySet is disposed. + public unloadPartial(rowIds: number[]): void { + // Remove the unneeded rows, reusing BulkRemoveRecord code. + this.onBulkRemoveRecord(['BulkRemoveRecord', this.tableId, rowIds], this.tableId, rowIds); + } + + /** + * Read-only tableId. + */ + public get tableId(): string { return this._tableId; } + + /** + * Boolean flag for whether the data for this table is already loaded. + */ + public get isLoaded(): boolean { return this._isLoaded; } + + /** + * The number of records loaded in this table. + */ + public numRecords(): number { return this._rowIdCol.length; } + + /** + * Returns the specified value from this table. + */ + public getValue(rowId: number, colId: string): CellValue|undefined { + const colData = this._columns.get(colId); + const index = this._rowMap.get(rowId); + return colData && index !== undefined ? colData.values[index] : undefined; + } + + /** + * Given a column name, returns a function that takes a rowId and returns the value for that + * column of that row. The returned function is faster than getValue() calls. + */ + public getRowPropFunc(colId: string): undefined | ((rowId: number|"new") => CellValue|undefined) { + const colData = this._columns.get(colId); + if (!colData) { return undefined; } + const values = colData.values; + const rowMap = this._rowMap; + return function(rowId: number|"new") { return rowId === "new" ? "new" : values[rowMap.get(rowId)!]; }; + } + + /** + * Returns the list of all rowIds in this table, in unspecified and unstable order. Equivalent + * to getColValues('id'). + */ + public getRowIds(): ReadonlyArray { + return this._rowIdCol; + } + + /** + * Sort and returns the list of all rowIds in this table. + */ + public getSortedRowIds(): number[] { + return this._rowIdCol.slice(0).sort((a, b) => a - b); + } + + /** + * Returns the list of colIds in this table, including 'id'. + */ + public getColIds(): string[] { + return Array.from(this._columns.keys()); + } + + /** + * Returns an unsorted list of all values in the given column. With no intervening actions, + * all arrays returned by getColValues() and getRowIds() are parallel to each other, i.e. the + * values at the same index correspond to the same record. + */ + public getColValues(colId: string): ReadonlyArray|undefined { + const colData = this._columns.get(colId); + return colData ? colData.values : undefined; + } + + /** + * Returns a limited-sized set of distinct values from a column. If count is given, limits how many + * distinct values are returned. + */ + public getDistinctValues(colId: string, count: number = Infinity): Set|undefined { + const valColumn = this.getColValues(colId); + if (!valColumn) { return undefined; } + const distinct: Set = new Set(); + // Add values to the set until it reaches the desired size, or until there are no more values. + for (let i = 0; i < valColumn.length && distinct.size < count; i++) { + distinct.add(valColumn[i]); + } + return distinct; + } + + /** + * Return data in TableDataAction form ['TableData', tableId, [...rowIds], {...}] + */ + public getTableDataAction(): TableDataAction { + const rowIds = this.getRowIds(); + return ['TableData', + this.tableId, + rowIds as number[], + fromPairs( + this.getColIds() + .filter(colId => colId !== 'id') + .map(colId => [colId, this.getColValues(colId)! as CellValue[]]))]; + } + + /** + * Returns the given columns type, if the column exists, or undefined otherwise. + */ + public getColType(colId: string): string|undefined { + const colData = this._columns.get(colId); + return colData ? colData.type : undefined; + } + + /** + * Builds and returns a record object for the given rowId. + */ + public getRecord(rowId: number): undefined | RowRecord { + const index = this._rowMap.get(rowId); + if (index === undefined) { return undefined; } + const ret: RowRecord = { id: this._rowIdCol[index] }; + for (const colData of this._colArray) { + ret[colData.colId] = colData.values[index]; + } + return ret; + } + + /** + * Builds and returns the list of all records on this table, in unspecified and unstable order. + */ + public getRecords(): RowRecord[] { + const records: RowRecord[] = this._rowIdCol.map((id) => ({ id })); + for (const {colId, values} of this._colArray) { + for (let i = 0; i < records.length; i++) { + records[i][colId] = values[i]; + } + } + return records; + } + + /** + * Builds and returns the list of records in this table that match the given properties object. + * Properties may include 'id' and any table columns. Returned records are not sorted. + */ + public filterRecords(properties: {[key: string]: any}): RowRecord[] { + const rowIndices: number[] = []; + // Pairs of [valueToMatch, arrayOfColValues] + const props = Object.keys(properties).map(p => [properties[p], this._columns.get(p)]); + this._rowIdCol.forEach((id, i) => { + for (const p of props) { + if (p[1].values[i] !== p[0]) { return; } + } + // Collect the indices of the matching rows. + rowIndices.push(i); + }); + + // Convert the array of indices to an array of RowRecords. + const records: RowRecord[] = rowIndices.map(i => ({id: this._rowIdCol[i]})); + for (const {colId, values} of this._colArray) { + for (let i = 0; i < records.length; i++) { + records[i][colId] = values[rowIndices[i]]; + } + } + return records; + } + + /** + * Returns the rowId in the table where colValue is found in the column with the given colId. + */ + public findRow(colId: string, colValue: any): number { + const colData = this._columns.get(colId); + if (!colData) { + return 0; + } + const index = colData.values.indexOf(colValue); + return index < 0 ? 0 : this._rowIdCol[index]; + } + + /** + * Applies a DocAction received from the server; returns true, or false if it was skipped. + */ + public receiveAction(action: DocAction): boolean { + if (this._isLoaded || isSchemaAction(action)) { + this.dispatchAction(action); + return true; + } + return false; + } + + // ---- The following methods implement ActionDispatcher interface ---- + + protected onAddRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void { + const index: number = this._rowIdCol.length; + this._rowMap.set(rowId, index); + this._rowIdCol[index] = rowId; + for (const {colId, defl, values} of this._colArray) { + values[index] = colValues.hasOwnProperty(colId) ? colValues[colId] : defl; + } + } + + protected onBulkAddRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void { + const index: number = this._rowIdCol.length; + for (let i = 0; i < rowIds.length; i++) { + this._rowMap.set(rowIds[i], index + i); + this._rowIdCol[index + i] = rowIds[i]; + } + for (const {colId, defl, values} of this._colArray) { + for (let i = 0; i < rowIds.length; i++) { + values[index + i] = colValues.hasOwnProperty(colId) ? colValues[colId][i] : defl; + } + } + } + + protected onRemoveRecord(action: DocAction, tableId: string, rowId: number): void { + // Note that in this implementation, delete + undo will reorder the storage and the ordering + // of rows returned getRowIds() and similar methods. + const index = this._rowMap.get(rowId); + if (index !== undefined) { + const last: number = this._rowIdCol.length - 1; + // We keep the column-wise arrays dense by moving the last element into the freed-up spot. + for (const {values} of this._columns.values()) { // This adjusts _rowIdCol too. + values[index] = values[last]; + values.pop(); + } + this._rowMap.set(this._rowIdCol[index], index); + this._rowMap.delete(rowId); + } + } + + protected onUpdateRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void { + const index = this._rowMap.get(rowId); + if (index !== undefined) { + for (const colId in colValues) { + if (colValues.hasOwnProperty(colId)) { + const colData = this._columns.get(colId); + if (colData) { + colData.values[index] = colValues[colId]; + } + } + } + } + } + + protected onBulkUpdateRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void { + for (let i = 0; i < rowIds.length; i++) { + const index = this._rowMap.get(rowIds[i]); + if (index !== undefined) { + for (const colId in colValues) { + if (colValues.hasOwnProperty(colId)) { + const colData = this._columns.get(colId); + if (colData) { + colData.values[index] = colValues[colId][i]; + } + } + } + } + } + } + + protected onReplaceTableData(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void { + this.loadData(action as ReplaceTableData); + } + + protected onAddColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void { + if (this._columns.has(colId)) { return; } + const type = colInfo.type; + const defl = getDefaultForType(type); + const colData: ColData = { colId, type, defl, values: this._rowIdCol.map(() => defl) }; + this._columns.set(colId, colData); + this._colArray.push(colData); + } + + protected onRemoveColumn(action: DocAction, tableId: string, colId: string): void { + const colData = this._columns.get(colId); + if (!colData) { return; } + this._columns.delete(colId); + arrayRemove(this._colArray, colData); + } + + protected onRenameColumn(action: DocAction, tableId: string, oldColId: string, newColId: string): void { + const colData = this._columns.get(oldColId); + if (colData) { + colData.colId = newColId; + this._columns.set(newColId, colData); + this._columns.delete(oldColId); + } + } + + protected onModifyColumn(action: DocAction, tableId: string, oldColId: string, colInfo: ColInfo): void { + const colData = this._columns.get(oldColId); + if (colData && colInfo.hasOwnProperty('type')) { + colData.type = colInfo.type; + colData.defl = getDefaultForType(colInfo.type); + } + } + + protected onRenameTable(action: DocAction, oldTableId: string, newTableId: string): void { + this._tableId = newTableId; + } + + protected onAddTable(action: DocAction, tableId: string, columns: ColInfoWithId[]): void { + // A table processing its own addition is a noop + } + + protected onRemoveTable(action: DocAction, tableId: string): void { + // Stop dispatching actions if we've been deleted. We might also want to clean up in the future. + this._isLoaded = false; + } +} + +function reassignArray(targetArray: T[], sourceArray: T[]): void { + targetArray.length = 0; + arraySplice(targetArray, 0, sourceArray); +} diff --git a/app/common/TabularDiff.ts b/app/common/TabularDiff.ts new file mode 100644 index 00000000..e6395df5 --- /dev/null +++ b/app/common/TabularDiff.ts @@ -0,0 +1,31 @@ +/** + * + * Types for use when summarizing differences between versions of a table, with the + * diff itself presented in tabular form. + * + */ + + +/** + * Pairs of before/after values of cells. Values, when present, are nested in a trivial + * list since they can be literally anything - null, undefined, etc. Otherwise they + * are either null, meaning non-existent, or "?", meaning unknown. Non-existent values + * appear prior to a table/column being created, or after it has been destroyed. + * Unknown values appear when they are omitted from summaries of bulk actions, and those + * summaries are then merged with others. + */ +export type CellDelta = [[any]|"?"|null, [any]|"?"|null]; + +/** a special column indicating what changes happened on row (addition, update, removal) */ +export type RowChangeType = string; + +/** differences for an individual table */ +export interface TabularDiff { + header: string[]; /** labels for columns */ + cells: Array<[RowChangeType, number, CellDelta[]]>; // "number" is rowId +} + +/** differences for a collection of tables */ +export interface TabularDiffs { + [tableId: string]: TabularDiff; +} diff --git a/app/common/TestState.ts b/app/common/TestState.ts new file mode 100644 index 00000000..3e0ae38b --- /dev/null +++ b/app/common/TestState.ts @@ -0,0 +1,4 @@ +export interface TestState { + clipboard?: string; + anchorApplied?: boolean; +} diff --git a/app/common/UserAPI.ts b/app/common/UserAPI.ts new file mode 100644 index 00000000..e3bc86ea --- /dev/null +++ b/app/common/UserAPI.ts @@ -0,0 +1,667 @@ +import {ApplyUAResult} from 'app/common/ActiveDocAPI'; +import {BaseAPI, IOptions} from 'app/common/BaseAPI'; +import {BillingAPI, BillingAPIImpl} from 'app/common/BillingAPI'; +import {BrowserSettings} from 'app/common/BrowserSettings'; +import {BulkColValues, TableColValues, UserAction} from 'app/common/DocActions'; +import {DocCreationInfo} from 'app/common/DocListAPI'; +import {Features} from 'app/common/Features'; +import {isClient} from 'app/common/gristUrls'; +import {FullUser} from 'app/common/LoginSessionAPI'; +import * as roles from 'app/common/roles'; +import {addCurrentOrgToPath} from 'app/common/urlUtils'; + +// Nominal email address of the anonymous user. +export const ANONYMOUS_USER_EMAIL = 'anon@getgrist.com'; + +// A special user allowed to add/remove the EVERYONE_EMAIL to/from a resource. +export const SUPPORT_EMAIL = 'support@getgrist.com'; + +// A special 'docId' that means to create a new document. +export const NEW_DOCUMENT_CODE = 'new'; + +// Properties shared by org, workspace, and doc resources. +export interface CommonProperties { + name: string; + createdAt: string; // ISO date string + updatedAt: string; // ISO date string + removedAt?: string; // ISO date string - only can appear on docs and workspaces currently + public?: boolean; // If set, resource is available to the public +} +export const commonPropertyKeys = ['createdAt', 'name', 'updatedAt']; + +export interface OrganizationProperties extends CommonProperties { + domain: string|null; +} +export const organizationPropertyKeys = [...commonPropertyKeys, 'domain']; + +// Basic information about an organization, excluding the user's access level +export interface OrganizationWithoutAccessInfo extends OrganizationProperties { + id: number; + owner: FullUser|null; + billingAccount?: BillingAccount; + host: string|null; // if set, org's preferred domain (e.g. www.thing.com) +} + +// Organization information plus the user's access level +export interface Organization extends OrganizationWithoutAccessInfo { + access: roles.Role; +} + +// Basic information about a billing account associated with an org or orgs. +export interface BillingAccount { + id: number; + individual: boolean; + product: Product; + isManager: boolean; +} + +// Information about the product associated with an org or orgs. +export interface Product { + name: string; + features: Features; +} + +// The upload types vary based on which fetch implementation is in use. This is +// an incomplete list. For example, node streaming types are supported by node-fetch. +export type UploadType = string | Blob | Buffer; + +/** + * Returns a user-friendly org name, which is either org.name, or "@User Name" for personal orgs. + */ +export function getOrgName(org: Organization): string { + return org.owner ? `@` + org.owner.name : org.name; +} + +export type WorkspaceProperties = CommonProperties; +export const workspacePropertyKeys = ['createdAt', 'name', 'updatedAt']; + +export interface Workspace extends WorkspaceProperties { + id: number; + docs: Document[]; + org: Organization; + access: roles.Role; + owner?: FullUser; // Set when workspaces are in the "docs" pseudo-organization, + // assembled from multiple personal organizations. + // Not set when workspaces are all from the same organization. + + // Set when the workspace belongs to support@getgrist.com. We expect only one such workspace + // ("Examples & Templates"), containing sample documents. + isSupportWorkspace?: boolean; +} + +export interface DocumentProperties extends CommonProperties { + isPinned: boolean; + urlId: string|null; +} +export const documentPropertyKeys = [...commonPropertyKeys, 'isPinned', 'urlId']; + +export interface Document extends DocumentProperties { + id: string; + workspace: Workspace; + access: roles.Role; +} + +export interface PermissionDelta { + maxInheritedRole?: roles.BasicRole|null; + users?: { + // Maps from email to group name, or null to inherit. + [email: string]: roles.NonGuestRole|null + }; +} + +export interface PermissionData { + maxInheritedRole?: roles.BasicRole|null; + users: UserAccessData[]; +} + +// A structure for modifying managers of a billing account. +export interface ManagerDelta { + users: { + // To add a manager, link their email to 'managers'. + // To remove a manager, link their email to null. + // This format is used to rhyme with the ACL PermissionDelta format. + [email: string]: 'managers'|null + }; +} + +// Information about a user and their access to an unspecified resource of interest. +export interface UserAccessData { + id: number; + name: string; + email: string; + picture?: string|null; // When present, a url to a public image of unspecified dimensions. + // Represents the user's direct access to the resource of interest. Lack of access to a resource + // is represented by a null value. + access: roles.Role|null; + // A user's parentAccess represent their effective inheritable access to the direct parent of the resource + // of interest. The user's effective access to the resource of interest can be determined based + // on the user's parentAccess, the maxInheritedRole setting of the resource and the user's direct + // access to the resource. Lack of access to the parent resource is represented by a null value. + // If parent has non-inheritable access, this should be null. + parentAccess?: roles.BasicRole|null; +} + +export interface ActiveSessionInfo { + user: FullUser & {helpScoutSignature?: string}; + org: Organization|null; + orgError?: OrgError; +} + +export interface OrgError { + error: string; + status: number; +} + +/** + * Options to control the source of a document being replaced. For + * example, a document could be initialized from another document + * (e.g. a fork) or from a snapshot. + */ +export interface DocReplacementOptions { + sourceDocId?: string; // docId to copy from + snapshotId?: string; // s3 VersionId +} + +/** + * Information about a single document snapshot/backup. + */ +export interface DocSnapshot { + lastModified: string; // when the snapshot was made + snapshotId: string; // the id of the snapshot in the underlying store + docId: string; // an id for accessing the snapshot as a Grist document +} + +/** + * A list of document snapshots. + */ +export interface DocSnapshots { + snapshots: DocSnapshot[]; // snapshots, freshest first. +} + +/** + * Information about a single document state. + */ +export interface DocState { + n: number; // a sequential identifier + h: string; // a hash identifier +} + +/** + * A list of document states. Most recent is first. + */ +export interface DocStates { + states: DocState[]; +} + +/** + * A comparison between two documents, called "left" and "right". + * The comparison is based on the action histories in the documents. + * If those histories have been truncated, the comparison may report + * two documents as being unrelated even if they do in fact have some + * shared history. + */ +export interface DocStateComparison { + left: DocState; // left / local document + right: DocState; // right / remote document + parent: DocState|null; // most recent common ancestor of left and right + // summary of the relationship between the two documents. + // same: documents have the same most recent state + // left: the left document has actions not yet in the right + // right: the right document has actions not yet in the left + // both: both documents have changes (possible divergence) + // unrelated: no common history found + summary: 'same' | 'left' | 'right' | 'both' | 'unrelated'; +} + +export {UserProfile} from 'app/common/LoginSessionAPI'; + +export interface UserAPI { + getSessionActive(): Promise; + setSessionActive(email: string): Promise; + getSessionAll(): Promise<{users: FullUser[], orgs: Organization[]}>; + getOrgs(merged?: boolean): Promise; + getWorkspace(workspaceId: number): Promise; + getOrg(orgId: number|string): Promise; + getOrgWorkspaces(orgId: number|string): Promise; + getDoc(docId: string): Promise; + newOrg(props: Partial): Promise; + newWorkspace(props: Partial, orgId: number|string): Promise; + newDoc(props: Partial, workspaceId: number): Promise; + newUnsavedDoc(options?: {timezone?: string}): Promise; + renameOrg(orgId: number|string, name: string): Promise; + renameWorkspace(workspaceId: number, name: string): Promise; + renameDoc(docId: string, name: string): Promise; + updateDoc(docId: string, props: Partial): Promise; + deleteOrg(orgId: number|string): Promise; + deleteWorkspace(workspaceId: number): Promise; // delete workspace permanently + softDeleteWorkspace(workspaceId: number): Promise; // soft-delete workspace + undeleteWorkspace(workspaceId: number): Promise; // recover soft-deleted workspace + deleteDoc(docId: string): Promise; // delete doc permanently + softDeleteDoc(docId: string): Promise; // soft-delete doc + undeleteDoc(docId: string): Promise; // recover soft-deleted doc + updateOrgPermissions(orgId: number|string, delta: PermissionDelta): Promise; + updateWorkspacePermissions(workspaceId: number, delta: PermissionDelta): Promise; + updateDocPermissions(docId: string, delta: PermissionDelta): Promise; + getOrgAccess(orgId: number|string): Promise; + getWorkspaceAccess(workspaceId: number): Promise; + getDocAccess(docId: string): Promise; + pinDoc(docId: string): Promise; + unpinDoc(docId: string): Promise; + moveDoc(docId: string, workspaceId: number): Promise; + getUserProfile(): Promise; + updateUserName(name: string): Promise; + getWorker(key: string): Promise; + getWorkerAPI(key: string): Promise; + getBillingAPI(): BillingAPI; + getDocAPI(docId: string): DocAPI; + fetchApiKey(): Promise; + createApiKey(): Promise; + deleteApiKey(): Promise; + getTable(docId: string, tableName: string): Promise; + applyUserActions(docId: string, actions: UserAction[]): Promise; + importUnsavedDoc(material: UploadType, options?: { + filename?: string, + timezone?: string, + onUploadProgress?: (ev: ProgressEvent) => void, + }): Promise; + deleteUser(userId: number, name: string): Promise; + getBaseUrl(): string; // Get the prefix for all the endpoints this object wraps. + forRemoved(): UserAPI; // Get a version of the API that works on removed resources. +} + +/** + * Collect endpoints related to the content of a single document that we've been thinking + * of as the (restful) "Doc API". A few endpoints that could be here are not, for historical + * reasons, such as downloads. + */ +export interface DocAPI { + getRows(tableId: string): Promise; + updateRows(tableId: string, changes: TableColValues): Promise; + addRows(tableId: string, additions: BulkColValues): Promise; + replace(source: DocReplacementOptions): Promise; + getSnapshots(): Promise; + forceReload(): Promise; + compareState(remoteDocId: string): Promise; +} + +// Operations that are supported by a doc worker. +export interface DocWorkerAPI { + readonly url: string; + importDocToWorkspace(uploadId: number, workspaceId: number, settings?: BrowserSettings): Promise; + upload(material: UploadType, filename?: string): Promise; + downloadDoc(docId: string, template?: boolean): Promise; + copyDoc(docId: string, template?: boolean, name?: string): Promise; +} + +export class UserAPIImpl extends BaseAPI implements UserAPI { + constructor(private _homeUrl: string, private _options: IOptions = {}) { + super(_options); + } + + public forRemoved(): UserAPI { + const extraParameters = new Map([['showRemoved', '1']]); + return new UserAPIImpl(this._homeUrl, {...this._options, extraParameters}); + } + + public async getSessionActive(): Promise { + return this.requestJson(`${this._url}/api/session/access/active`, {method: 'GET'}); + } + + public async setSessionActive(email: string): Promise { + const body = JSON.stringify({ email }); + return this.requestJson(`${this._url}/api/session/access/active`, {method: 'POST', body}); + } + + public async getSessionAll(): Promise<{users: FullUser[], orgs: Organization[]}> { + return this.requestJson(`${this._url}/api/session/access/all`, {method: 'GET'}); + } + + public async getOrgs(merged: boolean = false): Promise { + return this.requestJson(`${this._url}/api/orgs?merged=${merged ? 1 : 0}`, { method: 'GET' }); + } + + public async getWorkspace(workspaceId: number): Promise { + return this.requestJson(`${this._url}/api/workspaces/${workspaceId}`, { method: 'GET' }); + } + + public async getOrg(orgId: number|string): Promise { + return this.requestJson(`${this._url}/api/orgs/${orgId}`, { method: 'GET' }); + } + + public async getOrgWorkspaces(orgId: number|string): Promise { + return this.requestJson(`${this._url}/api/orgs/${orgId}/workspaces?includeSupport=1`, + { method: 'GET' }); + } + + public async getDoc(docId: string): Promise { + return this.requestJson(`${this._url}/api/docs/${docId}`, { method: 'GET' }); + } + + public async newOrg(props: Partial): Promise { + return this.requestJson(`${this._url}/api/orgs`, { + method: 'POST', + body: JSON.stringify(props) + }); + } + + public async newWorkspace(props: Partial, orgId: number|string): Promise { + return this.requestJson(`${this._url}/api/orgs/${orgId}/workspaces`, { + method: 'POST', + body: JSON.stringify(props) + }); + } + + public async newDoc(props: Partial, workspaceId: number): Promise { + return this.requestJson(`${this._url}/api/workspaces/${workspaceId}/docs`, { + method: 'POST', + body: JSON.stringify(props) + }); + } + + public async newUnsavedDoc(options: {timezone?: string} = {}): Promise { + return this.requestJson(`${this._url}/api/docs`, { + method: 'POST', + body: JSON.stringify(options), + }); + } + + public async renameOrg(orgId: number|string, name: string): Promise { + await this.request(`${this._url}/api/orgs/${orgId}`, { + method: 'PATCH', + body: JSON.stringify({ name }) + }); + } + + public async renameWorkspace(workspaceId: number, name: string): Promise { + await this.request(`${this._url}/api/workspaces/${workspaceId}`, { + method: 'PATCH', + body: JSON.stringify({ name }) + }); + } + + public async renameDoc(docId: string, name: string): Promise { + return this.updateDoc(docId, {name}); + } + + public async updateDoc(docId: string, props: Partial): Promise { + await this.request(`${this._url}/api/docs/${docId}`, { + method: 'PATCH', + body: JSON.stringify(props) + }); + } + + public async deleteOrg(orgId: number|string): Promise { + await this.request(`${this._url}/api/orgs/${orgId}`, { method: 'DELETE' }); + } + + public async deleteWorkspace(workspaceId: number): Promise { + await this.request(`${this._url}/api/workspaces/${workspaceId}`, { method: 'DELETE' }); + } + + public async softDeleteWorkspace(workspaceId: number): Promise { + await this.request(`${this._url}/api/workspaces/${workspaceId}/remove`, { method: 'POST' }); + } + + public async undeleteWorkspace(workspaceId: number): Promise { + await this.request(`${this._url}/api/workspaces/${workspaceId}/unremove`, { method: 'POST' }); + } + + public async deleteDoc(docId: string): Promise { + await this.request(`${this._url}/api/docs/${docId}`, { method: 'DELETE' }); + } + + public async softDeleteDoc(docId: string): Promise { + await this.request(`${this._url}/api/docs/${docId}/remove`, { method: 'POST' }); + } + + public async undeleteDoc(docId: string): Promise { + await this.request(`${this._url}/api/docs/${docId}/unremove`, { method: 'POST' }); + } + + public async updateOrgPermissions(orgId: number|string, delta: PermissionDelta): Promise { + await this.request(`${this._url}/api/orgs/${orgId}/access`, { + method: 'PATCH', + body: JSON.stringify({ delta }) + }); + } + + public async updateWorkspacePermissions(workspaceId: number, delta: PermissionDelta): Promise { + await this.request(`${this._url}/api/workspaces/${workspaceId}/access`, { + method: 'PATCH', + body: JSON.stringify({ delta }) + }); + } + + public async updateDocPermissions(docId: string, delta: PermissionDelta): Promise { + await this.request(`${this._url}/api/docs/${docId}/access`, { + method: 'PATCH', + body: JSON.stringify({ delta }) + }); + } + + public async getOrgAccess(orgId: number|string): Promise { + return this.requestJson(`${this._url}/api/orgs/${orgId}/access`, { method: 'GET' }); + } + + public async getWorkspaceAccess(workspaceId: number): Promise { + return this.requestJson(`${this._url}/api/workspaces/${workspaceId}/access`, { method: 'GET' }); + } + + public async getDocAccess(docId: string): Promise { + return this.requestJson(`${this._url}/api/docs/${docId}/access`, { method: 'GET' }); + } + + public async pinDoc(docId: string): Promise { + await this.request(`${this._url}/api/docs/${docId}/pin`, { + method: 'PATCH' + }); + } + + public async unpinDoc(docId: string): Promise { + await this.request(`${this._url}/api/docs/${docId}/unpin`, { + method: 'PATCH' + }); + } + + public async moveDoc(docId: string, workspaceId: number): Promise { + await this.request(`${this._url}/api/docs/${docId}/move`, { + method: 'PATCH', + body: JSON.stringify({ workspace: workspaceId }) + }); + } + + public async getUserProfile(): Promise { + return this.requestJson(`${this._url}/api/profile/user`); + } + + public async updateUserName(name: string): Promise { + await this.request(`${this._url}/api/profile/user/name`, { + method: 'POST', + body: JSON.stringify({name}) + }); + } + + public async getWorker(key: string): Promise { + const json = await this.requestJson(`${this._url}/api/worker/${key}`, { + method: 'GET', + credentials: 'include' + }); + return json.docWorkerUrl; + } + + public async getWorkerAPI(key: string): Promise { + const docUrl = this._urlWithOrg(await this.getWorker(key)); + return new DocWorkerAPIImpl(docUrl, this._options); + } + + public getBillingAPI(): BillingAPI { + return new BillingAPIImpl(this._url, this._options); + } + + public getDocAPI(docId: string): DocAPI { + return new DocAPIImpl(this._url, docId, this._options); + } + + public async fetchApiKey(): Promise { + const resp = await this.fetch(`${this._url}/api/profile/apiKey`, { + credentials: 'include' + }); + return await resp.text(); + } + + public async createApiKey(): Promise { + const res = await this.fetch(`${this._url}/api/profile/apiKey`, {credentials: 'include', method: 'POST'}); + return await res.text(); + } + + public async deleteApiKey(): Promise { + await this.fetch(`${this._url}/api/profile/apiKey`, {credentials: 'include', method: 'DELETE'}); + } + + // This method is not strictly needed anymore, but is widely used by + // tests so supporting as a handy shortcut for getDocAPI(docId).getRows(tableName) + public async getTable(docId: string, tableName: string): Promise { + return this.getDocAPI(docId).getRows(tableName); + } + + public async applyUserActions(docId: string, actions: UserAction[]): Promise { + return this.requestJson(`${this._url}/api/docs/${docId}/apply`, { + method: 'POST', + body: JSON.stringify(actions) + }); + } + + public async importUnsavedDoc(material: UploadType, options?: { + filename?: string, + timezone?: string, + onUploadProgress?: (ev: ProgressEvent) => void, + }): Promise { + options = options || {}; + const formData = this.newFormData(); + formData.append('upload', material as any, options.filename); + if (options.timezone) { formData.append('timezone', options.timezone); } + const resp = await this.requestAxios(`${this._url}/api/docs`, { + headers: this._options.headers, + method: 'POST', + data: formData, + onUploadProgress: options.onUploadProgress, + }); + return resp.data; + } + + public async deleteUser(userId: number, name: string) { + await this.request(`${this._url}/api/users/${userId}`, + {method: 'DELETE', + body: JSON.stringify({name})}); + } + + public getBaseUrl(): string { return this._url; } + + // Recomputes the URL on every call to pick up changes in the URL when switching orgs. + // (Feels inefficient, but probably doesn't matter, and it's simpler than the alternatives.) + private get _url(): string { + return this._urlWithOrg(this._homeUrl); + } + + private _urlWithOrg(base: string): string { + return isClient() ? addCurrentOrgToPath(base) : base.replace(/\/$/, ''); + } +} + +export class DocWorkerAPIImpl extends BaseAPI implements DocWorkerAPI { + constructor(readonly url: string, private _options: IOptions = {}) { + super(_options); + } + + public async importDocToWorkspace(uploadId: number, workspaceId: number, browserSettings?: BrowserSettings) + : Promise { + return this.requestJson(`${this.url}/api/workspaces/${workspaceId}/import`, { + method: 'POST', + body: JSON.stringify({ uploadId, browserSettings }) + }); + } + + public async upload(material: UploadType, filename?: string): Promise { + const formData = this.newFormData(); + formData.append('upload', material as any, filename); + const json = await this.requestJson(`${this.url}/uploads`, { + headers: this._options.headers, + method: 'POST', + body: formData + }); + return json.uploadId; + } + + public async downloadDoc(docId: string, template: boolean = false): Promise { + const extra = template ? '&template=1' : ''; + const result = await this.request(`${this.url}/download?doc=${docId}${extra}`, { + headers: this._options.headers, + method: 'GET', + }); + if (!result.ok) { throw new Error(await result.text()); } + return result; + } + + public async copyDoc(docId: string, template: boolean = false, name?: string): Promise { + const url = new URL(`${this.url}/copy?doc=${docId}`); + if (template) { + url.searchParams.append('template', '1'); + } + if (name) { + url.searchParams.append('name', name); + } + const json = await this.requestJson(url.href, { + headers: this._options.headers, + method: 'POST', + }); + return json.uploadId; + } +} + +export class DocAPIImpl extends BaseAPI implements DocAPI { + private _url: string; + + constructor(url: string, readonly docId: string, options: IOptions = {}) { + super(options); + this._url = `${url}/api/docs/${docId}`; + } + + public async getRows(tableId: string): Promise { + return this.requestJson(`${this._url}/tables/${tableId}/data`); + } + + public async updateRows(tableId: string, changes: TableColValues): Promise { + return this.requestJson(`${this._url}/tables/${tableId}/data`, { + body: JSON.stringify(changes), + method: 'PATCH' + }); + } + + public async addRows(tableId: string, additions: BulkColValues): Promise { + return this.requestJson(`${this._url}/tables/${tableId}/data`, { + body: JSON.stringify(additions), + method: 'POST' + }); + } + + public async replace(source: DocReplacementOptions): Promise { + return this.requestJson(`${this._url}/replace`, { + body: JSON.stringify(source), + method: 'POST' + }); + } + + public async getSnapshots(): Promise { + return this.requestJson(`${this._url}/snapshots`); + } + + public async forceReload(): Promise { + await this.request(`${this._url}/force-reload`, { + method: 'POST' + }); + } + + public async compareState(remoteDocId: string): Promise { + return this.requestJson(`${this._url}/compare/${remoteDocId}`); + } +} diff --git a/app/common/UserConfig.ts b/app/common/UserConfig.ts new file mode 100644 index 00000000..c53d7fb7 --- /dev/null +++ b/app/common/UserConfig.ts @@ -0,0 +1,31 @@ +/* + * Interface for the user's config found in config.json. + */ +export interface UserConfig { + enableMetrics?: boolean; + docListSortBy?: string; + docListSortDir?: number; + features?: ISupportedFeatures; + + /* + * The host serving the untrusted content: on dev environment could be + * "http://getgrist.localtest.me". Port is added at runtime and should not be included. + */ + untrustedContentOrigin?: string; +} + +export interface ISupportedFeatures { + signin?: boolean; + sharing?: boolean; + proxy?: boolean; // If true, Grist will accept login information via http headers + // X-Forwarded-User and X-Forwarded-Email. Set to true only if + // Grist is behind a reverse proxy that is managing those headers, + // otherwise they could be spoofed. + formulaBar?: boolean; + + // Plugin views, REPL, and Validations all need work, but are exposed here to allow existing + // tests to continue running. These only affect client-side code. + customViewPlugin?: boolean; + replTool?: boolean; + validationsTool?: boolean; +} diff --git a/app/common/ValueFormatter.ts b/app/common/ValueFormatter.ts new file mode 100644 index 00000000..e0bddda5 --- /dev/null +++ b/app/common/ValueFormatter.ts @@ -0,0 +1,144 @@ +// tslint:disable:max-classes-per-file + +import {CellValue} from 'app/common/DocActions'; +import * as gristTypes from 'app/common/gristTypes'; +import * as gutil from 'app/common/gutil'; +import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat'; +import * as moment from 'moment-timezone'; + +// Some text to show on cells whose values are pending. +export const PENDING_DATA_PLACEHOLDER = "Loading..."; + +/** + * Formats a custom object received as a value in a DocAction, as "Constructor(args...)". + * E.g. ["Foo", 1, 2, 3] becomes the string "Foo(1, 2, 3)". + */ +export function formatObject(args: [string, ...any[]]): string { + const objType = args[0], objArgs = args.slice(1); + switch (objType) { + case 'L': return JSON.stringify(objArgs); + // First arg is seconds since epoch (moment takes ms), second arg is timezone + case 'D': return moment.tz(objArgs[0] * 1000, objArgs[1]).format("YYYY-MM-DD HH:mm:ssZ"); + case 'd': return moment.tz(objArgs[0] * 1000, 'UTC').format("YYYY-MM-DD"); + case 'R': return `${objArgs[0]}[${objArgs[1]}]`; + case 'E': return gristTypes.formatError(args); + case 'P': return PENDING_DATA_PLACEHOLDER; + } + return objType + "(" + JSON.stringify(objArgs).slice(1, -1) + ")"; +} + +/** + * Formats a value of unknown type, using formatObject() for encoded objects. + */ +export function formatUnknown(value: any): string { + return gristTypes.isObject(value) ? formatObject(value) : (value == null ? "" : String(value)); +} + +export type IsRightTypeFunc = (value: CellValue) => boolean; + +export class BaseFormatter { + public readonly isRightType: IsRightTypeFunc; + + constructor(public type: string, public opts: object) { + this.isRightType = gristTypes.isRightType(gristTypes.extractTypeFromColType(type)) || + gristTypes.isRightType('Any')!; + } + + /** + * Formats a value that matches the type of this formatter. This should be overridden by derived + * classes to handle values in formatter-specific ways. + */ + public format(value: any): string { + return value; + } + + /** + * Formats using this.format() if a value is of the right type for this formatter, or using + * AnyFormatter otherwise. This method the recommended API. There is no need to override it. + */ + public formatAny(value: any): string { + return this.isRightType(value) ? this.format(value) : formatUnknown(value); + } +} + +class AnyFormatter extends BaseFormatter { + public format(value: any): string { + return formatUnknown(value); + } +} + +export class NumericFormatter extends BaseFormatter { + private _numFormat: Intl.NumberFormat; + private _formatter: (val: number) => string; + + constructor(type: string, options: NumberFormatOptions) { + super(type, options); + this._numFormat = buildNumberFormat(options); + this._formatter = (options.numSign === 'parens') ? this._formatParens : this._formatPlain; + } + + public format(value: any): string { + return value === null ? '' : this._formatter(value); + } + + public _formatPlain(value: number): string { + return this._numFormat.format(value); + } + + public _formatParens(value: number): string { + // Surround positive numbers with spaces to align them visually to parenthesized numbers. + return (value >= 0) ? + ` ${this._numFormat.format(value)} ` : + `(${this._numFormat.format(-value)})`; + } +} + +class IntFormatter extends NumericFormatter { + constructor(type: string, opts: object) { + super(type, {decimals: 0, ...opts}); + } +} + +class DateFormatter extends BaseFormatter { + private _dateTimeFormat: string; + private _timezone: string; + + constructor(type: string, opts: {dateFormat?: string}, timezone: string = 'UTC') { + super(type, opts); + this._dateTimeFormat = opts.dateFormat || 'YYYY-MM-DD'; + this._timezone = timezone; + } + + public format(value: any): string { + if (value === null) { return ''; } + const time = moment.tz(value * 1000, this._timezone); + return time.format(this._dateTimeFormat); + } +} + +class DateTimeFormatter extends DateFormatter { + constructor(type: string, opts: {dateFormat?: string; timeFormat?: string}) { + const timezone = gutil.removePrefix(type, "DateTime:") || ''; + const timeFormat = opts.timeFormat === undefined ? 'h:mma' : opts.timeFormat; + const dateFormat = (opts.dateFormat || 'YYYY-MM-DD') + " " + timeFormat; + super(type, {dateFormat}, timezone); + } +} + +const formatters: {[name: string]: typeof BaseFormatter} = { + Numeric: NumericFormatter, + Int: IntFormatter, + Bool: BaseFormatter, + Date: DateFormatter, + DateTime: DateTimeFormatter, + // We don't list anything that maps to AnyFormatter, since that's the default. +}; + +/** + * Takes column type and widget options and returns a constructor with a format function that can + * properly convert a value passed to it into the right format for that column. + */ +export function createFormatter(type: string, opts: object): BaseFormatter { + const ctor = formatters[gristTypes.extractTypeFromColType(type)] || AnyFormatter; + return new ctor(type, opts); +} diff --git a/app/common/arrayToString.ts b/app/common/arrayToString.ts new file mode 100644 index 00000000..2d5f43be --- /dev/null +++ b/app/common/arrayToString.ts @@ -0,0 +1,28 @@ +/** + * Functions to convert between an array of bytes and a string. The implementations are + * different for Node and for the browser. + */ + +declare const TextDecoder: any, TextEncoder: any; + +export let arrayToString: (data: Uint8Array) => string; +export let stringToArray: (data: string) => Uint8Array; + +if (typeof TextDecoder !== 'undefined') { + // Note that constructing a TextEncoder/Decoder takes time, so it's faster to reuse. + const dec = new TextDecoder('utf8'); + const enc = new TextEncoder('utf8'); + arrayToString = function(uint8Array: Uint8Array): string { + return dec.decode(uint8Array); + }; + stringToArray = function(str: string): Uint8Array { + return enc.encode(str); + }; +} else { + arrayToString = function(uint8Array: Uint8Array): string { + return Buffer.from(uint8Array).toString('utf8'); + }; + stringToArray = function(str: string): Uint8Array { + return new Uint8Array(Buffer.from(str, 'utf8')); + }; +} diff --git a/app/common/declarations.d.ts b/app/common/declarations.d.ts new file mode 100644 index 00000000..c0d0611b --- /dev/null +++ b/app/common/declarations.d.ts @@ -0,0 +1,5 @@ +declare module "app/common/MemBuffer" { + const MemBuffer: any; + type MemBuffer = any; + export = MemBuffer; +} diff --git a/app/common/delay.ts b/app/common/delay.ts new file mode 100644 index 00000000..c509c42e --- /dev/null +++ b/app/common/delay.ts @@ -0,0 +1,7 @@ +/** + * Returns a promise that resolves in the given number of milliseconds. + * (A replica of bluebird.delay using native promises.) + */ +export function delay(msec: number): Promise { + return new Promise((resolve) => setTimeout(resolve, msec)); +} diff --git a/app/common/emails.ts b/app/common/emails.ts new file mode 100644 index 00000000..e966ba80 --- /dev/null +++ b/app/common/emails.ts @@ -0,0 +1,43 @@ +/** + * + * Utilities related to email normalization. Currently + * trivial, but could potentially need special per-domain + * rules in future. + * + * Email addresses are a bit slippery. Domain names are + * case insensitive, but user names may or may not be, + * depending on the mail server handling the domain. + * Other special treatment of user names may also be in + * place for particular domains (periods, plus sign, etc). + * + * We treat emails as case-insensitive for the purposes + * of determining equality of emails, and indexing users + * by email address. + * + */ + +/** + * + * Convert the supplied email address to a normalized form + * that we will use for indexing and equality tests. + * Many possible email addresses could map to the same + * normalized result; as far as we are concerned those + * addresses are equivalent. + * + * The normalization we do is a simple lowercase. This + * means we won't be able to treat both Jane@x.y and + * jane@x.y as separate email addresses, even through + * they may in fact be separate mailboxes on x.y. + * + * The normalized email is not something we should show + * the user in the UI, but is rather for internal purposes. + * + * The original non-normalized email is called a + * "display email" to distinguish it from a "normalized + * email" + * + */ +export function normalizeEmail(displayEmail: string): string { + // We take the lower case, without use of locale. + return displayEmail.toLowerCase(); +} diff --git a/app/common/gristTypes.ts b/app/common/gristTypes.ts new file mode 100644 index 00000000..9653b977 --- /dev/null +++ b/app/common/gristTypes.ts @@ -0,0 +1,242 @@ +import {CellValue} from 'app/common/DocActions'; +import isString = require('lodash/isString'); + +// tslint:disable:object-literal-key-quotes + +export type GristType = 'Any' | 'Attachments' | 'Blob' | 'Bool' | 'Choice' | 'Date' | 'DateTime' | + 'Id' | 'Int' | 'ManualSortPos' | 'Numeric' | 'PositionNumber' | 'Ref' | 'RefList' | 'Text'; + +// Letter codes for CellValue types encoded as [code, args...] tuples. +export type GristObjType = 'L' | 'D' | 'd' | 'R' | 'E' | 'P'; + +export const MANUALSORT = 'manualSort'; + +// This mapping includes both the default value, and its representation for SQLite. +const _defaultValues: {[key in GristType]: [CellValue, string]} = { + 'Any': [ null, "NULL" ], + 'Attachments': [ null, "NULL" ], + 'Blob': [ null, "NULL" ], + // Bool is only supported by SQLite as 0 and 1 values. + 'Bool': [ false, "0" ], + 'Choice': [ '', "''" ], + 'Date': [ null, "NULL" ], + 'DateTime': [ null, "NULL" ], + 'Id': [ 0, "0" ], + 'Int': [ 0, "0" ], + // Note that "1e999" is a way to store Infinity into SQLite. This is verified by "Defaults" + // tests in DocStorage.js. See also http://sqlite.1065341.n5.nabble.com/Infinity-td55327.html. + 'ManualSortPos': [ Number.POSITIVE_INFINITY, "1e999" ], + 'Numeric': [ 0, "0" ], + 'PositionNumber': [ Number.POSITIVE_INFINITY, "1e999" ], + 'Ref': [ 0, "0" ], + 'RefList': [ null, "NULL" ], + 'Text': [ '', "''" ], +}; + + +/** + * Given a grist column type (e.g Text, Numeric, ...) returns the default value for that type. + * If options.sqlFormatted is true, returns the representation of the value for SQLite. + */ +export function getDefaultForType(colType: string, options: {sqlFormatted?: boolean} = {}) { + const type = extractTypeFromColType(colType); + return (_defaultValues[type as GristType] || _defaultValues.Any)[options.sqlFormatted ? 1 : 0]; +} + +/** + * Returns whether a value (as received in a DocAction) represents a custom object. + */ +export function isObject(value: CellValue): value is [string, any?] { + return Array.isArray(value); +} + +/** + * Returns whether a value (as received in a DocAction) represents a raised exception. + */ +export function isRaisedException(value: CellValue): boolean { + return Array.isArray(value) && value[0] === 'E'; +} + +/** + * Returns whether a value (as received in a DocAction) represents a list or is null, + * which is a valid value for list types in grist. + */ +export function isListOrNull(value: CellValue): boolean { + return value === null || (Array.isArray(value) && value[0] === 'L'); +} + +/** + * Returns whether a value (as received in a DocAction) represents an empty list. + */ +export function isEmptyList(value: CellValue): boolean { + return Array.isArray(value) && value.length === 1 && value[0] === 'L'; +} + +/** + * Returns whether a value (as received in a DocAction) represents a "Pending" value. + */ +export function isPending(value: CellValue): boolean { + return Array.isArray(value) && value[0] === 'P'; +} + +/** + * Formats a raised exception (a value for which isRaisedException is true) for display in a cell. + * This is designed to look somewhat similar to Excel, e.g. #VALUE or #DIV/0!" + */ +export function formatError(value: [string, ...any[]]): string { + const errName = value[1]; + switch (errName) { + case 'ZeroDivisionError': return '#DIV/0!'; + case 'UnmarshallableError': return value[3] || ('#' + errName); + case 'InvalidTypedValue': return `#Invalid ${value[2]}: ${value[3]}`; + } + return '#' + errName; +} + +function isNumber(v: CellValue) { return typeof v === 'number' || typeof v === 'boolean'; } +function isNumberOrNull(v: CellValue) { return isNumber(v) || v === null; } +function isBoolean(v: CellValue) { return typeof v === 'boolean' || v === 1 || v === 0; } + +function isNormalValue(value: CellValue) { + return !(Array.isArray(value) && (value[0] === 'E' || value[0] === 'P')); +} + +/** + * Map of Grist type to an "isRightType" checker function, which determines if a given values type + * matches the declared type of the column. + */ +const rightType: {[key in GristType]: (value: CellValue) => boolean} = { + Any: isNormalValue, + Attachments: isListOrNull, + Text: isString, + Blob: isString, + Int: isNumberOrNull, + Bool: isBoolean, + Date: isNumberOrNull, + DateTime: isNumberOrNull, + Numeric: isNumberOrNull, + Id: isNumber, + PositionNumber: isNumber, + ManualSortPos: isNumber, + Ref: isNumber, + RefList: isListOrNull, + Choice: (v: CellValue, options?: any) => { + // TODO widgets options should not be used outside of the client. They are an instance of + // modelUtil.jsonObservable, passed in by FieldBuilder. + if (v === '') { + // Accept empty-string values as valid + return true; + } else if (options) { + const choices = options().choices; + return Array.isArray(choices) && choices.includes(v); + } else { + return false; + } + } +}; + +export function isRightType(type: string): undefined | ((value: CellValue) => boolean) { + return rightType[type as GristType]; +} + +export function extractTypeFromColType(type: string): string { + if (!type) { return type; } + const colon = type.indexOf(':'); + return (colon === -1 ? type : type.slice(0, colon)); +} + +/** + * Convert pureType to Grist python type name, e.g. 'Ref' to 'Reference'. + */ +export function getGristType(pureType: string): string { + switch (pureType) { + case 'Ref': return 'Reference'; + case 'RefList': return 'ReferenceList'; + default: return pureType; + } +} + +/** + * Converts SQL type strings produced by the Sequelize library into its corresponding + * Grist type. The list of types is based on an analysis of SQL type string outputs + * produced by the Sequelize library (mostly covered in lib/data-types.js). Some + * additional engine/dialect specific types are detailed in dialect directories. + * + * TODO: A handful of exotic SQL types (mostly from PostgreSQL) will currently throw an + * Error, rather than returning a type. Further testing is required to determine + * whether Grist can manage those data types. + * + * @param {String} sqlType A string produced by Sequelize's describeTable query + * @return {String} The corresponding Grist type string + * @throws {Error} If the sqlType is unrecognized or unsupported + */ +export function sequelizeToGristType(sqlType: string): GristType { + // Sequelize type strings can include parens (e.g., `CHAR(10)`). This function + // ignores those additional details when determining the Grist type. + let endMarker = sqlType.length; + const parensMarker = sqlType.indexOf('('); + endMarker = parensMarker > 0 ? parensMarker : endMarker; + + // Type strings might also include a space after the basic type description. + // The type `DOUBLE PRECISION` is one such example, but modifiers or attributes + // relevant to the type might also appear after the type itself (e.g., UNSIGNED, + // NONZERO). These are ignored when determining the Grist type. + const spaceMarker = sqlType.indexOf(' '); + endMarker = spaceMarker > 0 && spaceMarker < endMarker ? spaceMarker : endMarker; + + switch (sqlType.substring(0, endMarker)) { + case 'INTEGER': + case 'BIGINT': + case 'SMALLINT': + case 'INT': + return 'Int'; + case 'NUMBER': + case 'FLOAT': + case 'DECIMAL': + case 'NUMERIC': + case 'REAL': + case 'DOUBLE': + case 'DOUBLE PRECISION': + return 'Numeric'; + case 'BOOLEAN': + case 'TINYINT': + return 'Bool'; + case 'STRING': + case 'CHAR': + case 'TEXT': + case 'UUID': + case 'UUIDV1': + case 'UUIDV4': + case 'VARCHAR': + case 'NVARCHAR': + case 'TINYTEXT': + case 'MEDIUMTEXT': + case 'LONGTEXT': + case 'ENUM': + return 'Text'; + case 'TIME': + case 'DATE': + case 'DATEONLY': + case 'DATETIME': + case 'NOW': + return 'Text'; + case 'BLOB': + case 'TINYBLOB': + case 'MEDIUMBLOB': + case 'LONGBLOB': + // TODO: Passing binary data to the Sandbox is throwing Errors. Proper support + // for these Blob data types requires some more investigation. + throw new Error('SQL type: `' + sqlType + '` is currently unsupported'); + case 'NONE': + case 'HSTORE': + case 'JSON': + case 'JSONB': + case 'VIRTUAL': + case 'ARRAY': + case 'RANGE': + case 'GEOMETRY': + throw new Error('SQL type: `' + sqlType + '` is currently untested'); + default: + throw new Error('Unrecognized datatype: `' + sqlType + '`'); + } +} diff --git a/app/common/gristUrls.ts b/app/common/gristUrls.ts index 726a667e..ec80b91d 100644 --- a/app/common/gristUrls.ts +++ b/app/common/gristUrls.ts @@ -1 +1,583 @@ -export type ProductFlavor = 'grist'; +import {BillingPage, BillingSubPage, BillingTask} from 'app/common/BillingAPI'; +import {OpenDocMode} from 'app/common/DocListAPI'; +import {encodeQueryParams} from 'app/common/gutil'; +import {localhostRegex} from 'app/common/LoginState'; +import {Document} from 'app/common/UserAPI'; +import identity = require('lodash/identity'); +import pickBy = require('lodash/pickBy'); +import {StringUnion} from './StringUnion'; + +export type IDocPage = number | 'new' | 'code'; + +// What page to show in the user's home area. Defaults to 'workspace' if a workspace is set, and +// to 'all' otherwise. +export const HomePage = StringUnion('all', 'workspace', 'trash'); +export type IHomePage = typeof HomePage.type; + +export const WelcomePage = StringUnion('user', 'teams'); +export type WelcomePage = typeof WelcomePage.type; + +// Default subdomain for home api service if not otherwise specified. +export const DEFAULT_HOME_SUBDOMAIN = 'api'; + +// This is the minimum length a urlId may have if it is chosen +// as a prefix of the docId. +export const MIN_URLID_PREFIX_LENGTH = 12; + +/** + * Special ways to open a document, based on what the user intends to do. + * - view: Open document in read-only mode (even if user has edit rights) + * - fork: Open document in fork-ready mode. This means that while edits are + * permitted, those edits should go to a copy of the document rather than + * the original. + */ + +export const commonUrls = { + help: "https://support.getgrist.com", + plans: "https://www.getgrist.com/pricing", + + efcrConnect: 'https://efc-r.com/connect', + efcrHelp: 'https://www.nioxus.info/eFCR-Help', +}; + +/** + * Values representable in a URL. The current state is available as urlState().state observable + * in client. Updates to this state are expected by functions such as makeUrl() and setLinkUrl(). + */ +export interface IGristUrlState { + org?: string; + homePage?: IHomePage; + ws?: number; + doc?: string; + slug?: string; // if present, this is based on the document title, and is not a stable id + mode?: OpenDocMode; + fork?: UrlIdParts; + docPage?: IDocPage; + newui?: boolean; + billing?: BillingPage; + welcome?: WelcomePage; + params?: { + billingPlan?: string; + billingTask?: BillingTask; + }; + hash?: HashLink; // if present, this specifies an individual row within a section of a page. +} + +// Subset of GristLoadConfig used by getOrgUrlInfo(), which affects the interpretation of the +// current URL. +export interface OrgUrlOptions { + // The org associated with the current URL. + org?: string; + + // Base domain for constructing new URLs, should start with "." and not include port, e.g. + // ".getgrist.com". It should be unset for localhost operation and in single-org mode. + baseDomain?: string; + + // In single-org mode, this is the single well-known org. + singleOrg?: string; +} + +// Result of getOrgUrlInfo(). +export interface OrgUrlInfo { + hostname?: string; // If hostname should be changed to access the requested org. + orgInPath?: string; // If /o/{orgInPath} should be used to access the requested org. +} + +export function isCustomHost(hostname: string, baseDomain: string) { + // .localtest.me is used by plugin tests for arcane reasons. + return hostname !== 'localhost' && !hostname.endsWith(baseDomain) && !hostname.endsWith('.localtest.me'); +} + +export function getOrgUrlInfo(newOrg: string, currentHostname: string, options: OrgUrlOptions): OrgUrlInfo { + if (newOrg === options.singleOrg) { + return {}; + } + if (!options.baseDomain || currentHostname === 'localhost') { + return {orgInPath: newOrg}; + } + if (newOrg === options.org && isCustomHost(currentHostname, options.baseDomain)) { + return {}; + } + return {hostname: newOrg + options.baseDomain}; +} + +/** + * The actual serialization of a url state into a URL. The URL has the form + * / + * /ws// + * /doc/[/p/] + * + * where depends on whether subdomains are in use, e.g. + * .getgrist.com + * localhost:8080/o/ + */ +export function encodeUrl(gristConfig: Partial, + state: IGristUrlState, baseLocation: Location | URL): string { + const url = new URL(baseLocation.href); + const parts = ['/']; + + if (state.org) { + // We figure out where to stick the org using the gristConfig and the current host. + const {hostname, orgInPath} = getOrgUrlInfo(state.org, baseLocation.hostname, gristConfig); + if (hostname) { + url.hostname = hostname; + } + if (orgInPath) { + parts.push(`o/${orgInPath}/`); + } + } + + if (state.ws) { parts.push(`ws/${state.ws}/`); } + if (state.doc) { + if (state.slug) { + parts.push(`${encodeURIComponent(state.doc)}/${encodeURIComponent(state.slug)}`); + } else { + parts.push(`doc/${encodeURIComponent(state.doc)}`); + } + if (state.mode && parseOpenDocMode(state.mode)) { + parts.push(`/m/${state.mode}`); + } + if (state.docPage) { + parts.push(`/p/${state.docPage}`); + } + } else { + if (state.homePage === 'trash') { parts.push('p/trash'); } + } + + if (state.billing) { + parts.push(state.billing === 'billing' ? 'billing' : `billing/${state.billing}`); + } + + if (state.welcome) { + parts.push(`welcome/${state.welcome}`); + } + + const queryParams = pickBy(state.params, identity) as {[key: string]: string}; + if (state.newui !== undefined) { + queryParams.newui = state.newui ? '1' : '0'; + } + const hashParts: string[] = []; + if (state.hash && state.hash.rowId) { + const hash = state.hash; + hashParts.push(`a1`); + for (const key of ['sectionId', 'rowId', 'colRef'] as Array) { + if (hash[key]) { hashParts.push(`${key[0]}${hash[key]}`); } + } + } + const queryStr = encodeQueryParams(queryParams); + url.pathname = parts.join(''); + url.search = queryStr; + if (state.hash) { + // Project tests use hashes, so only set hash if there is an anchor. + url.hash = hashParts.join('.'); + } + return url.href; +} + +/** + * Parse a URL location into an IGristUrlState object. See encodeUrl() documentation. + */ +export function decodeUrl(gristConfig: Partial, location: Location | URL): IGristUrlState { + const parts = location.pathname.slice(1).split('/'); + const map = new Map(); + for (let i = 0; i < parts.length; i += 2) { + map.set(parts[i], decodeURIComponent(parts[i + 1])); + } + // When the urlId is a prefix of the docId, documents are identified + // as "/slug" instead of "doc/". We can detect that because + // the minimum length of a urlId prefix is longer than the maximum length + // of any of the valid keys in the url. + for (const key of map.keys()) { + if (key.length >= MIN_URLID_PREFIX_LENGTH) { + map.set('doc', key); + map.set('slug', map.get(key)!); + map.delete(key); + break; + } + } + + const state: IGristUrlState = {}; + const subdomain = parseSubdomain(location.host); + if (gristConfig.org || gristConfig.singleOrg) { + state.org = gristConfig.org || gristConfig.singleOrg; + } else if (!gristConfig.pathOnly && subdomain.org) { + state.org = subdomain.org; + } + const sp = new URLSearchParams(location.search); + if (location.search) { state.params = {}; } + if (map.has('o')) { state.org = map.get('o'); } + if (map.has('ws')) { state.ws = parseInt(map.get('ws')!, 10); } + if (map.has('doc')) { + state.doc = map.get('doc'); + const fork = parseUrlId(map.get('doc')!); + if (fork.forkId) { state.fork = fork; } + if (map.has('slug')) { state.slug = map.get('slug'); } + if (map.has('p')) { state.docPage = parseDocPage(map.get('p')!); } + } else { + if (map.has('p')) { + const p = map.get('p')!; + state.homePage = HomePage.guard(p) ? p : undefined; + } + } + if (map.has('m')) { state.mode = parseOpenDocMode(map.get('m')!); } + if (sp.has('newui')) { state.newui = useNewUI(sp.get('newui') ? sp.get('newui') === '1' : undefined); } + if (map.has('billing')) { state.billing = parseBillingPage(map.get('billing')!); } + if (map.has('welcome')) { state.welcome = parseWelcomePage(map.get('welcome')!); } + if (sp.has('billingPlan')) { state.params!.billingPlan = sp.get('billingPlan')!; } + if (sp.has('billingTask')) { + state.params!.billingTask = parseBillingTask(sp.get('billingTask')!); + } + if (location.hash) { + const hash = location.hash; + const hashParts = hash.split('.'); + const hashMap = new Map(); + for (const part of hashParts) { + hashMap.set(part.slice(0, 1), part.slice(1)); + } + if (hashMap.has('#') && hashMap.get('#') === 'a1') { + const link: HashLink = {}; + for (const key of ['sectionId', 'rowId', 'colRef'] as Array) { + const ch = key.substr(0, 1); + if (hashMap.has(ch)) { link[key] = parseInt(hashMap.get(ch)!, 10); } + } + state.hash = link; + } + } + return state; +} + +export function useNewUI(newui: boolean|undefined) { + return newui !== false; +} + +/** + * parseDocPage is a noop if p is 'new' or 'code', otherwise parse to integer + */ +function parseDocPage(p: string) { + if (['new', 'code'].includes(p)) { + return p as 'new'|'code'; + } + return parseInt(p, 10); +} + +/** + * parseBillingPage ensures that the billing page value is a valid BillingPageType. + */ +function parseBillingPage(p: string): BillingPage { + return BillingSubPage.guard(p) ? p : 'billing'; +} + +/** + * parseBillingTask ensures that the value is a valid BillingTask or undefined. + */ +function parseBillingTask(t: string): BillingTask|undefined { + return BillingTask.guard(t) ? t : undefined; +} + +/** + * parseOpenDocMode ensures that the value is a valid OpenDocMode or undefined. + */ +function parseOpenDocMode(p: string): OpenDocMode|undefined { + return OpenDocMode.guard(p) ? p : undefined; +} + +/** + * parse welcome page ensure that the value is a valid WelcomePage, default to 'user' if not. + */ +function parseWelcomePage(p: string): WelcomePage { + return WelcomePage.guard(p) ? p : 'user'; +} + +/** + * Parses the URL like "foo.bar.baz" into the pair {org: "foo", base: ".bar.baz"}. + * Port is allowed and included into base. + * + * The "base" part is required to have at least two periods. The "org" part must pass + * the subdomainRegex test. + * + * If there's no way to parse the URL into such a pair, then an empty object is returned. + */ +export function parseSubdomain(host: string|undefined): {org?: string, base?: string} { + if (!host) { return {}; } + const match = /^([^.]+)(\..+\..+)$/.exec(host.toLowerCase()); + if (match) { + const org = match[1]; + const base = match[2]; + if (subdomainRegex.exec(org)) { + return {org, base}; + } + } + // Host has nowhere to put a subdomain. + return {}; +} + +/** + * Like parseSubdomain, but throws an error if neither of these cases apply: + * - host can be parsed into a valid subdomain and a valid base domain. + * - host is localhost:NNNN + * An empty object is only returned when host is localhost:NNNN. + */ +export function parseSubdomainStrictly(host: string|undefined): {org?: string, base?: string} { + if (!host) { throw new Error('host not known'); } + const result = parseSubdomain(host); + if (result.org) { return result; } + if (!host.match(localhostRegex)) { + throw new Error(`host not understood: ${host}`); + } + // Host is localhost[:NNNN], no org available. + return {}; +} + +/** + * These settings get sent to the client along with the loaded page. At the minimum, the browser + * needs to know the URL of the home API server (e.g. api.getgrist.com). + */ +export interface GristLoadConfig { + // URL of the Home API server for the browser client to use. + homeUrl: string|null; + + // When loading /doc/{docId}, we include the id used to assign the document (this is the docId). + assignmentId?: string; + + // Org or "subdomain". When present, this overrides org information from the hostname. We rely + // on this for custom domains, but set it generally for all pages. + org?: string; + + // Base domain for constructing new URLs, should start with "." and not include port, e.g. + // ".getgrist.com". It should be unset for localhost operation and in single-org mode. + baseDomain?: string; + + // In single-org mode, this is the single well-known org. Suppress any org selection UI. + singleOrg?: string; + + // When set, this directs the client to encode org information in path, not in domain. + pathOnly?: boolean; + + // Type of error page to show. This is used for pages such as "signed-out" and "not-found", + // which don't include the full app. + errPage?: string; + + // When errPage is a generic "other-error", this is the message to show. + errMessage?: string; + + // URL for client to use for untrusted content. + pluginUrl?: string|null; + + // Stripe API key for use on the client. + stripeAPIKey?: string; + + // BeaconID for the support widget from HelpScout. + helpScoutBeaconId?: string; + + // If set, enable anonymous sharing UI elements. + supportAnon?: boolean; + + // Max upload allowed for imports (except .grist files), in bytes; 0 or omitted for unlimited. + maxUploadSizeImport?: number; + + // Max upload allowed for attachments, in bytes; 0 or omitted for unlimited. + maxUploadSizeAttachment?: number; + + // Pre-fetched call to getDoc for the doc being loaded. + getDoc?: {[id: string]: Document}; + + // Pre-fetched call to getWorker for the doc being loaded. + getWorker?: {[id: string]: string}; + + // The timestamp when this gristConfig was generated. + timestampMs: number; +} + +// Acceptable org subdomains are alphanumeric (hyphen also allowed) and of +// non-zero length. +const subdomainRegex = /^[-a-z0-9]+$/i; + +export interface OrgParts { + subdomain: string|null; + orgFromHost: string|null; + orgFromPath: string|null; + pathRemainder: string; + mismatch: boolean; +} + +/** + * Returns true if code is running in client, false if running in server. + */ +export function isClient() { + return (typeof window !== 'undefined') && window && window.location && window.location.hostname; +} + +/** + * Returns a known org "subdomain" if Grist is configured in single-org mode + * (GRIST_SINGLE_ORG= on the server) or if the page includes an org in gristConfig. + */ +export function getKnownOrg(): string|null { + if (isClient()) { + const gristConfig: GristLoadConfig = (window as any).gristConfig; + return (gristConfig && gristConfig.org) || null; + } else { + return process.env.GRIST_SINGLE_ORG || null; + } +} + +/** + * Returns true if org must be encoded in path, not in domain. Determined from + * gristConfig on the client. On on the server returns true if the host is + * supplied and is 'localhost', or if GRIST_ORG_IN_PATH is set to 'true'. + */ +export function isOrgInPathOnly(host?: string): boolean { + if (isClient()) { + const gristConfig: GristLoadConfig = (window as any).gristConfig; + return (gristConfig && gristConfig.pathOnly) || false; + } else { + if (host && host.match(/^localhost(:[0-9]+)?$/)) { return true; } + return (process.env.GRIST_ORG_IN_PATH === 'true'); + } +} + +// Extract an organization name from the host. Returns null if an organization name +// could not be recovered. Organization name may be overridden by server configuration. +export function getOrgFromHost(reqHost: string): string|null { + const singleOrg = getKnownOrg(); + if (singleOrg) { return singleOrg; } + if (isOrgInPathOnly()) { return null; } + return parseSubdomain(reqHost).org || null; +} + +/** + * Get any information about an organization that is embedded in the host name or the + * path. + * For example, on nasa.getgrist.com, orgFromHost and subdomain will be set to "nasa". + * On localhost:8000/o/nasa, orgFromPath and subdomain will be set to "nasa". + * On nasa.getgrist.com/o/nasa, orgFromHost, orgFromPath, and subdomain will all be "nasa". + * On spam.getgrist.com/o/nasa, orgFromHost will be "spam", orgFromPath will be "nasa", + * subdomain will be null, and mismatch will be true. + */ +export function extractOrgParts(reqHost: string|undefined, reqPath: string): OrgParts { + let orgFromHost: string|null = getKnownOrg(); + + if (!orgFromHost && reqHost) { + orgFromHost = getOrgFromHost(reqHost); + if (orgFromHost) { + // Some subdomains are shared, and do not reflect the name of an organization. + // See https://phab.getgrist.com/w/hosting/v1/urls/ for a list. + if (/^(api|v1-.*|doc-worker-.*)$/.test(orgFromHost)) { + orgFromHost = null; + } + } + } + + const part = parseFirstUrlPart('o', reqPath); + if (part.value) { + const orgFromPath = part.value.toLowerCase(); + const mismatch = Boolean(orgFromHost && orgFromPath && (orgFromHost !== orgFromPath)); + const subdomain = mismatch ? null : orgFromPath; + return {orgFromHost, orgFromPath, pathRemainder: part.path, mismatch, subdomain}; + } + return {orgFromHost, orgFromPath: null, pathRemainder: reqPath, mismatch: false, subdomain: orgFromHost}; +} + +/** + * When a prefix is extracted from the path, the remainder of the path may be empty. + * This method makes sure there is at least a "/". + */ +export function sanitizePathTail(path: string|undefined) { + path = path || '/'; + return (path.startsWith('/') ? '' : '/') + path; +} + +/* + * If path starts with /{tag}/{value}{/rest}, returns value and the remaining path (/rest). + * Otherwise, returns value of undefined and the path unchanged. + * E.g. parseFirstUrlPart('o', '/o/foo/bar') returns {value: 'foo', path: '/bar'}. + */ +export function parseFirstUrlPart(tag: string, path: string): {value?: string, path: string} { + const match = path.match(/^\/([^/?#]+)\/([^/?#]+)(.*)$/); + if (match && match[1] === tag) { + return {value: match[2], path: sanitizePathTail(match[3])}; + } else { + return {path}; + } +} + +/** + * The internal structure of a UrlId. There is no internal structure. unless the id is + * for a fork, in which case the fork has a separate id, and a user id may also be + * embedded to track ownership. + */ +export interface UrlIdParts { + trunkId: string; + forkId?: string; + forkUserId?: number; + snapshotId?: string; +} + +// Parse a string of the form trunkId or trunkId~forkId or trunkId~forkId~forkUserId +// or trunkId[....]~v=snapshotId +export function parseUrlId(urlId: string): UrlIdParts { + let snapshotId: string|undefined; + const parts = urlId.split('~'); + const bareParts = parts.filter(part => !part.includes('=')); + for (const part of parts) { + if (part.startsWith('v=')) { + snapshotId = decodeURIComponent(part.substr(2).replace(/_/g, '%')); + } + } + return { + trunkId: bareParts[0], + forkId: bareParts[1], + forkUserId: (bareParts[2] !== undefined) ? parseInt(bareParts[2], 10) : undefined, + snapshotId, + }; +} + +// Construct a string of the form trunkId or trunkId~forkId or trunkId~forkId~forkUserId +// or trunkId[....]~v=snapshotId +export function buildUrlId(parts: UrlIdParts): string { + let token = [parts.trunkId, parts.forkId, parts.forkUserId].filter(x => x !== undefined).join('~'); + if (parts.snapshotId) { + // This could be an S3 VersionId, about which AWS makes few promises. + // encodeURIComponent leaves untouched the following: + // alphabetic; decimal; any of: - _ . ! ~ * ' ( ) + // We further encode _.!~*'() to fit within existing limits on what characters + // may be in a docId (leaving just the hyphen, which is permitted). The limits + // could be loosened, but without much benefit. + const codedSnapshotId = encodeURIComponent(parts.snapshotId) + .replace(/[_.!~*'()]/g, ch => `_${ch.charCodeAt(0).toString(16).toUpperCase()}`) + .replace(/%/g, '_'); + token = `${token}~v=${codedSnapshotId}`; + } + return token; +} + +/** + * Values that may be encoded in a hash in a document url. + */ +export interface HashLink { + sectionId?: number; + rowId?: number; + colRef?: number; +} + +// Check whether a urlId is a prefix of the docId, and adequately long to be +// a candidate for use in prettier urls. +function shouldIncludeSlug(doc: {id: string, urlId: string|null}): boolean { + if (!doc.urlId || doc.urlId.length < MIN_URLID_PREFIX_LENGTH) { return false; } + return doc.id.startsWith(doc.urlId); +} + +// Convert the name of a document into a slug. Only alphanumerics are retained, +// and spaces are replaced with hyphens. +// TODO: investigate whether there's a better option with unicode than just +// deleting it, seems unfair to languages using anything other than unaccented +// Latin characters. +function nameToSlug(name: string): string { + return name.trim().replace(/ /g, '-').replace(/[^-a-zA-Z0-9]/g, '').replace(/---*/g, '-'); +} + +// Returns a slug for the given docId/urlId/name, or undefined if a slug should +// not be used. +export function getSlugIfNeeded(doc: {id: string, urlId: string|null, name: string}): string|undefined { + if (!shouldIncludeSlug(doc)) { return; } + return nameToSlug(doc.name); +} diff --git a/app/common/gutil.ts b/app/common/gutil.ts new file mode 100644 index 00000000..089d132a --- /dev/null +++ b/app/common/gutil.ts @@ -0,0 +1,782 @@ +import {delay} from 'app/common/delay'; +import {Listener, Observable} from 'grainjs'; +import {Observable as KoObservable} from 'knockout'; +import constant = require('lodash/constant'); +import identity = require('lodash/identity'); +import times = require('lodash/times'); + +export const UP_TRIANGLE = '\u25B2'; +export const DOWN_TRIANGLE = '\u25BC'; + +const EMAIL_RE = new RegExp("^\\w[\\w%+/='-]*(\\.[\\w%+/='-]+)*@([A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z" + + "0-9])?\\.)+[A-Za-z]{2,6}$", "u"); + +// Returns whether str starts with prefix. (Note that this implementation avoids creating a new +// string, and only checks a single location.) +export function startsWith(str: string, prefix: string): boolean { + return str.lastIndexOf(prefix, 0) === 0; +} + +// Returns whether str ends with suffix. +export function endsWith(str: string, suffix: string): boolean { + return str.indexOf(suffix, str.length - suffix.length) !== -1; +} + +// If str starts with prefix, removes it and returns what remains. Otherwise, returns null. +export function removePrefix(str: string, prefix: string): string|null { + return startsWith(str, prefix) ? str.slice(prefix.length) : null; +} + + +// If str ends with suffix, removes it and returns what remains. Otherwise, returns null. +export function removeSuffix(str: string, suffix: string): string|null { + return endsWith(str, suffix) ? str.slice(0, str.length - suffix.length) : null; +} + +export function removeTrailingSlash(str: string): string { + const result = removeSuffix(str, '/'); + return result === null ? str : result; +} + +// Expose .padStart. The version of node we use has it, but they typings +// need the es2017 typescript target. TODO: replace once typings in place. +export function padStart(str: string, targetLength: number, padString: string) { + return (str as any).padStart(targetLength, padString); +} + +// Capitalizes every word in a string. +export function capitalize(str: string): string { + return str.replace(/\b[a-z]/gi, c => c.toUpperCase()); +} + +// Returns whether the string n represents a valid number. +// http://stackoverflow.com/questions/18082/validate-numbers-in-javascript-isnumeric +export function isNumber(n: string): boolean { + // This wasn't right for a long time: isFinite() is key to failing on strings like "5a". + return !isNaN(parseFloat(n)) && isFinite(n as any); +} + +/** + * Returns a value clamped to the given min-max range. + * @param {Number} value - some numeric value. + * @param {Number} min - minimum value allowed. + * @param {Number} max - maximum value allowed. Must have min <= max. + * @returns {Number} - value restricted to the given range. + */ +export function clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); +} + +/** + * Checks if ele is contained within the given bounds. + * @param {Number} value + * @param {Number} bound1 - does not have to be less than/eqal to bound2 + * @param {Number} bound2 + * @returns {Boolean} - True/False + */ +export function between(value: number, bound1: number, bound2: number): boolean { + const lower = Math.min(bound1, bound2); + const upper = Math.max(bound1, bound2); + return lower <= value && value <= upper; +} + +/** + * Returns the positive modulo of x by n. (Javascript default allows negatives) + */ +export function mod(x: number, n: number): number { + return ((x % n) + n) % n; +} + +/** + * Returns a number that is n rounded down to the next nearest number divisible by m + */ + +export function roundDownToMultiple(n: number, m: number): number { + return Math.floor(n / m) * m; +} + +/** + * Returns the first argument unless it's undefined, in which case returns the second one. + */ +export function undefDefault(x: T|undefined, y: T): T { + return (x !== void 0) ? x : y; +} + +/** + * Parses json and returns the result, or returns defaultVal if parsing fails. + */ +export function safeJsonParse(json: string, defaultVal: any): any { + try { + return JSON.parse(json); + } catch (e) { + return defaultVal; + } +} + +/** + * Just like encodeURIComponent, but does not encode slashes. Slashes don't hurt to be included in + * URL parameters, and look much friendlier not encoded. + */ +export function encodeQueryParam(str: string|number|undefined): string { + return encodeURIComponent(String(str === undefined ? null : str)).replace(/%2F/g, '/'); +} + +/** + * Encode an object into a querystring ("key=value&key2=value2"). + * This is similar to JQuery's $.param, but only works on shallow objects. + */ +export function encodeQueryParams(obj: {[key: string]: string|number|undefined}): string { + return Object.keys(obj).map((k: string) => encodeQueryParam(k) + '=' + encodeQueryParam(obj[k])).join('&'); +} + +/** + * Return a list of the words in the string, using the given separator string. At most + * maxNumSplits splits are done, so the result will have at most maxNumSplits + 1 elements (this + * is the main difference from how JS built-in string.split() works, and similar to Python split). + * @param {String} str: String to split. + * @param {String} sep: Separator to split on. + * @param {Number} maxNumSplits: Maximum number of splits to do. + * @return {Array[String]} Array of words, of length at most maxNumSplits + 1. + */ +export function maxsplit(str: string, sep: string, maxNumSplits: number): string[] { + const result: string[] = []; + let start = 0, pos; + for (let i = 0; i < maxNumSplits; i++) { + pos = str.indexOf(sep, start); + if (pos === -1) { + break; + } + result.push(str.slice(start, pos)); + start = pos + sep.length; + } + result.push(str.slice(start)); + return result; +} + + +// Compare arrays of scalars for equality. +export function arraysEqual(a: any[], b: any[]): boolean { + if (a === b) { + return true; + } + if (!a || !b) { + return false; + } + if (a.length !== b.length) { + return false; + } + + for (let i = 0; i < a.length; i++) { + if (a[i] !== b[i]) { return false; } + } + return true; +} + + +// Gives a set representing the set difference a - b. +export function setDifference(a: Set, b: Set): Set { + const c = new Set(); + for (const ai of a) { + if (!b.has(ai)) { c.add(ai); } + } + return c; +} + +// Like array.indexOf, but works with array-like objects like HTMLCollection. +export function indexOf(arrayLike: ArrayLike, item: T): number { + return Array.prototype.indexOf.call(arrayLike, item); +} + + +/** + * Removes a value from the given array. Only the first instance is removed. + * Returns true on success, false if the value was not found. + */ +export function arrayRemove(array: T[], value: T): boolean { + const index = array.indexOf(value); + if (index === -1) { + return false; + } + array.splice(index, 1); + return true; +} + + +/** + * Inserts value into the array before nextValue, or at the end if nextValue is not found. + */ +export function arrayInsertBefore(array: T[], value: T, nextValue: T): void { + const index = array.indexOf(nextValue); + if (index === -1) { + array.push(value); + } else { + array.splice(index, 0, value); + } +} + + +/** + * Extends the first array with the second. Like native push, but adds all values in anotherArray. + */ +export function arrayExtend(array: T[], anotherArray: T[]): void { + for (let i = 0, len = anotherArray.length; i < len; i++) { + array.push(anotherArray[i]); + } +} + + +/** + * Copies count items from fromArray to toArray, copying in a forward direction (which matters + * when the arrays are the same and source and destination indices overlap). + * + * See test/common/arraySplice.js for alternative implementations with timings, from which this + * one is chosen as consistently among the faster ones. + */ +export function arrayCopyForward(toArray: T[], toStart: number, + fromArray: ArrayLike, fromStart: number, count: number): void { + const end = toStart + count; + for (const xend = end - 7; toStart < xend; fromStart += 8, toStart += 8) { + toArray[toStart] = fromArray[fromStart]; + toArray[toStart + 1] = fromArray[fromStart + 1]; + toArray[toStart + 2] = fromArray[fromStart + 2]; + toArray[toStart + 3] = fromArray[fromStart + 3]; + toArray[toStart + 4] = fromArray[fromStart + 4]; + toArray[toStart + 5] = fromArray[fromStart + 5]; + toArray[toStart + 6] = fromArray[fromStart + 6]; + toArray[toStart + 7] = fromArray[fromStart + 7]; + } + for (; toStart < end; ++fromStart, ++toStart) { + toArray[toStart] = fromArray[fromStart]; + } +} + + +/** + * Copies count items from fromArray to toArray, copying in a backward direction (which matters + * when the arrays are the same and source and destination indices overlap). + * + * See test/common/arraySplice.js for alternative implementations with timings, from which this + * one is chosen as consistently among the faster ones. + */ +export function arrayCopyBackward(toArray: T[], toStart: number, + fromArray: ArrayLike, fromStart: number, count: number): void { + let i = toStart + count - 1, j = fromStart + count - 1; + for (const xStart = toStart + 7; i >= xStart; i -= 8, j -= 8) { + toArray[i] = fromArray[j]; + toArray[i - 1] = fromArray[j - 1]; + toArray[i - 2] = fromArray[j - 2]; + toArray[i - 3] = fromArray[j - 3]; + toArray[i - 4] = fromArray[j - 4]; + toArray[i - 5] = fromArray[j - 5]; + toArray[i - 6] = fromArray[j - 6]; + toArray[i - 7] = fromArray[j - 7]; + } + for ( ; i >= toStart; --i, --j) { + toArray[i] = fromArray[j]; + } +} + + +/** + * Appends a slice of fromArray to the end of toArray. + * + * See test/common/arraySplice.js for alternative implementations with timings, from which this + * one is chosen as consistently among the faster ones. + */ +export function arrayAppend(toArray: T[], fromArray: ArrayLike, fromStart: number, count: number): void { + if (count === 1) { + toArray.push(fromArray[fromStart]); + } else { + const len = toArray.length; + toArray.length = len + count; + arrayCopyForward(toArray, len, fromArray, fromStart, count); + } +} + + +/** + * Splices array arrToInsert into target starting at the given start index. + * This implementation tries to be smart by avoiding allocations, appending to the array + * contiguously, then filling in the gap. + * + * See test/common/arraySplice.js for alternative implementations with timings, from which this + * one is chosen as consistently among the faster ones. + */ +export function arraySplice(target: T[], start: number, arrToInsert: ArrayLike): T[] { + const origLen = target.length; + const tailLen = origLen - start; + const insLen = arrToInsert.length; + target.length = origLen + insLen; + if (insLen > tailLen) { + arrayCopyForward(target, origLen, arrToInsert, tailLen, insLen - tailLen); + arrayCopyForward(target, start + insLen, target, start, tailLen); + arrayCopyForward(target, start, arrToInsert, 0, tailLen); + } else { + arrayCopyForward(target, origLen, target, origLen - insLen, insLen); + arrayCopyBackward(target, start + insLen, target, start, tailLen - insLen); + arrayCopyForward(target, start, arrToInsert, 0, insLen); + } + return target; +} + + +/** + * Returns a new array of length count, filled with the given value. + */ +export function arrayRepeat(count: number, value: T): T[] { + return times(count, constant(value)); +} + +// Type for a compare func that returns a positive, negative, or zero value, as used for sorting. +export type CompareFunc = (a: T, b: T) => number; + +/** + * Returns the index at which the given element can be inserted to keep the array sorted. + * This is equivalent to underscore's sortedIndex and python's bisect_left. + * @param {Array} array - sorted array of elements based on the given compareFunc + * @param {object} elem - object to be inserted in the given array + * @param {function} compareFunc - compares 2 elements. Returns a pos value if the 1st element is + * larger, 0 if they're equal, a neg value if the 2nd is larger. + */ +export function sortedIndex(array: ArrayLike, elem: T, compareFunc: CompareFunc): number { + let lo = 0, mid; + let hi = array.length; + + if (array.length === 0) { return 0; } + while (lo < hi) { + mid = Math.floor((lo + hi) / 2); + if (compareFunc(array[mid], elem) < 0) { // mid < elem + lo = mid + 1; + } else { + hi = mid; + } + } + return lo; +} + +/** + * Returns true if an array contains duplicate values. + * Values are considered equal if their toString() representations are equal. + */ +export function hasDuplicates(array: any[]): boolean { + const prevVals = Object.create(null); + for (const value of array) { + if (value in prevVals) { + return true; + } + prevVals[value] = true; + } + return false; +} + +/** + * Counts the number of items in array which satisfy the callback. + */ +export function countIf(array: ReadonlyArray, callback: (item: T) => boolean): number { + let count = 0; + array.forEach(item => { + if (callback(item)) { count++; } + }); + return count; +} + + +/** + * For two parallel arrays, calls mapFunc(a[i], b[i]) for each pair of corresponding elements, and + * returns an array of the results. + */ +export function map2(array1: ArrayLike, array2: ArrayLike, mapFunc: (a: T, b: U) => V): V[] { + const len = array1.length; + const result: V[] = new Array(len); + for (let i = 0; i < len; i++) { + result[i] = mapFunc(array1[i], array2[i]); + } + return result; +} + +/** + * Takes a 2d array returns a new matrix with r rows and c columns + * @param [Array] dataMatrix: a 2d array + * @param [Number] r: final row length + * @param [Number] c: final column length + */ +export function growMatrix(dataMatrix: T[][], r: number, c: number): T[][] { + const colArr = dataMatrix.map(colVals => + Array.from({length: c}, (_v, k) => colVals[k % colVals.length]) + ); + return Array.from({length: r}, (_v, k) => colArr[k % colArr.length]); +} + +/** + * Returns a function that compares two elements based on multiple sort keys and the + * given compare functions. + * Elements are compared using the sort key functions with index 0 having the greatest priority. + * Subsequent sort key functions are used as tie breakers. + * @param {function Array} sortKeyFuncs - a list of sort key functions. + * @param {function Array} compareKeyFuncs - a list of comparison functions parallel to sortKeyFuncs + * Each compare function must satisfy the comparison invariant: + * If compare(a, b) > 0 then a > b, + * If compare(a, b) < 0 then a < b, + * If compare(a, b) == 0 then a == b, + * @param {Array of 1/-1's} optAscending - Comparison on sortKeyFuncs[i] is inverted if optAscending[i] == -1 + */ +export function multiCompareFunc(sortKeyFuncs: ReadonlyArray<(a: T) => U>, + compareFuncs: ArrayLike>, + optAscending?: number[]): CompareFunc { + if (sortKeyFuncs.length !== compareFuncs.length) { + throw new Error('Number of sort key funcs must be the same as the number of compare funcs'); + } + const ascending = optAscending || sortKeyFuncs.map(() => 1); + return function(a: T, b: T): number { + let compareOutcome, keyA, keyB; + for (let i = 0; i < compareFuncs.length; i++) { + keyA = sortKeyFuncs[i](a); + keyB = sortKeyFuncs[i](b); + compareOutcome = compareFuncs[i](keyA, keyB); + if (compareOutcome !== 0) { return ascending[i] * compareOutcome; } + } + return 0; + }; +} + + +export function nativeCompare(a: T, b: T): number { + return (a < b ? -1 : (a > b ? 1 : 0)); +} + +/** + * A copy of python`s `setdefault` function. + * Sets key in mapInst to value, if key is not already set. + * @param {Map} mapInst: Instance of Map. + * @param {Object} key: Key into the map. + * @param {Object} value: Value to insert, possibly. + */ +export function setDefault(mapInst: Map, key: K, val: V): V { + if (!mapInst.has(key)) { mapInst.set(key, val); } + return mapInst.get(key)!; +} + + +/** + * Similar to Python's `setdefault`: returns the key `key` from `mapInst`, or if it's not there, sets + * it to the result buildValue(). + */ +export function getSetMapValue(mapInst: Map, key: K, buildValue: () => V): V { + if (!mapInst.has(key)) { mapInst.set(key, buildValue()); } + return mapInst.get(key)!; +} + + +/** + * If key is in mapInst, remove it and return its value, else return `undefined`. + * @param {Map} mapInst: Instance of Map. + * @param {Object} key: Key into the map to remove. + */ +export function popFromMap(mapInst: Map, key: K): V|undefined { + const value = mapInst.get(key); + mapInst.delete(key); + return value; +} + +/** + * For each encountered value in `values`, increment the corresponding counter in `valueCounts`. + */ +export function addCountsToMap(valueCounts: Map, values: Iterable, + mapFunc: (v: any) => any = identity) { + for (const v of values) { + const mappedValue = mapFunc(v); + valueCounts.set(mappedValue, (valueCounts.get(mappedValue) || 0) + 1); + } +} + +/** + * Returns whether one Set is a subset of another. + */ +export function isSubset(smaller: Set, larger: Set): boolean { + for (const value of smaller) { + if (!larger.has(value)) { + return false; + } + } + return true; +} + + +/** + * Merges the contents of two or more objects together into the first object, recursing into + * nested objects and arrays (like jquery.extend(true, ...)). + * @param {Object} target - The object to modify. Use {} to create a new merged object. + * @param {Object} ... - Additional objects from which to copy properties into target. + * @returns {Object} The first argument, target, modified. + */ +export function deepExtend(target: any, _varArgObjects: any): any { + for (let i = 1; i < arguments.length; i++) { + const object = arguments[i]; + // Extend the base object + for (const name in object) { + if (!object.hasOwnProperty(name)) { continue; } + let src = object[name]; + if (src === target || src === undefined) { + // Prevent one kind of infinite loop, as JQuery's extend does, and skip undefined values. + continue; + } + + if (src) { + // Recurse if we're merging plain objects or arrays + const tgt = target[name]; + if (Array.isArray(src)) { + src = deepExtend(tgt && Array.isArray(tgt) ? tgt : [], src); + } else if (typeof src === 'object') { + src = deepExtend(tgt && typeof tgt === 'object' ? tgt : {}, src); + } + } + target[name] = src; + } + } + // Return the modified object + return target; +} + + +/** + * Returns a human-readable string containing a number of bytes, KB, or MB. + * @param {Number} bytes. Number of bytes. + * @returns {String} A description such as "4.1KB". + */ +export function byteString(bytes: number): string { + if (bytes < 1024) { + return bytes + 'B'; + } else if (bytes < 1024 * 1024) { + return (bytes / 1024).toFixed(1) + 'KB'; + } else { + return (bytes / 1024 / 1024).toFixed(1) + 'MB'; + } +} + + +/** + * Creates a new object mapping each key in keysArray to the value returned by callback. + * @param {Array} keysArray - Array of strings to use as the properties of the returned object. + * @param {Function} callback - Function that produces the value for each key. Called in the same + * way as array.map() calls its callbacks. + * @param {Object} optThisArg - Value to use as `this` when executing callback. + * @returns {Object} - object mapping keys from `keysArray` to values returned by `callback`. + */ +export function mapToObject(keysArray: string[], callback: (key: string) => T, + optThisArg: any): {[key: string]: T} { + const values: T[] = keysArray.map(callback, optThisArg); + const map: {[key: string]: T} = {}; + for (let i = 0; i < keysArray.length; i++) { + map[keysArray[i]] = values[i]; + } + return map; +} + +/** + * A List of python identifiers; the result of running keywords.kwlist in Python 2.7.6, + * plus additional illegal identifiers None, False, True + * Using [] instead of new Array causes a "comprehension error" for some reason + */ +const _kwlist = ['and', 'as', 'assert', 'break', 'class', 'continue', 'def', + 'del', 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from', 'global', + 'if', 'import', 'in', 'is', 'lambda', 'not', 'or', 'pass', 'print', 'raise', + 'return', 'try', 'while', 'with', 'yield', 'None', 'False', 'True']; +/** + * Given an arbitrary string, makes substitutions to make it a valid SQL/Python identifier. + * Corresponds to sandbox/grist/gencode.sanitize_ident + */ +export function sanitizeIdent(ident: string, prefix?: string) { + prefix = prefix || 'c'; + // Remove non-alphanumeric non-_ chars + ident = ident.replace(/[^a-zA-Z0-9_]+/g, '_'); + // Remove leading and trailing _ + ident = ident.replace(/^_+|_+$/g, ''); + // Place prefix at front if the beginning isn't a number + ident = ident.replace(/^(?=[0-9])/g, prefix); + // Append prefix until it is not python keyword + while (_kwlist.includes(ident)) { + ident = prefix + ident; + } + return ident; +} + + +/** + * Clone a function, returning a function object that represents a brand new function with the + * same code. If the same function is used with different argument types, it would prevent JS V8 + * engine optimizations (or cause it to deoptimize it). If different clones are called with + * different argument types, they can be optimized independently. + * + * As with all micro-optimizations, only do this when the optimization matters. + */ +export function cloneFunc(fn: Function): Function { // tslint:disable-line:ban-types + /* jshint evil:true */ // suppress eval warning. + return eval('(' + fn.toString() + ')'); // tslint:disable-line:no-eval +} + + +/** + * Generates a random id using a sequence of uppercase alphanumeric characters + * preceeded by an optional prefix. + */ +export function genRandomId(len: number, optPrefix?: string): string { + const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'; + let ret = optPrefix || ''; + for (let i = 0; i < len; i++) { + ret += chars[Math.floor(Math.random() * chars.length)]; + } + return ret; +} + +/** + * Scans through two sorted arrays, calling a function on each item or pair of items + * for every present key in order. + * @param {Array} arrA - First array to scan. NOTE: Should be sorted by the key value. + * @param {Array} arrB - Second array to scan. NOTE: Should be sorted by the key value. + * @param {Function} callback - Called with an item from arrA as the first argument and an + * item from arrB as the second. Called for every unique key in order, either with one of the + * arguments null if the key is present only in one array, or both non-null if the key is + * present in both arrays. NOTE: Key values should not be null. + * @param {Function} optKeyFunc - Optional function to map each array value to a sort key. + * Defaults to the identity function. + */ +export function sortedScan(arrA: ArrayLike, arrB: ArrayLike, + callback: (a: T|null, B: U|null) => void, + optKeyFunc?: (item: T|U) => any) { + const keyFunc = optKeyFunc || identity; + let i = 0, j = 0; + while (i < arrA.length || j < arrB.length) { + const a = arrA[i], b = arrB[j]; + const keyA = i < arrA.length ? keyFunc(a) : null; + const keyB = j < arrB.length ? keyFunc(b) : null; + if (keyA !== null && (keyB === null || keyA < keyB)) { + callback(a, null); + i++; + } else if (keyA === null || keyA > keyB) { + callback(null, b); + j++; + } else { + callback(a, b); + i++; + j++; + } + } +} + +/** + * Returns the time in ms to wait until attempting another connection. + * @param {Number} attemptNumber - Reconnect attempt number starting at 0. + * @param {Array} intervals - Array of reconnect intervals in ms. + * @returns {Number} + */ +export function getReconnectTimeout(attemptNumber: number, intervals: ArrayLike): number { + if (attemptNumber >= intervals.length) { + // Add an additional wait time if already at max attempts. + const timeout = intervals[intervals.length - 1]; + return timeout + Math.random() * timeout; + } else { + return intervals[attemptNumber]; + } +} + +/** + * Returns whether the given email is a valid formatted email string. + * @param {String} email - Email to test. + * @returns {Boolean} + */ +export function isEmail(email: string): boolean { + return EMAIL_RE.test(email.toLowerCase()); +} + +/* + * Takes an observable and returns a promise for when the observable's value matches the given + * predicate. It then unsubscribes from the observable, and returns its value. + * If a predicate is not given, resolves to the observable values as soon as it's truthy. + */ +export function waitObs(observable: KoObservable, predicate: (value: T) => boolean = Boolean): Promise { + return new Promise((resolve, _reject) => { + const value = observable.peek(); + if (predicate(value)) { return resolve(value); } + const sub = observable.subscribe((val: T) => { + if (predicate(val)) { + sub.dispose(); + resolve(val); + } + }); + }); +} + +/** + * Same as waitObs but for grainjs observables. + */ +export async function waitGrainObs(observable: Observable): Promise>; +export async function waitGrainObs(observable: Observable, predicate?: (value: T) => boolean): Promise; +export async function waitGrainObs(observable: Observable, + predicate: (value: T) => boolean = Boolean): Promise { + let sub: Listener|undefined; + const res: T = await new Promise((resolve, _reject) => { + const value = observable.get(); + if (predicate(value)) { return resolve(value); } + sub = observable.addListener((val: T) => { + if (predicate(val)) { + resolve(val); + } + }); + }); + if (sub) { sub.dispose(); } + return res; +} + +/** + * Class to maintain a chain of promise-returning callbacks. All scheduled callbacks will be + * called in order as long as the previous one is successful. If a callback fails is rejected, + * already-scheduled callbacks will be skipped, but newly-scheduled ones will be run. + */ +export class PromiseChain { + private _last: Promise = Promise.resolve(); + + // Adds a callback to the chain. If the callback runs, the return value is the return value of + // the callback. If it's skipped due to a failure earlier in the chain, the return value is the + // rejection with the message "Skipped due to an earlier error". + public add(nextCB: () => Promise): Promise { + const next = this._last.catch(() => { throw new Error("Skipped due to an earlier error"); }).then(nextCB); + // If any callback fails, all queued ones will be skipped. Here we reset the chain, so that + // callbacks added later do get run. + next.catch(() => { this._last = Promise.resolve(); }); + this._last = next; + return next; + } +} + +/** + * Indicates if a hex color value, e.g. '#000000', is darker than the given value. + * Darkness is measured from 0..255, where 0 is the darkest and 255 is the lightest. + * + * Taken from: https://stackoverflow.com/questions/12043187/how-to-check-if-hex-color-is-too-black + */ +export function isColorDark(hexColor: string, isDarkBelow: number = 220): boolean { + const c = hexColor.substring(1); // strip # + const rgb = parseInt(c, 16); // convert rrggbb to decimal + // Extract RGB components + const r = (rgb >> 16) & 0xff; // tslint:disable-line:no-bitwise + const g = (rgb >> 8) & 0xff; // tslint:disable-line:no-bitwise + const b = (rgb >> 0) & 0xff; // tslint:disable-line:no-bitwise + + const luma = 0.2126 * r + 0.7152 * g + 0.0722 * b; // per ITU-R BT.709 + return luma < isDarkBelow; +} + + +/** + * Returns a promise that resolves to true if promise takes longer than timeoutMsec to resolve. If not + * or if promise throws returns false. + */ +export async function isLongerThan(promise: Promise, timeoutMsec: number): Promise { + let isPending = true; + const done = () => {isPending = false; }; + await Promise.race([ + promise.then(done, done), + delay(timeoutMsec) + ]); + return isPending; +} diff --git a/app/common/marshal.ts b/app/common/marshal.ts new file mode 100644 index 00000000..a54f4acd --- /dev/null +++ b/app/common/marshal.ts @@ -0,0 +1,502 @@ +/** + * Module for serializing data in the format of Python 'marshal' module. It's used for + * communicating with the Python-based formula engine running in a Pypy sandbox. It supports + * version 0 of python marshalling format, which is what the Pypy sandbox supports. + * + * Usage: + * Marshalling: + * const marshaller = new Marshaller({version: 2}); + * marshaller.marshal(value); + * marshaller.marshal(value); + * const buf = marshaller.dump(); // Leaves the marshaller empty. + * + * Unmarshalling: + * const unmarshaller = new Unmarshaller(); + * unmarshaller.on('value', function(value) { ... }); + * unmarshaller.push(buffer); + * unmarshaller.push(buffer); + * + * In Python, and in the marshalled format, there is a distinction between strings and unicode + * objects. In JS, there is a good correspondence to Uint8Array objects and strings, respectively. + * Python unicode objects always become JS strings. JS Uint8Arrays always become Python strings. + * + * JS strings become Python unicode objects, but can be marshalled to Python strings with + * 'stringToBuffer' option. Similarly, Python strings become JS Uint8Arrays, but can be + * unmarshalled to JS strings if 'bufferToString' option is set. + */ +import {BigInt} from 'app/common/BigInt'; +import * as MemBuffer from 'app/common/MemBuffer'; +import {EventEmitter} from 'events'; +import * as util from 'util'; + + +export interface MarshalOptions { + stringToBuffer?: boolean; + version?: number; +} + +export interface UnmarshalOptions { + bufferToString?: boolean; +} + +function ord(str: string): number { + return str.charCodeAt(0); +} + +/** + * Type codes used for python marshalling of values. + * See pypy: rpython/translator/sandbox/_marshal.py. + */ +const marshalCodes = { + NULL : ord('0'), + NONE : ord('N'), + FALSE : ord('F'), + TRUE : ord('T'), + STOPITER : ord('S'), + ELLIPSIS : ord('.'), + INT : ord('i'), + INT64 : ord('I'), + /* + BFLOAT, for 'binary float', is an encoding of float that just encodes the bytes of the + double in standard IEEE 754 float64 format. It is used by Version 2+ of Python's marshal + module. Previously (in versions 0 and 1), the FLOAT encoding is used, which stores floats + through their string representations. + + Version 0 (FLOAT) is mandatory for system calls within the sandbox, while Version 2 (BFLOAT) + is recommended for Grist's communication because it is more efficient and faster to + encode/decode + */ + BFLOAT : ord('g'), + FLOAT : ord('f'), + COMPLEX : ord('x'), + LONG : ord('l'), + STRING : ord('s'), + INTERNED : ord('t'), + STRINGREF: ord('R'), + TUPLE : ord('('), + LIST : ord('['), + DICT : ord('{'), + CODE : ord('c'), + UNICODE : ord('u'), + UNKNOWN : ord('?'), + SET : ord('<'), + FROZENSET: ord('>'), +}; + +type MarshalCode = keyof typeof marshalCodes; + +// A little hack to test if the value is a 32-bit integer. Actually, for Python, int might be up +// to 64 bits (if that's the native size), but this is simpler. +// See http://stackoverflow.com/questions/3885817/how-to-check-if-a-number-is-float-or-integer. +function isInteger(n: number): boolean { + // Float have +0.0 and -0.0. To represent -0.0 precisely, we have to use a float, not an int + // (see also https://stackoverflow.com/questions/7223359/are-0-and-0-the-same). + // tslint:disable-next-line:no-bitwise + return n === +n && n === (n | 0) && !Object.is(n, -0.0); +} + +// ---------------------------------------------------------------------- + +/** + * To force a value to be serialized using a particular representation (e.g. a number as INT64), + * wrap it into marshal.wrap('INT64', value) and serialize that. + */ +export function wrap(codeStr: MarshalCode, value: unknown) { + return new WrappedObj(marshalCodes[codeStr], value); +} + +export class WrappedObj { + constructor(public code: number, public value: unknown) {} + + public inspect() { + return util.inspect(this.value); + } +} + +// ---------------------------------------------------------------------- + +/** + * @param {Boolean} options.stringToBuffer - If set, JS strings will become Python strings rather + * than unicode objects (as if each JS string is wrapped into MemBuffer.stringToArray(str)). + * This flag becomes a same-named property of Marshaller, which can be set at any time. + * @param {Number} options.version - If version >= 2, uses binary representation for floats. The + * default version 0 formats floats as strings. + * + * TODO: The default should be version 2. (0 was used historically because it was needed for + * communication with PyPy-based sandbox.) + */ +export class Marshaller { + private memBuf: MemBuffer; + private readonly floatCode: number; + private readonly stringCode: number; + + constructor(options?: MarshalOptions) { + this.memBuf = new MemBuffer(undefined); + this.floatCode = options && options.version && options.version >= 2 ? marshalCodes.BFLOAT : marshalCodes.FLOAT; + this.stringCode = options && options.stringToBuffer ? marshalCodes.STRING : marshalCodes.UNICODE; + } + + public dump(): Uint8Array { + // asByteArray returns a view on the underlying data, and the constructor creates a new copy. + // For some usages, we may want to avoid making the copy. + const bytes = new Uint8Array(this.memBuf.asByteArray()); + this.memBuf.clear(); + return bytes; + } + + public dumpAsBuffer(): Buffer { + const bytes = Buffer.from(this.memBuf.asByteArray()); + this.memBuf.clear(); + return bytes; + } + + public getCode(value: any) { + switch (typeof value) { + case 'number': return isInteger(value) ? marshalCodes.INT : this.floatCode; + case 'string': return this.stringCode; + case 'boolean': return value ? marshalCodes.TRUE : marshalCodes.FALSE; + case 'undefined': return marshalCodes.NONE; + case 'object': { + if (value instanceof WrappedObj) { + return value.code; + } else if (value === null) { + return marshalCodes.NONE; + } else if (value instanceof Uint8Array) { + return marshalCodes.STRING; + } else if (Buffer.isBuffer(value)) { + return marshalCodes.STRING; + } else if (Array.isArray(value)) { + return marshalCodes.LIST; + } + return marshalCodes.DICT; + } + default: { + throw new Error("Marshaller: Unsupported value of type " + (typeof value)); + } + } + } + + public marshal(value: any): void { + const code = this.getCode(value); + if (value instanceof WrappedObj) { + value = value.value; + } + this.memBuf.writeUint8(code); + switch (code) { + case marshalCodes.NULL: return; + case marshalCodes.NONE: return; + case marshalCodes.FALSE: return; + case marshalCodes.TRUE: return; + case marshalCodes.INT: return this.memBuf.writeInt32LE(value); + case marshalCodes.INT64: return this._writeInt64(value); + case marshalCodes.FLOAT: return this._writeStringFloat(value); + case marshalCodes.BFLOAT: return this.memBuf.writeFloat64LE(value); + case marshalCodes.STRING: + return (value instanceof Uint8Array || Buffer.isBuffer(value) ? + this._writeByteArray(value) : + this._writeUtf8String(value)); + case marshalCodes.TUPLE: return this._writeList(value); + case marshalCodes.LIST: return this._writeList(value); + case marshalCodes.DICT: return this._writeDict(value); + case marshalCodes.UNICODE: return this._writeUtf8String(value); + // None of the following are supported. + case marshalCodes.STOPITER: + case marshalCodes.ELLIPSIS: + case marshalCodes.COMPLEX: + case marshalCodes.LONG: + case marshalCodes.INTERNED: + case marshalCodes.STRINGREF: + case marshalCodes.CODE: + case marshalCodes.UNKNOWN: + case marshalCodes.SET: + case marshalCodes.FROZENSET: throw new Error("Marshaller: Can't serialize code " + code); + default: throw new Error("Marshaller: Can't serialize code " + code); + } + } + + private _writeInt64(value: number) { + if (!isInteger(value)) { + // TODO We could actually support 53 bits or so. + throw new Error("Marshaller: int64 still only supports 32-bit ints for now: " + value); + } + this.memBuf.writeInt32LE(value); + this.memBuf.writeInt32LE(value >= 0 ? 0 : -1); + } + + private _writeStringFloat(value: number) { + // This could be optimized a bit, but it's only used in V0 marshalling, which is only used in + // sandbox system calls, which don't really ever use floats anyway. + const bytes = MemBuffer.stringToArray(value.toString()); + if (bytes.byteLength >= 127) { + throw new Error("Marshaller: Trying to write a float that takes " + bytes.byteLength + " bytes"); + } + this.memBuf.writeUint8(bytes.byteLength); + this.memBuf.writeByteArray(bytes); + } + + private _writeByteArray(value: Uint8Array|Buffer) { + // This works for both Uint8Arrays and Node Buffers. + this.memBuf.writeInt32LE(value.length); + this.memBuf.writeByteArray(value); + } + + private _writeUtf8String(value: string) { + const offset = this.memBuf.size(); + // We don't know the length until we write the value. + this.memBuf.writeInt32LE(0); + this.memBuf.writeString(value); + const byteLength = this.memBuf.size() - offset - 4; + // Overwrite the 0 length we wrote earlier with the correct byte length. + this.memBuf.asDataView.setInt32(this.memBuf.startPos + offset, byteLength, true); + } + + private _writeList(array: unknown[]) { + this.memBuf.writeInt32LE(array.length); + for (const item of array) { + this.marshal(item); + } + } + + private _writeDict(obj: {[key: string]: any}) { + const keys = Object.keys(obj); + keys.sort(); + for (const key of keys) { + this.marshal(key); + this.marshal(obj[key]); + } + this.memBuf.writeUint8(marshalCodes.NULL); + } +} + +// ---------------------------------------------------------------------- + +const TwoTo32 = 0x100000000; // 2**32 +const TwoTo15 = 0x8000; // 2**15 + +/** + * @param {Boolean} options.bufferToString - If set, Python strings will become JS strings rather + * than Buffers (as if each decoded buffer is wrapped into `buf.toString()`). + * This flag becomes a same-named property of Unmarshaller, which can be set at any time. + * Note that options.version isn't needed, since this will decode both formats. + * TODO: Integers (such as int64 and longs) that are too large for JS are currently represented as + * decimal strings. They may need a better representation, or a configurable option. + */ +export class Unmarshaller extends EventEmitter { + public memBuf: MemBuffer; + private consumer: any = null; + private _lastCode: number|null = null; + private readonly bufferToString: boolean; + private emitter: (v: any) => boolean; + private stringTable: Array = []; + + constructor(options?: UnmarshalOptions) { + super(); + this.memBuf = new MemBuffer(undefined); + this.bufferToString = Boolean(options && options.bufferToString); + this.emitter = this.emit.bind(this, 'value'); + } + + /** + * Adds more data for parsing. Parsed values will be emitted as 'value' events. + * @param {Uint8Array|Buffer} byteArray: Uint8Array or Node Buffer with bytes to parse. + */ + public push(byteArray: Uint8Array|Buffer) { + this.parse(byteArray, this.emitter); + } + + /** + * Adds data to parse, and calls valueCB(value) for each value parsed. If valueCB returns the + * Boolean false, stops parsing and returns. + */ + public parse(byteArray: Uint8Array|Buffer, valueCB: (val: any) => boolean|void) { + this.memBuf.writeByteArray(byteArray); + try { + while (this.memBuf.size() > 0) { + this.consumer = this.memBuf.makeConsumer(); + + // Have to reset stringTable for interned strings before each top-level parse call. + this.stringTable.length = 0; + + const value = this._parse(); + this.memBuf.consume(this.consumer); + if (valueCB(value) === false) { + return; + } + } + } catch (err) { + // If the error is `needMoreData`, we silently return. We'll retry by reparsing the message + // from scratch after the next push(). If buffers contain complete serialized messages, the + // cost should be minor. But this design might get very inefficient if we have big messages + // of arrays or dictionaries. + if (err.needMoreData) { + if (!err.consumedData || err.consumedData > 1024) { + // tslint:disable-next-line:no-console + console.log("Unmarshaller: Need more data; wasted parsing of %d bytes", err.consumedData); + } + } else { + err.message = "Unmarshaller: " + err.message; + throw err; + } + } + } + + private _parse(): unknown { + const code = this.memBuf.readUint8(this.consumer); + this._lastCode = code; + switch (code) { + case marshalCodes.NULL: return null; + case marshalCodes.NONE: return null; + case marshalCodes.FALSE: return false; + case marshalCodes.TRUE: return true; + case marshalCodes.INT: return this._parseInt(); + case marshalCodes.INT64: return this._parseInt64(); + case marshalCodes.FLOAT: return this._parseStringFloat(); + case marshalCodes.BFLOAT: return this._parseBinaryFloat(); + case marshalCodes.STRING: return this._parseByteString(); + case marshalCodes.TUPLE: return this._parseList(); + case marshalCodes.LIST: return this._parseList(); + case marshalCodes.DICT: return this._parseDict(); + case marshalCodes.UNICODE: return this._parseUnicode(); + case marshalCodes.INTERNED: return this._parseInterned(); + case marshalCodes.STRINGREF: return this._parseStringRef(); + case marshalCodes.LONG: return this._parseLong(); + // None of the following are supported. + // case marshalCodes.STOPITER: + // case marshalCodes.ELLIPSIS: + // case marshalCodes.COMPLEX: + // case marshalCodes.CODE: + // case marshalCodes.UNKNOWN: + // case marshalCodes.SET: + // case marshalCodes.FROZENSET: + default: + throw new Error(`Unmarshaller: unsupported code "${String.fromCharCode(code)}" (${code})`); + } + } + + private _parseInt() { + return this.memBuf.readInt32LE(this.consumer); + } + + private _parseInt64() { + const low = this.memBuf.readInt32LE(this.consumer); + const hi = this.memBuf.readInt32LE(this.consumer); + if ((hi === 0 && low >= 0) || (hi === -1 && low < 0)) { + return low; + } + const unsignedLow = low < 0 ? TwoTo32 + low : low; + if (hi >= 0) { + return new BigInt(TwoTo32, [unsignedLow, hi], 1).toNative(); + } else { + // This part is tricky. See unittests for check of correctness. + return new BigInt(TwoTo32, [TwoTo32 - unsignedLow, -hi - 1], -1).toNative(); + } + } + + private _parseLong() { + // The format is a 32-bit size whose sign is the sign of the result, followed by 16-bit digits + // in base 2**15. + const size = this.memBuf.readInt32LE(this.consumer); + const sign = size < 0 ? -1 : 1; + const numDigits = size < 0 ? -size : size; + const digits = []; + for (let i = 0; i < numDigits; i++) { + digits.push(this.memBuf.readInt16LE(this.consumer)); + } + return new BigInt(TwoTo15, digits, sign).toNative(); + } + + private _parseStringFloat() { + const len = this.memBuf.readUint8(this.consumer); + const buf = this.memBuf.readString(this.consumer, len); + return parseFloat(buf); + } + + private _parseBinaryFloat() { + return this.memBuf.readFloat64LE(this.consumer); + } + + private _parseByteString(): string|Uint8Array { + const len = this.memBuf.readInt32LE(this.consumer); + return (this.bufferToString ? + this.memBuf.readString(this.consumer, len) : + this.memBuf.readByteArray(this.consumer, len)); + } + + private _parseInterned() { + const s = this._parseByteString(); + this.stringTable.push(s); + return s; + } + + private _parseStringRef() { + const index = this._parseInt(); + return this.stringTable[index]; + } + + private _parseList() { + const len = this.memBuf.readInt32LE(this.consumer); + const value = []; + for (let i = 0; i < len; i++) { + value[i] = this._parse(); + } + return value; + } + + private _parseDict() { + const dict: {[key: string]: any} = {}; + while (true) { // eslint-disable-line no-constant-condition + let key = this._parse() as string|Uint8Array; + if (key === null && this._lastCode === marshalCodes.NULL) { + break; + } + const value = this._parse(); + if (key !== null) { + if (key instanceof Uint8Array) { + key = MemBuffer.arrayToString(key); + } + dict[key as string] = value; + } + } + return dict; + } + + private _parseUnicode() { + const len = this.memBuf.readInt32LE(this.consumer); + return this.memBuf.readString(this.consumer, len); + } +} + +/** + * Similar to python's marshal.loads(). Parses the given bytes and returns the parsed value. There + * must not be any trailing data beyond the single marshalled value. + */ +export function loads(byteArray: Uint8Array|Buffer, options?: UnmarshalOptions): any { + const unmarshaller = new Unmarshaller(options); + let parsedValue; + unmarshaller.parse(byteArray, function(value) { + parsedValue = value; + return false; + }); + if (typeof parsedValue === 'undefined') { + throw new Error("loads: input data truncated"); + } else if (unmarshaller.memBuf.size() > 0) { + throw new Error("loads: extra bytes past end of input"); + } + return parsedValue; +} + +/** + * Serializes arbitrary data by first marshalling then converting to a base64 string. + */ +export function dumpBase64(data: any, options?: MarshalOptions) { + const marshaller = new Marshaller(options || {version: 2}); + marshaller.marshal(data); + return marshaller.dumpAsBuffer().toString('base64'); +} + +/** + * Loads data from a base64 string, as serialized by dumpBase64(). + */ +export function loadBase64(data: string, options?: UnmarshalOptions) { + return loads(Buffer.from(data, 'base64'), options); +} diff --git a/app/common/metricConfig.js b/app/common/metricConfig.js new file mode 100644 index 00000000..417a40c1 --- /dev/null +++ b/app/common/metricConfig.js @@ -0,0 +1,252 @@ +/** + * File for configuring the metric collection bucket duration, data push intervals between client, server, + * and Grist Metrics EC2 instance, as well as individual metrics collected in the client and server. + */ + +// Time interval settings (ms) +exports.BUCKET_SIZE = 60 * 1000; + +exports.CLIENT_PUSH_INTERVAL = 120 * 1000; +exports.SERVER_PUSH_INTERVAL = 120 * 1000; +exports.MAX_PENDING_BUCKETS = 40; +exports.CONN_RETRY = 20 * 1000; + +// Metrics use the general form: +// . +// With prefixes, measurement type, and clientId/serverId added automatically on send. + +// 'type' is the measurement tool type, with options 'Switch', 'Counter', 'Gauge', 'Timer', and +// 'ExecutionTimer'. (See metricTools.js for details) +// Suffixes are added to the metric names depending on their measurement tool. +// 'Switch' => '.instances' +// 'Gauge' => '.total' +// 'Counter' => '.count' +// 'Timer' => '.time' +// 'ExecutionTimer' => '.execution_time', '.count' (Execution timer automatically records a count) + +exports.clientMetrics = [ + // General + { + name: 'sidepane.opens', + type: 'Counter', + desc: 'Number of times the side pane is opened' + }, + { + name: 'app.client_active_span', + type: 'Timer', + desc: 'Total client time spent using grist' + }, + { + name: 'app.connected_to_server_span', + type: 'Timer', + desc: 'Total time spent connected to the server' + }, + { + name: 'app.disconnected_from_server_span', + type: 'Timer', + desc: 'Total time spent disconnected from the server' + }, + // Docs + { + name: 'docs.num_open_6+_tables', + type: 'SamplingGauge', + desc: 'Number of open docs with more than 5 tables' + }, + { + name: 'docs.num_open_0-5_tables', + type: 'SamplingGauge', + desc: 'Number of open docs with 0-5 tables' + }, + // Tables + { + name: 'tables.num_tables', + type: 'SamplingGauge', + desc: 'Number of open tables' + }, + { + name: 'tables.num_summary_tables', + type: 'SamplingGauge', + desc: 'Number of open sections in the current view' + }, + // Views + { + name: 'views.code_view_open_span', + type: 'Timer', + desc: 'Time spent with code viewer open' + }, + // Sections + { + name: 'sections.grid_open_span', + type: 'Timer', + desc: 'Time spent with gridview open' + }, + { + name: 'sections.detail_open_span', + type: 'Timer', + desc: 'Time spent with gridview open' + }, + { + name: 'sections.num_grid_sections', + type: 'SamplingGauge', + desc: 'Number of open sections in the current view' + }, + { + name: 'sections.num_detail_sections', + type: 'SamplingGauge', + desc: 'Number of open sections in the current view' + }, + { + name: 'sections.num_chart_sections', + type: 'SamplingGauge', + desc: 'Number of open sections in the current view' + }, + { + name: 'sections.multiple_open_span', + type: 'Timer', + desc: 'Time spent with multiple sections open' + }, + // Performance + { + name: 'performance.server_action', + type: 'ExecutionTimer', + desc: 'Time for a server action to complete' + }, + { + name: 'performance.doc_load', + type: 'ExecutionTimer', + desc: 'Time to load a document' + }, + // Columns + { + name: 'cols.num_formula_cols', + type: 'SamplingGauge', + desc: 'Number of formula columns in open documents' + }, + { + name: 'cols.num_text_cols', + type: 'SamplingGauge', + desc: 'Number of text columns in open documents' + }, + { + name: 'cols.num_int_cols', + type: 'SamplingGauge', + desc: 'Number of integer columns in open documents' + }, + { + name: 'cols.num_numeric_cols', + type: 'SamplingGauge', + desc: 'Number of numeric columns in open documents' + }, + { + name: 'cols.num_date_cols', + type: 'SamplingGauge', + desc: 'Number of date columns in open documents' + }, + { + name: 'cols.num_datetime_cols', + type: 'SamplingGauge', + desc: 'Number of datetime columns in open documents' + }, + { + name: 'cols.num_ref_cols', + type: 'SamplingGauge', + desc: 'Number of reference columns in open documents' + }, + { + name: 'cols.num_attachments_cols', + type: 'SamplingGauge', + desc: 'Number of attachments columns in open documents' + }, + { + name: 'performance.front_end_errors', + type: 'Counter', + desc: 'Number of frontend errors' + } + // TODO: Implement the following: + // { + // name: 'grist-rt.performance.view_swap', + // type: 'ExecutionTimer', + // desc: 'Time to swap views' + // } +]; + +exports.serverMetrics = [ + // General + { + name: 'app.server_active', + type: 'Switch', + desc: 'Number of users currently using grist' + }, + { + name: 'app.server_active_span', + type: 'Timer', + desc: 'Total server time spent using grist' + }, + { + name: 'app.have_doc_open', + type: 'Switch', + desc: 'Number of users with at least one doc open' + }, + { + name: 'app.doc_open_span', + type: 'Timer', + desc: 'Total time spent with at least one doc open' + }, + // Docs + { + name: 'docs.num_open', + type: 'Gauge', + desc: 'Number of open docs' + }, + { + name: 'performance.node_memory_usage', + type: 'SamplingGauge', + desc: 'Memory utilization in bytes of the node process' + } + // TODO: Implement the following: + // { + // name: 'grist-rt.docs.total_size_open', + // type: 'Gauge', + // desc: 'Cumulative size of open docs' + // } + // { + // name: 'grist-rt.performance.open_standalone_app', + // type: 'ExecutionTimer', + // desc: 'Time to start standalone app' + // } + // { + // name: 'grist-rt.performance.sandbox_recalculation', + // type: 'ExecutionTimer', + // desc: 'Time for sandbox recalculation to occur' + // } + // { + // name: 'grist-rt.performance.open_standalone_app', + // type: 'ExecutionTimer', + // desc: 'Time to start standalone app' + // } + // { + // name: 'grist-rt.performance.node_cpu_usage', + // type: 'SamplingGauge', + // desc: 'Amount of time node was using the cpu in the interval' + // } + // { + // name: 'grist-rt.performance.sandbox_cpu_usage', + // type: 'SamplingGauge', + // desc: 'Amount of time the sandbox was using the cpu in the interval' + // } + // { + // name: 'grist-rt.performance.chrome_cpu_usage', + // type: 'SamplingGauge', + // desc: 'Amount of time chrome was using the cpu in the interval' + // } + // { + // name: 'grist-rt.performance.sandbox_memory_usage', + // type: 'SamplingGauge', + // desc: 'Memory utilization in bytes of the sandbox process' + // } + // { + // name: 'grist-rt.performance.chrome_memory_usage', + // type: 'SamplingGauge', + // desc: 'Memory utilization in bytes of the chrome process' + // } +]; diff --git a/app/common/metricTools.js b/app/common/metricTools.js new file mode 100644 index 00000000..146355c4 --- /dev/null +++ b/app/common/metricTools.js @@ -0,0 +1,261 @@ +const _ = require('underscore'); +const gutil = require('./gutil'); +const metricConfig = require('./metricConfig'); + +// TODO: Create a metric test class and write tests for each metric tool. + +/** + * Base class for tools to gather metrics. Should not be instantiated. + */ +function MetricTool(name) { + this.name = name; +} + +// Should be implemented by extending classes +MetricTool.prototype._getSuffix = function() { + throw new Error("Not implemented"); +}; + +// Should be overridden by extending classes depending on desired reset behavior +MetricTool.prototype.reset = _.noop; + +// Returns the name of the metric with its suffix appended to the end. +// NOTE: Should return names in the same order as getValues. +MetricTool.prototype.getName = function() { + return this.name + '.' + this._getSuffix(); +}; + +// Should be implemented by extending classes. Returns the value of the tool for a bucket. +// @param {Number} bucketEndTime - The desired bucket's end time in milliseconds +MetricTool.prototype.getValue = function(bucketEndTime) { + throw new Error("Not implemented"); +}; + +// Returns a list of all primitive metrics this tool is made up of. +// Only requires overridding by non-primitive metrics. +MetricTool.prototype.getPrimitiveMetrics = function() { + return [this]; +}; + +/** + * Counts the number of times an event has occurred in the current bucket. + */ +function Counter(name) { + MetricTool.call(this, name); + this.val = 0; +} +_.extend(Counter.prototype, MetricTool.prototype); + +Counter.prototype.inc = function() { + this.val += 1; +}; + +Counter.prototype._getSuffix = function() { + return 'count'; +}; + +Counter.prototype.getValue = function(bucketEndTime) { + // If the bucket is more recent than the last one where counting occurred, return 0 + return this.val; +}; + +Counter.prototype.reset = function(bucketEndTime) { + this.val = 0; +}; +exports.Counter = Counter; + + +/** + * Keeps track of a count that persists across buckets. + */ +function Gauge(name) { + MetricTool.call(this, name); + this.val = null; +} +_.extend(Gauge.prototype, MetricTool.prototype); + +Gauge.prototype.set = function(num) { + this.val = num; +}; + +Gauge.prototype.inc = function() { + this.val = (this.val ? this.val + 1 : 1); +}; + +Gauge.prototype.dec = function() { + this.val -= 1; +}; + +Gauge.prototype._getSuffix = function() { + return 'total'; +}; + +Gauge.prototype.getValue = function(bucketEndTime) { + return this.val; +}; +exports.Gauge = Gauge; + + +/** + * A gauge that pulls samples using a callback function + */ +function SamplingGauge(name) { + MetricTool.call(this, name); + this.callback = _.constant(null); +} +_.extend(SamplingGauge.prototype, MetricTool.prototype); + +SamplingGauge.prototype.assignCallback = function(callback) { + this.callback = callback; +}; + +SamplingGauge.prototype._getSuffix = function() { + return 'total'; +}; + +SamplingGauge.prototype.getValue = function(bucketEndTime) { + return this.callback(); +}; +exports.SamplingGauge = SamplingGauge; + + +/** + * Keeps track of whether or not a certain condition is met. Useful for statistics + * which measure the number of users who meet a certain criteria. Persists across buckets. + */ +function Switch(name) { + MetricTool.call(this, name); + this.val = null; +} +_.extend(Switch.prototype, Gauge.prototype); + +Switch.prototype.set = function(bool) { + this.val = bool ? 1 : 0; +}; + +Switch.prototype._getSuffix = function() { + return 'instances'; +}; + +Switch.prototype.getValue = function(bucketEndTime) { + return this.val; +}; +exports.Switch = Switch; + + +/** + * Keeps track of the amount of time in each bucket that an event is occurring (ms). + */ +function Timer(name) { + MetricTool.call(this, name); + this.val = 0; // The sum of all runtimes in the last updated bucket + this.startTime = 0; // The time (in ms since the bucket started) when the timer was started + this.running = false; +} +_.extend(Timer.prototype, MetricTool.prototype); + +Timer.prototype.setRunning = function(bool) { + return bool ? this.start() : this.stop(); +}; + +Timer.prototype.start = function() { + if (this.running) { + return; + } + // Record start time and set to running + this.startTime = Date.now(); + this.running = true; +}; + +Timer.prototype.stop = function() { + if (!this.running) { + return; + } + // Add time since start to value and set running to false + var stopTime = Date.now(); + this.val += stopTime - this.startTime; + this.running = false; +}; + +Timer.prototype._getSuffix = function() { + return 'time'; +}; + +Timer.prototype.getValue = function(bucketEndTime) { + // Add the value and the time to the end of the bucket if the timer is running + return this.val + (this.running ? Math.max(0, bucketEndTime - this.startTime) : 0); +}; + +Timer.prototype.reset = function(bucketEndTime) { + this.val = 0; + this.startTime = Math.max(bucketEndTime, this.startTime); +}; +exports.Timer = Timer; + + +/** + * Keeps track of the amount of time in an event takes, and the number of times that event occurs (ms). + */ +function ExecutionTimer(name) { + MetricTool.call(this, name); + this.startTime = 0; // The last time (in ms) the timer was started + this.val = 0; + this.running = false; + // Counter keeps track of the total number of executions in the current bucket. + // An execution is in a bucket if it ended in that bucket. + this.counter = new Counter(name); +} +_.extend(ExecutionTimer.prototype, MetricTool.prototype); + +ExecutionTimer.prototype.setRunning = function(bool) { + return bool ? this.start() : this.stop(); +}; + +ExecutionTimer.prototype.start = function() { + if (this.running) { + return; + } + this.startTime = Date.now(); + this.running = true; +}; + +ExecutionTimer.prototype.stop = function() { + if (!this.running) { + return; + } + var stopTime = Date.now(); + this.val += stopTime - this.startTime; + this.counter.inc(); + this.running = false; +}; + +ExecutionTimer.prototype._getSuffix = function() { + return 'execution_time'; +}; + +ExecutionTimer.prototype.getValue = function(bucketEndTime) { + return this.val; +}; + +ExecutionTimer.prototype.reset = function(bucketEndTime) { + this.val = 0; + this.counter.reset(); +}; + +ExecutionTimer.prototype.getPrimitiveMetrics = function() { + return [this, this.counter]; +}; +exports.ExecutionTimer = ExecutionTimer; + + +// Returns the time rounded down to the start of the current bucket's time window (in ms). +function getBucketStartTime(now) { + return gutil.roundDownToMultiple(now, metricConfig.BUCKET_SIZE); +} +exports.getBucketStartTime = getBucketStartTime; + +// Returns the time until the start of the next bucket (in ms). +function getDeltaMs(now) { + return getBucketStartTime(now) + metricConfig.BUCKET_SIZE - now; +} +exports.getDeltaMs = getDeltaMs; diff --git a/app/common/orgNameUtils.ts b/app/common/orgNameUtils.ts new file mode 100644 index 00000000..956c9fef --- /dev/null +++ b/app/common/orgNameUtils.ts @@ -0,0 +1,56 @@ +const BLACKLISTED_SUBDOMAINS = new Set([ + // from wiki page as of 2018-12-14 + 'aws', + 'gristlogin', + 'issues', + 'metrics', + 'phab', + 'releases', + 'test', + 'vpn', + 'www', + + // A few more reserved just in case. The minimum length requirement would eliminate + // some in any case, but specified here also in case that minimum changes. + 'w', 'ww', 'wwww', 'wwwww', + 'docs', 'api', 'static', + 'ftp', 'imap', 'pop', 'smtp', 'mail', 'git', 'blog', 'wiki', 'support', 'kb', 'help', + 'admin', 'store', 'dev', 'beta', 'dev', + + // a few random tech brands + 'google', 'apple', 'microsoft', 'ms', 'facebook', 'fb', 'twitter', 'youtube', 'yt', + + // updates for new special domains + 'current', 'staging', 'prod', 'login', 'login-dev', +]); + +/** + * + * Checks whether the subdomain is on the list of forbidden subdomains. + * See https://phab.getgrist.com/w/hosting/v1/urls/#organization-subdomains + * + * Also enforces various sanity checks. + * + * Throws if the subdomain is invalid. + * + */ +export function checkSubdomainValidity(subdomain: string): void { + // stick with limited alphanumeric subdomains. + if (!(/^[a-z][-a-z0-9]*$/.test(subdomain))) { + throw new Error('Domain must include letters, numbers, and dashes only.'); + } + // 'docs-*' is reserved for personal orgs. + if (subdomain.startsWith('docs-')) { throw new Error('Domain cannot use reserved prefix "docs-".'); } + // 'doc-worker-*' is reserved for doc workers. + if (subdomain.startsWith('doc-worker-')) { throw new Error('Domain cannot use reserved prefix "doc-worker-".'); } + // special subdomains like _domainkey. + if (subdomain.startsWith('_')) { throw new Error('Domain cannot use reserved prefix "_".'); } + // some domains are currently in use for testing v1. + if (subdomain.startsWith('v1-')) { throw new Error('Domain cannot use reserved prefix "v1-".'); } + // check limit of 63 characters on dns label. + if (subdomain.length > 63) { throw new Error('Domain must contain less than 64 characters.'); } + // check the subdomain isn't too short. + if (subdomain.length <= 2) { throw new Error('Domain must contain more than 2 characters.'); } + // a small blacklist prepared by hand. + if (BLACKLISTED_SUBDOMAINS.has(subdomain)) { throw new Error('Invalid domain value.'); } +} diff --git a/app/common/parseDate.ts b/app/common/parseDate.ts new file mode 100644 index 00000000..6bc4f276 --- /dev/null +++ b/app/common/parseDate.ts @@ -0,0 +1,107 @@ +import * as moment from 'moment-timezone'; + +// Order of formats to try if the date cannot be parsed as the currently set format. +// Formats are parsed in momentjs strict mode, but separator matching and the MM/DD +// two digit requirement are ignored. Also, partial completion is permitted, so formats +// may match even if only beginning elements are provided. +// TODO: These should be affected by the user's locale/settings. +// TODO: We may want to consider adding default time formats as well to support more +// time formats. +const PARSER_FORMATS: string[] = [ + 'M D YYYY', + 'M D YY', + 'M D', + 'M', + 'MMMM D YYYY', + 'MMMM D', + 'MMMM Do YYYY', + 'MMMM Do', + 'MMMM', + 'MMM D YYYY', + 'MMM D', + 'D MMM YYYY', + 'D MMM', + 'MMM', + 'YYYY M D', + 'YYYY M', + 'YYYY', + 'D M YYYY', + 'D M YY', + 'D M', + 'D' +]; + +interface ParseOptions { + time?: string; + dateFormat?: string; + timeFormat?: string; + timezone?: string; +} + +/** + * parseDate - Attempts to parse a date string using several common formats. Returns the + * timestamp of the parsed date in seconds since epoch, or returns null on failure. + * @param {String} date - The date string to parse. + * @param {String} options.dateFormat - The preferred momentjs format to use to parse the + * date. This is attempted before the default formats. + * @param {String} options.time - The time string to parse. + * @param {String} options.timeFormat - The momentjs format to use to parse the time. This + * must be given if options.time is given. + * @param {String} options.timezone - The timezone string for the date/time, which affects + * the resulting timestamp. + */ +export function parseDate(date: string, options: ParseOptions = {}): number | null { + // If no date, return null. + if (!date) { + return null; + } + // Not picky about separators, so replace them in the date and format strings to be spaces. + const separators = /\W+/g; + const dateFormats = PARSER_FORMATS.slice(); + // If a preferred parse format is given, set that to be the first parser used. + if (options.dateFormat) { + // Momentjs has an undesirable feature in strict mode where MM and DD + // matches require two digit numbers. Change MM, DD to M, D. + const format = options.dateFormat.replace(/\bMM\b/g, 'M') + .replace(/\bDD\b/g, 'D') + .replace(separators, ' '); + dateFormats.unshift(_getPartialFormat(date, format)); + } + const cleanDate = date.replace(separators, ' '); + const datetime = (options.time ? `${cleanDate} ${options.time}` : cleanDate).trim(); + for (const f of dateFormats) { + // Momentjs has an undesirable feature in strict mode where HH, mm, and ss + // matches require two digit numbers. Change HH, mm, and ss to H, m, and s. + const timeFormat = options.timeFormat ? options.timeFormat.replace(/\bHH\b/g, 'H') + .replace(/\bmm\b/g, 'm') + .replace(/\bss\b/g, 's') : null; + const fullFormat = options.time && timeFormat ? `${f} ${timeFormat}` : f; + const m = moment.tz(datetime, fullFormat, true, options.timezone || 'UTC'); + if (m.isValid()) { + return m.valueOf() / 1000; + } + } + return null; +} + +// Helper function to get the partial format string based on the input. Momentjs has a feature +// which allows defaulting to the current year, month and/or day if not accounted for in the +// parser. We remove any parts of the parser not given in the input to take advantage of this +// feature. +function _getPartialFormat(input: string, format: string): string { + // Define a regular expression to match contiguous separators. + const re = /\W+/g; + // Clean off any whitespace from the ends, and count the number of separators. + const inputMatch = input.trim().match(re); + const numInputSeps = inputMatch ? inputMatch.length : 0; + // Find the separator matches in the format string. + let formatMatch; + for (let i = 0; i < numInputSeps + 1; i++) { + formatMatch = re.exec(format); + if (!formatMatch) { + break; + } + } + // Get the format string up until the corresponding input ends. + return formatMatch ? format.slice(0, formatMatch.index) : format; +} diff --git a/app/common/plugin.ts b/app/common/plugin.ts new file mode 100644 index 00000000..1787f0f9 --- /dev/null +++ b/app/common/plugin.ts @@ -0,0 +1,77 @@ +/** + * Plugin's utilities common to server and client. + */ +import {BarePlugin, Implementation} from 'app/plugin/PluginManifest'; + +export type LocalPluginKind = "installed"|"builtIn"; + +export interface ImplDescription { + localPluginId: string; + implementation: Implementation; +} + +export interface FileParser { + fileExtensions: string[]; + parseOptions?: ImplDescription; + fileParser: ImplDescription; +} + +// Deprecated, use FileParser or ImportSource instead. +export interface FileImporter { + id: string; + fileExtensions?: string[]; + script?: string; + scriptFullPath?: string; + filePicker?: string; + filePickerFullPath?: string; +} + +/** + * Manifest parsing error. + */ +export interface ManifestParsingError { + yamlError?: any; + jsonError?: any; + cannotReadError?: any; + missingEntryErrors?: string; +} + +/** + * Whether the importer provides a file picker. + */ +export function isPicker(importer: FileImporter): boolean { + return importer.filePicker !== undefined; +} + +/** + * A Plugin that was found in the system, either installed or builtin. + */ +export interface LocalPlugin { + /** + * the plugin's manifest + */ + manifest: BarePlugin; + /** + * The path to the plugin's folder. + */ + path: string; + /** + * A name to uniquely identify a LocalPlugin. + */ + readonly id: string; +} + +export interface DirectoryScanEntry { + manifest?: BarePlugin; + /** + * User-friendly error messages. + */ + errors?: any[]; + path: string; + id: string; +} + +/** + * The contributions type. + */ +export type Contribution = "importSource" | "fileParser"; diff --git a/app/common/resetOrg.ts b/app/common/resetOrg.ts new file mode 100644 index 00000000..e10113f2 --- /dev/null +++ b/app/common/resetOrg.ts @@ -0,0 +1,44 @@ +import {ManagerDelta, PermissionDelta, UserAPI} from 'app/common/UserAPI'; + +/** + * A utility to reset an organization into the state it would have when first + * created - no docs, one workspace called "Home", a single user. Should be + * called by a user who is both an owner of the org and a billing manager. + */ +export async function resetOrg(api: UserAPI, org: string|number) { + const session = await api.getSessionActive(); + if (!(session.org && session.org.access === 'owners')) { + throw new Error('user must be an owner of the org to be reset'); + } + const billing = api.getBillingAPI(); + const account = await billing.getBillingAccount(); + if (!account.managers.some(manager => (manager.id === session.user.id))) { + throw new Error('user must be a billing manager'); + } + const wss = await api.getOrgWorkspaces(org); + for (const ws of wss) { + if (!ws.isSupportWorkspace) { + await api.deleteWorkspace(ws.id); + } + } + await api.newWorkspace({name: 'Home'}, org); + const permissions: PermissionDelta = { users: {} }; + for (const user of (await api.getOrgAccess(org)).users) { + if (user.id !== session.user.id) { + permissions.users![user.email] = null; + } + } + await api.updateOrgPermissions(org, permissions); + // For non-individual accounts, update billing managers (individual accounts will + // throw an error if we try to do this). + if (!account.individual) { + const managers: ManagerDelta = { users: {} }; + for (const user of account.managers) { + if (user.id !== session.user.id) { + managers.users[user.email] = null; + } + } + await billing.updateBillingManagers(managers); + } + return api; +} diff --git a/app/common/roles.ts b/app/common/roles.ts new file mode 100644 index 00000000..5e6736fe --- /dev/null +++ b/app/common/roles.ts @@ -0,0 +1,85 @@ +export const OWNER = 'owners'; +export const EDITOR = 'editors'; +export const VIEWER = 'viewers'; +export const GUEST = 'guests'; +export const MEMBER = 'members'; + +// Roles ordered from most to least permissive. +const roleOrder: Array = [OWNER, EDITOR, VIEWER, MEMBER, GUEST, null]; + +export type BasicRole = 'owners'|'editors'|'viewers'; +export type NonMemberRole = BasicRole|'guests'; +export type NonGuestRole = BasicRole|'members'; +export type Role = NonMemberRole|'members'; + +// Returns the BasicRole (or null) with the same effective access as the given role. +export function getEffectiveRole(role: Role|null): BasicRole|null { + if (role === GUEST || role === MEMBER) { + return VIEWER; + } else { + return role; + } +} + +export function canEditAccess(role: string|null): boolean { + return role === OWNER; +} + +// Note that while canEdit has the same return value as canDelete, the functions are +// kept separate as they may diverge in the future. +export function canEdit(role: string|null): boolean { + return role === OWNER || role === EDITOR; +} + +export function canDelete(role: string|null): boolean { + return role === OWNER || role === EDITOR; +} + +export function canView(role: string|null): boolean { + return role !== null; +} + +// Returns true if the role string is a valid role or null. +export function isValidRole(role: string|null): role is Role|null { + return (roleOrder as Array).includes(role); +} + +// Returns true if the role string is a valid non-Guest, non-Member, non-null role. +export function isBasicRole(role: string|null): role is BasicRole { + return Boolean(role && role !== GUEST && role !== MEMBER && isValidRole(role)); +} + +// Returns true if the role string is a valid non-Guest, non-null role. +export function isNonGuestRole(role: string|null): role is NonGuestRole { + return Boolean(role && role !== GUEST && isValidRole(role)); +} + +/** + * Returns out of any number of group role names the one that offers more permissions. The function + * is overloaded so that the output type matches the specificity of the input values. + */ +export function getStrongestRole(...args: T[]): T { + return getFirstMatchingRole(roleOrder, args); +} + +/** + * Returns out of any number of group role names the one that offers fewer permissions. The function + * is overloaded so that the output type matches the specificity of the input values. + */ +export function getWeakestRole(...args: T[]): T { + return getFirstMatchingRole(roleOrder.slice().reverse(), args); +} + +// Returns which of the `anyOf` args comes first in `array`. Helper for getStrongestRole +// and getWeakestRole. +function getFirstMatchingRole(array: Array, anyOf: T[]): T { + if (anyOf.length === 0) { + throw new Error(`getFirstMatchingRole: No roles given`); + } + for (const role of anyOf) { + if (!isValidRole(role)) { + throw new Error(`getFirstMatchingRole: Invalid role ${role}`); + } + } + return array.find((item) => anyOf.includes(item as T)) as T; +} diff --git a/app/common/schema.ts b/app/common/schema.ts new file mode 100644 index 00000000..60287a24 --- /dev/null +++ b/app/common/schema.ts @@ -0,0 +1,336 @@ +/*** THIS FILE IS AUTO-GENERATED BY sandbox/gen_js_schema.py ***/ +// tslint:disable:object-literal-key-quotes + +export const schema = { + + "_grist_DocInfo": { + docId : "Text", + peers : "Text", + basketId : "Text", + schemaVersion : "Int", + timezone : "Text", + }, + + "_grist_Tables": { + tableId : "Text", + primaryViewId : "Ref:_grist_Views", + summarySourceTable : "Ref:_grist_Tables", + onDemand : "Bool", + }, + + "_grist_Tables_column": { + parentId : "Ref:_grist_Tables", + parentPos : "PositionNumber", + colId : "Text", + type : "Text", + widgetOptions : "Text", + isFormula : "Bool", + formula : "Text", + label : "Text", + untieColIdFromLabel : "Bool", + summarySourceCol : "Ref:_grist_Tables_column", + displayCol : "Ref:_grist_Tables_column", + visibleCol : "Ref:_grist_Tables_column", + }, + + "_grist_Imports": { + tableRef : "Ref:_grist_Tables", + origFileName : "Text", + parseFormula : "Text", + delimiter : "Text", + doublequote : "Bool", + escapechar : "Text", + quotechar : "Text", + skipinitialspace : "Bool", + encoding : "Text", + hasHeaders : "Bool", + }, + + "_grist_External_database": { + host : "Text", + port : "Int", + username : "Text", + dialect : "Text", + database : "Text", + storage : "Text", + }, + + "_grist_External_table": { + tableRef : "Ref:_grist_Tables", + databaseRef : "Ref:_grist_External_database", + tableName : "Text", + }, + + "_grist_TableViews": { + tableRef : "Ref:_grist_Tables", + viewRef : "Ref:_grist_Views", + }, + + "_grist_TabItems": { + tableRef : "Ref:_grist_Tables", + viewRef : "Ref:_grist_Views", + }, + + "_grist_TabBar": { + viewRef : "Ref:_grist_Views", + tabPos : "PositionNumber", + }, + + "_grist_Pages": { + viewRef : "Ref:_grist_Views", + indentation : "Int", + pagePos : "PositionNumber", + }, + + "_grist_Views": { + name : "Text", + type : "Text", + layoutSpec : "Text", + }, + + "_grist_Views_section": { + tableRef : "Ref:_grist_Tables", + parentId : "Ref:_grist_Views", + parentKey : "Text", + title : "Text", + defaultWidth : "Int", + borderWidth : "Int", + theme : "Text", + options : "Text", + chartType : "Text", + layoutSpec : "Text", + filterSpec : "Text", + sortColRefs : "Text", + linkSrcSectionRef : "Ref:_grist_Views_section", + linkSrcColRef : "Ref:_grist_Tables_column", + linkTargetColRef : "Ref:_grist_Tables_column", + embedId : "Text", + }, + + "_grist_Views_section_field": { + parentId : "Ref:_grist_Views_section", + parentPos : "PositionNumber", + colRef : "Ref:_grist_Tables_column", + width : "Int", + widgetOptions : "Text", + displayCol : "Ref:_grist_Tables_column", + visibleCol : "Ref:_grist_Tables_column", + filter : "Text", + }, + + "_grist_Validations": { + formula : "Text", + name : "Text", + tableRef : "Int", + }, + + "_grist_REPL_Hist": { + code : "Text", + outputText : "Text", + errorText : "Text", + }, + + "_grist_Attachments": { + fileIdent : "Text", + fileName : "Text", + fileType : "Text", + fileSize : "Int", + imageHeight : "Int", + imageWidth : "Int", + timeUploaded : "DateTime", + }, + + "_grist_ACLRules": { + resource : "Ref:_grist_ACLResources", + permissions : "Int", + principals : "Text", + aclFormula : "Text", + aclColumn : "Ref:_grist_Tables_column", + }, + + "_grist_ACLResources": { + tableId : "Text", + colIds : "Text", + }, + + "_grist_ACLPrincipals": { + type : "Text", + userEmail : "Text", + userName : "Text", + groupName : "Text", + instanceId : "Text", + }, + + "_grist_ACLMemberships": { + parent : "Ref:_grist_ACLPrincipals", + child : "Ref:_grist_ACLPrincipals", + }, + +}; + +export interface SchemaTypes { + + "_grist_DocInfo": { + docId: string; + peers: string; + basketId: string; + schemaVersion: number; + timezone: string; + }; + + "_grist_Tables": { + tableId: string; + primaryViewId: number; + summarySourceTable: number; + onDemand: boolean; + }; + + "_grist_Tables_column": { + parentId: number; + parentPos: number; + colId: string; + type: string; + widgetOptions: string; + isFormula: boolean; + formula: string; + label: string; + untieColIdFromLabel: boolean; + summarySourceCol: number; + displayCol: number; + visibleCol: number; + }; + + "_grist_Imports": { + tableRef: number; + origFileName: string; + parseFormula: string; + delimiter: string; + doublequote: boolean; + escapechar: string; + quotechar: string; + skipinitialspace: boolean; + encoding: string; + hasHeaders: boolean; + }; + + "_grist_External_database": { + host: string; + port: number; + username: string; + dialect: string; + database: string; + storage: string; + }; + + "_grist_External_table": { + tableRef: number; + databaseRef: number; + tableName: string; + }; + + "_grist_TableViews": { + tableRef: number; + viewRef: number; + }; + + "_grist_TabItems": { + tableRef: number; + viewRef: number; + }; + + "_grist_TabBar": { + viewRef: number; + tabPos: number; + }; + + "_grist_Pages": { + viewRef: number; + indentation: number; + pagePos: number; + }; + + "_grist_Views": { + name: string; + type: string; + layoutSpec: string; + }; + + "_grist_Views_section": { + tableRef: number; + parentId: number; + parentKey: string; + title: string; + defaultWidth: number; + borderWidth: number; + theme: string; + options: string; + chartType: string; + layoutSpec: string; + filterSpec: string; + sortColRefs: string; + linkSrcSectionRef: number; + linkSrcColRef: number; + linkTargetColRef: number; + embedId: string; + }; + + "_grist_Views_section_field": { + parentId: number; + parentPos: number; + colRef: number; + width: number; + widgetOptions: string; + displayCol: number; + visibleCol: number; + filter: string; + }; + + "_grist_Validations": { + formula: string; + name: string; + tableRef: number; + }; + + "_grist_REPL_Hist": { + code: string; + outputText: string; + errorText: string; + }; + + "_grist_Attachments": { + fileIdent: string; + fileName: string; + fileType: string; + fileSize: number; + imageHeight: number; + imageWidth: number; + timeUploaded: number; + }; + + "_grist_ACLRules": { + resource: number; + permissions: number; + principals: string; + aclFormula: string; + aclColumn: number; + }; + + "_grist_ACLResources": { + tableId: string; + colIds: string; + }; + + "_grist_ACLPrincipals": { + type: string; + userEmail: string; + userName: string; + groupName: string; + instanceId: string; + }; + + "_grist_ACLMemberships": { + parent: number; + child: number; + }; + +} diff --git a/app/common/sharing.ts b/app/common/sharing.ts new file mode 100644 index 00000000..59f5b106 --- /dev/null +++ b/app/common/sharing.ts @@ -0,0 +1,43 @@ +import {EncActionBundleFromHub} from 'app/common/EncActionBundle'; + +export const allToken: string = '#ALL'; + +/** + * Messages received from SQS + */ +export interface Message { + type: MessageType; + content: Invite | EncActionBundleFromHub; + docId: string; // The docId to which the message pertains. +} + +export enum MessageType { + invite = 1, + accept, + decline, + action +} + +export interface Invite { + senderEmail: string; + senderName?: string; + docId: string; // Indicates the doc to which the user is being invited to join. + docName: string; // Indicates the docName at the time of sending for user doc recognition. + isUnread?: boolean; + isIgnored?: boolean; +} + +/** + * Contains information about someone who may or may not be a Grist user. + */ +export interface Peer { + email: string; + name?: string; + instIds?: string[]; +} + +export interface EmailResult { + email: string; + instIds: string[]; + name?: string; +} diff --git a/app/common/tbind.ts b/app/common/tbind.ts new file mode 100644 index 00000000..a066da66 --- /dev/null +++ b/app/common/tbind.ts @@ -0,0 +1,15 @@ +/** + * A version of Function.bind() that preserves types. + */ + +// tslint:disable:max-line-length + +// Bind just the context for a function of up to 4 args. +export function tbind(func: (this: T, ...a: Args) => R, context: T): (...a: Args) => R; + +// Bind context and first arg for a function of up to 5 args. +export function tbind(func: (this: T, x: X, ...a: Args) => R, context: T, x: X): (...a: Args) => R; + +export function tbind(func: any, context: any, ...boundArgs: any[]): any { + return func.bind(context, ...boundArgs); +} diff --git a/app/common/timeFormat.ts b/app/common/timeFormat.ts new file mode 100644 index 00000000..6bc38db3 --- /dev/null +++ b/app/common/timeFormat.ts @@ -0,0 +1,56 @@ +/** + * timeFormat(format, date) formats the passed-in Date object using the format string. The format + * string may contain the following: + * 'h': hour (00 - 23) + * 'm': minute (00 - 59) + * 's': second (00 - 59) + * 'd': day of the month (01 - 31) + * 'n': month (01 - 12) + * 'y': 4-digit year + * 'M': milliseconds (000 - 999) + * 'Y': date as 20140212 + * 'D': date as 2014-02-12 + * 'T': time as 00:51:06 + * 'A': full time and date, as 2014-02-12 00:51:06.123 + * @param {String} format The format string. + * @param {Date} date The date/time object to format. + * @returns {String} The formatted date and/or time. + */ + +function pad(num: number, len: number): string { + const s = num.toString(); + return s.length >= len ? s : "00000000".slice(0, len - s.length) + s; +} + +type FormatHelper = (out: string[], date: Date) => void; +const timeFormatKeys: {[spec: string]: FormatHelper} = { + h: (out, date) => out.push(pad(date.getHours(), 2)), + m: (out, date) => out.push(pad(date.getMinutes(), 2)), + s: (out, date) => out.push(pad(date.getSeconds(), 2)), + d: (out, date) => out.push(pad(date.getDate(), 2)), + n: (out, date) => out.push(pad(date.getMonth() + 1, 2)), + y: (out, date) => out.push("" + date.getFullYear()), + M: (out, date) => out.push(pad(date.getMilliseconds(), 3)), + Y: (out, date) => timeFormatHelper(out, 'ynd', date), + D: (out, date) => timeFormatHelper(out, 'y-n-d', date), + T: (out, date) => timeFormatHelper(out, 'h:m:s', date), + A: (out, date) => timeFormatHelper(out, 'D T.M', date), +}; + +function timeFormatHelper(out: string[], format: string, date: Date) { + for (let i = 0, len = format.length; i < len; i++) { + const c = format[i]; + const helper = timeFormatKeys[c]; + if (helper) { + helper(out, date); + } else { + out.push(c); + } + } +} + +export function timeFormat(format: string, date: Date): string { + const out: string[] = []; + timeFormatHelper(out, format, date); + return out.join(""); +} diff --git a/app/common/tpromisified.ts b/app/common/tpromisified.ts new file mode 100644 index 00000000..4ef7c6e3 --- /dev/null +++ b/app/common/tpromisified.ts @@ -0,0 +1,25 @@ +// tslint:disable:max-line-length +// credits: https://stackoverflow.com/questions/49998665/promisified-function-type + +// Generic Function definition +type AnyFunction = (...args: any[]) => any; + +// Extracts the type if wrapped by a Promise +type Unpacked = T extends Promise ? U : T; + +type PromisifiedFunction = + T extends () => infer U ? () => Promise> : + T extends (a1: infer A1) => infer U ? (a1: A1) => Promise> : + T extends (a1: infer A1, a2: infer A2) => infer U ? (a1: A1, a2: A2) => Promise> : + T extends (a1: infer A1, a2: infer A2, a3: infer A3) => infer U ? (a1: A1, a2: A2, a3: A3) => Promise> : + T extends (a1: infer A1, a2: infer A2, a3: infer A3, a4: infer A4) => infer U ? (a1: A1, a2: A2, a3: A3, a4: A4) => Promise> : + // ... + T extends (...args: any[]) => infer U ? (...args: any[]) => Promise> : T; + +/** + * `Promisified` has the same methods as `T` but they all return promises. This is useful when + * creating a stub with `grain-rpc` for an api which is synchronous. + */ +export type Promisified = { + [K in keyof T]: T[K] extends AnyFunction ? PromisifiedFunction : never +}; diff --git a/app/common/tsconfig.json b/app/common/tsconfig.json index 185bec10..44b48ccb 100644 --- a/app/common/tsconfig.json +++ b/app/common/tsconfig.json @@ -1,3 +1,6 @@ { "extends": "../../buildtools/tsconfig-base.json", + "references": [ + { "path": "../plugin" } + ] } diff --git a/app/common/tsvFormat.ts b/app/common/tsvFormat.ts new file mode 100644 index 00000000..a3861659 --- /dev/null +++ b/app/common/tsvFormat.ts @@ -0,0 +1,58 @@ +/** + * Given a 2D array of strings, encodes them in tab-separated format. + * Certain values are quoted; when quoted, internal quotes get doubled. The behavior attempts to + * match Excel's tsv encoding and parsing when using copy-paste. + */ +export function tsvEncode(data: any[][]): string { + return data.map(row => row.map(value => encode(value)).join("\t")).join("\n"); +} + +function encode(rawValue: any): string { + // For encoding-decoding symmetry, we should also encode any values that start with '"', + // but neither Excel nor Google Sheets do it. They both decode such values to something + // different than what produced them (e.g. `"foo""bar"` is encoded into `"foo""bar"`, and + // that is decoded into `foo"bar`). + const value: string = typeof rawValue === 'string' ? rawValue : + (rawValue == null ? "" : String(rawValue)); + if (value.includes("\t") || value.includes("\n")) { + return '"' + value.replace(/"/g, '""') + '"'; + } + return value; +} + +/** + * Given a tab-separated string, decodes it and returns a 2D array of strings. + * TODO: This does not yet deal with Windows line endings (\r or \r\n). + */ +export function tsvDecode(tsvString: string): string[][] { + const lines: string[][] = []; + let row: string[] = []; + + // This is a complex regexp but it does the job of a lot of parsing code. Here are the parts: + // A: [^\t\n]* Sequence of character that does not require the field to get quoted. + // B: ([^"]*"")*[^"]* Sequence of characters containing all double-quotes in pairs (i.e. `""`) + // C: "B"(?!") Quoted sequence, with all double-quotes inside paired up, and ending in a single quote. + // D: C?A A value for one field, a relaxation of C|A (to cope with not-quite expected data) + // E: D(\t|\n|$) Field value with field, line, or file terminator. + const fieldRegexp = /(("([^"]*"")*[^"]*"(?!"))?[^\t\n]*)(\t|\n|$)/g; + for (;;) { + const m = fieldRegexp.exec(tsvString); + if (!m) { break; } + const sep = m[4]; + let value = m[1]; + if (value.startsWith('"')) { + // It's a quoted value, so doubled-up quotes should became individual quotes, and individual + // quotes should be removed. + value = value.replace(/"([^"]*"")*[^"]*"(?!")/, q => q.slice(1, -1).replace(/""/g, '"')); + } + row.push(value); + if (sep !== '\t') { + lines.push(row); + row = []; + if (sep === '') { + break; + } + } + } + return lines; +} diff --git a/app/common/uploads.ts b/app/common/uploads.ts new file mode 100644 index 00000000..a5479565 --- /dev/null +++ b/app/common/uploads.ts @@ -0,0 +1,44 @@ +/** + * Code and declarations shared by browser and server-side code for handling uploads. + * + * Browser code has several functions available in app/client/lib/uploads.ts which return an + * UploadResult that represents an upload. An upload may contain multiple files. + * + * An upload is identified by a numeric uploadId which is unique within an UploadSet. An UploadSet + * is collection of uploads tied to a browser session (as maintained by app/server/lib/Client). + * When the session ends, all uploads are cleaned up. + * + * The uploadId is useful to identify the upload to the server, which can then consume the actual + * files there. It may also be used to clean up the upload once it is no longer needed. + * + * Files within an upload can be identified by their index in UploadResult.files array. The + * origName available for files is not guaranteed to be unique. + * + * Implementation detail: The upload is usually a temporary directory on the server, but may be a + * collection of non-temporary files when files are selected using Electron's native file picker. + */ + +/** + * Represents a single upload, containing one or more files. Empty uploads are never created. + */ +export interface UploadResult { + uploadId: number; + files: FileUploadResult[]; +} + +/** + * Represents a single file within an upload. This is the only information made available to the + * browser. (In particular, while the server knows also the actual path of the file on the server, + * the browser has no need for it and should not know it.) + */ +export interface FileUploadResult { + origName: string; // The filename that the user reports for the file (not guaranteed unique). + size: number; // The size of the file in bytes. + ext: string; // The extension of the file, starting with "." +} + +/** + * Path where the server accepts POST requests with uploads. Don't include a leading / so that + * the page's will be respected. + */ +export const UPLOAD_URL_PATH = 'uploads'; diff --git a/app/common/urlUtils.ts b/app/common/urlUtils.ts new file mode 100644 index 00000000..8dacdf8d --- /dev/null +++ b/app/common/urlUtils.ts @@ -0,0 +1,88 @@ +import {extractOrgParts, GristLoadConfig} from 'app/common/gristUrls'; + +export function getGristConfig(): GristLoadConfig { + return (window as any).gristConfig || {}; +} + +/** + * + * Adds /o/ORG to the supplied path, with ORG extracted from current URL if possible. + * If not, path is returned as is, but with any trailing / removed for consistency. + * + */ +export function addCurrentOrgToPath(path: string, skipIfInDomain: boolean = false) { + if (typeof window === 'undefined' || !window) { return path; } + return addOrgToPath(path, window.location.href, skipIfInDomain); +} + +/** + * + * Adds /o/ORG to the supplied path, with ORG extracted from the page URL if possible. + * If not, path is returned as is, but with any trailing / removed for consistency. + * + */ +export function addOrgToPath(path: string, page: string, skipIfInDomain: boolean = false) { + if (typeof window === 'undefined' || !window) { return path; } + if (path.includes('/o/')) { return path; } + const src = new URL(page); + const srcParts = extractOrgParts(src.host, src.pathname); + if (srcParts.mismatch) { + throw new Error('Cannot figure out what organization the URL is for.'); + } + path = path.replace(/\/$/, ''); + if (!srcParts.subdomain) { + return path; + } + if (skipIfInDomain && srcParts.orgFromHost) { + return path; + } + return `${path}/o/${srcParts.subdomain}`; +} + +/** + * Expands an endpoint path to a full url anchored to the given doc worker base url. + */ +export function docUrl(docWorkerUrl: string|null|undefined, path?: string) { + const base = document.querySelector('base'); + const baseHref = base && base.href; + const baseUrl = new URL(docWorkerUrl || baseHref || window.location.origin); + return baseUrl.toString().replace(/\/$/, '') + (path ? `/${path}` : ''); +} + +// Get a url on the same webserver as the current page, adding a prefix to encode +// the current organization if necessary. +export function getOriginUrl(path: string) { + return `${window.location.origin}${addCurrentOrgToPath('/', true)}${path}`; +} + +// Return a string docId if server has provided one (as in hosted Grist), otherwise null +// (as in classic Grist). +export function getInitialDocAssignment(): string|null { + return getGristConfig().assignmentId || null; +} + +// Return true if we are on a page that can send metrics. +// TODO: all pages should send suitable metrics. +export function pageHasMetrics(): boolean { + // No metric support on hosted grist. + return !getGristConfig().homeUrl; +} + +// Return true if we are on a page that can supply a doc list. +// TODO: the doclist object isn't relevant to hosted grist and should be factored out. +export function pageHasDocList(): boolean { + // No doc list support on hosted grist. + return !getGristConfig().homeUrl; +} + +// Return true if we are on a page that has access to home api. +export function pageHasHome(): boolean { + return Boolean(getGristConfig().homeUrl); +} + +// Construct a url by adding `path` to the home url (adding in the part to the current +// org if needed), and fetch from it. +export function fetchFromHome(path: string, opts: RequestInit): Promise { + const baseUrl = addCurrentOrgToPath(getGristConfig().homeUrl!); + return window.fetch(`${baseUrl}${path}`, opts); +} diff --git a/app/common/version.ts b/app/common/version.ts new file mode 100644 index 00000000..bfc7a98b --- /dev/null +++ b/app/common/version.ts @@ -0,0 +1,3 @@ +export const version = "1.0.2-dev"; +export const channel = "devtest"; +export const gitcommit = "e33c4e5aeM"; diff --git a/app/gen-server/ApiServer.ts b/app/gen-server/ApiServer.ts new file mode 100644 index 00000000..a58adb3c --- /dev/null +++ b/app/gen-server/ApiServer.ts @@ -0,0 +1,475 @@ +import * as crypto from 'crypto'; +import * as express from 'express'; +import {EntityManager} from 'typeorm'; + +import {ApiError} from 'app/common/ApiError'; +import {FullUser} from 'app/common/LoginSessionAPI'; +import {OrganizationProperties} from 'app/common/UserAPI'; +import {getAuthorizedUserId, getUserId, getUserProfiles, RequestWithLogin} from 'app/server/lib/Authorizer'; +import {getSessionUser, linkOrgWithEmail} from 'app/server/lib/BrowserSession'; +import {expressWrap} from 'app/server/lib/expressWrap'; +import {RequestWithOrg} from 'app/server/lib/extractOrg'; +import * as log from 'app/server/lib/log'; +import {getDocScope, getScope, integerParam, isParameterOn, sendOkReply, + sendReply, stringParam} from 'app/server/lib/requestUtils'; +import {Request} from 'express'; + +import {User} from './entity/User'; +import {HomeDBManager} from './lib/HomeDBManager'; + +// exposed for testing purposes +export const Deps = { + apiKeyGenerator: () => crypto.randomBytes(20).toString('hex') +}; + +// Fetch the org this request was made for, or null if it isn't tied to a particular org. +// Early middleware should have put the org in the request object for us. +export function getOrgFromRequest(req: Request): string|null { + return (req as RequestWithOrg).org || null; +} + +/** + * Compute the signature of the user's email address using HelpScout's secret key, to prove to + * HelpScout the user identity for identifying customer information and conversation history. + */ +function helpScoutSign(email: string): string|undefined { + const secretKey = process.env.HELP_SCOUT_SECRET_KEY; + if (!secretKey) { return undefined; } + return crypto.createHmac('sha256', secretKey).update(email).digest('hex'); +} + +/** + * Fetch an identifier for an organization from the "oid" parameter of the request. + * - Integers are accepted, and will be compared with values in orgs.id column + * - Strings are accepted, and will be compared with values in orgs.domain column + * (or, if they match the pattern docs-NNNN, will check orgs.owner_id) + * - The special string "current" is replaced with the current org domain embedded + * in the url + * - If there is no identifier available, a 400 error is thrown. + */ +export function getOrgKey(req: Request): string|number { + let orgKey: string|null = stringParam(req.params.oid); + if (orgKey === 'current') { + orgKey = getOrgFromRequest(req); + } + if (!orgKey) { + throw new ApiError("No organization chosen", 400); + } else if (/^\d+$/.test(orgKey)) { + return parseInt(orgKey, 10); + } + return orgKey; +} + +// Adds an non-personal org with a new billingAccout, with the given name and domain. +// Returns a QueryResult with the orgId on success. +export function addOrg( + dbManager: HomeDBManager, + userId: number, + props: Partial, +): Promise { + return dbManager.connection.transaction(async manager => { + const user = await manager.findOne(User, userId); + if (!user) { return handleDeletedUser(); } + const query = await dbManager.addOrg(user, props, false, true, manager); + if (query.status !== 200) { throw new ApiError(query.errMessage!, query.status); } + return query.data!; + }); +} + +/** + * Provides a REST API for the landing page, which returns user's workspaces, organizations and documents. + * Temporarily sqlite database is used. Later it will be changed to RDS Aurora or PostgreSQL. + */ +export class ApiServer { + /** + * Add API endpoints to the specified connection. An error handler is added to /api to make sure + * all error responses have a body in json format. + * + * Note that it expects bodyParser, userId, and jsonErrorHandler middleware to be set up outside + * to apply to these routes, and trustOrigin too for cross-domain requests. + */ + constructor( + private _app: express.Application, + private _dbManager: HomeDBManager, + ) { + this._addEndpoints(); + } + + private _addEndpoints(): void { + // GET /api/orgs + // Get all organizations user may have some access to. + this._app.get('/api/orgs', expressWrap(async (req, res) => { + const userId = getUserId(req); + const domain = getOrgFromRequest(req); + const merged = Boolean(req.query.merged); + const query = merged ? + await this._dbManager.getMergedOrgs(userId, userId, domain) : + await this._dbManager.getOrgs(userId, domain); + return sendReply(req, res, query); + })); + + // GET /api/workspace/:wid + // Get workspace by id, returning nested documents that user has access to. + this._app.get('/api/workspaces/:wid', expressWrap(async (req, res) => { + const wsId = integerParam(req.params.wid); + const query = await this._dbManager.getWorkspace(getScope(req), wsId); + return sendReply(req, res, query); + })); + + // GET /api/orgs/:oid + // Get organization by id + this._app.get('/api/orgs/:oid', expressWrap(async (req, res) => { + const org = getOrgKey(req); + const query = await this._dbManager.getOrg(getScope(req), org); + return sendReply(req, res, query); + })); + + // GET /api/orgs/:oid/workspaces + // Get all workspaces and nested documents of organization that user has access to. + this._app.get('/api/orgs/:oid/workspaces', expressWrap(async (req, res) => { + const org = getOrgKey(req); + const query = await this._dbManager.getOrgWorkspaces(getScope(req), org); + return sendReply(req, res, query); + })); + + // POST /api/orgs + // Body params: name (required), domain + // Create a new org. + this._app.post('/api/orgs', expressWrap(async (req, res) => { + // Don't let anonymous users end up owning organizations, it will be confusing. + // Maybe if the user has presented credentials this would be ok - but addOrg + // doesn't have access to that information yet, so punting on this. + // TODO: figure out who should be allowed to create organizations + const userId = getAuthorizedUserId(req); + const orgId = await addOrg(this._dbManager, userId, req.body); + return sendOkReply(req, res, orgId); + })); + + // PATCH /api/orgs/:oid + // Body params: name, domain + // Update the specified org. + this._app.patch('/api/orgs/:oid', expressWrap(async (req, res) => { + const org = getOrgKey(req); + const query = await this._dbManager.updateOrg(getScope(req), org, req.body); + return sendReply(req, res, query); + })); + + // // DELETE /api/orgs/:oid + // Delete the specified org and all included workspaces and docs. + this._app.delete('/api/orgs/:oid', expressWrap(async (req, res) => { + const org = getOrgKey(req); + const query = await this._dbManager.deleteOrg(getScope(req), org); + return sendReply(req, res, query); + })); + + // POST /api/orgs/:oid/workspaces + // Body params: name + // Create a new workspace owned by the specific organization. + this._app.post('/api/orgs/:oid/workspaces', expressWrap(async (req, res) => { + const org = getOrgKey(req); + const query = await this._dbManager.addWorkspace(getScope(req), org, req.body); + return sendReply(req, res, query); + })); + + // PATCH /api/workspaces/:wid + // Body params: name + // Update the specified workspace. + this._app.patch('/api/workspaces/:wid', expressWrap(async (req, res) => { + const wsId = integerParam(req.params.wid); + const query = await this._dbManager.updateWorkspace(getScope(req), wsId, req.body); + return sendReply(req, res, query); + })); + + // // DELETE /api/workspaces/:wid + // Delete the specified workspace and all included docs. + this._app.delete('/api/workspaces/:wid', expressWrap(async (req, res) => { + const wsId = integerParam(req.params.wid); + const query = await this._dbManager.deleteWorkspace(getScope(req), wsId); + return sendReply(req, res, query); + })); + + // POST /api/workspaces/:wid/remove + // Soft-delete the specified workspace. If query parameter "permanent" is set, + // delete permanently. + this._app.post('/api/workspaces/:wid/remove', expressWrap(async (req, res) => { + const wsId = integerParam(req.params.wid); + if (isParameterOn(req.query.permanent)) { + const query = await this._dbManager.deleteWorkspace(getScope(req), wsId); + return sendReply(req, res, query); + } else { + await this._dbManager.softDeleteWorkspace(getScope(req), wsId); + return sendOkReply(req, res); + } + })); + + // POST /api/workspaces/:wid/unremove + // Recover the specified workspace if it was previously soft-deleted and is + // still available. + this._app.post('/api/workspaces/:wid/unremove', expressWrap(async (req, res) => { + const wsId = integerParam(req.params.wid); + await this._dbManager.undeleteWorkspace(getScope(req), wsId); + return sendOkReply(req, res); + })); + + // POST /api/workspaces/:wid/docs + // Create a new doc owned by the specific workspace. + this._app.post('/api/workspaces/:wid/docs', expressWrap(async (req, res) => { + const wsId = integerParam(req.params.wid); + const query = await this._dbManager.addDocument(getScope(req), wsId, req.body); + return sendReply(req, res, query); + })); + + // PATCH /api/docs/:did + // Update the specified doc. + this._app.patch('/api/docs/:did', expressWrap(async (req, res) => { + const query = await this._dbManager.updateDocument(getDocScope(req), req.body); + return sendReply(req, res, query); + })); + + // POST /api/docs/:did/unremove + // Recover the specified doc if it was previously soft-deleted and is + // still available. + this._app.post('/api/docs/:did/unremove', expressWrap(async (req, res) => { + await this._dbManager.undeleteDocument(getDocScope(req)); + return sendOkReply(req, res); + })); + + // PATCH /api/orgs/:oid/access + // Update the specified org acl rules. + this._app.patch('/api/orgs/:oid/access', expressWrap(async (req, res) => { + const org = getOrgKey(req); + const delta = req.body.delta; + const query = await this._dbManager.updateOrgPermissions(getScope(req), org, delta); + return sendReply(req, res, query); + })); + + // PATCH /api/workspaces/:wid/access + // Update the specified workspace acl rules. + this._app.patch('/api/workspaces/:wid/access', expressWrap(async (req, res) => { + const workspaceId = integerParam(req.params.wid); + const delta = req.body.delta; + const query = await this._dbManager.updateWorkspacePermissions(getScope(req), workspaceId, delta); + return sendReply(req, res, query); + })); + + // GET /api/docs/:did + // Get information about a document. + this._app.get('/api/docs/:did', expressWrap(async (req, res) => { + const query = await this._dbManager.getDoc(getDocScope(req)); + return sendOkReply(req, res, query); + })); + + // PATCH /api/docs/:did/access + // Update the specified doc acl rules. + this._app.patch('/api/docs/:did/access', expressWrap(async (req, res) => { + const delta = req.body.delta; + const query = await this._dbManager.updateDocPermissions(getDocScope(req), delta); + return sendReply(req, res, query); + })); + + // PATCH /api/docs/:did/move + // Move the doc to the workspace specified in the body. + this._app.patch('/api/docs/:did/move', expressWrap(async (req, res) => { + const workspaceId = req.body.workspace; + const query = await this._dbManager.moveDoc(getDocScope(req), workspaceId); + return sendReply(req, res, query); + })); + + this._app.patch('/api/docs/:did/pin', expressWrap(async (req, res) => { + const query = await this._dbManager.pinDoc(getDocScope(req), true); + return sendReply(req, res, query); + })); + + this._app.patch('/api/docs/:did/unpin', expressWrap(async (req, res) => { + const query = await this._dbManager.pinDoc(getDocScope(req), false); + return sendReply(req, res, query); + })); + + // GET /api/orgs/:oid/access + // Get user access information regarding an org + this._app.get('/api/orgs/:oid/access', expressWrap(async (req, res) => { + const org = getOrgKey(req); + const query = await this._dbManager.getOrgAccess(getScope(req), org); + return sendReply(req, res, query); + })); + + // GET /api/workspaces/:wid/access + // Get user access information regarding a workspace + this._app.get('/api/workspaces/:wid/access', expressWrap(async (req, res) => { + const workspaceId = integerParam(req.params.wid); + const query = await this._dbManager.getWorkspaceAccess(getScope(req), workspaceId); + return sendReply(req, res, query); + })); + + // GET /api/docs/:did/access + // Get user access information regarding a doc + this._app.get('/api/docs/:did/access', expressWrap(async (req, res) => { + const query = await this._dbManager.getDocAccess(getDocScope(req)); + return sendReply(req, res, query); + })); + + // GET /api/profile/user + // Get user's profile + this._app.get('/api/profile/user', expressWrap(async (req, res) => { + const fullUser = await this._getFullUser(req); + return sendOkReply(req, res, fullUser); + })); + + // POST /api/profile/user/name + // Body params: string + // Update users profile. + this._app.post('/api/profile/user/name', expressWrap(async (req, res) => { + const userId = getUserId(req); + if (!(req.body && req.body.name)) { + throw new ApiError('Name expected in the body', 400); + } + const name = req.body.name; + await this._dbManager.updateUserName(userId, name); + res.sendStatus(200); + })); + + // GET /api/profile/apikey + // Get user's apiKey + this._app.get('/api/profile/apikey', expressWrap(async (req, res) => { + const userId = getUserId(req); + const user = await User.findOne(userId); + if (user) { + // The null value is of no interest to the user, let's show empty string instead. + res.send(user.apiKey || ''); + return; + } + handleDeletedUser(); + })); + + // POST /api/profile/apikey + // Update user's apiKey + this._app.post('/api/profile/apikey', expressWrap(async (req, res) => { + const userId = getAuthorizedUserId(req); + const force = req.body ? req.body.force : false; + const manager = this._dbManager.connection.manager; + let user = await manager.findOne(User, userId); + if (!user) { return handleDeletedUser(); } + if (!user.apiKey || force) { + user = await updateApiKeyWithRetry(manager, user); + res.status(200).send(user.apiKey); + } else { + res.status(400).send({error: "An apikey is already set, use `{force: true}` to override it."}); + } + })); + + // DELETE /api/profile/apiKey + // Delete apiKey + this._app.delete('/api/profile/apikey', expressWrap(async (req, res) => { + const userId = getAuthorizedUserId(req); + await this._dbManager.connection.transaction(async manager => { + const user = await manager.findOne(User, userId); + if (!user) {return handleDeletedUser(); } + user.apiKey = null; + await manager.save(User, user); + }); + res.sendStatus(200); + })); + + // GET /api/session/access/active + // Returns active user and active org (if any) + this._app.get('/api/session/access/active', expressWrap(async (req, res) => { + const fullUser = await this._getFullUser(req); + const domain = getOrgFromRequest(req); + const org = domain ? (await this._dbManager.getOrg(getScope(req), domain || null)) : null; + const orgError = (org && org.errMessage) ? {error: org.errMessage, status: org.status} : undefined; + return sendOkReply(req, res, { + user: {...fullUser, helpScoutSignature: helpScoutSign(fullUser.email)}, + org: (org && org.data) || null, + orgError + }); + })); + + // POST /api/session/access/active + // Body params: email (required) + // Sets active user for active org + this._app.post('/api/session/access/active', expressWrap(async (req, res) => { + const mreq = req as RequestWithLogin; + const domain = getOrgFromRequest(mreq); + const email = req.body.email; + if (!email) { throw new ApiError('email required', 400); } + try { + // Modify session copy in request. Will be saved to persistent storage before responding + // by express-session middleware. + linkOrgWithEmail(mreq.session, req.body.email, domain || ''); + return sendOkReply(req, res, {email}); + } catch (e) { + throw new ApiError('email not available', 403); + } + })); + + // GET /api/session/access/all + // Returns all user profiles (with ids) and all orgs they can access. + // Flattens personal orgs into a single org. + this._app.get('/api/session/access/all', expressWrap(async (req, res) => { + const domain = getOrgFromRequest(req); + const users = getUserProfiles(req); + const userId = getUserId(req); + const orgs = await this._dbManager.getMergedOrgs(userId, users, domain); + if (orgs.errMessage) { throw new ApiError(orgs.errMessage, orgs.status); } + return sendOkReply(req, res, { + users: await this._dbManager.completeProfiles(users), + orgs: orgs.data + }); + })); + + // DELETE /users/:uid + // Delete the specified user, their personal organization, removing them from all groups. + // Not available to the anonymous user. + // TODO: should orphan orgs, inaccessible by anyone else, get deleted when last user + // leaves? + this._app.delete('/api/users/:uid', expressWrap(async (req, res) => { + const userIdToDelete = parseInt(req.params.uid, 10); + if (!(req.body && req.body.name !== undefined)) { + throw new ApiError('to confirm deletion of a user, provide their name', 400); + } + const query = await this._dbManager.deleteUser(getScope(req), userIdToDelete, req.body.name); + return sendReply(req, res, query); + })); + } + + private async _getFullUser(req: Request): Promise { + const mreq = req as RequestWithLogin; + const userId = getUserId(mreq); + const fullUser = await this._dbManager.getFullUser(userId); + const domain = getOrgFromRequest(mreq); + const sessionUser = getSessionUser(mreq.session, domain || ''); + const loginMethod = sessionUser && sessionUser.profile ? sessionUser.profile.loginMethod : undefined; + return {...fullUser, loginMethod}; + } +} + +/** + * Throw the error for when a user has been deleted since point of call (very unlikely to happen). + */ +function handleDeletedUser(): never { + throw new ApiError("user not known", 401); +} + +/** + * Helper to update a user's apiKey. Update might fail because of the DB uniqueness constraint on + * the apiKey (although it is very unlikely according to `crypto`), we retry until success. Fails + * after 5 unsuccessful attempts. + */ +async function updateApiKeyWithRetry(manager: EntityManager, user: User): Promise { + const currentKey = user.apiKey; + for (let i = 0; i < 5; ++i) { + user.apiKey = Deps.apiKeyGenerator(); + try { + // if new key is the same as the current, the db update won't fail so we check it here (very + // unlikely to happen but but still better to handle) + if (user.apiKey === currentKey) { + throw new Error('the new key is the same as the current key'); + } + return await manager.save(User, user); + } catch (e) { + // swallow and retry + log.warn(`updateApiKeyWithRetry: failed attempt ${i}/5, %s`, e); + } + } + throw new Error('Could not generate a valid api key.'); +} diff --git a/app/gen-server/entity/AclRule.ts b/app/gen-server/entity/AclRule.ts new file mode 100644 index 00000000..011ff458 --- /dev/null +++ b/app/gen-server/entity/AclRule.ts @@ -0,0 +1,58 @@ +import {BaseEntity, ChildEntity, Column, Entity, JoinColumn, ManyToOne, OneToOne, + PrimaryGeneratedColumn, RelationId, TableInheritance} from "typeorm"; + +import {Document} from "./Document"; +import {Group} from "./Group"; +import {Organization} from "./Organization"; +import {Workspace} from "./Workspace"; + +@Entity('acl_rules') +@TableInheritance({ column: { type: "int", name: "type" } }) +export class AclRule extends BaseEntity { + + @PrimaryGeneratedColumn() + public id: number; + + @Column() + public permissions: number; + + @OneToOne(type => Group, group => group.aclRule) + @JoinColumn({name: "group_id"}) + public group: Group; +} + + +@ChildEntity() +export class AclRuleWs extends AclRule { + + @ManyToOne(type => Workspace, workspace => workspace.aclRules) + @JoinColumn({name: "workspace_id"}) + public workspace: Workspace; + + @RelationId((aclRule: AclRuleWs) => aclRule.workspace) + public workspaceId: number; +} + + +@ChildEntity() +export class AclRuleOrg extends AclRule { + + @ManyToOne(type => Organization, organization => organization.aclRules) + @JoinColumn({name: "org_id"}) + public organization: Organization; + + @RelationId((aclRule: AclRuleOrg) => aclRule.organization) + public orgId: number; +} + + +@ChildEntity() +export class AclRuleDoc extends AclRule { + + @ManyToOne(type => Document, document => document.aclRules) + @JoinColumn({name: "doc_id"}) + public document: Document; + + @RelationId((aclRule: AclRuleDoc) => aclRule.document) + public docId: number; +} diff --git a/app/gen-server/entity/Alias.ts b/app/gen-server/entity/Alias.ts new file mode 100644 index 00000000..f8e79b41 --- /dev/null +++ b/app/gen-server/entity/Alias.ts @@ -0,0 +1,27 @@ +import {BaseEntity, Column, CreateDateColumn, Entity, JoinColumn, ManyToOne, + PrimaryColumn} from 'typeorm'; +import {Document} from './Document'; +import {Organization} from './Organization'; + +@Entity({name: 'aliases'}) +export class Alias extends BaseEntity { + @PrimaryColumn({name: 'org_id'}) + public orgId: number; + + @PrimaryColumn({name: 'url_id'}) + public urlId: string; + + @Column({name: 'doc_id'}) + public docId: string; + + @ManyToOne(type => Document) + @JoinColumn({name: 'doc_id'}) + public doc: Document; + + @ManyToOne(type => Organization) + @JoinColumn({name: 'org_id'}) + public org: Organization; + + @CreateDateColumn({name: 'created_at'}) + public createdAt: Date; +} diff --git a/app/gen-server/entity/BillingAccount.ts b/app/gen-server/entity/BillingAccount.ts new file mode 100644 index 00000000..6ad07714 --- /dev/null +++ b/app/gen-server/entity/BillingAccount.ts @@ -0,0 +1,65 @@ +import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn} from 'typeorm'; +import {BillingAccountManager} from 'app/gen-server/entity/BillingAccountManager'; +import {Organization} from 'app/gen-server/entity/Organization'; +import {Product} from 'app/gen-server/entity/Product'; +import {nativeValues} from 'app/gen-server/lib/values'; + +// This type is for billing account status information. Intended for stuff +// like "free trial running out in N days". +interface BillingAccountStatus { + stripeStatus?: string; + currentPeriodEnd?: Date; + message?: string; +} + +/** + * This relates organizations to products. It holds any stripe information + * needed to be able to update and pay for the product that applies to the + * organization. It has a list of managers detailing which users have the + * right to view and edit these settings. + */ +@Entity({name: 'billing_accounts'}) +export class BillingAccount extends BaseEntity { + @PrimaryGeneratedColumn() + public id: number; + + @ManyToOne(type => Product) + @JoinColumn({name: 'product_id'}) + public product: Product; + + @Column() + public individual: boolean; + + // A flag for when all is well with the user's subscription. + // Probably shouldn't use this to drive whether service is provided or not. + // Strip recommends updating an end-of-service datetime every time payment + // is received, adding on a grace period of some days. + @Column({name: 'in_good_standing', default: nativeValues.trueValue}) + public inGoodStanding: boolean; + + @Column({type: nativeValues.jsonEntityType, nullable: true}) + public status: BillingAccountStatus; + + @Column({name: 'stripe_customer_id', type: String, nullable: true}) + public stripeCustomerId: string | null; + + @Column({name: 'stripe_subscription_id', type: String, nullable: true}) + public stripeSubscriptionId: string | null; + + @Column({name: 'stripe_plan_id', type: String, nullable: true}) + public stripePlanId: string | null; + + @OneToMany(type => BillingAccountManager, manager => manager.billingAccount) + public managers: BillingAccountManager[]; + + @OneToMany(type => Organization, org => org.billingAccount) + public orgs: Organization[]; + + // A calculated column that is true if it looks like there is a paid plan. + @Column({name: 'paid', type: 'boolean', insert: false, select: false}) + public paid?: boolean; + + // A calculated column summarizing whether active user is a manager of the billing account. + // (No @Column needed since calculation is done in javascript not sql) + public isManager?: boolean; +} diff --git a/app/gen-server/entity/BillingAccountManager.ts b/app/gen-server/entity/BillingAccountManager.ts new file mode 100644 index 00000000..7ba18d62 --- /dev/null +++ b/app/gen-server/entity/BillingAccountManager.ts @@ -0,0 +1,26 @@ +import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, PrimaryGeneratedColumn} from 'typeorm'; +import {BillingAccount} from 'app/gen-server/entity/BillingAccount'; +import {User} from 'app/gen-server/entity/User'; + +/** + * A list of users with the right to modify a giving billing account. + */ +@Entity({name: 'billing_account_managers'}) +export class BillingAccountManager extends BaseEntity { + @PrimaryGeneratedColumn() + public id: number; + + @Column({name: 'billing_account_id'}) + public billingAccountId: number; + + @ManyToOne(type => BillingAccount, { onDelete: 'CASCADE' }) + @JoinColumn({name: 'billing_account_id'}) + public billingAccount: BillingAccount; + + @Column({name: 'user_id'}) + public userId: number; + + @ManyToOne(type => User, { onDelete: 'CASCADE' }) + @JoinColumn({name: 'user_id'}) + public user: User; +} diff --git a/app/gen-server/entity/Document.ts b/app/gen-server/entity/Document.ts new file mode 100644 index 00000000..e256efd8 --- /dev/null +++ b/app/gen-server/entity/Document.ts @@ -0,0 +1,69 @@ +import {ApiError} from 'app/common/ApiError'; +import {Role} from 'app/common/roles'; +import {DocumentProperties, documentPropertyKeys, NEW_DOCUMENT_CODE} from "app/common/UserAPI"; +import {nativeValues} from 'app/gen-server/lib/values'; +import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryColumn} from "typeorm"; +import {AclRuleDoc} from "./AclRule"; +import {Alias} from "./Alias"; +import {Resource} from "./Resource"; +import {Workspace} from "./Workspace"; + +// Acceptable ids for use in document urls. +const urlIdRegex = /^[-a-z0-9]+$/i; + +function isValidUrlId(urlId: string) { + if (urlId === NEW_DOCUMENT_CODE) { return false; } + return urlIdRegex.exec(urlId); +} + +@Entity({name: 'docs'}) +export class Document extends Resource { + + @PrimaryColumn() + public id: string; + + @ManyToOne(type => Workspace) + @JoinColumn({name: 'workspace_id'}) + public workspace: Workspace; + + @OneToMany(type => AclRuleDoc, aclRule => aclRule.document) + public aclRules: AclRuleDoc[]; + + // Indicates whether the doc is pinned to the org it lives in. + @Column({name: 'is_pinned', default: false}) + public isPinned: boolean; + + // Property that may be returned when the doc is fetched to indicate the access the + // fetching user has on the doc, i.e. 'owners', 'editors', 'viewers' + public access: Role|null; + + // a computed column with permissions. + // {insert: false} makes sure typeorm doesn't try to put values into such + // a column when creating documents. + @Column({name: 'permissions', type: 'text', select: false, insert: false, update: false}) + public permissions?: any; + + @Column({name: 'url_id', type: 'text', nullable: true}) + public urlId: string|null; + + @Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true}) + public removedAt: Date|null; + + @OneToMany(type => Alias, alias => alias.doc) + public aliases: Alias[]; + + public checkProperties(props: any): props is Partial { + return super.checkProperties(props, documentPropertyKeys); + } + + public updateFromProperties(props: Partial) { + super.updateFromProperties(props); + if (props.isPinned !== undefined) { this.isPinned = props.isPinned; } + if (props.urlId !== undefined) { + if (props.urlId !== null && !isValidUrlId(props.urlId)) { + throw new ApiError('invalid urlId', 400); + } + this.urlId = props.urlId; + } + } +} diff --git a/app/gen-server/entity/Group.ts b/app/gen-server/entity/Group.ts new file mode 100644 index 00000000..b82f9c18 --- /dev/null +++ b/app/gen-server/entity/Group.ts @@ -0,0 +1,33 @@ +import {BaseEntity, Column, Entity, JoinTable, ManyToMany, OneToOne, PrimaryGeneratedColumn} from "typeorm"; + +import {AclRule} from "./AclRule"; +import {User} from "./User"; + +@Entity({name: 'groups'}) +export class Group extends BaseEntity { + + @PrimaryGeneratedColumn() + public id: number; + + @Column() + public name: string; + + @ManyToMany(type => User) + @JoinTable({ + name: 'group_users', + joinColumn: {name: 'group_id'}, + inverseJoinColumn: {name: 'user_id'} + }) + public memberUsers: User[]; + + @ManyToMany(type => Group) + @JoinTable({ + name: 'group_groups', + joinColumn: {name: 'group_id'}, + inverseJoinColumn: {name: 'subgroup_id'} + }) + public memberGroups: Group[]; + + @OneToOne(type => AclRule, aclRule => aclRule.group) + public aclRule: AclRule; +} diff --git a/app/gen-server/entity/Login.ts b/app/gen-server/entity/Login.ts new file mode 100644 index 00000000..7dec4295 --- /dev/null +++ b/app/gen-server/entity/Login.ts @@ -0,0 +1,25 @@ +import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, PrimaryColumn} from "typeorm"; + +import {User} from "./User"; + +@Entity({name: 'logins'}) +export class Login extends BaseEntity { + + @PrimaryColumn() + public id: number; + + // This is the normalized email address we use for equality and indexing. + @Column() + public email: string; + + // This is how the user's email address should be displayed. + @Column({name: 'display_email'}) + public displayEmail: string; + + @Column({name: 'user_id'}) + public userId: number; + + @ManyToOne(type => User) + @JoinColumn({name: 'user_id'}) + public user: User; +} diff --git a/app/gen-server/entity/Organization.ts b/app/gen-server/entity/Organization.ts new file mode 100644 index 00000000..6edbabfb --- /dev/null +++ b/app/gen-server/entity/Organization.ts @@ -0,0 +1,79 @@ +import {Column, Entity, JoinColumn, ManyToOne, OneToMany, OneToOne, + PrimaryGeneratedColumn, RelationId} from "typeorm"; +import {Role} from "app/common/roles"; +import {OrganizationProperties, organizationPropertyKeys} from "app/common/UserAPI"; +import {AclRuleOrg} from "./AclRule"; +import {BillingAccount} from "./BillingAccount"; +import {Resource} from "./Resource"; +import {User} from "./User"; +import {Workspace} from "./Workspace"; + +// Information about how an organization may be accessed. +export interface AccessOption { + id: number; // a user id + email: string; // a user email + name: string; // a user name + perms: number; // permissions the user would have on organization +} + +export interface AccessOptionWithRole extends AccessOption { + access: Role; // summary of permissions +} + +@Entity({name: 'orgs'}) +export class Organization extends Resource { + + @PrimaryGeneratedColumn() + public id: number; + + @Column({ + nullable: true + }) + public domain: string; + + @OneToOne(type => User, user => user.personalOrg) + @JoinColumn({name: 'owner_id'}) + public owner: User; + + @RelationId((org: Organization) => org.owner) + public ownerId: number; + + @OneToMany(type => Workspace, workspace => workspace.org) + public workspaces: Workspace[]; + + @OneToMany(type => AclRuleOrg, aclRule => aclRule.organization) + public aclRules: AclRuleOrg[]; + + @Column({name: 'billing_account_id'}) + public billingAccountId: number; + + @ManyToOne(type => BillingAccount) + @JoinColumn({name: 'billing_account_id'}) + public billingAccount: BillingAccount; + + // Property that may be returned when the org is fetched to indicate the access the + // fetching user has on the org, i.e. 'owners', 'editors', 'viewers' + public access: string; + + // Property that may be used internally to track multiple ways an org can be accessed + public accessOptions?: AccessOptionWithRole[]; + + // a computed column with permissions. + // {insert: false} makes sure typeorm doesn't try to put values into such + // a column when creating organizations. + @Column({name: 'permissions', type: 'text', select: false, insert: false}) + public permissions?: any; + + // For custom domains, this is the preferred host associated with this org/team. + @Column({name: 'host', type: 'text', nullable: true}) + public host: string|null; + + public checkProperties(props: any): props is Partial { + return super.checkProperties(props, organizationPropertyKeys); + } + + public updateFromProperties(props: Partial) { + super.updateFromProperties(props); + if (props.domain) { this.domain = props.domain; } + } +} diff --git a/app/gen-server/entity/Product.ts b/app/gen-server/entity/Product.ts new file mode 100644 index 00000000..a0bc8ced --- /dev/null +++ b/app/gen-server/entity/Product.ts @@ -0,0 +1,176 @@ +import {Features} from 'app/common/Features'; +import {nativeValues} from 'app/gen-server/lib/values'; +import * as assert from 'assert'; +import {BaseEntity, Column, Connection, Entity, PrimaryGeneratedColumn} from 'typeorm'; + +/** + * A summary of features used in 'starter' plans. + */ +export const starterFeatures: Features = { + workspaces: true, + // no vanity domain + maxDocsPerOrg: 10, + maxSharesPerDoc: 2, + maxWorkspacesPerOrg: 1 +}; + +/** + * A summary of features used in 'team' plans. + */ +export const teamFeatures: Features = { + workspaces: true, + vanityDomain: true, + maxSharesPerWorkspace: 0, // all workspace shares need to be org members. + maxSharesPerDoc: 2 +}; + +/** + * A summary of features used in unrestricted grandfathered accounts, and also + * in some test settings. + */ +export const grandfatherFeatures: Features = { + workspaces: true, + vanityDomain: true, +}; + +export const suspendedFeatures: Features = { + workspaces: true, + vanityDomain: true, + readOnlyDocs: true, + // clamp down on new docs/workspaces/shares + maxDocsPerOrg: 0, + maxSharesPerDoc: 0, + maxWorkspacesPerOrg: 0, +}; + +/** + * Basic fields needed for products supported by Grist. + */ +export interface IProduct { + name: string; + features: Features; +} + +/** + * + * Products are a bundle of enabled features. Most products in + * Grist correspond to products in stripe. The correspondence is + * established by a gristProduct metadata field on stripe plans. + * + * In addition, there are the following products in Grist that don't + * exist in stripe: + * - The product named 'Free'. This is a product used for organizations + * created prior to the billing system being set up. + * - The product named 'stub'. This is product assigned to new + * organizations that should not be usable until a paid plan + * is set up for them. + * + * TODO: change capitalization of name of grandfather product. + * + */ +const PRODUCTS: IProduct[] = [ + // This is a product for grandfathered accounts/orgs. + { + name: 'Free', + features: grandfatherFeatures, + }, + + // This is a product for newly created accounts/orgs. + { + name: 'stub', + features: {}, + }, + + // These are products set up in stripe. + { + name: 'starter', + features: starterFeatures, + }, + { + name: 'professional', // deprecated, can be removed once no longer referred to in stripe. + features: teamFeatures, + }, + { + name: 'team', + features: teamFeatures, + }, + + // This is a product for a team site that is no longer in good standing, but isn't yet + // to be removed / deactivated entirely. + { + name: 'suspended', + features: suspendedFeatures, + }, +]; + +/** + * Get names of products for different situations. + */ +export function getDefaultProductNames() { + return { + personal: 'starter', // Personal site start off on a functional plan. + teamInitial: 'stub', // Team site starts off on a limited plan, requiring subscription. + team: 'team', // Functional team site + }; +} + +/** + * A Grist product. Corresponds to a set of enabled features and a choice of limits. + */ +@Entity({name: 'products'}) +export class Product extends BaseEntity { + @PrimaryGeneratedColumn() + public id: number; + + @Column() + public name: string; + + @Column({type: nativeValues.jsonEntityType}) + public features: Features; +} + +/** + * Make sure the products defined for the current stripe setup are + * in the database and up to date. Other products in the database + * are untouched. + * + * If `apply` is set, the products are changed in the db, otherwise + * the are left unchanged. A summary of affected products is returned. + */ +export async function synchronizeProducts(connection: Connection, apply: boolean): Promise { + try { + await connection.query('select name, features, stripe_product_id from products limit 1'); + } catch (e) { + // No usable products table, do not try to synchronize. + return []; + } + const changingProducts: string[] = []; + await connection.transaction(async transaction => { + const desiredProducts = new Map(PRODUCTS.map(p => [p.name, p])); + const existingProducts = new Map((await transaction.find(Product)) + .map(p => [p.name, p])); + for (const product of desiredProducts.values()) { + if (existingProducts.has(product.name)) { + const p = existingProducts.get(product.name)!; + try { + assert.deepStrictEqual(p.features, product.features); + } catch (e) { + if (apply) { + p.features = product.features; + await transaction.save(p); + } + changingProducts.push(p.name); + } + } else { + if (apply) { + const p = new Product(); + p.name = product.name; + p.features = product.features; + await transaction.save(p); + } + changingProducts.push(product.name); + } + } + }); + return changingProducts; +} diff --git a/app/gen-server/entity/Resource.ts b/app/gen-server/entity/Resource.ts new file mode 100644 index 00000000..1958026e --- /dev/null +++ b/app/gen-server/entity/Resource.ts @@ -0,0 +1,46 @@ +import {BaseEntity, Column} from "typeorm"; +import {ApiError} from 'app/common/ApiError'; +import {CommonProperties} from "app/common/UserAPI"; + +export class Resource extends BaseEntity { + @Column() + public name: string; + + @Column({name: 'created_at', default: () => "CURRENT_TIMESTAMP"}) + public createdAt: Date; + + @Column({name: 'updated_at', default: () => "CURRENT_TIMESTAMP"}) + public updatedAt: Date; + + // a computed column which, when present, means the entity should be filtered out + // of results. + @Column({name: 'filtered_out', type: 'boolean', select: false, insert: false}) + public filteredOut?: boolean; + + public updateFromProperties(props: Partial) { + if (props.createdAt) { this.createdAt = _propertyToDate(props.createdAt); } + if (props.updatedAt) { + this.updatedAt = _propertyToDate(props.updatedAt); + } else { + this.updatedAt = new Date(); + } + if (props.name) { this.name = props.name; } + } + + protected checkProperties(props: any, keys: string[]): props is Partial { + for (const key of Object.keys(props)) { + if (!keys.includes(key)) { + throw new ApiError(`unrecognized property ${key}`, 400); + } + } + return true; + } +} + +// Ensure iso-string-or-date value is converted to a date. +function _propertyToDate(d: string|Date): Date { + if (typeof(d) === 'string') { + return new Date(d); + } + return d; +} diff --git a/app/gen-server/entity/User.ts b/app/gen-server/entity/User.ts new file mode 100644 index 00000000..bc661cd8 --- /dev/null +++ b/app/gen-server/entity/User.ts @@ -0,0 +1,54 @@ +import {BaseEntity, Column, Entity, JoinTable, ManyToMany, OneToMany, OneToOne, + PrimaryGeneratedColumn} from "typeorm"; + +import {Group} from "./Group"; +import {Login} from "./Login"; +import {Organization} from "./Organization"; + +@Entity({name: 'users'}) +export class User extends BaseEntity { + + @PrimaryGeneratedColumn() + public id: number; + + @Column() + public name: string; + + @Column({name: 'api_key', type: String, nullable: true}) + // Found how to make a type nullable in this discussion: https://github.com/typeorm/typeorm/issues/2567 + // todo: adds constraint for api_key not to equal '' + public apiKey: string | null; + + @Column({name: 'picture', type: String, nullable: true}) + public picture: string | null; + + @Column({name: 'first_login_at', type: Date, nullable: true}) + public firstLoginAt: Date | null; + + @OneToOne(type => Organization, organization => organization.owner) + public personalOrg: Organization; + + @OneToMany(type => Login, login => login.user) + public logins: Login[]; + + @ManyToMany(type => Group) + @JoinTable({ + name: 'group_users', + joinColumn: {name: 'user_id'}, + inverseJoinColumn: {name: 'group_id'} + }) + public groups: Group[]; + + @Column({name: 'is_first_time_user', default: false}) + public isFirstTimeUser: boolean; + + /** + * Get user's email. Returns undefined if logins has not been joined, or no login + * is available + */ + public get loginEmail(): string|undefined { + const login = this.logins && this.logins[0]; + if (!login) { return undefined; } + return login.email; + } +} diff --git a/app/gen-server/entity/Workspace.ts b/app/gen-server/entity/Workspace.ts new file mode 100644 index 00000000..a8d9ad9d --- /dev/null +++ b/app/gen-server/entity/Workspace.ts @@ -0,0 +1,49 @@ +import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn} from "typeorm"; +import {WorkspaceProperties, workspacePropertyKeys} from "app/common/UserAPI"; +import {nativeValues} from 'app/gen-server/lib/values'; +import {AclRuleWs} from "./AclRule"; +import {Document} from "./Document"; +import {Organization} from "./Organization"; +import {Resource} from "./Resource"; + +@Entity({name: 'workspaces'}) +export class Workspace extends Resource { + + @PrimaryGeneratedColumn() + public id: number; + + @ManyToOne(type => Organization) + @JoinColumn({name: 'org_id'}) + public org: Organization; + + @OneToMany(type => Document, document => document.workspace) + public docs: Document[]; + + @OneToMany(type => AclRuleWs, aclRule => aclRule.workspace) + public aclRules: AclRuleWs[]; + + // Property that may be returned when the workspace is fetched to indicate the access the + // fetching user has on the workspace, i.e. 'owners', 'editors', 'viewers' + public access: string; + + // A computed column that is true if the workspace is a support workspace. + @Column({name: 'support', type: 'boolean', insert: false, select: false}) + public isSupportWorkspace?: boolean; + + // a computed column with permissions. + // {insert: false} makes sure typeorm doesn't try to put values into such + // a column when creating workspaces. + @Column({name: 'permissions', type: 'text', select: false, insert: false}) + public permissions?: any; + + @Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true}) + public removedAt: Date|null; + + public checkProperties(props: any): props is Partial { + return super.checkProperties(props, workspacePropertyKeys); + } + + public updateFromProperties(props: Partial) { + super.updateFromProperties(props); + } +} diff --git a/app/gen-server/lib/DocApiForwarder.ts b/app/gen-server/lib/DocApiForwarder.ts new file mode 100644 index 00000000..3c6c3de3 --- /dev/null +++ b/app/gen-server/lib/DocApiForwarder.ts @@ -0,0 +1,103 @@ +import * as express from "express"; +import fetch, { RequestInit } from 'node-fetch'; + +import { ApiError } from 'app/common/ApiError'; +import { removeTrailingSlash } from 'app/common/gutil'; +import { HomeDBManager } from "app/gen-server/lib/HomeDBManager"; +import { assertAccess, getOrSetDocAuth, getTransitiveHeaders, RequestWithLogin } from 'app/server/lib/Authorizer'; +import { IDocWorkerMap } from "app/server/lib/DocWorkerMap"; +import { expressWrap } from "app/server/lib/expressWrap"; +import { getAssignmentId } from "app/server/lib/idUtils"; + +/** + * Forwards all /api/docs/:docId/tables requests to the doc worker handling the :docId document. Makes + * sure the user has at least view access to the document otherwise rejects the request. For + * performance reason we stream the body directly from the request, which requires that no-one reads + * the req before, in particular you should register DocApiForwarder before bodyParser. + * + * Use: + * const home = new ApiServer(false); + * const docApiForwarder = new DocApiForwarder(getDocWorkerMap(), home); + * app.use(docApiForwarder.getMiddleware()); + * + * Note that it expects userId, and jsonErrorHandler middleware to be set up outside + * to apply to these routes. + */ +export class DocApiForwarder { + + constructor(private _docWorkerMap: IDocWorkerMap, private _dbManager: HomeDBManager) { + } + + public addEndpoints(app: express.Application) { + // Middleware to forward a request about an existing document that user has access to. + // We do not check whether the document has been soft-deleted; that will be checked by + // the worker if needed. + const withDoc = expressWrap(this._forwardToDocWorker.bind(this, true)); + // Middleware to forward a request without a pre-existing document (for imports/uploads). + const withoutDoc = expressWrap(this._forwardToDocWorker.bind(this, false)); + app.use('/api/docs/:docId/tables', withDoc); + app.use('/api/docs/:docId/force-reload', withDoc); + app.use('/api/docs/:docId/remove', withDoc); + app.delete('/api/docs/:docId', withDoc); + app.use('/api/docs/:docId/download', withDoc); + app.use('/api/docs/:docId/apply', withDoc); + app.use('/api/docs/:docId/attachments', withDoc); + app.use('/api/docs/:docId/snapshots', withDoc); + app.use('/api/docs/:docId/replace', withDoc); + app.use('/api/docs/:docId/flush', withDoc); + app.use('/api/docs/:docId/states', withDoc); + app.use('/api/docs/:docId/compare', withDoc); + app.use('^/api/docs$', withoutDoc); + } + + private async _forwardToDocWorker(withDocId: boolean, req: express.Request, res: express.Response): Promise { + let docId: string|null = null; + if (withDocId) { + const docAuth = await getOrSetDocAuth(req as RequestWithLogin, this._dbManager, req.params.docId); + assertAccess('viewers', docAuth, {allowRemoved: true}); + docId = docAuth.docId; + } + // Use the docId for worker assignment, rather than req.params.docId, which could be a urlId. + const assignmentId = getAssignmentId(this._docWorkerMap, docId === null ? 'import' : docId); + + if (!this._docWorkerMap) { + throw new ApiError('no worker map', 404); + } + const docStatus = await this._docWorkerMap.assignDocWorker(assignmentId); + + // Construct new url by keeping only origin and path prefixes of `docWorker.internalUrl`, + // and otherwise reflecting fully the original url (remaining path, and query params). + const docWorkerUrl = new URL(docStatus.docWorker.internalUrl); + const url = new URL(req.originalUrl, docWorkerUrl.origin); + url.pathname = removeTrailingSlash(docWorkerUrl.pathname) + url.pathname; + + const headers: {[key: string]: string} = { + ...getTransitiveHeaders(req), + 'Content-Type': req.get('Content-Type') || 'application/json', + }; + for (const key of ['X-Sort', 'X-Limit']) { + const hdr = req.get(key); + if (hdr) { headers[key] = hdr; } + } + const options: RequestInit = { + method: req.method, + headers, + }; + if (['POST', 'PATCH'].includes(req.method)) { + // uses `req` as a stream + options.body = req; + } + const docWorkerRes = await fetch(url.href, options); + res.status(docWorkerRes.status); + for (const key of ['content-type', 'content-disposition', 'cache-control']) { + const value = docWorkerRes.headers.get(key); + if (value) { res.set(key, value); } + } + return new Promise((resolve, reject) => { + docWorkerRes.body.on('error', reject); + res.on('error', reject); + res.on('finish', resolve); + docWorkerRes.body.pipe(res); + }); + } +} diff --git a/app/gen-server/lib/DocWorkerMap.ts b/app/gen-server/lib/DocWorkerMap.ts new file mode 100644 index 00000000..c9dfa6a2 --- /dev/null +++ b/app/gen-server/lib/DocWorkerMap.ts @@ -0,0 +1,440 @@ +import {MapWithTTL} from 'app/common/AsyncCreate'; +import * as version from 'app/common/version'; +import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; +import * as log from 'app/server/lib/log'; +import {checkPermitKey, formatPermitKey, Permit} from 'app/server/lib/Permit'; +import {promisifyAll} from 'bluebird'; +import mapValues = require('lodash/mapValues'); +import {createClient, Multi, RedisClient} from 'redis'; +import * as Redlock from 'redlock'; +import * as uuidv4 from 'uuid/v4'; + +promisifyAll(RedisClient.prototype); +promisifyAll(Multi.prototype); + +// Max time for which we will hold a lock, by default. In milliseconds. +const LOCK_TIMEOUT = 3000; + +// How long do checksums stored in redis last. In milliseconds. +// Should be long enough to allow S3 to reach consistency with very high probability. +// Consistency failures shorter than this interval will be detectable, failures longer +// than this interval will not be detectable. +const CHECKSUM_TTL_MSEC = 24 * 60 * 60 * 1000; // 24 hours + +// How long do permits stored in redis last, in milliseconds. +const PERMIT_TTL_MSEC = 1 * 60 * 1000; // 1 minute + +class DummyDocWorkerMap implements IDocWorkerMap { + private _worker?: DocWorkerInfo; + private _available: boolean = false; + private _permits = new MapWithTTL(PERMIT_TTL_MSEC); + private _elections = new MapWithTTL(1); // default ttl never used + + public async getDocWorker(docId: string) { + if (!this._worker) { throw new Error('no workers'); } + return {docMD5: 'unknown', docWorker: this._worker, isActive: true}; + } + + public async assignDocWorker(docId: string) { + if (!this._worker || !this._available) { throw new Error('no workers'); } + return {docMD5: 'unknown', docWorker: this._worker, isActive: true}; + } + + public async getDocWorkerOrAssign(docId: string, workerId: string): Promise { + if (!this._worker || !this._available) { throw new Error('no workers'); } + if (this._worker.id !== workerId) { throw new Error('worker not known'); } + return {docMD5: 'unknown', docWorker: this._worker, isActive: true}; + } + + public async updateDocStatus(docId: string, checksum: string) { + // nothing to do + } + + public async addWorker(info: DocWorkerInfo): Promise { + this._worker = info; + } + + public async removeWorker(workerId: string): Promise { + this._worker = undefined; + } + + public async setWorkerAvailability(workerId: string, available: boolean): Promise { + this._available = available; + } + + public async releaseAssignment(workerId: string, docId: string): Promise { + // nothing to do + } + + public async getAssignments(workerId: string): Promise { + return []; + } + + public async setPermit(permit: Permit): Promise { + const key = formatPermitKey(uuidv4()); + this._permits.set(key, JSON.stringify(permit)); + return key; + } + + public async getPermit(key: string): Promise { + const result = this._permits.get(key); + return result ? JSON.parse(result) : null; + } + + public async removePermit(key: string): Promise { + this._permits.delete(key); + } + + public async close(): Promise { + this._permits.clear(); + this._elections.clear(); + } + + public async getElection(name: string, durationInMs: number): Promise { + if (this._elections.get(name)) { return null; } + const key = uuidv4(); + this._elections.setWithCustomTTL(name, key, durationInMs); + return key; + } + + public async removeElection(name: string, electionKey: string): Promise { + if (this._elections.get(name) === electionKey) { + this._elections.delete(name); + } + } +} + +/** + * Manage the relationship between document and workers. Backed by Redis. + * Can also assign workers to "groups" for serving particular documents. + * Keys used: + * workers - the set of known active workers, identified by workerId + * workers-available - the set of workers available for assignment (a subset of the workers set) + * workers-available-{group} - the set of workers available for a given group + * worker-{workerId} - a hash of contact information for a worker + * worker-{workerId}-docs - a set of docs assigned to a worker, identified by docId + * worker-{workerId}-group - if set, marks the worker as serving a particular group + * doc-${docId} - a hash containing (JSON serialized) DocStatus fields, other than docMD5. + * doc-${docId}-checksum - the docs docMD5, or 'null' if docMD5 is null + * doc-${docId}-group - if set, marks the doc as to be served by workers in a given group + * workers-lock - a lock used when working with the list of workers + * groups - a hash from groupIds (arbitrary strings) to desired number of workers in group + * elections-${deployment} - a hash, from groupId to a (serialized json) list of worker ids + * + * Assignments of documents to workers can end abruptly at any time. Clients + * should be prepared to retry if a worker is not responding or denies that a document + * is assigned to it. + * + * If the groups key is set, workers assign themselves to groupIds to + * fill the counts specified in groups (in order of groupIds), and + * once those are exhausted, get assigned to the special group + * "default". + */ +export class DocWorkerMap implements IDocWorkerMap { + private _client: RedisClient; + private _redlock: Redlock; + + // Optional deploymentKey argument supplies a key unique to the deployment (this is important + // for maintaining groups across redeployments only) + constructor(_clients?: RedisClient[], private _deploymentKey?: string, private _options?: { + permitMsec?: number + }) { + this._deploymentKey = this._deploymentKey || version.version; + _clients = _clients || [createClient(process.env.REDIS_URL)]; + this._redlock = new Redlock(_clients); + this._client = _clients[0]!; + } + + public async addWorker(info: DocWorkerInfo): Promise { + log.info(`DocWorkerMap.addWorker ${info.id}`); + const lock = await this._redlock.lock('workers-lock', LOCK_TIMEOUT); + try { + // Make a worker-{workerId} key with contact info, then add this worker to available set. + await this._client.hmsetAsync(`worker-${info.id}`, info); + await this._client.saddAsync('workers', info.id); + // Figure out if worker should belong to a group + const groups = await this._client.hgetallAsync('groups'); + if (groups) { + const elections = await this._client.hgetallAsync(`elections-${this._deploymentKey}`) || {}; + for (const group of Object.keys(groups).sort()) { + const count = parseInt(groups[group], 10) || 0; + if (count < 1) { continue; } + const elected: string[] = JSON.parse(elections[group] || '[]'); + if (elected.length >= count) { continue; } + elected.push(info.id); + await this._client.setAsync(`worker-${info.id}-group`, group); + await this._client.hsetAsync(`elections-${this._deploymentKey}`, group, JSON.stringify(elected)); + break; + } + } + } finally { + await lock.unlock(); + } + } + + public async removeWorker(workerId: string): Promise { + log.info(`DocWorkerMap.removeWorker ${workerId}`); + const lock = await this._redlock.lock('workers-lock', LOCK_TIMEOUT); + try { + // Drop out of available set first. + await this._client.sremAsync('workers-available', workerId); + const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default'; + await this._client.sremAsync(`workers-available-${group}`, workerId); + // At this point, this worker should no longer be receiving new doc assignments, though + // clients may still be directed to the worker. + + // If we were elected for anything, back out. + const elections = await this._client.hgetallAsync(`elections-${this._deploymentKey}`); + if (elections) { + if (group in elections) { + const elected: string[] = JSON.parse(elections[group]); + const newElected = elected.filter(worker => worker !== workerId); + if (elected.length !== newElected.length) { + if (newElected.length > 0) { + await this._client.hsetAsync(`elections-${this._deploymentKey}`, group, + JSON.stringify(newElected)); + } else { + await this._client.hdelAsync(`elections-${this._deploymentKey}`, group); + delete elections[group]; + } + } + // We're the last one involved in elections - remove the key entirely. + if (Object.keys(elected).length === 0) { + await this._client.delAsync(`elections-${this._deploymentKey}`); + } + } + } + + // Now, we start removing the assignments. + const assignments = await this._client.smembersAsync(`worker-${workerId}-docs`); + if (assignments) { + const op = this._client.multi(); + for (const doc of assignments) { op.del(`doc-${doc}`); } + await op.execAsync(); + } + + // Now remove worker-{workerId}* keys. + await this._client.delAsync(`worker-${workerId}-docs`); + await this._client.delAsync(`worker-${workerId}-group`); + await this._client.delAsync(`worker-${workerId}`); + + // Forget about this worker completely. + await this._client.sremAsync('workers', workerId); + } finally { + await lock.unlock(); + } + } + + public async setWorkerAvailability(workerId: string, available: boolean): Promise { + log.info(`DocWorkerMap.setWorkerAvailability ${workerId} ${available}`); + const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default'; + if (available) { + await this._client.saddAsync(`workers-available-${group}`, workerId); + await this._client.saddAsync('workers-available', workerId); + } else { + await this._client.sremAsync('workers-available', workerId); + await this._client.sremAsync(`workers-available-${group}`, workerId); + } + } + + public async releaseAssignment(workerId: string, docId: string): Promise { + const op = this._client.multi(); + op.del(`doc-${docId}`); + op.srem(`worker-${workerId}-docs`, docId); + await op.execAsync(); + } + + public async getAssignments(workerId: string): Promise { + return this._client.smembersAsync(`worker-${workerId}-docs`); + } + + /** + * Defined by IDocWorkerMap. + * + * Looks up which DocWorker is responsible for this docId. + * Responsibility could change at any time after this call, so it + * should be treated as a hint, and clients should be prepared to be + * refused and need to retry. + */ + public async getDocWorker(docId: string): Promise { + // Fetch the various elements that go into making a DocStatus + const props = await this._client.multi() + .hgetall(`doc-${docId}`) + .get(`doc-${docId}-checksum`) + .execAsync() as [{[key: string]: any}|null, string|null]|null; + if (!props) { return null; } + + // If there is no worker, return null. An alternative would be to modify + // DocStatus so that it is possible for it to not have a worker assignment. + if (!props[0]) { return null; } + + // Fields are JSON encoded since redis cannot store them directly. + const doc = mapValues(props[0], (val) => JSON.parse(val)); + + // Redis cannot store a null value, so we encode it as 'null', which does + // not match any possible MD5. + doc.docMD5 = props[1] === 'null' ? null : props[1]; + + // Ok, we have a valid DocStatus at this point. + return doc as DocStatus; + } + + /** + * + * Defined by IDocWorkerMap. + * + * Assigns a DocWorker to this docId if one is not yet assigned. + * Note that the assignment could be unmade at any time after this + * call if the worker dies, is brought down, or for other potential + * reasons in the future such as migration of individual documents + * between workers. + * + * A preferred doc worker can be specified, which will be assigned + * if no assignment is already made. + * + */ + public async assignDocWorker(docId: string, workerId?: string): Promise { + // Check if a DocWorker is already assigned; if so return result immediately + // without locking. + let docStatus = await this.getDocWorker(docId); + if (docStatus) { return docStatus; } + + // No assignment yet, so let's lock and set an assignment up. + const lock = await this._redlock.lock(`workers-lock`, LOCK_TIMEOUT); + + try { + // Now that we've locked, recheck that the worker hasn't been reassigned + // in the meantime. Return immediately if it has. + docStatus = await this.getDocWorker(docId); + if (docStatus) { return docStatus; } + + if (!workerId) { + // Check if document has a preferred worker group set. + const group = await this._client.getAsync(`doc-${docId}-group`) || 'default'; + + // Let's start off by assigning documents to available workers randomly. + // TODO: use a smarter algorithm. + workerId = await this._client.srandmemberAsync(`workers-available-${group}`) || undefined; + if (!workerId) { + // No workers available in the desired worker group. Rather than refusing to + // open the document, we fall back on assigning a worker from any of the workers + // available, regardless of grouping. + // This limits the impact of operational misconfiguration (bad redis setup, + // or not starting enough workers). It has the downside of potentially disguising + // problems, so we log a warning. + log.warn(`DocWorkerMap.assignDocWorker ${docId} found no workers for group ${group}`); + workerId = await this._client.srandmemberAsync('workers-available') || undefined; + } + if (!workerId) { throw new Error('no doc workers available'); } + } else { + if (!await this._client.sismemberAsync('workers-available', workerId)) { + throw new Error(`worker ${workerId} not known or not available`); + } + } + + // Look up how to contact the worker. + const docWorker = await this._client.hgetallAsync(`worker-${workerId}`) as DocWorkerInfo|null; + if (!docWorker) { throw new Error('no doc worker contact info available'); } + + // We can now construct a DocStatus. + const newDocStatus = {docMD5: null, docWorker, isActive: true}; + + // We add the assignment to worker-{workerId}-docs and save doc-{docId}. + const result = await this._client.multi() + .sadd(`worker-${workerId}-docs`, docId) + .hmset(`doc-${docId}`, { + docWorker: JSON.stringify(docWorker), // redis can't store nested objects, strings only + isActive: JSON.stringify(true) // redis can't store booleans, strings only + }) + .setex(`doc-${docId}-checksum`, CHECKSUM_TTL_MSEC / 1000.0, 'null') + .execAsync(); + if (!result) { throw new Error('failed to store new assignment'); } + return newDocStatus; + } finally { + await lock.unlock(); + } + } + + /** + * + * Defined by IDocWorkerMap. + * + * Assigns a specific DocWorker to this docId if one is not yet assigned. + * + */ + public async getDocWorkerOrAssign(docId: string, workerId: string): Promise { + return this.assignDocWorker(docId, workerId); + } + + public async updateDocStatus(docId: string, checksum: string): Promise { + await this._client.setexAsync(`doc-${docId}-checksum`, CHECKSUM_TTL_MSEC / 1000.0, checksum); + } + + public async setPermit(permit: Permit): Promise { + const key = formatPermitKey(uuidv4()); + const duration = (this._options && this._options.permitMsec) || PERMIT_TTL_MSEC; + // seems like only integer seconds are supported? + await this._client.setexAsync(key, Math.ceil(duration / 1000.0), + JSON.stringify(permit)); + return key; + } + + public async getPermit(key: string): Promise { + if (!checkPermitKey(key)) { throw new Error('permit could not be read'); } + const result = await this._client.getAsync(key); + return result && JSON.parse(result); + } + + public async removePermit(key: string): Promise { + if (!checkPermitKey(key)) { throw new Error('permit could not be read'); } + await this._client.delAsync(key); + } + + public async close(): Promise { + // nothing to do + } + + public async getElection(name: string, durationInMs: number): Promise { + // Could use "set nx" for election, but redis docs don't encourage that any more, + // favoring redlock: + // https://redis.io/commands/setnx#design-pattern-locking-with-codesetnxcode + const redisKey = `nomination-${name}`; + const lock = await this._redlock.lock(`${redisKey}-lock`, LOCK_TIMEOUT); + try { + if (await this._client.getAsync(redisKey) !== null) { return null; } + const electionKey = uuidv4(); + // seems like only integer seconds are supported? + await this._client.setexAsync(redisKey, Math.ceil(durationInMs / 1000.0), electionKey); + return electionKey; + } finally { + await lock.unlock(); + } + } + + public async removeElection(name: string, electionKey: string): Promise { + const redisKey = `nomination-${name}`; + const lock = await this._redlock.lock(`${redisKey}-lock`, LOCK_TIMEOUT); + try { + const current = await this._client.getAsync(redisKey); + if (current === electionKey) { + await this._client.delAsync(redisKey); + } else if (current !== null) { + throw new Error('could not remove election'); + } + } finally { + await lock.unlock(); + } + } +} + +// If we don't have redis available and use a DummyDocWorker, it should be a singleton. +let dummyDocWorkerMap: DummyDocWorkerMap|null = null; + +export function getDocWorkerMap(): IDocWorkerMap { + if (process.env.REDIS_URL) { + return new DocWorkerMap(); + } else { + dummyDocWorkerMap = dummyDocWorkerMap || new DummyDocWorkerMap(); + return dummyDocWorkerMap; + } +} diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts new file mode 100644 index 00000000..9cbde39e --- /dev/null +++ b/app/gen-server/lib/HomeDBManager.ts @@ -0,0 +1,3706 @@ +import {ApiError} from 'app/common/ApiError'; +import {mapGetOrSet, mapSetOrClear, MapWithTTL} from 'app/common/AsyncCreate'; +import {normalizeEmail} from 'app/common/emails'; +import {canAddOrgMembers, Features} from 'app/common/Features'; +import {buildUrlId, MIN_URLID_PREFIX_LENGTH, parseUrlId} from 'app/common/gristUrls'; +import {FullUser, UserProfile} from 'app/common/LoginSessionAPI'; +import {checkSubdomainValidity} from 'app/common/orgNameUtils'; +import * as roles from 'app/common/roles'; +// TODO: API should implement UserAPI +import {ANONYMOUS_USER_EMAIL, DocumentProperties, ManagerDelta, NEW_DOCUMENT_CODE, Organization as OrgInfo, + OrganizationProperties, PermissionData, PermissionDelta, SUPPORT_EMAIL, UserAccessData, + WorkspaceProperties} from "app/common/UserAPI"; +import {AclRule, AclRuleDoc, AclRuleOrg, AclRuleWs} from "app/gen-server/entity/AclRule"; +import {Alias} from "app/gen-server/entity/Alias"; +import {BillingAccount} from "app/gen-server/entity/BillingAccount"; +import {BillingAccountManager} from "app/gen-server/entity/BillingAccountManager"; +import {Document} from "app/gen-server/entity/Document"; +import {Group} from "app/gen-server/entity/Group"; +import {Login} from "app/gen-server/entity/Login"; +import {AccessOption, AccessOptionWithRole, Organization} from "app/gen-server/entity/Organization"; +import {getDefaultProductNames, Product, starterFeatures} from "app/gen-server/entity/Product"; +import {User} from "app/gen-server/entity/User"; +import {Workspace} from "app/gen-server/entity/Workspace"; +import {Permissions} from 'app/gen-server/lib/Permissions'; +import {scrubUserFromOrg} from "app/gen-server/lib/scrubUserFromOrg"; +import {applyPatch} from 'app/gen-server/lib/TypeORMPatches'; +import {bitOr, now, readJson} from 'app/gen-server/sqlUtils'; +import {makeId} from 'app/server/lib/idUtils'; +import * as log from 'app/server/lib/log'; +import {Permit} from 'app/server/lib/Permit'; +import {EventEmitter} from 'events'; +import flatten = require('lodash/flatten'); +import pick = require('lodash/pick'); +import {Brackets, Connection, createConnection, DatabaseType, EntityManager, + getConnection, SelectQueryBuilder, WhereExpression} from "typeorm"; + +// Support transactions in Sqlite in async code. This is a monkey patch, affecting +// the prototypes of various TypeORM classes. +// TODO: remove this patch if the issue is ever accepted as a problem in TypeORM and +// fixed. See https://github.com/typeorm/typeorm/issues/1884#issuecomment-380767213 +applyPatch(); + +// Nominal email address of a user who can view anything (for thumbnails). +export const PREVIEWER_EMAIL = 'thumbnail@getgrist.com'; + +// Nominal email address of a user who, if you share with them, everyone gets access. +export const EVERYONE_EMAIL = 'everyone@getgrist.com'; + +// A list of emails we don't expect to see logins for. +const NON_LOGIN_EMAILS = [PREVIEWER_EMAIL, EVERYONE_EMAIL, ANONYMOUS_USER_EMAIL]; + +// Name of a special workspace with examples in it. +export const EXAMPLE_WORKSPACE_NAME = 'Examples & Templates'; + +// A TTL in milliseconds for caching the result of looking up access level for a doc, +// which is a burden under heavy traffic. +const DOC_AUTH_CACHE_TTL = 5000; + +type Resource = Organization|Workspace|Document; + +export interface QueryResult { + status: number; + data?: T; + errMessage?: string; +} + +// Maps from userId to group name, or null to inherit. +export interface UserIdDelta { + [userId: string]: roles.NonGuestRole|null; +} + +// Options for certain create query helpers private to this file. +interface QueryOptions { + manager?: EntityManager; + markPermissions?: Permissions; + needRealOrg?: boolean; // Set if pseudo-org should be collapsed to user's personal org + allowSpecialPermit?: boolean; // Set if specialPermit in Scope object should be respected, + // potentially overriding markPermissions. +} + +interface GroupDescriptor { + readonly name: roles.Role; + readonly permissions: number; + readonly nestParent: boolean; + readonly orgOnly?: boolean; +} + +// Information about a change in billable users. +export interface UserChange { + userId: number; // who initiated the change + org: Organization; // organization changed + customerId: string|null; // stripe customer id + countBefore: number; // billable users before change + countAfter: number; // billable users after change + membersBefore: Map; + membersAfter: Map; +} + +// A specification of the users available during a request. This can be a single +// user, identified by a user id, or a collection of profiles (typically drawn from +// the session). +type AvailableUsers = number | UserProfile[]; + +// A type guard to check for single-user case. +function isSingleUser(users: AvailableUsers): users is number { + return typeof users === 'number'; +} + +// The context in which a query is being made. Includes what we know +// about the user, and for requests made from pages, the active organization. +export interface Scope { + userId: number; // The ID of the user for authentication purposes. + org?: string; // Org identified in request. + urlId?: string; // Set when accessing a document. May be a docId. + users?: AvailableUsers; // Set if available identities. + includeSupport?: boolean; // When set, include sample resources shared by support to scope. + showRemoved?: boolean; // When set, query is scoped to removed workspaces/docs. + showAll?: boolean; // When set, return both removed and regular resources. + specialPermit?: Permit; // When set, extra rights are granted on a specific resource. +} + +// A Scope for documents, with mandatory urlId. +export interface DocScope extends Scope { + urlId: string; +} + +type NonGuestGroup = Group & { name: roles.NonGuestRole }; + +// Returns whether the given group is a valid non-guest group. +function isNonGuestGroup(group: Group): group is NonGuestGroup { + return roles.isNonGuestRole(group.name); +} + +export interface UserProfileChange { + name?: string; + isFirstTimeUser?: boolean; +} + +// Identifies a request to access a document. This combination of values is also used for caching +// DocAuthResult for DOC_AUTH_CACHE_TTL. Other request scope information is passed along. +export interface DocAuthKey { + urlId: string; // May be docId. Must be unambiguous in the context of the org. + userId: number; // The user accessing this doc. (Could be the ID of Anonymous.) + org?: string; // Undefined if unknown (e.g. in API calls, but needs unique urlId). +} + +// Document auth info. This is the minimum needed to resolve user access checks. For anything else +// (e.g. doc title), the uncached getDoc() call should be used. +export interface DocAuthResult { + docId: string|null; // The unique identifier of the document. Null on error. + access: roles.Role|null; // The access level for the requesting user. Null on error. + removed: boolean|null; // Set if the doc is soft-deleted. Users may still have access + // to removed documents for some purposes. Null on error. + error?: ApiError; +} + +// Represent a DocAuthKey as a string. The format is ": ". +// flushSingleDocAuthCache() depends on this format. +function stringifyDocAuthKey(key: DocAuthKey): string { + return stringifyUrlIdOrg(key.urlId, key.org) + ` ${key.userId}`; +} + +function stringifyUrlIdOrg(urlId: string, org?: string): string { + return `${urlId}:${org}`; +} + +/** + * HomeDBManager handles interaction between the ApiServer and the Home database, + * encapsulating the typeorm logic. + */ +export class HomeDBManager extends EventEmitter { + private _connection: Connection; + private _dbType: DatabaseType; + private _specialUserIds: {[name: string]: number} = {}; // id for anonymous user, previewer, etc + private _exampleWorkspaceId: number; + private _exampleOrgId: number; + private _idPrefix: string = ""; // Place this before ids in subdomains, used in routing to + // deployments on same subdomain. + + private _docAuthCache = new MapWithTTL>(DOC_AUTH_CACHE_TTL); + + /** + * Five aclRules, each with one group (with the names 'owners', 'editors', 'viewers', + * 'guests', and 'members') are created by default on every new entity (Organization, + * Workspace, Document). These special groups are documented in the _defaultGroups + * constant below. + * + * When a child resource is created under a parent (i.e. when a new Workspace is created + * under an Organization), special groups with a truthy 'nestParent' property are set up + * to include in their memberGroups a single group on initialization - the parent's + * corresponding special group. Special groups with a falsy 'nextParent' property are + * empty on intialization. + * + * NOTE: The groups are ordered from most to least permissive, and should remain that way. + * TODO: app/common/roles already contains an ordering of the default roles. Usage should + * be consolidated. + */ + private readonly _defaultGroups: GroupDescriptor[] = [{ + name: roles.OWNER, + permissions: Permissions.OWNER, + nestParent: true + }, { + name: roles.EDITOR, + permissions: Permissions.EDITOR, + nestParent: true + }, { + name: roles.VIEWER, + permissions: Permissions.VIEW, + nestParent: true + }, { + name: roles.GUEST, + permissions: Permissions.VIEW, + nestParent: false + }, { + name: roles.MEMBER, + permissions: Permissions.VIEW, + nestParent: false, + orgOnly: true + }]; + + // All groups. + public get defaultGroups(): GroupDescriptor[] { + return this._defaultGroups; + } + + // Groups whose permissions are inherited from parent resource to child resources. + public get defaultBasicGroups(): GroupDescriptor[] { + return this._defaultGroups + .filter(_grpDesc => _grpDesc.nestParent); + } + + // Groups that are common to all resources. + public get defaultCommonGroups(): GroupDescriptor[] { + return this._defaultGroups + .filter(_grpDesc => !_grpDesc.orgOnly); + } + + public get defaultGroupNames(): roles.Role[] { + return this._defaultGroups.map(_grpDesc => _grpDesc.name); + } + + public get defaultBasicGroupNames(): roles.BasicRole[] { + return this.defaultBasicGroups + .map(_grpDesc => _grpDesc.name) as roles.BasicRole[]; + } + + public get defaultNonGuestGroupNames(): roles.NonGuestRole[] { + return this._defaultGroups + .filter(_grpDesc => _grpDesc.name !== roles.GUEST) + .map(_grpDesc => _grpDesc.name) as roles.NonGuestRole[]; + } + + public get defaultCommonGroupNames(): roles.NonMemberRole[] { + return this.defaultCommonGroups + .map(_grpDesc => _grpDesc.name) as roles.NonMemberRole[]; + } + + public setPrefix(prefix: string) { + this._idPrefix = prefix; + } + + public async connect(): Promise { + try { + // If multiple servers are started within the same process, we + // share the database connection. This saves locking trouble + // with Sqlite. + this._connection = getConnection(); + } catch (e) { + this._connection = await createConnection(); + } + this._dbType = this._connection.driver.options.type; + } + + // make sure special users and workspaces are available + public async initializeSpecialIds(): Promise { + await this._getSpecialUserId({ + email: ANONYMOUS_USER_EMAIL, + name: "Anonymous" + }); + await this._getSpecialUserId({ + email: PREVIEWER_EMAIL, + name: "Preview" + }); + await this._getSpecialUserId({ + email: EVERYONE_EMAIL, + name: "Everyone" + }); + await this._getSpecialUserId({ + email: SUPPORT_EMAIL, + name: "Support" + }); + + // Find the example workspace. If there isn't one named just right, take the first workspace + // belonging to the support user. This shouldn't happen in deployments but could happen + // in tests. + const supportWorkspaces = await this._workspaces() + .leftJoinAndSelect('workspaces.org', 'orgs') + .where('orgs.owner_id = :userId', { userId: this.getSupportUserId() }) + .orderBy('workspaces.created_at') + .getMany(); + const exampleWorkspace = supportWorkspaces.find(ws => ws.name === EXAMPLE_WORKSPACE_NAME) || supportWorkspaces[0]; + if (!exampleWorkspace) { throw new Error('No example workspace available'); } + if (exampleWorkspace.name !== EXAMPLE_WORKSPACE_NAME) { + log.warn('did not find an appropriately named example workspace in deployment'); + } + this._exampleWorkspaceId = exampleWorkspace.id; + this._exampleOrgId = exampleWorkspace.org.id; + } + + public get connection() { + return this._connection; + } + + public async testQuery(sql: string, args: any[]): Promise { + return this._connection.query(sql, args); + } + + /** + * Maps from the name of an entity to its id, for the purposes of + * unit tests only. It relies on test entities being named + * distinctly. It just runs through each model in turn by brute + * force, and returns the id of this first match it finds. + */ + public async testGetId(name: string): Promise { + const org = await Organization.findOne({name}); + if (org) { return org.id; } + const ws = await Workspace.findOne({name}); + if (ws) { return ws.id; } + const doc = await Document.findOne({name}); + if (doc) { return doc.id; } + const user = await User.findOne({name}); + if (user) { return user.id; } + const product = await Product.findOne({name}); + if (product) { return product.id; } + throw new Error(`Cannot testGetId(${name})`); + } + + public getUserByKey(apiKey: string): Promise { + return User.findOne({apiKey}); + } + + public getUser(userId: number): Promise { + return User.findOne(userId); + } + + public async getFullUser(userId: number): Promise { + const user = await User.findOne(userId, {relations: ["logins"]}); + if (!user) { throw new ApiError("unable to find user", 400); } + return this.makeFullUser(user); + } + + /** + * Convert a user record into the format specified in api. + */ + public makeFullUser(user: User): FullUser { + if (!(user.logins && user.logins[0].displayEmail)) { + throw new ApiError("unable to find mandatory user email", 400); + } + return { + id: user.id, + email: user.logins[0].displayEmail, + name: user.name, + picture: user.picture, + anonymous: this.getAnonymousUserId() === user.id + }; + } + + public async updateUser(userId: number, props: UserProfileChange): Promise { + let isWelcomed: boolean = false; + let user: User|undefined; + await this._connection.transaction(async manager => { + user = await manager.findOne(User, {relations: ['logins'], + where: {id: userId}}); + let needsSave = false; + if (!user) { throw new ApiError("unable to find user", 400); } + if (props.name && props.name !== user.name) { + user.name = props.name; + needsSave = true; + } + if (props.isFirstTimeUser !== undefined && props.isFirstTimeUser !== user.isFirstTimeUser) { + user.isFirstTimeUser = props.isFirstTimeUser; + needsSave = true; + // If we are turning off the isFirstTimeUser flag, then right + // after this transaction commits is a great time to trigger + // any automation for first logins - the user has logged in + // and gone through the welcome process (so they've had a + // chance to set a name) + if (!props.isFirstTimeUser) { isWelcomed = true; } + } + if (needsSave) { + await user.save(); + } + }); + if (user && isWelcomed) { + this.emit('firstLogin', this.makeFullUser(user)); + } + } + + public async updateUserName(userId: number, name: string) { + const user = await User.findOne(userId); + if (!user) { throw new ApiError("unable to find user", 400); } + user.name = name; + await user.save(); + } + + // Fetch user from login, creating the user if previously unseen, allowing one retry + // for an email key conflict failure. This is in case our transaction conflicts with a peer + // doing the same thing. This is quite likely if the first page visited by a previously + // unseen user fires off multiple api calls. + public async getUserByLoginWithRetry(email: string, profile: UserProfile): Promise { + try { + return await this.getUserByLogin(email, profile); + } catch (e) { + if (e.name === 'QueryFailedError' && e.detail && + e.detail.match(/Key \(email\)=[^ ]+ already exists/)) { + // This is a postgres-specific error message. This problem cannot arise in sqlite, + // because we have to serialize sqlite transactions in any case to get around a typeorm + // limitation. + return await this.getUserByLogin(email, profile); + } + throw e; + } + } + + /** + * + * Fetches a user record based on an email address. If a user record already + * exists linked to the email address supplied, that is the record returned. + * Otherwise a fresh record is created, linked to the supplied email address. + * The name supplied is used to create this fresh record - otherwise it is + * ignored. + * + */ + public async getUserByLogin( + email: string, + profile?: UserProfile, + transaction?: EntityManager + ): Promise { + const normalizedEmail = normalizeEmail(email); + const userByLogin = await this._runInTransaction(transaction, async manager => { + let needUpdate = false; + const userQuery = manager.createQueryBuilder() + .select('user') + .from(User, 'user') + .leftJoinAndSelect('user.logins', 'logins') + .leftJoinAndSelect('user.personalOrg', 'personalOrg') + .where('email = :email', {email: normalizedEmail}); + let user = await userQuery.getOne(); + let login: Login; + if (!user) { + user = new User(); + // Special users do not have first time user set so that they don't get redirected to the + // welcome page. + user.isFirstTimeUser = !NON_LOGIN_EMAILS.includes(normalizedEmail); + login = new Login(); + login.email = normalizedEmail; + login.user = user; + needUpdate = true; + } else { + login = user.logins[0]; + } + + // Check that user and login records are up to date. + if (!user.name) { + // Set the user's name if our provider knows it. Otherwise use their username + // from email, for lack of something better. If we don't have a profile at this + // time, then leave the name blank in the hopes of learning it when the user logs in. + user.name = (profile && (profile.name || email.split('@')[0])) || ''; + needUpdate = true; + } + if (profile && !user.firstLoginAt) { + // set first login time to now (remove milliseconds for compatibility with other + // timestamps in db set by typeorm, and since second level precision is fine) + const nowish = new Date(); + nowish.setMilliseconds(0); + user.firstLoginAt = nowish; + needUpdate = true; + } + if (!user.picture && profile && profile.picture) { + // Set the user's profile picture if our provider knows it. + user.picture = profile.picture; + needUpdate = true; + } + if (profile && profile.email && profile.email !== login.displayEmail) { + // Use provider's version of email address for display. + login.displayEmail = profile.email; + needUpdate = true; + } + if (!login.displayEmail) { + // Save some kind of display email if we don't have anything at all for it yet. + // This could be coming from how someone wrote it in a UserManager dialog, for + // instance. It will get overwritten when the user logs in if the provider's + // version is different. + login.displayEmail = email; + needUpdate = true; + } + if (needUpdate) { + login.user = user; + await manager.save([user, login]); + } + if (!user.personalOrg && !NON_LOGIN_EMAILS.includes(login.email)) { + // Add a personal organization for this user. + // We don't add a personal org for anonymous/everyone/previewer "users" as it could + // get a bit confusing. + const result = await this.addOrg(user, {name: "Personal"}, true, true, manager); + if (result.status !== 200) { + throw new Error(result.errMessage); + } + needUpdate = true; + } + if (needUpdate) { + // We changed the db - reload user in order to give consistent results. + // In principle this could be optimized, but this is simpler to maintain. + user = await userQuery.getOne(); + } + return user; + }); + return userByLogin; + } + + /** + * Returns true if the given domain string is available, and false if it is not available. + * NOTE that the endpoint only checks if the domain string is taken in the database, it does + * not check whether the string contains invalid characters. + */ + public async isDomainAvailable(domain: string): Promise { + let qb = this._orgs(); + qb = this._whereOrg(qb, domain); + const results = await qb.getRawAndEntities(); + return results.entities.length === 0; + } + + /** + * Returns the number of users in any non-guest role in the given org. + * Note that this does not require permissions and should not be exposed to the client. + * + * If an Organization is provided, all of orgs.acl_rules, orgs.acl_rules.group, + * and orgs.acl_rules.group.memberUsers should be included. + */ + public async getOrgMemberCount(org: string|number|Organization): Promise { + if (!(org instanceof Organization)) { + const orgQuery = this._org(null, false, org, { + needRealOrg: true + }) + // Join the org's ACL rules (with 1st level groups/users listed). + .leftJoinAndSelect('orgs.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'org_groups') + .leftJoinAndSelect('org_groups.memberUsers', 'org_member_users'); + const result = await orgQuery.getRawAndEntities(); + if (result.entities.length === 0) { + // If the query for the doc failed, return the failure result. + throw new ApiError('org not found', 404); + } + org = result.entities[0]; + } + return getResourceUsers(org, this.defaultNonGuestGroupNames).length; + } + + /** + * Deletes a user from the database. For the moment, the only person with the right + * to delete a user is the user themselves. + * Users have logins, a personal org, and entries in the group_users table. All are + * removed together in a transaction. All material in the personal org will be lost. + * + * @param scope: request scope, including the id of the user initiating this action + * @param userIdToDelete: the id of the user to delete from the database + * @param name: optional cross-check, delete only if user name matches this + */ + public async deleteUser(scope: Scope, userIdToDelete: number, + name?: string): Promise> { + const userIdDeleting = scope.userId; + if (userIdDeleting !== userIdToDelete) { + throw new ApiError('not permitted to delete this user', 403); + } + await this._connection.transaction(async manager => { + const user = await manager.findOne(User, {where: {id: userIdToDelete}, + relations: ["logins", "personalOrg"]}); + if (!user) { throw new ApiError('user not found', 404); } + if (name) { + if (user.name !== name) { + throw new ApiError(`user name did not match ('${name}' vs '${user.name}')`, 400); + } + } + if (user.personalOrg) { await this.deleteOrg(scope, user.personalOrg.id, manager); } + await manager.remove([...user.logins]); + // We don't have a GroupUser entity, and adding one tickles lots of TypeOrm quirkiness, + // so use a plain query to delete entries in the group_users table. + await manager.createQueryBuilder() + .delete() + .from('group_users') + .where('user_id = :userId', {userId: userIdToDelete}) + .execute(); + await manager.delete(User, userIdToDelete); + }); + return { + status: 200 + }; + } + + /** + * Returns a QueryResult for the given organization. The orgKey + * can be a string (the domain from url) or the id of an org. If it is + * null, the user's personal organization is returned. + */ + public async getOrg(scope: Scope, orgKey: string|number|null, + transaction?: EntityManager): Promise> { + const {userId} = scope; + // Anonymous access to the merged org is a special case. We return an + // empty organization, not backed by the database, and which can contain + // nothing but the example documents always added to the merged org. + if (this.isMergedOrg(orgKey) && userId === this.getAnonymousUserId()) { + const anonOrg: OrgInfo = { + id: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + domain: this.mergedOrgDomain(), + name: 'Anonymous', + owner: this.makeFullUser(this.getAnonymousUser()), + access: 'viewers', + billingAccount: { + id: 0, + individual: true, + product: { + name: 'anonymous', + features: starterFeatures, + }, + isManager: false, + }, + host: null + }; + return { status: 200, data: anonOrg as any }; + } + let qb = this.org(scope, orgKey, { + manager: transaction, + needRealOrg: true + }); + qb = this._addBillingAccount(qb, scope.userId); + qb = this._withAccess(qb, scope.userId, 'orgs'); + qb = qb.leftJoinAndSelect('orgs.owner', 'owner'); + const result = await this._verifyAclPermissions(qb); + if (result.status === 200) { + // Return the only org. + result.data = result.data[0]; + if (this.isMergedOrg(orgKey)) { + // The merged psuedo-organization is almost, but not quite, the user's personal + // org. We give it a distinct domain and id. + result.data.id = 0; + result.data.domain = this.mergedOrgDomain(); + } + } + return result; + } + + /** + * Gets the billing account for the specified org. Will throw errors if the org + * is not found, or if the user does not have access to its billing account. + * + * The special previewer user is given access to billing account information. + * + * The billing account includes fields such as stripeCustomerId. + * To include `managers` and `orgs` fields listing all billing account managers + * and organizations linked to the account, set `includeOrgsAndManagers`. + */ + public async getBillingAccount(userId: number, orgKey: string|number, + includeOrgsAndManagers: boolean, + transaction?: EntityManager): Promise { + const org = this.unwrapQueryResult(await this.getOrg({userId}, orgKey, transaction)); + if (!org.billingAccount.isManager && userId !== this.getPreviewerUserId()) { + throw new ApiError('User does not have access to billing account', 401); + } + if (!includeOrgsAndManagers) { return org.billingAccount; } + + // For full billing account information including all managers + // (for team accounts) and orgs (for individual accounts), we need + // to make a different query since what we've got so far is + // filtered by org and by user for authorization purposes. + // Also, filling out user information linked to orgs and managers + // requires a few extra joins. + return this.getFullBillingAccount(org.billingAccount.id, transaction); + } + + /** + * Gets all information about a billing account, without permission check. + */ + public getFullBillingAccount(billingAccountId: number, transaction?: EntityManager): Promise { + return this._runInTransaction(transaction, async tr => { + let qb = tr.createQueryBuilder() + .select('billing_accounts') + .from(BillingAccount, 'billing_accounts') + .leftJoinAndSelect('billing_accounts.product', 'products') + .leftJoinAndSelect('billing_accounts.managers', 'managers') + .leftJoinAndSelect('managers.user', 'manager_users') + .leftJoinAndSelect('manager_users.logins', 'manager_logins') + .leftJoinAndSelect('billing_accounts.orgs', 'orgs') + .leftJoinAndSelect('orgs.owner', 'org_users') + .leftJoinAndSelect('org_users.logins', 'org_logins') + .where('billing_accounts.id = :billingAccountId', {billingAccountId}); + qb = this._addBillingAccountCalculatedFields(qb); + // TODO: should reconcile with isManager field that stripped down results have. + const results = await qb.getRawAndEntities(); + const resources = this._normalizeQueryResults(results.entities); + if (!resources[0]) { + throw new ApiError('Cannot find billing account', 500); + } + return resources[0]; + }); + } + + /** + * Returns a QueryResult for an organization with nested workspaces. + */ + public async getOrgWorkspaces(scope: Scope, orgKey: string|number, + options: QueryOptions = {}): Promise> { + const {userId} = scope; + const supportId = this._specialUserIds[SUPPORT_EMAIL]; + let queryBuilder = this.org(scope, orgKey, options) + .leftJoinAndSelect('orgs.workspaces', 'workspaces') + .leftJoinAndSelect('workspaces.docs', 'docs', this._onDoc(scope)) + .leftJoin('orgs.billingAccount', 'account') + .leftJoin('account.product', 'product') + .addSelect('product.features') + .addSelect('account.id') + // order the support org (aka Samples/Examples) after other ones. + .orderBy('coalesce(orgs.owner_id = :supportId, false)') + .setParameter('supportId', supportId) + .addOrderBy('(orgs.owner_id = :userId)', 'DESC') + .setParameter('userId', userId) + // For consistency of results, particularly in tests, order workspaces by name. + .addOrderBy('workspaces.name') + .addOrderBy('docs.created_at') + .leftJoinAndSelect('orgs.owner', 'org_users'); + // If merged org, we need to take some special steps. + if (this.isMergedOrg(orgKey)) { + // Add information about owners of personal orgs. + queryBuilder = queryBuilder + .leftJoinAndSelect('org_users.logins', 'org_logins'); + // Add a direct, efficient filter to remove irrelevant personal orgs from consideration. + queryBuilder = this._filterByOrgGroups(queryBuilder, userId); + // The anonymous user is a special case; include only examples from support user. + if (userId === this.getAnonymousUserId()) { + queryBuilder = queryBuilder.andWhere('orgs.owner_id = :supportId', { supportId }); + } + } + queryBuilder = this._addIsSupportWorkspace(userId, queryBuilder, 'orgs', 'workspaces'); + // Add access information and query limits + // TODO: allow generic org limit once sample/support workspace is done differently + queryBuilder = this._applyLimit(queryBuilder, {...scope, org: undefined}, ['orgs', 'workspaces', 'docs']); + const result = await this._verifyAclPermissions(queryBuilder); + // Return the workspaces, not the org(s). + if (result.status === 200) { + // Place ownership information in workspaces, available for the merged org. + for (const o of result.data) { + for (const ws of o.workspaces) { + ws.owner = o.owner; + } + } + // For org-specific requests, we still have the org's workspaces, plus the Samples workspace + // from the support org. + result.data = [].concat(...result.data.map((o: Organization) => o.workspaces)); + } + return result; + } + + + /** + * Returns a QueryResult for the workspace with the given workspace id. The workspace + * includes nested Docs. + */ + public async getWorkspace(scope: Scope, wsId: number): Promise> { + const {userId} = scope; + let queryBuilder = this._workspaces() + .where('workspaces.id = :wsId', {wsId}) + // Nest the docs within the workspace object + .leftJoinAndSelect('workspaces.docs', 'docs', this._onDoc(scope)) + .leftJoinAndSelect('workspaces.org', 'orgs') + .leftJoinAndSelect('orgs.owner', 'owner') + // Define some order (spec doesn't promise anything though) + .orderBy('workspaces.created_at') + .addOrderBy('docs.created_at'); + queryBuilder = this._addIsSupportWorkspace(userId, queryBuilder, 'orgs', 'workspaces'); + // Add access information and query limits + // TODO: allow generic org limit once sample/support workspace is done differently + queryBuilder = this._applyLimit(queryBuilder, {...scope, org: undefined}, ['workspaces', 'docs']); + const result = await this._verifyAclPermissions(queryBuilder); + // Return a single workspace. + if (result.status === 200) { + result.data = result.data[0]; + } + return result; + } + + /** + * Compute the best access option for an organization, from the + * users available to the client. If none of the options can access + * the organization, returns null. If there are equally good + * options, an arbitrary one is returned. + * + * Comparison is made between roles rather than fine-grained + * permissions, since otherwise the result would not be well defined + * (permissions could in general overlap without one being a + * superset of the other). For the acl rules we've used so far, + * this problem does not arise and reasoning at the level of a + * hierarchy of roles is adequate. + */ + public async getBestUserForOrg(users: AvailableUsers, org: number|string): Promise { + if (this.isMergedOrg(org)) { + // Don't try to pick a best user for the merged personal org. + // If this changes in future, be sure to call this._filterByOrgGroups on the query + // below, otherwise it will include every users' personal org which is wasteful + // and parsing/mapping the results in TypeORM is slow. + return null; + } + let qb = this._orgs(); + qb = this._whereOrg(qb, org); + qb = this._withAccess(qb, users, 'orgs'); + const result = await this._verifyAclPermissions(qb, true); + if (!result.data) { + throw new ApiError(result.errMessage || 'failed to select user', result.status); + } + if (!result.data.length) { return null; } + const options: AccessOptionWithRole[] = result.data[0].accessOptions; + if (!options.length) { return null; } + const role = roles.getStrongestRole(...options.map(option => option.access)); + return options.find(option => option.access === role) || null; + } + + + /** + * Returns a SelectQueryBuilder which gives an array of orgs already filtered by + * the given user' (or users') access. + * If a domain is specified, only an org matching that domain and accessible by + * the user or users is returned. + * The anonymous user is treated specially, to avoid advertising organizations + * with anonymous access. + */ + public async getOrgs(users: AvailableUsers, domain: string|null): Promise> { + let queryBuilder = this._orgs() + .leftJoinAndSelect('orgs.owner', 'users', 'orgs.owner_id = users.id'); + if (isSingleUser(users)) { + // When querying with a single user in mind, we keep our api promise + // of returning their personal org first in the list. + queryBuilder = queryBuilder + .orderBy('(coalesce(users.id,0) = :userId)', 'DESC') + .setParameter('userId', users); + } + queryBuilder = queryBuilder + .addOrderBy('users.name') + .addOrderBy('orgs.name'); + queryBuilder = this._withAccess(queryBuilder, users, 'orgs'); + // Add a direct, efficient filter to remove irrelevant personal orgs from consideration. + queryBuilder = this._filterByOrgGroups(queryBuilder, users, domain); + if (this._isAnonymousUser(users)) { + // The anonymous user is a special case. It may have access to potentially + // many orgs, but listing them all would be kind of a misfeature. but reporting + // nothing would complicate the client. We compromise, and report at most + // the org of the site the user is on (or nothing when the api is accessed + // via a url that is unrelated to any particular org). + // This special processing is only needed for the isSingleUser case. Multiple + // users can only be presented when the user has proven login access to each. + if (domain && !this.isMergedOrg(domain)) { + queryBuilder = this._whereOrg(queryBuilder, domain); + } else { + return {status: 200, data: []}; + } + } + return this._verifyAclPermissions(queryBuilder, true); + } + + // As for getOrgs, but all personal orgs are merged into a single entry. + public async getMergedOrgs(userId: number, users: AvailableUsers, + domain: string|null): Promise> { + const result = await this.getOrgs(users, domain); + if (result.status === 200) { + return {status: 200, data: this._mergePersonalOrgs(userId, result.data!)}; + } + return result; + } + + // Returns the doc with access information for the calling user only. + // TODO: The return type of this function includes the workspace and org with the owner + // properties set, as documented in app/common/UserAPI. The return type of this function + // should reflect that. + public async getDocImpl(key: DocAuthKey): Promise { + const {userId} = key; + // Doc permissions of forks are based on the "trunk" document, so make sure + // we look up permissions of trunk if we are on a fork (we'll fix the permissions + // up for the fork immediately afterwards). + const {trunkId, forkId, forkUserId, snapshotId} = parseUrlId(key.urlId); + const urlId = trunkId; + if (forkId || snapshotId) { key = {...key, urlId}; } + let doc: Document; + if (urlId === NEW_DOCUMENT_CODE) { + if (!forkId) { throw new ApiError('invalid document identifier', 400); } + // We imagine current user owning trunk if there is no embedded userId, or + // the embedded userId matches the current user. + const access = (forkUserId === undefined || forkUserId === userId) ? 'owners' : null; + if (!access) { throw new ApiError("access denied", 403); } + doc = { + name: 'Untitled', + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + id: 'new', + isPinned: false, + urlId: null, + workspace: this.unwrapQueryResult( + await this.getWorkspace({userId: this.getSupportUserId()}, + this._exampleWorkspaceId)), + aliases: [], + access + } as any; + } else { + // We can't delegate filtering of removed documents to the db, since we'll be + // caching authentication. But we also don't need to delegate filtering, since + // it is very simple at the single-document level. So we direct the db to include + // everything with showAll flag, and let the getDoc() wrapper deal with the remaining + // work. + let qb = this._doc({...key, showAll: true}) + .leftJoinAndSelect('orgs.owner', 'org_users'); + qb = this._addIsSupportWorkspace(userId, qb, 'orgs', 'workspaces'); + qb = this._addFeatures(qb); // add features to determine whether we've gone readonly + const docs = this.unwrapQueryResult(await this._verifyAclPermissions(qb)); + if (docs.length === 0) { throw new ApiError('document not found', 404); } + if (docs.length > 1) { throw new ApiError('ambiguous document request', 400); } + doc = docs[0]; + const features = doc.workspace.org.billingAccount.product.features; + if (features.readOnlyDocs) { + // Don't allow any access to docs that is stronger than "viewers". + doc.access = roles.getWeakestRole('viewers', doc.access); + } + // Place ownership information in the doc's workspace. + (doc.workspace as any).owner = doc.workspace.org.owner; + } + if (forkId || snapshotId) { + // Fix up our reply to be correct for the fork, rather than the trunk. + // The "id" and "urlId" fields need updating. + doc.id = buildUrlId({trunkId: doc.id, forkId, forkUserId, snapshotId}); + if (doc.urlId) { + doc.urlId = buildUrlId({trunkId: doc.urlId, forkId, forkUserId, snapshotId}); + } + // Forks without a user id are editable by anyone with view access to the trunk. + if (forkUserId === undefined && doc.access === 'viewers') { doc.access = 'editors'; } + if (forkUserId !== undefined) { + // A fork user id is known, so only that user should get to edit the fork. + if (userId === forkUserId) { + // Promote to editor if just a viewer of the trunk. + if (doc.access === 'viewers') { doc.access = 'editors'; } + } else { + // reduce to viewer if not already viewer + doc.access = roles.getWeakestRole('viewers', doc.access); + } + } + // No-one may be an owner of a fork, since there's no way to set up ACLs for it. + if (doc.access === 'owners') { doc.access = 'editors'; } + + // Finally, if we are viewing a snapshot, we can't edit it. + if (snapshotId) { + doc.access = roles.getWeakestRole('viewers', doc.access); + } + } + return doc; + } + + // Calls getDocImpl() and returns the Document from that, caching a fresh DocAuthResult along + // the way. Note that we only cache the access level, not Document itself. + public async getDoc(scope: Scope): Promise { + const key = getDocAuthKeyFromScope(scope); + const promise = this.getDocImpl(key); + await mapSetOrClear(this._docAuthCache, stringifyDocAuthKey(key), makeDocAuthResult(promise)); + const doc = await promise; + // Filter the result for removed / non-removed documents. + if (!scope.showAll && scope.showRemoved ? + (doc.removedAt === null && doc.workspace.removedAt === null) : + (doc.removedAt || doc.workspace.removedAt)) { + throw new ApiError('document not found', 404); + } + return doc; + } + + // Returns access info for the given doc and user, caching the results for DOC_AUTH_CACHE_TTL + // ms. This helps reduce database load created by liberal authorization requests. + public async getDocAuthCached(key: DocAuthKey): Promise { + return mapGetOrSet(this._docAuthCache, stringifyDocAuthKey(key), + () => makeDocAuthResult(this.getDocImpl(key))); + } + + // Used in tests, and to clear all timeouts when exiting. + public flushDocAuthCache() { + this._docAuthCache.clear(); + } + + // Flush cached access information about a specific document + // (identified specifically by a docId, not a urlId). Any cached + // information under an alias will also be flushed. + // TODO: make a more efficient implementation if needed. + public async flushSingleDocAuthCache(scope: DocScope, docId: string) { + // Get all aliases of this document. + const aliases = await this._connection.manager.find(Alias, { docId }); + // Construct a set of possible prefixes for cache keys. + const names = new Set(aliases.map(a => stringifyUrlIdOrg(a.urlId, scope.org))); + names.add(stringifyUrlIdOrg(docId, scope.org)); + // Remove any cache keys that start with any of the prefixes. + for (const key of this._docAuthCache.keys()) { + const name = key.split(' ', 1)[0]; + if (names.has(name)) { this._docAuthCache.delete(key); } + } + } + + // Find a document by name. Limit name search to a specific organization. + // It is possible to hit ambiguities, e.g. with the same name of a doc + // in multiple workspaces, so this is not a general-purpose method. It + // is here to facilitate V0 -> V1 migration, so existing links to docs continue + // to work. + public async getDocByName(userId: number, orgId: number, docName: string): Promise> { + let qb = this._docs() + .innerJoin('docs.workspace', 'workspace') + .innerJoin('workspace.org', 'org') + .where('docs.name = :docName', {docName}) + .andWhere('org.id = :orgId', {orgId}); + qb = this._withAccess(qb, userId, 'docs'); + return this._single(await this._verifyAclPermissions(qb)); + } + + /** + * + * Adds an org with the given name. Returns a query result with the id of the added org. + * + * @param user: user doing the adding + * @param name: desired org name + * @param domain: desired org domain, or null not to set a domain + * @param setUserAsOwner: if this is the user's personal org (they will be made an + * owner in the ACL sense in any case) + * @param useNewPlan: by default, the individual billing account associated with the + * user's personal org will be used for all other orgs they create. Set useNewPlan + * to force a distinct non-individual billing account to be used for this org. + * + */ + public async addOrg(user: User, props: Partial, + setUserAsOwner: boolean, useNewPlan: boolean, + transaction?: EntityManager): Promise> { + const notifications: Array<() => void> = []; + const name = props.name; + const domain = props.domain; + if (!name) { + return { + status: 400, + errMessage: 'Bad request: name required' + }; + } + const orgResult = await this._runInTransaction(transaction, async manager => { + if (domain) { + try { + checkSubdomainValidity(domain); + } catch (e) { + return { + status: 400, + errMessage: `Domain is not permitted: ${e.message}` + }; + } + } + // Create or find a billing account to associate with this org. + const billingAccountEntities = []; + let billingAccount; + if (useNewPlan) { + const productNames = getDefaultProductNames(); + let productName = setUserAsOwner ? productNames.personal : productNames.teamInitial; + // A bit fragile: this is called during creation of support@ user, before + // getSupportUserId() is available, but with setUserAsOwner of true. + if (!setUserAsOwner && user.id === this.getSupportUserId()) { + // For teams created by support@getgrist.com, set the product to something + // good so payment not needed. This is useful for testing. + productName = productNames.team; + } + billingAccount = new BillingAccount(); + billingAccount.individual = setUserAsOwner; + const dbProduct = await manager.findOne(Product, {name: productName}); + if (!dbProduct) { + throw new Error('Cannot find product for new organization'); + } + billingAccount.product = dbProduct; + billingAccountEntities.push(billingAccount); + const billingAccountManager = new BillingAccountManager(); + billingAccountManager.user = user; + billingAccountManager.billingAccount = billingAccount; + billingAccountEntities.push(billingAccountManager); + } else { + // Use the billing account from the user's personal org to start with. + billingAccount = await manager.createQueryBuilder() + .select('billing_accounts') + .from(BillingAccount, 'billing_accounts') + .leftJoinAndSelect('billing_accounts.orgs', 'orgs') + .where('orgs.owner_id = :userId', {userId: user.id}) + .getOne(); + if (!billingAccount) { + throw new ApiError('Cannot find an initial plan for organization', 500); + } + } + // Create a new org. + const org = new Organization(); + org.checkProperties(props); + org.updateFromProperties(props); + org.billingAccount = billingAccount; + if (domain) { + org.domain = domain; + } + if (setUserAsOwner) { + org.owner = user; + } + // Create the special initial permission groups for the new org. + const groupMap = this._createGroups(); + org.aclRules = this.defaultGroups.map(_grpDesc => { + // Get the special group with the name needed for this ACL Rule + const group = groupMap[_grpDesc.name]; + // Note that the user is added to the owners group of an org when it is created. + if (_grpDesc.name === roles.OWNER) { + group.memberUsers = [user]; + } + // Add each of the special groups to the new workspace. + const aclRuleOrg = new AclRuleOrg(); + aclRuleOrg.permissions = _grpDesc.permissions; + aclRuleOrg.group = group; + aclRuleOrg.organization = org; + return aclRuleOrg; + }); + // Saves the workspace as well as its new ACL Rules and Group. + const groups = org.aclRules.map(rule => rule.group); + let savedOrg: Organization; + try { + const result = await manager.save([org, ...org.aclRules, ...groups, ...billingAccountEntities]); + savedOrg = result[0] as Organization; + } catch (e) { + if (e.name === 'QueryFailedError' && e.message && + e.message.match(/unique constraint/i)) { + throw new ApiError('Domain already in use', 400); + } + throw e; + } + // Add a starter workspace to the org. Any limits on org workspace + // count are not checked, this will succeed unconditionally. + await this._doAddWorkspace(savedOrg, {name: 'Home'}, manager); + + if (!setUserAsOwner) { + // This user just made a team site (once this transaction is applied). + // Emit a notification. + notifications.push(this._teamCreatorNotification(user.id)); + } + return { + status: 200, + data: savedOrg.id + }; + }); + for (const notification of notifications) { notification(); } + return orgResult; + } + + // Checks that the user has UPDATE permissions to the given org. If not, throws an + // error. Otherwise updates the given org with the given name. Returns an empty + // query result with status 200 on success. + public async updateOrg( + scope: Scope, + orgKey: string|number, + props: Partial + ): Promise> { + // TODO: Unsetting a domain will likely have to be supported. + return await this._connection.transaction(async manager => { + const orgQuery = this.org(scope, orgKey, { + manager, + markPermissions: Permissions.UPDATE + }); + const queryResult = await verifyIsPermitted(orgQuery); + if (queryResult.status !== 200) { + // If the query for the workspace failed, return the failure result. + return queryResult; + } + // Update the fields and save. + const org: Organization = queryResult.data; + if (props.domain) { + if (org.owner) { + throw new ApiError('Cannot set a domain for a personal organization', 400); + } + } + org.checkProperties(props); + org.updateFromProperties(props); + await manager.save(org); + return {status: 200}; + }); + } + + // Checks that the user has REMOVE permissions to the given org. If not, throws an + // error. Otherwise deletes the given org. Returns an empty query result with + // status 200 on success. + public async deleteOrg(scope: Scope, orgKey: string|number, + transaction?: EntityManager): Promise> { + return await this._runInTransaction(transaction, async manager => { + const orgQuery = this.org(scope, orgKey, { + manager, + markPermissions: Permissions.REMOVE + }) + // Join the org's workspaces (with ACLs and groups), docs (with ACLs and groups) + // and ACLs and groups so we can remove them. + .leftJoinAndSelect('orgs.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'groups') + .leftJoinAndSelect('orgs.workspaces', 'workspaces') + .leftJoinAndSelect('workspaces.aclRules', 'workspace_acl_rules') + .leftJoinAndSelect('workspace_acl_rules.group', 'workspace_group') + .leftJoinAndSelect('workspaces.docs', 'docs') + .leftJoinAndSelect('docs.aclRules', 'doc_acl_rules') + .leftJoinAndSelect('doc_acl_rules.group', 'doc_group') + .leftJoinAndSelect('orgs.billingAccount', 'billing_accounts'); + const queryResult = await verifyIsPermitted(orgQuery); + if (queryResult.status !== 200) { + // If the query for the org failed, return the failure result. + return queryResult; + } + const org: Organization = queryResult.data; + // Delete the org, org ACLs/groups, workspaces, workspace ACLs/groups, workspace docs + // and doc ACLs/groups. + const orgGroups = org.aclRules.map(orgAcl => orgAcl.group); + const wsAcls = ([] as AclRule[]).concat(...org.workspaces.map(ws => ws.aclRules)); + const wsGroups = wsAcls.map(wsAcl => wsAcl.group); + const docs = ([] as Document[]).concat(...org.workspaces.map(ws => ws.docs)); + const docAcls = ([] as AclRule[]).concat(...docs.map(doc => doc.aclRules)); + const docGroups = docAcls.map(docAcl => docAcl.group); + await manager.remove([org, ...org.aclRules, ...orgGroups, ...org.workspaces, + ...wsAcls, ...wsGroups, ...docs, ...docAcls, ...docGroups]); + + // Delete billing account if this was the last org using it. + const billingAccount = await manager.findOne(BillingAccount, org.billingAccount, + {relations: ['orgs']}); + if (billingAccount && billingAccount.orgs.length === 0) { + await manager.remove([billingAccount]); + } + return {status: 200}; + }); + } + + // Checks that the user has ADD permissions to the given org. If not, throws an error. + // Otherwise adds a workspace with the given name. Returns a query result with the id + // of the added workspace. + public async addWorkspace(scope: Scope, orgKey: string|number, + props: Partial): Promise> { + const name = props.name; + if (!name) { + return { + status: 400, + errMessage: 'Bad request: name required' + }; + } + return await this._connection.transaction(async manager => { + let orgQuery = this.org(scope, orgKey, { + manager, + markPermissions: Permissions.ADD, + needRealOrg: true + }) + // Join the org's ACL rules (with 1st level groups listed) so we can include them in the + // workspace. + .leftJoinAndSelect('orgs.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'org_group') + .leftJoinAndSelect('orgs.workspaces', 'workspaces'); // we may want to count workspaces. + orgQuery = this._addFeatures(orgQuery); // add features to access optional workspace limit. + const queryResult = await verifyIsPermitted(orgQuery); + if (queryResult.status !== 200) { + // If the query for the organization failed, return the failure result. + return queryResult; + } + const org: Organization = queryResult.data; + const features = org.billingAccount.product.features; + if (features.maxWorkspacesPerOrg !== undefined) { + // we need to count how many workspaces are in the current org, and if we + // are already at or above the limit, then fail. + const count = org.workspaces.length; + if (count >= features.maxWorkspacesPerOrg) { + throw new ApiError('No more workspaces permitted', 403, { + limit: { + quantity: 'workspaces', + maximum: features.maxWorkspacesPerOrg, + value: count, + projectedValue: count + 1 + } + }); + } + } + const workspace = await this._doAddWorkspace(org, props, manager); + return { + status: 200, + data: workspace.id + }; + }); + } + + // Checks that the user has UPDATE permissions to the given workspace. If not, throws an + // error. Otherwise updates the given workspace with the given name. Returns an empty + // query result with status 200 on success. + public async updateWorkspace(scope: Scope, wsId: number, + props: Partial): Promise> { + return await this._connection.transaction(async manager => { + const wsQuery = this._workspace(scope, wsId, { + manager, + markPermissions: Permissions.UPDATE + }); + const queryResult = await verifyIsPermitted(wsQuery); + if (queryResult.status !== 200) { + // If the query for the workspace failed, return the failure result. + return queryResult; + } + // Update the name and save. + const workspace: Workspace = queryResult.data; + workspace.checkProperties(props); + workspace.updateFromProperties(props); + await manager.save(workspace); + return {status: 200}; + }); + } + + // Checks that the user has REMOVE permissions to the given workspace. If not, throws an + // error. Otherwise deletes the given workspace. Returns an empty query result with + // status 200 on success. + public async deleteWorkspace(scope: Scope, wsId: number): Promise> { + return await this._connection.transaction(async manager => { + const wsQuery = this._workspace(scope, wsId, { + manager, + markPermissions: Permissions.REMOVE, + allowSpecialPermit: true + }) + // Join the workspace's docs (with ACLs and groups) and ACLs and groups so we can + // remove them. Also join the org to get the orgId. + .leftJoinAndSelect('workspaces.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'groups') + .leftJoinAndSelect('workspaces.docs', 'docs') + .leftJoinAndSelect('docs.aclRules', 'doc_acl_rules') + .leftJoinAndSelect('doc_acl_rules.group', 'doc_groups') + .leftJoinAndSelect('workspaces.org', 'orgs'); + const queryResult = await verifyIsPermitted(wsQuery); + if (queryResult.status !== 200) { + // If the query for the workspace failed, return the failure result. + return queryResult; + } + const workspace: Workspace = queryResult.data; + // Delete the workspace, workspace docs, doc ACLs/groups and workspace ACLs/groups. + const wsGroups = workspace.aclRules.map(wsAcl => wsAcl.group); + const docAcls = ([] as AclRule[]).concat(...workspace.docs.map(doc => doc.aclRules)); + const docGroups = docAcls.map(docAcl => docAcl.group); + await manager.remove([workspace, ...wsGroups, ...docAcls, ...workspace.docs, + ...workspace.aclRules, ...docGroups]); + // Update the guests in the org after removing this workspace. + await this._repairOrgGuests(scope, workspace.org.id, manager); + return {status: 200}; + }); + } + + public softDeleteWorkspace(scope: Scope, wsId: number): Promise { + return this._setWorkspaceRemovedAt(scope, wsId, new Date()); + } + + public async undeleteWorkspace(scope: Scope, wsId: number): Promise { + return this._setWorkspaceRemovedAt(scope, wsId, null); + } + + // Checks that the user has ADD permissions to the given workspace. If not, throws an + // error. Otherwise adds a doc with the given name. Returns a query result with the id + // of the added doc. + // The desired docId may be passed in. If passed in, it should have been generated + // by makeId(). The client should not be given control of the choice of docId. + // This option is used during imports, where it is convenient not to add a row to the + // document database until the document has actually been imported. + public async addDocument(scope: Scope, wsId: number, props: Partial, + docId?: string): Promise> { + const {userId} = scope; + const name = props.name; + if (!name) { + return { + status: 400, + errMessage: 'Bad request: name required' + }; + } + return await this._connection.transaction(async manager => { + let wsQuery = this._workspace(scope, wsId, { + manager, + markPermissions: Permissions.ADD + }) + .leftJoinAndSelect('workspaces.org', 'orgs') + // Join the workspaces's ACL rules (with 1st level groups listed) so we can include + // them in the doc. + .leftJoinAndSelect('workspaces.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'workspace_group'); + wsQuery = this._addFeatures(wsQuery); + const queryResult = await verifyIsPermitted(wsQuery); + if (queryResult.status !== 200) { + // If the query for the organization failed, return the failure result. + return queryResult; + } + const workspace: Workspace = queryResult.data; + await this._checkRoomForAnotherDoc(userId, workspace, manager); + // Create a new document. + const doc = new Document(); + doc.id = docId || makeId(); + doc.checkProperties(props); + doc.updateFromProperties(props); + // By default, assign a urlId that is a prefix of the docId. + // The urlId should be unique across all existing documents. + if (!doc.urlId) { + for (let i = MIN_URLID_PREFIX_LENGTH; i <= doc.id.length; i++) { + const candidate = doc.id.substr(0, i); + if (!await manager.findOne(Alias, { urlId: candidate })) { + doc.urlId = candidate; + break; + } + } + if (!doc.urlId) { + // This should happen only if UUIDs collide. + throw new Error('Could not find a free identifier for document'); + } + } + if (doc.urlId) { + await this._checkForUrlIdConflict(manager, workspace.org, doc.urlId); + const alias = new Alias(); + doc.aliases = [alias]; + alias.urlId = doc.urlId; + alias.orgId = workspace.org.id; + } else { + doc.aliases = []; + } + doc.workspace = workspace; + // Create the special initial permission groups for the new workspace. + const groupMap = this._createGroups(workspace); + doc.aclRules = this.defaultCommonGroups.map(_grpDesc => { + // Get the special group with the name needed for this ACL Rule + const group = groupMap[_grpDesc.name]; + // Add each of the special groups to the new doc. + const aclRuleDoc = new AclRuleDoc(); + aclRuleDoc.permissions = _grpDesc.permissions; + aclRuleDoc.group = group; + aclRuleDoc.document = doc; + return aclRuleDoc; + }); + // Saves the document as well as its new ACL Rules and Group. + const groups = doc.aclRules.map(rule => rule.group); + const result = await manager.save([doc, ...doc.aclRules, ...doc.aliases, ...groups]); + return { + status: 200, + data: (result[0] as Document).id + }; + }); + } + + // Checks that the user has UPDATE permissions to the given doc. If not, throws an + // error. Otherwise updates the given doc with the given name. Returns an empty + // query result with status 200 on success. + // NOTE: This does not update the updateAt date indicating the last modified time of the doc. + // We may want to make it do so. + public async updateDocument(scope: DocScope, + props: Partial): Promise> { + return await this._connection.transaction(async manager => { + const docQuery = this._doc(scope, { + manager, + markPermissions: Permissions.UPDATE + }); + + const queryResult = await verifyIsPermitted(docQuery); + if (queryResult.status !== 200) { + // If the query for the workspace failed, return the failure result. + return queryResult; + } + // Update the name and save. + const doc: Document = queryResult.data; + doc.checkProperties(props); + doc.updateFromProperties(props); + // Forcibly remove the aliases relation from the document object, so that TypeORM + // doesn't try to save it. It isn't safe to do that because it was filtered by + // a where clause. + // TODO: refactor to avoid using TypeORM's save method. + doc.aliases = undefined as any; + // TODO: if pinning does anything special in future, like triggering thumbnail + // processing, then we should probably call pinDoc. + await manager.save(doc); + if (props.urlId) { + // We accumulate old urlIds in order to correctly redirect them, so we need + // to do some extra bookwork when a doc's urlId is changed. First, throw + // an error if urlId is already in use by this org. + await this._checkForUrlIdConflict(manager, doc.workspace.org, props.urlId, doc.id); + // Otherwise, add an alias entry for this document. + await manager.createQueryBuilder() + .insert() + // if urlId has been used before, update it + .onConflict(`(org_id, url_id) DO UPDATE SET doc_id = :docId, created_at = ${now(this._dbType)}`) + .setParameter('docId', doc.id) + .into(Alias) + .values({orgId: doc.workspace.org.id, urlId: props.urlId, doc}) + .execute(); + // TODO: we could limit the max number of aliases stored per document. + } + return {status: 200}; + }); + } + + // Checks that the user has REMOVE permissions to the given document. If not, throws an + // error. Otherwise deletes the given document. Returns an empty query result with + // status 200 on success. + public async deleteDocument(scope: DocScope): Promise> { + return await this._connection.transaction(async manager => { + const docQuery = this._doc(scope, { + manager, + markPermissions: Permissions.REMOVE, + allowSpecialPermit: true + }) + // Join the docs's ACLs and groups so we can remove them. + // Join the workspace and org to get their ids. + .leftJoinAndSelect('docs.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'groups'); + const queryResult = await verifyIsPermitted(docQuery); + if (queryResult.status !== 200) { + // If the query for the workspace failed, return the failure result. + return queryResult; + } + const doc: Document = queryResult.data; + // Delete the doc and doc ACLs/groups. + const docGroups = doc.aclRules.map(docAcl => docAcl.group); + await manager.remove([doc, ...docGroups, ...doc.aclRules]); + // Update guests of the workspace and org after removing this doc. + await this._repairWorkspaceGuests(scope, doc.workspace.id, manager); + await this._repairOrgGuests(scope, doc.workspace.org.id, manager); + return {status: 200}; + }); + } + + public softDeleteDocument(scope: DocScope): Promise { + return this._setDocumentRemovedAt(scope, new Date()); + } + + public async undeleteDocument(scope: DocScope): Promise { + return this._setDocumentRemovedAt(scope, null); + } + + // Fetches and provides a callback with the billingAccount so it may be updated within + // a transaction. The billingAccount is saved after any changes applied in the callback. + // Will throw an error if the user does not have access to the org's billingAccount. + // + // Only certain properties of the billingAccount may be changed: + // 'inGoodStanding', 'status', 'stripeCustomerId','stripeSubscriptionId', 'stripePlanId' + // + // Returns an empty query result with status 200 on success. + public async updateBillingAccount( + userId: number, + orgKey: string|number, + callback: (billingAccount: BillingAccount, transaction: EntityManager) => void|Promise + ): Promise> { + return await this._connection.transaction(async transaction => { + const billingAccount = await this.getBillingAccount(userId, orgKey, false, transaction); + const billingAccountCopy = Object.assign({}, billingAccount); + await callback(billingAccountCopy, transaction); + // Pick out properties that are allowed to be changed, to prevent accidental updating + // of other information. + const updated = pick(billingAccountCopy, 'inGoodStanding', 'status', 'stripeCustomerId', + 'stripeSubscriptionId', 'stripePlanId', 'product'); + billingAccount.paid = undefined; // workaround for a typeorm bug fixed upstream in + // https://github.com/typeorm/typeorm/pull/4035 + await transaction.save(Object.assign(billingAccount, updated)); + return { status: 200 }; + }); + } + + // Updates the managers of a billing account. Returns an empty query result with + // status 200 on success. + public async updateBillingAccountManagers(userId: number, orgKey: string|number, + delta: ManagerDelta): Promise> { + const notifications: Array<() => void> = []; + // Translate our ManagerDelta to a PermissionDelta so that we can reuse existing + // methods for normalizing/merging emails and finding the user ids. + const permissionDelta: PermissionDelta = {users: {}}; + for (const key of Object.keys(delta.users)) { + const target = delta.users[key]; + if (target !== null && target !== 'managers') { + throw new ApiError("Only valid settings for billing account managers are 'managers' or null", 400); + } + permissionDelta.users![key] = delta.users[key] ? 'owners' : null; + } + + return await this._connection.transaction(async transaction => { + const billingAccount = await this.getBillingAccount(userId, orgKey, true, transaction); + // At this point, we'll have thrown an error if userId is not a billing account manager. + // Now check if the billing account has mutable managers (individual account does not). + if (billingAccount.individual) { + throw new ApiError('billing account managers cannot be added/removed for individual billing accounts', 400); + } + // Get the ids of users to update. + const billingAccountId = billingAccount.id; + const userIdDelta = await this._verifyAndLookupDeltaEmails(userId, permissionDelta, true, transaction); + if (!userIdDelta) { throw new ApiError('No userIdDelta', 500); } + // Any duplicated emails have been merged, and userIdDelta is now keyed by user ids. + // Now we iterate over users and add/remove them as managers. + for (const memberUserIdStr of Object.keys(userIdDelta)) { + const memberUserId = parseInt(memberUserIdStr, 10); + const add = Boolean(userIdDelta[memberUserIdStr]); + const manager = await transaction.findOne(BillingAccountManager, {where: {userId: memberUserId, + billingAccountId}}); + if (add) { + // Skip adding user if they are already a manager. + if (!manager) { + const newManager = new BillingAccountManager(); + newManager.userId = memberUserId; + newManager.billingAccountId = billingAccountId; + await transaction.save(newManager); + notifications.push(this._billingManagerNotification(userId, memberUserId, + billingAccount.orgs)); + } + } else { + if (manager) { + // Don't allow a user to remove themselves as a manager, to be consistent + // with ACL behavior. + if (memberUserId === userId) { + throw new ApiError('Users cannot remove themselves as billing managers', 400); + } + await transaction.remove(manager); + } + } + } + for (const notification of notifications) { notification(); } + return { status: 200 }; + }); + } + + // Updates the permissions of users on the given org according to the PermissionDelta. + public async updateOrgPermissions( + scope: Scope, + orgKey: string|number, + delta: PermissionDelta + ): Promise> { + const {userId} = scope; + const notifications: Array<() => void> = []; + const result = await this._connection.transaction(async manager => { + const userIdDelta = await this._verifyAndLookupDeltaEmails(userId, delta, true, manager); + let orgQuery = this.org(scope, orgKey, { + manager, + markPermissions: Permissions.ACL_EDIT, + needRealOrg: true + }) + // Join the org's ACL rules (with 1st level groups/users listed) so we can edit them. + .leftJoinAndSelect('orgs.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'org_groups') + .leftJoinAndSelect('org_groups.memberUsers', 'org_member_users'); + orgQuery = this._addFeatures(orgQuery); + const queryResult = await verifyIsPermitted(orgQuery); + if (queryResult.status !== 200) { + // If the query for the organization failed, return the failure result. + return queryResult; + } + const org: Organization = queryResult.data; + const groups = getNonGuestGroups(org); + if (userIdDelta) { + const membersBefore = getUsersWithRole(groups, this.getExcludedUserIds()); + const countBefore = removeRole(membersBefore).length; + await this._updateUserPermissions(groups, userIdDelta, manager); + this._checkUserChangeAllowed(userId, groups); + await manager.save(groups); + // Fully remove any users being removed from the org. + for (const deltaUser in userIdDelta) { + // Any users removed from the org should be removed from everything in the org. + if (userIdDelta[deltaUser] === null) { + await scrubUserFromOrg(org.id, parseInt(deltaUser, 10), userId, manager); + } + } + // Emit an event if the number of org users is changing. + const membersAfter = getUsersWithRole(groups, this.getExcludedUserIds()); + const countAfter = removeRole(membersAfter).length; + notifications.push(this._userChangeNotification(userId, org, countBefore, countAfter, + membersBefore, membersAfter)); + // Notify any added users that they've been added to this resource. + notifications.push(this._inviteNotification(userId, org, userIdDelta, membersBefore)); + } + return {status: 200}; + }); + for (const notification of notifications) { notification(); } + return result; + } + + // Updates the permissions of users on the given workspace according to the PermissionDelta. + public async updateWorkspacePermissions( + scope: Scope, + wsId: number, + delta: PermissionDelta + ): Promise> { + const {userId} = scope; + const notifications: Array<() => void> = []; + const result = await this._connection.transaction(async manager => { + let userIdDelta = await this._verifyAndLookupDeltaEmails(userId, delta, false, manager); + let wsQuery = this._workspace(scope, wsId, { + manager, + markPermissions: Permissions.ACL_EDIT + }) + // Join the workspace's ACL rules and groups/users so we can edit them. + .leftJoinAndSelect('workspaces.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'workspace_groups') + .leftJoinAndSelect('workspace_groups.memberUsers', 'workspace_users') + // Join the workspace's org and org member groups so we know what should be inherited. + .leftJoinAndSelect('workspaces.org', 'org') + .leftJoinAndSelect('org.aclRules', 'org_acl_rules') + .leftJoinAndSelect('org_acl_rules.group', 'org_groups') + .leftJoinAndSelect('org_groups.memberUsers', 'org_users'); + wsQuery = this._addFeatures(wsQuery, 'org'); + const queryResult = await verifyIsPermitted(wsQuery); + if (queryResult.status !== 200) { + // If the query for the workspace failed, return the failure result. + return queryResult; + } + const ws: Workspace = queryResult.data; + // Get all the non-guest groups on the org. + const orgGroups = getNonGuestGroups(ws.org); + // Get all the non-guest groups to be updated by the delta. + const groups = getNonGuestGroups(ws); + if ('maxInheritedRole' in delta) { + // Honor the maxInheritedGroups delta setting. + this._moveInheritedGroups(groups, orgGroups, delta.maxInheritedRole); + if (delta.maxInheritedRole !== roles.OWNER) { + // If the maxInheritedRole was lowered from 'owners', add the calling user + // back as an owner so that their acl edit access is not revoked. + userIdDelta = userIdDelta || {}; + userIdDelta[userId] = roles.OWNER; + } + } + const membersBefore = this._withoutExcludedUsers(new Map(groups.map(grp => [grp.name, grp.memberUsers]))); + if (userIdDelta) { + // To check limits on shares, we track group members before and after call + // to _updateUserPermissions. Careful, that method mutates groups. + const nonOrgMembersBefore = this._getUserDifference(groups, orgGroups); + await this._updateUserPermissions(groups, userIdDelta, manager); + this._checkUserChangeAllowed(userId, groups); + const nonOrgMembersAfter = this._getUserDifference(groups, orgGroups); + const features = ws.org.billingAccount.product.features; + const limit = features.maxSharesPerWorkspace; + if (limit !== undefined) { + this._restrictShares(null, limit, removeRole(nonOrgMembersBefore), + removeRole(nonOrgMembersAfter), true, 'workspace', features); + } + } + await manager.save(groups); + // If the users in workspace were changed, make a call to repair the guests in the org. + if (userIdDelta) { + await this._repairOrgGuests(scope, ws.org.id, manager); + notifications.push(this._inviteNotification(userId, ws, userIdDelta, membersBefore)); + } + return {status: 200}; + }); + for (const notification of notifications) { notification(); } + return result; + } + + // Updates the permissions of users on the given doc according to the PermissionDelta. + public async updateDocPermissions( + scope: DocScope, + delta: PermissionDelta + ): Promise> { + const notifications: Array<() => void> = []; + const result = await this._connection.transaction(async manager => { + const {userId} = scope; + let userIdDelta = await this._verifyAndLookupDeltaEmails(userId, delta, false, manager); + const doc = await this._loadDocAccess(scope, Permissions.ACL_EDIT, manager); + // Get all the non-guest doc groups to be updated by the delta. + const groups = getNonGuestGroups(doc); + if ('maxInheritedRole' in delta) { + const wsGroups = getNonGuestGroups(doc.workspace); + // Honor the maxInheritedGroups delta setting. + this._moveInheritedGroups(groups, wsGroups, delta.maxInheritedRole); + if (delta.maxInheritedRole !== roles.OWNER) { + // If the maxInheritedRole was lowered from 'owners', add the calling user + // back as an owner so that their acl edit access is not revoked. + userIdDelta = userIdDelta || {}; + userIdDelta[userId] = roles.OWNER; + } + } + const membersBefore = new Map(groups.map(grp => [grp.name, grp.memberUsers])); + if (userIdDelta) { + // To check limits on shares, we track group members before and after call + // to _updateUserPermissions. Careful, that method mutates groups. + const org = doc.workspace.org; + const orgGroups = getNonGuestGroups(org); + const nonOrgMembersBefore = this._getUserDifference(groups, orgGroups); + await this._updateUserPermissions(groups, userIdDelta, manager); + this._checkUserChangeAllowed(userId, groups); + const nonOrgMembersAfter = this._getUserDifference(groups, orgGroups); + const features = org.billingAccount.product.features; + this._restrictAllDocShares(features, nonOrgMembersBefore, nonOrgMembersAfter); + } + await manager.save(groups); + if (userIdDelta) { + // If the users in the doc were changed, make calls to repair workspace then org guests. + await this._repairWorkspaceGuests(scope, doc.workspace.id, manager); + await this._repairOrgGuests(scope, doc.workspace.org.id, manager); + notifications.push(this._inviteNotification(userId, doc, userIdDelta, membersBefore)); + } + return {status: 200}; + }); + for (const notification of notifications) { notification(); } + return result; + } + + // Returns UserAccessData for all users with any permissions on the org. + public async getOrgAccess(scope: Scope, orgKey: string|number): Promise> { + const orgQuery = this.org(scope, orgKey, { + markPermissions: Permissions.VIEW, + needRealOrg: true + }) + // Join the org's ACL rules (with 1st level groups/users listed). + .leftJoinAndSelect('orgs.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'org_groups') + .leftJoinAndSelect('org_groups.memberUsers', 'org_member_users') + .leftJoinAndSelect('org_member_users.logins', 'user_logins'); + const queryResult = await verifyIsPermitted(orgQuery); + if (queryResult.status !== 200) { + // If the query for the doc failed, return the failure result. + return queryResult; + } + const org: Organization = queryResult.data; + const userRoleMap = getMemberUserRoles(org, this.defaultGroupNames); + const users = getResourceUsers(org).filter(u => userRoleMap[u.id]).map(u => ({ + id: u.id, + name: u.name, + email: u.logins.map((login: Login) => login.displayEmail)[0], + picture: u.picture, + access: userRoleMap[u.id] as roles.Role + })); + return { + status: 200, + data: { + users + } + }; + } + + // Returns UserAccessData for all users with any permissions on the ORG, as well as the + // maxInheritedRole set on the workspace. Note that information for all users in the org + // is given to indicate which users have access to the org but not to this particular workspace. + public async getWorkspaceAccess(scope: Scope, wsId: number): Promise> { + const wsQuery = this._workspace(scope, wsId, { + markPermissions: Permissions.VIEW + }) + // Join the workspace's ACL rules (with 1st level groups/users listed). + .leftJoinAndSelect('workspaces.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'workspace_groups') + .leftJoinAndSelect('workspace_groups.memberUsers', 'workspace_group_users') + .leftJoinAndSelect('workspace_groups.memberGroups', 'workspace_group_groups') + .leftJoinAndSelect('workspace_group_users.logins', 'workspace_user_logins') + // Join the org and groups/users. + .leftJoinAndSelect('workspaces.org', 'org') + .leftJoinAndSelect('org.aclRules', 'org_acl_rules') + .leftJoinAndSelect('org_acl_rules.group', 'org_groups') + .leftJoinAndSelect('org_groups.memberUsers', 'org_group_users') + .leftJoinAndSelect('org_group_users.logins', 'org_user_logins'); + const queryResult = await verifyIsPermitted(wsQuery); + if (queryResult.status !== 200) { + // If the query for the doc failed, return the failure result. + return queryResult; + } + const workspace: Workspace = queryResult.data; + const wsMap = getMemberUserRoles(workspace, this.defaultCommonGroupNames); + // The orgMap gives the org access inherited by each user. + const orgMap = getMemberUserRoles(workspace.org, this.defaultBasicGroupNames); + // Iterate through the org since all users will be in the org. + const users: UserAccessData[] = getResourceUsers(workspace.org).map(u => { + return { + id: u.id, + name: u.name, + email: u.logins.map((login: Login) => login.email)[0], + picture: u.picture, + access: wsMap[u.id] || null, + parentAccess: roles.getEffectiveRole(orgMap[u.id] || null) + }; + }); + return { + status: 200, + data: { + maxInheritedRole: this._getMaxInheritedRole(workspace), + users + } + }; + } + + // Returns UserAccessData for all users with any permissions on the ORG, as well as the + // maxInheritedRole set on the doc. Note that information for all users in the org is given + // to indicate which users have access to the org but not to this particular doc. + // TODO: Consider updating to traverse through the doc groups and their nested groups for + // a more straightforward way of determining inheritance. The difficulty here is that all users + // in the org and their logins are needed for inclusion in the result, which would require an + // extra lookup step when traversing from the doc. + public async getDocAccess(scope: DocScope): Promise> { + const doc = await this._loadDocAccess(scope, Permissions.VIEW); + const docMap = getMemberUserRoles(doc, this.defaultCommonGroupNames); + // The wsMap gives the ws access inherited by each user. + const wsMap = getMemberUserRoles(doc.workspace, this.defaultBasicGroupNames); + // The orgMap gives the org access inherited by each user. + const orgMap = getMemberUserRoles(doc.workspace.org, this.defaultBasicGroupNames); + const wsMaxInheritedRole = this._getMaxInheritedRole(doc.workspace); + // Iterate through the org since all users will be in the org. + const users: UserAccessData[] = getResourceUsers(doc.workspace.org).map(u => { + // Merge the strongest roles from the resource and parent resources. Note that the parent + // resource access levels must be tempered by the maxInheritedRole values of their children. + const inheritFromOrg = roles.getWeakestRole(orgMap[u.id] || null, wsMaxInheritedRole); + return { + id: u.id, + name: u.name, + email: u.logins.map((login: Login) => login.email)[0], + picture: u.picture, + access: docMap[u.id] || null, + parentAccess: roles.getEffectiveRole( + roles.getStrongestRole(wsMap[u.id] || null, inheritFromOrg) + ) + }; + }); + return { + status: 200, + data: { + maxInheritedRole: this._getMaxInheritedRole(doc), + users + } + }; + } + + public async moveDoc( + scope: DocScope, + wsId: number + ): Promise> { + return await this._connection.transaction(async manager => { + const {userId} = scope; + // Get the doc + const docQuery = this._doc(scope, { + manager, + markPermissions: Permissions.OWNER + }) + .leftJoinAndSelect('docs.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'doc_groups') + .leftJoinAndSelect('doc_groups.memberUsers', 'doc_users') + .leftJoinAndSelect('workspaces.aclRules', 'workspace_acl_rules') + .leftJoinAndSelect('workspace_acl_rules.group', 'workspace_groups') + .leftJoinAndSelect('workspace_groups.memberUsers', 'workspace_users') + .leftJoinAndSelect('orgs.aclRules', 'org_acl_rules') + .leftJoinAndSelect('org_acl_rules.group', 'org_groups') + .leftJoinAndSelect('org_groups.memberUsers', 'org_users'); + const docQueryResult = await verifyIsPermitted(docQuery); + if (docQueryResult.status !== 200) { + // If the query for the doc failed, return the failure result. + return docQueryResult; + } + const doc: Document = docQueryResult.data; + if (doc.workspace.id === wsId) { + return { + status: 400, + errMessage: `Bad request: doc is already in destination workspace` + }; + } + // Get the destination workspace + let wsQuery = this._workspace(scope, wsId, { + manager, + markPermissions: Permissions.ADD + }) + // Join the workspaces's ACL rules (with 1st level groups listed) so we can include + // them in the doc. + .leftJoinAndSelect('workspaces.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'workspace_groups') + .leftJoinAndSelect('workspace_groups.memberUsers', 'workspace_users') + .leftJoinAndSelect('workspaces.org', 'orgs') + .leftJoinAndSelect('orgs.aclRules', 'org_acl_rules') + .leftJoinAndSelect('org_acl_rules.group', 'org_groups') + .leftJoinAndSelect('org_groups.memberUsers', 'org_users'); + wsQuery = this._addFeatures(wsQuery); + const wsQueryResult = await verifyIsPermitted(wsQuery); + if (wsQueryResult.status !== 200) { + // If the query for the organization failed, return the failure result. + return wsQueryResult; + } + const workspace: Workspace = wsQueryResult.data; + // Collect all first-level users of the doc being moved. + const firstLevelUsers = getResourceUsers(doc); + const docGroups = doc.aclRules.map(rule => rule.group); + if (doc.workspace.org.id !== workspace.org.id) { + // Doc is going to a new org. Check that there is room for it there. + await this._checkRoomForAnotherDoc(userId, workspace, manager); + // Check also that doc doesn't have too many shares. + if (firstLevelUsers.length > 0) { + const sourceOrg = doc.workspace.org; + const sourceOrgGroups = getNonGuestGroups(sourceOrg); + const destOrg = workspace.org; + const destOrgGroups = getNonGuestGroups(destOrg); + const nonOrgMembersBefore = this._getUserDifference(docGroups, sourceOrgGroups); + const nonOrgMembersAfter = this._getUserDifference(docGroups, destOrgGroups); + const features = destOrg.billingAccount.product.features; + this._restrictAllDocShares(features, nonOrgMembersBefore, nonOrgMembersAfter, false); + } + } + // Update the doc workspace. + const oldWs = doc.workspace; + doc.workspace = workspace; + // The doc should have groups which properly inherit the permissions of the + // new workspace after it is moved. + // Update the doc groups to inherit the groups in the new workspace/org. + // Any previously custom added members remain in the doc groups. + doc.aclRules.forEach(aclRule => { + this._setInheritance(aclRule.group, workspace); + }); + // If the org is changing, remove all urlIds for this doc, since there could be + // conflicts in the new org. + // TODO: could try recreating/keeping the urlIds in the new org if there is in fact + // no conflict. Be careful about the merged personal org. + if (oldWs.org.id !== doc.workspace.org.id) { + doc.urlId = null; + await manager.delete(Alias, { doc: doc.id }); + } + // Forcibly remove the aliases relation from the document object, so that TypeORM + // doesn't try to save it. It isn't safe to do that because it was filtered by + // a where clause. + doc.aliases = undefined as any; + // Saves the document as well as its new ACL Rules and Groups and the + // updated guest group in the workspace. + await manager.save([doc, ...doc.aclRules, ...docGroups]); + if (firstLevelUsers.length > 0) { + // If the doc has first-level users, update the source and destination workspaces. + await this._repairWorkspaceGuests(scope, oldWs.id, manager); + await this._repairWorkspaceGuests(scope, doc.workspace.id, manager); + if (oldWs.org.id !== doc.workspace.org.id) { + // Also if the org changed, update the source and destination org guest groups. + await this._repairOrgGuests(scope, oldWs.org.id, manager); + await this._repairOrgGuests(scope, doc.workspace.org.id, manager); + } + } + return { + status: 200 + }; + }); + } + + // Pin or unpin a doc. + public async pinDoc( + scope: DocScope, + setPinned: boolean + ): Promise> { + return await this._connection.transaction(async manager => { + // Find the doc to assert that it exists. Assert that the user has edit access to the + // parent org. + const permissions = Permissions.EDITOR; + const docQuery = this._doc(scope, { + manager + }) + .addSelect(this._markIsPermitted('orgs', scope.userId, permissions), 'is_permitted'); + const docQueryResult = await verifyIsPermitted(docQuery); + if (docQueryResult.status !== 200) { + // If the query for the doc failed, return the failure result. + return docQueryResult; + } + const doc: Document = docQueryResult.data; + if (doc.isPinned !== setPinned) { + doc.isPinned = setPinned; + // Forcibly remove the aliases relation from the document object, so that TypeORM + // doesn't try to save it. It isn't safe to do that because it was filtered by + // a where clause. + doc.aliases = undefined as any; + // Save and return success status. + await manager.save(doc); + } + return { status: 200 }; + }); + } + + /** + * Updates the updatedAt values for several docs. Takes a map where each entry maps a docId to + * an ISO date string representing the new updatedAt time. This is not a part of the API, it + * should be called only by the HostedMetadataManager when a change is made to a doc. + */ + public async setDocsUpdatedAt( + docUpdateMap: {[docId: string]: string} + ): Promise> { + if (!docUpdateMap || Object.keys(docUpdateMap).length === 0) { + return { + status: 400, + errMessage: `Bad request: missing argument` + }; + } + const docIds = Object.keys(docUpdateMap); + return this._connection.transaction(async manager => { + const updateTasks = docIds.map(docId => { + return manager.createQueryBuilder() + .update(Document) + .set({updatedAt: docUpdateMap[docId]}) + .where("id = :docId", {docId}) + .execute(); + }); + await Promise.all(updateTasks); + return { status: 200 }; + }); + } + + /** + * Get the anonymous user, as a constructed object rather than a database lookup. + */ + public getAnonymousUser(): User { + const user = new User(); + user.id = this.getAnonymousUserId(); + user.name = "Anonymous"; + user.isFirstTimeUser = false; + const login = new Login(); + login.displayEmail = login.email = ANONYMOUS_USER_EMAIL; + user.logins = [login]; + return user; + } + + /** + * + * Get the id of the anonymous user. + * + */ + public getAnonymousUserId(): number { + const id = this._specialUserIds[ANONYMOUS_USER_EMAIL]; + if (!id) { throw new Error("Anonymous user not available"); } + return id; + } + + /** + * Get the id of the thumbnail user. + */ + public getPreviewerUserId(): number { + const id = this._specialUserIds[PREVIEWER_EMAIL]; + if (!id) { throw new Error("Previewer user not available"); } + return id; + } + + /** + * Get the id of the 'everyone' user. + */ + public getEveryoneUserId(): number { + const id = this._specialUserIds[EVERYONE_EMAIL]; + if (!id) { throw new Error("'everyone' user not available"); } + return id; + } + + /** + * Get the id of the 'support' user. + */ + public getSupportUserId(): number { + const id = this._specialUserIds[SUPPORT_EMAIL]; + if (!id) { throw new Error("'support' user not available"); } + return id; + } + + /** + * Get ids of users to be excluded from member counts and emails. + */ + public getExcludedUserIds(): number[] { + return [this.getSupportUserId(), this.getAnonymousUserId(), this.getEveryoneUserId()]; + } + + /** + * + * Take a list of user profiles coming from the client's session, correlate + * them with Users and Logins in the database, and construct full profiles + * with user ids, standardized display emails, pictures, and anonymous flags. + * + */ + public async completeProfiles(profiles: UserProfile[]): Promise { + if (profiles.length === 0) { return []; } + const qb = this._connection.createQueryBuilder() + .select('logins') + .from(Login, 'logins') + .leftJoinAndSelect('logins.user', 'user') + .where('logins.email in (:...emails)', {emails: profiles.map(profile => normalizeEmail(profile.email))}); + const completedProfiles: {[email: string]: FullUser} = {}; + for (const login of await qb.getMany()) { + completedProfiles[login.email] = { + id: login.user.id, + email: login.displayEmail, + name: login.user.name, + picture: login.user.picture, + anonymous: login.user.id === this.getAnonymousUserId() + }; + } + return profiles.map(profile => completedProfiles[normalizeEmail(profile.email)]) + .filter(profile => profile); + } + + /** + * Calculate the public-facing subdomain for an org. + * + * If the domain is a personal org, the public-facing subdomain will + * be docs/docs-s (if `mergePersonalOrgs` is set), or docs-[s]NNN where NNN + * is the user id (if `mergePersonalOrgs` is not set). + * + * If a domain is set in the database, and `suppressDomain` is not + * set, we report that domain verbatim. The `suppressDomain` may + * be set in some key endpoints in order to enforce a `vanityDomain` + * feature flag. + * + * Otherwise, we report o-NNN (or o-sNNN in staging) where NNN is + * the org id. + */ + public normalizeOrgDomain(orgId: number, domain: string|null, + ownerId: number|undefined, mergePersonalOrgs: boolean = true, + suppressDomain: boolean = false): string { + if (!domain) { + if (ownerId) { + // This is an org with no domain set, and an owner set. + domain = mergePersonalOrgs ? this.mergedOrgDomain() : `docs-${this._idPrefix}${ownerId}`; + } else { + // This is an org with no domain or owner set. + domain = `o-${this._idPrefix}${orgId}`; + } + } else if (suppressDomain) { + domain = `o-${this._idPrefix}${orgId}`; + } + return domain; + } + + // Throw an error for query results that represent errors or have no data; otherwise unwrap + // the valid result it contains. + public unwrapQueryResult(qr: QueryResult): T { + if (qr.data) { return qr.data; } + throw new ApiError(qr.errMessage || 'an error occurred', qr.status); + } + + // Throw an error for query results that represent errors + public checkQueryResult(qr: QueryResult) { + if (qr.status !== 200) { + throw new ApiError(qr.errMessage || 'an error occurred', qr.status); + } + } + + // Get the domain name for the merged organization. In production, this is 'docs', + // in staging, it is 'docs-s'. + public mergedOrgDomain() { + if (this._idPrefix) { + return `docs-${this._idPrefix}`; + } + return 'docs'; + } + + // The merged organization is a special pseudo-organization + // patched together from all the material a given user has access + // to. The result is approximately, but not exactly, an organization, + // and so it treated a bit differently. + public isMergedOrg(orgKey: string|number|null) { + return orgKey === this.mergedOrgDomain() || orgKey === 0; + } + + /** + * Construct a QueryBuilder for a select query on a specific org given by orgId. + * Provides options for running in a transaction and adding permission info. + * See QueryOptions documentation above. + */ + public org(scope: Scope, org: string|number|null, + options: QueryOptions = {}): SelectQueryBuilder { + return this._org(scope.userId, scope.includeSupport || false, org, options); + } + + private _org(userId: number|null, includeSupport: boolean, org: string|number|null, + options: QueryOptions = {}): SelectQueryBuilder { + let query = this._orgs(options.manager); + // merged pseudo-org must become personal org. + if (org === null || (options.needRealOrg && this.isMergedOrg(org))) { + if (!userId) { throw new Error('_org: requires userId'); } + query = query.where('orgs.owner_id = :userId', {userId}); + } else { + query = this._whereOrg(query, org, includeSupport); + } + if (options.markPermissions) { + if (!userId) { + throw new Error(`_orgQuery error: userId must be set to mark permissions`); + } + // Compute whether we have access to the doc + query = query.addSelect( + this._markIsPermitted('orgs', userId, options.markPermissions), + 'is_permitted' + ); + } + return query; + } + + /** + * Check if urlId is already in use in the given org, and throw an error if so. + * If the org is a personal org, we check for use of the urlId in any personal org. + * If docId is set, we permit the urlId to be in use by that doc. + */ + private async _checkForUrlIdConflict(manager: EntityManager, org: Organization, urlId: string, docId?: string) { + // Prepare a query to see if there is an existing conflicting urlId. + let aliasQuery = this._docs(manager) + .leftJoinAndSelect('docs.aliases', 'aliases') + .leftJoinAndSelect('aliases.org', 'orgs') + .where('docs.urlId = :urlId', {urlId}); // Place restriction on active urlIds only. + // Older urlIds are best-effort, and subject to + // reuse (currently). + if (org.ownerId === this.getSupportUserId()) { + // This is the support user. Some of their documents end up as examples on team sites. + // so urlIds need to be checked globally, which corresponds to placing no extra where + // clause here. + } else if (org.ownerId) { + // This is a personal org, so look for conflicts in any personal org + // (needed to ensure consistency in merged personal org). + // We don't need to do anything special about examples since they are stored in a personal + // org. + aliasQuery = aliasQuery.andWhere('orgs.owner_id is not null'); + } else { + // For team sites, just check within the team site. + // We also need to check within the support@ org for conflict with examples, which + // currently have an existence within team sites. + aliasQuery = aliasQuery.andWhere('(aliases.orgId = :orgId OR aliases.orgId = :exampleOrgId)', + {orgId: org.id, exampleOrgId: this._exampleOrgId}); + } + if (docId) { + aliasQuery = aliasQuery.andWhere('docs.id <> :docId', {docId}); + } + if (await aliasQuery.getOne()) { + throw new ApiError('urlId already in use', 400); + } + // Also forbid any urlId that would match an existing docId, that is a recipe for confusion + // and mischief. + if (await this._docs(manager).where('docs.id = :urlId', {urlId}).getOne()) { + throw new ApiError('urlId already in use as document id', 400); + } + } + + /** + * Updates the workspace guests with any first-level users of docs inside the workspace. + */ + private async _repairWorkspaceGuests(scope: Scope, wsId: number, transaction?: EntityManager): Promise { + return await this._runInTransaction(transaction, async manager => { + const wsQuery = this._workspace(scope, wsId, {manager}) + .leftJoinAndSelect('workspaces.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'groups') + .leftJoinAndSelect('workspaces.docs', 'docs') + .leftJoinAndSelect('docs.aclRules', 'doc_acl_rules') + .leftJoinAndSelect('doc_acl_rules.group', 'doc_groups') + .leftJoinAndSelect('doc_groups.memberUsers', 'doc_users'); + const workspace: Workspace = (await wsQuery.getOne())!; + const wsGuestGroup = workspace.aclRules.map(aclRule => aclRule.group) + .find(_grp => _grp.name === roles.GUEST); + if (!wsGuestGroup) { + throw new Error(`_repairWorkspaceGuests error: could not find ${roles.GUEST} ACL group`); + } + wsGuestGroup.memberUsers = this._filterEveryone(getResourceUsers(workspace.docs)); + await manager.save(wsGuestGroup); + }); + } + + /** + * Updates the org guests with any first-level users of workspaces inside the org. + * NOTE: If repairing both workspace and org guests, this should always be called AFTER + * _repairWorkspaceGuests. + */ + private async _repairOrgGuests(scope: Scope, orgKey: string|number, transaction?: EntityManager): Promise { + return await this._runInTransaction(transaction, async manager => { + const orgQuery = this.org(scope, orgKey, {manager}) + .leftJoinAndSelect('orgs.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'groups') + .leftJoinAndSelect('groups.memberUsers', 'users') + .andWhere('groups.name = :role', {role: roles.GUEST}); + const org = await orgQuery.getOne(); + if (!org) { throw new Error('cannot find org'); } + const workspaceQuery = this._workspaces(manager) + .where('workspaces.org_id = :orgId', {orgId: org.id}) + .leftJoinAndSelect('workspaces.aclRules', 'workspace_acl_rules') + .leftJoinAndSelect('workspace_acl_rules.group', 'workspace_group') + .leftJoinAndSelect('workspace_group.memberUsers', 'workspace_users') + .leftJoinAndSelect('workspaces.org', 'org'); + org.workspaces = await workspaceQuery.getMany(); + const orgGroups = org.aclRules.map(aclRule => aclRule.group); + if (orgGroups.length !== 1) { + throw new Error(`_repairOrgGuests error: found ${orgGroups.length} ${roles.GUEST} ACL group(s)`); + } + const orgGuestGroup = orgGroups[0]!; + orgGuestGroup.memberUsers = this._filterEveryone(getResourceUsers(org.workspaces)); + await manager.save(orgGuestGroup); + }); + } + + /** + * Don't add everyone@ as a guest, unless also sharing with anon@. + * This means that material shared with everyone@ doesn't become + * listable/discoverable by default. + * + * This is a HACK to allow existing example doc setup to continue to + * work. It could be removed if we are willing to share the entire + * support org with users. E.g. move any material we don't want to + * share into a workspace that doesn't inherit ACLs. TODO: remove + * this hack, or enhance it up as a way to support discoverability / + * listing. It has the advantage of cloning well. + */ + private _filterEveryone(users: User[]): User[] { + const everyone = this.getEveryoneUserId(); + const anon = this.getAnonymousUserId(); + if (users.find(u => u.id === anon)) { return users; } + return users.filter(u => u.id !== everyone); + } + + /** + * Creates, initializes and saves a workspace in the given org with the given properties. + * Product limits on number of workspaces allowed in org are not checked. + */ + private async _doAddWorkspace(org: Organization, props: Partial, + transaction?: EntityManager): Promise { + if (!props.name) { throw new ApiError('Bad request: name required', 400); } + return await this._runInTransaction(transaction, async manager => { + // Create a new workspace. + const workspace = new Workspace(); + workspace.checkProperties(props); + workspace.updateFromProperties(props); + workspace.org = org; + // Create the special initial permission groups for the new workspace. + const groupMap = this._createGroups(org); + workspace.aclRules = this.defaultCommonGroups.map(_grpDesc => { + // Get the special group with the name needed for this ACL Rule + const group = groupMap[_grpDesc.name]; + // Add each of the special groups to the new workspace. + const aclRuleWs = new AclRuleWs(); + aclRuleWs.permissions = _grpDesc.permissions; + aclRuleWs.group = group; + aclRuleWs.workspace = workspace; + return aclRuleWs; + }); + // Saves the workspace as well as its new ACL Rules and Group. + const groups = workspace.aclRules.map(rule => rule.group); + const result = await manager.save([workspace, ...workspace.aclRules, ...groups]); + return result[0]; + }); + } + + /** + * If the user is a manager of the billing account associated with + * the domain, an extra `billingAccount` field is returned, + * containing a `inGoodStanding` flag, a `status` json field, and a + * `product.paid` flag which is true if on a paid plan or false + * otherwise. Other `billingAccount` fields are included (stripe ids in + * particular) but these will not be reported across the API. + */ + private _addBillingAccount(qb: SelectQueryBuilder, userId: number) { + qb = qb.leftJoinAndSelect('orgs.billingAccount', 'billing_accounts'); + qb = qb.leftJoinAndSelect('billing_accounts.product', 'products'); + qb = qb.leftJoinAndSelect('billing_accounts.managers', 'managers', + 'managers.billing_account_id = billing_accounts.id and ' + + 'managers.user_id = :userId'); + qb = qb.setParameter('userId', userId); + qb = this._addBillingAccountCalculatedFields(qb); + return qb; + } + + /** + * Adds any calculated fields related to billing accounts - currently just + * products.paid. + */ + private _addBillingAccountCalculatedFields(qb: SelectQueryBuilder) { + // We need to sum up whether the account is paid or not, so that UI can provide + // a "billing" vs "upgrade" link. For the moment, we just check if there is + // a subscription id. TODO: make sure this is correct in case of free plans. + qb = qb.addSelect(`(billing_accounts.stripe_subscription_id is not null)`, 'billing_accounts_paid'); + return qb; + } + + /** + * Makes sure that product features for orgs are available in query result. + */ + private _addFeatures(qb: SelectQueryBuilder, orgAlias: string = 'orgs') { + qb = qb.leftJoinAndSelect(`${orgAlias}.billingAccount`, 'billing_accounts'); + qb = qb.leftJoinAndSelect('billing_accounts.product', 'products'); + // orgAlias.billingAccount.product.features should now be available + return qb; + } + + private _addIsSupportWorkspace(users: AvailableUsers, qb: SelectQueryBuilder, + orgAlias: string, workspaceAlias: string) { + const supportId = this._specialUserIds[SUPPORT_EMAIL]; + + // We'll be selecting a boolean and naming it as *_support. This matches the + // SQL name `support` of a column in the Workspace entity whose javascript + // name is `isSupportWorkspace`. + const alias = `${workspaceAlias}_support`; + + // If we happen to be the support user, don't treat our workspaces as anything + // special, so we can work with them in the ordinary way. + if (isSingleUser(users) && users === supportId) { return qb.addSelect('false', alias); } + + // Otherwise, treat workspaces owned by support as special. + return qb.addSelect(`coalesce(${orgAlias}.owner_id = ${supportId}, false)`, alias); + } + + /** + * + * Get the id of a special user, creating that user if it is not already present. + * + */ + private async _getSpecialUserId(profile: UserProfile) { + let id = this._specialUserIds[profile.email]; + if (!id) { + // get or create user - with retry, since there'll be a race to create the + // user if a bunch of servers start simultaneously and the user doesn't exist + // yet. + const user = await this.getUserByLoginWithRetry(profile.email, profile); + if (user) { id = this._specialUserIds[profile.email] = user.id; } + } + if (!id) { throw new Error(`Could not find or create user ${profile.email}`); } + return id; + } + + // This deals with the problem posed by receiving a PermissionDelta specifying a + // role for both alice@x and Alice@x. We do not distinguish between such emails. + // If there are multiple indistinguishabe emails, we preserve just one of them, + // assigning it the most powerful permission specified. The email variant perserved + // is the earliest alphabetically. + private _mergeIndistinguishableEmails(delta: PermissionDelta) { + if (!delta.users) { return; } + // We normalize emails for comparison, but track how they were capitalized + // in order to preserve it. This is worth doing since for the common case + // of a user being added to a resource prior to ever logging in, their + // displayEmail will be seeded from this value. + const displayEmails: {[email: string]: string} = {}; + // This will be our output. + const users: {[email: string]: roles.NonGuestRole|null} = {}; + for (const displayEmail of Object.keys(delta.users).sort()) { + const email = normalizeEmail(displayEmail); + const role = delta.users[displayEmail]; + const key = displayEmails[email] = displayEmails[email] || displayEmail; + users[key] = users[key] ? roles.getStrongestRole(users[key], role) : role; + } + delta.users = users; + } + + // Looks up the emails in the permission delta and adds them to the users map in + // the delta object. + // Returns a QueryResult based on the validity of the passed in PermissionDelta object. + private async _verifyAndLookupDeltaEmails( + userId: number, + delta: PermissionDelta, + isOrg: boolean = false, + transaction?: EntityManager + ): Promise { + if (!delta) { + throw new ApiError('Bad request: missing permission delta', 400); + } + this._mergeIndistinguishableEmails(delta); + const hasInherit = 'maxInheritedRole' in delta; + const hasUsers = delta.users; // allow zero actual changes; useful to reduce special + // cases in scripts + if ((isOrg && (hasInherit || !hasUsers)) || (!isOrg && !hasInherit && !hasUsers)) { + throw new ApiError('Bad request: invalid permission delta', 400); + } + // Lookup the email access changes and move them to the users object. + const userIdMap: {[userId: string]: roles.NonGuestRole|null} = {}; + if (hasInherit) { + // Verify maxInheritedRole + const role = delta.maxInheritedRole; + const validRoles = new Set(this.defaultBasicGroupNames); + if (role && !validRoles.has(role)) { + throw new ApiError(`Invalid maxInheritedRole ${role}`, 400); + } + } + if (delta.users) { + // Verify roles + const deltaRoles = Object.keys(delta.users).map(_userId => delta.users![_userId]); + // Cannot set role "members" on workspace/doc. + const validRoles = new Set(isOrg ? this.defaultNonGuestGroupNames : this.defaultBasicGroupNames); + for (const role of deltaRoles) { + if (role && !validRoles.has(role)) { + throw new ApiError(`Invalid user role ${role}`, 400); + } + } + // Lookup emails + const emailMap = delta.users; + const emails = Object.keys(emailMap); + const emailUsers = await Promise.all( + emails.map(async email => await this.getUserByLogin(email, undefined, transaction)) + ); + emails.forEach((email, i) => { + const userIdAffected = emailUsers[i]!.id; + // Org-level sharing with everyone would allow serious spamming - forbid it. + if (emailMap[email] !== null && // allow removing anything + userId !== this.getSupportUserId() && // allow support user latitude + userIdAffected === this.getEveryoneUserId() && + isOrg) { + throw new ApiError('This user cannot share with everyone at top level', 403); + } + userIdMap[userIdAffected] = emailMap[email]; + }); + } + if (userId in userIdMap) { + // TODO: Consider when to allow updating own permissions - allowing updating own + // permissions indiscriminately could lead to orphaned resources. + throw new ApiError('Bad request: cannot update own permissions', 400); + } + return delta.users ? userIdMap : null; + } + + /** + * Helper for adjusting acl rules. Given an array of top-level groups from the resource + * of interest, returns the updated groups. The returned groups should be saved to + * update the group inheritance in the database. Updates the passed in groups. + * + * NOTE that all group memberUsers must be populated. + */ + private async _updateUserPermissions( + groups: NonGuestGroup[], + userDelta: UserIdDelta, + manager: EntityManager + ): Promise { + // Get the user objects which map to non-null values in the userDelta. + const userIds = Object.keys(userDelta).filter(userId => userDelta[userId]) + .map(userIdStr => parseInt(userIdStr, 10)); + const users = await this._getUsers(userIds, manager); + + // Add unaffected users to the delta so that we have a record of where they are. + groups.forEach(grp => { + grp.memberUsers.forEach(usr => { + if (!(usr.id in userDelta)) { + userDelta[usr.id] = grp.name; + users.push(usr); + } + }); + }); + + // Create mapping from group names to top-level groups (contain the inherited groups) + const topGroups: {[groupName: string]: NonGuestGroup} = {}; + groups.forEach(grp => { + // Note that this has a side effect of resetting the memberUsers arrays. + grp.memberUsers = []; + topGroups[grp.name] = grp; + }); + + // Add users to groups (this has a side-effect of updating the group memberUsers) + users.forEach(user => { + const groupName = userDelta[user.id]!; + // NOTE that the special names constant is ordered from least to most permissive. + // The destination must be a reserved inheritance group or null. + if (groupName && !this.defaultNonGuestGroupNames.includes(groupName)) { + throw new Error(`_updateUserPermissions userDelta contains invalid group`); + } + topGroups[groupName].memberUsers.push(user); + }); + } + + /** + * Run an operation in an existing transaction if available, otherwise create + * a new transaction for it. + * + * @param transaction: the manager of an existing transaction, or undefined. + * @param op: the operation to run in a transaction. + */ + private _runInTransaction(transaction: EntityManager|undefined, + op: (manager: EntityManager) => Promise): Promise { + if (transaction) { return op(transaction); } + return this._connection.transaction(op); + } + + /** + * Returns a Promise for an array of User entites for the given userIds. + */ + private async _getUsers(userIds: number[], optManager?: EntityManager): Promise { + if (userIds.length === 0) { + return []; + } + const manager = optManager || new EntityManager(this._connection); + const queryBuilder = manager.createQueryBuilder() + .select('users') + .from(User, 'users') + .where('users.id IN (:...userIds)', {userIds}); + return await queryBuilder.getMany(); + } + + /** + * Aggregate the given columns as a json object. The keys should be simple + * alphanumeric strings, and the values should be the names of sql columns - + * this method is not set up to quote concrete values. + */ + private _aggJsonObject(content: {[key: string]: string}): string { + const args = [...Object.keys(content).map(key => [`'${key}'`, content[key]])]; + if (this._dbType === 'postgres') { + return `json_agg(json_build_object(${args.join(',')}))`; + } else { + return `json_group_array(json_object(${args.join(',')}))`; + } + } + + private _docs(manager?: EntityManager) { + return (manager || this._connection).createQueryBuilder() + .select('docs') + .from(Document, 'docs'); + } + + /** + * Construct a QueryBuilder for a select query on a specific doc given by urlId. + * Provides options for running in a transaction and adding permission info. + * See QueryOptions documentation above. + * + * In order to accept urlIds, the aliases, workspaces, and orgs tables are joined. + */ + private _doc(scope: DocScope, options: QueryOptions = {}): SelectQueryBuilder { + const {urlId, userId} = scope; + // Check if doc is being accessed with a merged org url. If so, + // we will only filter urlId matches, and will allow docId matches + // for team site documents. This is for backwards compatibility, + // to support https://docs.getgrist.com/api/docs/ for team + // site documents. + const mergedOrg = this.isMergedOrg(scope.org || null); + let query = this._docs(options.manager) + .leftJoinAndSelect('docs.workspace', 'workspaces') + .leftJoinAndSelect('workspaces.org', 'orgs') + .leftJoinAndSelect('docs.aliases', 'aliases') + .where(new Brackets(cond => { + return cond + .where('docs.id = :urlId', {urlId}) + .orWhere(new Brackets(urlIdCond => { + let urlIdQuery = urlIdCond + .where('aliases.url_id = :urlId', {urlId}) + .andWhere('aliases.org_id = orgs.id'); + if (mergedOrg) { + // Filter specifically for merged org documents. + urlIdQuery = urlIdQuery.andWhere('orgs.owner_id is not null'); + } + return urlIdQuery; + })); + })); + // TODO includeSupport should really be false, and the support for it should be removed. + // (For this, example doc URLs should be under docs.getgrist.com rather than team domains.) + // Add access information and query limits + query = this._applyLimit(query, {...scope, includeSupport: true}, ['docs', 'workspaces', 'orgs']); + if (options.markPermissions) { + let effectiveUserId = userId; + let threshold = options.markPermissions; + if (options.allowSpecialPermit && scope.specialPermit && scope.specialPermit.docId) { + query = query.andWhere('docs.id = :docId', {docId: scope.specialPermit.docId!}); + effectiveUserId = this.getPreviewerUserId(); + threshold = Permissions.VIEW; + } + // Compute whether we have access to the doc + query = query.addSelect( + this._markIsPermitted('docs', effectiveUserId, threshold), + 'is_permitted' + ); + } + return query; + } + + private _workspaces(manager?: EntityManager) { + return (manager || this._connection).createQueryBuilder() + .select('workspaces') + .from(Workspace, 'workspaces'); + } + + /** + * Construct "ON" clause for joining docs. This clause takes care of filtering + * out any docs that are not to be listed due to soft deletion. This filtering + * is done in the "ON" clause rather than in a "WHERE" clause since we still + * want to list workspaces even if there are no docs within them. A "WHERE" clause + * would entirely remove information about a workspace with no docs. The "ON" + * clause, in combination with a "LEFT JOIN", preserves the workspace information + * and just sets doc information to NULL. + */ + private _onDoc(scope: Scope) { + const onDefault = 'docs.workspace_id = workspaces.id'; + if (scope.showAll) { + return onDefault; + } else if (scope.showRemoved) { + return `${onDefault} AND (workspaces.removed_at IS NOT NULL OR docs.removed_at IS NOT NULL)`; + } else { + return `${onDefault} AND (workspaces.removed_at IS NULL AND docs.removed_at IS NULL)`; + } + } + + /** + * Construct a QueryBuilder for a select query on a specific workspace given by + * wsId. Provides options for running in a transaction and adding permission info. + * See QueryOptions documentation above. + */ + private _workspace(scope: Scope, wsId: number, options: QueryOptions = {}): SelectQueryBuilder { + let query = this._workspaces(options.manager) + .where('workspaces.id = :wsId', {wsId}); + if (options.markPermissions) { + let effectiveUserId = scope.userId; + let threshold = options.markPermissions; + if (options.allowSpecialPermit && scope.specialPermit && + scope.specialPermit.workspaceId === wsId) { + effectiveUserId = this.getPreviewerUserId(); + threshold = Permissions.VIEW; + } + // Compute whether we have access to the doc + query = query.addSelect( + this._markIsPermitted('workspaces', effectiveUserId, threshold), + 'is_permitted' + ); + } + return query; + } + + private _orgs(manager?: EntityManager) { + return (manager || this._connection).createQueryBuilder() + .select('orgs') + .from(Organization, 'orgs'); + } + + // Adds a where clause to filter orgs by domain or id. + // If org is null, filter for user's personal org. + // if includeSupport is true, include the org of the support@ user (for the Samples workspace) + private _whereOrg(qb: T, org: string|number, includeSupport = false): T { + if (this.isMergedOrg(org)) { + // Select from universe of personal orgs. + // Don't panic though! While this means that SQL can't use an organization id + // to narrow down queries, it will still be filtering via joins against the user and + // groups the user belongs to. + qb = qb.andWhere('orgs.owner_id is not null'); + return qb; + } + // Always include the org of the support@ user, which contains the Samples workspace, + // which we always show. (For isMergedOrg case, it's already included.) + if (includeSupport) { + const supportId = this._specialUserIds[SUPPORT_EMAIL]; + return qb.andWhere(new Brackets((q) => + this._wherePlainOrg(q, org).orWhere('orgs.owner_id = :supportId', {supportId}))); + } else { + return this._wherePlainOrg(qb, org); + } + } + + private _wherePlainOrg(qb: T, org: string|number): T { + if (typeof org === 'number') { + return qb.andWhere('orgs.id = :org', {org}); + } + if (org.startsWith(`docs-${this._idPrefix}`)) { + // this is someone's personal org + const ownerId = org.split(`docs-${this._idPrefix}`)[1]; + qb = qb.andWhere('orgs.owner_id = :ownerId', {ownerId}); + } else if (org.startsWith(`o-${this._idPrefix}`)) { + // this is an org identified by org id + const orgId = org.split(`o-${this._idPrefix}`)[1]; + qb = qb.andWhere('orgs.id = :orgId', {orgId}); + } else { + // this is a regular domain + qb = qb.andWhere('orgs.domain = :org', {org}); + } + return qb; + } + + private _withAccess(qb: SelectQueryBuilder, users: AvailableUsers, + table: 'orgs'|'workspaces'|'docs') { + return qb + .addSelect(this._markIsPermitted(table, users, null), `${table}_permissions`); + } + + /** + * Filter for orgs for which the user is a member of a group (or which are shared + * with "everyone@"). For access to workspaces and docs, we rely on the fact that + * the user will be added to a guest group at the organization level. + * + * If AvailableUsers is a profile list, we do NOT include orgs accessible + * via "everyone@" (this affects the "api/session/access/all" endpoint). + * + * Otherwise, orgs shared with "everyone@" are candidates for inclusion. + * If an orgKey is supplied, it is the only org which will be considered + * for inclusion on the basis of sharing with "everyone@". TODO: consider + * whether this wrinkle is needed anymore, or can be safely removed. + */ + private _filterByOrgGroups(qb: SelectQueryBuilder, users: AvailableUsers, + orgKey: string|number|null = null) { + qb = qb + .leftJoin('orgs.aclRules', 'acl_rules') + .leftJoin('acl_rules.group', 'groups') + .leftJoin('groups.memberUsers', 'members'); + if (isSingleUser(users)) { + // Add an exception for the previewer user, if present. + const previewerId = this._specialUserIds[PREVIEWER_EMAIL]; + if (users === previewerId) { return qb; } + const everyoneId = this._specialUserIds[EVERYONE_EMAIL]; + return qb.andWhere(new Brackets(cond => { + // Accept direct membership, or via a share with "everyone@". + return cond + .where('members.id = :userId', {userId: users}) + .orWhere(new Brackets(everyoneCond => { + const everyoneQuery = everyoneCond.where('members.id = :everyoneId', {everyoneId}); + return orgKey ? this._whereOrg(everyoneQuery, orgKey) : everyoneQuery; + })); + })); + } + + // The user hasn't been narrowed down to one choice, so join against logins and + // check normalized email. + const emails = new Set(users.map(profile => normalizeEmail(profile.email))); + // Empty list needs to be special-cased since "in ()" isn't supported in postgres. + if (emails.size === 0) { return qb.andWhere('1 = 0'); } + return qb + .leftJoin('members.logins', 'memberLogins') + .andWhere('memberLogins.email in (:...emails)', {emails: [...emails]}); + } + + private _single(result: QueryResult) { + if (result.status === 200) { + // TODO: assert result is really singular. + result.data = result.data[0]; + } + return result; + } + + /** + * Helper for adjusting acl inheritance rules. Given an array of top-level groups from the + * resource of interest, and an array of inherited groups belonging to the parent resource, + * moves the inherited groups to the group with the destination name or lower, if their + * permission level is lower. If the destination group name is omitted, the groups are + * moved to their original inheritance locations. If the destination group name is null, + * the groups are all removed and there is no access inheritance to this resource. + * Returns the updated array of top-level groups. These returned groups should be saved + * to update the group inheritance in the database. + * + * For all passed-in groups, their .memberGroups will be reset. For + * the basic roles (owner | editor | viewer), these will get updated + * to include inheritedGroups, with roles reduced to dest when dest + * is given. All of the basic roles must be present among + * groups. Any non-basic roles present among inheritedGroups will be + * ignored. + * + * Does not modify inheritedGroups. + */ + private _moveInheritedGroups( + groups: NonGuestGroup[], inheritedGroups: Group[], dest?: roles.BasicRole|null + ): void { + // Limit scope to those inheritedGroups that have basic roles (viewers, editors, owners). + inheritedGroups = inheritedGroups.filter(group => roles.isBasicRole(group.name)); + + // NOTE that the special names constant is ordered from least to most permissive. + const reverseDefaultNames = this.defaultBasicGroupNames.reverse(); + + // The destination must be a reserved inheritance group or null. + if (dest && !reverseDefaultNames.includes(dest)) { + throw new Error('moveInheritedGroups called with invalid destination name'); + } + + // Mapping from group names to top-level groups + const topGroups: {[groupName: string]: NonGuestGroup} = {}; + groups.forEach(grp => { + // Note that this has a side effect of initializing the memberGroups arrays. + grp.memberGroups = []; + topGroups[grp.name] = grp; + }); + + // The destFunc maps from an inherited group to its required top-level group name. + const destFunc = (inherited: Group) => + dest === null ? null : reverseDefaultNames.find(sp => sp === inherited.name || sp === dest); + + // Place inherited groups (this has the side-effect of updating member groups) + inheritedGroups.forEach(grp => { + if (!roles.isBasicRole(grp.name)) { + // We filtered out such groups at the start of this method, but just in case... + throw new Error(`${grp.name} is not an inheritable group`); + } + const moveTo = destFunc(grp); + if (moveTo) { + topGroups[moveTo].memberGroups.push(grp); + } + }); + } + + /** + * Returns a name to group mapping for the standard groups. Useful when adding a new child + * entity. Finds and includes the correct parent groups as member groups. + */ + private _createGroups(inherit?: Organization|Workspace): {[name: string]: Group} { + const groupMap: {[name: string]: Group} = {}; + this.defaultGroups.forEach(groupProps => { + if (!groupProps.orgOnly || !inherit) { + // Skip this group if it's an org only group and the resource inherits from a parent. + const group = new Group(); + group.name = groupProps.name as roles.Role; + if (inherit) { + this._setInheritance(group, inherit); + } + groupMap[groupProps.name] = group; + } + }); + return groupMap; + } + + // Sets the given group to inherit the groups in the given parent resource. + private _setInheritance(group: Group, parent: Organization|Workspace) { + // Add the parent groups to the group + const groupProps = this.defaultGroups.find(special => special.name === group.name); + if (!groupProps) { + throw new Error(`Non-standard group passed to _addInheritance: ${group.name}`); + } + if (groupProps.nestParent) { + const parentGroups = (parent.aclRules as AclRule[]).map((_aclRule: AclRule) => _aclRule.group); + const inheritGroup = parentGroups.find((_parentGroup: Group) => _parentGroup.name === group.name); + if (!inheritGroup) { + throw new Error(`Special group ${group.name} not found in ${parent.name} for inheritance`); + } + group.memberGroups = [inheritGroup]; + } + } + + // Return a QueryResult reflecting the output of a query builder. + // Checks on all "permissions" fields which select queries set on + // resources to indicate whether the user has access. + // If the output is empty, and `emptyAllowed` is not set, we signal that the desired + // resource does not exist (404). + // If the overall permissions do not allow viewing, we signal that the resource is forbidden. + // Access fields are added to all entities giving the group name corresponding + // with the access level of the user. + // Returns the resource fetched by the queryBuilder. + private async _verifyAclPermissions( + queryBuilder: SelectQueryBuilder, + emptyAllowed: boolean = false + ): Promise> { + const results = await queryBuilder.getRawAndEntities(); + if (results.entities.length === 0 || + (results.entities.length === 1 && results.entities[0].filteredOut)) { + if (emptyAllowed) { return {status: 200, data: []}; } + return {errMessage: `${getFrom(queryBuilder)} not found`, status: 404}; + } + const resources = this._normalizeQueryResults(results.entities); + if (resources.length === 0 && !emptyAllowed) { + return {errMessage: "access denied", status: 403}; + } else { + return { + status: 200, + data: resources + }; + } + } + + // Normalize query results in the following ways: + // * Convert `permissions` fields to summary `access` fields. + // * Set appropriate `domain` fields for personal organizations. + // * Include `billingAccount` field only for a billing account manager. + // * Replace `user.logins` objects with user.email and user.anonymous. + // * Collapse fields from nested `manager.user` objects into the surrounding + // `manager` objects. + // + // Find any nested entities with a "permissions" field, and add to them an + // "access" field (if the permission is a simple number) or an "accessOptions" + // field (if the permission is json). Entities in a list that the user doesn't + // have the right to access are removed. + // + // When returning organizations, set the domain to docs-${userId} for personal orgs. + // We could also have simply stored that domain in the database, but have kept + // them out for now, for the flexibility to change how we want these kinds of orgs + // to be presented without having to do awkward migrations. + // + // The suppressDomain option ensures that any organization domains are given + // in ugly o-NNNN form. + private _normalizeQueryResults(value: any, + options: { + suppressDomain?: boolean + } = {}): any { + // We only need to examine objects, excluding null. + if (typeof value !== 'object' || value === null) { return value; } + // For arrays, add access information and remove anything user doesn't have access to. + if (Array.isArray(value)) { + return value.map(v => this._normalizeQueryResults(v, options)).filter(v => !this._isForbidden(v)); + } + // For hashes, iterate through key/values, adding access info if 'permissions' field is found. + if (value.billingAccount) { + // This is an organization with billing account information available. Check limits. + const org = value as Organization; + const features = org.billingAccount.product.features; + if (!features.vanityDomain) { + // Vanity domain not allowed for this org. + options = {...options, suppressDomain: true}; + } + } + for (const key of Object.keys(value)) { + const subValue = value[key]; + // When returning organizations, set the domain to docs-${userId} for personal orgs. + // We could also have simply stored that domain in the database. I'd prefer to keep + // them out for now, for the flexibility to change how we want these kinds of orgs + // to be presented without having to do awkward migrations. + if (key === 'domain') { + value[key] = this.normalizeOrgDomain(value.id, subValue, value.owner && value.owner.id, + false, options.suppressDomain); + continue; + } + if (key === 'billingAccount') { + if (value[key].managers) { + value[key].isManager = Boolean(value[key].managers.length); + delete value[key].managers; + } + continue; + } + if (key === 'logins') { + const logins = subValue; + delete value[key]; + if (logins.length !== 1) { + throw new ApiError('Cannot find unique login for user', 500); + } + value.email = logins[0].displayEmail; + value.anonymous = (logins[0].userId === this.getAnonymousUserId()); + continue; + } + if (key === 'managers') { + const managers = this._normalizeQueryResults(subValue, options); + for (const manager of managers) { + if (manager.user) { + Object.assign(manager, manager.user); + delete manager.user; + } + } + value[key] = managers; + continue; + } + if (key !== 'permissions') { + value[key] = this._normalizeQueryResults(subValue, options); + continue; + } + if (typeof subValue === 'number' || !subValue) { + // Find the first special group for which the user has all permissions. + value.access = this._getRoleFromPermissions(subValue || 0); + if (subValue & Permissions.PUBLIC) { // tslint:disable-line:no-bitwise + value.public = true; + } + } else { + // Resource may be accessed by multiple users, encoded in JSON. + const accessOptions: AccessOption[] = readJson(this._dbType, subValue); + value.accessOptions = accessOptions.map(option => ({ + access: this._getRoleFromPermissions(option.perms), ...option + })); + } + delete value.permissions; // permissions is not specified in the api, so we drop it. + } + return value; + } + + // entity is forbidden if it contains an access field set to null, or an accessOptions field + // that is the empty list. + private _isForbidden(entity: any): boolean { + if (!entity) { return false; } + if (entity.access === null) { return true; } + if (entity.filteredOut) { return true; } + if (!entity.accessOptions) { return false; } + return entity.accessOptions.length === 0; + } + + // Returns the most permissive default role that does not have more permissions than the passed + // in argument. + private _getRoleFromPermissions(permissions: number): roles.Role|null { + permissions &= ~Permissions.PUBLIC; // tslint:disable-line:no-bitwise + const group = this.defaultBasicGroups.find(grp => + (permissions & grp.permissions) === grp.permissions); // tslint:disable-line:no-bitwise + return group ? group.name : null; + } + + // Returns the maxInheritedRole group name set on a resource. + // The resource's aclRules, groups, and memberGroups must be populated. + private _getMaxInheritedRole(res: Workspace|Document): roles.BasicRole|null { + const groups = (res.aclRules as AclRule[]).map((_aclRule: AclRule) => _aclRule.group); + let maxInheritedRole: roles.NonGuestRole|null = null; + for (const name of this.defaultBasicGroupNames) { + const group = groups.find(_grp => _grp.name === name); + if (!group) { + throw new Error(`Error in _getMaxInheritedRole: group ${name} not found in ${res.name}`); + } + if (group.memberGroups.length > 0) { + maxInheritedRole = name; + break; + } + } + return roles.getEffectiveRole(maxInheritedRole); + } + + /** + * Return a query builder to check if we have access to the given resource. + * Tests the given permission-level access, defaulting to view permission. + * @param resType: type of resource (table name) + * @param userId: id of user accessing the resource + * @param permissions: permission to test for - if null, we return the permissions + */ + private _markIsPermitted( + resType: 'orgs'|'workspaces'|'docs', + users: AvailableUsers, + permissions: Permissions|null = Permissions.VIEW + ): (qb: SelectQueryBuilder) => SelectQueryBuilder { + const idColumn = resType.slice(0, -1) + "_id"; + return qb => { + const getBasicPermissions = (q: SelectQueryBuilder) => { + if (permissions !== null) { + q = q.select('acl_rules.permissions'); + } else { + const everyoneId = this._specialUserIds[EVERYONE_EMAIL]; + const anonId = this._specialUserIds[ANONYMOUS_USER_EMAIL]; + // Overall permissions are the bitwise-or of all individual + // permissions from ACL rules. We also include + // Permissions.PUBLIC if any of the ACL rules are for the + // public (shared with everyone@ or anon@). This could be + // optimized if we eliminate one of those users. The guN + // aliases are joining in _getUsersAcls, and refer to the + // group_users table at different levels of nesting. + q = q.select( + bitOr(this._dbType, `(acl_rules.permissions | (case when ` + + `${everyoneId} IN (gu0.user_id, gu1.user_id, gu2.user_id, gu3.user_id) OR ` + + `${anonId} IN (gu0.user_id, gu1.user_id, gu2.user_id, gu3.user_id) ` + + `then ${Permissions.PUBLIC} else 0 end))`, 8), 'permissions'); + } + q = q.from('acl_rules', 'acl_rules'); + q = this._getUsersAcls(q, users); + q = q.andWhere(`acl_rules.${idColumn} = ${resType}.id`); + if (permissions !== null) { + q = q.andWhere(`(acl_rules.permissions & ${permissions}) = ${permissions}`).limit(1); + } else if (!isSingleUser(users)) { + q = q.addSelect('profiles.id'); + q = q.addSelect('profiles.display_email'); + q = q.addSelect('profiles.name'); + // anything we select without aggregating, we must also group by (postgres is fussy + // about this) + q = q.groupBy('profiles.id'); + q = q.addGroupBy('profiles.display_email'); + q = q.addGroupBy('profiles.name'); + } + return q; + }; + if (isSingleUser(users)) { + return getBasicPermissions(qb.subQuery()); + } else { + return qb.subQuery() + .from(subQb => getBasicPermissions(subQb.subQuery()), 'options') + .select(this._aggJsonObject({id: 'options.id', + email: 'options.display_email', + perms: 'options.permissions', + name: 'options.name'})); + } + }; + } + + // Takes a query that includes acl_rules, and filters for just those acl_rules that apply + // to the user, either directly or via up to three layers of nested groups. Two layers are + // sufficient for our current ACL setup. A third is added as a low-cost preparation + // for implementing something like teams in the future. It has no measurable effect on + // speed. + private _getUsersAcls(qb: SelectQueryBuilder, users: AvailableUsers) { + // Every acl_rule is associated with a single group. A user may + // be a direct member of that group, via the group_users table. + // Or they may be a member of a group that is a member of that + // group, via group_groups. Or they may be even more steps + // removed. We unroll to a fixed number of steps, and use joins + // rather than a recursive query, since we need this step to be as + // fast as possible. + qb = qb + // filter for the specified user being a direct or indirect member of the acl_rule's group + .where(new Brackets(cond => { + if (isSingleUser(users)) { + // Users is an integer, so ok to insert into sql. It we + // didn't, we'd need to use distinct parameter names, since + // we may include this code with different user ids in the + // same query + cond = cond.where(`gu0.user_id = ${users}`); + cond = cond.orWhere(`gu1.user_id = ${users}`); + cond = cond.orWhere(`gu2.user_id = ${users}`); + cond = cond.orWhere(`gu3.user_id = ${users}`); + // Support the special "everyone" user. + const everyoneId = this._specialUserIds[EVERYONE_EMAIL]; + cond = cond.orWhere(`gu0.user_id = ${everyoneId}`); + cond = cond.orWhere(`gu1.user_id = ${everyoneId}`); + cond = cond.orWhere(`gu2.user_id = ${everyoneId}`); + cond = cond.orWhere(`gu3.user_id = ${everyoneId}`); + // Add an exception for the previewer user, if present. + const previewerId = this._specialUserIds[PREVIEWER_EMAIL]; + if (users === previewerId) { + // All acl_rules granting view access are available to previewer user. + cond = cond.orWhere('acl_rules.permissions = :permission', + {permission: Permissions.VIEW}); + } + } else { + cond = cond.where('gu0.user_id = profiles.id'); + cond = cond.orWhere('gu1.user_id = profiles.id'); + cond = cond.orWhere('gu2.user_id = profiles.id'); + cond = cond.orWhere('gu3.user_id = profiles.id'); + } + return cond; + })); + if (!isSingleUser(users)) { + // We need to join against a list of users. + const emails = new Set(users.map(profile => normalizeEmail(profile.email))); + if (emails.size > 0) { + // the 1 = 1 on clause seems the shortest portable way to do a cross join in postgres + // and sqlite via typeorm. + qb = qb.leftJoin('(select users.id, display_email, email, name from users inner join logins ' + + 'on users.id = logins.user_id where logins.email in (:...emails))', + 'profiles', '1 = 1'); + qb = qb.setParameter('emails', [...emails]); + } else { + // Add a dummy user with id 0, for simplicity. This user will + // not match any group. The casts are needed for a postgres 9.5 issue + // where type inference fails (we use 9.5 on jenkins). + qb = qb.leftJoin(`(select 0 as id, cast('none' as text) as display_email, ` + + `cast('none' as text) as email, cast('none' as text) as name)`, + 'profiles', '1 = 1'); + } + } + // join the relevant groups and subgroups + return qb + .leftJoin('group_groups', 'gg1', 'gg1.group_id = acl_rules.group_id') + .leftJoin('group_groups', 'gg2', 'gg2.group_id = gg1.subgroup_id') + .leftJoin('group_groups', 'gg3', 'gg3.group_id = gg2.subgroup_id') + // join the users in the relevant groups and subgroups. + .leftJoin('group_users', 'gu3', 'gg3.subgroup_id = gu3.group_id') + .leftJoin('group_users', 'gu2', 'gg2.subgroup_id = gu2.group_id') + .leftJoin('group_users', 'gu1', 'gg1.subgroup_id = gu1.group_id') + .leftJoin('group_users', 'gu0', 'acl_rules.group_id = gu0.group_id'); + } + + // Apply limits to the query. Results should be limited to a specific org + // if request is from a branded webpage; results should be limited to a + // specific user or set of users. + private _applyLimit(qb: SelectQueryBuilder, limit: Scope, + resources: Array<'docs'|'workspaces'|'orgs'>): SelectQueryBuilder { + if (limit.org) { + // Filtering on merged org is a special case, see urlIdQuery + const mergedOrg = this.isMergedOrg(limit.org || null); + if (!mergedOrg) { + qb = this._whereOrg(qb, limit.org, limit.includeSupport || false); + } + } + if (limit.users || limit.userId) { + for (const res of resources) { + qb = this._withAccess(qb, limit.users || limit.userId, res); + } + } + if (resources.includes('docs') && resources.includes('workspaces') && !limit.showAll) { + // Add Workspace.filteredOut column that is set for workspaces that should be filtered out. + // We don't use a WHERE clause directly since this would leave us unable to distinguish + // an empty result from insufficient access; and there's no straightforward way to do + // what we want in an ON clause. + // Filter out workspaces only if there are no docs in them (The "ON" clause from + // _onDocs will have taken care of including the right docs). If there are docs, + // then include the workspace regardless of whether it itself has been soft-deleted + // or not. + // TODO: if getOrgWorkspaces and getWorkspace were restructured to make two queries + // rather than a single query, this trickiness could be eliminated. + if (limit.showRemoved) { + qb = qb.addSelect('docs.id IS NULL AND workspaces.removed_at IS NULL', + 'workspaces_filtered_out'); + } else { + qb = qb.addSelect('docs.id IS NULL AND workspaces.removed_at IS NOT NULL', + 'workspaces_filtered_out'); + } + } + return qb; + } + + // Filter out all personal orgs, and add back in a single merged org. + private _mergePersonalOrgs(userId: number, orgs: Organization[]): Organization[] { + const regularOrgs = orgs.filter(org => org.owner === null); + const personalOrg = orgs.find(org => org.owner && org.owner.id === userId); + if (!personalOrg) { return regularOrgs; } + personalOrg.id = 0; + personalOrg.domain = this.mergedOrgDomain(); + return [personalOrg].concat(regularOrgs); + } + + // Check if shares are about to exceed a limit, and emit a meaningful + // ApiError if so. + // If checkChange is set, issue an error only if a new share is being + // made. + private _restrictShares(role: roles.NonGuestRole|null, limit: number, + before: User[], after: User[], checkChange: boolean, kind: string, + features: Features) { + const existingUserIds = new Set(before.map(user => user.id)); + // Do not emit error if users are not added, even if the number is past the limit. + if (after.length > limit && + (!checkChange || after.some(user => !existingUserIds.has(user.id)))) { + const more = limit > 0 ? ' more' : ''; + throw new ApiError( + checkChange ? `No${more} external ${kind} ${role || 'shares'} permitted` : + `Too many external ${kind} ${role || 'shares'}`, + 403, { + limit: { + quantity: 'collaborators', + subquantity: role || undefined, + maximum: limit, + value: before.length, + projectedValue: after.length + }, + tips: canAddOrgMembers(features) ? [{ + action: 'add-members', + message: 'add users as team members to the site first' + }] : [{ + action: 'upgrade', + message: 'pay for more team members' + }] + }); + } + } + + // Check if document shares exceed any of the share limits, and emit a meaningful + // ApiError if so. If both membersBefore and membersAfter are specified, fail + // only if a new share is being added, but otherwise don't complain even if limits + // are exceeded. If only membersBefore is specified, fail strictly if limits are + // exceeded. + private _restrictAllDocShares(features: Features, + nonOrgMembersBefore: Map, + nonOrgMembersAfter: Map, + checkChange: boolean = true) { + // Apply a limit to document shares that is not specific to a particular role. + if (features.maxSharesPerDoc !== undefined) { + this._restrictShares(null, features.maxSharesPerDoc, removeRole(nonOrgMembersBefore), + removeRole(nonOrgMembersAfter), checkChange, 'document', features); + } + if (features.maxSharesPerDocPerRole) { + for (const role of this.defaultBasicGroupNames) { + const limit = features.maxSharesPerDocPerRole[role]; + if (limit === undefined) { continue; } + // Apply a per-role limit to document shares. + this._restrictShares(role, limit, nonOrgMembersBefore.get(role) || [], + nonOrgMembersAfter.get(role) || [], checkChange, 'document', features); + } + } + } + + // Throw an error if there's no room for adding another document. + private async _checkRoomForAnotherDoc(userId: number, workspace: Workspace, manager: EntityManager) { + const features = workspace.org.billingAccount.product.features; + if (features.maxDocsPerOrg !== undefined) { + // we need to count how many docs are in the current org, and if we + // are already at or above the limit, then fail. + const wss = this.unwrapQueryResult(await this.getOrgWorkspaces({userId}, workspace.org.id, + {manager})); + const count = wss.map(ws => ws.docs.length).reduce((a, b) => a + b, 0); + if (count >= features.maxDocsPerOrg) { + throw new ApiError('No more documents permitted', 403, { + limit: { + quantity: 'docs', + maximum: features.maxDocsPerOrg, + value: count, + projectedValue: count + 1 + } + }); + } + } + } + + // For the moment only the support user can add both everyone@ and anon@ to a + // resource, since that allows spam. TODO: enhance or remove. + private _checkUserChangeAllowed(userId: number, groups: Group[]) { + if (userId === this.getSupportUserId()) { return; } + const ids = new Set(flatten(groups.map(g => g.memberUsers)).map(u => u.id)); + if (ids.has(this.getEveryoneUserId()) && ids.has(this.getAnonymousUserId())) { + throw new Error('this user cannot share with everyone and anonymous'); + } + } + + // Fetch a Document with all access information loaded. Make sure the user has the + // specified permissions on the doc. The Document's organization will have product + // feature information loaded also. + private async _loadDocAccess(scope: DocScope, markPermissions: Permissions, + transaction?: EntityManager): Promise { + return await this._runInTransaction(transaction, async manager => { + + const docQuery = this._doc(scope, {manager, markPermissions}) + // Join the doc's ACL rules and groups/users so we can edit them. + .leftJoinAndSelect('docs.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'doc_groups') + .leftJoinAndSelect('doc_groups.memberUsers', 'doc_group_users') + .leftJoinAndSelect('doc_groups.memberGroups', 'doc_group_groups') + .leftJoinAndSelect('doc_group_users.logins', 'doc_user_logins') + // Join the workspace so we know what should be inherited. We will join + // the workspace member groups/users as a separate query, since + // SQL results are flattened, and multiplying the number of rows we have already + // by the number of workspace users could get excessive. + .leftJoinAndSelect('docs.workspace', 'workspace'); + const queryResult = await verifyIsPermitted(docQuery); + const doc: Document = this.unwrapQueryResult(queryResult); + + // Load the workspace's member groups/users. + const workspaceQuery = this._workspace(scope, doc.workspace.id, {manager}) + .leftJoinAndSelect('workspaces.aclRules', 'workspace_acl_rules') + .leftJoinAndSelect('workspace_acl_rules.group', 'workspace_groups') + .leftJoinAndSelect('workspace_groups.memberUsers', 'workspace_group_users') + .leftJoinAndSelect('workspace_groups.memberGroups', 'workspace_group_groups') + .leftJoinAndSelect('workspace_group_users.logins', 'workspace_user_logins') + // We'll need the org as well. We will join its members as a separate query, since + // SQL results are flattened, and multiplying the number of rows we have already + // by the number of org users could get excessive. + .leftJoinAndSelect('workspaces.org', 'org'); + doc.workspace = (await workspaceQuery.getOne())!; + + // Load the org's member groups/users. + let orgQuery = this.org(scope, doc.workspace.org.id, {manager}) + .leftJoinAndSelect('orgs.aclRules', 'org_acl_rules') + .leftJoinAndSelect('org_acl_rules.group', 'org_groups') + .leftJoinAndSelect('org_groups.memberUsers', 'org_group_users') + .leftJoinAndSelect('org_group_users.logins', 'org_user_logins'); + orgQuery = this._addFeatures(orgQuery); + doc.workspace.org = (await orgQuery.getOne())!; + return doc; + }); + } + + // Emit an event indicating that the count of users with access to the org has changed, with + // the customerId and the updated number of users. + // The org argument must include the billingAccount. + private _userChangeNotification( + userId: number, + org: Organization, // Must include billingAccount + countBefore: number, + countAfter: number, + membersBefore: Map, + membersAfter: Map + ) { + return () => { + const customerId = org.billingAccount.stripeCustomerId; + const change: UserChange = {userId, org, customerId, + countBefore, countAfter, + membersBefore, membersAfter}; + this.emit('userChange', change); + }; + } + + // Create a notification function that emits an event when users may have been added to a resource. + private _inviteNotification(userId: number, resource: Organization|Workspace|Document, + userIdDelta: UserIdDelta, membersBefore: Map): () => void { + return () => this.emit('addUser', userId, resource, userIdDelta, membersBefore); + } + + // Given two arrays of groups, returns a map of users present in the first array but + // not the second, where the map is broken down by user role. + // This method is used for checking limits on shares. + // Excluded users are removed from the results. + private _getUserDifference(groupsA: Group[], groupsB: Group[]): Map { + const subtractSet: Set = + new Set(flatten(groupsB.map(grp => grp.memberUsers)).map(usr => usr.id)); + const result = new Map(); + for (const group of groupsA) { + const name = group.name; + if (!roles.isNonGuestRole(name)) { continue; } + result.set(name, group.memberUsers.filter(user => !subtractSet.has(user.id))); + } + return this._withoutExcludedUsers(result); + } + + private _withoutExcludedUsers(members: Map): Map { + const excludedUsers = this.getExcludedUserIds(); + for (const [role, users] of members.entries()) { + members.set(role, users.filter((user) => !excludedUsers.includes(user.id))); + } + return members; + } + + private _billingManagerNotification(userId: number, addUserId: number, orgs: Organization[]) { + return () => { + this.emit('addBillingManager', userId, addUserId, orgs); + }; + } + + private _teamCreatorNotification(userId: number) { + return () => { + this.emit('teamCreator', userId); + }; + } + + /** + * Check for anonymous user, either encoded directly as an id, or as a singular + * profile (this case arises during processing of the session/access/all endpoint + * whether we are checking for available orgs without committing yet to a particular + * choice of user). + */ + private _isAnonymousUser(users: AvailableUsers): boolean { + return isSingleUser(users) ? users === this.getAnonymousUserId() : + users.length === 1 && normalizeEmail(users[0].email) === ANONYMOUS_USER_EMAIL; + } + + // Set Workspace.removedAt to null (undeletion) or to a datetime (soft deletion) + private _setWorkspaceRemovedAt(scope: Scope, wsId: number, removedAt: Date|null) { + return this._connection.transaction(async manager => { + const wsQuery = this._workspace({...scope, showAll: true}, wsId, { + manager, + markPermissions: Permissions.REMOVE + }); + const workspace: Workspace = this.unwrapQueryResult(await verifyIsPermitted(wsQuery)); + await manager.createQueryBuilder() + .update(Workspace).set({removedAt}).where({id: workspace.id}) + .execute(); + }); + } + + // Set Document.removedAt to null (undeletion) or to a datetime (soft deletion) + private _setDocumentRemovedAt(scope: DocScope, removedAt: Date|null) { + return this._connection.transaction(async manager => { + let docQuery = this._doc({...scope, showAll: true}, { + manager, + markPermissions: Permissions.REMOVE + }); + if (!removedAt) { + docQuery = this._addFeatures(docQuery); // pull in billing information for doc count limits + } + const doc: Document = this.unwrapQueryResult(await verifyIsPermitted(docQuery)); + if (!removedAt) { + await this._checkRoomForAnotherDoc(scope.userId, doc.workspace, manager); + } + await manager.createQueryBuilder() + .update(Document).set({removedAt}).where({id: doc.id}) + .execute(); + }); + } +} + +// Return a QueryResult reflecting the output of a query builder. +// Checks on the "is_permitted" field which select queries set on resources to +// indicate whether the user has access. +// If the output is empty, we signal that the desired resource does not exist. +// If the "is_permitted" field is falsy, we signal that the resource is forbidden. +// Returns the resource fetched by the queryBuilder. +async function verifyIsPermitted( + queryBuilder: SelectQueryBuilder +): Promise> { + const results = await queryBuilder.getRawAndEntities(); + if (results.entities.length === 0) { + return { + status: 404, + errMessage: `${getFrom(queryBuilder)} not found` + }; + } else if (results.entities.length > 1) { + return { + status: 400, + errMessage: `ambiguous ${getFrom(queryBuilder)} request` + }; + } else if (!results.raw[0].is_permitted) { + return { + status: 403, + errMessage: "access denied" + }; + } + return { + status: 200, + data: results.entities[0] + }; +} + +// Returns all first-level memberUsers in the resources. Requires all resources' aclRules, groups +// and memberUsers to be populated. +// If optRoles is provided, only checks membership in resource groups with the given roles. +function getResourceUsers(res: Resource|Resource[], optRoles?: string[]): User[] { + res = Array.isArray(res) ? res : [res]; + const users: {[uid: string]: User} = {}; + let resAcls: AclRule[] = flatten(res.map(_res => _res.aclRules as AclRule[])); + if (optRoles) { + resAcls = resAcls.filter(_acl => optRoles.includes(_acl.group.name)); + } + resAcls.forEach((aclRule: AclRule) => { + aclRule.group.memberUsers.forEach((u: User) => users[u.id] = u); + }); + const userList = Object.keys(users).map(uid => users[uid]); + userList.sort((a, b) => a.id - b.id); + return userList; +} + +// Returns a map of userIds to the user's strongest default role on the given resource. +// The resource's aclRules, groups, and memberUsers must be populated. +function getMemberUserRoles(res: Resource, allowRoles: T[]): {[userId: string]: T} { + // Add the users to a map to ensure uniqueness. (A user may be present in + // more than one group) + const userMap: {[userId: string]: T} = {}; + (res.aclRules as AclRule[]).forEach((aclRule: AclRule) => { + const role = aclRule.group.name as T; + if (allowRoles.includes(role)) { + // Map the users to remove sensitive information from the result and + // to add the group names. + aclRule.group.memberUsers.forEach((u: User) => { + // If the user is already present in another group, use the more + // powerful role name. + userMap[u.id] = userMap[u.id] ? roles.getStrongestRole(userMap[u.id], role) : role; + }); + } + }); + return userMap; +} + +// Extract a human-readable name for the type of entity being selected. +function getFrom(queryBuilder: SelectQueryBuilder): string { + const alias = queryBuilder.expressionMap.mainAlias; + return (alias && alias.metadata && alias.metadata.name.toLowerCase()) || 'resource'; +} + +// Flatten a map of users per role into a simple list of users. +function removeRole(usersWithRoles: Map) { + return flatten([...usersWithRoles.values()]); +} + +function getNonGuestGroups(entity: Organization|Workspace|Document): NonGuestGroup[] { + return (entity.aclRules as AclRule[]).map(aclRule => aclRule.group).filter(isNonGuestGroup); +} + +// Returns a map of users indexed by their roles. Optionally excludes users whose ids are in +// excludeUsers. +function getUsersWithRole(groups: NonGuestGroup[], excludeUsers?: number[]): Map { + const members = new Map(); + for (const group of groups) { + let users = group.memberUsers; + if (excludeUsers) { + users = users.filter((user) => !excludeUsers.includes(user.id)); + } + members.set(group.name, users); + } + return members; +} + +export async function makeDocAuthResult(docPromise: Promise): Promise { + try { + const doc = await docPromise; + const removed = Boolean(doc.removedAt || doc.workspace.removedAt); + return {docId: doc.id, access: doc.access, removed}; + } catch (error) { + return {docId: null, access: null, removed: null, error}; + } +} + +/** + * Extracts DocAuthKey information from scope. This includes everything needed to + * identify the document to access. Throws if information is not present. + */ +export function getDocAuthKeyFromScope(scope: Scope): DocAuthKey { + const {urlId, userId, org} = scope; + if (!urlId) { throw new Error('document required'); } + return {urlId, userId, org}; +} diff --git a/app/gen-server/lib/Permissions.ts b/app/gen-server/lib/Permissions.ts new file mode 100644 index 00000000..45b00d61 --- /dev/null +++ b/app/gen-server/lib/Permissions.ts @@ -0,0 +1,21 @@ +export enum Permissions { + NONE = 0x0, + // Note that the view permission bit provides view access ONLY to the resource to which + // the aclRule belongs - it does not allow listing that resource's children. A resource's + // children may only be listed if those children also have the view permission set. + VIEW = 0x1, + UPDATE = 0x2, + ADD = 0x4, + // Note that the remove permission bit provides remove access to a resource AND all of + // its child resources/ACLs + REMOVE = 0x8, + SCHEMA_EDIT = 0x10, + ACL_EDIT = 0x20, + EDITOR = VIEW | UPDATE | ADD | REMOVE, // tslint:disable-line:no-bitwise + ADMIN = EDITOR | SCHEMA_EDIT, // tslint:disable-line:no-bitwise + OWNER = ADMIN | ACL_EDIT, // tslint:disable-line:no-bitwise + + // A virtual permission bit signifying that the general public has some access to + // the resource via ACLs involving the everyone@ user. + PUBLIC = 0x80 +} diff --git a/app/gen-server/lib/TypeORMPatches.ts b/app/gen-server/lib/TypeORMPatches.ts new file mode 100644 index 00000000..f4b78eb7 --- /dev/null +++ b/app/gen-server/lib/TypeORMPatches.ts @@ -0,0 +1,196 @@ +// This contains two TypeORM patches. + +// Patch 1: +// TypeORM Sqlite driver does not support using transactions in async code, if it is possible +// for two transactions to get called (one of the whole point of transactions). This +// patch adds support for that, based on a monkey patch published in: +// https://gist.github.com/keenondrums/556f8c61d752eff730841170cd2bc3f1 +// Explanation at https://github.com/typeorm/typeorm/issues/1884#issuecomment-380767213 + +// Patch 2: +// TypeORM parameters are global, and collisions in setting them are not detected. +// We add a patch to throw an exception if a parameter value is ever set and then +// changed during construction of a query. + +import * as sqlite3 from '@gristlabs/sqlite3'; +import isEqual = require('lodash/isEqual'); +import {EntityManager, QueryRunner} from 'typeorm'; +import {SqliteDriver} from 'typeorm/driver/sqlite/SqliteDriver'; +import {SqliteQueryRunner} from 'typeorm/driver/sqlite/SqliteQueryRunner'; +import { + QueryRunnerProviderAlreadyReleasedError +} from 'typeorm/error/QueryRunnerProviderAlreadyReleasedError'; +import {QueryBuilder} from 'typeorm/query-builder/QueryBuilder'; + + +/********************** + * Patch 1 + **********************/ + +type Releaser = () => void; +type Worker = () => Promise|T; + +interface MutexInterface { + acquire(): Promise; + runExclusive(callback: Worker): Promise; + isLocked(): boolean; +} + +class Mutex implements MutexInterface { + private _queue: Array<(release: Releaser) => void> = []; + private _pending = false; + + public isLocked(): boolean { + return this._pending; + } + + public acquire(): Promise { + const ticket = new Promise(resolve => this._queue.push(resolve)); + if (!this._pending) { + this._dispatchNext(); + } + return ticket; + } + + public runExclusive(callback: Worker): Promise { + return this + .acquire() + .then(release => { + let result: T|Promise; + + try { + result = callback(); + } catch (e) { + release(); + throw(e); + } + + return Promise + .resolve(result) + .then( + (x: T) => (release(), x), + e => { + release(); + throw e; + } + ); + } + ); + } + + private _dispatchNext(): void { + if (this._queue.length > 0) { + this._pending = true; + this._queue.shift()!(this._dispatchNext.bind(this)); + } else { + this._pending = false; + } + } + +} + +// A singleton mutex for all sqlite transactions. +const mutex = new Mutex(); + +class SqliteQueryRunnerPatched extends SqliteQueryRunner { + private _releaseMutex: Releaser | null; + + public async startTransaction(level?: any): Promise { + this._releaseMutex = await mutex.acquire(); + return super.startTransaction(level); + } + + public async commitTransaction(): Promise { + if (!this._releaseMutex) { + throw new Error('SqliteQueryRunnerPatched.commitTransaction -> mutex releaser unknown'); + } + await super.commitTransaction(); + this._releaseMutex(); + this._releaseMutex = null; + } + + public async rollbackTransaction(): Promise { + if (!this._releaseMutex) { + throw new Error('SqliteQueryRunnerPatched.rollbackTransaction -> mutex releaser unknown'); + } + await super.rollbackTransaction(); + this._releaseMutex(); + this._releaseMutex = null; + } + + public async connect(): Promise { + if (!this.isTransactionActive) { + const release = await mutex.acquire(); + release(); + } + return super.connect(); + } +} + +class SqliteDriverPatched extends SqliteDriver { + public createQueryRunner(): QueryRunner { + if (!this.queryRunner) { + this.queryRunner = new SqliteQueryRunnerPatched(this); + } + return this.queryRunner; + } + protected loadDependencies(): void { + // Use our own sqlite3 module, which is a fork of the original. + this.sqlite = sqlite3; + } +} + +// Patch the underlying SqliteDriver, since it's impossible to convince typeorm to use only our +// patched classes. (Previously we patched DriverFactory and Connection, but those would still +// create an unpatched SqliteDriver and then overwrite it.) +SqliteDriver.prototype.createQueryRunner = SqliteDriverPatched.prototype.createQueryRunner; +(SqliteDriver.prototype as any).loadDependencies = (SqliteDriverPatched.prototype as any).loadDependencies; + +export function applyPatch() { + // tslint: disable-next-line + EntityManager.prototype.transaction = async function (arg1: any, arg2?: any): Promise { + if (this.queryRunner && this.queryRunner.isReleased) { + throw new QueryRunnerProviderAlreadyReleasedError(); + } + if (this.queryRunner && this.queryRunner.isTransactionActive) { + throw new Error(`Cannot start transaction because its already started`); + } + const queryRunner = this.connection.createQueryRunner(); + const runInTransaction = typeof arg1 === "function" ? arg1 : arg2; + try { + await queryRunner.startTransaction(); + const result = await runInTransaction(queryRunner.manager); + await queryRunner.commitTransaction(); + return result; + } catch (err) { + try { + // we throw original error even if rollback thrown an error + await queryRunner.rollbackTransaction(); + // tslint: disable-next-line + } catch (rollbackError) { + // tslint: disable-next-line + } + throw err; + } finally { + await queryRunner.release(); + } + }; +} + + +/********************** + * Patch 2 + **********************/ + +abstract class QueryBuilderPatched extends QueryBuilder { + public setParameter(key: string, value: any): this { + const prev = this.expressionMap.parameters[key]; + if (prev !== undefined && !isEqual(prev, value)) { + throw new Error(`TypeORM parameter collision for key '${key}' ('${prev}' vs '${value}')`); + } + this.expressionMap.parameters[key] = value; + return this; + } +} + +(QueryBuilder.prototype as any).setParameter = (QueryBuilderPatched.prototype as any).setParameter; diff --git a/app/gen-server/lib/Usage.ts b/app/gen-server/lib/Usage.ts new file mode 100644 index 00000000..60f23a54 --- /dev/null +++ b/app/gen-server/lib/Usage.ts @@ -0,0 +1,62 @@ +import {Document} from 'app/gen-server/entity/Document'; +import {Organization} from 'app/gen-server/entity/Organization'; +import {User} from 'app/gen-server/entity/User'; +import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import * as log from 'app/server/lib/log'; + +// Frequency of logging usage information. Not something we need +// to track with much granularity. +const USAGE_PERIOD_MS = 1 * 60 * 60 * 1000; // log every 1 hour + +/** + * Occasionally log usage information - number of users, orgs, + * docs, etc. + */ +export class Usage { + private _interval: NodeJS.Timeout; + + public constructor(private _dbManager: HomeDBManager) { + this._interval = setInterval(() => this.apply().catch(log.warn.bind(log)), USAGE_PERIOD_MS); + // Log once at beginning, in case we roll over servers faster than + // the logging period for an extended length of time, + // and to raise the visibility of this logging step so if it gets + // slow devs notice. + this.apply().catch(log.warn.bind(log)); + } + + public close() { + clearInterval(this._interval); + } + + public async apply() { + const manager = this._dbManager.connection.manager; + // raw count of users + const userCount = await manager.count(User); + // users who have logged in at least once + const userWithLoginCount = await manager.createQueryBuilder() + .from(User, 'users') + .where('first_login_at is not null') + .getCount(); + // raw count of organizations (excluding personal orgs) + const orgCount = await manager.createQueryBuilder() + .from(Organization, 'orgs') + .where('owner_id is null') + .getCount(); + // organizations with subscriptions that are in a non-terminated state + const orgInGoodStandingCount = await manager.createQueryBuilder() + .from(Organization, 'orgs') + .leftJoin('orgs.billingAccount', 'billing_accounts') + .where('owner_id is null') + .andWhere('billing_accounts.in_good_standing = true') + .getCount(); + // raw count of documents + const docCount = await manager.count(Document); + log.rawInfo('activity', { + docCount, + orgCount, + orgInGoodStandingCount, + userCount, + userWithLoginCount, + }); + } +} diff --git a/app/gen-server/lib/scrubUserFromOrg.ts b/app/gen-server/lib/scrubUserFromOrg.ts new file mode 100644 index 00000000..408eae95 --- /dev/null +++ b/app/gen-server/lib/scrubUserFromOrg.ts @@ -0,0 +1,209 @@ +import {EntityManager} from "typeorm"; +import * as roles from 'app/common/roles'; +import {Document} from "app/gen-server/entity/Document"; +import {Group} from "app/gen-server/entity/Group"; +import {Organization} from "app/gen-server/entity/Organization"; +import {Workspace} from "app/gen-server/entity/Workspace"; +import pick = require('lodash/pick'); + +/** + * + * Remove the given user from the given org and every resource inside the org. + * If the user being removed is an owner of any resources in the org, the caller replaces + * them as the owner. This is to prevent complete loss of access to any resource. + * + * This method transforms ownership without regard to permissions. We all talked this + * over and decided this is what we wanted, but there's no denying it is funky and could + * be surprising. + * TODO: revisit user scrubbing when we can. + * + */ +export async function scrubUserFromOrg( + orgId: number, + removeUserId: number, + callerUserId: number, + manager: EntityManager +): Promise { + await addMissingGuestMemberships(callerUserId, orgId, manager); + + // This will be a list of all mentions of removeUser and callerUser in any resource + // within the org. + const mentions: Mention[] = []; + + // Base query for all group_users related to these two users and this org. + const q = manager.createQueryBuilder() + .select('group_users.group_id, group_users.user_id') + .from('group_users', 'group_users') + .leftJoin(Group, 'groups', 'group_users.group_id = groups.id') + .addSelect('groups.name as name') + .leftJoin('groups.aclRule', 'acl_rules') + .where('(group_users.user_id = :removeUserId or group_users.user_id = :callerUserId)', + {removeUserId, callerUserId}) + .andWhere('orgs.id = :orgId', {orgId}); + + // Pick out group_users related specifically to the org resource, in 'mentions' format + // (including resource id, a tag for the kind of resource, the group name, the user + // id, and the group id). + const orgs = q.clone() + .addSelect(`'org' as kind, orgs.id`) + .innerJoin(Organization, 'orgs', 'orgs.id = acl_rules.org_id'); + mentions.push(...await orgs.getRawMany()); + + // Pick out mentions related to any workspace within the org. + const wss = q.clone() + .innerJoin(Workspace, 'workspaces', 'workspaces.id = acl_rules.workspace_id') + .addSelect(`'ws' as kind, workspaces.id`) + .innerJoin('workspaces.org', 'orgs'); + mentions.push(...await wss.getRawMany()); + + // Pick out mentions related to any doc within the org. + const docs = q.clone() + .innerJoin(Document, 'docs', 'docs.id = acl_rules.doc_id') + .addSelect(`'doc' as kind, docs.id`) + .innerJoin('docs.workspace', 'workspaces') + .innerJoin('workspaces.org', 'orgs'); + mentions.push(...await docs.getRawMany()); + + // Prepare to add and delete group_users. + const toDelete: Mention[] = []; + const toAdd: Mention[] = []; + + // Now index the mentions by whether they are for the removeUser or the callerUser, + // and the resource they apply to. + const removeUserMentions = new Map(); + const callerUserMentions = new Map(); + for (const mention of mentions) { + const isGuest = mention.name === roles.GUEST; + if (mention.user_id === removeUserId) { + // We can safely remove any guest roles for the removeUser without any + // further inspection. + if (isGuest) { toDelete.push(mention); continue; } + removeUserMentions.set(getMentionKey(mention), mention); + } else { + if (isGuest) { continue; } + callerUserMentions.set(getMentionKey(mention), mention); + } + } + // Now iterate across the mentions of removeUser, and see what we need to do + // for each of them. + for (const [key, removeUserMention] of removeUserMentions) { + toDelete.push(removeUserMention); + if (removeUserMention.name !== roles.OWNER) { + // Nothing fancy needed for cases where the removeUser is not the owner. + // Just discard those. + continue; + } + // The removeUser was a direct owner on this resource, but the callerUser was + // not. We set the callerUser as a direct owner on this resource, to preserve + // access to it. + // TODO: the callerUser might inherit sufficient access, in which case this + // step is unnecessary and could be skipped. I believe it does no harm though. + const callerUserMention = callerUserMentions.get(key); + if (callerUserMention && callerUserMention.name === roles.OWNER) { continue; } + if (callerUserMention) { toDelete.push(callerUserMention); } + toAdd.push({...removeUserMention, user_id: callerUserId}); + } + if (toDelete.length > 0) { + await manager.createQueryBuilder() + .delete() + .from('group_users') + .whereInIds(toDelete.map(m => pick(m, ['user_id', 'group_id']))) + .execute(); + } + if (toAdd.length > 0) { + await manager.createQueryBuilder() + .insert() + .into('group_users') + .values(toAdd.map(m => pick(m, ['user_id', 'group_id']))) + .execute(); + } + + // TODO: At this point, we've removed removeUserId from every mention in group_users. + // The user may still be mentioned in billing_account_managers. If the billing_account + // is linked to just this single organization, perhaps it would make sense to remove + // the user there, if the callerUser is themselves a billing account manager? + + await addMissingGuestMemberships(callerUserId, orgId, manager); +} + +/** + * Adds specified user to any guest groups for the resources of an org where the + * user needs to be and is not already. + */ +export async function addMissingGuestMemberships(userId: number, orgId: number, + manager: EntityManager) { + // For workspaces: + // User should be in guest group if mentioned in a doc within that workspace. + let groupUsers = await manager.createQueryBuilder() + .select('workspace_groups.id as group_id, cast(:userId as int) as user_id') + .setParameter('userId', userId) + .from(Workspace, 'workspaces') + .where('workspaces.org_id = :orgId', {orgId}) + .innerJoin('workspaces.docs', 'docs') + .innerJoin('docs.aclRules', 'doc_acl_rules') + .innerJoin('doc_acl_rules.group', 'doc_groups') + .innerJoin('doc_groups.memberUsers', 'doc_group_users') + .andWhere('doc_group_users.id = :userId', {userId}) + .leftJoin('workspaces.aclRules', 'workspace_acl_rules') + .leftJoin('workspace_acl_rules.group', 'workspace_groups') + .leftJoin('group_users', 'workspace_group_users', + 'workspace_group_users.group_id = workspace_groups.id and ' + + 'workspace_group_users.user_id = :userId') + .andWhere('workspace_groups.name = :guestName', {guestName: roles.GUEST}) + .groupBy('workspaces.id, workspace_groups.id, workspace_group_users.user_id') + .having('workspace_group_users.user_id is null') + .getRawMany(); + if (groupUsers.length > 0) { + await manager.createQueryBuilder() + .insert() + .into('group_users') + .values(groupUsers) + .execute(); + } + + // For org: + // User should be in guest group if mentioned in a workspace within that org. + groupUsers = await manager.createQueryBuilder() + .select('org_groups.id as group_id, cast(:userId as int) as user_id') + .setParameter('userId', userId) + .from(Organization, 'orgs') + .where('orgs.id = :orgId', {orgId}) + .innerJoin('orgs.workspaces', 'workspaces') + .innerJoin('workspaces.aclRules', 'workspaces_acl_rules') + .innerJoin('workspaces_acl_rules.group', 'workspace_groups') + .innerJoin('workspace_groups.memberUsers', 'workspace_group_users') + .andWhere('workspace_group_users.id = :userId', {userId}) + .leftJoin('orgs.aclRules', 'org_acl_rules') + .leftJoin('org_acl_rules.group', 'org_groups') + .leftJoin('group_users', 'org_group_users', + 'org_group_users.group_id = org_groups.id and ' + + 'org_group_users.user_id = :userId') + .andWhere('org_groups.name = :guestName', {guestName: roles.GUEST}) + .groupBy('org_groups.id, org_group_users.user_id') + .having('org_group_users.user_id is null') + .getRawMany(); + if (groupUsers.length > 0) { + await manager.createQueryBuilder() + .insert() + .into('group_users') + .values(groupUsers) + .execute(); + } + + // For doc: + // Guest groups are not used. +} + +interface Mention { + id: string|number; // id of resource + kind: 'org'|'ws'|'doc'; // type of resource + user_id: number; // id of user in group + group_id: number; // id of group + name: string; // name of group +} + +type MentionKey = string; + +function getMentionKey(mention: Mention): MentionKey { + return `${mention.kind} ${mention.id}`; +} diff --git a/app/gen-server/lib/values.ts b/app/gen-server/lib/values.ts new file mode 100644 index 00000000..a9cb688c --- /dev/null +++ b/app/gen-server/lib/values.ts @@ -0,0 +1,36 @@ +/** + * This smoothes over some awkward differences between TypeORM treatment of + * booleans and json in sqlite and postgres. Booleans and json work fine + * with each db, but have different levels of driver-level support. + */ + +export interface NativeValues { + // Json columns are handled natively by the postgres driver, but for + // sqlite requires a typeorm wrapper (simple-json). + jsonEntityType: 'json' | 'simple-json'; + jsonType: 'json' | 'varchar'; + booleanType: 'boolean' | 'integer'; + dateTimeType: 'timestamp with time zone' | 'datetime'; + trueValue: boolean | number; + falseValue: boolean | number; +} + +const sqliteNativeValues: NativeValues = { + jsonEntityType: 'simple-json', + jsonType: 'varchar', + booleanType: 'integer', + dateTimeType: 'datetime', + trueValue: 1, + falseValue: 0 +}; + +const postgresNativeValues: NativeValues = { + jsonEntityType: 'json', + jsonType: 'json', + booleanType: 'boolean', + dateTimeType: 'timestamp with time zone', + trueValue: true, + falseValue: false +}; + +export const nativeValues = (process.env.TYPEORM_TYPE === 'postgres') ? postgresNativeValues : sqliteNativeValues; diff --git a/app/gen-server/migration/1536634251710-Initial.ts b/app/gen-server/migration/1536634251710-Initial.ts new file mode 100644 index 00000000..7d88a87f --- /dev/null +++ b/app/gen-server/migration/1536634251710-Initial.ts @@ -0,0 +1,304 @@ +import {MigrationInterface, QueryRunner, Table} from "typeorm"; + + +export class Initial1536634251710 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + // TypeORM doesn't currently help with types of created tables: + // https://github.com/typeorm/typeorm/issues/305 + // so we need to do a little smoothing over postgres and sqlite. + const sqlite = queryRunner.connection.driver.options.type === 'sqlite'; + const datetime = sqlite ? "datetime" : "timestamp with time zone"; + const now = "now()"; + + await queryRunner.createTable(new Table({ + name: "users", + columns: [ + { + name: "id", + type: "integer", + isGenerated: true, + generationStrategy: 'increment', + isPrimary: true + }, + { + name: "name", + type: "varchar", + }, + { + name: "api_key", + type: "varchar", + isNullable: true, + isUnique: true + } + ] + }), false); + + await queryRunner.createTable(new Table({ + name: "orgs", + columns: [ + { + name: "id", + type: "integer", + isGenerated: true, + generationStrategy: 'increment', + isPrimary: true + }, + { + name: "name", + type: "varchar", + }, + { + name: "domain", + type: "varchar", + isNullable: true, + }, + { + name: "created_at", + type: datetime, + default: now + }, + { + name: "updated_at", + type: datetime, + default: now + }, + { + name: "owner_id", + type: "integer", + isNullable: true, + isUnique: true + } + ], + foreignKeys: [ + { + columnNames: ["owner_id"], + referencedColumnNames: ["id"], + referencedTableName: "users" + } + ] + }), false); + + await queryRunner.createTable(new Table({ + name: "workspaces", + columns: [ + { + name: "id", + type: "integer", + isGenerated: true, + generationStrategy: 'increment', + isPrimary: true + }, + { + name: "name", + type: "varchar", + }, + { + name: "created_at", + type: datetime, + default: now + }, + { + name: "updated_at", + type: datetime, + default: now + }, + { + name: "org_id", + type: "integer", + isNullable: true + } + ], + foreignKeys: [ + { + columnNames: ["org_id"], + referencedColumnNames: ["id"], + referencedTableName: "orgs" + } + ] + }), false); + + await queryRunner.createTable(new Table({ + name: "docs", + columns: [ + { + name: "id", + type: "varchar", + isPrimary: true + }, + { + name: "name", + type: "varchar", + }, + { + name: "created_at", + type: datetime, + default: now + }, + { + name: "updated_at", + type: datetime, + default: now + }, + { + name: "workspace_id", + type: "integer", + isNullable: true + } + ], + foreignKeys: [ + { + columnNames: ["workspace_id"], + referencedColumnNames: ["id"], + referencedTableName: "workspaces" + } + ] + }), false); + + await queryRunner.createTable(new Table({ + name: "groups", + columns: [ + { + name: "id", + type: "integer", + isGenerated: true, + generationStrategy: 'increment', + isPrimary: true + }, + { + name: "name", + type: "varchar", + } + ] + }), false); + + await queryRunner.createTable(new Table({ + name: "acl_rules", + columns: [ + { + name: "id", + type: "integer", + isGenerated: true, + generationStrategy: 'increment', + isPrimary: true + }, + { + name: "permissions", + type: "integer" + }, + { + name: "type", + type: "varchar" + }, + { + name: "workspace_id", + type: "integer", + isNullable: true + }, + { + name: "org_id", + type: "integer", + isNullable: true + }, + { + name: "doc_id", + type: "varchar", + isNullable: true + }, + { + name: "group_id", + type: "integer", + isNullable: true + } + ], + foreignKeys: [ + { + columnNames: ["workspace_id"], + referencedColumnNames: ["id"], + referencedTableName: "workspaces" + }, + { + columnNames: ["org_id"], + referencedColumnNames: ["id"], + referencedTableName: "orgs" + }, + { + columnNames: ["doc_id"], + referencedColumnNames: ["id"], + referencedTableName: "docs" + }, + { + columnNames: ["group_id"], + referencedColumnNames: ["id"], + referencedTableName: "groups" + } + ] + }), false); + + await queryRunner.createTable(new Table({ + name: "group_users", + columns: [ + { + name: "group_id", + type: "integer", + isPrimary: true + }, + { + name: "user_id", + type: "integer", + isPrimary: true + }, + ], + foreignKeys: [ + { + columnNames: ["group_id"], + referencedColumnNames: ["id"], + referencedTableName: "groups" + }, + { + columnNames: ["user_id"], + referencedColumnNames: ["id"], + referencedTableName: "users" + } + ] + }), false); + + await queryRunner.createTable(new Table({ + name: "group_groups", + columns: [ + { + name: "group_id", + type: "integer", + isPrimary: true + }, + { + name: "subgroup_id", + type: "integer", + isPrimary: true + }, + ], + foreignKeys: [ + { + columnNames: ["group_id"], + referencedColumnNames: ["id"], + referencedTableName: "groups" + }, + { + columnNames: ["subgroup_id"], + referencedColumnNames: ["id"], + referencedTableName: "groups" + } + ] + }), false); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "group_groups"`); + await queryRunner.query(`DROP TABLE "group_users"`); + await queryRunner.query(`DROP TABLE "acl_rules"`); + await queryRunner.query(`DROP TABLE "groups"`); + await queryRunner.query(`DROP TABLE "docs"`); + await queryRunner.query(`DROP TABLE "workspaces"`); + await queryRunner.query(`DROP TABLE "orgs"`); + await queryRunner.query(`DROP TABLE "users"`); + } +} diff --git a/app/gen-server/migration/1539031763952-Login.ts b/app/gen-server/migration/1539031763952-Login.ts new file mode 100644 index 00000000..8eb7bf25 --- /dev/null +++ b/app/gen-server/migration/1539031763952-Login.ts @@ -0,0 +1,39 @@ +import {MigrationInterface, QueryRunner, Table} from "typeorm"; + +export class Login1539031763952 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable(new Table({ + name: 'logins', + columns: [ + { + name: "id", + type: "integer", + isGenerated: true, + generationStrategy: 'increment', + isPrimary: true + }, + { + name: 'user_id', + type: 'integer' + }, + { + name: 'email', + type: 'varchar', + isUnique: true + } + ], + foreignKeys: [ + { + columnNames: ["user_id"], + referencedColumnNames: ["id"], + referencedTableName: "users" + } + ] + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query('DROP TABLE logins'); + } +} diff --git a/app/gen-server/migration/1549313797109-PinDocs.ts b/app/gen-server/migration/1549313797109-PinDocs.ts new file mode 100644 index 00000000..dfc33588 --- /dev/null +++ b/app/gen-server/migration/1549313797109-PinDocs.ts @@ -0,0 +1,17 @@ +import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; + +export class PinDocs1549313797109 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + const sqlite = queryRunner.connection.driver.options.type === 'sqlite'; + await queryRunner.addColumn('docs', new TableColumn({ + name: 'is_pinned', + type: 'boolean', + default: sqlite ? 0 : false + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropColumn('docs', 'is_pinned'); + } +} diff --git a/app/gen-server/migration/1549381727494-UserPicture.ts b/app/gen-server/migration/1549381727494-UserPicture.ts new file mode 100644 index 00000000..40c5f42e --- /dev/null +++ b/app/gen-server/migration/1549381727494-UserPicture.ts @@ -0,0 +1,16 @@ +import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; + +export class UserPicture1549381727494 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.addColumn("users", new TableColumn({ + name: "picture", + type: "varchar", + isNullable: true, + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropColumn("users", "picture"); + } +} diff --git a/app/gen-server/migration/1551805156919-LoginDisplayEmail.ts b/app/gen-server/migration/1551805156919-LoginDisplayEmail.ts new file mode 100644 index 00000000..dc0a500f --- /dev/null +++ b/app/gen-server/migration/1551805156919-LoginDisplayEmail.ts @@ -0,0 +1,16 @@ +import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; + +export class LoginDisplayEmail1551805156919 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.addColumn('logins', new TableColumn({ + name: 'display_email', + type: 'varchar', + isNullable: true, + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropColumn('logins', 'display_email'); + } +} diff --git a/app/gen-server/migration/1552416614755-LoginDisplayEmailNonNull.ts b/app/gen-server/migration/1552416614755-LoginDisplayEmailNonNull.ts new file mode 100644 index 00000000..03845cfe --- /dev/null +++ b/app/gen-server/migration/1552416614755-LoginDisplayEmailNonNull.ts @@ -0,0 +1,32 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class LoginDisplayEmailNonNull1552416614755 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('update logins set display_email = email where display_email is null'); + // if our db will already heavily loaded, it might be better to add a check constraint + // rather than modifying the column properties. But for our case, this will be fast. + + // To work correctly with RDS version of postgres, it is important to clone + // and change typeorm's settings for the column, rather than the settings specified + // in previous migrations. Otherwise typeorm will fall back on a brutal method of + // drop-and-recreate that doesn't work for non-null in any case. + // + // The pg command is very simple, just alter table logins alter column display_email set not null + // but sqlite migration is tedious since table needs to be rebuilt, so still just + // marginally worthwhile letting typeorm deal with it. + const logins = (await queryRunner.getTable('logins'))!; + const displayEmail = logins.findColumnByName('display_email')!; + const displayEmailNonNull = displayEmail.clone(); + displayEmailNonNull.isNullable = false; + await queryRunner.changeColumn('logins', displayEmail, displayEmailNonNull); + } + + public async down(queryRunner: QueryRunner): Promise { + const logins = (await queryRunner.getTable('logins'))!; + const displayEmail = logins.findColumnByName('display_email')!; + const displayEmailNonNull = displayEmail.clone(); + displayEmailNonNull.isNullable = true; + await queryRunner.changeColumn('logins', displayEmail, displayEmailNonNull); + } +} diff --git a/app/gen-server/migration/1553016106336-Indexes.ts b/app/gen-server/migration/1553016106336-Indexes.ts new file mode 100644 index 00000000..bcdfea2c --- /dev/null +++ b/app/gen-server/migration/1553016106336-Indexes.ts @@ -0,0 +1,66 @@ +import {MigrationInterface, QueryRunner, TableIndex} from "typeorm"; + +export class Indexes1553016106336 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createIndex("acl_rules", new TableIndex({ + name: "acl_rules__org_id", + columnNames: ["org_id"] + })); + await queryRunner.createIndex("acl_rules", new TableIndex({ + name: "acl_rules__workspace_id", + columnNames: ["workspace_id"] + })); + await queryRunner.createIndex("acl_rules", new TableIndex({ + name: "acl_rules__doc_id", + columnNames: ["doc_id"] + })); + + await queryRunner.createIndex("group_groups", new TableIndex({ + name: "group_groups__group_id", + columnNames: ["group_id"] + })); + await queryRunner.createIndex("group_groups", new TableIndex({ + name: "group_groups__subgroup_id", + columnNames: ["subgroup_id"] + })); + + await queryRunner.createIndex("group_users", new TableIndex({ + name: "group_users__group_id", + columnNames: ["group_id"] + })); + await queryRunner.createIndex("group_users", new TableIndex({ + name: "group_users__user_id", + columnNames: ["user_id"] + })); + + await queryRunner.createIndex("workspaces", new TableIndex({ + name: "workspaces__org_id", + columnNames: ["org_id"] + })); + + await queryRunner.createIndex("docs", new TableIndex({ + name: "docs__workspace_id", + columnNames: ["workspace_id"] + })); + + await queryRunner.createIndex("logins", new TableIndex({ + name: "logins__user_id", + columnNames: ["user_id"] + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropIndex("acl_rules", "acl_rules__org_id"); + await queryRunner.dropIndex("acl_rules", "acl_rules__workspace_id"); + await queryRunner.dropIndex("acl_rules", "acl_rules__doc_id"); + await queryRunner.dropIndex("group_groups", "group_groups__group_id"); + await queryRunner.dropIndex("group_groups", "group_groups__subgroup_id"); + await queryRunner.dropIndex("group_users", "group_users__group_id"); + await queryRunner.dropIndex("group_users", "group_users__user_id"); + await queryRunner.dropIndex("workspaces", "workspaces__org_id"); + await queryRunner.dropIndex("docs", "docs__workspace_id"); + await queryRunner.dropIndex("logins", "logins__user_id"); + } + +} diff --git a/app/gen-server/migration/1556726945436-Billing.ts b/app/gen-server/migration/1556726945436-Billing.ts new file mode 100644 index 00000000..63f7ecae --- /dev/null +++ b/app/gen-server/migration/1556726945436-Billing.ts @@ -0,0 +1,225 @@ +import {MigrationInterface, QueryRunner, Table, TableColumn, TableForeignKey} from 'typeorm'; +import {BillingAccount} from 'app/gen-server/entity/BillingAccount'; +import {BillingAccountManager} from 'app/gen-server/entity/BillingAccountManager'; +import {Organization} from 'app/gen-server/entity/Organization'; +import {Product} from 'app/gen-server/entity/Product'; +import {nativeValues} from 'app/gen-server/lib/values'; + +export class Billing1556726945436 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + // Create table for products. + await queryRunner.createTable(new Table({ + name: 'products', + columns: [ + { + name: 'id', + type: 'integer', + isGenerated: true, + generationStrategy: 'increment', + isPrimary: true + }, + { + name: 'name', + type: 'varchar' + }, + { + name: 'stripe_product_id', + type: 'varchar', + isUnique: true, + isNullable: true + }, + { + name: 'features', + type: nativeValues.jsonType + } + ] + })); + + // Create a basic free product that existing orgs can use. + const product = new Product(); + product.name = 'Free'; + product.features = {}; + await queryRunner.manager.save(product); + + // Create billing accounts and billing account managers. + await queryRunner.createTable(new Table({ + name: 'billing_accounts', + columns: [ + { + name: 'id', + type: 'integer', + isGenerated: true, + generationStrategy: 'increment', + isPrimary: true + }, + { + name: 'product_id', + type: 'integer' + }, + { + name: 'individual', + type: nativeValues.booleanType + }, + { + name: 'in_good_standing', + type: nativeValues.booleanType, + default: nativeValues.trueValue + }, + { + name: 'status', + type: nativeValues.jsonType, + isNullable: true + }, + { + name: 'stripe_customer_id', + type: 'varchar', + isUnique: true, + isNullable: true + }, + { + name: 'stripe_subscription_id', + type: 'varchar', + isUnique: true, + isNullable: true + }, + { + name: 'stripe_plan_id', + type: 'varchar', + isNullable: true + } + ], + foreignKeys: [ + { + columnNames: ['product_id'], + referencedColumnNames: ['id'], + referencedTableName: 'products' + } + ] + })); + + await queryRunner.createTable(new Table({ + name: 'billing_account_managers', + columns: [ + { + name: 'id', + type: 'integer', + isGenerated: true, + generationStrategy: 'increment', + isPrimary: true + }, + { + name: 'billing_account_id', + type: 'integer' + }, + { + name: 'user_id', + type: 'integer' + } + ], + foreignKeys: [ + { + columnNames: ['billing_account_id'], + referencedColumnNames: ['id'], + referencedTableName: 'billing_accounts', + onDelete: 'CASCADE' // delete manager if referenced billing_account goes away + }, + { + columnNames: ['user_id'], + referencedColumnNames: ['id'], + referencedTableName: 'users', + onDelete: 'CASCADE' // delete manager if referenced user goes away + } + ] + })); + + // Add a reference to billing accounts from orgs. + await queryRunner.addColumn('orgs', new TableColumn({ + name: 'billing_account_id', + type: 'integer', + isNullable: true + })); + await queryRunner.createForeignKey('orgs', new TableForeignKey({ + columnNames: ['billing_account_id'], + referencedColumnNames: ['id'], + referencedTableName: 'billing_accounts' + })); + + // Let's add billing accounts to all existing orgs. + // Personal orgs are put on an individual billing account. + // Other orgs are put on a team billing account, with the + // list of payment managers seeded by owners of that account. + const query = + queryRunner.manager.createQueryBuilder() + .select('orgs.id') + .from(Organization, 'orgs') + .leftJoin('orgs.owner', 'owners') + .addSelect('orgs.owner.id') + .leftJoinAndSelect('orgs.aclRules', 'acl_rules') + .leftJoinAndSelect('acl_rules.group', 'groups') + .leftJoin('groups.memberUsers', 'users') + .addSelect('users.id') + .where('permissions & 8 = 8'); // seed managers with owners+editors, omitting guests+viewers + // (permission 8 is "Remove") + const orgs = await query.getMany(); + for (const org of orgs) { + const individual = Boolean(org.owner); + const billingAccountInsert = await queryRunner.manager.createQueryBuilder() + .insert() + .into(BillingAccount) + .values([{product, individual}]) + .execute(); + const billingAccountId = billingAccountInsert.identifiers[0].id; + if (individual) { + await queryRunner.manager.createQueryBuilder() + .insert() + .into(BillingAccountManager) + .values([{billingAccountId, userId: org.owner.id}]) + .execute(); + } else { + for (const rule of org.aclRules) { + for (const user of rule.group.memberUsers) { + await queryRunner.manager.createQueryBuilder() + .insert() + .into(BillingAccountManager) + .values([{billingAccountId, userId: user.id}]) + .execute(); + } + } + } + await queryRunner.manager.createQueryBuilder() + .update(Organization) + .set({billingAccountId}) + .where('id = :id', {id: org.id}) + .execute(); + } + + // TODO: in a future migration, orgs.billing_account_id could be constrained + // to be non-null. All code deployments linked to a database that will be + // migrated must have code that sets orgs.billing_account_id by that time, + // otherwise they would fail to create orgs (and remember creating a user + // involves creating an org). + /* + // Now that all orgs have a billing account (and this migration is running within + // a transaction), we can constrain orgs.billing_account_id to be non-null. + const orgTable = (await queryRunner.getTable('orgs'))!; + const billingAccountId = orgTable.findColumnByName('billing_account_id')!; + const billingAccountIdNonNull = billingAccountId.clone(); + billingAccountIdNonNull.isNullable = false; + await queryRunner.changeColumn('orgs', billingAccountId, billingAccountIdNonNull); + */ + } + + public async down(queryRunner: QueryRunner): Promise { + // this is a bit ugly, but is the documented way to remove a foreign key + const table = await queryRunner.getTable('orgs'); + const foreignKey = table!.foreignKeys.find(fk => fk.columnNames.indexOf('billing_account_id') !== -1); + await queryRunner.dropForeignKey('orgs', foreignKey!); + + await queryRunner.dropColumn('orgs', 'billing_account_id'); + await queryRunner.dropTable('billing_account_managers'); + await queryRunner.dropTable('billing_accounts'); + await queryRunner.dropTable('products'); + } + +} diff --git a/app/gen-server/migration/1557157922339-OrgDomainUnique.ts b/app/gen-server/migration/1557157922339-OrgDomainUnique.ts new file mode 100644 index 00000000..e8e7b8c4 --- /dev/null +++ b/app/gen-server/migration/1557157922339-OrgDomainUnique.ts @@ -0,0 +1,27 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class OrgDomainUnique1557157922339 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + const logins = (await queryRunner.getTable('orgs'))!; + const domain = logins.findColumnByName('domain')!; + const domainUnique = domain.clone(); + domainUnique.isUnique = true; + await queryRunner.changeColumn('orgs', domain, domainUnique); + + // On postgres, all of the above amounts to: + // ALTER TABLE "orgs" ADD CONSTRAINT "..." UNIQUE ("domain") + // On sqlite, the table gets regenerated. + } + + public async down(queryRunner: QueryRunner): Promise { + const logins = (await queryRunner.getTable('orgs'))!; + const domain = logins.findColumnByName('domain')!; + const domainNonUnique = domain.clone(); + domainNonUnique.isUnique = false; + await queryRunner.changeColumn('orgs', domain, domainNonUnique); + + // On postgres, all of the above amount to: + // ALTER TABLE "orgs" DROP CONSTRAINT "..." + } +} diff --git a/app/gen-server/migration/1561589211752-Aliases.ts b/app/gen-server/migration/1561589211752-Aliases.ts new file mode 100644 index 00000000..5346950d --- /dev/null +++ b/app/gen-server/migration/1561589211752-Aliases.ts @@ -0,0 +1,67 @@ +import {MigrationInterface, QueryRunner, Table, TableColumn, TableIndex} from 'typeorm'; +import {datetime, now} from 'app/gen-server/sqlUtils'; + +export class Aliases1561589211752 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + const dbType = queryRunner.connection.driver.options.type; + + // Make a table for document aliases. + await queryRunner.createTable(new Table({ + name: 'aliases', + columns: [ + { + name: 'url_id', + type: 'varchar', + isPrimary: true + }, + { + name: 'org_id', + type: 'integer', + isPrimary: true + }, + { + name: 'doc_id', + type: 'varchar', + isNullable: true // nullable in case in future we make aliases for other resources + }, + { + name: "created_at", + type: datetime(dbType), + default: now(dbType) + } + ], + foreignKeys: [ + { + columnNames: ['doc_id'], + referencedColumnNames: ['id'], + referencedTableName: 'docs', + onDelete: 'CASCADE' // delete alias if doc goes away + }, + { + columnNames: ['org_id'], + referencedColumnNames: ['id'], + referencedTableName: 'orgs' + // no CASCADE set - let deletions be triggered via docs + } + ] + })); + + // Add preferred alias to docs. Not quite a foreign key (we'd need org as well) + await queryRunner.addColumn('docs', new TableColumn({ + name: 'url_id', + type: 'varchar', + isNullable: true + })); + await queryRunner.createIndex("docs", new TableIndex({ + name: "docs__url_id", + columnNames: ["url_id"] + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropIndex('docs', 'docs__url_id'); + await queryRunner.dropColumn('docs', 'url_id'); + await queryRunner.dropTable('aliases'); + } +} diff --git a/app/gen-server/migration/1568238234987-TeamMembers.ts b/app/gen-server/migration/1568238234987-TeamMembers.ts new file mode 100644 index 00000000..7a462ca4 --- /dev/null +++ b/app/gen-server/migration/1568238234987-TeamMembers.ts @@ -0,0 +1,56 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; +import * as roles from "app/common/roles"; +import {AclRuleOrg} from "app/gen-server/entity/AclRule"; +import {Group} from "app/gen-server/entity/Group"; +import {Organization} from "app/gen-server/entity/Organization"; +import {Permissions} from "app/gen-server/lib/Permissions"; + +export class TeamMembers1568238234987 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + // Get all orgs and add a team member ACL (with group) to each. + const orgs = await queryRunner.manager.createQueryBuilder() + .select("orgs.id") + .from(Organization, "orgs") + .getMany(); + for (const org of orgs) { + const groupInsert = await queryRunner.manager.createQueryBuilder() + .insert() + .into(Group) + .values([{name: roles.MEMBER}]) + .execute(); + const groupId = groupInsert.identifiers[0].id; + await queryRunner.manager.createQueryBuilder() + .insert() + .into(AclRuleOrg) + .values([{ + permissions: Permissions.VIEW, + organization: {id: org.id}, + group: groupId + }]) + .execute(); + } + } + + public async down(queryRunner: QueryRunner): Promise { + // Remove all team member groups and corresponding ACLs. + const groups = await queryRunner.manager.createQueryBuilder() + .select("groups") + .from(Group, "groups") + .where('name = :name', {name: roles.MEMBER}) + .getMany(); + for (const group of groups) { + await queryRunner.manager.createQueryBuilder() + .delete() + .from(AclRuleOrg) + .where("group_id = :id", {id: group.id}) + .execute(); + } + await queryRunner.manager.createQueryBuilder() + .delete() + .from(Group) + .where("name = :name", {name: roles.MEMBER}) + .execute(); + } + +} diff --git a/app/gen-server/migration/1569593726320-FirstLogin.ts b/app/gen-server/migration/1569593726320-FirstLogin.ts new file mode 100644 index 00000000..b4b72c98 --- /dev/null +++ b/app/gen-server/migration/1569593726320-FirstLogin.ts @@ -0,0 +1,18 @@ +import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; + +export class FirstLogin1569593726320 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + const sqlite = queryRunner.connection.driver.options.type === 'sqlite'; + const datetime = sqlite ? "datetime" : "timestamp with time zone"; + await queryRunner.addColumn('users', new TableColumn({ + name: 'first_login_at', + type: datetime, + isNullable: true + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropColumn('users', 'first_login_at'); + } +} diff --git a/app/gen-server/migration/1569946508569-FirstTimeUser.ts b/app/gen-server/migration/1569946508569-FirstTimeUser.ts new file mode 100644 index 00000000..8b5afab4 --- /dev/null +++ b/app/gen-server/migration/1569946508569-FirstTimeUser.ts @@ -0,0 +1,18 @@ +import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; +import {nativeValues} from 'app/gen-server/lib/values'; + +export class FirstTimeUser1569946508569 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.addColumn("users", new TableColumn({ + name: "is_first_time_user", + type: nativeValues.booleanType, + default: nativeValues.falseValue, + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropColumn("users", "is_first_time_user"); + } + +} diff --git a/app/gen-server/migration/1573569442552-CustomerIndex.ts b/app/gen-server/migration/1573569442552-CustomerIndex.ts new file mode 100644 index 00000000..a630773a --- /dev/null +++ b/app/gen-server/migration/1573569442552-CustomerIndex.ts @@ -0,0 +1,15 @@ +import {MigrationInterface, QueryRunner, TableIndex} from "typeorm"; + +export class CustomerIndex1573569442552 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createIndex("billing_accounts", new TableIndex({ + name: "billing_accounts__stripe_customer_id", + columnNames: ["stripe_customer_id"] + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropIndex("billing_accounts", "billing_accounts__stripe_customer_id"); + } +} diff --git a/app/gen-server/migration/1579559983067-ExtraIndexes.ts b/app/gen-server/migration/1579559983067-ExtraIndexes.ts new file mode 100644 index 00000000..5e7bcffc --- /dev/null +++ b/app/gen-server/migration/1579559983067-ExtraIndexes.ts @@ -0,0 +1,45 @@ +import {MigrationInterface, QueryRunner, TableIndex} from "typeorm"; + +export class ExtraIndexes1579559983067 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createIndex("acl_rules", new TableIndex({ + name: "acl_rules__group_id", + columnNames: ["group_id"] + })); + await queryRunner.createIndex("orgs", new TableIndex({ + name: "orgs__billing_account_id", + columnNames: ["billing_account_id"] + })); + await queryRunner.createIndex("billing_account_managers", new TableIndex({ + name: "billing_account_managers__billing_account_id", + columnNames: ["billing_account_id"] + })); + await queryRunner.createIndex("billing_account_managers", new TableIndex({ + name: "billing_account_managers__user_id", + columnNames: ["user_id"] + })); + await queryRunner.createIndex("billing_accounts", new TableIndex({ + name: "billing_accounts__product_id", + columnNames: ["product_id"] + })); + await queryRunner.createIndex("aliases", new TableIndex({ + name: "aliases__org_id", + columnNames: ["org_id"] + })); + await queryRunner.createIndex("aliases", new TableIndex({ + name: "aliases__doc_id", + columnNames: ["doc_id"] + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropIndex("acl_rules", "acl_rules__group_id"); + await queryRunner.dropIndex("orgs", "orgs__billing_account_id"); + await queryRunner.dropIndex("billing_account_managers", "billing_account_managers__billing_account_id"); + await queryRunner.dropIndex("billing_account_managers", "billing_account_managers__user_id"); + await queryRunner.dropIndex("billing_accounts", "billing_accounts__product_id"); + await queryRunner.dropIndex("aliases", "aliases__org_id"); + await queryRunner.dropIndex("aliases", "aliases__doc_id"); + } +} diff --git a/app/gen-server/migration/1591755411755-OrgHost.ts b/app/gen-server/migration/1591755411755-OrgHost.ts new file mode 100644 index 00000000..10d6fb13 --- /dev/null +++ b/app/gen-server/migration/1591755411755-OrgHost.ts @@ -0,0 +1,17 @@ +import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; + +export class OrgHost1591755411755 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.addColumn('orgs', new TableColumn({ + name: 'host', + type: 'varchar', + isNullable: true, + isUnique: true + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropColumn('orgs', 'host'); + } +} diff --git a/app/gen-server/migration/1592261300044-DocRemovedAt.ts b/app/gen-server/migration/1592261300044-DocRemovedAt.ts new file mode 100644 index 00000000..e87b3e19 --- /dev/null +++ b/app/gen-server/migration/1592261300044-DocRemovedAt.ts @@ -0,0 +1,26 @@ +import {nativeValues} from "app/gen-server/lib/values"; +import {MigrationInterface, QueryRunner, TableColumn, TableIndex} from "typeorm"; + +export class DocRemovedAt1592261300044 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + for (const table of ['docs', 'workspaces']) { + await queryRunner.addColumn(table, new TableColumn({ + name: 'removed_at', + type: nativeValues.dateTimeType, + isNullable: true + })); + await queryRunner.createIndex(table, new TableIndex({ + name: `${table}__removed_at`, + columnNames: ['removed_at'] + })); + } + } + + public async down(queryRunner: QueryRunner): Promise { + for (const table of ['docs', 'workspaces']) { + await queryRunner.dropIndex(table, `${table}__removed_at`); + await queryRunner.dropColumn(table, 'removed_at'); + } + } +} diff --git a/app/gen-server/sqlUtils.ts b/app/gen-server/sqlUtils.ts new file mode 100644 index 00000000..14242ee1 --- /dev/null +++ b/app/gen-server/sqlUtils.ts @@ -0,0 +1,90 @@ +import {DatabaseType} from 'typeorm'; + +/** + * + * Generates an expression to simulate postgres's bit_or + * aggregate function in sqlite. The expression is verbose, + * and has a term for each bit in the permission bitmap, + * but this seems ok since sqlite is only used in the dev + * environment. + * @param column: the sql column to aggregate + * @param bits: the maximum number of bits to consider + * + */ +export function sqliteBitOr(column: string, bits: number): string { + const parts: string[] = []; + let mask: number = 1; + for (let b = 0; b < bits; b++) { + parts.push(`((sum(${column}&${mask})>0)<<${b})`); + mask *= 2; + } + return `(${parts.join('+')})`; +} + +/** + * Generates an expression to aggregate the named column + * by taking the bitwise-or of all the values it takes on. + * @param dbType: the type of database (sqlite and postgres are supported) + * @param column: the sql column to aggregate + * @param bits: the maximum number of bits to consider (used for sqlite variant) + */ +export function bitOr(dbType: DatabaseType, column: string, bits: number): string { + switch (dbType) { + case 'postgres': + return `bit_or(${column})`; + case 'sqlite': + return sqliteBitOr(column, bits); + default: + throw new Error(`bitOr not implemented for ${dbType}`); + } +} + +/** + * Convert a json value returned by the database into a javascript + * object. For postgres, the value is already unpacked, but for sqlite + * it is a string. + */ +export function readJson(dbType: DatabaseType, selection: any) { + switch (dbType) { + case 'postgres': + return selection; + case 'sqlite': + return JSON.parse(selection); + default: + throw new Error(`readJson not implemented for ${dbType}`); + } +} + +export function now(dbType: DatabaseType) { + switch (dbType) { + case 'postgres': + return 'now()'; + case 'sqlite': + return "datetime('now')"; + default: + throw new Error(`now not implemented for ${dbType}`); + } +} + +// Understands strings like: "-30 days" or "1 year" +export function fromNow(dbType: DatabaseType, relative: string) { + switch (dbType) { + case 'postgres': + return `(now() + interval '${relative}')`; + case 'sqlite': + return `datetime('now','${relative}')`; + default: + throw new Error(`fromNow not implemented for ${dbType}`); + } +} + +export function datetime(dbType: DatabaseType) { + switch (dbType) { + case 'postgres': + return 'timestamp with time zone'; + case 'sqlite': + return "datetime"; + default: + throw new Error(`now not implemented for ${dbType}`); + } +} diff --git a/app/plugin/CustomSectionAPI-ti.ts b/app/plugin/CustomSectionAPI-ti.ts new file mode 100644 index 00000000..7e90cf7e --- /dev/null +++ b/app/plugin/CustomSectionAPI-ti.ts @@ -0,0 +1,11 @@ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const CustomSectionAPI = t.iface([], { + "createSection": t.func("void", t.param("inlineTarget", "RenderTarget")), +}); + +const exportedTypeSuite: t.ITypeSuite = { + CustomSectionAPI, +}; +export default exportedTypeSuite; diff --git a/app/plugin/CustomSectionAPI.ts b/app/plugin/CustomSectionAPI.ts new file mode 100644 index 00000000..91920da1 --- /dev/null +++ b/app/plugin/CustomSectionAPI.ts @@ -0,0 +1,10 @@ +/** + * API definitions for CustomSection plugins. + */ + + +import {RenderTarget} from './RenderOptions'; + +export interface CustomSectionAPI { + createSection(inlineTarget: RenderTarget): Promise; +} diff --git a/app/plugin/FileParserAPI-ti.ts b/app/plugin/FileParserAPI-ti.ts new file mode 100644 index 00000000..821016bd --- /dev/null +++ b/app/plugin/FileParserAPI-ti.ts @@ -0,0 +1,41 @@ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const EditOptionsAPI = t.iface([], { + "getParseOptions": t.func("ParseOptions", t.param("parseOptions", "ParseOptions", true)), +}); + +export const ParseFileAPI = t.iface([], { + "parseFile": t.func("ParseFileResult", t.param("file", "FileSource"), t.param("parseOptions", "ParseOptions", true)), +}); + +export const ParseOptions = t.iface([], { + "NUM_ROWS": t.opt("number"), + "SCHEMA": t.opt(t.array("ParseOptionSchema")), +}); + +export const ParseOptionSchema = t.iface([], { + "name": "string", + "label": "string", + "type": "string", + "visible": "boolean", +}); + +export const FileSource = t.iface([], { + "path": "string", + "origName": "string", +}); + +export const ParseFileResult = t.iface(["GristTables"], { + "parseOptions": "ParseOptions", +}); + +const exportedTypeSuite: t.ITypeSuite = { + EditOptionsAPI, + ParseFileAPI, + ParseOptions, + ParseOptionSchema, + FileSource, + ParseFileResult, +}; +export default exportedTypeSuite; diff --git a/app/plugin/FileParserAPI.ts b/app/plugin/FileParserAPI.ts new file mode 100644 index 00000000..867f41b4 --- /dev/null +++ b/app/plugin/FileParserAPI.ts @@ -0,0 +1,52 @@ +/** + * API definitions for FileParser plugins. + */ + +import {GristTables} from './GristTable'; + +export interface EditOptionsAPI { + getParseOptions(parseOptions?: ParseOptions): Promise; +} + +export interface ParseFileAPI { + parseFile(file: FileSource, parseOptions?: ParseOptions): Promise; +} + +/** + * ParseOptions contains parse options depending on plugin, + * number of rows, which is special option that can be used for any plugin + * and schema for generating parse options UI + */ +export interface ParseOptions { + NUM_ROWS?: number; + SCHEMA?: ParseOptionSchema[]; +} + +/** + * ParseOptionSchema contains information for generaing parse options UI + */ +export interface ParseOptionSchema { + name: string; + label: string; + type: string; + visible: boolean; +} + +export interface FileSource { + /** + * The path is often a temporary file, so its name is meaningless. Access to the file depends on + * the type of plugin. For instance, for `safePython` plugins file is directly available at + * `/importDir/path`. + */ + path: string; + + /** + * Plugins that want to know the original filename should use origName. Depending on the source + * of the data, it may or may not be meaningful. + */ + origName: string; +} + +export interface ParseFileResult extends GristTables { + parseOptions: ParseOptions; +} diff --git a/app/plugin/GristAPI-ti.ts b/app/plugin/GristAPI-ti.ts new file mode 100644 index 00000000..096d0135 --- /dev/null +++ b/app/plugin/GristAPI-ti.ts @@ -0,0 +1,34 @@ +/** + * This module was automatically generated by `ts-interface-builder` + */ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const ComponentKind = t.union(t.lit("safeBrowser"), t.lit("safePython"), t.lit("unsafeNode")); + +export const GristAPI = t.iface([], { + "render": t.func("number", t.param("path", "string"), t.param("target", "RenderTarget"), + t.param("options", "RenderOptions", true)), + "dispose": t.func("void", t.param("procId", "number")), + "subscribe": t.func("void", t.param("tableId", "string")), + "unsubscribe": t.func("void", t.param("tableId", "string")), +}); + +export const GristDocAPI = t.iface([], { + "getDocName": t.func("string"), + "listTables": t.func(t.array("string")), + "fetchTable": t.func("any", t.param("tableId", "string")), + "applyUserActions": t.func("any", t.param("actions", t.array(t.array("any")))), +}); + +export const GristView = t.iface([], { + "fetchSelectedTable": t.func("any"), +}); + +const exportedTypeSuite: t.ITypeSuite = { + ComponentKind, + GristAPI, + GristDocAPI, + GristView, +}; +export default exportedTypeSuite; diff --git a/app/plugin/GristAPI.ts b/app/plugin/GristAPI.ts new file mode 100644 index 00000000..7560d613 --- /dev/null +++ b/app/plugin/GristAPI.ts @@ -0,0 +1,99 @@ +/** + * This file defines the interface for the grist api exposed to SafeBrowser plugins. Grist supports + * various ways to require it to cover various scenarios. If writing the main safeBrowser module + * (the one referenced by the components.safeBrowser key of the manifest) use + * `self.importScript('grist');`, if writing a view include the script in the html `` + * + * + * Example usage (let's assume that Grist let's plugin contributes to a Foo API defined as follow ): + * + * interface Foo { + * foo(name: string): Promise; + * } + * + * > main.ts: + * class MyFoo { + * public foo(name: string): Promise { + * return new Promise( async resolve => { + * grist.rpc.onMessage( e => { + * resolve(e.data + name); + * }); + * grist.ready(); + * await grist.api.render('view1.html', 'fullscreen'); + * }); + * } + * } + * grist.rpc.registerImpl('grist', new MyFoo()); // can add 3rd arg with type information + * + * > view1.html includes: + * grist.api.render('static/view2.html', 'fullscreen').then( view => { + * grist.rpc.onMessage(e => grist.rpc.postMessageForward("main.ts", e.data)); + * }); + * + * > view2.html includes: + * grist.rpc.postMessage('view1.html', 'foo '); + * + */ + +import {RenderOptions, RenderTarget} from './RenderOptions'; + +export type ComponentKind = "safeBrowser" | "safePython" | "unsafeNode"; + +export const RPC_GRISTAPI_INTERFACE = '_grist_api'; + +export interface GristAPI { + /** + * Render the file at `path` into the `target` location in Grist. `path` must be relative to the + * root of the plugin's directory and point to an html that is contained within the plugin's + * directory. `target` is a predifined location of the Grist UI, it could be `fullscreen` or + * identifier for an inline target. Grist provides inline target identifiers in certain call + * plugins. E.g. ImportSourceAPI.getImportSource is given a target identifier to allow rende UI + * inline in the import dialog. Returns the procId which can be used to dispose the view. + */ + render(path: string, target: RenderTarget, options?: RenderOptions): Promise; + + /** + * Dispose the process with id procId. If the process was embedded into the UI, removes the + * corresponding element from the view. + */ + dispose(procId: number): Promise; + + // Subscribes to actions for `tableId`. Actions of all subscribed tables are send as rpc's + // message. + // TODO: document format of messages that can be listened on `rpc.onMessage(...);` + subscribe(tableId: string): Promise; + + // Unsubscribe from actions for `tableId`. + unsubscribe(tableId: string): Promise; + +} + +/** + * GristDocAPI interface is implemented by Grist, and allows getting information from and + * interacting with the Grist document to which a plugin is attached. + */ +export interface GristDocAPI { + // Returns the docName that identifies the document. + getDocName(): Promise; + + // Returns a sorted list of table IDs. + listTables(): Promise; + + // Returns a complete table of data in the format {colId: [values]}, including the 'id' column. + // Do not modify the returned arrays in-place, especially if used directly (not over RPC). + // TODO: return type is Promise{[colId: string]: CellValue[]}> but cannot be specified because + // ts-interface-builder does not properly support index-signature. + fetchTable(tableId: string): Promise; + + // Applies an array of user actions. + // todo: return type should be Promise, but this requires importing modules from + // `app/common` which is not currently supported by the build. + applyUserActions(actions: any[][]): Promise; +} + +export interface GristView { + // Like fetchTable, but gets data for the custom section specifically, if there is any. + // TODO: return type is Promise{[colId: string]: CellValue[]}> but cannot be specified because + // ts-interface-builder does not properly support index-signature. + fetchSelectedTable(): Promise; +} diff --git a/app/plugin/GristTable-ti.ts b/app/plugin/GristTable-ti.ts new file mode 100644 index 00000000..e47f2103 --- /dev/null +++ b/app/plugin/GristTable-ti.ts @@ -0,0 +1,24 @@ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const GristTable = t.iface([], { + "table_name": t.union("string", "null"), + "column_metadata": t.array("GristColumn"), + "table_data": t.array(t.array("any")), +}); + +export const GristTables = t.iface([], { + "tables": t.array("GristTable"), +}); + +export const GristColumn = t.iface([], { + "id": "string", + "type": "string", +}); + +const exportedTypeSuite: t.ITypeSuite = { + GristTable, + GristTables, + GristColumn, +}; +export default exportedTypeSuite; diff --git a/app/plugin/GristTable.ts b/app/plugin/GristTable.ts new file mode 100644 index 00000000..f871b2b8 --- /dev/null +++ b/app/plugin/GristTable.ts @@ -0,0 +1,38 @@ +/** + * Common definitions for Grist plugin APIs. + */ + +/** + * + * Metadata and data for a table. This is documenting what is currently returned by the + * core plugins. Could be worth reconciling with: + * https://phab.getgrist.com/w/grist_data_format/ + * Capitalization is python-style. + * + */ +export interface GristTable { + table_name: string | null; // currently allow names to be null + column_metadata: GristColumn[]; + table_data: any[][]; +} + +export interface GristTables { + tables: GristTable[]; +} + +/** + * + * Metadata about a single column. + * + */ +export interface GristColumn { + id: string; + type: string; +} + +export enum APIType { + ImportSourceAPI, + ImportProcessorAPI, + ParseOptionsAPI, + ParseFileAPI, +} diff --git a/app/plugin/ImportSourceAPI-ti.ts b/app/plugin/ImportSourceAPI-ti.ts new file mode 100644 index 00000000..0a1529f2 --- /dev/null +++ b/app/plugin/ImportSourceAPI-ti.ts @@ -0,0 +1,44 @@ +/** + * This module was automatically generated by `ts-interface-builder` + */ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const ImportSourceAPI = t.iface([], { + "getImportSource": t.func(t.union("ImportSource", "undefined")), +}); + +export const ImportProcessorAPI = t.iface([], { + "processImport": t.func(t.array("GristTable"), t.param("source", "ImportSource")), +}); + +export const FileContent = t.iface([], { + "content": "any", + "name": "string", +}); + +export const FileListItem = t.iface([], { + "kind": t.lit("fileList"), + "files": t.array("FileContent"), +}); + +export const URL = t.iface([], { + "kind": t.lit("url"), + "url": "string", +}); + +export const ImportSource = t.iface([], { + "item": t.union("FileListItem", "URL"), + "options": t.opt(t.union("string", "Buffer")), + "description": t.opt("string"), +}); + +const exportedTypeSuite: t.ITypeSuite = { + ImportSourceAPI, + ImportProcessorAPI, + FileContent, + FileListItem, + URL, + ImportSource, +}; +export default exportedTypeSuite; diff --git a/app/plugin/ImportSourceAPI.ts b/app/plugin/ImportSourceAPI.ts new file mode 100644 index 00000000..0074409c --- /dev/null +++ b/app/plugin/ImportSourceAPI.ts @@ -0,0 +1,53 @@ +/** + * API definitions for ImportSource plugins. + */ + +import { GristTable } from './GristTable'; + +export interface ImportSourceAPI { + /** + * Returns a promise that resolves to an `ImportSource` which is then passed for import to the + * import modal dialog. `undefined` interrupts the workflow and prevent the modal from showing up, + * but not an empty list of `ImportSourceItem`. Which is a valid import source and is used in + * cases where only options are to be sent to an `ImportProcessAPI` implementation. + */ + getImportSource(): Promise; +} + +export interface ImportProcessorAPI { + processImport(source: ImportSource): Promise; +} + +export interface FileContent { + content: any; + name: string; +} + +export interface FileListItem { + kind: "fileList"; + // TODO: there're might be a better way to send file content. In particular for electron where + // file will then be send from client to server where it shouldn't be really. An idea could be to + // expose something similar to `client/lib/upload.ts` to let plugins create upload entries, and + // then send only uploads ids over the rpc. + files: FileContent[]; +} + +export interface URL { + kind: "url"; + url: string; +} + +export interface ImportSource { + item: FileListItem | URL; + + /** + * The options are only passed within this plugin, nothing else needs to know how they are + * serialized. Using JSON.stringify/JSON.parse is a simple approach. + */ + options?: string|Buffer; + + /** + * The short description that shows in the import dialog after source have been selected. + */ + description?: string; +} diff --git a/app/plugin/InternalImportSourceAPI-ti.ts b/app/plugin/InternalImportSourceAPI-ti.ts new file mode 100644 index 00000000..0b5ea84d --- /dev/null +++ b/app/plugin/InternalImportSourceAPI-ti.ts @@ -0,0 +1,11 @@ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const InternalImportSourceAPI = t.iface([], { + "getImportSource": t.func(t.union("ImportSource", "undefined"), t.param("inlineTarget", "RenderTarget")), +}); + +const exportedTypeSuite: t.ITypeSuite = { + InternalImportSourceAPI, +}; +export default exportedTypeSuite; diff --git a/app/plugin/InternalImportSourceAPI.ts b/app/plugin/InternalImportSourceAPI.ts new file mode 100644 index 00000000..349e3c16 --- /dev/null +++ b/app/plugin/InternalImportSourceAPI.ts @@ -0,0 +1,24 @@ +import { RenderTarget } from './RenderOptions'; + +import { ImportSource } from './ImportSourceAPI'; + +export * from './ImportSourceAPI'; + +/** + * This internal interface is implemented by grist-plugin-api.ts to support + * `grist.addImporter(...)`. This is this interface that grist stubs to calls + * `ImportSourceAPI`. However, some of the complexity (ie: rendering targets) is hidden from the + * plugin author which implements directly the simpler `ImportSourceAPI`. + * + * Reason for this interface is because we want to have the `inlineTarget` parameter but we don't + * want plugin author to have it. + */ +export interface InternalImportSourceAPI { + /** + * The `inlineTarget` argument which will be passed to the implementation of this method, can be + * used as follow `grist.api.render('index.html', inlineTarget)` to embbed `index.html` in the + * import panel. Or it can be ignored and use `'fullscreen'` in-place. It is used in + * `grist.addImporter(...)` according to the value of the `mode` argument. + */ + getImportSource(inlineTarget: RenderTarget): Promise; +} diff --git a/app/plugin/PluginManifest-ti.ts b/app/plugin/PluginManifest-ti.ts new file mode 100644 index 00000000..231f5600 --- /dev/null +++ b/app/plugin/PluginManifest-ti.ts @@ -0,0 +1,60 @@ +/** + * This module was automatically generated by `ts-interface-builder` + */ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const PublishedPlugin = t.iface(["BarePlugin"], { + "name": "string", + "version": "string", +}); + +export const BarePlugin = t.iface([], { + "components": t.iface([], { + "safeBrowser": t.opt("string"), + "safePython": t.opt("string"), + "unsafeNode": t.opt("string"), + "deactivate": t.opt(t.iface([], { + "inactivitySec": t.opt("number"), + })), + }), + "contributions": t.iface([], { + "importSources": t.opt(t.array("ImportSource")), + "fileParsers": t.opt(t.array("FileParser")), + "customSections": t.opt(t.array("CustomSection")), + }), + "experimental": t.opt("boolean"), +}); + +export const ImportSource = t.iface([], { + "label": "string", + "importSource": "Implementation", + "importProcessor": t.opt("Implementation"), +}); + +export const FileParser = t.iface([], { + "fileExtensions": t.array("string"), + "editOptions": t.opt("Implementation"), + "parseFile": "Implementation", +}); + +export const CustomSection = t.iface([], { + "path": "string", + "name": "string", +}); + +export const Implementation = t.iface([], { + "component": t.union(t.lit("safeBrowser"), t.lit("safePython"), t.lit("unsafeNode")), + "name": "string", + "path": t.opt("string"), +}); + +const exportedTypeSuite: t.ITypeSuite = { + PublishedPlugin, + BarePlugin, + ImportSource, + FileParser, + CustomSection, + Implementation, +}; +export default exportedTypeSuite; diff --git a/app/plugin/PluginManifest.ts b/app/plugin/PluginManifest.ts new file mode 100644 index 00000000..6bffef88 --- /dev/null +++ b/app/plugin/PluginManifest.ts @@ -0,0 +1,219 @@ +/** + * This file defines the interface for a plugin manifest. + * + * Note that it is possible to validate a manifest against a TypeScript interface as follows: + * (1) Convert the interface to a JSON schema at build time using + * https://www.npmjs.com/package/typescript-json-schema: + * bin/typescript-json-schema --required --noExtraProps PluginManifest.ts PublishedPlugin + * (2) Use a JSON schema validator like https://www.npmjs.com/package/ajv to validate manifests + * read at run-time and produce informative errors automatically. + * + * TODO [Proposal]: To save an ImportSource for reuse, we would save: + * { + * pluginId: string; + * importSource: ImportSource; + * importProcessor?: Implementation; + * parseOptions?: ParseOptions; // If importProcessor is omitted and fileParser is used. + * } + * This should suffice for database re-imports, as well as for re-imports from a URL, or from a + * saved path in the filesystem (which can be a builtIn plugin available for Electron version). + */ + +/** + * PublishedPlugin is a BarePlugin with additional attributes to identify and describe a plugin + * for publishing. + */ +export interface PublishedPlugin extends BarePlugin { + name: string; + version: string; +} + + +/** + * BarePlugin defines the functionality of a plugin. It is the only part required for a plugin to + * function, and is implemented by built-in plugins, published plugins, and private plugins (such + * as those being developed). + */ +export interface BarePlugin { + /** + * Components describe how the plugin runs. A plugin may provide UI and behavior that runs in + * the browser, Python code that runs in a secure sandbox, and arbitrary code that runs in Node. + */ + components: { + /** + * Relative path to the directory whose content will be served to the browser. Required for + * those plugins that need to render their own HTML or run in-browser Javascript. This + * directory should contain all html files referenced in the manifest. + * + * It is "safe" in that Grist offers protections that allow such plugins to be marked "safe". + */ + safeBrowser?: string; + + /** + * Relative path to a file with Python code that will be run in a python sandbox. This + * file is started on plugin activation, and should register any implemented APIs. + * Required for plugins that do Python processing. + * + * It is "safe" in that Grist offers protections that allow such plugins to be marked "safe". + */ + safePython?: string; + + /** + * Relative path to a file containing Javascript code that will be executed with Node.js. + * The code is called on plugin activation, and should register any implemented APIs + * once we've figured out how that should happen (TODO). Required for plugins that need + * to do any "unsafe" work, such as accessing the local filesystem or starting helper + * programs. + * + * It is "unsafe" in that it can do too much, and Grist marks such plugins as "unsafe". + * + * An unsafeNode component opens as separate process to run plugin node code, with the + * NODE_PATH set to the plugin directory. The node code can execute arbitrary actions - + * there is no sandboxing. + * + * The node child may communicate with the server via standard ChildProcess ipc + * (`process.send`, `process.on('message', ...)`). The child is expected to + * `process.send` a message to the server once it is listening to the `message` + * event. That message is expected to contain a `ready` field set to `true`. All + * other communication should follow the protocol implemented by the Rpc module. + * TODO: provide plugin authors with documentation + library to use that implements + * these requirements. + * + */ + unsafeNode?: string; + + /** + * Options for when to deactivate the plugin, i.e. when to stop any plugin processes. (Note + * that we may in the future also add options for when to activate the plugin, which is for + * now automatic and not configurable.) + */ + deactivate?: { + // Deactivate after this many seconds of inactivity. Defaults to 300 (5 minutes) if omitted. + inactivitySec?: number; + } + }; + + /** + * Contributions describe what new functionality the plugin contributes to the Grist + * application. See documentation for individual contribution types for details. Any plugin may + * provide multiple contributions. It is common to provide just one, in which case include a + * single property with a single-element array. + */ + contributions: { + importSources?: ImportSource[]; + fileParsers?: FileParser[]; + customSections?: CustomSection[]; + }; + + /** + * Experimental plugins run only if the environment variable GRIST_EXPERIMENTAL_PLUGINS is + * set. Otherwise they are ignored. This is useful for plugins that needs a bit of experimentation + * before being pushed to production (ie: production does not have GRIST_EXPERIMENTAL_PLUGINS set + * but staging does). Keep in mind that developers need to set this environment if they want to + * run them locally. + */ + experimental?: boolean; +} + +/** + * An ImportSource plugin creates a new source of imports, such as an external API, a file-sharing + * service, or a new type of database. It adds a new item for the user to select when importing. + */ +export interface ImportSource { + /** + * Label shows up as a new item for the user to select when starting an import. + */ + label: string; + + /** + * Implementation of ImportSourceAPI. Supports safeBrowser component, which allows you to create + * custom UI to show to the user. Or describe UI using a .json or .yml config file and use + * {component: "builtIn", name: "importSourceConfig", path: "your-config"}. + */ + importSource: Implementation; + + /** + * Implementation of ImportProcessorAPI. It receives the output of importSource, and produces + * Grist data. If omitted, uses the default ImportProcessor, which is equivalent to + * {component: "builtIn", name: "fileParser"}. + * + * The default ImportProcessor handles received ImportSourceItems as follows: + * (1) items of type "file" are saved to temp files. + * (2) items of type "url" are downloaded to temp files. + * (3) calls ParseFileAPI.parseFile() with all temp files, to produce Grist tables + * (4) returns those Grist tables along with all items of type "table". + * Note that the default ImportParser ignores ImportSource items of type "custom". + */ + importProcessor?: Implementation; + +} + +/** + * A FileParser plugin adds support to parse a new type of file data, such as "csv", "yml", or + * "ods". It then enables importing the new type of file via upload or from any other ImportSource + * that produces Files or URLs. + */ +export interface FileParser { + /** + * File extensions for which this FileParser should be considered, e.g. "csv", "yml". You may + * use "" for files with no extensions, and "*" to match any extension. + */ + fileExtensions: string[]; + + /** + * Implementation of EditOptionsAPI. Supports safeBrowser component, which allows you to create + * custom UI to show to the user. Or describe UI using a .json or .yml config file and use + * {component: "builtIn", name: "parseOptionsConfig", path: "your-config"}. + * + * If omitted, the user will be shown no parse options. + */ + editOptions?: Implementation; + + /** + * Implementation of ParseFileAPI, which converts Files to Grist data using parse options. + */ + parseFile: Implementation; +} + +/** + * A CustomSection plugin adds support to add new types of section to Grist, such as a calendar, + * maps, data visualizations. + */ +export interface CustomSection { + /** + * Path to an html file. + */ + path: string; + /** + * The name should uniquely identify the section in the plugin. + */ + name: string; +} + +/** + * A Plugin supplies one or more Implementation of some APIs. Components register implementation + * using a call such as: + * grist.register(SomeAPI, 'myName', impl). + * The manifest documentation describes which API must be implemented at any particular point, and + * it is the plugin's responsibility to register an implementation of the correct API and refer to + * it by Implementation.name. + */ +export interface Implementation { + /** + * Which component of the plugin provides this implementation. + */ + component: "safeBrowser" | "safePython" | "unsafeNode"; + + /** + * The name of the implementation registered by the chosen component. The same component can + * register any number of APIs at any names. + */ + name: string; + + /** + * Path is used by safeBrowser component for which page to load. Defaults to 'index.html'. + * It is also used by certain builtIn implementation, e.g. if name is 'parse-options-config', + * path is the path to JSON or YAML file containing the configuration. + */ + path?: string; +} diff --git a/app/plugin/RenderOptions-ti.ts b/app/plugin/RenderOptions-ti.ts new file mode 100644 index 00000000..a6302dd1 --- /dev/null +++ b/app/plugin/RenderOptions-ti.ts @@ -0,0 +1,14 @@ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const RenderTarget = t.union(t.lit("fullscreen"), "number"); + +export const RenderOptions = t.iface([], { + "height": t.opt("string"), +}); + +const exportedTypeSuite: t.ITypeSuite = { + RenderTarget, + RenderOptions, +}; +export default exportedTypeSuite; diff --git a/app/plugin/RenderOptions.ts b/app/plugin/RenderOptions.ts new file mode 100644 index 00000000..82a94156 --- /dev/null +++ b/app/plugin/RenderOptions.ts @@ -0,0 +1,11 @@ +/** + * Where to append the content that a plugin renders. + */ +export type RenderTarget = "fullscreen" | number; + +/** + * Options for the `grist.render` function. + */ +export interface RenderOptions { + height?: string; +} diff --git a/app/plugin/StorageAPI-ti.ts b/app/plugin/StorageAPI-ti.ts new file mode 100644 index 00000000..657d9efb --- /dev/null +++ b/app/plugin/StorageAPI-ti.ts @@ -0,0 +1,15 @@ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const Storage = t.iface([], { + "getItem": t.func("any", t.param("key", "string")), + "hasItem": t.func("boolean", t.param("key", "string")), + "setItem": t.func("void", t.param("key", "string"), t.param("value", "any")), + "removeItem": t.func("void", t.param("key", "string")), + "clear": t.func("void"), +}); + +const exportedTypeSuite: t.ITypeSuite = { + Storage, +}; +export default exportedTypeSuite; diff --git a/app/plugin/StorageAPI.ts b/app/plugin/StorageAPI.ts new file mode 100644 index 00000000..87e35023 --- /dev/null +++ b/app/plugin/StorageAPI.ts @@ -0,0 +1,8 @@ +// subset of WebStorage API +export interface Storage { + getItem(key: string): any; + hasItem(key: string): boolean; + setItem(key: string, value: any): void; + removeItem(key: string): void; + clear(): void; +} diff --git a/app/plugin/TypeCheckers.ts b/app/plugin/TypeCheckers.ts new file mode 100644 index 00000000..0e42a20f --- /dev/null +++ b/app/plugin/TypeCheckers.ts @@ -0,0 +1,50 @@ +import {createCheckers, ICheckerSuite} from 'ts-interface-checker'; +import CustomSectionAPITI from './CustomSectionAPI-ti'; +import FileParserAPITI from './FileParserAPI-ti'; +import GristAPITI from './GristAPI-ti'; +import GristTableTI from './GristTable-ti'; +import ImportSourceAPITI from './ImportSourceAPI-ti'; +import InternalImportSourceAPITI from './InternalImportSourceAPI-ti'; +import RenderOptionsTI from './RenderOptions-ti'; +import StorageAPITI from './StorageAPI-ti'; + +/** + * The ts-interface-checker type suites are all exported with the "TI" suffix. + */ +export { + CustomSectionAPITI, FileParserAPITI, GristAPITI, GristTableTI, ImportSourceAPITI, + InternalImportSourceAPITI, RenderOptionsTI, StorageAPITI}; + +const allTypes = [ + CustomSectionAPITI, FileParserAPITI, GristAPITI, GristTableTI, ImportSourceAPITI, + InternalImportSourceAPITI, RenderOptionsTI, StorageAPITI]; + +function checkDuplicates(types: Array<{[key: string]: object}>) { + const seen = new Set(); + for (const t of types) { + for (const key of Object.keys(t)) { + if (seen.has(key)) { throw new Error(`TypeCheckers: Duplicate type name ${key}`); } + seen.add(key); + // Uncomment the line below to generate updated list of included types. + // console.log(`'${key}' |`); + } + } +} + +checkDuplicates(allTypes); + +/** + * We also create and export a global checker object that includes all of the types above. + */ +export const checkers = createCheckers(...allTypes) as ( + // The following Pick typecast ensures that Typescript can only use correct properties of the + // checkers object (e.g. checkers.GristAPI will compile, but checkers.GristApi will not). + // TODO: The restrictive type of ICheckerSuite should be generated automatically. (Currently + // generated by commenting out console.log() in checkDuplicates() above.) + Pick); diff --git a/app/plugin/grist-plugin-api.ts b/app/plugin/grist-plugin-api.ts new file mode 100644 index 00000000..be168ca3 --- /dev/null +++ b/app/plugin/grist-plugin-api.ts @@ -0,0 +1,135 @@ +// Provide a way to acess grist for iframe, web worker (which runs the main safeBrowser script) and +// unsafeNode. WebView should work the same way as iframe, grist is exposed just the same way and +// necessary api is exposed using preload script. Here we bootstrap from channel capabilities to key +// parts of the grist API. + +// For iframe (and webview): +// user will add '' and get a window.grist + +// For web worker: +// use will add `self.importScripts('/grist-api.js');` + +// For node, user will do something like: +// const {grist} = require('grist-api'); +// grist.registerFunction(); +// In TypeScript: +// import {grist} from 'grist-api'; +// grist.registerFunction(); + +// tslint:disable:no-console + +import { GristAPI, GristDocAPI, GristView, RPC_GRISTAPI_INTERFACE } from './GristAPI'; +import { ImportSource, ImportSourceAPI, InternalImportSourceAPI } from './InternalImportSourceAPI'; +import { RenderOptions, RenderTarget } from './RenderOptions'; +import { checkers } from './TypeCheckers'; + +export * from './TypeCheckers'; +export * from './FileParserAPI'; +export * from './GristAPI'; +export * from './GristTable'; +export * from './ImportSourceAPI'; +export * from './StorageAPI'; +export * from './RenderOptions'; + +import {IRpcLogger, Rpc} from 'grain-rpc'; + +export const rpc: Rpc = new Rpc({logger: createRpcLogger()}); + +export const api = rpc.getStub(RPC_GRISTAPI_INTERFACE, checkers.GristAPI); +export const coreDocApi = rpc.getStub('GristDocAPI@grist', checkers.GristDocAPI); +export const viewApi = rpc.getStub('GristView', checkers.GristView); +export const docApi = { + ...coreDocApi, + ...viewApi, +}; + +export const on = rpc.on.bind(rpc); + +/** + * Calling `addImporter(...)` adds a safeBrowser importer. It is a short-hand for forwarding calls + * to an `ImportSourceAPI` implementation registered in the file at `path`. It takes care of + * creating the stub, registering an implementation that renders the file, forward the call and + * dispose the view properly. If `mode` is `'inline'` embeds the view in the import modal, ohterwise + * renders fullscreen. + * + * Notes: it assumes that file at `path` registers an `ImportSourceAPI` implementation under + * `name`. Calling `addImporter(...)` from another component than a `safeBrowser` component is not + * currently supported. + * + */ +export async function addImporter(name: string, path: string, mode: 'fullscreen' | 'inline', options?: RenderOptions) { + // checker is omitterd for implementation because call was alredy checked by grist. + rpc.registerImpl(name, { + async getImportSource(target: RenderTarget): Promise { + const procId = await api.render(path, mode === 'inline' ? target : 'fullscreen', options); + try { + // stubName for the interface `name` at forward destination `path` + const stubName = `${name}@${path}`; + // checker is omitted in stub because call will be checked just after in grist. + return await rpc.getStub(stubName).getImportSource(); + } finally { + await api.dispose(procId); + } + } + }); +} + +/** + * Declare that a component is prepared to receive messages from the outside world. + * Grist will not attempt to communicate with it until this method is called. + */ +export function ready(): void { + rpc.processIncoming(); + rpc.sendReadyMessage(); +} + +function getPluginPath(location: Location) { + return location.pathname.replace(/^\/plugins\//, ''); +} + +if (typeof window !== 'undefined') { + // Window or iframe. + const preloadWindow: any = window; + if (preloadWindow.isRunningUnderElectron) { + rpc.setSendMessage(msg => preloadWindow.sendToHost(msg)); + preloadWindow.onGristMessage((data: any) => rpc.receiveMessage(data)); + } else { + rpc.setSendMessage(msg => window.parent.postMessage(msg, "*")); + window.onmessage = (e: MessageEvent) => rpc.receiveMessage(e.data); + } +} else if (typeof process === 'undefined') { + // Web worker. We can't really bring in the types for WebWorker (available with --lib flag) + // without conflicting with a regular window, so use just use `self as any` here. + self.onmessage = (e: MessageEvent) => rpc.receiveMessage(e.data); + rpc.setSendMessage((mssg: any) => (self as any).postMessage(mssg)); +} else if (typeof process.send !== 'undefined') { + // Forked ChildProcess of node or electron. + // sendMessage callback returns void 0 because rpc process.send returns a boolean and rpc + // expecting void|Promise interprets truthy values as Promise which cause failure. + rpc.setSendMessage((data) => { process.send!(data); }); + process.on('message', (data: any) => rpc.receiveMessage(data)); + process.on('disconnect', () => { process.exit(0); }); +} else { + // Not a recognized environment, perhaps plain nodejs run independently of Grist, or tests + // running under mocha. For now, we only provide a disfunctional implementation. It allows + // plugins to call methods like registerFunction() without failing, so that plugin code may be + // imported, but the methods don't do anything useful. + rpc.setSendMessage((data) => {return; }); +} + +function createRpcLogger(): IRpcLogger { + let prefix: string; + if (typeof window !== 'undefined') { + prefix = `PLUGIN VIEW ${getPluginPath(window.location)}:`; + } else if (typeof process === 'undefined') { + prefix = `PLUGIN VIEW ${getPluginPath(self.location)}:`; + } else if (typeof process.send !== 'undefined') { + prefix = `PLUGIN NODE ${process.env.GRIST_PLUGIN_PATH || ""}:`; + } else { + return {}; + } + return { + info(msg: string) { console.log("%s %s", prefix, msg); }, + warn(msg: string) { console.warn("%s %s", prefix, msg); }, + }; +} diff --git a/app/plugin/tsconfig.json b/app/plugin/tsconfig.json new file mode 100644 index 00000000..185bec10 --- /dev/null +++ b/app/plugin/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "../../buildtools/tsconfig-base.json", +} diff --git a/app/server/declarations.d.ts b/app/server/declarations.d.ts new file mode 100644 index 00000000..4d92661c --- /dev/null +++ b/app/server/declarations.d.ts @@ -0,0 +1,102 @@ +declare module "app/server/lib/ActionLog"; +declare module "app/server/lib/sandboxUtil"; +declare module "app/server/lib/ServerMetrics"; +declare module "app/server/lib/User"; +declare module "app/server/serverMethods"; + +declare module "app/server/lib/Comm" { + import {Client, ClientMethod} from "app/server/lib/Client"; + import {LoginSession} from "app/server/lib/LoginSession"; + import * as http from "http"; + + class Comm { + constructor(server: http.Server, options: any); + public broadcastMessage(type: string, messageData: any): void; + public destroyAllClients(): void; + public setServerVersion(serverVersion: string|null): void; + public setServerActivation(active: boolean): void; + public getSessionIdFromCookie(gristSidCookie: string): string; + public getOrCreateSession(sessionId: string, req: any): LoginSession; + public registerMethods(methods: {[name: string]: ClientMethod}): void; + public getClient(clientId: string): Client; + public testServerShutdown(): Promise; + public testServerRestart(): Promise; + public testSetClientPersistence(ttlMs: number): void; + } + namespace Comm { + function sendDocMessage(client: Client, docFD: number, type: string, mesageData: any, fromSelf: boolean): void; + } + export = Comm; +} + +declare module "app/server/lib/shutdown" { + export function addCleanupHandler(context: T, method: (this: T) => void, timeout?: number, name?: string): void; + export function removeCleanupHandlers(context: T): void; + export function cleanupOnSignals(...signalNames: string[]): void; + export function exit(optExitCode?: number): void; +} + +// There is a @types/bluebird, but it's not great, and breaks for some of our usages. +declare module "bluebird"; + +// Redlock types refer to bluebird.Disposer. +declare module "bluebird" { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + class Disposer {} +} + +// TODO This is a module by Grist Labs; we should add index.d.ts to it. +declare module "@gristlabs/basket-api" { + interface Item { [colId: string]: any; } + interface ColValues { [colId: string]: any[]; } + interface AuthToken { [authProvider: string]: string; } + + class Basket { + public static addBasket(login: AuthToken): Promise; + public static getBaskets(login: AuthToken): Promise; + + public basketId: Readonly; + public apiKey: Readonly; + + constructor(basketId: string, apiKey?: string); + public addTable(optTableId: string): Promise; + public getTable(tableId: string): Promise; + public renameTable(oldTableId: string, newTableId: string): Promise; + public replaceTableData(tableId: string, columnValues: ColValues): Promise; + public deleteTable(tableId: string): Promise; + public getTables(): Promise; + public uploadAttachment(attachmentId: string, attachment: Buffer): Promise; + public delete(login: AuthToken): Promise; + } + namespace Basket {} + export = Basket; +} + +// Used in one place, and the typings are almost entirely unhelpful. +declare module "multiparty"; + +// Used in one place, for one call. +declare module "chokidar"; + +// Used in one place +declare module "mime-types"; + +// Used in one place +declare module "morgan"; +declare module "cookie"; +declare module "cookie-parser"; +declare module "@gristlabs/express-session"; + +// Used for command line path tweaks. +declare module "app-module-path" { + export function addPath(path: string): void; +} + +// Used in tests +declare module "ws"; + +// version of pidusage that has correct ctime on linux +declare module '@gristlabs/pidusage' { + import * as pidusage from 'pidusage'; + export = pidusage; +} diff --git a/app/server/devServerMain.ts b/app/server/devServerMain.ts new file mode 100644 index 00000000..52f17e44 --- /dev/null +++ b/app/server/devServerMain.ts @@ -0,0 +1,144 @@ +/** + * + * Run a home server, doc worker, and static server as a single process for regular + * development work. + * + * PORT -- this sets the main web server port (defaults to 8080) + * HOME_PORT -- this sets the main home server port (defaults to 9000) + * STATIC_PORT -- port for the static resource server (defaults to 9001) + * DOC_PORT -- comma separated ports for doc workers (defaults to 9002) + * TEST_CLEAN_DATABASE -- reset the database(s) before starting + * GRIST_SINGLE_PORT -- if set, just a single combined server on HOME_PORT + * DOC_WORKER_COUNT -- if set, makes sure there are at least this number of + * doc workers. Will add ports incrementally after the last + * worker added with DOC_PORT. + * + * If you run more than one doc worker, you'll need to have a redis server running + * and REDIS_URL set (e.g. to redis://localhost). + * + */ + +import {updateDb} from 'app/server/lib/dbUtils'; +import {FlexServer} from 'app/server/lib/FlexServer'; +import * as log from 'app/server/lib/log'; +import {main as mergedServerMain} from 'app/server/mergedServerMain'; +import {promisifyAll} from 'bluebird'; +import * as fse from 'fs-extra'; +import * as path from 'path'; +import {createClient, RedisClient} from 'redis'; + +promisifyAll(RedisClient.prototype); + +function getPort(envVarName: string, fallbackPort: number): number { + const val = process.env[envVarName]; + return val ? parseInt(val, 10) : fallbackPort; +} + +export async function main() { + log.info("=========================================================================="); + log.info("== devServer"); + log.info("devServer starting. Please do not set any ports in environment :-)"); + log.info("Server will be available at http://localhost:8080"); + + process.env.GRIST_HOSTED = "true"; + if (!process.env.GRIST_ADAPT_DOMAIN) { + process.env.GRIST_ADAPT_DOMAIN = "true"; + } + + // Experimental plugins are enabled by default for devs + if (!process.env.GRIST_EXPERIMENTAL_PLUGINS) { + process.env.GRIST_EXPERIMENTAL_PLUGINS = "1"; + } + + // For tests, it is useful to start with the database in a known state. + // If TEST_CLEAN_DATABASE is set, we reset the database before starting. + if (process.env.TEST_CLEAN_DATABASE) { + const {createInitialDb} = require('test/gen-server/seed'); + await createInitialDb(); + if (process.env.REDIS_URL) { + await createClient(process.env.REDIS_URL).flushdbAsync(); + } + } else { + await updateDb(); + } + + // In V1, we no longer create a config.json file automatically if it is missing. + // It is convenient to do that in the dev and test environment. + const appRoot = path.dirname(path.dirname(__dirname)); + const instDir = process.env.GRIST_INST_DIR || appRoot; + if (process.env.GRIST_INST_DIR) { + const fileName = path.join(instDir, 'config.json'); + if (!(await fse.pathExists(fileName))) { + const config = { + enableMetrics: false, + untrustedContentOrigin: 'notset', + }; + await fse.writeFile(fileName, JSON.stringify(config, null, 2)); + } + } + + if (process.env.GRIST_SINGLE_PORT) { + log.info("=========================================================================="); + log.info("== mergedServer"); + const port = getPort("HOME_PORT", 8080); + if (!process.env.APP_HOME_URL) { + process.env.APP_HOME_URL = `http://localhost:${port}`; + } + const server = await mergedServerMain(port, ["home", "docs", "static"]); + await server.addTestingHooks(); + return; + } + + const homeServerPort = getPort("HOME_PORT", 9000); + if (!process.env.APP_HOME_URL) { + process.env.APP_HOME_URL = `http://localhost:${homeServerPort}`; + } + + // Bring up the static resource server + log.info("=========================================================================="); + log.info("== staticServer"); + const staticPort = getPort("STATIC_PORT", 9001); + process.env.APP_STATIC_URL = `http://localhost:${staticPort}`; + await mergedServerMain(staticPort, ["static"]); + + // Bring up a home server + log.info("=========================================================================="); + log.info("== homeServer"); + const home = await mergedServerMain(homeServerPort, ["home"]); + + // If a distinct webServerPort is specified, we listen also on that port, though serving + // exactly the same content. This is handy for testing CORS issues. + const webServerPort = getPort("PORT", 8080); + if (webServerPort !== 0 && webServerPort !== homeServerPort) { + await home.startCopy('webServer', webServerPort); + } + + // Bring up the docWorker(s) + log.info("=========================================================================="); + log.info("== docWorker"); + const ports = (process.env.DOC_PORT || '9002').split(',').map(port => parseInt(port, 10)); + if (process.env.DOC_WORKER_COUNT) { + const n = parseInt(process.env.DOC_WORKER_COUNT, 10); + while (ports.length < n) { + ports.push(ports[ports.length - 1] + 1); + } + } + log.info(`== ports ${ports.join(',')}`); + if (ports.length > 1 && !process.env.REDIS_URL) { + throw new Error('Need REDIS_URL=redis://localhost or similar for multiple doc workers'); + } + const workers = new Array(); + for (const port of ports) { + workers.push(await mergedServerMain(port, ["docs"])); + } + + await home.addTestingHooks(workers); +} + + +if (require.main === module) { + main().catch((e) => { + log.error("devServer failed to start %s", e); + process.exit(1); + }); +} diff --git a/app/server/lib/ActionHistory.ts b/app/server/lib/ActionHistory.ts new file mode 100644 index 00000000..3af5467d --- /dev/null +++ b/app/server/lib/ActionHistory.ts @@ -0,0 +1,252 @@ +/** + * TODO For now, this is just a placeholder for an actual ActionHistory implementation that should + * replace today's ActionLog. It defines all the methods that are expected from it by Sharing.ts. + * + * In addition, it will need to support some methods to show action history to the user, which is + * the main purpose of ActionLog today. And it will need to allow querying a subset of history (at + * least by table or record). + * + * The main difference with today's ActionLog is that it needs to mark actions either with labels, + * or more likely with Git-like branches, so that we can distinguish shared, local-sent, and + * local-unsent actions. And it needs to work on LocalActionBundles, which include more + * information than what ActionLog stores. On the other hand, it can probably store actions as + * blobs, which can simplify the database storage. + */ + +import {LocalActionBundle} from 'app/common/ActionBundle'; +import {ActionGroup} from 'app/common/ActionGroup'; +import {createEmptyActionSummary} from 'app/common/ActionSummary'; +import {getSelectionDesc, UserAction} from 'app/common/DocActions'; +import {DocState} from 'app/common/UserAPI'; +import toPairs = require('lodash/toPairs'); +import {summarizeAction} from './ActionSummary'; + +export abstract class ActionHistory { + /** + * Initialize the ActionLog by reading the database. No other methods may be used until the + * initialization completes. If used, their behavior is undefined. + */ + public abstract initialize(): Promise; + + public abstract isInitialized(): boolean; + + /** Returns the actionNum of the next action we expect from the hub. */ + public abstract getNextHubActionNum(): number; + + /** Returns the actionNum of the next local action should have. */ + public abstract getNextLocalActionNum(): number; + + /** + * Act as if we have already seen actionNum. getNextHubActionNum will return 1 plus this. + * Only suitable for use if there are no unshared local actions. + */ + public abstract skipActionNum(actionNum: number): Promise; + + /** Returns whether we have local unsent actions. */ + public abstract haveLocalUnsent(): boolean; + + /** Returns whether we have any local actions that have been sent to the hub. */ + public abstract haveLocalSent(): boolean; + + /** Returns whether we have any locally-applied actions. */ + public abstract haveLocalActions(): boolean; + + /** Fetches and returns an array of all local unsent actions. */ + public abstract fetchAllLocalUnsent(): Promise; + + /** Fetches and returns an array of all local actions (sent and unsent). */ + public abstract fetchAllLocal(): Promise; + + /** Deletes all local-only actions, and resets the affected branch pointers. */ + // TODO Should we actually delete, or be more git-like, only reset local branch pointer, and let + // cleanup of unreferenced actions happen in a separate step? + public abstract clearLocalActions(): Promise; + + /** + * Marks all actions returned from fetchAllLocalUnsent() as sent. Actions must be consecutive + * starting with the the first local unsent action. + */ + public abstract markAsSent(actions: LocalActionBundle[]): Promise; + + /** + * Matches the action from the hub against the first sent local action. If it's the same action, + * marks our action as "shared", i.e. accepted by the hub, and returns true. Else returns false. + * If actionHash is null, accepts unconditionally. + */ + public abstract acceptNextSharedAction(actionHash: string|null): Promise; + + /** Records a new local unsent action, after setting action.actionNum appropriately. */ + public abstract recordNextLocalUnsent(action: LocalActionBundle): Promise; + + /** Records a new action received from the hub, after setting action.actionNum appropriately. */ + public abstract recordNextShared(action: LocalActionBundle): Promise; + + /** + * Get the most recent actions from the history. Results are ordered by + * earliest actions first, later actions later. If `maxActions` is supplied, + * at most that number of actions are returned. + */ + public abstract getRecentActions(maxActions?: number): Promise; + + /** + * Get the most recent states from the history. States are just + * actions without any content. Results are ordered by most recent + * states first (careful, this is the opposite to getRecentActions). + * If `maxStates` is supplied, at most that number of actions are + * returned. + */ + public abstract getRecentStates(maxStates?: number): Promise; + + /** + * Get a list of actions, identified by their actionNum. Any actions that could not be + * found are returned as undefined. + */ + public abstract getActions(actionNums: number[]): Promise>; + + /** + * Associates an action with a client. This association is expected to be transient, rather + * than persistent. It should survive a client-side reload but not a server-side restart. + */ + public abstract setActionClientId(actionHash: string, clientId: string): void; + + /** Check for any client associated with an action, identified by checksum */ + public abstract getActionClientId(actionHash: string): string | undefined; +} + + +/** + * Old helper to display the actionGroup in a human-readable way. Being maintained + * to avoid having to change too much at once. + */ +export function humanDescription(actions: UserAction[]): string { + const action = actions[0]; + if (!action) { return ""; } + let output = ''; + // Common names for various action parameters + const name = action[0]; + const table = action[1]; + const rows = action[2]; + const colId = action[2]; + const columns: any = action[3]; // TODO - better typing - but code may evaporate + switch (name) { + case 'UpdateRecord': + case 'BulkUpdateRecord': + case 'AddRecord': + case 'BulkAddRecord': + output = name + ' ' + getSelectionDesc(action, columns); + break; + case 'ApplyUndoActions': + // Currently cannot display information about what action was undone, as the action comes + // with the description of the "undo" message, which might be very different + // Also, cannot currently properly log redos as they are not distinguished from others in any way + // TODO: make an ApplyRedoActions type for redoing actions + output = 'Undo Previous Action'; + break; + case 'InitNewDoc': + output = 'Initialized new Document'; + break; + case 'AddColumn': + output = 'Added column ' + colId + ' to ' + table; + break; + case 'RemoveColumn': + output = 'Removed column ' + colId + ' from ' + table; + break; + case 'RemoveRecord': + case 'BulkRemoveRecord': + output = 'Removed record(s) ' + rows + ' from ' + table; + break; + case 'EvalCode': + output = 'Evaluated Code ' + action[1]; + break; + case 'AddTable': + output = 'Added table ' + table; + break; + case 'RemoveTable': + output = 'Removed table ' + table; + break; + case 'ModifyColumn': + // TODO: The Action Log currently only logs user actions, + // But ModifyColumn/Rename Column are almost always triggered from the client + // through a meta-table UpdateRecord. + // so, this is a case where making use of explicit sandbox engine 'looged' actions + // may be useful + output = 'Modify column ' + colId + ", "; + for (const [col, val] of toPairs(columns)) { + output += col + ": " + val + ", "; + } + output += ' in table ' + table; + break; + case 'RenameColumn': { + const newColId = action[3]; + output = 'Renamed Column ' + colId + ' to ' + newColId + ' in ' + table; + break; + } + default: + output = name + ' [No Description]'; + } + // A period for good grammar + output += '.'; + return output; +} + +/** + * Convert an ActionBundle into an ActionGroup. ActionGroups are the representation of + * actions on the client. + * @param history: interface to action history + * @param act: action to convert + * @param options.summarize: if set, inspect the action in detail in order to include a summary of + * changes made within the action. Otherwise, the actionSummary returned is empty. + * @param options.client: the client for which the action group is being prepared, if known. + * @param options.retValues: values returned by the action, if known. + */ +export function asActionGroup(history: ActionHistory, + act: LocalActionBundle, + options: { + summarize?: boolean + client?: {clientId: string}|null, + retValues?: any[], + }): ActionGroup { + const {summarize, client, retValues} = options; + const info = act.info[1]; + const fromSelf = (client && client.clientId && act.actionHash) ? + (history.getActionClientId(act.actionHash) === client.clientId) : false; + + let rowIdHint = 0; + if (retValues) { + // A hint for cursor position. This logic used to live on the client, but now trying to + // limit how much the client looks at the internals of userActions. + // In case of AddRecord, the returned value is rowId, which is the best cursorPos for Redo. + for (let i = 0; i < act.userActions.length; i++) { + const name = act.userActions[i][0]; + const retValue = retValues[i]; + if (name === 'AddRecord') { + rowIdHint = retValue; + break; + } else if (name === 'BulkAddRecord') { + rowIdHint = retValue[0]; + break; + } + } + } + + const primaryAction: string = String((act.userActions[0] || [""])[0]); + const isUndo = primaryAction === 'ApplyUndoActions'; + + return { + actionNum: act.actionNum, + actionHash: act.actionHash || "", + desc: info.desc || humanDescription(act.userActions), + actionSummary: summarize ? summarizeAction(act) : createEmptyActionSummary(), + fromSelf, + linkId: info.linkId, + otherId: info.otherId, + time: info.time, + user: info.user, + rowIdHint, + primaryAction, + isUndo, + internal: act.actionNum === 0 // Mark lazy-loading calculated columns. In future, + // synchronizing fields to today's date and other + // changes from external values may count as internal. + }; +} diff --git a/app/server/lib/ActionHistoryImpl.ts b/app/server/lib/ActionHistoryImpl.ts new file mode 100644 index 00000000..d4ddb365 --- /dev/null +++ b/app/server/lib/ActionHistoryImpl.ts @@ -0,0 +1,673 @@ +/** + * Minimal ActionHistory implementation + */ +import {LocalActionBundle} from 'app/common/ActionBundle'; +import * as marshaller from 'app/common/marshal'; +import {DocState} from 'app/common/UserAPI'; +import * as crypto from 'crypto'; +import keyBy = require('lodash/keyBy'); +import mapValues = require('lodash/mapValues'); +import {ActionHistory} from './ActionHistory'; +import {ISQLiteDB, ResultRow} from './SQLiteDB'; + +// History will from time to time be pruned back to within these limits +// on rows and the maximum total number of bytes in the "body" column. +// Pruning is done when the history has grown above these limits, to +// the specified factor. +const ACTION_HISTORY_MAX_ROWS = 1000; +const ACTION_HISTORY_MAX_BYTES = 1000 * 1000 * 1000; // 1 GB. +const ACTION_HISTORY_GRACE_FACTOR = 1.25; // allow growth to 1250 rows / 1.25 GB. +const ACTION_HISTORY_CHECK_PERIOD = 10; // number of actions between size checks. + +/** + * + * Encode an action as a buffer. + * + */ +export function encodeAction(action: LocalActionBundle): Buffer { + const encoder = new marshaller.Marshaller({version: 2}); + encoder.marshal(action); + return encoder.dumpAsBuffer(); +} + +/** + * + * Decode an action from a buffer. Throws an error if buffer doesn't look plausible. + * + */ +export function decodeAction(blob: Buffer | Uint8Array): LocalActionBundle { + return marshaller.loads(blob) as LocalActionBundle; +} + + +/** + * + * Decode an action from an ActionHistory row. Row must include body, actionNum, actionHash fields. + * + */ +function decodeActionFromRow(row: ResultRow): LocalActionBundle { + const body = decodeAction(row.body); + // Reset actionNum and actionHash, just to have one fewer thing to worry about. + body.actionNum = row.actionNum; + body.actionHash = row.actionHash; + return body; + } + +/** + * + * Generate an action checksum from a LocalActionBundle + * Needs to be in sync with Hub/Sharing. + * + */ +export function computeActionHash(action: LocalActionBundle): string { + const shaSum = crypto.createHash('sha256'); + const encoder = new marshaller.Marshaller({version: 2}); + encoder.marshal(action.actionNum); + encoder.marshal(action.parentActionHash); + encoder.marshal(action.info); + encoder.marshal(action.stored); + const buf = encoder.dumpAsBuffer(); + shaSum.update(buf); + return shaSum.digest('hex'); +} + + +/** The important identifiers associated with an action */ +interface ActionIdentifiers { + /** + * + * actionRef is the SQLite-allocated row id in the main ActionHistory table. + * See: + * https://www.sqlite.org/rowidtable.html + * https://sqlite.org/autoinc.html + * for background on how this works. + * + */ + actionRef: number|null; + + /** + * + * actionHash is a checksum computed from salient parts of an ActionBundle. + * + */ + actionHash: string|null; + + /** + * + * actionNum is the depth in history from the root, starting from 1 for the first + * action. + * + */ + actionNum: number|null; + + /** + * + * The name of a branch where we found this action. + * + */ + branchName: string; +} + +/** An organized view of the standard branches: shared, local_sent, local_unsent */ +interface StandardBranches { + shared: ActionIdentifiers; + local_sent: ActionIdentifiers; + local_unsent: ActionIdentifiers; +} + +/** Tweakable parameters for storing the action history */ +interface ActionHistoryOptions { + maxRows: number; // maximum number of rows to aim for + maxBytes: number; // maximum total "body" bytes to aim for + graceFactor: number; // allow this amount of slop in limits + checkPeriod: number; // number of actions between checks +} + +const defaultOptions: ActionHistoryOptions = { + maxRows: ACTION_HISTORY_MAX_ROWS, + maxBytes: ACTION_HISTORY_MAX_BYTES, + graceFactor: ACTION_HISTORY_GRACE_FACTOR, + checkPeriod: ACTION_HISTORY_CHECK_PERIOD, +}; + +/** + * + * An implementation of the ActionHistory interface, using SQLite tables. + * + * The history of Grist actions is essentially linear. We have a notion of + * action branches only to track certain "subhistories" of those actions, + * specifically: + * - those actions that have been "shared" + * - those actions that have been "sent" (but not yet declared "shared") + * The "shared" branch reaches from the beginning of history to the last known + * shared action. The "local_sent" branch reaches at least to that point, and + * potentially on to other actions that have been "sent" but not "shared". + * All remaining branches -- just one right now, called "local_unsent" -- + * continue on from there. We may in the future permit multiple such + * branches. In this case, this part of the action history could actually + * form a tree and not be linear. + * + * For all branches, we track their "tip", the most recent action on + * that branch. + * + * TODO: links to parent actions stored in bundles are not currently + * updated in the database when those parent actions are deleted. If this + * is an issue, it might be best to remove such information from the bundles + * when stored and add it back as it is retrieved, or treat it separately. + * + */ +export class ActionHistoryImpl implements ActionHistory { + + private _sharedActionNum: number = 1; // track depth in tree of shared actions + private _localActionNum: number = 1; // track depth in tree of local actions + private _haveLocalSent: boolean = false; // cache for this.haveLocalSent() + private _haveLocalUnsent: boolean = false; // cache for this.haveLocalUnsent() + private _initialized: boolean = false; // true when initialize() has completed + private _actionClient = new Map(); // transient cache of who created actions + + constructor(private _db: ISQLiteDB, private _options: ActionHistoryOptions = defaultOptions) { + } + + /** remove any existing data from ActionHistory - useful during testing. */ + public async wipe() { + await this._db.run("UPDATE _gristsys_ActionHistoryBranch SET actionRef = NULL"); + await this._db.run("DELETE FROM _gristsys_ActionHistory"); + this._actionClient.clear(); + } + + public async initialize(): Promise { + const branches = await this._getBranches(); + if (branches.shared.actionNum) { + this._sharedActionNum = branches.shared.actionNum + 1; + } + if (branches.local_unsent.actionNum) { + this._localActionNum = branches.local_unsent.actionNum + 1; + } + // Record whether we currently have local actions (sent or unsent). + const sharedActionNum = branches.shared.actionNum || -1; + const localSentActionNum = branches.local_sent.actionNum || -1; + const localUnsentActionNum = branches.local_unsent.actionNum || -1; + this._haveLocalUnsent = localUnsentActionNum > localSentActionNum; + this._haveLocalSent = localSentActionNum > sharedActionNum; + this._initialized = true; + // Apply any limits on action history size. + await this._pruneLargeHistory(sharedActionNum); + } + + public isInitialized(): boolean { + return this._initialized; + } + + public getNextHubActionNum(): number { + return this._sharedActionNum; + } + + public getNextLocalActionNum(): number { + return this._localActionNum; + } + + public async skipActionNum(actionNum: number): Promise { + if (this._localActionNum !== this._sharedActionNum) { + throw new Error("Tried to skip to an actionNum with unshared local actions"); + } + + if (actionNum < this._sharedActionNum) { + if (actionNum === this._sharedActionNum - 1) { + // that was easy + return; + } + throw new Error("Tried to skip to an actionNum we've already passed"); + } + + // Force the actionNum to the desired value + this._localActionNum = this._sharedActionNum = actionNum; + + // We store a row as we would for recordNextShared() + const action: LocalActionBundle = { + actionHash: null, + parentActionHash: null, + actionNum: this._sharedActionNum, + userActions: [], + undo: [], + envelopes: [], + info: [0, {time: 0, user: "grist", inst: "", desc: "root", otherId: 0, linkId: 0}], + stored: [], + calc: [] + }; + await this._db.execTransaction(async () => { + const branches = await this._getBranches(); + if (branches.shared.actionRef !== branches.local_sent.actionRef || + branches.shared.actionRef !== branches.local_unsent.actionRef) { + throw new Error("skipActionNum not defined when branches not in sync"); + } + const actionRef = await this._addAction(action, branches.shared); + this._noteSharedAction(action.actionNum); + await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ? + WHERE name IN ('local_unsent', 'local_sent')`, + actionRef); + }); + } + + public haveLocalUnsent(): boolean { + return this._haveLocalUnsent; + } + + public haveLocalSent(): boolean { + return this._haveLocalSent; + } + + public haveLocalActions(): boolean { + return this._haveLocalSent || this._haveLocalUnsent; + } + + public async fetchAllLocalUnsent(): Promise { + const branches = await this._getBranches(); + return this._fetchActions(branches.local_sent, branches.local_unsent); + } + + public async fetchAllLocal(): Promise { + const branches = await this._getBranches(); + return this._fetchActions(branches.shared, branches.local_unsent); + } + + public async clearLocalActions(): Promise { + await this._db.execTransaction(async () => { + const branches = await this._getBranches(); + const rows = await this._fetchParts(branches.shared, branches.local_unsent, + "_gristsys_ActionHistory.id, actionHash"); + await this._deleteRows(rows); + await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ? + WHERE name IN ('local_unsent', 'local_sent')`, + branches.shared.actionRef); + this._haveLocalSent = false; + this._haveLocalUnsent = false; + this._localActionNum = this._sharedActionNum; + }); + } + + public async markAsSent(actions: LocalActionBundle[]): Promise { + const branches = await this._getBranches(); + const candidates = await this._fetchParts(branches.local_sent, + branches.local_unsent, + "_gristsys_ActionHistory.id, actionHash"); + let tip: number|undefined; + try { + for (const act of actions) { + if (candidates.length === 0) { + throw new Error("markAsSent() called but nothing local and unsent"); + } + const candidate = candidates[0]; + // act and act2 must be one and the same + if (act.actionHash !== candidate.actionHash) { + throw new Error("markAsSent() got an unexpected action"); + } + tip = candidate.id; + candidates.shift(); + if (candidates.length === 0) { + this._haveLocalUnsent = false; + } + this._haveLocalSent = true; + } + } finally { + if (tip) { + await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ? + WHERE name = "local_sent"`, + tip); + } + } + } + + public async acceptNextSharedAction(actionHash: string|null): Promise { + const branches = await this._getBranches(); + const candidates = await this._fetchParts(branches.shared, + branches.local_sent, + "_gristsys_ActionHistory.id, actionHash, actionNum", + 2); + if (candidates.length === 0) { + return false; + } + const candidate = candidates[0]; + if (actionHash != null) { + if (candidate.actionHash !== actionHash) { + return false; + } + } + await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ? + WHERE name = "shared"`, + candidate.id); + if (candidates.length === 1) { + this._haveLocalSent = false; + } + this._noteSharedAction(candidate.actionNum); + await this._pruneLargeHistory(candidate.actionNum); + return true; + } + + /** This will populate action.actionHash and action.parentActionHash */ + public async recordNextLocalUnsent(action: LocalActionBundle): Promise { + const branches = await this._getBranches(); + await this._addAction(action, branches.local_unsent); + this._noteLocalAction(action.actionNum); + this._haveLocalUnsent = true; + } + + public async recordNextShared(action: LocalActionBundle): Promise { + // I think, reading Sharing.ts, that these actions should be added to all + // the system branches - it is just a shortcut for getting to shared + await this._db.execTransaction(async () => { + const branches = await this._getBranches(); + if (branches.shared.actionRef !== branches.local_sent.actionRef || + branches.shared.actionRef !== branches.local_unsent.actionRef) { + throw new Error("recordNextShared not defined when branches not in sync"); + } + const actionRef = await this._addAction(action, branches.shared); + this._noteSharedAction(action.actionNum); + await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ? + WHERE name IN ('local_unsent', 'local_sent')`, + actionRef); + }); + await this._pruneLargeHistory(action.actionNum); + } + + public async getRecentActions(maxActions?: number): Promise { + const branches = await this._getBranches(); + const actions = await this._fetchParts(null, + branches.local_unsent, + "_gristsys_ActionHistory.id, actionNum, actionHash, body", + maxActions, + true); + const result = actions.map(decodeActionFromRow); + result.reverse(); // Implementation note: this could be optimized away when `maxActions` + // is not specified, by simply asking _fetchParts for ascending order. + return result; + } + + public async getRecentStates(maxStates?: number): Promise { + const branches = await this._getBranches(); + const states = await this._fetchParts(null, + branches.local_unsent, + "_gristsys_ActionHistory.id, actionNum, actionHash", + maxStates, + true); + return states.map(row => ({n: row.actionNum, h: row.actionHash})); + } + + public async getActions(actionNums: number[]): Promise> { + const actions = await this._db.all(`SELECT actionHash, actionNum, body FROM _gristsys_ActionHistory + where actionNum in (${actionNums.map(x => '?').join(',')})`, + actionNums); + const actionsByActionNum = keyBy(actions, 'actionNum'); + return actionNums + .map(n => actionsByActionNum[n]) + .map((row) => row ? decodeActionFromRow(row) : undefined); + } + + /** + * Helper function to remove all stored actions except the last keepN and run the VACUUM command + * to reduce the size of the SQLite file. + * + * @param {Int} keepN - The number of most recent actions to keep. The value must be at least 1, and + * will default to 1 if not given. + * @returns {Promise} - A promise for the SQL execution. + * + * NOTE: Only keeps actions after maxActionNum - keepN, which might be less than keepN actions if + * actions are not sequential in the file. + */ + public async deleteActions(keepN: number): Promise { + await this._db.execTransaction(async () => { + const branches = await this._getBranches(); + const rows = await this._fetchParts(null, + branches.local_unsent, + "_gristsys_ActionHistory.id, actionHash", + keepN, + true); + const ids = await this._deleteRows(rows, true); + // By construction, we are removing all rows from the start of history to a certain point. + // So, if any of the removed actions are mentioned as the tip of a branch, that tip should + // now simply become null/empty. + await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = NULL WHERE actionRef NOT IN (${ids})`); + await this._db.requestVacuum(); + }); + } + + public setActionClientId(actionHash: string, clientId: string): void { + this._actionClient.set(actionHash, clientId); + } + + public getActionClientId(actionHash: string): string | undefined { + return this._actionClient.get(actionHash); + } + + /** Check if we need to update the next shared actionNum */ + private _noteSharedAction(actionNum: number): void { + if (actionNum >= this._sharedActionNum) { + this._sharedActionNum = actionNum + 1; + } + this._noteLocalAction(actionNum); + } + + /** Check if we need to update the next local actionNum */ + private _noteLocalAction(actionNum: number): void { + if (actionNum >= this._localActionNum) { + this._localActionNum = actionNum + 1; + } + } + + /** Append an action to a branch. */ + private async _addAction(action: LocalActionBundle, + branch: ActionIdentifiers): Promise { + action.parentActionHash = branch.actionHash; + if (!action.actionHash) { + action.actionHash = computeActionHash(action); + } + const buf = encodeAction(action); + return this._db.execTransaction(async () => { + // Add the action. We let SQLite fill in the "id" column, which is an alias for + // the SQLite rowid in this case: https://www.sqlite.org/rowidtable.html + const id = await this._db.runAndGetId(`INSERT INTO _gristsys_ActionHistory + (actionHash, parentRef, actionNum, body) + VALUES (?, ?, ?, ?)`, + action.actionHash, + branch.actionRef, + action.actionNum, + buf); + await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ? + WHERE name = ?`, + id, branch.branchName); + return id; + }); + } + + /** Get the current status of the standard branches: shared, local_sent, and local_unsent */ + private async _getBranches(): Promise { + const rows = await this._db.all(`SELECT name, actionNum, actionHash, Branch.actionRef + FROM _gristsys_ActionHistoryBranch as Branch + LEFT JOIN _gristsys_ActionHistory as History + ON History.id = Branch.actionRef + WHERE name in ("shared", "local_sent", "local_unsent")`); + const bits = mapValues(keyBy(rows, 'name'), this._asActionIdentifiers); + const missing = { actionHash: null, actionRef: null, actionNum: null } as ActionIdentifiers; + return { + shared: bits.shared || missing, + local_sent: bits.local_sent || missing, + local_unsent: bits.local_unsent || missing + }; + } + + /** Cast an sqlite result row into a structure with the IDs we care about */ + private _asActionIdentifiers(row: ResultRow|null): ActionIdentifiers|null { + if (!row) { + return null; + } + return { + actionRef: row.actionRef, + actionHash: row.actionHash, + actionNum: row.actionNum, + branchName: row.name + }; + } + + /** + * + * Fetch selected parts of a range of actions. We do a recursive query + * working backwards from the action identified by `end`, following a + * chain of ancestors via `parentRef` links, until we reach the action + * identified by `start` or run out of ancestors. The action identified + * by `start` is NOT included in the results. Results are returned in + * ascending order of `actionNum` - in other words results closer to the + * beginning of history are returned first. + * + * @param start - identifiers of an action not to include in the results. + * @param end - identifiers of an action to include in the results + * @param selection - SQLite SELECT result-columns to return + * @param limit - optional cap on the number of results to return. + * @param desc - optional - if true, invert order of results, starting + * from highest `actionNum` rather than lowest. + * + * @return a list of ResultRows, containing whatever was requested in + * the `selection` parameter for each action found. + * + */ + private async _fetchParts(start: ActionIdentifiers|null, + end: ActionIdentifiers|null, + selection: string, + limit?: number, + desc?: boolean): Promise { + if (!end) { return []; } + + // Collect all actions, Starting at the branch tip, and working + // backwards until we hit a delimiting actionNum. + // See https://sqlite.org/lang_with.html for details of recursive CTEs. + const rows = await this._db.all(`WITH RECURSIVE + actions(id) AS ( + VALUES(?) + UNION ALL + SELECT parentRef FROM _gristsys_ActionHistory, actions + WHERE _gristsys_ActionHistory.id = actions.id + AND parentRef IS NOT NULL + AND _gristsys_ActionHistory.id IS NOT ?) + SELECT ${selection} from actions + JOIN _gristsys_ActionHistory + ON actions.id = _gristsys_ActionHistory.id + WHERE _gristsys_ActionHistory.id IS NOT ? + ORDER BY actionNum ${desc ? "DESC " : ""} + ${limit ? ("LIMIT " + limit) : ""}`, + end.actionRef, + start ? start.actionRef : null, + start ? start.actionRef : null); + return rows; + } + + /** + * + * Fetch a range of actions as LocalActionBundles. We do a recursive query + * working backwards from the action identified by `end`, following a + * chain of ancestors via `parentRef` links, until we reach the action + * identified by `start` or run out of ancestors. The action identified + * by `start` is NOT included in the results. Results are returned in + * ascending order of `actionNum` - in other words results closer to the + * beginning of history are returned first. + * + * @param start - identifiers of an action not to include in the results. + * @param end - identifiers of an action to include in the results + * + * @return a list of LocalActionBundles. + * + */ + private async _fetchActions(start: ActionIdentifiers|null, + end: ActionIdentifiers|null): Promise { + const rows = await this._fetchParts(start, end, "body, actionNum, actionHash"); + return rows.map(decodeActionFromRow); + } + + /** + * Delete rows in the ActionHistory. Any client id association is also removed for + * the given rows. Branch information is not updated, it is the responsibility of + * the caller to keep that synchronized. + * + * @param rows: The rows to delete. Should have at least id and actionHash fields. + * @param invert: True if all but the listed rows should be deleted. + * + * Returns the list of ids of the supplied rows. + */ + private async _deleteRows(rows: ResultRow[], invert?: boolean): Promise { + // There's no great solution for passing a long list of numbers to sqlite for a + // single query. Here, we concatenate them with comma separators and embed them + // in the SQL string. + // TODO: deal with limit on max length of sql statement https://www.sqlite.org/limits.html + const ids = rows.map(row => row.id); + const idList = ids.join(','); + await this._db.run(`DELETE FROM _gristsys_ActionHistory + WHERE id ${invert ? 'NOT' : ''} IN (${idList})`); + for (const row of rows) { + this._actionClient.delete(row.actionHash); + } + return ids; + } + + /** + * Deletes rows in the ActionHistory if there are too many of them or they hold too + * much data. + */ + private async _pruneLargeHistory(actionNum: number): Promise { + // We check history size occasionally, not on every single action. The check + // requires summing a blob length over up to roughly ACTION_HISTORY_MAX_ROWS rows. + // For a 2GB test db with 3 times this number of rows, the check takes < 10 ms. + // But there's no need to add that tax to every action. + if (actionNum % this._options.checkPeriod !== 0) { + return; + } + // Do a quick check on the history size. We work on the "shared" branch, to + // avoid the possibility of deleting history that has not yet been shared. + let branches = await this._getBranches(); + const checks = (await this._fetchParts(null, + branches.shared, + "count(*) as count, sum(length(body)) as bytes", + undefined, + true))[0]; + if (checks.count <= this._options.maxRows * this._options.graceFactor && + checks.bytes <= this._options.maxBytes * this._options.graceFactor) { + return; // Nothing to do, size is ok. + } + // Too big! Check carefully what needs to be done. + await this._db.execTransaction(async () => { + // Make sure branches are up to date within this transaction. + branches = await this._getBranches(); + const rows = await this._fetchParts(null, + branches.shared, + "_gristsys_ActionHistory.id, actionHash, actionNum, length(body) as bytes", + undefined, + true); + // Scan to find the first row that pushes us over a limit. + let count: number = 0; + let bytes: number = 0; + let first: number = -1; + for (let i = 0; i < rows.length; i++) { + const row = rows[i]; + count++; + bytes += row.bytes; + if (count > 1 && (bytes > this._options.maxBytes || count > this._options.maxRows)) { + first = i; + break; + } + } + if (first === -1) { return; } + // Delete remaining rows - in batches because _deleteRows has limited capacity. + const batchLength: number = 100; + for (let i = first; i < rows.length; i += batchLength) { + const batch = rows.slice(i, i + batchLength); + const ids = await this._deleteRows(batch); + // We are removing all rows from the start of history to a certain point. + // So, if any of the removed actions are mentioned as the tip of a branch, + // that tip should now simply become null/empty. + await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = NULL WHERE actionRef IN (${ids})`); + } + // At this point, to recover the maximum memory, we could VACUUM the document. + // But vacuuming is an unacceptably slow operation for large documents (e.g. + // 30 secs for a 2GB doc) so it is obnoxious to do that while the user is waiting. + // Without vacuuming, the document will grow due to fragmentation, but this should + // be at a lower rate than it would grow if we were simply retaining full history. + // TODO: occasionally VACUUM large documents while they are not being used. + }); + } +} diff --git a/app/server/lib/ActionSummary.ts b/app/server/lib/ActionSummary.ts new file mode 100644 index 00000000..0fde4603 --- /dev/null +++ b/app/server/lib/ActionSummary.ts @@ -0,0 +1,434 @@ +import {getEnvContent, LocalActionBundle} from 'app/common/ActionBundle'; +import {ActionSummary, ColumnDelta, createEmptyActionSummary, + createEmptyTableDelta, defunctTableName, LabelDelta, TableDelta} from 'app/common/ActionSummary'; +import {DocAction} from 'app/common/DocActions'; +import * as Action from 'app/common/DocActions'; +import {arrayExtend} from 'app/common/gutil'; +import {CellDelta} from 'app/common/TabularDiff'; +import fromPairs = require('lodash/fromPairs'); +import keyBy = require('lodash/keyBy'); +import sortBy = require('lodash/sortBy'); +import toPairs = require('lodash/toPairs'); +import values = require('lodash/values'); + +/** + * The maximum number of rows in a single bulk change that will be recorded + * individually. Bulk changes that touch more than this number of rows + * will be summarized only by the number of rows touched. + */ +const MAXIMUM_INLINE_ROWS = 10; + +/** helper function to access summary changes for a specific table by name */ +function _forTable(summary: ActionSummary, tableId: string): TableDelta { + return summary.tableDeltas[tableId] || (summary.tableDeltas[tableId] = createEmptyTableDelta()); +} + +/** helper function to access summary changes for a specific cell by rowId and colId */ +function _forCell(td: TableDelta, rowId: number, colId: string): CellDelta { + const cd = td.columnDeltas[colId] || (td.columnDeltas[colId] = {}); + return cd[rowId] || (cd[rowId] = [null, null]); +} + +/** + * helper function to store detailed cell changes for a single row. + * Direction parameter is 0 if values are prior values of cells, 1 if values are new values. + */ +function _addRow(td: TableDelta, rowId: number, colValues: Action.ColValues, + direction: 0|1) { + for (const [colId, colChanges] of toPairs(colValues)) { + const cell = _forCell(td, rowId, colId); + cell[direction] = [colChanges]; + } +} + +/** helper function to store detailed cell changes for a set of rows */ +function _addRows(tableId: string, td: TableDelta, rowIds: number[], + colValues: Action.BulkColValues, direction: 0|1) { + let rows: Array<[number, number]>; + if (rowIds.length <= MAXIMUM_INLINE_ROWS || tableId.startsWith("_grist_")) { + rows = [...rowIds.entries()]; + } else { + // if many rows, just take some from start and one from end as examples + rows = [...rowIds.slice(0, MAXIMUM_INLINE_ROWS - 1).entries()]; + rows.push([rowIds.length - 1, rowIds[rowIds.length - 1]]); + } + + for (const [colId, colChanges] of toPairs(colValues)) { + rows.forEach(([idx, rowId]) => { + const cell = _forCell(td, rowId, colId); + cell[direction] = [colChanges[idx]]; + }); + } +} + +/** add a rename to a list, avoiding duplicates */ +function _addRename(renames: LabelDelta[], rename: LabelDelta) { + if (renames.find(r => r[0] === rename[0] && r[1] === rename[1])) { return; } + renames.push(rename); +} + +/** add information about an action based on the forward direction */ +function _addForwardAction(summary: ActionSummary, act: DocAction) { + const tableId = act[1]; + if (Action.isAddTable(act)) { + summary.tableRenames.push([null, tableId]); + for (const info of act[2]) { + _forTable(summary, tableId).columnRenames.push([null, info.id]); + } + } else if (Action.isRenameTable(act)) { + _addRename(summary.tableRenames, [tableId, act[2]]); + } else if (Action.isRenameColumn(act)) { + _addRename(_forTable(summary, tableId).columnRenames, [act[2], act[3]]); + } else if (Action.isAddColumn(act)) { + _forTable(summary, tableId).columnRenames.push([null, act[2]]); + } else if (Action.isRemoveColumn(act)) { + _forTable(summary, tableId).columnRenames.push([act[2], null]); + } else if (Action.isAddRecord(act)) { + const td = _forTable(summary, tableId); + td.addRows.push(act[2]); + _addRow(td, act[2], act[3], 1); + } else if (Action.isUpdateRecord(act)) { + const td = _forTable(summary, tableId); + td.updateRows.push(act[2]); + _addRow(td, act[2], act[3], 1); + } else if (Action.isBulkAddRecord(act)) { + const td = _forTable(summary, tableId); + arrayExtend(td.addRows, act[2]); + _addRows(tableId, td, act[2], act[3], 1); + } else if (Action.isBulkUpdateRecord(act)) { + const td = _forTable(summary, tableId); + arrayExtend(td.updateRows, act[2]); + _addRows(tableId, td, act[2], act[3], 1); + } else if (Action.isReplaceTableData(act)) { + const td = _forTable(summary, tableId); + arrayExtend(td.addRows, act[2]); + _addRows(tableId, td, act[2], act[3], 1); + } +} + +/** add information about an action based on undo information */ +function _addReverseAction(summary: ActionSummary, act: DocAction) { + const tableId = act[1]; + if (Action.isAddTable(act)) { // undoing, so this is a table removal + summary.tableRenames.push([tableId, null]); + for (const info of act[2]) { + _forTable(summary, tableId).columnRenames.push([info.id, null]); + } + } else if (Action.isAddRecord(act)) { // undoing, so this is a record removal + const td = _forTable(summary, tableId); + td.removeRows.push(act[2]); + _addRow(td, act[2], act[3], 0); + } else if (Action.isUpdateRecord(act)) { // undoing, so this is reversal of a record update + const td = _forTable(summary, tableId); + _addRow(td, act[2], act[3], 0); + } else if (Action.isBulkAddRecord(act)) { // undoing, this may be reversing a table delete + const td = _forTable(summary, tableId); + arrayExtend(td.removeRows, act[2]); + _addRows(tableId, td, act[2], act[3], 0); + } else if (Action.isBulkUpdateRecord(act)) { // undoing, so this is reversal of a bulk record update + const td = _forTable(summary, tableId); + arrayExtend(td.updateRows, act[2]); + _addRows(tableId, td, act[2], act[3], 0); + } else if (Action.isRenameTable(act)) { // undoing - sometimes renames only in undo info + _addRename(summary.tableRenames, [act[2], tableId]); + } else if (Action.isRenameColumn(act)) { // undoing - sometimes renames only in undo info + _addRename(_forTable(summary, tableId).columnRenames, [act[3], act[2]]); + } else if (Action.isReplaceTableData(act)) { // undoing + const td = _forTable(summary, tableId); + arrayExtend(td.removeRows, act[2]); + _addRows(tableId, td, act[2], act[3], 0); + } +} + +/** + * Summarize the tabular changes that a LocalActionBundle results in, in a form + * that will be suitable for composition. + */ +export function summarizeAction(body: LocalActionBundle): ActionSummary { + const summary = createEmptyActionSummary(); + for (const act of getEnvContent(body.stored)) { + _addForwardAction(summary, act); + } + for (const act of Array.from(body.undo).reverse()) { + _addReverseAction(summary, act); + } + // Name tables consistently, by their ultimate name, now we know it. + for (const renames of summary.tableRenames) { + const pre = renames[0]; + let post = renames[1]; + if (pre === null) { continue; } + if (post === null) { post = defunctTableName(pre); } + if (summary.tableDeltas[pre]) { + summary.tableDeltas[post] = summary.tableDeltas[pre]; + delete summary.tableDeltas[pre]; + } + } + for (const td of values(summary.tableDeltas)) { + // Name columns consistently, by their ultimate name, now we know it. + for (const renames of td.columnRenames) { + const pre = renames[0]; + let post = renames[1]; + if (pre === null) { continue; } + if (post === null) { post = defunctTableName(pre); } + if (td.columnDeltas[pre]) { + td.columnDeltas[post] = td.columnDeltas[pre]; + delete td.columnDeltas[pre]; + } + } + // remove any duplicates that crept in + td.addRows = Array.from(new Set(td.addRows)); + td.updateRows = Array.from(new Set(td.updateRows)); + td.removeRows = Array.from(new Set(td.removeRows)); + } + return summary; +} + +/** + * Once we can produce an ActionSummary for each LocalActionBundle, it is useful to be able + * to compose them. Take the case of an ActionSummary pair, part 1 and part 2. NameMerge + * is an internal structure to help merging table/column name changes across two parts. + */ +interface NameMerge { + dead1: Set; /** anything of this name in part 1 should be removed from merge */ + dead2: Set; /** anything of this name in part 2 should be removed from merge */ + rename1: Map; /** replace these names in part 1 */ + rename2: Map; /** replace these names in part 2 */ + merge: LabelDelta[]; /** a merged list of adds/removes/renames for the result */ +} + +/** + * Looks at a pair of name change lists (could be tables or columns) and figures out what + * changes would need to be made to a data structure keyed on those names in order to key + * it consistently on final names. + */ +function planNameMerge(names1: LabelDelta[], names2: LabelDelta[]): NameMerge { + const result: NameMerge = { + dead1: new Set(), + dead2: new Set(), + rename1: new Map(), + rename2: new Map(), + merge: new Array(), + }; + const names1ByFinalName: {[name: string]: LabelDelta} = keyBy(names1, p => p[1]!); + const names2ByInitialName: {[name: string]: LabelDelta} = keyBy(names2, p => p[0]!); + for (const [before1, after1] of names1) { + if (!after1) { + if (!before1) { throw new Error("invalid name change found"); } + // Table/column was deleted in part 1. + result.dead1.add(before1); + result.merge.push([before1, null]); + continue; + } + // At this point, we know the table/column existed at end of part 1. + const pair2 = names2ByInitialName[after1]; + if (!pair2) { + // Table/column's name was stable in part 2, so only change was in part 1. + result.merge.push([before1, after1]); + continue; + } + const after2 = pair2[1]; + if (!after2) { + // Table/column was deleted in part 2. + result.dead2.add(after1); + if (before1) { + // Table/column existed prior to part 1, so we need to expose its history. + result.dead1.add(before1); + result.merge.push([before1, null]); + } else { + // Table/column did not exist prior to part 1, so we erase it from history. + result.dead1.add(after1); + result.dead2.add(defunctTableName(after1)); + } + continue; + } + // It we made it this far, our table/column exists after part 2. Any information + // keyed to its name in part 1 will need to be rekeyed to its final name. + result.rename1.set(after1, after2); + result.merge.push([before1, after2]); + } + // Look through part 2 for any changes not already covered. We won't need to do any + // renaming since table/column names at end of part 2 are just what we want. + for (const [before2, after2] of names2) { + if (!before2 && !after2) { throw new Error("invalid name change found"); } + if (before2 && names1ByFinalName[before2]) { continue; } // Already handled + result.merge.push([before2, after2]); + } + // For neatness, sort the merge order. Not essential. + result.merge = sortBy(result.merge, ([a, b]) => [a || "", b || ""]); + return result; +} + +/** + * Re-key nested data to match name changes / removals. Needs to be done a little carefully + * since it is perfectly possible for names to be swapped or shuffled. + * + * Entries may be TableDeltas in the case of table renames or ColumnDeltas for column renames. + * + * @param entries: a dictionary of nested data - TableDeltas for tables, ColumnDeltas for columns. + * @param dead: a set of keys to remove from the dictionary. + * @param rename: changes of names to apply to the dictionary. + */ +function renameAndDelete(entries: {[name: string]: T}, dead: Set, + rename: Map) { + // Remove all entries marked as dead. + for (const key of dead) { delete entries[key]; } + // Move all entries that are going to be renamed out to a cache temporarily. + const cache: {[name: string]: any} = {}; + for (const key of rename.keys()) { + if (entries[key]) { + cache[key] = entries[key]; + delete entries[key]; + } + } + // Move all renamed entries back in with their new names. + for (const [key, val] of rename.entries()) { if (cache[key]) { entries[val] = cache[key]; } } +} + +/** + * Apply planned name changes to a pair of entries, and return a merged entry encorporating + * their composition. + * + * @param names: the planned name changes as calculated by planNameMerge() + * @param entries1: the first dictionary of nested data keyed on the names + * @param entries2: test second dictionary of nested data keyed on the names + * @param mergeEntry: a function to apply any further corrections needed to the entries + * + */ +function mergeNames(names: NameMerge, + entries1: {[name: string]: T}, + entries2: {[name: string]: T}, + mergeEntry: (e1: T, e2: T) => T): {[name: string]: T} { + // Update the keys of the entries1 and entries2 dictionaries to be consistent. + renameAndDelete(entries1, names.dead1, names.rename1); + renameAndDelete(entries2, names.dead2, names.rename2); + + // Prepare the composition of the two dictionaries. + const entries = entries2; // Start with the second dictionary. + for (const key of Object.keys(entries1)) { // Add material from the first. + const e1 = entries1[key]; + if (!entries[key]) { entries[key] = e1; continue; } // No overlap - just add and move on. + entries[key] = mergeEntry(e1, entries[key]); // Recursive merge if overlap. + } + return entries; +} + +/** + * Track whether a specific row was added, removed or updated. + */ +interface RowChange { + added: boolean; + removed: boolean; + updated: boolean; +} + +/** RowChange for each row in a table */ +export interface RowChanges { + [rowId: number]: RowChange; +} + + +/** + * This is used when we hit a cell that we know has changed but don't know its + * value due to it being part of a bulk input. This produces a cell that + * represents the unknowns. + */ +function bulkCellFor(rc: RowChange|undefined): CellDelta|undefined { + if (!rc) { return undefined; } + const result: CellDelta = [null, null]; + if (rc.removed || rc.updated) { result[0] = '?'; } + if (rc.added || rc.updated) { result[1] = '?'; } + return result; +} + +/** + * Merge changes that apply to a particular column. + * + * @param present1: affected rows in part 1 + * @param present2: affected rows in part 2 + * @param e1: cached cell values for the column in part 1 + * @param e2: cached cell values for the column in part 2 + */ +function mergeColumn(present1: RowChanges, present2: RowChanges, + e1: ColumnDelta, e2: ColumnDelta): ColumnDelta { + for (const key of (Object.keys(present1) as unknown as number[])) { + let v1 = e1[key]; + let v2 = e2[key]; + if (!v1 && !v2) { continue; } + v1 = v1 || bulkCellFor(present1[key]); + v2 = v2 || bulkCellFor(present2[key]); + if (!v2) { e2[key] = e1[key]; continue; } + if (!v1[1]) { continue; } // Deleted row. + e2[key] = [v1[0], v2[1]]; // Change is from initial value in e1 to final value in e2. + } + return e2; +} + + +/** Put list of numbers in ascending order, with duplicates removed. */ +function uniqueAndSorted(lst: number[]) { + return [...new Set(lst)].sort((a, b) => a - b); +} + +/** For each row changed, figure out whether it was added/removed/updated */ +/** TODO: need for this method suggests maybe a better core representation for this info */ +function getRowChanges(e: TableDelta): RowChanges { + const all = new Set([...e.addRows, ...e.removeRows, ...e.updateRows]); + const added = new Set(e.addRows); + const removed = new Set(e.removeRows); + const updated = new Set(e.updateRows); + return fromPairs([...all].map(x => { + return [x, {added: added.has(x), + removed: removed.has(x), + updated: updated.has(x)}] as [number, RowChange]; + })); +} + +/** + * Merge changes that apply to a particular table. For updating addRows and removeRows, care is + * needed, since it is fine to remove and add the same rowId within a single summary -- this is just + * rowId reuse. It needs to be tracked so we know lifetime of rows though. + */ +function mergeTable(e1: TableDelta, e2: TableDelta): TableDelta { + // First, sort out any changes to names of columns. + const names = planNameMerge(e1.columnRenames, e2.columnRenames); + mergeNames(names, e1.columnDeltas, e2.columnDeltas, + mergeColumn.bind(null, + getRowChanges(e1), + getRowChanges(e2))); + e2.columnRenames = names.merge; + // All the columnar data is now merged. What remains is to merge the summary lists of rowIds + // that we maintain. + const addRows1 = new Set(e1.addRows); // Non-transient rows we have clearly added. + const removeRows2 = new Set(e2.removeRows); // Non-transient rows we have clearly removed. + const transients = e1.addRows.filter(x => removeRows2.has(x)); + e2.addRows = uniqueAndSorted([...e2.addRows, ...e1.addRows.filter(x => !removeRows2.has(x))]); + e2.removeRows = uniqueAndSorted([...e2.removeRows.filter(x => !addRows1.has(x)), ...e1.removeRows]); + e2.updateRows = uniqueAndSorted([...e1.updateRows.filter(x => !removeRows2.has(x)), + ...e2.updateRows.filter(x => !addRows1.has(x))]); + // Remove all traces of transients (rows that were created and destroyed) from history. + for (const cols of values(e2.columnDeltas)) { + for (const key of transients) { delete cols[key]; } + } + return e2; +} + +/** Finally, merge a pair of summaries. */ +export function concatenateSummaryPair(sum1: ActionSummary, sum2: ActionSummary): ActionSummary { + const names = planNameMerge(sum1.tableRenames, sum2.tableRenames); + const rowChanges = mergeNames(names, sum1.tableDeltas, sum2.tableDeltas, mergeTable); + const sum: ActionSummary = { + tableRenames: names.merge, + tableDeltas: rowChanges + }; + return sum; +} + +/** Generalize to merging a list of summaries. */ +export function concatenateSummaries(sums: ActionSummary[]): ActionSummary { + if (sums.length === 0) { return createEmptyActionSummary(); } + let result = sums[0]; + for (let i = 1; i < sums.length; i++) { + result = concatenateSummaryPair(result, sums[i]); + } + return result; +} diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts new file mode 100644 index 00000000..e3ee0c4d --- /dev/null +++ b/app/server/lib/ActiveDoc.ts @@ -0,0 +1,1132 @@ +/** + * Module to manage "active" Grist documents, i.e. those loaded in-memory, with + * clients connected to them. It handles the incoming user actions, and outgoing + * change events. + */ + +import * as assert from 'assert'; +import * as bluebird from 'bluebird'; +import {EventEmitter} from 'events'; +import {Request} from 'express'; +import {IMessage, MsgType} from 'grain-rpc'; +import * as imageSize from 'image-size'; +import flatten = require('lodash/flatten'); +import remove = require('lodash/remove'); +import zipObject = require('lodash/zipObject'); +import * as moment from 'moment-timezone'; +import * as tmp from 'tmp'; +import * as util from 'util'; + +import {getEnvContent, LocalActionBundle} from 'app/common/ActionBundle'; +import {SandboxActionBundle, UserActionBundle} from 'app/common/ActionBundle'; +import {ActionGroup} from 'app/common/ActionGroup'; +import {ApplyUAOptions, ApplyUAResult, ForkResult} from 'app/common/ActiveDocAPI'; +import {DataSourceTransformed, ImportResult, Query, QueryResult} from 'app/common/ActiveDocAPI'; +import {ApiError} from 'app/common/ApiError'; +import {mapGetOrSet, MapWithTTL} from 'app/common/AsyncCreate'; +import {BulkColValues, CellValue, DocAction, RowRecord, TableDataAction, UserAction} from 'app/common/DocActions'; +import {toTableDataAction} from 'app/common/DocActions'; +import {DocData} from 'app/common/DocData'; +import {EncActionBundleFromHub} from 'app/common/EncActionBundle'; +import {byteString} from 'app/common/gutil'; +import {InactivityTimer} from 'app/common/InactivityTimer'; +import * as marshal from 'app/common/marshal'; +import {Peer} from 'app/common/sharing'; +import {UploadResult} from 'app/common/uploads'; +import {DocReplacementOptions} from 'app/common/UserAPI'; +import {ParseOptions} from 'app/plugin/FileParserAPI'; +import {GristDocAPI} from 'app/plugin/GristAPI'; +import {Authorizer, getUserId} from 'app/server/lib/Authorizer'; +import {checksumFile} from 'app/server/lib/checksumFile'; +import {Client} from 'app/server/lib/Client'; +import {DEFAULT_CACHE_TTL, DocManager} from 'app/server/lib/DocManager'; +import {DocSnapshots} from 'app/server/lib/DocSnapshots'; +import {makeForkIds} from 'app/server/lib/idUtils'; +import {ISandbox} from 'app/server/lib/ISandbox'; +import * as log from 'app/server/lib/log'; +import {shortDesc} from 'app/server/lib/shortDesc'; +import {fetchURL, FileUploadInfo, globalUploadSet, UploadInfo} from 'app/server/lib/uploads'; + +import {ActionHistory, asActionGroup} from './ActionHistory'; +import {ActionHistoryImpl} from './ActionHistoryImpl'; +import {ActiveDocImport} from './ActiveDocImport'; +import {DocClients} from './DocClients'; +import {DocPluginManager} from './DocPluginManager'; +import {DocSession, OptDocSession} from './DocSession'; +import {DocStorage} from './DocStorage'; +import {expandQuery} from './ExpandedQuery'; +import {OnDemandActions} from './OnDemandActions'; +import {findOrAddAllEnvelope, Sharing} from './Sharing'; + +bluebird.promisifyAll(tmp); + +const MAX_RECENT_ACTIONS = 100; + +const DEFAULT_TIMEZONE = (process.versions as any).electron ? moment.tz.guess() : "UTC"; + +// Number of seconds an ActiveDoc is retained without any clients. +// In dev environment, it is convenient to keep this low for quick tests. +// In production, it is reasonable to stretch it out a bit. +const ACTIVEDOC_TIMEOUT = (process.env.NODE_ENV === 'production') ? 30 : 5; + +// We'll wait this long between re-measuring sandbox memory. +const MEMORY_MEASUREMENT_INTERVAL_MS = 60 * 1000; + +// A hook for dependency injection. +export const Deps = {ACTIVEDOC_TIMEOUT}; + +/** + * Represents an active document with the given name. The document isn't actually open until + * either .loadDoc() or .createDoc() is called. + * @param {String} docName - The document's filename, without the '.grist' extension. + */ +export class ActiveDoc extends EventEmitter { + /** + * Decorator for ActiveDoc methods that prevents shutdown while the method is running, i.e. + * until the returned promise is resolved. + */ + public static keepDocOpen(target: ActiveDoc, propertyKey: string, descriptor: PropertyDescriptor) { + const origFunc = descriptor.value; + descriptor.value = function(this: ActiveDoc) { + return this._inactivityTimer.disableUntilFinish(origFunc.apply(this, arguments)); + }; + } + + public readonly docStorage: DocStorage; + public readonly docPluginManager: DocPluginManager; + public readonly docClients: DocClients; // Only exposed for Sharing.ts + public docData: DocData|null = null; + + protected _actionHistory: ActionHistory; + protected _docManager: DocManager; + protected _docName: string; + protected _sharing: Sharing; + private readonly _dataEngine: ISandbox; + private _activeDocImport: ActiveDocImport; + private _onDemandActions: OnDemandActions; + private _muted: boolean = false; // If set, changes to this document should not propagate + // to outside world + private _initializationPromise: Promise|null = null; + // If set, wait on this to be sure the ActiveDoc is fully + // initialized. True on success. + private _fullyLoaded: boolean = false; // Becomes true once all columns are loaded/computed. + private _lastMemoryMeasurement: number = 0; // Timestamp when memory was last measured. + private _fetchCache = new MapWithTTL>(DEFAULT_CACHE_TTL); + + // Timer for shutting down the ActiveDoc a bit after all clients are gone. + private _inactivityTimer = new InactivityTimer(() => this.shutdown(), Deps.ACTIVEDOC_TIMEOUT * 1000); + + constructor(docManager: DocManager, docName: string) { + super(); + this._docManager = docManager; + this._docName = docName; + this.docStorage = new DocStorage(docManager.storageManager, docName); + this.docClients = new DocClients(this); + this._actionHistory = new ActionHistoryImpl(this.docStorage); + this.docPluginManager = new DocPluginManager(docManager.pluginManager.getPlugins(), + docManager.pluginManager.appRoot!, this, this._docManager.gristServer); + + // Our DataEngine is a separate sandboxed process (one per open document). The data engine runs + // user-defined python code including formula calculations. It maintains all document data and + // metadata, and applies translates higher-level UserActions into lower-level DocActions. + this._dataEngine = this._docManager.gristServer.create.NSandbox({ + comment: docName, + logCalls: false, + logTimes: true, + logMeta: {docId: docName}, + }); + + this._activeDocImport = new ActiveDocImport(this); + + // Schedule shutdown immediately. If a client connects soon (normal case), it will get + // unscheduled. If not (e.g. abandoned import, network problems after creating a doc), then + // the ActiveDoc will get cleaned up. + this._inactivityTimer.enable(); + } + + public get docName(): string { return this._docName; } + + // Helpers to log a message along with metadata about the request. + public logDebug(c: Client|OptDocSession|null, msg: string, ...args: any[]) { this._log('debug', c, msg, ...args); } + public logInfo(c: Client|OptDocSession|null, msg: string, ...args: any[]) { this._log('info', c, msg, ...args); } + public logWarn(c: Client|OptDocSession|null, msg: string, ...args: any[]) { this._log('warn', c, msg, ...args); } + public logError(c: Client|OptDocSession|null, msg: string, ...args: any[]) { this._log('error', c, msg, ...args); } + + // Constructs metadata for logging, given a Client or an OptDocSession. + public getLogMeta(cli: Client|OptDocSession|null, docMethod?: string): log.ILogMeta { + const client = cli && ('getProfile' in cli ? cli : cli.client); + return { + docId: this._docName, + ...(docMethod ? {docMethod} : {}), + ...(client ? client.getLogMeta() : {}), + }; + } + + public setMuted() { + this._muted = true; + } + + public get muted() { + return this._muted; + } + + public getRecentActionsDirect(maxActions?: number): Promise { + return this._actionHistory.getRecentActions(maxActions); + } + + public getRecentStates(maxStates?: number) { + return this._actionHistory.getRecentStates(maxStates); + } + + /** + * Get the most recent actions from the history. Results are ordered by + * earliest actions first, later actions later. If `summarize` is set, + * action summaries are computed and included. + */ + public async getRecentActions(client: Client|null, summarize: boolean): Promise { + const actions = await this._actionHistory.getRecentActions(MAX_RECENT_ACTIONS); + return actions.map(act => asActionGroup(this._actionHistory, act, {client, summarize})); + } + + /** expose action history for tests */ + public getActionHistory(): ActionHistory { + return this._actionHistory; + } + + /** + * Adds a client of this doc to the list of connected clients. + * @param client: The client object maintaining the websocket connection. + * @param authorizer: The authorizer for the client/doc combination. + * @returns docSession + */ + public addClient(client: Client, authorizer: Authorizer): DocSession { + const docSession: DocSession = this.docClients.addClient(client, authorizer); + + // If we had a shutdown scheduled, unschedule it. + if (this._inactivityTimer.isEnabled()) { + this.logInfo(client, "will stay open"); + this._inactivityTimer.disable(); + } + return docSession; + } + + /** + * Shut down the ActiveDoc, and (by default) remove it from the docManager. + * @returns {Promise} Promise for when database and data engine are done shutting down. + */ + public async shutdown(removeThisActiveDoc: boolean = true): Promise { + this.logDebug(null, "shutdown starting"); + this._inactivityTimer.disable(); + if (this.docClients.clientCount() > 0) { + this.logWarn(null, `Doc being closed with ${this.docClients.clientCount()} clients left`); + this.docClients.broadcastDocMessage(null, 'docShutdown', null); + this.docClients.removeAllClients(); + } + + // Clear the MapWithTTL to remove all timers from the event loop. + this._fetchCache.clear(); + + if (removeThisActiveDoc) { await this._docManager.removeActiveDoc(this); } + try { + await this._docManager.storageManager.closeDocument(this.docName); + } catch (err) { + log.error('Problem shutting down document: %s %s', this.docName, err.message); + } + + try { + await Promise.all([ + this.docStorage.shutdown(), + this.docPluginManager.shutdown(), + this._dataEngine.shutdown() + ]); + // The this.waitForInitialization promise may not yet have resolved, but + // should do so quickly now we've killed everything it depends on. + try { + await this.waitForInitialization(); + } catch (err) { + // Initialization errors do not matter at this point. + } + this.logDebug(null, "shutdown complete"); + } catch (err) { + this.logError(null, "failed to shutdown some resources", err); + } + } + + /** + * Create a new blank document. Returns a promise for the ActiveDoc itself. + */ + @ActiveDoc.keepDocOpen + public async createDoc(docSession: OptDocSession): Promise { + this.logDebug(docSession, "createDoc"); + await this.docStorage.createFile(); + await this._dataEngine.pyCall('load_empty'); + const timezone = docSession.browserSettings ? docSession.browserSettings.timezone : DEFAULT_TIMEZONE; + // This init action is special. It creates schema tables, and is used to init the DB, but does + // not go through other steps of a regular action (no ActionHistory or broadcasting). + const initBundle = await this._dataEngine.pyCall('apply_user_actions', [["InitNewDoc", timezone]]); + await this.docStorage.execTransaction(() => + this.docStorage.applyStoredActions(getEnvContent(initBundle.stored))); + + await this._initDoc(docSession); + // Makes sure docPluginManager is ready in case new doc is used to import new data + await this.docPluginManager.ready; + this._fullyLoaded = true; + return this; + } + + /** + * Loads an existing document from storage, fetching all data from the database via DocStorage and + * loading it into the DataEngine. User tables are not immediately loaded (see use of + * this.waitForInitialization throughout this class to wait for that). + * @returns {Promise} Promise for this ActiveDoc itself. + */ + @ActiveDoc.keepDocOpen + public async loadDoc(docSession: OptDocSession): Promise { + const startTime = Date.now(); + this.logDebug(docSession, "loadDoc"); + try { + const isNew: boolean = await this._docManager.storageManager.prepareLocalDoc(this.docName, + docSession); + if (isNew) { + await this.createDoc(docSession); + await this.addInitialTable(); + } else { + await this.docStorage.openFile(); + const tableNames = await this._loadOpenDoc(docSession); + const desiredTableNames = tableNames.filter(name => name.startsWith('_grist_')); + await this._loadTables(docSession, desiredTableNames); + const pendingTableNames = tableNames.filter(name => !name.startsWith('_grist_')); + await this._initDoc(docSession); + this._initializationPromise = this._finishInitialization(docSession, pendingTableNames, startTime); + } + } catch (err) { + await this.shutdown(); + throw err; + } + return this; + } + + /** + * Replace this document with another, in-place so its id and other metadata does not change. + * This operation will leave the ActiveDoc it is called for unusable. It will mute it, + * shut it down, and unlist it via the DocManager. A fresh ActiveDoc can be acquired via the + * DocManager. + */ + public async replace(source: DocReplacementOptions) { + // During replacement, it is important for all hands to be off the document. So: + // - We set the "mute" flag. Setting this means that any operations in progress + // using this ActiveDoc should be ineffective (apart from the replacement). + // In other words, the operations shouldn't ultimately result in any changes in S3, + // and any related requests should result in a failure or be retried. TODO: + // review how well we do on meeting this goal. + // - We close the ActiveDoc, retaining its listing in DocManager but shutting down + // all its component parts. We retain it in DocManager to delay another + // ActiveDoc being opened for the same document if someone is trying to operate + // on it. + // - We replace the document. + // - We remove the ActiveDoc from DocManager, opening the way for the document to be + // freshly opened. + // The "mute" flag is borrowed from worker shutdown. Note this scenario is a little + // different, since the worker is not withdrawing from service, so fresh work may get + // assigned to it at any time. + this.setMuted(); + this.docClients.interruptAllClients(); + try { + await this.shutdown(false); + await this._docManager.storageManager.replace(this.docName, source); + } finally { + // Whatever happened, success or failure, there is nothing further we can do + // with this ActiveDoc. Unlist it. + await this._docManager.removeActiveDoc(this); + } + } + + /** + * Create a document given encrypted action bundles from the sharing hub. Part of the process + * of downloading a shared doc. + * TODO: Not only the snapshot but all actions shared to the hub before download are applied + * directly to the database, meaning they cannot be undone by this instance. We may want to + * consider applying actions following the snapshot differently. + */ + public async downloadSharedDoc( + docId: string, + instanceId: string, + encBundles: EncActionBundleFromHub[] + ): Promise { + throw new Error('downloadSharedDoc not implemented'); + } + + /** + * Finish initializing ActiveDoc, by initializing ActionHistory, Sharing, and docData. + */ + public async _initDoc(docSession: OptDocSession|null): Promise { + const metaTableData = await this._dataEngine.pyCall('fetch_meta_tables'); + this.docData = new DocData(tableId => this.fetchTable(null, tableId), metaTableData); + this._onDemandActions = new OnDemandActions(this.docStorage, this.docData); + + await this._actionHistory.initialize(); + this._sharing = new Sharing(this, this._actionHistory); + + await this.openSharedDoc(docSession); + } + + public async openSharedDoc(docSession: OptDocSession|null) { + // Doesn't do anything special in this base class. + } + + /** + * Adds a small table to start off a newly-created blank document. + */ + public addInitialTable() { + return this._applyUserActions(null, [["AddEmptyTable"]]); + } + + /** + * Imports files, removes previously created temporary hidden tables and creates the new ones. + * Param `prevTableIds` is an array of hiddenTableIds as received from previous `importFiles` + * call, or empty if there was no previous call. + */ + public importFiles(docSession: DocSession, dataSource: DataSourceTransformed, + parseOptions: ParseOptions, prevTableIds: string[]): Promise { + return this._activeDocImport.importFiles(docSession, dataSource, parseOptions, prevTableIds); + } + + /** + * Finishes import files, creates the new tables, and cleans up temporary hidden tables and uploads. + * Param `prevTableIds` is an array of hiddenTableIds as received from previous `importFiles` + * call, or empty if there was no previous call. + */ + public finishImportFiles(docSession: DocSession, dataSource: DataSourceTransformed, + parseOptions: ParseOptions, prevTableIds: string[]): Promise { + return this._activeDocImport.finishImportFiles(docSession, dataSource, parseOptions, prevTableIds); + } + + /** + * Cancels import files, cleans up temporary hidden tables and uploads. + * Param `prevTableIds` is an array of hiddenTableIds as received from previous `importFiles` + * call, or empty if there was no previous call. + */ + public cancelImportFiles(docSession: DocSession, dataSource: DataSourceTransformed, + prevTableIds: string[]): Promise { + return this._activeDocImport.cancelImportFiles(docSession, dataSource, prevTableIds); + } + + /** + * Close the current document. + */ + public async closeDoc(docSession: DocSession): Promise { + // Note that it's async only to satisfy the Rpc interface that expects a promise. + this.docClients.removeClient(docSession); + + // If no more clients, schedule a shutdown. + if (this.docClients.clientCount() === 0) { + this.logInfo(docSession, "will self-close in %ds", Deps.ACTIVEDOC_TIMEOUT); + this._inactivityTimer.enable(); + } + } + + /** + * Import the given upload as new tables in one step. + */ + @ActiveDoc.keepDocOpen + public async oneStepImport(docSession: OptDocSession, uploadInfo: UploadInfo): Promise { + await this._activeDocImport.oneStepImport(docSession, uploadInfo); + } + + /** + * This function saves attachments from a given upload and creates an entry for them in the database. + * It returns the list of rowIds for the rows created in the _grist_Attachments table. + */ + public async addAttachments(docSessOrReq: DocSession|Request, uploadId: number): Promise { + // TODO Refactor to accept Request generally when DocSession is absent (for API calls), and + // include user/org info in logging too. + const [docSession, userId] = ('authorizer' in docSessOrReq ? + [docSessOrReq, docSessOrReq.authorizer.getUserId()] : + [{client: null}, getUserId(docSessOrReq)]); + + const upload: UploadInfo = globalUploadSet.getUploadInfo(uploadId, this.makeAccessId(userId)); + try { + const userActions: UserAction[] = await Promise.all( + upload.files.map(file => this._prepAttachment(docSession, file))); + const result = await this._applyUserActions(docSession.client, userActions); + return result.retValues; + } finally { + await globalUploadSet.cleanup(uploadId); + } + } + + /** + * Returns the record from _grist_Attachments table for the given attachment ID, + * or throws an error if not found. + */ + public getAttachmentMetadata(attId: number|string): RowRecord { + // docData should always be available after loadDoc() or createDoc(). + if (!this.docData) { + throw new Error("No doc data"); + } + // Parse strings into numbers to make more convenient to call from route handlers. + const attachmentId: number = (typeof attId === 'string') ? parseInt(attId, 10) : attId; + const attRecord = this.docData.getTable('_grist_Attachments')!.getRecord(attachmentId); + if (!attRecord) { + throw new ApiError(`Attachment not found: ${attId}`, 404); + } + return attRecord; + } + + /** + * Given the fileIdent of an attachment, returns a promise for the attachment data. + * @param {String} fileIdent: The unique identifier of the attachment (as stored in fileIdent + * field of the _grist_Attachments table). + * @returns {Promise} Promise for the data of this attachment; rejected on error. + */ + public async getAttachmentData(client: Client|null, fileIdent: string): Promise { + const data = await this.docStorage.getFileData(fileIdent); + if (!data) { throw new ApiError("Invalid attachment identifier", 404); } + this.logInfo(client, "getAttachment: %s -> %s bytes", fileIdent, data.length); + return data; + } + + /** + * Fetches the meta tables to return to the client when first opening a document. + */ + public async fetchMetaTables(client: Client) { + this.logInfo(client, "fetchMetaTables"); + if (!this.docData) { throw new Error("No doc data"); } + // Get metadata from local cache rather than data engine, so that we can + // still get it even if data engine is busy calculating. + const tables: {[key: string]: TableDataAction} = {}; + for (const [tableId, tableData] of this.docData.getTables().entries()) { + if (!tableId.startsWith('_grist_')) { continue; } + tables[tableId] = tableData.getTableDataAction(); + } + return tables; +} + + /** + * Makes sure document is completely initialized. May throw if doc is broken. + */ + public async waitForInitialization() { + if (this._initializationPromise) { + if (!await this._initializationPromise) { + throw new Error('ActiveDoc initialization failed'); + } + } + return true; + } + + /** + * Fetches a particular table from the data engine to return to the client. + * @param {String} tableId: The string identifier of the table. + * @param {Boolean} waitForFormulas: If true, wait for all data to be loaded/calculated. + * @returns {Promise} Promise for the TableData object, which is a BulkAddRecord-like array of the + * form of the form ["TableData", table_id, row_ids, column_values]. + */ + public async fetchTable(docSession: DocSession|null, tableId: string, + waitForFormulas: boolean = false): Promise { + this.logInfo(docSession, "fetchTable(%s, %s)", docSession, tableId); + return this.fetchQuery(docSession, {tableId, filters: {}}, waitForFormulas); + } + + /** + * Fetches data according to the given query, which includes tableId and filters (see Query in + * app/common/ActiveDocAPI.ts). The data is fetched from the data engine for regular tables, or + * from the DocStorage directly for onDemand tables. + * @param {Boolean} waitForFormulas: If true, wait for all data to be loaded/calculated. If false, + * special "pending" values may be returned. + */ + public async fetchQuery(docSession: DocSession|null, query: Query, + waitForFormulas: boolean = false): Promise { + this._inactivityTimer.ping(); // The doc is in active use; ping it to stay open longer. + + // Some tests read _grist_ tables via the api. The _fetchQueryFromDB method + // currently cannot read those tables, so we load them from the data engine + // when ready. + const wantFull = waitForFormulas || query.tableId.startsWith('_grist_'); + const onDemand = this._onDemandActions.isOnDemand(query.tableId); + this.logInfo(docSession, "fetchQuery(%s, %s) %s", docSession, JSON.stringify(query), + onDemand ? "(onDemand)" : "(regular)"); + let data: TableDataAction; + if (onDemand) { + data = await this._fetchQueryFromDB(query, onDemand); + } else if (wantFull) { + await this.waitForInitialization(); + data = await this._fetchQueryFromDataEngine(query); + } else { + if (!this._fullyLoaded) { + data = await this._fetchQueryFromDB(query, false); + } + if (this._fullyLoaded) { // Already loaded or finished loading while fetching from DB + const key = JSON.stringify(query); + // TODO: cache longer if the underlying fetch takes longer to do. + data = await mapGetOrSet(this._fetchCache, key, () => this._fetchQueryFromDataEngine(query)); + } + } + this.logInfo(docSession, "fetchQuery -> %d rows, cols: %s", + data![2].length, Object.keys(data![3]).join(", ")); + return data!; + } + + /** + * Fetches the generated schema for a given table. + * @param {String} tableId: The string identifier of the table. + * @returns {Promise} Promise for a string representing the generated table schema. + */ + public async fetchTableSchema(docSession: DocSession): Promise { + this.logInfo(docSession, "fetchTableSchema(%s)", docSession); + await this.waitForInitialization(); + return this._dataEngine.pyCall('fetch_table_schema'); + } + + /** + * Makes a query (documented elsewhere) and subscribes to it, so that the client receives + * docActions that affect this query's results. + */ + public async useQuerySet(docSession: DocSession, query: Query): Promise { + this.logInfo(docSession, "useQuerySet(%s, %s)", docSession, query); + // TODO implement subscribing to the query. + // - Convert tableId+colIds to TableData/ColData references + // - Return a unique identifier for unsubscribing + // - Each call can create its own object, return own identifier. + // - Subscription should not be affected by renames (so don't hold on to query/tableId/colIds) + // - Table/column deletion should make subscription inactive, and unsubscribing an inactive + // subscription should not produce an error. + const tableData: TableDataAction = await this.fetchQuery(docSession, query); + return {querySubId: 0, tableData}; + } + + /** + * Removes all subscriptions to the given query from this client, so that it stops receiving + * docActions relevant only to this query. + */ + public async disposeQuerySet(docSession: DocSession, querySubId: number): Promise { + this.logInfo(docSession, "disposeQuerySet(%s, %s)", docSession, querySubId); + // TODO To-be-implemented + } + + /** + * Returns the most likely target column in the document for the given column. + * @param {Array} values: An array of values to search for in columns in the document. + * @param {Number} n: Number of results to return. + * @param {String} optTableId: If a valid tableId, search only that table. + * @returns {Promise} Promise for an array of colRefs describing matching columns ordered from + * best to worst. Match quality is determined by searching only a sample of column data. + * See engine.py find_col_from_values for implementation. + */ + public async findColFromValues(docSession: DocSession, values: any[], n: number, + optTableId?: string): Promise { + this.logInfo(docSession, "findColFromValues(%s, %s, %s)", docSession, values, n); + await this.waitForInitialization(); + return this._dataEngine.pyCall('find_col_from_values', values, n, optTableId); + } + + /** + * Returns error message (traceback) for one invalid formula cell. + * @param {String} tableId - Table name + * @param {String} colId - Column name + * @param {Integer} rowId - Row number + * @returns {Promise} Promise for a error message + */ + public async getFormulaError(docSession: DocSession, tableId: string, colId: string, + rowId: number): Promise { + this.logInfo(docSession, "getFormulaError(%s, %s, %s, %s)", + docSession, tableId, colId, rowId); + await this.waitForInitialization(); + return this._dataEngine.pyCall('get_formula_error', tableId, colId, rowId); + } + + /** + * Applies an array of user actions received from a browser client. + * + * @param {Object} docSession: The client session originating this action. + * @param {Array} action: The user action to apply, e.g. ["UpdateRecord", tableId, rowId, etc]. + * @param {Object} options: See _applyUserActions for documentation + * @returns {Promise:Array[Object]} Promise that's resolved when action is applied successfully. + * The array includes the retValue objects for each + * actionGroup. + */ + public async applyUserActions(docSession: OptDocSession, actions: UserAction[], + options?: ApplyUAOptions): Promise { + assert(Array.isArray(actions), "`actions` parameter should be an array."); + // Be careful not to sneak into user action queue before Calculate action, otherwise + // there'll be a deadlock. + await this.waitForInitialization(); + const newOptions = {linkId: docSession.linkId, ...options}; + const result: ApplyUAResult = await this._applyUserActions(docSession.client, actions, newOptions); + docSession.linkId = docSession.shouldBundleActions ? result.actionNum : 0; + return result; + } + + /** + * A variant of applyUserActions where actions are passed in by ids (actionNum, actionHash) + * rather than by value. + * + * @param docSession: The client session originating this action. + * @param actionNums: The user actions to do/undo, by actionNum. + * @param actionHashes: actionHash checksums for each listed actionNum. + * @param undo: Whether the actions are to be undone. + * @param options: As for applyUserActions. + * @returns Promise of retValues, see applyUserActions. + */ + public async applyUserActionsById(docSession: DocSession, + actionNums: number[], + actionHashes: string[], + undo: boolean, + options?: ApplyUAOptions): Promise { + const actionBundles = await this._actionHistory.getActions(actionNums); + for (const [index, bundle] of actionBundles.entries()) { + const actionNum = actionNums[index]; + const actionHash = actionHashes[index]; + if (!bundle) { throw new Error(`Could not find actionNum ${actionNum}`); } + if (actionHash !== bundle.actionHash) { + throw new Error(`Hash mismatch for actionNum ${actionNum}: ` + + `expected ${actionHash} but got ${bundle.actionHash}`); + } + } + let actions: UserAction[]; + if (undo) { + actions = [['ApplyUndoActions', flatten(actionBundles.map(a => a!.undo))]]; + } else { + actions = flatten(actionBundles.map(a => a!.userActions)); + } + return this.applyUserActions(docSession, actions, options); + } + + /** + * Called by Sharing class for every LocalActionBundle (of our own actions) that gets applied. + */ + public async processActionBundle(localActionBundle: LocalActionBundle): Promise { + const docData = this.docData; + if (!docData) { return; } // Happens on doc creation while processing InitNewDoc action. + localActionBundle.stored.forEach(da => docData.receiveAction(da[1])); + localActionBundle.calc.forEach(da => docData.receiveAction(da[1])); + const docActions = getEnvContent(localActionBundle.stored); + if (docActions.some(docAction => this._onDemandActions.isSchemaAction(docAction))) { + const indexes = this._onDemandActions.getDesiredIndexes(); + await this.docStorage.updateIndexes(indexes); + } + } + + /** + * Used by tests to force an update indexes. We don't otherwise update indexes until + * there is a schema change. + */ + public async testUpdateIndexes() { + const indexes = this._onDemandActions.getDesiredIndexes(); + await this.docStorage.updateIndexes(indexes); + } + + /** + * Shares the doc and invites peers. + * @param {Array} peers - Array of peer objects with which the doc should be shared. + * @returns {Promise} Return promise for docId on completion. + */ + public async shareDoc(docSession: DocSession, peers: Peer[]): Promise { + throw new Error('shareDoc not implemented'); + } + + public async removeInstanceFromDoc(docSession: DocSession): Promise { + const instanceId = await this._sharing.removeInstanceFromDoc(); + await this._applyUserActions(docSession.client, [['RemoveInstance', instanceId]]); + } + + public async renameDocTo(client: Client, newName: string): Promise { + this.logDebug(client, 'renameDoc', newName); + await this.docStorage.renameDocTo(newName); + this._docName = newName; + } + + /** + * Initiates user actions bandling for undo. + */ + public startBundleUserActions(docSession: OptDocSession) { + if (!docSession.shouldBundleActions) { + docSession.shouldBundleActions = true; + docSession.linkId = 0; + } + } + + /** + * Stops user actions bandling for undo. + */ + public stopBundleUserActions(docSession: OptDocSession) { + docSession.shouldBundleActions = false; + docSession.linkId = 0; + } + + public async autocomplete(docSession: DocSession, txt: string, tableId: string): Promise { + await this.waitForInitialization(); + return this._dataEngine.pyCall('autocomplete', txt, tableId); + } + + public fetchURL(docSession: DocSession, url: string): Promise { + return fetchURL(url, this.makeAccessId(docSession.authorizer.getUserId())); + } + + public forwardPluginRpc(docSession: DocSession, pluginId: string, msg: IMessage): Promise { + const pluginRpc = this.docPluginManager.plugins[pluginId].rpc; + switch (msg.mtype) { + case MsgType.RpcCall: return pluginRpc.forwardCall(msg); + case MsgType.Custom: return pluginRpc.forwardMessage(msg); + } + throw new Error(`Invalid message type for forwardPluginRpc: ${msg.mtype}`); + } + + /** + * Reload documents plugins. + */ + public async reloadPlugins(docSession: DocSession) { + // refresh the list plugins found on the system + await this._docManager.pluginManager.reloadPlugins(); + const plugins = this._docManager.pluginManager.getPlugins(); + // reload found plugins + await this.docPluginManager.reload(plugins); + } + + /** + * Immediately close the document and data engine, to be reloaded from scratch, and cause all + * browser clients to reopen it. + */ + public async reloadDoc(docSession?: DocSession) { + return this.shutdown(); + } + + /** + * Fork the current document. In fact, all that requires is calculating a good + * ID for the fork. TODO: reconcile the two ways there are now of preparing a fork. + */ + public async fork(docSession: DocSession): Promise { + const userId = docSession.client.getCachedUserId(); + const isAnonymous = docSession.client.isAnonymous(); + // Get fresh document metadata (the cached metadata doesn't include the urlId). + const doc = await docSession.authorizer.getDoc(); + if (!doc) { throw new Error('document id not known'); } + const trunkDocId = doc.id; + const trunkUrlId = doc.urlId || doc.id; + await this.flushDoc(); // Make sure fork won't be too out of date. + return makeForkIds({userId, isAnonymous, trunkDocId, trunkUrlId}); + } + + public getGristDocAPI(): GristDocAPI { + return this.docPluginManager.gristDocAPI; + } + + // Get recent actions in ActionGroup format with summaries included. + public async getActionSummaries(docSession: DocSession): Promise { + return this.getRecentActions(docSession.client, true); + } + + /** + * Applies normal actions to the data engine while processing onDemand actions separately. + */ + public async applyActionsToDataEngine(userActions: UserAction[]): Promise { + const [normalActions, onDemandActions] = this._onDemandActions.splitByOnDemand(userActions); + + let sandboxActionBundle: SandboxActionBundle; + if (normalActions.length > 0) { + // For all but the special 'Calculate' action, we wait for full initialization. + if (normalActions[0][0] !== 'Calculate') { + await this.waitForInitialization(); + } + sandboxActionBundle = await this._dataEngine.pyCall('apply_user_actions', normalActions); + await this._reportDataEngineMemory(); + } else { + // Create default SandboxActionBundle to use if the data engine is not called. + sandboxActionBundle = createEmptySandboxActionBundle(); + } + + if (onDemandActions.length > 0) { + const allIndex = findOrAddAllEnvelope(sandboxActionBundle.envelopes); + await this.docStorage.execTransaction(async () => { + for (const action of onDemandActions) { + const {stored, undo, retValues} = await this._onDemandActions.processUserAction(action); + // Note: onDemand stored/undo actions are arbitrarily processed/added after normal actions + // and do not support access control. + sandboxActionBundle.stored.push(...stored.map(a => [allIndex, a] as [number, DocAction])); + sandboxActionBundle.undo.push(...undo.map(a => [allIndex, a] as [number, DocAction])); + sandboxActionBundle.retValues.push(retValues); + } + }); + } + + return sandboxActionBundle; + } + + public async fetchSnapshot() { + await this.waitForInitialization(); + return this._dataEngine.pyCall('fetch_snapshot'); + } + + // Needed for test/server/migrations.js tests + public async testGetVersionFromDataEngine() { + return this._dataEngine.pyCall('get_version'); + } + + // Needed for test/server/lib/HostedStorageManager.ts tests + public async testKeepOpen() { + this._inactivityTimer.ping(); + } + + public async getSnapshots(): Promise { + return this._docManager.storageManager.getSnapshots(this.docName); + } + + /** + * Make sure the current version of the document has been pushed to persistent + * storage. + */ + public async flushDoc(): Promise { + return this._docManager.storageManager.flushDoc(this.docName); + } + + public makeAccessId(userId: number|null): string|null { + return this._docManager.makeAccessId(userId); + } + + /** + * Loads an open document from DocStorage. Returns a list of the tables it contains. + */ + protected async _loadOpenDoc(docSession: OptDocSession|null): Promise { + // Fetch the schema version of document and sandbox, and migrate if the sandbox is newer. + const [schemaVersion, docInfoData] = await Promise.all([ + this._dataEngine.pyCall('get_version'), + this.docStorage.fetchTable('_grist_DocInfo'), + ]); + + // Migrate the document if needed. + const values = marshal.loads(docInfoData!); + const versionCol = values.schemaVersion; + const docSchemaVersion = (versionCol && versionCol.length === 1 ? versionCol[0] : 0); + if (docSchemaVersion < schemaVersion) { + this.logInfo(docSession, "Doc needs migration from v%s to v%s", docSchemaVersion, schemaVersion); + await this._migrate(docSession); + } else if (docSchemaVersion > schemaVersion) { + // We do NOT attempt to down-migrate in this case. Migration code cannot down-migrate + // directly (since it doesn't know anything about newer documents). We could revert the + // migration action, but that requires merging and still may not be safe. For now, doing + // nothing seems best, as long as we follow the recommendations in migrations.py (never + // remove/modify/rename metadata tables or columns, or change their meaning). + this.logWarn(docSession, "Doc is newer (v%s) than this version of Grist (v%s); " + + "proceeding with fingers crossed", docSchemaVersion, schemaVersion); + } + + // Load the initial meta tables which determine the document schema. + const [tablesData, columnsData] = await Promise.all([ + this.docStorage.fetchTable('_grist_Tables'), + this.docStorage.fetchTable('_grist_Tables_column'), + ]); + + const tableNames: string[] = await this._dataEngine.pyCall('load_meta_tables', tablesData, columnsData); + + // Figure out which tables are on-demand. + const tablesParsed: BulkColValues = marshal.loads(tablesData!); + const onDemandMap = zipObject(tablesParsed.tableId as string[], tablesParsed.onDemand); + const onDemandNames = remove(tableNames, (t) => onDemandMap[t]); + + this.logDebug(docSession, "found %s tables: %s", tableNames.length, + tableNames.join(", ")); + this.logDebug(docSession, "skipping %s on-demand tables: %s", onDemandNames.length, + onDemandNames.join(", ")); + + return tableNames; + } + + /** + * Applies an array of user actions to the sandbox and broadcasts the results to doc's clients. + * + * @private + * @param {Object} client - The client originating this action. May be null. + * @param {Array} actions - The user actions to apply. + * @param {String} options.desc - Description of the action which overrides the default client + * description if provided. Should be used to describe bundled actions. + * @param {Int} options.otherId - Action number for the original useraction to which this undo/redo + * action applies. + * @param {Boolean} options.linkId - ActionNumber of the previous action in an undo/redo bundle. + * @returns {Promise} Promise that's resolved when all actions are applied successfully to { + * actionNum: number of the action that got recorded + * retValues: array of return values, one for each of the passed-in user actions. + * isModification: true if document was changed by one or more actions. + * } + */ + protected async _applyUserActions(client: Client|null, actions: UserAction[], + options: ApplyUAOptions = {}): Promise { + this.logDebug(client, "_applyUserActions(%s, %s)", client, shortDesc(actions)); + this._inactivityTimer.ping(); // The doc is in active use; ping it to stay open longer. + + const user = client && client.session ? (await client.session.getEmail()) : ""; + + // Create the UserActionBundle. + const action: UserActionBundle = { + info: { + time: Date.now(), + user, + inst: this._sharing.instanceId || "unset-inst", + desc: options.desc, + otherId: options.otherId || 0, + linkId: options.linkId || 0, + }, + userActions: actions, + }; + + const result: ApplyUAResult = await new Promise( + (resolve, reject) => + this._sharing!.addUserAction({action, client, resolve, reject})); + this.logDebug(client, "_applyUserActions returning %s", util.inspect(result)); + + if (result.isModification) { + this._fetchCache.clear(); // This could be more nuanced. + this._docManager.markAsChanged(this); + this._docManager.markAsEdited(this); + } + return result; + } + + /** + * Prepares a single attachment by adding it DocStorage and returns a UserAction to apply. + */ + private async _prepAttachment(docSession: OptDocSession, fileData: FileUploadInfo): Promise { + // Check that upload size is within the configured limits. + const limit = (Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024) || Infinity; + if (fileData.size > limit) { + throw new ApiError(`Attachments must not exceed ${byteString(limit)}`, 413); + } + + let dimensions: {width?: number, height?: number} = {}; + // imageSize returns an object with a width, height and type property if the file is an image. + // The width and height properties are integers representing width and height in pixels. + try { + dimensions = await bluebird.fromCallback((cb: any) => imageSize(fileData.absPath, cb)); + } catch (err) { + // Non-images will fail in some way, and that's OK. + dimensions.height = 0; + dimensions.width = 0; + } + const checksum = await checksumFile(fileData.absPath); + const fileIdent = checksum + fileData.ext; + const ret: boolean = await this.docStorage.findOrAttachFile(fileData.absPath, fileIdent); + this.logInfo(docSession, "addAttachment: file %s (image %sx%s) %s", fileIdent, + dimensions.width, dimensions.height, ret ? "attached" : "already exists"); + return ['AddRecord', '_grist_Attachments', null, { + fileIdent, + fileName: fileData.origName, + // We used to set fileType, but it's not easily available for native types. Since it's + // also entirely unused, we just skip it until it becomes relevant. + fileSize: fileData.size, + imageHeight: dimensions.height, + imageWidth: dimensions.width, + timeUploaded: Date.now() + }]; + } + + /** + * If the software is newer than the document, migrate the document by fetching all tables, and + * giving them to the sandbox so that it can produce migration actions. + * TODO: We haven't figured out how to do sharing between different Grist versions that + * expect different schema versions. The returned actions at the moment aren't even shared with + * collaborators. + */ + private async _migrate(docSession: OptDocSession|null): Promise { + // TODO: makeBackup should possibly be in docManager directly. + const backupPath = await this._docManager.storageManager.makeBackup(this._docName, "migrate"); + this.logInfo(docSession, "_migrate: backup made at %s", backupPath); + this.emit("backupMade", backupPath); + const allTables = await this.docStorage.fetchAllTables(); + const docActions: DocAction[] = await this._dataEngine.pyCall('create_migrations', allTables); + this.logInfo(docSession, "_migrate: applying %d migration actions", docActions.length); + docActions.forEach((action, i) => this.logInfo(docSession, "_migrate: docAction %s: %s", i, shortDesc(action))); + await this.docStorage.execTransaction(() => this.docStorage.applyStoredActions(docActions)); + } + + /** + * Load the specified tables into the data engine. + */ + private async _loadTables(docSession: OptDocSession, tableNames: string[]) { + this.logDebug(docSession, "loading %s tables: %s", tableNames.length, + tableNames.join(", ")); + // Pass the resulting array to `map`, which allows parallel processing of the tables. Database + // and DataEngine may still do things serially, but it allows them to be busy simultaneously. + await bluebird.map(tableNames, async (tableName: string) => + this._dataEngine.pyCall('load_table', tableName, await this._fetchTableIfPresent(tableName)), + // How many tables to query for and push to the data engine in parallel. + { concurrency: 3 }); + return this; + } + + // Fetches and returns the requested table, or null if it's missing. This allows documents to + // load with missing metadata tables (should only matter if migrations are also broken). + private async _fetchTableIfPresent(tableName: string): Promise { + try { + return await this.docStorage.fetchTable(tableName); + } catch (err) { + if (/no such table/.test(err.message)) { return null; } + throw err; + } + } + + // It's a bit risky letting "Calculate" (and other formula-dependent calls) to disable + // inactivityTimer, since a user formulas with an infinite loop can disable it forever. + // TODO find a solution to this issue. + @ActiveDoc.keepDocOpen + private async _finishInitialization(docSession: OptDocSession, pendingTableNames: string[], startTime: number) { + try { + await this._loadTables(docSession, pendingTableNames); + await this._applyUserActions(null, [['Calculate']]); + await this._reportDataEngineMemory(); + this._fullyLoaded = true; + const endTime = Date.now(); + const loadMs = endTime - startTime; + // Adjust the inactivity timer: if the load took under 1 sec, use the regular timeout; if it + // took longer, scale it up proportionately. + const closeTimeout = Math.max(loadMs, 1000) * Deps.ACTIVEDOC_TIMEOUT; + this._inactivityTimer.setDelay(closeTimeout); + this.logDebug(docSession, `loaded in ${loadMs} ms, InactivityTimer set to ${closeTimeout} ms`); + return true; + } catch (err) { + this.logWarn(docSession, "_finishInitialization stopped with %s", err); + this._fullyLoaded = true; + return false; + } + } + + private async _fetchQueryFromDB(query: Query, onDemand: boolean): Promise { + // Expand query to compute formulas (or include placeholders for them). + const expandedQuery = expandQuery(query, this.docData!, onDemand); + const marshalled = await this.docStorage.fetchQuery(expandedQuery); + const table = this.docStorage.decodeMarshalledData(marshalled, query.tableId); + + // Substitute in constant values for errors / placeholders. + if (expandedQuery.constants) { + for (const colId of Object.keys(expandedQuery.constants)) { + const constant = expandedQuery.constants[colId]; + table[colId] = table[colId].map(() => constant); + } + } + return toTableDataAction(query.tableId, table); + } + + private async _fetchQueryFromDataEngine(query: Query): Promise { + return this._dataEngine.pyCall('fetch_table', query.tableId, true, query.filters); + } + + private async _reportDataEngineMemory() { + const now = Date.now(); + if (now >= this._lastMemoryMeasurement + MEMORY_MEASUREMENT_INTERVAL_MS) { + this._lastMemoryMeasurement = now; + await this._dataEngine.reportMemoryUsage(); + } + } + + private _log(level: string, cli: Client|OptDocSession|null, msg: string, ...args: any[]) { + log.origLog(level, `ActiveDoc ` + msg, ...args, this.getLogMeta(cli)); + } +} + +// Helper to initialize a sandbox action bundle with no values. +function createEmptySandboxActionBundle(): SandboxActionBundle { + return { + envelopes: [], + stored: [], + calc: [], + undo: [], + retValues: [] + }; +} diff --git a/app/server/lib/ActiveDocImport.ts b/app/server/lib/ActiveDocImport.ts new file mode 100644 index 00000000..3d39b708 --- /dev/null +++ b/app/server/lib/ActiveDocImport.ts @@ -0,0 +1,311 @@ +/* Helper file to separate ActiveDoc import functions and convert them to TypeScript. */ + +import * as path from 'path'; +import * as _ from 'underscore'; + +import {DataSourceTransformed, ImportResult, ImportTableResult, TransformRuleMap} from 'app/common/ActiveDocAPI'; +import {ApplyUAResult} from 'app/common/ActiveDocAPI'; +import {ApiError} from 'app/common/ApiError'; +import * as gutil from 'app/common/gutil'; +import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI'; +import {GristTable} from 'app/plugin/GristTable'; +import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import {DocSession, OptDocSession} from 'app/server/lib/DocSession'; +import * as log from 'app/server/lib/log'; +import {globalUploadSet, moveUpload, UploadInfo} from 'app/server/lib/uploads'; + + +/* + * AddTableRetValue contains return value of user actions 'AddTable' +*/ +interface AddTableRetValue { + table_id: string; + id: number; + columns: string[]; + views: object[]; +} + +interface ReferenceDescription { + // the table index + tableIndex: number; + // the column index + colIndex: number; + // the id of the table which is referenced + refTableId: string; +} + +export class ActiveDocImport { + constructor(private _activeDoc: ActiveDoc) {} + /** + * Imports files, removes previously created temporary hidden tables and creates the new ones + */ + public async importFiles(docSession: DocSession, dataSource: DataSourceTransformed, + parseOptions: ParseOptions, prevTableIds: string[]): Promise { + this._activeDoc.startBundleUserActions(docSession); + await this._removeHiddenTables(docSession, prevTableIds); + const userId = docSession.authorizer.getUserId(); + const accessId = this._activeDoc.makeAccessId(userId); + const uploadInfo: UploadInfo = globalUploadSet.getUploadInfo(dataSource.uploadId, accessId); + return this._importFiles(docSession, uploadInfo, dataSource.transforms, parseOptions, true); + } + + /** + * Finishes import files, removes temporary hidden tables, temporary uploaded files and creates + * the new tables + */ + public async finishImportFiles(docSession: DocSession, dataSource: DataSourceTransformed, + parseOptions: ParseOptions, prevTableIds: string[]): Promise { + this._activeDoc.startBundleUserActions(docSession); + try { + await this._removeHiddenTables(docSession, prevTableIds); + const userId = docSession.authorizer.getUserId(); + const accessId = this._activeDoc.makeAccessId(userId); + const uploadInfo: UploadInfo = globalUploadSet.getUploadInfo(dataSource.uploadId, accessId); + const importResult = await this._importFiles(docSession, uploadInfo, dataSource.transforms, + parseOptions, false); + await globalUploadSet.cleanup(dataSource.uploadId); + return importResult; + } finally { + this._activeDoc.stopBundleUserActions(docSession); + } + } + + /** + * Cancels import files, removes temporary hidden tables and temporary uploaded files + * + * @param {ActiveDoc} activeDoc: Instance of ActiveDoc. + * @param {DataSourceTransformed} dataSource: an array of DataSource + * @param {Array} prevTableIds: Array of tableIds as received from previous `importFiles` + * call when re-importing with changed `parseOptions`. + * @returns {Promise} Promise that's resolved when all actions are applied successfully. + */ + public async cancelImportFiles(docSession: DocSession, + dataSource: DataSourceTransformed, + prevTableIds: string[]): Promise { + await this._removeHiddenTables(docSession, prevTableIds); + this._activeDoc.stopBundleUserActions(docSession); + await globalUploadSet.cleanup(dataSource.uploadId); + } + + /** + * Import the given upload as new tables in one step. This does not give the user a chance to + * modify parse options or transforms. The caller is responsible for cleaning up the upload. + */ + public async oneStepImport(docSession: OptDocSession, uploadInfo: UploadInfo): Promise { + this._activeDoc.startBundleUserActions(docSession); + try { + return this._importFiles(docSession, uploadInfo, [], {}, false); + } finally { + this._activeDoc.stopBundleUserActions(docSession); + } + } + + /** + * Imports all files as new tables, using the given transform rules and parse options. + * The isHidden flag indicates whether to create temporary hidden tables, or final ones. + */ + private async _importFiles(docSession: OptDocSession, upload: UploadInfo, transforms: TransformRuleMap[], + parseOptions: ParseOptions, isHidden: boolean): Promise { + + // Check that upload size is within the configured limits. + const limit = (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || Infinity; + const totalSize = upload.files.reduce((acc, f) => acc + f.size, 0); + if (totalSize > limit) { + throw new ApiError(`Imported files must not exceed ${gutil.byteString(limit)}`, 413); + } + + // The upload must be within the plugin-accessible directory. Once moved, subsequent calls to + // moveUpload() will return without having to do anything. + await moveUpload(upload, this._activeDoc.docPluginManager.tmpDir()); + + const importResult: ImportResult = {options: parseOptions, tables: []}; + for (const [index, file] of upload.files.entries()) { + // If we have a better guess for the file's extension, replace it in origName, to ensure + // that DocPluginManager has access to it to guess the best parser type. + let origName: string = file.origName; + if (file.ext) { + origName = path.basename(origName, path.extname(origName)) + file.ext; + } + const res = await this._importFileAsNewTable(docSession, index, file.absPath, origName, + parseOptions, isHidden, transforms[index] || {}); + if (index === 0) { + // Returned parse options from the first file should be used for all files in one upload. + importResult.options = parseOptions = res.options; + } + importResult.tables.push(...res.tables); + } + return importResult; + } + + /** + * Imports the data stored at tmpPath. + * + * Currently it starts a python parser (that relies on the messytables library) as a child process + * outside the sandbox, and supports xls(x), csv, txt, and perhaps some other formats. It may + * result in the import of multiple tables, in case of e.g. Excel formats. + * @param {ActiveDoc} activeDoc: Instance of ActiveDoc. + * @param {Number} dataSourceIdx: Index of original dataSourse corresponding to current imported file. + * @param {String} tmpPath: The path from of the original file. + * @param {String} originalFilename: Suggested name of the import file. It is sometimes used as a + * suggested table name, e.g. for csv imports. + * @param {String} options: Containing parseOptions as serialized JSON to pass to the import plugin. + * @param {Boolean} isHidden: Flag to indicate whether table is temporary and hidden or regular. + * @param {TransformRuleMap} transformRuleMap: Containing transform rules for each table in file such as + * `destTableId`, `destCols`, `sourceCols`. + * @returns {Promise} with `options` property containing parseOptions as serialized JSON as adjusted + * or guessed by the plugin, and `tables`, which is which is a list of objects with information about + * tables, such as `hiddenTableId`, `dataSourceIndex`, `origTableName`, `transformSectionRef`, `destTableId`. + */ + private async _importFileAsNewTable(docSession: OptDocSession, uploadFileIndex: number, tmpPath: string, + originalFilename: string, + options: ParseOptions, isHidden: boolean, + transformRuleMap: TransformRuleMap|undefined): Promise { + log.info("ActiveDoc._importFileAsNewTable(%s, %s)", tmpPath, originalFilename); + const optionsAndData: ParseFileResult = await this._activeDoc.docPluginManager.parseFile(tmpPath, + originalFilename, options); + options = optionsAndData.parseOptions; + + const parsedTables = optionsAndData.tables; + const references = this._encodeReferenceAsInt(parsedTables); + + const tables: ImportTableResult[] = []; + const fixedColumnIdsByTable: { [tableId: string]: string[]; } = {}; + + for (const table of parsedTables) { + const ext = path.extname(originalFilename); + const basename = path.basename(originalFilename, ext).trim(); + const hiddenTableName = 'GristHidden_import'; + const origTableName = table.table_name ? table.table_name : ''; + const transformRule = transformRuleMap && transformRuleMap.hasOwnProperty(origTableName) ? + transformRuleMap[origTableName] : null; + const result: ApplyUAResult = await this._activeDoc.applyUserActions(docSession, + [["AddTable", hiddenTableName, table.column_metadata]]); + const retValue: AddTableRetValue = result.retValues[0]; + const hiddenTableId = retValue.table_id; // The sanitized version of the table name. + const hiddenTableColIds = retValue.columns; // The sanitized names of the columns. + + // The table_data received from importFile is an array of columns of data, rather than a + // dictionary, so that it doesn't depend on column names. We instead construct the + // dictionary once we receive the sanitized column names from AddTable. + const dataLength = table.table_data[0] ? table.table_data[0].length : 0; + log.info("Importing table %s, %s rows, from %s", hiddenTableId, dataLength, table.table_name); + + const rowIdColumn = _.range(1, dataLength + 1); + const columnValues = _.object(hiddenTableColIds, table.table_data); + const destTableId = transformRule ? transformRule.destTableId : null; + const ruleCanBeApplied = (transformRule != null) && + _.difference(transformRule.sourceCols, hiddenTableColIds).length === 0; + await this._activeDoc.applyUserActions(docSession, + [["ReplaceTableData", hiddenTableId, rowIdColumn, columnValues]]); + + // data parsed and put into hiddenTableId + // For preview_table (isHidden) do GenImporterView to make views and formulas and cols + // For final import, call TransformAndFinishImport, which imports file using a transform rule (or blank) + + let createdTableId: string; + let transformSectionRef: number = -1; // TODO: we only have this if we genImporterView, is it necessary? + + if (isHidden) { + // Generate formula columns, view sections, etc + const results: ApplyUAResult = await this._activeDoc.applyUserActions(docSession, + [['GenImporterView', hiddenTableId, destTableId, ruleCanBeApplied ? transformRule : null]]); + + transformSectionRef = results.retValues[0]; + createdTableId = hiddenTableId; + + } else { + // Do final import + const intoNewTable: boolean = destTableId ? false : true; + const destTable = destTableId || table.table_name || basename; + const tableId = await this._activeDoc.applyUserActions(docSession, + [['TransformAndFinishImport', + hiddenTableId, destTable, intoNewTable, + ruleCanBeApplied ? transformRule : null]]); + + createdTableId = tableId.retValues[0]; // this is garbage for now I think? + + } + + fixedColumnIdsByTable[createdTableId] = hiddenTableColIds; + + + tables.push({ + hiddenTableId: createdTableId, // TODO: rename thing? + uploadFileIndex, + origTableName, + transformSectionRef, // TODO: this shouldnt always be needed, and we only get it if genimporttransform + destTableId + }); + } + + await this._fixReferences(docSession, parsedTables, tables, fixedColumnIdsByTable, references, isHidden); + + return ({options, tables}); + } + + /** + * This function removes temporary hidden tables which were created during the import process + * + * @param {Array[String]} hiddenTableIds: Array of hidden table ids + * @returns {Promise} Promise that's resolved when all actions are applied successfully. + */ + private async _removeHiddenTables(docSession: DocSession, hiddenTableIds: string[]) { + if (hiddenTableIds.length !== 0) { + await this._activeDoc.applyUserActions(docSession, hiddenTableIds.map(t => ['RemoveTable', t])); + } + } + + /** + * The methods changes every column of references into a column of integers in `parsedTables`. It + * returns `parsedTable` and a list of descriptors of all columns of references. + */ + private _encodeReferenceAsInt(parsedTables: GristTable[]): ReferenceDescription[] { + const references = []; + for (const [tableIndex, parsedTable] of parsedTables.entries()) { + for (const [colIndex, col] of parsedTable.column_metadata.entries()) { + const refTableId = gutil.removePrefix(col.type, "Ref:"); + if (refTableId) { + references.push({refTableId, colIndex, tableIndex}); + col.type = 'Int'; + } + } + } + return references; + } + + /** + * This function fix references that are broken by the change of table id. + */ + private async _fixReferences(docSession: OptDocSession, + parsedTables: GristTable[], + tables: ImportTableResult[], + fixedColumnIds: { [tableId: string]: string[]; }, + references: ReferenceDescription[], + isHidden: boolean) { + + // collect all new table ids + const tablesByOrigName = _.indexBy(tables, 'origTableName'); + + // gather all of the user actions + let userActions: any[] = references.map( ref => { + const fixedTableId = tables[ref.tableIndex].hiddenTableId; + return [ + 'ModifyColumn', + fixedTableId, + fixedColumnIds[fixedTableId][ref.colIndex], + { type: `Ref:${tablesByOrigName[ref.refTableId].hiddenTableId}` } + ]; + }); + + if (isHidden) { + userActions = userActions.concat(userActions.map(([, tableId, columnId, colInfo]) => [ + 'ModifyColumn', tableId, 'gristHelper_Import_' + columnId, colInfo ])); + } + + // apply user actions + if (userActions.length) { + await this._activeDoc.applyUserActions(docSession, userActions); + } + + } +} diff --git a/app/server/lib/AppEndpoint.ts b/app/server/lib/AppEndpoint.ts new file mode 100644 index 00000000..b42cc21b --- /dev/null +++ b/app/server/lib/AppEndpoint.ts @@ -0,0 +1,254 @@ +/** + * AppServer serves up the main app.html file to the browser. It is the first point of contact of + * a browser with Grist. It handles sessions, redirect-to-login, and serving up a suitable version + * of the client-side code. + */ +import * as express from 'express'; +import fetch, {RequestInit, Response as FetchResponse} from 'node-fetch'; + +import {ApiError} from 'app/common/ApiError'; +import {getSlugIfNeeded, isOrgInPathOnly, + parseSubdomainStrictly} from 'app/common/gristUrls'; +import {removeTrailingSlash} from 'app/common/gutil'; +import {Document as APIDocument} from 'app/common/UserAPI'; +import {Document} from "app/gen-server/entity/Document"; +import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import {assertAccess, getTransitiveHeaders, getUserId, RequestWithLogin} from 'app/server/lib/Authorizer'; +import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; +import {expressWrap} from 'app/server/lib/expressWrap'; +import {getAssignmentId} from 'app/server/lib/idUtils'; +import * as log from 'app/server/lib/log'; +import {adaptServerUrl, pruneAPIResult, trustOrigin} from 'app/server/lib/requestUtils'; +import {ISendAppPageOptions} from 'app/server/lib/sendAppPage'; + +export interface AttachOptions { + app: express.Application; // Express app to which to add endpoints + middleware: express.RequestHandler[]; // Middleware to apply for all endpoints + docWorkerMap: IDocWorkerMap|null; + sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise; + dbManager: HomeDBManager; +} + +/** + * This method transforms a doc worker's public url as needed based on the request. + * + * For historic reasons, doc workers are assigned a public url at the time + * of creation. In production/staging, this is of the form: + * https://doc-worker-NNN-NNN-NNN-NNN.getgrist.com/v/VVVV/ + * and in dev: + * http://localhost:NNNN/v/VVVV/ + * + * Prior to support for different base domains, this was fine. Now that different + * base domains are supported, a wrinkle arises. When a web client communicates + * with a doc worker, it is important that it accesses the doc worker via a url + * containing the same base domain as the web page the client is on (for cookie + * purposes). Hence this method. + * + * If both the request and docWorkerUrl contain identifiable base domains (not localhost), + * then the base domain of docWorkerUrl is replaced with that of the request. + * + * But wait, there's another wrinkle: custom domains. In this case, we have a single + * domain available to serve a particular org from. This method will use the origin of req + * and include a /dw/doc-worker-NNN-NNN-NNN-NNN/ + * (or /dw/local-NNNN/) prefix in all doc worker paths. Once this is in place, it + * will allow doc worker routing to be changed so it can be overlaid on a custom + * domain. + * + * TODO: doc worker registration could be redesigned to remove the assumption + * of a fixed base domain. + */ +function customizeDocWorkerUrl(docWorkerUrlSeed: string, req: express.Request) { + const docWorkerUrl = new URL(docWorkerUrlSeed); + const workerSubdomain = parseSubdomainStrictly(docWorkerUrl.hostname).org; + adaptServerUrl(docWorkerUrl, req); + + // We wish to migrate to routing doc workers by path, so insert a doc worker identifier + // in the path (if not already present). + if (!docWorkerUrl.pathname.startsWith('/dw/')) { + // When doc worker is localhost, the port number is necessary and sufficient for routing. + // Let's add a /dw/... prefix just for consistency. + const workerIdent = workerSubdomain || `local-${docWorkerUrl.port}`; + docWorkerUrl.pathname = `/dw/${workerIdent}${docWorkerUrl.pathname}`; + } + return docWorkerUrl.href; +} + +/** + * + * Gets the worker responsible for a given assignment, and fetches a url + * from the worker. + * + * If the fetch fails, we throw an exception, unless we see enough evidence + * to unassign the worker and try again. + * + * - If GRIST_MANAGED_WORKERS is set, we assume that we've arranged + * for unhealthy workers to be removed automatically, and that if a + * fetch returns a 404 with specific content, it is proof that the + * worker is no longer in existence. So if we see a 404 with that + * specific content, we can safely de-list the worker from redis, + * and repeat. + * - If GRIST_MANAGED_WORKERS is not set, we accept a broader set + * of failures as evidence of a missing worker. + * + * The specific content of a 404 that will be treated as evidence of + * a doc worker not being present is: + * - A json format body + * - With a key called "message" + * - With the value of "message" being "document worker not present" + * In production, this is provided by a special doc-worker-* load balancer + * rule. + * + */ +async function getWorker(docWorkerMap: IDocWorkerMap, assignmentId: string, + urlPath: string, config: RequestInit = {}) { + let docStatus: DocStatus|undefined; + const workersAreManaged = Boolean(process.env.GRIST_MANAGED_WORKERS); + for (;;) { + docStatus = await docWorkerMap.assignDocWorker(assignmentId); + const configWithTimeout = {timeout: 10000, ...config}; + const fullUrl = removeTrailingSlash(docStatus.docWorker.internalUrl) + urlPath; + try { + const resp: FetchResponse = await fetch(fullUrl, configWithTimeout); + if (resp.ok) { + return { + resp, + docStatus, + }; + } + if (resp.status === 403) { + throw new ApiError("You do not have access to this document.", resp.status); + } + if (resp.status !== 404) { + throw new ApiError(resp.statusText, resp.status); + } + let body: any; + try { + body = await resp.json(); + } catch (e) { + throw new ApiError(resp.statusText, resp.status); + } + if (!(body && body.message && body.message === 'document worker not present')) { + throw new ApiError(resp.statusText, resp.status); + } + // This is a 404 with the expected content for a missing worker. + } catch (e) { + // If workers are managed, no errors merit continuing except a 404. + // Otherwise, we continue if we see a system error (e.g. ECONNREFUSED). + // We don't accept timeouts since there is too much potential to + // bring down a single-worker deployment that has a hiccup. + if (workersAreManaged || !(e.type === 'system')) { + throw e; + } + } + log.warn(`fetch from ${fullUrl} failed convincingly, removing that worker`); + await docWorkerMap.removeWorker(docStatus.docWorker.id); + docStatus = undefined; + } +} + +export function attachAppEndpoint(options: AttachOptions): void { + const {app, middleware, docWorkerMap, sendAppPage, dbManager} = options; + // Per-workspace URLs open the same old Home page, and it's up to the client to notice and + // render the right workspace. + app.get(['/', '/ws/:wsId', '/p/:page'], ...middleware, expressWrap(async (req, res) => + sendAppPage(req, res, {path: 'app.html', status: 200, config: {}, googleTagManager: 'anon'}))); + + app.get('/api/worker/:assignmentId([^/]+)/?*', expressWrap(async (req, res) => { + if (!trustOrigin(req, res)) { throw new Error('Unrecognized origin'); } + res.header("Access-Control-Allow-Credentials", "true"); + + if (!docWorkerMap) { + return res.status(500).json({error: 'no worker map'}); + } + const assignmentId = getAssignmentId(docWorkerMap, req.params.assignmentId); + const {docStatus} = await getWorker(docWorkerMap, assignmentId, '/status'); + if (!docStatus) { + return res.status(500).json({error: 'no worker'}); + } + res.json({docWorkerUrl: customizeDocWorkerUrl(docStatus.docWorker.publicUrl, req)}); + })); + + // Handler for serving the document landing pages. Expects the following parameters: + // urlId, slug (optional), remainder + // This handler is used for both "doc/urlId" and "urlId/slug" style endpoints. + const docHandler = expressWrap(async (req, res, next) => { + if (req.params.slug && req.params.slug === 'app.html') { + // This can happen on a single-port configuration, since "docId/app.html" matches + // the "urlId/slug" pattern. Luckily the "." character is not allowed in slugs. + return next(); + } + if (!docWorkerMap) { + return await sendAppPage(req, res, {path: 'app.html', status: 200, config: {}, + googleTagManager: 'anon'}); + } + const mreq = req as RequestWithLogin; + const urlId = req.params.urlId; + let doc: Document|null = null; + try { + const userId = getUserId(mreq); + + // Query DB for the doc metadata, to include in the page (as a pre-fetch of getDoc() call), + // and to get fresh (uncached) access info. + doc = await dbManager.getDoc({userId, org: mreq.org, urlId}); + const slug = getSlugIfNeeded(doc); + + const slugMismatch = (req.params.slug || null) !== (slug || null); + const preferredUrlId = doc.urlId || doc.id; + if (urlId !== preferredUrlId || slugMismatch) { + // Prepare to redirect to canonical url for document. + // Preserve org in url path if necessary. + const prefix = isOrgInPathOnly(req.hostname) ? `/o/${mreq.org}` : ''; + // Preserve any query parameters or fragments. + const queryOrFragmentCheck = req.originalUrl.match(/([#?].*)/); + const queryOrFragment = (queryOrFragmentCheck && queryOrFragmentCheck[1]) || ''; + if (slug) { + res.redirect(`${prefix}/${preferredUrlId}/${slug}${req.params.remainder}${queryOrFragment}`); + } else { + res.redirect(`${prefix}/doc/${preferredUrlId}${req.params.remainder}${queryOrFragment}`); + } + return; + } + + // The docAuth value will be cached from the getDoc() above (or could be derived from doc). + const docAuth = await dbManager.getDocAuthCached({userId, org: mreq.org, urlId}); + assertAccess('viewers', docAuth); + + } catch (err) { + if (err.status === 404) { + log.info("/:urlId/app.html did not find doc", mreq.userId, urlId, doc && doc.access, mreq.org); + throw new ApiError('Document not found.', 404); + } else if (err.status === 403) { + log.info("/:urlId/app.html denied access", mreq.userId, urlId, doc && doc.access, mreq.org); + throw new ApiError('You do not have access to this document.', 403); + } + throw err; + } + + // The reason to pass through app.html fetched from docWorker is in case it is a different + // version of Grist (could be newer or older). + // TODO: More must be done for correct version tagging of URLs: assumes all + // links and static resources come from the same host, but we'll have Home API, DocWorker, + // and static resources all at hostnames different from where this page is served. + // TODO docWorkerMain needs to serve app.html, perhaps with correct base-href already set. + const docId = doc.id; + const headers = { + Accept: 'application/json', + ...getTransitiveHeaders(req), + }; + const {docStatus, resp} = await getWorker(docWorkerMap, docId, + `/${docId}/app.html`, {headers}); + const body = await resp.json(); + + await sendAppPage(req, res, {path: "", content: body.page, tag: body.tag, status: 200, + googleTagManager: 'anon', config: { + assignmentId: docId, + getWorker: {[docId]: customizeDocWorkerUrl(docStatus.docWorker.publicUrl, req)}, + getDoc: {[docId]: pruneAPIResult(doc as unknown as APIDocument)}, + }}); + }); + // The * is a wildcard in express 4, rather than a regex symbol. + // See https://expressjs.com/en/guide/routing.html + app.get('/doc/:urlId([^/]+):remainder(*)', ...middleware, docHandler); + app.get('/:urlId([^/]{12,})/:slug([^/]+):remainder(*)', + ...middleware, docHandler); +} diff --git a/app/server/lib/Authorizer.ts b/app/server/lib/Authorizer.ts new file mode 100644 index 00000000..1595ed2c --- /dev/null +++ b/app/server/lib/Authorizer.ts @@ -0,0 +1,429 @@ +import {ApiError} from 'app/common/ApiError'; +import {OpenDocMode} from 'app/common/DocListAPI'; +import {ErrorWithCode} from 'app/common/ErrorWithCode'; +import {UserProfile} from 'app/common/LoginSessionAPI'; +import {canEdit, canView, getWeakestRole, Role} from 'app/common/roles'; +import {Document} from 'app/gen-server/entity/Document'; +import {User} from 'app/gen-server/entity/User'; +import {DocAuthKey, DocAuthResult, HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import {getSessionProfiles, getSessionUser, linkOrgWithEmail, SessionObj, + SessionUserObj} from 'app/server/lib/BrowserSession'; +import {RequestWithOrg} from 'app/server/lib/extractOrg'; +import {COOKIE_MAX_AGE, getAllowedOrgForSessionID} from 'app/server/lib/gristSessions'; +import * as log from 'app/server/lib/log'; +import {IPermitStore, Permit} from 'app/server/lib/Permit'; +import {allowHost} from 'app/server/lib/requestUtils'; +import {NextFunction, Request, RequestHandler, Response} from 'express'; + +export interface RequestWithLogin extends Request { + sessionID: string; + session: SessionObj; + org?: string; + isCustomHost?: boolean; // when set, the request's domain is a recognized custom host linked + // with the specified org. + users?: UserProfile[]; + userId?: number; + user?: User; + userIsAuthorized?: boolean; // If userId is for "anonymous", this will be false. + docAuth?: DocAuthResult; // For doc requests, the docId and the user's access level. + specialPermit?: Permit; +} + +/** + * Extract the user id from a request, assuming we've added it via appropriate middleware. + * Throws ApiError with code 401 (unauthorized) if the user id is missing. + */ +export function getUserId(req: Request): number { + const userId = (req as RequestWithLogin).userId; + if (!userId) { + throw new ApiError("user not known", 401); + } + return userId; +} + +/** + * Extract the user object from a request, assuming we've added it via appropriate middleware. + * Throws ApiError with code 401 (unauthorized) if the user is missing. + */ +export function getUser(req: Request): User { + const user = (req as RequestWithLogin).user; + if (!user) { + throw new ApiError("user not known", 401); + } + return user; +} + +/** + * Extract the user profiles from a request, assuming we've added them via appropriate middleware. + * Throws ApiError with code 401 (unauthorized) if the profiles are missing. + */ +export function getUserProfiles(req: Request): UserProfile[] { + const users = (req as RequestWithLogin).users; + if (!users) { + throw new ApiError("user profile not found", 401); + } + return users; +} + +// Extract the user id from a request, requiring it to be authorized (not an anonymous session). +export function getAuthorizedUserId(req: Request) { + const userId = getUserId(req); + if (isAnonymousUser(req)) { + throw new ApiError("user not authorized", 401); + } + return userId; +} + +export function isAnonymousUser(req: Request) { + return !(req as RequestWithLogin).userIsAuthorized; +} + +// True if Grist is configured for a single user without specific authorization +// (classic standalone/electron mode). +export function isSingleUserMode(): boolean { + return process.env.GRIST_SINGLE_USER === '1'; +} + +/** + * Returns the express request object with user information added, if it can be + * found based on passed in headers or the session. Specifically, sets: + * - req.userId: the id of the user in the database users table + * - req.userIsAuthorized: set if user has presented credentials that were accepted + * (the anonymous user has a userId but does not have userIsAuthorized set if, + * as would typically be the case, credentials were not presented) + * - req.users: set for org-and-session-based logins, with list of profiles in session + */ +export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPermitStore, + fallbackEmail: string|null, + req: Request, res: Response, next: NextFunction) { + const mreq = req as RequestWithLogin; + let profile: UserProfile|undefined; + + // First, check for an apiKey + if (mreq.headers && mreq.headers.authorization) { + // header needs to be of form "Bearer XXXXXXXXX" to apply + const parts = String(mreq.headers.authorization).split(' '); + if (parts[0] === "Bearer") { + const user = parts[1] ? await dbManager.getUserByKey(parts[1]) : undefined; + if (!user) { + return res.status(401).send('Bad request: invalid API key'); + } + if (user.id === dbManager.getAnonymousUserId()) { + // We forbid the anonymous user to present an api key. That saves us + // having to think through the consequences of authorized access to the + // anonymous user's profile via the api (e.g. how should the api key be managed). + return res.status(401).send('Credentials cannot be presented for the anonymous user account via API key'); + } + mreq.user = user; + mreq.userId = user.id; + mreq.userIsAuthorized = true; + } + } + + // Special permission header for internal housekeeping tasks + if (mreq.headers && mreq.headers.permit) { + const permitKey = String(mreq.headers.permit); + try { + const permit = await permitStore.getPermit(permitKey); + if (!permit) { return res.status(401).send('Bad request: unknown permit'); } + mreq.user = dbManager.getAnonymousUser(); + mreq.userId = mreq.user.id; + mreq.specialPermit = permit; + } catch (err) { + log.error(`problem reading permit: ${err}`); + return res.status(401).send('Bad request: permit could not be read'); + } + } + + // A bit of extra info we'll add to the "Auth" log message when this request passes the check + // for custom-host-specific sessionID. + let customHostSession = ''; + + // If we haven't selected a user by other means, and have profiles available in the + // session, then select a user based on those profiles. + const session = mreq.session; + if (!mreq.userId && session && session.users && session.users.length > 0 && + mreq.org !== undefined) { + + // Prevent using custom-domain sessionID to authorize to a different domain, since + // custom-domain owner could hijack such sessions. + const allowedOrg = getAllowedOrgForSessionID(mreq.sessionID); + if (allowedOrg) { + if (allowHost(req, allowedOrg.host)) { + customHostSession = ` custom-host-match ${allowedOrg.host}`; + } else { + // We need an exception for internal forwarding from home server to doc-workers. These use + // internal hostnames, so we can't expect a custom domain. These requests do include an + // Organization header, which we'll use to grant the exception, but security issues remain. + // TODO Issue 1: an attacker can use a custom-domain request to get an API key, which is an + // open door to all orgs accessible by this user. + // TODO Issue 2: Organization header is easy for an attacker (who has stolen a session + // cookie) to include too; it does nothing to prove that the request is internal. + const org = req.header('organization'); + if (org && org === allowedOrg.org) { + customHostSession = ` custom-host-fwd ${org}`; + } else { + // Log error and fail. + log.warn("Auth[%s]: sessionID for host %s org %s; wrong for host %s org %s", mreq.method, + allowedOrg.host, allowedOrg.org, mreq.get('host'), mreq.org); + return res.status(403).send('Bad request: invalid session ID'); + } + } + } + + mreq.users = getSessionProfiles(session); + + // If we haven't set a maxAge yet, set it now. + if (session && session.cookie && !session.cookie.maxAge) { + session.cookie.maxAge = COOKIE_MAX_AGE; + } + + // See if we have a profile linked with the active organization already. + let sessionUser: SessionUserObj|null = getSessionUser(session, mreq.org); + + if (!sessionUser) { + // No profile linked yet, so let's elect one. + // Choose a profile that is no worse than the others available. + const option = await dbManager.getBestUserForOrg(mreq.users, mreq.org); + if (option) { + // Modify request session object to link the current org with our choice of + // profile. Express-session will save this change. + sessionUser = linkOrgWithEmail(session, option.email, mreq.org); + // In this special case of initially linking a profile, we need to look up the user's info. + mreq.user = await dbManager.getUserByLogin(option.email); + mreq.userId = option.id; + mreq.userIsAuthorized = true; + } else { + // No profile has access to this org. We could choose to + // link no profile, in which case user will end up + // immediately presented with a sign-in page, or choose to + // link an arbitrary profile (say, the first one the user + // logged in as), in which case user will end up with a + // friendlier page explaining the situation and offering to + // add an account to resolve it. We go ahead and pick an + // arbitrary profile. + sessionUser = session.users[0]; + if (!session.orgToUser) { throw new Error("Session misconfigured"); } + // Express-session will save this change. + session.orgToUser[mreq.org] = 0; + } + } + + profile = sessionUser && sessionUser.profile || undefined; + + // If we haven't computed a userId yet, check for one using an email address in the profile. + // A user record will be created automatically for emails we've never seen before. + if (profile && !mreq.userId) { + const user = await dbManager.getUserByLoginWithRetry(profile.email, profile); + if (user) { + mreq.user = user; + mreq.userId = user.id; + mreq.userIsAuthorized = true; + } + } + } + + if (!mreq.userId && fallbackEmail) { + const user = await dbManager.getUserByLogin(fallbackEmail); + if (user) { + mreq.user = user; + mreq.userId = user.id; + mreq.userIsAuthorized = true; + const fullUser = dbManager.makeFullUser(user); + mreq.users = [fullUser]; + profile = fullUser; + } + } + + // If no userId has been found yet, fall back on anonymous. + if (!mreq.userId) { + const anon = dbManager.getAnonymousUser(); + mreq.user = anon; + mreq.userId = anon.id; + mreq.userIsAuthorized = false; + mreq.users = [dbManager.makeFullUser(anon)]; + } + + log.debug("Auth[%s]: id %s email %s host %s path %s org %s%s", mreq.method, + mreq.userId, profile && profile.email, mreq.get('host'), mreq.path, mreq.org, + customHostSession); + + return next(); +} + + +/** + * Middleware to redirects user to a login page when the user is not + * logged in. If allowExceptions is set, then we make an exception + * for a team site allowing anonymous access, or a personal doc + * allowing anonymous access, or the merged org. + */ +export function redirectToLogin( + allowExceptions: boolean, + getLoginRedirectUrl: (redirectUrl: URL) => Promise, + getSignUpRedirectUrl: (redirectUrl: URL) => Promise, + dbManager: HomeDBManager +): RequestHandler { + return async (req: Request, resp: Response, next: NextFunction) => { + const mreq = req as RequestWithLogin; + mreq.session.alive = true; // This will ensure that express-session will set our cookie + // if it hasn't already - we'll need it if we redirect. + if (mreq.userIsAuthorized) { return next(); } + + try { + // Otherwise it's an anonymous user. Proceed normally only if the org allows anon access. + if (mreq.userId && mreq.org && allowExceptions) { + // Anonymous user has qualified access to merged org. + if (dbManager.isMergedOrg(mreq.org)) { return next(); } + const result = await dbManager.getOrg({userId: mreq.userId}, mreq.org || null); + if (result.status === 200) { return next(); } + } + + // In all other cases (including unknown org), redirect user to login or sign up. + // Redirect to sign up if it doesn't look like the user has ever logged in (on + // this browser) After logging in, `users` will be set in the session. Even after + // logging out again, `users` will still be set. + const signUp: boolean = (mreq.session.users === undefined); + log.debug(`Authorizer: redirecting to ${signUp ? 'sign up' : 'log in'}`); + const redirectUrl = new URL(req.protocol + '://' + req.get('host') + req.originalUrl); + if (signUp) { + return resp.redirect(await getSignUpRedirectUrl(redirectUrl)); + } else { + return resp.redirect(await getLoginRedirectUrl(redirectUrl)); + } + + } catch (err) { + log.info("Authorizer failed to redirect", err.message); + return resp.status(401).send(err.message); + } + }; +} + +/** + * Sets mreq.docAuth if not yet set, and returns it. + */ +export async function getOrSetDocAuth( + mreq: RequestWithLogin, dbManager: HomeDBManager, urlId: string +): Promise { + if (!mreq.docAuth) { + let effectiveUserId = getUserId(mreq); + if (mreq.specialPermit && mreq.userId === dbManager.getAnonymousUserId()) { + effectiveUserId = dbManager.getPreviewerUserId(); + } + mreq.docAuth = await dbManager.getDocAuthCached({urlId, userId: effectiveUserId, org: mreq.org}); + if (mreq.specialPermit && mreq.userId === dbManager.getAnonymousUserId() && + mreq.specialPermit.docId === mreq.docAuth.docId) { + mreq.docAuth = {...mreq.docAuth, access: 'owners'}; + } + } + return mreq.docAuth; +} + + +export interface ResourceSummary { + kind: 'doc'; + id: string|number; +} + +/** + * + * Handle authorization for a single resource accessed by a given user. + * + */ +export interface Authorizer { + // get the id of user, or null if no authorization in place. + getUserId(): number|null; + + // Fetch the doc metadata from HomeDBManager. + getDoc(): Promise; + + // Check access, throw error if the requested level of access isn't available. + assertAccess(role: 'viewers'|'editors'): Promise; +} + +/** + * + * Handle authorization for a single document and user. + * + */ +export class DocAuthorizer implements Authorizer { + constructor( + private _dbManager: HomeDBManager, + private _key: DocAuthKey, + public readonly openMode: OpenDocMode, + ) { + } + + public getUserId(): number { + return this._key.userId; + } + + public async getDoc(): Promise { + return this._dbManager.getDoc(this._key); + } + + public async assertAccess(role: 'viewers'|'editors'): Promise { + const docAuth = await this._dbManager.getDocAuthCached(this._key); + assertAccess(role, docAuth, {openMode: this.openMode}); + } +} + +export class DummyAuthorizer implements Authorizer { + constructor(public role: Role|null) {} + public getUserId() { return null; } + public async getDoc(): Promise { throw new Error("Not supported in standalone"); } + public async assertAccess() { /* noop */ } +} + + +export function assertAccess( + role: 'viewers'|'editors', docAuth: DocAuthResult, options: { + openMode?: OpenDocMode, + allowRemoved?: boolean, + } = {}) { + const openMode = options.openMode || 'default'; + const details = {status: 403, accessMode: openMode}; + if (docAuth.error) { + if ([400, 401, 403].includes(docAuth.error.status)) { + // For these error codes, we know our access level - forbidden. Make errors more uniform. + throw new ErrorWithCode("AUTH_NO_VIEW", "No view access", details); + } + throw docAuth.error; + } + + if (docAuth.removed && !options.allowRemoved) { + throw new ErrorWithCode("AUTH_NO_VIEW", "Document is deleted", {status: 404}); + } + + // If docAuth has no error, the doc is accessible, but we should still check the level (in case + // it's possible to access the doc with a level less than "viewer"). + if (!canView(docAuth.access)) { + throw new ErrorWithCode("AUTH_NO_VIEW", "No view access", details); + } + + if (role === 'editors') { + // If opening in a fork or view mode, treat user as viewer and deny write access. + const access = (openMode === 'fork' || openMode === 'view') ? + getWeakestRole('viewers', docAuth.access) : docAuth.access; + if (!canEdit(access)) { + throw new ErrorWithCode("AUTH_NO_EDIT", "No write access", details); + } + } +} + +/** + * Pull out headers to pass along to a proxied service. Focussed primarily on + * authentication. + */ +export function getTransitiveHeaders(req: Request): {[key: string]: string} { + const Authorization = req.get('Authorization'); + const Cookie = req.get('Cookie'); + const PermitHeader = req.get('Permit'); + const Organization = (req as RequestWithOrg).org; + return { + ...(Authorization ? { Authorization } : undefined), + ...(Cookie ? { Cookie } : undefined), + ...(Organization ? { Organization } : undefined), + ...(PermitHeader ? { Permit: PermitHeader } : undefined), + }; +} diff --git a/app/server/lib/BrowserSession.ts b/app/server/lib/BrowserSession.ts new file mode 100644 index 00000000..e2e038ee --- /dev/null +++ b/app/server/lib/BrowserSession.ts @@ -0,0 +1,225 @@ +import {normalizeEmail} from 'app/common/emails'; +import {UserProfile} from 'app/common/LoginSessionAPI'; +import {SessionStore} from 'app/server/lib/gristSessions'; +import * as log from 'app/server/lib/log'; + +// Part of a session related to a single user. +export interface SessionUserObj { + // a grist-internal identify for the user, if known. + userId?: number; + + // The user profile object. When updated, all clients get a message with the update. + profile?: UserProfile; + + // Authentication provider string indicating the login method used. + authProvider?: string; + + // Login ID token used to access AWS services. + idToken?: string; + + // Login access token used to access other AWS services. + accessToken?: string; + + // Login refresh token used to retrieve new ID and access tokens. + refreshToken?: string; +} + +// Session state maintained for a particular browser. It is identified by a cookie. There may be +// several browser windows/tabs that share this cookie and this state. +export interface SessionObj { + // Session cookie. + // This is marked optional to reflect the reality of pre-existing code. + cookie?: any; + + // A list of users we have logged in as. + // This is optional since the session may already exist. + users?: SessionUserObj[]; + + // map from org to an index into users[] + // This is optional since the session may already exist. + orgToUser?: {[org: string]: number}; + + // This gets set to encourage express-session to set a cookie. + alive?: boolean; +} + +/** + * Extract the available user profiles from the session. + * + */ +export function getSessionProfiles(session: SessionObj): UserProfile[] { + if (!session.users) { return []; } + return session.users.filter(user => user && user.profile).map(user => user.profile!); +} + +/** + * + * Gets user profile from the session for a given org, returning null if no profile is + * found specific to that org. + * + */ +export function getSessionUser(session: SessionObj, org: string): SessionUserObj|null { + if (!session.users) { return null; } + if (!session.users.length) { return null; } + + if (session.orgToUser && session.orgToUser[org] !== undefined && + session.users.length > session.orgToUser[org]) { + return session.users[session.orgToUser[org]] || null; + } + return null; +} + +/** + * + * Record which user to use by default for a given org in future. + * This method mutates the session object passed to it. It does not save it, + * that is up to the caller. + * + */ +export function linkOrgWithEmail(session: SessionObj, email: string, org: string): SessionUserObj { + if (!session.users || !session.orgToUser) { throw new Error("Session not set up"); } + email = normalizeEmail(email); + for (let i = 0; i < session.users.length; i++) { + const iUser = session.users[i]; + if (iUser && iUser.profile && normalizeEmail(iUser.profile.email) === email) { + session.orgToUser[org] = i; + return iUser; + } + } + throw new Error("Failed to link org with email"); +} + +/** + * + * This is a view of the session object, for a single organization (the "scope"). + * + * Local caching is disabled in an enviroment where there is a home server (or we are + * the home server). In hosted Grist, per-instance caching would be a problem. + * + * We retain local caching for situations with a single server - especially electron. + * + */ +export class ScopedSession { + private _sessionCache?: SessionObj; + private _live: boolean; // if set, never cache session in memory. + + /** + * Create an interface to the session identified by _sessionId, in the store identified + * by _sessionStore, for the organization identified by _scope. + */ + constructor(private _sessionId: string, + private _sessionStore: SessionStore, + private _org: string) { + // Assume we need to skip cache in a hosted environment. GRIST_HOST is always set there. + // TODO: find a cleaner way to configure this flag. + this._live = Boolean(process.env.GRIST_HOST || process.env.GRIST_HOSTED); + } + + /** + * Get the user entry from the current session. + * @param prev: if supplied, this session object is used rather than querying the session again. + * @return the user entry + */ + public async getScopedSession(prev?: SessionObj): Promise { + const session = prev || await this._getSession(); + return getSessionUser(session, this._org) || {}; + } + + /** + * + * This performs an operation on the session object, limited to a single user entry. The state of that + * user entry before and after the operation are returned. LoginSession relies heavily on this method, + * to determine whether the change made by an operation merits certain follow-up work. + * + * @param op: Operation to perform. Given a single user entry, and should return a single user entry. + * It is fine to modify the supplied user entry in place. + * + * @return a pair [prev, current] with the state of the single user entry before and after the operation. + * + */ + public async operateOnScopedSession(op: (user: SessionUserObj) => + Promise): Promise<[SessionUserObj, SessionUserObj]> { + const session = await this._getSession(); + const user = await this.getScopedSession(session); + const oldUser = JSON.parse(JSON.stringify(user)); // Old version to compare against. + const newUser = await op(JSON.parse(JSON.stringify(user))); // Modify a scratch version. + if (Object.keys(newUser).length === 0) { + await this.clearScopedSession(session); + } else { + await this._updateScopedSession(newUser, session); + } + return [oldUser, newUser]; + } + + /** + * This clears the current user entry from the session. + * @param prev: if supplied, this session object is used rather than querying the session again. + */ + public async clearScopedSession(prev?: SessionObj): Promise { + const session = prev || await this._getSession(); + this._clearUser(session); + await this._setSession(session); + } + + /** + * Read the state of the session. + */ + private async _getSession(): Promise { + if (this._sessionCache) { return this._sessionCache; } + const session = ((await this._sessionStore.getAsync(this._sessionId)) || {}) as SessionObj; + if (!this._live) { this._sessionCache = session; } + return session; + } + + /** + * Set the session to the supplied object. + */ + private async _setSession(session: SessionObj): Promise { + try { + await this._sessionStore.setAsync(this._sessionId, session); + if (!this._live) { this._sessionCache = session; } + } catch (e) { + // (I've copied this from old code, not sure if continuing after a session save error is + // something existing code depends on?) + // Report and keep going. This ensures that the session matches what's in the sessionStore. + log.error(`ScopedSession[${this._sessionId}]: Error updating sessionStore: ${e}`); + } + } + + /** + * Update the session with the supplied user entry, replacing anything for that user already there. + * @param user: user entry to insert in session + * @param prev: if supplied, this session object is used rather than querying the session again. + * + */ + private async _updateScopedSession(user: SessionUserObj, prev?: SessionObj): Promise { + const profile = user.profile; + if (!profile) { + throw new Error("No profile available"); + } + // We used to also check profile.email_verified, but we no longer create UserProfile objects + // unless the email is verified, so this check is no longer needed. + if (!profile.email) { + throw new Error("Profile has no email address"); + } + + const session = prev || await this._getSession(); + if (!session.users) { session.users = []; } + if (!session.orgToUser) { session.orgToUser = {}; } + let index = session.users.findIndex(u => Boolean(u.profile && u.profile.email === profile.email)); + if (index < 0) { index = session.users.length; } + session.orgToUser[this._org] = index; + session.users[index] = user; + await this._setSession(session); + } + + /** + * This clears all user logins (not just the current login). + * In future, we may want to be able to log in and out selectively, slack style, + * but right now it seems confusing. + */ + private _clearUser(session: SessionObj): void { + session.users = []; + session.orgToUser = {}; + } +} diff --git a/app/server/lib/Client.ts b/app/server/lib/Client.ts new file mode 100644 index 00000000..692e39fc --- /dev/null +++ b/app/server/lib/Client.ts @@ -0,0 +1,362 @@ +import {ApiError} from 'app/common/ApiError'; +import {BrowserSettings} from 'app/common/BrowserSettings'; +import {ErrorWithCode} from 'app/common/ErrorWithCode'; +import {UserProfile} from 'app/common/LoginSessionAPI'; +import {getLoginState, LoginState} from 'app/common/LoginState'; +import {User} from 'app/gen-server/entity/User'; +import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import {Authorizer} from 'app/server/lib/Authorizer'; +import {DocSession} from 'app/server/lib/DocSession'; +import * as log from 'app/server/lib/log'; +import {ILoginSession} from 'app/server/lib/ILoginSession'; +import {shortDesc} from 'app/server/lib/shortDesc'; +import * as crypto from 'crypto'; +import * as moment from 'moment'; + +/// How many messages to accumulate for a disconnected client before booting it. +const clientMaxMissedMessages = 100; + +export type ClientMethod = (client: Client, ...args: any[]) => Promise; + +/** + * Generates and returns a random string to use as a clientId. This is better + * than numbering clients with consecutive integers; otherwise a reconnecting + * client presenting the previous clientId to a restarted (new) server may + * accidentally associate itself with a wrong session that happens to share the + * same clientId. In other words, we need clientIds to be unique across server + * restarts. + * @returns {String} - random string to use as a new clientId. + */ +function generateClientId(): string { + // Non-blocking version of randomBytes may fail if insufficient entropy is available without + // blocking. If we encounter that, we could either block, or maybe use less random values. + return crypto.randomBytes(8).toString('hex'); +} + +/** + * These are the types of messages that are allowed to be sent to the client even if the client is + * not authorized to use this instance (e.g. not a member of the team for this subdomain). + */ +const MESSAGE_TYPES_NO_AUTH = new Set([ + 'clientConnect', + 'profileFetch', + 'userSettings', + 'clientLogout', +]); + +// tslint:disable-next-line:no-unused-expression Silence "unused variable" warning. +void(MESSAGE_TYPES_NO_AUTH); + +/** + * Class that encapsulates the information for a client. A Client may survive + * across multiple websocket reconnects. + * TODO: this could provide a cleaner interface. + * + * @param comm: parent Comm object + * @param websocket: websocket connection, promisified to have a sendAsync method + * @param methods: a mapping from method names to server methods (must return promises) + */ +export class Client { + public readonly clientId: string; + + public session: ILoginSession|null = null; + + public browserSettings: BrowserSettings = {}; + + // Maps docFDs to DocSession objects. + private _docFDs: Array = []; + + private _missedMessages: any = []; + private _destroyTimer: NodeJS.Timer|null = null; + private _destroyed: boolean = false; + private _websocket: any; + private _loginState: LoginState|null = null; + private _org: string|null = null; + private _profile: UserProfile|null = null; + private _userId: number|null = null; + private _firstLoginAt: Date|null = null; + private _isAnonymous: boolean = false; + // Identifier for the current GristWSConnection object connected to this client. + private _counter: string|null = null; + + constructor( + private _comm: any, + private _methods: any, + private _host: string + ) { + this.clientId = generateClientId(); + } + + public toString() { return `Client ${this.clientId} #${this._counter}`; } + + // Returns the LoginState object that's encoded and passed via login pages to login-connect. + public getLoginState(): LoginState|null { return this._loginState; } + + public setCounter(counter: string) { + this._counter = counter; + } + + public get host(): string { + return this._host; + } + + public setConnection(websocket: any, reqHost: string, browserSettings: BrowserSettings) { + this._websocket = websocket; + // Set this._loginState, used by CognitoClient to construct login/logout URLs. + this._loginState = getLoginState(reqHost); + this.browserSettings = browserSettings; + } + + /** + * Returns DocSession for the given docFD, or throws an exception if this doc is not open. + */ + public getDocSession(fd: number): DocSession { + const docSession = this._docFDs[fd]; + if (!docSession) { + throw new Error(`Invalid docFD ${fd}`); + } + return docSession; + } + + // Adds a new DocSession to this Client, and returns the new FD for it. + public addDocSession(activeDoc: ActiveDoc, authorizer: Authorizer): DocSession { + const fd = this._getNextDocFD(); + const docSession = new DocSession(activeDoc, this, fd, authorizer); + this._docFDs[fd] = docSession; + return docSession; + } + + // Removes a DocSession from this Client, called when a doc is closed. + public removeDocSession(fd: number): void { + this._docFDs[fd] = null; + } + + // Check that client still has access to all documents. Used to determine whether + // a Comm client can be safely reused after a reconnect. Without this check, the client + // would be reused even if access to a document has been lost (although an error would be + // issued later, on first use of the document). + public async isAuthorized(): Promise { + for (const docFD of this._docFDs) { + try { + if (docFD !== null) { await docFD.authorizer.assertAccess('viewers'); } + } catch (e) { + return false; + } + } + return true; + } + + /** + * Closes all docs. + */ + public closeAllDocs() { + let count = 0; + for (let fd = 0; fd < this._docFDs.length; fd++) { + const docSession = this._docFDs[fd]; + if (docSession && docSession.activeDoc) { + // Note that this indirectly calls to removeDocSession(docSession.fd) + docSession.activeDoc.closeDoc(docSession) + .catch((e) => { log.warn("%s: error closing docFD %d", this, fd); }); + count++; + } + this._docFDs[fd] = null; + } + log.debug("%s: closeAllDocs() closed %d doc(s)", this, count); + } + + public interruptConnection() { + if (this._websocket) { + this._websocket.removeAllListeners(); + this._websocket.terminate(); // close() is inadequate when ws routed via loadbalancer + this._websocket = null; + } + } + + /** + * Sends a message to the client, queuing it up on failure or if the client is disconnected. + */ + public async sendMessage(messageObj: any): Promise { + if (this._destroyed) { + return; + } + + const message: string = JSON.stringify(messageObj); + + // Log something useful about the message being sent. + if (messageObj.type) { + log.info("%s: sending %s: %d bytes", this, messageObj.type, message.length); + } else if (messageObj.error) { + log.warn("%s: responding to #%d ERROR %s", this, messageObj.reqId, messageObj.error); + } else { + log.info("%s: responding to #%d OK: %d bytes", this, messageObj.reqId, message.length); + } + + if (this._websocket) { + // If we have a websocket, send the message. + try { + await this._websocket.sendAsync(message); + } catch (err) { + // Sending failed. Presumably we should be getting onClose around now too. + // NOTE: if this handler is run after onClose, we could have messages end up out of order. + // Let's check to make sure. If this can happen, we need to refactor for correct ordering. + if (!this._websocket) { + log.error("%s sendMessage: UNEXPECTED ORDER OF CALLBACKS", this); + } + log.warn("%s sendMessage: queuing after send error: %s", this, err.toString()); + this._missedMessages.push(message); + } + } else if (this._missedMessages.length < clientMaxMissedMessages) { + // Queue up the message. + this._missedMessages.push(message); + } else { + // Too many messages queued. Boot the client now, to make it reset when/if it reconnects. + log.error("%s sendMessage: too many messages queued; booting client", this); + if (this._destroyTimer) { + clearTimeout(this._destroyTimer); + this._destroyTimer = null; + } + this._comm._destroyClient(this); + } + } + + // Assigns the client to the given login session and the session to the client. + public setSession(session: ILoginSession): void { + this.unsetSession(); + this.session = session; + session.clients.add(this); + } + + // Unsets the current login session and removes the client from it. + public unsetSession(): void { + if (this.session) { this.session.clients.delete(this); } + this.session = null; + } + + public destroy() { + this.unsetSession(); + this._destroyed = true; + } + + /** + * Processes a request from a client. All requests from a client get a response, at least to + * indicate success or failure. + */ + public async onMessage(message: string): Promise { + const clientId = this.clientId; + const request = JSON.parse(message); + if (request.beat) { + // this is a heart beat, to keep the websocket alive. No need to reply. + log.rawInfo('heartbeat', {clientId, counter: this._counter, url: request.url}); + return; + } else { + log.info("%s: onMessage", this, shortDesc(message)); + } + const response: any = {reqId: request.reqId}; + const method = this._methods[request.method]; + if (!method) { + response.error = `Unknown method ${request.method}`; + } else { + try { + response.data = await method(this, ...request.args); + } catch (error) { + const err: ErrorWithCode = error; + // Print the error stack, except for SandboxErrors, for which the JS stack isn't that useful. + // Also not helpful is the stack of AUTH_NO_VIEW|EDIT errors produced by the Authorizer. + const code: unknown = err.code; + const skipStack = ( + !err.stack || + err.stack.match(/^SandboxError:/) || + (typeof code === 'string' && code.startsWith('AUTH_NO')) + ); + + log.warn("%s: Error %s %s", this, skipStack ? err : err.stack, code || ''); + response.error = err.message; + if (err.code) { + response.errorCode = err.code; + } + if (typeof code === 'string' && code === 'AUTH_NO_EDIT' && err.accessMode === 'fork') { + response.shouldFork = true; + } + } + } + await this.sendMessage(response); + } + + public setOrg(org: string): void { + this._org = org; + } + + public getOrg(): string { + return this._org!; + } + + public setProfile(profile: UserProfile|null): void { + this._profile = profile; + // Unset userId, so that we look it up again on demand. (Not that userId could change in + // practice via a change to profile, but let's not make any assumptions here.) + this._userId = null; + this._firstLoginAt = null; + this._isAnonymous = false; + } + + public getProfile(): UserProfile|null { + return this._profile; + } + + public getCachedUserId(): number|null { + return this._userId; + } + + public isAnonymous(): boolean { + return this._isAnonymous; + } + + // Returns the userId for profile.email, or null when profile is not set; with caching. + public async getUserId(dbManager: HomeDBManager): Promise { + if (!this._userId) { + const user = await this._fetchUser(dbManager); + this._userId = (user && user.id) || null; + this._isAnonymous = this._userId && dbManager.getAnonymousUserId() === this._userId || false; + this._firstLoginAt = (user && user.firstLoginAt) || null; + } + return this._userId; + } + + // Returns the userId for profile.email, or throws 403 error when profile is not set. + public async requireUserId(dbManager: HomeDBManager): Promise { + const userId = await this.getUserId(dbManager); + if (userId) { return userId; } + throw new ApiError(this._profile ? `user not known: ${this._profile.email}` : 'user not set', 403); + } + + public getLogMeta() { + const meta: {[key: string]: any} = {}; + if (this._profile) { meta.email = this._profile.email; } + // We assume the _userId has already been cached, which will be true always (for all practical + // purposes) because it's set when the Authorizer checks this client. + if (this._userId) { meta.userId = this._userId; } + // Likewise for _firstLoginAt, which we learn along with _userId. + if (this._firstLoginAt) { + meta.age = Math.floor(moment.duration(moment().diff(this._firstLoginAt)).asDays()); + } + if (this._org) { meta.org = this._org; } + meta.clientId = this.clientId; // identifies a client connection, essentially a websocket + meta.counter = this._counter; // identifies a GristWSConnection in the connected browser tab + return meta; + } + + // Fetch the user database record from profile.email, or null when profile is not set. + private async _fetchUser(dbManager: HomeDBManager): Promise { + return this._profile && this._profile.email ? + await dbManager.getUserByLogin(this._profile.email) : + undefined; + } + + // Returns the next unused docFD number. + private _getNextDocFD(): number { + let fd = 0; + while (this._docFDs[fd]) { fd++; } + return fd; + } +} diff --git a/app/server/lib/Comm.js b/app/server/lib/Comm.js new file mode 100644 index 00000000..38dfb871 --- /dev/null +++ b/app/server/lib/Comm.js @@ -0,0 +1,395 @@ +/** + * The server's Comm object implements communication with the client. + * + * The server receives requests, to which it sends a response (or an error). The server can + * also send asynchronous messages to the client. Available methods should be provided via + * comm.registerMethods(). + * + * To send async messages, you may call broadcastMessage() or sendDocMessage(). + * + * In practice, requests which modify the document are done via UserActions.js, and result in an + * asynchronous message updating the document (which is sent to all clients who have the document + * open), and the response could return some useful value, but does not have to. + * + * See app/client/components/Comm.js for other details of the communication protocol. + * + * + * Currently, this module also implements the concept of a "Client". A Client corresponds to a + * browser window, and should persist across brief disconnects. A Client has a 'clientId' + * property, which uniquely identifies a client within the currently running server. Method + * registered with Comm always receive a Client object as the first argument. + * + * In the future, we may want to have a separate Client.js file with documentation of the various + * properties that may be associated with a client. + * + * Note that users of this module should never use the websocket of a Client, since that's an + * implementation detail of Comm.js. + */ + +/** + * Event for DocList changes. + * @event docListAction Emitted when the document list changes in any way. + * @property {Array[String]} [addDocs] Array of names of documents to add to the docList. + * @property {Array[String]} [removeDocs] Array of names of documents that got removed. + * @property {Array[String]} [renameDocs] Array of [oldName, newName] pairs for renamed docs. + * @property {Array[String]} [addInvites] Array of document invite names to add. + * @property {Array[String]} [removeInvites] Array of documents invite names to remove. + */ + + + +var events = require('events'); +var url = require('url'); +var util = require('util'); +var ws = require('ws'); +var Promise = require('bluebird'); + +var log = require('./log'); +var gutil = require('app/common/gutil'); +const {parseFirstUrlPart} = require('app/common/gristUrls'); +const version = require('app/common/version'); +const {Client} = require('./Client'); + +// Bluebird promisification, to be able to use e.g. websocket.sendAsync method. +Promise.promisifyAll(ws.prototype); + +/// How long the client state persists after a disconnect. +var clientRemovalTimeoutMsDefault = 300 * 1000; // 300s = 5 minutes. +var clientRemovalTimeoutMs = clientRemovalTimeoutMsDefault; + +/** + * Constructs a Comm object. + * @param {Object} server - The HTTP server. + * @param {Object} options.sessions - A collection of sessions + * @param {Object} options.settings - The config object containing instance settings + * including features. + * @param {Object} options.instanceManager - Instance manager, giving access to InstanceStore + * and per-instance objects. If null, HubUserClient will not be created. + * @param {Object} options.hosts - Hosts object from extractOrg.ts. if set, we use + * hosts.getOrgInfo(req) to extract an organization from a (possibly versioned) url. + */ +function Comm(server, options) { + events.EventEmitter.call(this); + this._server = server; + this._httpsServer = options.httpsServer; + this.wss = this._startServer(); + + // Maps client IDs to websocket objects. + this._clients = {}; // Maps clientIds to Client objects. + this.clientList = []; // List of all active Clients, ordered by clientId. + + // Maps sessionIds to LoginSession objects. + this.sessions = options.sessions; + + this._settings = options.settings; + this._instanceManager = options.instanceManager; + this._hosts = options.hosts; + + // This maps method names to their implementation. + this.methods = {}; + + // For testing, we need a way to override the server version reported. + // For upgrading, we use this to set the server version for a defunct server + // to "dead" so that a client will know that it needs to periodically recheck + // for a valid server. + this._serverVersion = null; +} +util.inherits(Comm, events.EventEmitter); + + +/** + * Registers server methods. + * @param {Object[String:Function]} Mapping of method name to their implementations. All methods + * receive the client as the first argument, and the arguments from the request. + */ +Comm.prototype.registerMethods = function(serverMethods) { + // Wrap methods to translate return values and exceptions to promises. + for (var methodName in serverMethods) { + this.methods[methodName] = Promise.method(serverMethods[methodName]); + } +}; + +/** + * Returns the Client object associated with the given clientId, or throws an Error if not found. + */ +Comm.prototype.getClient = function(clientId) { + const client = this._clients[clientId]; + if (!client) { throw new Error('Unrecognized clientId'); } + return client; +}; + +/** + * Returns a LoginSession object with the given session id from the list of sessions, + * or adds a new one and returns that. + */ +Comm.prototype.getOrCreateSession = function(sid, req) { + // LoginSessions are specific to a session id / org combination. + const org = req.org || ""; + return this.sessions.getOrCreateLoginSession(sid, org, this, this._instanceManager); +}; + + +/** + * Returns the sessionId from the signed grist cookie. + */ +Comm.prototype.getSessionIdFromCookie = function(gristCookie) { + return this.sessions.getSessionIdFromCookie(gristCookie); +}; + + +/** + * Broadcasts an app-level message to all clients. + * @param {String} type - Type of message, e.g. 'docListAction'. + * @param {Object} messageData - The data for this type of message. + */ +Comm.prototype.broadcastMessage = function(type, messageData) { + return this._broadcastMessage(type, messageData, this.clientList); +}; + +Comm.prototype._broadcastMessage = function(type, data, clients) { + clients.forEach(client => client.sendMessage({type, data})); +}; + +/** + * Sends a per-doc message to the given client. + * @param {Object} client - The client object, as passed to all per-doc methods. + * @param {Number} docFD - The document's file descriptor in the given client. + * @param {String} type - The type of the message, e.g. 'docUserAction'. + * @param {Object} messageData - The data for this type of message. + * @param {Boolean} fromSelf - Whether `client` is the originator of this message. + */ +Comm.sendDocMessage = function(client, docFD, type, data, fromSelf = undefined) { + client.sendMessage({type, docFD, data, fromSelf}); +}; + +/** + * Processes a new websocket connection. + * TODO: Currently it always creates a new client, but in the future the creation of a client + * should possibly be delayed until some hello message, so that a previous client may reconnect + * without losing state. + */ +Comm.prototype._onWebSocketConnection = async function(websocket, req) { + log.info("Comm: Got WebSocket connection: %s", req.url); + if (this._hosts) { + // DocWorker ID (/dw/) and version tag (/v/) may be present in this request but are not + // needed. addOrgInfo assumes req.url starts with /o/ if present. + req.url = parseFirstUrlPart('dw', req.url).path; + req.url = parseFirstUrlPart('v', req.url).path; + await this._hosts.addOrgInfo(req); + } + + websocket.on('error', this.onError.bind(this, websocket)); + websocket.on('close', this.onClose.bind(this, websocket)); + // message handler is added later, after we create a Client but before any async operations + + // Parse the cookie in the request to get the sessionId. + var sessionId = this.sessions.getSessionIdFromRequest(req); + var urlObj = url.parse(req.url, true); + var existingClientId = urlObj.query.clientId; + var browserSettings = urlObj.query.browserSettings ? JSON.parse(urlObj.query.browserSettings) : {}; + var newClient = (parseInt(urlObj.query.newClient, 10) === 1); + const counter = urlObj.query.counter; + + // Associate an ID with each websocket, reusing the supplied one if it's valid. + var client; + if (existingClientId && this._clients.hasOwnProperty(existingClientId) && + !this._clients[existingClientId]._websocket && + await this._clients[existingClientId].isAuthorized()) { + client = this._clients[existingClientId]; + client.setCounter(counter); + log.info("Comm %s: existing client reconnected (%d missed messages)", client, + client._missedMessages.length); + if (client._destroyTimer) { + log.warn("Comm %s: clearing scheduled destruction", client); + clearTimeout(client._destroyTimer); + client._destroyTimer = null; + } + if (newClient) { + // If this isn't a reconnect, then we assume that the browser client lost its state (e.g. + // reloaded the page), so we treat it as a disconnect followed by a new connection to the + // same state. At the moment, this only means that we close all docs. + if (client._missedMessages.length) { + log.warn("Comm %s: clearing missed messages for new client", client); + } + client._missedMessages.length = 0; + client.closeAllDocs(); + } + client.setConnection(websocket, req.headers.host, browserSettings); + } else { + client = new Client(this, this.methods, req.headers.host); + client.setCounter(counter); + client.setConnection(websocket, req.headers.host, browserSettings); + this._clients[client.clientId] = client; + this.clientList.push(client); + log.info("Comm %s: new client", client); + } + + websocket._commClient = client; + websocket.clientId = client.clientId; + + // Add a Session object to the client. + log.info(`Comm ${client}: using session ${sessionId}`); + const loginSession = this.getOrCreateSession(sessionId, req); + client.setSession(loginSession); + + // Delegate message handling to the client + websocket.on('message', client.onMessage.bind(client)); + + loginSession.getSessionProfile() + .then((profile) => { + log.debug(`Comm ${client}: sending clientConnect with ` + + `${client._missedMessages.length} missed messages`); + // Don't use sendMessage here, since we don't want to queue up this message on failure. + client.setOrg(req.org || ""); + client.setProfile(profile); + const clientConnectMsg = { + type: 'clientConnect', + clientId: client.clientId, + serverVersion: this._serverVersion || version.gitcommit, + missedMessages: client._missedMessages.slice(0), + settings: this._settings, + profile, + }; + // If reconnecting a client with missed messages, clear them now. + client._missedMessages.length = 0; + return websocket.sendAsync(JSON.stringify(clientConnectMsg)) + // A heavy-handed fix to T396, since 'clientConnect' is sometimes not seen in the browser, + // (seemingly when the 'message' event is triggered before 'open' on the native WebSocket.) + // See also my report at https://stackoverflow.com/a/48411315/328565 + .delay(250).then(() => { + if (client._destroyed) { return; } // object is already closed - don't show messages + if (websocket.readyState === websocket.OPEN) { + return websocket.sendAsync(JSON.stringify(Object.assign(clientConnectMsg, {dup: true}))); + } else { + log.debug(`Comm ${client}: websocket closed right after clientConnect`); + } + }); + }) + .then(() => { + if (!client._destroyed) { log.debug(`Comm ${client}: clientConnect sent successfully`); } + }) + .catch(err => { + log.error(`Comm ${client}: failed to prepare or send clientConnect:`, err); + }); +}; + +/** + * Processes an error on the websocket. + */ +Comm.prototype.onError = function(websocket, err) { + log.warn("Comm cid %s: onError", websocket.clientId, err); + // TODO Make sure that this is followed by onClose when the connection is lost. +}; + +/** + * Processes the closing of a websocket. + */ +Comm.prototype.onClose = function(websocket) { + log.info("Comm cid %s: onClose", websocket.clientId); + websocket.removeAllListeners(); + + var client = websocket._commClient; + if (client) { + // Remove all references to the websocket. + client._websocket = null; + + // Schedule the client to be destroyed after a timeout. The timer gets cleared if the same + // client reconnects in the interim. + if (client._destroyTimer) { + log.warn("Comm cid %s: clearing previously scheduled destruction", websocket.clientId); + clearTimeout(client._destroyTimer); + } + log.warn("Comm cid %s: will discard client in %s sec", + websocket.clientId, clientRemovalTimeoutMs / 1000); + client._destroyTimer = setTimeout(this._destroyClient.bind(this, client), + clientRemovalTimeoutMs); + } +}; + +Comm.prototype._startServer = function() { + const servers = [this._server]; + if (this._httpsServer) { servers.push(this._httpsServer); } + const wss = []; + for (const server of servers) { + const wssi = new ws.Server({server}); + wssi.on('connection', async (websocket, req) => { + try { + await this._onWebSocketConnection(websocket, req); + } catch (e) { + log.error("Comm connection for %s threw exception: %s", req.url, e.message); + websocket.removeAllListeners(); + websocket.terminate(); // close() is inadequate when ws routed via loadbalancer + } + }); + wss.push(wssi); + } + return wss; +}; + +Comm.prototype.testServerShutdown = async function() { + if (this.wss) { + for (const wssi of this.wss) { + await Promise.fromCallback((cb) => wssi.close(cb)); + } + this.wss = null; + } +}; + +Comm.prototype.testServerRestart = async function() { + await this.testServerShutdown(); + this.wss = this._startServer(); +}; + +/** + * Destroy all clients, forcing reconnections. + */ +Comm.prototype.destroyAllClients = function() { + // Iterate over all clients. Take a copy of the list of clients since it will be changing + // during the loop as we remove them one by one. + for (const client of this.clientList.slice()) { + client.interruptConnection(); + this._destroyClient(client); + } +}; + +/** + * Destroys a client. If the same browser window reconnects later, it will get a new Client + * object and clientId. + */ +Comm.prototype._destroyClient = function(client) { + log.info("Comm %s: client gone", client); + client.closeAllDocs(); + if (client._destroyTimer) { + clearTimeout(client._destroyTimer); + } + delete this._clients[client.clientId]; + gutil.arrayRemove(this.clientList, client); + client.destroy(); +}; + +/** + * Override the version string Comm will report to clients. + * Call with null to reset the override. + * + */ +Comm.prototype.setServerVersion = function (serverVersion) { + this._serverVersion = serverVersion; +}; + +/** + * Mark the server as active or inactive. If inactive, any client that manages to + * connect to it will read a server version of "dead". + */ +Comm.prototype.setServerActivation = function (active) { + this._serverVersion = active ? null : 'dead'; +}; + +/** + * Set how long clients persist on the server after disconnection. Call with + * 0 to return to the default. + */ +Comm.prototype.testSetClientPersistence = function (ttlMs) { + clientRemovalTimeoutMs = ttlMs || clientRemovalTimeoutMsDefault; +} + +module.exports = Comm; diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts new file mode 100644 index 00000000..80a5d76b --- /dev/null +++ b/app/server/lib/DocApi.ts @@ -0,0 +1,560 @@ +import { Application, NextFunction, Request, RequestHandler, Response } from "express"; + +import { ApiError } from 'app/common/ApiError'; +import { BrowserSettings } from "app/common/BrowserSettings"; +import { fromTableDataAction, TableColValues } from 'app/common/DocActions'; +import { arrayRepeat } from "app/common/gutil"; +import { SortFunc } from 'app/common/SortFunc'; +import { DocReplacementOptions, DocState, DocStateComparison, DocStates, NEW_DOCUMENT_CODE} from 'app/common/UserAPI'; +import { HomeDBManager, makeDocAuthResult } from 'app/gen-server/lib/HomeDBManager'; +import { ActiveDoc } from "app/server/lib/ActiveDoc"; +import { assertAccess, getOrSetDocAuth, getTransitiveHeaders, getUserId, isAnonymousUser, + RequestWithLogin } from 'app/server/lib/Authorizer'; +import { DocManager } from "app/server/lib/DocManager"; +import { DocWorker } from "app/server/lib/DocWorker"; +import { expressWrap } from 'app/server/lib/expressWrap'; +import { GristServer } from 'app/server/lib/GristServer'; +import { makeForkIds } from "app/server/lib/idUtils"; +import { getDocId, getDocScope, integerParam, isParameterOn, optStringParam, + sendOkReply, sendReply } from 'app/server/lib/requestUtils'; +import { SandboxError } from "app/server/lib/sandboxUtil"; +import { handleOptionalUpload, handleUpload } from "app/server/lib/uploads"; +import * as contentDisposition from 'content-disposition'; +import fetch from 'node-fetch'; +import * as path from 'path'; + +// Cap on the number of requests that can be outstanding on a single document via the +// rest doc api. When this limit is exceeded, incoming requests receive an immediate +// reply with status 429. +const MAX_PARALLEL_REQUESTS_PER_DOC = 10; + +type WithDocHandler = (activeDoc: ActiveDoc, req: RequestWithLogin, resp: Response) => Promise; + +/** + * Middleware to track the number of requests outstanding on each document, and to + * throw an exception when the maximum number of requests are already outstanding. + * Access to a document must already have been authorized. + */ +function apiThrottle(usage: Map, + callback: (req: RequestWithLogin, + resp: Response, + next: NextFunction) => Promise): RequestHandler { + return async (req, res, next) => { + const docId = getDocId(req); + try { + const count = usage.get(docId) || 0; + usage.set(docId, count + 1); + if (count + 1 > MAX_PARALLEL_REQUESTS_PER_DOC) { + throw new ApiError(`Too many backlogged requests for document ${docId} - ` + + `try again later?`, 429); + } + await callback(req as RequestWithLogin, res, next); + } catch (err) { + next(err); + } finally { + const count = usage.get(docId); + if (count) { + if (count === 1) { + usage.delete(docId); + } else { + usage.set(docId, count - 1); + } + } + } + }; +} + +export class DocWorkerApi { + constructor(private _app: Application, private _docWorker: DocWorker, private _docManager: DocManager, + private _dbManager: HomeDBManager, private _grist: GristServer) {} + + /** + * Adds endpoints for the doc api. + * + * Note that it expects bodyParser, userId, and jsonErrorHandler middleware to be set up outside + * to apply to these routes. + */ + public addEndpoints() { + + // check document exists (not soft deleted) and user can view it + const canView = expressWrap(this._assertAccess.bind(this, 'viewers', false)); + // check document exists (not soft deleted) and user can edit it + const canEdit = expressWrap(this._assertAccess.bind(this, 'editors', false)); + // check user can edit document, with soft-deleted documents being acceptable + const canEditMaybeRemoved = expressWrap(this._assertAccess.bind(this, 'editors', true)); + + // Middleware to limit number of outstanding requests per document. Will also + // handle errors like expressWrap would. + const throttled = apiThrottle.bind(null, new Map()); + const withDoc = (callback: WithDocHandler) => throttled(this._requireActiveDoc(callback)); + + // Apply user actions to a document. + this._app.post('/api/docs/:docId/apply', canEdit, withDoc(async (activeDoc, req, res) => { + res.json(await activeDoc.applyUserActions({ client: null, req }, req.body)); + })); + + // Get the specified table. + this._app.get('/api/docs/:docId/tables/:tableId/data', canView, withDoc(async (activeDoc, req, res) => { + const filters = req.query.filter ? JSON.parse(String(req.query.filter)) : {}; + if (!Object.keys(filters).every(col => Array.isArray(filters[col]))) { + throw new ApiError("Invalid query: filter values must be arrays", 400); + } + const tableId = req.params.tableId; + const tableData = await handleSandboxError(tableId, [], activeDoc.fetchQuery(null, {tableId, filters}, true)); + // Apply sort/limit parameters, if set. TODO: move sorting/limiting into data engine + // and sql. + const params = getQueryParameters(req); + res.json(applyQueryParameters(fromTableDataAction(tableData), params)); + })); + + // The upload should be a multipart post with an 'upload' field containing one or more files. + // Returns the list of rowIds for the rows created in the _grist_Attachments table. + this._app.post('/api/docs/:docId/attachments', canEdit, withDoc(async (activeDoc, req, res) => { + const uploadResult = await handleUpload(req, res); + res.json(await activeDoc.addAttachments(req, uploadResult.uploadId)); + })); + + // Returns the metadata for a given attachment ID (i.e. a rowId in _grist_Attachments table). + this._app.get('/api/docs/:docId/attachments/:attId', canView, withDoc(async (activeDoc, req, res) => { + const attRecord = activeDoc.getAttachmentMetadata(req.params.attId as string); + const {fileName, fileSize, timeUploaded: t} = attRecord; + const timeUploaded = (typeof t === 'number') ? new Date(t).toISOString() : undefined; + res.json({fileName, fileSize, timeUploaded}); + })); + + // Responds with attachment contents, with suitable Content-Type and Content-Disposition. + this._app.get('/api/docs/:docId/attachments/:attId/download', canView, withDoc(async (activeDoc, req, res) => { + const attRecord = activeDoc.getAttachmentMetadata(req.params.attId as string); + const fileIdent = attRecord.fileIdent as string; + const ext = path.extname(fileIdent); + const origName = attRecord.fileName as string; + const fileName = ext ? path.basename(origName, path.extname(origName)) + ext : origName; + const fileData = await activeDoc.getAttachmentData(null, fileIdent); + res.status(200) + .type(ext) + // Construct a content-disposition header of the form 'attachment; filename="NAME"' + .set('Content-Disposition', contentDisposition(fileName, {type: 'attachment'})) + .set('Cache-Control', 'private, max-age=3600') + .send(fileData); + })); + + // Adds records. + this._app.post('/api/docs/:docId/tables/:tableId/data', canEdit, withDoc(async (activeDoc, req, res) => { + const tableId = req.params.tableId; + const columnValues = req.body; + const colNames = Object.keys(columnValues); + // user actions expect [null, ...] as row ids, first let's figure the number of items to add by + // looking at the length of a column + const count = columnValues[colNames[0]].length; + // then, let's create [null, ...] + const rowIds = arrayRepeat(count, null); + const sandboxRes = await handleSandboxError(tableId, colNames, activeDoc.applyUserActions({client: null, req}, + [['BulkAddRecord', tableId, rowIds, columnValues]])); + res.json(sandboxRes.retValues[0]); + })); + + this._app.post('/api/docs/:docId/tables/:tableId/data/delete', canEdit, withDoc(async (activeDoc, req, res) => { + const tableId = req.params.tableId; + const rowIds = req.body; + const sandboxRes = await handleSandboxError(tableId, [], activeDoc.applyUserActions({client: null, req}, + [['BulkRemoveRecord', tableId, rowIds]])); + res.json(sandboxRes.retValues[0]); + })); + + // Download full document + // TODO: look at download behavior if ActiveDoc is shutdown during call (cannot + // use withDoc wrapper) + this._app.get('/api/docs/:docId/download', canView, throttled(async (req, res) => { + try { + // We carefully avoid creating an ActiveDoc for the document being downloaded, + // in case it is broken in some way. It is convenient to be able to download + // broken files for diagnosis/recovery. + return await this._docWorker.downloadDoc(req, res, this._docManager.storageManager); + } catch (e) { + if (e.message && e.message.match(/does not exist yet/)) { + // The document has never been seen on file system / s3. It may be new, so + // we try again after having created an ActiveDoc for the document. + await this._getActiveDoc(req); + return this._docWorker.downloadDoc(req, res, this._docManager.storageManager); + } else { + throw e; + } + } + })); + + // Update records. The records to update are identified by their id column. Any invalid id fails + // the request and returns a 400 error code. + this._app.patch('/api/docs/:docId/tables/:tableId/data', canEdit, withDoc(async (activeDoc, req, res) => { + const tableId = req.params.tableId; + const columnValues = req.body; + const colNames = Object.keys(columnValues); + const rowIds = columnValues.id; + // sandbox expects no id column + delete columnValues.id; + await handleSandboxError(tableId, colNames, activeDoc.applyUserActions({client: null, req}, + [['BulkUpdateRecord', tableId, rowIds, columnValues]])); + res.json(null); + })); + + // Reload a document forcibly (in fact this closes the doc, it will be automatically + // reopened on use). + this._app.post('/api/docs/:docId/force-reload', canEdit, withDoc(async (activeDoc, req, res) => { + await activeDoc.reloadDoc(); + res.json(null); + })); + + // DELETE /api/docs/:docId + // Delete the specified doc. + this._app.delete('/api/docs/:docId', canEditMaybeRemoved, throttled(async (req, res) => { + await this._removeDoc(req, res, true); + })); + + // POST /api/docs/:docId/remove + // Soft-delete the specified doc. If query parameter "permanent" is set, + // delete permanently. + this._app.post('/api/docs/:docId/remove', canEditMaybeRemoved, throttled(async (req, res) => { + await this._removeDoc(req, res, isParameterOn(req.query.permanent)); + })); + + this._app.get('/api/docs/:docId/snapshots', canView, withDoc(async (activeDoc, req, res) => { + const {snapshots} = await activeDoc.getSnapshots(); + res.json({snapshots}); + })); + + this._app.post('/api/docs/:docId/flush', canEdit, throttled(async (req, res) => { + const activeDocPromise = this._getActiveDocIfAvailable(req); + if (!activeDocPromise) { + // Only need to flush if doc is actually open. + res.json(false); + return; + } + const activeDoc = await activeDocPromise; + await activeDoc.flushDoc(); + res.json(true); + })); + + // This endpoint cannot use withDoc since it is expected behavior for the ActiveDoc it + // starts with to become muted. + this._app.post('/api/docs/:docId/replace', canEdit, throttled(async (req, res) => { + const activeDoc = await this._getActiveDoc(req); + const options: DocReplacementOptions = {}; + if (req.body.sourceDocId) { + options.sourceDocId = await this._confirmDocIdForRead(req, String(req.body.sourceDocId)); + // We should make sure the source document has flushed recently. + // It may not be served by the same worker, so work through the api. + await fetch(this._grist.getHomeUrl(req, `/api/docs/${options.sourceDocId}/flush`), { + method: 'POST', + headers: { + ...getTransitiveHeaders(req), + 'Content-Type': 'application/json', + } + }); + } + if (req.body.snapshotId) { + options.snapshotId = String(req.body.snapshotId); + } + await activeDoc.replace(options); + res.json(null); + })); + + this._app.get('/api/docs/:docId/states', canView, withDoc(async (activeDoc, req, res) => { + res.json(await this._getStates(activeDoc)); + })); + + this._app.get('/api/docs/:docId/compare/:docId2', canView, withDoc(async (activeDoc, req, res) => { + const {states} = await this._getStates(activeDoc); + const ref = await fetch(this._grist.getHomeUrl(req, `/api/docs/${req.params.docId2}/states`), { + headers: { + ...getTransitiveHeaders(req), + 'Content-Type': 'application/json', + } + }); + const states2: DocState[] = (await ref.json()).states; + const left = states[0]; + const right = states2[0]; + if (!left || !right) { + // This should not arise unless there's a bug. + throw new Error('document with no history'); + } + const rightHashes = new Set(states2.map(state => state.h)); + const parent = states.find(state => rightHashes.has(state.h )) || null; + const leftChanged = parent && parent.h !== left.h; + const rightChanged = parent && parent.h !== right.h; + const summary = leftChanged ? (rightChanged ? 'both' : 'left') : + (rightChanged ? 'right' : (parent ? 'same' : 'unrelated')); + const comparison: DocStateComparison = { + left, right, parent, summary + }; + res.json(comparison); + })); + + // Do an import targeted at a specific workspace. Although the URL fits ApiServer, this + // endpoint is handled only by DocWorker, so is handled here. (Note: this does not handle + // actual file uploads, so no worries here about large request bodies.) + this._app.post('/api/workspaces/:wid/import', expressWrap(async (req, res) => { + const userId = getUserId(req); + const wsId = integerParam(req.params.wid); + const uploadId = integerParam(req.body.uploadId); + const result = await this._docManager.importDocToWorkspace(userId, uploadId, wsId, req.body.browserSettings); + res.json(result); + })); + + // Create a document. When an upload is included, it is imported as the initial + // state of the document. Otherwise a fresh empty document is created. + // A "timezone" option can be supplied. + // Documents are created "unsaved". + // TODO: support workspaceId option for creating regular documents, at which point + // existing import endpoint and doc creation endpoint can share implementation + // with this. + // Returns the id of the created document. + this._app.post('/api/docs', expressWrap(async (req, res) => { + const userId = getUserId(req); + let uploadId: number|undefined; + let parameters: {[key: string]: any}; + if (req.is('multipart/form-data')) { + const formResult = await handleOptionalUpload(req, res); + if (formResult.upload) { + uploadId = formResult.upload.uploadId; + } + parameters = formResult.parameters || {}; + } else { + parameters = req.body; + } + if (parameters.workspaceId) { throw new Error('workspaceId not supported'); } + const browserSettings: BrowserSettings = {}; + if (parameters.timezone) { browserSettings.timezone = parameters.timezone; } + if (uploadId !== undefined) { + const result = await this._docManager.importDocToWorkspace(userId, uploadId, null, + browserSettings); + return res.json(result.id); + } + const isAnonymous = isAnonymousUser(req); + const {docId} = makeForkIds({userId, isAnonymous, trunkDocId: NEW_DOCUMENT_CODE, + trunkUrlId: NEW_DOCUMENT_CODE}); + await this._docManager.fetchDoc({ client: null, req: req as RequestWithLogin, browserSettings }, docId); + return res.status(200).json(docId); + })); + } + + /** + * Check for read access to the given document, and return its + * canonical docId. Throws error if read access not available. + * This method is used for documents that are not the main document + * associated with the request, but are rather an extra source to be + * read from, so the access information is not cached in the + * request. + */ + private async _confirmDocIdForRead(req: Request, urlId: string): Promise { + const userId = getUserId(req); + const org = (req as RequestWithLogin).org; + const docAuth = await makeDocAuthResult(this._dbManager.getDoc({urlId, userId, org})); + if (docAuth.error) { throw docAuth.error; } + assertAccess('viewers', docAuth); + return docAuth.docId!; + } + + private _getActiveDoc(req: RequestWithLogin): Promise { + return this._docManager.fetchDoc({ client: null, req }, getDocId(req)); + } + + private _getActiveDocIfAvailable(req: RequestWithLogin): Promise|undefined { + return this._docManager.getActiveDoc(getDocId(req)); + } + + private async _assertAccess(role: 'viewers'|'editors', allowRemoved: boolean, + req: Request, res: Response, next: NextFunction) { + const scope = getDocScope(req); + allowRemoved = scope.showAll || scope.showRemoved || allowRemoved; + const docAuth = await getOrSetDocAuth(req as RequestWithLogin, this._dbManager, scope.urlId); + assertAccess(role, docAuth, {allowRemoved}); + next(); + } + + // Helper to generate a 503 if the ActiveDoc has been muted. + private _checkForMute(activeDoc: ActiveDoc|undefined) { + if (activeDoc && activeDoc.muted) { + throw new ApiError('Document in flux - try again later', 503); + } + } + + /** + * Throws an error if, during processing, the ActiveDoc becomes "muted". Also replaces any + * other error that may have occurred if the ActiveDoc becomes "muted", since the document + * shutting down during processing may have caused a variety of errors. + * + * Expects to be called within a handler that catches exceptions. + */ + private _requireActiveDoc(callback: WithDocHandler): RequestHandler { + return async (req, res) => { + let activeDoc: ActiveDoc|undefined; + try { + activeDoc = await this._getActiveDoc(req as RequestWithLogin); + await callback(activeDoc, req as RequestWithLogin, res); + if (!res.headersSent) { this._checkForMute(activeDoc); } + } catch (err) { + this._checkForMute(activeDoc); + throw err; + } + }; + } + + private async _getStates(activeDoc: ActiveDoc): Promise { + const states = await activeDoc.getRecentStates(); + return { + states, + }; + } + + private async _removeDoc(req: Request, res: Response, permanent: boolean) { + const scope = getDocScope(req); + const docId = getDocId(req); + if (permanent) { + const query = await this._dbManager.deleteDocument(scope); + this._dbManager.checkQueryResult(query); // fail immediately if deletion denied. + await this._docManager.deleteDoc(null, docId, true); + await sendReply(req, res, query); + } else { + await this._dbManager.softDeleteDocument(scope); + await sendOkReply(req, res); + } + await this._dbManager.flushSingleDocAuthCache(scope, docId); + await this._docManager.interruptDocClients(docId); + } +} + +export function addDocApiRoutes( + app: Application, docWorker: DocWorker, docManager: DocManager, dbManager: HomeDBManager, + grist: GristServer +) { + const api = new DocWorkerApi(app, docWorker, docManager, dbManager, grist); + api.addEndpoints(); +} + +/** + * Catches the errors thrown by the sandbox, and converts to more descriptive ones (such as for + * invalid table names, columns, or rowIds) with better status codes. Accepts the table name, a + * list of column names in that table, and a promise for the result of the sandbox call. + */ +async function handleSandboxError(tableId: string, colNames: string[], p: Promise): Promise { + try { + return await p; + } catch (e) { + if (e instanceof SandboxError) { + let match = e.message.match(/non-existent record #([0-9]+)/); + if (match) { + throw new ApiError(`Invalid row id ${match[1]}`, 400); + } + match = e.message.match(/\[Sandbox\] KeyError '(.*?)'/); + if (match) { + if (match[1] === tableId) { + throw new ApiError(`Table not found "${tableId}"`, 404); + } else if (colNames.includes(match[1])) { + throw new ApiError(`Invalid column "${match[1]}"`, 400); + } + } + throw new ApiError(`Error doing API call: ${e.message}`, 400); + } + throw e; + } +} + +/** + * Options for returning results from a query about document data. + * Currently these option don't affect the query itself, only the + * results returned to the user. + */ +export interface QueryParameters { + sort?: string[]; // Columns to sort by (ascending order by default, + // prepend "-" for descending order). + limit?: number; // Limit on number of rows to return. +} + + +/** + * Extract a sort parameter from a request, if present. Follows + * https://jsonapi.org/format/#fetching-sorting for want of a better + * standard - comma separated, defaulting to ascending order, keys + * prefixed by "-" for descending order. + * + * The sort parameter can either be given as a query parameter, or + * as a header. + */ +function getSortParameter(req: Request): string[]|undefined { + const sortString: string|undefined = optStringParam(req.query.sort) || req.get('X-Sort'); + if (!sortString) { return undefined; } + return sortString.split(','); +} + +/** + * Extract a limit parameter from a request, if present. Should be a + * simple integer. The limit parameter can either be given as a query + * parameter, or as a header. + */ +function getLimitParameter(req: Request): number|undefined { + const limitString: string|undefined = optStringParam(req.query.limit) || req.get('X-Limit'); + if (!limitString) { return undefined; } + const limit = parseInt(limitString, 10); + if (isNaN(limit)) { throw new Error('limit is not a number'); } + return limit; +} + +/** + * Extract sort and limit parameters from request, if they are present. + */ +function getQueryParameters(req: Request): QueryParameters { + return { + sort: getSortParameter(req), + limit: getLimitParameter(req), + }; +} + +/** + * Sort table contents being returned. Sort keys with a '-' prefix + * are sorted in descending order, otherwise ascending. Contents are + * modified in place. + */ +function applySort(values: TableColValues, sort: string[]) { + if (!sort) { return values; } + const sortKeys = sort.map(key => key.replace(/^-/, '')); + const iteratees = sortKeys.map(key => { + if (!(key in values)) { + throw new Error(`unknown key ${key}`); + } + const col = values[key]; + return (i: number) => col[i]; + }); + const sortSpec = sort.map((key, i) => (key.startsWith('-') ? -i - 1 : i + 1)); + const index = values.id.map((_, i) => i); + const sortFunc = new SortFunc({ + getColGetter(i) { return iteratees[i - 1]; }, + getManualSortGetter() { return null; } + }); + sortFunc.updateSpec(sortSpec); + index.sort(sortFunc.compare.bind(sortFunc)); + for (const key of Object.keys(values)) { + const col = values[key]; + values[key] = index.map(i => col[i]); + } + return values; +} + +/** + * Truncate columns to the first N values. Columns are modified in place. + */ +function applyLimit(values: TableColValues, limit: number) { + // for no limit, or 0 limit, do not apply any restriction + if (!limit) { return values; } + for (const key of Object.keys(values)) { + values[key].splice(limit); + } + return values; +} + +/** + * Apply query parameters to table contents. Contents are modified in place. + */ +export function applyQueryParameters(values: TableColValues, params: QueryParameters): TableColValues { + if (params.sort) { applySort(values, params.sort); } + if (params.limit) { applyLimit(values, params.limit); } + return values; +} diff --git a/app/server/lib/DocClients.ts b/app/server/lib/DocClients.ts new file mode 100644 index 00000000..f523b9ba --- /dev/null +++ b/app/server/lib/DocClients.ts @@ -0,0 +1,90 @@ +/** + * Module to manage the clients of an ActiveDoc. It keeps track of how many clients have the doc + * open, and what FD they are using. + */ + +import {arrayRemove} from 'app/common/gutil'; +import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import {Authorizer} from 'app/server/lib/Authorizer'; +import {Client} from 'app/server/lib/Client'; +import {sendDocMessage} from 'app/server/lib/Comm'; +import {DocSession} from 'app/server/lib/DocSession'; +import * as log from 'app/server/lib/log'; + +export class DocClients { + private _docSessions: DocSession[] = []; + + constructor( + public readonly activeDoc: ActiveDoc + ) {} + + /** + * Returns the number of connected clients. + */ + public clientCount(): number { + return this._docSessions.length; + } + + /** + * Adds a client's open file to the list of connected clients. + */ + public addClient(client: Client, authorizer: Authorizer): DocSession { + const docSession = client.addDocSession(this.activeDoc, authorizer); + this._docSessions.push(docSession); + log.debug("DocClients (%s) now has %d clients; new client is %s (fd %s)", this.activeDoc.docName, + this._docSessions.length, client.clientId, docSession.fd); + return docSession; + } + + /** + * Removes a client from the list of connected clients for this document. In other words, closes + * this DocSession. + */ + public removeClient(docSession: DocSession): void { + log.debug("DocClients.removeClient", docSession.client.clientId); + docSession.client.removeDocSession(docSession.fd); + + if (arrayRemove(this._docSessions, docSession)) { + log.debug("DocClients (%s) now has %d clients", this.activeDoc.docName, this._docSessions.length); + } + } + + /** + * Removes all active clients from this document, i.e. closes all DocSessions. + */ + public removeAllClients(): void { + log.debug("DocClients.removeAllClients() removing %s docSessions", this._docSessions.length); + const docSessions = this._docSessions.splice(0); + for (const docSession of docSessions) { + docSession.client.removeDocSession(docSession.fd); + } + } + + public interruptAllClients() { + log.debug("DocClients.interruptAllClients() interrupting %s docSessions", this._docSessions.length); + for (const docSession of this._docSessions) { + docSession.client.interruptConnection(); + } + } + + /** + * Broadcasts a message to all clients of this document using Comm.sendDocMessage. Also sends all + * docAction to active doc's plugin manager. + * @param {Object} client: Originating client used to set the `fromSelf` flag in the message. + * @param {String} type: The type of the message, e.g. 'docUserAction'. + * @param {Object} messageData: The data for this type of message. + */ + public broadcastDocMessage(client: Client|null, type: string, messageData: any): void { + for (let i = 0, len = this._docSessions.length; i < len; i++) { + const curr = this._docSessions[i]; + const fromSelf = (curr.client === client); + + sendDocMessage(curr.client, curr.fd, type, messageData, fromSelf); + } + if (type === "docUserAction" && messageData.docActions) { + for (const action of messageData.docActions) { + this.activeDoc.docPluginManager.receiveAction(action); + } + } + } +} diff --git a/app/server/lib/DocManager.ts b/app/server/lib/DocManager.ts new file mode 100644 index 00000000..e98edcbf --- /dev/null +++ b/app/server/lib/DocManager.ts @@ -0,0 +1,481 @@ +import * as pidusage from '@gristlabs/pidusage'; +import * as bluebird from 'bluebird'; +import {EventEmitter} from 'events'; +import noop = require('lodash/noop'); +import * as path from 'path'; + +import {ApiError} from 'app/common/ApiError'; +import {mapSetOrClear} from 'app/common/AsyncCreate'; +import {BrowserSettings} from 'app/common/BrowserSettings'; +import {DocCreationInfo, DocEntry, DocListAPI, OpenDocMode, OpenLocalDocResult} from 'app/common/DocListAPI'; +import {EncActionBundleFromHub} from 'app/common/EncActionBundle'; +import {Invite} from 'app/common/sharing'; +import {tbind} from 'app/common/tbind'; +import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI'; +import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import {assertAccess, Authorizer, DocAuthorizer, DummyAuthorizer, + isSingleUserMode} from 'app/server/lib/Authorizer'; +import {Client} from 'app/server/lib/Client'; +import {makeOptDocSession, OptDocSession} from 'app/server/lib/DocSession'; +import * as docUtils from 'app/server/lib/docUtils'; +import {GristServer} from 'app/server/lib/GristServer'; +import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; +import {makeForkIds, makeId} from 'app/server/lib/idUtils'; +import * as log from 'app/server/lib/log'; +import * as ServerMetrics from 'app/server/lib/ServerMetrics'; +import {ActiveDoc} from './ActiveDoc'; +import {PluginManager} from './PluginManager'; +import {getFileUploadInfo, globalUploadSet, makeAccessId, UploadInfo} from './uploads'; + +// A TTL in milliseconds to use for material that can easily be recomputed / refetched +// but is a bit of a burden under heavy traffic. +export const DEFAULT_CACHE_TTL = 10000; + +/** + * DocManager keeps track of "active" Grist documents, i.e. those loaded + * in-memory, with clients connected to them. + */ +export class DocManager extends EventEmitter { + // Maps docName to promise for ActiveDoc object. Most of the time the promise + // will be long since resolved, with the resulting document cached. + private _activeDocs: Map> = new Map(); + + constructor( + public readonly storageManager: IDocStorageManager, + public readonly pluginManager: PluginManager, + private _homeDbManager: HomeDBManager|null, + public gristServer: GristServer + ) { + super(); + } + + // attach a home database to the DocManager. During some tests, it + // is awkward to have this set up at the point of construction. + public testSetHomeDbManager(dbManager: HomeDBManager) { + this._homeDbManager = dbManager; + } + + /** + * Returns an implementation of the DocListAPI for the given Client object. + */ + public getDocListAPIImpl(client: Client): DocListAPI { + return { + getDocList: tbind(this.listDocs, this, client), + createNewDoc: tbind(this.createNewDoc, this, client), + importSampleDoc: tbind(this.importSampleDoc, this, client), + importDoc: tbind(this.importDoc, this, client), + deleteDoc: tbind(this.deleteDoc, this, client), + renameDoc: tbind(this.renameDoc, this, client), + openDoc: tbind(this.openDoc, this, client), + }; + } + + /** + * Returns the number of currently open docs. + */ + public numOpenDocs(): number { + return this._activeDocs.size; + } + + /** + * Returns a Map from docId to number of connected clients for each doc. + */ + public async getDocClientCounts(): Promise> { + const values = await Promise.all(Array.from(this._activeDocs.values(), async (adocPromise) => { + const adoc = await adocPromise; + return [adoc.docName, adoc.docClients.clientCount()] as [string, number]; + })); + return new Map(values); + } + + /** + * Returns a promise for all known Grist documents and document invites to show in the doc list. + */ + public async listDocs(client: Client): Promise<{docs: DocEntry[], docInvites: DocEntry[]}> { + const docs = await this.storageManager.listDocs(); + return {docs, docInvites: []}; + } + + /** + * Returns a promise for invites to docs which have not been downloaded. + */ + public async getLocalInvites(client: Client): Promise { + return []; + } + + /** + * Creates a new document, fetches it, and adds a table to it. + * @returns {Promise:String} The name of the new document. + */ + public async createNewDoc(client: Client): Promise { + log.debug('DocManager.createNewDoc'); + const activeDoc: ActiveDoc = await this.createNewEmptyDoc(makeOptDocSession(client), 'Untitled'); + await activeDoc.addInitialTable(); + return activeDoc.docName; + } + + /** + * Download a shared doc by creating a new doc and applying to it the shared doc snapshot actions. + * Also marks the invite to the doc as ignored, since it has already been accepted. + * @returns {Promise:String} The name of the new document. + */ + public async downloadSharedDoc(client: Client, docId: string, docName: string): Promise { + throw new Error('downloadSharedDoc not implemented'); + } + + /** + * Creates a new document, fetches it, and adds a table to it. + * @param {String} sampleDocName: Doc name of a sample document. + * @returns {Promise:String} The name of the new document. + */ + public async importSampleDoc(client: Client, sampleDocName: string): Promise { + const sourcePath = this.storageManager.getSampleDocPath(sampleDocName); + if (!sourcePath) { + throw new Error(`no path available to sample ${sampleDocName}`); + } + log.info('DocManager.importSampleDoc importing', sourcePath); + const basenameHint = path.basename(sampleDocName); + const targetName = await docUtils.createNumbered(basenameHint, '-', + (name: string) => docUtils.createExclusive(this.storageManager.getPath(name))); + + const targetPath = this.storageManager.getPath(targetName); + log.info('DocManager.importSampleDoc saving as', targetPath); + await docUtils.copyFile(sourcePath, targetPath); + return targetName; + } + + /** + * Processes an upload, containing possibly multiple files, to create a single new document, and + * returns the new document's name/id. + */ + public async importDoc(client: Client, uploadId: number): Promise { + const userId = this._homeDbManager ? await client.requireUserId(this._homeDbManager) : null; + const result = await this._doImportDoc(makeOptDocSession(client), + globalUploadSet.getUploadInfo(uploadId, this.makeAccessId(userId)), {naming: 'classic'}); + return result.id; + } + + // Import a document, assigning it a unique id distinct from its title. Cleans up uploadId. + public importDocWithFreshId(docSession: OptDocSession, userId: number, uploadId: number): Promise { + const accessId = this.makeAccessId(userId); + return this._doImportDoc(docSession, globalUploadSet.getUploadInfo(uploadId, accessId), + {naming: 'saved'}); + } + + // Do an import targeted at a specific workspace. Cleans up uploadId. + // UserId should correspond to the user making the request. + // A workspaceId of null results in an import to an unsaved doc, not + // associated with a specific workspace. + public async importDocToWorkspace( + userId: number, uploadId: number, workspaceId: number|null, browserSettings?: BrowserSettings, + ): Promise { + if (!this._homeDbManager) { throw new Error("HomeDbManager not available"); } + + const accessId = this.makeAccessId(userId); + const result = await this._doImportDoc(makeOptDocSession(null, browserSettings), + globalUploadSet.getUploadInfo(uploadId, accessId), { + naming: workspaceId ? 'saved' : 'unsaved', + userId, + }); + if (workspaceId) { + const queryResult = await this._homeDbManager.addDocument({userId}, workspaceId, + {name: result.title}, result.id); + if (queryResult.status !== 200) { + // TODO The ready-to-add document is not yet in storageManager, but is in the filesystem. It + // should get cleaned up in case of error here. + throw new ApiError(queryResult.errMessage || 'unable to add imported document', queryResult.status); + } + } + + // Ship the import to S3, since it isn't associated with any particular worker at this time. + // We could associate it with the current worker, but that is not necessarily desirable. + await this.storageManager.addToStorage(result.id); + return result; + } + + /** + * Imports file at filepath into the app by creating a new document and adding the file to + * the documents directory. + * @param {String} filepath - Path to the current location of the file on the server. + * @returns {Promise:String} The name of the new document. + */ + public async importNewDoc(filepath: string): Promise { + const uploadId = globalUploadSet.registerUpload([await getFileUploadInfo(filepath)], null, noop, null); + return await this._doImportDoc(makeOptDocSession(null), globalUploadSet.getUploadInfo(uploadId, null), + {naming: 'classic'}); + } + + /** + * Deletes the Grist files and directories for a given document name. + * @param {String} docName - The name of the Grist document to be deleted. + * @returns {Promise:String} The name of the deleted Grist document. + * + */ + public async deleteDoc(client: Client|null, docName: string, deletePermanently: boolean): Promise { + log.debug('DocManager.deleteDoc starting for %s', docName); + const docPromise = this._activeDocs.get(docName); + if (docPromise) { + // Call activeDoc's shutdown method first, to remove the doc from internal structures. + const doc: ActiveDoc = await docPromise; + await doc.shutdown(); + } + await this.storageManager.deleteDoc(docName, deletePermanently); + return docName; + } + + /** + * Interrupt all clients, forcing them to reconnect. Handy when a document has changed + * status in some major way that affects access rights, such as being deleted. + */ + public async interruptDocClients(docName: string) { + const docPromise = this._activeDocs.get(docName); + if (docPromise) { + const doc: ActiveDoc = await docPromise; + doc.docClients.interruptAllClients(); + } + } + + /** + * Opens a document. Adds the client as a subscriber to the document, and fetches and returns the + * document's metadata. + * @returns {Promise:Object} An object with properties: + * `docFD` - the descriptor to use in further methods and messages about this document, + * `doc` - the object with metadata tables. + */ + public async openDoc(client: Client, docId: string, + mode: OpenDocMode = 'default'): Promise { + let auth: Authorizer; + const dbManager = this._homeDbManager; + if (!isSingleUserMode()) { + if (!dbManager) { throw new Error("HomeDbManager not available"); } + // Sets up authorization of the document. + const org = client.getOrg(); + if (!org) { throw new Error('Documents can only be opened in the context of a specific organization'); } + const userId = await client.getUserId(dbManager) || dbManager.getAnonymousUserId(); + + // We use docId in the key, and disallow urlId, so we can be sure that we are looking at the + // right doc when we re-query the DB over the life of the websocket. + const key = {urlId: docId, userId, org}; + log.debug("DocManager.openDoc Authorizer key", key); + const docAuth = await dbManager.getDocAuthCached(key); + assertAccess('viewers', docAuth); + + if (docAuth.docId !== docId) { + // The only plausible way to end up here is if we called openDoc with a urlId rather + // than a docId. + throw new Error(`openDoc expected docId ${docAuth.docId} not urlId ${docId}`); + } + auth = new DocAuthorizer(dbManager, key, mode); + } else { + log.debug(`DocManager.openDoc not using authorization for ${docId} because GRIST_SINGLE_USER`); + auth = new DummyAuthorizer('owners'); + } + + // Fetch the document, and continue when we have the ActiveDoc (which may be immediately). + const activeDoc: ActiveDoc = await this.fetchDoc(makeOptDocSession(client), docId); + + if (activeDoc.muted) { + log.debug('DocManager.openDoc interrupting, called for a muted doc', docId); + client.interruptConnection(); + throw new Error(`document ${docId} cannot be opened right now`); + } + + const docSession = activeDoc.addClient(client, auth); + const [metaTables, recentActions] = await Promise.all([ + activeDoc.fetchMetaTables(client), + activeDoc.getRecentActions(client, false) + ]); + this.emit('open-doc', this.storageManager.getPath(activeDoc.docName)); + + ServerMetrics.get('docs.num_open').set(this._activeDocs.size); + ServerMetrics.get('app.have_doc_open').set(true); + ServerMetrics.get('app.doc_open_span').start(); + + return { + docFD: docSession.fd, + clientId: docSession.client.clientId, + doc: metaTables, + log: recentActions, + plugins: activeDoc.docPluginManager.getPlugins() + }; + } + + /** + * Shut down all open docs. This is called, in particular, on server shutdown. + */ + public async shutdownAll() { + await Promise.all(Array.from(this._activeDocs.values(), + adocPromise => adocPromise.then(adoc => adoc.shutdown()))); + try { + await this.storageManager.closeStorage(); + } catch (err) { + log.error('DocManager had problem shutting down storage: %s', err.message); + } + + // Clear the setInterval that the pidusage module sets up internally. + pidusage.clear(); + } + + // Access a document by name. + public getActiveDoc(docName: string): Promise|undefined { + return this._activeDocs.get(docName); + } + + public async removeActiveDoc(activeDoc: ActiveDoc): Promise { + this._activeDocs.delete(activeDoc.docName); + ServerMetrics.get('docs.num_open').set(this._activeDocs.size); + ServerMetrics.get('app.have_doc_open').set(this._activeDocs.size > 0); + ServerMetrics.get('app.doc_open_span').setRunning(this._activeDocs.size > 0); + } + + public async renameDoc(client: Client, oldName: string, newName: string): Promise { + log.debug('DocManager.renameDoc %s -> %s', oldName, newName); + const docPromise = this._activeDocs.get(oldName); + if (docPromise) { + const adoc: ActiveDoc = await docPromise; + await adoc.renameDocTo(client, newName); + this._activeDocs.set(newName, docPromise); + this._activeDocs.delete(oldName); + } else { + await this.storageManager.renameDoc(oldName, newName); + } + } + + public markAsChanged(activeDoc: ActiveDoc) { + if (!activeDoc.muted) { + this.storageManager.markAsChanged(activeDoc.docName); + } + } + + public markAsEdited(activeDoc: ActiveDoc) { + if (!activeDoc.muted) { + this.storageManager.markAsEdited(activeDoc.docName); + } + } + + /** + * Helper function for creating a new empty document that also emits an event. + * @param docSession The client session. + * @param basenameHint Suggested base name to use (no directory, no extension). + */ + public async createNewEmptyDoc(docSession: OptDocSession, basenameHint: string): Promise { + const docName = await this._createNewDoc(basenameHint); + return mapSetOrClear(this._activeDocs, docName, + this.gristServer.create.ActiveDoc(this, docName).createDoc(docSession)); + } + + /** + * Fetches an ActiveDoc object. Used by openDoc. + */ + public async fetchDoc(docSession: OptDocSession, docName: string): Promise { + log.debug('DocManager.fetchDoc', docName); + // Repeat until we acquire an ActiveDoc that is not muted (shutting down). + for (;;) { + if (!this._activeDocs.has(docName)) { + const newDoc = this.gristServer.create.ActiveDoc(this, docName); + // Propagate backupMade events from newly opened activeDocs (consolidate all to DocMan) + newDoc.on('backupMade', (bakPath: string) => { + this.emit('backupMade', bakPath); + }); + return mapSetOrClear(this._activeDocs, docName, newDoc.loadDoc(docSession)); + } + const activeDoc = await this._activeDocs.get(docName)!; + if (!activeDoc.muted) { return activeDoc; } + log.debug('DocManager.fetchDoc waiting because doc is muted', docName); + await bluebird.delay(1000); + } + } + + public makeAccessId(userId: number|null): string|null { + return makeAccessId(this.gristServer, userId); + } + + /** + * Helper function for creating a new shared document given the doc snapshot bundles received + * from the sharing hub. + * @param {String} basenameHint: Suggested base name to use (no directory, no extension). + * @param {String} docId: The docId of the doc received from the hub. + * @param {String} instanceId: The user instanceId creating the doc. + * @param {EncActionBundleFromHub[]} encBundles: The action bundles making up the doc snapshot. + * @returns {Promise:ActiveDoc} ActiveDoc for the newly created document. + */ + protected async _createNewSharedDoc(basenameHint: string, docId: string, instanceId: string, + encBundles: EncActionBundleFromHub[]): Promise { + const docName = await this._createNewDoc(basenameHint); + return mapSetOrClear(this._activeDocs, docName, + this.gristServer.create.ActiveDoc(this, docName).downloadSharedDoc(docId, instanceId, encBundles)); + } + + /** + * Helper that implements doing the actual import of an uploaded set of files to create a new + * document. + */ + private async _doImportDoc(docSession: OptDocSession, uploadInfo: UploadInfo, + options: { + naming: 'classic'|'saved'|'unsaved', + userId?: number, + }): Promise { + try { + const fileCount = uploadInfo.files.length; + const hasGristDoc = Boolean(uploadInfo.files.find(f => path.extname(f.origName) === '.grist')); + if (hasGristDoc && fileCount > 1) { + throw new Error('Grist docs must be uploaded individually'); + } + const first = uploadInfo.files[0].origName; + const ext = path.extname(first); + const basename = path.basename(first, ext).trim() || "Untitled upload"; + let id: string; + switch (options.naming) { + case 'saved': + id = makeId(); + break; + case 'unsaved': { + const {userId} = options; + if (!userId) { throw new Error('unsaved import requires userId'); } + if (!this._homeDbManager) { throw new Error("HomeDbManager not available"); } + const isAnonymous = userId === this._homeDbManager.getAnonymousUserId(); + id = makeForkIds({userId, isAnonymous, trunkDocId: NEW_DOCUMENT_CODE, + trunkUrlId: NEW_DOCUMENT_CODE}).docId; + break; + } + case 'classic': + id = basename; + break; + default: + throw new Error('naming mode not recognized'); + } + if (ext === '.grist') { + // If the import is a grist file, copy it to the docs directory. + // TODO: We should be skeptical of the upload file to close a possible + // security vulnerability. See https://phab.getgrist.com/T457. + const docName = await this._createNewDoc(id); + const docPath = await this.storageManager.getPath(docName); + await docUtils.copyFile(uploadInfo.files[0].absPath, docPath); + return {title: basename, id: docName}; + } else { + const doc = await this.createNewEmptyDoc(docSession, id); + await doc.oneStepImport(docSession, uploadInfo); + return {title: basename, id: doc.docName}; + } + } catch (err) { + throw new ApiError(err.message, err.status || 400, { + tips: [{action: 'ask-for-help', message: 'Ask for help'}] + }); + } finally { + await globalUploadSet.cleanup(uploadInfo.uploadId); + } + } + + // Returns the name for a new doc, based on basenameHint. + private async _createNewDoc(basenameHint: string): Promise { + const docName: string = await docUtils.createNumbered(basenameHint, '-', async (name: string) => { + if (this._activeDocs.has(name)) { + throw new Error("Existing entry in active docs for: " + name); + } + return docUtils.createExclusive(this.storageManager.getPath(name)); + }); + log.debug('DocManager._createNewDoc picked name', docName); + await this.pluginManager.pluginsLoaded; + return docName; + } +} diff --git a/app/server/lib/DocPluginData.ts b/app/server/lib/DocPluginData.ts new file mode 100644 index 00000000..1dbacab9 --- /dev/null +++ b/app/server/lib/DocPluginData.ts @@ -0,0 +1,32 @@ +import {Promisified} from 'app/common/tpromisified'; +import {Storage} from 'app/plugin/StorageAPI'; +import {DocStorage} from 'app/server/lib/DocStorage'; + +/** + * DocPluginData implements a document's `Storage` for plugin. + */ +export class DocPluginData implements Promisified { + constructor(private _docStorage: DocStorage, private _pluginId: string) { + // nothing to do here + } + public async getItem(key: string): Promise { + const res = await this._docStorage.getPluginDataItem(this._pluginId, key); + if (typeof res === 'string') { + return JSON.parse(res); + } + return res; + } + public hasItem(key: string): Promise { + return this._docStorage.hasPluginDataItem(this._pluginId, key); + } + public setItem(key: string, value: any): Promise { + return this._docStorage.setPluginDataItem(this._pluginId, key, JSON.stringify(value)); + } + public removeItem(key: string): Promise { + return this._docStorage.removePluginDataItem(this._pluginId, key); + } + public clear(): Promise { + return this._docStorage.clearPluginDataItem(this._pluginId); + } + +} diff --git a/app/server/lib/DocPluginManager.ts b/app/server/lib/DocPluginManager.ts new file mode 100644 index 00000000..e1ed8d16 --- /dev/null +++ b/app/server/lib/DocPluginManager.ts @@ -0,0 +1,218 @@ +import {ApplyUAResult} from 'app/common/ActiveDocAPI'; +import {fromTableDataAction, TableColValues} from 'app/common/DocActions'; +import * as gutil from 'app/common/gutil'; +import {LocalPlugin} from 'app/common/plugin'; +import {createRpcLogger, PluginInstance} from 'app/common/PluginInstance'; +import {Promisified} from 'app/common/tpromisified'; +import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI'; +import {checkers, GristTable} from "app/plugin/grist-plugin-api"; +import {GristDocAPI} from "app/plugin/GristAPI"; +import {Storage} from 'app/plugin/StorageAPI'; +import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import {DocPluginData} from 'app/server/lib/DocPluginData'; +import {FileParserElement} from 'app/server/lib/FileParserElement'; +import {GristServer} from 'app/server/lib/GristServer'; +import * as log from 'app/server/lib/log'; +import { SafePythonComponent } from 'app/server/lib/SafePythonComponent'; +import { UnsafeNodeComponent } from 'app/server/lib/UnsafeNodeComponent'; +import { promisifyAll } from 'bluebird'; +import * as fse from 'fs-extra'; +import * as path from 'path'; +import * as tmp from 'tmp'; + + +promisifyAll(tmp); + +/** + * Implements GristDocAPI interface. + */ +class GristDocAPIImpl implements GristDocAPI { + constructor(private _activeDoc: ActiveDoc) {} + + public async getDocName() { return this._activeDoc.docName; } + + public async listTables(): Promise { + const table = this._activeDoc.docData!.getTable('_grist_Tables')!; + return (table.getColValues('tableId') as string[]) + .filter(id => !id.startsWith("GristSummary_")).sort(); + } + + public async fetchTable(tableId: string): Promise { + return fromTableDataAction(await this._activeDoc.fetchTable(null, tableId)); + } + + public applyUserActions(actions: any[][]): Promise { + return this._activeDoc.applyUserActions({client: null}, actions); + } +} + +/** + * DocPluginManager manages plugins for a document. + * + * DocPluginManager instanciates asynchronously. Wait for the `ready` to resolve before using any + * plugin. + * + */ +export class DocPluginManager { + + public readonly plugins: {[s: string]: PluginInstance} = {}; + public readonly ready: Promise; + public readonly gristDocAPI: GristDocAPI; + + private _tmpDir: string; + private _pluginInstances: PluginInstance[]; + + + constructor(private _localPlugins: LocalPlugin[], private _appRoot: string, private _activeDoc: ActiveDoc, private _server: GristServer) { + this.gristDocAPI = new GristDocAPIImpl(_activeDoc); + this._pluginInstances = []; + this.ready = this._initialize(); + } + + public tmpDir(): string { + return this._tmpDir; + } + + /** + * To be moved in ActiveDoc.js as a new implementation for ActiveDoc.importFile. + * Throws if no importers can parse the file. + */ + public async parseFile(filePath: string, fileName: string, parseOptions: ParseOptions): Promise { + + // Support an existing grist json format directly for files with a "jgrist" + // extension. + if (path.extname(fileName) === '.jgrist') { + try { + const result = JSON.parse(await fse.readFile(filePath, 'utf8')) as ParseFileResult; + result.parseOptions = {}; + // The parseOptions component isn't checked here, since it seems free-form. + checkers.ParseFileResult.check(result); + checkReferences(result.tables); + return result; + } catch (err) { + throw new Error('Grist json format could not be parsed: ' + err); + } + } + + const matchingFileParsers: FileParserElement[] = FileParserElement.getMatching(this._pluginInstances, fileName); + + if (!this._tmpDir) { + throw new Error("DocPluginManager: initialization has not completed"); + } + + // TODO: PluginManager shouldn't patch path here. Instead it should expose a method to create + // dataSources, that would move the file to under _tmpDir and return an object with the relative + // path. + filePath = path.relative(this._tmpDir, filePath); + log.debug(`parseFile: found ${matchingFileParsers.length} fileParser with matching file extensions`); + const messages = []; + for (const {plugin, parseFileStub} of matchingFileParsers) { + const name = plugin.definition.id; + try { + log.info(`DocPluginManager.parseFile: calling to ${name} with ${filePath}`); + const result = await parseFileStub.parseFile({path: filePath, origName: fileName}, parseOptions); + checkers.ParseFileResult.check(result); + checkReferences(result.tables); + return result; + } catch (err) { + const cleanerMessage = err.message.replace(/^\[Sandbox\] (Exception)?/, '').trim(); + messages.push(cleanerMessage); + log.warn(`DocPluginManager.parseFile: ${name} Failed parseFile `, err.message); + continue; + } + } + const details = messages.length ? ": " + messages.join("; ") : ""; + throw new Error(`Cannot parse this data${details}`); + } + + /** + * Returns a promise which resolves with the list of plugins definitions. + */ + public getPlugins(): LocalPlugin[] { + return this._localPlugins; + } + + /** + * Shut down all plugins for this document. + */ + public async shutdown(): Promise { + const names = Object.keys(this.plugins); + log.debug("DocPluginManager.shutdown cleaning up %s plugins", names.length); + await Promise.all(names.map(name => this.plugins[name].shutdown())); + if (this._tmpDir) { + log.debug("DocPluginManager.shutdown removing tmpDir %s", this._tmpDir); + await fse.remove(this._tmpDir); + } + } + + /** + * Reload plugins: shutdown all plugins, clear list of plugins and load new ones. Returns a + * promise that resolves when initialisation is done. + */ + public async reload(plugins: LocalPlugin[]): Promise { + await this.shutdown(); + this._pluginInstances = []; + this._localPlugins = plugins; + await this._initialize(); + } + + public receiveAction(action: any[]): void { + for (const plugin of this._pluginInstances) { + const unsafeNode = plugin.unsafeNode as UnsafeNodeComponent; + if (unsafeNode) { + unsafeNode.receiveAction(action); + } + } + } + + private async _initialize(): Promise { + this._tmpDir = await tmp.dirAsync({prefix: 'grist-tmp-', unsafeCleanup: true}); + for (const plugin of this._localPlugins) { + try { + // todo: once Comm has been replaced by grain-rpc, pluginInstance.rpc should forward '*' to client + const pluginInstance = new PluginInstance(plugin, createRpcLogger(log, `PLUGIN ${plugin.id}:`)); + pluginInstance.rpc.registerForwarder('grist', pluginInstance.rpc, ''); + pluginInstance.rpc.registerImpl("GristDocAPI", this.gristDocAPI, checkers.GristDocAPI); + pluginInstance.rpc.registerImpl>("DocStorage", + new DocPluginData(this._activeDoc.docStorage, plugin.id), checkers.Storage); + const components = plugin.manifest.components; + if (components) { + const {safePython, unsafeNode} = components; + if (safePython) { + const comp = pluginInstance.safePython = new SafePythonComponent(plugin, safePython, this._tmpDir, + this._activeDoc.docName, this._server); + pluginInstance.rpc.registerForwarder(safePython, comp); + } + if (unsafeNode) { + const gristDocPath = this._activeDoc.docStorage.docPath; + const comp = pluginInstance.unsafeNode = new UnsafeNodeComponent(plugin, pluginInstance.rpc, unsafeNode, + this._appRoot, gristDocPath); + pluginInstance.rpc.registerForwarder(unsafeNode, comp); + } + } + this._pluginInstances.push(pluginInstance); + } catch (err) { + log.info(`DocPluginInstance: failed to create instance ${plugin.id}: ${err.message}`); + } + } + for (const instance of this._pluginInstances) { + this.plugins[instance.definition.id] = instance; + } + } +} + +/** + * Checks that tables include all the tables referenced by tables columns. Throws an exception + * otherwise. + */ +function checkReferences(tables: GristTable[]) { + const tableIds = tables.map(table => table.table_name); + for (const table of tables) { + for (const col of table.column_metadata) { + const refTableId = gutil.removePrefix(col.type, "Ref:"); + if (refTableId && !tableIds.includes(refTableId)) { + throw new Error(`Column type: ${col.type}, references an unknown table`); + } + } + } +} diff --git a/app/server/lib/DocSession.ts b/app/server/lib/DocSession.ts new file mode 100644 index 00000000..98409bd4 --- /dev/null +++ b/app/server/lib/DocSession.ts @@ -0,0 +1,49 @@ +import {BrowserSettings} from 'app/common/BrowserSettings'; +import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import {Authorizer, RequestWithLogin} from 'app/server/lib/Authorizer'; +import {Client} from 'app/server/lib/Client'; + +/** + * OptDocSession allows for certain ActiveDoc operations to work with or without an open document. + * It is useful in particular for actions when importing a file to create a new document. + */ +export interface OptDocSession { + client: Client|null; + shouldBundleActions?: boolean; + linkId?: number; + browserSettings?: BrowserSettings; + req?: RequestWithLogin; +} + +export function makeOptDocSession(client: Client|null, browserSettings?: BrowserSettings): OptDocSession { + if (client && !browserSettings) { browserSettings = client.browserSettings; } + return {client, browserSettings}; +} + +/** + * DocSession objects maintain information for a single session<->doc instance. + */ +export class DocSession implements OptDocSession { + /** + * Flag to indicate that user actions 'bundle' process is started and in progress (`true`), + * otherwise it's `false` + */ + public shouldBundleActions?: boolean; + + /** + * Indicates the actionNum of the previously applied action + * to which the first action in actions should be linked. + * Linked actions appear as one action and can be undone/redone in a single step. + */ + public linkId?: number; + + constructor( + public readonly activeDoc: ActiveDoc, + public readonly client: Client, + public readonly fd: number, + public readonly authorizer: Authorizer + ) {} + + // Browser settings (like timezone) obtained from the Client. + public get browserSettings(): BrowserSettings { return this.client.browserSettings; } +} diff --git a/app/server/lib/DocSnapshots.ts b/app/server/lib/DocSnapshots.ts new file mode 100644 index 00000000..f3fea923 --- /dev/null +++ b/app/server/lib/DocSnapshots.ts @@ -0,0 +1,151 @@ +import { KeyedOps } from 'app/common/KeyedOps'; +import { ExternalStorage } from 'app/server/lib/ExternalStorage'; +import * as log from 'app/server/lib/log'; +import * as moment from 'moment'; + +/** + * Metadata about a single document version. + */ +export interface ObjSnapshot { + lastModified: Date; + snapshotId: string; +} + +/** + * Information about a single document snapshot in S3, including a Grist docId. + * Similar to a type in app/common/UserAPI, but with lastModified as a Date + * rather than a string. + */ +export interface DocSnapshot extends ObjSnapshot { + docId: string; +} + +/** + * A collection of document snapshots. Most recent snapshots first. + */ +export interface DocSnapshots { + snapshots: DocSnapshot[]; +} + +/** + * A utility for pruning snapshots, so the number of snapshots doesn't get out of hand. + */ +export class DocSnapshotPruner { + private _closing: boolean = false; // when set, should ignore prune requests + private _prunes: KeyedOps; + + // Specify store to be pruned, and delay before pruning. + constructor(private _ext: ExternalStorage, _options: { + delayBeforeOperationMs?: number, + minDelayBetweenOperationsMs?: number + } = {}) { + this._prunes = new KeyedOps((key) => this.prune(key), { + ..._options, + retry: false, + logError: (key, failureCount, err) => log.error(`Pruning document ${key} gave error ${err}`) + }); + } + + // Shut down. Prunes scheduled for the future are run immediately. + // Can be called repeated safely. + public async close() { + this._closing = true; + this._prunes.expediteOperations(); + await this.wait(); + } + + // Wait for all in-progress prunes to finish up in an orderly fashion. + public async wait() { + await this._prunes.wait(() => 'waiting for pruning to finish'); + } + + // Note that a document has changed, and should be pruned (or repruned). Pruning operation + // done as a background operation. + public requestPrune(key: string) { + // If closing down, do not accept any prune requests. + if (this._closing) { return; } + // Mark the key as needing work. + this._prunes.addOperation(key); + } + + // Get all snapshots for a document, and whether they should be kept or pruned. + public async classify(key: string): Promise> { + const versions = await this._ext.versions(key); + return shouldKeepSnapshots(versions).map((keep, index) => ({keep, snapshot: versions[index]})); + } + + // Prune the specified document immediately. + public async prune(key: string) { + const versions = await this.classify(key); + const redundant = versions.filter(v => !v.keep); + await this._ext.remove(key, redundant.map(r => r.snapshot.snapshotId)); + log.info(`Pruned ${redundant.length} versions of ${versions.length} for document ${key}`); + } +} + +/** + * Calculate which snapshots to keep. Expects most recent snapshots to be first. + * We keep: + * - The five most recent versions (including the current version) + * - The most recent version in every hour, for up to 25 hours before the current version + * - The most recent version in every day, for up to 32 days before the current version + * - The most recent version in every week, for up to 12 weeks before the current version + * - The most recent version in every month, for up to 36 months before the current version + * - The most recent version in every year, for up to 1000 years before the current version + * Calculations done in UTC, Gregorian calendar, ISO weeks (week starts with Monday). + */ +export function shouldKeepSnapshots(snapshots: ObjSnapshot[]): boolean[] { + // Get current version + const current = snapshots[0]; + if (!current) { return []; } + + // Get time of current version + const start = moment.utc(current.lastModified); + + // Track saved version per hour, day, week, month, year, and number of times a version + // has been saved based on a corresponding rule. + const buckets: TimeBucket[] = [ + {range: 'hour', prev: start, usage: 0, cap: 25}, + {range: 'day', prev: start, usage: 0, cap: 32}, + {range: 'isoWeek', prev: start, usage: 0, cap: 12}, + {range: 'month', prev: start, usage: 0, cap: 36}, + {range: 'year', prev: start, usage: 0, cap: 1000} + ]; + // For each snapshot starting with newest, check if it is worth saving by comparing + // it with the last saved snapshot based on hour, day, week, month, year + return snapshots.map((snapshot, index) => { + let keep = index < 5; // Keep 5 most recent versions + const date = moment.utc(snapshot.lastModified); + for (const bucket of buckets) { + if (updateAndCheckRange(date, bucket)) { keep = true; } + } + return keep; + }); +} + +/** + * Check whether time `t` is in the same time-bucket as the time + * stored in `prev` for that time-bucket, and the time-bucket has not + * been used to its limit to justify saving versions. + * + * If all is good, we return true, store `t` in the appropriate + * time-bucket in `prev`, and increment the usage count. Note keeping + * a single version can increment usage on several buckets. This is + * easy to change, but other variations have results that feel + * counter-intuitive. + */ +function updateAndCheckRange(t: moment.Moment, bucket: TimeBucket) { + if (bucket.usage < bucket.cap && !t.isSame(bucket.prev, bucket.range)) { + bucket.prev = t; + bucket.usage++; + return true; + } + return false; +} + +interface TimeBucket { + range: 'hour' | 'day' | 'isoWeek' | 'month' | 'year', + prev: moment.Moment; // last time stored in this bucket + usage: number; // number of times this bucket justified saving a snapshot + cap: number; // maximum number of usages permitted +} diff --git a/app/server/lib/DocStorage.ts b/app/server/lib/DocStorage.ts new file mode 100644 index 00000000..03586daf --- /dev/null +++ b/app/server/lib/DocStorage.ts @@ -0,0 +1,1353 @@ +/** + * Module to handle the storage of Grist documents. + * + * A Grist document is stored as a SQLite database file. We keep everything in a single database + * file, including attachments, for the sake of having a single file represent a single "document" + * or "data set". + */ + + +import * as sqlite3 from '@gristlabs/sqlite3'; +import {LocalActionBundle} from 'app/common/ActionBundle'; +import {DocAction, TableColValues, TableDataAction, toTableDataAction} from 'app/common/DocActions'; +import * as gristTypes from 'app/common/gristTypes'; +import * as marshal from 'app/common/marshal'; +import * as schema from 'app/common/schema'; +import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl'; +import {ExpandedQuery} from 'app/server/lib/ExpandedQuery'; +import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; +import * as log from 'app/server/lib/log'; +import * as assert from 'assert'; +import * as bluebird from 'bluebird'; +import * as fse from 'fs-extra'; +import chunk = require('lodash/chunk'); +import groupBy = require('lodash/groupBy'); +import * as _ from 'underscore'; +import * as util from 'util'; +import * as uuidv4 from "uuid/v4"; +import {ISQLiteDB, OpenMode, quoteIdent, ResultRow, SchemaInfo, SQLiteDB} from './SQLiteDB'; + + +// Run with environment variable NODE_DEBUG=db (may include additional comma-separated sections) +// for verbose logging. +const debuglog = util.debuglog('db'); + +const maxSQLiteVariables = 500; // Actually could be 999, so this is playing it safe. + +export class DocStorage implements ISQLiteDB { + + // ====================================================================== + // Static fields + // ====================================================================== + + /** + * Schema for all system tables, i.e. those that are NOT known by the data engine. Regular + * metadata tables (such as _grist_DocInfo) are created via DocActions received from + * InitNewDoc useraction. + * + * The current "Storage Version" used by Grist is the length of the migrations list in its + * Schema. We use it to track changes to how data is stored on disk, and changes to the schema + * of non-data-engine tables (such as _gristsys_* tables). By contrast, "Schema Version" keeps + * track of the version of data-engine metadata. In SQLite, we use "PRAGMA user_version" to + * store the storage version number. + */ + public static docStorageSchema: SchemaInfo = { + async create(db: SQLiteDB): Promise { + await db.exec(`CREATE TABLE _gristsys_Files ( + id INTEGER PRIMARY KEY, + ident TEXT UNIQUE, + data BLOB + )`); + await db.exec(`CREATE TABLE _gristsys_Action ( + id INTEGER PRIMARY KEY, + "actionNum" BLOB DEFAULT 0, + "time" BLOB DEFAULT 0, + "user" BLOB DEFAULT '', + "desc" BLOB DEFAULT '', + "otherId" BLOB DEFAULT 0, + "linkId" BLOB DEFAULT 0, + "json" BLOB DEFAULT '' + )`); + await db.exec(`CREATE TABLE _gristsys_Action_step ( + id INTEGER PRIMARY KEY, + "parentId" BLOB DEFAULT 0, + "type" BLOB DEFAULT '', + "name" BLOB DEFAULT '', + "tableId" BLOB DEFAULT '', + "colIds" BLOB DEFAULT '', + "rowIds" BLOB DEFAULT '', + "values" BLOB DEFAULT '', + "json" BLOB DEFAULT '' + )`); + await db.exec(`CREATE TABLE _gristsys_ActionHistory ( + id INTEGER PRIMARY KEY, -- Plain integer action ID ("actionRef") + actionHash TEXT UNIQUE, -- Action checksum + parentRef INTEGER, -- id of parent of this action + actionNum INTEGER, -- distance from root of tree in actions + body BLOB -- content of action + )`); + await db.exec(`CREATE TABLE _gristsys_ActionHistoryBranch ( + id INTEGER PRIMARY KEY, -- Numeric branch ID + name TEXT UNIQUE, -- Branch name + actionRef INTEGER -- Latest action on branch + )`); + for (const branchName of ['shared', 'local_sent', 'local_unsent']) { + await db.run("INSERT INTO _gristsys_ActionHistoryBranch(name) VALUES(?)", + branchName); + } + // This is a single row table (enforced by the CHECK on 'id'), containing non-shared info. + // - ownerInstanceId is the id of the instance which owns this copy of the Grist doc. + // - docId is also kept here because it should not be changeable by UserActions. + await db.exec(`CREATE TABLE _gristsys_FileInfo ( + id INTEGER PRIMARY KEY CHECK (id = 0), + docId TEXT DEFAULT '', + ownerInstanceId TEXT DEFAULT '' + )`); + await db.exec("INSERT INTO _gristsys_FileInfo (id) VALUES (0)"); + await db.exec(`CREATE TABLE _gristsys_PluginData ( + id INTEGER PRIMARY KEY, -- Plain integer plugin data id + pluginId TEXT NOT NULL, -- Plugin id + key TEXT NOT NULL, -- the key + value BLOB DEFAULT '' -- the value associated with the key + ); + -- Plugins have unique keys. + CREATE UNIQUE INDEX _gristsys_PluginData_unique_key on _gristsys_PluginData(pluginId, key);`); + }, + migrations: [ + async function(db: SQLiteDB): Promise { + // Storage version 1 does not require a migration. Docs at v1 (or before) may not all + // be the same, and are only made uniform by v2. + }, + async function(db: SQLiteDB): Promise { + // Storage version 2. We change the types of all columns to BLOBs. + // This applies to all Grist tables, including metadata. + const migrationLabel = "DocStorage.docStorageSchema.migrations[v1->v2]"; + const oldMaxPosDefault = String(Math.pow(2, 31) - 1); + + function _upgradeTable(tableId: string) { + log.debug(`${migrationLabel}: table ${tableId}`); + // This returns rows with (at least) {name, type, dflt_value}. + return db.all(`PRAGMA table_info(${quoteIdent(tableId)})`) + .then(infoRows => { + const colListSql = infoRows.map(info => quoteIdent(info.name)).join(', '); + const colSpecSql = infoRows.map(_sqlColSpec).join(', '); + const tmpTableId = DocStorage._makeTmpTableId(tableId); + debuglog(`${migrationLabel}: ${tableId} (${colSpecSql})`); + return db.runEach( + `CREATE TABLE ${quoteIdent(tmpTableId)} (${colSpecSql})`, + `INSERT INTO ${quoteIdent(tmpTableId)} SELECT ${colListSql} FROM ${quoteIdent(tableId)}`, + `DROP TABLE ${quoteIdent(tableId)}`, + `ALTER TABLE ${quoteIdent(tmpTableId)} RENAME TO ${quoteIdent(tableId)}` + ); + }); + } + + function _sqlColSpec(info: ResultRow): string { + if (info.name === 'id') { return 'id INTEGER PRIMARY KEY'; } + // Fix the default for PositionNumber and ManualPos types, if set to a wrong old value. + const dfltValue = (info.type === 'REAL' && info.dflt_value === oldMaxPosDefault) ? + DocStorage._formattedDefault('PositionNumber') : + // The string "undefined" is also an invalid default; fix that too. + (info.dflt_value === 'undefined' ? 'NULL' : info.dflt_value); + + return DocStorage._sqlColSpecFromDBInfo(Object.assign({}, info, { + type: 'BLOB', + dflt_value: dfltValue + })); + } + + // Some migration-type steps pre-date storage migrations. We can do them once for the first + // proper migration (i.e. this one, to v2), and then never worry about them for upgraded docs. + + // Create table for files that wasn't always created in the past. + await db.exec(`CREATE TABLE IF NOT EXISTS _gristsys_Files ( + id INTEGER PRIMARY KEY, + ident TEXT UNIQUE, + data BLOB + )`); + // Create _gristsys_Action.linkId column that wasn't always created in the past. + try { + await db.exec('ALTER TABLE _gristsys_Action ADD COLUMN linkId INTEGER'); + log.debug("${migrationLabel}: Column linkId added to _gristsys_Action"); + } catch (err) { + if (!(/duplicate/.test(err.message))) { + // ok if column already existed + throw err; + } + } + // Deal with the transition to blob types + const tblRows = await db.all("SELECT name FROM sqlite_master WHERE type='table'"); + for (const tblRow of tblRows) { + // Note that _gristsys_Action tables in the past used Grist actions to create appropriate + // tables, so docs from that period would use BLOBs. For consistency, we upgrade those tables + // too. + if (tblRow.name.startsWith('_grist_') || !tblRow.name.startsWith('_') || + tblRow.name.startsWith('_gristsys_Action')) { + await _upgradeTable(tblRow.name); + } + } + }, + async function(db: SQLiteDB): Promise { + // Storage version 3. Convert old _gristsys_Action* tables to _gristsys_ActionHistory*. + await db.exec(`CREATE TABLE IF NOT EXISTS _gristsys_ActionHistory ( + id INTEGER PRIMARY KEY, + actionHash TEXT UNIQUE, + parentRef INTEGER, + actionNum INTEGER, + body BLOB + )`); + await db.exec(`CREATE TABLE IF NOT EXISTS _gristsys_ActionHistoryBranch ( + id INTEGER PRIMARY KEY, + name TEXT UNIQUE, + actionRef INTEGER + )`); + for (const branchName of ['shared', 'local_sent', 'local_unsent']) { + await db.run("INSERT OR IGNORE INTO _gristsys_ActionHistoryBranch(name) VALUES(?)", + branchName); + } + // Migrate any ActionLog information as best we can + const actions = await db.all("SELECT * FROM _gristsys_Action ORDER BY actionNum"); + const steps = groupBy(await db.all("SELECT * FROM _gristsys_Action_step ORDER BY id"), + 'parentId'); + await db.execTransaction(async () => { + const history = new ActionHistoryImpl(db); + await history.initialize(); + for (const action of actions) { + const step = steps[action.actionNum] || []; + const crudeTranslation: LocalActionBundle = { + actionNum: history.getNextHubActionNum(), + actionHash: null, + parentActionHash: null, + envelopes: [], + info: [ + 0, + { + time: action.time, + user: action.user, + inst: "", + desc: action.desc, + otherId: action.otherId, + linkId: action.linkId + } + ], + // Take what was logged as a UserAction and treat it as a DocAction. Summarization + // currently depends on stored+undo fields to understand what changed in an ActionBundle. + // DocActions were not logged prior to this version, so we have to fudge things a little. + stored: [[0, JSON.parse(action.json) as DocAction]], + calc: [], + userActions: [JSON.parse(action.json)], + undo: step.map(row => JSON.parse(row.json)) + }; + await history.recordNextShared(crudeTranslation); + } + await db.run("DELETE FROM _gristsys_Action_step"); + await db.run("DELETE FROM _gristsys_Action"); + }); + }, + async function(db: SQLiteDB): Promise { + // Storage version 4. Maintain docId and ownerInstanceId in a single-row special table; + // for standalone sharing. + await db.exec(`CREATE TABLE _gristsys_FileInfo ( + id INTEGER PRIMARY KEY CHECK (id = 0), + docId TEXT DEFAULT '', + ownerInstanceId TEXT DEFAULT '' + )`); + await db.exec("INSERT INTO _gristsys_FileInfo (id) VALUES (0)"); + }, + async function(db: SQLiteDB): Promise { + // Storage version 5. Add a table to maintain per-plugin data, for plugins' Storage API. + await db.exec(`CREATE TABLE _gristsys_PluginData ( + id INTEGER PRIMARY KEY, + pluginId TEXT NOT NULL, + key TEXT NOT NULL, + value BLOB DEFAULT '' + ); + CREATE UNIQUE INDEX IF NOT EXISTS _gristsys_PluginData_unique_key on _gristsys_PluginData(pluginId, key);`); + }, + + async function(db: SQLiteDB): Promise { + // Storage version 6. Migration to fix columns in user tables which have an incorrect + // DEFAULT for their Grist type, due to bug T462. + const migrationLabel = "DocStorage.docStorageSchema.migrations[v5->v6]"; + + const colRows: ResultRow[] = await db.all('SELECT t.tableId, c.colId, c.type ' + + 'FROM _grist_Tables_column c JOIN _grist_Tables t ON c.parentId=t.id'); + const docSchema = new Map(); // Maps tableId.colId to grist type. + for (const {tableId, colId, type} of colRows) { + docSchema.set(`${tableId}.${colId}`, type); + } + + // Fixes defaults and affected null values in a particular table. + async function _fixTable(tableId: string) { + log.debug(`${migrationLabel}: table ${tableId}`); + // This returns rows with (at least) {name, type, dflt_value}. + const infoRows: ResultRow[] = await db.all(`PRAGMA table_info(${quoteIdent(tableId)})`); + const origColSpecSql = infoRows.map(_sqlColSpec).join(', '); + + // Get the column SQL for what the columns should be, and the value SQL for how to + // prepare the values to fill them in. + const fixes = infoRows.map((r) => _getInfoAndValuesSql(r, tableId)); + const newColSpecSql = fixes.map(pair => pair[0]).map(_sqlColSpec).join(', '); + const valuesSql = fixes.map(pair => pair[1]).join(', '); + + // Rebuild the table only if any column's SQL (e.g. DEFAULT values) have changed. + if (newColSpecSql === origColSpecSql) { + debuglog(`${migrationLabel}: ${tableId} unchanged: (${newColSpecSql})`); + } else { + debuglog(`${migrationLabel}: ${tableId} changed: (${newColSpecSql})`); + const tmpTableId = DocStorage._makeTmpTableId(tableId); + return db.runEach( + `CREATE TABLE ${quoteIdent(tmpTableId)} (${newColSpecSql})`, + `INSERT INTO ${quoteIdent(tmpTableId)} SELECT ${valuesSql} FROM ${quoteIdent(tableId)}`, + `DROP TABLE ${quoteIdent(tableId)}`, + `ALTER TABLE ${quoteIdent(tmpTableId)} RENAME TO ${quoteIdent(tableId)}` + ); + } + } + + // Look up the type for a single column, and if the default changed to non-NULL, construct + // the updated column SQL and the value SQL for how to prepare values. + function _getInfoAndValuesSql(info: ResultRow, tableId: string): [ResultRow, string] { + const qColId = quoteIdent(info.name); + const gristType = docSchema.get(`${tableId}.${info.name}`); + if (gristType) { + const dflt = DocStorage._formattedDefault(gristType); + if (info.dflt_value === 'NULL' && dflt !== 'NULL') { + return [{...info, dflt_value: dflt}, `IFNULL(${qColId}, ${dflt}) as ${qColId}`]; + } + } + return [info, qColId]; + } + + function _sqlColSpec(info: ResultRow): string { + if (info.name === 'id') { return 'id INTEGER PRIMARY KEY'; } + return DocStorage._sqlColSpecFromDBInfo(info); + } + + // Go through all user tables and fix them. + const tblRows = await db.all("SELECT name FROM sqlite_master WHERE type='table'"); + for (const tblRow of tblRows) { + if (!tblRow.name.startsWith('_')) { + await _fixTable(tblRow.name); + } + } + }, + ] + }; + + /** + * Decodes a database row object, returning a new object with decoded values. This is needed for + * Grist data, which is encoded. + */ + public static decodeRowValues(dbRow: ResultRow): any { + return _.mapObject(dbRow, val => DocStorage._decodeValue(val)); + } + + /** + * Internal helper to distinguish which tables contain information about the metadata + * that docstorage needs to keep track of + */ + private static _isMetadataTable(tableId: string): boolean { + return tableId === "_grist_Tables" || tableId === "_grist_Tables_column"; + } + + /** + * Shortcut to get the SQL default for the given Grist type. + */ + private static _formattedDefault(colType: string): any { + return gristTypes.getDefaultForType(colType, {sqlFormatted: true}); + } + + /** + * Join array of strings by prefixing each one with sep. + */ + private static _prefixJoin(sep: string, array: string[]): string { + return array.length ? sep + array.join(sep) : ''; + } + + /** + * Internal helper to make a tmp table given a tableId + * + * @param {String} tableId + * @returns {String} + */ + private static _makeTmpTableId(tableId: string): string { + return '_tmp_' + tableId; + } + + private static _sqlColSpecFromDBInfo(info: ResultRow): string { + return `${quoteIdent(info.name)} ${info.type} DEFAULT ${info.dflt_value}`; + } + + /** + * Converts an array of columns to an array of rows (suitable to use as sqlParams), encoding all + * values as needed, according to an array of Grist type strings (must be parallel to columns). + */ + private static _encodeColumnsToRows(types: string[], valueColumns: any[]): any[] { + const marshaller = new marshal.Marshaller({version: 2}); + const rows = _.unzip(valueColumns); + for (const row of rows) { + for (let i = 0; i < row.length; i++) { + row[i] = DocStorage._encodeValue(marshaller, types[i], row[i]); + } + } + return rows; + } + + /** + * Encodes a single value for storing in SQLite. Numbers and text are stored as is, but complex + * types are marshalled and stored as BLOBs. We also marshal binary data, so that for encoded + * data, all BLOBs consistently contain marshalled data. + * + * Note that SQLite may contain tables that aren't used for Grist data (e.g. attachments), for + * which such encoding/marshalling is not used, and e.g. binary data is stored to BLOBs directly. + */ + private static _encodeValue(marshaller: marshal.Marshaller, type: string, val: any): Uint8Array { + if (Array.isArray(val) || val instanceof Uint8Array || Buffer.isBuffer(val) || + Number.isNaN(val) || + (type === 'Bool' && (val === 0 || val === 1)) || + (type !== 'Bool' && (val === true || val === false))) { + marshaller.marshal(val); + return marshaller.dump(); + } else { + return val; + } + } + + /** + * Decodes Grist data received from SQLite; the inverse of _encodeValue(). + */ + private static _decodeValue(val: any, type: string = 'Any'): any { + if (val instanceof Uint8Array || Buffer.isBuffer(val)) { + val = marshal.loads(val); + } + if (type === 'Bool' && (val === 0 || val === 1)) { + // Boolean values come in as 0/1. If the column is of type "Bool", interpret those as + // true/false (note that the data engine does this too). + return Boolean(val); + } else { + return val; + } + } + + /** + * Helper to return SQL snippet for column definition, using its colId and Grist type. + */ + private static _columnDef(colId: string, colType: string): string { + return `${quoteIdent(colId)} BLOB DEFAULT ${DocStorage._formattedDefault(colType)}`; + } + + + // ====================================================================== + // Instance fields + // ====================================================================== + + public docPath: string; // path to document file on disk + private _db: SQLiteDB|null; // database handle + + // Maintains { tableId: { colId: gristType } } mapping for all tables, including grist metadata + // tables (obtained from auto-generated schema.js). + private _docSchema: {[tableId: string]: {[colId: string]: string}}; + + public constructor(public storageManager: IDocStorageManager, public docName: string) { + this.docPath = this.storageManager.getPath(docName); + this._db = null; + this._docSchema = Object.assign({}, schema.schema); + } + + /** + * Opens an existing SQLite database and prepares it for use. + */ + public openFile(): Promise { + // It turns out to be important to return a bluebird promise, a lot of code outside + // of DocStorage ultimately depends on this. + return bluebird.Promise.resolve(this._openFile(OpenMode.OPEN_EXISTING)) + .then(() => this._initDB()) + .then(() => this._updateMetadata()); + } + + /** + * Creates a new SQLite database. Will throw an error if the database already exists. + * After a database is created it should be initialized by applying the InitNewDoc action. + */ + public createFile(): Promise { + // It turns out to be important to return a bluebird promise, a lot of code outside + // of DocStorage ultimately depends on this. + return bluebird.Promise.resolve(this._openFile(OpenMode.CREATE_EXCL)) + .then(() => this._initDB()); + // Note that we don't call _updateMetadata() as there are no metadata tables yet anyway. + } + + /** + * Creates a backup and calls cb() within a transaction. Returns the backup path. In case of + * failure, adds .backupPath property to the error object (and transaction is rolled back). + */ + public execWithBackup(cb: (db: SQLiteDB) => Promise): Promise { + let backupPath: string; + return this.storageManager.makeBackup(this.docName, "migrate-db") + .then((_backupPath: string) => { + backupPath = _backupPath; + log.info(`DocStorage[${this.docName}]: backup made at ${backupPath}`); + return this.execTransaction(cb); + }) + .then(() => backupPath) + .catch((err: any) => { + // TODO: deal with typing for this kind of error (although nothing seems to depend + // on it yet. + err.backupPath = backupPath; + throw err; + }); + } + + /** + * Initializes the database with proper settings. + */ + public _initDB(): Promise { + // Set options for speed across multiple OSes/Filesystems. + // WAL is fast and safe (guarantees consistency accross crashes), but has disadvantages + // including generating unwanted extra files that can be tricky to deal with in renaming, etc + // the options for WAL are commented out + // Setting synchronous to OFF is the fastest method, but is not as safe, and could lead to + // a database being corrupted if the computer it is running on crashes. + // TODO: Switch setting to FULL, but don't wait for SQLite transactions to finish before + // returning responses to the user. Instead send error messages on unexpected errors. + return this._getDB().exec( + // "PRAGMA wal_autochceckpoint = 1000;" + + // "PRAGMA page_size = 4096;" + + // "PRAGMA journal_size_limit = 0;" + + // "PRAGMA journal_mode = WAL;" + + // "PRAGMA auto_vacuum = 0;" + + // "PRAGMA synchronous = NORMAL" + "PRAGMA synchronous = OFF;" + ); + } + + /** + * Queries the database for Grist metadata and updates this._docSchema. It extends the auto- + * generated mapping in app/common/schema.js, to all tables, as `{tableId: {colId: gristType}}`. + */ + public _updateMetadata(): Promise { + return this.all('SELECT t.tableId, c.colId, c.type ' + + 'FROM _grist_Tables_column c JOIN _grist_Tables t ON c.parentId=t.id') + .then((rows: ResultRow[]) => { + const s: {[key: string]: any} = {}; + for (const {tableId, colId, type} of rows) { + const table = s.hasOwnProperty(tableId) ? s[tableId] : (s[tableId] = {}); + table[colId] = type; + } + // Note that schema is what's imported from app/common/schema.js + this._docSchema = Object.assign(s, schema.schema); + }) + .catch(err => { + // This replicates previous logic for _updateMetadata. + if (err.message.startsWith('SQLITE_ERROR: no such table')) { + err.message = `NO_METADATA_ERROR: ${this.docName} has no metadata`; + err.cause.code = 'NO_METADATA_ERROR'; + } + throw err; + }); + } + + /** + * Closes the SQLite database. + */ + public async shutdown(): Promise { + if (!this._db) { + log.debug('DocStorage shutdown (trivial) success'); + return; + } + const db = this._getDB(); + this._db = null; + await db.close(); + log.debug('DocStorage shutdown success'); + } + + + /** + * Attaches the file to the document. + * + * TODO: This currently does not make the attachment available to the sandbox code. This is likely + * to be needed in the future, and a suitable API will need to be provided. Note that large blobs + * would be (very?) inefficient until node-sqlite3 adds support for incremental reading from a + * blob: https://github.com/mapbox/node-sqlite3/issues/424. + * + * @param {String} sourcePath: The path of the file containing the attachment data. + * @param {String} fileIdent: The unique identifier of the file in the database. ActiveDoc uses the + * checksum of the file's contents with the original extension. + * @returns {Promise[Boolean]} True if the file got attached; false if this ident already exists. + */ + public findOrAttachFile(sourcePath: string, fileIdent: string): Promise { + return this.execTransaction(db => { + // Try to insert a new record with the given ident. It'll fail UNIQUE constraint if exists. + return db.run('INSERT INTO _gristsys_Files (ident) VALUES (?)', fileIdent) + // Only if this succeeded, do the work of reading the file and inserting its data. + .then(() => fse.readFile(sourcePath)) + .then(data => + db.run('UPDATE _gristsys_Files SET data=? WHERE ident=?', data, fileIdent)) + .then(() => true) + // If UNIQUE constraint failed, this ident must already exists, so return false. + .catch(err => { + if (/^SQLITE_CONSTRAINT: UNIQUE constraint failed/.test(err.message)) { + return false; + } + throw err; + }); + }); + } + + /** + * Reads and returns the data for the given attachment. + * @param {String} fileIdent: The unique identifier of a file, as used by findOrAttachFile. + * @returns {Promise[Buffer]} The data buffer associated with fileIdent. + */ + public getFileData(fileIdent: string): Promise { + return this.get('SELECT data FROM _gristsys_Files WHERE ident=?', fileIdent) + .then(row => row && row.data); + } + + + /** + * Fetches the given table from the database. See fetchQuery() for return value. + */ + public fetchTable(tableId: string): Promise { + return this.fetchQuery({tableId, filters: {}}); + } + + /** + * Returns as a number the next row id for the given table. + */ + public async getNextRowId(tableId: string): Promise { + const colData = await this.get(`SELECT MAX(id) as maxId FROM ${quoteIdent(tableId)}`); + if (!colData) { + throw new Error(`Error in DocStorage.getNextRowId: no table ${tableId}`); + } + return colData.maxId ? colData.maxId + 1 : 1; + } + + /** + * Fetches all rows of the table with the given rowIds. + */ + public async fetchActionData(tableId: string, rowIds: number[], colIds?: string[]): Promise { + const colSpec = colIds ? ['id', ...colIds].map((c) => quoteIdent(c)).join(', ') : '*'; + let fullValues: TableColValues|undefined; + + // There is a limit to the number of arguments that may be passed in, so fetch data in chunks. + for (const rowIdChunk of chunk(rowIds, maxSQLiteVariables)) { + const sqlArg = rowIdChunk.map(() => '?').join(','); + const marshalled: Buffer = await this._getDB().allMarshal( + `SELECT ${colSpec} FROM ${quoteIdent(tableId)} WHERE id IN (${sqlArg})`, rowIdChunk); + + const colValues: TableColValues = marshal.loads(marshalled); + if (!fullValues) { + fullValues = colValues; + } else { + for (const col of Object.keys(colValues)) { + fullValues[col].push(...colValues[col]); + } + } + } + return toTableDataAction(tableId, fullValues || {id: []}); // Return empty TableColValues if rowIds was empty. + } + + /** + * Fetches a subset of the data specified by the given query, and returns an encoded TableData + * object, which is a marshalled dict mapping column ids (including 'id') to arrays of values. + * + * This now essentially subsumes the old fetchTable() method. + * Note that text is marshalled as unicode and blobs as binary strings (used to be binary strings + * for both before 2017-11-09). This allows blobs to be used exclusively for encoding types that + * are not easily stored as sqlite's native types. + */ + public async fetchQuery(query: ExpandedQuery): Promise { + // Convert query to SQL. + const params: any[] = []; + const whereParts: string[] = []; + for (const colId of Object.keys(query.filters)) { + const values = query.filters[colId]; + // If values is empty, "IN ()" works in SQLite (always false), but wouldn't work in Postgres. + whereParts.push(`${quoteIdent(query.tableId)}.${quoteIdent(colId)} IN (${values.map(() => '?').join(', ')})`); + params.push(...values); + } + const whereClause = whereParts.length > 0 ? `WHERE ${whereParts.join(' AND ')}` : ''; + const limitClause = (typeof query.limit === 'number') ? `LIMIT ${query.limit}` : ''; + const joinClauses = query.joins ? query.joins.join(' ') : ''; + const selects = query.selects ? query.selects.join(', ') : '*'; + const sql = `SELECT ${selects} FROM ${quoteIdent(query.tableId)} ` + + `${joinClauses} ${whereClause} ${limitClause}`; + return this._getDB().allMarshal(sql, params); + } + + /** + * Fetches all tables from the database, and returns a dictionary mapping table names to encoded + * TableData objects (marshalled dicts mapping column ids to arrays of values). + */ + public fetchAllTables(): Promise<{[key: string]: Buffer|null}> { + const tables: {[key: string]: Buffer|null} = {}; + return bluebird.Promise.each( + this.all("SELECT name FROM sqlite_master WHERE type='table'"), + (row: ResultRow) => { + if (!row.name.startsWith('_gristsys_')) { + return this.fetchTable(row.name) + .then(data => { + tables[row.name] = data; + }); + } + }) + .return(tables); + } + + /** + * Unmarshals and decodes data received from db.allMarshal() method (which we added to node-sqlite3). + * The data is a dictionary mapping column ids (including 'id') to arrays of values. This should + * be used for Grist data, which is encoded. For non-Grist data, use `marshal.loads()`. + * + * Note that we do NOT use this when loading data from a document, since the whole point of + * db.allMarshal() is to pass data directly to Python data engine without parsing in Node. + */ + public decodeMarshalledData(marshalledData: Buffer | Uint8Array, tableId: string): TableColValues { + const columnValues: TableColValues = marshal.loads(marshalledData); + // Decode in-place to avoid unnecessary array creation. + for (const col of Object.keys(columnValues)) { + const type = this._getGristType(tableId, col); + const column = columnValues[col]; + for (let i = 0; i < column.length; i++) { + column[i] = DocStorage._decodeValue(column[i], type); + } + } + return columnValues; + } + + /** + * Applies stored actions received from data engine to the database by converting them to SQL + * statements and executing a serialized transaction. + * @param {Array[DocAction]} docActions - Array of doc actions from DataEngine. + * @returns {Promise} - An empty promise, resolved if successfully committed to db. + */ + public async applyStoredActions(docActions: DocAction[]): Promise { + debuglog('DocStorage.applyStoredActions'); + + return bluebird.Promise.each(docActions, (action: DocAction) => { + const actionType = action[0]; + const f = (this as any)["_process_" + actionType]; + if (!_.isFunction(f)) { + log.error("Unknown action: " + actionType); + } else { + return f.apply(this, action.slice(1)) + .then(() => { + const tableId = action[1]; // The first argument is always tableId; + if (DocStorage._isMetadataTable(tableId) && actionType !== 'AddTable') { + // We only need to update the metadata for actions that change + // the metadata. We don't update on AddTable actions + // because the additional of a table gives no additional data + // and if we tried to update when only _grist_Tables was added + // without _grist_Tables_column, we would get an error + return this._updateMetadata(); + } + }); + } + }); + } + + /** + * Internal helper to process AddTable action. + * + * @param {String} tableId - Table ID. + * @param {Array[Object]} columns - List of column objects with schema attributes. + * @returns {Promise} - A promise for the SQL execution. + */ + public _process_AddTable(tableId: string, columns: any[]): Promise { + const colSpecSql = + DocStorage._prefixJoin(', ', + columns.filter(c => + !c.isFormula).map(c => + DocStorage._columnDef(c.id, c.type))); + + // Every table needs an "id" column, and it should be an "integer primary key" type so that it + // serves as the alias for the SQLite built-in "rowid" column. See + // https://www.sqlite.org/lang_createtable.html#rowid for details. + const sql = `CREATE TABLE ${quoteIdent(tableId)} (id INTEGER PRIMARY KEY${colSpecSql})`; + log.debug('AddTable SQL : ' + sql); + + return this.exec(sql); + } + + /** + * Internal helper to process UpdateRecord action. + * + * @param {String} tableId - Table Id. + * @param {String} rowId - Row Id. + * @param {Object} columnValues - Column object with keys as column names. + * @returns {Promise} - A promise for the SQL execution. + */ + public _process_UpdateRecord(tableId: string, rowId: string, columnValues: any): Promise { + // Do some small preprocessing to make this look like a BulkUpdateRecord + return this._process_BulkUpdateRecord(tableId, [rowId], _.mapObject(columnValues, (val: any) => [val])); + } + + /** + * Internal helper to process AddRecord action. + * + * @param {String} tableId - Table ID. + * @param {Integer} rowId - Row ID. + * @param {Object} columnValues - Column object with keys as column names. + * @returns {Promise} - A promise for the SQL execution. + */ + public _process_AddRecord(tableId: string, rowId: number, columnValues: any): Promise { + // Do some small preprocessing to make this look like a BulkAddRecord + return this._process_BulkAddRecord(tableId, [rowId], _.mapObject(columnValues, (val: any) => [val])); + } + + /** + * Internal helper to process BulkUpdateRecord action. + * + * @param {String} tableId - Table Id. + * @param {Array[String]} rowIds - List of Row Ids. + * @param {Object} columnValues - Column object with keys as column names and arrays of values. + * @returns {Promise} - Promise for SQL execution. + */ + public _process_BulkUpdateRecord(tableId: string, rowIds: string[], columnValues: any): Promise { + const cols = Object.keys(columnValues); + if (!rowIds.length || !cols.length) { return Promise.resolve(); } // Nothing to do. + + const colListSql = cols.map(c => quoteIdent(c) + '=?').join(', '); + const sql = `UPDATE ${quoteIdent(tableId)} SET ${colListSql} WHERE id=?`; + + const types = cols.map(c => this._getGristType(tableId, c)); + const sqlParams = DocStorage._encodeColumnsToRows(types, cols.map(c => columnValues[c]).concat([rowIds])); + + debuglog("DocStorage._maybeBulkUpdateRecord SQL: %s (%s rows)", sql, sqlParams.length); + return this._applyMaybeBulkUpdateOrAddSql(sql, sqlParams); + } + + /** + * Internal helper to process BulkAddRecord action. + * + * @param {String} tableId - Table ID. + * @param {Array[Integer]} rowIds - Array of row IDs to be inserted. + * @param {Array[Object]} columnValues - Array of column info objects. + * @returns {Promise} - Promise for SQL execution. + */ + public _process_BulkAddRecord(tableId: string, rowIds: number[], columnValues: {[key: string]: any}): Promise { + if (rowIds.length === 0) { return Promise.resolve(); } // no rows means nothing to do + + const cols = Object.keys(columnValues); + const colListSql = cols.map(c => quoteIdent(c) + ', ').join(''); + const placeholders = cols.map(c => '?, ').join(''); + const sql = `INSERT INTO ${quoteIdent(tableId)} (${colListSql}id) VALUES (${placeholders}?)`; + + const types = cols.map(c => this._getGristType(tableId, c)); + const sqlParams = + DocStorage._encodeColumnsToRows(types, + cols.map(c => columnValues[c]).concat([rowIds])); + + debuglog("DocStorage._maybeBulkAddRecord SQL: %s (%s rows)", sql, sqlParams.length); + return this._applyMaybeBulkUpdateOrAddSql(sql, sqlParams); + } + + /** + * Internal helper to process RemoveRecord action. + * + * @param {String} tableId - Table ID. + * @param {String} rowId - Row ID. + * @returns {Promise} - A promise for the SQL execution. + */ + public _process_RemoveRecord(tableId: string, rowId: string): Promise { + const sql = "DELETE FROM " + quoteIdent(tableId) + " WHERE id=?"; + debuglog("RemoveRecord SQL: " + sql, [rowId]); + return this.run(sql, [rowId]); + } + + + /** + * Internal helper to process ReplaceTableData action. It is identical to BulkAddRecord, but + * deletes all data from the table first. + */ + public _process_ReplaceTableData(tableId: string, rowIds: number[], columnValues: any[]): Promise { + return this.exec("DELETE FROM " + quoteIdent(tableId)) + .then(() => this._process_BulkAddRecord(tableId, rowIds, columnValues)); + } + + /** + * Internal helper to process BulkRemoveRecord action. + * + * @param {String} tableId - Table ID. + * @param {Array[Integer]} rowIds - Array of row IDs to be deleted. + * @returns {Promise} - Promise for SQL execution. + */ + public _process_BulkRemoveRecord(tableId: string, rowIds: number[]): Promise { + if (rowIds.length === 0) { return Promise.resolve(); }// If we have nothing to remove, done. + + const chunkSize = 10; + const preSql = "DELETE FROM " + quoteIdent(tableId) + " WHERE id IN ("; + const postSql = ")"; + const q = _.constant('?'); + const chunkParams = _.range(chunkSize).map(q).join(','); + const numChunks = Math.floor(rowIds.length / chunkSize); + const numLeftovers = rowIds.length % chunkSize; + + let chunkPromise; + + if (numChunks > 0) { + debuglog("DocStorage.BulkRemoveRecord: splitting " + rowIds.length + + " deletes into chunks of size " + chunkSize); + chunkPromise = this.prepare(preSql + chunkParams + postSql) + .then(function(stmt) { + return bluebird.Promise.each(_.range(0, numChunks * chunkSize, chunkSize), function(index: number) { + debuglog("DocStorage.BulkRemoveRecord: chunk delete " + index + "-" + (index + chunkSize - 1)); + return bluebird.Promise.fromCallback((cb: any) => stmt.run(rowIds.slice(index, index + chunkSize), cb)); + }) + .then(function() { + return bluebird.Promise.fromCallback((cb: any) => stmt.finalize(cb)); + }); + }); + } else { + chunkPromise = Promise.resolve(); + } + + return chunkPromise.then(() => { + if (numLeftovers > 0) { + debuglog("DocStorage.BulkRemoveRecord: leftover delete " + (numChunks * chunkSize) + "-" + (rowIds.length - 1)); + const leftoverParams = _.range(numLeftovers).map(q).join(','); + return this.run(preSql + leftoverParams + postSql, + rowIds.slice(numChunks * chunkSize, rowIds.length)); + } + }); + } + + /** + * Internal helper to process AddColumn action. + * + * @param {String} tableId - Table Id. + * @param {String} colId - Column Id. + * @param {Object} colInfo - Column info object. + * @returns {Promise} - A promise for the SQL execution. + */ + public async _process_AddColumn(tableId: string, colId: string, colInfo: any): Promise { + // No need to ALTER TABLE for formula columns + if (colInfo.isFormula) { + return; + } + await this.exec( + `ALTER TABLE ${quoteIdent(tableId)} ADD COLUMN ${DocStorage._columnDef(colId, colInfo.type)}`); + } + + /** + * Internal helper to process RenameColumn action. + * + * @param {String} tableId - Table ID. + * @param {String} fromColId - Column ID to rename. + * @param {String} toColId - New Column ID. + * @returns {Promise} - A promise for the SQL execution. + */ + public async _process_RenameColumn(tableId: string, fromColId: string, toColId: string): Promise { + if (fromColId === 'id' || fromColId === 'manualSort' || tableId.startsWith('_grist')) { + throw new Error('Cannot rename internal Grist column'); + } + try { + await this.exec( + `ALTER TABLE ${quoteIdent(tableId)} RENAME COLUMN ${quoteIdent(fromColId)} TO ${quoteIdent(toColId)}`); + } catch (error) { + if (!String(error).match(/SQLITE_ERROR: no such column/)) { throw error; } + // Accept no-such-column, because this may be a formula column. + // TODO: tighten constraint by getting access to grist schema info and isFormula flag. + } + } + + /** + * Internal helper to process ModifyColumn action. + * + * Note that this requires access to the _grist_ tables, unlike many of the other actions. + * + * @param {String} tableId - Table ID. + * @param {String} colId - Column ID. + * @param {Object} colInfo - Column info object. + * @returns {Promise} - A promise for the SQL execution. + */ + public async _process_ModifyColumn(tableId: string, colId: string, colInfo: any): Promise { + if (!colInfo) { + log.error("ModifyColumn action called without params."); + return; + } + + if (colInfo.isFormula) { + return this._process_RemoveColumn(tableId, colId); + } else { + const colExists = await this._colExistsInDB(tableId, colId); + const toNonFormula = colInfo.hasOwnProperty('isFormula') && !colInfo.isFormula; + if (!colExists && toNonFormula) { + // It's important to have the original type here, when colInfo does not include it. + if (!colInfo.type) { colInfo.type = this._getGristType(tableId, colId); } + return this._process_AddColumn(tableId, colId, colInfo); + } else { + return this._alterColumn(tableId, colId, colId, colInfo.type); + } + } + } + + + /** + * Internal helper to process RemoveColumn action. + * + * @param {String} tableId - Table ID. + * @param {String} colId - Column ID to rename. + * @returns {Promise} - A promise for the SQL execution. + */ + public _process_RemoveColumn(tableId: string, colId: string): Promise { + const quote = quoteIdent; + const tmpTableId = DocStorage._makeTmpTableId(tableId); + + // Note that SQLite does not support easily dropping columns. To drop a column from a table, we + // need to follow the instructions at https://sqlite.org/lang_altertable.html Since we don't use + // indexes or triggers, we skip a few steps. + + // This returns rows with (at least) {name, type, dflt_value}. + return this.all(`PRAGMA table_info(${quote(tableId)})`) + .then(infoRows => { + const newInfoRows = infoRows.filter(row => (row.name !== colId && row.name !== 'id')); + if (newInfoRows.length === infoRows.length) { + // Column was not found. That's ok, and happens when deleting formula column. + return; + } + const colListSql = DocStorage._prefixJoin(', ', newInfoRows.map(info => quote(info.name))); + const colSpecSql = DocStorage._prefixJoin(', ', newInfoRows.map(DocStorage._sqlColSpecFromDBInfo)); + return this._getDB().runEach( + `CREATE TABLE ${quote(tmpTableId)} (id INTEGER PRIMARY KEY${colSpecSql})`, + `INSERT INTO ${quote(tmpTableId)} SELECT id${colListSql} FROM ${quote(tableId)}`, + `DROP TABLE ${quote(tableId)}`, + `ALTER TABLE ${quote(tmpTableId)} RENAME TO ${quote(tableId)}` + ); + }); + } + + + /** + * Internal helper to process RenameTable action. + * + * @param {string} fromTableId - Old table id + * @param {string} toTableId - New table id + * @returns {Promise} - A promise for the SQL execution. + */ + public _process_RenameTable(fromTableId: string, toTableId: string): Promise { + const sql: string[] = []; + + if (fromTableId === toTableId) { + return Promise.resolve(); + } else if (fromTableId.toLowerCase() === toTableId.toLowerCase()) { + const tmpTableId = DocStorage._makeTmpTableId(fromTableId); + sql.push("ALTER TABLE " + quoteIdent(fromTableId) + + " RENAME TO " + quoteIdent(tmpTableId)); + fromTableId = tmpTableId; + } + + sql.push("ALTER TABLE " + quoteIdent(fromTableId) + + " RENAME TO " + quoteIdent(toTableId)); + + log.debug("RenameTable SQL: " + sql); + return bluebird.Promise.each(sql, (stmt: string) => this.exec(stmt)); + } + + /** + * Internal helper to process RemoveTable action. + * + * @param {String} tableId - Table ID. + * @returns {Promise} - A promise for the SQL execution. + */ + public _process_RemoveTable(tableId: string): Promise { + const sql = "DROP TABLE " + quoteIdent(tableId); + + log.debug("RemoveTable SQL: " + sql); + + return this.exec(sql); + } + + public renameDocTo(newName: string): Promise { + log.debug('DocStorage.renameDocTo: %s -> %s', this.docName, newName); + return this.shutdown() + .then(() => this.storageManager.renameDoc(this.docName, newName)) + .catch(err => { + log.error("DocStorage: renameDocTo %s -> %s failed: %s", this.docName, newName, err.message); + return this.openFile() + .then(function() { + throw err; + }); + }) + .then(() => { + this.docName = newName; + this.docPath = this.storageManager.getPath(newName); + return this.openFile(); + }); + } + + public all(sql: string, ...args: any[]): Promise { + return this._getDB().all(sql, ...args); + } + + public run(sql: string, ...args: any[]): Promise { + return this._markAsChanged(this._getDB().run(sql, ...args)); + } + + public exec(sql: string): Promise { + return this._markAsChanged(this._getDB().exec(sql)); + } + + public prepare(sql: string, ...args: any[]): Promise { + return this._getDB().prepare(sql, ...args); + } + + public get(sql: string, ...args: any[]): Promise { + return this._getDB().get(sql, ...args); + } + + public execTransaction(transx: (db1: SQLiteDB) => Promise): Promise { + const db = this._getDB(); + return this._markAsChanged(db.execTransaction(() => transx(db))); + } + + public runAndGetId(sql: string, ...params: any[]): Promise { + const db = this._getDB(); + return this._markAsChanged(db.runAndGetId(sql, ...params)); + } + + public requestVacuum(): Promise { + const db = this._getDB(); + return this._markAsChanged(db.requestVacuum()); + } + + public async getPluginDataItem(pluginId: string, key: string): Promise { + const row = await this.get('SELECT value from _gristsys_PluginData WHERE pluginId = ? and key = ?', pluginId, key); + if (row) { + return row.value; + } + return undefined; + } + + public async hasPluginDataItem(pluginId: string, key: string): Promise { + const row = await this.get('SELECT value from _gristsys_PluginData WHERE pluginId=? and key=?', pluginId, key); + return typeof row !== 'undefined'; + } + + public setPluginDataItem(pluginId: string, key: string, value: string): Promise { + return this.run('INSERT OR REPLACE into _gristsys_PluginData (pluginId, key, value) values (?, ?, ?)', + pluginId, key, value); + } + + public removePluginDataItem(pluginId: string, key: string): Promise { + return this.run('DELETE from _gristsys_PluginData where pluginId = ? and key = ?', pluginId, key); + } + + public clearPluginDataItem(pluginId: string): Promise { + return this.run('DELETE from _gristsys_PluginData where pluginId = ?', pluginId); + } + + /** + * Get a list of indexes. For use in tests. + */ + public async testGetIndexes(): Promise { + return this._getIndexes(); + } + + /** + * Create the specified indexes if they don't already exist. Remove indexes we + * created in the past that are not listed (leaving other indexes untouched). + */ + public async updateIndexes(desiredIndexes: IndexColumns[]) { + // Find all indexes on user tables. + const indexes = await this._getIndexes(); + // Keep track of indexes prior to calling this method and after the call to this method + // as two sets of "tableId.colId" strings. + const pre = new Set(indexes.map(index => `${index.tableId}.${index.colId}`)); + const post = new Set(); + for (const index of desiredIndexes) { + const idx = `${index.tableId}.${index.colId}`; + if (!pre.has(idx)) { + const name = `auto_index_${uuidv4().replace(/-/g, '_')}`; + log.debug(`DocStorage.updateIndexes: doc ${this.docName} adding index ${name} for ` + + `table ${index.tableId}, column ${index.colId}`); + await this.exec(`CREATE INDEX ${name} ON ${quoteIdent(index.tableId)}(${quoteIdent(index.colId)})`); + log.debug(`DocStorage.updateIndexes: doc ${this.docName} added index ${name} for ` + + `table ${index.tableId}, column ${index.colId}`); + } + post.add(idx); + } + for (const index of indexes) { + const idx = `${index.tableId}.${index.colId}`; + if (!post.has(idx) && index.indexId.startsWith('auto_index_')) { + log.debug(`DocStorage.updateIndexes: doc ${this.docName} dropping index ${index.indexId} for ` + + `table ${index.tableId}, column ${index.colId}`); + await this.exec(`DROP INDEX ${index.indexId}`); + log.debug(`DocStorage.updateIndexes: doc ${this.docName} dropped index ${index.indexId} for ` + + `table ${index.tableId}, column ${index.colId}`); + } + } + } + + private async _markAsChanged(promise: Promise): Promise { + try { + return await promise; + } finally { + this.storageManager.markAsChanged(this.docName); + } + } + + /** + * Creates a new or opens an existing SQLite database, depending on mode. + * @return {Promise} Promise for user_version stored in the database. + */ + private async _openFile(mode: number): Promise { + try { + this._db = await SQLiteDB.openDB(this.docPath, DocStorage.docStorageSchema, mode); + log.debug("DB %s open successfully", this.docName); + return this._db.getMigrationVersion(); + } catch (err) { + log.debug("DB %s open error: %s", this.docName, err); + throw err; + } + } + + /** + * Internal helper for applying Bulk Update or Add Record sql + */ + private _applyMaybeBulkUpdateOrAddSql(sql: string, sqlParams: any[]): Promise { + if (sqlParams.length === 1) { + return this.run(sql, sqlParams[0]); + } else { + return this.prepare(sql) + .then(function(stmt) { + return bluebird.Promise.each(sqlParams, function(param: string) { + return bluebird.Promise.fromCallback((cb: any) => stmt.run(param, cb)); + }) + .then(function() { + return bluebird.Promise.fromCallback((cb: any) => stmt.finalize(cb)); + }); + }); + } + } + + /** + * Read SQLite's metadata for tableId, and generate SQL for the altered version of the table. + * @param {string} colId: Existing colId to change or delete. We'll return null if it's missing. + * @param {string} newColId: New colId. + * @param {string|null} newColType: New grist type, or null to keep unchanged. + * @return {Promise} New table SQL, or null when nothing changed or colId is missing. + */ + private async _rebuildTableSql(tableId: string, colId: string, newColId: string, + newColType: string|null = null): Promise { + // This returns rows with (at least) {name, type, dflt_value}. + assert(newColId, 'newColId required'); + let infoRows = await this.all(`PRAGMA table_info(${quoteIdent(tableId)})`); + + // Skip "id" column, and find the column we are modifying. + infoRows = infoRows.filter(row => (row.name !== 'id')); + const colInfo = infoRows.find(info => (info.name === colId)); + if (!colInfo) { + return null; // Column not found. + } + const oldDefault = colInfo.dflt_value; + const newDefault = newColType ? DocStorage._formattedDefault(newColType) : oldDefault; + const newInfo = {name: newColId, type: 'BLOB', dflt_value: newDefault}; + // Check if anything actually changed, and only rebuild the table then. + if (Object.keys(newInfo).every(p => ((newInfo as any)[p] === colInfo[p]))) { + return null; // No changes. + } + Object.assign(colInfo, newInfo); + const colSpecSql = DocStorage._prefixJoin(', ', infoRows.map(DocStorage._sqlColSpecFromDBInfo)); + return { + sql: `CREATE TABLE ${quoteIdent(tableId)} (id INTEGER PRIMARY KEY${colSpecSql})`, + oldDefault, + newDefault, + }; + } + + /** + * Helper to alter a table to new table SQL, which is appropriate for renaming columns, or + * changing default values for a column, i.e. changes that don't affect on-disk content in any + * way. See https://sqlite.org/lang_altertable.html. + */ + private async _alterTableSoft(tableId: string, newTableSql: string): Promise { + // Procedure according to https://sqlite.org/lang_altertable.html: "appropriate for ... renaming + // columns, or adding or removing or changing default values on a column." + const row = await this.get("PRAGMA schema_version"); + assert(row && row.schema_version, "Could not retrieve schema_version."); + const newSchemaVersion = row!.schema_version + 1; + const tmpTableId = DocStorage._makeTmpTableId(tableId); + await this._getDB().runEach( + "PRAGMA writable_schema=ON", + ["UPDATE sqlite_master SET sql=? WHERE type='table' and name=?", [newTableSql, tableId]], + `PRAGMA schema_version=${newSchemaVersion}`, + "PRAGMA writable_schema=OFF", + // The following are not in the instructions, but are needed for SQLite to notice the + // changes for subsequent queries. + `ALTER TABLE ${quoteIdent(tableId)} RENAME TO ${quoteIdent(tmpTableId)}`, + `ALTER TABLE ${quoteIdent(tmpTableId)} RENAME TO ${quoteIdent(tableId)}` + ); + } + + private async _alterColumn(tableId: string, colId: string, newColId: string, + newColType: string|null = null): Promise { + const result = await this._rebuildTableSql(tableId, colId, newColId, newColType); + if (result) { + if (result.oldDefault !== result.newDefault) { + // This isn't strictly necessary, but addresses a SQLite quirk that breaks our tests + // (although likely unnoticeable in practice): an added column has "holes" for existing + // records that show up as the default value but don't actually store that default. When + // we do the soft-alter here, those values reflect the new default, i.e. change + // unexpectedly. Setting the default values explicitly prevents this unexpected change. + const dflt = result.oldDefault; + const q = quoteIdent; + // (Note that comparison below must use "IS" rather than "=" to work for NULLs.) + await this.exec(`UPDATE ${q(tableId)} SET ${q(colId)}=${dflt} WHERE ${q(colId)} IS ${dflt}`); + } + await this._alterTableSoft(tableId, result.sql); + } + } + + /** + * Returns a promise for a boolean for whether the given column exists in the database. + */ + private _colExistsInDB(tableId: string, colId: string): Promise { + return this.all(`PRAGMA table_info(${quoteIdent(tableId)})`) + .then(infoRows => infoRows.some(row => (row.name === colId))); + } + + private _getGristType(tableId: string, colId: string): string { + return (this._docSchema[tableId] && this._docSchema[tableId][colId]) || 'Any'; + } + + private _getDB(): SQLiteDB { + if (!this._db) { + throw new Error("Tried to use DocStorage database before it was opened"); + } + return this._db; + } + + /** + * Get a list of user indexes + */ + private async _getIndexes(): Promise { + // Find all indexes on user tables. + return await this.all("SELECT tbl_name as tableId, il.name as indexId, ii.name as colId " + + "FROM sqlite_master AS m, " + + "pragma_index_list(m.name) AS il, " + + "pragma_index_info(il.name) AS ii " + + "WHERE m.type='table' " + + "AND tbl_name NOT LIKE '_grist%' " + + "ORDER BY tableId, colId") as any; + } +} + +interface RebuildResult { + sql: string; + oldDefault: string; + newDefault: string; +} + +// A summary of columns a database index is covering or should cover. +export interface IndexColumns { + tableId: string; // name of table + colId: string; // column indexed (only single-column indexes supported for now) +} + +// A summary of a database index, including its name. +export interface IndexInfo extends IndexColumns { + indexId: string; // name of index +} diff --git a/app/server/lib/DocStorageManager.ts b/app/server/lib/DocStorageManager.ts new file mode 100644 index 00000000..5f5dbd60 --- /dev/null +++ b/app/server/lib/DocStorageManager.ts @@ -0,0 +1,352 @@ +import * as bluebird from 'bluebird'; +import * as chokidar from 'chokidar'; +import * as fse from 'fs-extra'; +import * as moment from 'moment'; +import * as path from 'path'; + +import {DocEntry, DocEntryTag} from 'app/common/DocListAPI'; +import * as gutil from 'app/common/gutil'; +import * as Comm from 'app/server/lib/Comm'; +import {OptDocSession} from 'app/server/lib/DocSession'; +import {DocSnapshots} from 'app/server/lib/DocSnapshots'; +import * as docUtils from 'app/server/lib/docUtils'; +import {GristServer} from 'app/server/lib/GristServer'; +import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; +import {IShell} from 'app/server/lib/IShell'; +import * as log from 'app/server/lib/log'; +import * as uuidv4 from "uuid/v4"; + + +/** + * DocStorageManager manages Grist documents. This implementation deals with files in the file + * system. An alternative implementation could provide the same public methods to implement + * storage management for the hosted version of Grist. + * + * This file-based DocStorageManager uses file path as the docName identifying a document, with + * one exception. For files in the docsRoot directory, the basename of the document is used + * instead, with .grist extension stripped; primarily to maintain previous behavior and keep + * clean-looking URLs. In all other cases, the realpath of the file (including .grist extension) + * is the canonical docName. + * + */ +export class DocStorageManager implements IDocStorageManager { + private _watcher: any; // chokidar filesystem watcher + private _shell: IShell; + + /** + * Initialize with the given root directory, which should be a fully-resolved path (i.e. using + * fs.realpath or docUtils.realPath). + * The file watcher is created if the optComm argument is given. + */ + constructor(private _docsRoot: string, private _samplesRoot?: string, + private _comm?: Comm, gristServer?: GristServer) { + // If we have a way to communicate with clients, watch the docsRoot for changes. + this._watcher = null; + this._shell = (gristServer && gristServer.create.Shell()) || { + moveItemToTrash() { throw new Error('Unable to move document to trash'); }, + showItemInFolder() { throw new Error('Unable to show item in folder'); } + }; + if (_comm) { + this._initFileWatcher(); + } + } + + /** + * Returns the path to the given document. This is used by DocStorage.js, and is specific to the + * file-based storage implementation. + * @param {String} docName: The canonical docName. + * @returns {String} path: Filesystem path. + */ + public getPath(docName: string): string { + docName += (path.extname(docName) === '.grist' ? '' : '.grist'); + return path.resolve(this._docsRoot, docName); + } + + /** + * Returns the path to the given sample document. + */ + public getSampleDocPath(sampleDocName: string): string|null { + return this._samplesRoot ? this.getPath(path.resolve(this._samplesRoot, sampleDocName)) : null; + } + + /** + * Translates a possibly non-canonical docName to a canonical one (e.g. adds .grist to a path + * without .grist extension, and canonicalizes the path). All other functions deal with + * canonical docNames. + * @param {String} altDocName: docName which may not be the canonical one. + * @returns {Promise:String} Promise for the canonical docName. + */ + public async getCanonicalDocName(altDocName: string): Promise { + const p = await docUtils.realPath(this.getPath(altDocName)); + return path.dirname(p) === this._docsRoot ? path.basename(p, '.grist') : p; + } + + /** + * Prepares a document for use locally. Returns whether the document is new (needs to be + * created). This is a no-op in the local DocStorageManager case. + */ + public async prepareLocalDoc(docName: string, docSession: OptDocSession): Promise { return false; } + + /** + * Returns a promise for the list of docNames to show in the doc list. For the file-based + * storage, this will include all .grist files under the docsRoot. + * @returns {Promise:Array} Promise for an array of objects with `name`, `size`, + * and `mtime`. + */ + public listDocs(): Promise { + return bluebird.Promise.all([ + this._listDocs(this._docsRoot, ""), + this._samplesRoot ? this._listDocs(this._samplesRoot, "sample") : [], + ]) + .spread((docsEntries: DocEntry[], samplesEntries: DocEntry[]) => { + return [...docsEntries, ...samplesEntries]; + }); + } + + /** + * Deletes a document. + * @param {String} docName: docName of the document to delete. + * @returns {Promise} Resolved on success. + */ + public deleteDoc(docName: string, deletePermanently?: boolean): Promise { + const docPath = this.getPath(docName); + // Keep this check, to protect against wiping out the whole disk or the user's home. + if (path.extname(docPath) !== '.grist') { + return Promise.reject(new Error("Refusing to delete path which does not end in .grist")); + } else if (deletePermanently) { + return fse.remove(docPath); + } else { + this._shell.moveItemToTrash(docPath); // this is a synchronous action + return Promise.resolve(); + } + } + + /** + * Renames a closed document. In the file-system case, moves files to reflect the new paths. For + * a document already open, use `docStorageInstance.renameDocTo()` instead. + * @param {String} oldName: original docName. + * @param {String} newName: new docName. + * @returns {Promise} Resolved on success. + */ + public renameDoc(oldName: string, newName: string): Promise { + const oldPath = this.getPath(oldName); + const newPath = this.getPath(newName); + return docUtils.createExclusive(newPath) + .catch(async (e: any) => { + if (e.code !== 'EEXIST') { throw e; } + const isSame = await docUtils.isSameFile(oldPath, newPath); + if (!isSame) { throw e; } + }) + .then(() => fse.rename(oldPath, newPath)) + // Send 'renameDocs' event immediately after the rename. Previously, this used to be sent by + // DocManager after reopening the renamed doc. The extra delay caused issue T407, where + // chokidar.watch() triggered 'removeDocs' before 'renameDocs'. + .then(() => { this._sendDocListAction('renameDocs', oldPath, [oldName, newName]); }) + .catch((err: Error) => { + log.warn("DocStorageManager: rename %s -> %s failed: %s", oldPath, newPath, err.message); + throw err; + }); + } + + /** + * Should create a backup of the file + * @param {String} docName - docName to backup + * @param {String} backupTag - string to identify backup, like foo.grist.$DATE.$TAG.bak + * @returns {Promise} Resolved on success, returns path to backup (to show user) + */ + public makeBackup(docName: string, backupTag: string): Promise { + // this need to persist between calling createNumbered and + // getting it's return value, to re-add the extension again (._.) + let ext: string; + let finalBakPath: string; // holds final value of path, with numbering + + return bluebird.Promise.try(() => this._generateBackupFilePath(docName, backupTag)) + .then((bakPath: string) => { // make a numbered migration if necessary + + log.debug(`DocStorageManager: trying to make backup at ${bakPath}`); + + // create a file at bakPath, adding numbers if necessary + ext = path.extname(bakPath); // persists to makeBackup closure + const bakPathPrefix = bakPath.slice(0, -ext.length); + return docUtils.createNumbered(bakPathPrefix, '-', + (pathPrefix: string) => docUtils.createExclusive(pathPrefix + ext) + ); + }).tap((numberedBakPathPrefix: string) => { // do the copying, but return bakPath anyway + finalBakPath = numberedBakPathPrefix + ext; + const docPath = this.getPath(docName); + log.info(`Backing up ${docName} to ${finalBakPath}`); + return docUtils.copyFile(docPath, finalBakPath); + }).then(() => { + log.debug("DocStorageManager: Backup made successfully at: %s", finalBakPath); + return finalBakPath; + }).catch((err: Error) => { + log.error("DocStorageManager: Backup %s %s failed: %s", docName, err.message); + throw err; + }); + } + + /** + * Electron version only. Shows the given doc in the file explorer. + */ + public async showItemInFolder(docName: string): Promise { + this._shell.showItemInFolder(await this.getPath(docName)); + } + + public async closeStorage() { + // nothing to do + } + + public async closeDocument(docName: string) { + // nothing to do + } + + public markAsChanged(docName: string): void { + // nothing to do + } + + public markAsEdited(docName: string): void { + // nothing to do + } + + public testReopenStorage(): void { + // nothing to do + } + + public addToStorage(id: string): void { + // nothing to do + } + + public prepareToCloseStorage(): void { + // nothing to do + } + + public async flushDoc(docName: string): Promise { + // nothing to do + } + + public async getCopy(docName: string): Promise { + const srcPath = this.getPath(docName); + const postfix = uuidv4(); + const tmpPath = `${srcPath}-${postfix}`; + await docUtils.copyFile(srcPath, tmpPath); + return tmpPath; + } + + public async getSnapshots(docName: string): Promise { + throw new Error('getSnapshots not implemented'); + } + + public async replace(docName: string, options: any): Promise { + throw new Error('replacement not implemented'); + } + + /** + * Returns a promise for the list of docNames for all docs in the given directory. + * @returns {Promise:Array} Promise for an array of objects with `name`, `size`, + * and `mtime`. + */ + private _listDocs(dirPath: string, tag: DocEntryTag): Promise { + return fse.readdir(dirPath) + // Filter out for .grist files, and strip the .grist extension. + .then(entries => Promise.all( + entries.filter(e => (path.extname(e) === '.grist')) + .map(e => { + const docPath = path.resolve(dirPath, e); + return fse.stat(docPath) + .then(stat => getDocListFileInfo(docPath, stat, tag)); + }) + )) + // Sort case-insensitively. + .then(entries => entries.sort((a, b) => a.name.toLowerCase().localeCompare(b.name.toLowerCase()))) + // If the root directory is missing, just return an empty array. + .catch(err => { + if (err.cause && err.cause.code === 'ENOENT') { return []; } + throw err; + }); + } + + /** + * Generates the filename for the given document backup + * Backup names should look roughly like: + * ${basefilename}.grist.${YYYY-MM-DD}.${tag}.bak + * + * @returns {Promise} backup filepath (might need to createNumbered) + */ + private _generateBackupFilePath(docName: string, backupTag: string): Promise { + const dateString = moment().format("YYYY-MM-DD"); + + return docUtils.realPath(this.getPath(docName)) + .then((filePath: string) => { + const fileName = path.basename(filePath); + const fileDir = path.dirname(filePath); + + const bakName = `${fileName}.${dateString}.${backupTag}.bak`; + return path.join(fileDir, bakName); + }); + } + + /** + * Creates the file watcher and begins monitoring the docsRoot. Returns the created watcher. + */ + private _initFileWatcher(): void { + // NOTE: The chokidar watcher reports file renames as unlink then add events. + this._watcher = chokidar.watch(this._docsRoot, { + ignoreInitial: true, // Prevent messages for initial adds of all docs when watching begins + depth: 0, // Ignore changes in subdirectories of docPath + alwaysStat: true, // Tells the watcher to always include the stats arg + // Waits for a file to remain constant for a short time after changing before triggering + // an action. Prevents reporting of incomplete writes. + awaitWriteFinish: { + stabilityThreshold: 100, // Waits for the file to remain constant for 100ms + pollInterval: 10 // Polls the file every 10ms after a change + } + }); + this._watcher.on('add', (docPath: string, fsStats: any) => { + this._sendDocListAction('addDocs', docPath, getDocListFileInfo(docPath, fsStats, "")); + }); + this._watcher.on('change', (docPath: string, fsStats: any) => { + this._sendDocListAction('changeDocs', docPath, getDocListFileInfo(docPath, fsStats, "")); + }); + this._watcher.on('unlink', (docPath: string) => { + this._sendDocListAction('removeDocs', docPath, getDocName(docPath)); + }); + } + + /** + * Helper to broadcast a docListAction for a single doc to clients. If the action is not on a + * '.grist' file, it is not sent. + * @param {String} actionType - DocListAction type to send, 'addDocs' | 'removeDocs' | 'changeDocs'. + * @param {String} docPath - System path to the doc including the filename. + * @param {Any} data - Data to send as the message. + */ + private _sendDocListAction(actionType: string, docPath: string, data: any): void { + if (this._comm && gutil.endsWith(docPath, '.grist')) { + log.debug(`Sending ${actionType} action for doc ${getDocName(docPath)}`); + this._comm.broadcastMessage('docListAction', { [actionType]: [data] }); + } + } +} + +/** + * Helper to return the docname (without .grist) given the path to the .grist file. + */ +function getDocName(docPath: string): string { + return path.basename(docPath, '.grist'); +} + +/** + * Helper to get the stats used by the Grist DocList for a document. + * @param {String} docPath - System path to the doc including the doc filename. + * @param {Object} fsStat - fs.Stats object describing the file metadata. + * @param {String} tag - The tag indicating the type of doc. + * @return {Promise:Object} Promise for an object containing stats for the requested doc. + */ +function getDocListFileInfo(docPath: string, fsStat: any, tag: DocEntryTag): DocEntry { + return { + docId: undefined, // TODO: Should include docId if it exists + name: getDocName(docPath), + mtime: fsStat.mtime, + size: fsStat.size, + tag + }; +} diff --git a/app/server/lib/DocWorker.ts b/app/server/lib/DocWorker.ts new file mode 100644 index 00000000..ade13d4f --- /dev/null +++ b/app/server/lib/DocWorker.ts @@ -0,0 +1,197 @@ +/** + * DocWorker collects the methods and endpoints that relate to a single Grist document. + * In hosted environment, this comprises the functionality of the DocWorker instance type. + */ +import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl'; +import {ActiveDoc} from 'app/server/lib/ActiveDoc'; +import {assertAccess, getOrSetDocAuth, getUserId, RequestWithLogin} from 'app/server/lib/Authorizer'; +import {Client} from 'app/server/lib/Client'; +import * as Comm from 'app/server/lib/Comm'; +import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; +import * as log from 'app/server/lib/log'; +import {integerParam, optStringParam, stringParam} from 'app/server/lib/requestUtils'; +import {OpenMode, quoteIdent, SQLiteDB} from 'app/server/lib/SQLiteDB'; +import {generateCSV} from 'app/server/serverMethods'; +import * as contentDisposition from 'content-disposition'; +import * as express from 'express'; +import * as fse from 'fs-extra'; +import * as mimeTypes from 'mime-types'; +import * as path from 'path'; + +export interface AttachOptions { + comm: Comm; // Comm object for methods called via websocket +} + +export class DocWorker { + private _comm: Comm; + constructor(private _dbManager: HomeDBManager, {comm}: AttachOptions) { + this._comm = comm; + } + + public getCSV(req: express.Request, res: express.Response): void { + return generateCSV(req, res, this._comm); + } + + public async getAttachment(req: express.Request, res: express.Response): Promise { + try { + const client = this._comm.getClient(stringParam(req.query.clientId)); + const activeDoc = this._getActiveDoc(stringParam(req.query.clientId), + integerParam(req.query.docFD)); + const ext = path.extname(stringParam(req.query.ident)); + const type = mimeTypes.lookup(ext); + + let inline = Boolean(req.query.inline); + // Serving up user-uploaded HTML files inline is an open door to XSS attacks. + if (type === "text/html") { inline = false; } + + // Construct a content-disposition header of the form 'inline|attachment; filename="NAME"' + const contentDispType = inline ? "inline" : "attachment"; + const contentDispHeader = contentDisposition(stringParam(req.query.name), {type: contentDispType}); + const data = await activeDoc.getAttachmentData(client, stringParam(req.query.ident)); + res.status(200) + .type(ext) + .set('Content-Disposition', contentDispHeader) + .set('Cache-Control', 'private, max-age=3600') + .send(data); + } catch (err) { + res.status(404).send({error: err.toString()}); + } + } + + public async downloadDoc(req: express.Request, res: express.Response, + storageManager: IDocStorageManager): Promise { + const mreq = req as RequestWithLogin; + if (!mreq.docAuth || !mreq.docAuth.docId) { throw new Error('Cannot find document'); } + const docId = mreq.docAuth.docId; + + // Query DB for doc metadata to get the doc title. + const doc = await this._dbManager.getDoc({userId: getUserId(req), org: mreq.org, urlId: docId}); + const docTitle = doc.name; + + // Get a copy of document for downloading. + const tmpPath = await storageManager.getCopy(docId); + if (req.query.template === '1') { + // If template flag is on, remove data and history from the download. + await removeData(tmpPath); + } + // NOTE: We may want to reconsider the mimeType used for Grist files. + return res.type('application/x-sqlite3') + .download(tmpPath, (optStringParam(req.query.title) || docTitle || 'document') + ".grist", async (err: any) => { + if (err) { + log.error(`Download failure for doc ${docId}`, err); + } + await fse.unlink(tmpPath); + }); + } + + // Register main methods related to documents. + public registerCommCore(): void { + const comm = this._comm; + comm.registerMethods({ + closeDoc: activeDocMethod.bind(null, null, 'closeDoc'), + fetchTable: activeDocMethod.bind(null, 'viewers', 'fetchTable'), + fetchTableSchema: activeDocMethod.bind(null, 'viewers', 'fetchTableSchema'), + useQuerySet: activeDocMethod.bind(null, 'viewers', 'useQuerySet'), + disposeQuerySet: activeDocMethod.bind(null, 'viewers', 'disposeQuerySet'), + applyUserActions: activeDocMethod.bind(null, 'editors', 'applyUserActions'), + applyUserActionsById: activeDocMethod.bind(null, 'editors', 'applyUserActionsById'), + findColFromValues: activeDocMethod.bind(null, 'viewers', 'findColFromValues'), + getFormulaError: activeDocMethod.bind(null, 'viewers', 'getFormulaError'), + importFiles: activeDocMethod.bind(null, 'editors', 'importFiles'), + finishImportFiles: activeDocMethod.bind(null, 'editors', 'finishImportFiles'), + cancelImportFiles: activeDocMethod.bind(null, 'editors', 'cancelImportFiles'), + addAttachments: activeDocMethod.bind(null, 'editors', 'addAttachments'), + removeInstanceFromDoc: activeDocMethod.bind(null, 'editors', 'removeInstanceFromDoc'), + startBundleUserActions: activeDocMethod.bind(null, 'editors', 'startBundleUserActions'), + stopBundleUserActions: activeDocMethod.bind(null, 'editors', 'stopBundleUserActions'), + autocomplete: activeDocMethod.bind(null, 'viewers', 'autocomplete'), + fetchURL: activeDocMethod.bind(null, 'viewers', 'fetchURL'), + getActionSummaries: activeDocMethod.bind(null, 'viewers', 'getActionSummaries'), + reloadDoc: activeDocMethod.bind(null, 'editors', 'reloadDoc'), + fork: activeDocMethod.bind(null, 'viewers', 'fork'), + }); + } + + // Register methods related to plugins. + public registerCommPlugin(): void { + this._comm.registerMethods({ + forwardPluginRpc: activeDocMethod.bind(null, 'editors', 'forwardPluginRpc'), + // TODO: consider not providing reloadPlugins on hosted grist, since it affects the + // plugin manager shared across docs on a given doc worker, and seems useful only in + // standalone case. + reloadPlugins: activeDocMethod.bind(null, 'editors', 'reloadPlugins'), + }); + } + + // Checks that document is accessible, and adds docAuth information to request. + // Otherwise issues a 403 access denied. + // (This is used for endpoints like /download, /gen-csv, /attachment.) + public async assertDocAccess( + req: express.Request, + res: express.Response, + next: express.NextFunction + ) { + const mreq = req as RequestWithLogin; + let urlId: string|undefined; + try { + if (optStringParam(req.query.clientId)) { + const activeDoc = this._getActiveDoc(stringParam(req.query.clientId), + integerParam(req.query.docFD)); + // TODO: The docId should be stored in the ActiveDoc class. Currently docName is + // used instead, which will coincide with the docId for hosted grist but not for + // standalone grist. + urlId = activeDoc.docName; + } else { + // Otherwise, if being used without a client, expect the doc query parameter to + // be the docId. + urlId = stringParam(req.query.doc); + } + if (!urlId) { return res.status(403).send({error: 'missing document id'}); } + + const docAuth = await getOrSetDocAuth(mreq, this._dbManager, urlId); + assertAccess('viewers', docAuth); + next(); + } catch (err) { + log.info(`DocWorker can't access document ${urlId} with userId ${mreq.userId}: ${err}`); + res.status(err.status || 404).send({error: err.toString()}); + } + } + + private _getActiveDoc(clientId: string, docFD: number): ActiveDoc { + const client = this._comm.getClient(clientId); + const docSession = client.getDocSession(docFD); + return docSession.activeDoc; + } +} + +/** + * Translates calls from the browser client into calls of the form + * `activeDoc.method(docSession, ...args)`. + */ +async function activeDocMethod(role: 'viewers'|'editors'|null, methodName: string, client: Client, + docFD: number, ...args: any[]): Promise { + const docSession = client.getDocSession(docFD); + const activeDoc = docSession.activeDoc; + if (role) { await docSession.authorizer.assertAccess(role); } + // Include a basic log record for each ActiveDoc method call. + log.rawDebug('activeDocMethod', activeDoc.getLogMeta(client, methodName)); + return (activeDoc as any)[methodName](docSession, ...args); +} + +/** + * Remove rows from all user tables, and wipe as much history as we can. + */ +async function removeData(filename: string) { + const db = await SQLiteDB.openDBRaw(filename, OpenMode.OPEN_EXISTING); + const tableIds = (await db.all("SELECT name FROM sqlite_master WHERE type='table'")) + .map(row => row.name as string) + .filter(name => !name.startsWith('_grist')); + for (const tableId of tableIds) { + await db.run(`DELETE FROM ${quoteIdent(tableId)}`); + } + const history = new ActionHistoryImpl(db); + await history.deleteActions(1); + await db.run('VACUUM'); + await db.close(); +} diff --git a/app/server/lib/DocWorkerMap.ts b/app/server/lib/DocWorkerMap.ts new file mode 100644 index 00000000..87412271 --- /dev/null +++ b/app/server/lib/DocWorkerMap.ts @@ -0,0 +1,62 @@ +/** + * Defines the IDocWorkerMap interface we need to assign a DocWorker to a doc, and to look it up. + * TODO This is not yet implemented, there is only a hard-coded stub. + */ + +import { IElectionStore } from 'app/server/lib/IElectionStore'; +import { IPermitStore } from 'app/server/lib/Permit'; + +export interface DocWorkerInfo { + id: string; + + // The public base URL for the docWorker, which tells the browser how to connect to it. E.g. + // https://docworker-17.getgrist.com/ or http://localhost:8080/v/gtag/ + publicUrl: string; + + // The internal base URL for the docWorker. + internalUrl: string; +} + +export interface DocStatus { + // MD5 hash of the SQLite file for this document as stored on S3. We use MD5 because it is + // automatically computed by S3 (except for multipart uploads). Null indicates a new file. + docMD5: string|null; + + // DocWorker most recently, or currently, responsible for the file. + docWorker: DocWorkerInfo; + + // Whether the file is currently open on this DocWorker. + isActive: boolean; +} + +/** + * Assignment of documents to workers, and other storage related to distributed work. + */ +export interface IDocWorkerMap extends IPermitStore, IElectionStore { + // Looks up which DocWorker is responsible for this docId. + getDocWorker(docId: string): Promise; + + // Assigns a DocWorker to this docId if one is not yet assigned. + assignDocWorker(docId: string): Promise; + + // Assigns a particular DocWorker to this docId if one is not yet assigned. + getDocWorkerOrAssign(docId: string, workerId: string): Promise; + + updateDocStatus(docId: string, checksum: string): Promise; + + addWorker(info: DocWorkerInfo): Promise; + + removeWorker(workerId: string): Promise; + + // Set whether worker is accepting new assignments. This does not automatically + // release existing assignments. + setWorkerAvailability(workerId: string, available: boolean): Promise; + + // Releases doc from worker, freeing it to be assigned elsewhere. + // Assigments should only be released for workers that are now unavailable. + releaseAssignment(workerId: string, docId: string): Promise; + + // Get all assignments for a worker. Should only be queried for a worker that + // is currently unavailable. + getAssignments(workerId: string): Promise; +} diff --git a/app/server/lib/ExpandedQuery.ts b/app/server/lib/ExpandedQuery.ts new file mode 100644 index 00000000..37ab8dae --- /dev/null +++ b/app/server/lib/ExpandedQuery.ts @@ -0,0 +1,135 @@ +import {Query} from 'app/common/ActiveDocAPI'; +import {ApiError} from 'app/common/ApiError'; +import {DocData} from 'app/common/DocData'; +import {parseFormula} from 'app/common/Formula'; +import {removePrefix} from 'app/common/gutil'; +import {quoteIdent} from 'app/server/lib/SQLiteDB'; + +/** + * Represents a query for Grist data with support for SQL-based + * formulas. Use of this representation should be limited to within a + * trusted part of Grist since it assembles SQL strings. + */ +export interface ExpandedQuery extends Query { + // Errors detected for given columns because of formula issues. We + // need to make sure the result of the query contains these error + // objects. It is awkward to write a sql selection that constructs + // an error object, so instead we select 0 in the case of an error, + // and substitute in the error object in javascript after the SQL + // step. That means we need to pass the error message along + // explicitly. + constants?: { + [colId: string]: ['E', string] | ['P']; + }; + + // A list of join clauses to bring in data from other tables. + joins?: string[]; + + // A list of selections for regular data and data computed via formulas. + selects?: string[]; +} + +/** + * Add JOINs and SELECTs to a query in order to implement formulas via SQL. + * + * Supports simple formulas that load a column via a reference. + * The referenced column itself cannot (yet) be a formula. + * Filtered columns cannot (yet) be a formula. + * + * If formulas is not set, we simply mark formula columns as pending. + */ +export function expandQuery(iquery: Query, docData: DocData, formulas: boolean = true): ExpandedQuery { + const query: ExpandedQuery = { + tableId: iquery.tableId, + filters: iquery.filters, + limit: iquery.limit + }; + + // Look up the main table for the query. + const tables = docData.getTable('_grist_Tables')!; + const columns = docData.getTable('_grist_Tables_column')!; + const tableRef = tables.findRow('tableId', query.tableId); + if (!tableRef) { throw new ApiError('table not found', 404); } + + // Find any references to other tables. + const dataColumns = columns.filterRecords({parentId: tableRef, isFormula: false}); + const references = new Map(); + for (const column of dataColumns) { + const refTableId = removePrefix(column.type as string, 'Ref:'); + if (refTableId) { references.set(column.colId as string, refTableId); } + } + + // Start accumulating a set of joins and selects needed for the query. + const joins = new Set(); + const selects = new Set(); + + // Select all data columns + selects.add(`${quoteIdent(query.tableId)}.*`); + + // Iterate through all formulas, adding joins and selects as we go. + if (formulas) { + const formulaColumns = columns.filterRecords({parentId: tableRef, isFormula: true}); + for (const column of formulaColumns) { + const formula = parseFormula(column.formula as string); + const colId = column.colId as string; + let sqlFormula = ""; + let error = ""; + if (formula.kind === 'foreignColumn') { + const altTableId = references.get(formula.refColId); + const altTableRef = tables.findRow('tableId', altTableId); + if (altTableId && altTableRef) { + const altColumn = columns.filterRecords({parentId: altTableRef, isFormula: false, colId: formula.colId}); + // TODO: deal with a formula column in the other table. + if (altColumn.length > 0) { + const alias = `${query.tableId}_${formula.refColId}`; + joins.add(`LEFT JOIN ${quoteIdent(altTableId)} AS ${quoteIdent(alias)} ` + + `ON ${quoteIdent(alias)}.id = ` + + `${quoteIdent(query.tableId)}.${quoteIdent(formula.refColId)}`); + sqlFormula = `${quoteIdent(alias)}.${quoteIdent(formula.colId)}`; + } else { + error = "Cannot find column"; + } + } else { + error = "Cannot find table"; + } + } else if (formula.kind === 'column') { + const altColumn = columns.filterRecords({parentId: tableRef, isFormula: false, colId: formula.colId}); + // TODO: deal with a formula column. + if (altColumn.length > 0) { + sqlFormula = `${quoteIdent(query.tableId)}.${quoteIdent(formula.colId)}`; + } else { + error = "Cannot find column"; + } + } else if (formula.kind === 'literalNumber') { + sqlFormula = `${formula.value}`; + } else if (formula.kind === 'error') { + error = formula.msg; + } else { + throw new Error('Unrecognized type of formula'); + } + if (error) { + // We add a trivial selection, and store errors in the query for substitution later. + sqlFormula = '0'; + if (!query.constants) { query.constants = {}; } + query.constants[colId] = ['E', error]; + } + if (sqlFormula) { + selects.add(`${sqlFormula} as ${quoteIdent(colId)}`); + } + } + } else { + const formulaColumns = columns.filterRecords({parentId: tableRef, isFormula: true}); + for (const column of formulaColumns) { + if (!column.formula) { continue; } // Columns like this won't get calculated, so skip. + const colId = column.colId as string; + if (!query.constants) { query.constants = {}; } + query.constants[colId] = ['P']; + selects.add(`0 as ${quoteIdent(colId)}`); + } + } + + // Copy decisions to the query object, and return. + query.joins = [...joins]; + query.selects = [...selects]; + return query; +} diff --git a/app/server/lib/ExternalStorage.ts b/app/server/lib/ExternalStorage.ts new file mode 100644 index 00000000..3f58e917 --- /dev/null +++ b/app/server/lib/ExternalStorage.ts @@ -0,0 +1,320 @@ +import {ObjSnapshot} from 'app/server/lib/DocSnapshots'; +import * as log from 'app/server/lib/log'; +import {createTmpDir} from 'app/server/lib/uploads'; +import {delay} from 'bluebird'; +import * as fse from 'fs-extra'; +import * as path from 'path'; + +// A special token representing a deleted document, used in places where a +// checksum is expected otherwise. +export const DELETED_TOKEN = '*DELETED*'; + +/** + * An external store for the content of files. The store may be either consistent + * or eventually consistent. Specifically, the `exists`, `download`, and `versions` + * methods may return somewhat stale data from time to time. + * + * The store should be versioned; that is, uploads to a `key` should be assigned + * a `snapshotId`, and be accessible later with that `key`/`snapshotId` pair. + * When data is accessed by `snapshotId`, results should be immediately consistent. + */ +export interface ExternalStorage { + // Check if content exists in the store for a given key. + exists(key: string): Promise; + + // Upload content from file to the given key. Returns a snapshotId if store supports that. + upload(key: string, fname: string): Promise; + + // Download content from key to given file. Can download a specific version of the key + // if store supports that (should throw a fatal exception if not). + download(key: string, fname: string, snapshotId?: string): Promise; + + // Remove content for this key from the store, if it exists. Can delete specific versions + // if specified. If no version specified, all versions are removed. If versions specified, + // newest should be given first. + remove(key: string, snapshotIds?: string[]): Promise; + + // List content versions that exist for the given key. More recent versions should + // come earlier in the result list. + versions(key: string): Promise; + + // Render the given key as something url-like, for log messages (e.g. "s3://bucket/path") + url(key: string): string; + + // Check if an exception thrown by a store method should be treated as fatal. + // Non-fatal exceptions are those that may result from eventual consistency, and + // where a retry could help -- specifically "not found" exceptions. + isFatalError(err: any): boolean; + + // Close the storage object. + close(): Promise; +} + +/** + * Convenience wrapper to transform keys for an external store. + * E.g. this could convert "" to "v1/.grist" + */ +export class KeyMappedExternalStorage implements ExternalStorage { + constructor(private _ext: ExternalStorage, + private _map: (key: string) => string) { + } + + public exists(key: string): Promise { + return this._ext.exists(this._map(key)); + } + + public upload(key: string, fname: string) { + return this._ext.upload(this._map(key), fname); + } + + public download(key: string, fname: string, snapshotId?: string) { + return this._ext.download(this._map(key), fname, snapshotId); + } + + public remove(key: string, snapshotIds?: string[]): Promise { + return this._ext.remove(this._map(key), snapshotIds); + } + + public versions(key: string) { + return this._ext.versions(this._map(key)); + } + + public url(key: string) { + return this._ext.url(this._map(key)); + } + + public isFatalError(err: any) { + return this._ext.isFatalError(err); + } + + public async close() { + // nothing to do + } +} + +/** + * A wrapper for an external store that uses checksums and retries + * to compensate for eventual consistency. With this wrapper, the + * store either returns consistent results or fails with an error. + * + * This wrapper works by tracking what is in the external store, + * using content hashes and ids placed in consistent stores. These + * consistent stores are: + * + * - sharedHash: a key/value store containing expected checksums + * of content in the external store. In our setup, this is + * implemented using Redis. Populated on upload and checked on + * download. + * - localHash: a key/value store containing checksums of uploaded + * content. In our setup, this is implemented on the worker's + * disk. This is used to skip unnecessary uploads. Populated + * on download and checked on upload. + * - latestVersion: a key/value store containing snapshotIds of + * uploads. In our setup, this is implemented in the worker's + * memory. Only affects the consistency of the `versions` method. + * Populated on upload and checked on `versions` calls. + * TODO: move to Redis if consistency of `versions` during worker + * transitions becomes important. + * + * It is not important for all this side information to persist very + * long, just long enough to give the store time to become + * consistent. + * + * Keys presented to this class should be file-system safe. + */ +export class ChecksummedExternalStorage implements ExternalStorage { + private _closed: boolean = false; + + constructor(private _ext: ExternalStorage, private _options: { + maxRetries: number, // how many time to retry inconsistent downloads + initialDelayMs: number, // how long to wait before retrying + localHash: PropStorage, // key/value store for hashes of downloaded content + sharedHash: PropStorage, // key/value store for hashes of external content + latestVersion: PropStorage, // key/value store for snapshotIds of uploads + computeFileHash: (fname: string) => Promise, // compute hash for file + }) { + } + + public async exists(key: string): Promise { + return this._retry('exists', async () => { + const hash = await this._options.sharedHash.load(key); + const expected = hash !== null && hash !== DELETED_TOKEN; + const reported = await this._ext.exists(key); + // If we expect an object but store doesn't seem to have it, retry. + if (expected && !reported) { return undefined; } + // If store says there is an object but that is not what we expected (if we + // expected anything), retry. + if (hash && !expected && reported) { return undefined; } + // If expectations are matched, or we don't have expectations, return. + return reported; + }); + } + + public async upload(key: string, fname: string) { + try { + const checksum = await this._options.computeFileHash(fname); + const prevChecksum = await this._options.localHash.load(key); + if (prevChecksum && prevChecksum === checksum) { + // nothing to do, checksums match + log.info("ext upload: %s unchanged, not sending", key); + return this._options.latestVersion.load(key); + } + const snapshotId = await this._ext.upload(key, fname); + log.info("ext upload: %s checksum %s", this._ext.url(key), checksum); + if (snapshotId) { await this._options.latestVersion.save(key, snapshotId); } + await this._options.localHash.save(key, checksum); + await this._options.sharedHash.save(key, checksum); + return snapshotId; + } catch (err) { + log.error("ext upload: %s failure to send, error %s", key, err.message); + throw err; + } + } + + public async remove(key: string, snapshotIds?: string[]) { + try { + // Removing most recent version by id is not something we should be doing, and + // if we want to do it it would need to be done carefully - so just forbid it. + if (snapshotIds && snapshotIds.includes(await this._options.latestVersion.load(key) || '')) { + throw new Error('cannot remove most recent version of a document by id'); + } + await this._ext.remove(key, snapshotIds); + log.info("ext remove: %s version %s", this._ext.url(key), snapshotIds || 'ALL'); + if (!snapshotIds) { + await this._options.latestVersion.save(key, DELETED_TOKEN); + await this._options.sharedHash.save(key, DELETED_TOKEN); + } + } catch (err) { + log.error("ext delete: %s failure to remove, error %s", key, err.message); + throw err; + } + } + + public download(key: string, fname: string, snapshotId?: string) { + return this.downloadTo(key, key, fname, snapshotId); + } + + /** + * We may want to download material from one key and henceforth treat it as another + * key (specifically for forking a document). Since this class crossreferences the + * key in the external store with other consistent stores, it needs to know we are + * doing that. So we add a downloadTo variant that takes before and after keys. + */ + public async downloadTo(fromKey: string, toKey: string, fname: string, snapshotId?: string) { + await this._retry('download', async () => { + const {tmpDir, cleanupCallback} = await createTmpDir({}); + const tmpPath = path.join(tmpDir, `${toKey}.grist-tmp`); // NOTE: assumes key is file-system safe. + try { + await this._ext.download(fromKey, tmpPath, snapshotId); + + const checksum = await this._options.computeFileHash(tmpPath); + + // Check for consistency if mutable data fetched. + if (!snapshotId) { + const expectedChecksum = await this._options.sharedHash.load(fromKey); + // Let null docMD5s pass. Otherwise we get stuck if redis is cleared. + // Otherwise, make sure what we've got matches what we expect to get. + // S3 is eventually consistent - if you overwrite an object in it, and then read from it, + // you may get an old version for some time. + // If a snapshotId was specified, we can skip this check. + if (expectedChecksum && expectedChecksum !== checksum) { + log.error("ext download: data for %s has wrong checksum: %s (expected %s)", fromKey, + checksum, + expectedChecksum); + return undefined; + } + } + + // If successful, rename the temporary file to its proper name. The destination should NOT + // exist in this case, and this should fail if it does. + await fse.move(tmpPath, fname, {overwrite: false}); + await this._options.localHash.save(toKey, checksum); + + log.info("ext download: %s%s%s with checksum %s", fromKey, + snapshotId ? ` [VersionId ${snapshotId}]` : '', + fromKey !== toKey ? ` as ${toKey}` : '', + checksum); + + return true; + } catch (err) { + log.error("ext download: failed to fetch data (%s): %s", fromKey, err.message); + throw err; + } finally { + await cleanupCallback(); + } + }); + } + + public async versions(key: string) { + return this._retry('versions', async () => { + const snapshotId = await this._options.latestVersion.load(key); + if (snapshotId === DELETED_TOKEN) { return []; } + const result = await this._ext.versions(key); + if (snapshotId && (result.length === 0 || result[0].snapshotId !== snapshotId)) { + // Result is not consistent yet. + return undefined; + } + return result; + }); + } + + public url(key: string): string { + return this._ext.url(key); + } + + public isFatalError(err: any): boolean { + return this._ext.isFatalError(err); + } + + public async close() { + this._closed = true; + } + + /** + * Call an operation until it returns a value other than undefined. + * + * While the operation returns undefined, it will be retried for some period. + * This period is chosen to be long enough for S3 to become consistent. + * + * If the operation throws an error, and that error is not fatal (as determined + * by `isFatalError`, then it will also be retried. Fatal errors are thrown + * immediately. + * + * Once the operation returns a result, we pass that along. If it fails to + * return a result after all the allowed retries, a special exception is thrown. + */ + private async _retry(name: string, operation: () => Promise): Promise { + let backoffCount = 1; + let backoffFactor = this._options.initialDelayMs; + const problems = new Array<[number, string|Error]>(); + const start = Date.now(); + while (backoffCount <= this._options.maxRetries) { + try { + const result = await operation(); + if (result !== undefined) { return result; } + problems.push([Date.now() - start, 'not ready']); + } catch (err) { + if (this._ext.isFatalError(err)) { + throw err; + } + problems.push([Date.now() - start, err]); + } + // Wait some time before attempting to reload from s3. The longer we wait, the greater + // the odds of success. In practice, a second should be more than enough almost always. + await delay(Math.round(backoffFactor)); + if (this._closed) { throw new Error('storage closed'); } + backoffCount++; + backoffFactor *= 1.7; + } + log.error(`operation failed to become consistent: ${name} - ${problems}`); + throw new Error(`operation failed to become consistent: ${name} - ${problems}`); + } +} + +/** + * Small interface for storing hashes and ids. + */ +export interface PropStorage { + save(key: string, val: string): Promise; + load(key: string): Promise; +} diff --git a/app/server/lib/FileParserElement.ts b/app/server/lib/FileParserElement.ts new file mode 100644 index 00000000..43175c3b --- /dev/null +++ b/app/server/lib/FileParserElement.ts @@ -0,0 +1,49 @@ +import {PluginInstance} from 'app/common/PluginInstance'; +import {ParseFileAPI} from 'app/plugin/FileParserAPI'; +import {checkers} from 'app/plugin/TypeCheckers'; + +import {FileParser} from 'app/plugin/PluginManifest'; + +import * as path from 'path'; + +/** + * Encapsulates together a file parse contribution with its plugin instance and callable stubs for + * `parseFile` implementation provided by the plugin. + * + * Implements as well a `getMatching` static method to get all file parsers matching a filename from + * the list of plugin instances. + * + */ +export class FileParserElement { + + /** + * Get all file parser that matches fileName from the list of plugins instances. + */ + public static getMatching(pluginInstances: PluginInstance[], fileName: string): FileParserElement[] { + const fileParserElements: FileParserElement[] = []; + for (const plugin of pluginInstances) { + const fileParsers = plugin.definition.manifest.contributions.fileParsers; + if (fileParsers) { + for (const fileParser of fileParsers) { + if (matchFileParser(fileParser, fileName)) { + fileParserElements.push(new FileParserElement(plugin, fileParser)); + } + } + } + } + return fileParserElements; + } + + public parseFileStub: ParseFileAPI; + + private constructor(public plugin: PluginInstance, public fileParser: FileParser) { + this.parseFileStub = plugin.getStub(fileParser.parseFile, checkers.ParseFileAPI); + } + +} + +function matchFileParser(fileParser: FileParser, fileName: string): boolean { + const ext = path.extname(fileName).slice(1), + fileExtensions = fileParser.fileExtensions; + return fileExtensions && fileExtensions.includes(ext); +} diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts new file mode 100644 index 00000000..9c3c7bf8 --- /dev/null +++ b/app/server/lib/FlexServer.ts @@ -0,0 +1,1423 @@ +import {BillingTask} from 'app/common/BillingAPI'; +import {delay} from 'app/common/delay'; +import {DocCreationInfo} from 'app/common/DocListAPI'; +import {isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls'; +import {getOrgUrlInfo} from 'app/common/gristUrls'; +import {UserProfile} from 'app/common/LoginSessionAPI'; +import {tbind} from 'app/common/tbind'; +import {UserConfig} from 'app/common/UserConfig'; +import * as version from 'app/common/version'; +import {ApiServer} from 'app/gen-server/ApiServer'; +import {DocApiForwarder} from 'app/gen-server/lib/DocApiForwarder'; +import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap'; +import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import {Housekeeper} from 'app/gen-server/lib/Housekeeper'; +import {Usage} from 'app/gen-server/lib/Usage'; +import {attachAppEndpoint} from 'app/server/lib/AppEndpoint'; +import {addRequestUser, getUser, getUserId, isSingleUserMode} from 'app/server/lib/Authorizer'; +import {redirectToLogin, RequestWithLogin} from 'app/server/lib/Authorizer'; +import {SessionUserObj} from 'app/server/lib/BrowserSession'; +import * as Comm from 'app/server/lib/Comm'; +import {create} from 'app/server/lib/create'; +import {addDocApiRoutes} from 'app/server/lib/DocApi'; +import {DocManager} from 'app/server/lib/DocManager'; +import {DocStorageManager} from 'app/server/lib/DocStorageManager'; +import {DocWorker} from 'app/server/lib/DocWorker'; +import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; +import {expressWrap, jsonErrorHandler} from 'app/server/lib/expressWrap'; +import {Hosts, RequestWithOrg} from 'app/server/lib/extractOrg'; +import {GristLoginMiddleware, GristServer} from 'app/server/lib/GristServer'; +import {initGristSessions} from 'app/server/lib/gristSessions'; +import {HostedStorageManager} from 'app/server/lib/HostedStorageManager'; +import {IBilling} from 'app/server/lib/IBilling'; +import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; +import {INotifier} from 'app/server/lib/INotifier'; +import * as log from 'app/server/lib/log'; +import {getLoginMiddleware} from 'app/server/lib/logins'; +import {getAppPathTo, getAppRoot, getUnpackedAppRoot} from 'app/server/lib/places'; +import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint'; +import {PluginManager} from 'app/server/lib/PluginManager'; +import {adaptServerUrl, optStringParam, RequestWithGristInfo, stringParam, TEST_HTTPS_OFFSET, + trustOrigin} from 'app/server/lib/requestUtils'; +import {ISendAppPageOptions, makeSendAppPage} from 'app/server/lib/sendAppPage'; +import * as ServerMetrics from 'app/server/lib/ServerMetrics'; +import {getDatabaseUrl} from 'app/server/lib/serverUtils'; +import {Sessions} from 'app/server/lib/Sessions'; +import * as shutdown from 'app/server/lib/shutdown'; +import {TagChecker} from 'app/server/lib/TagChecker'; +import {startTestingHooks} from 'app/server/lib/TestingHooks'; +import {addUploadRoute} from 'app/server/lib/uploads'; +import axios from 'axios'; +import * as bodyParser from 'body-parser'; +import * as express from 'express'; +import * as fse from 'fs-extra'; +import * as http from 'http'; +import * as https from 'https'; +import * as morganLogger from 'morgan'; +import {AddressInfo} from 'net'; +import * as path from 'path'; +import * as serveStatic from "serve-static"; + +// Health checks are a little noisy in the logs, so we don't show them all. +// We show the first N health checks: +const HEALTH_CHECK_LOG_SHOW_FIRST_N = 10; +// And we show every Nth health check: +const HEALTH_CHECK_LOG_SHOW_EVERY_N = 100; + +export interface FlexServerOptions { + dataDir?: string; + + // Base domain for org hostnames, starting with ".". Defaults to the base domain of APP_HOME_URL. + baseDomain?: string; +} + +export interface RequestWithGrist extends express.Request { + gristServer?: GristServer; +} + +export class FlexServer implements GristServer { + public readonly create = create; + public tagChecker: TagChecker; + public app: express.Express; + public deps: Set = new Set(); + public appRoot: string; + public host: string; + public tag: string; + public info = new Array<[string, any]>(); + public sessions: Sessions; + public dbManager: HomeDBManager; + public notifier: INotifier; + public usage: Usage; + public housekeeper: Housekeeper; + public server: http.Server; + public httpsServer?: https.Server; + public comm: Comm; + public settings: any; + public worker: DocWorkerInfo; + public electronServerMethods: ElectronServerMethods; + public readonly docsRoot: string; + private _defaultBaseDomain: string|undefined; + private _billing: IBilling; + private _instanceRoot: string; + private _docManager: DocManager; + private _docWorker: DocWorker; + private _hosts: Hosts; + private _pluginManager: PluginManager; + private _storageManager: IDocStorageManager; + private _docWorkerMap: IDocWorkerMap; + private _serverMetrics: any; + private _disabled: boolean = false; + private _disableS3: boolean = false; + private _healthy: boolean = true; // becomes false if a serious error has occurred and + // server cannot do its work. + private _healthCheckCounter: number = 0; + private _hasTestingHooks: boolean = false; + private _loginMiddleware: GristLoginMiddleware; + private _userIdMiddleware: express.RequestHandler; + private _trustOriginsMiddleware: express.RequestHandler; + private _docPermissionsMiddleware: express.RequestHandler; + // This middleware redirects to signin/signup for anon, except on merged org or for + // a team site that allows anon access. + private _redirectToLoginWithExceptionsMiddleware: express.RequestHandler; + // This unconditionally redirects to signin/signup for anon, for pages where anon access + // is never desired. + private _redirectToLoginWithoutExceptionsMiddleware: express.RequestHandler; + private _redirectToOrgMiddleware: express.RequestHandler; + private _redirectToHostMiddleware: express.RequestHandler; + private _getLoginRedirectUrl: (target: URL) => Promise; + private _getSignUpRedirectUrl: (target: URL) => Promise; + private _getLogoutRedirectUrl: (nextUrl: URL, userSession: SessionUserObj) => Promise; + private _sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise; + + constructor(public port: number, public name: string = 'flexServer', + readonly options: FlexServerOptions = {}) { + this.app = express(); + this.app.set('port', port); + this.appRoot = getAppRoot(); + this.host = process.env.GRIST_HOST || "localhost"; + log.info(`== Grist version is ${version.version} (commit ${version.gitcommit})`); + this.info.push(['appRoot', this.appRoot]); + // This directory hold Grist documents. + this.docsRoot = fse.realpathSync(path.resolve((this.options && this.options.dataDir) || + process.env.GRIST_DATA_DIR || + getAppPathTo(this.appRoot, 'samples'))); + this.info.push(['docsRoot', this.docsRoot]); + + const homeUrl = process.env.APP_HOME_URL; + this._defaultBaseDomain = options.baseDomain || (homeUrl && parseSubdomain(new URL(homeUrl).hostname).base); + this.info.push(['defaultBaseDomain', this._defaultBaseDomain]); + + this.app.use((req, res, next) => { + (req as RequestWithGrist).gristServer = this; + next(); + }); + } + + public getHost(): string { + return `${this.host}:${this.getOwnPort()}`; + } + + // Get a url for this server, based on the protocol it speaks (http), the host it + // runs on, and the port it listens on. The url the client uses to communicate with + // the server may be different if there are intermediaries (such as a load-balancer + // terminating TLS). + public getOwnUrl(): string { + const port = this.getOwnPort(); + return `http://${this.host}:${port}`; + } + + /** + * Get a url for the home server api. Called without knowledge of a specific + * request, so will default to a generic url. Use of this method can render + * code incompatible with custom base domains (currently, sendgrid notifications + * via Notifier are incompatible for this reason). + */ + public getDefaultHomeUrl(): string { + const homeUrl = process.env.APP_HOME_URL || (this._has('api') && this.getOwnUrl()); + if (!homeUrl) { throw new Error("need APP_HOME_URL"); } + return homeUrl; + } + + /** + * Get a url for the home server api, adapting it to match the base domain in the + * requested url. This adaptation is important for cookie-based authentication. + * + * If relPath is given, returns that path relative to homeUrl. If omitted, note that + * getHomeUrl() will still return a URL ending in "/". + */ + public getHomeUrl(req: express.Request, relPath: string = ''): string { + // Get the default home url. + const homeUrl = new URL(relPath, this.getDefaultHomeUrl()); + adaptServerUrl(homeUrl, req as RequestWithOrg); + return homeUrl.href; + } + + /** + * Get a home url that is appropriate for the given document. For now, this + * returns a default that works for all documents. That could change in future, + * specifically with custom domains (perhaps we might limit which docs can be accessed + * based on domain). + */ + public async getHomeUrlByDocId(docId: string, relPath: string = ''): Promise { + return new URL(relPath, this.getDefaultHomeUrl()).href; + } + + // Get the port number the server listens on. This may be different from the port + // number the client expects when communicating with the server if there are intermediaries. + public getOwnPort(): number { + // Get the port from the server in case it was started with port 0. + return this.server ? (this.server.address() as AddressInfo).port : this.port; + } + + public addLogging() { + if (this._check('logging')) { return; } + // Add a timestamp token that matches exactly the formatting of non-morgan logs. + morganLogger.token('logTime', (req: Request) => log.timestamp()); + // Add an optional gristInfo token that can replace the url, if the url is sensitive + // (this is the case for some cognito login urls). + morganLogger.token('gristInfo', (req: RequestWithGristInfo) => + req.gristInfo || req.originalUrl || req.url); + morganLogger.token('host', (req: express.Request) => req.get('host')); + const msg = ':logTime :host :method :gristInfo :status :response-time ms - :res[content-length]'; + // In hosted Grist, render json so logs retain more organization. + function outputJson(tokens: any, req: any, res: any) { + return JSON.stringify({ + timestamp: tokens.logTime(req, res), + method: tokens.method(req, res), + path: tokens.gristInfo(req, res), + status: tokens.status(req, res), + timeMs: parseFloat(tokens['response-time'](req, res)) || undefined, + contentLength: parseInt(tokens.res(req, res, 'content-length'), 10) || undefined, + host: tokens.host(req, res) + }); + } + this.app.use(morganLogger(process.env.GRIST_HOSTED_VERSION ? outputJson : msg, { + skip: this._shouldSkipRequestLogging.bind(this) + })); + } + + public addHealthCheck() { + if (this._check('health')) { return; } + // Health check endpoint. if called with /hooks, testing hooks are required in order to be + // considered healthy. Testing hooks are used only in server started for tests, and + // /status/hooks allows the tests to wait for them to be ready. + this.app.get('/status(/hooks)?', (req, res) => { + if (this._healthy && (this._hasTestingHooks || !req.url.endsWith('/hooks'))) { + this._healthCheckCounter++; + res.status(200).send(`Grist ${this.name} is alive.`); + } else { + this._healthCheckCounter = 0; // reset counter if we ever go internally unhealthy. + res.status(500).send(`Grist ${this.name} is unhealthy.`); + } + }); + } + + public testAddRouter() { + if (this._check('router')) { return; } + this.app.get('/test/router', (req, res) => { + const act = optStringParam(req.query.act) || 'none'; + const port = stringParam(req.query.port); // port is trusted in mock; in prod it is not. + if (act === 'add' || act === 'remove') { + const host = `localhost:${port}`; + return res.status(200).json({ + act, + host, + url: `http://${host}`, + message: 'ok', + }); + } + return res.status(500).json({error: 'unrecognized action'}); + }); + } + + public addCleanup() { + if (this._check('cleanup')) { return; } + // Set up signal handlers. Note that nodemon sends SIGUSR2 to restart node. + shutdown.cleanupOnSignals('SIGINT', 'SIGTERM', 'SIGHUP', 'SIGUSR2'); + } + + public addTagChecker() { + if (this._check('tag', '!org')) { return; } + // Handle requests that start with /v/TAG/ and set .tag property on them. + this.tag = version.gitcommit; + this.info.push(['tag', this.tag]); + this.tagChecker = new TagChecker(this.tag); + this.app.use(this.tagChecker.inspectTag); + } + + /** + * To allow routing to doc workers via the path, doc workers remove any + * path prefix of the form /dw/...../ if present. The prefix is not checked, + * just removed unconditionally. + * TODO: determine what the prefix should be, and check it, to catch bugs. + */ + public stripDocWorkerIdPathPrefixIfPresent() { + if (this._check('strip_dw', '!tag', '!org')) { return; } + this.app.use((req, resp, next) => { + const match = req.url.match(/^\/dw\/([-a-zA-Z0-9]+)([/?].*)?$/); + if (match) { req.url = sanitizePathTail(match[2]); } + next(); + }); + } + + public addOrg() { + if (this._check('org', 'homedb', 'hosts')) { return; } + this.app.use(this._hosts.extractOrg); + } + + public setDirectory() { + if (this._check('dir')) { return; } + process.chdir(getUnpackedAppRoot(this.appRoot)); + } + + public get instanceRoot() { + if (!this._instanceRoot) { + this._instanceRoot = path.resolve(process.env.GRIST_INST_DIR || this.appRoot); + this.info.push(['instanceRoot', this._instanceRoot]); + } + return this._instanceRoot; + } + + public addStaticAndBowerDirectories() { + if (this._check('static_and_bower', 'dir')) { return; } + this.addTagChecker(); + // Allow static files to be requested from any origin. + const options: serveStatic.ServeStaticOptions = { + setHeaders: (res, filepath, stat) => { + res.header("Access-Control-Allow-Origin", "*"); + } + }; + // Grist has static help files, which may be useful for standalone app, + // but for hosted grist the latest help is at support.getgrist.com. Redirect + // to this page for the benefit of crawlers which currently rank the static help + // page link highly for historic reasons. + this.app.use(/^\/help\//, expressWrap(async (req, res) => { + res.redirect('https://support.getgrist.com'); + })); + const staticApp = express.static(getAppPathTo(this.appRoot, 'static'), options); + const bowerApp = express.static(getAppPathTo(this.appRoot, 'bower_components'), options); + this.app.use(this.tagChecker.withTag(staticApp)); + this.app.use(this.tagChecker.withTag(bowerApp)); + } + + // Some tests rely on testFOO.html files being served. + public addAssetsForTests() { + if (this._check('testAssets', 'dir')) { return; } + // Serve test[a-z]*.html for test purposes. + this.app.use(/^\/(test[a-z]*.html)$/i, expressWrap(async (req, res) => + res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')}))); + } + + // Plugin operation relies currently on grist-plugin-api.js being available, + // and with Grist's static assets to be also available on the untrusted + // host. The assets should be available without version tags. + public async addAssetsForPlugins() { + if (this._check('pluginUntaggedAssets', 'dir')) { return; } + this.app.use(/^\/(grist-plugin-api.js)$/, expressWrap(async (req, res) => + res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')}))); + // Plugins get access to static resources without a tag + this.app.use(limitToPlugins(express.static(getAppPathTo(this.appRoot, 'static')))); + this.app.use(limitToPlugins(express.static(getAppPathTo(this.appRoot, 'bower_components')))); + this.addOrg(); + addPluginEndpoints(this, await this._addPluginManager()); + } + + // Prepare cache for managing org-to-host relationship. + public addHosts() { + if (this._check('hosts', 'homedb')) { return; } + this._hosts = new Hosts(this._defaultBaseDomain, this.dbManager); + } + + public async initHomeDBManager() { + if (this._check('homedb')) { return; } + this.dbManager = new HomeDBManager(); + this.dbManager.setPrefix(process.env.GRIST_ID_PREFIX || ""); + await this.dbManager.connect(); + await this.dbManager.initializeSpecialIds(); + + // Report which database we are using, without sensitive credentials. + this.info.push(['database', getDatabaseUrl(this.dbManager.connection.options, false)]); + } + + public addDocWorkerMap() { + if (this._check('map')) { return; } + this._docWorkerMap = getDocWorkerMap(); + } + + // Set up the main express middleware used. For a single user setup, without logins, + // all this middleware is currently a no-op. + public addAccessMiddleware() { + if (this._check('middleware', 'map', isSingleUserMode() ? null : 'hosts')) { return; } + + if (!isSingleUserMode()) { + // Middleware to redirect landing pages to preferred host + this._redirectToHostMiddleware = this._hosts.redirectHost; + // Middleware to add the userId to the express request object. + // If GRIST_DEFAULT_EMAIL is set, login as that user when no other credentials + // presented. + const fallbackEmail = process.env.GRIST_DEFAULT_EMAIL || null; + this._userIdMiddleware = expressWrap(addRequestUser.bind(null, this.dbManager, this._docWorkerMap, + fallbackEmail)); + this._trustOriginsMiddleware = expressWrap(trustOriginHandler); + // middleware to authorize doc access to the app. Note that this requires the userId + // to be set on the request by _userIdMiddleware. + this._docPermissionsMiddleware = expressWrap((...args) => this._docWorker.assertDocAccess(...args)); + this._redirectToLoginWithExceptionsMiddleware = redirectToLogin(true, + this._getLoginRedirectUrl, + this._getSignUpRedirectUrl, + this.dbManager); + this._redirectToLoginWithoutExceptionsMiddleware = redirectToLogin(false, + this._getLoginRedirectUrl, + this._getSignUpRedirectUrl, + this.dbManager); + this._redirectToOrgMiddleware = tbind(this._redirectToOrg, this); + } else { + const noop: express.RequestHandler = (req, res, next) => next(); + this._userIdMiddleware = noop; + this._trustOriginsMiddleware = noop; + this._docPermissionsMiddleware = (req, res, next) => { + // For standalone single-user Grist, documents are stored on-disk + // with their filename equal to the document title, no document + // aliases are possible, and there is no access control. + // The _docPermissionsMiddleware is a no-op. + // TODO We might no longer have any tests for isSingleUserMode, or modes of operation. + next(); + }; + this._redirectToLoginWithExceptionsMiddleware = noop; + this._redirectToLoginWithoutExceptionsMiddleware = noop; + this._redirectToOrgMiddleware = noop; + this._redirectToHostMiddleware = noop; + } + } + + /** + * Add middleware common to all API endpoints (including forwarding ones). + */ + public addApiMiddleware() { + if (this._check('api-mw', 'middleware')) { return; } + // API endpoints need req.userId and need to support requests from different subdomains. + this.app.use("/api", this._userIdMiddleware); + this.app.use("/api", this._trustOriginsMiddleware); + this.app.use("/api", noCaching); + } + + /** + * Add error-handling middleware common to all API endpoints. + */ + public addApiErrorHandlers() { + if (this._check('api-error', 'api-mw')) { return; } + + // add a final not-found handler for api + this.app.use("/api", (req, res) => { + res.status(404).send({error: `not found: ${req.originalUrl}`}); + }); + + // Add a final error handler for /api endpoints that reports errors as JSON. + this.app.use('/api', jsonErrorHandler); + } + + public addHomeApi() { + if (this._check('api', 'homedb', 'json', 'api-mw')) { return; } + + // ApiServer's constructor adds endpoints to the app. + // tslint:disable-next-line:no-unused-expression + new ApiServer(this.app, this.dbManager); + } + + public addBillingApi() { + if (this._check('billing-api', 'homedb', 'json', 'api-mw')) { return; } + this._getBilling(); + this._billing.addEndpoints(this.app); + this._billing.addEventHandlers(); + } + + /** + * Add a /api/log endpoint that simply outputs client errors to our + * logs. This is a minimal placeholder for a special-purpose + * service for dealing with client errors. + */ + public addLogEndpoint() { + if (this._check('log-endpoint', 'json', 'api-mw')) { return; } + this.app.post('/api/log', expressWrap(async (req, resp) => { + const mreq = req as RequestWithLogin; + log.rawWarn('client error', { + event: req.body.event, + page: req.body.page, + browser: req.body.browser, + org: mreq.org, + email: mreq.user && mreq.user.loginEmail, + userId: mreq.userId, + }); + return resp.status(200).send(); + })); + } + + public async close() { + if (this._hosts) { this._hosts.close(); } + if (this.dbManager) { + this.dbManager.removeAllListeners(); + this.dbManager.flushDocAuthCache(); + } + if (this.server) { this.server.close(); } + if (this.httpsServer) { this.httpsServer.close(); } + if (this.usage) { this.usage.close(); } + if (this.housekeeper) { await this.housekeeper.stop(); } + await this._shutdown(); + // Do this last, DocWorkerMap is used during shutdown. + if (this._docWorkerMap) { await this._docWorkerMap.close(); } + } + + public addDocApiForwarder() { + if (this._check('doc_api_forwarder', '!json', 'homedb', 'api-mw', 'map')) { return; } + const docApiForwarder = new DocApiForwarder(this._docWorkerMap, this.dbManager); + docApiForwarder.addEndpoints(this.app); + } + + public addJsonSupport() { + if (this._check('json')) { return; } + this.app.use(bodyParser.json({limit: '1mb'})); // Increase from the default 100kb + } + + public addSessions() { + if (this._check('sessions', 'config')) { return; } + this.addTagChecker(); + this.addOrg(); + + // Create the sessionStore and related objects. + const {sessions, sessionMiddleware} = initGristSessions(this.instanceRoot, this); + this.app.use(sessionMiddleware); + + // Create an endpoint for making cookies during testing. + this.app.get('/test/session', async (req, res) => { + (req as any).session.alive = true; + res.status(200).send(`Grist ${this.name} is alive and is interested in you.`); + }); + + this.sessions = sessions; + } + + // Close connections and stop accepting new connections. Remove server from any lists + // it may be in. + public async stopListening(mode: 'crash'|'clean' = 'clean') { + if (!this._disabled) { + if (mode === 'clean') { + await this._shutdown(); + this._disabled = true; + } else { + this._disabled = true; + if (this.comm) { + this.comm.setServerActivation(false); + this.comm.destroyAllClients(); + } + } + this.server.close(); + if (this.httpsServer) { this.httpsServer.close(); } + } + } + + public async createWorkerUrl(): Promise<{url: string, host: string}> { + if (!process.env.GRIST_ROUTER_URL) { + throw new Error('No service available to create worker url'); + } + const w = await axios.get(process.env.GRIST_ROUTER_URL, + {params: {act: 'add', port: this.getOwnPort()}}); + log.info(`DocWorker registered itself via ${process.env.GRIST_ROUTER_URL} as ${w.data.url}`); + const statusUrl = `${w.data.url}/status`; + // We now wait for the worker to be available from the url that clients will + // use to connect to it. This may take some time. The main delay is the + // new target group and load balancer rule taking effect - typically 10-20 seconds. + // If we don't wait, the worker will end up registered for work and clients + // could end up trying to reach it to open documents - but the url they have + // won't work. + for (let tries = 0; tries < 600; tries++) { + await delay(1000); + try { + await axios.get(statusUrl); + return w.data; + } catch (err) { + log.debug(`While waiting for ${statusUrl} got error ${err.message}`); + } + } + throw new Error(`Cannot connect to ${statusUrl}`); + } + + // Accept new connections again. Add server to any lists it needs to be in to get work. + public async restartListening() { + if (!this._docWorkerMap) { throw new Error('expected to have DocWorkerMap'); } + await this.stopListening('clean'); + if (this._disabled) { + if (this._storageManager) { + this._storageManager.testReopenStorage(); + } + this.comm.setServerActivation(true); + if (this.worker) { + await this._startServers(this.server, this.httpsServer, this.name, this.port, false); + await this._addSelfAsWorker(this._docWorkerMap); + } + this._disabled = false; + } + } + + public addLandingPages() { + // TODO: check if isSingleUserMode() path can be removed from this method + if (this._check('landing', 'map', isSingleUserMode() ? null : 'homedb')) { return; } + this.addSessions(); + + // Initialize _sendAppPage helper. + this._sendAppPage = makeSendAppPage({ + server: isSingleUserMode() ? null : this, + staticDir: getAppPathTo(this.appRoot, 'static'), + tag: this.tag, + testLogin: allowTestLogin(), + baseDomain: this._defaultBaseDomain, + }); + + const welcomeNewUser: express.RequestHandler = isSingleUserMode() ? + (req, res, next) => next() : + expressWrap(async (req, res, next) => { + const mreq = req as RequestWithLogin; + const user = getUser(req); + if (user && user.isFirstTimeUser) { + log.debug(`welcoming user: ${user.name}`); + const prefix = isOrgInPathOnly(req.hostname) ? `/o/${mreq.org}` : ''; + return res.redirect(`${prefix}/welcome/user`); + } + next(); + }); + + attachAppEndpoint({ + app: this.app, + middleware: [ + this._redirectToHostMiddleware, + this._userIdMiddleware, + this._redirectToLoginWithExceptionsMiddleware, + this._redirectToOrgMiddleware, + welcomeNewUser + ], + docWorkerMap: isSingleUserMode() ? null : this._docWorkerMap, + sendAppPage: this._sendAppPage, + dbManager: this.dbManager, + }); + } + + // Load user config file from standard location. Alternatively, a config object + // can be supplied, in which case no file is needed. The notion of a user config + // file doesn't mean much in hosted grist, so it is convenient to be able to skip it. + public async loadConfig(settings?: UserConfig) { + if (this._check('config')) { return; } + if (!settings) { + const settingsPath = path.join(this.instanceRoot, 'config.json'); + if (await fse.pathExists(settingsPath)) { + log.info(`Loading config from ${settingsPath}`); + this.settings = JSON.parse(await fse.readFile(settingsPath, 'utf8')); + } else { + log.info(`Loading empty config because ${settingsPath} missing`); + this.settings = {}; + } + } else { + this.settings = settings; + } + + // TODO: We could include a third mock provider of login/logout URLs for better tests. Or we + // could create a mock SAML identity provider for testing this using the SAML flow. + this._loginMiddleware = await getLoginMiddleware(); + this._getLoginRedirectUrl = tbind(this._loginMiddleware.getLoginRedirectUrl, this._loginMiddleware); + this._getSignUpRedirectUrl = tbind(this._loginMiddleware.getSignUpRedirectUrl, this._loginMiddleware); + this._getLogoutRedirectUrl = tbind(this._loginMiddleware.getLogoutRedirectUrl, this._loginMiddleware); + } + + public addComm() { + if (this._check('comm', 'start')) { return; } + this.comm = new Comm(this.server, { + settings: this.settings, + sessions: this.sessions, + instanceStore: null, // no instanceStore + hosts: this._hosts, + httpsServer: this.httpsServer, + }); + } + + public addLoginRoutes() { + if (this._check('login', 'org', 'sessions')) { return; } + // TODO: We do NOT want Comm here at all, it's only being used for handling sessions, which + // should be factored out of it. + this.addComm(); + + async function redirectToLoginOrSignup( + this: FlexServer, signUp: boolean|null, req: express.Request, resp: express.Response, + ) { + const mreq = req as RequestWithLogin; + mreq.session.alive = true; // This will ensure that express-session will set our cookie + // if it hasn't already - we'll need it when we come back + // from Cognito. + // Redirect to "/" on our requested hostname (in test env, this will redirect further) + const next = req.protocol + '://' + req.get('host') + '/'; + if (signUp === null) { + // Like redirectToLogin in Authorizer, redirect to sign up if it doesn't look like the + // user has ever logged in on this browser. + signUp = (mreq.session.users === undefined); + } + const getRedirectUrl = signUp ? this._getSignUpRedirectUrl : this._getLoginRedirectUrl; + resp.redirect(await getRedirectUrl(new URL(next))); + } + + this.app.get('/login', expressWrap(redirectToLoginOrSignup.bind(this, false))); + this.app.get('/signup', expressWrap(redirectToLoginOrSignup.bind(this, true))); + this.app.get('/signin', expressWrap(redirectToLoginOrSignup.bind(this, null))); + + if (allowTestLogin()) { + // This is an endpoint for the dev environment that lets you log in as anyone. + // For a standard dev environment, it will be accessible at localhost:8080/test/login + // and localhost:8080/o//test/login. Only available when GRIST_TEST_LOGIN is set. + // Handy when without network connectivity to reach Cognito. + + log.warn("Adding a /test/login endpoint because GRIST_TEST_LOGIN is set. " + + "Users will be able to login as anyone."); + + this.app.get('/test/login', expressWrap(async (req, res) => { + log.warn("Serving unauthenticated /test/login endpoint, made available because GRIST_TEST_LOGIN is set."); + + const session = this.sessions.getOrCreateSessionFromRequest(req); + const profile: UserProfile = { + email: optStringParam(req.query.email) || 'chimpy@getgrist.com', + name: optStringParam(req.query.name) || 'Chimpy McBanana', + }; + await session.scopedSession.operateOnScopedSession(async user => { + user.profile = profile; + return user; + }); + res.send(` + +

Logged in as ${JSON.stringify(profile)}.

+

+ + + +
+ + `); + })); + } + + this.app.get('/logout', expressWrap(async (req, resp) => { + const session = this.sessions.getOrCreateSessionFromRequest(req); + const userSession = await session.scopedSession.getScopedSession(); + + // If 'next' param is missing, redirect to "/" on our requested hostname. + const next = optStringParam(req.query.next) || (req.protocol + '://' + req.get('host') + '/'); + const redirectUrl = await this._getLogoutRedirectUrl(new URL(next), userSession); + + // Clear session so that user needs to log in again at the next request. + // SAML logout in theory uses userSession, so clear it AFTER we compute the URL. + // Express-session will save these changes. + const expressSession = (req as any).session; + if (expressSession) { expressSession.users = []; expressSession.orgToUser = {}; } + if (session.loginSession) { + await session.loginSession.clearSession(); + } + resp.redirect(redirectUrl); + })); + + // Add a static "signed-out" page. This is where logout typically lands (after redirecting + // through Cognito or SAML). + this.app.get('/signed-out', expressWrap((req, resp) => + this._sendAppPage(req, resp, {path: 'error.html', status: 200, config: {errPage: 'signed-out'}}))); + + // Add a static "verified" page. This is where an email verification link will land, + // on success. TODO: rename simple static pages from "error" to something more generic. + this.app.get('/verified', expressWrap((req, resp) => + this._sendAppPage(req, resp, {path: 'error.html', status: 200, config: {errPage: 'verified'}}))); + + const comment = this._loginMiddleware.addEndpoints(this.app, this.comm, this.sessions, this._hosts); + this.info.push(['loginMiddlewareComment', comment]); + } + + public async addTestingHooks(workerServers?: FlexServer[]) { + if (process.env.GRIST_TESTING_SOCKET) { + await startTestingHooks(process.env.GRIST_TESTING_SOCKET, this.port, null!, this.comm, this, + workerServers || []); + this._hasTestingHooks = true; + } + } + + // Returns a Map from docId to number of connected clients for each doc. + public async getDocClientCounts(): Promise> { + return this._docManager ? this._docManager.getDocClientCounts() : new Map(); + } + + // allow the document manager to be specified externally, for convenience in testing. + public testSetDocManager(docManager: DocManager) { + this._docManager = docManager; + } + + // Add document-related endpoints and related support. A testAvoidMetrics flag + // is added to disable touching metrics if tests have taken care of stubbing them. + public async addDoc(testAvoidMetrics: boolean = false) { + this._check('doc', 'start', 'tag', 'json', isSingleUserMode() ? null : 'homedb', 'api-mw', 'map'); + // add handlers for cleanup, if we are in charge of the doc manager. + if (!this._docManager) { this.addCleanup(); } + await this.loadConfig(); + this.addComm(); + + // TODO: metrics collection might be best left to a separate service, but we need an + // instantiated object because various code attempts to report metrics. + if (!testAvoidMetrics) { + this._serverMetrics = new ServerMetrics(); + this._serverMetrics.handlePreferences(this.settings); + } + + if (!isSingleUserMode()) { + const s3Bucket = this._disableS3 ? '' : (process.env.GRIST_DOCS_S3_BUCKET || ''); + const s3Prefix = process.env.GRIST_DOCS_S3_PREFIX || "docs/"; + + this.info.push(['s3Bucket', `${s3Bucket}/${s3Prefix}`]); + + const workers = this._docWorkerMap; + const docWorkerId = await this._addSelfAsWorker(workers); + + const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, s3Bucket, s3Prefix, workers, + this.dbManager, this.create); + this._storageManager = storageManager; + } else { + const samples = getAppPathTo(this.appRoot, 'public_samples'); + const storageManager = new DocStorageManager(this.docsRoot, samples, this.comm, this); + this._storageManager = storageManager; + } + + const pluginManager = await this._addPluginManager(); + this._docManager = this._docManager || new DocManager(this._storageManager, pluginManager, + this.dbManager, this); + const docManager = this._docManager; + + shutdown.addCleanupHandler(null, this._shutdown.bind(this), 25000, 'FlexServer._shutdown'); + + if (isSingleUserMode()) { + // Load standalone stuff only if needed. Hosted grist doesn't currently need basket, + // sharing, etc. + const extras = await import('app/server/lib/StandaloneExtras'); + this.electronServerMethods = extras.addStandaloneMethods({ + flexServer: this, + docManager: this._docManager, + storageManager: this._storageManager, + }); + } else { + this.comm.registerMethods({ + openDoc: docManager.openDoc.bind(docManager), + }); + this._serveDocPage(); + } + + // Attach docWorker endpoints and Comm methods. + const docWorker = new DocWorker(this.dbManager, {comm: this.comm}); + this._docWorker = docWorker; + + // Register the websocket comm functions associated with the docworker. + docWorker.registerCommCore(); + docWorker.registerCommPlugin(); + + // Doc-specific endpoints require authorization; collect the relevant middleware in one list. + const docAccessMiddleware = [ + this._userIdMiddleware, + this._docPermissionsMiddleware, + this.tagChecker.requireTag + ]; + + this.addSupportPaths(docAccessMiddleware); + + if (!isSingleUserMode()) { + addDocApiRoutes(this.app, docWorker, docManager, this.dbManager, this); + } + } + + // Must be called *after* metrics have been created. + public disableMetrics() { + if (!this._serverMetrics) { + throw new Error('disableMetrics called too early'); + } + this._serverMetrics.disable(); + } + + public disableS3() { + if (this.deps.has('doc')) { + throw new Error('disableS3 called too late'); + } + this._disableS3 = true; + } + + public addBillingPages() { + const middleware = [ + this._redirectToHostMiddleware, + this._userIdMiddleware, + this._redirectToLoginWithoutExceptionsMiddleware + ]; + + this.app.get('/billing', ...middleware, expressWrap(async (req, resp, next) => { + const mreq = req as RequestWithLogin; + const orgDomain = mreq.org; + if (!orgDomain) { + return this._sendAppPage(req, resp, {path: 'error.html', status: 404, config: {errPage: 'not-found'}}); + } + const query = await this.dbManager.getOrg({userId: mreq.userId!}, orgDomain); + const org = this.dbManager.unwrapQueryResult(query); + // This page isn't availabe for personal site. + if (org.owner) { + return this._sendAppPage(req, resp, {path: 'error.html', status: 404, config: {errPage: 'not-found'}}); + } + return this._sendAppPage(req, resp, {path: 'billing.html', status: 200, config: {}}); + })); + + this.app.get('/billing/payment', ...middleware, expressWrap(async (req, resp, next) => { + const task = optStringParam(req.query.billingTask) || ''; + const planRequired = task === 'signup' || task === 'updatePlan'; + if (!BillingTask.guard(task) || (planRequired && !req.query.billingPlan)) { + // If the payment task/plan are invalid, redirect to the summary page. + return resp.redirect(req.protocol + '://' + req.get('host') + `/billing`); + } else { + return this._sendAppPage(req, resp, {path: 'billing.html', status: 200, config: {}}); + } + })); + + // This endpoint is used only during testing, to support existing tests that + // depend on a page that has been removed. + this.app.get('/test/support/billing/plans', expressWrap(async (req, resp, next) => { + return this._sendAppPage(req, resp, {path: 'billing.html', status: 200, config: {}}); + })); + } + + /** + * Add billing webhooks. Strip signatures sign the raw body of the message, so + * we need to get these webhooks in before the bodyParser is added to parse json. + */ + public addEarlyWebhooks() { + if (this._check('webhooks', 'homedb')) { return; } + if (this.deps.has('json')) { + throw new Error('addEarlyWebhooks called too late'); + } + this._getBilling(); + this._billing.addWebhooks(this.app); + } + + public addWelcomePaths() { + const middleware = [ + bodyParser.urlencoded({ extended: true }), + this._redirectToHostMiddleware, + this._userIdMiddleware, + this._redirectToLoginWithoutExceptionsMiddleware, + ]; + + this.app.get('/welcome/user', ...middleware, expressWrap(async (req, resp, next) => { + return this._sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}, googleTagManager: true}); + })); + + this.app.post('/welcome/user', ...middleware, expressWrap(async (req, resp, next) => { + const mreq = req as RequestWithLogin; + const userId = getUserId(req); + const domain = mreq.org; + const result = await this.dbManager.getMergedOrgs(userId, userId, domain || null); + const orgs = (result.status === 200) ? result.data : null; + + const name: string|undefined = req.body && req.body.username || undefined; + await this.dbManager.updateUser(userId, {name, isFirstTimeUser: false}); + + // redirect to teams page if users has access to more than one org. Otherwise redirect to + // personal org. + const pathname = orgs && orgs.length > 1 ? '/welcome/teams' : '/'; + const mergedOrgDomain = this.dbManager.mergedOrgDomain(); + const redirectUrl = this._getOrgRedirectUrl(mreq, mergedOrgDomain, pathname); + resp.redirect(redirectUrl); + })); + + this.app.get('/welcome/teams', ...middleware, expressWrap(async (req, resp, next) => { + return this._sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}}); + })); + } + + public finalize() { + this.addApiErrorHandlers(); + + // add a final non-found handler for other content. + this.app.use("/", expressWrap((req, resp) => { + if (this._sendAppPage) { + return this._sendAppPage(req, resp, {path: 'error.html', status: 404, config: {errPage: 'not-found'}}); + } else { + return resp.status(404).json({error: 'not found'}); + } + })); + + // add a final error handler + this.app.use(async (err: any, req: express.Request, resp: express.Response, next: express.NextFunction) => { + // Delegate to default error handler when headers have already been sent, as express advises + // at https://expressjs.com/en/guide/error-handling.html#the-default-error-handler. + // Also delegates if no _sendAppPage method has been configured. + if (resp.headersSent || !this._sendAppPage) { return next(err); } + try { + const errPage = ( + err.status === 403 ? 'access-denied' : + err.status === 404 ? 'not-found' : + 'other-error' + ); + const config = {errPage, errMessage: err.message || err}; + await this._sendAppPage(req, resp, {path: 'error.html', status: err.status || 400, config}); + } catch (err) { + return next(err); + } + }); + } + + public summary() { + for (const [label, value] of this.info) { + log.info("== %s: %s", label, value); + } + } + + public async start() { + if (this._check('start')) { return; } + + const servers = this._createServers(); + this.server = servers.server; + this.httpsServer = servers.httpsServer; + await this._startServers(this.server, this.httpsServer, this.name, this.port, true); + } + + public addNotifier() { + if (this._check('notifier', 'start', 'homedb')) { return; } + // TODO: Disable notifications for Nioxus orgs, until they are ready to deal with them. + // TODO: make Notifier aware of base domains, rather than sending emails with default + // base domain. + // Most notifications are ultimately triggered by requests with a base domain in them, + // and all that is needed is a refactor to pass that info along. But there is also the + // case of notification(s) from stripe. May need to associate a preferred base domain + // with org/user and persist that? + this.notifier = this.create.Notifier(this.dbManager, this.getDefaultHomeUrl()); + } + + public addUsage() { + if (this._check('usage', 'start', 'homedb')) { return; } + this.usage = new Usage(this.dbManager); + } + + public async addHousekeeper() { + if (this._check('housekeeper', 'start', 'homedb', 'map')) { return; } + const store = this._docWorkerMap; + this.housekeeper = new Housekeeper(this.dbManager, this, store, store); + await this.housekeeper.start(); + } + + public async startCopy(name2: string, port2: number) { + const servers = this._createServers(); + await this._startServers(servers.server, servers.httpsServer, name2, port2, true); + } + + /** + * Close all documents currently held open. + */ + public async closeDocs(): Promise { + if (this._docManager) { + return this._docManager.shutdownAll(); + } + } + + // Adds endpoints that support imports and exports. + private addSupportPaths(docAccessMiddleware: express.RequestHandler[]) { + if (!this._docWorker) { throw new Error("need DocWorker"); } + + // TODO: Need a way to translate docName from query to filesystem path. Use OpenDocManager. + this.app.get('/download', ...docAccessMiddleware, expressWrap(async (req, res) => { + return this._docWorker.downloadDoc(req, res, this._storageManager); + })); + + const basicMiddleware = [this._userIdMiddleware, this.tagChecker.requireTag]; + + // Add the handling for the /upload route. Most uploads are meant for a DocWorker: they are put + // in temporary files, and the DocWorker needs to be on the same machine to have access to them. + // This doesn't check for doc access permissions because the request isn't tied to a document. + addUploadRoute(this, this.app, ...basicMiddleware); + + this.app.get('/gen_csv', ...docAccessMiddleware, (req, res) => this._docWorker.getCSV(req, res)); + + this.app.get('/attachment', ...docAccessMiddleware, + expressWrap(async (req, res) => this._docWorker.getAttachment(req, res))); + } + + private _check(part: string, ...precedents: Array) { + if (this.deps.has(part)) { return true; } + for (const precedent of precedents) { + if (!precedent) { continue; } + if (precedent[0] === '!') { + const antecedent = precedent.slice(1); + if (this._has(antecedent)) { + throw new Error(`${part} is needed before ${antecedent}`); + } + } else if (!this._has(precedent)) { + throw new Error(`${precedent} is needed before ${part}`); + } + } + this.deps.add(part); + return false; + } + + private _has(part: string) { + return this.deps.has(part); + } + + private async _addSelfAsWorker(workers: IDocWorkerMap): Promise { + try { + this._healthy = true; + // Check if this is the first time calling this method. In production, + // it always will be. In testing, we may disconnect and reconnect the + // worker. We only need to determine docWorkerId and this.worker once. + if (!this.worker) { + + if (process.env.GRIST_ROUTER_URL) { + // register ourselves with the load balancer first. + const w = await this.createWorkerUrl(); + const url = `${w.url}/v/${this.tag}/`; + // TODO: we could compute a distinct internal url here. + this.worker = { + id: w.host, + publicUrl: url, + internalUrl: url, + }; + } else { + const url = (process.env.APP_DOC_URL || this.getOwnUrl()) + `/v/${this.tag}/`; + this.worker = { + // The worker id should be unique to this worker. + id: process.env.GRIST_DOC_WORKER_ID || `testDocWorkerId_${this.port}`, + publicUrl: url, + internalUrl: process.env.APP_DOC_INTERNAL_URL || url, + }; + } + this.info.push(['docWorkerId', this.worker.id]); + + } else { + if (process.env.GRIST_ROUTER_URL) { + await this.createWorkerUrl(); + } + } + await workers.addWorker(this.worker); + await workers.setWorkerAvailability(this.worker.id, true); + } catch (err) { + this._healthy = false; + throw err; + } + return this.worker.id; + } + + private async _removeSelfAsWorker(workers: IDocWorkerMap, docWorkerId: string) { + this._healthy = false; + await workers.removeWorker(docWorkerId); + if (process.env.GRIST_ROUTER_URL) { + await axios.get(process.env.GRIST_ROUTER_URL, + {params: {act: 'remove', port: this.getOwnPort()}}); + log.info(`DocWorker unregistered itself via ${process.env.GRIST_ROUTER_URL}`); + } + } + + // Called when server is shutting down. Save any state that needs saving, and + // disentangle ourselves from outside world. + private async _shutdown(): Promise { + if (!this.worker) { return; } + if (!this._storageManager) { return; } + if (!this._docWorkerMap) { return; } // but this should never happen. + + const workers = this._docWorkerMap; + + // Pick up the pace on saving documents. + this._storageManager.prepareToCloseStorage(); + + // We urgently want to disable any new assignments. + await workers.setWorkerAvailability(this.worker.id, false); + + // Enumerate the documents we are responsible for. + let assignments = await workers.getAssignments(this.worker.id); + let retries: number = 0; + while (assignments.length > 0 && retries < 3) { + await Promise.all(assignments.map(async assignment => { + log.info("FlexServer shutdown assignment", assignment); + try { + // Start sending the doc to S3 if needed. + const flushOp = this._storageManager.closeDocument(assignment); + + // Get access to the clients of this document. This has the side + // effect of waiting for the ActiveDoc to finish initialization. + // This could include loading it from S3, an operation we could + // potentially abort as an optimization. + // TODO: abort any s3 loading as an optimization. + const docPromise = this._docManager.getActiveDoc(assignment); + const doc = docPromise && await docPromise; + + await flushOp; + // At this instant, S3 and local document should be the same. + + // We'd now like to make sure (synchronously) that: + // - we never output anything new to S3 about this document. + // - we never output anything new to user about this document. + // There could be asynchronous operations going on related to + // these documents, but if we can make sure that their effects + // do not reach the outside world then we can ignore them. + if (doc) { + doc.docClients.interruptAllClients(); + doc.setMuted(); + } + + // Release this document for other workers to pick up. + // There is a small window of time here in which a client + // could reconnect to us. The muted ActiveDoc will result + // in them being dropped again. + await workers.releaseAssignment(this.worker.id, assignment); + } catch (err) { + log.info("problem dealing with assignment", assignment, err); + } + })); + // Check for any assignments that slipped through at the last minute. + assignments = await workers.getAssignments(this.worker.id); + retries++; + } + if (assignments.length > 0) { + log.error("FlexServer shutdown failed to release assignments:", assignments); + } + + await this._removeSelfAsWorker(workers, this.worker.id); + try { + await this._docManager.shutdownAll(); + } catch (err) { + log.error("FlexServer shutdown problem", err); + } + if (this.comm) { + this.comm.destroyAllClients(); + } + log.info("FlexServer shutdown is complete"); + } + + /** + * Middleware that redirects a request with a userId but without an org to an org-specific URL, + * after looking up the first org for this userId in DB. + */ + private async _redirectToOrg(req: express.Request, resp: express.Response, next: express.NextFunction) { + const mreq = req as RequestWithLogin; + if (mreq.org || !mreq.userId || !mreq.userIsAuthorized) { return next(); } + + // We have a userId, but the request is for an unknown org. Redirect to an org that's + // available to the user. This matters in dev, and in prod when visiting a generic URL, which + // will here redirect to e.g. the user's personal org. + const result = await this.dbManager.getMergedOrgs(mreq.userId, mreq.userId, null); + const orgs = (result.status === 200) ? result.data : null; + const subdomain = orgs && orgs.length > 0 ? orgs[0].domain : null; + const redirectUrl = subdomain && this._getOrgRedirectUrl(mreq, subdomain); + if (redirectUrl) { + log.debug(`Redirecting userId ${mreq.userId} to: ${redirectUrl}`); + return resp.redirect(redirectUrl); + } + next(); + } + + /** + * Given a Request and a desired subdomain, returns a URL for a similar request that specifies that + * subdomain either in the hostname or in the path. Optionally passing pathname overrides url's + * path. + */ + private _getOrgRedirectUrl(req: RequestWithLogin, subdomain: string, pathname: string = req.originalUrl): string { + const {hostname, orgInPath} = getOrgUrlInfo(subdomain, req.hostname, { + org: req.org, + baseDomain: this._defaultBaseDomain, + singleOrg: process.env.GRIST_SINGLE_ORG, + }); + const redirectUrl = new URL(pathname, `${req.protocol}://${req.get('host')}`); + if (hostname) { + redirectUrl.hostname = hostname; + } + if (orgInPath) { + redirectUrl.pathname = `/o/${orgInPath}` + redirectUrl.pathname; + } + return redirectUrl.href; + } + + + // Create and initialize the plugin manager + private async _addPluginManager() { + if (this._pluginManager) { return this._pluginManager; } + // Only used as {userRoot}/plugins as a place for plugins in addition to {appRoot}/plugins + const userRoot = path.resolve(process.env.GRIST_USER_ROOT || getAppPathTo(this.appRoot, '.grist')); + this.info.push(['userRoot', userRoot]); + + const pluginManager = new PluginManager(this.appRoot, userRoot); + // `initialize()` is asynchronous and reads plugins manifests; if PluginManager is used before it + // finishes, it will act as if there are no plugins. + // ^ I think this comment was here to justify calling initialize without waiting for + // the result. I'm just going to wait, for determinism. + await pluginManager.initialize(); + this._pluginManager = pluginManager; + return pluginManager; + } + + // Serve the static app.html proxied for a document. + private _serveDocPage() { + // Serve the static app.html file. + // TODO: We should be the ones to fill in the base href here to ensure that the browser fetches + // the correct version of static files for this app.html. + this.app.get('/:docId/app.html', this._userIdMiddleware, expressWrap(async (req, res) => { + const page = await fse.readFile(path.join(getAppPathTo(this.appRoot, 'static'), + 'app.html'), 'utf8'); + res.json({ + page, + tag: this.tag + }); + })); + } + + private _getBilling(): IBilling { + if (!this._billing) { + if (!this.dbManager) { throw new Error("need dbManager"); } + this._billing = this.create.Billing(this.dbManager); + } + return this._billing; + } + + // Check whether logger should skip a line. Careful, req and res are morgan-specific + // types, not Express. + private _shouldSkipRequestLogging(req: {url: string}, res: {statusCode: number}) { + if (req.url === '/status' && [200, 304].includes(res.statusCode) && + this._healthCheckCounter > HEALTH_CHECK_LOG_SHOW_FIRST_N && + this._healthCheckCounter % HEALTH_CHECK_LOG_SHOW_EVERY_N !== 1) { + return true; + } + return false; + } + + private _createServers() { + // Start the app. + const server = configServer(http.createServer(this.app)); + let httpsServer; + if (TEST_HTTPS_OFFSET) { + const certFile = process.env.GRIST_TEST_SSL_CERT; + const privateKeyFile = process.env.GRIST_TEST_SSL_KEY; + if (!certFile) { throw new Error('Set GRIST_TEST_SSL_CERT to location of certificate file'); } + if (!privateKeyFile) { throw new Error('Set GRIST_TEST_SSL_KEY to location of private key file'); } + log.debug(`https support: reading cert from ${certFile}`); + log.debug(`https support: reading private key from ${privateKeyFile}`); + httpsServer = configServer(https.createServer({ + key: fse.readFileSync(privateKeyFile, 'utf8'), + cert: fse.readFileSync(certFile, 'utf8'), + }, this.app)); + } + return {server, httpsServer}; + } + + private async _startServers(server: http.Server, httpsServer: https.Server|undefined, + name: string, port: number, verbose: boolean) { + await new Promise((resolve, reject) => server.listen(port, this.host, resolve).on('error', reject)); + if (verbose) { log.info(`${name} available at ${this.host}:${port}`); } + if (TEST_HTTPS_OFFSET && httpsServer) { + const httpsPort = port + TEST_HTTPS_OFFSET; + await new Promise((resolve, reject) => { + httpsServer.listen(httpsPort, this.host, resolve) + .on('error', reject); + }); + if (verbose) { log.info(`${name} available at https://${this.host}:${httpsPort}`); } + } + } +} + +/** + * Returns the passed-in server, with some options adjusted. Specifically, removes the default + * socket timeout. + */ +function configServer(server: T): T { + // Remove the socket timeout, which causes node to close socket for long-running requests + // (like imports), triggering browser retry. (The default is 2 min; removed starting node v13.) + // See also https://nodejs.org/docs/latest-v10.x/api/http.html#http_server_settimeout_msecs_callback.) + server.setTimeout(0); + + // The server's keepAlive timeout should be longer than the load-balancer's. Otherwise LB will + // produce occasional 502 errors when it sends a request to node just as node closes a + // connection. See https://adamcrowder.net/posts/node-express-api-and-aws-alb-502/. + const lbTimeoutSec = 300; + + // Ensure all inactive connections are terminated by the ALB, by setting this a few seconds + // higher than the ALB idle timeout + server.keepAliveTimeout = (lbTimeoutSec + 5) * 1000; + + // Ensure the headersTimeout is set higher than the keepAliveTimeout due to this nodejs + // regression bug: https://github.com/nodejs/node/issues/27363 + server.headersTimeout = (lbTimeoutSec + 6) * 1000; + + log.info("Server timeouts: keepAliveTimeout %s headersTimeout %s", + server.keepAliveTimeout, server.headersTimeout); + + return server; +} + +// Returns true if environment is configured to allow unauthenticated test logins. +function allowTestLogin() { + return Boolean(process.env.GRIST_TEST_LOGIN); +} + +// Check OPTIONS requests for allowed origins, and return heads to allow the browser to proceed +// with a POST (or other method) request. +function trustOriginHandler(req: express.Request, res: express.Response, next: express.NextFunction) { + if (trustOrigin(req, res)) { + res.header("Access-Control-Allow-Credentials", "true"); + res.header("Access-Control-Allow-Methods", "GET, PATCH, POST, DELETE, OPTIONS"); + res.header("Access-Control-Allow-Headers", "Authorization, Content-Type"); + } + if ('OPTIONS' === req.method) { + res.sendStatus(200); + } else { + next(); + } +} + +// Set Cache-Control header to "no-cache" +function noCaching(req: express.Request, res: express.Response, next: express.NextFunction) { + res.header("Cache-Control", "no-cache"); + next(); +} + +// Methods that Electron app relies on. +export interface ElectronServerMethods { + importDoc(filepath: string): Promise; + onDocOpen(cb: () => void): void; + getUserConfig(): Promise; + updateUserConfig(obj: any): Promise; + onBackupMade(cb: () => void): void; +} diff --git a/app/server/lib/GristServer.ts b/app/server/lib/GristServer.ts new file mode 100644 index 00000000..a022369c --- /dev/null +++ b/app/server/lib/GristServer.ts @@ -0,0 +1,26 @@ +import {SessionUserObj} from 'app/server/lib/BrowserSession'; +import * as Comm from 'app/server/lib/Comm'; +import {Hosts} from 'app/server/lib/extractOrg'; +import {ICreate} from 'app/server/lib/ICreate'; +import {Sessions} from 'app/server/lib/Sessions'; +import * as express from 'express'; + +/** + * Basic information about a Grist server. Accessible in many + * contexts, including request handlers and ActiveDoc methods. + */ +export interface GristServer { + readonly create: ICreate; + getHost(): string; + getHomeUrl(req: express.Request, relPath?: string): string; + getHomeUrlByDocId(docId: string, relPath?: string): Promise; +} + +export interface GristLoginMiddleware { + getLoginRedirectUrl(target: URL): Promise; + getSignUpRedirectUrl(target: URL): Promise; + getLogoutRedirectUrl(nextUrl: URL, userSession: SessionUserObj): Promise; + + // Returns arbitrary string for log. + addEndpoints(app: express.Express, comm: Comm, sessions: Sessions, hosts: Hosts): string; +} diff --git a/app/server/lib/HostedMetadataManager.ts b/app/server/lib/HostedMetadataManager.ts new file mode 100644 index 00000000..c0f77b46 --- /dev/null +++ b/app/server/lib/HostedMetadataManager.ts @@ -0,0 +1,89 @@ +import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import * as log from 'app/server/lib/log'; + +/** + * HostedMetadataManager handles pushing document metadata changes to the Home database when + * a doc is updated. Currently only updates doc updatedAt time. + */ +export class HostedMetadataManager { + + // updatedAt times as UTC ISO strings mapped by docId. + private _updatedAt: {[docId: string]: string} = {}; + + // Set if the class holder is closing and no further pushes should be scheduled. + private _closing: boolean = false; + + // Last push time in ms since epoch. + private _lastPushTime: number = 0.0; + + // Callback for next opportunity to push changes. + private _timeout: any = null; + + // Mantains the update Promise to wait on it if the class is closing. + private _push: Promise|null; + + /** + * Create an instance of HostedMetadataManager. + * The minPushDelay is the delay in seconds between metadata pushes to the database. + */ + constructor(private _dbManager: HomeDBManager, private _minPushDelay: number = 60) {} + + /** + * Close the manager. Send out any pending updates and prevent more from being scheduled. + */ + public async close(): Promise { + // Finish up everything outgoing + this._closing = true; // Pushes will no longer be scheduled. + if (this._timeout) { + clearTimeout(this._timeout); + this._timeout = null; + // Since an update was scheduled, perform one final update now. + this._update(); + } + if (this._push) { await this._push; } + } + + /** + * Schedule a call to _update some time from now. + */ + public scheduleUpdate(docId: string): void { + // Update updatedAt even if an update is already scheduled - if the update has not yet occurred, + // the more recent updatedAt time will be used. + this._updatedAt[docId] = new Date().toISOString(); + if (this._timeout || this._closing) { return; } + const minDelay = this._minPushDelay * 1000; + // Set the push to occur at least the minDelay after the last push time. + const delay = Math.round(minDelay - (Date.now() - this._lastPushTime)); + this._timeout = setTimeout(() => this._update(), delay < 0 ? 0 : delay); + } + + public setDocsUpdatedAt(docUpdateMap: {[docId: string]: string}): Promise { + return this._dbManager.setDocsUpdatedAt(docUpdateMap); + } + + /** + * Push all metadata updates to the databse. + */ + private _update(): void { + if (this._push) { return; } + if (this._timeout) { + clearTimeout(this._timeout); + this._timeout = null; + } + this._push = this._performUpdate() + .catch(err => { log.error("HostedMetadataManager error performing update: ", err); }) + .then(() => { this._push = null; }); + } + + /** + * This is called by the update function to actually perform the update. This should not + * be called unless to force an immediate update. + */ + private async _performUpdate(): Promise { + // Await the database if it is not yet connected. + const docUpdates = this._updatedAt; + this._updatedAt = {}; + this._lastPushTime = Date.now(); + await this.setDocsUpdatedAt(docUpdates); + } +} diff --git a/app/server/lib/HostedStorageManager.ts b/app/server/lib/HostedStorageManager.ts new file mode 100644 index 00000000..8ca77593 --- /dev/null +++ b/app/server/lib/HostedStorageManager.ts @@ -0,0 +1,713 @@ +import * as sqlite3 from '@gristlabs/sqlite3'; +import {mapGetOrSet} from 'app/common/AsyncCreate'; +import {delay} from 'app/common/delay'; +import {DocEntry} from 'app/common/DocListAPI'; +import {buildUrlId, parseUrlId} from 'app/common/gristUrls'; +import {KeyedOps} from 'app/common/KeyedOps'; +import {DocReplacementOptions, NEW_DOCUMENT_CODE} from 'app/common/UserAPI'; +import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import {getUserId} from 'app/server/lib/Authorizer'; +import {checksumFile} from 'app/server/lib/checksumFile'; +import {OptDocSession} from 'app/server/lib/DocSession'; +import {DocSnapshotPruner, DocSnapshots} from 'app/server/lib/DocSnapshots'; +import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; +import {ChecksummedExternalStorage, DELETED_TOKEN, ExternalStorage} from 'app/server/lib/ExternalStorage'; +import {HostedMetadataManager} from 'app/server/lib/HostedMetadataManager'; +import {ICreate} from 'app/server/lib/ICreate'; +import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; +import * as log from 'app/server/lib/log'; +import {fromCallback} from 'app/server/lib/serverUtils'; +import * as fse from 'fs-extra'; +import * as path from 'path'; +import * as uuidv4 from "uuid/v4"; + +// Check for a valid document id. +const docIdRegex = /^[-=_\w~%]+$/; + +// Wait this long after a change to the document before trying to make a backup of it. +const GRIST_BACKUP_DELAY_SECS = parseInt(process.env.GRIST_BACKUP_DELAY_SECS || '15', 10); + +// This constant controls how many pages of the database we back up in a single step. +// The larger it is, the faster the backup overall, but the slower each step is. +// Slower steps result in longer periods when the database is locked, without any +// opportunity for a waiting client to get in and make a write. +// The size of a page, as far as sqlite is concerned, is 4096 bytes. +const PAGES_TO_BACKUP_PER_STEP = 1024; // Backup is made in 4MB chunks. + +// Between steps of the backup, we pause in case a client is waiting to make a write. +// The shorter the pause, the greater the odds that the client won't be able to make +// its write, but the faster the backup will complete. +const PAUSE_BETWEEN_BACKUP_STEPS_IN_MS = 10; + +function checkValidDocId(docId: string): void { + if (!docIdRegex.test(docId)) { + throw new Error(`Invalid docId ${docId}`); + } +} + +interface HostedStorageOptions { + secondsBeforePush: number; + secondsBeforeFirstRetry: number; + pushDocUpdateTimes: boolean; + testExternalStorage?: ExternalStorage; +} + +const defaultOptions: HostedStorageOptions = { + secondsBeforePush: GRIST_BACKUP_DELAY_SECS, + secondsBeforeFirstRetry: 3.0, + pushDocUpdateTimes: true +}; + +/** + * HostedStorageManager manages Grist files in the hosted environment for a particular DocWorker. + * These files are stored on S3 and synced to the local file system. It matches the interface of + * DocStorageManager (used for standalone Grist), but is more limited, e.g. does not expose the + * list of local files. + * + * In hosted environment, documents are uniquely identified by docId, which serves as the + * canonical docName. This ID does not change on renaming. HostedStorageManager knows nothing of + * friendlier doc names. + * + * TODO: Listen (to Redis?) to find out when a local document has been deleted or renamed. + * (In case of rename, something on the DocWorker needs to inform the client about the rename) + * TODO: Do something about the active flag in redis DocStatus. + * TODO: Add an explicit createFlag in DocStatus for clarity and simplification. + */ +export class HostedStorageManager implements IDocStorageManager { + + // Handles pushing doc metadata changes when the doc is updated. + private _metadataManager: HostedMetadataManager|null = null; + + // Maps docId to the promise for when the document is present on the local filesystem. + private _localFiles = new Map>(); + + // Access external storage. + private _ext: ChecksummedExternalStorage; + + // Prune external storage. + private _pruner: DocSnapshotPruner; + + // If _disableS3 is set, don't actually communicate with S3 - keep everything local. + private _disableS3 = (process.env.GRIST_DISABLE_S3 === 'true'); + + // A set of filenames currently being created or downloaded. + private _prepareFiles = new Set(); + + // Ongoing and scheduled uploads for documents. + private _uploads: KeyedOps; + + // Set once the manager has been closed. + private _closed: boolean = false; + + private _baseStore: ExternalStorage; // External store for documents, without checksumming. + + /** + * Initialize with the given root directory, which should be a fully-resolved path. + * If s3Bucket is blank, S3 storage will be disabled. + */ + constructor( + private _docsRoot: string, + private _docWorkerId: string, + s3Bucket: string, + s3Prefix: string, // Should end in / if non-empty. + private _docWorkerMap: IDocWorkerMap, + dbManager: HomeDBManager, + create: ICreate, + options: HostedStorageOptions = defaultOptions + ) { + if (s3Bucket === '') { this._disableS3 = true; } + // We store documents either in a test store, or in an s3 store + // at s3:///.grist + const externalStore = options.testExternalStorage || + (this._disableS3 ? undefined : create.ExternalStorage(s3Bucket, s3Prefix)); + if (!externalStore) { this._disableS3 = true; } + const secondsBeforePush = options.secondsBeforePush; + const secondsBeforeFirstRetry = options.secondsBeforeFirstRetry; + if (options.pushDocUpdateTimes) { + this._metadataManager = new HostedMetadataManager(dbManager); + } + this._uploads = new KeyedOps(key => this._pushToS3(key), { + delayBeforeOperationMs: secondsBeforePush * 1000, + retry: true, + logError: (key, failureCount, err) => { + log.error("HostedStorageManager: error pushing %s (%d): %s", key, failureCount, err); + } + }); + + if (!this._disableS3) { + this._baseStore = externalStore!; + // Whichever store we have, we use checksums to deal with + // eventual consistency. + const versions = new Map(); + this._ext = new ChecksummedExternalStorage(this._baseStore, { + maxRetries: 4, + initialDelayMs: secondsBeforeFirstRetry * 1000, + computeFileHash: this._getHash.bind(this), + sharedHash: { + save: async (key, checksum) => { + await this._docWorkerMap.updateDocStatus(key, checksum); + }, + load: async (key) => { + const docStatus = await this._docWorkerMap.getDocWorker(key); + return docStatus && docStatus.docMD5 || null; + } + }, + localHash: { + save: async (key, checksum) => { + const fname = this._getHashFile(this.getPath(key)); + await fse.writeFile(fname, checksum); + }, + load: async (key) => { + const fname = this._getHashFile(this.getPath(key)); + if (!await fse.pathExists(fname)) { return null; } + return await fse.readFile(fname, 'utf8'); + } + }, + latestVersion: { + save: async (key, ver) => { + versions.set(key, ver); + }, + load: async (key) => versions.get(key) || null + } + }); + // The pruner could use an inconsistent store without any real loss overall, + // but tests are easier if it is consistent. + this._pruner = new DocSnapshotPruner(this._ext, { + delayBeforeOperationMs: 0, // prune as soon as we've made a first upload. + minDelayBetweenOperationsMs: secondsBeforePush * 4000, // ... but wait awhile before + // pruning again. + }); + } + } + + /** + * Send a document to S3, without doing anything fancy. Assumes this is the first time + * the object is written in S3 - so no need to worry about consistency. + */ + public async addToStorage(docId: string) { + if (this._disableS3) { return; } + await this._ext.upload(docId, this.getPath(docId)); + } + + public getPath(docName: string): string { + // docName should just be a docId; we use basename to protect against some possible hack attempts. + checkValidDocId(docName); + return path.join(this._docsRoot, `${path.basename(docName, '.grist')}.grist`); + } + + // We don't deal with sample docs + public getSampleDocPath(sampleDocName: string): string|null { return null; } + + /** + * Translates a possibly non-canonical docName to a canonical one. Returns a bare docId, + * stripping out any possible path components or .grist extension. (We don't expect these to + * ever be used, but stripping seems better than asserting.) + */ + public async getCanonicalDocName(altDocName: string): Promise { + return path.basename(altDocName, '.grist'); + } + + /** + * Prepares a document for use locally. Here we sync the doc from S3 to the local filesystem. + * Returns whether the document is new (needs to be created). + * Calling this method multiple times in parallel for the same document is treated as a sign + * of a bug. + */ + public async prepareLocalDoc(docName: string, docSession: OptDocSession): Promise { + // We could be reopening a document that is still closing down. + // Wait for that to happen. TODO: we could also try to interrupt the closing-down process. + await this.closeDocument(docName); + + if (this._prepareFiles.has(docName)) { + throw new Error(`Tried to call prepareLocalDoc('${docName}') twice in parallel`); + } + + try { + this._prepareFiles.add(docName); + const isNew = !(await this._ensureDocumentIsPresent(docName, docSession)); + return isNew; + } finally { + this._prepareFiles.delete(docName); + } + } + + // Gets a copy of the document, eg. for downloading. Returns full file path. + // Copy won't change if edits are made to the document. It is caller's responsibility + // to delete the result. + public async getCopy(docName: string): Promise { + const present = await this._ensureDocumentIsPresent(docName, {client: null}); + if (!present) { + throw new Error('cannot copy document that does not exist yet'); + } + return await this._prepareBackup(docName, uuidv4()); + } + + public async replace(docId: string, options: DocReplacementOptions): Promise { + // Make sure the current version of the document is flushed. + await this.flushDoc(docId); + + // Figure out the source s3 key to copy from. For this purpose, we need to + // remove any snapshotId embedded in the document id. + const rawSourceDocId = options.sourceDocId || docId; + const parts = parseUrlId(rawSourceDocId); + const sourceDocId = buildUrlId({...parts, snapshotId: undefined}); + const snapshotId = options.snapshotId || parts.snapshotId; + + if (sourceDocId === docId && !snapshotId) { return; } + + // Basic implementation for when S3 is not available. + if (this._disableS3) { + if (snapshotId) { + throw new Error('snapshots not supported without S3'); + } + if (await fse.pathExists(this.getPath(sourceDocId))) { + await fse.copy(this.getPath(sourceDocId), this.getPath(docId)); + return; + } else { + throw new Error(`cannot find ${docId}`); + } + } + + // While replacing, move the current version of the document aside. If a problem + // occurs, move it back. + const docPath = this.getPath(docId); + const tmpPath = `${docPath}-replacing`; + // NOTE: fse.remove succeeds also when the file does not exist. + await fse.remove(tmpPath); + if (await fse.pathExists(docPath)) { + await fse.move(docPath, tmpPath); + } + try { + // Fetch new content from S3. + if (!await this._fetchFromS3(docId, {sourceDocId, snapshotId})) { + throw new Error('Cannot fetch document'); + } + // Make sure the new content is considered new. + // NOTE: fse.remove succeeds also when the file does not exist. + await fse.remove(this._getHashFile(this.getPath(docId))); + this.markAsChanged(docId); + this.markAsEdited(docId); + } catch (err) { + log.error("HostedStorageManager: problem replacing %s: %s", docId, err); + await fse.move(tmpPath, docPath, {overwrite: true}); + throw err; + } finally { + // NOTE: fse.remove succeeds also when the file does not exist. + await fse.remove(tmpPath); + } + // Flush the document immediately if it has been changed. + await this.flushDoc(docId); + } + + // We don't deal with listing documents. + public async listDocs(): Promise { return []; } + + public async deleteDoc(docName: string, deletePermanently?: boolean): Promise { + if (!deletePermanently) { + throw new Error("HostedStorageManager only implements permanent deletion in deleteDoc"); + } + await this.closeDocument(docName); + if (!this._disableS3) { + await this._ext.remove(docName); + } + // NOTE: fse.remove succeeds also when the file does not exist. + await fse.remove(this.getPath(docName)); + await fse.remove(this._getHashFile(this.getPath(docName))); + } + + // We don't implement document renames. + public async renameDoc(oldName: string, newName: string): Promise { + throw new Error("HostedStorageManager does not implement renameDoc"); + } + + /** + * We handle backups by syncing the current version of the file as a new object version in S3, + * with the requested backupTag as an S3 tag. + */ + public async makeBackup(docName: string, backupTag: string): Promise { + // TODO Must implement backups: currently this will prevent open docs that need migration. + // TODO: This method isn't used by SQLiteDB when migrating DB versions, but probably should be. + return "I_totally_did_not_back_up_your_document_sorry_not_sorry"; + } + + /** + * Electron version only. Shows the given doc in the file explorer. + */ + public async showItemInFolder(docName: string): Promise { + throw new Error("HostedStorageManager does not implement showItemInFolder"); + } + + /** + * Close the storage manager. Make sure any pending changes reach S3 first. + */ + public async closeStorage(): Promise { + await this._uploads.wait(() => log.info('HostedStorageManager: waiting for closeStorage to finish')); + + // Close metadata manager. + if (this._metadataManager) { await this._metadataManager.close(); } + + // Finish up everything incoming. This is most relevant for tests. + // Wait for any downloads to wind up, since there's no easy way to cancel them. + while (this._prepareFiles.size > 0) { await delay(100); } + await Promise.all(this._localFiles.values()); + + this._closed = true; + if (this._ext) { await this._ext.close(); } + if (this._pruner) { await this._pruner.close(); } + } + + /** + * Allow storage manager to be used again - used in tests. + */ + public testReopenStorage() { + this._closed = false; + } + + public async testWaitForPrunes() { + if (this._pruner) { await this._pruner.wait(); } + } + + /** + * Get direct access to the external store - used in tests. + */ + public testGetExternalStorage(): ExternalStorage { + return this._baseStore; + } + + // return true if document is backed up to s3. + public isSaved(docName: string): boolean { + return !this._uploads.hasPendingOperation(docName); + } + + // pick up the pace of pushing to s3, from leisurely to urgent. + public prepareToCloseStorage() { + if (this._pruner) { + this._pruner.close().catch(e => log.error("HostedStorageManager: pruning error %s", e)); + } + this._uploads.expediteOperations(); + } + + /** + * Finalize any operations involving the named document. + */ + public async closeDocument(docName: string): Promise { + if (this._localFiles.has(docName)) { + await this._localFiles.get(docName); + } + this._localFiles.delete(docName); + return this.flushDoc(docName); + } + + /** + * Make sure document is backed up to s3. + */ + public async flushDoc(docName: string): Promise { + while (!this.isSaved(docName)) { + log.info('HostedStorageManager: waiting for document to finish: %s', docName); + await this._uploads.expediteOperationAndWait(docName); + } + } + + /** + * This is called when a document may have been changed, via edits or migrations etc. + */ + public markAsChanged(docName: string): void { + if (parseUrlId(docName).snapshotId) { return; } + if (this._localFiles.has(docName)) { + // Make sure the file is marked as locally present (it may be newly created). + this._localFiles.set(docName, Promise.resolve(true)); + } + if (this._disableS3) { return; } + if (this._closed) { throw new Error("HostedStorageManager.markAsChanged called after closing"); } + this._uploads.addOperation(docName); + } + + /** + * This is called when a document was edited by the user. + */ + public markAsEdited(docName: string): void { + if (parseUrlId(docName).snapshotId) { return; } + // Schedule a metadata update for the modified doc. + if (this._metadataManager) { this._metadataManager.scheduleUpdate(docName); } + } + + /** + * Check if there is a pending change to be pushed to S3. + */ + public needsUpdate(): boolean { + return this._uploads.hasPendingOperations(); + } + + public async getSnapshots(docName: string): Promise { + if (this._disableS3) { + return { + snapshots: [{ + snapshotId: 'current', + lastModified: new Date(), + docId: docName, + }] + }; + } + const versions = await this._ext.versions(docName); + const parts = parseUrlId(docName); + return { + snapshots: versions + .map(v => ({ + lastModified: v.lastModified, + snapshotId: v.snapshotId, + docId: buildUrlId({...parts, snapshotId: v.snapshotId}), + })) + }; + } + + /** + * Makes sure a document is present locally, fetching it from S3 if necessary. + * Returns true on success, false if document not found. It is safe to call + * this method multiple times in parallel. + */ + private async _ensureDocumentIsPresent(docName: string, + docSession: OptDocSession): Promise { + // AsyncCreate.mapGetOrSet ensures we don't start multiple promises to talk to S3/Redis + // and that we clean up the failed key in case of failure. + return mapGetOrSet(this._localFiles, docName, async () => { + if (this._closed) { throw new Error("HostedStorageManager._ensureDocumentIsPresent called after closing"); } + checkValidDocId(docName); + + const {trunkId, forkId, forkUserId, snapshotId} = parseUrlId(docName); + + // If forkUserId is set to a valid user id, we can only create a fork if we know the + // requesting user and their id matches the forkUserId. + const userId = (docSession.client && docSession.client.getCachedUserId()) || + (docSession.req && getUserId(docSession.req)); + const canCreateFork = forkUserId ? (forkUserId === userId) : true; + + const docStatus = await this._docWorkerMap.getDocWorkerOrAssign(docName, this._docWorkerId); + if (!docStatus.isActive) { throw new Error(`Doc is not active on a DocWorker: ${docName}`); } + if (docStatus.docWorker.id !== this._docWorkerId) { + throw new Error(`Doc belongs to a different DocWorker (${docStatus.docWorker.id}): ${docName}`); + } + + if (this._disableS3) { + // skip S3, just use file system + let present: boolean = await fse.pathExists(this.getPath(docName)); + if (forkId && !present) { + if (!canCreateFork) { throw new Error(`Cannot create fork`); } + if (snapshotId && snapshotId !== 'current') { + throw new Error(`cannot find snapshot ${snapshotId} of ${docName}`); + } + if (await fse.pathExists(this.getPath(trunkId))) { + await fse.copy(this.getPath(trunkId), this.getPath(docName)); + present = true; + } + } + return present; + } + + const existsLocally = await fse.pathExists(this.getPath(docName)); + if (existsLocally) { + if (!docStatus.docMD5 || docStatus.docMD5 === DELETED_TOKEN) { + // New doc appears to already exist, but not in S3 (according to redis). + // Go ahead and use local version. + return true; + } else { + // Doc exists locally and in S3 (according to redis). + // Make sure the checksum matches. + const checksum = await this._getHash(await this._prepareBackup(docName)); + if (checksum === docStatus.docMD5) { + // Fine, accept the doc as existing on our file system. + return true; + } else { + log.info("Local hash does not match redis: %s vs %s", checksum, docStatus.docMD5); + // The file that exists locally does not match S3. But S3 is the canonical version. + // On the assumption that the local file is outdated, delete it. + // TODO: may want to be more careful in case the local file has modifications that + // simply never made it to S3 due to some kind of breakage. + // NOTE: fse.remove succeeds also when the file does not exist. + await fse.remove(this.getPath(docName)); + } + } + } + return this._fetchFromS3(docName, { + trunkId: forkId ? trunkId : undefined, snapshotId, canCreateFork + }); + }); + } + + /** + * Fetch a document from s3 and save it locally as destId.grist + * + * If the document is not present in s3: + * + If it has a trunk: + * - If we do not not have permission to create a fork, we throw an error + * - Else we fetch the document from the trunk instead + * + Otherwise return false + * + * Forks of fork will not spark joy at this time. An attempt to + * fork a fork will result in a new fork of the original trunk. + */ + private async _fetchFromS3(destId: string, options: {sourceDocId?: string, + trunkId?: string, + snapshotId?: string, + canCreateFork?: boolean}): Promise { + const destIdWithoutSnapshot = buildUrlId({...parseUrlId(destId), snapshotId: undefined}); + let sourceDocId = options.sourceDocId || destIdWithoutSnapshot; + if (!await this._ext.exists(destIdWithoutSnapshot)) { + if (!options.trunkId) { return false; } // Document not found in S3 + // No such fork in s3 yet, try from trunk (if we are allowed to create the fork). + if (!options.canCreateFork) { throw new Error('Cannot create fork'); } + // The special NEW_DOCUMENT_CODE trunk means we should create an empty document. + if (options.trunkId === NEW_DOCUMENT_CODE) { return false; } + if (!await this._ext.exists(options.trunkId)) { throw new Error('Cannot find original'); } + sourceDocId = options.trunkId; + } + await this._ext.downloadTo(sourceDocId, destId, this.getPath(destId), options.snapshotId); + return true; + } + + /** + * Get a checksum for the given file (absolute path). + */ + private _getHash(srcPath: string): Promise { + return checksumFile(srcPath, 'md5'); + } + + /** + * We'll save hashes in a file with the suffix -hash. + */ + private _getHashFile(docPath: string): string { + return docPath + "-hash"; + } + + /** + * Makes a copy of a document to a file with the suffix -backup. The copy is + * made using Sqlite's backup API. The backup is made incrementally so the db + * is never locked for long by the backup. The backup process will survive + * transient locks on the db. + */ + private async _prepareBackup(docId: string, postfix: string = 'backup'): Promise { + const docPath = this.getPath(docId); + const tmpPath = `${docPath}-${postfix}`; + return backupSqliteDatabase(docPath, tmpPath, undefined, postfix); + } + + /** + * Send a document to S3. + */ + private async _pushToS3(docId: string): Promise { + let tmpPath: string|null = null; + + try { + if (this._prepareFiles.has(docId)) { + throw new Error('too soon to consider pushing'); + } + tmpPath = await this._prepareBackup(docId); + await this._ext.upload(docId, tmpPath); + this._pruner.requestPrune(docId); + } finally { + // Clean up backup. + // NOTE: fse.remove succeeds also when the file does not exist. + if (tmpPath) { await fse.remove(tmpPath); } + } + } +} + + +/** + * Make a copy of a sqlite database safely and without locking it for long periods, using the + * sqlite backup api. + * @param src: database to copy + * @param dest: file to which we copy the database + * @param testProgress: a callback used for test purposes to monitor detailed timing of backup. + * @param label: a tag to add to log messages + * @return dest + */ +export async function backupSqliteDatabase(src: string, dest: string, + testProgress?: (e: BackupEvent) => void, + label?: string): Promise { + log.debug(`backupSqliteDatabase: starting copy of ${src} (${label})`); + let db: sqlite3.DatabaseWithBackup|null = null; + let success: boolean = false; + try { + // NOTE: fse.remove succeeds also when the file does not exist. + await fse.remove(dest); // Just in case some previous process terminated very badly. + // Sqlite will try to open any existing material at this + // path prior to overwriting it. + await fromCallback(cb => { db = new sqlite3.Database(dest, cb) as sqlite3.DatabaseWithBackup; }); + // Turn off protections that can slow backup steps. If the app or OS + // crashes, the backup may be corrupt. In Grist use case, if app or OS + // crashes, no use will be made of backup, so we're OK. + // This sets flags matching the --async option to .backup in the sqlite3 + // shell program: https://www.sqlite.org/src/info/7b6a605b1883dfcb + await fromCallback(cb => db!.exec("PRAGMA synchronous=OFF; PRAGMA journal_mode=OFF;", cb)); + if (testProgress) { testProgress({action: 'open', phase: 'before'}); } + const backup: sqlite3.Backup = db!.backup(src, 'main', 'main', false); + if (testProgress) { testProgress({action: 'open', phase: 'after'}); } + let remaining: number = -1; + let prevError: Error|null = null; + let errorMsgTime: number = 0; + let restartMsgTime: number = 0; + for (;;) { + // For diagnostic purposes, issue a message if the backup appears to have been + // restarted by sqlite. The symptom of a restart we use is that the number of + // pages remaining in the backup increases rather than decreases. That number + // is reported by backup.remaining (after an initial period of where sqlite + // doesn't yet know how many pages there are and reports -1). + // So as not to spam the log if the user is making a burst of changes, we report + // this message at most once a second. + // See https://www.sqlite.org/c3ref/backup_finish.html and + // https://github.com/mapbox/node-sqlite3/pull/1116 for api details. + if (remaining >= 0 && backup.remaining > remaining && Date.now() - restartMsgTime > 1000) { + log.info(`backupSqliteDatabase: copy of ${src} (${label}) restarted`); + restartMsgTime = Date.now(); + } + remaining = backup.remaining; + if (testProgress) { testProgress({action: 'step', phase: 'before'}); } + let isCompleted: boolean = false; + try { + isCompleted = Boolean(await fromCallback(cb => backup.step(PAGES_TO_BACKUP_PER_STEP, cb))); + } catch (err) { + if (String(err) !== String(prevError) || Date.now() - errorMsgTime > 1000) { + log.info(`backupSqliteDatabase (${src} ${label}): ${err}`); + errorMsgTime = Date.now(); + } + prevError = err; + if (backup.failed) { throw new Error(`backupSqliteDatabase (${src} ${label}): internal copy failed`); } + } + if (testProgress) { testProgress({action: 'step', phase: 'after'}); } + if (isCompleted) { + log.info(`backupSqliteDatabase: copy of ${src} (${label}) completed successfully`); + success = true; + break; + } + await delay(PAUSE_BETWEEN_BACKUP_STEPS_IN_MS); + } + } finally { + if (testProgress) { testProgress({action: 'close', phase: 'before'}); } + try { + if (db) { await fromCallback(cb => db!.close(cb)); } + } catch (err) { + log.debug(`backupSqliteDatabase: problem stopping copy of ${src} (${label}): ${err}`); + } + if (!success) { + // Something went wrong, remove backup if it was started. + try { + // NOTE: fse.remove succeeds also when the file does not exist. + await fse.remove(dest); + } catch (err) { + log.debug(`backupSqliteDatabase: problem removing copy of ${src} (${label}): ${err}`); + } + } + if (testProgress) { testProgress({action: 'close', phase: 'after'}); } + log.debug(`backupSqliteDatabase: stopped copy of ${src} (${label})`); + } + return dest; +} + + +/** + * A summary of an event during a backup. Emitted for test purposes, to check timing. + */ +export interface BackupEvent { + action: 'step' | 'close' | 'open'; + phase: 'before' | 'after'; +} diff --git a/app/server/lib/IBilling.ts b/app/server/lib/IBilling.ts new file mode 100644 index 00000000..2a1d6196 --- /dev/null +++ b/app/server/lib/IBilling.ts @@ -0,0 +1,7 @@ +import * as express from 'express'; + +export interface IBilling { + addEndpoints(app: express.Express): void; + addEventHandlers(): void; + addWebhooks(app: express.Express): void; +} diff --git a/app/server/lib/ICreate.ts b/app/server/lib/ICreate.ts new file mode 100644 index 00000000..e6f6b347 --- /dev/null +++ b/app/server/lib/ICreate.ts @@ -0,0 +1,30 @@ +import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager'; +import { ActiveDoc } from 'app/server/lib/ActiveDoc'; +import { ScopedSession } from 'app/server/lib/BrowserSession'; +import * as Comm from 'app/server/lib/Comm'; +import { DocManager } from 'app/server/lib/DocManager'; +import { ExternalStorage } from 'app/server/lib/ExternalStorage'; +import { GristServer } from 'app/server/lib/GristServer'; +import { IBilling } from 'app/server/lib/IBilling'; +import { IDocStorageManager } from 'app/server/lib/IDocStorageManager'; +import { IInstanceManager } from 'app/server/lib/IInstanceManager'; +import { ILoginSession } from 'app/server/lib/ILoginSession'; +import { INotifier } from 'app/server/lib/INotifier'; +import { ISandbox, ISandboxCreationOptions } from 'app/server/lib/ISandbox'; +import { IShell } from 'app/server/lib/IShell'; +import { PluginManager } from 'app/server/lib/PluginManager'; + +export interface ICreate { + LoginSession(comm: Comm, sid: string, domain: string, scopeSession: ScopedSession, + instanceManager: IInstanceManager|null): ILoginSession; + Billing(dbManager: HomeDBManager): IBilling; + Notifier(dbManager: HomeDBManager, homeUrl: string): INotifier; + Shell(): IShell|undefined; + ExternalStorage(bucket: string, prefix: string): ExternalStorage|undefined; + ActiveDoc(docManager: DocManager, docName: string): ActiveDoc; + DocManager(storageManager: IDocStorageManager, pluginManager: PluginManager, + homeDbManager: HomeDBManager|null, gristServer: GristServer): DocManager; + NSandbox(options: ISandboxCreationOptions): ISandbox; + + sessionSecret(): string; +} diff --git a/app/server/lib/IDocStorageManager.ts b/app/server/lib/IDocStorageManager.ts new file mode 100644 index 00000000..2affbed2 --- /dev/null +++ b/app/server/lib/IDocStorageManager.ts @@ -0,0 +1,35 @@ +import {DocEntry} from 'app/common/DocListAPI'; +import {DocReplacementOptions} from 'app/common/UserAPI'; +import {OptDocSession} from 'app/server/lib/DocSession'; +import {DocSnapshots} from 'app/server/lib/DocSnapshots'; + +export interface IDocStorageManager { + getPath(docName: string): string; + getSampleDocPath(sampleDocName: string): string|null; + getCanonicalDocName(altDocName: string): Promise; + + // This method must not be called for the same docName twice in parallel. + // In the current implementation, it is called in the context of an + // AsyncCreate[docName]. + prepareLocalDoc(docName: string, docSession: OptDocSession): Promise; + + listDocs(): Promise; + deleteDoc(docName: string, deletePermanently?: boolean): Promise; + renameDoc(oldName: string, newName: string): Promise; + makeBackup(docName: string, backupTag: string): Promise; + showItemInFolder(docName: string): Promise; + closeStorage(): Promise; + closeDocument(docName: string): Promise; + markAsChanged(docName: string): void; // document needs a backup (edits, migrations, etc) + markAsEdited(docName: string): void; // document was edited by a user + + testReopenStorage(): void; // restart storage during tests + addToStorage(docName: string): void; // add a new local document to storage + prepareToCloseStorage(): void; // speed up sync with remote store + getCopy(docName: string): Promise; // get an immutable copy of a document + + flushDoc(docName: string): Promise; // flush a document to persistent storage + + getSnapshots(docName: string): Promise; + replace(docName: string, options: DocReplacementOptions): Promise; +} diff --git a/app/server/lib/IInstanceManager.ts b/app/server/lib/IInstanceManager.ts new file mode 100644 index 00000000..51cea7c4 --- /dev/null +++ b/app/server/lib/IInstanceManager.ts @@ -0,0 +1,5 @@ +import { ILoginSession } from 'app/server/lib/ILoginSession'; + +export interface IInstanceManager { + getLoginSession(instanceId: string): ILoginSession; +} diff --git a/app/server/lib/ILoginSession.ts b/app/server/lib/ILoginSession.ts new file mode 100644 index 00000000..f20e26f6 --- /dev/null +++ b/app/server/lib/ILoginSession.ts @@ -0,0 +1,15 @@ +import {UserProfile} from 'app/common/LoginSessionAPI'; +import {Client} from 'app/server/lib/Client'; + +export interface ILoginSession { + clients: Set; + getEmail(): Promise; + // Log out + clearSession(): Promise; + + // For testing only. If no email address, profile is wiped, otherwise it is set. + testSetProfile(profile: UserProfile|null): Promise; + updateTokenForTesting(idToken: string): Promise; + getCurrentTokenForTesting(): Promise; + useTestToken(idToken: string): Promise; +} diff --git a/app/server/lib/INotifier.ts b/app/server/lib/INotifier.ts new file mode 100644 index 00000000..44f84b57 --- /dev/null +++ b/app/server/lib/INotifier.ts @@ -0,0 +1,4 @@ +export interface INotifier { + // for test purposes, check if any notifications are in progress + readonly testPending: boolean; +} diff --git a/app/server/lib/ISandbox.ts b/app/server/lib/ISandbox.ts new file mode 100644 index 00000000..c3e322bb --- /dev/null +++ b/app/server/lib/ISandbox.ts @@ -0,0 +1,28 @@ +import * as log from 'app/server/lib/log'; + +/** + * Starting to whittle down the options used when creating a sandbox, to leave more + * freedom in how the sandbox works. + */ +export interface ISandboxCreationOptions { + comment?: string; // an argument to add in command line when possible, so it shows in `ps` + + logCalls?: boolean; + logMeta?: log.ILogMeta; + logTimes?: boolean; + + // This batch of options is used by SafePythonComponent, so are important for importers. + entryPoint?: string; // main script to call - leave undefined for default + sandboxMount?: string; // if defined, make this path available read-only as "/sandbox" + importMount?: string; // if defined, make this path available read-only as "/importdir" +} + +export interface ISandbox { + shutdown(): Promise; // TODO: tighten up this type. + pyCall(funcName: string, ...varArgs: unknown[]): Promise; + reportMemoryUsage(): Promise; +} + +export interface ISandboxCreator { + create(options: ISandboxCreationOptions): ISandbox; +} diff --git a/app/server/lib/IShell.ts b/app/server/lib/IShell.ts new file mode 100644 index 00000000..4a3a6e0d --- /dev/null +++ b/app/server/lib/IShell.ts @@ -0,0 +1,4 @@ +export interface IShell { + moveItemToTrash(docPath: string): void; + showItemInFolder(docPath: string): void; +} diff --git a/app/server/lib/ITestingHooks-ti.ts b/app/server/lib/ITestingHooks-ti.ts new file mode 100644 index 00000000..a0c67f4d --- /dev/null +++ b/app/server/lib/ITestingHooks-ti.ts @@ -0,0 +1,34 @@ +/** + * This module was automatically generated by `ts-interface-builder` + */ +import * as t from "ts-interface-checker"; +// tslint:disable:object-literal-key-quotes + +export const ITestingHooks = t.iface([], { + "getOwnPort": t.func("number"), + "getPort": t.func("number"), + "updateAuthToken": t.func("void", t.param("instId", "string"), t.param("authToken", "string")), + "getAuthToken": t.func(t.union("string", "null"), t.param("instId", "string")), + "useTestToken": t.func("void", t.param("instId", "string"), t.param("token", "string")), + "setLoginSessionProfile": t.func("void", t.param("gristSidCookie", "string"), + t.param("profile", t.union("UserProfile", "null")), t.param("org", "string", true)), + "setServerVersion": t.func("void", t.param("version", t.union("string", "null"))), + "disconnectClients": t.func("void"), + "commShutdown": t.func("void"), + "commRestart": t.func("void"), + "commSetClientPersistence": t.func("void", t.param("ttlMs", "number")), + "closeDocs": t.func("void"), + "setDocWorkerActivation": t.func("void", t.param("workerId", "string"), + t.param("active", t.union(t.lit('active'), + t.lit('inactive'), + t.lit('crash')))), + "flushAuthorizerCache": t.func("void"), + "getDocClientCounts": t.func(t.array(t.tuple("string", "number"))), + "setActiveDocTimeout": t.func("number", t.param("seconds", "number")), +}); + +const exportedTypeSuite: t.ITypeSuite = { + ITestingHooks, + UserProfile: t.name("object"), +}; +export default exportedTypeSuite; diff --git a/app/server/lib/ITestingHooks.ts b/app/server/lib/ITestingHooks.ts new file mode 100644 index 00000000..9a3a587f --- /dev/null +++ b/app/server/lib/ITestingHooks.ts @@ -0,0 +1,20 @@ +import {UserProfile} from 'app/common/LoginSessionAPI'; + +export interface ITestingHooks { + getOwnPort(): number; + getPort(): number; + updateAuthToken(instId: string, authToken: string): Promise; + getAuthToken(instId: string): Promise; + useTestToken(instId: string, token: string): Promise; + setLoginSessionProfile(gristSidCookie: string, profile: UserProfile|null, org?: string): Promise; + setServerVersion(version: string|null): Promise; + disconnectClients(): Promise; + commShutdown(): Promise; + commRestart(): Promise; + commSetClientPersistence(ttlMs: number): Promise; + closeDocs(): Promise; + setDocWorkerActivation(workerId: string, active: 'active'|'inactive'|'crash'): Promise; + flushAuthorizerCache(): Promise; + getDocClientCounts(): Promise>; + setActiveDocTimeout(seconds: number): Promise; +} diff --git a/app/server/lib/NSandbox.ts b/app/server/lib/NSandbox.ts new file mode 100644 index 00000000..159929e3 --- /dev/null +++ b/app/server/lib/NSandbox.ts @@ -0,0 +1,354 @@ +/** + * JS controller for the pypy sandbox. + */ +import * as pidusage from '@gristlabs/pidusage'; +import * as marshal from 'app/common/marshal'; +import {ISandbox, ISandboxCreationOptions, ISandboxCreator} from 'app/server/lib/ISandbox'; +import * as log from 'app/server/lib/log'; +import * as sandboxUtil from 'app/server/lib/sandboxUtil'; +import * as shutdown from 'app/server/lib/shutdown'; +import {Throttle} from 'app/server/lib/Throttle'; +import {ChildProcess, spawn, SpawnOptions} from 'child_process'; +import * as path from 'path'; +import {Stream, Writable} from 'stream'; + +type SandboxMethod = (...args: any[]) => any; + +export interface ISandboxCommand { + process: string; +} + +export interface ISandboxOptions { + args: string[]; // The arguments to pass to the python process. + exports?: {[name: string]: SandboxMethod}; // Functions made available to the sandboxed process. + logCalls?: boolean; // (Not implemented) Whether to log all system calls from the python sandbox. + logTimes?: boolean; // Whether to log time taken by calls to python sandbox. + unsilenceLog?: boolean; // Don't silence the sel_ldr logging. + selLdrArgs?: string[]; // Arguments passed to selLdr, for instance the following sets an + // environment variable `{ ... selLdrArgs: ['-E', 'PYTHONPATH=grist'] ... }`. + logMeta?: log.ILogMeta; // Log metadata (e.g. including docId) to report in all log messages. + command?: ISandboxCommand; +} + +// Options for low-level spawning of selLdr sandbox process. +export interface ISpawnOptions extends SpawnOptions { + unsilenceLog?: boolean; // Don't silence the sel_ldr logging. + command?: ISandboxCommand; +} + +type ResolveRejectPair = [(value?: any) => void, (reason?: unknown) => void]; + +// Type for basic message identifiers, available as constants in sandboxUtil. +type MsgCode = null | true | false; + +export class NSandbox implements ISandbox { + /** + * Helper function to run the nacl sandbox. It takes care of most arguments, similarly to + * nacl/bin/run script, but without the reliance on bash. We can't use bash when -r/-w options + * because on Windows it doesn't pass along the open file descriptors. Bash is also unavailable + * when installing a standalone version on Windows. + * @param selLdrArgs: Arguments to pass to sel_ldr; + * @param pythonArgs: Arguments to pass to python within the sandbox. + * @param spawnOptions: extra options for child_process.spawn(), such as 'stdio'. + */ + public static spawn(selLdrArgs: string[], pythonArgs: string[], spawnOptions: ISpawnOptions = {}): ChildProcess { + const unsilenceLog = spawnOptions.unsilenceLog; + delete spawnOptions.unsilenceLog; + const command = spawnOptions.command; + delete spawnOptions.command; + + if (command) { + return spawn(command.process, pythonArgs, + {env: {PYTHONPATH: 'grist:thirdparty'}, + cwd: path.join(process.cwd(), 'sandbox'), ...spawnOptions}); + } + + const noLog = unsilenceLog ? [] : + (process.env.OS === 'Windows_NT' ? ['-l', 'NUL'] : ['-l', '/dev/null']); + return spawn('sandbox/nacl/bin/sel_ldr', [ + '-B', './sandbox/nacl/lib/irt_core.nexe', '-m', './sandbox/nacl/root:/:ro', + ...noLog, + ...selLdrArgs, + './sandbox/nacl/lib/runnable-ld.so', + '--library-path', '/slib', '/python/bin/python2.7.nexe', + ...pythonArgs + ], + {env: {}, ...spawnOptions}, + ); + } + + public readonly childProc: ChildProcess; + private _logTimes: boolean; + private _exportedFunctions: {[name: string]: SandboxMethod}; + private _marshaller = new marshal.Marshaller({stringToBuffer: true, version: 2}); + private _unmarshaller = new marshal.Unmarshaller({ bufferToString: false }); + + // Members used for reading from the sandbox process. + private _pendingReads: ResolveRejectPair[] = []; + private _isReadClosed = false; + private _isWriteClosed = false; + + private _logMeta: log.ILogMeta; + private _streamToSandbox: Writable; + private _streamFromSandbox: Stream; + + private _throttle: Throttle | undefined; + + /* + * Callers may listen to events from sandbox.childProc (a ChildProcess), e.g. 'close' and 'error'. + * The sandbox listens for 'aboutToExit' event on the process, to properly shut down. + */ + constructor(options: ISandboxOptions) { + this._logTimes = Boolean(options.logTimes || options.logCalls); + this._exportedFunctions = options.exports || {}; + + const selLdrArgs = options.selLdrArgs || []; + + // We use these options to set up communication with the sandbox: + // -r 3:3 to associate a file descriptor 3 on the outside of the sandbox with FD 3 on the + // inside, for reading from the inside. This becomes `this._streamToSandbox`. + // -w 4:4 to associate FD 4 on the outside with FD 4 on the inside for writing from the inside. + // This becomes `this._streamFromSandbox` + this.childProc = NSandbox.spawn(['-r', '3:3', '-w', '4:4', ...selLdrArgs], options.args, { + stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'], + unsilenceLog: options.unsilenceLog, + command: options.command + }); + + this._logMeta = {sandboxPid: this.childProc.pid, ...options.logMeta}; + log.rawDebug("Sandbox started", this._logMeta); + + this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable; + this._streamFromSandbox = (this.childProc.stdio as Stream[])[4]; + + this.childProc.on('close', this._onExit.bind(this)); + this.childProc.on('error', this._onError.bind(this)); + + this.childProc.stdout.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta)); + this.childProc.stderr.on('data', sandboxUtil.makeLinePrefixer('Sandbox stderr: ', this._logMeta)); + + this._streamFromSandbox.on('data', (data) => this._onSandboxData(data)); + this._streamFromSandbox.on('end', () => this._onSandboxClose()); + this._streamFromSandbox.on('error', (err) => { + log.rawError(`Sandbox error reading: ${err}`, this._logMeta); + this._onSandboxClose(); + }); + + this._streamToSandbox.on('error', (err) => { + if (!this._isWriteClosed) { + log.rawError(`Sandbox error writing: ${err}`, this._logMeta); + } + }); + + // On shutdown, shutdown the child process cleanly, and wait for it to exit. + shutdown.addCleanupHandler(this, this.shutdown); + + if (process.env.GRIST_THROTTLE_CPU) { + this._throttle = new Throttle({ + pid: this.childProc.pid, + logMeta: this._logMeta, + }); + } + } + + /** + * Shuts down the sandbox process cleanly, and wait for it to exit. + * @return {Promise} Promise that's resolved with [code, signal] when the sandbox exits. + */ + public async shutdown() { + log.rawDebug("Sandbox shutdown starting", this._logMeta); + shutdown.removeCleanupHandlers(this); + + // The signal ensures the sandbox process exits even if it's hanging in an infinite loop or + // long computation. It doesn't get a chance to clean up, but since it is sandboxed, there is + // nothing it needs to clean up anyway. + const timeoutID = setTimeout(() => { + log.rawWarn("Sandbox sending SIGKILL", this._logMeta); + this.childProc.kill('SIGKILL'); + }, 1000); + + const result = await new Promise((resolve, reject) => { + if (this._isWriteClosed) { resolve(); } + this.childProc.on('error', reject); + this.childProc.on('close', resolve); + this._close(); + }); + + // In the normal case, the kill timer is pending when the process exits, and we can clear it. If + // the process got killed, the timer is invalid, and clearTimeout() does nothing. + clearTimeout(timeoutID); + return result; + } + + /** + * Makes a call to the python process implementing our calling convention on stdin/stdout. + * @param funcName The name of the python RPC function to call. + * @param args Arguments to pass to the given function. + * @returns A promise for the return value from the Python function. + */ + public pyCall(funcName: string, ...varArgs: unknown[]): Promise { + const startTime = Date.now(); + this._sendData(sandboxUtil.CALL, Array.from(arguments)); + return this._pyCallWait(funcName, startTime); + } + + /** + * Returns the RSS (resident set size) of the sandbox process, in bytes. + */ + public async reportMemoryUsage() { + const memory = (await pidusage(this.childProc.pid)).memory; + log.rawDebug('Sandbox memory', {memory, ...this._logMeta}); + } + + private async _pyCallWait(funcName: string, startTime: number): Promise { + try { + return await new Promise((resolve, reject) => { + this._pendingReads.push([resolve, reject]); + }); + } finally { + if (this._logTimes) { + log.rawDebug(`Sandbox pyCall[${funcName}] took ${Date.now() - startTime} ms`, this._logMeta); + } + } + } + + + private _close() { + if (this._throttle) { this._throttle.stop(); } + if (!this._isWriteClosed) { + // Close the pipe to the sandbox, which should cause the sandbox to exit cleanly. + this._streamToSandbox.end(); + this._isWriteClosed = true; + } + } + + private _onExit(code: number, signal: string) { + this._close(); + log.rawDebug(`Sandbox exited with code ${code} signal ${signal}`, this._logMeta); + } + + + private _onError(err: Error) { + this._close(); + log.rawWarn(`Sandbox could not be spawned: ${err}`, this._logMeta); + } + + + /** + * Send a message to the sandbox process with the given message code and data. + */ + private _sendData(msgCode: MsgCode, data: any) { + if (this._isReadClosed) { + throw new sandboxUtil.SandboxError("PipeToSandbox is closed"); + } + this._marshaller.marshal(msgCode); + this._marshaller.marshal(data); + return this._streamToSandbox.write(this._marshaller.dumpAsBuffer()); + } + + + /** + * Process a buffer of data received from the sandbox process. + */ + private _onSandboxData(data: any) { + this._unmarshaller.parse(data, buf => { + const value = marshal.loads(buf, { bufferToString: true }); + this._onSandboxMsg(value[0], value[1]); + }); + } + + + /** + * Process the closing of the pipe by the sandboxed process. + */ + private _onSandboxClose() { + if (this._throttle) { this._throttle.stop(); } + this._isReadClosed = true; + // Clear out all reads pending on PipeFromSandbox, rejecting them with the given error. + const err = new sandboxUtil.SandboxError("PipeFromSandbox is closed"); + this._pendingReads.forEach(resolvePair => resolvePair[1](err)); + this._pendingReads = []; + } + + + /** + * Process a parsed message from the sandboxed process. + */ + private _onSandboxMsg(msgCode: MsgCode, data: any) { + if (msgCode === sandboxUtil.CALL) { + // Handle calls FROM the sandbox. + if (!Array.isArray(data) || data.length === 0) { + log.rawWarn("Sandbox invalid call from the sandbox", this._logMeta); + } else { + const fname = data[0]; + const args = data.slice(1); + log.rawDebug(`Sandbox got call to ${fname} (${args.length} args)`, this._logMeta); + Promise.resolve() + .then(() => { + const func = this._exportedFunctions[fname]; + if (!func) { throw new Error("No such exported function: " + fname); } + return func(...args); + }) + .then((ret) => { + this._sendData(sandboxUtil.DATA, ret); + }, (err) => { + this._sendData(sandboxUtil.EXC, err.toString()); + }) + .catch((err) => { + log.rawDebug(`Sandbox sending response failed: ${err}`, this._logMeta); + }); + } + } else { + // Handle return values for calls made to the sandbox. + const resolvePair = this._pendingReads.shift(); + if (resolvePair) { + if (msgCode === sandboxUtil.EXC) { + resolvePair[1](new sandboxUtil.SandboxError(data)); + } else if (msgCode === sandboxUtil.DATA) { + resolvePair[0](data); + } else { + log.rawWarn("Sandbox invalid message from sandbox", this._logMeta); + } + } + } + } +} + +export class NSandboxCreator implements ISandboxCreator { + public constructor(private _flavor: 'pynbox' | 'unsandboxed') { + } + + public create(options: ISandboxCreationOptions): ISandbox { + const defaultEntryPoint = this._flavor === 'pynbox' ? 'grist/main.pyc' : 'grist/main.py'; + const args = [options.entryPoint || defaultEntryPoint]; + if (!options.entryPoint && options.comment) { + // When using default entry point, we can add on a comment as an argument - it isn't + // used, but will show up in `ps` output for the sandbox process. Comment is intended + // to be a document name/id. + args.push(options.comment); + } + const selLdrArgs: string[] = []; + if (options.sandboxMount) { + selLdrArgs.push( + // TODO: Only modules that we share with plugins should be mounted. They could be gathered in + // a "$APPROOT/sandbox/plugin" folder, only which get mounted. + '-E', 'PYTHONPATH=grist:thirdparty', + '-m', `${options.sandboxMount}:/sandbox:ro`); + } + if (options.importMount) { + selLdrArgs.push('-m', `${options.importMount}:/importdir:ro`); + } + return new NSandbox({ + args, + logCalls: options.logCalls, + logMeta: options.logMeta, + logTimes: options.logTimes, + selLdrArgs, + ...(this._flavor === 'pynbox' ? {} : { + command: { + process: "python2.7" + } + }) + }); + } +} diff --git a/app/server/lib/OnDemandActions.ts b/app/server/lib/OnDemandActions.ts new file mode 100644 index 00000000..3e9f0b36 --- /dev/null +++ b/app/server/lib/OnDemandActions.ts @@ -0,0 +1,206 @@ +import {BulkColValues, ColValues, DocAction, isSchemaAction, TableDataAction, UserAction} from 'app/common/DocActions'; +import {DocData} from 'app/common/DocData'; +import {TableData} from 'app/common/TableData'; +import {IndexColumns} from 'app/server/lib/DocStorage'; + +const ACTION_TYPES = new Set(['AddRecord', 'BulkAddRecord', 'UpdateRecord', 'BulkUpdateRecord', + 'RemoveRecord', 'BulkRemoveRecord']); + +export interface ProcessedAction { + stored: DocAction[]; + undo: DocAction[]; + retValues: any; +} + +export interface OnDemandStorage { + getNextRowId(tableId: string): Promise; + fetchActionData(tableId: string, rowIds: number[], colIds?: string[]): Promise; +} + +/** + * Handle converting UserActions to DocActions for onDemand tables. + */ +export class OnDemandActions { + + private _tablesMeta: TableData = this._docData.getTable('_grist_Tables')!; + private _columnsMeta: TableData = this._docData.getTable('_grist_Tables_column')!; + + constructor(private _storage: OnDemandStorage, private _docData: DocData) {} + + // TODO: Ideally a faster data structure like an index by tableId would be used to decide whether + // the table is onDemand. + public isOnDemand(tableId: string): boolean { + const tableRef = this._tablesMeta.findRow('tableId', tableId); + // OnDemand tables must have a record in the _grist_Tables metadata table. + return tableRef ? Boolean(this._tablesMeta.getValue(tableRef, 'onDemand')) : false; + } + + /** + * Convert a UserAction into stored and undo DocActions as well as return values. + */ + public processUserAction(action: UserAction): Promise { + const a = action.map(item => item as any); + switch (a[0]) { + case "ApplyUndoActions": return this._doApplyUndoActions(a[1]); + case "AddRecord": return this._doAddRecord (a[1], a[2], a[3]); + case "BulkAddRecord": return this._doBulkAddRecord (a[1], a[2], a[3]); + case "UpdateRecord": return this._doUpdateRecord (a[1], a[2], a[3]); + case "BulkUpdateRecord": return this._doBulkUpdateRecord(a[1], a[2], a[3]); + case "RemoveRecord": return this._doRemoveRecord (a[1], a[2]); + case "BulkRemoveRecord": return this._doBulkRemoveRecord(a[1], a[2]); + default: throw new Error(`Received unknown action ${action[0]}`); + } + } + + /** + * Splits an array of UserActions into two separate arrays of normal and onDemand actions. + */ + public splitByOnDemand(actions: UserAction[]): [UserAction[], UserAction[]] { + const normal: UserAction[] = []; + const onDemand: UserAction[] = []; + actions.forEach(a => { + // Check that the actionType can be applied without the sandbox and also that the action + // is on a data table. + const isOnDemandAction = ACTION_TYPES.has(a[0] as string); + const isDataTableAction = typeof a[1] === 'string' && !(a[1] as string).startsWith('_grist_'); + if (a[0] === 'ApplyUndoActions') { + // Split actions inside the undo action array. + const [undoNormal, undoOnDemand] = this.splitByOnDemand(a[1] as UserAction[]); + if (undoNormal.length > 0) { + normal.push(['ApplyUndoActions', undoNormal]); + } + if (undoOnDemand.length > 0) { + onDemand.push(['ApplyUndoActions', undoOnDemand]); + } + } else if (isDataTableAction && isOnDemandAction && this.isOnDemand(a[1] as string)) { + // Check whether the tableId belongs to an onDemand table. + onDemand.push(a); + } else { + normal.push(a); + } + }); + return [normal, onDemand]; + } + + /** + * Compute the indexes we would like to have, given the current schema. + */ + public getDesiredIndexes(): IndexColumns[] { + const desiredIndexes: IndexColumns[] = []; + for (const c of this._columnsMeta.getRecords()) { + const t = this._tablesMeta.getRecord(c.parentId as number); + if (t && t.onDemand && c.type && (c.type as string).startsWith('Ref:')) { + desiredIndexes.push({tableId: t.tableId as string, colId: c.colId as string}); + } + } + return desiredIndexes; + } + + /** + * Check if an action represents a schema change on an onDemand table. + */ + public isSchemaAction(docAction: DocAction): boolean { + return isSchemaAction(docAction) && this.isOnDemand(docAction[1]); + } + + private async _doApplyUndoActions(actions: DocAction[]) { + const undo: DocAction[] = []; + for (const a of actions) { + const converted = await this.processUserAction(a); + undo.concat(converted.undo); + } + return { + stored: actions, + undo, + retValues: null + }; + } + + private async _doAddRecord( + tableId: string, + rowId: number|null, + colValues: ColValues + ): Promise { + if (rowId === null) { + rowId = await this._storage.getNextRowId(tableId); + } + // Set the manualSort to be the same as the rowId. This forces new rows to always be added + // at the end of the table. + colValues.manualSort = rowId; + return { + stored: [['AddRecord', tableId, rowId, colValues]], + undo: [['RemoveRecord', tableId, rowId]], + retValues: rowId + }; + } + + private async _doBulkAddRecord( + tableId: string, + rowIds: Array, + colValues: BulkColValues + ): Promise { + + // When unset, we will set the rowId values to count up from the greatest + // values already in the table. + if (rowIds[0] === null) { + const nextRowId = await this._storage.getNextRowId(tableId); + for (let i = 0; i < rowIds.length; i++) { + rowIds[i] = nextRowId + i; + } + } + // Set the manualSort values to be the same as the rowIds. This forces new rows to always be + // added at the end of the table. + colValues.manualSort = rowIds; + return { + stored: [['BulkAddRecord', tableId, rowIds as number[], colValues]], + undo: [['BulkRemoveRecord', tableId, rowIds as number[]]], + retValues: rowIds + }; + } + + private async _doUpdateRecord( + tableId: string, + rowId: number, + colValues: ColValues + ): Promise { + const [, , oldRowIds, oldColValues] = + await this._storage.fetchActionData(tableId, [rowId], Object.keys(colValues)); + return { + stored: [['UpdateRecord', tableId, rowId, colValues]], + undo: [['BulkUpdateRecord', tableId, oldRowIds, oldColValues]], + retValues: null + }; + } + + private async _doBulkUpdateRecord( + tableId: string, + rowIds: number[], + colValues: BulkColValues + ): Promise { + const [, , oldRowIds, oldColValues] = + await this._storage.fetchActionData(tableId, rowIds, Object.keys(colValues)); + return { + stored: [['BulkUpdateRecord', tableId, rowIds, colValues]], + undo: [['BulkUpdateRecord', tableId, oldRowIds, oldColValues]], + retValues: null + }; + } + + private async _doRemoveRecord(tableId: string, rowId: number): Promise { + const [, , oldRowIds, oldColValues] = await this._storage.fetchActionData(tableId, [rowId]); + return { + stored: [['RemoveRecord', tableId, rowId]], + undo: [['BulkAddRecord', tableId, oldRowIds, oldColValues]], + retValues: null + }; + } + + private async _doBulkRemoveRecord(tableId: string, rowIds: number[]): Promise { + const [, , oldRowIds, oldColValues] = await this._storage.fetchActionData(tableId, rowIds); + return { + stored: [['BulkRemoveRecord', tableId, rowIds]], + undo: [['BulkAddRecord', tableId, oldRowIds, oldColValues]], + retValues: null + }; + } +} diff --git a/app/server/lib/PluginEndpoint.ts b/app/server/lib/PluginEndpoint.ts new file mode 100644 index 00000000..549a0a9d --- /dev/null +++ b/app/server/lib/PluginEndpoint.ts @@ -0,0 +1,78 @@ +import {FlexServer} from 'app/server/lib/FlexServer'; +import * as log from 'app/server/lib/log'; +import {PluginManager} from 'app/server/lib/PluginManager'; +import * as express from 'express'; +import * as mimeTypes from 'mime-types'; +import * as path from 'path'; + +// Get the url where plugin material should be served from. +export function getUntrustedContentOrigin(): string|undefined { + return process.env.APP_UNTRUSTED_URL; +} + +// Get the host serving plugin material +export function getUntrustedContentHost(): string|undefined { + const origin = getUntrustedContentOrigin(); + if (!origin) { return; } + return new URL(origin).host; +} + +// Add plugin endpoints to be served on untrusted host +export function addPluginEndpoints(server: FlexServer, pluginManager: PluginManager) { + const host = getUntrustedContentHost(); + if (host) { + server.app.get(/^\/plugins\/(installed|builtIn)\/([^/]+)\/(.+)/, (req, res) => + servePluginContent(req, res, pluginManager, host)); + } +} + +// Serve content for plugins with various checks that it is being accessed as we expect. +function servePluginContent(req: express.Request, res: express.Response, + pluginManager: PluginManager, untrustedContentHost: string) { + const pluginKind = req.params[0]; + const pluginId = req.params[1]; + const pluginPath = req.params[2]; + + // We should not serve untrusted content (as from plugins) from the same domain as the main app + // (at least not html pages), as it's an open door to XSS attacks. + // - For hosted version, we serve it from a separate domain name. + // - For electron version, we give access to protected content based on a special header. + // - We also allow "application/javascript" content from the main domain for serving the + // WebWorker main script, since that's hard to distinguish in electron case, and should not + // enable XSS. + if (matchHost(req.get('host'), untrustedContentHost) || + req.get('X-From-Plugin-WebView') === "true" || + mimeTypes.lookup(path.extname(pluginPath)) === "application/javascript") { + const dirs = pluginManager.dirs(); + const contentRoot = pluginKind === "installed" ? dirs.installed : dirs.builtIn; + // Note that pluginPath may not be safe, but `sendFile` with the "root" option restricts + // relative paths to be within the root folder (see the 3rd party library unit-test: + // https://github.com/pillarjs/send/blob/3daa901cf731b86187e4449fa2c52f971e0b3dbc/test/send.js#L1363) + return res.sendFile(`${pluginId}/${pluginPath}`, {root: contentRoot}); + } + + log.warn(`Refusing to serve untrusted plugin content on ${req.get('host')}`); + res.status(403).end('Plugin content is not accessible to this request'); +} + +// Middleware to restrict some assets to untrusted host. +export function limitToPlugins(handler: express.RequestHandler) { + const host = getUntrustedContentHost(); + return function(req: express.Request, resp: express.Response, next: express.NextFunction) { + if (!host) { return next(); } + if (matchHost(req.get('host'), host) || req.get('X-From-Plugin-WebView') === "true") { + return handler(req, resp, next); + } + return next(); + }; +} + +// Compare hosts, bearing in mind that if they happen to be on port 443 the +// port number may or may not be included. This assumes we are serving over https. +function matchHost(host1: string|undefined, host2: string) { + if (!host1) { return false; } + if (host1 === host2) { return true; } + if (host1.indexOf(':') === -1) { host1 += ":443"; } + if (host2.indexOf(':') === -1) { host2 += ":443"; } + return host1 === host2; +} diff --git a/app/server/lib/PluginManager.ts b/app/server/lib/PluginManager.ts new file mode 100644 index 00000000..bb78debb --- /dev/null +++ b/app/server/lib/PluginManager.ts @@ -0,0 +1,165 @@ +import {DirectoryScanEntry, LocalPlugin} from 'app/common/plugin'; +import * as log from 'app/server/lib/log'; +import {readManifest} from 'app/server/lib/manifest'; +import {getAppPathTo} from 'app/server/lib/places'; +import * as fse from 'fs-extra'; +import * as path from 'path'; + +/** + * Various plugins' related directories. + */ +export interface PluginDirectories { + /** + * Directory where built in plugins are located. + */ + readonly builtIn?: string; + /** + * Directory where user installed plugins are localted. + */ + readonly installed?: string; +} + +/** + * + * The plugin manager class is responsible for providing both built in and installed plugins and + * spawning server side plugins's. + * + * Usage: + * + * const pluginManager = new PluginManager(appRoot, userRoot); + * await pluginManager.initialize(); + * + */ +export class PluginManager { + + public pluginsLoaded: Promise; + + // ========== Instance members and methods ========== + private _dirs: PluginDirectories; + private _validPlugins: LocalPlugin[] = []; + private _entries: DirectoryScanEntry[] = []; + + + /** + * @param {string} userRoot: path to user's grist directory; `null` is allowed, to only uses built in plugins. + * + */ + public constructor(public appRoot?: string, userRoot?: string) { + this._dirs = { + installed: userRoot ? path.join(userRoot, 'plugins') : undefined, + builtIn: appRoot ? getAppPathTo(appRoot, 'plugins') : undefined + }; + } + + public dirs(): PluginDirectories {return this._dirs; } + + /** + * Create tmp dir and load plugins. + */ + public async initialize(): Promise { + try { + await (this.pluginsLoaded = this.loadPlugins()); + } catch (err) { + log.error("PluginManager's initialization failed: ", err); + throw err; + } + } + + /** + * Re-load plugins (litterally re-run `loadPlugins`). + */ + // TODO: it's not clear right now what we do on reload. Do we deactivate plugins that were removed + // from the fs? Do we update plugins that have changed on the fs ? + public async reloadPlugins(): Promise { + return await this.loadPlugins(); + } + + /** + * Discover both builtIn and user installed plugins. Logs any failures that happens when scanning + * a directory (ie: manifest missing or manifest validation errors etc...) + */ + public async loadPlugins(): Promise { + this._entries = []; + + // Load user installed plugins + if (this._dirs.installed) { + this._entries.push(...await scanDirectory(this._dirs.installed, "installed")); + } + + // Load builtIn plugins + if (this._dirs.builtIn) { + this._entries.push(...await scanDirectory(this._dirs.builtIn, "builtIn")); + } + + if (!process.env.GRIST_EXPERIMENTAL_PLUGINS || + process.env.GRIST_EXPERIMENTAL_PLUGINS === '0') { + // Remove experimental plugins + this._entries = this._entries.filter(entry => { + if (entry.manifest && entry.manifest.experimental) { + log.warn("Ignoring experimental plugin %s", entry.id); + return false; + } + return true; + }); + } + + this._validPlugins = this._entries.filter(entry => !entry.errors).map(entry => entry as LocalPlugin); + + this._logScanningReport(); + } + + public getPlugins(): LocalPlugin[] { + return this._validPlugins; + } + + + private _logScanningReport() { + const invalidPlugins = this._entries.filter( entry => entry.errors); + if (invalidPlugins.length) { + for (const plugin of invalidPlugins) { + log.warn(`Error loading plugins: Failed to load extension from ${plugin.path}\n` + + (plugin.errors!).map(m => " - " + m).join("\n ") + ); + } + } + log.info(`Found ${this._validPlugins.length} valid plugins on the system`); + for (const p of this._validPlugins) { + log.debug("PLUGIN %s -- %s", p.id, p.path); + } + } +} + + +async function scanDirectory(dir: string, kind: "installed"|"builtIn"): Promise { + const plugins: DirectoryScanEntry[] = []; + let listDir; + + try { + listDir = await fse.readdir(dir); + } catch (e) { + // non existing dir is treated as an empty dir + log.info(`No plugins directory: ${e.message}`); + return []; + } + + for (const id of listDir) { + const folderPath = path.join(dir, id), + plugin: DirectoryScanEntry = { + path: folderPath, + id: `${kind}/${id}` + }; + try { + plugin.manifest = await readManifest(folderPath); + } catch (e) { + plugin.errors = []; + if (e.message) { + plugin.errors.push(e.message); + } + if (e.notices) { + plugin.errors.push(...e.notices); + } + } + plugins.push(plugin); + } + return plugins; +} diff --git a/app/server/lib/SQLiteDB.ts b/app/server/lib/SQLiteDB.ts new file mode 100644 index 00000000..f4d50760 --- /dev/null +++ b/app/server/lib/SQLiteDB.ts @@ -0,0 +1,534 @@ +/** + * SQLiteDB provides a clean Promise-based interface to SQLite along with an organized way to + * specify the initial structure of the database and migrations when this structure changes. + * + * Here's a simple example, + * + * const schemaInfo: SQLiteDB.SchemaInfo = { + * async create(db: SQLiteDB.SQLiteDB) { + * await db.exec("CREATE TABLE Foo (A TEXT)"); + * }, + * migrations: [ + * async function(db: SQLiteDB.SQLiteDB) { + * await db.exec("CREATE TABLE Foo (A TEXT)"); + * } + * ], + * } + * const db = await SQLiteDB.openDB("pathToDB", schemaInfo, SQLiteDB.OpenMode.OPEN_CREATE); + * + * Note how the create() function and the first migration are identical here. But they'll diverge + * once we make a change to the schema. E.g. the next change could look like this: + * + * const schemaInfo: SQLiteDB.SchemaInfo = { + * async create(db: SQLiteDB.SQLiteDB) { + * await db.exec("CREATE TABLE Foo (A TEXT, B NUMERIC)"); + * }, + * migrations: [ + * async function(db: SQLiteDB.SQLiteDB) { + * await db.exec("CREATE TABLE Foo (A TEXT)"); + * }, + * async function(db: SQLiteDB.SQLiteDB) { + * await db.exec("ALTER TABLE Foo ADD COLUMN B NUMERIC"); + * } + * ], + * } + * const db = await SQLiteDB.openDB("pathToDB", schemaInfo, SQLiteDB.OpenMode.OPEN_CREATE); + * + * Now a new document will have two columns. A document created with the first version of the code + * will gain a second column when opened with the new code. If a migration happened during open, + * you may examine two properties of the returned db object: + * + * db.migrationBackupPath -- set to the path of the pre-migration backup file. + * db.migrationError -- set to the Error object if the migration failed. + * + * This module uses SQLite's "user_version" pragma to keep track of the version number of a + * migration. It does not require, support, or record backwards migrations, but it will warn of + * inconsistencies that may arise during development. In that case, remember you have a backup + * from each migration. + * + * If you are starting with an existing unversioned DB, the first migration should have code to + * bring such DBs to a common state. + * + * const schemaInfo: SQLiteDB.SchemaInfo = { + * async create(db: SQLiteDB.SQLiteDB) { + * await db.exec("CREATE TABLE Foo (A TEXT)"); + * await db.exec("CREATE TABLE Bar (B TEXT)"); + * }, + * migrations: [ + * async function(db: SQLiteDB.SQLiteDB) { + * await db.exec("CREATE TABLE IF NOT EXISTS Foo (A TEXT)"); + * await db.exec("CREATE TABLE IF NOT EXISTS Bar (B TEXT)"); + * } + * ], + * } + * const db = await SQLiteDB.openDB("pathToDB", schemaInfo, SQLiteDB.OpenMode.OPEN_CREATE); + * + * Once using this module with versioning, future changes would be made by adding one item to the + * "migrations" array, and modifying create() to create correct new documents. + */ + +import {ErrorWithCode} from 'app/common/ErrorWithCode'; +import {timeFormat} from 'app/common/timeFormat'; +import * as docUtils from 'app/server/lib/docUtils'; +import * as log from 'app/server/lib/log'; +import {fromCallback} from 'app/server/lib/serverUtils'; + +import * as sqlite3 from '@gristlabs/sqlite3'; +import * as assert from 'assert'; +import {each} from 'bluebird'; +import * as fse from 'fs-extra'; +import fromPairs = require('lodash/fromPairs'); +import isEqual = require('lodash/isEqual'); +import noop = require('lodash/noop'); +import range = require('lodash/range'); + +// Describes the result of get() and all() database methods. +export interface ResultRow { + [column: string]: any; +} + +// Describes how to create a new DB or migrate an old one. Any changes to the DB must be reflected +// in the 'create' function, and added as new entries in the 'migrations' array. Existing +// 'migration' entries may not be modified; they are used to migrate older DBs. +export interface SchemaInfo { + // Creates a structure for a new DB (i.e. execs CREATE TABLE statements). + readonly create: DBFunc; + + // List of functions that perform DB migrations from one version to the next. This array's + // length determines the schema version, which is stored in user_version SQLite property. + // + // The very first migration should normally be identical to the original version of create(). + // I.e. initially SchemaInfo should be { create: X, migrations: [X] }, where the two X's + // represent two copies of the same code. Don't go for code reuse here. When the schema is + // modified, you will change it to { create: X2, migrations: [X, Y] }. Keeping the unchanged + // copy of X is important as a reference to see that X + Y produces the same DB as X2. + // + // If you may open DBs created without versioning (e.g. predate use of this module), such DBs + // will go through all migrations including the very first one. In this case, the first + // migration's job is to bring any older DB to the same consistent state. + readonly migrations: ReadonlyArray; +} + +export type DBFunc = (db: SQLiteDB) => Promise; + +export enum OpenMode { + OPEN_CREATE, // Open DB or create if doesn't exist (the default mode for sqlite3 module) + OPEN_EXISTING, // Open DB or fail if doesn't exist + OPEN_READONLY, // Open DB in read-only mode or fail if doens't exist. + CREATE_EXCL, // Create new DB or fail if it already exists. +} + +/** + * An interface implemented both by SQLiteDB and DocStorage (by forwarding). Methods + * documented in SQLiteDB. + */ +export interface ISQLiteDB { + exec(sql: string): Promise; + run(sql: string, ...params: any[]): Promise; + get(sql: string, ...params: any[]): Promise; + all(sql: string, ...params: any[]): Promise; + prepare(sql: string, ...params: any[]): Promise; + execTransaction(callback: () => Promise): Promise; + runAndGetId(sql: string, ...params: any[]): Promise; + requestVacuum(): Promise; +} + +/** + * Wrapper around sqlite3.Database. This class provides many of the same methods, but promisified. + * In addition, it offers: + * + * SQLiteDB.openDB(): Opens a DB, and initialize or migrate it to correct schema. + * db.execTransaction(cb): Runs a callback in the context of a new DB transaction. + */ +export class SQLiteDB { + /** + * Opens a database or creates a new one, according to OpenMode enum. The schemaInfo specifies + * how to initialize a new database, and how to migrate an existing one from an older version. + * If the database was migrated, its "migrationBackupPath" property will be set. + * + * If a migration was needed but failed, the DB remains unchanged, and gets opened anyway. + * We report the migration error, and expose it via .migrationError property. + */ + public static async openDB(dbPath: string, schemaInfo: SchemaInfo, + mode: OpenMode = OpenMode.OPEN_CREATE): Promise { + const db = await SQLiteDB.openDBRaw(dbPath, mode); + const userVersion: number = await db.getMigrationVersion(); + + // It's possible that userVersion is 0 for a non-empty DB if it was created without this + // module. In that case, we apply migrations starting with the first one. + if (userVersion === 0 && (await isEmpty(db))) { + await db._initNewDB(schemaInfo); + } else if (mode === OpenMode.CREATE_EXCL) { + await db.close(); + throw new ErrorWithCode('EEXISTS', `EEXISTS: Database already exists: ${dbPath}`); + } else { + // Don't attempt migrations in OPEN_READONLY mode. + if (mode === OpenMode.OPEN_READONLY) { + const targetVer: number = schemaInfo.migrations.length; + if (userVersion < targetVer) { + db._migrationError = new Error(`SQLiteDB[${dbPath}] needs migration but is readonly`); + } + } else { + try { + db._migrationBackupPath = await db._migrate(userVersion, schemaInfo); + } catch (err) { + db._migrationError = err; + } + } + await db._reportSchemaDiscrepancies(schemaInfo); + } + return db; + } + + /** + * Opens a database or creates a new one according to OpenMode value. Does not check for or do + * any migrations. + */ + public static async openDBRaw(dbPath: string, + mode: OpenMode = OpenMode.OPEN_CREATE): Promise { + const sqliteMode: number = + // tslint:disable-next-line:no-bitwise + (mode === OpenMode.OPEN_READONLY ? sqlite3.OPEN_READONLY : sqlite3.OPEN_READWRITE) | + (mode === OpenMode.OPEN_CREATE || mode === OpenMode.CREATE_EXCL ? sqlite3.OPEN_CREATE : 0); + + let _db: sqlite3.Database; + await fromCallback(cb => { _db = new sqlite3.Database(dbPath, sqliteMode, cb); }); + + if (SQLiteDB._addOpens(dbPath, 1) > 1) { + log.warn("SQLiteDB[%s] avoid opening same DB more than once", dbPath); + } + return new SQLiteDB(_db!, dbPath); + } + + /** + * Reads the migration version from the database without any attempts to migrate it. + */ + public static async getMigrationVersion(dbPath: string): Promise { + const db = await SQLiteDB.openDBRaw(dbPath, OpenMode.OPEN_READONLY); + try { + return await db.getMigrationVersion(); + } finally { + await db.close(); + } + } + + // It is a bad idea to open the same database file multiple times, because simultaneous use can + // cause SQLITE_BUSY errors, and artificial delays (default of 1 sec) when there is contention. + // We keep track of open DB paths, and warn if one is opened multiple times. + private static _openPaths: Map = new Map(); + + // Convert the "create" function from schemaInfo into a DBMetadata object that describes the + // tables, columns, and types. This is used for checking if an open database matches the + // schema we expect, including after a migration, and reporting discrepancies. + private static async _getExpectedMetadata(schemaInfo: SchemaInfo): Promise { + // We cache the result and associate it with the create function, since it's not that cheap to + // build. To build the metadata, we open an in-memory DB and apply "create" function to it. + // Note that for tiny DBs it takes <10ms. + if (!dbMetadataCache.has(schemaInfo.create)) { + const db = await SQLiteDB.openDB(':memory:', schemaInfo, OpenMode.CREATE_EXCL); + dbMetadataCache.set(schemaInfo.create, await db.collectMetadata()); + await db.close(); + } + return dbMetadataCache.get(schemaInfo.create)!; + } + + // Private helper to keep track of opens for the same path. Returns the number of times this + // path is open, after adding the delta. Use delta of +1 for open, -1 for close. + private static _addOpens(dbPath: string, delta: number): number { + const newCount = (SQLiteDB._openPaths.get(dbPath) || 0) + delta; + if (newCount > 0) { + SQLiteDB._openPaths.set(dbPath, newCount); + } else { + SQLiteDB._openPaths.delete(dbPath); + } + return newCount; + } + + + private _prevTransaction: Promise = Promise.resolve(); + private _inTransaction: boolean = false; + private _migrationBackupPath: string|null = null; + private _migrationError: Error|null = null; + private _needVacuum: boolean = false; + + private constructor(private _db: sqlite3.Database, private _dbPath: string) { + // Default database to serialized execution. See https://github.com/mapbox/node-sqlite3/wiki/Control-Flow + // This isn't enough for transactions, which we serialize explicitly. + this._db.serialize(); + } + + + /** + * If a DB was migrated on open, this will be set to the path of the pre-migration backup copy. + * If migration failed, open throws with unchanged DB and no backup file. + */ + public get migrationBackupPath(): string|null { return this._migrationBackupPath; } + + /** + * If a needed migration failed, the DB will be opened anyway, with this property set to the + * error. E.g. you may use it like so: + * sdb = await SQLiteDB.openDB(...) + * if (sdb.migrationError) { throw sdb.migrationError; } + */ + public get migrationError(): Error|null { return this._migrationError; } + + // The following methods mirror https://github.com/mapbox/node-sqlite3/wiki/API, but return + // Promises. We use fromCallback() rather than use promisify, to get better type-checking. + + public exec(sql: string): Promise { + return fromCallback(cb => this._db.exec(sql, cb)); + } + + public run(sql: string, ...params: any[]): Promise { + return fromCallback(cb => this._db.run(sql, ...params, cb)); + } + + public get(sql: string, ...params: any[]): Promise { + return fromCallback(cb => this._db.get(sql, ...params, cb)); + } + + public all(sql: string, ...params: any[]): Promise { + return fromCallback(cb => this._db.all(sql, ...params, cb)); + } + + public allMarshal(sql: string, ...params: any[]): Promise { + // allMarshal isn't in the typings, because it is our addition to our fork of sqlite3 JS lib. + return fromCallback(cb => (this._db as any).allMarshal(sql, ...params, cb)); + } + + public prepare(sql: string, ...params: any[]): Promise { + let stmt: sqlite3.Statement; + // The original interface is a little strange; we resolve to Statement if prepare() succeeded. + return fromCallback(cb => { stmt = this._db.prepare(sql, ...params, cb); }).then(() => stmt); + } + + /** + * VACUUM the DB either immediately or, if in a transaction, after that transaction. + */ + public async requestVacuum(): Promise { + if (this._inTransaction) { + this._needVacuum = true; + return false; + } + await this.exec("VACUUM"); + log.info("SQLiteDB[%s]: DB VACUUMed", this._dbPath); + this._needVacuum = false; + return true; + } + + /** + * Run each of the statements in turn. Each statement is either a string, or an array of arguments + * to db.run, e.g. [sqlString, [params...]]. + */ + public runEach(...statements: Array): Promise { + return each(statements, (stmt: any) => { + return (Array.isArray(stmt) ? this.run(stmt[0], ...stmt[1]) : + this.exec(stmt)) + .catch(err => { log.warn(`SQLiteDB: Failed to run ${stmt}`); throw err; }); + }); + } + + public close(): Promise { + return fromCallback(cb => this._db.close(cb)) + .then(() => { SQLiteDB._addOpens(this._dbPath, -1); }); + } + + /** + * As for run(), but captures the last_insert_rowid after the statement executes. This + * is sqlite's rowid for the last insert made on this database connection. This method + * is only useful if the sql is actually an INSERT operation, but we don't check this. + */ + public runAndGetId(sql: string, ...params: any[]): Promise { + return new Promise((resolve, reject) => { + this._db.run(sql, ...params, function(this: any, err: any) { + if (err) { + reject(err); + } else { + resolve(this.lastID); + } + }); + }); + } + + /** + * Runs callback() in the context of a new DB transaction, committing on success and rolling + * back on error in the callback. The callback may return a promise, which will be waited for. + * The callback is called with no arguments. + * + * This method can be nested. The result is one big merged transaction that will succeed or + * roll back as a single unit. + */ + public async execTransaction(callback: () => Promise): Promise { + if (this._inTransaction) { + return callback(); + } + let outerResult; + try { + outerResult = await (this._prevTransaction = this._execTransactionImpl(async () => { + this._inTransaction = true; + let innerResult; + try { + innerResult = await callback(); + } finally { + this._inTransaction = false; + } + return innerResult; + })); + } finally { + if (this._needVacuum) { + await this.requestVacuum(); + } + } + return outerResult; + } + + /** + * Returns the 'user_version' saved in the database that reflects the current DB schema. It is 0 + * initially, and we update it to 1 or higher when initializing or migrating the database. + */ + public async getMigrationVersion(): Promise { + const row = await this.get("PRAGMA user_version"); + return (row && row.user_version) || 0; + } + + /** + * Creates a DBMetadata object mapping DB's table names to column names to column types. Used + * for reporting discrepancies in DB schema, and exposed for tests. + * + * Optionally, a list of table names can be supplied, and metadata will be omitted for any + * tables not named in that list. + */ + public async collectMetadata(names?: string[]): Promise { + const tables = await this.all("SELECT name FROM sqlite_master WHERE type='table'"); + const metadata: DBMetadata = {}; + for (const t of tables) { + if (names && !names.includes(t.name)) { continue; } + const infoRows = await this.all(`PRAGMA table_info(${quoteIdent(t.name)})`); + const columns = fromPairs(infoRows.map(r => [r.name, r.type])); + metadata[t.name] = columns; + } + return metadata; + } + + // Implementation of execTransction. + private async _execTransactionImpl(callback: () => Promise): Promise { + // We need to swallow errors, so that one failed transaction doesn't cause the next one to fail. + await this._prevTransaction.catch(noop); + await this.exec("BEGIN"); + try { + const value = await callback(); + await this.exec("COMMIT"); + return value; + } catch (err) { + try { + await this.exec("ROLLBACK"); + } catch (rollbackErr) { + log.error("SQLiteDB[%s]: Rollback failed: %s", this._dbPath, rollbackErr); + } + throw err; // Throw the original error from the transaction. + } + } + + /** + * Applies schemaInfo.create function to initialize a new DB. + */ + private async _initNewDB(schemaInfo: SchemaInfo): Promise { + await this.execTransaction(async () => { + const targetVer: number = schemaInfo.migrations.length; + await schemaInfo.create(this); + await this.exec(`PRAGMA user_version = ${targetVer}`); + }); + } + + /** + * Applies migrations to this database according to MigrationInfo. In all cases, checks the + * database schema against MigrationInfo.currentSchema, and warns of discrepancies. + * + * If migration succeeded, it leaves a backup file and returns its path. If no migration was + * needed, returns null. If migration failed, leaves DB unchanged and throws Error. + */ + private async _migrate(actualVer: number, schemaInfo: SchemaInfo): Promise { + const targetVer: number = schemaInfo.migrations.length; + let backupPath: string|null = null; + + if (actualVer > targetVer) { + log.warn("SQLiteDB[%s]: DB is at version %s ahead of target version %s", + this._dbPath, actualVer, targetVer); + } else if (actualVer < targetVer) { + log.info("SQLiteDB[%s]: DB needs migration from version %s to %s", + this._dbPath, actualVer, targetVer); + const versions = range(actualVer, targetVer); + backupPath = await createBackupFile(this._dbPath, actualVer); + try { + await this.execTransaction(async () => { + for (const versionNum of versions) { + await schemaInfo.migrations[versionNum](this); + } + await this.exec(`PRAGMA user_version = ${targetVer}`); + }); + // After a migration, reduce the sqlite file size. This must be run outside a transaction. + await this.run("VACUUM"); + + log.info("SQLiteDB[%s]: DB backed up to %s, migrated to %s", + this._dbPath, backupPath, targetVer); + } catch (err) { + // If the transaction failed, we trust SQLite to have left the DB in unmodified state, so + // we remove the pointless backup. + await fse.remove(backupPath); + backupPath = null; + log.warn("SQLiteDB[%s]: DB migration from %s to %s failed: %s", + this._dbPath, actualVer, targetVer, err); + err.message = `SQLiteDB[${this._dbPath}] migration to ${targetVer} failed: ${err.message}`; + throw err; + } + } + return backupPath; + } + + private async _reportSchemaDiscrepancies(schemaInfo: SchemaInfo): Promise { + // Regardless of where we started, warn if DB doesn't match expected schema. + const expected = await SQLiteDB._getExpectedMetadata(schemaInfo); + const metadata = await this.collectMetadata(Object.keys(expected)); + for (const tname in expected) { + if (expected.hasOwnProperty(tname) && !isEqual(metadata[tname], expected[tname])) { + log.warn("SQLiteDB[%s]: table %s does not match schema: %s != %s", + this._dbPath, tname, JSON.stringify(metadata[tname]), JSON.stringify(expected[tname])); + } + } + } +} + +// Every SchemaInfo.create function determines a DB structure. We can get it by initializing a +// dummy DB, and we use it to do sanity checking, in particular after migrations. To avoid +// creating dummy DBs multiple times, the result is cached, keyed by the "create" function itself. +const dbMetadataCache: Map = new Map(); +interface DBMetadata { + [tableName: string]: { + [colName: string]: string; // Maps column name to SQLite type, e.g. "TEXT". + }; +} + +// Helper to see if a database is empty. +async function isEmpty(db: SQLiteDB): Promise { + return (await db.get("SELECT count(*) as count FROM sqlite_master"))!.count === 0; +} + +/** + * Copies filePath to "filePath.YYYY-MM-DD.V0[-N].bak", adding "-N" suffix (starting at "-2") if + * needed to ensure the path is new. Returns the backup path. + */ +async function createBackupFile(filePath: string, versionNum: number): Promise { + const backupPath = await docUtils.createNumberedTemplate( + `${filePath}.${timeFormat('D', new Date())}.V${versionNum}{NUM}.bak`, + docUtils.createExclusive); + await docUtils.copyFile(filePath, backupPath); + return backupPath; +} + +/** + * Validate and quote SQL identifiers such as table and column names. + */ +export function quoteIdent(ident: string): string { + assert(/^\w+$/.test(ident), `SQL identifier is not valid: ${ident}`); + return `"${ident}"`; +} diff --git a/app/server/lib/SafePythonComponent.ts b/app/server/lib/SafePythonComponent.ts new file mode 100644 index 00000000..4223e73c --- /dev/null +++ b/app/server/lib/SafePythonComponent.ts @@ -0,0 +1,73 @@ +import {LocalPlugin} from 'app/common/plugin'; +import {BaseComponent, createRpcLogger} from 'app/common/PluginInstance'; +import {GristServer} from 'app/server/lib/GristServer'; +import {ISandbox} from 'app/server/lib/ISandbox'; +import * as log from 'app/server/lib/log'; +import {IMsgCustom, IMsgRpcCall} from 'grain-rpc'; +import * as path from 'path'; + +// TODO safePython component should be able to call other components function +// TODO calling a function on safePython component with a name that was not register chould fail +// gracefully. + +/** + * The safePython component used by a PluginInstance. + * + * It uses `NSandbox` implementation of rpc for calling methods within the sandbox. + */ +export class SafePythonComponent extends BaseComponent { + + private _sandbox: ISandbox; + private _logMeta: log.ILogMeta; + + // safe python component does not need pluginInstance.rpc because it is not possible to forward + // calls to other component from within python + constructor(private _localPlugin: LocalPlugin, + private _mainPath: string, private _tmpDir: string, + docName: string, private _server: GristServer, + rpcLogger = createRpcLogger(log, `PLUGIN ${_localPlugin.id}/${_mainPath} SafePython:`)) { + super(_localPlugin.manifest, rpcLogger); + this._logMeta = {plugin: _localPlugin.id, docId: docName}; + } + + /** + * `SafePythonComponent` activation creates the Sandbox. Throws if the plugin has no `safePyton` + * components. + */ + protected async activateImplementation(): Promise { + if (!this._tmpDir) { + throw new Error("Sanbox should have a tmpDir"); + } + this._sandbox = this._server.create.NSandbox({ + entryPoint: this._mainPath, + sandboxMount: path.join(this._localPlugin.path, 'sandbox'), + importMount: this._tmpDir, + logTimes: true, + logMeta: this._logMeta, + }); + } + + protected async deactivateImplementation(): Promise { + log.info('SafePython deactivating ...'); + if (!this._sandbox) { + log.info(' sandbox is undefined'); + } + if (this._sandbox) { + await this._sandbox.shutdown(); + log.info('SafePython done deactivating the sandbox'); + delete this._sandbox; + } + } + + protected doForwardCall(c: IMsgRpcCall): Promise { + if (!this._sandbox) { throw new Error("Component should have be activated"); } + const {meth, iface, args} = c; + const funcName = meth === "invoke" ? iface : iface + "." + meth; + return this._sandbox.pyCall(funcName, ...args); + } + + protected doForwardMessage(c: IMsgCustom): Promise { + throw new Error("Forwarding messages to python sandbox is not supported"); + } + +} diff --git a/app/server/lib/ServerColumnGetters.ts b/app/server/lib/ServerColumnGetters.ts new file mode 100644 index 00000000..6679b247 --- /dev/null +++ b/app/server/lib/ServerColumnGetters.ts @@ -0,0 +1,41 @@ +import {ColumnGetters} from 'app/common/ColumnGetters'; +import * as gristTypes from 'app/common/gristTypes'; + +/** + * + * An implementation of ColumnGetters for the server, currently + * drawing on the data and metadata prepared for CSV export. + * + */ +export class ServerColumnGetters implements ColumnGetters { + private _rowIndices: Map; + private _colIndices: Map; + + constructor(rowIds: number[], private _dataByColId: {[colId: string]: any}, private _columns: any[]) { + this._rowIndices = new Map(rowIds.map((rowId, r) => [rowId, r] as [number, number])); + this._colIndices = new Map(_columns.map(col => [col.id, col.colId] as [number, string])); + } + + public getColGetter(colRef: number): ((rowId: number) => any) | null { + const colId = this._colIndices.get(colRef); + if (colId === undefined) { + return null; + } + const col = this._dataByColId[colId]; + return rowId => { + const idx = this._rowIndices.get(rowId); + if (idx === undefined) { + return null; + } + return col[idx]; + }; + } + + public getManualSortGetter(): ((rowId: number) => any) | null { + const manualSortCol = this._columns.find(c => c.colId === gristTypes.MANUALSORT); + if (!manualSortCol) { + return null; + } + return this.getColGetter(manualSortCol.id); + } +} diff --git a/app/server/lib/ServerMetrics.js b/app/server/lib/ServerMetrics.js new file mode 100644 index 00000000..2bbaee85 --- /dev/null +++ b/app/server/lib/ServerMetrics.js @@ -0,0 +1,205 @@ +const _ = require('underscore'); +const net = require('net'); +const Promise = require('bluebird'); +const log = require('./log'); +const MetricCollector = require('app/common/MetricCollector'); +const metricConfig = require('app/common/metricConfig'); +const shutdown = require('./shutdown'); +const version = require('app/common/version'); +const crypto = require('crypto'); + +// Grist Metrics EC2 instance host and port +const host = 'metrics.getgrist.com'; +const port = '2023'; // Plain-text port of carbon-aggregator + +// Global reference to an instance of this class established in the constuctor. +var globalServerMetrics = null; + +/** + * Server-facing class for initializing server metrics collection. + * Establishes interval attempts to push measured server metrics to the prometheus PushGateway + * on creation. + * @param {Object} user - Instance of User.js server class, which contains config settings. + */ +function ServerMetrics() { + MetricCollector.call(this); + this.socket = null; + // Randomly generated id to differentiate between metrics from this server and others. + this.serverId = crypto.randomBytes(8).toString('hex'); + this.serverMetrics = this.initMetricTools(metricConfig.serverMetrics); + this.clientNames = null; + this.enabled = false; + // Produce the prefix string for all metrics. + // NOTE: If grist-rt is used instead of grist-raw for some metrics, this must be changed. + let versionStr = version.version.replace(/\W/g, '-'); + let channelStr = version.channel.replace(/\W/g, '-'); + this._prefix = `grist-raw.instance.${channelStr}.${versionStr}`; + + globalServerMetrics = this; + + // This will not send metrics when they are disabled since there is a check in pushMetrics. + shutdown.addCleanupHandler(null, () => this.attemptPush()); +} +_.extend(ServerMetrics.prototype, MetricCollector.prototype); + +/** + * Checks the given preferences object from the user configuration and starts pushing metrics + * to carbon if metrics are enabled. Otherwise, ends the socket connection if there is one. + */ +ServerMetrics.prototype.handlePreferences = function(config) { + config = config || {}; + this.enabled = config.enableMetrics; + Promise.resolve(this.enabled && this._connectSocket()) + .then(() => { + if (this.enabled) { + this._push = setTimeout(() => this.attemptPush(), metricConfig.SERVER_PUSH_INTERVAL); + } else if (this.socket) { + this.socket.end(); + } + }); +}; + +ServerMetrics.prototype.disable = function() { + this.enabled = false; + if (this._push) { + clearTimeout(this._push); + this._push = null; + } + if (this._collect) { + clearTimeout(this._collect); + this._collect = null; + } +} + + +/** + * Returns a promise for a socket connection to the Carbon metrics collection server. + * The promise will not fail because of connection errors, rather it will be continuously + * re-evaluated until it connects. The retry rate is specified in metricConfig.js + */ +ServerMetrics.prototype._connectSocket = function() { + if (!this.enabled) { return Promise.resolve(); } + var socket = null; + log.info('Attempting connection to Carbon metrics server'); + return new Promise((resolve, reject) => { + socket = net.connect({host: host, port: port}, () => { + log.info('Connected to Carbon metrics server'); + this.socket = socket; + resolve(); + }); + socket.setEncoding('utf8'); + socket.on('error', err => { + log.warn('Carbon metrics connection error: %s', err); + if (this.socket) { + this.socket.end(); + this.socket = null; + } + reject(err); + }); + }) + .catch(() => { + return Promise.delay(metricConfig.CONN_RETRY) + .then(() => this._connectSocket()); + }); +}; + +// Returns a map from metric names (as entered in metricConfig.js) to their metricTools. +ServerMetrics.prototype.getMetrics = function() { + return this.serverMetrics; +}; + +// Pushes ready server and client metrics to the aggregator +ServerMetrics.prototype.pushMetrics = function(metrics) { + if (this.enabled) { + return this._request(metrics.join("")) + .finally(() => { + this._push = setTimeout(() => this.attemptPush(), metricConfig.SERVER_PUSH_INTERVAL); + }); + } +}; + +ServerMetrics.prototype._request = function(text) { + return new Promise(resolve => { + if (!this.enabled) { + resolve(); + return; + } + this.socket.write(text, 'utf8', () => { + log.info('Pushed metrics to Carbon'); + resolve(); + }); + }) + .catch(() => { + return this._connectSocket() + .then(() => this._request(text)); + }); +}; + +/** + * Function exposed to comm interface to provide server with client list of metrics. + * Used so that ServerMetrics can associate indices to client metric names. + * @param {Array} metricNames - A list of client metric names in the order in which values will be sent. + */ +ServerMetrics.prototype.registerClientMetrics = function(client, metricNames) { + this.clientNames = metricNames; +}; + +/** + * Function exposed to comm interface to allow client metrics to be pushed to this file, + * so that they may in turn be pushed to Carbon with the server metrics. + * @param {Array} data - A list of client buckets as defined in ClientMetrics.js's createBucket + */ +ServerMetrics.prototype.pushClientMetrics = function(client, data) { + // Merge ready client bucket metrics into ready server buckets. + if (!this.clientNames) { + throw new Error("Client metrics must be registered"); + } + data.forEach(clientBucket => { + // Label the bucket with the client id so that clients' metrics do not replace one another + let clientData = clientBucket.values.map((val, i) => { + return this._stringifyMetric(this.clientNames[i], client.clientId, val, clientBucket.startTime); + }).join(""); + this.queueBucket(clientData); + }); +}; + +ServerMetrics.prototype.get = function(name) { + this.prepareCompletedBuckets(Date.now()); + return this.serverMetrics[name]; +}; + +/** + * Creates string bucket with metrics in carbon's text format. + * For details, see phriction documentation: https://phab.getgrist.com/w/metrics/ + */ +ServerMetrics.prototype.createBucket = function(bucketStart) { + var data = []; + var bucketEnd = bucketStart + metricConfig.BUCKET_SIZE; + this.forEachBucketMetric(bucketEnd, tool => { + if (tool.getValue(bucketEnd) !== null) { + data.push(this._stringifyMetric(tool.getName(), this.serverId, tool.getValue(bucketEnd), bucketStart)); + } + }); + return data.join(""); +}; + +// Helper to stringify individual metrics for carbon's text format. +ServerMetrics.prototype._stringifyMetric = function(name, id, val, startTime) { + // Server/client id is added to name for differentiating inputs to aggregator + return `${this._prefix}.${name}.${id} ${val} ${startTime/1000}\n`; +}; + +/** + * Static get method to retreive server metric recording tools. + * IMPORTANT: Usage involves the side effect of updating completed buckets and + * adding them to a ready object. get() results should not be assigned to variables and + * reused, rather get() should be called each time a metric is needed. + */ +ServerMetrics.get = function(name) { + if (!globalServerMetrics) { + throw new Error('Must create ServerMetrics instance to access server metrics.'); + } + return globalServerMetrics.get(name); +}; + +module.exports = ServerMetrics; diff --git a/app/server/lib/Sessions.ts b/app/server/lib/Sessions.ts new file mode 100644 index 00000000..3f780a16 --- /dev/null +++ b/app/server/lib/Sessions.ts @@ -0,0 +1,107 @@ +import {ScopedSession} from 'app/server/lib/BrowserSession'; +import * as Comm from 'app/server/lib/Comm'; +import {GristServer} from 'app/server/lib/GristServer'; +import {cookieName, SessionStore} from 'app/server/lib/gristSessions'; +import {IInstanceManager} from 'app/server/lib/IInstanceManager'; +import {ILoginSession} from 'app/server/lib/ILoginSession'; +import * as cookie from 'cookie'; +import * as cookieParser from 'cookie-parser'; +import {Request} from 'express'; + +interface Session { + scopedSession: ScopedSession; + loginSession?: ILoginSession; +} + +/** + * + * A collection of all the sessions relevant to this instance of Grist. + * + * This collection was previously maintained by the Comm object. This + * class is added as a stepping stone to disentangling session management + * from code related to websockets. + * + * The collection caches all existing interfaces to sessions. + * LoginSessions play an important role in standalone Grist and address + * end-to-end sharing concerns. ScopedSessions play an important role in + * hosted Grist and address per-organization scoping of identity. + * + * TODO: now this is separated out, we could refactor to share sessions + * across organizations. Currently, when a user moves between organizations, + * the session interfaces are not shared. This was for simplicity in working + * with existing code. + * + */ +export class Sessions { + private _sessions = new Map(); + + constructor(private _sessionSecret: string, private _sessionStore: SessionStore, private _server: GristServer) { + } + + /** + * Get the session id and organization from the request, and return the + * identified session. + */ + public getOrCreateSessionFromRequest(req: Request): Session { + const sid = this.getSessionIdFromRequest(req); + const org = (req as any).org; + if (!sid) { throw new Error("session not found"); } + return this.getOrCreateSession(sid, org); + } + + /** + * Get or create a session given the session id and organization name. + */ + public getOrCreateSession(sid: string, domain: string): Session { + const key = this._getSessionOrgKey(sid, domain); + if (!this._sessions.has(key)) { + const scopedSession = new ScopedSession(sid, this._sessionStore, domain); + this._sessions.set(key, {scopedSession}); + } + return this._sessions.get(key)!; + } + + /** + * Access a LoginSession interface, creating it if necessary. For creation, + * purposes, Comm, and optionally InstanceManager objects are needed. + * + */ + public getOrCreateLoginSession(sid: string, domain: string, comm: Comm, + instanceManager: IInstanceManager|null): ILoginSession { + const sess = this.getOrCreateSession(sid, domain); + if (!sess.loginSession) { + sess.loginSession = this._server.create.LoginSession(comm, sid, domain, sess.scopedSession, + instanceManager); + } + return sess.loginSession; + } + + /** + * Returns the sessionId from the signed grist cookie. + */ + public getSessionIdFromCookie(gristCookie: string) { + return cookieParser.signedCookie(gristCookie, this._sessionSecret); + } + + /** + * Get the session id from the grist cookie. Returns null if no cookie found. + */ + public getSessionIdFromRequest(req: Request): string|null { + if (req.headers.cookie) { + const cookies = cookie.parse(req.headers.cookie); + const sessionId = this.getSessionIdFromCookie(cookies[cookieName]); + return sessionId; + } + return null; + } + + /** + * Get a per-organization, per-session key. + * Grist has historically cached sessions in memory by their session id. + * With the introduction of per-organization identity, that cache is now + * needs to be keyed by the session id and organization name. + */ + private _getSessionOrgKey(sid: string, domain: string): string { + return `${sid}__${domain}`; + } +} diff --git a/app/server/lib/Sharing.ts b/app/server/lib/Sharing.ts new file mode 100644 index 00000000..89412d03 --- /dev/null +++ b/app/server/lib/Sharing.ts @@ -0,0 +1,387 @@ +import {ActionBundle, LocalActionBundle, UserActionBundle} from 'app/common/ActionBundle'; +import {ActionInfo, Envelope, getEnvContent} from 'app/common/ActionBundle'; +import {DocAction, UserAction} from 'app/common/DocActions'; +import {allToken, Peer} from 'app/common/sharing'; +import {timeFormat} from 'app/common/timeFormat'; +import * as log from 'app/server/lib/log'; +import {shortDesc} from 'app/server/lib/shortDesc'; +import * as assert from 'assert'; +import * as Deque from 'double-ended-queue'; +import {ActionHistory, asActionGroup} from './ActionHistory'; +import {ActiveDoc} from './ActiveDoc'; +import {Client} from './Client'; +import {WorkCoordinator} from './WorkCoordinator'; + +// Describes the request to apply a UserActionBundle. It includes a Client (so that broadcast +// message can set `.fromSelf` property), and methods to resolve or reject the promise for when +// the action is applied. Note that it may not be immediate in case we are in the middle of +// processing hub actions or rebasing. +interface UserRequest { + action: UserActionBundle; + client: Client|null; + resolve(result: UserResult): void; + reject(err: Error): void; +} + +// The result of applying a UserRequest, used to resolve the promise. It includes the retValues +// (one for each UserAction in the bundle) and the actionNum of the applied LocalActionBundle. +interface UserResult { + actionNum: number; + retValues: any[]; + isModification: boolean; +} + +// Internally-used enum to distinguish if applied actions should be logged as local or shared. +enum Branch { Local, Shared } + +export class Sharing { + protected _activeDoc: ActiveDoc; + protected _actionHistory: ActionHistory; + protected _hubQueue: Deque = new Deque(); + protected _pendingQueue: Deque = new Deque(); + protected _workCoordinator: WorkCoordinator; + + constructor(activeDoc: ActiveDoc, actionHistory: ActionHistory) { + // TODO actionHistory is currently unused (we use activeDoc.actionLog). + assert(actionHistory.isInitialized()); + + this._activeDoc = activeDoc; + this._actionHistory = actionHistory; + this._workCoordinator = new WorkCoordinator(() => this._doNextStep()); + } + + /** Initialize the sharing for a previously-shared doc. */ + public async openSharedDoc(hub: any, docId: string): Promise { + throw new Error('openSharedDoc not implemented'); + } + + /** Initialize the sharing for a newly-shared doc. */ + public async createSharedDoc(hub: any, docId: string, docName: string, peers: Peer[]): Promise { + throw new Error('openSharedDoc not implemented'); + } + + /** + * Returns whether this doc is shared. It's shared if and only if HubDocClient is set (though it + * may be disconnected). + */ + public isShared(): boolean { return false; } + + public isSharingActivated(): boolean { return false; } + + /** Returns the instanceId if the doc is shared or null otherwise. */ + public get instanceId(): string|null { return null; } + + public isOwnEnvelope(recipients: string[]): boolean { return true; } + + public async sendLocalAction(): Promise { + throw new Error('sendLocalAction not implemented'); + } + + public async shareDoc(docName: string, peers: Peer[]): Promise { + throw new Error('shareDoc not implemented'); + } + + public async removeInstanceFromDoc(): Promise { + throw new Error('removeInstanceFromDoc not implemented'); + } + + /** + * The only public interface. This may be called at any time, including while rebasing. + * WorkCoordinator ensures that actual work will only happen once other work finishes. + */ + public addUserAction(userRequest: UserRequest) { + this._pendingQueue.push(userRequest); + this._workCoordinator.ping(); + } + + // Returns a promise if there is some work happening, or null if there isn't. + private _doNextStep(): Promise|null { + if (this._hubQueue.isEmpty()) { + if (!this._pendingQueue.isEmpty()) { + return this._applyLocalAction(); + } else if (this.isSharingActivated() && this._actionHistory.haveLocalUnsent()) { + return this.sendLocalAction(); + } else { + return null; + } + } else { + if (!this._actionHistory.haveLocalActions()) { + return this._applyHubAction(); + } else { + return this._mergeInHubAction(); + } + } + } + + private async _applyLocalAction(): Promise { + assert(this._hubQueue.isEmpty() && !this._pendingQueue.isEmpty()); + const userRequest: UserRequest = this._pendingQueue.shift()!; + try { + const ret = await this.doApplyUserActionBundle(userRequest.action, userRequest.client); + userRequest.resolve(ret); + } catch (e) { + log.warn("Unable to apply action...", e); + userRequest.reject(e); + } + } + + private async _applyHubAction(): Promise { + assert(!this._hubQueue.isEmpty() && !this._actionHistory.haveLocalActions()); + const action: ActionBundle = this._hubQueue.shift()!; + try { + await this.doApplySharedActionBundle(action); + } catch (e) { + log.error("Unable to apply hub action... skipping"); + } + } + + private async _mergeInHubAction(): Promise { + assert(!this._hubQueue.isEmpty() && this._actionHistory.haveLocalActions()); + + const action: ActionBundle = this._hubQueue.peekFront()!; + try { + const accepted = await this._actionHistory.acceptNextSharedAction(action.actionHash); + if (accepted) { + this._hubQueue.shift(); + } else { + await this._rebaseLocalActions(); + } + } catch (e) { + log.error("Unable to apply hub action... skipping"); + } + } + + private async _rebaseLocalActions(): Promise { + const rebaseQueue: Deque = new Deque(); + try { + await this.createCheckpoint(); + const actions: LocalActionBundle[] = await this._actionHistory.fetchAllLocal(); + assert(actions.length > 0); + await this.doApplyUserActionBundle(this._createUndo(actions), null); + rebaseQueue.push(...actions.map((a) => getUserActionBundle(a))); + await this._actionHistory.clearLocalActions(); + } catch (e) { + log.error("Can't undo local actions; sharing is off"); + await this.rollbackToCheckpoint(); + // TODO this.disconnect(); + // TODO errorState = true; + return; + } + assert(!this._actionHistory.haveLocalActions()); + + while (!this._hubQueue.isEmpty()) { + await this._applyHubAction(); + } + const rebaseFailures: Array<[UserActionBundle, UserActionBundle]> = []; + while (!rebaseQueue.isEmpty()) { + const action: UserActionBundle = rebaseQueue.shift()!; + const adjusted: UserActionBundle = this._mergeAdjust(action); + try { + await this.doApplyUserActionBundle(adjusted, null); + } catch (e) { + log.warn("Unable to apply rebased action..."); + rebaseFailures.push([action, adjusted]); + } + } + if (rebaseFailures.length > 0) { + await this.createBackupAtCheckpoint(); + // TODO we should notify the user too. + log.error('Rebase failed to reapply some of your actions, backup of local at...'); + } + await this.releaseCheckpoint(); + } + + // ====================================================================== + + private doApplySharedActionBundle(action: ActionBundle): Promise { + const userActions: UserAction[] = [ + ['ApplyDocActions', action.stored.map(envContent => envContent[1])] + ]; + return this.doApplyUserActions(action.info[1], userActions, Branch.Shared, null); + } + + private doApplyUserActionBundle(action: UserActionBundle, client: Client|null): Promise { + return this.doApplyUserActions(action.info, action.userActions, Branch.Local, client); + } + + private async doApplyUserActions(info: ActionInfo, userActions: UserAction[], + branch: Branch, client: Client|null): Promise { + const sandboxActionBundle = await this._activeDoc.applyActionsToDataEngine(userActions); + // A trivial action does not merit allocating an actionNum, + // logging, and sharing. Since we currently don't store + // calculated values in the database, it is best not to log the + // action that initializes them when the document is opened cold + // (without cached ActiveDoc) - otherwise we'll end up with spam + // log entries for each time the document is opened cold. + const trivial = (userActions.length === 1 && + userActions[0][0] === 'Calculate' && + sandboxActionBundle.stored.length === 0); + + const actionNum = trivial ? 0 : + (branch === Branch.Shared ? this._actionHistory.getNextHubActionNum() : + this._actionHistory.getNextLocalActionNum()); + + const localActionBundle: LocalActionBundle = { + actionNum, + // The ActionInfo should go into the envelope that includes all recipients. + info: [findOrAddAllEnvelope(sandboxActionBundle.envelopes), info], + envelopes: sandboxActionBundle.envelopes, + stored: sandboxActionBundle.stored, + calc: sandboxActionBundle.calc, + undo: getEnvContent(sandboxActionBundle.undo), + userActions, + actionHash: null, // Gets set below by _actionHistory.recordNext... + parentActionHash: null, // Gets set below by _actionHistory.recordNext... + }; + this._logActionBundle(`doApplyUserActions (${Branch[branch]})`, localActionBundle); + + // TODO Note that the sandbox may produce actions which are not addressed to us (e.g. when we + // have EDIT permission without VIEW). These are not sent to the browser or the database. But + // today they are reflected in the sandbox. Should we (or the sandbox) immediately undo the + // full change, and then redo only the actions addressed to ourselves? Let's cross that bridge + // when we come to it. For now we only log skipped envelopes as "alien" in _logActionBundle(). + const ownActionBundle: LocalActionBundle = this._filterOwnActions(localActionBundle); + + // Apply the action to the database, and record in the action log. + if (!trivial) { + await this._activeDoc.docStorage.execTransaction(async () => { + await this._activeDoc.docStorage.applyStoredActions(getEnvContent(ownActionBundle.stored)); + if (this.isShared() && branch === Branch.Local) { + // this call will compute an actionHash for localActionBundle + await this._actionHistory.recordNextLocalUnsent(localActionBundle); + } else { + // Before sharing is enabled, actions are immediately marked as "shared" (as if accepted + // by the hub). The alternative of keeping actions on the "local" branch until sharing is + // enabled is less suitable, because such actions could have empty envelopes, and cannot + // be shared. Once sharing is enabled, we would share a snapshot at that time. + await this._actionHistory.recordNextShared(localActionBundle); + } + if (client && client.clientId) { + this._actionHistory.setActionClientId(localActionBundle.actionHash!, client.clientId); + } + }); + } + await this._activeDoc.processActionBundle(ownActionBundle); + + // In the future, we'll save (and share) the result of applying one bundle of UserActions + // as a single ActionBundle with one actionNum. But the old ActionLog saves on UserAction + // per actionNum, using linkId to "bundle" them for the purpose of undo-redo. We simulate + // it here by breaking up ActionBundle into as many old-style ActionGroups as there are + // UserActions, and associating all DocActions with the first of these ActionGroups. + + // Broadcast the action to connected browsers. + const actionGroup = asActionGroup(this._actionHistory, localActionBundle, { + client, + retValues: sandboxActionBundle.retValues, + summarize: true, + }); + await this._activeDoc.docClients.broadcastDocMessage(client || null, 'docUserAction', { + actionGroup, + docActions: getEnvContent(localActionBundle.stored).concat( + getEnvContent(localActionBundle.calc)) + }); + return { + actionNum: localActionBundle.actionNum, + retValues: sandboxActionBundle.retValues, + isModification: sandboxActionBundle.stored.length > 0 + }; + } + + private _mergeAdjust(action: UserActionBundle): UserActionBundle { + // TODO: This is where we adjust actions after rebase, e.g. add delta to rowIds and such. + return action; + } + + /** + * Creates a UserActionBundle with a single 'ApplyUndoActions' action, which combines the undo + * actions addressed to ourselves from all of the passed-in LocalActionBundles. + */ + private _createUndo(localActions: LocalActionBundle[]): UserActionBundle { + assert(localActions.length > 0); + const undo: DocAction[] = []; + for (const local of localActions) { + undo.push(...local.undo); + } + const first = localActions[0]; + return { + info: { + time: Date.now(), + user: first.info[1].user, + inst: first.info[1].inst, + desc: "UNDO BEFORE REBASE", + otherId: 0, + linkId: 0, + }, + userActions: [['ApplyUndoActions', undo]] + }; + } + + // Our beautiful little checkpointing interface, used to handle errors during rebase. + private createCheckpoint() { /* TODO */ } + private releaseCheckpoint() { /* TODO */ } + private rollbackToCheckpoint() { /* TODO */ } + private createBackupAtCheckpoint() { /* TODO */ } + + /** + * Reduces a LocalActionBundle down to only those actions addressed to ourselves. + */ + private _filterOwnActions(localActionBundle: LocalActionBundle): LocalActionBundle { + const includeEnv: boolean[] = localActionBundle.envelopes.map( + (e) => this.isOwnEnvelope(e.recipients)); + + return Object.assign({}, localActionBundle, { + stored: localActionBundle.stored.filter((ea) => includeEnv[ea[0]]), + calc: localActionBundle.calc.filter((ea) => includeEnv[ea[0]]), + }); + } + + /** Log an action bundle to the debug log. */ + private _logActionBundle(prefix: string, actionBundle: ActionBundle) { + const includeEnv = actionBundle.envelopes.map((e) => this.isOwnEnvelope(e.recipients)); + log.debug("%s: ActionBundle #%s with #%s envelopes: %s", + prefix, actionBundle.actionNum, actionBundle.envelopes.length, + infoDesc(actionBundle.info[1])); + actionBundle.envelopes.forEach((env, i) => + log.debug("%s: env #%s: %s", prefix, i, env.recipients.join(' '))); + actionBundle.stored.forEach((envAction, i) => + log.debug("%s: stored #%s [%s%s]: %s", prefix, i, envAction[0], + (includeEnv[envAction[0]] ? "" : " alien"), + shortDesc(envAction[1]))); + actionBundle.calc.forEach((envAction, i) => + log.debug("%s: calc #%s [%s%s]: %s", prefix, i, envAction[0], + (includeEnv[envAction[0]] ? "" : " alien"), + shortDesc(envAction[1]))); + } +} + +/** + * Returns the index of the envelope containing the '#ALL' recipient, adding such an envelope to + * the provided array if it wasn't already there. + */ +export function findOrAddAllEnvelope(envelopes: Envelope[]): number { + const i = envelopes.findIndex(e => e.recipients.includes(allToken)); + if (i >= 0) { return i; } + envelopes.push({recipients: [allToken]}); + return envelopes.length - 1; +} + +/** + * Convert actionInfo to a concise human-readable description, for debugging. + */ +function infoDesc(info: ActionInfo): string { + const timestamp = timeFormat('A', new Date(info.time)); + const desc = info.desc ? ` desc=[${info.desc}]` : ''; + const otherId = info.otherId ? ` [otherId=${info.otherId}]` : ''; + const linkId = info.linkId ? ` [linkId=${info.linkId}]` : ''; + return `${timestamp} on ${info.inst} by ${info.user}${desc}${otherId}${linkId}`; +} + +/** + * Extract a UserActionBundle from a LocalActionBundle, which contains a superset of data. + */ +function getUserActionBundle(localAction: LocalActionBundle): UserActionBundle { + return { + info: localAction.info[1], + userActions: localAction.userActions + }; +} diff --git a/app/server/lib/TagChecker.ts b/app/server/lib/TagChecker.ts new file mode 100644 index 00000000..23abcb31 --- /dev/null +++ b/app/server/lib/TagChecker.ts @@ -0,0 +1,74 @@ +import {tbind} from 'app/common/tbind'; +import {NextFunction, Request, RequestHandler, Response} from 'express'; + +export type RequestWithTag = Request & {tag: string|null}; + +/** + * + * Middleware to handle a /v/TAG/ prefix on urls. + * + */ +export class TagChecker { + + // Use app.use(tagChecker.inspectTag) to strip /v/TAG/ from urls (if it is present). + // If the tag is present and matches what is expected, then `tag` is set on the request. + // If the tag is present but does not match what is expected, a 400 response is returned. + // If the tag is absent, `tag` is not set on the request. + public readonly inspectTag: RequestHandler = tbind(this._inspectTag, this); + + // Use app.get('/path', tagChecker.requireTag, ...) to serve something only if the tag was + // present in the url. If the tag was not present, the route will not match and express will + // look further. + public readonly requireTag: RequestHandler = tbind(this._requireTag, this); + + // pass in the tag to expect. + public constructor(public tag: string) { + } + + // Like requireTag but for use wrapping other handlers in app.use(). + // Whatever it wraps will be skipped if that tag was not set. + // See https://github.com/expressjs/express/issues/2591 + public withTag(handler: RequestHandler) { + return function fn(req: Request, resp: Response, next: NextFunction) { + if (!(req as RequestWithTag).tag) { return next(); } + return handler(req, resp, next); + }; + } + + // Removes tag from url if present. + // Returns [remainder, tagInUrl, isMatch] + private _removeTag(url: string): [string, string|null, boolean] { + if (url.startsWith('/v/')) { + const taggedUrl = url.match(/^\/v\/([a-zA-Z0-9.\-_]+)(\/.*)/); + if (taggedUrl) { + const tag = taggedUrl[1]; + // Turn off tag matching as we transition to serving + // static resources from CDN. We don't have version-sensitive + // routing, so under ordinary operation landing page html served + // by one home server could have its assets served by another home server. + // Once the CDN is active, those asset requests won't reach the home + // servers. TODO: turn tag matching back on when tag mismatches + // imply a bug. + return [taggedUrl[2], tag, true /* tag === this.tag */]; + } + } + return [url, null, true]; + } + + private async _inspectTag(req: Request, resp: Response, next: NextFunction) { + const [newUrl, urlTag, isOk] = this._removeTag(req.url); + if (!isOk) { + return resp.status(400).send({error: "Tag mismatch", + expected: this.tag, + received: urlTag}); + } + req.url = newUrl; + (req as RequestWithTag).tag = urlTag; + return next(); + } + + private async _requireTag(req: Request, resp: Response, next: NextFunction) { + if ((req as RequestWithTag).tag) { return next(); } + return next('route'); + } +} diff --git a/app/server/lib/TestingHooks.ts b/app/server/lib/TestingHooks.ts new file mode 100644 index 00000000..15cebf32 --- /dev/null +++ b/app/server/lib/TestingHooks.ts @@ -0,0 +1,205 @@ +import * as net from 'net'; + +import {UserProfile} from 'app/common/LoginSessionAPI'; +import {Deps as ActiveDocDeps} from 'app/server/lib/ActiveDoc'; +import * as Comm from 'app/server/lib/Comm'; +import {ILoginSession} from 'app/server/lib/ILoginSession'; +import * as log from 'app/server/lib/log'; +import {IMessage, Rpc} from 'grain-rpc'; +import {createCheckers} from 'ts-interface-checker'; +import {FlexServer} from './FlexServer'; +import {IInstanceManager} from './IInstanceManager'; +import {ITestingHooks} from './ITestingHooks'; +import ITestingHooksTI from './ITestingHooks-ti'; +import {connect, fromCallback} from './serverUtils'; + +export function startTestingHooks(socketPath: string, port: number, instanceManager: IInstanceManager, + comm: Comm, flexServer: FlexServer, + workerServers: FlexServer[]): Promise { + // Create socket server listening on the given path for testing connections. + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.on('error', reject); + server.on('listening', () => resolve(server)); + server.on('connection', socket => { + // On connection, create an Rpc object communicating over that socket. + const rpc = connectToSocket(new Rpc({logger: {}}), socket); + // Register the testing implementation. + rpc.registerImpl('testing', + new TestingHooks(port, instanceManager, comm, flexServer, workerServers), + createCheckers(ITestingHooksTI).ITestingHooks); + }); + server.listen(socketPath); + }); +} + +function connectToSocket(rpc: Rpc, socket: net.Socket): Rpc { + socket.setEncoding('utf8'); + socket.on('data', (buf: string) => rpc.receiveMessage(JSON.parse(buf))); + rpc.setSendMessage((m: IMessage) => fromCallback(cb => socket.write(JSON.stringify(m), 'utf8', cb))); + return rpc; +} + +export interface TestingHooksClient extends ITestingHooks { + close(): void; +} + +export async function connectTestingHooks(socketPath: string): Promise { + const socket = await connect(socketPath); + const rpc = connectToSocket(new Rpc({logger: {}}), socket); + return Object.assign(rpc.getStub('testing', createCheckers(ITestingHooksTI).ITestingHooks), { + close: () => socket.end(), + }); +} + +export class TestingHooks implements ITestingHooks { + constructor( + private _port: number, + private _instanceManager: IInstanceManager, + private _comm: Comm, + private _server: FlexServer, + private _workerServers: FlexServer[] + ) {} + + public getOwnPort(): number { + log.info("TestingHooks.getOwnPort called"); + return this._server.getOwnPort(); + } + + public getPort(): number { + log.info("TestingHooks.getPort called"); + return this._port; + } + + public async updateAuthToken(instId: string, authToken: string): Promise { + log.info("TestingHooks.updateAuthToken called with", instId, authToken); + const loginSession = this._getLoginSession(instId); + await loginSession.updateTokenForTesting(authToken); + } + + public async getAuthToken(instId: string): Promise { + log.info("TestingHooks.getAuthToken called with", instId); + const loginSession = this._getLoginSession(instId); + return await loginSession.getCurrentTokenForTesting(); + } + + public async useTestToken(instId: string, token: string): Promise { + log.info("TestingHooks.useTestToken called with", token); + const loginSession = this._getLoginSession(instId); + return await loginSession.useTestToken(token); + } + + public async setLoginSessionProfile(gristSidCookie: string, profile: UserProfile|null, org?: string): Promise { + log.info("TestingHooks.setLoginSessionProfile called with", gristSidCookie, profile, org); + const sessionId = this._comm.getSessionIdFromCookie(gristSidCookie); + const loginSession = this._comm.getOrCreateSession(sessionId, {org}); + return await loginSession.testSetProfile(profile); + } + + public async setServerVersion(version: string|null): Promise { + log.info("TestingHooks.setServerVersion called with", version); + this._comm.setServerVersion(version); + for (const server of this._workerServers) { + await server.comm.setServerVersion(version); + } + } + + public async disconnectClients(): Promise { + log.info("TestingHooks.disconnectClients called"); + this._comm.destroyAllClients(); + for (const server of this._workerServers) { + await server.comm.destroyAllClients(); + } + } + + public async commShutdown(): Promise { + log.info("TestingHooks.commShutdown called"); + await this._comm.testServerShutdown(); + for (const server of this._workerServers) { + await server.comm.testServerShutdown(); + } + } + + public async commRestart(): Promise { + log.info("TestingHooks.commRestart called"); + await this._comm.testServerRestart(); + for (const server of this._workerServers) { + await server.comm.testServerRestart(); + } + } + + // Set how long new clients will persist after disconnection. + // Call with 0 to return to default duration. + public async commSetClientPersistence(ttlMs: number) { + log.info("TestingHooks.setClientPersistence called with", ttlMs); + this._comm.testSetClientPersistence(ttlMs); + for (const server of this._workerServers) { + await server.comm.testSetClientPersistence(ttlMs); + } + } + + public async closeDocs(): Promise { + log.info("TestingHooks.closeDocs called"); + if (this._server) { + await this._server.closeDocs(); + } + for (const server of this._workerServers) { + await server.closeDocs(); + } + } + + public async setDocWorkerActivation(workerId: string, active: 'active'|'inactive'|'crash'): + Promise { + log.info("TestingHooks.setDocWorkerActivation called with", workerId, active); + for (const server of this._workerServers) { + if (server.worker.id === workerId || server.worker.publicUrl === workerId) { + switch (active) { + case 'active': + await server.restartListening(); + break; + case 'inactive': + await server.stopListening(); + break; + case 'crash': + await server.stopListening('crash'); + break; + } + return; + } + } + throw new Error(`could not find worker: ${workerId}`); + } + + public async flushAuthorizerCache(): Promise { + log.info("TestingHooks.flushAuthorizerCache called"); + this._server.dbManager.flushDocAuthCache(); + for (const server of this._workerServers) { + await server.dbManager.flushDocAuthCache(); + } + } + + // Returns a Map from docId to number of connected clients for all open docs across servers, + // but represented as an array of pairs, to be serializable. + public async getDocClientCounts(): Promise> { + log.info("TestingHooks.getDocClientCounts called"); + const counts = new Map(); + for (const server of [this._server, ...this._workerServers]) { + const c = await server.getDocClientCounts(); + for (const [key, val] of c) { + counts.set(key, (counts.get(key) || 0) + val); + } + } + return Array.from(counts); + } + + // Sets the seconds for ActiveDoc timeout, and returns the previous value. + public async setActiveDocTimeout(seconds: number): Promise { + const prev = ActiveDocDeps.ACTIVEDOC_TIMEOUT; + ActiveDocDeps.ACTIVEDOC_TIMEOUT = seconds; + return prev; + } + + private _getLoginSession(instId: string): ILoginSession { + return this._instanceManager.getLoginSession(instId); + } +} diff --git a/app/server/lib/Throttle.ts b/app/server/lib/Throttle.ts new file mode 100644 index 00000000..54a89984 --- /dev/null +++ b/app/server/lib/Throttle.ts @@ -0,0 +1,252 @@ +/** + * + * Simple CPU throttling implementation. + * + * For this setup, a sandbox attempting to use 100% of cpu over an + * extended period will end up throttled, in the steady-state, to + * 10% of cpu. + * + * Very simple mechanism to begin with. "ctime" is measured for the + * sandbox, being the cumulative time charged to the user (directly or + * indirectly) by the OS for that process. If the average increase in + * ctime over a time period is over 10% (targetRate) of that time period, + * throttling kicks in, and the process will be paused/unpaused via + * signals on a duty cycle. + * + * Left for future work: more careful shaping of CPU throttling, and + * factoring in a team-site level credit system or similar. + * + */ + +import * as pidusage from '@gristlabs/pidusage'; +import * as log from 'app/server/lib/log'; + +/** + * Parameters related to throttling. + */ +export interface ThrottleTiming { + dutyCyclePositiveMs: number; // when throttling, how much uninterrupted time to give + // the process before pausing it. The length of the + // non-positive cycle is chosen to achieve the desired + // cpu usage. + samplePeriodMs: number; // how often to sample cpu usage and update throttling + targetAveragingPeriodMs: number; // (rough) time span to average cpu usage over. + minimumAveragingPeriodMs: number; // minimum time span before throttling is considered. + // No throttling will occur before a process has run + // for at least this length of time. + minimumLogPeriodMs: number; // minimum time between log messages about throttling. + targetRate: number; // when throttling, aim for this fraction of cpu usage + // per unit time. + maxThrottle: number; // maximum ratio of negative duty cycle phases to + // positive. +} + +/** + * Some parameters that seem reasonable defaults. + */ +const defaultThrottleTiming: ThrottleTiming = { + dutyCyclePositiveMs: 50, + samplePeriodMs: 1000, + targetAveragingPeriodMs: 20000, + minimumAveragingPeriodMs: 6000, + minimumLogPeriodMs: 10000, + targetRate: 0.25, + maxThrottle: 10, +}; + +/** + * A sample of cpu usage. + */ +interface MeterSample { + time: number; // time at which sample was made (as reported by Date.now()) + cpuDuration: number; // accumulated "ctime" measured by pidusage + offDuration: number; // accumulated clock time for which process was paused (approximately) +} + +/** + * A throttling implementation for a process. Supply a pid, and it will try to keep that + * process from consuming too much cpu until stop() is called. + */ +export class Throttle { + private _timing: ThrottleTiming; // overall timing parameters + private _meteringInterval: NodeJS.Timeout | undefined; // timer for cpu measurements + private _dutyCycleTimeout: NodeJS.Timeout | undefined; // driver for throttle duty cycle + private _throttleFactor: number = 0; // relative length of paused phase + private _sample: MeterSample | undefined; // latest measurement. + private _anchor: MeterSample | undefined; // sample from past for averaging + private _nextAnchor: MeterSample | undefined; // upcoming replacement for _anchor + private _lastLogTime: number | undefined; // time of last throttle log message + private _offDuration: number = 0; // cumulative time spent paused + private _stopped: boolean = false; // set when stop has been called + + /** + * Start monitoring the given process and throttle as needed. + */ + constructor(private readonly _options: { + pid: number, + logMeta: log.ILogMeta, + timing?: ThrottleTiming + }) { + this._timing = this._options.timing || defaultThrottleTiming; + this._meteringInterval = setInterval(() => this._update(), this._timing.samplePeriodMs); + } + + /** + * Stop all activity. + */ + public stop() { + this._stopped = true; + this._stopMetering(); + this._stopThrottling(); + } + + /** + * Read the last cpu usage sample made, for test purposes. + */ + public get testStats(): MeterSample|undefined { + return this._sample; + } + + /** + * Measure cpu usage and update whether and how much we are throttling the process. + */ + private async _update() { + // Measure cpu usage to date. + let cpuDuration: number; + try { + cpuDuration = (await pidusage(this._options.pid)).ctime; + } catch (e) { + // process may have disappeared. + log.rawDebug(`Throttle measurement error: ${e}`, this._options.logMeta); + return; + } + const now = Date.now(); + const current: MeterSample = { time: now, cpuDuration, offDuration: this._offDuration }; + this._sample = current; + + // Measuring cpu usage was an async operation, so check that we haven't been stopped + // in the meantime. Otherwise we could sneak in and restart a throttle duty cycle. + if (this._stopped) { return; } + + // We keep a reference point in the past called the "anchor". Whenever the anchor + // becomes sufficiently old, we replace it with something newer. + if (!this._anchor) { this._anchor = current; } + if (this._nextAnchor && now - this._anchor.time > this._timing.targetAveragingPeriodMs * 2) { + this._anchor = this._nextAnchor; + this._nextAnchor = undefined; + } + // Keep a replacement for the current anchor in mind. + if (!this._nextAnchor && now - this._anchor.time > this._timing.targetAveragingPeriodMs) { + this._nextAnchor = current; + } + // Check if the anchor is sufficiently old for averages to be meaningful enough + // to support throttling. + const dt = current.time - this._anchor.time; + if (dt < this._timing.minimumAveragingPeriodMs) { return; } + + // Calculate the average cpu use per second since the anchor. + const rate = (current.cpuDuration - this._anchor.cpuDuration) / dt; + + // If that rate is less than our target rate, don't bother throttling. + const targetRate = this._timing.targetRate; + if (rate <= targetRate) { + this._updateThrottle(0); + return; + } + + // Calculate how much time the sandbox was paused since the anchor. This is + // approximate, since we don't line up duty cycles with this update function, + // but it should be good enough for throttling purposes. + const off = current.offDuration - this._anchor.offDuration; + // If the sandbox was never allowed to run, wait a bit longer for a duty cycle to complete. + // This should never happen unless time constants are set too tight relative to the + // maximum length of duty cycle. + const on = dt - off; + if (on <= 0) { return; } + + // Calculate the average cpu use per second while the sandbox is unpaused. + const rateWithoutThrottling = (current.cpuDuration - this._anchor.cpuDuration) / on; + + // Now pick a throttle level such that, if the sandbox continues using cpu + // at rateWithoutThrottling when it is unpaused, the overall rate matches + // the targetRate. + // one duty cycle lasts: quantum * (1 + throttleFactor) + // (positive cycle lasts 1 quantum; non-positive cycle duration is that of + // positive cycle scaled by throttleFactor) + // cpu use for this cycle is: quantum * rateWithoutThrottling + // cpu use per second is therefore: rateWithoutThrottling / (1 + throttleFactor) + // so: throttleFactor = (rateWithoutThrottling / targetRate) - 1 + const throttleFactor = rateWithoutThrottling / targetRate - 1; + + // Apply the throttle. Place a cap on it so the duty cycle does not get too long. + // This cap means that low targetRates could be unobtainable. + this._updateThrottle(Math.min(throttleFactor, this._timing.maxThrottle)); + + if (!this._lastLogTime || now - this._lastLogTime > this._timing.minimumLogPeriodMs) { + this._lastLogTime = now; + log.rawDebug('throttle', {...this._options.logMeta, + throttle: Math.round(this._throttleFactor), + throttledRate: Math.round(rate * 100), + rate: Math.round(rateWithoutThrottling * 100)}); + } + } + + /** + * Start/stop the throttling duty cycle as necessary. + */ + private _updateThrottle(factor: number) { + // For small factors, let the process run continuously. + if (factor < 0.001) { + if (this._dutyCycleTimeout) { this._stopThrottling(); } + this._throttleFactor = 0; + return; + } + // Set the throttle factor to apply and make sure the duty cycle is running. + this._throttleFactor = factor; + if (!this._dutyCycleTimeout) { this._throttle(true); } + } + + /** + * Send CONTinue or STOP signal to process. + */ + private _letProcessRun(on: boolean) { + try { + process.kill(this._options.pid, on ? 'SIGCONT' : 'SIGSTOP'); + } catch (e) { + // process may have disappeared + log.rawDebug(`Throttle error: ${e}`, this._options.logMeta); + } + } + + /** + * Send CONTinue or STOP signal to process, and schedule next step + * in duty cycle. + */ + private _throttle(on: boolean) { + this._letProcessRun(on); + const dt = this._timing.dutyCyclePositiveMs * (on ? 1.0 : this._throttleFactor); + if (!on) { this._offDuration += dt; } + this._dutyCycleTimeout = setTimeout(() => this._throttle(!on), dt); + } + + /** + * Make sure measurement of cpu is stopped. + */ + private _stopMetering() { + if (this._meteringInterval) { + clearInterval(this._meteringInterval); + this._meteringInterval = undefined; + } + } + + /** + * Make sure duty cycle is stopped and process is left in running state. + */ + private _stopThrottling() { + if (this._dutyCycleTimeout) { + clearTimeout(this._dutyCycleTimeout); + this._dutyCycleTimeout = undefined; + this._letProcessRun(true); + } + } +} diff --git a/app/server/lib/TimeQuery.ts b/app/server/lib/TimeQuery.ts new file mode 100644 index 00000000..c2f26565 --- /dev/null +++ b/app/server/lib/TimeQuery.ts @@ -0,0 +1,164 @@ +import {ActionSummary, ColumnDelta, createEmptyActionSummary, createEmptyTableDelta} from 'app/common/ActionSummary'; +import {CellDelta} from 'app/common/TabularDiff'; +import {concatenateSummaries} from 'app/server/lib/ActionSummary'; +import {ISQLiteDB, quoteIdent, ResultRow} from 'app/server/lib/SQLiteDB'; +import keyBy = require('lodash/keyBy'); +import matches = require('lodash/matches'); +import sortBy = require('lodash/sortBy'); +import toPairs = require('lodash/toPairs'); + +/** + * We can combine an ActionSummary with the current state of the database + * to answer questions about the state of the database in the past. This + * is particularly useful for grist metadata tables, which are needed to + * interpret the content of user tables fully. + * - TimeCursor is a simple container for the db and an ActionSummary + * - TimeQuery offers a db-like interface for a given table and set of columns + * - TimeLayout answers a couple of concrete questions about table meta-data using a + * set of TimeQuery objects hooked up to _grist_* tables. It could be used to + * improve the rendering of the ActionLog, for example, although it is not (yet). + */ + +/** Track the state of the database at a particular time. */ +export class TimeCursor { + public summary: ActionSummary; + + constructor(public db: ISQLiteDB) { + this.summary = createEmptyActionSummary(); + } + + /** add a summary of an action just before the last action applied to the TimeCursor */ + public prepend(prevSummary: ActionSummary) { + this.summary = concatenateSummaries([prevSummary, this.summary]); + } + + /** add a summary of an action just after the last action applied to the TimeCursor */ + public append(nextSummary: ActionSummary) { + this.summary = concatenateSummaries([this.summary, nextSummary]); + } +} + +/** internal class for storing a ResultRow dictionary, keyed by rowId */ +class ResultRows { + [rowId: number]: ResultRow; +} + +/** + * Query the state of a particular table in the past, given a TimeCursor holding the + * current db and a summary of all changes between that past time and now. + * For the moment, for simplicity, names of tables and columns are assumed not to + * change, and TimeQuery should only be used for _grist_* tables. + */ +export class TimeQuery { + private _currentRows: ResultRow[]; + private _pastRows: ResultRow[]; + + constructor(public tc: TimeCursor, public tableId: string, public colIds: string[]) { + } + + /** Get fresh data from DB and overlay with any past data */ + public async update(): Promise { + this._currentRows = await this.tc.db.all( + `select ${['id', ...this.colIds].map(quoteIdent).join(',')} from ${quoteIdent(this.tableId)}`); + + // Let's see everything the summary has accumulated about the table back then. + const td = this.tc.summary.tableDeltas[this.tableId] || createEmptyTableDelta(); + + // Now rewrite the summary as a ResultRow dictionary, to make it comparable + // with database. + const summaryRows: ResultRows = {}; + for (const [colId, columns] of toPairs(td.columnDeltas)) { + for (const [rowId, cell] of toPairs(columns) as unknown as Array<[keyof ColumnDelta, CellDelta]>) { + if (!summaryRows[rowId]) { summaryRows[rowId] = {}; } + const val = cell[0]; + summaryRows[rowId][colId] = (val !== null && typeof val === 'object' ) ? val[0] : null; + } + } + + // Prepare to access the current database state by rowId. + const rowsById = keyBy(this._currentRows, r => (r.id as number)); + + // Prepare a list of rowIds at the time of interest. + // The past rows are whatever the db has now, omitting rows that were added + // since the past time, and adding back any rows that were removed since then. + // Careful about the order of this, since rows could be replaced. + const additions = new Set(td.addRows); + const pastRowIds = + new Set([...this._currentRows.map(r => r.id as number).filter(r => !additions.has(r)), + ...td.removeRows]); + + // Now prepare a row for every expected rowId, using current db data if available + // and relevant, and overlaying past data when available. + this._pastRows = new Array(); + const colIdsOfInterest = new Set(this.colIds); + for (const id of Array.from(pastRowIds).sort()) { + const row: ResultRow = rowsById[id] || {id}; + if (summaryRows[id] && !additions.has(id)) { + for (const [colId, val] of toPairs(summaryRows[id])) { + if (colIdsOfInterest.has(colId)) { row[colId] = val; } + } + } + this._pastRows.push(row); + } + return this._pastRows; + } + + /** + * Do a query with a single result, specifying any desired filters. Exception thrown + * if there is no result. + */ + public one(args: {[name: string]: any}): ResultRow { + const result = this._pastRows.find(matches(args)); + if (!result) { + throw new Error(`could not find: ${JSON.stringify(args)} for ${this.tableId}`); + } + return result; + } + + /** Get all results for a query. */ + public all(args?: {[name: string]: any}): ResultRow[] { + if (!args) { return this._pastRows; } + return this._pastRows.filter(matches(args)); + } +} + +/** + * Put some TimeQuery queries to work answering questions about column order and + * user-facing name of tables. + */ +export class TimeLayout { + public tables: TimeQuery; + public fields: TimeQuery; + public columns: TimeQuery; + public views: TimeQuery; + + constructor(public tc: TimeCursor) { + this.tables = new TimeQuery(tc, '_grist_Tables', ['tableId', 'primaryViewId']); + this.fields = new TimeQuery(tc, '_grist_Views_section_field', + ['parentId', 'parentPos', 'colRef']); + this.columns = new TimeQuery(tc, '_grist_Tables_column', ['parentId', 'colId']); + this.views = new TimeQuery(tc, '_grist_Views', ['id', 'name']); + } + + /** update from TimeCursor */ + public async update() { + await this.tables.update(); + await this.columns.update(); + await this.fields.update(); + await this.views.update(); + } + + public getColumnOrder(tableId: string): string[] { + const primaryViewId = this.tables.one({tableId}).primaryViewId; + const preorder = this.fields.all({parentId: primaryViewId}); + const precol = keyBy(this.columns.all(), 'id'); + const ordered = sortBy(preorder, 'parentPos'); + const names = ordered.map(r => precol[r.colRef].colId); + return names; + } + + public getTableName(tableId: string): string { + const primaryViewId = this.tables.one({tableId}).primaryViewId; + return this.views.one({id: primaryViewId}).name; + } +} diff --git a/app/server/lib/UnsafeNodeComponent.ts b/app/server/lib/UnsafeNodeComponent.ts new file mode 100644 index 00000000..72b2e758 --- /dev/null +++ b/app/server/lib/UnsafeNodeComponent.ts @@ -0,0 +1,161 @@ +import { ActionRouter } from 'app/common/ActionRouter'; +import { LocalPlugin } from 'app/common/plugin'; +import { BaseComponent, createRpcLogger, warnIfNotReady } from 'app/common/PluginInstance'; +import { GristAPI, RPC_GRISTAPI_INTERFACE } from 'app/plugin/GristAPI'; +import * as log from 'app/server/lib/log'; +import { getAppPathTo } from 'app/server/lib/places'; +import { makeLinePrefixer } from 'app/server/lib/sandboxUtil'; +import { exitPromise, timeoutReached } from 'app/server/lib/serverUtils'; +import { ChildProcess, fork, ForkOptions } from 'child_process'; +import * as fse from 'fs-extra'; +import { IMessage, IMsgCustom, IMsgRpcCall, Rpc } from 'grain-rpc'; +import * as path from 'path'; + +// Error for not yet implemented api. +class NotImplemented extends Error { + constructor(name: string) { + super(`calling ${name} from UnsafeNode is not yet implemented`); + } +} + +/** + * The unsafeNode component used by a PluginInstance. + * + */ +export class UnsafeNodeComponent extends BaseComponent { + private _child?: ChildProcess; /* plugin node code will run as separate process */ + private _exited: Promise; /* fulfulled when process has completed */ + private _rpc: Rpc; + private _pluginPath: string; + private _pluginId: string; + private _actionRouter: ActionRouter; + + private _gristAPI: GristAPI = { + render() { throw new NotImplemented('render'); }, + dispose() { throw new NotImplemented('dispose'); }, + subscribe: (tableId: string) => this._actionRouter.subscribeTable(tableId), + unsubscribe: (tableId: string) => this._actionRouter.unsubscribeTable(tableId), + }; + + /** + * + * @arg parent: the plugin instance this component is part of + * @arg _mainPath: main script file to run + * @arg appRoot: root path for application (important for setting a good NODE_PATH) + * @arg _gristDocPath: path to the current Grist doc (to which this plugin applies). + * + */ + constructor(plugin: LocalPlugin, pluginRpc: Rpc, private _mainPath: string, public appRoot: string, + private _gristDocPath: string, + rpcLogger = createRpcLogger(log, `PLUGIN ${plugin.id}/${_mainPath} UnsafeNode:`)) { + super(plugin.manifest, rpcLogger); + this._pluginPath = plugin.path; + this._pluginId = plugin.id; + this._rpc = new Rpc({ + sendMessage: (msg) => this.sendMessage(msg), + logger: rpcLogger, + }); + this._rpc.registerForwarder('*', pluginRpc); + this._rpc.registerImpl(RPC_GRISTAPI_INTERFACE, this._gristAPI); + this._actionRouter = new ActionRouter(this._rpc); + } + + public async sendMessage(data: IMessage): Promise { + if (!this._child) { + await this.activateImplementation(); + } + this._child!.send(data); + return Promise.resolve(); + } + + public receiveAction(action: any[]) { + this._actionRouter.process(action) + .catch((err: any) => log.warn('unsafeNode[%s] receiveAction failed with %s', + this._child ? this._child.pid : "NULL", err)); + } + + /** + * + * Create the child node process needed for this component. + * + */ + protected async activateImplementation(): Promise { + log.info(`unsafeNode operating in ${this._pluginPath}`); + const base = this._pluginPath; + const script = path.resolve(base, this._mainPath); + await fse.access(script, fse.constants.R_OK); + // Time to set up the node search path the client will see. + // We take our own, via Module.globalPaths, a poorly documented + // method listing the search path for the active node program + // https://github.com/nodejs/node/blob/master/test/parallel/test-module-globalpaths-nodepath.js + const paths = require('module').globalPaths.slice().concat([ + // add the path to the plugin itself + path.resolve(base), + // add the path to grist's public api + getAppPathTo(this.appRoot, 'public-api'), + // add the path to the node_modules packaged with grist, in electron form + getAppPathTo(this.appRoot, 'node_modules') + ]); + const env = Object.assign({}, process.env, { + NODE_PATH: paths.join(path.delimiter), + GRIST_PLUGIN_PATH: `${this._pluginId}/${this._mainPath}`, + GRIST_DOC_PATH: this._gristDocPath, + }); + const electronVersion: string = (process.versions as any).electron; + if (electronVersion) { + // Pass along the fact that we are running under an electron-ified node, for the purposes of + // finding binaries (sqlite3 in particular). + env.ELECTRON_VERSION = electronVersion; + } + const child = this._child = fork(script, [], { + env, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + } as ForkOptions); // Explicit cast only because node-6 typings mistakenly omit stdio property + + log.info("unsafeNode[%s] started %s", child.pid, script); + + // Important to use exitPromise() before events from child may be received, so don't call + // yield or await between fork and here. + this._exited = exitPromise(child) + .then(code => log.info("unsafeNode[%s] exited with %s", child.pid, code)) + .catch(err => log.warn("unsafeNode[%s] failed with %s", child.pid, err)) + .then(() => { this._child = undefined; }); + + child.stdout.on('data', makeLinePrefixer('PLUGIN stdout: ')); + child.stderr.on('data', makeLinePrefixer('PLUGIN stderr: ')); + + warnIfNotReady(this._rpc, 3000, "Plugin isn't ready; be sure to call grist.ready() from plugin"); + child.on('message', this._rpc.receiveMessage.bind(this._rpc)); + } + + /** + * + * Remove the child node process needed for this component. + * + */ + protected async deactivateImplementation(): Promise { + if (!this._child) { + log.info('unsafeNode deactivating: no child process'); + } else { + log.info('unsafeNode[%s] deactivate: disconnecting child', this._child.pid); + this._child.disconnect(); + if (await timeoutReached(2000, this._exited)) { + log.info("unsafeNode[%s] deactivate: sending SIGTERM", this._child.pid); + this._child.kill('SIGTERM'); + } + if (await timeoutReached(5000, this._exited)) { + log.warn("unsafeNode[%s] deactivate: child still has not exited", this._child.pid); + } else { + log.info("unsafeNode deactivate: child exited"); + } + } + } + + protected doForwardCall(c: IMsgRpcCall): Promise { + return this._rpc.forwardCall({...c, mdest: ''}); + } + + protected async doForwardMessage(c: IMsgCustom): Promise { + return this._rpc.forwardMessage({...c, mdest: ''}); + } +} diff --git a/app/server/lib/WorkCoordinator.ts b/app/server/lib/WorkCoordinator.ts new file mode 100644 index 00000000..b787fa10 --- /dev/null +++ b/app/server/lib/WorkCoordinator.ts @@ -0,0 +1,66 @@ +import * as log from 'app/server/lib/log'; + +/** + * WorkCoordinator is a helper to do work serially. It takes a doWork() callback which may either + * do some work and return a Promise, or report no work to be done by returning null. After work + * completes, doWork() will be called again; when idle, ping() should be called to retry doWork(). + */ +export class WorkCoordinator { + private _doWorkCB: () => Promise|null; + private _tryNextStepCB: () => void; + private _isStepRunning: boolean = false; + private _isStepScheduled: boolean = false; + + /** + * The doWork() callback will be called on ping() and whenever previous doWork() promise + * succeeds. If doWork() had nothing to do, it should return null, and will not be called again + * until the next ping(). + * + * Note that doWork() should never fail. If it does, exceptions and rejections will be caught + * and logged, and WorkCoordinator will not be called again until the next ping(). + */ + constructor(doWork: () => Promise|null) { + this._doWorkCB = doWork; + this._tryNextStepCB = () => this._tryNextStep(); // bound version of _tryNextStep. + } + + /** + * Attempt doWork() again. If doWork() is currently running, it will be attempted again on + * completion even if the current run fails. + */ + public ping(): void { + if (!this._isStepScheduled) { + this._isStepScheduled = true; + this._maybeSchedule(); + } + } + + private async _tryNextStep(): Promise { + this._isStepScheduled = false; + if (!this._isStepRunning) { + this._isStepRunning = true; + try { + const work = this._doWorkCB(); + if (work) { + await work; + // Only schedule the next step if some work was done. If _doWorkCB() did nothing, or + // failed, _doWorkCB() will only be called when an external ping() triggers it. + this._isStepScheduled = true; + } + } catch (err) { + // doWork() should NOT fail. If it does, we log the error here, and stop scheduling work + // as if there is no more work to be done. + log.error("WorkCoordinator: error in doWork()", err); + } finally { + this._isStepRunning = false; + this._maybeSchedule(); + } + } + } + + private _maybeSchedule() { + if (this._isStepScheduled && !this._isStepRunning) { + setImmediate(this._tryNextStepCB); + } + } +} diff --git a/app/server/lib/checksumFile.ts b/app/server/lib/checksumFile.ts new file mode 100644 index 00000000..dfd6241a --- /dev/null +++ b/app/server/lib/checksumFile.ts @@ -0,0 +1,21 @@ +import {createHash} from 'crypto'; +import * as fs from 'fs'; + +/** + * Computes hash of the file at the given path, using 'sha1' by default, or any algorithm + * supported by crypto.createHash(). + */ +export async function checksumFile(filePath: string, algorithm: string = 'sha1'): Promise { + const shaSum = createHash(algorithm); + const stream = fs.createReadStream(filePath); + try { + stream.on('data', (data) => shaSum.update(data)); + await new Promise((resolve, reject) => { + stream.on('end', resolve); + stream.on('error', reject); + }); + return shaSum.digest('hex'); + } finally { + stream.removeAllListeners(); // Isn't strictly necessary. + } +} diff --git a/app/server/lib/dbUtils.ts b/app/server/lib/dbUtils.ts new file mode 100644 index 00000000..c0cf76fc --- /dev/null +++ b/app/server/lib/dbUtils.ts @@ -0,0 +1,67 @@ +import {synchronizeProducts} from 'app/gen-server/entity/Product'; +import {Connection, createConnection} from 'typeorm'; + +// Summary of migrations found in database and in code. +interface MigrationSummary { + migrationsInDb: string[]; + migrationsInCode: string[]; + pendingMigrations: string[]; +} + +// Find the migrations in the database, the migrations in the codebase, and compare the two. +export async function getMigrations(connection: Connection): Promise { + let migrationsInDb: string[]; + try { + migrationsInDb = (await connection.query('select name from migrations')).map((rec: any) => rec.name); + } catch (e) { + // If no migrations have run, there'll be no migrations table - which is fine, + // it just means 0 migrations run yet. Sqlite+Postgres report this differently, + // so any query error that mentions the name of our table is treated as ok. + // Everything else is unexpected. + if (!(e.name === 'QueryFailedError' && e.message.includes('migrations'))) { + throw e; + } + migrationsInDb = []; + } + // get the migration names in codebase. + // They are a bit hidden, see typeorm/src/migration/MigrationExecutor::getMigrations + const migrationsInCode: string[] = connection.migrations.map(m => (m.constructor as any).name); + const pendingMigrations = migrationsInCode.filter(m => !migrationsInDb.includes(m)); + return { + migrationsInDb, + migrationsInCode, + pendingMigrations, + }; +} + +/** + * Run any needed migrations, and make sure products are up to date. + */ +export async function updateDb(connection?: Connection) { + connection = connection || await createConnection(); + await runMigrations(connection); + await synchronizeProducts(connection, true); +} + +export async function runMigrations(connection: Connection) { + // on SQLite, migrations fail if we don't temporarily disable foreign key + // constraint checking. This is because for sqlite typeorm copies each + // table and rebuilds it from scratch for each schema change. + // Also, we need to disable foreign key constraint checking outside of any + // transaction, or it has no effect. + const sqlite = connection.driver.options.type === 'sqlite'; + if (sqlite) { await connection.query("PRAGMA foreign_keys = OFF;"); } + await connection.transaction(async tr => { + await tr.connection.runMigrations(); + }); + if (sqlite) { await connection.query("PRAGMA foreign_keys = ON;"); } +} + +export async function undoLastMigration(connection: Connection) { + const sqlite = connection.driver.options.type === 'sqlite'; + if (sqlite) { await connection.query("PRAGMA foreign_keys = OFF;"); } + await connection.transaction(async tr => { + await tr.connection.undoLastMigration(); + }); + if (sqlite) { await connection.query("PRAGMA foreign_keys = ON;"); } +} diff --git a/app/server/lib/docUtils.d.ts b/app/server/lib/docUtils.d.ts new file mode 100644 index 00000000..7d00f959 --- /dev/null +++ b/app/server/lib/docUtils.d.ts @@ -0,0 +1,9 @@ +export function makeIdentifier(name: string): string; +export function copyFile(src: string, dest: string): Promise; +export function createNumbered(name: string, separator: string, creator: (path: string) => Promise, + startNum?: number): Promise; +export function createNumberedTemplate(template: string, creator: (path: string) => Promise): Promise; +export function createExclusive(path: string): Promise; +export function realPath(path: string): Promise; +export function pathExists(path: string): Promise; +export function isSameFile(path1: string, path2: string): Promise; diff --git a/app/server/lib/docUtils.js b/app/server/lib/docUtils.js new file mode 100644 index 00000000..8820b61e --- /dev/null +++ b/app/server/lib/docUtils.js @@ -0,0 +1,145 @@ +/** + * Functions generally useful when dealing with Grist documents. + */ + + + +var fs = require('fs'); +var fsPath = require('path'); +var Promise = require('bluebird'); +Promise.promisifyAll(fs); + +var nonIdentRegex = /[^\w_]+/g; + +/** + * Given a string, converts it to a Grist identifier. Identifiers consist of lowercase + * alphanumeric characters and the underscore. + * @param {String} name The name to convert. + * @returns {String} Identifier. + */ +function makeIdentifier(name) { + // Lowercase and replace consecutive invalid characters with underscores. + return name.toLowerCase().replace(nonIdentRegex, '_'); +} +exports.makeIdentifier = makeIdentifier; + + +/** + * Copies a file, returning a promise that is resolved (with no value) when the copy is complete. + * TODO This needs a unittest. + */ +function copyFile(sourcePath, destPath) { + var sourceStream, destStream; + return new Promise(function(resolve, reject) { + sourceStream = fs.createReadStream(sourcePath); + destStream = fs.createWriteStream(destPath); + + sourceStream.on('error', reject); + destStream.on('error', reject); + destStream.on('finish', resolve); + + sourceStream.pipe(destStream); + }) + .finally(function() { + if (destStream) { destStream.destroy(); } + if (sourceStream) { sourceStream.destroy(); } + }); +} +exports.copyFile = copyFile; + + +/** + * Helper for creating numbered files. Tries to call creator() with name, then (name + separator + + * "2") and so on with incrementing numbers, as long as the promise returned by creator() is + * rejected with err.code of 'EEXIST'. Creator() must return a promise. + * @param {String} name The first name to try. + * @param {String} separator The separator between name and appended numbers. + * @param {Function} creator The function to call with successive names. Must return a promise. + * @param {Number} startNum Optional number to start with; omit to try an unnumbered name first. + * @returns {Promise} Promise for the first name for which creator() succeeded. + */ +function createNumbered(name, separator, creator, startNum) { + var fullName = name + (startNum === undefined ? '' : separator + startNum); + var nextNum = (startNum === undefined ? 2 : startNum + 1); + return creator(fullName) + .then(() => fullName) + .catch(function(err) { + if (err.cause && err.cause.code !== 'EEXIST') + throw err; + return createNumbered(name, separator, creator, nextNum); + }); +} +exports.createNumbered = createNumbered; + +/** + * An easier-to-use alternative to createNumbered. Pass in a template string containing the + * special token "{NUM}". It will first call creator() with "{NUM}" removed, then with "{NUM}" + * replcaced by "-2", "-3", etc, until creator() succeeds, and will return the value for which it + * suceeded. + */ +function createNumberedTemplate(template, creator) { + const [prefix, suffix] = template.split("{NUM}"); + if (typeof prefix !== "string" || typeof suffix !== "string") { + throw new Error(`createNumberedTemplate: invalid template ${template}`); + } + return createNumbered(prefix, "-", (uniqPrefix) => creator(uniqPrefix + suffix)) + .then((uniqPrefix) => uniqPrefix + suffix); +} +exports.createNumberedTemplate = createNumberedTemplate; + +/** + * Creates a new file, failing if the path already exists. + * @param {String} path: The path to try creating. + * @returns {Promise} Resolved if the path was created, rejected if it already existed (with + * err.cause.code === EEXIST) or if there was another error creating it. + */ +function createExclusive(path) { + return fs.openAsync(path, 'wx').then(fd => fs.closeAsync(fd)); +} +exports.createExclusive = createExclusive; + + +/** + * Returns the canonicalized absolute path for the given path, using fs.realpath, but allowing + * non-existent paths. In case of non-existent path, the longest existing prefix is resolved and + * the rest kept unchanged. + * @param {String} path: Path to resolve. + * @return {Promise:String} Promise for the resolved path. + */ +function realPath(path) { + return fs.realpathAsync(path) + .catch(() => + realPath(fsPath.dirname(path)) + .then(dir => fsPath.join(dir, fsPath.basename(path))) + ); +} +exports.realPath = realPath; + + +/** + * Returns a promise that resolves to true or false based on whether the path exists. If other + * errors occur, this promise may still be rejected. + */ +function pathExists(path) { + return fs.accessAsync(path) + .then(() => true) + .catch({code: 'ENOENT'}, () => false) + .catch({code: 'ENOTDIR'}, () => false); +} +exports.pathExists = pathExists; + +/** + * Returns a promise that resolves to true or false based on whether the two paths point to the + * same file. If errors occur, this promise may be rejected. + */ +function isSameFile(path1, path2) { + return Promise.join(fs.lstatAsync(path1), fs.lstatAsync(path2), (stat1, stat2) => { + if (stat1.dev === stat2.dev && stat1.ino === stat2.ino) { + return true; + } + return false; + }) + .catch({code: 'ENOENT'}, () => false) + .catch({code: 'ENOTDIR'}, () => false); +} +exports.isSameFile = isSameFile; diff --git a/app/server/lib/expressWrap.ts b/app/server/lib/expressWrap.ts new file mode 100644 index 00000000..c332a5fe --- /dev/null +++ b/app/server/lib/expressWrap.ts @@ -0,0 +1,36 @@ +import {RequestWithLogin} from 'app/server/lib/Authorizer'; +import * as log from 'app/server/lib/log'; +import * as express from 'express'; + +/** + * Wrapper for async express endpoints to catch errors and forward them to the error handler. + */ +export function expressWrap(callback: express.RequestHandler): express.RequestHandler { + return async (req, res, next) => { + try { + await callback(req, res, next); + } catch (err) { + next(err); + } + }; +} + +/** + * Error-handling middleware that responds to errors in json. The status code is taken from + * error.status property (for which ApiError is convenient), and defaults to 500. + */ +export const jsonErrorHandler: express.ErrorRequestHandler = (err, req, res, next) => { + const mreq = req as RequestWithLogin; + log.warn("Error during api call to %s: (%s) user %d params %s body %s", req.path, err.message, + mreq.userId, + JSON.stringify(req.params), JSON.stringify(req.body)); + res.status(err.status || 500).json({error: err.message || 'internal error', + details: err.details}); +}; + +/** + * Middleware that responds with a 404 status and a json error object. + */ +export const jsonNotFoundHandler: express.RequestHandler = (req, res, next) => { + res.status(404).json({error: `not found: ${req.url}`}); +}; diff --git a/app/server/lib/extractOrg.ts b/app/server/lib/extractOrg.ts new file mode 100644 index 00000000..02fb931f --- /dev/null +++ b/app/server/lib/extractOrg.ts @@ -0,0 +1,166 @@ +import { ApiError } from 'app/common/ApiError'; +import { mapGetOrSet, MapWithTTL } from 'app/common/AsyncCreate'; +import { extractOrgParts, getKnownOrg, isCustomHost } from 'app/common/gristUrls'; +import { Organization } from 'app/gen-server/entity/Organization'; +import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager'; +import { NextFunction, Request, RequestHandler, Response } from 'express'; +import { IncomingMessage } from 'http'; + +// How long we cache information about the relationship between +// orgs and custom hosts. The higher this is, the fewer requests +// to the DB needed, but the longer it will take for changes +// to custom host setting to take effect. Also, since the caching +// is done on individual servers/workers, it could be inconsistent +// between servers/workers for some time. During this period, +// redirect cycles are possible. +// Units are milliseconds. +const ORG_HOST_CACHE_TTL = 60 * 1000; + +export interface RequestOrgInfo { + org: string; + isCustomHost: boolean; // when set, the request's domain is a recognized custom host linked + // with the specified org. + + // path remainder after stripping /o/{org} if any. + url: string; +} + +export type RequestWithOrg = Request & Partial; + +/** + * Manage the relationship between orgs and custom hosts in the url. + */ +export class Hosts { + + // Cache of orgs (e.g. "fancy" of "fancy.getgrist.com") associated with custom hosts + // (e.g. "www.fancypants.com") + private _host2org = new MapWithTTL>(ORG_HOST_CACHE_TTL); + // Cache of custom hosts associated with orgs. + private _org2host = new MapWithTTL>(ORG_HOST_CACHE_TTL); + + // baseDomain should start with ".". It may be undefined for localhost or single-org mode. + constructor(private _baseDomain: string|undefined, private _dbManager: HomeDBManager) { + } + + /** + * Use app.use(hosts.extractOrg) to set req.org, req.isCustomHost, and to strip + * /o/ORG/ from urls (when present). + * + * If Host header has a getgrist.com subdomain, then it must match the value in /o/ORG (when + * present), and req.org will be set to the subdomain. On mismatch, a 400 response is returned. + * + * If Host header is a localhost domain, then req.org is set to the value in /o/ORG when + * present, and to "" otherwise. + * + * If Host header is something else, we query the db for an org whose host value matches. + * If found, req.org is set appropriately, and req.isCustomHost is set to true. + * If not found, a 'Domain not recognized' error is thrown, showing an error page. + */ + public get extractOrg(): RequestHandler { + return this._extractOrg.bind(this); + } + + // Extract org info in a request. This applies to the low-level IncomingMessage type (rather + // than express.Request that derives from it) to be usable with websocket requests too. + public async getOrgInfo(req: IncomingMessage): Promise { + const host = req.headers.host || ''; + const hostname = host.split(':')[0]; // Strip out port (ignores IPv6 but is OK for us). + const info = await this.getOrgInfoFromParts(hostname, req.url!); + // "Organization" header is used in proxying to doc worker, so respect it if + // no org info found in url. + if (!info.org && req.headers.organization) { + info.org = req.headers.organization as string; + } + return info; + } + + // Extract org, isCustomHost, and the URL with /o/ORG stripped away. Throws ApiError for + // mismatching org or invalid custom domain. Hostname should not include port. + public async getOrgInfoFromParts(hostname: string, urlPath: string): Promise { + // Extract the org from the host and URL path. + const parts = extractOrgParts(hostname, urlPath); + + // If the server is configured to serve a single hard-wired org, respect that. + const singleOrg = getKnownOrg(); + if (singleOrg) { + return {org: singleOrg, url: parts.pathRemainder, isCustomHost: false}; + } + + // Fake the protocol; it doesn't matter for parsing out the hostname. + if (this._isNativeDomain(hostname)) { + if (parts.mismatch) { + throw new ApiError(`Wrong org for this domain: ` + + `'${parts.orgFromPath}' does not match '${parts.orgFromHost}'`, 400); + } + return {org: parts.subdomain || '', url: parts.pathRemainder, isCustomHost: false}; + + } else { + // Otherwise check for a custom host. + const org = await mapGetOrSet(this._host2org, hostname, async () => { + const o = await this._dbManager.connection.manager.findOne(Organization, {host: hostname}); + return o && o.domain || undefined; + }); + if (!org) { throw new ApiError(`Domain not recognized: ${hostname}`, 404); } + + // Strip any stray /o/.... that has been added to a url with a custom host. + // TODO: it would eventually be cleaner to make sure we don't make those + // additions in the first place. + + // To check for mismatch, compare to org, since orgFromHost is not expected to match. + if (parts.orgFromPath && parts.orgFromPath !== org) { + throw new ApiError(`Wrong org for this domain: ` + + `'${parts.orgFromPath}' does not match '${org}'`, 400); + } + return {org, isCustomHost: true, url: parts.pathRemainder}; + } + } + + public async addOrgInfo(req: Request): Promise { + return Object.assign(req, await this.getOrgInfo(req)); + } + + /** + * Use app.use(hosts.redirectHost) to ensure (by redirecting if necessary) + * that the domain in the url matches the preferred domain for the current org. + * Expects that the extractOrg has been used first. + */ + public get redirectHost(): RequestHandler { + return this._redirectHost.bind(this); + } + + public close() { + this._host2org.clear(); + this._org2host.clear(); + } + + private async _extractOrg(req: Request, resp: Response, next: NextFunction) { + try { + await this.addOrgInfo(req); + return next(); + } catch (err) { + return resp.status(err.status || 500).send({error: err.message}); + } + } + + private async _redirectHost(req: Request, resp: Response, next: NextFunction) { + const {org} = req as RequestWithOrg; + + if (org && this._isNativeDomain(req.hostname) && !this._dbManager.isMergedOrg(org)) { + // Check if the org has a preferred host. + const orgHost = await mapGetOrSet(this._org2host, org, async () => { + const o = await this._dbManager.connection.manager.findOne(Organization, {domain: org}); + return o && o.host || undefined; + }); + if (orgHost && orgHost !== req.hostname) { + const url = new URL(`${req.protocol}://${req.get('host')}${req.path}`); + url.hostname = orgHost; // assigning hostname rather than host preserves port. + return resp.redirect(url.href); + } + } + return next(); + } + + private _isNativeDomain(hostname: string) { + return !this._baseDomain || !isCustomHost(hostname, this._baseDomain); + } +} diff --git a/app/server/lib/gristSessions.ts b/app/server/lib/gristSessions.ts new file mode 100644 index 00000000..6ea7939f --- /dev/null +++ b/app/server/lib/gristSessions.ts @@ -0,0 +1,138 @@ +import * as session from '@gristlabs/express-session'; +import {parseSubdomain} from 'app/common/gristUrls'; +import {RequestWithOrg} from 'app/server/lib/extractOrg'; +import {GristServer} from 'app/server/lib/GristServer'; +import {Sessions} from 'app/server/lib/Sessions'; +import {promisifyAll} from 'bluebird'; +import * as express from 'express'; +import * as path from 'path'; +import * as shortUUID from "short-uuid"; + + +export const cookieName = process.env.GRIST_SESSION_COOKIE || 'grist_sid'; + +export const COOKIE_MAX_AGE = 90 * 24 * 60 * 60 * 1000; // 90 days in milliseconds + +// RedisStore and SqliteStore are expected to provide a set/get interface for sessions. +export interface SessionStore { + getAsync(sid: string): Promise; + setAsync(sid: string, session: any): Promise; +} + +/** + * + * A V1 session. A session can be associated with a number of users. + * There may be a preferred association between users and organizations: + * specifically, if from the url we can tell that we are showing material + * for a given organization, we should pick a user that has access to that + * organization. + * + * This interface plays no role at all yet! Working on refactoring existing + * sessions step by step to get closer to this. + * + */ +export interface IGristSession { + + // V1 Hosted Grist - known available users. + users: Array<{ + userId?: number; + }>; + + // V1 Hosted Grist - known user/org relationships. + orgs: Array<{ + orgId: number; + userId: number; + }>; +} + +function createSessionStoreFactory(sessionsDB: string): () => SessionStore { + if (process.env.REDIS_URL) { + // Note that ./build excludes this module from the electron build. + const RedisStore = require('connect-redis')(session); + promisifyAll(RedisStore.prototype); + return () => new RedisStore({ + url: process.env.REDIS_URL, + }); + } else { + const SQLiteStore = require('@gristlabs/connect-sqlite3')(session); + promisifyAll(SQLiteStore.prototype); + return () => new SQLiteStore({ + dir: path.dirname(sessionsDB), + db: path.basename(sessionsDB), // SQLiteStore no longer appends a .db suffix. + table: 'sessions' + }); + } +} + +export function getAllowedOrgForSessionID(sessionID: string): {org: string, host: string}|null { + if (sessionID.startsWith('c-') && sessionID.includes('@')) { + const [, org, host] = sessionID.split('@'); + if (!host) { throw new Error('Invalid session ID'); } + return {org, host}; + } + // Otherwise sessions start with 'g-', but we also accept older sessions without a prefix. + return null; +} + +/** + * Set up Grist Sessions, either in a sqlite db or via redis. + * @param instanceRoot: path to storage area in case we need to make a sqlite db. + */ +export function initGristSessions(instanceRoot: string, server: GristServer) { + // TODO: We may need to evaluate the usage of space in the SQLite store grist-sessions.db + // since entries are created on the first get request. + const sessionsDB: string = path.join(instanceRoot, 'grist-sessions.db'); + + // The extra step with the creator function is used in server.js to create a new session store + // after unpausing the server. + const sessionStoreCreator = createSessionStoreFactory(sessionsDB); + const sessionStore = sessionStoreCreator(); + + const adaptDomain = process.env.GRIST_ADAPT_DOMAIN === 'true'; + const fixedDomain = process.env.GRIST_SESSION_DOMAIN || process.env.GRIST_DOMAIN; + + const getCookieDomain = (req: express.Request) => { + const mreq = req as RequestWithOrg; + if (mreq.isCustomHost) { + // For custom hosts, omit the domain to make it a "host-only" cookie, to avoid it being + // included into subdomain requests (since we would not control all the subdomains). + return undefined; + } + if (adaptDomain) { + const reqDomain = parseSubdomain(req.get('host')); + if (reqDomain.base) { return reqDomain.base.split(':')[0]; } + } + return fixedDomain; + }; + // Use a separate session IDs for custom domains than for native ones. Because a custom domain + // cookie could be stolen (with some effort) by the custom domain's owner, we limit the damage + // by only honoring custom-domain cookies for requests to that domain. + const generateId = (req: RequestWithOrg) => { + const uid = shortUUID.generate(); + return req.isCustomHost ? `c-${uid}@${req.org}@${req.get('host')}` : `g-${uid}`; + }; + const sessionSecret = server.create.sessionSecret(); + const sessionMiddleware = session({ + secret: sessionSecret, + resave: false, + saveUninitialized: false, + name: cookieName, + requestDomain: getCookieDomain, + genid: generateId, + cookie: { + // We do not initially set max-age, leaving the cookie as a + // session cookie until there's a successful login. On the + // redis back-end, the session associated with the cookie will + // persist for 24 hours if there is no successful login. Once + // there is a successful login, max-age will be set to + // COOKIE_MAX_AGE, making the cookie a persistent cookie. The + // session associated with the cookie will receive an updated + // time-to-live, so that it persists for COOKIE_MAX_AGE. + }, + store: sessionStore + }); + + const sessions = new Sessions(sessionSecret, sessionStore, server); + + return {sessions, sessionSecret, sessionStore, sessionMiddleware, sessionStoreCreator}; +} diff --git a/app/server/lib/guessExt.ts b/app/server/lib/guessExt.ts new file mode 100644 index 00000000..00ed9319 --- /dev/null +++ b/app/server/lib/guessExt.ts @@ -0,0 +1,51 @@ +import {fromFile} from 'file-type'; +import {extension, lookup} from 'mime-types'; +import * as path from 'path'; + +/** + * Get our best guess of the file extension, based on its original extension (as received from the + * user), mimeType (as reported by the browser upload, or perhaps some API), and the file + * contents. + * + * The resulting extension is used to choose a parser for imports, and to present the file back + * to the user for attachments. + */ +export async function guessExt(filePath: string, fileName: string, mimeType: string|null): Promise { + const origExt = path.extname(fileName).toLowerCase(); // Has the form ".xls" + + let mimeExt = extension(mimeType); // Has the form "xls" + mimeExt = mimeExt ? "." + mimeExt : null; // Use the more comparable form ".xls" + if (mimeExt === ".json") { + // It's common for JSON APIs to specify MIME type, but origExt might come from a URL with + // periods that don't indicate a meaningful extension. Trust mime-type here. + return mimeExt; + } + + if (origExt === ".csv" || origExt === ".xls") { + // File type detection doesn't work for these, and mime type can't be trusted. E.g. Windows + // may report "application/vnd.ms-excel" for .csv files. See + // https://github.com/ManifoldScholar/manifold/issues/2409#issuecomment-545152220 + return origExt; + } + + // If extension and mime type agree, let's call it a day. + if (origExt && (origExt === mimeExt || lookup(origExt.slice(1)) === mimeType)) { + return origExt; + } + + // If not, let's take a look at the file contents. + const detected = await fromFile(filePath); + const detectedExt = detected ? "." + detected.ext : null; + if (detectedExt) { + // For the types for which detection works, we think we should prefer it. + return detectedExt; + } + + if (mimeExt === '.txt' || mimeExt === '.bin') { + // text/plain (txt) and application/octet-stream (bin) are too generic, only use them if we + // don't have anything better. + return origExt || mimeExt; + } + // In other cases, it's a tough call. + return origExt || mimeExt; +} diff --git a/app/server/lib/idUtils.ts b/app/server/lib/idUtils.ts new file mode 100644 index 00000000..a700ba1c --- /dev/null +++ b/app/server/lib/idUtils.ts @@ -0,0 +1,48 @@ +import {ForkResult} from 'app/common/ActiveDocAPI'; +import {buildUrlId, parseUrlId} from 'app/common/gristUrls'; +import {padStart} from 'app/common/gutil'; +import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; +import * as shortUUID from 'short-uuid'; + +// make an id that is a standard UUID compressed into fewer characters. +export function makeId(): string { + // Generate a flickr-style id, by converting a regular uuid interpreted + // as a hex number (without dashes) into a number expressed in a bigger + // base. That number is encoded as characters chosen for url safety and + // lack of confusability. The character encoding zero is '1'. We pad the + // result so that the length of the id remains consistent, since there is + // routing that depends on the id length exceeding a minimum threshold. + return padStart(shortUUID.generate(), 22, '1'); +} + +/** + * Construct an id for a fork, given the userId, whether the user is the anonymous user, + * and the id of a reference document (the trunk). + * If the userId is null, the user will be treated as the anonymous user. + */ +export function makeForkIds(options: { userId: number|null, isAnonymous: boolean, + trunkDocId: string, trunkUrlId: string }): ForkResult { + const forkId = makeId(); + const forkUserId = options.isAnonymous ? undefined : + (options.userId !== null ? options.userId : undefined); + // TODO: we will want to support forks of forks, but for now we do not - + // forks are always forks of the trunk. + const docId = parseUrlId(options.trunkDocId).trunkId; + const urlId = parseUrlId(options.trunkUrlId).trunkId; + return { + docId: buildUrlId({trunkId: docId, forkId, forkUserId}), + urlId: buildUrlId({trunkId: urlId, forkId, forkUserId}), + }; +} + +// For importing, we can assign any worker to the job. As a hack, we reuse the document +// assignment mechanism. To spread the work around a bit if we have several doc workers, +// we use a fake document id between import0 and import9. +// This method takes a DocWorkerMap to allow for something smarter in future. +export function getAssignmentId(docWorkerMap: IDocWorkerMap, docId: string): string { + let assignmentId = docId; + if (assignmentId === 'import') { + assignmentId = `import${Math.round(Math.random() * 10)}`; + } + return assignmentId; +} diff --git a/app/server/lib/log.ts b/app/server/lib/log.ts new file mode 100644 index 00000000..11019ec6 --- /dev/null +++ b/app/server/lib/log.ts @@ -0,0 +1,89 @@ +/** + * Configures grist logging. This is merely a customization of the 'winston' logging module, + * and all winston methods are available. Additionally provides log.timestamp() function. + * Usage: + * var log = require('./lib/log'); + * log.info(...); + */ + +import {timeFormat} from 'app/common/timeFormat'; +import * as winston from 'winston'; + +interface LogWithTimestamp extends winston.LoggerInstance { + timestamp(): string; + // We'd like to log raw json, for convenience of parsing downstream. + // We have a customization that interferes with meta arguments, and + // existing log messages that depend on that customization. For + // clarity then, we just add "raw" flavors of the primary level + // methods that pass their object argument through to winston. + rawError(msg: string, meta: ILogMeta): void; + rawInfo(msg: string, meta: ILogMeta): void; + rawWarn(msg: string, meta: ILogMeta): void; + rawDebug(msg: string, meta: ILogMeta): void; + origLog(level: string, msg: string, ...args: any[]): void; +} + +/** + * Hack winston to provide a saner behavior with regard to its optional arguments. Winston allows + * two optional arguments at the end: "meta" (if object) and "callback" (if function). We don't + * use them, but we do use variable number of arguments as in log.info("foo %s", foo). If foo is + * an object, winston dumps it in an ugly way, not at all as intended. We fix by always appending + * {} to the end of the arguments, so that winston sees an empty meta object. + * We can add support for callback if ever needed. + */ +const origLog = winston.Logger.prototype.log; +winston.Logger.prototype.log = function(level: string, msg: string, ...args: any[]) { + return origLog.call(this, level, msg, ...args, {}); +}; + +const rawLog = new (winston.Logger)(); +const log: LogWithTimestamp = Object.assign(rawLog, { + timestamp, + /** + * Versions of log.info etc that take a meta parameter. For + * winston, logs are streams of info objects. Info objects + * have two mandatory fields, level and message. They can + * have other fields, called "meta" fields. When logging + * in json, those fields are added directly to the json, + * rather than stringified into the message field, which + * is what we want and why we are adding these variants. + */ + rawError: (msg: string, meta: ILogMeta) => origLog.call(log, 'error', msg, meta), + rawInfo: (msg: string, meta: ILogMeta) => origLog.call(log, 'info', msg, meta), + rawWarn: (msg: string, meta: ILogMeta) => origLog.call(log, 'warn', msg, meta), + rawDebug: (msg: string, meta: ILogMeta) => origLog.call(log, 'debug', msg, meta), + origLog, +}); + +/** + * Returns the current timestamp as a string in the same format as used in logging. + */ +function timestamp() { + return timeFormat("A", new Date()); +} + +const fileTransportOptions = { + stream: process.stderr, + level: 'debug', + timestamp: log.timestamp, + colorize: true, + json: process.env.GRIST_HOSTED_VERSION ? true : false +}; + +// Configure logging to use console and simple timestamps. +log.add(winston.transports.File, fileTransportOptions); + +// Also update the default logger to use the same format. +winston.remove(winston.transports.Console); +winston.add(winston.transports.File, fileTransportOptions); + +// It's a little tricky to export a type when the top-level export is an object. +// tslint:disable-next-line:no-namespace +declare namespace log { // eslint-disable-line @typescript-eslint/no-namespace + interface ILogMeta { + [key: string]: any; + } +} +type ILogMeta = log.ILogMeta; + +export = log; diff --git a/app/server/lib/manifest.ts b/app/server/lib/manifest.ts new file mode 100644 index 00000000..de2a8a94 --- /dev/null +++ b/app/server/lib/manifest.ts @@ -0,0 +1,79 @@ +import {BarePlugin} from 'app/plugin/PluginManifest'; +import PluginManifestTI from 'app/plugin/PluginManifest-ti'; +import * as fse from 'fs-extra'; +import * as yaml from 'js-yaml'; +import * as path from 'path'; +import {createCheckers} from "ts-interface-checker"; + +const manifestChecker = createCheckers(PluginManifestTI).BarePlugin; +/** + * Validate the manifest and generate appropriate errors. + */ +// TODO: should validate that the resources referenced within the manifest are located within the +// plugin folder +// TODO: Need a comprehensive test that triggers every notices; +function isValidManifest(manifest: any, notices: string[]): boolean { + if (!manifest) { + notices.push("missing manifest"); + return false; + } + try { + manifestChecker.check(manifest); + } catch (e) { + notices.push(`Invalid manifest: ${e.message}`); + return false; + } + try { + manifestChecker.strictCheck(manifest); + } catch (e) { + notices.push(`WARNING: ${e.message}` ); + /* but don't fail */ + } + if (Object.keys(manifest.contributions).length === 0) { + notices.push("WARNING: no valid contributions"); + } + return true; +} + +/** + * A ManifestError is an error caused by a wrongly formatted manifest or missing manifest. The + * `notices` property holds a user-friendly description of the error(s). + */ +export class ManifestError extends Error { + constructor(public notices: string[], message: string = "") { + super(message); + } +} + +/** + * Parse the manifest. Look first for a Yaml manifest and then if missing for a Json manifest. + */ +export async function readManifest(pluginPath: string): Promise { + const notices: string[] = []; + const manifest = await _readManifest(pluginPath); + if (isValidManifest(manifest, notices)) { + return manifest as BarePlugin; + } + throw new ManifestError(notices); +} + +async function _readManifest(pluginPath: string): Promise { + try { + return yaml.safeLoad(await readManifestFile("yml")); + } catch (e) { + if (e instanceof yaml.YAMLException) { + throw new Error('error parsing yaml manifest: ' + e.message); + } + } + try { + return JSON.parse(await readManifestFile("json")); + } catch (e) { + if (e instanceof SyntaxError) { + throw new Error('error parsing json manifest' + e.message); + } + throw new Error('cannot read manifest file: ' + e.message); + } + async function readManifestFile(fileExtension: string): Promise { + return await fse.readFile(path.join(pluginPath, "manifest." + fileExtension), "utf8"); + } +} diff --git a/app/server/lib/places.ts b/app/server/lib/places.ts new file mode 100644 index 00000000..adf83db7 --- /dev/null +++ b/app/server/lib/places.ts @@ -0,0 +1,46 @@ +/** + * Utilities related to the layout of the application and where parts are stored. + */ + +import * as path from 'path'; + +/** + * codeRoot is the directory containing ./app with all the JS code. + */ +export const codeRoot = path.dirname(path.dirname(path.dirname(__dirname))); + +/** + * Returns the appRoot, i.e. the directory containing ./sandbox, ./node_modules, ./ormconfig.js, + * etc. + */ +export function getAppRoot(): string { + if (codeRoot.endsWith('/_build/core')) { return path.dirname(path.dirname(codeRoot)); } + return codeRoot.endsWith('/_build') ? path.dirname(codeRoot) : codeRoot; +} + +/** + * When packaged as an electron application, most files are stored in a .asar + * archive. Most, but not all. This method takes the "application root" + * which is that .asar file in packaged form, and returns a directory where + * remaining files are available on the regular filesystem. + */ +export function getUnpackedAppRoot(appRoot: string): string { + return path.resolve(path.dirname(appRoot), path.basename(appRoot, '.asar')); +} + +/** + * Return the correct root for a given subdirectory. + */ +export function getAppRootFor(appRoot: string, subdirectory: string): string { + if (['sandbox', 'plugins', 'public-api'].includes(subdirectory)) { + return getUnpackedAppRoot(appRoot); + } + return appRoot; +} + +/** + * Return the path to a given subdirectory, from the correct appRoot. + */ +export function getAppPathTo(appRoot: string, subdirectory: string): string { + return path.resolve(getAppRootFor(appRoot, subdirectory), subdirectory); +} diff --git a/app/server/lib/requestUtils.ts b/app/server/lib/requestUtils.ts new file mode 100644 index 00000000..cb9921cb --- /dev/null +++ b/app/server/lib/requestUtils.ts @@ -0,0 +1,204 @@ +import {ApiError} from 'app/common/ApiError'; +import {DEFAULT_HOME_SUBDOMAIN, parseSubdomain} from 'app/common/gristUrls'; +import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/HomeDBManager'; +import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer'; +import {RequestWithOrg} from 'app/server/lib/extractOrg'; +import * as log from 'app/server/lib/log'; +import {Request, Response} from 'express'; +import {URL} from 'url'; + +// log api details outside of dev environment (when GRIST_HOSTED_VERSION is set) +const shouldLogApiDetails = Boolean(process.env.GRIST_HOSTED_VERSION); + +// Offset to https ports in dev/testing environment. +export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ? + parseInt(process.env.GRIST_TEST_HTTPS_OFFSET, 10) : undefined; + +// Database fields that we permit in entities but don't want to cross the api. +const INTERNAL_FIELDS = new Set(['apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId', + 'stripeCustomerId', 'stripeSubscriptionId', 'stripePlanId', + 'stripeProductId', 'userId', 'isFirstTimeUser']); + +/** + * Adapt a home-server or doc-worker URL to match the hostname in the request URL. For custom + * domains and when GRIST_SERVE_SAME_ORIGIN is set, we replace the full hostname; otherwise just + * the base of the hostname. The changes to url are made in-place. + * + * For dev purposes, port is kept but possibly adjusted for TEST_HTTPS_OFFSET. Note that if port + * is different from req's port, it is not considered same-origin for CORS purposes, but would + * still receive cookies. + */ +export function adaptServerUrl(url: URL, req: RequestWithOrg): void { + const reqBaseDomain = parseSubdomain(req.hostname).base; + + if (process.env.GRIST_SERVE_SAME_ORIGIN === 'true' || req.isCustomHost) { + url.hostname = req.hostname; + } else if (reqBaseDomain) { + const subdomain: string|undefined = parseSubdomain(url.hostname).org || DEFAULT_HOME_SUBDOMAIN; + url.hostname = `${subdomain}${reqBaseDomain}`; + } + + // In dev/test environment we can turn on a flag to adjust URLs to use https. + if (TEST_HTTPS_OFFSET && url.port && url.protocol === 'http:') { + url.port = String(parseInt(url.port, 10) + TEST_HTTPS_OFFSET); + url.protocol = 'https:'; + } +} + +/** + * Returns true for requests from permitted origins. For such requests, an + * "Access-Control-Allow-Origin" header is added to the response. Vary: Origin + * is also set to reflect the fact that the headers are a function of the origin, + * to prevent inappropriate caching on the browser's side. + */ +export function trustOrigin(req: Request, resp: Response): boolean { + // TODO: We may want to consider changing allowed origin values in the future. + // Note that the request origin is undefined for non-CORS requests. + const origin = req.get('origin'); + if (!origin) { return true; } // Not a CORS request. + if (!allowHost(req, new URL(origin))) { return false; } + + // For a request to a custom domain, the full hostname must match. + resp.header("Access-Control-Allow-Origin", origin); + resp.header("Vary", "Origin"); + return true; +} + +// Returns whether req satisfies the given allowedHost. Unless req is to a custom domain, it is +// enough if only the base domains match. Differing ports are allowed, which helps in dev/testing. +export function allowHost(req: Request, allowedHost: string|URL) { + const mreq = req as RequestWithOrg; + const proto = req.protocol; + const actualUrl = new URL(`${proto}://${req.get('host')}`); + const allowedUrl = (typeof allowedHost === 'string') ? new URL(`${proto}://${allowedHost}`) : allowedHost; + if (mreq.isCustomHost) { + // For a request to a custom domain, the full hostname must match. + return actualUrl.hostname === allowedUrl.hostname; + } else { + // For requests to a native subdomains, only the base domain needs to match. + const allowedDomain = parseSubdomain(allowedUrl.hostname); + const actualDomain = parseSubdomain(actualUrl.hostname); + return (actualDomain.base === allowedDomain.base); + } +} + +export function isParameterOn(parameter: any): boolean { + return ['1', 'on', 'true'].includes(String(parameter).toLowerCase()); +} + +/** + * Get Scope from request, and make sure it has everything needed for a document. + */ +export function getDocScope(req: Request): DocScope { + const scope = getScope(req); + if (!scope.urlId) { throw new Error('document required'); } + return scope as DocScope; +} + +/** + * Extract information included in the request that may restrict the scope of + * that request. Not all requests will support all restrictions. + * + * - userId - Mandatory. Produced by authentication middleware. + * Information returned and actions taken will be limited by what + * that user has access to. + * + * - org - Optional. Extracted by middleware. Limits + * information/action to the given org. Not every endpoint + * respects this limit. Possible exceptions include endpoints for + * listing orgs a user has access to, and endpoints with an org id + * encoded in them. + * + * - urlId - Optional. Embedded as "did" (or "docId") path parameter in endpoints related + * to documents. Specifies which document the request pertains to. Can + * be a urlId or a docId. + * + * - includeSupport - Optional. Embedded as "includeSupport" query parameter. + * Just a few endpoints support this, it is a very specific "hack" for including + * an example workspace in org listings. + * + * - showRemoved - Optional. Embedded as "showRemoved" query parameter. + * Supported by many endpoints. When absent, request is limited + * to docs/workspaces that have not been removed. When present, request + * is limited to docs/workspaces that have been removed. + */ +export function getScope(req: Request): Scope { + const urlId = req.params.did || req.params.docId; + const userId = getUserId(req); + const org = (req as RequestWithOrg).org; + const {specialPermit} = (req as RequestWithLogin); + const includeSupport = isParameterOn(req.query.includeSupport); + const showRemoved = isParameterOn(req.query.showRemoved); + return {urlId, userId, org, includeSupport, showRemoved, specialPermit}; +} + +// Return a JSON response reflecting the output of a query. +// Filter out keys we don't want crossing the api. +// Set req to null to not log any information about request. +export async function sendReply(req: Request|null, res: Response, result: QueryResult) { + const data = pruneAPIResult(result.data || null); + if (shouldLogApiDetails && req) { + const mreq = req as RequestWithLogin; + log.rawDebug('api call', { + url: req.url, + userId: mreq.userId, + email: mreq.user && mreq.user.loginEmail, + org: mreq.org, + params: req.params, + body: req.body, + result: data, + }); + } + if (result.status === 200) { + return res.json(data); + } else { + return res.status(result.status).json({error: result.errMessage}); + } +} + +export async function sendOkReply(req: Request|null, res: Response, result?: T) { + return sendReply(req, res, {status: 200, data: result}); +} + +export function pruneAPIResult(data: T): T { + // TODO: This can be optimized by pruning data recursively without serializing in between. But + // it's fairly fast even with serializing (on the order of 15usec/kb). + const output = JSON.stringify(data, + (key: string, value: any) => { + // Do not include removedAt field if it is not set. It is not relevant to regular + // situations where the user is working with non-deleted resources. + if (key === 'removedAt' && value === null) { return undefined; } + return INTERNAL_FIELDS.has(key) ? undefined : value; + }); + return JSON.parse(output); +} + +/** + * Access the canonical docId associated with the request. Must have already authorized. + */ +export function getDocId(req: Request) { + const mreq = req as RequestWithLogin; + // We should always have authorized by now. + if (!mreq.docAuth || !mreq.docAuth.docId) { throw new ApiError(`unknown document`, 500); } + return mreq.docAuth.docId; +} + +export function optStringParam(p: any): string|undefined { + if (typeof p === 'string') { return p; } + return undefined; +} + +export function stringParam(p: any): string { + if (typeof p === 'string') { return p; } + throw new Error(`parameter should be a string: ${p}`); +} + +export function integerParam(p: any): number { + if (typeof p === 'number') { return Math.floor(p); } + if (typeof p === 'string') { return parseInt(p, 10); } + throw new Error(`parameter should be an integer: ${p}`); +} + +export interface RequestWithGristInfo extends Request { + gristInfo?: string; +} diff --git a/app/server/lib/sandboxUtil.js b/app/server/lib/sandboxUtil.js new file mode 100644 index 00000000..00e68b2b --- /dev/null +++ b/app/server/lib/sandboxUtil.js @@ -0,0 +1,58 @@ +/** + * Various utilities and constants for communicating with the python sandbox. + */ + + +var MemBuffer = require('app/common/MemBuffer'); +var log = require('./log'); + + +/** + * SandboxError is an error type for reporting errors forwarded from the sandbox. + */ +function SandboxError(message) { + // Poorly documented node feature, required to make the derived error keep a proper stack trace. + Error.captureStackTrace(this, this.constructor); + this.name = 'SandboxError'; + this.message = "[Sandbox] " + (message || 'Python reported an error'); +} +SandboxError.prototype = new Error(); +// We need to set the .constructor property for Error.captureStackTrace to work correctly. +SandboxError.prototype.constructor = SandboxError; + +exports.SandboxError = SandboxError; + + +/** + * Special msgCode values that precede msgBody to indicate what kind of message it is. + * These all cost one byte. If we needed more, we should probably switch to a number (5 bytes) + * CALL = call to the other side. The data must be an array of [func_name, arguments...] + * DATA = data must be a value to return to a call from the other side + * EXC = data must be an exception to return to a call from the other side + */ +exports.CALL = null; +exports.DATA = true; +exports.EXC = false; + + +/** + * Returns a function that takes data buffers and logs them to log.info() with the given prefix. + * The logged output is line-oriented, so that the prefix is only inserted at the start of a line. + * Binary data is encoded as with JSON.stringify. + */ +function makeLinePrefixer(prefix, logMeta) { + var partial = ''; + return data => { + partial += MemBuffer.arrayToString(data); + var newline; + while ((newline = partial.indexOf("\n")) !== -1) { + var line = partial.slice(0, newline); + partial = partial.slice(newline + 1); + // Escape some parts of the string by serializing it to JSON (without the quotes). + log.rawInfo(prefix + JSON.stringify(line).slice(1, -1).replace(/\\"/g, '"') + .replace(/\\\\/g, '\\'), + logMeta); + } + }; +} +exports.makeLinePrefixer = makeLinePrefixer; diff --git a/app/server/lib/sendAppPage.ts b/app/server/lib/sendAppPage.ts new file mode 100644 index 00000000..49fa48c3 --- /dev/null +++ b/app/server/lib/sendAppPage.ts @@ -0,0 +1,105 @@ +import {GristLoadConfig} from 'app/common/gristUrls'; +import {isAnonymousUser} from 'app/server/lib/Authorizer'; +import {RequestWithOrg} from 'app/server/lib/extractOrg'; +import {GristServer} from 'app/server/lib/GristServer'; +import * as express from 'express'; +import * as fse from 'fs-extra'; +import * as path from 'path'; + +export interface ISendAppPageOptions { + path: string; // Ignored if .content is present (set to "" for clarity). + content?: string; + status: number; + config: Partial; + tag?: string; // If present, override version tag. + + // If present, enable Google Tag Manager on this page (if GOOGLE_TAG_MANAGER_ID env var is set). + // Used on the welcome page to track sign-ups. We don't intend to use it for in-app analytics. + // Set to true to insert tracker unconditionally; false to omit it; "anon" to insert + // it only when the user is not logged in. + googleTagManager?: true | false | 'anon'; +} + +export function makeGristConfig(homeUrl: string|null, extra: Partial, + baseDomain?: string, req?: express.Request +): GristLoadConfig { + // .invalid is a TLD the IETF promises will never exist. + const pluginUrl = process.env.APP_UNTRUSTED_URL || 'plugins.invalid'; + const pathOnly = (process.env.GRIST_ORG_IN_PATH === "true") || + (homeUrl && new URL(homeUrl).hostname === 'localhost') || false; + const mreq = req as RequestWithOrg|undefined; + return { + homeUrl, + org: process.env.GRIST_SINGLE_ORG || (mreq && mreq.org), + baseDomain, + singleOrg: process.env.GRIST_SINGLE_ORG, + pathOnly, + supportAnon: shouldSupportAnon(), + pluginUrl, + stripeAPIKey: process.env.STRIPE_PUBLIC_API_KEY, + helpScoutBeaconId: process.env.HELP_SCOUT_BEACON_ID, + maxUploadSizeImport: (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || undefined, + maxUploadSizeAttachment: (Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024) || undefined, + timestampMs: Date.now(), + ...extra, + }; +} + +/** + * Send a simple template page, read from file at pagePath (relative to static/), with certain + * placeholders replaced. + */ +export function makeSendAppPage(opts: { + server: GristServer|null, staticDir: string, tag: string, testLogin?: boolean, + baseDomain?: string +}) { + const {server, staticDir, tag, testLogin} = opts; + return async (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => { + // .invalid is a TLD the IETF promises will never exist. + const config = makeGristConfig(server ? server.getHomeUrl(req) : null, options.config, + opts.baseDomain, req); + + // We could cache file contents in memory, but the filesystem does caching too, and compared + // to that, the performance gain is unlikely to be meaningful. So keep it simple here. + const fileContent = options.content || await fse.readFile(path.join(staticDir, options.path), 'utf8'); + + const needTagManager = (options.googleTagManager === 'anon' && isAnonymousUser(req)) || + options.googleTagManager === true; + const tagManagerSnippet = needTagManager ? getTagManagerSnippet() : ''; + const staticOrigin = process.env.APP_STATIC_URL || ""; + const staticBaseUrl = `${staticOrigin}/v/${options.tag || tag}/`; + const warning = testLogin ? "
Authentication is not enforced
" : ""; + const content = fileContent + .replace("", warning) + .replace("", `` + tagManagerSnippet) + .replace("", ``); + resp.status(options.status).type('html').send(content); + }; +} + +function shouldSupportAnon() { + // Enable UI for anonymous access if a flag is explicitly set in the environment + return process.env.GRIST_SUPPORT_ANON === "true"; +} + +/** + * Returns the Google Tag Manager snippet to insert into of the page, if + * GOOGLE_TAG_MANAGER_ID env var is set to a non-empty value. Otherwise returns the empty string. + */ +function getTagManagerSnippet() { + // Note also that we only insert the snippet for the . The second recommended part (for + // ) is for