Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'main' into spoffy/webdriver-logs

Browse Source
This commit is contained in:
Spoffy 2024-09-20 01:26:56 +01:00 committed by GitHub
commit 0bfc61507a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
159 changed files with 6301 additions and 1473 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/docker_latest.yml vendored
View File

@ -46,6 +46,7 @@ jobs:
push_to_registry:
name: Push latest Docker image to Docker Hub
runs-on: ubuntu-latest
if: ${{ vars.RUN_DAILY_BUILD }}
strategy:
matrix:
python-version: [3.11]
@ -123,6 +124,9 @@ jobs:
if: ${{ !inputs.disable_tests }}
run: yarn run build:prod
- name: Install Google Chrome for Testing
run: ./test/test_env.sh node_modules/selenium-webdriver/bin/linux/selenium-manager
- name: Run tests
if: ${{ !inputs.disable_tests }}
run: TEST_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }} VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:docker

4
.github/workflows/main.yml vendored
View File

@ -69,9 +69,9 @@ jobs:
- name: Build Node.js code
run: yarn run build:prod
- name: Install chromedriver
- name: Install Google Chrome for Testing
if: contains(matrix.tests, ':nbrowser-') || contains(matrix.tests, ':smoke:') || contains(matrix.tests, ':stubs:')
run: ./node_modules/selenium-webdriver/bin/linux/selenium-manager --driver chromedriver
run: ./test/test_env.sh ./node_modules/selenium-webdriver/bin/linux/selenium-manager
- name: Run smoke test
if: contains(matrix.tests, ':smoke:')

8
.gitignore vendored
View File

@ -12,6 +12,11 @@
/sandbox_venv*
/.vscode/
# Files created by grist-desktop setup
/cpython.tar.gz
/python
/static_ext
# Build helper files.
/.build*
@ -82,7 +87,8 @@ xunit.xml
**/_build
# ext directory can be overwritten
ext/**
/ext
/ext/**
# Docker compose examples - persistent values and secrets
/docker-compose-examples/*/persist

2
README.md
View File

@ -312,6 +312,8 @@ Grist can be configured in many ways. Here are the main environment variables it
| GRIST_UI_FEATURES | comma-separated list of UI features to enable. Allowed names of parts: `helpCenter,billing,templates,createSite,multiSite,multiAccounts,sendToDrive,tutorials,supportGrist`. If a part also exists in GRIST_HIDE_UI_ELEMENTS, it won't be enabled. |
| GRIST_UNTRUSTED_PORT | if set, plugins will be served from the given port. This is an alternative to setting APP_UNTRUSTED_URL. |
| GRIST_WIDGET_LIST_URL | a url pointing to a widget manifest, by default `https://github.com/gristlabs/grist-widget/releases/download/latest/manifest.json` is used |
| GRIST_LOG_HTTP | When set to `true`, log HTTP requests and responses information. Defaults to `false`. |
| GRIST_LOG_HTTP_BODY | When this variable and `GRIST_LOG_HTTP` are set to `true` , log the body along with the HTTP requests. :warning: Be aware it may leak confidential information in the logs.:warning: Defaults to `false`. |
| COOKIE_MAX_AGE | session cookie max age, defaults to 90 days; can be set to "none" to make it a session cookie |
| HOME_PORT | port number to listen on for REST API server; if set to "share", add API endpoints to regular grist port. |
| PORT | port number to listen on for Grist server |

4
app/client/components/DataTables.ts
View File

@ -12,8 +12,9 @@ import {icon} from 'app/client/ui2018/icons';
import {loadingDots} from 'app/client/ui2018/loaders';
import {menu, menuDivider, menuIcon, menuItem, menuItemAsync, menuText} from 'app/client/ui2018/menus';
import {confirmModal} from 'app/client/ui2018/modals';
import {Computed, Disposable, dom, fromKo, makeTestId, observable, Observable, styled} from 'grainjs';
import {Computed, Disposable, dom, fromKo, observable, Observable, styled} from 'grainjs';
import {makeT} from 'app/client/lib/localization';
import {makeTestId} from 'app/client/lib/domUtils';
import * as weasel from 'popweasel';
const testId = makeTestId('test-raw-data-');
@ -109,6 +110,7 @@ export class DataTables extends Disposable {
),
cssDotsButton(
testId('table-menu'),
testId(use => `table-menu-${use(tableRec.tableId)}`),
icon('Dots'),
menu(() => this._menuItems(tableRec, isEditingName), {placement: 'bottom-start'}),
dom.on('click', (ev) => { ev.stopPropagation(); ev.preventDefault(); }),

17
app/client/components/DocumentUsage.ts
View File

@ -7,7 +7,7 @@ import {withInfoTooltip} from 'app/client/ui/tooltips';
import {mediaXSmall, theme} from 'app/client/ui2018/cssVars';
import {icon} from 'app/client/ui2018/icons';
import {loadingDots, loadingSpinner} from 'app/client/ui2018/loaders';
import {APPROACHING_LIMIT_RATIO, DataLimitStatus} from 'app/common/DocUsage';
import {APPROACHING_LIMIT_RATIO, DataLimitInfo} from 'app/common/DocUsage';
import {Features, isFreePlan} from 'app/common/Features';
import {capitalizeFirstWord} from 'app/common/gutil';
import {canUpgradeOrg} from 'app/common/roles';
@ -40,8 +40,8 @@ export class DocumentUsage extends Disposable {
// TODO: Update this whenever the rest of the UI is internationalized.
private readonly _rowCountFormatter = new Intl.NumberFormat('en-US');
private readonly _dataLimitStatus = Computed.create(this, this._currentDocUsage, (_use, usage) => {
return usage?.dataLimitStatus ?? null;
private readonly _dataLimitInfo = Computed.create(this, this._currentDocUsage, (_use, usage) => {
return usage?.dataLimitInfo;
});
private readonly _rowCount = Computed.create(this, this._currentDocUsage, (_use, usage) => {
@ -158,11 +158,11 @@ export class DocumentUsage extends Disposable {
const org = use(this._currentOrg);
const product = use(this._currentProduct);
const features = use(this._currentFeatures);
const status = use(this._dataLimitStatus);
if (!org || !status) { return null; }
const usageInfo = use(this._dataLimitInfo);
if (!org || !usageInfo?.status) { return null; }
return buildMessage([
buildLimitStatusMessage(status, features, {
buildLimitStatusMessage(usageInfo, features, {
disableRawDataLink: true
}),
(product && isFreePlan(product.name)
@ -196,13 +196,14 @@ export class DocumentUsage extends Disposable {
}
export function buildLimitStatusMessage(
status: NonNullable<DataLimitStatus>,
usageInfo: NonNullable<DataLimitInfo>,
features?: Features|null,
options: {
disableRawDataLink?: boolean;
} = {}
) {
const {disableRawDataLink = false} = options;
const {status, daysRemaining} = usageInfo;
switch (status) {
case 'approachingLimit': {
return [
@ -224,7 +225,7 @@ export function buildLimitStatusMessage(
return [
'Document limits ',
disableRawDataLink ? 'exceeded' : buildRawDataPageLink('exceeded'),
`. In ${gracePeriodDays} days, this document will be read-only.`
`. In ${daysRemaining} days, this document will be read-only.`
];
}
case 'deleteOnly': {

4
app/client/components/FormRenderer.ts
View File

@ -135,7 +135,9 @@ class ParagraphRenderer extends FormRenderer {
return css.paragraph(
css.paragraph.cls(`-alignment-${this.layoutNode.alignment || 'left'}`),
el => {
el.innerHTML = sanitizeHTML(marked(this.layoutNode.text || '**Lorem** _ipsum_ dolor'));
el.innerHTML = sanitizeHTML(marked(this.layoutNode.text || '**Lorem** _ipsum_ dolor', {
async: false,
}));
},
);
}

2
app/client/components/Forms/styles.ts
View File

@ -505,7 +505,7 @@ export const cssMarkdownRender = styled('div', `
export function markdown(obs: BindableValue<string>, ...args: IDomArgs<HTMLDivElement>) {
return cssMarkdownRender(el => {
dom.autoDisposeElem(el, subscribeBindable(obs, val => {
el.innerHTML = sanitizeHTML(marked(val));
el.innerHTML = sanitizeHTML(marked(val, {async: false}));
}));
}, ...args);
}

12
app/client/components/Printing.css
View File

@ -5,6 +5,10 @@
display: none;
}
.print-force-hide {
display: none !important;
}
.print-parent {
display: block !important;
position: relative !important;
@ -51,6 +55,10 @@
.ui-resizable-handle {
display: none !important;
}
.viewsection_content .filter_bar {
display: none !important;
}
}
/*
@ -68,4 +76,8 @@
.print-all-rows {
display: none;
}
.screen-force-hide {
display: none !important;
}
}

4
app/client/declarations.d.ts vendored
View File

@ -335,3 +335,7 @@ interface Location {
// historical accident than an intentional choice.
reload(forceGet?: boolean): void;
}
interface JQuery {
datepicker(options: unknown): JQuery;
}

2
app/client/lib/markdown.ts
View File

@ -25,5 +25,5 @@ export function markdown(markdownObs: BindableValue<string>): DomElementMethod {
}
function setMarkdownValue(elem: Element, markdownValue: string): void {
elem.innerHTML = sanitizeHTML(marked(markdownValue));
elem.innerHTML = sanitizeHTML(marked(markdownValue, {async: false}));
}

3
app/client/models/DataRowModel.ts
View File

@ -21,6 +21,9 @@ export class DataRowModel extends BaseRowModel {
public _validationFailures: ko.PureComputed<Array<IRowModel<'_grist_Validations'>>>;
public _isAddRow: ko.Observable<boolean>;
// Observable that's set whenever a change to a row model is likely to be real, and unset when a
// row model is being reassigned to a different row. If a widget uses CSS transitions for
// changes, those should only be enabled when _isRealChange is true.
public _isRealChange: ko.Observable<boolean>;
public constructor(dataTableModel: DataTableModel, colNames: string[]) {

10
app/client/ui/AccountWidget.ts
View File

@ -196,13 +196,17 @@ export class AccountWidget extends Disposable {
if (deploymentType !== 'saas') { return null; }
const {currentValidUser, currentOrg, isTeamSite} = this._appModel;
const isBillingManager = Boolean(currentOrg && currentOrg.billingAccount &&
(currentOrg.billingAccount.isManager || currentValidUser?.isSupport));
const canViewBillingPage = Boolean(
currentOrg && // have accecc to org
currentOrg.billingAccount && // have access to billing account
(currentOrg.billingAccount.isManager // is billing manager
|| currentValidUser?.isSupport // or support
|| this._appModel.isInstallAdmin())); // or install admin
return isTeamSite ?
// For links, disabling with just a class is hard; easier to just not make it a link.
// TODO weasel menus should support disabling menuItemLink.
(isBillingManager ?
(canViewBillingPage ?
menuItemLink(urlState().setLinkUrl({billing: 'billing'}), t('Billing Account')) :
menuItem(() => null, t('Billing Account'), dom.cls('disabled', true))
) :

66
app/client/ui/HomeImports.ts → app/client/ui/CoreHomeImports.ts
View File

@ -1,13 +1,13 @@
import {AppModel, reportError} from 'app/client/models/AppModel';
import {AxiosProgressEvent} from 'axios';
import {PluginScreen} from 'app/client/components/PluginScreen';
import {guessTimezone} from 'app/client/lib/guessTimezone';
import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
import {IMPORTABLE_EXTENSIONS, uploadFiles} from 'app/client/lib/uploads';
import {AppModel, reportError} from 'app/client/models/AppModel';
import {IProgress} from 'app/client/models/NotifyModel';
import {ImportProgress} from 'app/client/ui/ImportProgress';
import {IMPORTABLE_EXTENSIONS} from 'app/client/lib/uploads';
import {openFilePicker} from 'app/client/ui/FileDialog';
import {byteString} from 'app/common/gutil';
import { AxiosProgressEvent } from 'axios';
import {Disposable} from 'grainjs';
import {uploadFiles} from 'app/client/lib/uploads';
/**
* Imports a document and returns its docId, or null if no files were selected.
@ -66,62 +66,6 @@ export async function fileImport(
progressUI.dispose();
}
}
export class ImportProgress extends Disposable {
// Import does upload first, then import. We show a single indicator, estimating which fraction
// of the time should be given to upload (whose progress we can report well), and which to the
// subsequent import (whose progress indicator is mostly faked).
private _uploadFraction: number;
private _estImportSeconds: number;
private _importTimer: null | ReturnType<typeof setInterval> = null;
private _importStart: number = 0;
constructor(private _progressUI: IProgress, file: File) {
super();
// We'll assume that for .grist files, the upload takes 90% of the total time, and for other
// files, 40%.
this._uploadFraction = file.name.endsWith(".grist") ? 0.9 : 0.4;
// TODO: Import step should include a progress callback, to be combined with upload progress.
// Without it, we estimate import to take 2s per MB (non-scientific unreliable estimate), and
// use an asymptotic indicator which keeps moving without ever finishing. Not terribly useful,
// but does slow down for larger files, and is more comforting than a stuck indicator.
this._estImportSeconds = file.size / 1024 / 1024 * 2;
this._progressUI.setProgress(0);
this.onDispose(() => this._importTimer && clearInterval(this._importTimer));
}
// Once this reaches 100, the import stage begins.
public setUploadProgress(percentage: number) {
this._progressUI.setProgress(percentage * this._uploadFraction);
if (percentage >= 100 && !this._importTimer) {
this._importStart = Date.now();
this._importTimer = setInterval(() => this._onImportTimer(), 100);
}
}
public finish() {
if (this._importTimer) {
clearInterval(this._importTimer);
}
this._progressUI.setProgress(100);
}
/**
* Calls _progressUI.setProgress(percent) with percentage increasing from 0 and asymptotically
* approaching 100, reaching 50% after estSeconds. It's intended to look reasonable when the
* estimate is good, and to keep showing slowing progress even if it's not.
*/
private _onImportTimer() {
const elapsedSeconds = (Date.now() - this._importStart) / 1000;
const importProgress = elapsedSeconds / (elapsedSeconds + this._estImportSeconds);
const progress = this._uploadFraction + importProgress * (1 - this._uploadFraction);
this._progressUI.setProgress(100 * progress);
}
}
/**
* Imports document through a plugin from a home/welcome screen.
*/

47
app/client/ui/CoreNewDocMethods.ts Normal file
View File

@ -0,0 +1,47 @@
import {homeImports} from 'app/client/ui/HomeImports';
import {docUrl, urlState} from 'app/client/models/gristUrlState';
import {HomeModel} from 'app/client/models/HomeModel';
import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
import {reportError} from 'app/client/models/AppModel';
export async function createDocAndOpen(home: HomeModel) {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
try {
const docId = await home.createDoc("Untitled document", destWS === "unsaved" ? "unsaved" : destWS.id);
// Fetch doc information including urlId.
// TODO: consider changing API to return same response as a GET when creating an
// object, which is a semi-standard.
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
} catch (err) {
reportError(err);
}
}
export async function importDocAndOpen(home: HomeModel) {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
const docId = await homeImports.docImport(home.app, destWS === "unsaved" ? "unsaved" : destWS.id);
if (docId) {
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
}
}
export async function importFromPluginAndOpen(home: HomeModel, source: ImportSourceElement) {
try {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
const docId = await homeImports.importFromPlugin(
home.app,
destWS === "unsaved" ? "unsaved" : destWS.id,
source);
if (docId) {
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
}
} catch (err) {
reportError(err);
}
}

10
app/client/ui/DocTutorial.ts
View File

@ -4,7 +4,7 @@ import {logTelemetryEvent} from 'app/client/lib/telemetry';
import {getWelcomeHomeUrl, urlState} from 'app/client/models/gristUrlState';
import {renderer} from 'app/client/ui/DocTutorialRenderer';
import {cssPopupBody, FLOATING_POPUP_TOOLTIP_KEY, FloatingPopup} from 'app/client/ui/FloatingPopup';
import {sanitizeHTML} from 'app/client/ui/sanitizeHTML';
import {sanitizeTutorialHTML} from 'app/client/ui/sanitizeHTML';
import {hoverTooltip, setHoverTooltip} from 'app/client/ui/tooltips';
import {basicButton, primaryButton, textButton} from 'app/client/ui2018/buttons';
import {mediaXSmall, theme, vars} from 'app/client/ui2018/cssVars';
@ -13,7 +13,7 @@ import {loadingSpinner} from 'app/client/ui2018/loaders';
import {confirmModal, modal} from 'app/client/ui2018/modals';
import {parseUrlId} from 'app/common/gristUrls';
import {dom, makeTestId, Observable, styled} from 'grainjs';
import {marked} from 'marked';
import {marked, Token} from 'marked';
import debounce = require('lodash/debounce');
import range = require('lodash/range');
import sortBy = require('lodash/sortBy');
@ -219,7 +219,7 @@ export class DocTutorial extends FloatingPopup {
return value ? String(value) : undefined;
};
const walkTokens = (token: marked.Token) => {
const walkTokens = (token: Token) => {
if (token.type === 'image') {
imageUrls.push(token.href);
}
@ -231,13 +231,13 @@ export class DocTutorial extends FloatingPopup {
let slideContent = getValue('slide_content');
if (!slideContent) { return null; }
slideContent = sanitizeHTML(await marked.parse(slideContent, {
slideContent = sanitizeTutorialHTML(await marked.parse(slideContent, {
async: true, renderer, walkTokens
}));
let boxContent = getValue('box_content');
if (boxContent) {
boxContent = sanitizeHTML(await marked.parse(boxContent, {
boxContent = sanitizeTutorialHTML(await marked.parse(boxContent, {
async: true, renderer, walkTokens
}));
}

4
app/client/ui/DocTutorialRenderer.ts
View File

@ -2,7 +2,7 @@ import {marked} from 'marked';
export const renderer = new marked.Renderer();
renderer.image = (href: string | null, title: string | null, _text: string) => {
renderer.image = ({href, title}) => {
let classes = 'doc-tutorial-popup-thumbnail';
const hash = href?.split('#')?.[1];
if (hash) {
@ -17,6 +17,6 @@ renderer.image = (href: string | null, title: string | null, _text: string) => {
</div>`;
};
renderer.link = (href: string | null, _title: string | null, text: string) => {
renderer.link = ({href, text}) => {
return `<a href="${href}" target="_blank">${text}</a>`;
};

6
app/client/ui/HomeIntro.ts
View File

@ -3,7 +3,7 @@ import {getLoginOrSignupUrl, getLoginUrl, getSignupUrl, urlState} from 'app/clie
import {HomeModel} from 'app/client/models/HomeModel';
import {productPill} from 'app/client/ui/AppHeader';
import * as css from 'app/client/ui/DocMenuCss';
import {createDocAndOpen, importDocAndOpen} from 'app/client/ui/HomeLeftPane';
import {newDocMethods} from 'app/client/ui/NewDocMethods';
import {manageTeamUsersApp} from 'app/client/ui/OpenUserManager';
import {bigBasicButton, cssButton} from 'app/client/ui2018/buttons';
import {testId, theme, vars} from 'app/client/ui2018/cssVars';
@ -177,11 +177,11 @@ function buildButtons(homeModel: HomeModel, options: {
),
!options.import ? null :
cssBtn(cssBtnIcon('Import'), t("Import Document"), testId('intro-import-doc'),
dom.on('click', () => importDocAndOpen(homeModel)),
dom.on('click', () => newDocMethods.importDocAndOpen(homeModel)),
),
!options.empty ? null :
cssBtn(cssBtnIcon('Page'), t("Create Empty Document"), testId('intro-create-doc'),
dom.on('click', () => createDocAndOpen(homeModel)),
dom.on('click', () => newDocMethods.createDocAndOpen(homeModel)),
),
);
}

72
app/client/ui/HomeLeftPane.ts
View File

@ -1,29 +1,27 @@
import {makeT} from 'app/client/lib/localization';
import {loadUserManager} from 'app/client/lib/imports';
import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
import {reportError} from 'app/client/models/AppModel';
import {docUrl, urlState} from 'app/client/models/gristUrlState';
import {urlState} from 'app/client/models/gristUrlState';
import {HomeModel} from 'app/client/models/HomeModel';
import {getWorkspaceInfo, workspaceName} from 'app/client/models/WorkspaceInfo';
import {getAdminPanelName} from 'app/client/ui/AdminPanelName';
import * as roles from 'app/common/roles';
import {addNewButton, cssAddNewButton} from 'app/client/ui/AddNewButton';
import {docImport, importFromPlugin} from 'app/client/ui/HomeImports';
import {commonUrls, isFeatureEnabled} from 'app/common/gristUrls';
import {computed, dom, domComputed, DomElementArg, observable, Observable, styled} from 'grainjs';
import {newDocMethods} from 'app/client/ui/NewDocMethods';
import {createHelpTools, cssLeftPanel, cssScrollPane,
cssSectionHeader, cssTools} from 'app/client/ui/LeftPanelCommon';
import {
cssLinkText, cssMenuTrigger, cssPageEntry, cssPageIcon, cssPageLink, cssSpacer
} from 'app/client/ui/LeftPanelCommon';
import {createVideoTourToolsButton} from 'app/client/ui/OpenVideoTour';
import {transientInput} from 'app/client/ui/transientInput';
import {testId, theme} from 'app/client/ui2018/cssVars';
import {icon} from 'app/client/ui2018/icons';
import {menu, menuIcon, menuItem, upgradableMenuItem, upgradeText} from 'app/client/ui2018/menus';
import {testId, theme} from 'app/client/ui2018/cssVars';
import {confirmModal} from 'app/client/ui2018/modals';
import {commonUrls, isFeatureEnabled} from 'app/common/gristUrls';
import * as roles from 'app/common/roles';
import {createVideoTourToolsButton} from 'app/client/ui/OpenVideoTour';
import {getGristConfig} from 'app/common/urlUtils';
import {icon} from 'app/client/ui2018/icons';
import {transientInput} from 'app/client/ui/transientInput';
import {Workspace} from 'app/common/UserAPI';
import {computed, dom, domComputed, DomElementArg, observable, Observable, styled} from 'grainjs';
import {createHelpTools, cssLeftPanel, cssScrollPane,
cssSectionHeader, cssTools} from 'app/client/ui/LeftPanelCommon';
const t = makeT('HomeLeftPane');
@ -160,65 +158,23 @@ export function createHomeLeftPane(leftPanelOpen: Observable<boolean>, home: Hom
);
}
export async function createDocAndOpen(home: HomeModel) {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
try {
const docId = await home.createDoc("Untitled document", destWS === "unsaved" ? "unsaved" : destWS.id);
// Fetch doc information including urlId.
// TODO: consider changing API to return same response as a GET when creating an
// object, which is a semi-standard.
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
} catch (err) {
reportError(err);
}
}
export async function importDocAndOpen(home: HomeModel) {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
const docId = await docImport(home.app, destWS === "unsaved" ? "unsaved" : destWS.id);
if (docId) {
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
}
}
export async function importFromPluginAndOpen(home: HomeModel, source: ImportSourceElement) {
try {
const destWS = home.newDocWorkspace.get();
if (!destWS) { return; }
const docId = await importFromPlugin(
home.app,
destWS === "unsaved" ? "unsaved" : destWS.id,
source);
if (docId) {
const doc = await home.app.api.getDoc(docId);
await urlState().pushUrl(docUrl(doc));
}
} catch (err) {
reportError(err);
}
}
function addMenu(home: HomeModel, creating: Observable<boolean>): DomElementArg[] {
const org = home.app.currentOrg;
const orgAccess: roles.Role|null = org ? org.access : null;
const needUpgrade = home.app.currentFeatures?.maxWorkspacesPerOrg === 1;
return [
menuItem(() => createDocAndOpen(home), menuIcon('Page'), t("Create Empty Document"),
menuItem(() => newDocMethods.createDocAndOpen(home), menuIcon('Page'), t("Create Empty Document"),
dom.cls('disabled', !home.newDocWorkspace.get()),
testId("dm-new-doc")
),
menuItem(() => importDocAndOpen(home), menuIcon('Import'), t("Import Document"),
menuItem(() => newDocMethods.importDocAndOpen(home), menuIcon('Import'), t("Import Document"),
dom.cls('disabled', !home.newDocWorkspace.get()),
testId("dm-import")
),
domComputed(home.importSources, importSources => ([
...importSources.map((source, i) =>
menuItem(() => importFromPluginAndOpen(home, source),
menuItem(() => newDocMethods.importFromPluginAndOpen(home, source),
menuIcon('Import'),
source.importSource.label,
dom.cls('disabled', !home.newDocWorkspace.get()),

58
app/client/ui/ImportProgress.ts Normal file
View File

@ -0,0 +1,58 @@
import {IProgress} from 'app/client/models/NotifyModel';
import {Disposable} from 'grainjs';
export class ImportProgress extends Disposable {
// Import does upload first, then import. We show a single indicator, estimating which fraction
// of the time should be given to upload (whose progress we can report well), and which to the
// subsequent import (whose progress indicator is mostly faked).
private _uploadFraction: number;
private _estImportSeconds: number;
private _importTimer: null | ReturnType<typeof setInterval> = null;
private _importStart: number = 0;
constructor(private _progressUI: IProgress, file: File) {
super();
// We'll assume that for .grist files, the upload takes 90% of the total time, and for other
// files, 40%.
this._uploadFraction = file.name.endsWith(".grist") ? 0.9 : 0.4;
// TODO: Import step should include a progress callback, to be combined with upload progress.
// Without it, we estimate import to take 2s per MB (non-scientific unreliable estimate), and
// use an asymptotic indicator which keeps moving without ever finishing. Not terribly useful,
// but does slow down for larger files, and is more comforting than a stuck indicator.
this._estImportSeconds = file.size / 1024 / 1024 * 2;
this._progressUI.setProgress(0);
this.onDispose(() => this._importTimer && clearInterval(this._importTimer));
}
// Once this reaches 100, the import stage begins.
public setUploadProgress(percentage: number) {
this._progressUI.setProgress(percentage * this._uploadFraction);
if (percentage >= 100 && !this._importTimer) {
this._importStart = Date.now();
this._importTimer = setInterval(() => this._onImportTimer(), 100);
}
}
public finish() {
if (this._importTimer) {
clearInterval(this._importTimer);
}
this._progressUI.setProgress(100);
}
/**
* Calls _progressUI.setProgress(percent) with percentage increasing from 0 and asymptotically
* approaching 100, reaching 50% after estSeconds. It's intended to look reasonable when the
* estimate is good, and to keep showing slowing progress even if it's not.
*/
private _onImportTimer() {
const elapsedSeconds = (Date.now() - this._importStart) / 1000;
const importProgress = elapsedSeconds / (elapsedSeconds + this._estImportSeconds);
const progress = this._uploadFraction + importProgress * (1 - this._uploadFraction);
this._progressUI.setProgress(100 * progress);
}
}

12
app/client/ui/MarkdownCellRenderer.ts Normal file
View File

@ -0,0 +1,12 @@
import {gristIconLink} from 'app/client/ui2018/links';
import escape from 'lodash/escape';
import {marked} from 'marked';
export const renderer = new marked.Renderer();
renderer.link = ({href, text}) => gristIconLink(href, text).outerHTML;
// Disable Markdown features that we aren't ready to support yet.
renderer.hr = ({raw}) => raw;
renderer.html = ({raw}) => escape(raw);
renderer.image = ({raw}) => raw;

12
app/client/ui/PageWidgetPicker.ts
View File

@ -1,5 +1,6 @@
import {BehavioralPromptsManager} from 'app/client/components/BehavioralPromptsManager';
import {GristDoc} from 'app/client/components/GristDoc';
import {FocusLayer} from 'app/client/lib/FocusLayer';
import {makeT} from 'app/client/lib/localization';
import {reportError} from 'app/client/models/AppModel';
import {ColumnRec, TableRec, ViewSectionRec} from 'app/client/models/DocModel';
@ -260,8 +261,7 @@ export function buildPageWidgetPicker(
dom.create(PageWidgetSelect,
value, tables, columns, onSaveCB, behavioralPromptsManager, options),
// gives focus and binds keydown events
(elem: any) => { setTimeout(() => elem.focus(), 0); },
elem => { FocusLayer.create(ctl, {defaultFocusElem: elem, pauseMousetrap: true}); },
onKeyDown({
Escape: () => ctl.close(),
Enter: () => isValid() && onSaveCB()
@ -328,6 +328,8 @@ export class PageWidgetSelect extends Disposable {
private _isNewTableDisabled = Computed.create(this, this._value.type, (use, type) => !isValidSelection(
'New Table', type, {isNewPage: this._options.isNewPage, summarize: use(this._value.summarize)}));
private _isSummaryDisabled = Computed.create(this, this._value.type, (_use, type) => !isSummaryCompatible(type));
constructor(
private _value: IWidgetValueObs,
private _tables: Observable<TableRec[]>,
@ -389,8 +391,9 @@ export class PageWidgetSelect extends Disposable {
cssEntry.cls('-selected', (use) => use(this._value.summarize) &&
use(this._value.table) === table.id()
),
cssEntry.cls('-disabled', (use) => !isSummaryCompatible(use(this._value.type))),
dom.on('click', (ev, el) => this._selectPivot(table.id(), el as HTMLElement)),
cssEntry.cls('-disabled', this._isSummaryDisabled),
dom.on('click', (_ev, el) =>
!this._isSummaryDisabled.get() && this._selectPivot(table.id(), el as HTMLElement)),
testId('pivot'),
),
testId('table'),
@ -572,7 +575,6 @@ const cssEntry = styled('div', `
&-disabled {
color: ${theme.widgetPickerItemDisabledBg};
cursor: default;
pointer-events: none;
}
&-disabled&-selected {
background-color: inherit;

8
app/client/ui/UserImage.ts
View File

@ -1,5 +1,5 @@
import {colors, theme} from 'app/client/ui2018/cssVars';
import {FullUser} from 'app/common/LoginSessionAPI';
import {UserProfile} from 'app/common/LoginSessionAPI';
import {dom, DomElementArg, styled} from 'grainjs';
import {icon} from 'app/client/ui2018/icons';
@ -9,7 +9,9 @@ export type Size = 'small' | 'medium' | 'large';
* Returns a DOM element showing a circular icon with a user's picture, or the user's initials if
* picture is missing. Also varies the color of the circle when using initials.
*/
export function createUserImage(user: FullUser|'exampleUser'|null, size: Size, ...args: DomElementArg[]): HTMLElement {
export function createUserImage(
user: UserProfile|'exampleUser'|null, size: Size, ...args: DomElementArg[]
): HTMLElement {
let initials: string;
return cssUserImage(
cssUserImage.cls('-' + size),
@ -39,7 +41,7 @@ export function getInitials(user: {name?: string, email?: string}) {
/**
* Hashes the username to return a color.
*/
function pickColor(user: FullUser): string {
function pickColor(user: UserProfile): string {
let c = hashCode(user.name + ':' + user.email) % someColors.length;
if (c < 0) { c += someColors.length; }
return someColors[c];

36
app/client/ui/sanitizeHTML.ts
View File

@ -1,16 +1,30 @@
import DOMPurify from 'dompurify';
import createDOMPurifier from 'dompurify';
const config = {
ADD_TAGS: ['iframe'],
ADD_ATTR: ['allowFullscreen'],
};
export function sanitizeHTML(source: string | Node): string {
return defaultPurifier.sanitize(source);
}
DOMPurify.addHook('uponSanitizeAttribute', (node) => {
export function sanitizeTutorialHTML(source: string | Node): string {
return tutorialPurifier.sanitize(source, {
ADD_TAGS: ['iframe'],
ADD_ATTR: ['allowFullscreen'],
});
}
const defaultPurifier = createDOMPurifier();
defaultPurifier.addHook('uponSanitizeAttribute', handleSanitizeAttribute);
const tutorialPurifier = createDOMPurifier();
tutorialPurifier.addHook('uponSanitizeAttribute', handleSanitizeAttribute);
tutorialPurifier.addHook('uponSanitizeElement', handleSanitizeTutorialElement);
function handleSanitizeAttribute(node: Element) {
if (!('target' in node)) { return; }
node.setAttribute('target', '_blank');
});
DOMPurify.addHook('uponSanitizeElement', (node, data) => {
}
function handleSanitizeTutorialElement(node: Element, data: createDOMPurifier.SanitizeElementHookEvent) {
if (data.tagName !== 'iframe') { return; }
const src = node.getAttribute('src');
@ -18,9 +32,5 @@ DOMPurify.addHook('uponSanitizeElement', (node, data) => {
return;
}
return node.parentNode?.removeChild(node);
});
export function sanitizeHTML(source: string | Node): string {
return DOMPurify.sanitize(source, config);
node.parentNode?.removeChild(node);
}

2
app/client/ui2018/IconList.ts
View File

@ -23,6 +23,7 @@ export type IconName = "ChartArea" |
"FieldFunctionEqual" |
"FieldInteger" |
"FieldLink" |
"FieldMarkdown" |
"FieldNumeric" |
"FieldReference" |
"FieldSpinner" |
@ -185,6 +186,7 @@ export const IconList: IconName[] = ["ChartArea",
"FieldFunctionEqual",
"FieldInteger",
"FieldLink",
"FieldMarkdown",
"FieldNumeric",
"FieldReference",
"FieldSpinner",

7
app/client/ui2018/cssVars.ts
View File

@ -895,6 +895,13 @@ export const theme = {
undefined, colors.slate),
widgetGallerySecondaryHeaderBgHover: new CustomProp(
'theme-widget-gallery-secondary-header-bg-hover', undefined, '#7E7E85'),
/* Markdown Cell */
markdownCellLightBg: new CustomProp('theme-markdown-cell-light-bg', undefined, colors.lightGrey),
markdownCellLightBorder: new CustomProp('theme-markdown-cell-light-border', undefined,
colors.mediumGreyOpaque),
markdownCellMediumBorder: new CustomProp('theme-markdown-cell-medium-border', undefined,
colors.darkGrey),
};
const cssColors = values(colors).map(v => v.decl()).join('\n');

33
app/client/ui2018/icons.ts
View File

@ -56,7 +56,7 @@ import { IconName } from './IconList';
/**
* Defaults for all icons.
*/
const iconDiv = styled('div', `
const iconStyles = `
position: relative;
display: inline-block;
vertical-align: middle;
@ -66,24 +66,35 @@ const iconDiv = styled('div', `
width: 16px;
height: 16px;
background-color: var(--icon-color, var(--grist-theme-text, black));
`);
`;
export const cssIconBackground = styled(iconDiv, `
background-color: var(--icon-background, inherit);
-webkit-mask: none;
& .${iconDiv.className} {
transition: inherit;
display: block;
}
`);
const cssIconDiv = styled('div', iconStyles);
const cssIconSpan = styled('span', iconStyles);
export function icon(name: IconName, ...domArgs: DomElementArg[]): HTMLElement {
return iconDiv(
return cssIconDiv(
dom.style('-webkit-mask-image', `var(--icon-${name})`),
...domArgs
);
}
export function iconSpan(name: IconName, ...domArgs: DomElementArg[]): HTMLElement {
return cssIconSpan(
dom.style('-webkit-mask-image', `var(--icon-${name})`),
...domArgs
);
}
export const cssIconSpanBackground = styled(cssIconSpan, `
background-color: var(--icon-background, inherit);
-webkit-mask: none;
& .${cssIconSpan.className} {
transition: inherit;
display: block;
}
`);
/**
* Container box for an icon to serve as a button..
*/

35
app/client/ui2018/links.ts
View File

@ -1,9 +1,9 @@
import {findLinks} from 'app/client/lib/textUtils';
import { sameDocumentUrlState, urlState } from 'app/client/models/gristUrlState';
import { hideInPrintView, testId, theme } from 'app/client/ui2018/cssVars';
import {cssIconBackground, icon} from 'app/client/ui2018/icons';
import { CellValue } from 'app/plugin/GristData';
import { dom, DomArg, IDomArgs, Observable, styled } from 'grainjs';
import {sameDocumentUrlState, urlState} from 'app/client/models/gristUrlState';
import {hideInPrintView, testId, theme} from 'app/client/ui2018/cssVars';
import {cssIconSpanBackground, iconSpan} from 'app/client/ui2018/icons';
import {CellValue} from 'app/plugin/GristData';
import {dom, DomArg, IDomArgs, Observable, styled} from 'grainjs';
/**
* Styling for a simple <A HREF> link.
@ -37,6 +37,19 @@ export function gristLink(href: string|Observable<string>, ...args: IDomArgs<HTM
);
}
export function gristIconLink(href: string, label = href) {
return cssMaybeWrap(
gristLink(href,
cssIconSpanBackground(
iconSpan("FieldLink", testId('tb-link-icon')),
dom.cls(cssHoverInText.className),
),
),
linkColor(label),
testId("text-link"),
);
}
/**
* If possible (i.e. if `url` points to somewhere in the current document)
* use pushUrl to navigate without reloading or opening a new tab
@ -60,17 +73,7 @@ export function makeLinks(text: string) {
const domElements: DomArg[] = [];
for (const {value, isLink} of findLinks(text)) {
if (isLink) {
// Wrap link with a span to provide hover on and to override wrapping.
domElements.push(cssMaybeWrap(
gristLink(value,
cssIconBackground(
icon("FieldLink", testId('tb-link-icon')),
dom.cls(cssHoverInText.className),
),
),
linkColor(value),
testId("text-link")
));
domElements.push(gristIconLink(value));
} else {
domElements.push(value);
}

172
app/client/widgets/DateEditor.js
View File

@ -1,172 +0,0 @@
/* global $, document */
const moment = require('moment-timezone');
const _ = require('underscore');
const gutil = require('app/common/gutil');
const commands = require('../components/commands');
const dispose = require('../lib/dispose');
const dom = require('../lib/dom');
const kd = require('../lib/koDom');
const TextEditor = require('./TextEditor');
const { parseDate, TWO_DIGIT_YEAR_THRESHOLD } = require('app/common/parseDate');
// DatePicker unfortunately requires an <input> (not <textarea>). But textarea is better for us,
// because sometimes it's taller than a line, and an <input> looks worse. The following
// unconsionable hack tricks Datepicker into thinking anything it's attached to is an input.
// It's more reasonable to just modify boostrap-datepicker, but that has its own downside (with
// upgrading and minification). This hack, however, is simpler than other workarounds.
var Datepicker = $.fn.datepicker.Constructor;
// datepicker.isInput can now be set to anything, but when read, always returns true. Tricksy.
Object.defineProperty(Datepicker.prototype, 'isInput', {
get: function() { return true; },
set: function(v) {},
});
/**
* DateEditor - Editor for Date type. Includes a dropdown datepicker.
* See reference: http://bootstrap-datepicker.readthedocs.org/en/latest/index.html
*
* @param {String} options.timezone: Optional timezone to use instead of UTC.
*/
function DateEditor(options) {
// A string that is always `UTC` in the DateEditor, eases DateTimeEditor inheritance.
this.timezone = options.timezone || 'UTC';
this.dateFormat = options.field.widgetOptionsJson.peek().dateFormat;
this.locale = options.field.documentSettings.peek().locale;
// Update moment format string to represent a date unambiguously.
this.safeFormat = makeFullMomentFormat(this.dateFormat);
// Use the default local timezone to format the placeholder date.
const defaultTimezone = moment.tz.guess();
let placeholder = moment.tz(defaultTimezone).format(this.safeFormat);
if (options.readonly) {
// clear placeholder for readonly mode
placeholder = null;
}
TextEditor.call(this, _.defaults(options, { placeholder: placeholder }));
const cellValue = this.formatValue(options.cellValue, this.safeFormat, true);
// Set the edited value, if not explicitly given, to the formatted version of cellValue.
this.textInput.value = gutil.undef(options.state, options.editValue, cellValue);
if (!options.readonly) {
// Indicates whether keyboard navigation is active for the datepicker.
this._keyboardNav = false;
// Attach the datepicker.
this._datePickerWidget = $(this.textInput).datepicker({
keyboardNavigation: false,
forceParse: false,
todayHighlight: true,
todayBtn: 'linked',
assumeNearbyYear: TWO_DIGIT_YEAR_THRESHOLD,
// Datepicker supports most of the languages. They just need to be included in the bundle
// or by script tag, i.e.
// <script src="bootstrap-datepicker/dist/locales/bootstrap-datepicker.pl.min.js"></script>
language : this.getLanguage(),
// Use the stripped format converted to one suitable for the datepicker.
format: {
toDisplay: (date, format, language) => moment.utc(date).format(this.safeFormat),
toValue: (date, format, language) => {
const timestampSec = parseDate(date, {
dateFormat: this.safeFormat,
// datepicker reads date in utc (ie: using date.getUTCDate()).
timezone: 'UTC',
});
return (timestampSec === null) ? null : new Date(timestampSec * 1000);
},
},
});
this.autoDisposeCallback(() => this._datePickerWidget.datepicker('destroy'));
// NOTE: Datepicker interferes with normal enter and escape functionality. Add an event handler
// to the DatePicker to prevent interference with normal behavior.
this._datePickerWidget.on('keydown', e => {
// If enter or escape is pressed, destroy the datepicker and re-dispatch the event.
if (e.keyCode === 13 || e.keyCode === 27) {
this._datePickerWidget.datepicker('destroy');
// The current target of the event will be the textarea.
setTimeout(() => e.currentTarget.dispatchEvent(e.originalEvent), 0);
}
});
// When the up/down arrow is pressed, modify the datepicker options to take control of
// the arrow keys for date selection.
let datepickerCommands = Object.assign({}, options.commands, {
datepickerFocus: () => { this._allowKeyboardNav(true); }
});
this._datepickerCommands = this.autoDispose(commands.createGroup(datepickerCommands, this, true));
this._datePickerWidget.on('show', () => {
// A workaround to allow clicking in the datepicker without losing focus.
dom(document.querySelector('.datepicker'),
kd.attr('tabIndex', 0), // allows datepicker to gain focus
kd.toggleClass('clipboard_focus', true) // tells clipboard to not steal focus from us
);
// Attach command group to the input to allow switching keyboard focus to the datepicker.
dom(this.textInput,
// If the user inputs text into the textbox, take keyboard focus from the datepicker.
dom.on('input', () => { this._allowKeyboardNav(false); }),
this._datepickerCommands.attach()
);
});
}
}
dispose.makeDisposable(DateEditor);
_.extend(DateEditor.prototype, TextEditor.prototype);
/** @inheritdoc */
DateEditor.prototype.getCellValue = function() {
let timestamp = parseDate(this.textInput.value, {
dateFormat: this.safeFormat,
timezone: this.timezone
});
return timestamp !== null ? timestamp : this.textInput.value;
};
// Helper to allow/disallow keyboard navigation within the datepicker.
DateEditor.prototype._allowKeyboardNav = function(bool) {
if (this._keyboardNav !== bool) {
this._keyboardNav = bool;
$(this.textInput).data().datepicker.o.keyboardNavigation = bool;
// Force parse must be turned on with keyboard navigation, since it forces the highlighted date
// to be used when enter is pressed. Otherwise, keyboard date selection will have no effect.
$(this.textInput).data().datepicker.o.forceParse = bool;
}
};
// Moment value formatting helper.
DateEditor.prototype.formatValue = function(value, formatString, shouldFallBackToValue) {
if (_.isNumber(value) && formatString) {
return moment.tz(value*1000, this.timezone).format(formatString);
} else {
// If value is AltText, return it unchanged. This way we can see it and edit in the editor.
return (shouldFallBackToValue && typeof value === 'string') ? value : "";
}
};
// Gets the language based on the current locale.
DateEditor.prototype.getLanguage = function() {
// this requires a polyfill, i.e. https://www.npmjs.com/package/@formatjs/intl-locale
// more info about ts: https://github.com/microsoft/TypeScript/issues/37326
// return new Intl.Locale(locale).language;
return this.locale.substr(0, this.locale.indexOf("-"));
}
// Updates the given Moment format to specify a complete date, so that the datepicker sees an
// unambiguous date in the textbox input. If the format is incomplete, fall back to YYYY-MM-DD.
function makeFullMomentFormat(mFormat) {
let safeFormat = mFormat;
if (!safeFormat.includes('Y')) {
safeFormat += " YYYY";
}
if (!safeFormat.includes('D') || !safeFormat.includes('M')) {
safeFormat = 'YYYY-MM-DD';
}
return safeFormat;
}
module.exports = DateEditor;

216
app/client/widgets/DateEditor.ts Normal file
View File

@ -0,0 +1,216 @@
import {CommandGroup, createGroup} from 'app/client/components/commands';
import {loadScript} from 'app/client/lib/loadScript';
import {detectCurrentLang} from 'app/client/lib/localization';
import {FieldOptions} from 'app/client/widgets/NewBaseEditor';
import {NTextEditor} from 'app/client/widgets/NTextEditor';
import {CellValue} from "app/common/DocActions";
import {parseDate, TWO_DIGIT_YEAR_THRESHOLD} from 'app/common/parseDate';
import moment from 'moment-timezone';
import {dom} from 'grainjs';
// These are all the locales available for the datepicker. Having a prepared list lets us find a
// suitable one without trying combinations that don't exist. This list can be rebuilt using:
// ls bower_components/bootstrap-datepicker/dist/locales/bootstrap-datepicker.* | cut -d. -f2 | xargs echo
// eslint-disable-next-line max-len
const availableLocales = 'ar-tn ar az bg bm bn br bs ca cs cy da de el en-AU en-CA en-GB en-IE en-NZ en-ZA eo es et eu fa fi fo fr-CH fr gl he hi hr hu hy id is it-CH it ja ka kh kk km ko kr lt lv me mk mn ms nl-BE nl no oc pl pt-BR pt ro rs-latin rs ru si sk sl sq sr-latin sr sv sw ta tg th tk tr uk uz-cyrl uz-latn vi zh-CN zh-TW';
monkeyPatchDatepicker();
/**
* DateEditor - Editor for Date type. Includes a dropdown datepicker.
* See reference: http://bootstrap-datepicker.readthedocs.org/en/latest/index.html
*/
export class DateEditor extends NTextEditor {
protected safeFormat: string; // Format that specifies a complete date.
private _dateFormat: string|undefined = this.options.field.widgetOptionsJson.peek().dateFormat;
private _locale = detectCurrentLang();
private _keyboardNav = false; // Whether keyboard navigation is active for the datepicker.
constructor(
options: FieldOptions,
protected timezone: string = 'UTC', // For use by the derived DateTimeEditor.
) {
super(options);
// Update moment format string to represent a date unambiguously.
this.safeFormat = makeFullMomentFormat(this._dateFormat || '');
// Set placeholder to current date(time), unless in read-only mode.
if (!options.readonly) {
// Use the default local timezone to format the placeholder date.
// TODO: this.timezone is better for DateTime; gristDoc.docInfo.timezone.peek() is better for Date.
const defaultTimezone = moment.tz.guess();
const placeholder = moment.tz(defaultTimezone).format(this.safeFormat);
this.textInput.setAttribute('placeholder', placeholder);
}
const cellValue = this.formatValue(options.cellValue, this.safeFormat, true);
// Set the edited value, if not explicitly given, to the formatted version of cellValue.
this.textInput.value = options.state ?? options.editValue ?? cellValue;
if (!options.readonly) {
// When the up/down arrow is pressed, modify the datepicker options to take control of
// the arrow keys for date selection.
const datepickerCommands = {
...options.commands,
datepickerFocus: () => { this._allowKeyboardNav(true); }
};
const datepickerCommandGroup = this.autoDispose(createGroup(datepickerCommands, this, true));
this._attachDatePicker(datepickerCommandGroup)
.catch(e => console.error("Error attaching datepicker", e));
}
}
public getCellValue() {
const timestamp = parseDate(this.textInput.value, {
dateFormat: this.safeFormat,
timezone: this.timezone
});
return timestamp !== null ? timestamp : this.textInput.value;
}
// Moment value formatting helper.
protected formatValue(value: CellValue, formatString: string|undefined, shouldFallBackToValue: boolean) {
if (typeof value === 'number' && formatString) {
return moment.tz(value*1000, this.timezone).format(formatString);
} else {
// If value is AltText, return it unchanged. This way we can see it and edit in the editor.
return (shouldFallBackToValue && typeof value === 'string') ? value : "";
}
}
// Helper to allow/disallow keyboard navigation within the datepicker.
private _allowKeyboardNav(bool: boolean) {
if (this._keyboardNav !== bool) {
this._keyboardNav = bool;
$(this.textInput).data().datepicker.o.keyboardNavigation = bool;
// Force parse must be turned on with keyboard navigation, since it forces the highlighted date
// to be used when enter is pressed. Otherwise, keyboard date selection will have no effect.
$(this.textInput).data().datepicker.o.forceParse = bool;
}
}
// Attach the datepicker.
private async _attachDatePicker(datepickerCommands: CommandGroup) {
const localeToUse = await loadLocale(this._locale);
if (this.isDisposed()) { return; } // Good idea to check after 'await'.
const datePickerWidget = $(this.textInput).datepicker({
keyboardNavigation: false,
forceParse: false,
todayHighlight: true,
todayBtn: 'linked',
assumeNearbyYear: TWO_DIGIT_YEAR_THRESHOLD,
language: localeToUse,
// Use the stripped format converted to one suitable for the datepicker.
format: {
toDisplay: (date: string, format: unknown, lang: unknown) => moment.utc(date).format(this.safeFormat),
toValue: (date: string, format: unknown, lang: unknown) => {
const timestampSec = parseDate(date, {
dateFormat: this.safeFormat,
// datepicker reads date in utc (ie: using date.getUTCDate()).
timezone: 'UTC',
});
return (timestampSec === null) ? null : new Date(timestampSec * 1000);
},
},
});
this.onDispose(() => datePickerWidget.datepicker('destroy'));
// NOTE: Datepicker interferes with normal enter and escape functionality. Add an event handler
// to the DatePicker to prevent interference with normal behavior.
datePickerWidget.on('keydown', (e) => {
// If enter or escape is pressed, destroy the datepicker and re-dispatch the event.
if (e.keyCode === 13 || e.keyCode === 27) {
datePickerWidget.datepicker('destroy');
// The current target of the event will be the textarea.
setTimeout(() => e.currentTarget?.dispatchEvent(e.originalEvent!), 0);
}
});
datePickerWidget.on('show', () => {
// A workaround to allow clicking in the datepicker without losing focus.
const datepickerElem: HTMLElement|null = document.querySelector('.datepicker');
if (datepickerElem) {
dom.update(datepickerElem,
dom.attr('tabIndex', '0'), // allows datepicker to gain focus
dom.cls('clipboard_focus') // tells clipboard to not steal focus from us
);
}
// Attach command group to the input to allow switching keyboard focus to the datepicker.
dom.update(this.textInput,
// If the user inputs text into the textbox, take keyboard focus from the datepicker.
dom.on('input', () => { this._allowKeyboardNav(false); }),
datepickerCommands.attach()
);
});
datePickerWidget.datepicker('show');
}
}
// Updates the given Moment format to specify a complete date, so that the datepicker sees an
// unambiguous date in the textbox input. If the format is incomplete, fall back to YYYY-MM-DD.
function makeFullMomentFormat(mFormat: string): string {
let safeFormat = mFormat;
if (!safeFormat.includes('Y')) {
safeFormat += " YYYY";
}
if (!safeFormat.includes('D') || !safeFormat.includes('M')) {
safeFormat = 'YYYY-MM-DD';
}
return safeFormat;
}
let availableLocaleSet: Set<string>|undefined;
const loadedLocaleMap = new Map<string, string>(); // Maps requested locale to the one to use.
// Datepicker supports many languages. They just need to be loaded. Here we load the language we
// need on-demand, taking care not to load any language more than once (we don't need to assume
// there is only one language being used on the page, though in practice that may well be true).
async function loadLocale(locale: string): Promise<string> {
return loadedLocaleMap.get(locale) ||
loadedLocaleMap.set(locale, await doLoadLocale(locale)).get(locale)!;
}
async function doLoadLocale(locale: string): Promise<string> {
if (!availableLocaleSet) {
availableLocaleSet = new Set(availableLocales.split(/\s+/));
}
if (!availableLocaleSet.has(locale)) {
const shortLocale = locale.split("-")[0]; // If "xx-YY" is not available, try "xx"
if (!availableLocaleSet.has(shortLocale)) {
// No special locale available. (This is even true for "en", which is fine since that's
// loaded by default.)
return locale;
}
locale = shortLocale;
}
console.debug(`DateEditor: loading locale ${locale}`);
try {
await loadScript(`bootstrap-datepicker/dist/locales/bootstrap-datepicker.${locale}.min.js`);
} catch (e) {
console.warn(`DateEditor: failed to load ${locale}`);
}
return locale;
}
// DatePicker unfortunately requires an <input> (not <textarea>). But textarea is better for us,
// because sometimes it's taller than a line, and an <input> looks worse. The following
// unconsionable hack tricks Datepicker into thinking anything it's attached to is an input.
// It's more reasonable to just modify boostrap-datepicker, but that has its own downside (with
// upgrading and minification). This hack, however, is simpler than other workarounds.
function monkeyPatchDatepicker() {
const Datepicker = ($.fn as any).datepicker?.Constructor;
if (Datepicker?.prototype) {
// datepicker.isInput can now be set to anything, but when read, always returns true. Tricksy.
Object.defineProperty(Datepicker.prototype, 'isInput', {
get: function() { return true; },
set: function(v) {},
});
}
}

173
app/client/widgets/DateTimeEditor.js
View File

@ -1,173 +0,0 @@
/* global document */
const moment = require('moment-timezone');
const _ = require('underscore');
const dom = require('../lib/dom');
const dispose = require('../lib/dispose');
const kd = require('../lib/koDom');
const DateEditor = require('./DateEditor');
const gutil = require('app/common/gutil');
const { parseDate } = require('app/common/parseDate');
const TextEditor = require('./TextEditor');
/**
* DateTimeEditor - Editor for DateTime type. Includes a dropdown datepicker.
* See reference: http://bootstrap-datepicker.readthedocs.org/en/latest/index.html
*/
function DateTimeEditor(options) {
// Get the timezone from the end of the type string.
options.timezone = gutil.removePrefix(options.field.column().type(), "DateTime:");
// Adjust the command group.
var origCommands = options.commands;
// don't modify navigation for readonly mode
if (!options.readonly) {
options.commands = Object.assign({}, origCommands, {
prevField: () => this._focusIndex() === 1 ? this._setFocus(0) : origCommands.prevField(),
nextField: () => this._focusIndex() === 0 ? this._setFocus(1) : origCommands.nextField(),
});
}
// Call the superclass.
DateEditor.call(this, options);
this._timeFormat = options.field.widgetOptionsJson.peek().timeFormat;
// To reuse code, this knows all about the DOM that DateEditor builds (using TextEditor), and
// modifies that to be two side-by-side textareas.
this._dateSizer = this.contentSizer; // For consistency with _timeSizer and _timeInput.
this._dateInput = this.textInput;
const isValid = _.isNumber(options.cellValue);
const formatted = this.formatValue(options.cellValue, this._timeFormat, false);
// Use a placeholder of 12:00am, since that is the autofill time value.
const placeholder = moment.tz('0', 'H', this.timezone).format(this._timeFormat);
// for readonly
if (options.readonly) {
if (!isValid) {
// do nothing - DateEditor will show correct error
} else {
// append time format or a placeholder
const time = (formatted || placeholder);
const sep = time ? ' ' : '';
this.textInput.value = this.textInput.value + sep + time;
}
} else {
dom(this.dom, kd.toggleClass('celleditor_datetime', true));
dom(this.dom.firstChild, kd.toggleClass('celleditor_datetime_editor', true));
this.dom.appendChild(
dom('div.celleditor_cursor_editor.celleditor_datetime_editor',
this._timeSizer = dom('div.celleditor_content_measure'),
this._timeInput = dom('textarea.celleditor_text_editor',
kd.attr('placeholder', placeholder),
kd.value(formatted),
this.commandGroup.attach(),
dom.on('input', () => this.onChange())
)
)
);
}
// If the edit value is encoded json, use those values as a starting point
if (typeof options.state == 'string') {
try {
const { date, time } = JSON.parse(options.state);
this._dateInput.value = date;
this._timeInput.value = time;
this.onChange();
} catch(e) {
console.error("DateTimeEditor can't restore its previous state")
}
}
}
dispose.makeDisposable(DateTimeEditor);
_.extend(DateTimeEditor.prototype, DateEditor.prototype);
DateTimeEditor.prototype.setSizerLimits = function() {
var maxSize = this.editorPlacement.calcSize({width: Infinity, height: Infinity}, {calcOnly: true});
if (this.options.readonly) {
return;
}
this._dateSizer.style.maxWidth =
this._timeSizer.style.maxWidth = Math.ceil(maxSize.width / 2 - 6) + 'px';
};
/**
* Returns which element has focus: 0 if date, 1 if time, null if neither.
*/
DateTimeEditor.prototype._focusIndex = function() {
return document.activeElement === this._dateInput ? 0 :
(document.activeElement === this._timeInput ? 1 : null);
};
/**
* Sets focus to date if index is 0, or time if index is 1.
*/
DateTimeEditor.prototype._setFocus = function(index) {
var elem = (index === 0 ? this._dateInput : (index === 1 ? this._timeInput : null));
if (elem) {
elem.focus();
elem.selectionStart = 0;
elem.selectionEnd = elem.value.length;
}
};
/**
* Occurs when user types something into the editor
*/
DateTimeEditor.prototype.onChange = function() {
this._resizeInput();
// store editor state as an encoded JSON string
const date = this._dateInput.value;
const time = this._timeInput.value;
this.editorState.set(JSON.stringify({ date, time}));
}
DateTimeEditor.prototype.getCellValue = function() {
let date = this._dateInput.value;
let time = this._timeInput.value;
let timestamp = parseDate(date, {
dateFormat: this.safeFormat,
time: time,
timeFormat: this._timeFormat,
timezone: this.timezone
});
return timestamp !== null ? timestamp :
(date && time ? `${date} ${time}` : date || time);
};
/**
* Overrides the resizing function in TextEditor.
*/
DateTimeEditor.prototype._resizeInput = function() {
// for readonly field, we will use logic from a super class
if (this.options.readonly) {
TextEditor.prototype._resizeInput.call(this);
return;
}
// Use the size calculation provided in options.calcSize (that takes into account cell size and
// screen size), with both date and time parts as the input. The resulting size is applied to
// the parent (containing date + time), with date and time each expanding or shrinking from the
// measured sizes using flexbox logic.
this._dateSizer.textContent = this._dateInput.value;
this._timeSizer.textContent = this._timeInput.value;
var dateRect = this._dateSizer.getBoundingClientRect();
var timeRect = this._timeSizer.getBoundingClientRect();
// Textboxes get 3px of padding on left/right/top (see TextEditor.css); we specify it manually
// since editorPlacement can't do a good job figuring it out with the flexbox arrangement.
var size = this.editorPlacement.calcSize({
width: dateRect.width + timeRect.width + 12,
height: Math.max(dateRect.height, timeRect.height) + 3
});
this.dom.style.width = size.width + 'px';
this._dateInput.parentNode.style.flexBasis = (dateRect.width + 6) + 'px';
this._timeInput.parentNode.style.flexBasis = (timeRect.width + 6) + 'px';
this._dateInput.style.height = Math.ceil(size.height - 3) + 'px';
this._timeInput.style.height = Math.ceil(size.height - 3) + 'px';
};
module.exports = DateTimeEditor;

173
app/client/widgets/DateTimeEditor.ts Normal file
View File

@ -0,0 +1,173 @@
import {DateEditor} from 'app/client/widgets/DateEditor';
import {FieldOptions} from 'app/client/widgets/NewBaseEditor';
import {removePrefix} from 'app/common/gutil';
import {parseDate} from 'app/common/parseDate';
import moment from 'moment-timezone';
import {dom} from 'grainjs';
/**
* DateTimeEditor - Editor for DateTime type. Includes a dropdown datepicker.
* See reference: http://bootstrap-datepicker.readthedocs.org/en/latest/index.html
*/
export class DateTimeEditor extends DateEditor {
private _timeFormat: string|undefined;
private _dateSizer: HTMLElement;
private _timeSizer: HTMLElement;
private _dateInput: HTMLTextAreaElement;
private _timeInput: HTMLTextAreaElement;
constructor(options: FieldOptions) {
// Get the timezone from the end of the type string.
const timezone = removePrefix(options.field.column().type(), "DateTime:");
// Adjust the command group, but not for readonly mode.
if (!options.readonly) {
const origCommands = options.commands;
options.commands = {
...origCommands,
prevField: () => this._focusIndex() === 1 ? this._setFocus(0) : origCommands.prevField(),
nextField: () => this._focusIndex() === 0 ? this._setFocus(1) : origCommands.nextField(),
};
}
// Call the superclass.
super(options, timezone || 'UTC');
this._timeFormat = this.options.field.widgetOptionsJson.peek().timeFormat;
// To reuse code, this knows all about the DOM that DateEditor builds (using TextEditor), and
// modifies that to be two side-by-side textareas.
this._dateSizer = this.contentSizer; // For consistency with _timeSizer.
this._dateInput = this.textInput; // For consistency with _timeInput.
const isValid = (typeof options.cellValue === 'number');
const formatted = this.formatValue(options.cellValue, this._timeFormat, false);
// Use a placeholder of 12:00am, since that is the autofill time value.
const placeholder = moment.tz('0', 'H', this.timezone).format(this._timeFormat);
// for readonly
if (options.readonly) {
if (!isValid) {
// do nothing - DateEditor will show correct error
} else {
// append time format or a placeholder
const time = (formatted || placeholder);
const sep = time ? ' ' : '';
this.textInput.value = this.textInput.value + sep + time;
}
} else {
const widgetElem = this.getDom();
dom.update(widgetElem, dom.cls('celleditor_datetime'));
dom.update(this.cellEditorDiv, dom.cls('celleditor_datetime_editor'));
widgetElem.appendChild(
dom('div',
dom.cls('celleditor_cursor_editor'),
dom.cls('celleditor_datetime_editor'),
this._timeSizer = dom('div', dom.cls('celleditor_content_measure')),
this._timeInput = dom('textarea', dom.cls('celleditor_text_editor'),
dom.attr('placeholder', placeholder),
dom.prop('value', formatted),
this.commandGroup.attach(),
dom.on('input', () => this._onChange())
)
)
);
}
// If the edit value is encoded json, use those values as a starting point
if (typeof options.state == 'string') {
try {
const { date, time } = JSON.parse(options.state);
this._dateInput.value = date;
this._timeInput.value = time;
this._onChange();
} catch(e) {
console.error("DateTimeEditor can't restore its previous state");
}
}
}
public getCellValue() {
const date = this._dateInput.value;
const time = this._timeInput.value;
const timestamp = parseDate(date, {
dateFormat: this.safeFormat,
time: time,
timeFormat: this._timeFormat,
timezone: this.timezone
});
return timestamp !== null ? timestamp :
(date && time ? `${date} ${time}` : date || time);
}
public setSizerLimits() {
const maxSize = this.editorPlacement.calcSize({width: Infinity, height: Infinity}, {calcOnly: true});
if (this.options.readonly) {
return;
}
this._dateSizer.style.maxWidth =
this._timeSizer.style.maxWidth = Math.ceil(maxSize.width / 2 - 6) + 'px';
}
/**
* Overrides the resizing function in TextEditor.
*/
protected resizeInput() {
// for readonly field, we will use logic from a super class
if (this.options.readonly) {
return super.resizeInput();
}
// Use the size calculation provided in options.calcSize (that takes into account cell size and
// screen size), with both date and time parts as the input. The resulting size is applied to
// the parent (containing date + time), with date and time each expanding or shrinking from the
// measured sizes using flexbox logic.
this._dateSizer.textContent = this._dateInput.value;
this._timeSizer.textContent = this._timeInput.value;
const dateRect = this._dateSizer.getBoundingClientRect();
const timeRect = this._timeSizer.getBoundingClientRect();
// Textboxes get 3px of padding on left/right/top (see TextEditor.css); we specify it manually
// since editorPlacement can't do a good job figuring it out with the flexbox arrangement.
const size = this.editorPlacement.calcSize({
width: dateRect.width + timeRect.width + 12,
height: Math.max(dateRect.height, timeRect.height) + 3
});
this.getDom().style.width = size.width + 'px';
this._dateInput.parentElement!.style.flexBasis = (dateRect.width + 6) + 'px';
this._timeInput.parentElement!.style.flexBasis = (timeRect.width + 6) + 'px';
this._dateInput.style.height = Math.ceil(size.height - 3) + 'px';
this._timeInput.style.height = Math.ceil(size.height - 3) + 'px';
}
/**
* Returns which element has focus: 0 if date, 1 if time, null if neither.
*/
private _focusIndex() {
return document.activeElement === this._dateInput ? 0 :
(document.activeElement === this._timeInput ? 1 : null);
}
/**
* Sets focus to date if index is 0, or time if index is 1.
*/
private _setFocus(index: 0|1) {
const elem = (index === 0 ? this._dateInput : (index === 1 ? this._timeInput : null));
if (elem) {
elem.focus();
elem.selectionStart = 0;
elem.selectionEnd = elem.value.length;
}
}
/**
* Occurs when user types something into the editor
*/
private _onChange() {
this.resizeInput();
// store editor state as an encoded JSON string
const date = this._dateInput.value;
const time = this._timeInput.value;
this.editorState.set(JSON.stringify({ date, time}));
}
}

4
app/client/widgets/FieldBuilder.ts
View File

@ -29,7 +29,7 @@ import { FieldEditor, saveWithoutEditor } from 'app/client/widgets/FieldEditor';
import { CellDiscussionPopup, EmptyCell } from 'app/client/widgets/DiscussionEditor';
import { openFormulaEditor } from 'app/client/widgets/FormulaEditor';
import { NewAbstractWidget } from 'app/client/widgets/NewAbstractWidget';
import { NewBaseEditor } from "app/client/widgets/NewBaseEditor";
import { IEditorConstructor } from "app/client/widgets/NewBaseEditor";
import * as UserType from 'app/client/widgets/UserType';
import * as UserTypeImpl from 'app/client/widgets/UserTypeImpl';
import * as gristTypes from 'app/common/gristTypes';
@ -758,7 +758,7 @@ export class FieldBuilder extends Disposable {
return clearOwn();
}
const editorCtor: typeof NewBaseEditor =
const editorCtor: IEditorConstructor =
UserTypeImpl.getEditorConstructor(this.options().widget, this._readOnlyPureType());
// constructor may be null for a read-only non-formula field, though not today.
if (!editorCtor) {

4
app/client/widgets/FieldEditor.ts
View File

@ -8,7 +8,7 @@ import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec';
import {reportError} from 'app/client/models/errors';
import {showTooltipToCreateFormula} from 'app/client/widgets/EditorTooltip';
import {FormulaEditor, getFormulaError} from 'app/client/widgets/FormulaEditor';
import {IEditorCommandGroup, NewBaseEditor} from 'app/client/widgets/NewBaseEditor';
import {IEditorCommandGroup, IEditorConstructor, NewBaseEditor} from 'app/client/widgets/NewBaseEditor';
import {asyncOnce} from "app/common/AsyncCreate";
import {CellValue} from "app/common/DocActions";
import * as gutil from 'app/common/gutil';
@ -18,8 +18,6 @@ import {CursorPos} from 'app/plugin/GristAPI';
import isEqual = require('lodash/isEqual');
import {Disposable, dom, Emitter, Holder, MultiHolder, Observable} from 'grainjs';
type IEditorConstructor = typeof NewBaseEditor;
const t = makeT('FieldEditor');
/**

33
app/client/widgets/FormulaAssistant.ts
View File

@ -31,10 +31,11 @@ import {Computed, Disposable, dom, DomElementArg, makeTestId, MutableObsArray,
obsArray, Observable, styled, subscribeElem} from 'grainjs';
import debounce from 'lodash/debounce';
import noop from 'lodash/noop';
import {marked} from 'marked';
import {Marked} from 'marked';
import {markedHighlight} from 'marked-highlight';
import {v4 as uuidv4} from 'uuid';
const t = makeT('FormulaEditor');
const t = makeT('FormulaAssistant');
const testId = makeTestId('test-formula-editor-');
const LOW_CREDITS_WARNING_BANNER_THRESHOLD = 10;
@ -802,6 +803,7 @@ class ChatHistory extends Disposable {
public lastSuggestedFormula: Computed<string|null>;
private _element: HTMLElement;
private _marked: Marked;
constructor(private _options: {
column: ColumnRec,
@ -845,6 +847,17 @@ class ChatHistory extends Disposable {
this.lastSuggestedFormula = Computed.create(this, use => {
return [...use(this.conversationHistory)].reverse().find(({formula}) => formula)?.formula ?? null;
});
const highlightCodePromise = buildCodeHighlighter({maxLines: 60});
this._marked = new Marked(
markedHighlight({
async: true,
highlight: async (code) => {
const highlightCode = await highlightCodePromise;
return highlightCode(code);
},
})
);
}
public thinking(on = true) {
@ -864,10 +877,6 @@ class ChatHistory extends Disposable {
}
}
public supportsMarkdown() {
return this._options.column.chatHistory.peek().get().state !== undefined;
}
public addResponse(message: ChatMessage) {
// Clear any thinking from messages.
this.thinking(false);
@ -958,6 +967,10 @@ class ChatHistory extends Disposable {
);
}
private _supportsMarkdown() {
return this._options.column.chatHistory.peek().get().state !== undefined;
}
private _buildIntroMessage() {
return cssAiIntroMessage(
cssAvatar(cssAiImage()),
@ -1010,14 +1023,10 @@ class ChatHistory extends Disposable {
* Renders the message as markdown if possible, otherwise as a code block.
*/
private _render(message: string, ...args: DomElementArg[]) {
if (this.supportsMarkdown()) {
if (this._supportsMarkdown()) {
return dom('div',
(el) => subscribeElem(el, gristThemeObs(), async () => {
const highlightCode = await buildCodeHighlighter({maxLines: 60});
const content = sanitizeHTML(marked(message, {
highlight: (code) => highlightCode(code)
}));
el.innerHTML = content;
el.innerHTML = sanitizeHTML(await this._marked.parse(message));
}),
...args
);

16
app/client/widgets/FormulaEditor.ts
View File

@ -118,11 +118,12 @@ export class FormulaEditor extends NewBaseEditor {
const hideErrDetails = Observable.create(this, true);
const raisedException = Computed.create(this, use => {
if (!options.formulaError || !use(options.formulaError)) {
const formulaError = options.formulaError && use(options.formulaError);
if (!formulaError) {
return null;
}
const error = isRaisedException(use(options.formulaError)!) ?
decodeObject(use(options.formulaError)!) as RaisedException:
const error = isRaisedException(formulaError) ?
decodeObject(formulaError) as RaisedException:
new RaisedException(["Unknown error"]);
return error;
});
@ -382,7 +383,7 @@ export class FormulaEditor extends NewBaseEditor {
// If we have an error to show, ask for a larger size for formulaEditor.
const desiredSize = {
width: Math.max(desiredElemSize.width, (this.options.formulaError.get() ? minFormulaErrorWidth : 0)),
width: Math.max(desiredElemSize.width, (this.options.formulaError?.get() ? minFormulaErrorWidth : 0)),
// Ask for extra space for the error; we'll decide how to allocate it below.
height: desiredElemSize.height + (errorBoxDesiredHeight - errorBoxStartHeight),
};
@ -488,7 +489,7 @@ export function openFormulaEditor(options: {
column?: ColumnRec,
// Associated formula from a view field. If provided together with column, this field is used
field?: ViewFieldRec,
editingFormula?: ko.Computed<boolean>,
editingFormula: ko.Computed<boolean>,
// Needed to get exception value, if any.
editRow?: DataRowModel,
// Element over which to position the editor.
@ -555,7 +556,7 @@ export function openFormulaEditor(options: {
column,
field: options.field,
}) : undefined;
const editor = FormulaEditor.create(null, {
const editorOptions: IFormulaEditorOptions = {
gristDoc,
column,
field: options.field,
@ -569,7 +570,8 @@ export function openFormulaEditor(options: {
cssClass: 'formula_editor_sidepane',
readonly : false,
canDetach: options.canDetach
} as IFormulaEditorOptions) as FormulaEditor;
};
const editor = FormulaEditor.create(null, editorOptions);
editor.autoDispose(attachedHolder);
editor.attach(refElem);

6
app/client/widgets/HyperLinkTextBox.ts
View File

@ -2,7 +2,7 @@ import { DataRowModel } from 'app/client/models/DataRowModel';
import { ViewFieldRec } from 'app/client/models/entities/ViewFieldRec';
import { constructUrl } from 'app/client/models/gristUrlState';
import { testId, theme } from 'app/client/ui2018/cssVars';
import { cssIconBackground, icon } from 'app/client/ui2018/icons';
import { cssIconSpanBackground, iconSpan } from 'app/client/ui2018/icons';
import { cssHoverIn, gristLink } from 'app/client/ui2018/links';
import { NTextBox } from 'app/client/widgets/NTextBox';
import { CellValue } from 'app/common/DocActions';
@ -27,8 +27,8 @@ export class HyperLinkTextBox extends NTextBox {
dom.cls('text_wrapping', this.wrapping),
dom.maybe((use) => Boolean(use(value)), () =>
gristLink(url,
cssIconBackground(
icon("FieldLink", testId('tb-link-icon')),
cssIconSpanBackground(
iconSpan("FieldLink", testId('tb-link-icon')),
dom.cls(cssHoverOnField.className),
),
testId('tb-link'),

176
app/client/widgets/MarkdownTextBox.ts Normal file
View File

@ -0,0 +1,176 @@
import { DataRowModel } from 'app/client/models/DataRowModel';
import { ViewFieldRec } from 'app/client/models/entities/ViewFieldRec';
import { buildCodeHighlighter } from 'app/client/ui/CodeHighlight';
import { renderer } from 'app/client/ui/MarkdownCellRenderer';
import { sanitizeHTML } from 'app/client/ui/sanitizeHTML';
import { theme, vars } from 'app/client/ui2018/cssVars';
import { gristThemeObs } from 'app/client/ui2018/theme';
import { NTextBox } from 'app/client/widgets/NTextBox';
import { dom, styled, subscribeBindable } from 'grainjs';
import { Marked } from 'marked';
import { markedHighlight } from 'marked-highlight';
import markedLinkifyIt from 'marked-linkify-it';
/**
* Creates a widget for displaying Markdown-formatted text.
*/
export class MarkdownTextBox extends NTextBox {
private _marked: Marked;
constructor(field: ViewFieldRec) {
super(field);
const highlightCodePromise = buildCodeHighlighter({maxLines: 60});
this._marked = new Marked(
markedHighlight({
async: true,
highlight: async (code) => {
const highlightCode = await highlightCodePromise;
return highlightCode(code);
},
}),
markedLinkifyIt(),
);
}
public buildDom(row: DataRowModel) {
const value = row.cells[this.field.colId()];
return cssFieldClip(
cssFieldClip.cls('-text-wrap', this.wrapping),
dom.style('text-align', this.alignment),
(el) => dom.autoDisposeElem(el, subscribeBindable(value, async () => {
el.innerHTML = sanitizeHTML(await this._marked.parse(String(value.peek()), {gfm: false, renderer}));
this.field.viewSection().events.trigger('rowHeightChange');
})),
// Note: the DOM needs to be rebuilt on theme change, as Ace needs to switch between
// light and dark themes. If we switch to using a custom Grist Ace theme (with CSS
// variables used for highlighting), we can remove the listener below (and elsewhere).
(el) => dom.autoDisposeElem(el, subscribeBindable(gristThemeObs(), async () => {
el.innerHTML = sanitizeHTML(await this._marked.parse(String(value.peek()), {gfm: false, renderer}));
})),
);
}
}
const cssFieldClip = styled('div.field_clip', `
white-space: nowrap;
&-text-wrap {
white-space: normal;
word-break: break-word;
}
&:not(&-text-wrap) p,
&:not(&-text-wrap) h1,
&:not(&-text-wrap) h2,
&:not(&-text-wrap) h3,
&:not(&-text-wrap) h4,
&:not(&-text-wrap) h5,
&:not(&-text-wrap) h6
{
overflow: hidden;
text-overflow: ellipsis;
}
& > *:first-child {
margin-top: 0px !important;
}
& > :not(blockquote, ol, pre, ul):last-child {
margin-bottom: 0px !important;
}
& h1, & h2, & h3, & h4, & h5, & h6 {
margin-top: 24px;
margin-bottom: 16px;
font-weight: 600;
line-height: 1.25;
}
& h1 {
padding-bottom: .3em;
font-size: 2em;
}
& h2 {
padding-bottom: .3em;
font-size: 1.5em;
}
& h3 {
font-size: 1.25em;
}
& h4 {
font-size: 1em;
}
& h5 {
font-size: .875em;
}
& h6 {
color: ${theme.lightText};
font-size: .85em;
}
& p, & blockquote, & ul, & ol, & dl, & pre {
margin-top: 0px;
margin-bottom: 10px;
}
& code, & pre {
color: ${theme.text};
font-size: 85%;
background-color: ${theme.markdownCellLightBg};
border: 0;
border-radius: 6px;
}
& code {
padding: .2em .4em;
margin: 0;
white-space: nowrap;
}
&-text-wrap code {
white-space: break-spaces;
}
& pre {
padding: 16px;
overflow: auto;
line-height: 1.45;
}
& pre code {
font-size: 100%;
display: inline;
max-width: auto;
margin: 0;
padding: 0;
overflow: visible;
line-height: inherit;
word-wrap: normal;
background: transparent;
}
& pre > code {
background: transparent;
white-space: nowrap;
word-break: normal;
margin: 0;
padding: 0;
}
&-text-wrap pre > code {
white-space: normal;
}
& pre .ace-chrome, & pre .ace-dracula {
background: ${theme.markdownCellLightBg} !important;
}
& .ace_indent-guide {
background: none;
}
& .ace_static_highlight {
white-space: nowrap;
}
& ul, & ol {
padding-left: 2em;
}
& li > ol, & li > ul {
margin: 0;
}
& li + li,
& li > ol > li:first-child,
& li > ul > li:first-child {
margin-top: .25em;
}
& blockquote {
font-size: ${vars.mediumFontSize};
border-left: .25em solid ${theme.markdownCellMediumBorder};
padding: 0 1em;
}
`);

12
app/client/widgets/NTextBox.ts
View File

@ -28,9 +28,7 @@ export class NTextBox extends NewAbstractWidget {
this.alignment = fromKo(this.options.prop('alignment'));
this.wrapping = fromKo(this.field.wrap);
this.autoDispose(this.wrapping.addListener(() => {
this.field.viewSection().events.trigger('rowHeightChange');
}));
this._addRowHeightListeners();
}
public buildConfigDom(_gristDoc: GristDoc): DomContents {
@ -112,4 +110,12 @@ export class NTextBox extends NewAbstractWidget {
makeLinks(use(this.valueFormatter).formatAny(use(value), t)))
);
}
private _addRowHeightListeners() {
for (const obs of [this.wrapping, fromKo(this.field.config.widget)]) {
this.autoDispose(obs.addListener(() => {
this.field.viewSection().events.trigger('rowHeightChange');
}));
}
}
}

8
app/client/widgets/NTextEditor.ts
View File

@ -96,6 +96,14 @@ export class NTextEditor extends NewBaseEditor {
this._contentSizer.style.maxWidth = Math.ceil(maxSize.width) + 'px';
}
public get contentSizer(): HTMLElement {
return this._contentSizer;
}
public get editorPlacement(): EditorPlacement {
return this._editorPlacement;
}
/**
* Occurs when user types text in the textarea
*

13
app/client/widgets/NewBaseEditor.ts
View File

@ -13,11 +13,13 @@ export interface IEditorCommandGroup {
[cmd: string]: () => void;
}
// Usually an editor is created for a field and provided FieldOptions, but it's possible to have
// no field object, e.g. for a FormulaEditor for a conditional style rule.
export interface Options {
gristDoc: GristDoc;
cellValue: CellValue;
rowId: number;
formulaError: Observable<CellValue|undefined>;
formulaError?: Observable<CellValue|undefined>;
editValue?: string;
cursorPos: number;
commands: IEditorCommandGroup;
@ -29,6 +31,9 @@ export interface FieldOptions extends Options {
field: ViewFieldRec;
}
// This represents any of the derived editor classes; the part after "&" restricts to non-abstract ones.
export type IEditorConstructor = typeof NewBaseEditor & { new (...args: any[]): NewBaseEditor };
/**
* Required parameters:
* @param {RowModel} options.field: ViewSectionField (i.e. column) being edited.
@ -44,8 +49,10 @@ export abstract class NewBaseEditor extends Disposable {
* Editors and provided by FieldBuilder. TODO: remove this method once all editors have been
* updated to new-style Disposables.
*/
public static create<Opt extends Options>(owner: IDisposableOwner|null, options: Opt): NewBaseEditor;
public static create<Opt extends Options>(options: Opt): NewBaseEditor;
public static create<T extends new (...args: any[]) => any, Opt extends Options>(
this: T, owner: IDisposableOwner|null, options: Opt): InstanceType<T>;
public static create<T extends new (...args: any[]) => any, Opt extends Options>(
this: T, options: Opt): InstanceType<T>;
public static create(ownerOrOptions: any, options?: any): NewBaseEditor {
return options ?
Disposable.create.call(this as any, ownerOrOptions, options) :

49
app/client/widgets/Toggle.ts
View File

@ -10,7 +10,7 @@ import { cssLabel, cssRow } from 'app/client/ui/RightPanelStyles';
import { buttonSelect } from 'app/client/ui2018/buttonSelect';
import { theme } from 'app/client/ui2018/cssVars';
import { NewAbstractWidget, Options } from 'app/client/widgets/NewAbstractWidget';
import { dom, DomContents, makeTestId } from 'grainjs';
import { dom, DomContents, DomElementArg, makeTestId } from 'grainjs';
const t = makeT('Toggle');
@ -74,14 +74,7 @@ export class ToggleCheckBox extends ToggleBase {
public buildDom(row: DataRowModel) {
const value = row.cells[this.field.colId.peek()] as KoSaveableObservable<boolean>;
return dom('div.field_clip',
dom('div.widget_checkbox',
dom('div.widget_checkmark',
dom.show(value),
dom('div.checkmark_kick'),
dom('div.checkmark_stem')
),
this._addClickEventHandlers(row),
)
buildCheckbox(value, this._addClickEventHandlers(row))
);
}
}
@ -91,16 +84,42 @@ export class ToggleSwitch extends ToggleBase {
super(field, {defaultTextColor: '#2CB0AF'});
}
public buildDom(row: DataRowModel) {
public override buildDom(row: DataRowModel) {
const value = row.cells[this.field.colId.peek()] as KoSaveableObservable<boolean>;
return dom('div.field_clip',
dom('div.widget_switch',
dom.cls('switch_on', value),
dom.cls('switch_transition', row._isRealChange),
dom('div.switch_slider'),
dom('div.switch_circle'),
// For printing, we will show this as a checkbox (without handlers).
buildCheckbox(value, dom.cls('screen-force-hide')),
// For screen, we will show this as a switch (with handlers).
buildSwitch(
value,
row._isRealChange,
this._addClickEventHandlers(row),
dom.cls('print-force-hide')
)
);
}
}
function buildCheckbox(value: KoSaveableObservable<boolean>, ...args: DomElementArg[]) {
return dom('div.widget_checkbox',
dom('div.widget_checkmark',
dom.show(value),
dom('div.checkmark_kick'),
dom('div.checkmark_stem')
),
...args
);
}
function buildSwitch(
value: KoSaveableObservable<boolean>,
isTransitionEnabled: ko.Observable<boolean>,
...args: DomElementArg[]) {
return dom('div.widget_switch',
dom.cls('switch_on', value),
dom.cls('switch_transition', isTransitionEnabled),
dom('div.switch_slider'),
dom('div.switch_circle'),
...args
);
}

9
app/client/widgets/UserType.ts
View File

@ -66,6 +66,15 @@ export const typeDefs: any = {
wrap: undefined,
}
},
Markdown: {
cons: 'MarkdownTextBox',
editCons: 'TextEditor',
icon: 'FieldMarkdown',
options: {
alignment: 'left',
wrap: undefined,
}
},
HyperLink: {
cons: 'HyperLinkTextBox',
editCons: 'HyperLinkEditor',

10
app/client/widgets/UserTypeImpl.ts
View File

@ -5,14 +5,15 @@ import ChoiceEditor from 'app/client/widgets/ChoiceEditor';
import {ChoiceListCell} from 'app/client/widgets/ChoiceListCell';
import {ChoiceListEditor} from 'app/client/widgets/ChoiceListEditor';
import {ChoiceTextBox} from 'app/client/widgets/ChoiceTextBox';
import DateEditor from 'app/client/widgets/DateEditor';
import {DateEditor} from 'app/client/widgets/DateEditor';
import DateTextBox from 'app/client/widgets/DateTextBox';
import DateTimeEditor from 'app/client/widgets/DateTimeEditor';
import {DateTimeEditor} from 'app/client/widgets/DateTimeEditor';
import DateTimeTextBox from 'app/client/widgets/DateTimeTextBox';
import {HyperLinkEditor} from 'app/client/widgets/HyperLinkEditor';
import {HyperLinkTextBox} from 'app/client/widgets/HyperLinkTextBox';
import {MarkdownTextBox} from 'app/client/widgets/MarkdownTextBox';
import {NewAbstractWidget} from 'app/client/widgets/NewAbstractWidget';
import {NewBaseEditor} from 'app/client/widgets/NewBaseEditor';
import {IEditorConstructor} from 'app/client/widgets/NewBaseEditor';
import {NTextBox} from 'app/client/widgets/NTextBox';
import {NTextEditor} from 'app/client/widgets/NTextEditor';
import {NumericEditor} from 'app/client/widgets/NumericEditor';
@ -36,6 +37,7 @@ export const nameToWidget = {
'NumericEditor': NumericEditor,
'HyperLinkTextBox': HyperLinkTextBox,
'HyperLinkEditor': HyperLinkEditor,
'MarkdownTextBox': MarkdownTextBox,
'Spinner': Spinner,
'CheckBox': ToggleCheckBox,
'CheckBoxEditor': CheckBoxEditor,
@ -72,7 +74,7 @@ export function getFormWidgetConstructor(widget: string, type: string): WidgetCo
}
/** return a good class to instantiate for editing a widget/type combination */
export function getEditorConstructor(widget: string, type: string): typeof NewBaseEditor {
export function getEditorConstructor(widget: string, type: string): IEditorConstructor {
const {config} = getWidgetConfiguration(widget, type as GristType);
return nameToWidget[config.editCons as keyof typeof nameToWidget] as any;
}

31
app/common/AuditEvent.ts Normal file
View File

@ -0,0 +1,31 @@
export interface AuditEvent<Name extends AuditEventName> {
event: {
/** The event name. */
name: Name;
/** The user that triggered the event. */
user: AuditEventUser | null;
/** Additional event details. */
details: AuditEventDetails[Name] | null;
};
/** ISO 8601 timestamp of when the event was logged. */
timestamp: string;
}
export type AuditEventName =
| 'createDocument';
export interface AuditEventUser {
/** The user's id. */
id: number | null;
/** The user's email address. */
email: string | null;
/** The user's name. */
name: string | null;
}
export interface AuditEventDetails {
createDocument: {
/** The ID of the document. */
id: string;
};
}

21
app/common/DocLimits.ts
View File

@ -1,5 +1,6 @@
import {ApiError} from 'app/common/ApiError';
import {APPROACHING_LIMIT_RATIO, DataLimitStatus, DocumentUsage, getUsageRatio} from 'app/common/DocUsage';
import {APPROACHING_LIMIT_RATIO, DataLimitInfo, DataLimitStatus,
DocumentUsage, getUsageRatio} from 'app/common/DocUsage';
import {Features} from 'app/common/Features';
import moment from 'moment-timezone';
@ -22,22 +23,24 @@ export interface GetDataLimitStatusParams {
* Given a set of params that includes document usage, current product features, and
* a grace-period start (if any), returns the data limit status of a document.
*/
export function getDataLimitStatus(params: GetDataLimitStatusParams): DataLimitStatus {
export function getDataLimitInfo(params: GetDataLimitStatusParams): DataLimitInfo {
const {docUsage, productFeatures, gracePeriodStart} = params;
const ratio = getDataLimitRatio(docUsage, productFeatures);
if (ratio > 1) {
const start = gracePeriodStart;
const days = productFeatures?.gracePeriodDays;
if (start && days && moment().diff(moment(start), 'days') >= days) {
return 'deleteOnly';
// In case we forgot to define a grace period, we'll default to two weeks.
const days = productFeatures?.gracePeriodDays ?? 14;
const daysRemaining = start && days ? days - moment().diff(moment(start), 'days') : NaN;
if (daysRemaining > 0) {
return {status: 'gracePeriod', daysRemaining};
} else {
return 'gracePeriod';
return {status: 'deleteOnly'};
}
} else if (ratio > APPROACHING_LIMIT_RATIO) {
return 'approachingLimit';
} else {
return null;
return {status: 'approachingLimit'};
}
return {status: null};
}
/**

8
app/common/DocUsage.ts
View File

@ -10,13 +10,17 @@ export interface RowCounts {
}
export type DataLimitStatus = 'approachingLimit' | 'gracePeriod' | 'deleteOnly' | null;
export interface DataLimitInfo {
status: DataLimitStatus;
daysRemaining?: number;
}
type DocUsageOrPending = {
[Metric in keyof Required<DocumentUsage>]: Required<DocumentUsage>[Metric] | 'pending'
}
export interface DocUsageSummary extends DocUsageOrPending {
dataLimitStatus: DataLimitStatus;
dataLimitInfo: DataLimitInfo;
}
// Count of non-removed documents in an org, grouped by data limit status.
@ -27,7 +31,7 @@ type FilteredDocUsage = {
}
export interface FilteredDocUsageSummary extends FilteredDocUsage {
dataLimitStatus: DataLimitStatus;
dataLimitInfo: DataLimitInfo;
}
// Ratio of usage at which we start telling users that they're approaching limits.

16
app/common/LoginSessionAPI.ts
View File

@ -1,3 +1,4 @@
import {normalizeEmail} from 'app/common/emails';
import {UserPrefs} from 'app/common/Prefs';
// User profile info for the user. When using Cognito, it is fetched during login.
@ -12,6 +13,21 @@ export interface UserProfile {
locale?: string|null;
}
/**
* Tries to compare two user profiles to see if they represent the same user.
* Note: if you have access to FullUser objects, comparing ids is more reliable.
*/
export function sameUser(a: UserProfile|FullUser, b: UserProfile|FullUser): boolean {
if ('id' in a && 'id' in b) {
return a.id === b.id;
}
if (a.loginEmail && b.loginEmail) {
return a.loginEmail === b.loginEmail;
} else {
return normalizeEmail(a.email) === normalizeEmail(b.email);
}
}
// User profile including user id and user ref. All information in it should
// have been validated against database.
export interface FullUser extends UserProfile {

3
app/common/ThemePrefs-ti.ts
View File

@ -447,6 +447,9 @@ export const ThemeColors = t.iface([], {
"widget-gallery-secondary-header-fg": "string",
"widget-gallery-secondary-header-bg": "string",
"widget-gallery-secondary-header-bg-hover": "string",
"markdown-cell-light-bg": "string",
"markdown-cell-light-border": "string",
"markdown-cell-medium-border": "string",
});
const exportedTypeSuite: t.ITypeSuite = {

5
app/common/ThemePrefs.ts
View File

@ -583,6 +583,11 @@ export interface ThemeColors {
'widget-gallery-secondary-header-fg': string;
'widget-gallery-secondary-header-bg': string;
'widget-gallery-secondary-header-bg-hover': string;
/* Markdown Cell */
'markdown-cell-light-bg': string;
'markdown-cell-light-border': string;
'markdown-cell-medium-border': string;
}
export const ThemePrefsChecker = createCheckers(ThemePrefsTI).ThemePrefs as CheckerT<ThemePrefs>;

4
app/common/schema.ts
View File

@ -4,7 +4,7 @@ import { GristObjCode } from "app/plugin/GristData";
// tslint:disable:object-literal-key-quotes
export const SCHEMA_VERSION = 42;
export const SCHEMA_VERSION = 43;
export const schema = {
@ -41,6 +41,7 @@ export const schema = {
displayCol : "Ref:_grist_Tables_column",
visibleCol : "Ref:_grist_Tables_column",
rules : "RefList:_grist_Tables_column",
reverseCol : "Ref:_grist_Tables_column",
recalcWhen : "Int",
recalcDeps : "RefList:_grist_Tables_column",
},
@ -264,6 +265,7 @@ export interface SchemaTypes {
displayCol: number;
visibleCol: number;
rules: [GristObjCode.List, ...number[]]|null;
reverseCol: number;
recalcWhen: number;
recalcDeps: [GristObjCode.List, ...number[]]|null;
};

5
app/common/themes/GristDark.ts
View File

@ -562,4 +562,9 @@ export const GristDark: ThemeColors = {
'widget-gallery-secondary-header-fg': '#FFFFFF',
'widget-gallery-secondary-header-bg': '#70707D',
'widget-gallery-secondary-header-bg-hover': '#60606D',
/* Markdown Cell */
'markdown-cell-light-bg': '#494958',
'markdown-cell-light-border': '#32323F',
'markdown-cell-medium-border': '#555563',
};

5
app/common/themes/GristLight.ts
View File

@ -562,4 +562,9 @@ export const GristLight: ThemeColors = {
'widget-gallery-secondary-header-fg': '#FFFFFF',
'widget-gallery-secondary-header-bg': '#929299',
'widget-gallery-secondary-header-bg-hover': '#7E7E85',
/* Markdown Cell */
'markdown-cell-light-bg': '#F7F7F7',
'markdown-cell-light-border': '#E8E8E8',
'markdown-cell-medium-border': '#D9D9D9',
};

34
app/gen-server/ApiServer.ts
View File

@ -17,7 +17,7 @@ import {RequestWithOrg} from 'app/server/lib/extractOrg';
import {GristServer} from 'app/server/lib/GristServer';
import {getTemplateOrg} from 'app/server/lib/gristSettings';
import log from 'app/server/lib/log';
import {addPermit, clearSessionCacheIfNeeded, getDocScope, getScope, integerParam,
import {clearSessionCacheIfNeeded, getDocScope, getScope, integerParam,
isParameterOn, optStringParam, sendOkReply, sendReply, stringParam} from 'app/server/lib/requestUtils';
import {IWidgetRepository} from 'app/server/lib/WidgetRepository';
import {getCookieDomain} from 'app/server/lib/gristSessions';
@ -283,6 +283,12 @@ export class ApiServer {
altSessionId: mreq.altSessionId,
},
});
this._gristServer.getAuditLogger().logEvent(mreq, {
event: {
name: 'createDocument',
details: {id: docId},
},
});
return sendReply(req, res, query);
}));
@ -392,7 +398,7 @@ export class ApiServer {
// Get user access information regarding an org
this._app.get('/api/orgs/:oid/access', expressWrap(async (req, res) => {
const org = getOrgKey(req);
const query = await this._withSupportUserAllowedToView(
const query = await this._withPrivilegedViewForUser(
org, req, (scope) => this._dbManager.getOrgAccess(scope, org)
);
return sendReply(req, res, query);
@ -429,7 +435,7 @@ export class ApiServer {
throw new ApiError('Name expected in the body', 400);
}
const name = req.body.name;
await this._dbManager.updateUserName(userId, name);
await this._dbManager.updateUser(userId, { name });
res.sendStatus(200);
}));
@ -534,7 +540,7 @@ export class ApiServer {
this._app.get('/api/session/access/active', expressWrap(async (req, res) => {
const fullUser = await this._getFullUser(req, {includePrefs: true});
const domain = getOrgFromRequest(req);
const org = domain ? (await this._withSupportUserAllowedToView(
const org = domain ? (await this._withPrivilegedViewForUser(
domain, req, (scope) => this._dbManager.getOrg(scope, domain)
)) : null;
const orgError = (org && org.errMessage) ? {error: org.errMessage, status: org.status} : undefined;
@ -617,26 +623,32 @@ export class ApiServer {
/**
* Run a query, and, if it is denied and the user is the support
* Run a query, and, if it is denied and the user is the support or admin
* user, rerun the query with permission to view the current
* org. This is a bit inefficient, but only affects the support
* org. This is a bit inefficient, but only affects the support/admin
* user. We wait to add the special permission only if needed, since
* it will in fact override any other access the support user has
* it will in fact override any other access the special user has
* been granted, which could reduce their apparent access if that is
* part of what is returned by the query.
*/
private async _withSupportUserAllowedToView<T>(
private async _withPrivilegedViewForUser<T>(
org: string|number, req: express.Request,
op: (scope: Scope) => Promise<QueryResult<T>>
): Promise<QueryResult<T>> {
const scope = getScope(req);
const userId = getUserId(req);
const result = await op(scope);
if (result.status === 200 || userId !== this._dbManager.getSupportUserId()) {
if (result.status === 200) {
return result;
}
const extendedScope = addPermit(scope, this._dbManager.getSupportUserId(), {org});
return await op(extendedScope);
if (userId === this._dbManager.getSupportUserId() ||
await this._gristServer.getInstallAdmin()?.isAdminReq(req)) {
const extendedScope: Scope = {...scope, specialPermit: {org}};
return await op(extendedScope);
}
return result;
}
private _logInvitedDocUserTelemetryEvents(mreq: RequestWithLogin, delta: PermissionDelta) {

4
app/gen-server/entity/User.ts
View File

@ -26,10 +26,10 @@ export class User extends BaseEntity {
@Column({name: 'picture', type: String, nullable: true})
public picture: string | null;
@Column({name: 'first_login_at', type: Date, nullable: true})
@Column({name: 'first_login_at', type: nativeValues.dateTimeType, nullable: true})
public firstLoginAt: Date | null;
@Column({name: 'last_connection_at', type: Date, nullable: true})
@Column({name: 'last_connection_at', type: nativeValues.dateTimeType, nullable: true})
public lastConnectionAt: Date | null;
@OneToOne(type => Organization, organization => organization.owner)

48
app/gen-server/lib/homedb/HomeDBManager.ts
View File

@ -1,7 +1,7 @@
import {ShareInfo} from 'app/common/ActiveDocAPI';
import {ApiError, LimitType} from 'app/common/ApiError';
import {mapGetOrSet, mapSetOrClear, MapWithTTL} from 'app/common/AsyncCreate';
import {getDataLimitStatus} from 'app/common/DocLimits';
import {getDataLimitInfo} from 'app/common/DocLimits';
import {createEmptyOrgUsageSummary, DocumentUsage, OrgUsageSummary} from 'app/common/DocUsage';
import {normalizeEmail} from 'app/common/emails';
import {ANONYMOUS_PLAN, canAddOrgMembers, Features} from 'app/common/Features';
@ -68,12 +68,12 @@ import {Request} from "express";
import {defaultsDeep, flatten, pick} from 'lodash';
import {
Brackets,
Connection,
DatabaseType,
DataSource,
EntityManager,
ObjectLiteral,
SelectQueryBuilder,
WhereExpression
WhereExpressionBuilder
} from "typeorm";
import uuidv4 from "uuid/v4";
@ -247,8 +247,7 @@ export type BillingOptions = Partial<Pick<BillingAccount,
*/
export class HomeDBManager extends EventEmitter {
private _usersManager = new UsersManager(this, this._runInTransaction.bind(this));
private _connection: Connection;
private _dbType: DatabaseType;
private _connection: DataSource;
private _exampleWorkspaceId: number;
private _exampleOrgId: number;
private _idPrefix: string = ""; // Place this before ids in subdomains, used in routing to
@ -258,6 +257,10 @@ export class HomeDBManager extends EventEmitter {
// In restricted mode, documents should be read-only.
private _restrictedMode: boolean = false;
private get _dbType(): DatabaseType {
return this._connection.driver.options.type;
}
/**
* Five aclRules, each with one group (with the names 'owners', 'editors', 'viewers',
* 'guests', and 'members') are created by default on every new entity (Organization,
@ -348,7 +351,10 @@ export class HomeDBManager extends EventEmitter {
public async connect(): Promise<void> {
this._connection = await getOrCreateConnection();
this._dbType = this._connection.driver.options.type;
}
public connectTo(connection: DataSource) {
this._connection = connection;
}
// make sure special users and workspaces are available
@ -451,7 +457,7 @@ export class HomeDBManager extends EventEmitter {
* @see UsersManager.prototype.ensureExternalUser
*/
public async ensureExternalUser(profile: UserProfile) {
return this._usersManager.ensureExternalUser(profile);
return await this._usersManager.ensureExternalUser(profile);
}
public async updateUser(userId: number, props: UserProfileChange) {
@ -461,10 +467,6 @@ export class HomeDBManager extends EventEmitter {
}
}
public async updateUserName(userId: number, name: string) {
return this._usersManager.updateUserName(userId, name);
}
public async updateUserOptions(userId: number, props: Partial<UserOptions>) {
return this._usersManager.updateUserOptions(userId, props);
}
@ -472,14 +474,14 @@ export class HomeDBManager extends EventEmitter {
/**
* @see UsersManager.prototype.getUserByLoginWithRetry
*/
public async getUserByLoginWithRetry(email: string, options: GetUserOptions = {}): Promise<User|undefined> {
public async getUserByLoginWithRetry(email: string, options: GetUserOptions = {}): Promise<User> {
return this._usersManager.getUserByLoginWithRetry(email, options);
}
/**
* @see UsersManager.prototype.getUserByLogin
*/
public async getUserByLogin(email: string, options: GetUserOptions = {}): Promise<User|undefined> {
public async getUserByLogin(email: string, options: GetUserOptions = {}): Promise<User> {
return this._usersManager.getUserByLogin(email, options);
}
@ -488,7 +490,7 @@ export class HomeDBManager extends EventEmitter {
* Find a user by email. Don't create the user if it doesn't already exist.
*/
public async getExistingUserByLogin(email: string, manager?: EntityManager): Promise<User|undefined> {
return this._usersManager.getExistingUserByLogin(email, manager);
return await this._usersManager.getExistingUserByLogin(email, manager);
}
/**
@ -764,7 +766,7 @@ export class HomeDBManager extends EventEmitter {
// Return an aggregate count of documents, grouped by data limit status.
const summary = createEmptyOrgUsageSummary();
for (const {usage: docUsage, gracePeriodStart} of docs) {
const dataLimitStatus = getDataLimitStatus({docUsage, gracePeriodStart, productFeatures});
const dataLimitStatus = getDataLimitInfo({docUsage, gracePeriodStart, productFeatures}).status;
if (dataLimitStatus) { summary[dataLimitStatus] += 1; }
}
return summary;
@ -985,6 +987,10 @@ export class HomeDBManager extends EventEmitter {
return doc;
}
public async getAllDocs() {
return this.connection.getRepository(Document).find();
}
public async getRawDocById(docId: string, transaction?: EntityManager) {
return await this.getDoc({
urlId: docId,
@ -1808,12 +1814,13 @@ export class HomeDBManager extends EventEmitter {
//
// Returns an empty query result with status 200 on success.
public async updateBillingAccount(
userId: number,
scopeOrUser: number|Scope,
orgKey: string|number,
callback: (billingAccount: BillingAccount, transaction: EntityManager) => void|Promise<void>
): Promise<QueryResult<void>> {
): Promise<QueryResult<void>> {
return await this._connection.transaction(async transaction => {
const billingAccount = await this.getBillingAccount({userId}, orgKey, false, transaction);
const scope = typeof scopeOrUser === 'number' ? {userId: scopeOrUser} : scopeOrUser;
const billingAccount = await this.getBillingAccount(scope, orgKey, false, transaction);
const billingAccountCopy = Object.assign({}, billingAccount);
await callback(billingAccountCopy, transaction);
// Pick out properties that are allowed to be changed, to prevent accidental updating
@ -3435,7 +3442,7 @@ export class HomeDBManager extends EventEmitter {
// Adds a where clause to filter orgs by domain or id.
// If org is null, filter for user's personal org.
// if includeSupport is true, include the org of the support@ user (for the Samples workspace)
private _whereOrg<T extends WhereExpression>(qb: T, org: string|number, includeSupport = false): T {
private _whereOrg<T extends WhereExpressionBuilder>(qb: T, org: string|number, includeSupport = false): T {
if (this.isMergedOrg(org)) {
// Select from universe of personal orgs.
// Don't panic though! While this means that SQL can't use an organization id
@ -3455,7 +3462,7 @@ export class HomeDBManager extends EventEmitter {
}
}
private _wherePlainOrg<T extends WhereExpression>(qb: T, org: string|number): T {
private _wherePlainOrg<T extends WhereExpressionBuilder>(qb: T, org: string|number): T {
if (typeof org === 'number') {
return qb.andWhere('orgs.id = :org', {org});
}
@ -4362,7 +4369,6 @@ export class HomeDBManager extends EventEmitter {
});
return verifyEntity(orgQuery);
}
}
// Return a QueryResult reflecting the output of a query builder.

6
app/gen-server/lib/homedb/Interfaces.ts
View File

@ -34,7 +34,7 @@ export type NonGuestGroup = Group & { name: roles.NonGuestRole };
export type Resource = Organization|Workspace|Document;
export type RunInTransaction = (
export type RunInTransaction = <T>(
transaction: EntityManager|undefined,
op: ((manager: EntityManager) => Promise<any>)
) => Promise<any>;
op: ((manager: EntityManager) => Promise<T>)
) => Promise<T>;

152
app/gen-server/lib/homedb/UsersManager.ts
View File

@ -97,7 +97,7 @@ export class UsersManager {
public async testClearUserPrefs(emails: string[]) {
return await this._connection.transaction(async manager => {
for (const email of emails) {
const user = await this.getUserByLogin(email, {manager});
const user = await this.getExistingUserByLogin(email, manager);
if (user) {
await manager.delete(Pref, {userId: user.id});
}
@ -116,7 +116,7 @@ export class UsersManager {
*/
public getAnonymousUserId(): number {
const id = this._specialUserIds[ANONYMOUS_USER_EMAIL];
if (!id) { throw new Error("Anonymous user not available"); }
if (!id) { throw new Error("'Anonymous' user not available"); }
return id;
}
@ -125,7 +125,7 @@ export class UsersManager {
*/
public getPreviewerUserId(): number {
const id = this._specialUserIds[PREVIEWER_EMAIL];
if (!id) { throw new Error("Previewer user not available"); }
if (!id) { throw new Error("'Previewer' user not available"); }
return id;
}
@ -134,7 +134,7 @@ export class UsersManager {
*/
public getEveryoneUserId(): number {
const id = this._specialUserIds[EVERYONE_EMAIL];
if (!id) { throw new Error("'everyone' user not available"); }
if (!id) { throw new Error("'Everyone' user not available"); }
return id;
}
@ -143,7 +143,7 @@ export class UsersManager {
*/
public getSupportUserId(): number {
const id = this._specialUserIds[SUPPORT_EMAIL];
if (!id) { throw new Error("'support' user not available"); }
if (!id) { throw new Error("'Support' user not available"); }
return id;
}
@ -221,12 +221,9 @@ export class UsersManager {
profile,
manager
});
if (!newUser) {
throw new ApiError("Unable to create user", 500);
}
// No need to survey this user.
newUser.isFirstTimeUser = false;
await newUser.save();
await manager.save(newUser);
} else {
// Else update profile and login information from external profile.
let updated = false;
@ -280,26 +277,20 @@ export class UsersManager {
if (!props.isFirstTimeUser) { isWelcomed = true; }
}
if (needsSave) {
await user.save();
await manager.save(user);
}
});
return { user, isWelcomed };
}
public async updateUserName(userId: number, name: string) {
const user = await User.findOne({where: {id: userId}});
if (!user) { throw new ApiError("unable to find user", 400); }
user.name = name;
await user.save();
}
// TODO: rather use the updateUser() method, if that makes sense?
public async updateUserOptions(userId: number, props: Partial<UserOptions>) {
const user = await User.findOne({where: {id: userId}});
if (!user) { throw new ApiError("unable to find user", 400); }
const newOptions = {...(user.options ?? {}), ...props};
user.options = newOptions;
await user.save();
await this._runInTransaction(undefined, async manager => {
const user = await manager.findOne(User, {where: {id: userId}});
if (!user) { throw new ApiError("unable to find user", 400); }
user.options = {...(user.options ?? {}), ...props};
await manager.save(user);
});
}
/**
@ -321,7 +312,7 @@ export class UsersManager {
// for an email key conflict failure. This is in case our transaction conflicts with a peer
// doing the same thing. This is quite likely if the first page visited by a previously
// unseen user fires off multiple api calls.
public async getUserByLoginWithRetry(email: string, options: GetUserOptions = {}): Promise<User|undefined> {
public async getUserByLoginWithRetry(email: string, options: GetUserOptions = {}): Promise<User> {
try {
return await this.getUserByLogin(email, options);
} catch (e) {
@ -361,10 +352,10 @@ export class UsersManager {
* unset/outdated fields of an existing record.
*
*/
public async getUserByLogin(email: string, options: GetUserOptions = {}): Promise<User|undefined> {
public async getUserByLogin(email: string, options: GetUserOptions = {}) {
const {manager: transaction, profile, userOptions} = options;
const normalizedEmail = normalizeEmail(email);
const userByLogin = await this._runInTransaction(transaction, async manager => {
return await this._runInTransaction(transaction, async manager => {
let needUpdate = false;
const userQuery = manager.createQueryBuilder()
.select('user')
@ -473,9 +464,8 @@ export class UsersManager {
// In principle this could be optimized, but this is simpler to maintain.
user = await userQuery.getOne();
}
return user;
return user!;
});
return userByLogin;
}
/**
@ -520,6 +510,63 @@ export class UsersManager {
};
}
public async initializeSpecialIds(): Promise<void> {
await this._maybeCreateSpecialUserId({
email: ANONYMOUS_USER_EMAIL,
name: "Anonymous"
});
await this._maybeCreateSpecialUserId({
email: PREVIEWER_EMAIL,
name: "Preview"
});
await this._maybeCreateSpecialUserId({
email: EVERYONE_EMAIL,
name: "Everyone"
});
await this._maybeCreateSpecialUserId({
email: SUPPORT_EMAIL,
name: "Support"
});
}
/**
*
* Take a list of user profiles coming from the client's session, correlate
* them with Users and Logins in the database, and construct full profiles
* with user ids, standardized display emails, pictures, and anonymous flags.
*
*/
public async completeProfiles(profiles: UserProfile[]): Promise<FullUser[]> {
if (profiles.length === 0) { return []; }
const qb = this._connection.createQueryBuilder()
.select('logins')
.from(Login, 'logins')
.leftJoinAndSelect('logins.user', 'user')
.where('logins.email in (:...emails)', {emails: profiles.map(profile => normalizeEmail(profile.email))});
const completedProfiles: {[email: string]: FullUser} = {};
for (const login of await qb.getMany()) {
completedProfiles[login.email] = {
id: login.user.id,
email: login.displayEmail,
name: login.user.name,
picture: login.user.picture,
anonymous: login.user.id === this.getAnonymousUserId(),
locale: login.user.options?.locale
};
}
return profiles.map(profile => completedProfiles[normalizeEmail(profile.email)])
.filter(fullProfile => fullProfile);
}
/**
* ==================================
*
* Below methods are public but not exposed by HomeDBManager
*
* They are meant to be used internally (i.e. by homedb/ modules)
*
*/
// Looks up the emails in the permission delta and adds them to the users map in
// the delta object.
// Returns a QueryResult based on the validity of the passed in PermissionDelta object.
@ -589,25 +636,6 @@ export class UsersManager {
};
}
public async initializeSpecialIds(): Promise<void> {
await this._maybeCreateSpecialUserId({
email: ANONYMOUS_USER_EMAIL,
name: "Anonymous"
});
await this._maybeCreateSpecialUserId({
email: PREVIEWER_EMAIL,
name: "Preview"
});
await this._maybeCreateSpecialUserId({
email: EVERYONE_EMAIL,
name: "Everyone"
});
await this._maybeCreateSpecialUserId({
email: SUPPORT_EMAIL,
name: "Support"
});
}
/**
* Check for anonymous user, either encoded directly as an id, or as a singular
* profile (this case arises during processing of the session/access/all endpoint
@ -684,34 +712,6 @@ export class UsersManager {
return members;
}
/**
*
* Take a list of user profiles coming from the client's session, correlate
* them with Users and Logins in the database, and construct full profiles
* with user ids, standardized display emails, pictures, and anonymous flags.
*
*/
public async completeProfiles(profiles: UserProfile[]): Promise<FullUser[]> {
if (profiles.length === 0) { return []; }
const qb = this._connection.createQueryBuilder()
.select('logins')
.from(Login, 'logins')
.leftJoinAndSelect('logins.user', 'user')
.where('logins.email in (:...emails)', {emails: profiles.map(profile => normalizeEmail(profile.email))});
const completedProfiles: {[email: string]: FullUser} = {};
for (const login of await qb.getMany()) {
completedProfiles[login.email] = {
id: login.user.id,
email: login.displayEmail,
name: login.user.name,
picture: login.user.picture,
anonymous: login.user.id === this.getAnonymousUserId(),
locale: login.user.options?.locale
};
}
return profiles.map(profile => completedProfiles[normalizeEmail(profile.email)])
.filter(profile => profile);
}
// For the moment only the support user can add both everyone@ and anon@ to a
// resource, since that allows spam. TODO: enhance or remove.
@ -735,7 +735,7 @@ export class UsersManager {
// user if a bunch of servers start simultaneously and the user doesn't exist
// yet.
const user = await this.getUserByLoginWithRetry(profile.email, {profile});
if (user) { id = this._specialUserIds[profile.email] = user.id; }
id = this._specialUserIds[profile.email] = user.id;
}
if (!id) { throw new Error(`Could not find or create user ${profile.email}`); }
return id;

237
app/server/MergedServer.ts Normal file
View File

@ -0,0 +1,237 @@
/**
*
* A version of hosted grist that recombines a home server,
* a doc worker, and a static server on a single port.
*
*/
import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
import log from 'app/server/lib/log';
import {getGlobalConfig} from "app/server/lib/globalConfig";
// Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS
// environment variable.
export type ServerType = "home" | "docs" | "static" | "app";
const allServerTypes: ServerType[] = ["home", "docs", "static", "app"];
// Parse a comma-separate list of server types into an array, with validation.
export function parseServerTypes(serverTypes: string|undefined): ServerType[] {
// Split and filter out empty strings (including the one we get when splitting "").
const types = (serverTypes || "").trim().split(',').filter(part => Boolean(part));
// Check that parts is non-empty and only contains valid options.
if (!types.length) {
throw new Error(`No server types; should be a comma-separated list of ${allServerTypes.join(", ")}`);
}
for (const t of types) {
if (!allServerTypes.includes(t as ServerType)) {
throw new Error(`Invalid server type '${t}'; should be in ${allServerTypes.join(", ")}`);
}
}
return types as ServerType[];
}
function checkUserContentPort(): number | null {
// Check whether a port is explicitly set for user content.
if (process.env.GRIST_UNTRUSTED_PORT) {
return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10);
}
// Checks whether to serve user content on same domain but on different port
if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
const homeUrl = new URL(process.env.APP_HOME_URL);
const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
// If the hostname of both home and plugin url are the same,
// but the ports are different
if (homeUrl.hostname === pluginUrl.hostname &&
homeUrl.port !== pluginUrl.port) {
const port = parseInt(pluginUrl.port || '80', 10);
return port;
}
}
return null;
}
interface ServerOptions extends FlexServerOptions {
// If set, messages logged to console (default: false)
// (but if options are not given at all in call to main, logToConsole is set to true)
logToConsole?: boolean;
// If set, documents saved to external storage such as s3 (default is to check environment variables,
// which get set in various ways in dev/test entry points)
externalStorage?: boolean;
}
export class MergedServer {
public static async create(port: number, serverTypes: ServerType[], options: ServerOptions = {}) {
options.settings ??= getGlobalConfig();
const ms = new MergedServer(port, serverTypes, options);
// We need to know early on whether we will be serving plugins or not.
if (ms.hasComponent("home")) {
const userPort = checkUserContentPort();
ms.flexServer.setServesPlugins(userPort !== undefined);
} else {
ms.flexServer.setServesPlugins(false);
}
ms.flexServer.addCleanup();
ms.flexServer.setDirectory();
if (process.env.GRIST_TEST_ROUTER) {
// Add a mock api for adding/removing doc workers from load balancer.
ms.flexServer.testAddRouter();
}
if (ms._options.logToConsole !== false) { ms.flexServer.addLogging(); }
if (ms._options.externalStorage === false) { ms.flexServer.disableExternalStorage(); }
await ms.flexServer.addLoginMiddleware();
if (ms.hasComponent("docs")) {
// It is important that /dw and /v prefixes are accepted (if present) by health check
// in ms case, since they are included in the url registered for the doc worker.
ms.flexServer.stripDocWorkerIdPathPrefixIfPresent();
ms.flexServer.addTagChecker();
}
ms.flexServer.addHealthCheck();
if (ms.hasComponent("home") || ms.hasComponent("app")) {
ms.flexServer.addBootPage();
}
ms.flexServer.denyRequestsIfNotReady();
if (ms.hasComponent("home") || ms.hasComponent("static") || ms.hasComponent("app")) {
ms.flexServer.setDirectory();
}
if (ms.hasComponent("home") || ms.hasComponent("static")) {
ms.flexServer.addStaticAndBowerDirectories();
}
await ms.flexServer.initHomeDBManager();
ms.flexServer.addHosts();
ms.flexServer.addDocWorkerMap();
if (ms.hasComponent("home") || ms.hasComponent("static")) {
await ms.flexServer.addAssetsForPlugins();
}
if (ms.hasComponent("home")) {
ms.flexServer.addEarlyWebhooks();
}
if (ms.hasComponent("home") || ms.hasComponent("docs") || ms.hasComponent("app")) {
ms.flexServer.addSessions();
}
ms.flexServer.addAccessMiddleware();
ms.flexServer.addApiMiddleware();
await ms.flexServer.addBillingMiddleware();
return ms;
}
public readonly flexServer: FlexServer;
private readonly _serverTypes: ServerType[];
private readonly _options: ServerOptions;
private constructor(port: number, serverTypes: ServerType[], options: ServerOptions = {}) {
this._serverTypes = serverTypes;
this._options = options;
this.flexServer = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
}
public hasComponent(serverType: ServerType) {
return this._serverTypes.includes(serverType);
}
public async run() {
try {
await this.flexServer.start();
if (this.hasComponent("home")) {
this.flexServer.addUsage();
if (!this.hasComponent("docs")) {
this.flexServer.addDocApiForwarder();
}
this.flexServer.addJsonSupport();
this.flexServer.addUpdatesCheck();
await this.flexServer.addLandingPages();
// todo: add support for home api to standalone app
this.flexServer.addHomeApi();
this.flexServer.addBillingApi();
this.flexServer.addNotifier();
this.flexServer.addAuditLogger();
await this.flexServer.addTelemetry();
await this.flexServer.addHousekeeper();
await this.flexServer.addLoginRoutes();
this.flexServer.addAccountPage();
this.flexServer.addBillingPages();
this.flexServer.addWelcomePaths();
this.flexServer.addLogEndpoint();
this.flexServer.addGoogleAuthEndpoint();
this.flexServer.addInstallEndpoints();
this.flexServer.addConfigEndpoints();
}
if (this.hasComponent("docs")) {
this.flexServer.addJsonSupport();
this.flexServer.addAuditLogger();
await this.flexServer.addTelemetry();
await this.flexServer.addDoc();
}
if (this.hasComponent("home")) {
this.flexServer.addClientSecrets();
}
this.flexServer.finalizeEndpoints();
await this.flexServer.finalizePlugins(this.hasComponent("home") ? checkUserContentPort() : null);
this.flexServer.checkOptionCombinations();
this.flexServer.summary();
this.flexServer.ready();
// Some tests have their timing perturbed by having this earlier
// TODO: update those tests.
if (this.hasComponent("docs")) {
await this.flexServer.checkSandbox();
}
} catch(e) {
await this.flexServer.close();
throw e;
}
}
}
export async function startMain() {
try {
const serverTypes = parseServerTypes(process.env.GRIST_SERVERS);
// No defaults for a port, since this server can serve very different purposes.
if (!process.env.GRIST_PORT) {
throw new Error("GRIST_PORT must be specified");
}
const port = parseInt(process.env.GRIST_PORT, 10);
const server = await MergedServer.create(port, serverTypes);
await server.run();
const opt = process.argv[2];
if (opt === '--testingHooks') {
await server.flexServer.addTestingHooks();
}
return server.flexServer;
} catch (e) {
log.error('mergedServer failed to start', e);
process.exit(1);
}
}
if (require.main === module) {
startMain().catch((e) => log.error('mergedServer failed to start', e));
}

4
app/server/companion.ts
View File

@ -166,10 +166,6 @@ export function addSiteCommand(program: commander.Command,
const profile = {email, name: email};
const db = await getHomeDBManager();
const user = await db.getUserByLogin(email, {profile});
if (!user) {
// This should not happen.
throw new Error('failed to create user');
}
db.unwrapQueryResult(await db.addOrg(user, {
name: domain,
domain,

18
app/server/devServerMain.ts
View File

@ -21,7 +21,7 @@
import {updateDb} from 'app/server/lib/dbUtils';
import {FlexServer} from 'app/server/lib/FlexServer';
import log from 'app/server/lib/log';
import {main as mergedServerMain} from 'app/server/mergedServerMain';
import {MergedServer} from 'app/server/MergedServer';
import {promisifyAll} from 'bluebird';
import * as fse from 'fs-extra';
import * as path from 'path';
@ -96,8 +96,9 @@ export async function main() {
if (!process.env.APP_HOME_URL) {
process.env.APP_HOME_URL = `http://localhost:${port}`;
}
const server = await mergedServerMain(port, ["home", "docs", "static"]);
await server.addTestingHooks();
const mergedServer = await MergedServer.create(port, ["home", "docs", "static"]);
await mergedServer.flexServer.addTestingHooks();
await mergedServer.run();
return;
}
@ -118,17 +119,18 @@ export async function main() {
log.info("== staticServer");
const staticPort = getPort("STATIC_PORT", 9001);
process.env.APP_STATIC_URL = `http://localhost:${staticPort}`;
await mergedServerMain(staticPort, ["static"]);
await MergedServer.create(staticPort, ["static"]).then((s) => s.run());
// Bring up a home server
log.info("==========================================================================");
log.info("== homeServer");
const home = await mergedServerMain(homeServerPort, ["home"]);
const homeServer = await MergedServer.create(homeServerPort, ["home"]);
await homeServer.run();
// If a distinct webServerPort is specified, we listen also on that port, though serving
// exactly the same content. This is handy for testing CORS issues.
if (webServerPort !== 0 && webServerPort !== homeServerPort) {
await home.startCopy('webServer', webServerPort);
await homeServer.flexServer.startCopy('webServer', webServerPort);
}
// Bring up the docWorker(s)
@ -147,10 +149,10 @@ export async function main() {
}
const workers = new Array<FlexServer>();
for (const port of ports) {
workers.push(await mergedServerMain(port, ["docs"]));
workers.push((await MergedServer.create(port, ["docs"])).flexServer);
}
await home.addTestingHooks(workers);
await homeServer.flexServer.addTestingHooks(workers);
}

3
app/server/generateInitialDocSql.ts
View File

@ -3,7 +3,7 @@ import { create } from 'app/server/lib/create';
import { DocManager } from 'app/server/lib/DocManager';
import { makeExceptionalDocSession } from 'app/server/lib/DocSession';
import { DocStorageManager } from 'app/server/lib/DocStorageManager';
import { createDummyTelemetry } from 'app/server/lib/GristServer';
import { createDummyAuditLogger, createDummyTelemetry } from 'app/server/lib/GristServer';
import { PluginManager } from 'app/server/lib/PluginManager';
import * as childProcess from 'child_process';
@ -34,6 +34,7 @@ export async function main(baseName: string) {
}
const docManager = new DocManager(storageManager, pluginManager, null as any, {
create,
getAuditLogger() { return createDummyAuditLogger(); },
getTelemetry() { return createDummyTelemetry(); },
} as any);
const activeDoc = new ActiveDoc(docManager, baseName);

49
app/server/lib/ActiveDoc.ts
View File

@ -50,12 +50,12 @@ import {
UserAction
} from 'app/common/DocActions';
import {DocData} from 'app/common/DocData';
import {getDataLimitRatio, getDataLimitStatus, getSeverity, LimitExceededError} from 'app/common/DocLimits';
import {getDataLimitInfo, getDataLimitRatio, getSeverity, LimitExceededError} from 'app/common/DocLimits';
import {DocSnapshots} from 'app/common/DocSnapshot';
import {DocumentSettings} from 'app/common/DocumentSettings';
import {
APPROACHING_LIMIT_RATIO,
DataLimitStatus,
DataLimitInfo,
DocumentUsage,
DocUsageSummary,
FilteredDocUsageSummary,
@ -303,7 +303,7 @@ export class ActiveDoc extends EventEmitter {
),
// Update the time in formulas every hour.
new Interval(
() => this._applyUserActions(makeExceptionalDocSession('system'), [['UpdateCurrentTime']]),
() => this._updateCurrentTime(),
Deps.UPDATE_CURRENT_TIME_DELAY,
{onError: (e) => this._log.error(null, 'failed to update current time', e)},
),
@ -421,8 +421,8 @@ export class ActiveDoc extends EventEmitter {
return getDataLimitRatio(this._docUsage, this._product?.features);
}
public get dataLimitStatus(): DataLimitStatus {
return getDataLimitStatus({
public get dataLimitInfo(): DataLimitInfo {
return getDataLimitInfo({
docUsage: this._docUsage,
productFeatures: this._product?.features,
gracePeriodStart: this._gracePeriodStart,
@ -431,7 +431,7 @@ export class ActiveDoc extends EventEmitter {
public getDocUsageSummary(): DocUsageSummary {
return {
dataLimitStatus: this.dataLimitStatus,
dataLimitInfo: this.dataLimitInfo,
rowCount: this._docUsage?.rowCount ?? 'pending',
dataSizeBytes: this._docUsage?.dataSizeBytes ?? 'pending',
attachmentsSizeBytes: this._docUsage?.attachmentsSizeBytes ?? 'pending',
@ -1696,7 +1696,7 @@ export class ActiveDoc extends EventEmitter {
const timeDeleted = changes.map(r => r.used ? null : now);
const action: BulkUpdateRecord = ["BulkUpdateRecord", "_grist_Attachments", rowIds, {timeDeleted}];
// Don't use applyUserActions which may block the update action in delete-only mode
await this._applyUserActions(makeExceptionalDocSession('system'), [action]);
await this._applyUserActionsAsSystem([action]);
return true;
}
@ -2011,6 +2011,16 @@ export class ActiveDoc extends EventEmitter {
return [tableNames, onDemandNames];
}
/**
* Applies an array of user actions initiated by Grist itself, using a DocSession with "system"
* access rights. These bypass access rules.
*
* They also do not count as "user activity" for the purpose of keeping the document open.
*/
protected async _applyUserActionsAsSystem(actions: UserAction[]): Promise<ApplyUAResult> {
return this._applyUserActions(makeExceptionalDocSession('system'), actions, {});
}
/**
* Applies an array of user actions to the sandbox and broadcasts the results to doc's clients.
*
@ -2028,14 +2038,12 @@ export class ActiveDoc extends EventEmitter {
* isModification: true if document was changed by one or more actions.
* }
*/
@ActiveDoc.keepDocOpen
protected async _applyUserActions(docSession: OptDocSession, actions: UserAction[],
options: ApplyUAExtendedOptions = {}): Promise<ApplyUAResult> {
const client = docSession.client;
this._log.debug(docSession, "_applyUserActions(%s, %s)%s", client, shortDesc(actions),
options.parseStrings ? ' (will parse)' : '');
this._inactivityTimer.ping(); // The doc is in active use; ping it to stay open longer.
if (options.parseStrings) {
actions = actions.map(ua => parseUserAction(ua, this.docData!));
@ -2159,6 +2167,7 @@ export class ActiveDoc extends EventEmitter {
this._log.debug(docSession, "shutdown complete");
}
@ActiveDoc.keepDocOpen
private async _applyUserActionsWithExtendedOptions(docSession: OptDocSession, actions: UserAction[],
options?: ApplyUAExtendedOptions): Promise<ApplyUAResult> {
assert(Array.isArray(actions), "`actions` parameter should be an array.");
@ -2167,7 +2176,7 @@ export class ActiveDoc extends EventEmitter {
await this.waitForInitialization();
if (
this.dataLimitStatus === "deleteOnly" &&
this.dataLimitInfo.status === "deleteOnly" &&
!actions.every(action => [
'RemoveTable', 'RemoveColumn', 'RemoveRecord', 'BulkRemoveRecord',
'RemoveViewSection', 'RemoveView', 'ApplyUndoActions', 'RespondToRequests',
@ -2222,6 +2231,20 @@ export class ActiveDoc extends EventEmitter {
};
}
/**
* Update the time in formulas; this is called via Interval every hour.
*/
private async _updateCurrentTime() {
const dataEngine = await this._getEngine();
if (dataEngine.isProcessDown()) {
// Don't attempt to update time if data engine is down, as this can't help, and leads to
// spurious errors. Instead, report as a warning, more clearly and concisely.
this._log.warn(null, 'failed to update current time: data engine is down');
return;
}
return this._applyUserActionsAsSystem([['UpdateCurrentTime']]);
}
/**
* Applies all metrics from `usage` to the current document usage state.
*
@ -2230,13 +2253,13 @@ export class ActiveDoc extends EventEmitter {
*/
private async _updateDocUsage(usage: Partial<DocumentUsage>, options: UpdateUsageOptions = {}) {
const {syncUsageToDatabase = true, broadcastUsageToClients = true} = options;
const oldStatus = this.dataLimitStatus;
const oldStatus = this.dataLimitInfo.status;
this._docUsage = {...(this._docUsage || {}), ...usage};
if (syncUsageToDatabase) {
/* If status decreased, we'll update usage in the database with minimal delay, so site usage
* banners show up-to-date statistics. If status increased or stayed the same, we'll schedule
* a delayed update, since it's less critical for banners to update immediately. */
const didStatusDecrease = getSeverity(this.dataLimitStatus) < getSeverity(oldStatus);
const didStatusDecrease = getSeverity(this.dataLimitInfo.status) < getSeverity(oldStatus);
this._syncDocUsageToDatabase(didStatusDecrease);
}
if (broadcastUsageToClients) {
@ -2462,7 +2485,7 @@ export class ActiveDoc extends EventEmitter {
// Calculations are not associated specifically with the user opening the document.
// TODO: be careful with which users can create formulas.
await this._applyUserActions(makeExceptionalDocSession('system'), [['Calculate']]);
await this._applyUserActionsAsSystem([['Calculate']]);
await this._reportDataEngineMemory();
}

2
app/server/lib/ActiveDocImport.ts
View File

@ -382,7 +382,7 @@ export class ActiveDocImport {
* @param {String} tmpPath: The path from of the original file.
* @param {FileImportOptions} importOptions: File import options.
* @returns {Promise<ImportResult>} with `options` property containing parseOptions as serialized JSON as adjusted
* or guessed by the plugin, and `tables`, which is which is a list of objects with information about
* or guessed by the plugin, and `tables`, which is a list of objects with information about
* tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`.
*/
private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string,

4
app/server/lib/Assistance.ts
View File

@ -148,8 +148,8 @@ class RetryableError extends Error {
* An optional ASSISTANT_MAX_TOKENS can be specified.
*/
export class OpenAIAssistant implements Assistant {
public static DEFAULT_MODEL = "gpt-3.5-turbo-0613";
public static DEFAULT_LONGER_CONTEXT_MODEL = "gpt-3.5-turbo-16k-0613";
public static DEFAULT_MODEL = "gpt-4o-2024-08-06";
public static DEFAULT_LONGER_CONTEXT_MODEL = "";
private _apiKey?: string;
private _model?: string;

40
app/server/lib/AuditLogger.ts Normal file
View File

@ -0,0 +1,40 @@
import {AuditEventDetails, AuditEventName} from 'app/common/AuditEvent';
import {RequestOrSession} from 'app/server/lib/requestUtils';
export interface IAuditLogger {
/**
* Logs an audit event.
*/
logEvent<Name extends AuditEventName>(
requestOrSession: RequestOrSession,
props: AuditEventProperties<Name>
): void;
/**
* Asynchronous variant of `logEvent`.
*
* Throws on failure to log an event.
*/
logEventAsync<Name extends AuditEventName>(
requestOrSession: RequestOrSession,
props: AuditEventProperties<Name>
): Promise<void>;
}
export interface AuditEventProperties<Name extends AuditEventName> {
event: {
/**
* The event name.
*/
name: Name;
/**
* Additional event details.
*/
details?: AuditEventDetails[Name];
};
/**
* ISO 8601 timestamp (e.g. `2024-09-04T14:54:50Z`) of when the event occured.
*
* Defaults to now.
*/
timestamp?: string;
}

25
app/server/lib/DocApi.ts
View File

@ -972,11 +972,12 @@ export class DocWorkerApi {
// Reload a document forcibly (in fact this closes the doc, it will be automatically
// reopened on use).
this._app.post('/api/docs/:docId/force-reload', canEdit, throttled(async (req, res) => {
const activeDoc = await this._getActiveDoc(req);
this._app.post('/api/docs/:docId/force-reload', canEdit, async (req, res) => {
const mreq = req as RequestWithLogin;
const activeDoc = await this._getActiveDoc(mreq);
await activeDoc.reloadDoc();
res.json(null);
}));
});
this._app.post('/api/docs/:docId/recover', canEdit, throttled(async (req, res) => {
const recoveryModeRaw = req.body.recoveryMode;
@ -1729,6 +1730,12 @@ export class DocWorkerApi {
docIdDigest: docId,
},
});
this._grist.getAuditLogger().logEvent(req as RequestWithLogin, {
event: {
name: 'createDocument',
details: {id: docId},
},
});
return docId;
}
@ -1736,6 +1743,7 @@ export class DocWorkerApi {
userId: number,
browserSettings?: BrowserSettings,
}): Promise<string> {
const mreq = req as RequestWithLogin;
const {userId, browserSettings} = options;
const isAnonymous = isAnonymousUser(req);
const result = makeForkIds({
@ -1746,10 +1754,7 @@ export class DocWorkerApi {
});
const docId = result.docId;
await this._docManager.createNamedDoc(
makeExceptionalDocSession('nascent', {
req: req as RequestWithLogin,
browserSettings,
}),
makeExceptionalDocSession('nascent', {req: mreq, browserSettings}),
docId
);
this._logDocumentCreatedTelemetryEvent(req, {
@ -1766,6 +1771,12 @@ export class DocWorkerApi {
docIdDigest: docId,
},
});
this._grist.getAuditLogger().logEvent(mreq, {
event: {
name: 'createDocument',
details: {id: docId},
},
});
return docId;
}

6
app/server/lib/DocManager.ts
View File

@ -254,6 +254,12 @@ export class DocManager extends EventEmitter {
isSaved: workspaceId !== undefined,
},
}, telemetryMetadata));
this.gristServer.getAuditLogger().logEvent(mreq, {
event: {
name: 'createDocument',
details: {id: docCreationInfo.id},
},
});
return docCreationInfo;
// The imported document is associated with the worker that did the import.

5
app/server/lib/DocStorageManager.ts
View File

@ -10,7 +10,6 @@ import {DocumentUsage} from 'app/common/DocUsage';
import * as gutil from 'app/common/gutil';
import {Comm} from 'app/server/lib/Comm';
import * as docUtils from 'app/server/lib/docUtils';
import {GristServer} from 'app/server/lib/GristServer';
import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
import {IShell} from 'app/server/lib/IShell';
import log from 'app/server/lib/log';
@ -39,10 +38,10 @@ export class DocStorageManager implements IDocStorageManager {
* The file watcher is created if the optComm argument is given.
*/
constructor(private _docsRoot: string, private _samplesRoot?: string,
private _comm?: Comm, gristServer?: GristServer) {
private _comm?: Comm, shell?: IShell) {
// If we have a way to communicate with clients, watch the docsRoot for changes.
this._watcher = null;
this._shell = gristServer?.create.Shell?.() || {
this._shell = shell ?? {
trashItem() { throw new Error('Unable to move document to trash'); },
showItemInFolder() { throw new Error('Unable to show item in folder'); }
};

9
app/server/lib/ExternalStorage.ts
View File

@ -377,6 +377,15 @@ export interface ExternalStorageSettings {
extraPrefix?: string;
}
/**
* Function returning the core ExternalStorage implementation,
* which may then be wrapped in additional layer(s) of ExternalStorage.
* See ICreate.ExternalStorage.
* Uses S3 by default in hosted Grist.
*/
export type ExternalStorageCreator =
(purpose: ExternalStorageSettings["purpose"], extraPrefix: string) => ExternalStorage | undefined;
/**
* The storage mapping we use for our SaaS. A reasonable default, but relies
* on appropriate lifecycle rules being set up in the bucket.

86
app/server/lib/FlexServer.ts
View File

@ -1,7 +1,6 @@
import {ApiError} from 'app/common/ApiError';
import {ICustomWidget} from 'app/common/CustomWidget';
import {delay} from 'app/common/delay';
import {DocCreationInfo} from 'app/common/DocListAPI';
import {commonUrls, encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
GristLoadConfig, IGristUrlState, isOrgInPathOnly, parseSubdomain,
sanitizePathTail} from 'app/common/gristUrls';
@ -27,6 +26,7 @@ import {AccessTokens, IAccessTokens} from 'app/server/lib/AccessTokens';
import {createSandbox} from 'app/server/lib/ActiveDoc';
import {attachAppEndpoint} from 'app/server/lib/AppEndpoint';
import {appSettings} from 'app/server/lib/AppSettings';
import {IAuditLogger} from 'app/server/lib/AuditLogger';
import {addRequestUser, getTransitiveHeaders, getUser, getUserId, isAnonymousUser,
isSingleUserMode, redirectToLoginUnconditionally} from 'app/server/lib/Authorizer';
import {redirectToLogin, RequestWithLogin, signInStatusMiddleware} from 'app/server/lib/Authorizer';
@ -37,7 +37,6 @@ import {create} from 'app/server/lib/create';
import {addDiscourseConnectEndpoints} from 'app/server/lib/DiscourseConnect';
import {addDocApiRoutes} from 'app/server/lib/DocApi';
import {DocManager} from 'app/server/lib/DocManager';
import {DocStorageManager} from 'app/server/lib/DocStorageManager';
import {DocWorker} from 'app/server/lib/DocWorker';
import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap';
@ -46,13 +45,11 @@ import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth";
import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer,
RequestWithGrist} from 'app/server/lib/GristServer';
import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions';
import {HostedStorageManager} from 'app/server/lib/HostedStorageManager';
import {IBilling} from 'app/server/lib/IBilling';
import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier';
import {InstallAdmin} from 'app/server/lib/InstallAdmin';
import log from 'app/server/lib/log';
import {getLoginSystem} from 'app/server/lib/logins';
import {IPermitStore} from 'app/server/lib/Permit';
import {getAppPathTo, getAppRoot, getInstanceRoot, getUnpackedAppRoot} from 'app/server/lib/places';
import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint';
@ -151,6 +148,7 @@ export class FlexServer implements GristServer {
private _sessions: Sessions;
private _sessionStore: SessionStore;
private _storageManager: IDocStorageManager;
private _auditLogger: IAuditLogger;
private _telemetry: ITelemetry;
private _processMonitorStop?: () => void; // Callback to stop the ProcessMonitor
private _docWorkerMap: IDocWorkerMap;
@ -183,7 +181,7 @@ export class FlexServer implements GristServer {
private _getSignUpRedirectUrl: (req: express.Request, target: URL) => Promise<string>;
private _getLogoutRedirectUrl: (req: express.Request, nextUrl: URL) => Promise<string>;
private _sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>;
private _getLoginSystem?: () => Promise<GristLoginSystem>;
private _getLoginSystem: () => Promise<GristLoginSystem>;
// Set once ready() is called
private _isReady: boolean = false;
private _updateManager: UpdateManager;
@ -191,6 +189,7 @@ export class FlexServer implements GristServer {
constructor(public port: number, public name: string = 'flexServer',
public readonly options: FlexServerOptions = {}) {
this._getLoginSystem = create.getLoginSystem;
this.settings = options.settings;
this.app = express();
this.app.set('port', port);
@ -248,7 +247,6 @@ export class FlexServer implements GristServer {
recentItems: [],
};
this.electronServerMethods = {
async importDoc() { throw new Error('not implemented'); },
onDocOpen(cb) {
// currently only a stub.
cb('');
@ -270,11 +268,6 @@ export class FlexServer implements GristServer {
});
}
// Allow overridding the login system.
public setLoginSystem(loginSystem: () => Promise<GristLoginSystem>) {
this._getLoginSystem = loginSystem;
}
public getHost(): string {
return `${this.host}:${this.getOwnPort()}`;
}
@ -398,6 +391,16 @@ export class FlexServer implements GristServer {
return this._storageManager;
}
public getAuditLogger(): IAuditLogger {
if (!this._auditLogger) { throw new Error('no audit logger available'); }
return this._auditLogger;
}
public getDocManager(): DocManager {
if (!this._docManager) { throw new Error('no document manager available'); }
return this._docManager;
}
public getTelemetry(): ITelemetry {
if (!this._telemetry) { throw new Error('no telemetry available'); }
return this._telemetry;
@ -442,24 +445,33 @@ export class FlexServer implements GristServer {
public addLogging() {
if (this._check('logging')) { return; }
if (process.env.GRIST_LOG_SKIP_HTTP) { return; }
if (!this._httpLoggingEnabled()) { return; }
// Add a timestamp token that matches exactly the formatting of non-morgan logs.
morganLogger.token('logTime', (req: Request) => log.timestamp());
// Add an optional gristInfo token that can replace the url, if the url is sensitive.
morganLogger.token('gristInfo', (req: RequestWithGristInfo) =>
req.gristInfo || req.originalUrl || req.url);
morganLogger.token('host', (req: express.Request) => req.get('host'));
const msg = ':logTime :host :method :gristInfo :status :response-time ms - :res[content-length]';
morganLogger.token('body', (req: express.Request) =>
req.is('application/json') ? JSON.stringify(req.body) : undefined
);
// For debugging, be careful not to enable logging in production (may log sensitive data)
const shouldLogBody = isAffirmative(process.env.GRIST_LOG_HTTP_BODY);
const msg = `:logTime :host :method :gristInfo ${shouldLogBody ? ':body ' : ''}` +
":status :response-time ms - :res[content-length]";
// In hosted Grist, render json so logs retain more organization.
function outputJson(tokens: any, req: any, res: any) {
return JSON.stringify({
timestamp: tokens.logTime(req, res),
host: tokens.host(req, res),
method: tokens.method(req, res),
path: tokens.gristInfo(req, res),
...(shouldLogBody ? { body: tokens.body(req, res) } : {}),
status: tokens.status(req, res),
timeMs: parseFloat(tokens['response-time'](req, res)) || undefined,
contentLength: parseInt(tokens.res(req, res, 'content-length'), 10) || undefined,
host: tokens.host(req, res),
altSessionId: req.altSessionId,
});
}
@ -902,6 +914,12 @@ export class FlexServer implements GristServer {
});
}
public addAuditLogger() {
if (this._check('audit-logger')) { return; }
this._auditLogger = this.create.AuditLogger();
}
public async addTelemetry() {
if (this._check('telemetry', 'homedb', 'json', 'api-mw')) { return; }
@ -1319,12 +1337,15 @@ export class FlexServer implements GristServer {
const workers = this._docWorkerMap;
const docWorkerId = await this._addSelfAsWorker(workers);
const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableExternalStorage, workers,
this._dbManager, this.create);
const storageManager = await this.create.createHostedDocStorageManager(
this.docsRoot, docWorkerId, this._disableExternalStorage, workers, this._dbManager, this.create.ExternalStorage
);
this._storageManager = storageManager;
} else {
const samples = getAppPathTo(this.appRoot, 'public_samples');
const storageManager = new DocStorageManager(this.docsRoot, samples, this._comm, this);
const storageManager = await this.create.createLocalDocStorageManager(
this.docsRoot, samples, this._comm, this.create.Shell?.()
);
this._storageManager = storageManager;
}
@ -1990,8 +2011,7 @@ export class FlexServer implements GristServer {
public resolveLoginSystem() {
return isTestLoginAllowed() ?
getTestLoginSystem() :
(this._getLoginSystem?.() || getLoginSystem());
getTestLoginSystem() : this._getLoginSystem();
}
public addUpdatesCheck() {
@ -2489,6 +2509,33 @@ export class FlexServer implements GristServer {
[];
return [...pluggedMiddleware, sessionClearMiddleware];
}
/**
* Returns true if GRIST_LOG_HTTP="true" (or any truthy value).
* Returns true if GRIST_LOG_SKIP_HTTP="" (empty string).
* Returns false otherwise.
*
* Also displays a deprecation warning if GRIST_LOG_SKIP_HTTP is set to any value ("", "true", whatever...),
* and throws an exception if GRIST_LOG_SKIP_HTTP and GRIST_LOG_HTTP are both set to make the server crash.
*/
private _httpLoggingEnabled(): boolean {
const deprecatedOptionEnablesLog = process.env.GRIST_LOG_SKIP_HTTP === '';
const isGristLogHttpEnabled = isAffirmative(process.env.GRIST_LOG_HTTP);
if (process.env.GRIST_LOG_HTTP !== undefined && process.env.GRIST_LOG_SKIP_HTTP !== undefined) {
throw new Error('Both GRIST_LOG_HTTP and GRIST_LOG_SKIP_HTTP are set. ' +
'Please remove GRIST_LOG_SKIP_HTTP and set GRIST_LOG_HTTP to the value you actually want.');
}
if (process.env.GRIST_LOG_SKIP_HTTP !== undefined) {
const expectedGristLogHttpVal = deprecatedOptionEnablesLog ? "true" : "false";
log.warn(`Setting env variable GRIST_LOG_SKIP_HTTP="${process.env.GRIST_LOG_SKIP_HTTP}" `
+ `is deprecated in favor of GRIST_LOG_HTTP="${expectedGristLogHttpVal}"`);
}
return isGristLogHttpEnabled || deprecatedOptionEnablesLog;
}
}
/**
@ -2560,7 +2607,6 @@ function noCaching(req: express.Request, res: express.Response, next: express.Ne
// Methods that Electron app relies on.
export interface ElectronServerMethods {
importDoc(filepath: string): Promise<DocCreationInfo>;
onDocOpen(cb: (filePath: string) => void): void;
getUserConfig(): Promise<any>;
updateUserConfig(obj: any): Promise<void>;

25
app/server/lib/GranularAccess.ts
View File

@ -160,6 +160,7 @@ const OTHER_RECOGNIZED_ACTIONS = new Set([
'RemoveColumn',
'RenameColumn',
'ModifyColumn',
'AddReverseColumn',
// Table-level schema changes.
'AddEmptyTable',
@ -792,7 +793,7 @@ export class GranularAccess implements GranularAccessForBundle {
}
const role = options.role ?? await this.getNominalAccess(docSession);
const hasEditRole = canEdit(role);
if (!hasEditRole) { result.dataLimitStatus = null; }
if (!hasEditRole) { result.dataLimitInfo.status = null; }
const hasFullReadAccess = await this.canReadEverything(docSession);
if (!hasEditRole || !hasFullReadAccess) {
result.rowCount = 'hidden';
@ -913,14 +914,20 @@ export class GranularAccess implements GranularAccessForBundle {
*/
public needEarlySchemaPermission(a: UserAction|DocAction): boolean {
const name = a[0] as string;
if (name === 'ModifyColumn' || name === 'SetDisplayFormula' ||
// ConvertFromColumn and CopyFromColumn are hard to reason
// about, especially since they appear in bundles with other
// actions. We throw up our hands a bit here, and just make
// sure the user has schema permissions. Today, in Grist, that
// gives a lot of power. If this gets narrowed down in future,
// we'll have to rethink this.
name === 'ConvertFromColumn' || name === 'CopyFromColumn') {
// ConvertFromColumn and CopyFromColumn are hard to reason
// about, especially since they appear in bundles with other
// actions. We throw up our hands a bit here, and just make
// sure the user has schema permissions. Today, in Grist, that
// gives a lot of power. If this gets narrowed down in future,
// we'll have to rethink this.
const actionNames = [
'ModifyColumn',
'SetDisplayFormula',
'ConvertFromColumn',
'CopyFromColumn',
'AddReverseColumn',
];
if (actionNames.includes(name)) {
return true;
} else if (isDataAction(a)) {
const tableId = getTableId(a);

9
app/server/lib/GristAuditLogger.ts Normal file
View File

@ -0,0 +1,9 @@
import {AuditEvent, AuditEventName} from 'app/common/AuditEvent';
import {IAuditLogger} from 'app/server/lib/AuditLogger';
import {HTTPAuditLogger} from 'app/server/lib/HTTPAuditLogger';
export class GristAuditLogger extends HTTPAuditLogger implements IAuditLogger {
protected toJSON<Name extends AuditEventName>(event: AuditEvent<Name>): string {
return JSON.stringify(event);
}
}

10
app/server/lib/GristServer.ts
View File

@ -9,6 +9,7 @@ import { User } from 'app/gen-server/entity/User';
import { Workspace } from 'app/gen-server/entity/Workspace';
import { Activations } from 'app/gen-server/lib/Activations';
import { HomeDBManager } from 'app/gen-server/lib/homedb/HomeDBManager';
import { IAuditLogger } from 'app/server/lib/AuditLogger';
import { IAccessTokens } from 'app/server/lib/AccessTokens';
import { RequestWithLogin } from 'app/server/lib/Authorizer';
import { Comm } from 'app/server/lib/Comm';
@ -54,6 +55,7 @@ export interface GristServer {
getInstallAdmin(): InstallAdmin;
getHomeDBManager(): HomeDBManager;
getStorageManager(): IDocStorageManager;
getAuditLogger(): IAuditLogger;
getTelemetry(): ITelemetry;
hasNotifier(): boolean;
getNotifier(): INotifier;
@ -147,6 +149,7 @@ export function createDummyGristServer(): GristServer {
getInstallAdmin() { throw new Error('no install admin'); },
getHomeDBManager() { throw new Error('no db'); },
getStorageManager() { throw new Error('no storage manager'); },
getAuditLogger() { return createDummyAuditLogger(); },
getTelemetry() { return createDummyTelemetry(); },
getNotifier() { throw new Error('no notifier'); },
hasNotifier() { return false; },
@ -165,6 +168,13 @@ export function createDummyGristServer(): GristServer {
};
}
export function createDummyAuditLogger(): IAuditLogger {
return {
logEvent() { /* do nothing */ },
logEventAsync() { return Promise.resolve(); },
};
}
export function createDummyTelemetry(): ITelemetry {
return {
addEndpoints() { /* do nothing */ },

135
app/server/lib/HTTPAuditLogger.ts Normal file
View File

@ -0,0 +1,135 @@
import {AuditEvent, AuditEventName, AuditEventUser} from 'app/common/AuditEvent';
import {AuditEventProperties, IAuditLogger} from 'app/server/lib/AuditLogger';
import {getDocSessionUser} from 'app/server/lib/DocSession';
import {ILogMeta, LogMethods} from 'app/server/lib/LogMethods';
import {RequestOrSession} from 'app/server/lib/requestUtils';
import {getLogMetaFromDocSession} from 'app/server/lib/serverUtils';
import moment from 'moment-timezone';
import fetch from 'node-fetch';
interface HTTPAuditLoggerOptions {
/**
* The HTTP endpoint to send audit events to.
*/
endpoint: string;
/**
* If set, the value to include in the `Authorization` header of each
* request to `endpoint`.
*/
authorizationHeader?: string;
}
const MAX_PENDING_REQUESTS = 25;
/**
* Base class for an audit event logger that logs events by sending them to an JSON-based HTTP
* endpoint.
*
* Subclasses are expected to provide a suitable `toJSON` implementation to handle serialization
* of audit events to JSON.
*
* See `GristAuditLogger` for an example.
*/
export abstract class HTTPAuditLogger implements IAuditLogger {
private _endpoint = this._options.endpoint;
private _authorizationHeader = this._options.authorizationHeader;
private _numPendingRequests = 0;
private readonly _logger = new LogMethods('AuditLogger ', (requestOrSession: RequestOrSession | undefined) =>
getLogMeta(requestOrSession));
constructor(private _options: HTTPAuditLoggerOptions) {}
/**
* Logs an audit event.
*/
public logEvent<Name extends AuditEventName>(
requestOrSession: RequestOrSession,
event: AuditEventProperties<Name>
): void {
this._logEventOrThrow(requestOrSession, event)
.catch((e) => this._logger.error(requestOrSession, `failed to log audit event`, event, e));
}
/**
* Asynchronous variant of `logEvent`.
*
* Throws on failure to log an event.
*/
public async logEventAsync<Name extends AuditEventName>(
requestOrSession: RequestOrSession,
event: AuditEventProperties<Name>
): Promise<void> {
await this._logEventOrThrow(requestOrSession, event);
}
/**
* Serializes an audit event to JSON.
*/
protected abstract toJSON<Name extends AuditEventName>(event: AuditEvent<Name>): string;
private async _logEventOrThrow<Name extends AuditEventName>(
requestOrSession: RequestOrSession,
{event: {name, details}, timestamp}: AuditEventProperties<Name>
) {
if (this._numPendingRequests === MAX_PENDING_REQUESTS) {
throw new Error(`exceeded the maximum number of pending audit event calls (${MAX_PENDING_REQUESTS})`);
}
try {
this._numPendingRequests += 1;
const resp = await fetch(this._endpoint, {
method: 'POST',
headers: {
...(this._authorizationHeader ? {'Authorization': this._authorizationHeader} : undefined),
'Content-Type': 'application/json',
},
body: this.toJSON({
event: {
name,
user: getAuditEventUser(requestOrSession),
details: details ?? null,
},
timestamp: timestamp ?? moment().toISOString(),
}),
});
if (!resp.ok) {
throw new Error(`received a non-200 response from ${resp.url}: ${resp.status} ${await resp.text()}`);
}
} finally {
this._numPendingRequests -= 1;
}
}
}
function getAuditEventUser(requestOrSession: RequestOrSession): AuditEventUser | null {
if (!requestOrSession) { return null; }
if ('get' in requestOrSession) {
return {
id: requestOrSession.userId ?? null,
email: requestOrSession.user?.loginEmail ?? null,
name: requestOrSession.user?.name ?? null,
};
} else {
const user = getDocSessionUser(requestOrSession);
if (!user) { return null; }
const {id, email, name} = user;
return {id, email, name};
}
}
function getLogMeta(requestOrSession?: RequestOrSession): ILogMeta {
if (!requestOrSession) { return {}; }
if ('get' in requestOrSession) {
return {
org: requestOrSession.org,
email: requestOrSession.user?.loginEmail,
userId: requestOrSession.userId,
altSessionId: requestOrSession.altSessionId,
};
} else {
return getLogMetaFromDocSession(requestOrSession);
}
}

18
app/server/lib/HostedStorageManager.ts
View File

@ -12,9 +12,14 @@ import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
import {checksumFile} from 'app/server/lib/checksumFile';
import {DocSnapshotInventory, DocSnapshotPruner} from 'app/server/lib/DocSnapshots';
import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {ChecksummedExternalStorage, DELETED_TOKEN, ExternalStorage, Unchanged} from 'app/server/lib/ExternalStorage';
import {
ChecksummedExternalStorage,
DELETED_TOKEN,
ExternalStorage,
ExternalStorageCreator, ExternalStorageSettings,
Unchanged
} from 'app/server/lib/ExternalStorage';
import {HostedMetadataManager} from 'app/server/lib/HostedMetadataManager';
import {ICreate} from 'app/server/lib/ICreate';
import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
import {LogMethods} from "app/server/lib/LogMethods";
import {fromCallback} from 'app/server/lib/serverUtils';
@ -51,11 +56,6 @@ export interface HostedStorageOptions {
secondsBeforePush: number;
secondsBeforeFirstRetry: number;
pushDocUpdateTimes: boolean;
// A function returning the core ExternalStorage implementation,
// which may then be wrapped in additional layer(s) of ExternalStorage.
// See ICreate.ExternalStorage.
// Uses S3 by default in hosted Grist.
externalStorageCreator?: (purpose: 'doc'|'meta') => ExternalStorage;
}
const defaultOptions: HostedStorageOptions = {
@ -134,10 +134,10 @@ export class HostedStorageManager implements IDocStorageManager {
private _disableS3: boolean,
private _docWorkerMap: IDocWorkerMap,
dbManager: HomeDBManager,
create: ICreate,
createExternalStorage: ExternalStorageCreator,
options: HostedStorageOptions = defaultOptions
) {
const creator = options.externalStorageCreator || ((purpose) => create.ExternalStorage(purpose, ''));
const creator = ((purpose: ExternalStorageSettings['purpose']) => createExternalStorage(purpose, ''));
// We store documents either in a test store, or in an s3 store
// at s3://<s3Bucket>/<s3Prefix><docId>.grist
const externalStoreDoc = this._disableS3 ? undefined : creator('doc');

77
app/server/lib/ICreate.ts
View File

@ -1,9 +1,11 @@
import {GristDeploymentType} from 'app/common/gristUrls';
import {getCoreLoginSystem} from 'app/server/lib/coreLogins';
import {getThemeBackgroundSnippet} from 'app/common/Themes';
import {Document} from 'app/gen-server/entity/Document';
import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
import {ExternalStorage} from 'app/server/lib/ExternalStorage';
import {createDummyTelemetry, GristServer} from 'app/server/lib/GristServer';
import {IAuditLogger} from 'app/server/lib/AuditLogger';
import {ExternalStorage, ExternalStorageCreator} from 'app/server/lib/ExternalStorage';
import {createDummyAuditLogger, createDummyTelemetry, GristLoginSystem, GristServer} from 'app/server/lib/GristServer';
import {IBilling} from 'app/server/lib/IBilling';
import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier';
import {InstallAdmin, SimpleInstallAdmin} from 'app/server/lib/InstallAdmin';
@ -12,6 +14,11 @@ import {IShell} from 'app/server/lib/IShell';
import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox';
import {SqliteVariant} from 'app/server/lib/SqliteCommon';
import {ITelemetry} from 'app/server/lib/Telemetry';
import {IDocStorageManager} from './IDocStorageManager';
import { Comm } from "./Comm";
import { IDocWorkerMap } from "./DocWorkerMap";
import { HostedStorageManager, HostedStorageOptions } from "./HostedStorageManager";
import { DocStorageManager } from "./DocStorageManager";
// In the past, the session secret was used as an additional
// protection passed on to expressjs-session for security when
@ -36,19 +43,36 @@ import {ITelemetry} from 'app/server/lib/Telemetry';
export const DEFAULT_SESSION_SECRET =
'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh';
export type LocalDocStorageManagerCreator =
(docsRoot: string, samplesRoot?: string, comm?: Comm, shell?: IShell) => Promise<IDocStorageManager>;
export type HostedDocStorageManagerCreator = (
docsRoot: string,
docWorkerId: string,
disableS3: boolean,
docWorkerMap: IDocWorkerMap,
dbManager: HomeDBManager,
createExternalStorage: ExternalStorageCreator,
options?: HostedStorageOptions
) => Promise<IDocStorageManager>;
export interface ICreate {
Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry;
Shell?(): IShell; // relevant to electron version of Grist only.
// Create a space to store files externally, for storing either:
// - documents. This store should be versioned, and can be eventually consistent.
// - meta. This store need not be versioned, and can be eventually consistent.
// For test purposes an extra prefix may be supplied. Stores with different prefixes
// should not interfere with each other.
ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage | undefined;
ExternalStorage: ExternalStorageCreator;
// Creates a IDocStorageManager for storing documents on the local machine.
createLocalDocStorageManager: LocalDocStorageManagerCreator;
// Creates a IDocStorageManager for storing documents on an external storage (e.g S3)
createHostedDocStorageManager: HostedDocStorageManagerCreator;
Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
AuditLogger(): IAuditLogger;
Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry;
Shell?(): IShell; // relevant to electron version of Grist only.
NSandbox(options: ISandboxCreationOptions): ISandbox;
@ -67,6 +91,8 @@ export interface ICreate {
getStorageOptions?(name: string): ICreateStorageOptions|undefined;
getSqliteVariant?(): SqliteVariant;
getSandboxVariants?(): Record<string, SpawnFn>;
getLoginSystem(): Promise<GristLoginSystem>;
}
export interface ICreateActiveDocOptions {
@ -91,6 +117,12 @@ export interface ICreateBillingOptions {
create(dbManager: HomeDBManager, gristConfig: GristServer): IBilling|undefined;
}
export interface ICreateAuditLoggerOptions {
name: 'grist'|'hec';
check(): boolean;
create(): IAuditLogger|undefined;
}
export interface ICreateTelemetryOptions {
create(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry|undefined;
}
@ -110,6 +142,7 @@ export function makeSimpleCreator(opts: {
storage?: ICreateStorageOptions[],
billing?: ICreateBillingOptions,
notifier?: ICreateNotifierOptions,
auditLogger?: ICreateAuditLoggerOptions[],
telemetry?: ICreateTelemetryOptions,
sandboxFlavor?: string,
shell?: IShell,
@ -117,8 +150,11 @@ export function makeSimpleCreator(opts: {
getSqliteVariant?: () => SqliteVariant,
getSandboxVariants?: () => Record<string, SpawnFn>,
createInstallAdmin?: (dbManager: HomeDBManager) => Promise<InstallAdmin>,
getLoginSystem?: () => Promise<GristLoginSystem>,
createHostedDocStorageManager?: HostedDocStorageManagerCreator,
createLocalDocStorageManager?: LocalDocStorageManagerCreator,
}): ICreate {
const {deploymentType, sessionSecret, storage, notifier, billing, telemetry} = opts;
const {deploymentType, sessionSecret, storage, notifier, billing, auditLogger, telemetry} = opts;
return {
deploymentType() { return deploymentType; },
Billing(dbManager, gristConfig) {
@ -141,6 +177,9 @@ export function makeSimpleCreator(opts: {
}
return undefined;
},
AuditLogger() {
return auditLogger?.find(({check}) => check())?.create() ?? createDummyAuditLogger();
},
Telemetry(dbManager, gristConfig) {
return telemetry?.create(dbManager, gristConfig) ?? createDummyTelemetry();
},
@ -187,5 +226,23 @@ export function makeSimpleCreator(opts: {
getSqliteVariant: opts.getSqliteVariant,
getSandboxVariants: opts.getSandboxVariants,
createInstallAdmin: opts.createInstallAdmin || (async (dbManager) => new SimpleInstallAdmin(dbManager)),
getLoginSystem: opts.getLoginSystem || getCoreLoginSystem,
createLocalDocStorageManager: opts.createLocalDocStorageManager ?? createDefaultLocalStorageManager,
createHostedDocStorageManager: opts.createHostedDocStorageManager ?? createDefaultHostedStorageManager,
};
}
const createDefaultHostedStorageManager: HostedDocStorageManagerCreator = async (
docsRoot,
docWorkerId,
disableS3,
docWorkerMap,
dbManager,
createExternalStorage, options
) =>
new HostedStorageManager(docsRoot, docWorkerId, disableS3, docWorkerMap, dbManager, createExternalStorage, options);
const createDefaultLocalStorageManager: LocalDocStorageManagerCreator = async (
docsRoot, samplesRoot, comm, shell
) => new DocStorageManager(docsRoot, samplesRoot, comm, shell);

1
app/server/lib/ISandbox.ts
View File

@ -27,6 +27,7 @@ export interface ISandbox {
pyCall(funcName: string, ...varArgs: unknown[]): Promise<any>;
reportMemoryUsage(): Promise<void>;
getFlavor(): string;
isProcessDown(): boolean;
}
export interface ISandboxCreator {

2
app/server/lib/InstallAdmin.ts
View File

@ -18,7 +18,7 @@ export abstract class InstallAdmin {
// the Grist installation. This should not fail, only return true or false.
public async isAdminReq(req: express.Request): Promise<boolean> {
const user = (req as RequestWithLogin).user;
return user ? this.isAdminUser(user) : false;
return user ? (await this.isAdminUser(user)) : false;
}
// Returns middleware that fails unless the request includes an authenticated user and this user

4
app/server/lib/NSandbox.ts
View File

@ -230,6 +230,10 @@ export class NSandbox implements ISandbox {
log.rawDebug('Sandbox memory', {memory, ...this._logMeta});
}
public isProcessDown() {
return this._isReadClosed || this._isWriteClosed;
}
public getFlavor() {
return this._logMeta.flavor;
}

2
app/server/lib/NullSandbox.ts
View File

@ -22,4 +22,6 @@ export class NullSandbox implements ISandbox {
public getFlavor() {
return 'null';
}
public isProcessDown() { return true; }
}

12
app/server/lib/OIDCConfig.ts
View File

@ -47,7 +47,9 @@
* A JSON object with extra client metadata to pass to openid-client. Optional.
* Be aware that setting this object may override any other values passed to the openid client.
* More info: https://github.com/panva/node-openid-client/tree/main/docs#new-clientmetadata-jwks-options
*
* env GRIST_OIDC_SP_HTTP_TIMEOUT
* The timeout in milliseconds for HTTP requests to the IdP. The default value is set to 3500 by the
* openid-client library. See: https://github.com/panva/node-openid-client/blob/main/docs/README.md#customizing-http-requests
*
* This version of OIDCConfig has been tested with Keycloak OIDC IdP following the instructions
* at:
@ -66,7 +68,7 @@
import * as express from 'express';
import { GristLoginSystem, GristServer } from './GristServer';
import {
Client, ClientMetadata, Issuer, errors as OIDCError, TokenSet, UserinfoResponse
Client, ClientMetadata, custom, Issuer, errors as OIDCError, TokenSet, UserinfoResponse
} from 'openid-client';
import { Sessions } from './Sessions';
import log from 'app/server/lib/log';
@ -137,6 +139,9 @@ export class OIDCConfig {
envVar: 'GRIST_OIDC_IDP_CLIENT_SECRET',
censor: true,
});
const httpTimeout = section.flag('httpTimeout').readInt({
envVar: 'GRIST_OIDC_SP_HTTP_TIMEOUT',
});
this._namePropertyKey = section.flag('namePropertyKey').readString({
envVar: 'GRIST_OIDC_SP_PROFILE_NAME_ATTR',
});
@ -173,6 +178,9 @@ export class OIDCConfig {
this._protectionManager = new ProtectionsManager(enabledProtections);
this._redirectUrl = new URL(CALLBACK_URL, spHost).href;
custom.setHttpOptionsDefaults({
...(httpTimeout !== undefined ? {timeout: httpTimeout} : {}),
});
await this._initClient({ issuerUrl, clientId, clientSecret, extraMetadata });
if (this._client.issuer.metadata.end_session_endpoint === undefined &&

6
app/server/lib/Telemetry.ts
View File

@ -19,12 +19,12 @@ import {Activation} from 'app/gen-server/entity/Activation';
import {Activations} from 'app/gen-server/lib/Activations';
import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
import {RequestWithLogin} from 'app/server/lib/Authorizer';
import {getDocSessionUser, OptDocSession} from 'app/server/lib/DocSession';
import {getDocSessionUser} from 'app/server/lib/DocSession';
import {expressWrap} from 'app/server/lib/expressWrap';
import {GristServer} from 'app/server/lib/GristServer';
import {hashId} from 'app/server/lib/hashingUtils';
import {LogMethods} from 'app/server/lib/LogMethods';
import {stringParam} from 'app/server/lib/requestUtils';
import {RequestOrSession, stringParam} from 'app/server/lib/requestUtils';
import {getLogMetaFromDocSession} from 'app/server/lib/serverUtils';
import * as cookie from 'cookie';
import * as express from 'express';
@ -32,8 +32,6 @@ import fetch from 'node-fetch';
import merge = require('lodash/merge');
import pickBy = require('lodash/pickBy');
type RequestOrSession = RequestWithLogin | OptDocSession | null;
interface RequestWithMatomoVisitorId extends RequestWithLogin {
/**
* Extracted from a cookie set by Matomo.

6
app/server/lib/TestLogin.ts
View File

@ -25,10 +25,8 @@ export async function getTestLoginSystem(): Promise<GristLoginSystem> {
if (process.env.TEST_SUPPORT_API_KEY) {
const dbManager = gristServer.getHomeDBManager();
const user = await dbManager.getUserByLogin(SUPPORT_EMAIL);
if (user) {
user.apiKey = process.env.TEST_SUPPORT_API_KEY;
await user.save();
}
user.apiKey = process.env.TEST_SUPPORT_API_KEY;
await user.save();
}
return "test-login";
},

30
app/server/lib/configureGristAuditLogger.ts Normal file
View File

@ -0,0 +1,30 @@
import {appSettings} from 'app/server/lib/AppSettings';
import {GristAuditLogger} from 'app/server/lib/GristAuditLogger';
export function configureGristAuditLogger() {
const options = checkGristAuditLogger();
if (!options) { return undefined; }
return new GristAuditLogger(options);
}
export function checkGristAuditLogger() {
const settings = appSettings.section('auditLogger').section('http');
const endpoint = settings.flag('endpoint').readString({
envVar: 'GRIST_AUDIT_HTTP_ENDPOINT',
});
if (!endpoint) { return undefined; }
const payloadFormat = settings.flag('payloadFormat').readString({
envVar: 'GRIST_AUDIT_HTTP_PAYLOAD_FORMAT',
defaultValue: 'grist',
});
if (payloadFormat !== 'grist') { return undefined; }
const authorizationHeader = settings.flag('authorizationHeader').readString({
envVar: 'GRIST_AUDIT_HTTP_AUTHORIZATION_HEADER',
censor: true,
});
return {endpoint, authorizationHeader};
}

8
app/server/lib/coreCreator.ts
View File

@ -1,3 +1,4 @@
import { checkGristAuditLogger, configureGristAuditLogger } from 'app/server/lib/configureGristAuditLogger';
import { checkMinIOBucket, checkMinIOExternalStorage,
configureMinIOExternalStorage } from 'app/server/lib/configureMinIOExternalStorage';
import { makeSimpleCreator } from 'app/server/lib/ICreate';
@ -13,6 +14,13 @@ export const makeCoreCreator = () => makeSimpleCreator({
create: configureMinIOExternalStorage,
},
],
auditLogger: [
{
name: 'grist',
check: () => checkGristAuditLogger() !== undefined,
create: configureGristAuditLogger,
},
],
telemetry: {
create: (dbManager, gristServer) => new Telemetry(dbManager, gristServer),
}

14
app/server/lib/dbUtils.ts
View File

@ -45,11 +45,16 @@ export async function updateDb(connection?: Connection) {
await synchronizeProducts(connection, true);
}
export function getConnectionName() {
return process.env.TYPEORM_NAME || 'default';
}
/**
* Get a connection to db if one exists, or create one. Serialized to
* avoid duplication.
*/
const connectionMutex = new Mutex();
export async function getOrCreateConnection(): Promise<Connection> {
return connectionMutex.runExclusive(async() => {
try {
@ -85,9 +90,7 @@ export async function runMigrations(connection: Connection) {
// transaction, or it has no effect.
const sqlite = connection.driver.options.type === 'sqlite';
if (sqlite) { await connection.query("PRAGMA foreign_keys = OFF;"); }
await connection.transaction(async tr => {
await tr.connection.runMigrations();
});
await connection.runMigrations({ transaction: "all" });
if (sqlite) { await connection.query("PRAGMA foreign_keys = ON;"); }
}
@ -103,7 +106,7 @@ export async function undoLastMigration(connection: Connection) {
// Replace the old janky ormconfig.js file, which was always a source of
// pain to use since it wasn't properly integrated into the typescript
// project.
export function getTypeORMSettings(): DataSourceOptions {
export function getTypeORMSettings(overrideConf?: Partial<DataSourceOptions>): DataSourceOptions {
// If we have a redis server available, tell typeorm. Then any queries built with
// .cache() called on them will be cached via redis.
// We use a separate environment variable for the moment so that we don't have to
@ -120,7 +123,7 @@ export function getTypeORMSettings(): DataSourceOptions {
} : undefined;
return {
"name": process.env.TYPEORM_NAME || "default",
"name": getConnectionName(),
"type": (process.env.TYPEORM_TYPE as any) || "sqlite", // officially, TYPEORM_CONNECTION -
// but if we use that, this file will never
// be read, and we can't configure
@ -144,5 +147,6 @@ export function getTypeORMSettings(): DataSourceOptions {
],
...JSON.parse(process.env.TYPEORM_EXTRA || "{}"),
...cache,
...overrideConf,
};
}

16
app/server/lib/initialDocSql.ts
View File

@ -6,9 +6,9 @@ export const GRIST_DOC_SQL = `
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT '');
INSERT INTO _grist_DocInfo VALUES(1,'','','',42,'','');
INSERT INTO _grist_DocInfo VALUES(1,'','','',43,'','');
CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0, "rawViewSectionRef" INTEGER DEFAULT 0, "recordCardViewSectionRef" INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "description" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "rules" TEXT DEFAULT NULL, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "description" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "rules" TEXT DEFAULT NULL, "reverseCol" INTEGER DEFAULT 0, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "_grist_Imports" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "origFileName" TEXT DEFAULT '', "parseFormula" TEXT DEFAULT '', "delimiter" TEXT DEFAULT '', "doublequote" BOOLEAN DEFAULT 0, "escapechar" TEXT DEFAULT '', "quotechar" TEXT DEFAULT '', "skipinitialspace" BOOLEAN DEFAULT 0, "encoding" TEXT DEFAULT '', "hasHeaders" BOOLEAN DEFAULT 0);
CREATE TABLE IF NOT EXISTS "_grist_External_database" (id INTEGER PRIMARY KEY, "host" TEXT DEFAULT '', "port" INTEGER DEFAULT 0, "username" TEXT DEFAULT '', "dialect" TEXT DEFAULT '', "database" TEXT DEFAULT '', "storage" TEXT DEFAULT '');
CREATE TABLE IF NOT EXISTS "_grist_External_table" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "databaseRef" INTEGER DEFAULT 0, "tableName" TEXT DEFAULT '');
@ -44,14 +44,14 @@ export const GRIST_DOC_WITH_TABLE1_SQL = `
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT '');
INSERT INTO _grist_DocInfo VALUES(1,'','','',42,'','');
INSERT INTO _grist_DocInfo VALUES(1,'','','',43,'','');
CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0, "rawViewSectionRef" INTEGER DEFAULT 0, "recordCardViewSectionRef" INTEGER DEFAULT 0);
INSERT INTO _grist_Tables VALUES(1,'Table1',1,0,0,2,3);
CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "description" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "rules" TEXT DEFAULT NULL, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
INSERT INTO _grist_Tables_column VALUES(1,1,1,'manualSort','ManualSortPos','',0,'','manualSort','',0,0,0,0,NULL,0,NULL);
INSERT INTO _grist_Tables_column VALUES(2,1,2,'A','Any','',1,'','A','',0,0,0,0,NULL,0,NULL);
INSERT INTO _grist_Tables_column VALUES(3,1,3,'B','Any','',1,'','B','',0,0,0,0,NULL,0,NULL);
INSERT INTO _grist_Tables_column VALUES(4,1,4,'C','Any','',1,'','C','',0,0,0,0,NULL,0,NULL);
CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "description" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "rules" TEXT DEFAULT NULL, "reverseCol" INTEGER DEFAULT 0, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
INSERT INTO _grist_Tables_column VALUES(1,1,1,'manualSort','ManualSortPos','',0,'','manualSort','',0,0,0,0,NULL,0,0,NULL);
INSERT INTO _grist_Tables_column VALUES(2,1,2,'A','Any','',1,'','A','',0,0,0,0,NULL,0,0,NULL);
INSERT INTO _grist_Tables_column VALUES(3,1,3,'B','Any','',1,'','B','',0,0,0,0,NULL,0,0,NULL);
INSERT INTO _grist_Tables_column VALUES(4,1,4,'C','Any','',1,'','C','',0,0,0,0,NULL,0,0,NULL);
CREATE TABLE IF NOT EXISTS "_grist_Imports" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "origFileName" TEXT DEFAULT '', "parseFormula" TEXT DEFAULT '', "delimiter" TEXT DEFAULT '', "doublequote" BOOLEAN DEFAULT 0, "escapechar" TEXT DEFAULT '', "quotechar" TEXT DEFAULT '', "skipinitialspace" BOOLEAN DEFAULT 0, "encoding" TEXT DEFAULT '', "hasHeaders" BOOLEAN DEFAULT 0);
CREATE TABLE IF NOT EXISTS "_grist_External_database" (id INTEGER PRIMARY KEY, "host" TEXT DEFAULT '', "port" INTEGER DEFAULT 0, "username" TEXT DEFAULT '', "dialect" TEXT DEFAULT '', "database" TEXT DEFAULT '', "storage" TEXT DEFAULT '');
CREATE TABLE IF NOT EXISTS "_grist_External_table" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "databaseRef" INTEGER DEFAULT 0, "tableName" TEXT DEFAULT '');

9
app/server/lib/requestUtils.ts
View File

@ -1,16 +1,19 @@
import {ApiError} from 'app/common/ApiError';
import { DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail } from 'app/common/gristUrls';
import {DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
import * as gutil from 'app/common/gutil';
import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/homedb/HomeDBManager';
import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
import {OptDocSession} from 'app/server/lib/DocSession';
import {RequestWithOrg} from 'app/server/lib/extractOrg';
import {RequestWithGrist} from 'app/server/lib/GristServer';
import log from 'app/server/lib/log';
import {Permit} from 'app/server/lib/Permit';
import {Request, Response} from 'express';
import { IncomingMessage } from 'http';
import {IncomingMessage} from 'http';
import {Writable} from 'stream';
import { TLSSocket } from 'tls';
import {TLSSocket} from 'tls';
export type RequestOrSession = RequestWithLogin | OptDocSession | null;
// log api details outside of dev environment (when GRIST_HOSTED_VERSION is set)
const shouldLogApiDetails = Boolean(process.env.GRIST_HOSTED_VERSION);

228
app/server/mergedServerMain.ts
View File

@ -1,228 +0,0 @@
/**
*
* A version of hosted grist that recombines a home server,
* a doc worker, and a static server on a single port.
*
*/
import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
import {GristLoginSystem} from 'app/server/lib/GristServer';
import log from 'app/server/lib/log';
import {getGlobalConfig} from "app/server/lib/globalConfig";
// Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS
// environment variable.
export type ServerType = "home" | "docs" | "static" | "app";
const allServerTypes: ServerType[] = ["home", "docs", "static", "app"];
// Parse a comma-separate list of server types into an array, with validation.
export function parseServerTypes(serverTypes: string|undefined): ServerType[] {
// Split and filter out empty strings (including the one we get when splitting "").
const types = (serverTypes || "").trim().split(',').filter(part => Boolean(part));
// Check that parts is non-empty and only contains valid options.
if (!types.length) {
throw new Error(`No server types; should be a comma-separated list of ${allServerTypes.join(", ")}`);
}
for (const t of types) {
if (!allServerTypes.includes(t as ServerType)) {
throw new Error(`Invalid server type '${t}'; should be in ${allServerTypes.join(", ")}`);
}
}
return types as ServerType[];
}
function checkUserContentPort(): number | null {
// Check whether a port is explicitly set for user content.
if (process.env.GRIST_UNTRUSTED_PORT) {
return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10);
}
// Checks whether to serve user content on same domain but on different port
if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
const homeUrl = new URL(process.env.APP_HOME_URL);
const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
// If the hostname of both home and plugin url are the same,
// but the ports are different
if (homeUrl.hostname === pluginUrl.hostname &&
homeUrl.port !== pluginUrl.port) {
const port = parseInt(pluginUrl.port || '80', 10);
return port;
}
}
return null;
}
interface ServerOptions extends FlexServerOptions {
logToConsole?: boolean; // If set, messages logged to console (default: false)
// (but if options are not given at all in call to main,
// logToConsole is set to true)
externalStorage?: boolean; // If set, documents saved to external storage such as s3 (default is to check environment
// variables, which get set in various ways in dev/test entry points)
loginSystem?: () => Promise<GristLoginSystem>;
}
/**
* Start a server on the given port, including the functionality specified in serverTypes.
*/
export async function main(port: number, serverTypes: ServerType[],
options: ServerOptions = {}) {
const includeHome = serverTypes.includes("home");
const includeDocs = serverTypes.includes("docs");
const includeStatic = serverTypes.includes("static");
const includeApp = serverTypes.includes("app");
options.settings ??= getGlobalConfig();
const server = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
// We need to know early on whether we will be serving plugins or not.
if (includeHome) {
const userPort = checkUserContentPort();
server.setServesPlugins(userPort !== undefined);
} else {
server.setServesPlugins(false);
}
if (options.loginSystem) {
server.setLoginSystem(options.loginSystem);
}
server.addCleanup();
server.setDirectory();
if (process.env.GRIST_TEST_ROUTER) {
// Add a mock api for adding/removing doc workers from load balancer.
server.testAddRouter();
}
if (options.logToConsole !== false) { server.addLogging(); }
if (options.externalStorage === false) { server.disableExternalStorage(); }
await server.addLoginMiddleware();
if (includeDocs) {
// It is important that /dw and /v prefixes are accepted (if present) by health check
// in this case, since they are included in the url registered for the doc worker.
server.stripDocWorkerIdPathPrefixIfPresent();
server.addTagChecker();
}
server.addHealthCheck();
if (includeHome || includeApp) {
server.addBootPage();
}
server.denyRequestsIfNotReady();
if (includeHome || includeStatic || includeApp) {
server.setDirectory();
}
if (includeHome || includeStatic) {
server.addStaticAndBowerDirectories();
}
await server.initHomeDBManager();
server.addHosts();
server.addDocWorkerMap();
if (includeHome || includeStatic) {
await server.addAssetsForPlugins();
}
if (includeHome) {
server.addEarlyWebhooks();
}
if (includeHome || includeDocs || includeApp) {
server.addSessions();
}
server.addAccessMiddleware();
server.addApiMiddleware();
await server.addBillingMiddleware();
try {
await server.start();
if (includeHome) {
server.addUsage();
if (!includeDocs) {
server.addDocApiForwarder();
}
server.addJsonSupport();
server.addUpdatesCheck();
await server.addLandingPages();
// todo: add support for home api to standalone app
server.addHomeApi();
server.addBillingApi();
server.addNotifier();
await server.addTelemetry();
await server.addHousekeeper();
await server.addLoginRoutes();
server.addAccountPage();
server.addBillingPages();
server.addWelcomePaths();
server.addLogEndpoint();
server.addGoogleAuthEndpoint();
server.addInstallEndpoints();
server.addConfigEndpoints();
}
if (includeDocs) {
server.addJsonSupport();
await server.addTelemetry();
await server.addDoc();
}
if (includeHome) {
server.addClientSecrets();
}
server.finalizeEndpoints();
await server.finalizePlugins(includeHome ? checkUserContentPort() : null);
server.checkOptionCombinations();
server.summary();
server.ready();
// Some tests have their timing perturbed by having this earlier
// TODO: update those tests.
if (includeDocs) {
await server.checkSandbox();
}
return server;
} catch(e) {
await server.close();
throw e;
}
}
export async function startMain() {
try {
const serverTypes = parseServerTypes(process.env.GRIST_SERVERS);
// No defaults for a port, since this server can serve very different purposes.
if (!process.env.GRIST_PORT) {
throw new Error("GRIST_PORT must be specified");
}
const port = parseInt(process.env.GRIST_PORT, 10);
const server = await main(port, serverTypes);
const opt = process.argv[2];
if (opt === '--testingHooks') {
await server.addTestingHooks();
}
return server;
} catch (e) {
log.error('mergedServer failed to start', e);
process.exit(1);
}
}
if (require.main === module) {
startMain().catch((e) => log.error('mergedServer failed to start', e));
}

4
app/server/utils/pruneActionHistory.ts
View File

@ -1,9 +1,9 @@
import * as gutil from 'app/common/gutil';
import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl';
import {DocStorage} from 'app/server/lib/DocStorage';
import {DocStorageManager} from 'app/server/lib/DocStorageManager';
import * as docUtils from 'app/server/lib/docUtils';
import log from 'app/server/lib/log';
import {create} from "app/server/lib/create";
/**
* A utility script for cleaning up the action log.
@ -18,7 +18,7 @@ export async function pruneActionHistory(docPath: string, keepN: number) {
throw new Error('Invalid document: Document should be a valid .grist file');
}
const storageManager = new DocStorageManager(".", ".");
const storageManager = await create.createLocalDocStorageManager(".", ".");
const docStorage = new DocStorage(storageManager, docPath);
const backupPath = gutil.removeSuffix(docPath, '.grist') + "-backup.grist";

2
buildtools/.grist-ee-version
View File

@ -1 +1 @@
0.9.8
0.9.9

24
package.json
View File

@ -1,6 +1,6 @@
{
"name": "grist-core",
"version": "1.1.17",
"version": "1.1.18",
"license": "Apache-2.0",
"description": "Grist is the evolution of spreadsheets",
"homepage": "https://github.com/gristlabs/grist-core",
@ -14,13 +14,13 @@
"install:python3": "buildtools/prepare_python3.sh",
"build:prod": "buildtools/build.sh",
"start:prod": "sandbox/run.sh",
"test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true LANGUAGE=en_US mocha ${DEBUG:+-b --no-exit} --slow 8000 $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" '_build/test/common/*.js' '_build/test/client/*.js' '_build/test/nbrowser/*.js' '_build/test/nbrowser_with_stubs/**/*.js' '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
"test:nbrowser": "TEST_SUITE=nbrowser TEST_SUITE_FOR_TIMINGS=nbrowser TIMINGS_FILE=test/timings/nbrowser.txt GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true LANGUAGE=en_US mocha ${DEBUG:+-b --no-exit} $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" --slow 8000 -R test/xunit-file '_build/test/nbrowser/**/*.js'",
"test:stubs": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true LANGUAGE=en_US mocha ${DEBUG:+-b --no-exit} $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" --slow 8000 -R test/xunit-file '_build/test/nbrowser_with_stubs/**/*.js'",
"test:client": "GRIST_SESSION_COOKIE=grist_test_cookie mocha ${DEBUG:+'-b'} '_build/test/client/**/*.js'",
"test:common": "GRIST_SESSION_COOKIE=grist_test_cookie mocha ${DEBUG:+'-b'} '_build/test/common/**/*.js'",
"test:server": "TEST_CLEAN_DATABASE=true TEST_SUITE=server TEST_SUITE_FOR_TIMINGS=server TIMINGS_FILE=test/timings/server.txt GRIST_SESSION_COOKIE=grist_test_cookie mocha ${DEBUG:+'-b'} -g \"${GREP_TESTS}\" -R test/xunit-file '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
"test:smoke": "LANGUAGE=en_US mocha _build/test/nbrowser/Smoke.js",
"test": "GRIST_TEST_LOGIN=1 ./test/test_env.sh mocha ${DEBUG:+-b --no-exit} --slow 8000 $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" '_build/test/common/*.js' '_build/test/client/*.js' '_build/test/nbrowser/*.js' '_build/test/nbrowser_with_stubs/**/*.js' '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
"test:nbrowser": "GRIST_TEST_LOGIN=1 TEST_SUITE=nbrowser TEST_SUITE_FOR_TIMINGS=nbrowser TIMINGS_FILE=test/timings/nbrowser.txt ./test/test_env.sh mocha ${DEBUG:+-b --no-exit} $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" --slow 8000 -R test/xunit-file '_build/test/nbrowser/**/*.js'",
"test:stubs": "GRIST_TEST_LOGIN=1 ./test/test_env.sh mocha ${DEBUG:+-b --no-exit} $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" --slow 8000 -R test/xunit-file '_build/test/nbrowser_with_stubs/**/*.js'",
"test:client": "./test/test_env.sh mocha ${DEBUG:+'-b'} '_build/test/client/**/*.js'",
"test:common": "./test/test_env.sh mocha ${DEBUG:+'-b'} '_build/test/common/**/*.js'",
"test:server": "TEST_SUITE=server TEST_SUITE_FOR_TIMINGS=server TIMINGS_FILE=test/timings/server.txt ./test/test_env.sh mocha ${DEBUG:+'-b'} -g \"${GREP_TESTS}\" -R test/xunit-file '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
"test:smoke": "./test/test_env.sh mocha _build/test/nbrowser/Smoke.js",
"test:docker": "./test/test_under_docker.sh",
"test:python": "sandbox_venv3/bin/python sandbox/grist/runtests.py ${GREP_TESTS:+discover -p \"test*${GREP_TESTS}*.py\"}",
"cli": "NODE_PATH=_build:_build/stubs:_build/ext node _build/app/server/companion.js",
@ -63,7 +63,6 @@
"@types/jsonwebtoken": "7.2.8",
"@types/lodash": "4.14.117",
"@types/lru-cache": "5.1.1",
"@types/marked": "4.0.8",
"@types/mime-types": "2.1.0",
"@types/mocha": "10.0.1",
"@types/moment-timezone": "0.5.9",
@ -96,8 +95,9 @@
"http-proxy": "1.18.1",
"i18next-scanner": "4.4.0",
"mocha": "10.2.0",
"mocha-webdriver": "git+https://github.com/gristlabs/mocha-webdriver.git#118f396e8e2e720e586d6ecd131fdf1d581bb12e",
"mocha-webdriver": "0.3.4",
"moment-locales-webpack-plugin": "^1.2.0",
"nock": "13.5.5",
"nodemon": "^2.0.4",
"otplib": "12.0.1",
"proper-lockfile": "4.1.2",
@ -165,7 +165,9 @@
"knockout": "3.5.0",
"locale-currency": "0.0.2",
"lodash": "4.17.21",
"marked": "4.2.12",
"marked": "14.0.0",
"marked-highlight": "2.1.4",
"marked-linkify-it": "3.1.11",
"minio": "8.0.0",
"moment": "2.29.4",
"moment-timezone": "0.5.35",

151
sandbox/grist/column.py
View File

@ -6,11 +6,13 @@ from numbers import Number
import six
import actions
import depend
import objtypes
import usertypes
import relabeling
import relation
import reverse_references
import moment
from sortedcontainers import SortedListWithKey
@ -221,17 +223,23 @@ class BaseColumn(object):
"""
return self.type_obj.convert(value_to_convert)
def prepare_new_values(self, values, ignore_data=False, action_summary=None):
def prepare_new_values(self, row_ids, values, ignore_data=False, action_summary=None):
"""
This allows us to modify values and also produce adjustments to existing records. This
currently is only used by PositionColumn. Returns two lists: new_values, and
[(row_id, new_value)] list of adjustments to existing records.
This allows us to modify values and also produce adjustments to existing records.
Returns the pair (new_values, adjustments), where new_values is a list to replace `values`
(one for each row_id), and adjustments is a list of additional docactions to apply, e.g. to
adjust other rows.
If ignore_data is True, makes adjustments without regard to the existing data; this is used
for processing ReplaceTableData actions.
"""
# pylint: disable=no-self-use, unused-argument
return values, []
def recalc_from_reverse_values(self):
pass # Only two-way references implement this
class DataColumn(BaseColumn):
"""
@ -253,6 +261,7 @@ class ChoiceColumn(DataColumn):
return row_ids, values
def _rename_cell_choice(self, renames, value):
# pylint: disable=no-self-use
return renames.get(value)
@ -373,7 +382,7 @@ class PositionColumn(NumericColumn):
self._sorted_rows = SortedListWithKey(other_column._sorted_rows[:],
key=lambda x: SafeSortKey(self.raw_get(x)))
def prepare_new_values(self, values, ignore_data=False, action_summary=None):
def prepare_new_values(self, row_ids, values, ignore_data=False, action_summary=None):
# This does the work of adjusting positions and relabeling existing rows with new position
# (without changing sort order) to make space for the new positions. Note that this is also
# used for updating a position for an existing row: we'll find a new value for it; later when
@ -385,7 +394,9 @@ class PositionColumn(NumericColumn):
# prepare_inserts expects floats as keys, not MixedTypesKeys
rows = SortedListWithKey(rows, key=self.raw_get)
adjustments, new_values = relabeling.prepare_inserts(rows, values)
return new_values, [(self._sorted_rows[i], pos) for (i, pos) in adjustments]
adj_action = _adjustments_to_action(self.node,
[(self._sorted_rows[i], pos) for (i, pos) in adjustments])
return new_values, ([adj_action] if adj_action else [])
class ChoiceListColumn(ChoiceColumn):
@ -410,6 +421,7 @@ class ChoiceListColumn(ChoiceColumn):
def _rename_cell_choice(self, renames, value):
if any((v in renames) for v in value):
return tuple(renames.get(choice, choice) for choice in value)
return None
class BaseReferenceColumn(BaseColumn):
@ -420,24 +432,45 @@ class BaseReferenceColumn(BaseColumn):
super(BaseReferenceColumn, self).__init__(table, col_id, col_info)
# We can assume that all tables have been instantiated, but not all initialized.
target_table_id = self.type_obj.table_id
self._table = table
self._target_table = table._engine.tables.get(target_table_id, None)
self._relation = relation.ReferenceRelation(table.table_id, target_table_id, col_id)
# Note that we need to remove these back-references when the column is removed.
if self._target_table:
self._target_table._back_references.add(self)
self._reverse_source_node = self.type_obj.reverse_source_node()
if self._reverse_source_node:
_multimap_add(self._table._reverse_cols_by_source_node, self._reverse_source_node, self)
def destroy(self):
# Destroy the column and remove the back-reference we created in the constructor.
super(BaseReferenceColumn, self).destroy()
if self._reverse_source_node:
_multimap_remove(self._table._reverse_cols_by_source_node, self._reverse_source_node, self)
if self._target_table:
self._target_table._back_references.remove(self)
def _update_references(self, row_id, old_value, new_value):
for r in self._value_iterable(old_value):
self._relation.remove_reference(row_id, r)
for r in self._value_iterable(new_value):
self._relation.add_reference(row_id, r)
def _clean_up_value(self, value):
raise NotImplementedError()
def _value_iterable(self, value):
raise NotImplementedError()
def _list_to_value(self, value_as_list):
raise NotImplementedError()
def set(self, row_id, value):
old = self.safe_get(row_id)
super(BaseReferenceColumn, self).set(row_id, value)
super(BaseReferenceColumn, self).set(row_id, self._clean_up_value(value))
new = self.safe_get(row_id)
self._update_references(row_id, old, new)
@ -462,6 +495,39 @@ class BaseReferenceColumn(BaseColumn):
affected_rows = sorted(self._relation.get_affected_rows(target_row_ids))
return [(row_id, self._raw_get_without(row_id, target_row_ids)) for row_id in affected_rows]
def prepare_new_values(self, row_ids, values, ignore_data=False, action_summary=None):
values = [self._clean_up_value(v) for v in values]
reverse_cols = self._target_table._reverse_cols_by_source_node.get(self.node, [])
adjustments = []
if reverse_cols:
old_values = [self.raw_get(r) for r in row_ids]
reverse_adjustments = reverse_references.get_reverse_adjustments(
row_ids, old_values, values, self._value_iterable, self._relation)
if reverse_adjustments:
for reverse_col in reverse_cols:
adjustments.append(_adjustments_to_action(
reverse_col.node,
[(row_id, reverse_col._list_to_value(value)) for (row_id, value) in reverse_adjustments]
))
return values, adjustments
def recalc_from_reverse_values(self):
"""
Generates actions to update reverse column based on this column.
"""
if not self._reverse_source_node:
return None
rev_table_id, rev_col_id = self._reverse_source_node
reverse_col = self._target_table.get_column(rev_col_id)
reverse_adjustments = []
for target_row_id in self._target_table.row_ids:
reverse_value = self._relation.get_affected_rows((target_row_id,))
reverse_adjustments.append((target_row_id, sorted(reverse_value)))
return _adjustments_to_action(reverse_col.node,
[(row_id, reverse_col._list_to_value(value)) for (row_id, value) in reverse_adjustments])
def _raw_get_without(self, _row_id, _target_row_ids):
"""
Returns a Ref or RefList cell value with the specified target_row_ids removed, assuming one of
@ -493,28 +559,33 @@ class ReferenceColumn(BaseReferenceColumn):
# the 0 index will contain the all-defaults record.
return self._target_table.Record(typed_value, self._relation)
def _update_references(self, row_id, old_value, new_value):
if old_value:
self._relation.remove_reference(row_id, old_value)
if new_value:
self._relation.add_reference(row_id, new_value)
def _value_iterable(self, value):
return (value,) if value and self.type_obj.is_right_type(value) else ()
def set(self, row_id, value):
def _list_to_value(self, value_as_list):
if len(value_as_list) > 1:
raise ValueError("UNIQUE reference constraint failed for action")
return value_as_list[0] if value_as_list else 0
def _clean_up_value(self, value):
# Allow float values that are small integers. In practice, this only turns out to be relevant
# in rare cases (such as undo of Ref->Numeric conversion).
if type(value) == float and value.is_integer(): # pylint:disable=unidiomatic-typecheck
if value > 0 and objtypes.is_int_short(int(value)):
value = int(value)
super(ReferenceColumn, self).set(row_id, value)
return int(value)
return value
def prepare_new_values(self, values, ignore_data=False, action_summary=None):
def prepare_new_values(self, row_ids, values, ignore_data=False, action_summary=None):
if action_summary and values:
values = action_summary.translate_new_row_ids(self._target_table.table_id, values)
return values, []
return super(ReferenceColumn, self).prepare_new_values(row_ids, values,
ignore_data=ignore_data, action_summary=action_summary)
def convert(self, val):
if isinstance(val, objtypes.ReferenceLookup):
val = self._lookup(val, val.value) or self._alt_text(val.alt_text)
elif isinstance(val, list):
val = val[0] if val else 0
return super(ReferenceColumn, self).convert(val)
@ -523,7 +594,7 @@ class ReferenceListColumn(BaseReferenceColumn):
ReferenceListColumn maintains for each row a list of references (row IDs) into another table.
Accessing them yields RecordSets.
"""
def set(self, row_id, value):
def _clean_up_value(self, value):
if isinstance(value, six.string_types):
# This is second part of a "hack" we have to do when we rename tables. During
# the rename, we briefly change all Ref columns to Int columns (to lose the table
@ -535,20 +606,27 @@ class ReferenceListColumn(BaseReferenceColumn):
try:
# If it's a string that looks like JSON, try to parse it as such.
if value.startswith('['):
value = json.loads(value)
parsed = json.loads(value)
# It must be list of integers.
if not isinstance(parsed, list):
return value
# All of them must be positive integers
if all(isinstance(v, int) and v > 0 for v in parsed):
return parsed
else:
# Else try to parse it as a RecordList
value = objtypes.RecordList.from_repr(value)
# Else try to parse it as a RecordList
return objtypes.RecordList.from_repr(value)
except Exception:
pass
return value
super(ReferenceListColumn, self).set(row_id, value)
def _value_iterable(self, value):
return value if value and self.type_obj.is_right_type(value) else ()
def _update_references(self, row_id, old_list, new_list):
for old_value in old_list or ():
self._relation.remove_reference(row_id, old_value)
for new_value in new_list or ():
self._relation.add_reference(row_id, new_value)
def _list_to_value(self, value_as_list):
return value_as_list or None
def _make_rich_value(self, typed_value):
if typed_value is None:
@ -579,8 +657,27 @@ class ReferenceListColumn(BaseReferenceColumn):
return self._alt_text(val.alt_text)
result.append(lookup_value)
val = result
if isinstance(val, int) and val:
val = [val]
return super(ReferenceListColumn, self).convert(val)
def _multimap_add(mapping, key, value):
mapping.setdefault(key, []).append(value)
def _multimap_remove(mapping, key, value):
if key in mapping and value in mapping[key]:
mapping[key].remove(value)
if not mapping[key]:
del mapping[key]
def _adjustments_to_action(node, row_value_pairs):
# Takes a depend.Node and a list of (row_id, value) pairs, and returns a BulkUpdateRecord action.
if not row_value_pairs:
return None
row_ids, values = zip(*row_value_pairs)
return actions.BulkUpdateRecord(node.table_id, row_ids, {node.col_id: values})
# Set up the relationship between usertypes objects and column objects.
usertypes.BaseColumnType.ColType = DataColumn

9
sandbox/grist/docactions.py
View File

@ -166,8 +166,7 @@ class DocActions(object):
# Replace the renamed column in the schema object.
schema_table_info = self._engine.schema[table_id]
colinfo = schema_table_info.columns.pop(old_col_id)
schema_table_info.columns[new_col_id] = schema.SchemaColumn(
new_col_id, colinfo.type, colinfo.isFormula, colinfo.formula)
schema_table_info.columns[new_col_id] = colinfo._replace(colId=new_col_id)
self._engine.rebuild_usercode()
self._engine.new_column_name(table)
@ -192,12 +191,14 @@ class DocActions(object):
new = schema.SchemaColumn(col_id,
col_info.get('type', old.type),
bool(col_info.get('isFormula', old.isFormula)),
col_info.get('formula', old.formula))
col_info.get('formula', old.formula),
col_info.get('reverseColId', old.reverseColId))
if new == old:
log.info("ModifyColumn called which was a noop")
return
undo_col_info = {k: v for k, v in six.iteritems(schema.col_to_dict(old, include_id=False))
undo_col_info = {k: v for k, v in six.iteritems(
schema.col_to_dict(old, include_id=False, include_default=True))
if k in col_info}
# Remove the column from the schema, then re-add it, to force creation of a new column object.

2
sandbox/grist/dropdown_condition.py
View File

@ -30,6 +30,8 @@ def perform_dropdown_condition_renames(useractions, renames):
updates = []
for col in useractions.get_docmodel().columns.all:
if not col.widgetOptions:
continue
# Find all columns in the document that have dropdown conditions.
try:

17
sandbox/grist/engine.py
View File

@ -507,9 +507,11 @@ class Engine(object):
if col_parent_ids > valid_table_refs:
collist = sorted(actions.transpose_bulk_action(meta_columns),
key=lambda c: (c.parentId, c.parentPos))
reverse_col_id = schema.get_reverse_col_id_lookup_func(collist)
raise AssertionError("Internal schema inconsistent; extra columns in metadata:\n"
+ "\n".join(' #%s %s' %
(c.id, schema.SchemaColumn(c.colId, c.type, bool(c.isFormula), c.formula))
(c.id, schema.SchemaColumn(c.colId, c.type, bool(c.isFormula), c.formula,
reverse_col_id(c)))
for c in collist if c.parentId not in valid_table_refs))
def dump_state(self):
@ -1035,11 +1037,9 @@ class Engine(object):
# If there are values for any PositionNumber columns, ensure PositionNumbers are ordered as
# intended but are all unique, which may require updating other positions.
nvalues, adjustments = col_obj.prepare_new_values(values,
nvalues, adjustments = col_obj.prepare_new_values(row_ids, values,
action_summary=self.out_actions.summary)
if adjustments:
extra_actions.append(actions.BulkUpdateRecord(
action.table_id, [r for r,v in adjustments], {col_id: [v for r,v in adjustments]}))
extra_actions.extend(adjustments)
new_values[col_id] = nvalues
@ -1054,11 +1054,10 @@ class Engine(object):
defaults = [col_obj.getdefault() for r in row_ids]
# We use defaults to get new values or adjustments. If we are replacing data, we'll make
# the adjustments without regard to the existing data.
nvalues, adjustments = col_obj.prepare_new_values(defaults, ignore_data=ignore_data,
nvalues, adjustments = col_obj.prepare_new_values(row_ids, defaults,
ignore_data=ignore_data,
action_summary=self.out_actions.summary)
if adjustments:
extra_actions.append(actions.BulkUpdateRecord(
action.table_id, [r for r,v in adjustments], {col_id: [v for r,v in adjustments]}))
extra_actions.extend(adjustments)
if nvalues != defaults:
new_values[col_id] = nvalues

14
sandbox/grist/gencode.py
View File

@ -42,7 +42,7 @@ def indent(body, levels=1):
#----------------------------------------------------------------------
def get_grist_type(col_type):
def get_grist_type(col_type, reverse_col_id=None):
"""Returns code for a grist usertype object given a column type string."""
col_type_split = col_type.split(':', 1)
typename = col_type_split[0]
@ -54,7 +54,12 @@ def get_grist_type(col_type):
arg = col_type_split[1] if len(col_type_split) > 1 else ''
arg = arg.strip().replace("'", "\\'")
return "grist.%s(%s)" % (typename, ("'%s'" % arg) if arg else '')
args = []
if arg:
args.append("'%s'" % arg)
if reverse_col_id and typename in ('Reference', 'ReferenceList'):
args.append('reverse_of=' + repr(reverse_col_id))
return "grist.%s(%s)" % (typename, ", ".join(args))
class GenCode(object):
@ -99,7 +104,7 @@ class GenCode(object):
decorator = ''
if include_type and col_info.type != 'Any':
decorator = '@grist.formulaType(%s)\n' % get_grist_type(col_info.type)
decorator = '@grist.formulaType(%s)\n' % get_grist_type(col_info.type, col_info.reverseColId)
return textbuilder.Combiner(['\n' + decorator + decl, indent(body), '\n'])
@ -111,7 +116,8 @@ class GenCode(object):
name=table.get_default_func_name(col_info.colId),
include_type=False,
additional_params=['value', 'user']))
parts.append("%s = %s\n" % (col_info.colId, get_grist_type(col_info.type)))
parts.append("%s = %s\n" % (col_info.colId,
get_grist_type(col_info.type, col_info.reverseColId)))
return textbuilder.Combiner(parts)

8
sandbox/grist/migrations.py
View File

@ -1317,3 +1317,11 @@ def migration42(tdset):
add_column('_grist_Triggers', 'watchedColRefList', 'RefList:_grist_Tables_column'),
add_column('_grist_Triggers', 'options', 'Text'),
])
@migration(schema_version=43)
def migration43(tdset):
"""
Adds reverseCol for two-way references.
"""
return tdset.apply_doc_actions([
add_column('_grist_Tables_column', 'reverseCol', 'Ref:_grist_Tables_column')])

17
sandbox/grist/objtypes.py
View File

@ -167,15 +167,26 @@ def encode_object(value):
Produces a Grist-encoded version of the value, e.g. turning a Date into ['d', timestamp].
Returns ['U', repr(value)] if it fails to encode otherwise.
"""
# pylint: disable=unidiomatic-typecheck
try:
if isinstance(value, (six.text_type, float, bool)) or value is None:
# A primitive type can be returned directly.
if type(value) in (six.text_type, float, bool) or value is None:
return value
# Other instances of these types must be derived; cast these to the primitive type to ensure
# they are marshallable.
elif isinstance(value, six.text_type):
return six.text_type(value)
elif isinstance(value, float):
return float(value)
elif isinstance(value, bool):
return bool(value)
elif isinstance(value, six.binary_type):
return value.decode('utf8')
elif isinstance(value, six.integer_types):
if not is_int_short(value):
raise UnmarshallableError("Integer too large")
return value
return ['U', str(value)]
# Cast to a primitive type to ensure it's marshallable (e.g. enum.IntEnum would not be).
return int(value)
elif isinstance(value, AltText):
return six.text_type(value)
elif isinstance(value, records.Record):

Some files were not shown because too many files have changed in this diff Show More