diff --git a/app/client/aclui/ACLFormulaEditor.ts b/app/client/aclui/ACLFormulaEditor.ts index 849b19dd..a99bc9ee 100644 --- a/app/client/aclui/ACLFormulaEditor.ts +++ b/app/client/aclui/ACLFormulaEditor.ts @@ -1,13 +1,12 @@ import ace, {Ace} from 'ace-builds'; import {setupAceEditorCompletions} from 'app/client/components/AceEditorCompletions'; import {theme} from 'app/client/ui2018/cssVars'; +import {gristThemeObs} from 'app/client/ui2018/theme'; import {Theme} from 'app/common/ThemePrefs'; -import {getGristConfig} from 'app/common/urlUtils'; -import {Computed, dom, DomArg, Listener, Observable, styled} from 'grainjs'; +import {dom, DomArg, Observable, styled} from 'grainjs'; import debounce from 'lodash/debounce'; export interface ACLFormulaOptions { - gristTheme: Computed; initialValue: string; readOnly: boolean; placeholder: DomArg; @@ -22,19 +21,15 @@ export function aclFormulaEditor(options: ACLFormulaOptions) { const editor: Ace.Editor = ace.edit(editorElem); // Set various editor options. - function setAceTheme(gristTheme: Theme) { - const {enableCustomCss} = getGristConfig(); - const gristAppearance = gristTheme.appearance; - const aceTheme = gristAppearance === 'dark' && !enableCustomCss ? 'dracula' : 'chrome'; + function setAceTheme(newTheme: Theme) { + const {appearance} = newTheme; + const aceTheme = appearance === 'dark' ? 'dracula' : 'chrome'; editor.setTheme(`ace/theme/${aceTheme}`); } - setAceTheme(options.gristTheme.get()); - let themeListener: Listener | undefined; - if (!getGristConfig().enableCustomCss) { - themeListener = options.gristTheme.addListener((gristTheme) => { - setAceTheme(gristTheme); - }); - } + setAceTheme(gristThemeObs().get()); + const themeListener = gristThemeObs().addListener((newTheme) => { + setAceTheme(newTheme); + }); // ACE editor resizes automatically when maxLines is set. editor.setOptions({enableLiveAutocompletion: true, maxLines: 10}); editor.renderer.setShowGutter(false); // Default line numbers to hidden diff --git a/app/client/aclui/AccessRules.ts b/app/client/aclui/AccessRules.ts index 010d05b1..167f27df 100644 --- a/app/client/aclui/AccessRules.ts +++ b/app/client/aclui/AccessRules.ts @@ -36,14 +36,13 @@ import {ACLRuleCollection, isSchemaEditResource, SPECIAL_RULES_TABLE_ID} from 'a import {AclRuleProblem, AclTableDescription, getTableTitle} from 'app/common/ActiveDocAPI'; import {BulkColValues, getColValues, RowRecord, UserAction} from 'app/common/DocActions'; import { - FormulaProperties, - getFormulaProperties, RulePart, RuleSet, UserAttributeRule } from 'app/common/GranularAccessClause'; import {isHiddenCol} from 'app/common/gristTypes'; import {isNonNullish, unwrap} from 'app/common/gutil'; +import {getPredicateFormulaProperties, PredicateFormulaProperties} from 'app/common/PredicateFormula'; import {SchemaTypes} from 'app/common/schema'; import {MetaRowRecord} from 'app/common/TableData'; import { @@ -496,7 +495,7 @@ export class AccessRules extends Disposable { removeItem(this._userAttrRules, userAttr); } - public async checkAclFormula(text: string): Promise { + public async checkAclFormula(text: string): Promise { if (text) { return this.gristDoc.docComm.checkAclFormula(text); } @@ -1465,7 +1464,6 @@ class ObsUserAttributeRule extends Disposable { cssColumnGroup( cssCell1( aclFormulaEditor({ - gristTheme: this._accessRules.gristDoc.currentTheme, initialValue: this._charId.get(), readOnly: false, setValue: (text) => this._setUserAttr(text), @@ -1598,7 +1596,8 @@ class ObsRulePart extends Disposable { // If the formula failed validation, the error message to show. Blank if valid. private _formulaError = Observable.create(this, ''); - private _formulaProperties = Observable.create(this, getAclFormulaProperties(this._rulePart)); + private _formulaProperties = Observable.create(this, + getAclFormulaProperties(this._rulePart)); // Error message if any validation failed. private _error: Computed; @@ -1618,7 +1617,7 @@ class ObsRulePart extends Disposable { this._error = Computed.create(this, (use) => { return use(this._formulaError) || - this._warnInvalidColIds(use(this._formulaProperties).usedColIds) || + this._warnInvalidColIds(use(this._formulaProperties).recColIds) || ( !this._ruleSet.isLastCondition(use, this) && use(this._aclFormula) === '' && permissionSetToText(use(this._permissions)) !== '' ? @@ -1690,7 +1689,6 @@ class ObsRulePart extends Disposable { cssCell2( wide ? cssCell4.cls('') : null, aclFormulaEditor({ - gristTheme: this._ruleSet.accessRules.gristDoc.currentTheme, initialValue: this._aclFormula.get(), readOnly: this.isBuiltIn(), setValue: (value) => this._setAclFormula(value), @@ -1913,9 +1911,9 @@ function getChangedStatus(value: boolean): RuleStatus { return value ? RuleStatus.ChangedValid : RuleStatus.Unchanged; } -function getAclFormulaProperties(part?: RulePart): FormulaProperties { +function getAclFormulaProperties(part?: RulePart): PredicateFormulaProperties { const aclFormulaParsed = part?.origRecord?.aclFormulaParsed; - return aclFormulaParsed ? getFormulaProperties(JSON.parse(String(aclFormulaParsed))) : {}; + return aclFormulaParsed ? getPredicateFormulaProperties(JSON.parse(String(aclFormulaParsed))) : {}; } // Return a rule set if it applies to one of the specified columns. diff --git a/app/client/components/AceEditor.js b/app/client/components/AceEditor.js index afac581b..59f31fe2 100644 --- a/app/client/components/AceEditor.js +++ b/app/client/components/AceEditor.js @@ -7,10 +7,10 @@ require('ace-builds/src-noconflict/theme-chrome'); require('ace-builds/src-noconflict/theme-dracula'); require('ace-builds/src-noconflict/ext-language_tools'); var {setupAceEditorCompletions} = require('./AceEditorCompletions'); -var {getGristConfig} = require('../../common/urlUtils'); var dom = require('../lib/dom'); var dispose = require('../lib/dispose'); var modelUtil = require('../models/modelUtil'); +var {gristThemeObs} = require('../ui2018/theme'); /** * A class to help set up the ace editor with standard formatting and convenience functions @@ -28,10 +28,9 @@ function AceEditor(options) { this.observable = options.observable || null; this.saveValueOnBlurEvent = !(options.saveValueOnBlurEvent === false); this.calcSize = options.calcSize || ((_elem, size) => size); - this.gristDoc = options.gristDoc || null; - this.column = options.column || null; this.editorState = options.editorState || null; this._readonly = options.readonly || false; + this._getSuggestions = options.getSuggestions || null; this.editor = null; this.editorDom = null; @@ -185,19 +184,8 @@ AceEditor.prototype.setFontSize = function(pxVal) { AceEditor.prototype._setup = function() { // Standard editor setup this.editor = this.autoDisposeWith('destroy', ace.edit(this.editorDom)); - if (this.gristDoc && this.column) { - const getSuggestions = (prefix) => { - const section = this.gristDoc.viewModel.activeSection(); - // If section is disposed or is pointing to an empty row, don't try to autocomplete. - if (!section?.getRowId()) { - return []; - } - const tableId = section.table().tableId(); - const columnId = this.column.colId(); - const rowId = section.activeRowId(); - return this.gristDoc.docComm.autocomplete(prefix, tableId, columnId, rowId); - }; - setupAceEditorCompletions(this.editor, {getSuggestions}); + if (this._getSuggestions) { + setupAceEditorCompletions(this.editor, {getSuggestions: this._getSuggestions}); } this.editor.setOptions({ enableLiveAutocompletion: true, // use autocompletion without needing special activation. @@ -205,13 +193,10 @@ AceEditor.prototype._setup = function() { this.session = this.editor.getSession(); this.session.setMode('ace/mode/python'); - const gristTheme = this.gristDoc?.currentTheme; - this._setAceTheme(gristTheme?.get()); - if (!getGristConfig().enableCustomCss && gristTheme) { - this.autoDispose(gristTheme.addListener((theme) => { - this._setAceTheme(theme); - })); - } + this._setAceTheme(gristThemeObs().get()); + this.autoDispose(gristThemeObs().addListener((newTheme) => { + this._setAceTheme(newTheme); + })); // Default line numbers to hidden this.editor.renderer.setShowGutter(false); @@ -283,10 +268,9 @@ AceEditor.prototype._getContentHeight = function() { return Math.max(1, this.session.getScreenLength()) * this.editor.renderer.lineHeight; }; -AceEditor.prototype._setAceTheme = function(gristTheme) { - const {enableCustomCss} = getGristConfig(); - const gristAppearance = gristTheme?.appearance; - const aceTheme = gristAppearance === 'dark' && !enableCustomCss ? 'dracula' : 'chrome'; +AceEditor.prototype._setAceTheme = function(newTheme) { + const {appearance} = newTheme; + const aceTheme = appearance === 'dark' ? 'dracula' : 'chrome'; this.editor.setTheme(`ace/theme/${aceTheme}`); }; diff --git a/app/client/components/ChartView.ts b/app/client/components/ChartView.ts index 7b33169b..3c49e2dd 100644 --- a/app/client/components/ChartView.ts +++ b/app/client/components/ChartView.ts @@ -17,6 +17,7 @@ import {cssFieldEntry, cssFieldLabel, IField, VisibleFieldsConfig } from 'app/cl import {IconName} from 'app/client/ui2018/IconList'; import {squareCheckbox} from 'app/client/ui2018/checkbox'; import {theme, vars} from 'app/client/ui2018/cssVars'; +import {gristThemeObs} from 'app/client/ui2018/theme'; import {cssDragger} from 'app/client/ui2018/draggableList'; import {icon} from 'app/client/ui2018/icons'; import {IOptionFull, linkSelect, menu, menuItem, menuText, select} from 'app/client/ui2018/menus'; @@ -229,7 +230,7 @@ export class ChartView extends Disposable { this.listenTo(this.sortedRows, 'rowNotify', this._update); this.autoDispose(this.sortedRows.getKoArray().subscribe(this._update)); this.autoDispose(this._formatterComp.subscribe(this._update)); - this.autoDispose(this.gristDoc.currentTheme.addListener(() => this._update())); + this.autoDispose(gristThemeObs().addListener(() => this._update())); } public prepareToPrint(onOff: boolean) { @@ -387,8 +388,7 @@ export class ChartView extends Disposable { } private _getPlotlyTheme(): Partial { - const appModel = this.gristDoc.docPageModel.appModel; - const {colors} = appModel.currentTheme.get(); + const {colors} = gristThemeObs().get(); return { paper_bgcolor: colors['chart-bg'], plot_bgcolor: colors['chart-bg'], diff --git a/app/client/components/ColumnTransform.ts b/app/client/components/ColumnTransform.ts index 127caae2..3e2881a7 100644 --- a/app/client/components/ColumnTransform.ts +++ b/app/client/components/ColumnTransform.ts @@ -91,9 +91,9 @@ export class ColumnTransform extends Disposable { protected buildEditorDom(optInit?: string) { if (!this.editor) { this.editor = this.autoDispose(AceEditor.create({ - gristDoc: this.gristDoc, observable: this.transformColumn.formula, saveValueOnBlurEvent: false, + // TODO: set `getSuggestions` (see `FormulaEditor.ts` for an example). })); } return this.editor.buildDom((aceObj: any) => { diff --git a/app/client/components/CustomView.ts b/app/client/components/CustomView.ts index 8c05db3c..168aea10 100644 --- a/app/client/components/CustomView.ts +++ b/app/client/components/CustomView.ts @@ -321,7 +321,7 @@ export class CustomView extends Disposable { }), new MinimumLevel(AccessLevel.none)); // none access is enough frame.useEvents( - ThemeNotifier.create(frame, this.gristDoc.currentTheme), + ThemeNotifier.create(frame), new MinimumLevel(AccessLevel.none)); }, onElem: (iframe) => onFrameFocus(iframe, () => { diff --git a/app/client/components/DropdownConditionConfig.ts b/app/client/components/DropdownConditionConfig.ts new file mode 100644 index 00000000..c3cb2e08 --- /dev/null +++ b/app/client/components/DropdownConditionConfig.ts @@ -0,0 +1,197 @@ +import {buildDropdownConditionEditor} from 'app/client/components/DropdownConditionEditor'; +import {makeT} from 'app/client/lib/localization'; +import {ViewFieldRec} from 'app/client/models/DocModel'; +import {cssLabel, cssRow} from 'app/client/ui/RightPanelStyles'; +import {withInfoTooltip} from 'app/client/ui/tooltips'; +import {textButton } from 'app/client/ui2018/buttons'; +import {testId, theme} from 'app/client/ui2018/cssVars'; +import {ISuggestionWithValue} from 'app/common/ActiveDocAPI'; +import {getPredicateFormulaProperties} from 'app/common/PredicateFormula'; +import {Computed, Disposable, dom, Observable, styled} from 'grainjs'; + +const t = makeT('DropdownConditionConfig'); + +/** + * Right panel configuration for dropdown conditions. + * + * Contains an instance of `DropdownConditionEditor`, the class responsible + * for setting dropdown conditions. + */ +export class DropdownConditionConfig extends Disposable { + private _text = Computed.create(this, use => { + const dropdownCondition = use(this._field.dropdownCondition); + if (!dropdownCondition) { return ''; } + + return dropdownCondition.text; + }); + + private _saveError = Observable.create(this, null); + + private _properties = Computed.create(this, use => { + const dropdownCondition = use(this._field.dropdownCondition); + if (!dropdownCondition?.parsed) { return null; } + + return getPredicateFormulaProperties(JSON.parse(dropdownCondition.parsed)); + }); + + private _column = Computed.create(this, use => use(this._field.column)); + + private _columns = Computed.create(this, use => use(use(use(this._column).table).visibleColumns)); + + private _refColumns = Computed.create(this, use => { + const refTable = use(use(this._column).refTable); + if (!refTable) { return null; } + + return use(refTable.visibleColumns); + }); + + private _propertiesError = Computed.create(this, use => { + const properties = use(this._properties); + if (!properties) { return null; } + + const {recColIds = [], choiceColIds = []} = properties; + const columns = use(this._columns); + const validRecColIds = new Set(columns.map((({colId}) => use(colId)))); + const invalidRecColIds = recColIds.filter(colId => !validRecColIds.has(colId)); + if (invalidRecColIds.length > 0) { + return t('Invalid columns: {{colIds}}', {colIds: invalidRecColIds.join(', ')}); + } + + const refColumns = use(this._refColumns); + if (refColumns) { + const validChoiceColIds = new Set(['id', ...refColumns.map((({colId}) => use(colId)))]); + const invalidChoiceColIds = choiceColIds.filter(colId => !validChoiceColIds.has(colId)); + if (invalidChoiceColIds.length > 0) { + return t('Invalid columns: {{colIds}}', {colIds: invalidChoiceColIds.join(', ')}); + } + } + + return null; + }); + + private _error = Computed.create(this, (use) => { + const maybeSaveError = use(this._saveError); + if (maybeSaveError) { return maybeSaveError; } + + const maybeCompiled = use(this._field.dropdownConditionCompiled); + if (maybeCompiled?.kind === 'failure') { return maybeCompiled.error; } + + const maybePropertiesError = use(this._propertiesError); + if (maybePropertiesError) { return maybePropertiesError; } + + return null; + }); + + private _disabled = Computed.create(this, use => + use(this._field.disableModify) || + use(use(this._column).disableEditData) || + use(this._field.config.multiselect) + ); + + private _isEditingCondition = Observable.create(this, false); + + private _isRefField = Computed.create(this, (use) => + ['Ref', 'RefList'].includes(use(use(this._column).pureType))); + + private _tooltip = Computed.create(this, use => use(this._isRefField) + ? 'setRefDropdownCondition' + : 'setChoiceDropdownCondition'); + + private _editorElement: HTMLElement; + + constructor(private _field: ViewFieldRec) { + super(); + + this.autoDispose(this._text.addListener(() => { + this._saveError.set(''); + })); + } + + public buildDom() { + return [ + dom.maybe((use) => !(use(this._isEditingCondition) || Boolean(use(this._text))), () => [ + cssSetDropdownConditionRow( + dom.domComputed(use => withInfoTooltip( + textButton( + t('Set dropdown condition'), + dom.on('click', () => { + this._isEditingCondition.set(true); + setTimeout(() => this._editorElement.focus(), 0); + }), + dom.prop('disabled', this._disabled), + testId('field-set-dropdown-condition'), + ), + use(this._tooltip), + )), + ), + ]), + dom.maybe((use) => use(this._isEditingCondition) || Boolean(use(this._text)), () => [ + cssLabel(t('Dropdown Condition')), + cssRow( + dom.create(buildDropdownConditionEditor, + { + value: this._text, + disabled: this._disabled, + getAutocompleteSuggestions: () => this._getAutocompleteSuggestions(), + onSave: async (value) => { + try { + const widgetOptions = this._field.widgetOptionsJson.peek(); + if (value.trim() === '') { + delete widgetOptions.dropdownCondition; + } else { + widgetOptions.dropdownCondition = {text: value}; + } + await this._field.widgetOptionsJson.setAndSave(widgetOptions); + } catch (e) { + if (e?.code === 'ACL_DENY') { + reportError(e); + } else { + this._saveError.set(e.message.replace(/^\[Sandbox\]/, '').trim()); + } + } + }, + onDispose: () => { + this._isEditingCondition.set(false); + }, + }, + (el) => { this._editorElement = el; }, + testId('field-dropdown-condition'), + ), + ), + dom.maybe(this._error, (error) => cssRow( + cssDropdownConditionError(error), testId('field-dropdown-condition-error')), + ), + ]), + ]; + } + + private _getAutocompleteSuggestions(): ISuggestionWithValue[] { + const variables = ['choice']; + const refColumns = this._refColumns.get(); + if (refColumns) { + variables.push('choice.id', ...refColumns.map(({colId}) => `choice.${colId.peek()}`)); + } + const columns = this._columns.get(); + variables.push( + ...columns.map(({colId}) => `$${colId.peek()}`), + ...columns.map(({colId}) => `rec.${colId.peek()}`), + ); + + const suggestions = [ + 'and', 'or', 'not', 'in', 'is', 'True', 'False', 'None', + 'OWNER', 'EDITOR', 'VIEWER', + ...variables, + ]; + return suggestions.map(suggestion => [suggestion, null]); + } +} + +const cssSetDropdownConditionRow = styled(cssRow, ` + margin-top: 16px; +`); + +const cssDropdownConditionError = styled('div', ` + color: ${theme.errorText}; + margin-top: 4px; + width: 100%; +`); diff --git a/app/client/components/DropdownConditionEditor.ts b/app/client/components/DropdownConditionEditor.ts new file mode 100644 index 00000000..1091ae44 --- /dev/null +++ b/app/client/components/DropdownConditionEditor.ts @@ -0,0 +1,234 @@ +import * as AceEditor from 'app/client/components/AceEditor'; +import {createGroup} from 'app/client/components/commands'; +import {makeT} from 'app/client/lib/localization'; +import {buildHighlightedCode} from 'app/client/ui/CodeHighlight'; +import {theme} from 'app/client/ui2018/cssVars'; +import {createMobileButtons, getButtonMargins} from 'app/client/widgets/EditorButtons'; +import {EditorPlacement, ISize} from 'app/client/widgets/EditorPlacement'; +import {initializeAceOptions} from 'app/client/widgets/FormulaEditor'; +import {IEditorCommandGroup} from 'app/client/widgets/NewBaseEditor'; +import {ISuggestionWithValue} from 'app/common/ActiveDocAPI'; +import { + Computed, + Disposable, + dom, + DomElementArg, + Holder, + IDisposableOwner, + Observable, + styled, +} from 'grainjs'; + +const t = makeT('DropdownConditionEditor'); + +interface BuildDropdownConditionEditorOptions { + value: Computed; + disabled: Computed; + onSave(value: string): Promise; + onDispose(): void; + getAutocompleteSuggestions(prefix: string): ISuggestionWithValue[]; +} + +/** + * Builds an editor for dropdown conditions. + * + * Dropdown conditions are client-evaluated predicate formulas used to filter + * items shown in autocomplete dropdowns for Choice and Reference type columns. + * + * Unlike Python formulas, dropdown conditions only support a very limited set of + * features. They are a close relative of ACL formulas, sharing the same underlying + * parser and compiler. + * + * See `sandbox/grist/predicate_formula.py` and `app/common/PredicateFormula.ts` for + * more details on parsing and compiling, respectively. + */ +export function buildDropdownConditionEditor( + owner: IDisposableOwner, + options: BuildDropdownConditionEditorOptions, + ...args: DomElementArg[] +) { + const {value, disabled, onSave, onDispose, getAutocompleteSuggestions} = options; + return dom.create(buildHighlightedCode, + value, + {maxLines: 1}, + dom.cls(cssDropdownConditionField.className), + dom.cls('disabled'), + cssDropdownConditionField.cls('-disabled', disabled), + {tabIndex: '-1'}, + dom.on('focus', (_, refElem) => openDropdownConditionEditor(owner, { + refElem, + value, + onSave, + onDispose, + getAutocompleteSuggestions, + })), + ...args, + ); +} + +function openDropdownConditionEditor(owner: IDisposableOwner, options: { + refElem: Element; + value: Computed; + onSave: (value: string) => Promise; + onDispose: () => void; + getAutocompleteSuggestions(prefix: string): ISuggestionWithValue[]; +}) { + const {refElem, value, onSave, onDispose, getAutocompleteSuggestions} = options; + + const saveAndDispose = async () => { + const editorValue = editor.getValue(); + if (editorValue !== value.get()) { + await onSave(editorValue); + } + if (editor.isDisposed()) { return; } + + editor.dispose(); + }; + + const commands: IEditorCommandGroup = { + fieldEditCancel: () => editor.dispose(), + fieldEditSaveHere: () => editor.blur(), + fieldEditSave: () => editor.blur(), + }; + + const editor = DropdownConditionEditor.create(owner, { + editValue: value.get(), + commands, + onBlur: saveAndDispose, + getAutocompleteSuggestions, + }); + editor.attach(refElem); + editor.onDispose(() => onDispose()); +} + +interface DropdownConditionEditorOptions { + editValue: string; + commands: IEditorCommandGroup; + onBlur(): Promise; + getAutocompleteSuggestions(prefix: string): ISuggestionWithValue[]; +} + +class DropdownConditionEditor extends Disposable { + private _aceEditor: any; + private _dom: HTMLElement; + private _editorPlacement!: EditorPlacement; + private _placementHolder = Holder.create(this); + private _isEmpty: Computed; + + constructor(private _options: DropdownConditionEditorOptions) { + super(); + + const initialValue = _options.editValue; + const editorState = Observable.create(this, initialValue); + + this._aceEditor = this.autoDispose(AceEditor.create({ + calcSize: this._calcSize.bind(this), + editorState, + getSuggestions: _options.getAutocompleteSuggestions, + })); + + this._isEmpty = Computed.create(this, editorState, (_use, state) => state === ''); + this.autoDispose(this._isEmpty.addListener(() => this._updateEditorPlaceholder())); + + const commandGroup = this.autoDispose(createGroup({ + ..._options.commands, + }, this, true)); + + this._dom = cssDropdownConditionEditorWrapper( + cssDropdownConditionEditor( + createMobileButtons(_options.commands), + this._aceEditor.buildDom((aceObj: any) => { + initializeAceOptions(aceObj); + const val = initialValue; + const pos = val.length; + this._aceEditor.setValue(val, pos); + this._aceEditor.attachCommandGroup(commandGroup); + if (val === '') { + this._updateEditorPlaceholder(); + } + }) + ), + ); + } + + public attach(cellElem: Element): void { + this._editorPlacement = EditorPlacement.create(this._placementHolder, this._dom, cellElem, { + margins: getButtonMargins(), + }); + this.autoDispose(this._editorPlacement.onReposition.addListener(this._aceEditor.resize, this._aceEditor)); + this._aceEditor.onAttach(); + this._updateEditorPlaceholder(); + this._aceEditor.resize(); + this._aceEditor.getEditor().focus(); + this._aceEditor.getEditor().on('blur', () => this._options.onBlur()); + } + + public getValue(): string { + return this._aceEditor.getValue(); + } + + public blur() { + this._aceEditor.getEditor().blur(); + } + + private _updateEditorPlaceholder() { + const editor = this._aceEditor.getEditor(); + const shouldShowPlaceholder = editor.session.getValue().length === 0; + if (editor.renderer.emptyMessageNode) { + // Remove the current placeholder if one is present. + editor.renderer.scroller.removeChild(editor.renderer.emptyMessageNode); + } + if (!shouldShowPlaceholder) { + editor.renderer.emptyMessageNode = null; + } else { + editor.renderer.emptyMessageNode = cssDropdownConditionPlaceholder(t('Enter condition.')); + editor.renderer.scroller.appendChild(editor.renderer.emptyMessageNode); + } + } + + private _calcSize(elem: HTMLElement, desiredElemSize: ISize) { + const placeholder: HTMLElement | undefined = this._aceEditor.getEditor().renderer.emptyMessageNode; + if (placeholder) { + return this._editorPlacement.calcSizeWithPadding(elem, { + width: placeholder.scrollWidth, + height: placeholder.scrollHeight, + }); + } else { + return this._editorPlacement.calcSizeWithPadding(elem, { + width: desiredElemSize.width, + height: desiredElemSize.height, + }); + } + } +} + +const cssDropdownConditionField = styled('div', ` + flex: auto; + cursor: pointer; + margin-top: 4px; + + &-disabled { + opacity: 0.4; + pointer-events: none; + } +`); + +const cssDropdownConditionEditorWrapper = styled('div.default_editor.formula_editor_wrapper', ` + border-radius: 3px; +`); + +const cssDropdownConditionEditor = styled('div', ` + background-color: ${theme.aceEditorBg}; + padding: 5px; + z-index: 10; + overflow: hidden; + flex: none; + min-height: 22px; + border-radius: 3px; +`); + +const cssDropdownConditionPlaceholder = styled('div', ` + color: ${theme.lightText}; + font-style: italic; + white-space: nowrap; +`); diff --git a/app/client/components/GristDoc.ts b/app/client/components/GristDoc.ts index 89e08d36..bbbab48b 100644 --- a/app/client/components/GristDoc.ts +++ b/app/client/components/GristDoc.ts @@ -192,8 +192,6 @@ export class GristDoc extends DisposableWithEvents { // Holder for the popped up formula editor. public readonly formulaPopup = Holder.create(this); - public readonly currentTheme = this.docPageModel.appModel.currentTheme; - public get docApi() { return this.docPageModel.appModel.api.getDocAPI(this.docPageModel.currentDocId.get()!); } @@ -238,7 +236,6 @@ export class GristDoc extends DisposableWithEvents { untrustedContentOrigin: app.topAppModel.getUntrustedContentOrigin(), docComm: this.docComm, clientScope: app.clientScope, - theme: this.currentTheme, }); // Maintain the MetaRowModel for the global document info, including docId and peers. diff --git a/app/client/components/Importer.ts b/app/client/components/Importer.ts index da779d6c..5e450d58 100644 --- a/app/client/components/Importer.ts +++ b/app/client/components/Importer.ts @@ -1345,8 +1345,9 @@ export class Importer extends DisposableWithEvents { const column = use(field.column); return use(column.formula); }); - const codeOptions = {gristTheme: this._gristDoc.currentTheme, placeholder: 'Skip', maxLines: 1}; - return cssFieldFormula(formula, codeOptions, + const codeOptions = {placeholder: 'Skip', maxLines: 1}; + return dom.create(buildHighlightedCode, formula, codeOptions, + dom.cls(cssFieldFormula.className), dom.cls('disabled'), dom.cls('formula_field_sidepane'), {tabIndex: '-1'}, @@ -1701,7 +1702,7 @@ const cssColumnMatchRow = styled('div', ` } `); -const cssFieldFormula = styled(buildHighlightedCode, ` +const cssFieldFormula = styled('div', ` flex: auto; cursor: pointer; margin-top: 1px; diff --git a/app/client/components/Printing.ts b/app/client/components/Printing.ts index 3dccfff1..7835d90e 100644 --- a/app/client/components/Printing.ts +++ b/app/client/components/Printing.ts @@ -2,7 +2,7 @@ import {CustomView} from 'app/client/components/CustomView'; import {DataRowModel} from 'app/client/models/DataRowModel'; import DataTableModel from 'app/client/models/DataTableModel'; import {ViewSectionRec} from 'app/client/models/DocModel'; -import {prefersDarkMode, prefersDarkModeObs} from 'app/client/ui2018/cssVars'; +import {prefersColorSchemeDark, prefersColorSchemeDarkObs} from 'app/client/ui2018/theme'; import {dom} from 'grainjs'; type RowId = number|'new'; @@ -45,7 +45,7 @@ export async function printViewSection(layout: any, viewSection: ViewSectionRec) // is Grist temporarily reverting to the light theme until the print dialog is dismissed. // As a workaround, we'll temporarily pause our listener, and unpause after the print dialog // is dismissed. - prefersDarkModeObs().pause(); + prefersColorSchemeDarkObs().pause(); // Hide all layout boxes that do NOT contain the section to be printed. layout?.forEachBox((box: any) => { @@ -87,10 +87,10 @@ export async function printViewSection(layout: any, viewSection: ViewSectionRec) prepareToPrint(false); } delete (window as any).afterPrintCallback; - prefersDarkModeObs().pause(false); + prefersColorSchemeDarkObs().pause(false); // This may have changed while window.print() was blocking. - prefersDarkModeObs().set(prefersDarkMode()); + prefersColorSchemeDarkObs().set(prefersColorSchemeDark()); }); // Running print on a timeout makes it possible to test printing using selenium, and doesn't diff --git a/app/client/components/WidgetFrame.ts b/app/client/components/WidgetFrame.ts index bc6228cb..8a2b1bb8 100644 --- a/app/client/components/WidgetFrame.ts +++ b/app/client/components/WidgetFrame.ts @@ -7,11 +7,11 @@ import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals'; import {makeTestId} from 'app/client/lib/domUtils'; import {ColumnRec, ViewSectionRec} from 'app/client/models/DocModel'; import {reportError} from 'app/client/models/errors'; +import {gristThemeObs} from 'app/client/ui2018/theme'; import {AccessLevel, ICustomWidget, isSatisfied, matchWidget} from 'app/common/CustomWidget'; import {DisposableWithEvents} from 'app/common/DisposableWithEvents'; import {BulkColValues, fromTableDataAction, RowRecord} from 'app/common/DocActions'; import {extractInfoFromColType, reencodeAsAny} from 'app/common/gristTypes'; -import {Theme} from 'app/common/ThemePrefs'; import {getGristConfig} from 'app/common/urlUtils'; import { AccessTokenOptions, CursorPos, CustomSectionAPI, FetchSelectedOptions, GristDocAPI, GristView, @@ -696,10 +696,10 @@ export class ConfigNotifier extends BaseEventSource { * Notifies about theme changes. Exposed in the API as `onThemeChange`. */ export class ThemeNotifier extends BaseEventSource { - constructor(private _theme: Computed) { + constructor() { super(); this.autoDispose( - this._theme.addListener((newTheme, oldTheme) => { + gristThemeObs().addListener((newTheme, oldTheme) => { if (isEqual(newTheme, oldTheme)) { return; } this._update(); @@ -715,7 +715,7 @@ export class ThemeNotifier extends BaseEventSource { if (this.isDisposed()) { return; } this._notify({ - theme: this._theme.get(), + theme: gristThemeObs().get(), fromReady, }); } diff --git a/app/client/lib/ACIndex.ts b/app/client/lib/ACIndex.ts index 96adb4c5..b8bd65c8 100644 --- a/app/client/lib/ACIndex.ts +++ b/app/client/lib/ACIndex.ts @@ -51,6 +51,9 @@ export interface ACResults { // Matching items in order from best match to worst. items: Item[]; + // Additional items to show (e.g. the "Add New" item, for Choice and Reference fields). + extraItems: Item[]; + // May be used to highlight matches using buildHighlightedDom(). highlightFunc: HighlightFunc; @@ -159,7 +162,7 @@ export class ACIndexImpl implements ACIndex { if (!cleanedSearchText) { // In this case we are just returning the first few items. - return {items, highlightFunc: highlightNone, selectIndex: -1}; + return {items, extraItems: [], highlightFunc: highlightNone, selectIndex: -1}; } const highlightFunc = highlightMatches.bind(null, searchWords); @@ -170,7 +173,7 @@ export class ACIndexImpl implements ACIndex { if (selectIndex >= 0 && !startsWithText(items[selectIndex], cleanedSearchText, searchWords)) { selectIndex = -1; } - return {items, highlightFunc, selectIndex}; + return {items, extraItems: [], highlightFunc, selectIndex}; } /** diff --git a/app/client/lib/ACUserManager.ts b/app/client/lib/ACUserManager.ts index ca35d4b7..af873d69 100644 --- a/app/client/lib/ACUserManager.ts +++ b/app/client/lib/ACUserManager.ts @@ -98,7 +98,7 @@ export function buildACMemberEmail( label: text, id: 0, }; - results.items.push(newObject); + results.extraItems.push(newObject); } return results; }; diff --git a/app/client/lib/DocPluginManager.ts b/app/client/lib/DocPluginManager.ts index a43edef5..45012116 100644 --- a/app/client/lib/DocPluginManager.ts +++ b/app/client/lib/DocPluginManager.ts @@ -4,8 +4,6 @@ import {SafeBrowser} from 'app/client/lib/SafeBrowser'; import {ActiveDocAPI} from 'app/common/ActiveDocAPI'; import {LocalPlugin} from 'app/common/plugin'; import {createRpcLogger, PluginInstance} from 'app/common/PluginInstance'; -import {Theme} from 'app/common/ThemePrefs'; -import {Computed} from 'grainjs'; import {Rpc} from 'grain-rpc'; /** @@ -18,7 +16,6 @@ export class DocPluginManager { private _clientScope = this._options.clientScope; private _docComm = this._options.docComm; private _localPlugins = this._options.plugins; - private _theme = this._options.theme; private _untrustedContentOrigin = this._options.untrustedContentOrigin; constructor(private _options: { @@ -26,7 +23,6 @@ export class DocPluginManager { untrustedContentOrigin: string, docComm: ActiveDocAPI, clientScope: ClientScope, - theme: Computed, }) { this.pluginsList = []; for (const plugin of this._localPlugins) { @@ -38,7 +34,6 @@ export class DocPluginManager { clientScope: this._clientScope, untrustedContentOrigin: this._untrustedContentOrigin, mainPath: components.safeBrowser, - theme: this._theme, }); if (components.safeBrowser) { pluginInstance.rpc.registerForwarder(components.safeBrowser, safeBrowser); diff --git a/app/client/lib/HomePluginManager.ts b/app/client/lib/HomePluginManager.ts index 0f5f31b7..28e6bac1 100644 --- a/app/client/lib/HomePluginManager.ts +++ b/app/client/lib/HomePluginManager.ts @@ -2,8 +2,6 @@ import {ClientScope} from 'app/client/components/ClientScope'; import {SafeBrowser} from 'app/client/lib/SafeBrowser'; import {LocalPlugin} from 'app/common/plugin'; import {createRpcLogger, PluginInstance} from 'app/common/PluginInstance'; -import {Theme} from 'app/common/ThemePrefs'; -import {Computed} from 'grainjs'; /** * Home plugins are all plugins that contributes to a general Grist management tasks. @@ -19,9 +17,8 @@ export class HomePluginManager { localPlugins: LocalPlugin[], untrustedContentOrigin: string, clientScope: ClientScope, - theme: Computed, }) { - const {localPlugins, untrustedContentOrigin, clientScope, theme} = options; + const {localPlugins, untrustedContentOrigin, clientScope} = options; this.pluginsList = []; for (const plugin of localPlugins) { try { @@ -41,7 +38,6 @@ export class HomePluginManager { clientScope, untrustedContentOrigin, mainPath: components.safeBrowser, - theme, }); if (components.safeBrowser) { pluginInstance.rpc.registerForwarder(components.safeBrowser, safeBrowser); diff --git a/app/client/lib/ReferenceUtils.ts b/app/client/lib/ReferenceUtils.ts index 5b393ccd..04bf6716 100644 --- a/app/client/lib/ReferenceUtils.ts +++ b/app/client/lib/ReferenceUtils.ts @@ -1,22 +1,36 @@ +import {ACIndex, ACResults} from 'app/client/lib/ACIndex'; +import {makeT} from 'app/client/lib/localization'; +import {ICellItem} from 'app/client/models/ColumnACIndexes'; +import {ColumnCache} from 'app/client/models/ColumnCache'; import {DocData} from 'app/client/models/DocData'; import {ColumnRec} from 'app/client/models/entities/ColumnRec'; import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec'; import {TableData} from 'app/client/models/TableData'; import {getReferencedTableId, isRefListType} from 'app/common/gristTypes'; +import {EmptyRecordView} from 'app/common/PredicateFormula'; import {BaseFormatter} from 'app/common/ValueFormatter'; +import {Disposable, dom, Observable} from 'grainjs'; + +const t = makeT('ReferenceUtils'); /** * Utilities for common operations involving Ref[List] fields. */ -export class ReferenceUtils { +export class ReferenceUtils extends Disposable { public readonly refTableId: string; public readonly tableData: TableData; public readonly visibleColFormatter: BaseFormatter; public readonly visibleColModel: ColumnRec; public readonly visibleColId: string; public readonly isRefList: boolean; + public readonly hasDropdownCondition = Boolean(this.field.dropdownCondition.peek()?.text); + + private readonly _columnCache: ColumnCache>; + private _dropdownConditionError = Observable.create(this, null); + + constructor(public readonly field: ViewFieldRec, private readonly _docData: DocData) { + super(); - constructor(public readonly field: ViewFieldRec, docData: DocData) { const colType = field.column().type(); const refTableId = getReferencedTableId(colType); if (!refTableId) { @@ -24,7 +38,7 @@ export class ReferenceUtils { } this.refTableId = refTableId; - const tableData = docData.getTable(refTableId); + const tableData = _docData.getTable(refTableId); if (!tableData) { throw new Error("Invalid referenced table " + refTableId); } @@ -34,6 +48,8 @@ export class ReferenceUtils { this.visibleColModel = field.visibleColModel(); this.visibleColId = this.visibleColModel.colId() || 'id'; this.isRefList = isRefListType(colType); + + this._columnCache = new ColumnCache>(this.tableData); } public idToText(value: unknown) { @@ -43,10 +59,86 @@ export class ReferenceUtils { return String(value || ''); } - public autocompleteSearch(text: string) { - const acIndex = this.tableData.columnACIndexes.getColACIndex(this.visibleColId, this.visibleColFormatter); + /** + * Searches the autocomplete index for the given `text`, returning + * all matching results and related metadata. + * + * If a dropdown condition is set, results are dependent on the `rowId` + * that the autocomplete dropdown is open in. Otherwise, `rowId` has no + * effect. + */ + public autocompleteSearch(text: string, rowId: number): ACResults { + let acIndex: ACIndex; + if (this.hasDropdownCondition) { + try { + acIndex = this._getDropdownConditionACIndex(rowId); + } catch (e) { + this._dropdownConditionError?.set(e); + return {items: [], extraItems: [], highlightFunc: () => [], selectIndex: -1}; + } + } else { + acIndex = this.tableData.columnACIndexes.getColACIndex( + this.visibleColId, + this.visibleColFormatter, + ); + } return acIndex.search(text); } + + public buildNoItemsMessage() { + return dom.domComputed(use => { + const error = use(this._dropdownConditionError); + if (error) { return t('Error in dropdown condition'); } + + return this.hasDropdownCondition + ? t('No choices matching condition') + : t('No choices to select'); + }); + } + + /** + * Returns a column index for the visible column, filtering the items in the + * index according to the set dropdown condition. + * + * This method is similar to `this.tableData.columnACIndexes.getColACIndex`, + * but whereas that method caches indexes globally, this method does so + * locally (as a new instances of this class is created each time a Reference + * or Reference List editor is created). + * + * It's important that this method be used when a dropdown condition is set, + * as items in indexes that don't satisfy the dropdown condition need to be + * filtered. + */ + private _getDropdownConditionACIndex(rowId: number) { + return this._columnCache.getValue( + this.visibleColId, + () => this.tableData.columnACIndexes.buildColACIndex( + this.visibleColId, + this.visibleColFormatter, + this._buildDropdownConditionACFilter(rowId) + ) + ); + } + + private _buildDropdownConditionACFilter(rowId: number) { + const dropdownConditionCompiled = this.field.dropdownConditionCompiled.get(); + if (dropdownConditionCompiled?.kind !== 'success') { + throw new Error('Dropdown condition is not compiled'); + } + + const tableId = this.field.tableId.peek(); + const table = this._docData.getTable(tableId); + if (!table) { throw new Error(`Table ${tableId} not found`); } + + const {result: predicate} = dropdownConditionCompiled; + const rec = table.getRecord(rowId) || new EmptyRecordView(); + return (item: ICellItem) => { + const choice = item.rowId === 'new' ? new EmptyRecordView() : this.tableData.getRecord(item.rowId); + if (!choice) { throw new Error(`Reference ${item.rowId} not found`); } + + return predicate({rec, choice}); + }; + } } export function nocaseEqual(a: string, b: string) { diff --git a/app/client/lib/SafeBrowser.ts b/app/client/lib/SafeBrowser.ts index 35182cb9..602d2bc2 100644 --- a/app/client/lib/SafeBrowser.ts +++ b/app/client/lib/SafeBrowser.ts @@ -33,6 +33,7 @@ import { ClientScope } from 'app/client/components/ClientScope'; import { get as getBrowserGlobals } from 'app/client/lib/browserGlobals'; import dom from 'app/client/lib/dom'; import * as Mousetrap from 'app/client/lib/Mousetrap'; +import { gristThemeObs } from 'app/client/ui2018/theme'; import { ActionRouter } from 'app/common/ActionRouter'; import { BaseComponent, BaseLogger, createRpcLogger, PluginInstance, warnIfNotReady } from 'app/common/PluginInstance'; import { tbind } from 'app/common/tbind'; @@ -41,7 +42,7 @@ import { getOriginUrl } from 'app/common/urlUtils'; import { GristAPI, RPC_GRISTAPI_INTERFACE } from 'app/plugin/GristAPI'; import { RenderOptions, RenderTarget } from 'app/plugin/RenderOptions'; import { checkers } from 'app/plugin/TypeCheckers'; -import { Computed, dom as grainjsDom, Observable } from 'grainjs'; +import { dom as grainjsDom, Observable } from 'grainjs'; import { IMsgCustom, IMsgRpcCall, IRpcLogger, MsgType, Rpc } from 'grain-rpc'; import { Disposable } from './dispose'; import isEqual from 'lodash/isEqual'; @@ -73,8 +74,6 @@ export class SafeBrowser extends BaseComponent { new IframeProcess(safeBrowser, rpc, src); } - public theme = this._options.theme; - // All view processes. This is not used anymore to dispose all processes on deactivation (this is // now achieved using `this._mainProcess.autoDispose(...)`) but rather to be able to dispatch // events to all processes (such as doc actions which will need soon). @@ -94,7 +93,6 @@ export class SafeBrowser extends BaseComponent { pluginInstance: PluginInstance, clientScope: ClientScope, untrustedContentOrigin: string, - theme: Computed, mainPath?: string, baseLogger?: BaseLogger, rpcLogger?: IRpcLogger, @@ -312,7 +310,7 @@ class IframeProcess extends ViewProcess { const listener = async (event: MessageEvent) => { if (event.source === iframe.contentWindow) { if (event.data.mtype === MsgType.Ready) { - await this._sendTheme({theme: safeBrowser.theme.get(), fromReady: true}); + await this._sendTheme({theme: gristThemeObs().get(), fromReady: true}); } if (event.data.data?.message === 'themeInitialized') { @@ -328,15 +326,11 @@ class IframeProcess extends ViewProcess { }); this.rpc.setSendMessage(msg => iframe.contentWindow!.postMessage(msg, '*')); - if (safeBrowser.theme) { - this.autoDispose( - safeBrowser.theme.addListener(async (newTheme, oldTheme) => { - if (isEqual(newTheme, oldTheme)) { return; } + this.autoDispose(gristThemeObs().addListener(async (newTheme, oldTheme) => { + if (isEqual(newTheme, oldTheme)) { return; } - await this._sendTheme({theme: newTheme}); - }) - ); - } + await this._sendTheme({theme: newTheme}); + })); } private async _sendTheme({theme, fromReady = false}: {theme: Theme, fromReady?: boolean}) { diff --git a/app/client/lib/autocomplete.ts b/app/client/lib/autocomplete.ts index 8ad4a828..9f5871d3 100644 --- a/app/client/lib/autocomplete.ts +++ b/app/client/lib/autocomplete.ts @@ -4,7 +4,8 @@ import {createPopper, Modifier, Instance as Popper, Options as PopperOptions} from '@popperjs/core'; import {ACItem, ACResults, HighlightFunc} from 'app/client/lib/ACIndex'; import {reportError} from 'app/client/models/errors'; -import {Disposable, dom, EventCB, IDisposable} from 'grainjs'; +import {testId, theme} from 'app/client/ui2018/cssVars'; +import {Disposable, dom, DomContents, EventCB, IDisposable} from 'grainjs'; import {obsArray, onKeyElem, styled} from 'grainjs'; import merge = require('lodash/merge'); import maxSize from 'popper-max-size-modifier'; @@ -26,6 +27,9 @@ export interface IAutocompleteOptions { // Defaults to the document body. attach?: Element|string|null; + // If provided, builds and shows the message when there are no items (excluding any extra items). + buildNoItemsMessage?: () => DomContents; + // Given a search term, return the list of Items to render. search(searchText: string): Promise>; @@ -46,7 +50,7 @@ export class Autocomplete extends Disposable { // The UL element containing the actual menu items. protected _menuContent: HTMLElement; - // Index into _items as well as into _menuContent, -1 if nothing selected. + // Index into _menuContent, -1 if nothing selected. protected _selectedIndex: number = -1; // Currently selected element. @@ -56,6 +60,7 @@ export class Autocomplete extends Disposable { private _mouseOver: {reset(): void}; private _lastAsTyped: string; private _items = this.autoDispose(obsArray([])); + private _extraItems = this.autoDispose(obsArray([])); private _highlightFunc: HighlightFunc; constructor( @@ -65,14 +70,19 @@ export class Autocomplete extends Disposable { super(); const content = cssMenuWrap( - this._menuContent = cssMenu({class: _options.menuCssClass || ''}, - dom.forEach(this._items, (item) => _options.renderItem(item, this._highlightFunc)), + cssMenu( + {class: _options.menuCssClass || ''}, dom.style('min-width', _triggerElem.getBoundingClientRect().width + 'px'), - dom.on('mouseleave', (ev) => this._setSelected(-1, true)), - dom.on('click', (ev) => { - this._setSelected(this._findTargetItem(ev.target), true); - if (_options.onClick) { _options.onClick(); } - }) + this._maybeShowNoItemsMessage(), + this._menuContent = dom('div', + dom.forEach(this._items, (item) => _options.renderItem(item, this._highlightFunc)), + dom.forEach(this._extraItems, (item) => _options.renderItem(item, this._highlightFunc)), + dom.on('mouseleave', (ev) => this._setSelected(-1, true)), + dom.on('click', (ev) => { + this._setSelected(this._findTargetItem(ev.target), true); + if (_options.onClick) { _options.onClick(); } + }), + ), ), // Prevent trigger element from being blurred on click. dom.on('mousedown', (ev) => ev.preventDefault()), @@ -104,7 +114,7 @@ export class Autocomplete extends Disposable { } public getSelectedItem(): Item|undefined { - return this._items.get()[this._selectedIndex]; + return this._allItems[this._selectedIndex]; } public search(findMatch?: (items: Item[]) => number) { @@ -145,7 +155,7 @@ export class Autocomplete extends Disposable { private _getNext(step: 1 | -1): number { // Pretend there is an extra element at the end to mean "nothing selected". - const xsize = this._items.get().length + 1; + const xsize = this._allItems.length + 1; const next = (this._selectedIndex + step + xsize) % xsize; return (next === xsize - 1) ? -1 : next; } @@ -157,6 +167,7 @@ export class Autocomplete extends Disposable { const acResults = await this._options.search(inputVal); this._highlightFunc = acResults.highlightFunc; this._items.set(acResults.items); + this._extraItems.set(acResults.extraItems); // Plain update() (which is deferred) may be better, but if _setSelected() causes scrolling // before the positions are updated, it causes the entire page to scroll horizontally. @@ -166,12 +177,24 @@ export class Autocomplete extends Disposable { let index: number; if (findMatch) { - index = findMatch(this._items.get()); + index = findMatch(this._allItems); } else { index = inputVal ? acResults.selectIndex : -1; } this._setSelected(index, false); } + + private get _allItems() { + return [...this._items.get(), ...this._extraItems.get()]; + } + + private _maybeShowNoItemsMessage() { + const {buildNoItemsMessage} = this._options; + if (!buildNoItemsMessage) { return null; } + + return dom.maybe(use => use(this._items).length === 0, () => + cssNoItemsMessage(buildNoItemsMessage(), testId('autocomplete-no-items-message'))); + } } @@ -253,3 +276,10 @@ const cssMenuWrap = styled('div', ` flex-direction: column; outline: none; `); + +const cssNoItemsMessage = styled('div', ` + color: ${theme.lightText}; + padding: var(--weaseljs-menu-item-padding, 8px 24px); + text-align: center; + user-select: none; +`); diff --git a/app/client/lib/imports.d.ts b/app/client/lib/imports.d.ts index 7dd719d8..942d224c 100644 --- a/app/client/lib/imports.d.ts +++ b/app/client/lib/imports.d.ts @@ -6,17 +6,20 @@ import * as GristDocModule from 'app/client/components/GristDoc'; import * as ViewPane from 'app/client/components/ViewPane'; import * as UserManagerModule from 'app/client/ui/UserManager'; import * as searchModule from 'app/client/ui2018/search'; +import * as ace from 'ace-builds'; import * as momentTimezone from 'moment-timezone'; import * as plotly from 'plotly.js'; -export type PlotlyType = typeof plotly; +export type Ace = typeof ace; export type MomentTimezone = typeof momentTimezone; +export type PlotlyType = typeof plotly; export function loadAccountPage(): Promise; export function loadActivationPage(): Promise; export function loadBillingPage(): Promise; export function loadAdminPanel(): Promise; export function loadGristDoc(): Promise; +export function loadAce(): Promise; export function loadMomentTimezone(): Promise; export function loadPlotly(): Promise; export function loadSearch(): Promise; diff --git a/app/client/lib/imports.js b/app/client/lib/imports.js index 7747d77d..5218aa27 100644 --- a/app/client/lib/imports.js +++ b/app/client/lib/imports.js @@ -13,6 +13,17 @@ exports.loadAdminPanel = () => import('app/client/ui/AdminPanel' /* webpackChunk exports.loadGristDoc = () => import('app/client/components/GristDoc' /* webpackChunkName: "GristDoc" */); // When importing this way, the module is under the "default" member, not sure why (maybe // esbuild-loader's doing). +exports.loadAce = () => import('ace-builds') + .then(async (m) => { + await Promise.all([ + import('ace-builds/src-noconflict/ext-static_highlight'), + import('ace-builds/src-noconflict/mode-python'), + import('ace-builds/src-noconflict/theme-chrome'), + import('ace-builds/src-noconflict/theme-dracula'), + ]); + + return m.default; + }); exports.loadMomentTimezone = () => import('moment-timezone').then(m => m.default); exports.loadPlotly = () => import('plotly.js-basic-dist' /* webpackChunkName: "plotly" */); exports.loadSearch = () => import('app/client/ui2018/search' /* webpackChunkName: "search" */); diff --git a/app/client/lib/nameUtils.ts b/app/client/lib/nameUtils.ts new file mode 100644 index 00000000..86037d27 --- /dev/null +++ b/app/client/lib/nameUtils.ts @@ -0,0 +1,14 @@ +/** + * We allow alphanumeric characters and certain common whitelisted characters (except at the start), + * plus everything non-ASCII (for non-English alphabets, which we want to allow but it's hard to be + * more precise about what exactly to allow). + */ +// eslint-disable-next-line no-control-regex +const VALID_NAME_REGEXP = /^(\w|[^\u0000-\u007F])(\w|[- ./'"()]|[^\u0000-\u007F])*$/; + +/** + * Test name against various rules to check if it is a valid username. + */ +export function checkName(name: string): boolean { + return VALID_NAME_REGEXP.test(name); +} diff --git a/app/client/lib/timeUtils.ts b/app/client/lib/timeUtils.ts new file mode 100644 index 00000000..637b2a97 --- /dev/null +++ b/app/client/lib/timeUtils.ts @@ -0,0 +1,21 @@ +import moment from 'moment'; + +/** + * Given a UTC Date ISO 8601 string (the doc updatedAt string), gives a reader-friendly + * relative time to now - e.g. 'yesterday', '2 days ago'. + */ +export function getTimeFromNow(utcDateISO: string): string { + const time = moment.utc(utcDateISO); + const now = moment(); + const diff = now.diff(time, 's'); + if (diff < 0 && diff > -60) { + // If the time appears to be in the future, but less than a minute + // in the future, chalk it up to a difference in time + // synchronization and don't claim the resource will be changed in + // the future. For larger differences, just report them + // literally, there's a more serious problem or lack of + // synchronization. + return now.fromNow(); + } + return time.fromNow(); +} diff --git a/app/client/models/AppModel.ts b/app/client/models/AppModel.ts index a2c12811..30ea4799 100644 --- a/app/client/models/AppModel.ts +++ b/app/client/models/AppModel.ts @@ -10,7 +10,7 @@ import {Notifier} from 'app/client/models/NotifyModel'; import {getFlavor, ProductFlavor} from 'app/client/ui/CustomThemes'; import {buildNewSiteModal, buildUpgradeModal} from 'app/client/ui/ProductUpgrades'; import {SupportGristNudge} from 'app/client/ui/SupportGristNudge'; -import {prefersDarkModeObs} from 'app/client/ui2018/cssVars'; +import {gristThemePrefs} from 'app/client/ui2018/theme'; import {AsyncCreate} from 'app/common/AsyncCreate'; import {ICustomWidget} from 'app/common/CustomWidget'; import {OrgUsageSummary} from 'app/common/DocUsage'; @@ -21,9 +21,7 @@ import {LocalPlugin} from 'app/common/plugin'; import {DismissedPopup, DismissedReminder, UserPrefs} from 'app/common/Prefs'; import {isOwner, isOwnerOrEditor} from 'app/common/roles'; import {getTagManagerScript} from 'app/common/tagManager'; -import {getDefaultThemePrefs, Theme, ThemeColors, ThemePrefs, - ThemePrefsChecker} from 'app/common/ThemePrefs'; -import {getThemeColors} from 'app/common/Themes'; +import {getDefaultThemePrefs, ThemePrefs, ThemePrefsChecker} from 'app/common/ThemePrefs'; import {getGristConfig} from 'app/common/urlUtils'; import {ExtendedUser} from 'app/common/UserAPI'; import {getOrgName, isTemplatesOrg, Organization, OrgError, UserAPI, UserAPIImpl} from 'app/common/UserAPI'; @@ -118,7 +116,6 @@ export interface AppModel { userPrefsObs: Observable; themePrefs: Observable; - currentTheme: Computed; /** * Popups that user has seen. */ @@ -170,8 +167,9 @@ export class TopAppModelImpl extends Disposable implements TopAppModel { private readonly _widgets: AsyncCreate; constructor(window: {gristConfig?: GristLoadConfig}, - public readonly api: UserAPI = newUserAPIImpl(), - public readonly options: TopAppModelOptions = {}) { + public readonly api: UserAPI = newUserAPIImpl(), + public readonly options: TopAppModelOptions = {} + ) { super(); setErrorNotifier(this.notifier); this.isSingleOrg = Boolean(window.gristConfig && window.gristConfig.singleOrg); @@ -307,7 +305,6 @@ export class AppModelImpl extends Disposable implements AppModel { defaultValue: getDefaultThemePrefs(), checker: ThemePrefsChecker, }) as Observable; - public readonly currentTheme = this._getCurrentThemeObs(); public readonly dismissedPopups = getUserPrefObs(this.userPrefsObs, 'dismissedPopups', { defaultValue: [] }) as Observable; @@ -359,6 +356,11 @@ export class AppModelImpl extends Disposable implements AppModel { public readonly orgError?: OrgError, ) { super(); + + // Whenever theme preferences change, update the global `gristThemePrefs` observable; this triggers + // an automatic update to the global `gristThemeObs` computed observable. + this.autoDispose(subscribe(this.themePrefs, (_use, themePrefs) => gristThemePrefs.set(themePrefs))); + this._recordSignUpIfIsNewUser(); const state = urlState().state.get(); @@ -493,41 +495,14 @@ export class AppModelImpl extends Disposable implements AppModel { dataLayer.push({event: 'new-sign-up'}); getUserPrefObs(this.userPrefsObs, 'recordSignUpEvent').set(undefined); } +} - private _getCurrentThemeObs() { - return Computed.create(this, this.themePrefs, prefersDarkModeObs(), - (_use, themePrefs, prefersDarkMode) => { - let {appearance, syncWithOS} = themePrefs; - - const urlParams = urlState().state.get().params; - if (urlParams?.themeAppearance) { - appearance = urlParams?.themeAppearance; - } - - if (urlParams?.themeSyncWithOs !== undefined) { - syncWithOS = urlParams?.themeSyncWithOs; - } - - if (syncWithOS) { - appearance = prefersDarkMode ? 'dark' : 'light'; - } - - let nameOrColors = themePrefs.colors[appearance]; - if (urlParams?.themeName) { - nameOrColors = urlParams?.themeName; - } - - let colors: ThemeColors; - if (typeof nameOrColors === 'string') { - colors = getThemeColors(nameOrColors); - } else { - colors = nameOrColors; - } - - return {appearance, colors}; - }, - ); +export function getOrgNameOrGuest(org: Organization|null, user: FullUser|null) { + if (!org) { return ''; } + if (user && user.anonymous && org.owner && org.owner.id === user.id) { + return "@Guest"; } + return getOrgName(org); } export function getHomeUrl(): string { @@ -541,11 +516,3 @@ export function newUserAPIImpl(): UserAPIImpl { fetch: hooks.fetch, }); } - -export function getOrgNameOrGuest(org: Organization|null, user: FullUser|null) { - if (!org) { return ''; } - if (user && user.anonymous && org.owner && org.owner.id === user.id) { - return "@Guest"; - } - return getOrgName(org); -} diff --git a/app/client/models/ColumnACIndexes.ts b/app/client/models/ColumnACIndexes.ts index bd323a1e..3df7431d 100644 --- a/app/client/models/ColumnACIndexes.ts +++ b/app/client/models/ColumnACIndexes.ts @@ -21,7 +21,6 @@ export interface ICellItem { cleanText: string; // Trimmed lowercase text for searching. } - export class ColumnACIndexes { private _columnCache = new ColumnCache>(this._tableData); @@ -33,22 +32,28 @@ export class ColumnACIndexes { * getColACIndex() is called for the same column with the the same formatter. */ public getColACIndex(colId: string, formatter: BaseFormatter): ACIndex { - return this._columnCache.getValue(colId, () => this._buildColACIndex(colId, formatter)); + return this._columnCache.getValue(colId, () => this.buildColACIndex(colId, formatter)); } - private _buildColACIndex(colId: string, formatter: BaseFormatter): ACIndex { + public buildColACIndex( + colId: string, + formatter: BaseFormatter, + filter?: (item: ICellItem) => boolean + ): ACIndex { const rowIds = this._tableData.getRowIds(); const valColumn = this._tableData.getColValues(colId); if (!valColumn) { throw new UserError(`Invalid column ${this._tableData.tableId}.${colId}`); } - const items: ICellItem[] = valColumn.map((val, i) => { - const rowId = rowIds[i]; - const text = formatter.formatAny(val); - const cleanText = normalizeText(text); - return {rowId, text, cleanText}; - }); - items.sort(itemCompare); + const items: ICellItem[] = valColumn + .map((val, i) => { + const rowId = rowIds[i]; + const text = formatter.formatAny(val); + const cleanText = normalizeText(text); + return {rowId, text, cleanText}; + }) + .filter((item) => filter?.(item) ?? true) + .sort(itemCompare); return new ACIndexImpl(items); } } diff --git a/app/client/models/HomeModel.ts b/app/client/models/HomeModel.ts index ee62e38e..ba4bd1d7 100644 --- a/app/client/models/HomeModel.ts +++ b/app/client/models/HomeModel.ts @@ -14,30 +14,11 @@ import * as roles from 'app/common/roles'; import {getGristConfig} from 'app/common/urlUtils'; import {Document, Organization, Workspace} from 'app/common/UserAPI'; import {bundleChanges, Computed, Disposable, Observable, subscribe} from 'grainjs'; -import moment from 'moment'; import flatten = require('lodash/flatten'); import sortBy = require('lodash/sortBy'); const DELAY_BEFORE_SPINNER_MS = 500; -// Given a UTC Date ISO 8601 string (the doc updatedAt string), gives a reader-friendly -// relative time to now - e.g. 'yesterday', '2 days ago'. -export function getTimeFromNow(utcDateISO: string): string { - const time = moment.utc(utcDateISO); - const now = moment(); - const diff = now.diff(time, 's'); - if (diff < 0 && diff > -60) { - // If the time appears to be in the future, but less than a minute - // in the future, chalk it up to a difference in time - // synchronization and don't claim the resource will be changed in - // the future. For larger differences, just report them - // literally, there's a more serious problem or lack of - // synchronization. - return now.fromNow(); - } - return time.fromNow(); -} - export interface HomeModel { // PageType value, one of the discriminated union values used by AppModel. pageType: "home"; @@ -190,7 +171,6 @@ export class HomeModelImpl extends Disposable implements HomeModel, ViewSettings localPlugins: _app.topAppModel.plugins, untrustedContentOrigin: _app.topAppModel.getUntrustedContentOrigin()!, clientScope, - theme: _app.currentTheme, }); const importSources = ImportSourceElement.fromArray(pluginManager.pluginsList); this.importSources.set(importSources); diff --git a/app/client/models/entities/ViewFieldRec.ts b/app/client/models/entities/ViewFieldRec.ts index 5f0c045a..fed7d80d 100644 --- a/app/client/models/entities/ViewFieldRec.ts +++ b/app/client/models/entities/ViewFieldRec.ts @@ -2,12 +2,15 @@ import {ColumnRec, DocModel, IRowModel, refListRecords, refRecord, ViewSectionRe import {formatterForRec} from 'app/client/models/entities/ColumnRec'; import * as modelUtil from 'app/client/models/modelUtil'; import {removeRule, RuleOwner} from 'app/client/models/RuleOwner'; -import { HeaderStyle, Style } from 'app/client/models/Styles'; +import {HeaderStyle, Style} from 'app/client/models/Styles'; import {ViewFieldConfig} from 'app/client/models/ViewFieldConfig'; import * as UserType from 'app/client/widgets/UserType'; import {DocumentSettings} from 'app/common/DocumentSettings'; +import {DropdownCondition, DropdownConditionCompilationResult} from 'app/common/DropdownCondition'; +import {compilePredicateFormula} from 'app/common/PredicateFormula'; import {BaseFormatter} from 'app/common/ValueFormatter'; import {createParser} from 'app/common/ValueParser'; +import {Computed} from 'grainjs'; import * as ko from 'knockout'; // Represents a page entry in the tree of pages. @@ -106,6 +109,9 @@ export interface ViewFieldRec extends IRowModel<"_grist_Views_section_field">, R /** Label in FormView. By default FormView uses label, use this to override it. */ question: modelUtil.KoSaveableObservable; + dropdownCondition: modelUtil.KoSaveableObservable; + dropdownConditionCompiled: Computed; + createValueParser(): (value: string) => any; // Helper which adds/removes/updates field's displayCol to match the formula. @@ -316,4 +322,21 @@ export function createViewFieldRec(this: ViewFieldRec, docModel: DocModel): void this.disableModify = this.autoDispose(ko.pureComputed(() => this.column().disableModify())); this.disableEditData = this.autoDispose(ko.pureComputed(() => this.column().disableEditData())); + + this.dropdownCondition = this.widgetOptionsJson.prop('dropdownCondition'); + this.dropdownConditionCompiled = Computed.create(this, use => { + const dropdownCondition = use(this.dropdownCondition); + if (!dropdownCondition?.parsed) { return null; } + + try { + return { + kind: 'success', + result: compilePredicateFormula(JSON.parse(dropdownCondition.parsed), { + variant: 'dropdown-condition', + }), + }; + } catch (e) { + return {kind: 'failure', error: e.message}; + } + }); } diff --git a/app/client/ui/AccountPage.ts b/app/client/ui/AccountPage.ts index 5eaafeb9..e68be43e 100644 --- a/app/client/ui/AccountPage.ts +++ b/app/client/ui/AccountPage.ts @@ -1,4 +1,5 @@ import {detectCurrentLang, makeT} from 'app/client/lib/localization'; +import {checkName} from 'app/client/lib/nameUtils'; import {AppModel, reportError} from 'app/client/models/AppModel'; import {urlState} from 'app/client/models/gristUrlState'; import * as css from 'app/client/ui/AccountPageCss'; @@ -249,23 +250,6 @@ designed to ensure that you're the only person who can access your account, even } } -/** - * We allow alphanumeric characters and certain common whitelisted characters (except at the start), - * plus everything non-ASCII (for non-English alphabets, which we want to allow but it's hard to be - * more precise about what exactly to allow). - */ -// eslint-disable-next-line no-control-regex -const VALID_NAME_REGEXP = /^(\w|[^\u0000-\u007F])(\w|[- ./'"()]|[^\u0000-\u007F])*$/; - -/** - * Test name against various rules to check if it is a valid username. - */ -export function checkName(name: string): boolean { - return VALID_NAME_REGEXP.test(name); -} - - - const cssWarnings = styled(css.warning, ` margin: -8px 0 0 110px; `); diff --git a/app/client/ui/App.ts b/app/client/ui/App.ts index 322c2fd6..2b2b1484 100644 --- a/app/client/ui/App.ts +++ b/app/client/ui/App.ts @@ -14,6 +14,7 @@ import {setUpErrorHandling} from 'app/client/models/errors'; import {createAppUI} from 'app/client/ui/AppUI'; import {addViewportTag} from 'app/client/ui/viewport'; import {attachCssRootVars} from 'app/client/ui2018/cssVars'; +import {attachTheme} from 'app/client/ui2018/theme'; import {BaseAPI} from 'app/common/BaseAPI'; import {CommDocError} from 'app/common/CommTypes'; import {DisposableWithEvents} from 'app/common/DisposableWithEvents'; @@ -183,6 +184,7 @@ export class App extends DisposableWithEvents { // Add the cssRootVars class to enable the variables in cssVars. attachCssRootVars(this.topAppModel.productFlavor); + attachTheme(); addViewportTag(); this.autoDispose(createAppUI(this.topAppModel, this)); } diff --git a/app/client/ui/AppUI.ts b/app/client/ui/AppUI.ts index ccbe272b..10e7b840 100644 --- a/app/client/ui/AppUI.ts +++ b/app/client/ui/AppUI.ts @@ -17,7 +17,7 @@ import {pagePanels} from 'app/client/ui/PagePanels'; import {RightPanel} from 'app/client/ui/RightPanel'; import {createTopBarDoc, createTopBarHome} from 'app/client/ui/TopBar'; import {WelcomePage} from 'app/client/ui/WelcomePage'; -import {attachTheme, testId} from 'app/client/ui2018/cssVars'; +import {testId} from 'app/client/ui2018/cssVars'; import {getPageTitleSuffix} from 'app/common/gristUrls'; import {getGristConfig} from 'app/common/urlUtils'; import {Computed, dom, IDisposable, IDisposableOwner, Observable, replaceContent, subscribe} from 'grainjs'; @@ -27,9 +27,7 @@ import {Computed, dom, IDisposable, IDisposableOwner, Observable, replaceContent // TODO once #newui is gone, we don't need to worry about this being disposable. // appObj is the App object from app/client/ui/App.ts export function createAppUI(topAppModel: TopAppModel, appObj: App): IDisposable { - const content = dom.maybeOwned(topAppModel.appObs, (owner, appModel) => { - owner.autoDispose(attachTheme(appModel.currentTheme)); - + const content = dom.maybe(topAppModel.appObs, (appModel) => { return [ createMainPage(appModel, appObj), buildSnackbarDom(appModel.notifier, appModel), diff --git a/app/client/ui/CodeHighlight.ts b/app/client/ui/CodeHighlight.ts index f80a4907..09aa21f8 100644 --- a/app/client/ui/CodeHighlight.ts +++ b/app/client/ui/CodeHighlight.ts @@ -1,45 +1,112 @@ -import * as ace from 'ace-builds'; +import {Ace, loadAce} from 'app/client/lib/imports'; import {theme, vars} from 'app/client/ui2018/cssVars'; -import {Theme} from 'app/common/ThemePrefs'; -import {getGristConfig} from 'app/common/urlUtils'; -import {BindableValue, Computed, dom, DomElementArg, Observable, styled, subscribeElem} from 'grainjs'; +import {gristThemeObs} from 'app/client/ui2018/theme'; +import { + BindableValue, + Disposable, + DomElementArg, + Observable, + styled, + subscribeElem, +} from 'grainjs'; -// ace-builds also has a minified build (src-min-noconflict), but we don't -// use it since webpack already handles minification. -require('ace-builds/src-noconflict/ext-static_highlight'); -require('ace-builds/src-noconflict/mode-python'); -require('ace-builds/src-noconflict/theme-chrome'); -require('ace-builds/src-noconflict/theme-dracula'); - -export interface ICodeOptions { - gristTheme: Computed; - placeholder?: string; +interface BuildCodeHighlighterOptions { maxLines?: number; } +let _ace: Ace; +let _highlighter: any; +let _PythonMode: any; +let _aceDom: any; +let _chrome: any; +let _dracula: any; +let _mode: any; + +async function fetchAceModules() { + return { + ace: _ace || (_ace = await loadAce()), + highlighter: _highlighter || (_highlighter = _ace.require('ace/ext/static_highlight')), + PythonMode: _PythonMode || (_PythonMode = _ace.require('ace/mode/python').Mode), + aceDom: _aceDom || (_aceDom = _ace.require('ace/lib/dom')), + chrome: _chrome || (_chrome = _ace.require('ace/theme/chrome')), + dracula: _dracula || (_dracula = _ace.require('ace/theme/dracula')), + mode: _mode || (_mode = new _PythonMode()), + }; +} + +/** + * Returns a function that accepts a string of text representing code and returns + * a highlighted version of it as an HTML string. + * + * This is useful for scenarios where highlighted code needs to be displayed outside of + * grainjs. For example, when using `marked`'s `highlight` option to highlight code + * blocks in a Markdown string. + */ +export async function buildCodeHighlighter(options: BuildCodeHighlighterOptions = {}) { + const {maxLines} = options; + const {highlighter, aceDom, chrome, dracula, mode} = await fetchAceModules(); + + return (code: string) => { + if (maxLines) { + // If requested, trim to maxLines, and add an ellipsis at the end. + // (Long lines are also truncated with an ellpsis via text-overflow style.) + const lines = code.split(/\n/); + if (lines.length > maxLines) { + code = lines.slice(0, maxLines).join("\n") + " \u2026"; // Ellipsis + } + } + + let aceThemeName: 'chrome' | 'dracula'; + let aceTheme: any; + if (gristThemeObs().get().appearance === 'dark') { + aceThemeName = 'dracula'; + aceTheme = dracula; + } else { + aceThemeName = 'chrome'; + aceTheme = chrome; + } + + // Rendering highlighted code gives you back the HTML to insert into the DOM, as well + // as the CSS styles needed to apply the theme. The latter typically isn't included in + // the document until an Ace editor is opened, so we explicitly import it here to avoid + // leaving highlighted code blocks without a theme applied. + const {html, css} = highlighter.render(code, mode, aceTheme, 1, true); + aceDom.importCssString(css, `${aceThemeName}-highlighted-code`); + return html; + }; +} + +interface BuildHighlightedCodeOptions extends BuildCodeHighlighterOptions { + placeholder?: string; +} + +/** + * Builds a block of highlighted `code`. + * + * Highlighting applies an appropriate Ace theme (Chrome or Dracula) based on + * the current Grist theme, and automatically re-applies it whenever the Grist + * theme changes. + */ export function buildHighlightedCode( - code: BindableValue, options: ICodeOptions, ...args: DomElementArg[] + owner: Disposable, + code: BindableValue, + options: BuildHighlightedCodeOptions, + ...args: DomElementArg[] ): HTMLElement { - const {gristTheme, placeholder, maxLines} = options; - const {enableCustomCss} = getGristConfig(); + const {placeholder, maxLines} = options; + const codeText = Observable.create(owner, ''); + const codeTheme = Observable.create(owner, gristThemeObs().get()); - const highlighter = ace.require('ace/ext/static_highlight'); - const PythonMode = ace.require('ace/mode/python').Mode; - const aceDom = ace.require('ace/lib/dom'); - const chrome = ace.require('ace/theme/chrome'); - const dracula = ace.require('ace/theme/dracula'); - const mode = new PythonMode(); - - const codeText = Observable.create(null, ''); - const codeTheme = Observable.create(null, gristTheme.get()); - - function updateHighlightedCode(elem: HTMLElement) { + async function updateHighlightedCode(elem: HTMLElement) { let text = codeText.get(); if (!text) { elem.textContent = placeholder || ''; return; } + const {highlighter, aceDom, chrome, dracula, mode} = await fetchAceModules(); + if (owner.isDisposed()) { return; } + if (maxLines) { // If requested, trim to maxLines, and add an ellipsis at the end. // (Long lines are also truncated with an ellpsis via text-overflow style.) @@ -51,7 +118,7 @@ export function buildHighlightedCode( let aceThemeName: 'chrome' | 'dracula'; let aceTheme: any; - if (codeTheme.get().appearance === 'dark' && !enableCustomCss) { + if (codeTheme.get().appearance === 'dark') { aceThemeName = 'dracula'; aceTheme = dracula; } else { @@ -69,15 +136,13 @@ export function buildHighlightedCode( } return cssHighlightedCode( - dom.autoDispose(codeText), - dom.autoDispose(codeTheme), - elem => subscribeElem(elem, code, (newCodeText) => { + elem => subscribeElem(elem, code, async (newCodeText) => { codeText.set(newCodeText); - updateHighlightedCode(elem); + await updateHighlightedCode(elem); }), - elem => subscribeElem(elem, gristTheme, (newCodeTheme) => { + elem => subscribeElem(elem, gristThemeObs(), async (newCodeTheme) => { codeTheme.set(newCodeTheme); - updateHighlightedCode(elem); + await updateHighlightedCode(elem); }), ...args, ); @@ -95,9 +160,7 @@ export const cssCodeBlock = styled('div', ` const cssHighlightedCode = styled(cssCodeBlock, ` position: relative; - white-space: pre; overflow: hidden; - text-overflow: ellipsis; border: 1px solid ${theme.highlightedCodeBorder}; border-radius: 3px; min-height: 28px; @@ -110,20 +173,6 @@ const cssHighlightedCode = styled(cssCodeBlock, ` & .ace_line { overflow: hidden; text-overflow: ellipsis; - } -`); - -export const cssFieldFormula = styled(buildHighlightedCode, ` - flex: auto; - cursor: pointer; - margin-top: 4px; - padding-left: 24px; - --icon-color: ${theme.accentIcon}; - - &-disabled-icon.formula_field_sidepane::before { - --icon-color: ${theme.iconDisabled}; - } - &-disabled { - pointer-events: none; + white-space: nowrap; } `); diff --git a/app/client/ui/DocHistory.ts b/app/client/ui/DocHistory.ts index 30ac2ff2..3c3f9a20 100644 --- a/app/client/ui/DocHistory.ts +++ b/app/client/ui/DocHistory.ts @@ -1,9 +1,9 @@ import {makeT} from 'app/client/lib/localization'; import {createSessionObs} from 'app/client/lib/sessionObs'; +import {getTimeFromNow} from 'app/client/lib/timeUtils'; import {DocPageModel} from 'app/client/models/DocPageModel'; import {reportError} from 'app/client/models/errors'; import {urlState} from 'app/client/models/gristUrlState'; -import {getTimeFromNow} from 'app/client/models/HomeModel'; import {buildConfigContainer} from 'app/client/ui/RightPanel'; import {buttonSelect} from 'app/client/ui2018/buttonSelect'; import {testId, theme, vars} from 'app/client/ui2018/cssVars'; diff --git a/app/client/ui/DocMenu.ts b/app/client/ui/DocMenu.ts index 90d5b26f..7ecdfb5e 100644 --- a/app/client/ui/DocMenu.ts +++ b/app/client/ui/DocMenu.ts @@ -4,9 +4,10 @@ * Orgs, workspaces and docs are fetched asynchronously on build via the passed in API. */ import {loadUserManager} from 'app/client/lib/imports'; +import {getTimeFromNow} from 'app/client/lib/timeUtils'; import {reportError} from 'app/client/models/AppModel'; import {docUrl, urlState} from 'app/client/models/gristUrlState'; -import {getTimeFromNow, HomeModel, makeLocalViewSettings, ViewSettings} from 'app/client/models/HomeModel'; +import {HomeModel, makeLocalViewSettings, ViewSettings} from 'app/client/models/HomeModel'; import {getWorkspaceInfo, workspaceName} from 'app/client/models/WorkspaceInfo'; import {attachAddNewTip} from 'app/client/ui/AddNewTip'; import * as css from 'app/client/ui/DocMenuCss'; diff --git a/app/client/ui/FieldConfig.ts b/app/client/ui/FieldConfig.ts index 606664fe..934eeadf 100644 --- a/app/client/ui/FieldConfig.ts +++ b/app/client/ui/FieldConfig.ts @@ -2,7 +2,7 @@ import {makeT} from 'app/client/lib/localization'; import {GristDoc} from 'app/client/components/GristDoc'; import {BEHAVIOR, ColumnRec} from 'app/client/models/entities/ColumnRec'; import {buildHighlightedCode, cssCodeBlock} from 'app/client/ui/CodeHighlight'; -import {cssBlockedCursor, cssLabel, cssRow} from 'app/client/ui/RightPanelStyles'; +import {cssBlockedCursor, cssFieldFormula, cssLabel, cssRow} from 'app/client/ui/RightPanelStyles'; import {withInfoTooltip} from 'app/client/ui/tooltips'; import {buildFormulaTriggers} from 'app/client/ui/TriggerFormulas'; import {textButton} from 'app/client/ui2018/buttons'; @@ -13,7 +13,6 @@ import {IconName} from 'app/client/ui2018/IconList'; import {selectMenu, selectOption, selectTitle} from 'app/client/ui2018/menus'; import {createFormulaErrorObs, cssError} from 'app/client/widgets/FormulaEditor'; import {sanitizeIdent} from 'app/common/gutil'; -import {Theme} from 'app/common/ThemePrefs'; import {CursorPos} from 'app/plugin/GristAPI'; import {bundleChanges, Computed, dom, DomContents, DomElementArg, fromKo, MultiHolder, Observable, styled} from 'grainjs'; @@ -139,6 +138,8 @@ export function buildFormulaConfig( // And close it dispose it when user opens up behavior menu. let formulaField: HTMLElement|null = null; + const focusFormulaField = () => setTimeout(() => formulaField?.focus(), 0); + // Helper function to clear temporary state (will be called when column changes or formula editor closes) const clearState = () => bundleChanges(() => { // For a detached editor, we may have already been disposed when user switched page. @@ -242,7 +243,7 @@ export function buildFormulaConfig( // Converts data column to formula column. const convertDataColumnToFormulaOption = () => selectOption( - () => (maybeFormula.set(true), formulaField?.focus()), + () => (maybeFormula.set(true), focusFormulaField()), t("Clear and make into formula"), 'Script'); // Converts to empty column and opens up the editor. (label is the same, but this is used when we have no formula) @@ -270,15 +271,15 @@ export function buildFormulaConfig( const convertDataColumnToTriggerColumn = () => { maybeTrigger.set(true); // Open the formula editor. - formulaField?.focus(); + focusFormulaField(); }; // Converts formula column to trigger formula column. const convertFormulaToTrigger = () => gristDoc.convertIsFormula([origColumn.id.peek()], {toFormula: false, noRecalc: false}); - const setFormula = () => (maybeFormula.set(true), formulaField?.focus()); - const setTrigger = () => (maybeTrigger.set(true), formulaField?.focus()); + const setFormula = () => { maybeFormula.set(true); focusFormulaField(); }; + const setTrigger = () => { maybeTrigger.set(true); focusFormulaField(); }; // Actions on save formula. Those actions are using column that comes from FormulaEditor. // Formula editor scope is broader then RightPanel, it can be disposed after RightPanel is closed, @@ -325,16 +326,19 @@ export function buildFormulaConfig( const errorMessage = createFormulaErrorObs(owner, gristDoc, origColumn); // Helper that will create different flavors for formula builder. const formulaBuilder = (onSave: SaveHandler, canDetach?: boolean) => [ - cssRow(formulaField = buildFormula( - origColumn, - buildEditor, - { - gristTheme: gristDoc.currentTheme, - disabled: disableOtherActions, - canDetach, - onSave, - onCancel: clearState, - })), + cssRow( + buildFormula( + origColumn, + buildEditor, + { + disabled: disableOtherActions, + canDetach, + onSave, + onCancel: clearState, + }, + (el) => { formulaField = el; }, + ) + ), dom.maybe(errorMessage, errMsg => cssRow(cssError(errMsg), testId('field-error-count'))), ]; @@ -419,7 +423,6 @@ export function buildFormulaConfig( } interface BuildFormulaOptions { - gristTheme: Computed; disabled: Observable; canDetach?: boolean; onSave?: SaveHandler; @@ -429,10 +432,12 @@ interface BuildFormulaOptions { function buildFormula( column: ColumnRec, buildEditor: BuildEditor, - options: BuildFormulaOptions + options: BuildFormulaOptions, + ...args: DomElementArg[] ) { - const {gristTheme, disabled, canDetach = true, onSave, onCancel} = options; - return cssFieldFormula(column.formula, {gristTheme, maxLines: 2}, + const {disabled, canDetach = true, onSave, onCancel} = options; + return dom.create(buildHighlightedCode, column.formula, {maxLines: 2}, + dom.cls(cssFieldFormula.className), dom.cls('formula_field_sidepane'), cssFieldFormula.cls('-disabled', disabled), cssFieldFormula.cls('-disabled-icon', use => !use(column.formula)), @@ -447,24 +452,10 @@ function buildFormula( onSave, onCancel, })), + ...args, ); } -export const cssFieldFormula = styled(buildHighlightedCode, ` - flex: auto; - cursor: pointer; - margin-top: 4px; - padding-left: 24px; - --icon-color: ${theme.accentIcon}; - - &-disabled-icon.formula_field_sidepane::before { - --icon-color: ${theme.lightText}; - } - &-disabled { - pointer-events: none; - } -`); - const cssToggleButton = styled(cssIconButton, ` margin-left: 8px; background-color: ${theme.rightPanelToggleButtonDisabledBg}; diff --git a/app/client/ui/GristTooltips.ts b/app/client/ui/GristTooltips.ts index 8f26142d..9b7b2554 100644 --- a/app/client/ui/GristTooltips.ts +++ b/app/client/ui/GristTooltips.ts @@ -1,5 +1,6 @@ import * as commands from 'app/client/components/commands'; import {makeT} from 'app/client/lib/localization'; +import {buildHighlightedCode} from 'app/client/ui/CodeHighlight'; import {ShortcutKey, ShortcutKeyContent} from 'app/client/ui/ShortcutKey'; import {icon} from 'app/client/ui2018/icons'; import {cssLink} from 'app/client/ui2018/links'; @@ -39,7 +40,9 @@ export type Tooltip = | 'uuid' | 'lookups' | 'formulaColumn' - | 'accessRulesTableWide'; + | 'accessRulesTableWide' + | 'setChoiceDropdownCondition' + | 'setRefDropdownCondition'; export type TooltipContentFunc = (...domArgs: DomElementArg[]) => DomContents; @@ -125,7 +128,26 @@ see or edit which parts of your document.') ...args, ), accessRulesTableWide: (...args: DomElementArg[]) => cssTooltipContent( - dom('div', t('These rules are applied after all column rules have been processed, if applicable.')) + dom('div', t('These rules are applied after all column rules have been processed, if applicable.')), + ...args, + ), + setChoiceDropdownCondition: (...args: DomElementArg[]) => cssTooltipContent( + dom('div', + t('Filter displayed dropdown values with a condition.') + ), + dom('div', {style: 'margin-top: 8px;'}, t('Example: {{example}}', { + example: dom.create(buildHighlightedCode, 'choice not in $Categories', {}, {style: 'margin-top: 8px;'}), + })), + ...args, + ), + setRefDropdownCondition: (...args: DomElementArg[]) => cssTooltipContent( + dom('div', + t('Filter displayed dropdown values with a condition.') + ), + dom('div', {style: 'margin-top: 8px;'}, t('Example: {{example}}', { + example: dom.create(buildHighlightedCode, 'choice.Role == "Manager"', {}, {style: 'margin-top: 8px;'}), + })), + ...args, ), }; diff --git a/app/client/ui/PinnedDocs.ts b/app/client/ui/PinnedDocs.ts index e84f3f1e..7077749c 100644 --- a/app/client/ui/PinnedDocs.ts +++ b/app/client/ui/PinnedDocs.ts @@ -1,5 +1,6 @@ +import {getTimeFromNow} from 'app/client/lib/timeUtils'; import {docUrl, urlState} from 'app/client/models/gristUrlState'; -import {getTimeFromNow, HomeModel} from 'app/client/models/HomeModel'; +import {HomeModel} from 'app/client/models/HomeModel'; import {makeDocOptionsMenu, makeRemovedDocOptionsMenu} from 'app/client/ui/DocMenu'; import {transientInput} from 'app/client/ui/transientInput'; import {colors, theme, vars} from 'app/client/ui2018/cssVars'; diff --git a/app/client/ui/RightPanelStyles.ts b/app/client/ui/RightPanelStyles.ts index b6ec7c0f..8e7a4723 100644 --- a/app/client/ui/RightPanelStyles.ts +++ b/app/client/ui/RightPanelStyles.ts @@ -94,3 +94,18 @@ export const cssPinButton = styled('div', ` export const cssNumericSpinner = styled(numericSpinner, ` height: 28px; `); + +export const cssFieldFormula = styled('div', ` + flex: auto; + cursor: pointer; + margin-top: 4px; + padding-left: 24px; + --icon-color: ${theme.accentIcon}; + + &-disabled-icon.formula_field_sidepane::before { + --icon-color: ${theme.iconDisabled}; + } + &-disabled { + pointer-events: none; + } +`); diff --git a/app/client/ui/ThemeConfig.ts b/app/client/ui/ThemeConfig.ts index 7b371a62..b9492d92 100644 --- a/app/client/ui/ThemeConfig.ts +++ b/app/client/ui/ThemeConfig.ts @@ -2,7 +2,7 @@ import {makeT} from 'app/client/lib/localization'; import {AppModel} from 'app/client/models/AppModel'; import * as css from 'app/client/ui/AccountPageCss'; import {labeledSquareCheckbox} from 'app/client/ui2018/checkbox'; -import {prefersDarkModeObs} from 'app/client/ui2018/cssVars'; +import {prefersColorSchemeDarkObs} from 'app/client/ui2018/theme'; import {select} from 'app/client/ui2018/menus'; import {ThemeAppearance} from 'app/common/ThemePrefs'; import {Computed, Disposable, dom, makeTestId, styled} from 'grainjs'; @@ -20,10 +20,10 @@ export class ThemeConfig extends Disposable { private _appearance = Computed.create(this, this._themePrefs, this._syncWithOS, - prefersDarkModeObs(), - (_use, prefs, syncWithOS, prefersDarkMode) => { + prefersColorSchemeDarkObs(), + (_use, prefs, syncWithOS, prefersColorSchemeDark) => { if (syncWithOS) { - return prefersDarkMode ? 'dark' : 'light'; + return prefersColorSchemeDark ? 'dark' : 'light'; } else { return prefs.appearance; } diff --git a/app/client/ui/createAppPage.ts b/app/client/ui/createAppPage.ts index f5840d70..28dc51a3 100644 --- a/app/client/ui/createAppPage.ts +++ b/app/client/ui/createAppPage.ts @@ -4,7 +4,8 @@ import {AppModel, TopAppModelImpl, TopAppModelOptions} from 'app/client/models/A import {reportError, setUpErrorHandling} from 'app/client/models/errors'; import {buildSnackbarDom} from 'app/client/ui/NotifyUI'; import {addViewportTag} from 'app/client/ui/viewport'; -import {attachCssRootVars, attachTheme} from 'app/client/ui2018/cssVars'; +import {attachCssRootVars} from 'app/client/ui2018/cssVars'; +import {attachTheme} from 'app/client/ui2018/theme'; import {BaseAPI} from 'app/common/BaseAPI'; import {dom, DomContents} from 'grainjs'; @@ -16,22 +17,22 @@ const G = getBrowserGlobals('document', 'window'); */ export function createAppPage( buildAppPage: (appModel: AppModel) => DomContents, - modelOptions: TopAppModelOptions = {}) { + modelOptions: TopAppModelOptions = {} +) { setUpErrorHandling(); const topAppModel = TopAppModelImpl.create(null, {}, undefined, modelOptions); addViewportTag(); attachCssRootVars(topAppModel.productFlavor); + attachTheme(); setupLocale().catch(reportError); // Add globals needed by test utils. G.window.gristApp = { testNumPendingApiRequests: () => BaseAPI.numPendingRequests(), }; - dom.update(document.body, dom.maybeOwned(topAppModel.appObs, (owner, appModel) => { - owner.autoDispose(attachTheme(appModel.currentTheme)); - + dom.update(document.body, dom.maybe(topAppModel.appObs, (appModel) => { return [ buildAppPage(appModel), buildSnackbarDom(appModel.notifier, appModel), diff --git a/app/client/ui/createPage.ts b/app/client/ui/createPage.ts index a5fa5c69..027af0a8 100644 --- a/app/client/ui/createPage.ts +++ b/app/client/ui/createPage.ts @@ -4,7 +4,8 @@ import {reportError, setErrorNotifier, setUpErrorHandling} from 'app/client/mode import {Notifier} from 'app/client/models/NotifyModel'; import {buildSnackbarDom} from 'app/client/ui/NotifyUI'; import {addViewportTag} from 'app/client/ui/viewport'; -import {attachCssRootVars, attachTheme, prefersColorSchemeThemeObs} from 'app/client/ui2018/cssVars'; +import {attachCssRootVars} from 'app/client/ui2018/cssVars'; +import {attachTheme} from 'app/client/ui2018/theme'; import {BaseAPI} from 'app/common/BaseAPI'; import {dom, DomContents} from 'grainjs'; @@ -21,6 +22,7 @@ export function createPage(buildPage: () => DomContents, options: {disableTheme? addViewportTag(); attachCssRootVars('grist'); + if (!disableTheme) { attachTheme(); } setupLocale().catch(reportError); // Add globals needed by test utils. @@ -32,7 +34,6 @@ export function createPage(buildPage: () => DomContents, options: {disableTheme? setErrorNotifier(notifier); dom.update(document.body, () => [ - disableTheme ? null : dom.autoDispose(attachTheme(prefersColorSchemeThemeObs())), buildPage(), buildSnackbarDom(notifier, null), ]); diff --git a/app/client/ui/searchDropdown.ts b/app/client/ui/searchDropdown.ts index b618f7bd..901feb3b 100644 --- a/app/client/ui/searchDropdown.ts +++ b/app/client/ui/searchDropdown.ts @@ -2,16 +2,17 @@ // keyboard. Dropdown features a search input and reoders the list of // items to bring best matches at the top. -import { Disposable, dom, DomElementMethod, IOptionFull, makeTestId, Observable, styled } from "grainjs"; -import { theme, vars } from 'app/client/ui2018/cssVars'; import { ACIndexImpl, ACIndexOptions, ACItem, buildHighlightedDom, HighlightFunc, normalizeText } from "app/client/lib/ACIndex"; -import { menuDivider } from "app/client/ui2018/menus"; -import { icon } from "app/client/ui2018/icons"; -import { cssMenuItem, defaultMenuOptions, IOpenController, IPopupOptions, setPopupToFunc } from "popweasel"; -import { mergeWith } from "lodash"; -import { getOptionFull, SimpleList } from "../lib/simpleList"; import { makeT } from 'app/client/lib/localization'; +import { getOptionFull, SimpleList } from "app/client/lib/simpleList"; +import { theme, vars } from 'app/client/ui2018/cssVars'; +import { icon } from "app/client/ui2018/icons"; +import { menuDivider } from "app/client/ui2018/menus"; +import { Disposable, dom, DomElementMethod, IOptionFull, makeTestId, Observable, styled } from "grainjs"; +import mergeWith from "lodash/mergeWith"; +import { cssMenuItem, defaultMenuOptions, IOpenController, IPopupOptions, setPopupToFunc } from "popweasel"; + const t = makeT('searchDropdown'); diff --git a/app/client/ui/tooltips.ts b/app/client/ui/tooltips.ts index 4398a52f..778dd3a8 100644 --- a/app/client/ui/tooltips.ts +++ b/app/client/ui/tooltips.ts @@ -582,7 +582,7 @@ const cssInfoTooltipPopup = styled('div', ` display: flex; flex-direction: column; background-color: ${theme.popupBg}; - max-width: 200px; + max-width: 240px; margin: 4px; padding: 0px; `); diff --git a/app/client/ui2018/cssVars.ts b/app/client/ui2018/cssVars.ts index deab8233..5cd706e5 100644 --- a/app/client/ui2018/cssVars.ts +++ b/app/client/ui2018/cssVars.ts @@ -6,16 +6,10 @@ * https://css-tricks.com/snippets/css/system-font-stack/ * */ -import {createPausableObs, PausableObservable} from 'app/client/lib/pausableObs'; -import {getStorage} from 'app/client/lib/storage'; import {urlState} from 'app/client/models/gristUrlState'; import {getTheme, ProductFlavor} from 'app/client/ui/CustomThemes'; -import {Theme, ThemeAppearance} from 'app/common/ThemePrefs'; -import {getThemeColors} from 'app/common/Themes'; -import {getGristConfig} from 'app/common/urlUtils'; -import {Computed, dom, DomElementMethod, makeTestId, Observable, styled, TestId} from 'grainjs'; +import {dom, DomElementMethod, makeTestId, Observable, styled, TestId} from 'grainjs'; import debounce = require('lodash/debounce'); -import isEqual = require('lodash/isEqual'); import values = require('lodash/values'); const VAR_PREFIX = 'grist'; @@ -1021,51 +1015,6 @@ export function isScreenResizing(): Observable { return _isScreenResizingObs; } -export function prefersDarkMode(): boolean { - return window.matchMedia('(prefers-color-scheme: dark)').matches; -} - -let _prefersDarkModeObs: PausableObservable|undefined; - -/** - * Returns a singleton observable for whether the user agent prefers dark mode. - */ -export function prefersDarkModeObs(): PausableObservable { - if (!_prefersDarkModeObs) { - const query = window.matchMedia('(prefers-color-scheme: dark)'); - const obs = createPausableObs(null, query.matches); - query.addEventListener('change', event => obs.set(event.matches)); - _prefersDarkModeObs = obs; - } - return _prefersDarkModeObs; -} - -let _prefersColorSchemeThemeObs: Computed|undefined; - -/** - * Returns a singleton observable for the Grist theme matching the current - * user agent color scheme preference ("light" or "dark"). - */ -export function prefersColorSchemeThemeObs(): Computed { - if (!_prefersColorSchemeThemeObs) { - const obs = Computed.create(null, prefersDarkModeObs(), (_use, prefersDarkTheme) => { - if (prefersDarkTheme) { - return { - appearance: 'dark', - colors: getThemeColors('GristDark'), - } as const; - } else { - return { - appearance: 'light', - colors: getThemeColors('GristLight'), - } as const; - } - }); - _prefersColorSchemeThemeObs = obs; - } - return _prefersColorSchemeThemeObs; -} - /** * Attaches the global css properties to the document's root to make them available in the page. */ @@ -1081,96 +1030,6 @@ export function attachCssRootVars(productFlavor: ProductFlavor, varsOnly: boolea document.body.classList.add(`interface-${interfaceStyle}`); } -export function attachTheme(themeObs: Observable) { - // Attach the current theme to the DOM. - attachCssThemeVars(themeObs.get()); - - // Whenever the theme changes, re-attach it to the DOM. - return themeObs.addListener((newTheme, oldTheme) => { - if (isEqual(newTheme, oldTheme)) { return; } - - attachCssThemeVars(newTheme); - }); -} - -/** - * Attaches theme-related css properties to the theme style element. - */ -function attachCssThemeVars({appearance, colors: themeColors}: Theme) { - // Custom CSS is incompatible with custom themes. - if (getGristConfig().enableCustomCss) { return; } - - // Prepare the custom properties needed for applying the theme. - const properties = Object.entries(themeColors) - .map(([name, value]) => `--grist-theme-${name}: ${value};`); - - // Include properties for styling the scrollbar. - properties.push(...getCssScrollbarProperties(appearance)); - - // Include properties for picking an appropriate background image. - properties.push(...getCssThemeBackgroundProperties(appearance)); - - // Apply the properties to the theme style element. - getOrCreateStyleElement('grist-theme').textContent = `:root { -${properties.join('\n')} - }`; - - // Make the browser aware of the color scheme. - document.documentElement.style.setProperty(`color-scheme`, appearance); - - // Cache the appearance in local storage; this is currently used to apply a suitable - // background image that's shown while the application is loading. - getStorage().setItem('appearance', appearance); -} - -/** - * Gets scrollbar-related css properties that are appropriate for the given `appearance`. - * - * Note: Browser support for customizing scrollbars is still a mixed bag; the bulk of customization - * is non-standard and unsupported by Firefox. If support matures, we could expose some of these in - * custom themes, but for now we'll just go with reasonable presets. - */ -function getCssScrollbarProperties(appearance: ThemeAppearance) { - return [ - '--scroll-bar-fg: ' + - (appearance === 'dark' ? '#6B6B6B;' : '#A8A8A8;'), - '--scroll-bar-hover-fg: ' + - (appearance === 'dark' ? '#7B7B7B;' : '#8F8F8F;'), - '--scroll-bar-active-fg: ' + - (appearance === 'dark' ? '#8B8B8B;' : '#7C7C7C;'), - '--scroll-bar-bg: ' + - (appearance === 'dark' ? '#2B2B2B;' : '#F0F0F0;'), - ]; -} - -/** - * Gets background-related css properties that are appropriate for the given `appearance`. - * - * Currently, this sets a property for showing a background image that's visible while a page - * is loading. - */ -function getCssThemeBackgroundProperties(appearance: ThemeAppearance) { - const value = appearance === 'dark' - ? 'url("img/prismpattern.png")' - : 'url("img/gplaypattern.png")'; - return [`--grist-theme-bg: ${value};`]; -} - -/** - * Gets or creates a style element in the head of the document with the given `id`. - * - * Useful for grouping CSS values such as theme custom properties without needing to - * pollute the document with in-line styles. - */ -function getOrCreateStyleElement(id: string) { - let style = document.head.querySelector(`#${id}`); - if (style) { return style; } - style = document.createElement('style'); - style.setAttribute('id', id); - document.head.append(style); - return style; -} - // A dom method to hide element in print view export function hideInPrintView(): DomElementMethod { return cssHideInPrint.cls(''); diff --git a/app/client/ui2018/theme.ts b/app/client/ui2018/theme.ts new file mode 100644 index 00000000..642732ad --- /dev/null +++ b/app/client/ui2018/theme.ts @@ -0,0 +1,191 @@ +import { createPausableObs, PausableObservable } from 'app/client/lib/pausableObs'; +import { getStorage } from 'app/client/lib/storage'; +import { urlState } from 'app/client/models/gristUrlState'; +import { Theme, ThemeAppearance, ThemeColors, ThemePrefs } from 'app/common/ThemePrefs'; +import { getThemeColors } from 'app/common/Themes'; +import { getGristConfig } from 'app/common/urlUtils'; +import { Computed, Observable } from 'grainjs'; +import isEqual from 'lodash/isEqual'; + +const DEFAULT_LIGHT_THEME: Theme = {appearance: 'light', colors: getThemeColors('GristLight')}; +const DEFAULT_DARK_THEME: Theme = {appearance: 'dark', colors: getThemeColors('GristDark')}; + +/** + * A singleton observable for the current user's Grist theme preferences. + * + * Set by `AppModel`, which populates it from `UserPrefs`. + */ +export const gristThemePrefs = Observable.create(null, null); + +/** + * Returns `true` if the user agent prefers a dark color scheme. + */ +export function prefersColorSchemeDark(): boolean { + return window.matchMedia('(prefers-color-scheme: dark)').matches; +} + +let _prefersColorSchemeDarkObs: PausableObservable | undefined; + +/** + * Returns a singleton observable for whether the user agent prefers a + * dark color scheme. + */ +export function prefersColorSchemeDarkObs(): PausableObservable { + if (!_prefersColorSchemeDarkObs) { + const query = window.matchMedia('(prefers-color-scheme: dark)'); + const obs = createPausableObs(null, query.matches); + query.addEventListener('change', event => obs.set(event.matches)); + _prefersColorSchemeDarkObs = obs; + } + return _prefersColorSchemeDarkObs; +} + +let _gristThemeObs: Computed | undefined; + +/** + * A singleton observable for the current Grist theme. + */ +export function gristThemeObs() { + if (!_gristThemeObs) { + _gristThemeObs = Computed.create(null, (use) => { + // Custom CSS is incompatible with custom themes. + if (getGristConfig().enableCustomCss) { return DEFAULT_LIGHT_THEME; } + + // If a user's preference is known, return it. + const themePrefs = use(gristThemePrefs); + const userAgentPrefersDarkTheme = use(prefersColorSchemeDarkObs()); + if (themePrefs) { return getThemeFromPrefs(themePrefs, userAgentPrefersDarkTheme); } + + // Otherwise, fall back to the user agent's preference. + return userAgentPrefersDarkTheme ? DEFAULT_DARK_THEME : DEFAULT_LIGHT_THEME; + }); + } + return _gristThemeObs; +} + +/** + * Attaches the current theme's CSS variables to the document, and + * re-attaches them whenever the theme changes. + */ +export function attachTheme() { + // Custom CSS is incompatible with custom themes. + if (getGristConfig().enableCustomCss) { return; } + + // Attach the current theme's variables to the DOM. + attachCssThemeVars(gristThemeObs().get()); + + // Whenever the theme changes, re-attach its variables to the DOM. + gristThemeObs().addListener((newTheme, oldTheme) => { + if (isEqual(newTheme, oldTheme)) { return; } + + attachCssThemeVars(newTheme); + }); +} + +/** + * Returns the `Theme` from the given `themePrefs`. + * + * If theme query parameters are present (`themeName`, `themeAppearance`, `themeSyncWithOs`), + * they will take precedence over their respective values in `themePrefs`. + */ +function getThemeFromPrefs(themePrefs: ThemePrefs, userAgentPrefersDarkTheme: boolean): Theme { + let {appearance, syncWithOS} = themePrefs; + + const urlParams = urlState().state.get().params; + if (urlParams?.themeAppearance) { + appearance = urlParams?.themeAppearance; + } + if (urlParams?.themeSyncWithOs !== undefined) { + syncWithOS = urlParams?.themeSyncWithOs; + } + + if (syncWithOS) { + appearance = userAgentPrefersDarkTheme ? 'dark' : 'light'; + } + + let nameOrColors = themePrefs.colors[appearance]; + if (urlParams?.themeName) { + nameOrColors = urlParams?.themeName; + } + + let colors: ThemeColors; + if (typeof nameOrColors === 'string') { + colors = getThemeColors(nameOrColors); + } else { + colors = nameOrColors; + } + + return {appearance, colors}; +} + +function attachCssThemeVars({appearance, colors: themeColors}: Theme) { + // Prepare the custom properties needed for applying the theme. + const properties = Object.entries(themeColors) + .map(([name, value]) => `--grist-theme-${name}: ${value};`); + + // Include properties for styling the scrollbar. + properties.push(...getCssThemeScrollbarProperties(appearance)); + + // Include properties for picking an appropriate background image. + properties.push(...getCssThemeBackgroundProperties(appearance)); + + // Apply the properties to the theme style element. + getOrCreateStyleElement('grist-theme').textContent = `:root { +${properties.join('\n')} + }`; + + // Make the browser aware of the color scheme. + document.documentElement.style.setProperty(`color-scheme`, appearance); + + // Cache the appearance in local storage; this is currently used to apply a suitable + // background image that's shown while the application is loading. + getStorage().setItem('appearance', appearance); +} + +/** + * Gets scrollbar-related css properties that are appropriate for the given `appearance`. + * + * Note: Browser support for customizing scrollbars is still a mixed bag; the bulk of customization + * is non-standard and unsupported by Firefox. If support matures, we could expose some of these in + * custom themes, but for now we'll just go with reasonable presets. + */ +function getCssThemeScrollbarProperties(appearance: ThemeAppearance) { + return [ + '--scroll-bar-fg: ' + + (appearance === 'dark' ? '#6B6B6B;' : '#A8A8A8;'), + '--scroll-bar-hover-fg: ' + + (appearance === 'dark' ? '#7B7B7B;' : '#8F8F8F;'), + '--scroll-bar-active-fg: ' + + (appearance === 'dark' ? '#8B8B8B;' : '#7C7C7C;'), + '--scroll-bar-bg: ' + + (appearance === 'dark' ? '#2B2B2B;' : '#F0F0F0;'), + ]; +} + +/** + * Gets background-related css properties that are appropriate for the given `appearance`. + * + * Currently, this sets a property for showing a background image that's visible while a page + * is loading. + */ +function getCssThemeBackgroundProperties(appearance: ThemeAppearance) { + const value = appearance === 'dark' + ? 'url("img/prismpattern.png")' + : 'url("img/gplaypattern.png")'; + return [`--grist-theme-bg: ${value};`]; +} + +/** + * Gets or creates a style element in the head of the document with the given `id`. + * + * Useful for grouping CSS values such as theme custom properties without needing to + * pollute the document with in-line styles. + */ +function getOrCreateStyleElement(id: string) { + let style = document.head.querySelector(`#${id}`); + if (style) { return style; } + style = document.createElement('style'); + style.setAttribute('id', id); + document.head.append(style); + return style; +} diff --git a/app/client/widgets/ChoiceEditor.js b/app/client/widgets/ChoiceEditor.js index 19ebfcba..940c8d53 100644 --- a/app/client/widgets/ChoiceEditor.js +++ b/app/client/widgets/ChoiceEditor.js @@ -4,13 +4,22 @@ var TextEditor = require('app/client/widgets/TextEditor'); const {Autocomplete} = require('app/client/lib/autocomplete'); const {ACIndexImpl, buildHighlightedDom} = require('app/client/lib/ACIndex'); -const {ChoiceItem, cssChoiceList, cssMatchText, cssPlusButton, - cssPlusIcon} = require('app/client/widgets/ChoiceListEditor'); +const {makeT} = require('app/client/lib/localization'); +const { + buildDropdownConditionFilter, + ChoiceItem, + cssChoiceList, + cssMatchText, + cssPlusButton, + cssPlusIcon, +} = require('app/client/widgets/ChoiceListEditor'); +const {icon} = require('app/client/ui2018/icons'); const {menuCssClass} = require('app/client/ui2018/menus'); const {testId, theme} = require('app/client/ui2018/cssVars'); const {choiceToken, cssChoiceACItem} = require('app/client/widgets/ChoiceToken'); const {dom, styled} = require('grainjs'); -const {icon} = require('../ui2018/icons'); + +const t = makeT('ChoiceEditor'); /** * ChoiceEditor - TextEditor with a dropdown for possible choices. @@ -18,15 +27,46 @@ const {icon} = require('../ui2018/icons'); function ChoiceEditor(options) { TextEditor.call(this, options); - this.choices = options.field.widgetOptionsJson.peek().choices || []; - this.choiceOptions = options.field.widgetOptionsJson.peek().choiceOptions || {}; + this.widgetOptionsJson = options.field.widgetOptionsJson; + this.choices = this.widgetOptionsJson.peek().choices || []; + this.choicesSet = new Set(this.choices); + this.choiceOptions = this.widgetOptionsJson.peek().choiceOptions || {}; + + this.hasDropdownCondition = Boolean(options.field.dropdownCondition.peek()?.text); + this.dropdownConditionError; + + let acItems = this.choices.map(c => new ChoiceItem(c, false, false)); + if (this.hasDropdownCondition) { + try { + const dropdownConditionFilter = this.buildDropdownConditionFilter(); + acItems = acItems.filter((item) => dropdownConditionFilter(item)); + } catch (e) { + acItems = []; + this.dropdownConditionError = e.message; + } + } + + const acIndex = new ACIndexImpl(acItems); + this._acOptions = { + popperOptions: { + placement: 'bottom' + }, + menuCssClass: `${menuCssClass} ${cssChoiceList.className} test-autocomplete`, + buildNoItemsMessage: this.buildNoItemsMessage.bind(this), + search: (term) => this.maybeShowAddNew(acIndex.search(term), term), + renderItem: (item, highlightFunc) => this.renderACItem(item, highlightFunc), + getItemText: (item) => item.label, + onClick: () => this.options.commands.fieldEditSave(), + }; + if (!options.readonly && options.field.viewSection().parentKey() === "single") { this.cellEditorDiv.classList.add(cssChoiceEditor.className); this.cellEditorDiv.appendChild(cssChoiceEditIcon('Dropdown')); } + // Whether to include a button to show a new choice. // TODO: Disable when the user cannot change column configuration. - this.enableAddNew = true; + this.enableAddNew = !this.hasDropdownCondition; } dispose.makeDisposable(ChoiceEditor); @@ -66,20 +106,7 @@ ChoiceEditor.prototype.attach = function(cellElem) { // Don't create autocomplete if readonly. if (this.options.readonly) { return; } - const acItems = this.choices.map(c => new ChoiceItem(c, false, false)); - const acIndex = new ACIndexImpl(acItems); - const acOptions = { - popperOptions: { - placement: 'bottom' - }, - menuCssClass: `${menuCssClass} ${cssChoiceList.className} test-autocomplete`, - search: (term) => this.maybeShowAddNew(acIndex.search(term), term), - renderItem: (item, highlightFunc) => this.renderACItem(item, highlightFunc), - getItemText: (item) => item.label, - onClick: () => this.options.commands.fieldEditSave(), - }; - - this.autocomplete = Autocomplete.create(this, this.textInput, acOptions); + this.autocomplete = Autocomplete.create(this, this.textInput, this._acOptions); } /** @@ -89,11 +116,35 @@ ChoiceEditor.prototype.attach = function(cellElem) { ChoiceEditor.prototype.prepForSave = async function() { const selectedItem = this.autocomplete && this.autocomplete.getSelectedItem(); if (selectedItem && selectedItem.isNew) { - const choices = this.options.field.widgetOptionsJson.prop('choices'); + const choices = this.widgetOptionsJson.prop('choices'); await choices.saveOnly([...(choices.peek() || []), selectedItem.label]); } } +ChoiceEditor.prototype.buildDropdownConditionFilter = function() { + const dropdownConditionCompiled = this.options.field.dropdownConditionCompiled.get(); + if (dropdownConditionCompiled?.kind !== 'success') { + throw new Error('Dropdown condition is not compiled'); + } + + return buildDropdownConditionFilter({ + dropdownConditionCompiled: dropdownConditionCompiled.result, + docData: this.options.gristDoc.docData, + tableId: this.options.field.tableId(), + rowId: this.options.rowId, + }); +} + +ChoiceEditor.prototype.buildNoItemsMessage = function() { + if (this.dropdownConditionError) { + return t('Error in dropdown condition'); + } else if (this.hasDropdownCondition) { + return t('No choices matching condition'); + } else { + return t('No choices to select'); + } +} + /** * If the search text does not match anything exactly, adds 'new' item to it. * @@ -103,15 +154,21 @@ ChoiceEditor.prototype.maybeShowAddNew = function(result, text) { // TODO: This logic is also mostly duplicated in ChoiceListEditor and ReferenceEditor. // See if there's anything common we can factor out and re-use. this.showAddNew = false; + if (!this.enableAddNew) { + return result; + } + const trimmedText = text.trim(); - if (!this.enableAddNew || !trimmedText) { return result; } + if (!trimmedText || this.choicesSet.has(trimmedText)) { + return result; + } const addNewItem = new ChoiceItem(trimmedText, false, false, true); if (result.items.find((item) => item.cleanText === addNewItem.cleanText)) { return result; } - result.items.push(addNewItem); + result.extraItems.push(addNewItem); this.showAddNew = true; return result; diff --git a/app/client/widgets/ChoiceListEditor.ts b/app/client/widgets/ChoiceListEditor.ts index 60329b63..238f47f0 100644 --- a/app/client/widgets/ChoiceListEditor.ts +++ b/app/client/widgets/ChoiceListEditor.ts @@ -2,7 +2,9 @@ import {createGroup} from 'app/client/components/commands'; import {ACIndexImpl, ACItem, ACResults, buildHighlightedDom, HighlightFunc, normalizeText} from 'app/client/lib/ACIndex'; import {IAutocompleteOptions} from 'app/client/lib/autocomplete'; +import {makeT} from 'app/client/lib/localization'; import {IToken, TokenField, tokenFieldStyles} from 'app/client/lib/TokenField'; +import {DocData} from 'app/client/models/DocData'; import {colors, testId, theme} from 'app/client/ui2018/cssVars'; import {menuCssClass} from 'app/client/ui2018/menus'; import {createMobileButtons, getButtonMargins} from 'app/client/widgets/EditorButtons'; @@ -10,12 +12,15 @@ import {EditorPlacement} from 'app/client/widgets/EditorPlacement'; import {FieldOptions, NewBaseEditor} from 'app/client/widgets/NewBaseEditor'; import {csvEncodeRow} from 'app/common/csvFormat'; import {CellValue} from "app/common/DocActions"; +import {CompiledPredicateFormula, EmptyRecordView} from 'app/common/PredicateFormula'; import {decodeObject, encodeObject} from 'app/plugin/objtypes'; import {ChoiceOptions, getRenderFillColor, getRenderTextColor} from 'app/client/widgets/ChoiceTextBox'; import {choiceToken, cssChoiceACItem, cssChoiceToken} from 'app/client/widgets/ChoiceToken'; import {icon} from 'app/client/ui2018/icons'; import {dom, styled} from 'grainjs'; +const t = makeT('ChoiceListEditor'); + export class ChoiceItem implements ACItem, IToken { public cleanText: string = normalizeText(this.label); constructor( @@ -38,25 +43,37 @@ export class ChoiceListEditor extends NewBaseEditor { private _inputSizer!: HTMLElement; // Part of _contentSizer to size the text input private _alignment: string; + private _widgetOptionsJson = this.options.field.widgetOptionsJson.peek(); + private _choices: string[] = this._widgetOptionsJson.choices || []; + private _choicesSet: Set = new Set(this._choices); + private _choiceOptionsByName: ChoiceOptions = this._widgetOptionsJson.choiceOptions || {}; + // Whether to include a button to show a new choice. // TODO: Disable when the user cannot change column configuration. - private _enableAddNew: boolean = true; + private _enableAddNew: boolean; private _showAddNew: boolean = false; - private _choiceOptionsByName: ChoiceOptions; + private _hasDropdownCondition = Boolean(this.options.field.dropdownCondition.peek()?.text); + private _dropdownConditionError: string | undefined; constructor(protected options: FieldOptions) { super(options); - const choices: string[] = options.field.widgetOptionsJson.peek().choices || []; - this._choiceOptionsByName = options.field.widgetOptionsJson - .peek().choiceOptions || {}; - const acItems = choices.map(c => new ChoiceItem(c, false, false)); - const choiceSet = new Set(choices); + let acItems = this._choices.map(c => new ChoiceItem(c, false, false)); + if (this._hasDropdownCondition) { + try { + const dropdownConditionFilter = this._buildDropdownConditionFilter(); + acItems = acItems.filter((item) => dropdownConditionFilter(item)); + } catch (e) { + acItems = []; + this._dropdownConditionError = e.message; + } + } const acIndex = new ACIndexImpl(acItems); const acOptions: IAutocompleteOptions = { menuCssClass: `${menuCssClass} ${cssChoiceList.className} test-autocomplete`, + buildNoItemsMessage: this._buildNoItemsMessage.bind(this), search: async (term: string) => this._maybeShowAddNew(acIndex.search(term), term), renderItem: (item, highlightFunc) => this._renderACItem(item, highlightFunc), getItemText: (item) => item.label, @@ -65,12 +82,13 @@ export class ChoiceListEditor extends NewBaseEditor { this.commandGroup = this.autoDispose(createGroup(options.commands, null, true)); this._alignment = options.field.widgetOptionsJson.peek().alignment || 'left'; + // If starting to edit by typing in a string, ignore previous tokens. const cellValue = decodeObject(options.cellValue); const startLabels: unknown[] = options.editValue !== undefined || !Array.isArray(cellValue) ? [] : cellValue; const startTokens = startLabels.map(label => new ChoiceItem( String(label), - !choiceSet.has(String(label)), + !this._choicesSet.has(String(label)), String(label).trim() === '' )); @@ -87,7 +105,7 @@ export class ChoiceListEditor extends NewBaseEditor { cssChoiceToken.cls('-invalid', item.isInvalid), cssChoiceToken.cls('-blank', item.isBlank), ], - createToken: label => new ChoiceItem(label, !choiceSet.has(label), label.trim() === ''), + createToken: label => new ChoiceItem(label, !this._choicesSet.has(label), label.trim() === ''), acOptions, openAutocompleteOnFocus: true, readonly : options.readonly, @@ -118,6 +136,8 @@ export class ChoiceListEditor extends NewBaseEditor { dom.prop('value', options.editValue || ''), this.commandGroup.attach(), ); + + this._enableAddNew = !this._hasDropdownCondition; } public attach(cellElem: Element): void { @@ -150,7 +170,7 @@ export class ChoiceListEditor extends NewBaseEditor { } public getTextValue() { - const values = this._tokenField.tokensObs.get().map(t => t.label); + const values = this._tokenField.tokensObs.get().map(token => token.label); return csvEncodeRow(values, {prettier: true}); } @@ -164,7 +184,7 @@ export class ChoiceListEditor extends NewBaseEditor { */ public async prepForSave() { const tokens = this._tokenField.tokensObs.get(); - const newChoices = tokens.filter(t => t.isNew).map(t => t.label); + const newChoices = tokens.filter(({isNew}) => isNew).map(({label}) => label); if (newChoices.length > 0) { const choices = this.options.field.widgetOptionsJson.prop('choices'); await choices.saveOnly([...(choices.peek() || []), ...new Set(newChoices)]); @@ -218,6 +238,30 @@ export class ChoiceListEditor extends NewBaseEditor { this._textInput.style.width = this._inputSizer.getBoundingClientRect().width + 'px'; } + private _buildDropdownConditionFilter() { + const dropdownConditionCompiled = this.options.field.dropdownConditionCompiled.get(); + if (dropdownConditionCompiled?.kind !== 'success') { + throw new Error('Dropdown condition is not compiled'); + } + + return buildDropdownConditionFilter({ + dropdownConditionCompiled: dropdownConditionCompiled.result, + docData: this.options.gristDoc.docData, + tableId: this.options.field.tableId(), + rowId: this.options.rowId, + }); + } + + private _buildNoItemsMessage(): string { + if (this._dropdownConditionError) { + return t('Error in dropdown condition'); + } else if (this._hasDropdownCondition) { + return t('No choices matching condition'); + } else { + return t('No choices to select'); + } + } + /** * If the search text does not match anything exactly, adds 'new' item to it. * @@ -225,15 +269,21 @@ export class ChoiceListEditor extends NewBaseEditor { */ private _maybeShowAddNew(result: ACResults, text: string): ACResults { this._showAddNew = false; + if (!this._enableAddNew) { + return result; + } + const trimmedText = text.trim(); - if (!this._enableAddNew || !trimmedText) { return result; } + if (!trimmedText || this._choicesSet.has(trimmedText)) { + return result; + } const addNewItem = new ChoiceItem(trimmedText, false, false, true); if (result.items.find((item) => item.cleanText === addNewItem.cleanText)) { return result; } - result.items.push(addNewItem); + result.extraItems.push(addNewItem); this._showAddNew = true; return result; @@ -259,6 +309,24 @@ export class ChoiceListEditor extends NewBaseEditor { } } +export interface GetACFilterFuncParams { + dropdownConditionCompiled: CompiledPredicateFormula; + docData: DocData; + tableId: string; + rowId: number; +} + +export function buildDropdownConditionFilter( + params: GetACFilterFuncParams +): (item: ChoiceItem) => boolean { + const {dropdownConditionCompiled, docData, tableId, rowId} = params; + const table = docData.getTable(tableId); + if (!table) { throw new Error(`Table ${tableId} not found`); } + + const rec = table.getRecord(rowId) || new EmptyRecordView(); + return (item: ChoiceItem) => dropdownConditionCompiled({rec, choice: item.label}); +} + const cssCellEditor = styled('div', ` background-color: ${theme.cellEditorBg}; font-family: var(--grist-font-family-data); diff --git a/app/client/widgets/ChoiceListEntry.ts b/app/client/widgets/ChoiceListEntry.ts index 9e83197d..450282f0 100644 --- a/app/client/widgets/ChoiceListEntry.ts +++ b/app/client/widgets/ChoiceListEntry.ts @@ -605,7 +605,6 @@ const cssButtonRow = styled('div', ` gap: 8px; display: flex; margin-top: 8px; - margin-bottom: 16px; `); const cssDeleteButton = styled('div', ` diff --git a/app/client/widgets/ChoiceTextBox.ts b/app/client/widgets/ChoiceTextBox.ts index f5a4208d..d2740947 100644 --- a/app/client/widgets/ChoiceTextBox.ts +++ b/app/client/widgets/ChoiceTextBox.ts @@ -3,6 +3,7 @@ import { FormOptionsSortConfig, FormSelectConfig, } from 'app/client/components/Forms/FormConfig'; +import {DropdownConditionConfig} from 'app/client/components/DropdownConditionConfig'; import {makeT} from 'app/client/lib/localization'; import {DataRowModel} from 'app/client/models/DataRowModel'; import {ViewFieldRec} from 'app/client/models/entities/ViewFieldRec'; @@ -82,11 +83,15 @@ export class ChoiceTextBox extends NTextBox { return [ super.buildConfigDom(), this.buildChoicesConfigDom(), + dom.create(DropdownConditionConfig, this.field), ]; } public buildTransformConfigDom() { - return this.buildConfigDom(); + return [ + super.buildConfigDom(), + this.buildChoicesConfigDom(), + ]; } public buildFormConfigDom() { diff --git a/app/client/widgets/ConditionalStyle.ts b/app/client/widgets/ConditionalStyle.ts index c17ef929..f2ccdf23 100644 --- a/app/client/widgets/ConditionalStyle.ts +++ b/app/client/widgets/ConditionalStyle.ts @@ -4,7 +4,8 @@ import {ColumnRec} from 'app/client/models/DocModel'; import {KoSaveableObservable} from 'app/client/models/modelUtil'; import {RuleOwner} from 'app/client/models/RuleOwner'; import {Style} from 'app/client/models/Styles'; -import {cssFieldFormula} from 'app/client/ui/FieldConfig'; +import {buildHighlightedCode} from 'app/client/ui/CodeHighlight'; +import {cssFieldFormula} from 'app/client/ui/RightPanelStyles'; import {withInfoTooltip} from 'app/client/ui/tooltips'; import {textButton} from 'app/client/ui2018/buttons'; import {ColorOption, colorSelect} from 'app/client/ui2018/ColorSelect'; @@ -180,10 +181,11 @@ export class ConditionalStyle extends Disposable { column: ColumnRec, hasError: Observable ) { - return cssFieldFormula( + return dom.create(buildHighlightedCode, formula, - { gristTheme: this._gristDoc.currentTheme, maxLines: 1 }, + { maxLines: 1 }, dom.cls('formula_field_sidepane'), + dom.cls(cssFieldFormula.className), dom.cls(cssErrorBorder.className, hasError), { tabIndex: '-1' }, dom.on('focus', (_, refElem) => { diff --git a/app/client/widgets/EditorButtons.ts b/app/client/widgets/EditorButtons.ts index 71f28648..ac08fd09 100644 --- a/app/client/widgets/EditorButtons.ts +++ b/app/client/widgets/EditorButtons.ts @@ -10,8 +10,8 @@ import {dom, styled} from 'grainjs'; export function createMobileButtons(commands: IEditorCommandGroup) { // TODO A better check may be to detect a physical keyboard or touch support. return isDesktop() ? null : [ - cssCancelBtn(cssIconWrap(cssFinishIcon('CrossSmall')), dom.on('click', commands.fieldEditCancel)), - cssSaveBtn(cssIconWrap(cssFinishIcon('Tick')), dom.on('click', commands.fieldEditSaveHere)), + cssCancelBtn(cssIconWrap(cssFinishIcon('CrossSmall')), dom.on('mousedown', commands.fieldEditCancel)), + cssSaveBtn(cssIconWrap(cssFinishIcon('Tick')), dom.on('mousedown', commands.fieldEditSaveHere)), ]; } diff --git a/app/client/widgets/FormulaAssistant.ts b/app/client/widgets/FormulaAssistant.ts index 2006dfe6..4aa2378f 100644 --- a/app/client/widgets/FormulaAssistant.ts +++ b/app/client/widgets/FormulaAssistant.ts @@ -9,12 +9,13 @@ import {ColumnRec, ViewFieldRec} from 'app/client/models/DocModel'; import {ChatMessage} from 'app/client/models/entities/ColumnRec'; import {HAS_FORMULA_ASSISTANT, WHICH_FORMULA_ASSISTANT} from 'app/client/models/features'; import {getLoginOrSignupUrl, urlState} from 'app/client/models/gristUrlState'; -import {buildHighlightedCode} from 'app/client/ui/CodeHighlight'; +import {buildCodeHighlighter, buildHighlightedCode} from 'app/client/ui/CodeHighlight'; import {autoGrow} from 'app/client/ui/forms'; import {sanitizeHTML} from 'app/client/ui/sanitizeHTML'; import {createUserImage} from 'app/client/ui/UserImage'; import {basicButton, bigPrimaryButtonLink, primaryButton} from 'app/client/ui2018/buttons'; import {theme, vars} from 'app/client/ui2018/cssVars'; +import {gristThemeObs} from 'app/client/ui2018/theme'; import {icon} from 'app/client/ui2018/icons'; import {cssLink} from 'app/client/ui2018/links'; import {loadingDots} from 'app/client/ui2018/loaders'; @@ -1009,26 +1010,19 @@ class ChatHistory extends Disposable { * Renders the message as markdown if possible, otherwise as a code block. */ private _render(message: string, ...args: DomElementArg[]) { - const doc = this._options.gristDoc; if (this.supportsMarkdown()) { return dom('div', - (el) => subscribeElem(el, doc.currentTheme, () => { + (el) => subscribeElem(el, gristThemeObs(), async () => { + const highlightCode = await buildCodeHighlighter({maxLines: 60}); const content = sanitizeHTML(marked(message, { - highlight: (code) => { - const codeBlock = buildHighlightedCode(code, { - gristTheme: doc.currentTheme, - maxLines: 60, - }); - return codeBlock.innerHTML; - }, + highlight: (code) => highlightCode(code) })); el.innerHTML = content; }), ...args ); } else { - return buildHighlightedCode(message, { - gristTheme: doc.currentTheme, + return dom.create(buildHighlightedCode, message, { maxLines: 100, }); } diff --git a/app/client/widgets/FormulaEditor.ts b/app/client/widgets/FormulaEditor.ts index c30681e3..010c1e93 100644 --- a/app/client/widgets/FormulaEditor.ts +++ b/app/client/widgets/FormulaEditor.ts @@ -74,15 +74,13 @@ export class FormulaEditor extends NewBaseEditor { this._aceEditor = AceEditor.create({ // A bit awkward, but we need to assume calcSize is not used until attach() has been called // and _editorPlacement created. - column: options.column, calcSize: this._calcSize.bind(this), - gristDoc: options.gristDoc, saveValueOnBlurEvent: !options.readonly, editorState : this.editorState, - readonly: options.readonly + readonly: options.readonly, + getSuggestions: this._getSuggestions.bind(this), }); - // For editable editor we will grab the cursor when we are in the formula editing mode. const cursorCommands = options.readonly ? {} : { setCursor: this._onSetCursor }; const isActive = Computed.create(this, use => Boolean(use(editingFormula))); @@ -201,10 +199,7 @@ export class FormulaEditor extends NewBaseEditor { cssFormulaEditor.cls('-detached', this.isDetached), dom('div.formula_editor.formula_field_edit', testId('formula-editor'), this._aceEditor.buildDom((aceObj: any) => { - aceObj.setFontSize(11); - aceObj.setHighlightActiveLine(false); - aceObj.getSession().setUseWrapMode(false); - aceObj.renderer.setPadding(0); + initializeAceOptions(aceObj); const val = initialValue; const pos = Math.min(options.cursorPos, val.length); this._aceEditor.setValue(val, pos); @@ -405,6 +400,17 @@ export class FormulaEditor extends NewBaseEditor { return result; } + private _getSuggestions(prefix: string) { + const section = this.options.gristDoc.viewModel.activeSection(); + // If section is disposed or is pointing to an empty row, don't try to autocomplete. + if (!section?.getRowId()) { return []; } + + const tableId = section.table().tableId(); + const columnId = this.options.column.colId(); + const rowId = section.activeRowId(); + return this.options.gristDoc.docComm.autocomplete(prefix, tableId, columnId, rowId); + } + // TODO: update regexes to unicode? private _onSetCursor(row?: DataRowModel, col?: ViewFieldRec) { // Don't do anything when we are readonly. @@ -714,6 +720,13 @@ export function createFormulaErrorObs(owner: MultiHolder, gristDoc: GristDoc, or return errorMessage; } +export function initializeAceOptions(aceObj: any) { + aceObj.setFontSize(11); + aceObj.setHighlightActiveLine(false); + aceObj.getSession().setUseWrapMode(false); + aceObj.renderer.setPadding(0); +} + const cssCollapseIcon = styled(icon, ` margin: -3px 4px 0 4px; --icon-color: ${colors.slate}; diff --git a/app/client/widgets/Reference.ts b/app/client/widgets/Reference.ts index 282388ac..81da964b 100644 --- a/app/client/widgets/Reference.ts +++ b/app/client/widgets/Reference.ts @@ -3,6 +3,7 @@ import { FormOptionsSortConfig, FormSelectConfig } from 'app/client/components/Forms/FormConfig'; +import {DropdownConditionConfig} from 'app/client/components/DropdownConditionConfig'; import {makeT} from 'app/client/lib/localization'; import {DataRowModel} from 'app/client/models/DataRowModel'; import {TableRec} from 'app/client/models/DocModel'; @@ -55,6 +56,7 @@ export class Reference extends NTextBox { public buildConfigDom() { return [ this.buildTransformConfigDom(), + dom.create(DropdownConditionConfig, this.field), cssLabel(t('CELL FORMAT')), super.buildConfigDom(), ]; diff --git a/app/client/widgets/ReferenceEditor.ts b/app/client/widgets/ReferenceEditor.ts index 71ee3254..79358917 100644 --- a/app/client/widgets/ReferenceEditor.ts +++ b/app/client/widgets/ReferenceEditor.ts @@ -11,7 +11,6 @@ import { nocaseEqual, ReferenceUtils } from 'app/client/lib/ReferenceUtils'; import { undef } from 'app/common/gutil'; import { styled } from 'grainjs'; - /** * A ReferenceEditor offers an autocomplete of choices from the referenced table. */ @@ -28,7 +27,12 @@ export class ReferenceEditor extends NTextEditor { this._utils = new ReferenceUtils(options.field, docData); const vcol = this._utils.visibleColModel; - this._enableAddNew = vcol && !vcol.isRealFormula() && !!vcol.colId(); + this._enableAddNew = ( + vcol && + !vcol.isRealFormula() && + !!vcol.colId() && + !this._utils.hasDropdownCondition + ); // Decorate the editor to look like a reference column value (with a "link" icon). // But not on readonly mode - here we will reuse default decoration @@ -65,7 +69,8 @@ export class ReferenceEditor extends NTextEditor { // don't create autocomplete for readonly mode if (this.options.readonly) { return; } this._autocomplete = this.autoDispose(new Autocomplete(this.textInput, { - menuCssClass: menuCssClass + ' ' + cssRefList.className, + menuCssClass: `${menuCssClass} ${cssRefList.className} test-autocomplete`, + buildNoItemsMessage: () => this._utils.buildNoItemsMessage(), search: this._doSearch.bind(this), renderItem: this._renderItem.bind(this), getItemText: (item) => item.text, @@ -110,7 +115,7 @@ export class ReferenceEditor extends NTextEditor { * Also see: prepForSave. */ private async _doSearch(text: string): Promise> { - const result = this._utils.autocompleteSearch(text); + const result = this._utils.autocompleteSearch(text, this.options.rowId); this._showAddNew = false; if (!this._enableAddNew || !text) { return result; } @@ -120,7 +125,7 @@ export class ReferenceEditor extends NTextEditor { return result; } - result.items.push({rowId: 'new', text, cleanText}); + result.extraItems.push({rowId: 'new', text, cleanText}); this._showAddNew = true; return result; diff --git a/app/client/widgets/ReferenceListEditor.ts b/app/client/widgets/ReferenceListEditor.ts index be6f7426..c2060199 100644 --- a/app/client/widgets/ReferenceListEditor.ts +++ b/app/client/widgets/ReferenceListEditor.ts @@ -59,10 +59,16 @@ export class ReferenceListEditor extends NewBaseEditor { this._utils = new ReferenceUtils(options.field, docData); const vcol = this._utils.visibleColModel; - this._enableAddNew = vcol && !vcol.isRealFormula() && !!vcol.colId(); + this._enableAddNew = ( + vcol && + !vcol.isRealFormula() && + !!vcol.colId() && + !this._utils.hasDropdownCondition + ); const acOptions: IAutocompleteOptions = { - menuCssClass: `${menuCssClass} ${cssRefList.className}`, + menuCssClass: `${menuCssClass} ${cssRefList.className} test-autocomplete`, + buildNoItemsMessage: () => this._utils.buildNoItemsMessage(), search: this._doSearch.bind(this), renderItem: this._renderItem.bind(this), getItemText: (item) => item.text, @@ -166,12 +172,14 @@ export class ReferenceListEditor extends NewBaseEditor { } public getCellValue(): CellValue { - const rowIds = this._tokenField.tokensObs.get().map(t => typeof t.rowId === 'number' ? t.rowId : t.text); + const rowIds = this._tokenField.tokensObs.get() + .map(token => typeof token.rowId === 'number' ? token.rowId : token.text); return encodeObject(rowIds); } public getTextValue(): string { - const rowIds = this._tokenField.tokensObs.get().map(t => typeof t.rowId === 'number' ? String(t.rowId) : t.text); + const rowIds = this._tokenField.tokensObs.get() + .map(token => typeof token.rowId === 'number' ? String(token.rowId) : token.text); return csvEncodeRow(rowIds, {prettier: true}); } @@ -184,19 +192,19 @@ export class ReferenceListEditor extends NewBaseEditor { */ public async prepForSave() { const tokens = this._tokenField.tokensObs.get(); - const newValues = tokens.filter(t => t.rowId === 'new'); + const newValues = tokens.filter(({rowId})=> rowId === 'new'); if (newValues.length === 0) { return; } // Add the new items to the referenced table. - const colInfo = {[this._utils.visibleColId]: newValues.map(t => t.text)}; + const colInfo = {[this._utils.visibleColId]: newValues.map(({text}) => text)}; const rowIds = await this._utils.tableData.sendTableAction( ["BulkAddRecord", new Array(newValues.length).fill(null), colInfo] ); // Update the TokenField tokens with the returned row ids. let i = 0; - const newTokens = tokens.map(t => { - return t.rowId === 'new' ? new ReferenceItem(t.text, rowIds[i++]) : t; + const newTokens = tokens.map(token => { + return token.rowId === 'new' ? new ReferenceItem(token.text, rowIds[i++]) : token; }); this._tokenField.setTokens(newTokens); } @@ -254,11 +262,12 @@ export class ReferenceListEditor extends NewBaseEditor { * Also see: prepForSave. */ private async _doSearch(text: string): Promise> { - const {items, selectIndex, highlightFunc} = this._utils.autocompleteSearch(text); + const {items, selectIndex, highlightFunc} = this._utils.autocompleteSearch(text, this.options.rowId); const result: ACResults = { selectIndex, highlightFunc, - items: items.map(i => new ReferenceItem(i.text, i.rowId)) + items: items.map(i => new ReferenceItem(i.text, i.rowId)), + extraItems: [], }; this._showAddNew = false; @@ -269,7 +278,7 @@ export class ReferenceListEditor extends NewBaseEditor { return result; } - result.items.push(new ReferenceItem(text, 'new')); + result.extraItems.push(new ReferenceItem(text, 'new')); this._showAddNew = true; return result; diff --git a/app/common/ACLRuleCollection.ts b/app/common/ACLRuleCollection.ts index f2f004dd..ce1f8234 100644 --- a/app/common/ACLRuleCollection.ts +++ b/app/common/ACLRuleCollection.ts @@ -3,14 +3,15 @@ import {AVAILABLE_BITS_COLUMNS, AVAILABLE_BITS_TABLES, trimPermissions} from 'ap import {ACLRulesReader} from 'app/common/ACLRulesReader'; import {AclRuleProblem} from 'app/common/ActiveDocAPI'; import {DocData} from 'app/common/DocData'; -import {AclMatchFunc, ParsedAclFormula, RulePart, RuleSet, UserAttributeRule} from 'app/common/GranularAccessClause'; +import {RulePart, RuleSet, UserAttributeRule} from 'app/common/GranularAccessClause'; import {getSetMapValue, isNonNullish} from 'app/common/gutil'; +import {CompiledPredicateFormula, ParsedPredicateFormula} from 'app/common/PredicateFormula'; import {MetaRowRecord} from 'app/common/TableData'; import {decodeObject} from 'app/plugin/objtypes'; export type ILogger = Pick; -const defaultMatchFunc: AclMatchFunc = () => true; +const defaultMatchFunc: CompiledPredicateFormula = () => true; export const SPECIAL_RULES_TABLE_ID = '*SPECIAL'; @@ -20,12 +21,12 @@ const DEFAULT_RULE_SET: RuleSet = { colIds: '*', body: [{ aclFormula: "user.Access in [EDITOR, OWNER]", - matchFunc: (input) => ['editors', 'owners'].includes(String(input.user.Access)), + matchFunc: (input) => ['editors', 'owners'].includes(String(input.user!.Access)), permissions: parsePermissions('all'), permissionsText: 'all', }, { aclFormula: "user.Access in [VIEWER]", - matchFunc: (input) => ['viewers'].includes(String(input.user.Access)), + matchFunc: (input) => ['viewers'].includes(String(input.user!.Access)), permissions: parsePermissions('+R-CUDS'), permissionsText: '+R', }, { @@ -48,7 +49,7 @@ const SPECIAL_RULE_SETS: Record = { colIds: ['SchemaEdit'], body: [{ aclFormula: "user.Access in [EDITOR, OWNER]", - matchFunc: (input) => ['editors', 'owners'].includes(String(input.user.Access)), + matchFunc: (input) => ['editors', 'owners'].includes(String(input.user!.Access)), permissions: parsePermissions('+S'), permissionsText: '+S', }, { @@ -63,7 +64,7 @@ const SPECIAL_RULE_SETS: Record = { colIds: ['AccessRules'], body: [{ aclFormula: "user.Access in [OWNER]", - matchFunc: (input) => ['owners'].includes(String(input.user.Access)), + matchFunc: (input) => ['owners'].includes(String(input.user!.Access)), permissions: parsePermissions('+R'), permissionsText: '+R', }, { @@ -78,7 +79,7 @@ const SPECIAL_RULE_SETS: Record = { colIds: ['FullCopies'], body: [{ aclFormula: "user.Access in [OWNER]", - matchFunc: (input) => ['owners'].includes(String(input.user.Access)), + matchFunc: (input) => ['owners'].includes(String(input.user!.Access)), permissions: parsePermissions('+R'), permissionsText: '+R', }, { @@ -102,7 +103,7 @@ const EMERGENCY_RULE_SET: RuleSet = { colIds: '*', body: [{ aclFormula: "user.Access in [OWNER]", - matchFunc: (input) => ['owners'].includes(String(input.user.Access)), + matchFunc: (input) => ['owners'].includes(String(input.user!.Access)), permissions: parsePermissions('all'), permissionsText: 'all', }, { @@ -381,7 +382,7 @@ export class ACLRuleCollection { export interface ReadAclOptions { log: ILogger; // For logging warnings during rule processing. - compile?: (parsed: ParsedAclFormula) => AclMatchFunc; + compile?: (parsed: ParsedPredicateFormula) => CompiledPredicateFormula; // If true, add and modify access rules in some special ways. // Specifically, call addHelperCols to add helper columns of restricted columns to rule sets, // and use ACLShareRules to implement any special shares as access rules. diff --git a/app/common/ACLRulesReader.ts b/app/common/ACLRulesReader.ts index 741a719e..97355861 100644 --- a/app/common/ACLRulesReader.ts +++ b/app/common/ACLRulesReader.ts @@ -3,7 +3,8 @@ import { getSetMapValue } from 'app/common/gutil'; import { SchemaTypes } from 'app/common/schema'; import { ShareOptions } from 'app/common/ShareOptions'; import { MetaRowRecord, MetaTableData } from 'app/common/TableData'; -import { isEqual, sortBy } from 'lodash'; +import isEqual from 'lodash/isEqual'; +import sortBy from 'lodash/sortBy'; /** * For special shares, we need to refer to resources that may not diff --git a/app/common/ActiveDocAPI.ts b/app/common/ActiveDocAPI.ts index f10a5804..0dbec7e8 100644 --- a/app/common/ActiveDocAPI.ts +++ b/app/common/ActiveDocAPI.ts @@ -1,6 +1,6 @@ import {ActionGroup} from 'app/common/ActionGroup'; import {BulkAddRecord, CellValue, TableDataAction, UserAction} from 'app/common/DocActions'; -import {FormulaProperties} from 'app/common/GranularAccessClause'; +import {PredicateFormulaProperties} from 'app/common/PredicateFormula'; import {FetchUrlOptions, UploadResult} from 'app/common/uploads'; import {DocStateComparison, PermissionData, UserAccessData} from 'app/common/UserAPI'; import {ParseOptions} from 'app/plugin/FileParserAPI'; @@ -421,7 +421,7 @@ export interface ActiveDocAPI { * Find and return a list of auto-complete suggestions that start with `txt`, when editing a * formula in table `tableId` and column `columnId`. */ - autocomplete(txt: string, tableId: string, columnId: string, rowId: UIRowId): Promise; + autocomplete(txt: string, tableId: string, columnId: string, rowId: UIRowId | null): Promise; /** * Removes the current instance from the doc. @@ -467,7 +467,7 @@ export interface ActiveDocAPI { /** * Check if an ACL formula is valid. If not, will throw an error with an explanation. */ - checkAclFormula(text: string): Promise; + checkAclFormula(text: string): Promise; /** * Get a token for out-of-band access to the document. diff --git a/app/common/ColumnFilterFunc.ts b/app/common/ColumnFilterFunc.ts index 8d62f4f3..742c0737 100644 --- a/app/common/ColumnFilterFunc.ts +++ b/app/common/ColumnFilterFunc.ts @@ -4,7 +4,7 @@ import {decodeObject} from "app/plugin/objtypes"; import moment, { Moment } from "moment-timezone"; import {extractInfoFromColType, isDateLikeType, isList, isListType, isNumberType} from "app/common/gristTypes"; import {isRelativeBound, relativeDateToUnixTimestamp} from "app/common/RelativeDates"; -import {noop} from "lodash"; +import noop from "lodash/noop"; export type ColumnFilterFunc = (value: CellValue) => boolean; diff --git a/app/common/DropdownCondition.ts b/app/common/DropdownCondition.ts new file mode 100644 index 00000000..b9262295 --- /dev/null +++ b/app/common/DropdownCondition.ts @@ -0,0 +1,20 @@ +import { CompiledPredicateFormula } from 'app/common/PredicateFormula'; + +export interface DropdownCondition { + text: string; + parsed: string; +} + +export type DropdownConditionCompilationResult = + | DropdownConditionCompilationSuccess + | DropdownConditionCompilationFailure; + +interface DropdownConditionCompilationSuccess { + kind: 'success'; + result: CompiledPredicateFormula; +} + +interface DropdownConditionCompilationFailure { + kind: 'failure'; + error: string; +} diff --git a/app/common/GranularAccessClause.ts b/app/common/GranularAccessClause.ts index b5ed6707..3f3b7769 100644 --- a/app/common/GranularAccessClause.ts +++ b/app/common/GranularAccessClause.ts @@ -1,7 +1,8 @@ import {PartialPermissionSet} from 'app/common/ACLPermissions'; import {CellValue, RowRecord} from 'app/common/DocActions'; +import {CompiledPredicateFormula} from 'app/common/PredicateFormula'; +import {Role} from 'app/common/roles'; import {MetaRowRecord} from 'app/common/TableData'; -import {Role} from './roles'; export interface RuleSet { tableId: '*' | string; @@ -18,7 +19,7 @@ export interface RulePart { permissionsText: string; // The text version of PermissionSet, as stored. // Compiled version of aclFormula. - matchFunc?: AclMatchFunc; + matchFunc?: CompiledPredicateFormula; // Optional memo, currently extracted from comment in formula. memo?: string; @@ -53,35 +54,6 @@ export interface UserInfo { toJSON(): {[key: string]: any}; } -/** - * Input into the AclMatchFunc. Compiled formulas evaluate AclMatchInput to produce a boolean. - */ -export interface AclMatchInput { - user: UserInfo; - rec?: InfoView; - newRec?: InfoView; - docId?: string; -} - -/** - * The actual boolean function that can evaluate a request. The result of compiling ParsedAclFormula. - */ -export type AclMatchFunc = (input: AclMatchInput) => boolean; - -/** - * Representation of a parsed ACL formula. - */ -type PrimitiveCellValue = number|string|boolean|null; -export type ParsedAclFormula = [string, ...(ParsedAclFormula|PrimitiveCellValue)[]]; - -/** - * Observations about a formula. - */ -export interface FormulaProperties { - hasRecOrNewRec?: boolean; - usedColIds?: string[]; -} - export interface UserAttributeRule { origRecord?: RowRecord; // Original record used to create this UserAttributeRule. name: string; // Should be unique among UserAttributeRules. @@ -89,45 +61,3 @@ export interface UserAttributeRule { lookupColId: string; // Column in tableId in which to do the lookup. charId: string; // Attribute to look up, possibly a path. E.g. 'Email' or 'office.city'. } - -/** - * Check some key facts about the formula. - */ -export function getFormulaProperties(formula: ParsedAclFormula) { - const result: FormulaProperties = {}; - if (usesRec(formula)) { result.hasRecOrNewRec = true; } - const colIds = new Set(); - collectRecColIds(formula, colIds); - result.usedColIds = Array.from(colIds); - return result; -} - -/** - * Check whether a formula mentions `rec` or `newRec`. - */ -export function usesRec(formula: ParsedAclFormula): boolean { - if (!Array.isArray(formula)) { throw new Error('expected a list'); } - if (isRecOrNewRec(formula)) { - return true; - } - return formula.some(el => { - if (!Array.isArray(el)) { return false; } - return usesRec(el); - }); -} - -function isRecOrNewRec(formula: ParsedAclFormula|PrimitiveCellValue): boolean { - return Array.isArray(formula) && - formula[0] === 'Name' && - (formula[1] === 'rec' || formula[1] === 'newRec'); -} - -function collectRecColIds(formula: ParsedAclFormula, colIds: Set): void { - if (!Array.isArray(formula)) { throw new Error('expected a list'); } - if (formula[0] === 'Attr' && isRecOrNewRec(formula[1])) { - const colId = formula[2]; - colIds.add(String(colId)); - return; - } - formula.forEach(el => Array.isArray(el) && collectRecColIds(el, colIds)); -} diff --git a/app/common/PredicateFormula.ts b/app/common/PredicateFormula.ts new file mode 100644 index 00000000..2d9d5f30 --- /dev/null +++ b/app/common/PredicateFormula.ts @@ -0,0 +1,223 @@ +/** + * Representation and compilation of predicate formulas. + * + * An example of a predicate formula is: "rec.office == 'Seattle' and user.email in ['sally@', 'xie@']". + * These formulas are parsed in Python into a tree with nodes of the form [NODE_TYPE, ...args]. + * See sandbox/grist/predicate_formula.py for details. + * + * This module includes typings for the nodes, and the compilePredicateFormula() function that + * turns such trees into actual predicate functions. + */ +import {CellValue, RowRecord} from 'app/common/DocActions'; +import {ErrorWithCode} from 'app/common/ErrorWithCode'; +import {InfoView, UserInfo} from 'app/common/GranularAccessClause'; +import {decodeObject} from 'app/plugin/objtypes'; +import constant = require('lodash/constant'); + +/** + * Representation of a parsed predicate formula. + */ +export type PrimitiveCellValue = number|string|boolean|null; +export type ParsedPredicateFormula = [string, ...(ParsedPredicateFormula|PrimitiveCellValue)[]]; + +/** + * Inputs to a predicate formula function. + */ +export interface PredicateFormulaInput { + user?: UserInfo; + rec?: RowRecord|InfoView; + newRec?: InfoView; + docId?: string; + choice?: string|RowRecord|InfoView; +} + +export class EmptyRecordView implements InfoView { + public get(_colId: string): CellValue { return null; } + public toJSON() { return {}; } +} + +/** + * The result of compiling ParsedPredicateFormula. + */ +export type CompiledPredicateFormula = (input: PredicateFormulaInput) => boolean; + +const GRIST_CONSTANTS: Record = { + EDITOR: 'editors', + OWNER: 'owners', + VIEWER: 'viewers', +}; + +/** + * An intermediate predicate formula returned during compilation, which may return + * a non-boolean value. + */ +type IntermediatePredicateFormula = (input: PredicateFormulaInput) => any; + +export interface CompilePredicateFormulaOptions { + /** Defaults to `'acl'`. */ + variant?: 'acl'|'dropdown-condition'; +} + +/** + * Compiles a parsed predicate formula and returns it. + */ +export function compilePredicateFormula( + parsedPredicateFormula: ParsedPredicateFormula, + options: CompilePredicateFormulaOptions = {} +): CompiledPredicateFormula { + const {variant = 'acl'} = options; + + function compileNode(node: ParsedPredicateFormula): IntermediatePredicateFormula { + const rawArgs = node.slice(1); + const args = rawArgs as ParsedPredicateFormula[]; + switch (node[0]) { + case 'And': { const parts = args.map(compileNode); return (input) => parts.every(p => p(input)); } + case 'Or': { const parts = args.map(compileNode); return (input) => parts.some(p => p(input)); } + case 'Add': return compileAndCombine(args, ([a, b]) => a + b); + case 'Sub': return compileAndCombine(args, ([a, b]) => a - b); + case 'Mult': return compileAndCombine(args, ([a, b]) => a * b); + case 'Div': return compileAndCombine(args, ([a, b]) => a / b); + case 'Mod': return compileAndCombine(args, ([a, b]) => a % b); + case 'Not': return compileAndCombine(args, ([a]) => !a); + case 'Eq': return compileAndCombine(args, ([a, b]) => a === b); + case 'NotEq': return compileAndCombine(args, ([a, b]) => a !== b); + case 'Lt': return compileAndCombine(args, ([a, b]) => a < b); + case 'LtE': return compileAndCombine(args, ([a, b]) => a <= b); + case 'Gt': return compileAndCombine(args, ([a, b]) => a > b); + case 'GtE': return compileAndCombine(args, ([a, b]) => a >= b); + case 'Is': return compileAndCombine(args, ([a, b]) => a === b); + case 'IsNot': return compileAndCombine(args, ([a, b]) => a !== b); + case 'In': return compileAndCombine(args, ([a, b]) => Boolean(b?.includes(a))); + case 'NotIn': return compileAndCombine(args, ([a, b]) => !b?.includes(a)); + case 'List': return compileAndCombine(args, (values) => values); + case 'Const': return constant(node[1] as CellValue); + case 'Name': { + const name = rawArgs[0] as keyof PredicateFormulaInput; + if (GRIST_CONSTANTS[name]) { return constant(GRIST_CONSTANTS[name]); } + + let validNames: string[]; + switch (variant) { + case 'acl': { + validNames = ['newRec', 'rec', 'user']; + break; + } + case 'dropdown-condition': { + validNames = ['rec', 'choice']; + break; + } + } + if (!validNames.includes(name)) { throw new Error(`Unknown variable '${name}'`); } + + return (input) => input[name]; + } + case 'Attr': { + const attrName = rawArgs[1] as string; + return compileAndCombine([args[0]], ([value]) => getAttr(value, attrName, args[0])); + } + case 'Comment': return compileNode(args[0]); + } + throw new Error(`Unknown node type '${node[0]}'`); + } + + /** + * Helper for operators: compile a list of nodes, then when evaluating, evaluate them all and + * combine the array of results using the given combine() function. + */ + function compileAndCombine( + args: ParsedPredicateFormula[], + combine: (values: any[]) => any + ): IntermediatePredicateFormula { + const compiled = args.map(compileNode); + return (input: PredicateFormulaInput) => combine(compiled.map(c => c(input))); + } + + const compiledPredicateFormula = compileNode(parsedPredicateFormula); + return (input) => Boolean(compiledPredicateFormula(input)); +} + +function describeNode(node: ParsedPredicateFormula): string { + if (node[0] === 'Name') { + return node[1] as string; + } else if (node[0] === 'Attr') { + return describeNode(node[1] as ParsedPredicateFormula) + '.' + (node[2] as string); + } else { + return 'value'; + } +} + +function getAttr(value: any, attrName: string, valueNode: ParsedPredicateFormula): any { + if (value == null) { + if (valueNode[0] === 'Name' && (valueNode[1] === 'rec' || valueNode[1] === 'newRec')) { + // This code is recognized by GranularAccess to know when an ACL rule is row-specific. + throw new ErrorWithCode('NEED_ROW_DATA', `Missing row data '${valueNode[1]}'`); + } + throw new Error(`No value for '${describeNode(valueNode)}'`); + } + return typeof value.get === 'function' + ? decodeObject(value.get(attrName)) // InfoView + : value[attrName]; +} + +/** + * Predicate formula properties. + */ +export interface PredicateFormulaProperties { + /** + * List of column ids that are referenced by either `$` or `rec.` notation. + */ + recColIds?: string[]; + /** + * List of column ids that are referenced by `choice.` notation. + * + * Only applies to the `dropdown-condition` variant of predicate formulas, + * and only for Reference and Reference List columns. + */ + choiceColIds?: string[]; +} + +/** + * Returns properties about a predicate `formula`. + * + * Properties include the list of column ids referenced in the formula. + * Currently, this information is used for error validation; specifically, to + * report when invalid column ids are referenced in ACL formulas and dropdown + * conditions. + */ +export function getPredicateFormulaProperties( + formula: ParsedPredicateFormula +): PredicateFormulaProperties { + return { + recColIds: [...getRecColIds(formula)], + choiceColIds: [...getChoiceColIds(formula)], + }; +} + +function isRecOrNewRec(formula: ParsedPredicateFormula|PrimitiveCellValue): boolean { + return Array.isArray(formula) && + formula[0] === 'Name' && + (formula[1] === 'rec' || formula[1] === 'newRec'); +} + +function getRecColIds(formula: ParsedPredicateFormula): string[] { + return [...new Set(collectColIds(formula, isRecOrNewRec))]; +} + +function isChoice(formula: ParsedPredicateFormula|PrimitiveCellValue): boolean { + return Array.isArray(formula) && formula[0] === 'Name' && formula[1] === 'choice'; +} + +function getChoiceColIds(formula: ParsedPredicateFormula): string[] { + return [...new Set(collectColIds(formula, isChoice))]; +} + +function collectColIds( + formula: ParsedPredicateFormula, + isIdentifierWithColIds: (formula: ParsedPredicateFormula|PrimitiveCellValue) => boolean, +): string[] { + if (!Array.isArray(formula)) { throw new Error('expected a list'); } + if (formula[0] === 'Attr' && isIdentifierWithColIds(formula[1])) { + const colId = String(formula[2]); + return [colId]; + } + return formula.flatMap(el => Array.isArray(el) ? collectColIds(el, isIdentifierWithColIds) : []); +} diff --git a/app/common/RelativeDates.ts b/app/common/RelativeDates.ts index c3012dd0..2803d6b6 100644 --- a/app/common/RelativeDates.ts +++ b/app/common/RelativeDates.ts @@ -2,9 +2,12 @@ // time defined as a series of periods. Hence, starting from the current date, each one of the // periods gets applied successively which eventually yields to the final date. Typical relative -import { isEqual, isNumber, isUndefined, omitBy } from "lodash"; -import moment from "moment-timezone"; import getCurrentTime from "app/common/getCurrentTime"; +import isEqual from "lodash/isEqual"; +import isNumber from "lodash/isNumber"; +import isUndefined from "lodash/isUndefined"; +import omitBy from "lodash/omitBy"; +import moment from "moment-timezone"; // Relative date uses one or two periods. When relative dates are defined by two periods, they are // applied successively to the start date to resolve the target date. In practice in grist, as of diff --git a/app/server/lib/ACLFormula.ts b/app/server/lib/ACLFormula.ts deleted file mode 100644 index 41a8deca..00000000 --- a/app/server/lib/ACLFormula.ts +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Representation and compilation of ACL formulas. - * - * An example of an ACL formula is: "rec.office == 'Seattle' and user.email in ['sally@', 'xie@']". - * These formulas are parsed in Python into a tree with nodes of the form [NODE_TYPE, ...args]. - * See sandbox/grist/acl_formula.py for details. - * - * This modules includes typings for the nodes, and compileAclFormula() function that turns such a - * tree into an actual boolean function. - */ -import {CellValue} from 'app/common/DocActions'; -import {ErrorWithCode} from 'app/common/ErrorWithCode'; -import {AclMatchFunc, AclMatchInput, ParsedAclFormula} from 'app/common/GranularAccessClause'; -import {decodeObject} from "app/plugin/objtypes"; -import constant = require('lodash/constant'); - -const GRIST_CONSTANTS: Record = { - EDITOR: 'editors', - OWNER: 'owners', - VIEWER: 'viewers', -}; - -/** - * Compile a parsed ACL formula into an actual function that can evaluate a request. - */ -export function compileAclFormula(parsedAclFormula: ParsedAclFormula): AclMatchFunc { - const compiled = _compileNode(parsedAclFormula); - return (input) => Boolean(compiled(input)); -} - -/** - * Type for intermediate functions, which may return values other than booleans. - */ -type AclEvalFunc = (input: AclMatchInput) => any; - -/** - * Compile a single node of the parsed formula tree. - */ -function _compileNode(parsedAclFormula: ParsedAclFormula): AclEvalFunc { - const rawArgs = parsedAclFormula.slice(1); - const args = rawArgs as ParsedAclFormula[]; - switch (parsedAclFormula[0]) { - case 'And': { const parts = args.map(_compileNode); return (input) => parts.every(p => p(input)); } - case 'Or': { const parts = args.map(_compileNode); return (input) => parts.some(p => p(input)); } - case 'Add': return _compileAndCombine(args, ([a, b]) => a + b); - case 'Sub': return _compileAndCombine(args, ([a, b]) => a - b); - case 'Mult': return _compileAndCombine(args, ([a, b]) => a * b); - case 'Div': return _compileAndCombine(args, ([a, b]) => a / b); - case 'Mod': return _compileAndCombine(args, ([a, b]) => a % b); - case 'Not': return _compileAndCombine(args, ([a]) => !a); - case 'Eq': return _compileAndCombine(args, ([a, b]) => a === b); - case 'NotEq': return _compileAndCombine(args, ([a, b]) => a !== b); - case 'Lt': return _compileAndCombine(args, ([a, b]) => a < b); - case 'LtE': return _compileAndCombine(args, ([a, b]) => a <= b); - case 'Gt': return _compileAndCombine(args, ([a, b]) => a > b); - case 'GtE': return _compileAndCombine(args, ([a, b]) => a >= b); - case 'Is': return _compileAndCombine(args, ([a, b]) => a === b); - case 'IsNot': return _compileAndCombine(args, ([a, b]) => a !== b); - case 'In': return _compileAndCombine(args, ([a, b]) => Boolean(b?.includes(a))); - case 'NotIn': return _compileAndCombine(args, ([a, b]) => !b?.includes(a)); - case 'List': return _compileAndCombine(args, (values) => values); - case 'Const': return constant(parsedAclFormula[1] as CellValue); - case 'Name': { - const name = rawArgs[0] as keyof AclMatchInput; - if (GRIST_CONSTANTS[name]) { return constant(GRIST_CONSTANTS[name]); } - if (!['user', 'rec', 'newRec'].includes(name)) { - throw new Error(`Unknown variable '${name}'`); - } - return (input) => input[name]; - } - case 'Attr': { - const attrName = rawArgs[1] as string; - return _compileAndCombine([args[0]], ([value]) => getAttr(value, attrName, args[0])); - } - case 'Comment': return _compileNode(args[0]); - } - throw new Error(`Unknown node type '${parsedAclFormula[0]}'`); -} - -function describeNode(node: ParsedAclFormula): string { - if (node[0] === 'Name') { - return node[1] as string; - } else if (node[0] === 'Attr') { - return describeNode(node[1] as ParsedAclFormula) + '.' + (node[2] as string); - } else { - return 'value'; - } -} - -function getAttr(value: any, attrName: string, valueNode: ParsedAclFormula): any { - if (value == null) { - if (valueNode[0] === 'Name' && (valueNode[1] === 'rec' || valueNode[1] === 'newRec')) { - // This code is recognized by GranularAccess to know when a rule is row-specific. - throw new ErrorWithCode('NEED_ROW_DATA', `Missing row data '${valueNode[1]}'`); - } - throw new Error(`No value for '${describeNode(valueNode)}'`); - } - return (typeof value.get === 'function' ? decodeObject(value.get(attrName)) : // InfoView - value[attrName]); -} - -/** - * Helper for operators: compile a list of nodes, then when evaluating, evaluate them all and - * combine the array of results using the given combine() function. - */ -function _compileAndCombine(args: ParsedAclFormula[], combine: (values: any[]) => any): AclEvalFunc { - const compiled = args.map(_compileNode); - return (input: AclMatchInput) => combine(compiled.map(c => c(input))); -} diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 41acb51a..b65ba14b 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -64,12 +64,16 @@ import { } from 'app/common/DocUsage'; import {normalizeEmail} from 'app/common/emails'; import {Product} from 'app/common/Features'; -import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause'; import {isHiddenCol} from 'app/common/gristTypes'; import {commonUrls, parseUrlId} from 'app/common/gristUrls'; import {byteString, countIf, retryOnce, safeJsonParse, timeoutReached} from 'app/common/gutil'; import {InactivityTimer} from 'app/common/InactivityTimer'; import {Interval} from 'app/common/Interval'; +import { + compilePredicateFormula, + getPredicateFormulaProperties, + PredicateFormulaProperties, +} from 'app/common/PredicateFormula'; import * as roles from 'app/common/roles'; import {schema, SCHEMA_VERSION} from 'app/common/schema'; import {MetaRowRecord, SingleCell} from 'app/common/TableData'; @@ -84,7 +88,6 @@ import {Share} from 'app/gen-server/entity/Share'; import {RecordWithStringId} from 'app/plugin/DocApiTypes'; import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI'; import {AccessTokenOptions, AccessTokenResult, GristDocAPI, UIRowId} from 'app/plugin/GristAPI'; -import {compileAclFormula} from 'app/server/lib/ACLFormula'; import {AssistanceSchemaPromptV1Context} from 'app/server/lib/Assistance'; import {AssistanceContext} from 'app/common/AssistancePrompts'; import {Authorizer, RequestWithLogin} from 'app/server/lib/Authorizer'; @@ -1289,7 +1292,11 @@ export class ActiveDoc extends EventEmitter { } public async autocomplete( - docSession: DocSession, txt: string, tableId: string, columnId: string, rowId: UIRowId + docSession: DocSession, + txt: string, + tableId: string, + columnId: string, + rowId: UIRowId | null ): Promise { // Autocompletion can leak names of tables and columns. if (!await this._granularAccess.canScanData(docSession)) { return []; } @@ -1479,16 +1486,16 @@ export class ActiveDoc extends EventEmitter { /** * Check if an ACL formula is valid. If not, will throw an error with an explanation. */ - public async checkAclFormula(docSession: DocSession, text: string): Promise { + public async checkAclFormula(docSession: DocSession, text: string): Promise { // Checks can leak names of tables and columns. if (await this._granularAccess.hasNuancedAccess(docSession)) { return {}; } await this.waitForInitialization(); try { - const parsedAclFormula = await this._pyCall('parse_acl_formula', text); - compileAclFormula(parsedAclFormula); + const parsedAclFormula = await this._pyCall('parse_predicate_formula', text); + compilePredicateFormula(parsedAclFormula); // TODO We also need to check the validity of attributes, and of tables and columns // mentioned in resources and userAttribute rules. - return getFormulaProperties(parsedAclFormula); + return getPredicateFormulaProperties(parsedAclFormula); } catch (e) { e.message = e.message?.replace('[Sandbox] ', ''); throw e; diff --git a/app/server/lib/GranularAccess.ts b/app/server/lib/GranularAccess.ts index 73241aab..e6116d36 100644 --- a/app/server/lib/GranularAccess.ts +++ b/app/server/lib/GranularAccess.ts @@ -26,16 +26,15 @@ import { UserOverride } from 'app/common/DocListAPI'; import { DocUsageSummary, FilteredDocUsageSummary } from 'app/common/DocUsage'; import { normalizeEmail } from 'app/common/emails'; import { ErrorWithCode } from 'app/common/ErrorWithCode'; -import { AclMatchInput, InfoEditor, InfoView } from 'app/common/GranularAccessClause'; -import { UserInfo } from 'app/common/GranularAccessClause'; +import { InfoEditor, InfoView, UserInfo } from 'app/common/GranularAccessClause'; import * as gristTypes from 'app/common/gristTypes'; import { getSetMapValue, isNonNullish, pruneArray } from 'app/common/gutil'; +import { compilePredicateFormula, EmptyRecordView, PredicateFormulaInput } from 'app/common/PredicateFormula'; import { MetaRowRecord, SingleCell } from 'app/common/TableData'; import { canEdit, canView, isValidRole, Role } from 'app/common/roles'; import { FullUser, UserAccessData } from 'app/common/UserAPI'; import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager'; import { GristObjCode } from 'app/plugin/GristData'; -import { compileAclFormula } from 'app/server/lib/ACLFormula'; import { DocClients } from 'app/server/lib/DocClients'; import { getDocSessionAccess, getDocSessionAltSessionId, getDocSessionShare, getDocSessionUser, OptDocSession } from 'app/server/lib/DocSession'; @@ -344,7 +343,7 @@ export class GranularAccess implements GranularAccessForBundle { * Represent fields from the session in an input object for ACL rules. * Just one field currently, "user". */ - public async inputs(docSession: OptDocSession): Promise { + public async inputs(docSession: OptDocSession): Promise { return { user: await this._getUser(docSession), docId: this._docId @@ -401,7 +400,7 @@ export class GranularAccess implements GranularAccessForBundle { } const rec = new RecordView(rows, 0); if (!hasExceptionalAccess) { - const input: AclMatchInput = {...await this.inputs(docSession), rec, newRec: rec}; + const input: PredicateFormulaInput = {...await this.inputs(docSession), rec, newRec: rec}; const rowPermInfo = new PermissionInfo(this._ruler.ruleCollection, input); const rowAccess = rowPermInfo.getTableAccess(cell.tableId).perms.read; if (rowAccess === 'deny') { fail(); } @@ -560,7 +559,7 @@ export class GranularAccess implements GranularAccessForBundle { // Use the post-actions data to process the rules collection, and throw error if that fails. const ruleCollection = new ACLRuleCollection(); - await ruleCollection.update(tmpDocData, {log, compile: compileAclFormula}); + await ruleCollection.update(tmpDocData, {log, compile: compilePredicateFormula}); if (ruleCollection.ruleError) { throw new ApiError(ruleCollection.ruleError.message, 400); } @@ -1664,7 +1663,7 @@ export class GranularAccess implements GranularAccessForBundle { const rec = new RecordView(rowsRec, undefined); const newRec = new RecordView(rowsNewRec, undefined); - const input: AclMatchInput = {...await this.inputs(docSession), rec, newRec}; + const input: PredicateFormulaInput = {...await this.inputs(docSession), rec, newRec}; const [, tableId, , colValues] = action; let filteredColValues: ColValues | BulkColValues | undefined | null = null; @@ -1746,7 +1745,7 @@ export class GranularAccess implements GranularAccessForBundle { colId?: string): Promise { const ruler = await this._getRuler(cursor); const rec = new RecordView(data, undefined); - const input: AclMatchInput = {...await this.inputs(cursor.docSession), rec}; + const input: PredicateFormulaInput = {...await this.inputs(cursor.docSession), rec}; const [, tableId, rowIds] = data; const toRemove: number[] = []; @@ -2561,7 +2560,7 @@ export class GranularAccess implements GranularAccessForBundle { } } const rec = rows ? new RecordView(rows, 0) : undefined; - const input: AclMatchInput = {...inputs, rec, newRec: rec}; + const input: PredicateFormulaInput = {...inputs, rec, newRec: rec}; const rowPermInfo = new PermissionInfo(ruler.ruleCollection, input); const rowAccess = rowPermInfo.getTableAccess(cell.tableId).perms.read; if (rowAccess === 'deny') { return false; } @@ -2635,7 +2634,11 @@ export class Ruler { * Update granular access from DocData. */ public async update(docData: DocData) { - await this.ruleCollection.update(docData, {log, compile: compileAclFormula, enrichRulesForImplementation: true}); + await this.ruleCollection.update(docData, { + log, + compile: compilePredicateFormula, + enrichRulesForImplementation: true, + }); // Also clear the per-docSession cache of rule evaluations. this.clearCache(); @@ -2652,7 +2655,7 @@ export class Ruler { export interface RulerOwner { getUser(docSession: OptDocSession): Promise; - inputs(docSession: OptDocSession): Promise; + inputs(docSession: OptDocSession): Promise; } /** @@ -2762,11 +2765,6 @@ class RecordEditor implements InfoEditor { } } -class EmptyRecordView implements InfoView { - public get(colId: string): CellValue { return null; } - public toJSON() { return {}; } -} - /** * Cache information about user attributes. */ @@ -2840,7 +2838,7 @@ class CellAccessHelper { private _tableAccess: Map = new Map(); private _rowPermInfo: Map> = new Map(); private _rows: Map = new Map(); - private _inputs!: AclMatchInput; + private _inputs!: PredicateFormulaInput; constructor( private _granular: GranularAccess, @@ -2864,7 +2862,7 @@ class CellAccessHelper { for(const [idx, rowId] of rows[2].entries()) { if (rowIds.has(rowId) === false) { continue; } const rec = new RecordView(rows, idx); - const input: AclMatchInput = {...this._inputs, rec, newRec: rec}; + const input: PredicateFormulaInput = {...this._inputs, rec, newRec: rec}; const rowPermInfo = new PermissionInfo(this._ruler.ruleCollection, input); if (!this._rowPermInfo.has(tableId)) { this._rowPermInfo.set(tableId, new Map()); diff --git a/app/server/lib/PermissionInfo.ts b/app/server/lib/PermissionInfo.ts index 164db7ad..7ef7860f 100644 --- a/app/server/lib/PermissionInfo.ts +++ b/app/server/lib/PermissionInfo.ts @@ -3,7 +3,8 @@ import { ALL_PERMISSION_PROPS, emptyPermissionSet, MixedPermissionSet, PartialPermissionSet, PermissionSet, TablePermissionSet, toMixed } from 'app/common/ACLPermissions'; import { ACLRuleCollection } from 'app/common/ACLRuleCollection'; -import { AclMatchInput, RuleSet, UserInfo } from 'app/common/GranularAccessClause'; +import { RuleSet, UserInfo } from 'app/common/GranularAccessClause'; +import { PredicateFormulaInput } from 'app/common/PredicateFormula'; import { getSetMapValue } from 'app/common/gutil'; import log from 'app/server/lib/log'; import { mapValues } from 'lodash'; @@ -59,7 +60,7 @@ abstract class RuleInfo { // Construct a RuleInfo for a particular input, which is a combination of user and // optionally a record. - constructor(protected _acls: ACLRuleCollection, protected _input: AclMatchInput) {} + constructor(protected _acls: ACLRuleCollection, protected _input: PredicateFormulaInput) {} public getColumnAspect(tableId: string, colId: string): MixedT { const ruleSet: RuleSet|undefined = this._acls.getColumnRuleSet(tableId, colId); @@ -80,7 +81,7 @@ abstract class RuleInfo { } public getUser(): UserInfo { - return this._input.user; + return this._input.user!; } protected abstract _processRule(ruleSet: RuleSet, defaultAccess?: () => MixedT): MixedT; @@ -205,7 +206,7 @@ export class PermissionInfo extends RuleInfo= n.A - 1 or r.B < n.B * 2.5 or r.B > n.B / 2.5 or r.C % 2 != 0"), - ['Or', - ['LtE', - ['Attr', ['Name', 'r'], 'A'], - ['Add', ['Attr', ['Name', 'n'], 'A'], ['Const', 1]]], - ['GtE', - ['Attr', ['Name', 'r'], 'A'], - ['Sub', ['Attr', ['Name', 'n'], 'A'], ['Const', 1]]], - ['Lt', - ['Attr', ['Name', 'r'], 'B'], - ['Mult', ['Attr', ['Name', 'n'], 'B'], ['Const', 2.5]]], - ['Gt', - ['Attr', ['Name', 'r'], 'B'], - ['Div', ['Attr', ['Name', 'n'], 'B'], ['Const', 2.5]]], - ['NotEq', - ['Mod', ['Attr', ['Name', 'r'], 'C'], ['Const', 2]], - ['Const', 0]] - ]) - - self.assertEqual(parse_acl_formula( - "rec.A is True or rec.A is not False"), - ['Or', - ['Is', ['Attr', ['Name', 'rec'], 'A'], ['Const', True]], - ['IsNot', ['Attr', ['Name', 'rec'], 'A'], ['Const', False]] - ]) - - self.assertEqual(parse_acl_formula( - "$A is True or $A is not False"), - ['Or', - ['Is', ['Attr', ['Name', 'rec'], 'A'], ['Const', True]], - ['IsNot', ['Attr', ['Name', 'rec'], 'A'], ['Const', False]] - ]) - - self.assertEqual(parse_acl_formula( - "user.Office.City == 'Seattle' and user.Status.IsActive"), - ['And', - ['Eq', - ['Attr', ['Attr', ['Name', 'user'], 'Office'], 'City'], - ['Const', 'Seattle']], - ['Attr', ['Attr', ['Name', 'user'], 'Status'], 'IsActive'] - ]) - - self.assertEqual(parse_acl_formula( - "True # Comment! "), - ['Comment', ['Const', True], 'Comment!']) - - self.assertEqual(parse_acl_formula( - "\"#x\" == \" # Not a comment \"#Comment!"), - ['Comment', - ['Eq', ['Const', '#x'], ['Const', ' # Not a comment ']], - 'Comment!' - ]) - - self.assertEqual(parse_acl_formula( - "# Allow owners\nuser.Access == 'owners' # ignored\n# comment ignored"), - ['Comment', - ['Eq', ['Attr', ['Name', 'user'], 'Access'], ['Const', 'owners']], - 'Allow owners' - ]) - - def test_unsupported(self): - # Test a few constructs we expect to fail - # Not an expression - self.assertRaises(SyntaxError, parse_acl_formula, "return 1") - self.assertRaises(SyntaxError, parse_acl_formula, "def foo(): pass") - - # Unsupported node type - self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "max(rec)") - self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "user.id in {1, 2, 3}") - self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 if user.IsAnon else 2") - - # Unsupported operation - self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 | 2") - self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "1 << 2") - self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_acl_formula, "~test") - - # Syntax error - self.assertRaises(SyntaxError, parse_acl_formula, "[(]") - self.assertRaises(SyntaxError, parse_acl_formula, "user.id in (1,2))") - self.assertRaisesRegex(SyntaxError, r'invalid syntax on line 1 col 9', parse_acl_formula, "foo and !bar") - class TestACLFormulaUserActions(test_engine.EngineTestCase): def test_acl_actions(self): # Adding or updating ACLRules automatically includes aclFormula compilation. diff --git a/sandbox/grist/test_dropdown_condition.py b/sandbox/grist/test_dropdown_condition.py new file mode 100644 index 00000000..c7add3e5 --- /dev/null +++ b/sandbox/grist/test_dropdown_condition.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +# pylint:disable=line-too-long +import json + +import test_engine + +class TestDropdownConditionUserActions(test_engine.EngineTestCase): + def test_dropdown_condition_col_actions(self): + self.apply_user_action(['AddTable', 'Table1', [ + {'id': 'A', 'type': 'Text'}, + {'id': 'B', 'type': 'Text'}, + {'id': 'C', 'type': 'Text'}, + ]]) + + # Check that setting dropdownCondition.text automatically sets a parsed version. + out_actions = self.apply_user_action(['UpdateRecord', '_grist_Tables_column', 1, { + "widgetOptions": json.dumps({ + "dropdownCondition": { + "text": 'choice.Role == "Manager"', + }, + }), + }]) + self.assertPartialOutActions(out_actions, { "stored": [ + ["UpdateRecord", "_grist_Tables_column", 1, { + "widgetOptions": "{\"dropdownCondition\": {\"text\": " + + "\"choice.Role == \\\"Manager\\\"\", \"parsed\": " + + "\"[\\\"Eq\\\", [\\\"Attr\\\", [\\\"Name\\\", \\\"choice\\\"], " + + "\\\"Role\\\"], [\\\"Const\\\", \\\"Manager\\\"]]\"}}" + }] + ]}) + out_actions = self.apply_user_action(['BulkUpdateRecord', '_grist_Tables_column', [2, 3], { + "widgetOptions": [ + json.dumps({ + "dropdownCondition": { + "text": 'choice == "Manager"', + }, + }), + json.dumps({ + "dropdownCondition": { + "text": '$Role == "Manager"', + }, + }), + ], + }]) + self.assertPartialOutActions(out_actions, { "stored": [ + ["BulkUpdateRecord", "_grist_Tables_column", [2, 3], { + "widgetOptions": [ + "{\"dropdownCondition\": {\"text\": \"choice == " + + "\\\"Manager\\\"\", \"parsed\": \"[\\\"Eq\\\", " + + "[\\\"Name\\\", \\\"choice\\\"], [\\\"Const\\\", \\\"Manager\\\"]]\"}}", + "{\"dropdownCondition\": {\"text\": \"$Role == " + + "\\\"Manager\\\"\", \"parsed\": \"[\\\"Eq\\\", " + + "[\\\"Attr\\\", [\\\"Name\\\", \\\"rec\\\"], \\\"Role\\\"], " + + "[\\\"Const\\\", \\\"Manager\\\"]]\"}}", + ] + }] + ]}) + + def test_dropdown_condition_field_actions(self): + self.apply_user_action(['AddTable', 'Table1', [ + {'id': 'A', 'type': 'Text'}, + {'id': 'B', 'type': 'Text'}, + {'id': 'C', 'type': 'Text'}, + ]]) + + # Check that setting dropdownCondition.text automatically sets a parsed version. + out_actions = self.apply_user_action(['UpdateRecord', '_grist_Views_section_field', 1, { + "widgetOptions": json.dumps({ + "dropdownCondition": { + "text": 'choice.Role == "Manager"', + }, + }), + }]) + self.assertPartialOutActions(out_actions, { "stored": [ + ["UpdateRecord", "_grist_Views_section_field", 1, { + "widgetOptions": "{\"dropdownCondition\": {\"text\": " + + "\"choice.Role == \\\"Manager\\\"\", \"parsed\": " + + "\"[\\\"Eq\\\", [\\\"Attr\\\", [\\\"Name\\\", \\\"choice\\\"], " + + "\\\"Role\\\"], [\\\"Const\\\", \\\"Manager\\\"]]\"}}" + }] + ]}) + out_actions = self.apply_user_action(['BulkUpdateRecord', '_grist_Views_section_field', [2, 3], { + "widgetOptions": [ + json.dumps({ + "dropdownCondition": { + "text": 'choice == "Manager"', + }, + }), + json.dumps({ + "dropdownCondition": { + "text": '$Role == "Manager"', + }, + }), + ], + }]) + self.assertPartialOutActions(out_actions, { "stored": [ + ["BulkUpdateRecord", "_grist_Views_section_field", [2, 3], { + "widgetOptions": [ + "{\"dropdownCondition\": {\"text\": \"choice == " + + "\\\"Manager\\\"\", \"parsed\": \"[\\\"Eq\\\", " + + "[\\\"Name\\\", \\\"choice\\\"], [\\\"Const\\\", \\\"Manager\\\"]]\"}}", + "{\"dropdownCondition\": {\"text\": \"$Role == " + + "\\\"Manager\\\"\", \"parsed\": \"[\\\"Eq\\\", " + + "[\\\"Attr\\\", [\\\"Name\\\", \\\"rec\\\"], \\\"Role\\\"], " + + "[\\\"Const\\\", \\\"Manager\\\"]]\"}}", + ] + }] + ]}) diff --git a/sandbox/grist/test_predicate_formula.py b/sandbox/grist/test_predicate_formula.py new file mode 100644 index 00000000..34914c2e --- /dev/null +++ b/sandbox/grist/test_predicate_formula.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +# pylint:disable=line-too-long + +import unittest +from predicate_formula import parse_predicate_formula + +class TestPredicateFormula(unittest.TestCase): + def test_basic(self): + # Test a few basic formulas and structures, hitting everything we expect to support + # in ACL formulas and dropdown conditions. + self.assertEqual(parse_predicate_formula( + "user.Email == 'X@'"), + ["Eq", ["Attr", ["Name", "user"], "Email"], + ["Const", "X@"]]) + + self.assertEqual(parse_predicate_formula( + "user.Role in ('editors', 'owners')"), + ["In", ["Attr", ["Name", "user"], "Role"], + ["List", ["Const", "editors"], ["Const", "owners"]]]) + + self.assertEqual(parse_predicate_formula( + "user.Role not in ('editors', 'owners')"), + ["NotIn", ["Attr", ["Name", "user"], "Role"], + ["List", ["Const", "editors"], ["Const", "owners"]]]) + + self.assertEqual(parse_predicate_formula( + "rec.office == 'Seattle' and user.email in ['sally@', 'xie@']"), + ['And', + ['Eq', ['Attr', ['Name', 'rec'], 'office'], ['Const', 'Seattle']], + ['In', + ['Attr', ['Name', 'user'], 'email'], + ['List', ['Const', 'sally@'], ['Const', 'xie@']] + ]]) + + self.assertEqual(parse_predicate_formula( + "$office == 'Seattle' and user.email in ['sally@', 'xie@']"), + ['And', + ['Eq', ['Attr', ['Name', 'rec'], 'office'], ['Const', 'Seattle']], + ['In', + ['Attr', ['Name', 'user'], 'email'], + ['List', ['Const', 'sally@'], ['Const', 'xie@']] + ]]) + + self.assertEqual(parse_predicate_formula( + "user.IsAdmin or rec.assigned is None or (not newRec.HasDuplicates and rec.StatusIndex <= newRec.StatusIndex)"), + ['Or', + ['Attr', ['Name', 'user'], 'IsAdmin'], + ['Is', ['Attr', ['Name', 'rec'], 'assigned'], ['Const', None]], + ['And', + ['Not', ['Attr', ['Name', 'newRec'], 'HasDuplicates']], + ['LtE', ['Attr', ['Name', 'rec'], 'StatusIndex'], ['Attr', ['Name', 'newRec'], 'StatusIndex']] + ] + ]) + + self.assertEqual(parse_predicate_formula( + "user.IsAdmin or $assigned is None or (not newRec.HasDuplicates and $StatusIndex <= newRec.StatusIndex)"), + ['Or', + ['Attr', ['Name', 'user'], 'IsAdmin'], + ['Is', ['Attr', ['Name', 'rec'], 'assigned'], ['Const', None]], + ['And', + ['Not', ['Attr', ['Name', 'newRec'], 'HasDuplicates']], + ['LtE', ['Attr', ['Name', 'rec'], 'StatusIndex'], ['Attr', ['Name', 'newRec'], 'StatusIndex']] + ] + ]) + + self.assertEqual(parse_predicate_formula( + "r.A <= n.A + 1 or r.A >= n.A - 1 or r.B < n.B * 2.5 or r.B > n.B / 2.5 or r.C % 2 != 0"), + ['Or', + ['LtE', + ['Attr', ['Name', 'r'], 'A'], + ['Add', ['Attr', ['Name', 'n'], 'A'], ['Const', 1]]], + ['GtE', + ['Attr', ['Name', 'r'], 'A'], + ['Sub', ['Attr', ['Name', 'n'], 'A'], ['Const', 1]]], + ['Lt', + ['Attr', ['Name', 'r'], 'B'], + ['Mult', ['Attr', ['Name', 'n'], 'B'], ['Const', 2.5]]], + ['Gt', + ['Attr', ['Name', 'r'], 'B'], + ['Div', ['Attr', ['Name', 'n'], 'B'], ['Const', 2.5]]], + ['NotEq', + ['Mod', ['Attr', ['Name', 'r'], 'C'], ['Const', 2]], + ['Const', 0]] + ]) + + self.assertEqual(parse_predicate_formula( + "rec.A is True or rec.A is not False"), + ['Or', + ['Is', ['Attr', ['Name', 'rec'], 'A'], ['Const', True]], + ['IsNot', ['Attr', ['Name', 'rec'], 'A'], ['Const', False]] + ]) + + self.assertEqual(parse_predicate_formula( + "$A is True or $A is not False"), + ['Or', + ['Is', ['Attr', ['Name', 'rec'], 'A'], ['Const', True]], + ['IsNot', ['Attr', ['Name', 'rec'], 'A'], ['Const', False]] + ]) + + self.assertEqual(parse_predicate_formula( + "user.Office.City == 'Seattle' and user.Status.IsActive"), + ['And', + ['Eq', + ['Attr', ['Attr', ['Name', 'user'], 'Office'], 'City'], + ['Const', 'Seattle']], + ['Attr', ['Attr', ['Name', 'user'], 'Status'], 'IsActive'] + ]) + + self.assertEqual(parse_predicate_formula( + "True # Comment! "), + ['Comment', ['Const', True], 'Comment!']) + + self.assertEqual(parse_predicate_formula( + "\"#x\" == \" # Not a comment \"#Comment!"), + ['Comment', + ['Eq', ['Const', '#x'], ['Const', ' # Not a comment ']], + 'Comment!' + ]) + + self.assertEqual(parse_predicate_formula( + "# Allow owners\nuser.Access == 'owners' # ignored\n# comment ignored"), + ['Comment', + ['Eq', ['Attr', ['Name', 'user'], 'Access'], ['Const', 'owners']], + 'Allow owners' + ]) + + self.assertEqual(parse_predicate_formula( + "choice not in $Categories"), + ['NotIn', ['Name', 'choice'], ['Attr', ['Name', 'rec'], 'Categories']]) + + self.assertEqual(parse_predicate_formula( + "choice.role == \"Manager\""), + ['Eq', ['Attr', ['Name', 'choice'], 'role'], ['Const', 'Manager']]) + + def test_unsupported(self): + # Test a few constructs we expect to fail + # Not an expression + self.assertRaises(SyntaxError, parse_predicate_formula, "return 1") + self.assertRaises(SyntaxError, parse_predicate_formula, "def foo(): pass") + + # Unsupported node type + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "max(rec)") + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "user.id in {1, 2, 3}") + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "1 if user.IsAnon else 2") + + # Unsupported operation + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "1 | 2") + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "1 << 2") + self.assertRaisesRegex(ValueError, r'Unsupported syntax', parse_predicate_formula, "~test") + + # Syntax error + self.assertRaises(SyntaxError, parse_predicate_formula, "[(]") + self.assertRaises(SyntaxError, parse_predicate_formula, "user.id in (1,2))") + self.assertRaisesRegex(SyntaxError, r'invalid syntax on line 1 col 9', parse_predicate_formula, "foo and !bar") diff --git a/sandbox/grist/useractions.py b/sandbox/grist/useractions.py index 4a8ec78e..ac9261ef 100644 --- a/sandbox/grist/useractions.py +++ b/sandbox/grist/useractions.py @@ -12,7 +12,8 @@ from six.moves import xrange import acl import depend import gencode -from acl_formula import parse_acl_formula_json +from acl_formula import parse_acl_formulas +from dropdown_condition import parse_dropdown_conditions import actions import column import sort_specs @@ -437,9 +438,7 @@ class UserActions(object): @override_action('BulkAddRecord', '_grist_ACLRules') def _addACLRules(self, table_id, row_ids, col_values): - # Automatically populate aclFormulaParsed value by parsing aclFormula. - if 'aclFormula' in col_values: - col_values['aclFormulaParsed'] = [parse_acl_formula_json(v) for v in col_values['aclFormula']] + parse_acl_formulas(col_values) return self.doBulkAddOrReplace(table_id, row_ids, col_values) #---------------------------------------- @@ -672,6 +671,7 @@ class UserActions(object): # columns for all summary tables of the same source table). # (4) Updates to the source columns of summary group-by columns (including renaming and type # changes) should be copied to those group-by columns. + parse_dropdown_conditions(col_values) # A list of individual (col_rec, values) updates, where values is a per-column dict. col_updates = OrderedDict() @@ -781,11 +781,14 @@ class UserActions(object): self.doBulkUpdateRecord(table_id, row_ids, col_values) + @override_action('BulkUpdateRecord', '_grist_Views_section_field') + def _updateViewSectionFields(self, table_id, row_ids, col_values): + parse_dropdown_conditions(col_values) + return self.doBulkUpdateRecord(table_id, row_ids, col_values) + @override_action('BulkUpdateRecord', '_grist_ACLRules') def _updateACLRules(self, table_id, row_ids, col_values): - # Automatically populate aclFormulaParsed value by parsing aclFormula. - if 'aclFormula' in col_values: - col_values['aclFormulaParsed'] = [parse_acl_formula_json(v) for v in col_values['aclFormula']] + parse_acl_formulas(col_values) return self.doBulkUpdateRecord(table_id, row_ids, col_values) def _prepare_formula_renames(self, renames): diff --git a/test/client/lib/ACIndex.ts b/test/client/lib/ACIndex.ts index 75eb7c20..1650e7a8 100644 --- a/test/client/lib/ACIndex.ts +++ b/test/client/lib/ACIndex.ts @@ -378,7 +378,12 @@ class BruteForceACIndexImpl implements ACIndex { public search(searchText: string): ACResults { const cleanedSearchText = searchText.trim().toLowerCase(); if (!cleanedSearchText) { - return {items: this._allItems.slice(0, this._maxResults), highlightFunc: highlightNone, selectIndex: -1}; + return { + items: this._allItems.slice(0, this._maxResults), + extraItems: [], + highlightFunc: highlightNone, + selectIndex: -1, + }; } const searchWords = cleanedSearchText.split(/\s+/); @@ -397,7 +402,7 @@ class BruteForceACIndexImpl implements ACIndex { matches.sort((a, b) => nativeCompare(b[0], a[0]) || nativeCompare(a[1], b[1])); const items = matches.slice(0, this._maxResults).map((m) => m[2]); - return {items, highlightFunc: highlightNone, selectIndex: -1}; + return {items, extraItems: [], highlightFunc: highlightNone, selectIndex: -1}; } } diff --git a/test/client/lib/SafeBrowser.ts b/test/client/lib/SafeBrowser.ts index 66390338..442c6f1b 100644 --- a/test/client/lib/SafeBrowser.ts +++ b/test/client/lib/SafeBrowser.ts @@ -3,13 +3,11 @@ import { Disposable } from 'app/client/lib/dispose'; import { ClientProcess, SafeBrowser } from 'app/client/lib/SafeBrowser'; import { LocalPlugin } from 'app/common/plugin'; import { PluginInstance } from 'app/common/PluginInstance'; -import { GristLight } from 'app/common/themes/GristLight'; import { GristAPI, RPC_GRISTAPI_INTERFACE } from 'app/plugin/GristAPI'; import { Storage } from 'app/plugin/StorageAPI'; import { checkers } from 'app/plugin/TypeCheckers'; import { assert } from 'chai'; import { Rpc } from 'grain-rpc'; -import { Computed } from 'grainjs'; import { noop } from 'lodash'; import { basename } from 'path'; import * as sinon from 'sinon'; @@ -188,7 +186,6 @@ describe('SafeBrowser', function() { untrustedContentOrigin: '', mainPath, baseLogger: {}, - theme: Computed.create(null, () => ({appearance: 'light', colors: GristLight})), }); cleanup.push(() => safeBrowser.deactivate()); pluginInstance.rpc.registerForwarder(mainPath, safeBrowser); diff --git a/test/client/ui/AccountPage.ts b/test/client/lib/nameUtils.ts similarity index 91% rename from test/client/ui/AccountPage.ts rename to test/client/lib/nameUtils.ts index 1f701664..438e1de2 100644 --- a/test/client/ui/AccountPage.ts +++ b/test/client/lib/nameUtils.ts @@ -1,11 +1,9 @@ -import {checkName} from 'app/client/ui/AccountPage'; +import {checkName} from 'app/client/lib/nameUtils'; import {assert} from 'chai'; - -describe("AccountPage", function() { +describe("nameUtils", function() { describe("isValidName", function() { it("should detect invalid name", function() { - assert.equal(checkName('santa'), true); assert.equal(checkName('_santa'), true); assert.equal(checkName("O'Neil"), true); diff --git a/test/client/models/HomeModel.ts b/test/client/lib/timeUtils.ts similarity index 87% rename from test/client/models/HomeModel.ts rename to test/client/lib/timeUtils.ts index 77e6ef13..1f3f7058 100644 --- a/test/client/models/HomeModel.ts +++ b/test/client/lib/timeUtils.ts @@ -1,8 +1,8 @@ -import {getTimeFromNow} from 'app/client/models/HomeModel'; +import {getTimeFromNow} from 'app/client/lib/timeUtils'; import {assert} from 'chai'; import moment from 'moment'; -describe("HomeModel", function() { +describe("timeUtils", function() { describe("getTimeFromNow", function() { it("should give good summary of time that just passed", function() { const t = moment().subtract(10, 's'); @@ -18,5 +18,5 @@ describe("HomeModel", function() { const t = moment().add(2, 'minutes'); assert.equal(getTimeFromNow(t.toISOString()), 'in 2 minutes'); }); -}); + }); }); diff --git a/test/fixtures/docs/DropdownCondition.grist b/test/fixtures/docs/DropdownCondition.grist new file mode 100644 index 00000000..50c7b189 Binary files /dev/null and b/test/fixtures/docs/DropdownCondition.grist differ diff --git a/test/nbrowser/DropdownConditionEditor.ts b/test/nbrowser/DropdownConditionEditor.ts new file mode 100644 index 00000000..d12527b6 --- /dev/null +++ b/test/nbrowser/DropdownConditionEditor.ts @@ -0,0 +1,272 @@ +import {assert, driver, Key} from 'mocha-webdriver'; +import * as gu from 'test/nbrowser/gristUtils'; +import {setupTestSuite} from 'test/nbrowser/testUtils'; + +describe('DropdownConditionEditor', function () { + this.timeout(20000); + const cleanup = setupTestSuite(); + + before(async () => { + const session = await gu.session().login(); + await session.tempDoc(cleanup, 'DropdownCondition.grist'); + await gu.openColumnPanel(); + }); + + afterEach(() => gu.checkForErrors()); + + describe(`in choice columns`, function() { + it('creates dropdown conditions', async function() { + await gu.getCell(1, 1).click(); + assert.isFalse(await driver.find('.test-field-dropdown-condition').isPresent()); + await driver.find('.test-field-set-dropdown-condition').click(); + await gu.sendKeys('c'); + await gu.waitToPass(async () => { + const completions = await driver.findAll('.ace_autocomplete .ace_line', el => el.getText()); + assert.deepEqual(completions, [ + 'c\nhoice\n ', + 're\nc\n.Name\n ', + 're\nc\n.Role\n ', + 're\nc\n.Supervisor\n ', + ]); + }); + await gu.sendKeys('hoice not in $'); + await gu.waitToPass(async () => { + const completions = await driver.findAll('.ace_autocomplete .ace_line', el => el.getText()); + assert.deepEqual(completions, [ + '$\nName\n ', + '$\nRole\n ', + '$\nSupervisor\n ', + ]); + }); + await gu.sendKeys('Role', Key.ENTER); + await gu.waitForServer(); + assert.equal( + await driver.find('.test-field-dropdown-condition').getText(), + 'choice not in $Role' + ); + + // Check that autocomplete values are filtered. + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [ + 'Supervisor', + ]); + await gu.sendKeys(Key.ESCAPE); + await gu.getCell(1, 4).click(); + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [ + 'Trainee', + ]); + await gu.sendKeys(Key.ESCAPE); + await gu.getCell(1, 6).click(); + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [ + 'Trainee', + 'Supervisor', + ]); + await gu.sendKeys(Key.ESCAPE); + + // Change the column type to Choice List and check values are still filtered. + await gu.setType('Choice List', {apply: true}); + assert.equal( + await driver.find('.test-field-dropdown-condition').getText(), + 'choice not in $Role' + ); + await gu.getCell(1, 4).click(); + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [ + 'Trainee', + ]); + await gu.sendKeys(Key.ESCAPE); + }); + + it('removes dropdown conditions', async function() { + await driver.find('.test-field-dropdown-condition').click(); + await gu.sendKeys(await gu.selectAllKey(), Key.DELETE, Key.ENTER); + await gu.waitForServer(); + + // Check that autocomplete values are no longer filtered. + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [ + 'Trainee', + 'Supervisor', + ]); + await gu.sendKeys(Key.ESCAPE); + + // Change the column type back to Choice and check values are still no longer filtered. + await gu.setType('Choice', {apply: true}); + assert.isFalse(await driver.find('.test-field-dropdown-condition').isPresent()); + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [ + 'Supervisor', + 'Trainee', + ]); + await gu.sendKeys(Key.ESCAPE); + }); + + it('reports errors', async function() { + // Check syntax errors are reported, but not saved. + await driver.find('.test-field-set-dropdown-condition').click(); + await gu.sendKeys('!@#$%^', Key.ENTER); + await gu.waitForServer(); + assert.equal( + await driver.find('.test-field-dropdown-condition-error').getText(), + 'SyntaxError invalid syntax on line 1 col 1' + ); + await gu.reloadDoc(); + assert.isFalse(await driver.find('.test-field-dropdown-condition-error').isPresent()); + + // Check compilation errors are reported and saved. + await driver.find('.test-field-set-dropdown-condition').click(); + await gu.sendKeys('foo', Key.ENTER); + await gu.waitForServer(); + assert.equal( + await driver.find('.test-field-dropdown-condition-error').getText(), + "Unknown variable 'foo'" + ); + await gu.reloadDoc(); + assert.equal( + await driver.find('.test-field-dropdown-condition-error').getText(), + "Unknown variable 'foo'" + ); + + // Check that the autocomplete dropdown also reports an error. + await gu.sendKeys(Key.ENTER); + assert.equal( + await driver.find('.test-autocomplete-no-items-message').getText(), + 'Error in dropdown condition' + ); + await gu.sendKeys(Key.ESCAPE); + }); + }); + + describe(`in reference columns`, function() { + it('creates dropdown conditions', async function() { + await gu.getCell(2, 1).click(); + assert.isFalse(await driver.find('.test-field-dropdown-condition').isPresent()); + await driver.find('.test-field-set-dropdown-condition').click(); + await gu.sendKeys('choice'); + await gu.waitToPass(async () => { + const completions = await driver.findAll('.ace_autocomplete .ace_line', el => el.getText()); + assert.deepEqual(completions, [ + 'choice\n ', + 'choice\n.id\n ', + 'choice\n.Name\n ', + 'choice\n.Role\n ', + 'choice\n.Supervisor\n ' + ]); + }); + await gu.sendKeys('.Role == "Supervisor" and $Role != "Supervisor"', Key.ENTER); + await gu.waitForServer(); + assert.equal( + await driver.find('.test-field-dropdown-condition .ace_line').getAttribute('textContent'), + 'choice.Role == "Supervisor" and $Role != "Supervisor"\n' + ); + + // Check that autocomplete values are filtered. + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [ + 'Pavan Madilyn', + 'Marie Ziyad', + ]); + await gu.sendKeys(Key.ESCAPE); + await gu.getCell(2, 4).click(); + await gu.sendKeys(Key.ENTER); + assert.isEmpty(await driver.findAll('.test-autocomplete li', (el) => el.getText())); + await gu.sendKeys(Key.ESCAPE); + await gu.getCell(2, 6).click(); + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [ + 'Marie Ziyad', + 'Pavan Madilyn', + ]); + await gu.sendKeys(Key.ESCAPE); + + // Change the column type to Reference List and check values are still filtered. + await gu.setType('Reference List', {apply: true}); + assert.equal( + await driver.find('.test-field-dropdown-condition .ace_line').getAttribute('textContent'), + 'choice.Role == "Supervisor" and $Role != "Supervisor"\n' + ); + await gu.getCell(2, 4).click(); + await gu.sendKeys(Key.ENTER); + assert.isEmpty(await driver.findAll('.test-autocomplete li', (el) => el.getText())); + await gu.sendKeys(Key.ESCAPE); + }); + + it('removes dropdown conditions', async function() { + await driver.find('.test-field-dropdown-condition').click(); + await gu.sendKeys(await gu.selectAllKey(), Key.DELETE, Key.ENTER); + await gu.waitForServer(); + + // Check that autocomplete values are no longer filtered. + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [ + 'Emma Thamir', + 'Holger Klyment', + 'Marie Ziyad', + 'Olivier Bipin', + 'Pavan Madilyn', + ]); + await gu.sendKeys(Key.ESCAPE); + + // Change the column type back to Reference and check values are still no longer filtered. + await gu.setType('Reference', {apply: true}); + assert.isFalse(await driver.find('.test-field-dropdown-condition').isPresent()); + await gu.sendKeys(Key.ENTER); + assert.deepEqual(await driver.findAll('.test-autocomplete li', (el) => el.getText()), [ + 'Emma Thamir', + 'Holger Klyment', + 'Marie Ziyad', + 'Olivier Bipin', + 'Pavan Madilyn', + ]); + await gu.sendKeys(Key.ESCAPE); + }); + + it('reports errors', async function() { + // Check syntax errors are reported, but not saved. + await driver.find('.test-field-set-dropdown-condition').click(); + await gu.sendKeys('!@#$%^', Key.ENTER); + await gu.waitForServer(); + assert.equal( + await driver.find('.test-field-dropdown-condition-error').getText(), + 'SyntaxError invalid syntax on line 1 col 1' + ); + await gu.reloadDoc(); + assert.isFalse(await driver.find('.test-field-dropdown-condition-error').isPresent()); + + // Check compilation errors are reported and saved. + await driver.find('.test-field-set-dropdown-condition').click(); + await gu.sendKeys('foo', Key.ENTER); + await gu.waitForServer(); + assert.equal( + await driver.find('.test-field-dropdown-condition-error').getText(), + "Unknown variable 'foo'" + ); + await gu.reloadDoc(); + assert.equal( + await driver.find('.test-field-dropdown-condition-error').getText(), + "Unknown variable 'foo'" + ); + + // Check that the autocomplete dropdown also reports an error. + await gu.sendKeys(Key.ENTER); + assert.equal( + await driver.find('.test-autocomplete-no-items-message').getText(), + 'Error in dropdown condition' + ); + await gu.sendKeys(Key.ESCAPE); + + // Check evaluation errors are also reported in the dropdown. + await driver.find('.test-field-dropdown-condition').click(); + await gu.sendKeys(await gu.selectAllKey(), Key.DELETE, '[] not in 5', Key.ENTER); + await gu.waitForServer(); + await gu.sendKeys(Key.ENTER); + assert.equal( + await driver.find('.test-autocomplete-no-items-message').getText(), + 'Error in dropdown condition' + ); + await gu.sendKeys(Key.ESCAPE); + }); + }); +}); diff --git a/test/nbrowser/Importer.ts b/test/nbrowser/Importer.ts new file mode 100644 index 00000000..daf9c275 --- /dev/null +++ b/test/nbrowser/Importer.ts @@ -0,0 +1,1008 @@ +/** + * Test of the Importer dialog (part 1), for imports inside an open doc. + * (See Import.ts for tests from the DocMenu page.) + */ +import {assert, driver, Key} from 'mocha-webdriver'; +import * as gu from 'test/nbrowser/gristUtils'; +import {getColumnMatchingRows, getParseOptionInput, getPreviewDiffCellValues, + openTableMapping, waitForColumnMapping, waitForDiffPreviewToLoad} from 'test/nbrowser/importerTestUtils'; +import {setupTestSuite} from 'test/nbrowser/testUtils'; + +describe('Importer', function() { + this.timeout(70000); // Imports can take some time, especially in tests that import larger files. + const cleanup = setupTestSuite(); + + let docUrl: string|undefined; + + beforeEach(async function() { + // Log in and import a sample document. If this is already done, we can skip these tests, to + // have tests go faster. Each successful test case should leave the document unchanged. + if (!docUrl || !await gu.testCurrentUrl(docUrl)) { + const session = await gu.session().teamSite.login(); + await session.tempDoc(cleanup, 'Hello.grist'); + docUrl = await driver.getCurrentUrl(); + } + }); + + afterEach(() => gu.checkForErrors()); + + it('should show correct preview', async function() { + await gu.importFileDialog('./uploads/UploadedData1.csv'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + assert.lengthOf(await driver.findAll('.test-importer-source'), 1); + + assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3]), + [ 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor' ]); + + // Check that the preview table cannot be edited by double-clicking a cell or via keyboard. + const cell = await (await gu.getPreviewCell(0, 1)).doClick(); + await driver.withActions(a => a.doubleClick(cell)); + assert(await driver.find(".default_editor.readonly_editor").isPresent()); + await gu.sendKeys(Key.ESCAPE); + assert.isFalse(await driver.find(".default_editor.readonly_editor").isPresent()); + await gu.sendKeys(Key.DELETE); + await gu.waitForServer(); + assert.equal(await cell.getText(), 'Lily'); + + // Check that the column matching section is not shown for new tables. + assert.isFalse(await driver.find('.test-importer-column-match-options').isPresent()); + + // Check that the preview table doesn't show formula icons in cells. + assert.isFalse(await cell.find('.formula_field').isPresent()); + + // Check that we have "Import Options" link and click it. + assert.equal(await driver.find('.test-importer-options-link').isPresent(), true); + await driver.find('.test-importer-options-link').click(); + + // Check that initially we see a button "Close" (nothing to update) + assert.equal(await driver.findWait('.test-parseopts-back', 500).getText(), 'Close'); + assert.equal(await driver.find('.test-parseopts-update').isPresent(), false); + + // After a change to parse options, button should change to 'Update Preview' + await getParseOptionInput(/Field separator/).doClear().sendKeys("|"); + assert.equal(await driver.findWait('.test-parseopts-update', 500).getText(), 'Update preview'); + assert.equal(await driver.find('.test-parseopts-back').isPresent(), false); + + // Changing the parse option back to initial state reverts the button back too. + await getParseOptionInput(/Field separator/).doClear().sendKeys(","); + assert.equal(await driver.findWait('.test-parseopts-back', 500).getText(), 'Close'); + assert.equal(await driver.find('.test-parseopts-update').isPresent(), false); + + // ensure that option 'First row contains headers' is checked if headers were guessed + let useHeaders = await getParseOptionInput(/First row/); + assert.equal(await useHeaders.getAttribute('checked'), 'true'); + + // Uncheck the option and update the preview. + await useHeaders.click(); + assert.equal(await useHeaders.getAttribute('checked'), null); + await driver.find('.test-parseopts-update').click(); + await gu.waitForServer(); + + // Ensure that column names become the first row in preview data. + assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3, 4]), + [ 'Name', 'Phone', 'Title', + 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor' ]); + + // Check the option again and update the preview. + await driver.find('.test-importer-options-link').click(); + useHeaders = await getParseOptionInput(/First row/); + assert.equal(await useHeaders.getAttribute('checked'), null); + await useHeaders.click(); + await driver.find('.test-parseopts-update').click(); + await gu.waitForServer(); + + // Ensure that column names are used as headers again. + assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3]), + [ 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor' ]); + + // Right-click a column header, to ensure we don't get a JS error in this case. + const colHeader = await driver.findContent('.test-importer-preview .column_name', /Name/); + await driver.withActions(actions => actions.contextClick(colHeader)); + await gu.checkForErrors(); + + // Change Field separator and update the preview. + await driver.find('.test-importer-options-link').click(); + await getParseOptionInput(/Field separator/).doClick().sendKeys("|"); + assert.equal(await getParseOptionInput(/Field separator/).value(), "|"); + assert.equal(await getParseOptionInput(/Line terminator/).value(), "\\n"); + await driver.find('.test-parseopts-update').click(); + await gu.waitForServer(); + + assert.deepEqual(await gu.getPreviewContents([0], [1, 2, 3]), + [ 'Lily,Jones,director', + 'Kathy,Mills,student', + 'Karen,Gold,professor' ]); + + // Close the dialog. + await driver.find('.test-modal-cancel').click(); + await gu.waitForServer(); + assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); + + // No new pages should be present. + assert.deepEqual(await gu.getPageNames(), ['Table1']); + }); + + it('should show correct preview for multiple tables', async function() { + await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv'); + assert.equal(await driver.findWait('.test-importer-preview', 8000).isPresent(), true); + assert.lengthOf(await driver.findAll('.test-importer-source'), 2); + assert.equal(await driver.find('.test-importer-source-selected .test-importer-from').getText(), + 'UploadedData1.csv'); + + assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3]), + [ 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor' ]); + + // Select another table + await driver.findContent('.test-importer-from', /UploadedData2/).click(); + await gu.waitForServer(); + assert.equal(await driver.find('.test-importer-source-selected .test-importer-from').getText(), + 'UploadedData2.csv'); + assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), + [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', + 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', + 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', + 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', + 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', + 'BUS540', 'Capstone', '', '01/13/2021', '' ]); + + // Check that changing a parse option (Field Separator to "|") affects both tables. + await driver.find('.test-importer-options-link').click(); + await getParseOptionInput(/Field separator/).doClick().sendKeys("|"); + await driver.find('.test-parseopts-update').click(); + await gu.waitForServer(); + + assert.deepEqual(await gu.getPreviewContents([0], [1, 2, 3]), + [ 'Lily,Jones,director', + 'Kathy,Mills,student', + 'Karen,Gold,professor' ]); + + await driver.findContent('.test-importer-from', /UploadedData2/).click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getPreviewContents([0], [1, 2, 3, 4, 5, 6]), + [ 'BUS100,Intro to Business,,01/13/2021,false', + 'BUS102,Business Law,Nathalie Patricia,01/13/2021,false', + 'BUS300,Business Operations,Michael Rian,01/14/2021,false', + 'BUS301,History of Business,Mariyam Melania,01/14/2021,false', + 'BUS500,Ethics and Law,Filip Andries,01/13/2021,false', + 'BUS540,Capstone,,01/13/2021,true' ]); + + // Close the dialog. + await driver.find('.test-modal-cancel').click(); + await gu.waitForServer(); + assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); + }); + + it('should not show preview for single empty file', async function() { + await gu.importFileDialog('./uploads/UploadedDataEmpty.csv'); + assert.match(await driver.findWait('.test-importer-error', 1000).getText(), + /Import failed: No data was imported/); + + await driver.find('.test-modal-cancel').click(); + await gu.waitForServer(); + }); + + it('should not show preview for empty file when importing with non empty files', async function() { + await gu.importFileDialog( + './uploads/UploadedData1.csv,./uploads/UploadedData2.csv,./uploads/UploadedDataEmpty.csv'); + + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + + // Ensure that there are no empty tables shown. + assert.deepEqual(await driver.findAll('.test-importer-from', (el) => el.getText()), + ['UploadedData1.csv', 'UploadedData2.csv']); + + assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3]), + [ 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor' ]); + + await driver.findContent('.test-importer-from', /UploadedData2/).click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), + [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', + 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', + 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', + 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', + 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', + 'BUS540', 'Capstone', '', '01/13/2021', '' ]); + + await driver.find('.test-modal-cancel').click(); + await gu.waitForServer(); + }); + + it('should finish import into an existing table', async function() { + // First import the file into a new table, which is the default import action. + await gu.importFileDialog('./uploads/UploadedData1.csv'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4], cols: [0, 1, 2] }), + [ 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor', + '', '', '']); + assert.deepEqual(await gu.getPageNames(), ['Table1', 'UploadedData1']); + + // Now import the same file again, choosing the same table as the first time. + await gu.importFileDialog('./uploads/UploadedData1.csv'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + await driver.findContent('.test-importer-target-existing-table', /UploadedData1/).click(); + await gu.waitForServer(); + + // The preview content should be the same, since all columns match. + assert.deepEqual(await gu.getPreviewContents([0, 1, 2], [1, 2, 3]), + [ 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor' ]); + + await waitForColumnMapping(); + assert.deepEqual(await getColumnMatchingRows(), [ + { destination: 'Name', source: 'Name' }, + { destination: 'Phone', source: 'Phone' }, + { destination: 'Title', source: 'Title' }, + ]); + + // Complete this second import. + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5, 6, 7], cols: [0, 1, 2] }), + [ 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor', + 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor', + '', '', '']); + assert.deepEqual(await gu.getPageNames(), ['Table1', 'UploadedData1']); + + // Undo the import + await gu.undo(2); + + // Ensure that imported table is removed, and we are back to the original one. + assert.deepEqual(await gu.getPageNames(), ['Table1']); + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1], cols: [0, 1, 2] }), + [ 'hello', '', '']); + }); + + it('should finish import multiple files', async function() { + // Import two files together. + await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv'); + await driver.findWait('.test-modal-confirm', 2000).click(); + await gu.waitForServer(); + + assert.deepEqual(await gu.getPageNames(), ['Table1', 'UploadedData1', 'UploadedData2']); + assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2], rowNums: [1, 2, 3]}), + [ 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor' ]); + + await gu.getPageItem('UploadedData2').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5, 6]}), + [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', + 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', + 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', + 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', + 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', + 'BUS540', 'Capstone', '', '01/13/2021', '' ]); + + // Undo and check that we are back to the original state. + await gu.undo(); + assert.deepEqual(await gu.getPageNames(), ['Table1']); + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1], cols: [0, 1, 2] }), + [ 'hello', '', '']); + }); + + it('should import empty dates', async function() { + await gu.importFileDialog('./uploads/EmptyDate.csv'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + + // Finish import and check that the dialog gets closed. + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); + + assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3], cols: [0, 1]}), + [ "Bob", "2018-01-01", + "Alice", "", + "Carol", "2017-01-01" ]); + + assert.deepEqual(await gu.getPageNames(), ['Table1', 'EmptyDate']); + + // Add a new column, with a formula to examine the first. + await gu.openColumnMenu('Birthday', 'Insert column to the right'); + await driver.find('.test-new-columns-menu-add-new').click(); + await gu.waitForServer(); + await driver.sendKeys(Key.ESCAPE); + await gu.getCell({col: 2, rowNum: 1}).click(); + await driver.sendKeys('=type($Birthday).__name__', Key.ENTER); + await gu.waitForServer(); + // Ensure that there is no ValueError in second row + assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3], cols: [0, 1, 2]}), + [ "Bob", "2018-01-01", "date", + "Alice", "", "NoneType", + "Carol", "2017-01-01", "date" ]); + }); + + it('should finish import xlsx file', async function() { + await gu.importFileDialog('./uploads/homicide_rates.xlsx'); + assert.equal(await driver.findWait('.test-importer-preview', 5000).isPresent(), true); + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(5000); + assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); + // Look at a small subset of the imported table. + assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3], cols: [0, 1, 2]}), + [ 'Africa', 'Eastern Africa', 'Burundi', + 'Africa', 'Eastern Africa', 'Burundi', + 'Africa', 'Eastern Africa', 'Comoros']); + }); + + it('should import correctly in prefork mode', async function() { + await driver.get(`${docUrl}/m/fork`); + await gu.waitForDocToLoad(); + + await gu.importFileDialog('./uploads/homicide_rates.xlsx'); + assert.equal(await driver.findWait('.test-importer-preview', 5000).isPresent(), true); + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(5000); + assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); + // Look at a small subset of the imported table. + assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3], cols: [0, 1, 2]}), + [ 'Africa', 'Eastern Africa', 'Burundi', + 'Africa', 'Eastern Africa', 'Burundi', + 'Africa', 'Eastern Africa', 'Comoros']); + await driver.get(`${docUrl}`); + await gu.acceptAlert(); + await gu.waitForDocToLoad(); + }); + + it('should support importing into on-demand tables', async function() { + // Mark EmptyDate as on-demand. + await gu.getPageItem('EmptyDate').click(); + await gu.waitForServer(); + await gu.toggleSidePanel('right', 'open'); + await driver.find('.test-config-data').click(); + await driver.find('[data-test-id=ViewConfig_advanced').click(); + await driver.find('[data-test-id=ViewConfig_onDemandBtn').click(); + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + await gu.waitForDocToLoad(); + + // Import EmptyDate.csv into EmptyDate and check the import was successful. + await gu.importFileDialog('./uploads/EmptyDate.csv'); + assert.equal(await driver.findWait('.test-importer-preview', 5000).isPresent(), true); + await driver.findContent('.test-importer-target-existing-table', /EmptyDate/).click(); + await gu.waitForServer(); + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(5000); + assert.equal(await driver.find('.test-importer-dialog').isPresent(), false); + + // Check that the imported file contents were added to the end of EmptyDate. + assert.deepEqual(await gu.getVisibleGridCells({rowNums: [4, 5, 6], cols: [0, 1]}), + [ "Bob", "2018-01-01", + "Alice", "", + "Carol", "2017-01-01" ]); + assert.equal(await gu.getGridRowCount(), 7); + }); + + describe('when updating existing records', async function() { + it('should populate merge columns/fields menu with columns from preview', async function() { + // First import a file into a new table, so that we have a base for merging. + await gu.importFileDialog('./uploads/UploadedData1.csv'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + + // Now import the same file again. + await gu.importFileDialog('./uploads/UploadedData1.csv'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + + // Check that the 'Update existing records' checkbox is not visible (since destination is 'New Table'). + assert.isNotTrue(await driver.find('.test-importer-update-existing-records').isPresent()); + assert.isNotTrue(await driver.find('.test-importer-merge-fields-select').isPresent()); + assert.isNotTrue(await driver.find('.test-importer-merge-fields-message').isPresent()); + + // Change the destination to the table we created earlier ('UploadedData1'). + await driver.findContent('.test-importer-target-existing-table', /UploadedData1/).click(); + await gu.waitForServer(); + + // Check that the 'Update existing records' checkbox is now visible and unchecked. + assert(await driver.find('.test-importer-update-existing-records').isPresent()); + assert.isNotTrue(await driver.find('.test-importer-merge-fields-select').isPresent()); + assert.isNotTrue(await driver.find('.test-importer-merge-fields-message').isPresent()); + + // Click 'Update existing records' and verify that additional merge options are shown. + await waitForColumnMapping(); + await driver.find('.test-importer-update-existing-records').click(); + assert.equal( + await driver.find('.test-importer-merge-fields-message').getText(), + 'Merge rows that match these fields:' + ); + assert.equal( + await driver.find('.test-importer-merge-fields-select').getText(), + 'Select fields to match on' + ); + + // Open the field select menu and check that all the preview table columns are available options. + await driver.find('.test-importer-merge-fields-select').click(); + assert.deepEqual( + await driver.findAll('.test-multi-select-menu .test-multi-select-menu-option-text', e => e.getText()), + ['Name', 'Phone', 'Title'] + ); + + // Close the field select menu. + await gu.sendKeys(Key.ESCAPE); + }); + + it('should display an error when clicking Import with no merge fields selected', async function() { + // No merge fields are currently selected. Click Import and check that nothing happened. + await driver.find('.test-modal-confirm').click(); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + + // Check that the merge field select button has a red outline. + assert.equal( + await driver.find('.test-importer-merge-fields-select').getCssValue('border'), + '1px solid rgb(208, 2, 27)' + ); + + // Select a merge field, and check that the red outline is gone. + await driver.find('.test-importer-merge-fields-select').click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /Name/ + ).click(); + assert.equal( + await driver.find('.test-importer-merge-fields-select').getCssValue('border'), + '1px solid rgb(217, 217, 217)' + ); + // Hide dropdown + await gu.sendKeys(Key.ESCAPE); + + await gu.checkForErrors(); + }); + + + it('should not throw an error when a column in the preview is clicked', async function() { + // A bug was previosuly causing an error to be thrown whenever a column header was + // clicked while merge columns were set. + await driver.findContent('.test-importer-preview .column_name', /Name/).click(); + await gu.checkForErrors(); + }); + + it('should merge fields of matching records when Import is clicked', async function() { + // The 'Name' field is selected as the only merge field. Click Import. + assert.equal(await driver.find('.test-importer-merge-fields-select').getText(), 'Name'); + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + + // Check that the destination table is unchanged since we imported the same file. + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4], cols: [0, 1, 2] }), + [ 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor', + '', '', '' + ] + ); + + // Undo the import, and check that the destination table is still unchanged. + await gu.undo(); + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4], cols: [0, 1, 2] }), + [ 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor', + '', '', '' + ] + ); + + // Import from another file containing some duplicates (with new values). + await gu.importFileDialog('./uploads/UploadedData1Extended.csv'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + await driver.findContent('.test-importer-target-existing-table', /UploadedData1/).click(); + await gu.waitForServer(); + + // Set the merge fields to 'Name' and 'Phone'. + await waitForColumnMapping(); + await driver.find('.test-importer-update-existing-records').click(); + await driver.find('.test-importer-merge-fields-select').click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /Name/ + ).click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /Phone/ + ).click(); + + // Close the merge fields menu. + await gu.sendKeys(Key.ESCAPE); + assert.equal(await driver.find('.test-importer-merge-fields-select').getText(), 'Name, Phone'); + + // Check the preview shows a diff of the changes importing will make. + await waitForDiffPreviewToLoad(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2], [1, 2, 3, 4, 5, 6]), + [ 'Lily', 'Jones', ['director', 'student', undefined], + 'Kathy', 'Mills', ['student', 'professor', undefined], + 'Karen', 'Gold', ['professor', 'director', undefined], + [undefined, 'Michael', undefined], [undefined, 'Smith', undefined], [undefined, 'student', undefined], + [undefined, 'Lily', undefined], [undefined, 'James', undefined], [undefined, 'student', undefined], + '', '', '', + ] + ); + + // Complete the import, and verify that incoming data was merged into matching records in UploadedData1. + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5, 6], cols: [0, 1, 2] }), + [ 'Lily', 'Jones', 'student', + 'Kathy', 'Mills', 'professor', + 'Karen', 'Gold', 'director', + 'Michael', 'Smith', 'student', + 'Lily', 'James', 'student', + '', '', '' + ] + ); + + // Undo the import, and check the table is back to how it was pre-import. + await gu.undo(); + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4], cols: [0, 1, 2] }), + [ 'Lily', 'Jones', 'director', + 'Kathy', 'Mills', 'student', + 'Karen', 'Gold', 'professor', + '', '', '' + ] + ); + }); + + it('should support merging multiple CSV files into multiple tables', async function() { + // Import a second table, so we have 2 destinations to incrementally import into. + await gu.importFileDialog('./uploads/UploadedData2.csv'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + + // Now import new versions of both files together. + await gu.importFileDialog('./uploads/UploadedData1Extended.csv,./uploads/UploadedData2Extended.csv'); + + // For UploadedData1Extended.csv, check 'Update existing records', but don't pick any merge fields yet. + await driver.findContent('.test-importer-target-existing-table', /UploadedData1/).click(); + + await gu.waitForServer(); + await waitForColumnMapping(); + await driver.find('.test-importer-update-existing-records').click(); + + // Try to click on UploadedData2.csv. + await driver.findContent('.test-importer-source', /UploadedData2Extended.csv/).click(); + + // Check that it failed, and that the merge fields select button is outlined in red. + assert.equal( + await driver.find('.test-importer-merge-fields-select').getCssValue('border'), + '1px solid rgb(208, 2, 27)' + ); + assert.equal( + await driver.find('.test-importer-source-selected .test-importer-from').getText(), + 'UploadedData1Extended.csv' + ); + + // Now pick the merge fields, and check that the preview diff looks correct. + await driver.find('.test-importer-merge-fields-select').click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /Name/ + ).click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /Phone/ + ).click(); + await gu.sendKeys(Key.ESCAPE); + + await waitForDiffPreviewToLoad(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2], [1, 2, 3, 4, 5, 6]), + [ 'Lily', 'Jones', ['director', 'student', undefined], + 'Kathy', 'Mills', ['student', 'professor', undefined], + 'Karen', 'Gold', ['professor', 'director', undefined], + [undefined, 'Michael', undefined], [undefined, 'Smith', undefined], [undefined, 'student', undefined], + [undefined, 'Lily', undefined], [undefined, 'James', undefined], [undefined, 'student', undefined], + '', '', '', + ] + ); + + // Check that clicking UploadedData2 now works. + await driver.findContent('.test-importer-source', /UploadedData2Extended.csv/).click(); + await gu.waitForServer(); + await driver.findContent('.test-importer-target-existing-table', /UploadedData2/).click(); + await gu.waitForServer(); + assert.equal( + await driver.find('.test-importer-source-selected .test-importer-from').getText(), + 'UploadedData2Extended.csv' + ); + + // Set the merge fields for UploadedData2 to 'CourseId'. + await waitForColumnMapping(); + await driver.find('.test-importer-update-existing-records').click(); + await driver.find('.test-importer-merge-fields-select').click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /CourseId/ + ).click(); + + // Close the merge fields menu. + await gu.sendKeys(Key.ESCAPE); + + assert.equal(await driver.find('.test-importer-merge-fields-select').getText(), 'CourseId'); + + // Check that the preview diff looks correct for UploadedData2. + await waitForDiffPreviewToLoad(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7, 8, 9]), + [ 'BUS100', 'Intro to Business', [undefined, 'Mariyam Melania', undefined], '01/13/2021', '', + 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', + 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', + 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', + 'BUS500', [undefined, undefined, 'Ethics and Law'], 'Filip Andries', '01/13/2021', '', + [undefined, 'BUS501', undefined], [undefined, 'Marketing', undefined], [undefined, 'Michael Rian', undefined], + [undefined, '01/13/2021', undefined], [undefined, 'false', undefined], + [undefined, 'BUS539', undefined], [undefined, 'Independent Study', undefined], '', + [undefined, '01/13/2021', undefined], [undefined, 'true', undefined], + 'BUS540', 'Capstone', '', '01/13/2021', ['true', 'false', undefined], + '', '', '', '', '' + ] + ); + + // Complete the import, and verify that incoming data was merged into both UploadedData1 and UploadedData2. + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getPageNames(), [ + 'Table1', + 'EmptyDate', + 'Homicide counts and rates (2000', + 'Sheet1', + 'UploadedData1', + 'UploadedData2' + ]); + + // Check the contents of UploadedData1. + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5, 6], cols: [0, 1, 2] }), + [ 'Lily', 'Jones', 'student', + 'Kathy', 'Mills', 'professor', + 'Karen', 'Gold', 'director', + 'Michael', 'Smith', 'student', + 'Lily', 'James', 'student', + '', '', '' + ] + ); + + // Check the contents of UploadedData2. + await gu.getPageItem('UploadedData2').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5, 6, 7, 8, 9]}), + [ 'BUS100', 'Intro to Business', 'Mariyam Melania', '01/13/2021', '', + 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', + 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', + 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', + 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', + 'BUS540', 'Capstone', '', '01/13/2021', '', + 'BUS501', 'Marketing', 'Michael Rian', '01/13/2021', '', + 'BUS539', 'Independent Study', '', '01/13/2021', '', + '', '', '', '', '' ]); + }); + + it('should support merging multiple Excel sheets into multiple tables', async function() { + this.timeout(90000); + + // Import an Excel file with multiple sheets into new tables. + await gu.importFileDialog('./uploads/World-v0.xlsx'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + + // Now import a new version of the Excel file with updated data. + await gu.importFileDialog('./uploads/World-v1.xlsx'); + + // For sheet Table1, don't pick any merge fields and import into the existing table (Table1_2). + await driver.findContent('.test-importer-target-existing-table', /Table1_2/).click(); + + await gu.waitForServer(); + + // For sheet City, merge on Name, District and Country. + await driver.findContent('.test-importer-source', /City/).click(); + await gu.waitForServer(); + await driver.findContent('.test-importer-target-existing-table', /City/).click(); + await gu.waitForServer(); + await waitForColumnMapping(); + await driver.find('.test-importer-update-existing-records').click(); + await driver.find('.test-importer-merge-fields-select').click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /Name/ + ).click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /District/ + ).click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /Country/ + ).click(); + await gu.sendKeys(Key.ESCAPE); + + // Check the preview diff of City. The population should have doubled in every row. + await waitForDiffPreviewToLoad(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5]), + [ + 'Kabul', 'Kabol', ['1780000', '3560000', undefined], '2', ['1780', '3560', undefined], + 'Qandahar', 'Qandahar', ['237500', '475000', undefined], '2', ['237.5', '475', undefined], + 'Herat', 'Herat', ['186800', '373600', undefined], '2', ['186.8', '373.6', undefined], + 'Mazar-e-Sharif', 'Balkh', ['127800', '255600', undefined], '2', ['127.8', '255.6', undefined], + 'Amsterdam', 'Noord-Holland', ['731200', '1462400', undefined], '159', ['731.2', '1462.4', undefined], + ] + ); + + // For sheet Country, merge on Code. + await driver.findContent('.test-importer-source', /Country/).click(); + await gu.waitForServer(); + await driver.findContent('.test-importer-target-existing-table', /Country/).click(); + await gu.waitForServer(); + await waitForColumnMapping(); + await driver.find('.test-importer-update-existing-records').click(); + await driver.find('.test-importer-merge-fields-select').click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /Code/ + ).click(); + await gu.sendKeys(Key.ESCAPE); + + // Check the preview diff of Country. The population should have doubled in every row. + await waitForDiffPreviewToLoad(); + assert.deepEqual( + await getPreviewDiffCellValues([0, 6], [1, 2, 3, 4, 5]), + [ 'ABW', ['103000', '206000', undefined], + 'AFG', ['22720000', '45440000', undefined], + 'AGO', ['12878000', '25756000', undefined], + 'AIA', ['8000', '16000', undefined], + 'ALB', [ '3401200', '6802400', undefined] + ] + ); + + // For sheet CountryLanguage, merge on Country and Language. + await driver.findContent('.test-importer-source', /CountryLanguage/).click(); + await gu.waitForServer(); + await driver.findContent('.test-importer-target-existing-table', /CountryLanguage/).click(); + await gu.waitForServer(); + + await waitForColumnMapping(); + await driver.find('.test-importer-update-existing-records').click(); + await driver.find('.test-importer-merge-fields-select').click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /Country/ + ).click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /Language/ + ).click(); + await gu.sendKeys(Key.ESCAPE); + + // Check the preview diff of CountryLanguage. The first few percentages should be slightly different. + await waitForDiffPreviewToLoad(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3], [1, 2, 3, 4, 5]), + [ 'Dutch', ['5.3', '5.5', undefined], 'ABW', '', + 'English', ['9.5', '9.3', undefined], 'ABW', '', + 'Papiamento', ['76.7', '76.3', undefined], 'ABW', '', + 'Spanish', ['7.4', '7.8', undefined], 'ABW', '', + 'Balochi', ['0.9', '1.1', undefined], 'AFG', '' + ] + ); + + // Complete the import, and verify that incoming data was merged correctly. + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getPageNames(), [ + 'Table1', + 'EmptyDate', + 'Homicide counts and rates (2000', + 'Sheet1', + 'UploadedData1', + 'UploadedData2', + 'Table1', + 'City', + 'Country', + 'CountryLanguage' + ]); + + // Check the contents of Table1; it should have duplicates of the original 2 rows. + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5], cols: [0, 1, 2, 3, 4] }), + [ + 'hello', '', '', '', 'HELLO', + '', 'world', '', '', '', + 'hello', '', '', '', 'HELLO', + '', 'world', '', '', '', + '', '', '', '', '', + ] + ); + + // Check the contents of City. The population should have doubled in every row. + await gu.getPageItem('City').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5]}), + [ + 'Kabul', 'Kabol', '3560000', '2', '3560', + 'Qandahar', 'Qandahar', '475000', '2', '475', + 'Herat', 'Herat', '373600', '2', '373.6', + 'Mazar-e-Sharif', 'Balkh', '255600', '2', '255.6', + 'Amsterdam', 'Noord-Holland', '1462400', '159', '1462.4', + ] + ); + + // Check that no new rows were added to City. + assert.equal(await gu.getGridRowCount(), 4080); + + // Check the contents of Country. The population should have doubled in every row. + await gu.getPageItem('Country').click(); + await gu.waitForServer(); + assert.deepEqual( + await gu.getVisibleGridCells({ + cols: [0, 6], + rowNums: [1, 2, 3, 4, 5] + }), + [ 'ABW', '206000', + 'AFG', '45440000', + 'AGO', '25756000', + 'AIA', '16000', + 'ALB', '6802400' + ] + ); + + // Check that no new rows were added to Country. + assert.equal(await gu.getGridRowCount(), 240); + + // Check the contents of CountryLanguage. The first few percentages should be slightly different. + await gu.getPageItem('CountryLanguage').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3], rowNums: [1, 2, 3, 4, 5]}), + [ 'Dutch', '5.5', 'ABW', '', + 'English', '9.3', 'ABW', '', + 'Papiamento', '76.3', 'ABW', '', + 'Spanish', '7.8', 'ABW', '', + 'Balochi', '1.1', 'AFG', '' + ] + ); + + // Check that no new rows were added to CountryLanguage. + assert.equal(await gu.getGridRowCount(), 985); + }); + + it('should show diff of changes in preview', async function() { + // Import UploadedData2.csv again, and change the destination to UploadedData2. + await gu.importFileDialog('./uploads/UploadedData2.csv'); + await driver.findContent('.test-importer-target-existing-table', /UploadedData2/).click(); + await gu.waitForServer(); + await waitForColumnMapping(); + + // Click 'Update existing records', and check the preview does not yet show a diff. + await driver.find('.test-importer-update-existing-records').click(); + await gu.waitForServer(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7]), + [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', + 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', + 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', + 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', + 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', + 'BUS540', 'Capstone', '', '01/13/2021', '', + '', '', '', '', '' ]); + + // Select 'CourseId' as the merge column, and check that the preview now contains a diff of old/new values. + await driver.find('.test-importer-merge-fields-select').click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /CourseId/ + ).click(); + await gu.sendKeys(Key.ESCAPE); + await gu.waitForServer(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7]), + [ 'BUS100', 'Intro to Business', [undefined, undefined, 'Mariyam Melania'], '01/13/2021', '', + 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', + 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', + 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', + 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', + 'BUS540', 'Capstone', '', '01/13/2021', ['false', 'true', undefined], + '', '', '', '', '' ]); + + + // Uncheck 'Update existing records', and check that the preview no longer shows a diff. + await driver.find('.test-importer-update-existing-records').click(); + await waitForDiffPreviewToLoad(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7]), + [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', + 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', + 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', + 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', + 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', + 'BUS540', 'Capstone', '', '01/13/2021', '', + '', '', '', '', '' ]); + + // Check that the column matching section is correct. + assert.deepEqual(await getColumnMatchingRows(), [ + { destination: 'CourseId', source: 'CourseId' }, + { destination: 'CourseName', source: 'CourseName' }, + { destination: 'Instructor', source: 'Instructor' }, + { destination: 'StartDate', source: 'StartDate' }, + { destination: 'PassFail', source: 'PassFail' }, + ]); + + // Click 'Update existing records' again, and edit the formula for CourseId to append a suffix. + await driver.find('.test-importer-update-existing-records').click(); + await waitForDiffPreviewToLoad(); + await driver.findContent('.test-importer-column-match-source-destination', /CourseId/) + .find('.test-importer-column-match-formula').click(); + await driver.find('.test-importer-apply-formula').click(); + await gu.sendKeys(' + "-NEW"'); + + // Before saving the formula, check that the preview isn't showing the hidden helper column ids. + assert.deepEqual( + await driver.find('.test-importer-preview').findAll('.g-column-label', el => el.getText()), + ['CourseId', 'CourseName', 'Instructor', 'StartDate', 'PassFail'] + ); + await gu.sendKeys(Key.ENTER); + await gu.waitForServer(); + + // Check that the preview diff was updated and now shows that all 6 rows are new rows. + await waitForDiffPreviewToLoad(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7]), + [ + [undefined, 'BUS100-NEW', undefined], [undefined, 'Intro to Business', undefined], '', + [undefined, '01/13/2021', undefined], [undefined, 'false', undefined], + [undefined, 'BUS102-NEW', undefined], [undefined, 'Business Law', undefined], + [undefined, 'Nathalie Patricia', undefined], [undefined, '01/13/2021', undefined], + [undefined, 'false', undefined], + [undefined, 'BUS300-NEW', undefined], [undefined, 'Business Operations', undefined], + [undefined, 'Michael Rian', undefined], [undefined, '01/14/2021', undefined], + [undefined, 'false', undefined], + [undefined, 'BUS301-NEW', undefined], [undefined, 'History of Business', undefined], + [undefined, 'Mariyam Melania', undefined], [undefined, '01/14/2021', undefined], + [undefined, 'false', undefined], + [undefined, 'BUS500-NEW', undefined], [undefined, 'Ethics and Law', undefined], + [undefined, 'Filip Andries', undefined], [undefined, '01/13/2021', undefined], + [undefined, 'false', undefined], + [undefined, 'BUS540-NEW', undefined], [undefined, 'Capstone', undefined], '', + [undefined, '01/13/2021', undefined], [undefined, 'true', undefined], + '', '', '', '', '' + ] + ); + + // Check the column mapping section updated with the new formula. + assert.deepEqual(await getColumnMatchingRows(), [ + { destination: 'CourseId', source: '$CourseId + "-NEW"\n' }, + { destination: 'CourseName', source: 'CourseName' }, + { destination: 'Instructor', source: 'Instructor' }, + { destination: 'StartDate', source: 'StartDate' }, + { destination: 'PassFail', source: 'PassFail' }, + ]); + + // Change the destination back to new table, and check that the preview no longer shows a diff. + await openTableMapping(); + await driver.find('.test-importer-target-new-table').click(); + await gu.waitForServer(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6, 7]), + [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', + 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', + 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', + 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', + 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', + 'BUS540', 'Capstone', '', '01/13/2021', '', + '', '', '', '', '' ]); + + // Close the dialog. + await driver.find('.test-modal-cancel').click(); + await gu.waitForServer(); + }); + }); +}); diff --git a/test/nbrowser/Importer2.ts b/test/nbrowser/Importer2.ts new file mode 100644 index 00000000..5cb9c179 --- /dev/null +++ b/test/nbrowser/Importer2.ts @@ -0,0 +1,818 @@ +/** + * Test of the Importer dialog (part 2), for imports inside an open doc. + */ +import {DocAPI} from 'app/common/UserAPI'; +import {DocCreationInfo} from 'app/common/DocListAPI'; +import * as _ from 'lodash'; +import {assert, driver, Key} from 'mocha-webdriver'; +import * as gu from 'test/nbrowser/gristUtils'; +import {getColumnMatchingRows, getPreviewDiffCellValues, openSource as openSourceFor, + openTableMapping, waitForColumnMapping, waitForDiffPreviewToLoad} from 'test/nbrowser/importerTestUtils'; +import {setupTestSuite} from 'test/nbrowser/testUtils'; + +describe('Importer2', function() { + this.timeout(60000); + const cleanup = setupTestSuite(); + let doc: DocCreationInfo; + let api: DocAPI; + + before(async function() { + // Log in and import a sample document. + const session = await gu.session().teamSite.login(); + doc = await session.tempDoc(cleanup, 'Hello.grist'); + api = session.createHomeApi().getDocAPI(doc.id); + }); + + afterEach(() => gu.checkForErrors()); + + it("should import new tables losslessly", async function() { + // Import mixed_dates.csv into a new table + await gu.importFileDialog('./uploads/mixed_dates.csv'); + await waitForDiffPreviewToLoad(); + await driver.find('.test-modal-confirm').click(); + await gu.waitAppFocus(); + + // Import the same file again into the same table + await gu.importFileDialog('./uploads/mixed_dates.csv'); + await driver.findContent('.test-importer-target-existing-table', /Mixed_dates/).click(); + await waitForDiffPreviewToLoad(); + await driver.find('.test-modal-confirm').click(); + await gu.waitAppFocus(); + + assert.deepEqual( + await gu.getVisibleGridCells({cols: [0], rowNums: _.range(1, 21)}), + [ + // mixed_dates.csv contains 10 dates. The first 9 are YYYY-MM-DD so that's the guessed date format. + // The last date '01/02/03' doesn't fit this format. + // Since 90% of the values fit the guessed format, the column is guessed to have type Date. + // The dates are parsed by DateGuesser which uses moment's strict parsing directly, not parseDate. + // So '01/02/03' isn't parsed and remains a string, and the column is imported losslessly, + // i.e. converting it back to text yields the original strings in the file unchanged. + '2020-03-04', + '2020-03-05', + '2020-03-06', + '2020-03-04', + '2020-03-05', + '2020-03-06', + '2020-03-04', + '2020-03-05', + '2020-03-06', + '01/02/03', + + // When the file is imported again into the same table, things go differently. + // The intermediate hidden table goes through the same process and stores '01/02/03' as a string. + // But for existing tables we set parseStrings to true when applying the final BulkAddRecord. + // So '01/02/03' is parsed by parseDate according to the existing column's date format which gives 2001-02-03. + '2020-03-04', + '2020-03-05', + '2020-03-06', + '2020-03-04', + '2020-03-05', + '2020-03-06', + '2020-03-04', + '2020-03-05', + '2020-03-06', + '2001-02-03', + ], + ); + + await gu.undo(2); + }); + + it("should set widget options for formatted numbers", async function() { + // Import formatted_numbers.csv into a new table + await gu.importFileDialog('./uploads/formatted_numbers.csv'); + await waitForDiffPreviewToLoad(); + await driver.find('.test-modal-confirm').click(); + await gu.waitAppFocus(); + + // Numbers appear formatted as in the CSV file + assert.deepEqual( + await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1]}), + ["$1.00", "1.20E3", "2,000,000", "43%", "(56)"], + ); + + const records = await api.getRecords('Formatted_numbers'); + const cols = await api.getRecords('_grist_Tables_column'); + + // Actual data has correct values, e.g. 43% -> 0.43 + assert.deepEqual(records, [{ + id: 1, + fields: { + fn_currency: 1, + fn_scientific: 1200, + fn_decimal: 2000000, + fn_percent: 0.43, + fn_parens: -56, + }, + }]); + + // Get the fields we care about describing the columns to allow comparison. + // All column names in the CSV file start with "fn_" + const colFields = cols.map( + ({fields: {colId, type, widgetOptions}}) => + ({colId, type, widgetOptions: JSON.parse(widgetOptions as string || "{}")}) + ).filter(f => (f.colId as string).startsWith("fn_")); + + // All the columns are numeric and have some kind of formatting + assert.deepEqual(colFields, [ + { + colId: 'fn_currency', + type: 'Numeric', + widgetOptions: {decimals: 2, numMode: 'currency'} + }, + { + colId: 'fn_scientific', + type: 'Numeric', + widgetOptions: {decimals: 2, numMode: 'scientific'} + }, + { + colId: 'fn_decimal', + type: 'Numeric', + widgetOptions: {numMode: 'decimal'} + }, + { + colId: 'fn_percent', + type: 'Numeric', + widgetOptions: {numMode: 'percent'} + }, + { + colId: 'fn_parens', + type: 'Numeric', + widgetOptions: {numSign: 'parens'} + }, + ]); + + // Remove the imported table + await gu.undo(); + }); + + it("should not show skip option for single table", async function() { + async function noSkip() { + await waitForDiffPreviewToLoad(); + assert.isFalse(await driver.find('.test-importer-target-skip').isPresent()); + await driver.sendKeys(Key.ESCAPE); + await driver.find('.test-modal-cancel').click(); + await gu.waitAppFocus(); + } + await gu.importFileDialog('./uploads/UploadedData1.csv'); + await noSkip(); + await gu.importFileDialog('./uploads/BooleanData.xlsx'); + await noSkip(); + }); + + it("should show skip option for multiple tables", async function() { + async function hasSkip() { + await waitForDiffPreviewToLoad(); + assert.isTrue(await driver.find('.test-importer-target-skip').isDisplayed()); + await driver.find('.test-importer-source-not-selected').click(); + assert.isTrue(await driver.find('.test-importer-target-skip').isDisplayed()); + await driver.find('.test-modal-cancel').click(); + await gu.waitAppFocus(); + } + await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv'); + await hasSkip(); + await gu.importFileDialog('./uploads/homicide_rates.xlsx'); + await hasSkip(); + }); + + it("should skip importing", async function() { + await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv'); + await waitForDiffPreviewToLoad(); + // Skip the first table. + await driver.find('.test-importer-target-skip').click(); + // Make sure preview is grayed out. + assert.isTrue(await driver.find(".test-importer-preview-overlay").isPresent()); + await driver.find('.test-modal-confirm').click(); + await gu.waitAppFocus(); + // Make sure only second table is visible. + assert.deepEqual(await gu.getPageNames(), ['Table1', 'UploadedData2']); + // And data is valid. + await gu.getPageItem('UploadedData2').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5, 6]}), + [ 'BUS100', 'Intro to Business', '', '01/13/2021', '', + 'BUS102', 'Business Law', 'Nathalie Patricia', '01/13/2021', '', + 'BUS300', 'Business Operations', 'Michael Rian', '01/14/2021', '', + 'BUS301', 'History of Business', 'Mariyam Melania', '01/14/2021', '', + 'BUS500', 'Ethics and Law', 'Filip Andries', '01/13/2021', '', + 'BUS540', 'Capstone', '', '01/13/2021', '' ]); + await gu.undo(); + }); + + it("should clean mapping when skipped", async function() { + // Import UploadedData2 to have a destination table. + await gu.importFileDialog('./uploads/UploadedData2.csv'); + await waitForDiffPreviewToLoad(); + await driver.find('.test-modal-confirm').click(); + await gu.waitAppFocus(); + + // Reimport + await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv'); + await waitForDiffPreviewToLoad(); + + // Skip first table + await driver.find('.test-importer-target-skip').click(); + + // Select second table and add mapping to update existing records. + await driver.find('.test-importer-source-not-selected').click(); + await driver.findContent('.test-importer-target-existing-table', /UploadedData2/).click(); + + await waitForDiffPreviewToLoad(); + await waitForColumnMapping(); + await driver.find('.test-importer-update-existing-records').click(); + await driver.find('.test-importer-merge-fields-select').click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /CourseId/ + ).click(); + await gu.sendKeys(Key.ESCAPE); + await gu.waitForServer(); + + // Now skip and make sure options are hidden + await openTableMapping(); + await driver.find('.test-importer-target-skip').click(); + + // And unskip, and make sure options are back, but not filled + await driver.findContent('.test-importer-target-existing-table', /UploadedData2/).click(); + await waitForDiffPreviewToLoad(); + + await waitForColumnMapping(); + assert.isTrue(await driver.find('.test-importer-update-existing-records').isPresent()); + assert.isTrue(await driver.find('.test-importer-merge-fields-select').isPresent()); + assert.isTrue(await driver.find('.test-importer-merge-fields-message').isPresent()); + assert.equal(await driver.find('.test-importer-merge-fields-select').getText(), + 'Select fields to match on'); + + await driver.find('.test-modal-cancel').click(); + await gu.waitAppFocus(); + await gu.undo(2); // Press two times, as we cancelled and import hasn't cleaned temps. + }); + + it("should disable import button when all tables are skipped", async function() { + await gu.importFileDialog('./uploads/UploadedData1.csv,./uploads/UploadedData2.csv'); + await waitForDiffPreviewToLoad(); + // Make sure both previews are available + for(const source of await driver.findAll(".test-importer-source")) { + await source.click(); + assert.isFalse(await driver.find(".test-importer-preview-overlay").isPresent()); + } + const sources = await driver.findAll(".test-importer-source"); + // Skip both tables. + for(const source of sources) { + await source.click(); + await gu.waitForServer(); + await driver.find('.test-importer-target-skip').click(); + await gu.waitForServer(); + } + assert.equal(await driver.find('.test-modal-confirm').getAttribute('disabled'), 'true'); + // Make sure both previews are grayed out + for(const source of sources) { + await source.click(); + assert.isTrue(await driver.find(".test-importer-preview-overlay").isPresent()); + } + + // Enable first, and test if one is grayed out and the second is not. + await sources[0].click(); + await gu.waitForServer(); + await driver.find(".test-importer-target-new-table").click(); + await gu.waitForServer(); + await waitForDiffPreviewToLoad(); + assert.isFalse(await driver.find(".test-importer-preview-overlay").isPresent()); + assert.equal(await driver.find('.test-modal-confirm').getAttribute('disabled'), null); + + // Second should be still grayed out + await sources[1].click(); + assert.isTrue(await driver.find(".test-importer-preview-overlay").isPresent()); + + await driver.find('.test-modal-cancel').click(); + await gu.waitAppFocus(); + }); + + describe('when importing JSON', async function() { + // A previous bug caused an error to be thrown when finishing importing a nested JSON file. + it('should import successfully to new tables', async function() { + // Import a nested JSON file. + await gu.importFileDialog('./uploads/names.json'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + + // Check that two preview tables were created. + assert.lengthOf(await driver.findAll('.test-importer-source'), 2); + assert.equal( + await driver.find('.test-importer-source[class*=-selected] .test-importer-from').getText(), + 'names - names.json' + ); + assert.deepEqual( + await driver.findAll('.test-importer-source .test-importer-from', (e) => e.getText()), + ['names - names.json', 'names_name - names.json'] + ); + + // Check that the first table looks ok. + assert.deepEqual(await gu.getPreviewContents([0], [1, 2, 3]), [ '[1]', '[2]', '']); + + // Check that the second table looks ok. + await driver.findContent('.test-importer-source', /names_name/).click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getPreviewContents([0], [1, 2, 3]), [ 'Bob', 'Alice', '']); + + // Finish import, and verify the import succeeded. + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getPageNames(), [ + 'Table1', + 'names', + 'names_name', + ]); + + // Verify data was imported to Names correctly. + assert.deepEqual( + await gu.getVisibleGridCells({ rowNums: [1, 2, 3], cols: [0] }), + ['Names_name[1]', 'Names_name[2]', ''] + ); + + // Open the side panel and check that the column type for 'name' is Reference (pointing to 'first'). + await gu.toggleSidePanel('right', 'open'); + await driver.find('.test-right-tab-field').click(); + assert.equal(await driver.find('.test-fbuilder-type-select').getText(), 'Reference'); + assert.equal(await gu.getRefTable(), 'names_name'); + assert.equal(await gu.getRefShowColumn(), 'Row ID'); + + // Verify data was imported to Names_name correctly. + await gu.getPageItem('names_name').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3], cols: [0] }), ['Bob', 'Alice', '']); + }); + + it('should import successfully to existing tables with references', async function() { + // Import the same nested JSON file again. + await gu.importFileDialog('./uploads/names.json'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + + // Change the destination of both source tables to the existing destination ones. + await driver.findContent('.test-importer-target-existing-table', /Names/).click(); + await gu.waitForServer(); + // Now on the second tab. + await driver.findContent('.test-importer-source', /names_name/).click(); + await driver.findContent('.test-importer-target-existing-table', /Names_name/).click(); + await gu.waitForServer(); + + // Finish import, and verify the import succeeded. + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getPageNames(), [ + 'Table1', + 'names', + 'names_name', + ]); + + // Verify data was imported to Names correctly. + assert.deepEqual( + await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5], cols: [0] }), + ['Names_name[1]', 'Names_name[2]', 'Names_name[1]', 'Names_name[2]', ''] + ); + + // Open the side panel and check that the column type for 'name' is Reference (pointing to 'first'). + await gu.toggleSidePanel('right', 'open'); + await driver.find('.test-right-tab-field').click(); + assert.equal(await driver.find('.test-fbuilder-type-select').getText(), 'Reference'); + assert.equal(await gu.getRefTable(), 'names_name'); + assert.equal(await gu.getRefShowColumn(), 'Row ID'); + + // Verify data was imported to Names_name correctly. + await gu.getPageItem('names_name').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getVisibleGridCells( + { rowNums: [1, 2, 3, 4, 5], cols: [0] }), + ['Bob', 'Alice', 'Bob', 'Alice', ''] + ); + + // Undo the last 2 imports. + await gu.undo(2); + }); + }); + + describe('when matching columns', async function() { + it('should not display column matching section for new destinations', async function() { + // Import an Excel file with multiple sheets. + await gu.importFileDialog('./uploads/World-v0.xlsx'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + + // Check that the column matching section is not shown. + assert.isFalse(await driver.find('.test-importer-column-match-options').isPresent()); + }); + + it('should display column matching section for existing destinations', async function() { + // From the previous test: finish importing World-v1.xlsx so we have tables to import to. + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(10_000); + + // Import the same file again. + await gu.importFileDialog('./uploads/World-v0.xlsx'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + + // Change the destination of the selected sheet to the table created earlier. + await driver.findContent('.test-importer-target-existing-table', /Table1_2/).click(); + await gu.waitForServer(); + + await waitForColumnMapping(); + // Check that source and destination are populated for each column from the first sheet. + assert.deepEqual(await getColumnMatchingRows(), [ + { destination: 'a', source: 'a' }, + { destination: 'b', source: 'b' }, + { destination: 'c', source: 'c' }, + { destination: 'd', source: 'd' }, + { destination: 'E', source: 'E' }, + ]); + assert.isFalse(await driver.find('.test-importer-unmatched-fields').isPresent()); + + // Switch to the City sheet, and check that the column matching section is no longer shown. + await driver.findContent('.test-importer-source', /City/).click(); + await gu.waitForServer(); + assert.isFalse(await driver.find('.test-importer-column-match-options').isPresent()); + + // Change the destination to 'City', and now check that the section is shown. + await driver.findContent('.test-importer-target-existing-table', /City/).click(); + await gu.waitForServer(); + + await waitForColumnMapping(); + assert.deepEqual(await getColumnMatchingRows(), [ + { destination: 'Name', source: 'Name' }, + { destination: 'District', source: 'District' }, + { destination: 'Population', source: 'Population' }, + { destination: 'Country', source: 'Country' }, + { destination: 'Pop. \'000', source: 'Pop. \'000' }, + ]); + assert.isFalse(await driver.find('.test-importer-unmatched-fields').isPresent()); + }); + + it('should allow skipping importing columns', async function() { + // Starting from the City sheet, open the menu for "Pop. '000". + await driver.findContent('.test-importer-column-match-source', /Pop\. '000/).click(); + + // Check that the menu contains only the selected source column, plus a 'Skip' option. + const menu = driver.find('.test-select-menu'); + assert.deepEqual( + await menu.findAll('.test-importer-column-match-menu-item', el => el.getText()), + ['Skip', 'Pop. \'000'] + ); + + // Click 'Skip', and check that the column mapping section and preview both updated. + await menu.findContent('.test-importer-column-match-menu-item', /Skip/).click(); + await gu.waitForServer(); + assert.deepEqual(await getColumnMatchingRows(), [ + { destination: 'Name', source: 'Name' }, + { destination: 'District', source: 'District' }, + { destination: 'Population', source: 'Population' }, + { destination: 'Country', source: 'Country' }, + { destination: 'Pop. \'000', source: 'Skip' }, + ]); + assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [ + 'Kabul', 'Kabol', '1780000', '2', '0', + 'Qandahar', 'Qandahar', '237500', '2', '0', + 'Herat', 'Herat', '186800', '2', '0', + 'Mazar-e-Sharif', 'Balkh', '127800', '2', '0', + 'Amsterdam', 'Noord-Holland', '731200', '159', '0', + 'Rotterdam', 'Zuid-Holland', '593321', '159', '0', + ]); + + // Check that a message is now shown about there being 1 unmapped field. + assert.equal( + await driver.find('.test-importer-unmatched-fields').getText(), + '1 unmatched field in import:\nPop. \'000' + ); + + // Click Country in the column mapping section, and clear the formula. + await driver.findContent('.test-importer-column-match-source', /Country/).click(); + await driver.find('.test-importer-apply-formula').click(); + await gu.sendKeys(await gu.selectAllKey(), Key.DELETE, Key.ENTER); + await gu.waitForServer(); + + // Check that the column mapping section and preview now show that it will be skipped. + assert.deepEqual(await getColumnMatchingRows(), [ + { destination: 'Name', source: 'Name' }, + { destination: 'District', source: 'District' }, + { destination: 'Population', source: 'Population' }, + { destination: 'Country', source: 'Skip' }, + { destination: 'Pop. \'000', source: 'Skip' }, + ]); + assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [ + 'Kabul', 'Kabol', '1780000', '0', '0', + 'Qandahar', 'Qandahar', '237500', '0', '0', + 'Herat', 'Herat', '186800', '0', '0', + 'Mazar-e-Sharif', 'Balkh', '127800', '0', '0', + 'Amsterdam', 'Noord-Holland', '731200', '0', '0', + 'Rotterdam', 'Zuid-Holland', '593321', '0', '0', + ]); + assert.equal( + await driver.find('.test-importer-unmatched-fields').getText(), + '2 unmatched fields in import:\nCountry, Pop. \'000' + ); + }); + + it('should autocomplete formula in source', async function() { + // Starting from the City sheet, open the menu for "Pop. '000". + await openSourceFor(/Pop\. '000/); + + // We want to map the same column twice, which is not possible through the menu, so we will + // use the formula. + await driver.find('.test-importer-apply-formula').click(); + await gu.sendKeys(await gu.selectAllKey(), Key.DELETE, '$Population', Key.ENTER); + await gu.waitForServer(); + assert.deepEqual(await getColumnMatchingRows(), [ + { source: 'Name', destination: 'Name' }, + { source: 'District', destination: 'District' }, + { source: 'Population', destination: 'Population' }, + { source: 'Skip', destination: 'Country' }, + { source: '$Population\n', destination: 'Pop. \'000' }, + ]); + assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [ + 'Kabul', 'Kabol', '1780000', '0', '1780000', + 'Qandahar', 'Qandahar', '237500', '0', '237500', + 'Herat', 'Herat', '186800', '0', '186800', + 'Mazar-e-Sharif', 'Balkh', '127800', '0', '127800', + 'Amsterdam', 'Noord-Holland', '731200', '0', '731200', + 'Rotterdam', 'Zuid-Holland', '593321', '0', '593321', + ]); + assert.equal( + await driver.find('.test-importer-unmatched-fields').getText(), + '1 unmatched field in import:\nCountry' + ); + + // Click Country (with formula 'Skip') in the column mapping section, and start typing a formula. + await openSourceFor(/Country/); + await driver.find('.test-importer-apply-formula').click(); + await gu.sendKeys('$'); + await gu.waitForServer(); + + // Wait until the Ace autocomplete menu is shown. + await driver.wait(() => driver.find('div.ace_autocomplete').isDisplayed(), 2000); + + // Check that the autocomplete is suggesting column ids from the imported table. + const completions = await driver.findAll( + 'div.ace_autocomplete div.ace_line', async el => (await el.getText()).split(' ')[0] + ); + assert.deepEqual( + completions.slice(0, 6), + [ + "$\nCountry", + "$\nDistrict", + "$\nid", + "$\nName", + "$\nPop_000", + "$\nPopulation", + ] + ); + + // Set a constant value for the formula. + await gu.sendKeys(Key.BACK_SPACE, '123', Key.ENTER); + await gu.waitForServer(); + + // Check that the formula code is shown, as well as the evaluation result in the preview. + assert.deepEqual(await getColumnMatchingRows(), [ + { source: 'Name', destination: 'Name' }, + { source: 'District', destination: 'District' }, + { source: 'Population', destination: 'Population' }, + { source: '123\n', destination: 'Country' }, + { source: '$Population\n', destination: 'Pop. \'000' }, + ]); + + assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [ + 'Kabul', 'Kabol', '1780000', '123', '1780000', + 'Qandahar', 'Qandahar', '237500', '123', '237500', + 'Herat', 'Herat', '186800', '123', '186800', + 'Mazar-e-Sharif', 'Balkh', '127800', '123', '127800', + 'Amsterdam', 'Noord-Holland', '731200', '123', '731200', + 'Rotterdam', 'Zuid-Holland', '593321', '123', '593321', + ]); + assert.isFalse(await driver.find('.test-importer-unmatched-fields').isPresent()); + }); + + it('should reflect mappings when import to new table is finished', async function() { + // Skip 'Population', so that we can test imports with skipped columns. + await openSourceFor(/Population/); + await driver.findContent('.test-importer-column-match-menu-item', 'Skip').click(); + await gu.waitForServer(); + + // Finish importing, and check that the destination tables have the correct data. + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getPageNames(), [ + 'Table1', + 'Table1', + 'City', + 'Country', + 'CountryLanguage', + 'Country', + 'CountryLanguage' + ]); + + // Check the contents of Table1; it should have duplicates of the original 2 rows. + assert.deepEqual(await gu.getVisibleGridCells({ rowNums: [1, 2, 3, 4, 5], cols: [0, 1, 2, 3, 4] }), + [ + 'hello', '', '', '', 'HELLO', + '', 'world', '', '', '', + 'hello', '', '', '', 'HELLO', + '', 'world', '', '', '', + '', '', '', '', '', + ] + ); + + await gu.getPageItem('City').click(); + await gu.waitForServer(); + + // The first half should be the original imported rows. + assert.deepEqual(await gu.getVisibleGridCells( + {cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5, 6]}), + [ + 'Kabul', 'Kabol', '1780000', '2', '1780', + 'Qandahar', 'Qandahar', '237500', '2', '237.5', + 'Herat', 'Herat', '186800', '2', '186.8', + 'Mazar-e-Sharif', 'Balkh', '127800', '2', '127.8', + 'Amsterdam', 'Noord-Holland', '731200', '159', '731.2', + 'Rotterdam', 'Zuid-Holland', '593321', '159', '593.321', + ] + ); + + // The second half should be the newly imported rows with custom mappings. + assert.equal(await gu.getGridRowCount(), 8159); + assert.deepEqual(await gu.getVisibleGridCells( + {cols: [0, 1, 2, 3, 4], rowNums: [8152, 8153, 8154, 8155, 8156, 8157]}), + [ + 'Gweru', 'Midlands', '0', '123', '128037', + 'Gaza', 'Gaza', '0', '123', '353632', + 'Khan Yunis', 'Khan Yunis', '0', '123', '123175', + 'Hebron', 'Hebron', '0', '123', '119401', + 'Jabaliya', 'North Gaza', '0', '123', '113901', + 'Nablus', 'Nablus', '0', '123', '100231', + ] + ); + }); + + it('should reflect mappings in previews of incremental imports', async function() { + // Delete the first row of the Country column. (Needed for a later assertion.) + await gu.sendKeys(Key.chord(await gu.modKey(), Key.UP)); + await gu.getCell(3, 1).click(); + await gu.sendKeys(Key.DELETE); + await gu.waitForServer(); + + // Import a CSV file containing city data, with column names that differ from the City table. + await gu.importFileDialog('./uploads/Cities.csv'); + assert.equal(await driver.findWait('.test-importer-preview', 2000).isPresent(), true); + + // Change the destination to City, and check that column mapping defaults to skipping all columns. + await driver.findContent('.test-importer-target-existing-table', /City/).click(); + await gu.waitForServer(); + await waitForColumnMapping(); + assert.deepEqual(await getColumnMatchingRows(), [ + { source: 'Skip', destination: 'Name' }, + { source: 'Skip', destination: 'District' }, + { source: 'Skip', destination: 'Population' }, + { source: 'Skip', destination: 'Country' }, + { source: 'Skip', destination: 'Pop. \'000' }, + ]); + + assert.equal( + await driver.find('.test-importer-unmatched-fields').getText(), + '5 unmatched fields in import:\nName, District, Population, Country, Pop. \'000' + ); + + // Set formula for 'Name' to 'city_name' by typing in the formula. + await openSourceFor(/Name/); + await driver.find('.test-importer-apply-formula').click(); + await gu.sendKeys('$city_name', Key.ENTER); + await gu.waitForServer(); + + // Map 'District' to 'city_district' via the column mapping menu. + await openSourceFor('District'); + const menu = driver.find('.test-select-menu'); + await menu.findContent('.test-importer-column-match-menu-item', /city_district/).click(); + await gu.waitForServer(); + + // Check the column mapping section and preview both updated correctly. + assert.deepEqual(await getColumnMatchingRows(), [ + { source: '$city_name\n', destination: 'Name' }, + { source: 'city_district', destination: 'District' }, + { source: 'Skip', destination: 'Population' }, + { source: 'Skip', destination: 'Country' }, + { source: 'Skip', destination: 'Pop. \'000' }, + ]); + assert.deepEqual(await gu.getPreviewContents([0, 1, 2, 3, 4], [1, 2, 3, 4, 5, 6]), [ + 'Kabul', 'Kabol', '0', '0', '0', + 'Qandahar', 'Qandahar', '0', '0', '0', + 'Herat', 'Herat', '0', '0', '0', + 'Mazar-e-Sharif', 'Balkh', '0', '0', '0', + 'Amsterdam', 'Noord-Holland', '0', '0', '0', + 'Rotterdam', 'Zuid-Holland', '0', '0', '0', + ]); + assert.equal( + await driver.find('.test-importer-unmatched-fields').getText(), + '3 unmatched fields in import:\nPopulation, Country, Pop. \'000' + ); + + // Now toggle 'Update existing records', and merge on 'Name' and 'District'. + await driver.find('.test-importer-update-existing-records').click(); + await driver.find('.test-importer-merge-fields-select').click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /Name/ + ).click(); + await driver.findContent( + '.test-multi-select-menu .test-multi-select-menu-option', + /District/ + ).click(); + await gu.sendKeys(Key.ESCAPE); + await gu.waitForServer(); + + // Check that the column mapping section and preview updated correctly. + assert.deepEqual(await getColumnMatchingRows(), [ + { source: '$city_name\n', destination: 'Name' }, + { source: 'city_district', destination: 'District' }, + { source: 'Skip', destination: 'Population' }, + { source: 'Skip', destination: 'Country' }, + { source: 'Skip', destination: 'Pop. \'000' }, + ]); + await waitForDiffPreviewToLoad(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5]), [ + 'Kabul', 'Kabol', [undefined, undefined, '1780000'], '', [undefined, undefined, '1780'], + 'Qandahar', 'Qandahar', [undefined, undefined, '237500'], [undefined, undefined, '2'], + [undefined, undefined, '237.5'], + 'Herat', 'Herat', [undefined, undefined, '186800'], [undefined, undefined, '2'], + [undefined, undefined, '186.8'], + 'Mazar-e-Sharif', 'Balkh', [undefined, undefined, '127800'], [undefined, undefined, '2'], + [undefined, undefined, '127.8'], + 'Amsterdam', 'Noord-Holland', [undefined, undefined, '731200'], [undefined, undefined, '159'], + [undefined, undefined, '731.2'], + ]); + assert.equal( + await driver.find('.test-importer-unmatched-fields').getText(), + '3 unmatched fields in import:\nPopulation, Country, Pop. \'000' + ); + + // Map the remaining columns, except "Country"; we'll leave it skipped to check that + // we don't overwrite any values in the destination table. (A previous bug caused non-text + // skipped columns to overwrite data with default values, like 0.) + await openSourceFor(/Population/); + await driver.find('.test-importer-apply-formula').click(); + await gu.sendKeys('$city_pop', Key.ENTER); + await gu.waitForServer(); + + // For "Pop. '000", deliberately map a duplicate column (so we can later check if import succeeded). + await openSourceFor(/Pop\. '000/); + await driver.find('.test-importer-apply-formula').click(); + await gu.sendKeys('$city_pop', Key.ENTER); + await gu.waitForServer(); + + assert.deepEqual(await getColumnMatchingRows(), [ + { source: '$city_name\n', destination: 'Name' }, + { source: 'city_district', destination: 'District' }, + { source: '$city_pop\n', destination: 'Population' }, + { source: 'Skip', destination: 'Country' }, + { source: '$city_pop\n', destination: 'Pop. \'000' }, + ]); + + await waitForDiffPreviewToLoad(); + assert.deepEqual(await getPreviewDiffCellValues([0, 1, 2, 3, 4], [1, 2, 3, 4, 5]), [ + // Kabul's Country column should appear blank, since we deleted it earlier. + 'Kabul', 'Kabol', ['1780000', '3560000', undefined], '', ['1780', '3560000', undefined], + 'Qandahar', 'Qandahar', ['237500', '475000', undefined], [undefined, undefined, '2'], + ['237.5', '475000', undefined], + 'Herat', 'Herat', ['186800', '373600', undefined], [undefined, undefined, '2'], + ['186.8', '373600', undefined], + 'Mazar-e-Sharif', 'Balkh', ['127800', '255600', undefined], [undefined, undefined, '2'], + ['127.8', '255600', undefined], + 'Amsterdam', 'Noord-Holland', ['731200', '1462400', undefined], [undefined, undefined, '159'], + ['731.2', '1462400', undefined], + ]); + assert.equal( + await driver.find('.test-importer-unmatched-fields').getText(), + '1 unmatched field in import:\nCountry' + ); + }); + + it('should reflect mappings when incremental import is finished', async function() { + // Finish importing, and check that the destination table has the correct data. + await driver.find('.test-modal-confirm').click(); + await gu.waitForServer(); + assert.deepEqual(await gu.getPageNames(), [ + 'Table1', + 'Table1', + 'City', + 'Country', + 'CountryLanguage', + 'Country', + 'CountryLanguage', + ]); + + assert.deepEqual(await gu.getVisibleGridCells({cols: [0, 1, 2, 3, 4], rowNums: [1, 2, 3, 4, 5]}), + [ + // Kabul's Country column should still be blank, since we skipped it earlier. + 'Kabul', 'Kabol', '3560000', '', '3560000', + 'Qandahar', 'Qandahar', '475000', '2', '475000', + 'Herat', 'Herat', '373600', '2', '373600', + 'Mazar-e-Sharif', 'Balkh', '255600', '2', '255600', + 'Amsterdam', 'Noord-Holland', '1462400', '159', '1462400', + ] + ); + }); + }); +}); diff --git a/test/nbrowser/importerTestUtils.ts b/test/nbrowser/importerTestUtils.ts index 81040d41..5715ccec 100644 --- a/test/nbrowser/importerTestUtils.ts +++ b/test/nbrowser/importerTestUtils.ts @@ -41,7 +41,7 @@ export const waitForDiffPreviewToLoad = stackWrapFunc(async (): Promise => // Helper that gets the list of visible column matching rows to the left of the preview. export const getColumnMatchingRows = stackWrapFunc(async (): Promise<{source: string, destination: string}[]> => { return await driver.findAll('.test-importer-column-match-source-destination', async (el) => { - const source = await el.find('.test-importer-column-match-formula').getText(); + const source = await el.find('.test-importer-column-match-formula').getAttribute('textContent'); const destination = await el.find('.test-importer-column-match-destination').getText(); return {source, destination}; }); diff --git a/test/server/lib/ACLFormula.ts b/test/server/lib/ACLFormula.ts index 4b4ac634..8951c0f9 100644 --- a/test/server/lib/ACLFormula.ts +++ b/test/server/lib/ACLFormula.ts @@ -1,7 +1,7 @@ import {CellValue} from 'app/common/DocActions'; -import {AclMatchFunc, InfoView} from 'app/common/GranularAccessClause'; +import {InfoView} from 'app/common/GranularAccessClause'; import {GristObjCode} from 'app/plugin/GristData'; -import {compileAclFormula} from 'app/server/lib/ACLFormula'; +import {CompiledPredicateFormula, compilePredicateFormula} from 'app/common/PredicateFormula'; import {makeExceptionalDocSession} from 'app/server/lib/DocSession'; import {User} from 'app/server/lib/GranularAccess'; import {assert} from 'chai'; @@ -26,7 +26,7 @@ describe('ACLFormula', function() { const V = getInfoView; // A shortcut. - type SetAndCompile = (aclFormula: string) => Promise; + type SetAndCompile = (aclFormula: string) => Promise; let setAndCompile: SetAndCompile; before(async function () { @@ -44,7 +44,7 @@ describe('ACLFormula', function() { fakeSession, {tableId: '_grist_ACLRules', filters: {id: [ruleRef]}}); assert(tableData[3].aclFormulaParsed, "Expected aclFormulaParsed to be populated"); const parsedFormula = String(tableData[3].aclFormulaParsed[0]); - return compileAclFormula(JSON.parse(parsedFormula)); + return compilePredicateFormula(JSON.parse(parsedFormula)); }; });