From 7be0ee289d4327293bd9947e089284b0859d79bd Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Tue, 23 May 2023 15:17:28 -0400 Subject: [PATCH] support other SQLite wrappers, and various hooks needed by grist-static (#516) --- app/client/DefaultHooks.ts | 11 + app/client/Hooks.ts | 3 + app/client/components/GristWSConnection.ts | 5 +- app/client/components/WidgetFrame.ts | 2 + app/client/exposeModulesForTests.js | 2 +- app/client/lib/localization.ts | 3 +- app/client/models/AppModel.ts | 9 +- app/client/models/gristUrlState.ts | 9 +- app/client/ui/App.ts | 2 +- app/common/gristUrls.ts | 42 +++- app/common/marshal.ts | 11 +- app/gen-server/entity/AclRule.ts | 2 +- app/gen-server/entity/Alias.ts | 6 +- app/gen-server/entity/BillingAccount.ts | 4 +- .../entity/BillingAccountManager.ts | 4 +- app/gen-server/entity/Document.ts | 4 +- app/gen-server/entity/Group.ts | 2 +- app/gen-server/entity/Login.ts | 8 +- app/gen-server/entity/Organization.ts | 3 +- app/gen-server/entity/Pref.ts | 4 +- app/gen-server/entity/Product.ts | 2 +- app/gen-server/entity/Resource.ts | 6 +- app/gen-server/entity/Secret.ts | 4 +- app/gen-server/entity/User.ts | 4 +- app/server/lib/ActionHistoryImpl.ts | 13 +- app/server/lib/ActiveDoc.ts | 33 ++- app/server/lib/DocStorage.ts | 76 ++++--- app/server/lib/GranularAccess.ts | 13 +- app/server/lib/ICreate.ts | 11 +- app/server/lib/NSandbox.ts | 205 ++++++++++++------ app/server/lib/SQLiteDB.ts | 136 +++++------- app/server/lib/SqliteCommon.ts | 126 +++++++++++ app/server/lib/SqliteNode.ts | 104 +++++++++ app/server/lib/WorkCoordinator.ts | 7 +- app/server/lib/requestUtils.ts | 1 - app/server/lib/shutdown.js | 2 +- buildtools/build.sh | 8 +- sandbox/MANIFEST.in | 3 + sandbox/bundle_as_wheel.sh | 17 ++ sandbox/setup.py | 15 ++ sandbox/watch.sh | 6 +- test/server/lib/HostedStorageManager.ts | 5 +- 42 files changed, 684 insertions(+), 249 deletions(-) create mode 100644 app/client/DefaultHooks.ts create mode 100644 app/client/Hooks.ts create mode 100644 app/server/lib/SqliteCommon.ts create mode 100644 app/server/lib/SqliteNode.ts create mode 100644 sandbox/MANIFEST.in create mode 100755 sandbox/bundle_as_wheel.sh create mode 100644 sandbox/setup.py diff --git a/app/client/DefaultHooks.ts b/app/client/DefaultHooks.ts new file mode 100644 index 00000000..216d80d3 --- /dev/null +++ b/app/client/DefaultHooks.ts @@ -0,0 +1,11 @@ +import { UrlTweaks } from 'app/common/gristUrls'; + +export interface IHooks { + iframeAttributes?: Record, + fetch?: typeof fetch, + baseURI?: string, + urlTweaks?: UrlTweaks, +} + +export const defaultHooks: IHooks = { +}; diff --git a/app/client/Hooks.ts b/app/client/Hooks.ts new file mode 100644 index 00000000..5b5d5adf --- /dev/null +++ b/app/client/Hooks.ts @@ -0,0 +1,3 @@ +import {defaultHooks} from 'app/client/DefaultHooks'; + +export const hooks = defaultHooks; diff --git a/app/client/components/GristWSConnection.ts b/app/client/components/GristWSConnection.ts index 5a465412..1538ea59 100644 --- a/app/client/components/GristWSConnection.ts +++ b/app/client/components/GristWSConnection.ts @@ -1,11 +1,12 @@ import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals'; import {guessTimezone} from 'app/client/lib/guessTimezone'; import {getSessionStorage} from 'app/client/lib/storage'; +import {newUserAPIImpl} from 'app/client/models/AppModel'; import {getWorker} from 'app/client/models/gristConfigCache'; import {CommResponseBase} from 'app/common/CommTypes'; import * as gutil from 'app/common/gutil'; import {addOrgToPath, docUrl, getGristConfig} from 'app/common/urlUtils'; -import {UserAPI, UserAPIImpl} from 'app/common/UserAPI'; +import {UserAPI} from 'app/common/UserAPI'; import {Events as BackboneEvents} from 'backbone'; import {Disposable} from 'grainjs'; @@ -25,7 +26,7 @@ async function getDocWorkerUrl(assignmentId: string|null): Promise // never changes. if (assignmentId === null) { return docUrl(null); } - const api: UserAPI = new UserAPIImpl(getGristConfig().homeUrl!); + const api: UserAPI = newUserAPIImpl(); return getWorker(api, assignmentId); } diff --git a/app/client/components/WidgetFrame.ts b/app/client/components/WidgetFrame.ts index 2a2ba799..b5eeb7b4 100644 --- a/app/client/components/WidgetFrame.ts +++ b/app/client/components/WidgetFrame.ts @@ -1,5 +1,6 @@ import BaseView from 'app/client/components/BaseView'; import {GristDoc} from 'app/client/components/GristDoc'; +import {hooks} from 'app/client/Hooks'; import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals'; import {ColumnRec, ViewSectionRec} from 'app/client/models/DocModel'; import {AccessLevel, isSatisfied} from 'app/common/CustomWidget'; @@ -157,6 +158,7 @@ export class WidgetFrame extends DisposableWithEvents { return onElem( (this._iframe = dom('iframe', dom.cls('clipboard_focus'), dom.cls('custom_view'), { src: fullUrl, + ...hooks.iframeAttributes, })) ); } diff --git a/app/client/exposeModulesForTests.js b/app/client/exposeModulesForTests.js index 7fba8607..b364f22d 100644 --- a/app/client/exposeModulesForTests.js +++ b/app/client/exposeModulesForTests.js @@ -6,7 +6,7 @@ Object.assign(window.exposedModules, { grainjs: require('grainjs'), ko: require('knockout'), moment: require('moment-timezone'), - Comm: require('./components/Comm'), + Comm: require('app/client/components/Comm'), _loadScript: require('./lib/loadScript'), ConnectState: require('./models/ConnectState'), }); diff --git a/app/client/lib/localization.ts b/app/client/lib/localization.ts index 39d69541..4780b73c 100644 --- a/app/client/lib/localization.ts +++ b/app/client/lib/localization.ts @@ -1,3 +1,4 @@ +import {hooks} from 'app/client/Hooks'; import {getGristConfig} from 'app/common/urlUtils'; import {DomContents} from 'grainjs'; import i18next from 'i18next'; @@ -34,7 +35,7 @@ export async function setupLocale() { // Detect what is resolved languages to load. const languages = i18next.languages; // Fetch all json files (all of which should be already preloaded); - const loadPath = `${document.baseURI}locales/{{lng}}.{{ns}}.json`; + const loadPath = `${hooks.baseURI || document.baseURI}locales/{{lng}}.{{ns}}.json`; const pathsToLoad: Promise[] = []; async function load(lang: string, n: string) { const resourceUrl = loadPath.replace('{{lng}}', lang.replace("-", "_")).replace('{{ns}}', n); diff --git a/app/client/models/AppModel.ts b/app/client/models/AppModel.ts index c5860db4..24bbc694 100644 --- a/app/client/models/AppModel.ts +++ b/app/client/models/AppModel.ts @@ -1,4 +1,5 @@ import {BehavioralPromptsManager} from 'app/client/components/BehavioralPromptsManager'; +import {hooks} from 'app/client/Hooks'; import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals'; import {makeT} from 'app/client/lib/localization'; import {sessionStorageObs} from 'app/client/lib/localStorageObs'; @@ -131,7 +132,7 @@ export class TopAppModelImpl extends Disposable implements TopAppModel { constructor( window: {gristConfig?: GristLoadConfig}, - public readonly api: UserAPI = new UserAPIImpl(getHomeUrl()), + public readonly api: UserAPI = newUserAPIImpl(), ) { super(); setErrorNotifier(this.notifier); @@ -436,6 +437,12 @@ export function getHomeUrl(): string { return (gristConfig && gristConfig.homeUrl) || `${protocol}//${host}`; } +export function newUserAPIImpl(): UserAPIImpl { + return new UserAPIImpl(getHomeUrl(), { + fetch: hooks.fetch, + }); +} + export function getOrgNameOrGuest(org: Organization|null, user: FullUser|null) { if (!org) { return ''; } if (user && user.anonymous && org.owner && org.owner.id === user.id) { diff --git a/app/client/models/gristUrlState.ts b/app/client/models/gristUrlState.ts index a95f2776..0295b5f8 100644 --- a/app/client/models/gristUrlState.ts +++ b/app/client/models/gristUrlState.ts @@ -23,6 +23,7 @@ * Note that the form of URLs depends on the settings in window.gristConfig object. */ import {unsavedChanges} from 'app/client/components/UnsavedChanges'; +import {hooks} from 'app/client/Hooks'; import {UrlState} from 'app/client/lib/UrlState'; import {decodeUrl, encodeUrl, getSlugIfNeeded, GristLoadConfig, IGristUrlState, parseFirstUrlPart} from 'app/common/gristUrls'; @@ -134,7 +135,9 @@ export class UrlStateImpl { */ public encodeUrl(state: IGristUrlState, baseLocation: Location | URL): string { const gristConfig = this._window.gristConfig || {}; - return encodeUrl(gristConfig, state, baseLocation); + return encodeUrl(gristConfig, state, baseLocation, { + tweaks: hooks.urlTweaks, + }); } /** @@ -142,7 +145,9 @@ export class UrlStateImpl { */ public decodeUrl(location: Location | URL): IGristUrlState { const gristConfig = this._window.gristConfig || {}; - return decodeUrl(gristConfig, location); + return decodeUrl(gristConfig, location, { + tweaks: hooks.urlTweaks, + }); } /** diff --git a/app/client/ui/App.ts b/app/client/ui/App.ts index cd5f323c..5fe1f246 100644 --- a/app/client/ui/App.ts +++ b/app/client/ui/App.ts @@ -84,7 +84,7 @@ export class App extends DisposableWithEvents { const isHelpPaneVisible = ko.observable(false); - G.document.querySelector('#grist-logo-wrapper').remove(); + G.document.querySelector('#grist-logo-wrapper')?.remove(); // Help pop-up pane const helpDiv = document.body.appendChild( diff --git a/app/common/gristUrls.ts b/app/common/gristUrls.ts index 679d9a1d..da8725f9 100644 --- a/app/common/gristUrls.ts +++ b/app/common/gristUrls.ts @@ -198,7 +198,8 @@ export function encodeUrl(gristConfig: Partial, options: { // make an api url - warning: just barely works, and // only for documents - api?: boolean + api?: boolean, + tweaks?: UrlTweaks, } = {}): string { const url = new URL(baseLocation.href); const parts = ['/']; @@ -269,8 +270,10 @@ export function encodeUrl(gristConfig: Partial, } } const queryStr = encodeQueryParams(queryParams); + url.pathname = parts.join(''); url.search = queryStr; + if (state.hash) { // Project tests use hashes, so only set hash if there is an anchor. url.hash = hashParts.join('.'); @@ -285,13 +288,23 @@ export function encodeUrl(gristConfig: Partial, } else { url.hash = ''; } + options.tweaks?.postEncode?.({ + url, + parts, + state, + baseLocation, + }); return url.href; } /** * Parse a URL location into an IGristUrlState object. See encodeUrl() documentation. */ -export function decodeUrl(gristConfig: Partial, location: Location | URL): IGristUrlState { +export function decodeUrl(gristConfig: Partial, location: Location | URL, options?: { + tweaks?: UrlTweaks, +}): IGristUrlState { + location = new URL(location.href); // Make sure location is a URL. + options?.tweaks?.preDecode?.({ url: location }); const parts = location.pathname.slice(1).split('/'); const map = new Map(); for (let i = 0; i < parts.length; i += 2) { @@ -871,3 +884,28 @@ export function getSlugIfNeeded(doc: {id: string, urlId: string|null, name: stri if (!shouldIncludeSlug(doc)) { return; } return nameToSlug(doc.name); } + +/** + * It is possible we want to remap Grist URLs in some way - specifically, + * grist-static does this. We allow for a hook that is called after + * encoding state as a URL, and a hook that is called before decoding + * state from a URL. + */ +export interface UrlTweaks { + /** + * Tweak an encoded URL. Operates on the URL directly, in place. + */ + postEncode?(options: { + url: URL, + parts: string[], + state: IGristUrlState, + baseLocation: Location | URL, + }): void; + + /** + * Tweak a URL prior to decoding it. Operates on the URL directly, in place. + */ + preDecode?(options: { + url: URL, + }): void; +} diff --git a/app/common/marshal.ts b/app/common/marshal.ts index 60443ac9..13486ba6 100644 --- a/app/common/marshal.ts +++ b/app/common/marshal.ts @@ -33,6 +33,13 @@ import * as util from 'util'; export interface MarshalOptions { stringToBuffer?: boolean; version?: number; + + // True if we want keys in dicts to be buffers. + // It is convenient to have some freedom here to simplify implementation + // of marshaling for some SQLite wrappers. This flag was initially + // introduced for a fork of Grist using better-sqlite3, and I don't + // remember exactly what the issues were. + keysAreBuffers?: boolean; } export interface UnmarshalOptions { @@ -129,11 +136,13 @@ export class Marshaller { private _memBuf: MemBuffer; private readonly _floatCode: number; private readonly _stringCode: number; + private readonly _keysAreBuffers: boolean; constructor(options?: MarshalOptions) { this._memBuf = new MemBuffer(undefined); this._floatCode = options && options.version && options.version >= 2 ? marshalCodes.BFLOAT : marshalCodes.FLOAT; this._stringCode = options && options.stringToBuffer ? marshalCodes.STRING : marshalCodes.UNICODE; + this._keysAreBuffers = Boolean(options?.keysAreBuffers); } public dump(): Uint8Array { @@ -261,7 +270,7 @@ export class Marshaller { const keys = Object.keys(obj); keys.sort(); for (const key of keys) { - this.marshal(key); + this.marshal(this._keysAreBuffers ? Buffer.from(key) : key); this.marshal(obj[key]); } this._memBuf.writeUint8(marshalCodes.NULL); diff --git a/app/gen-server/entity/AclRule.ts b/app/gen-server/entity/AclRule.ts index 6232b654..59b82746 100644 --- a/app/gen-server/entity/AclRule.ts +++ b/app/gen-server/entity/AclRule.ts @@ -13,7 +13,7 @@ export class AclRule extends BaseEntity { @PrimaryGeneratedColumn() public id: number; - @Column() + @Column({type: Number}) public permissions: number; @OneToOne(type => Group, group => group.aclRule) diff --git a/app/gen-server/entity/Alias.ts b/app/gen-server/entity/Alias.ts index f8e79b41..127212e2 100644 --- a/app/gen-server/entity/Alias.ts +++ b/app/gen-server/entity/Alias.ts @@ -5,13 +5,13 @@ import {Organization} from './Organization'; @Entity({name: 'aliases'}) export class Alias extends BaseEntity { - @PrimaryColumn({name: 'org_id'}) + @PrimaryColumn({name: 'org_id', type: Number}) public orgId: number; - @PrimaryColumn({name: 'url_id'}) + @PrimaryColumn({name: 'url_id', type: String}) public urlId: string; - @Column({name: 'doc_id'}) + @Column({name: 'doc_id', type: String}) public docId: string; @ManyToOne(type => Document) diff --git a/app/gen-server/entity/BillingAccount.ts b/app/gen-server/entity/BillingAccount.ts index a4a5ac83..64806d93 100644 --- a/app/gen-server/entity/BillingAccount.ts +++ b/app/gen-server/entity/BillingAccount.ts @@ -34,14 +34,14 @@ export class BillingAccount extends BaseEntity { @JoinColumn({name: 'product_id'}) public product: Product; - @Column() + @Column({type: Boolean}) public individual: boolean; // A flag for when all is well with the user's subscription. // Probably shouldn't use this to drive whether service is provided or not. // Strip recommends updating an end-of-service datetime every time payment // is received, adding on a grace period of some days. - @Column({name: 'in_good_standing', default: nativeValues.trueValue}) + @Column({name: 'in_good_standing', type: Boolean, default: nativeValues.trueValue}) public inGoodStanding: boolean; @Column({type: nativeValues.jsonEntityType, nullable: true}) diff --git a/app/gen-server/entity/BillingAccountManager.ts b/app/gen-server/entity/BillingAccountManager.ts index 7ba18d62..1a432640 100644 --- a/app/gen-server/entity/BillingAccountManager.ts +++ b/app/gen-server/entity/BillingAccountManager.ts @@ -10,14 +10,14 @@ export class BillingAccountManager extends BaseEntity { @PrimaryGeneratedColumn() public id: number; - @Column({name: 'billing_account_id'}) + @Column({name: 'billing_account_id', type: Number}) public billingAccountId: number; @ManyToOne(type => BillingAccount, { onDelete: 'CASCADE' }) @JoinColumn({name: 'billing_account_id'}) public billingAccount: BillingAccount; - @Column({name: 'user_id'}) + @Column({name: 'user_id', type: Number}) public userId: number; @ManyToOne(type => User, { onDelete: 'CASCADE' }) diff --git a/app/gen-server/entity/Document.ts b/app/gen-server/entity/Document.ts index 0d4bf08e..2cbda145 100644 --- a/app/gen-server/entity/Document.ts +++ b/app/gen-server/entity/Document.ts @@ -24,7 +24,7 @@ function isValidUrlId(urlId: string) { @Entity({name: 'docs'}) export class Document extends Resource { - @PrimaryColumn() + @PrimaryColumn({type: String}) public id: string; @ManyToOne(type => Workspace) @@ -35,7 +35,7 @@ export class Document extends Resource { public aclRules: AclRuleDoc[]; // Indicates whether the doc is pinned to the org it lives in. - @Column({name: 'is_pinned', default: false}) + @Column({name: 'is_pinned', type: Boolean, default: false}) public isPinned: boolean; // Property that may be returned when the doc is fetched to indicate the access the diff --git a/app/gen-server/entity/Group.ts b/app/gen-server/entity/Group.ts index b82f9c18..63fc7473 100644 --- a/app/gen-server/entity/Group.ts +++ b/app/gen-server/entity/Group.ts @@ -9,7 +9,7 @@ export class Group extends BaseEntity { @PrimaryGeneratedColumn() public id: number; - @Column() + @Column({type: String}) public name: string; @ManyToMany(type => User) diff --git a/app/gen-server/entity/Login.ts b/app/gen-server/entity/Login.ts index 7dec4295..98b7d858 100644 --- a/app/gen-server/entity/Login.ts +++ b/app/gen-server/entity/Login.ts @@ -5,18 +5,18 @@ import {User} from "./User"; @Entity({name: 'logins'}) export class Login extends BaseEntity { - @PrimaryColumn() + @PrimaryColumn({type: Number}) public id: number; // This is the normalized email address we use for equality and indexing. - @Column() + @Column({type: String}) public email: string; // This is how the user's email address should be displayed. - @Column({name: 'display_email'}) + @Column({name: 'display_email', type: String}) public displayEmail: string; - @Column({name: 'user_id'}) + @Column({name: 'user_id', type: Number}) public userId: number; @ManyToOne(type => User) diff --git a/app/gen-server/entity/Organization.ts b/app/gen-server/entity/Organization.ts index 321055bd..afa02dd9 100644 --- a/app/gen-server/entity/Organization.ts +++ b/app/gen-server/entity/Organization.ts @@ -29,6 +29,7 @@ export class Organization extends Resource { public id: number; @Column({ + type: String, nullable: true }) public domain: string; @@ -46,7 +47,7 @@ export class Organization extends Resource { @OneToMany(type => AclRuleOrg, aclRule => aclRule.organization) public aclRules: AclRuleOrg[]; - @Column({name: 'billing_account_id'}) + @Column({name: 'billing_account_id', type: Number}) public billingAccountId: number; @ManyToOne(type => BillingAccount) diff --git a/app/gen-server/entity/Pref.ts b/app/gen-server/entity/Pref.ts index deca87aa..80812b13 100644 --- a/app/gen-server/entity/Pref.ts +++ b/app/gen-server/entity/Pref.ts @@ -11,10 +11,10 @@ export class Pref { // one, but we haven't marked them as so in the DB since the SQL standard frowns // on nullable primary keys (and Postgres doesn't support them). We could add // another primary key, but we don't actually need one. - @PrimaryColumn({name: 'user_id'}) + @PrimaryColumn({name: 'user_id', type: Number}) public userId: number|null; - @PrimaryColumn({name: 'org_id'}) + @PrimaryColumn({name: 'org_id', type: Number}) public orgId: number|null; @ManyToOne(type => User) diff --git a/app/gen-server/entity/Product.ts b/app/gen-server/entity/Product.ts index 84f4358c..aac39042 100644 --- a/app/gen-server/entity/Product.ts +++ b/app/gen-server/entity/Product.ts @@ -169,7 +169,7 @@ export class Product extends BaseEntity { @PrimaryGeneratedColumn() public id: number; - @Column() + @Column({type: String}) public name: string; @Column({type: nativeValues.jsonEntityType}) diff --git a/app/gen-server/entity/Resource.ts b/app/gen-server/entity/Resource.ts index 1958026e..bb9d234a 100644 --- a/app/gen-server/entity/Resource.ts +++ b/app/gen-server/entity/Resource.ts @@ -3,13 +3,13 @@ import {ApiError} from 'app/common/ApiError'; import {CommonProperties} from "app/common/UserAPI"; export class Resource extends BaseEntity { - @Column() + @Column({type: String}) public name: string; - @Column({name: 'created_at', default: () => "CURRENT_TIMESTAMP"}) + @Column({name: 'created_at', type: Date, default: () => "CURRENT_TIMESTAMP"}) public createdAt: Date; - @Column({name: 'updated_at', default: () => "CURRENT_TIMESTAMP"}) + @Column({name: 'updated_at', type: Date, default: () => "CURRENT_TIMESTAMP"}) public updatedAt: Date; // a computed column which, when present, means the entity should be filtered out diff --git a/app/gen-server/entity/Secret.ts b/app/gen-server/entity/Secret.ts index d38d7466..b591961f 100644 --- a/app/gen-server/entity/Secret.ts +++ b/app/gen-server/entity/Secret.ts @@ -3,10 +3,10 @@ import {Document} from "./Document"; @Entity({name: 'secrets'}) export class Secret extends BaseEntity { - @PrimaryColumn() + @PrimaryColumn({type: String}) public id: string; // generally a UUID - @Column({name: 'value'}) + @Column({name: 'value', type: String}) public value: string; @ManyToOne(_type => Document, { onDelete: 'CASCADE' }) diff --git a/app/gen-server/entity/User.ts b/app/gen-server/entity/User.ts index 2079b867..2ed10169 100644 --- a/app/gen-server/entity/User.ts +++ b/app/gen-server/entity/User.ts @@ -15,7 +15,7 @@ export class User extends BaseEntity { @PrimaryGeneratedColumn() public id: number; - @Column() + @Column({type: String}) public name: string; @Column({name: 'api_key', type: String, nullable: true}) @@ -46,7 +46,7 @@ export class User extends BaseEntity { }) public groups: Group[]; - @Column({name: 'is_first_time_user', default: false}) + @Column({name: 'is_first_time_user', type: Boolean, default: false}) public isFirstTimeUser: boolean; @Column({name: 'options', type: nativeValues.jsonEntityType, nullable: true}) diff --git a/app/server/lib/ActionHistoryImpl.ts b/app/server/lib/ActionHistoryImpl.ts index 2896ba04..3930d864 100644 --- a/app/server/lib/ActionHistoryImpl.ts +++ b/app/server/lib/ActionHistoryImpl.ts @@ -314,7 +314,7 @@ export class ActionHistoryImpl implements ActionHistory { } finally { if (tip) { await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ? - WHERE name = "local_sent"`, + WHERE name = 'local_sent'`, tip); } } @@ -336,7 +336,7 @@ export class ActionHistoryImpl implements ActionHistory { } } await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ? - WHERE name = "shared"`, + WHERE name = 'shared'`, candidate.id); if (candidates.length === 1) { this._haveLocalSent = false; @@ -405,9 +405,10 @@ export class ActionHistoryImpl implements ActionHistory { } public async getActions(actionNums: number[]): Promise> { - const actions = await this._db.all(`SELECT actionHash, actionNum, body FROM _gristsys_ActionHistory - where actionNum in (${actionNums.map(x => '?').join(',')})`, - actionNums); + const actions = await this._db.all( + `SELECT actionHash, actionNum, body FROM _gristsys_ActionHistory + where actionNum in (${actionNums.map(x => '?').join(',')})`, + ...actionNums); return reportTimeTaken("getActions", () => { const actionsByActionNum = keyBy(actions, 'actionNum'); return actionNums @@ -516,7 +517,7 @@ export class ActionHistoryImpl implements ActionHistory { FROM _gristsys_ActionHistoryBranch as Branch LEFT JOIN _gristsys_ActionHistory as History ON History.id = Branch.actionRef - WHERE name in ("shared", "local_sent", "local_unsent")`); + WHERE name in ('shared', 'local_sent', 'local_unsent')`); const bits = mapValues(keyBy(rows, 'name'), this._asActionIdentifiers); const missing = { actionHash: null, actionRef: null, actionNum: null } as ActionIdentifiers; return { diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 107d3e82..22f02bb8 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -169,7 +169,10 @@ const UPDATE_DATA_SIZE_DELAY = {delayMs: 5 * 60 * 1000, varianceMs: 30 * 1000}; const LOG_DOCUMENT_METRICS_DELAY = {delayMs: 60 * 60 * 1000, varianceMs: 30 * 1000}; // A hook for dependency injection. -export const Deps = {ACTIVEDOC_TIMEOUT}; +export const Deps = { + ACTIVEDOC_TIMEOUT, + ACTIVEDOC_TIMEOUT_ACTION: 'shutdown' as 'shutdown'|'ignore', +}; interface UpdateUsageOptions { // Whether usage should be synced to the home database. Defaults to true. @@ -242,7 +245,7 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc { // Timer for shutting down the ActiveDoc a bit after all clients are gone. private _inactivityTimer = new InactivityTimer(() => { this._log.debug(null, 'inactivity timeout'); - return this.shutdown(); + return this._onInactive(); }, Deps.ACTIVEDOC_TIMEOUT * 1000); private _recoveryMode: boolean = false; private _shuttingDown: boolean = false; @@ -1509,8 +1512,7 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc { */ public async getUsersForViewAs(docSession: OptDocSession): Promise { // Make sure we have rights to view access rules. - const db = this.getHomeDbManager(); - if (!db || !await this._granularAccess.hasAccessRulesPermission(docSession)) { + if (!await this._granularAccess.hasAccessRulesPermission(docSession)) { throw new Error('Cannot list ACL users'); } @@ -1525,12 +1527,15 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc { // Collect users the document is shared with. const userId = getDocSessionUserId(docSession); if (!userId) { throw new Error('Cannot determine user'); } - const access = db.unwrapQueryResult( - await db.getDocAccess({userId, urlId: this.docName}, { - flatten: true, excludeUsersWithoutAccess: true, - })); - result.users = access.users; - result.users.forEach(user => isShared.add(normalizeEmail(user.email))); + const db = this.getHomeDbManager(); + if (db) { + const access = db.unwrapQueryResult( + await db.getDocAccess({userId, urlId: this.docName}, { + flatten: true, excludeUsersWithoutAccess: true, + })); + result.users = access.users; + result.users.forEach(user => isShared.add(normalizeEmail(user.email))); + } // Collect users from user attribute tables. Omit duplicates with users the document is // shared with. @@ -2048,7 +2053,7 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc { documentSettings.engine = (pythonVersion === '2') ? 'python2' : 'python3'; } await this.docStorage.run('UPDATE _grist_DocInfo SET timezone = ?, documentSettings = ?', - [timezone, JSON.stringify(documentSettings)]); + timezone, JSON.stringify(documentSettings)); } private _makeInfo(docSession: OptDocSession, options: ApplyUAOptions = {}) { @@ -2657,6 +2662,12 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc { } return this._attachmentColumns; } + + private async _onInactive() { + if (Deps.ACTIVEDOC_TIMEOUT_ACTION === 'shutdown') { + await this.shutdown(); + } + } } // Helper to initialize a sandbox action bundle with no values. diff --git a/app/server/lib/DocStorage.ts b/app/server/lib/DocStorage.ts index 6a07d0d3..d6a46834 100644 --- a/app/server/lib/DocStorage.ts +++ b/app/server/lib/DocStorage.ts @@ -7,7 +7,6 @@ */ -import * as sqlite3 from '@gristlabs/sqlite3'; import {LocalActionBundle} from 'app/common/ActionBundle'; import {BulkColValues, DocAction, TableColValues, TableDataAction, toTableDataAction} from 'app/common/DocActions'; import * as gristTypes from 'app/common/gristTypes'; @@ -23,12 +22,12 @@ import log from 'app/server/lib/log'; import assert from 'assert'; import * as bluebird from 'bluebird'; import * as fse from 'fs-extra'; -import {RunResult} from 'sqlite3'; import * as _ from 'underscore'; import * as util from 'util'; import uuidv4 from "uuid/v4"; import {OnDemandStorage} from './OnDemandActions'; -import {ISQLiteDB, MigrationHooks, OpenMode, quoteIdent, ResultRow, SchemaInfo, SQLiteDB} from './SQLiteDB'; +import {ISQLiteDB, MigrationHooks, OpenMode, PreparedStatement, quoteIdent, + ResultRow, RunResult, SchemaInfo, SQLiteDB} from 'app/server/lib/SQLiteDB'; import chunk = require('lodash/chunk'); import cloneDeep = require('lodash/cloneDeep'); import groupBy = require('lodash/groupBy'); @@ -447,7 +446,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { * Converts an array of columns to an array of rows (suitable to use as sqlParams), encoding all * values as needed, according to an array of Grist type strings (must be parallel to columns). */ - private static _encodeColumnsToRows(types: string[], valueColumns: any[]): any[] { + private static _encodeColumnsToRows(types: string[], valueColumns: any[]): any[][] { const marshaller = new marshal.Marshaller({version: 2}); const rows = _.unzip(valueColumns); for (const row of rows) { @@ -734,8 +733,11 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { }) .catch(err => { // This replicates previous logic for _updateMetadata. - if (err.message.startsWith('SQLITE_ERROR: no such table')) { + // It matches errors from node-sqlite3 and better-sqlite3 + if (err.message.startsWith('SQLITE_ERROR: no such table') || + err.message.startsWith('no such table:')) { err.message = `NO_METADATA_ERROR: ${this.docName} has no metadata`; + if (!err.cause) { err.cause = {}; } err.cause.code = 'NO_METADATA_ERROR'; } throw err; @@ -781,7 +783,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { .then(() => true) // If UNIQUE constraint failed, this ident must already exists, so return false. .catch(err => { - if (/^SQLITE_CONSTRAINT: UNIQUE constraint failed/.test(err.message)) { + if (/^(SQLITE_CONSTRAINT: )?UNIQUE constraint failed/.test(err.message)) { return false; } throw err; @@ -879,7 +881,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { } whereParts = whereParts.concat(query.wheres ?? []); const sql = this._getSqlForQuery(query, whereParts); - return this._getDB().allMarshal(sql, params); + return this._getDB().allMarshal(sql, ...params); } /** @@ -1125,16 +1127,12 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { if (numChunks > 0) { debuglog("DocStorage.BulkRemoveRecord: splitting " + rowIds.length + " deletes into chunks of size " + chunkSize); - await this.prepare(preSql + chunkParams + postSql) - .then(function(stmt) { - return bluebird.Promise.each(_.range(0, numChunks * chunkSize, chunkSize), function(index: number) { - debuglog("DocStorage.BulkRemoveRecord: chunk delete " + index + "-" + (index + chunkSize - 1)); - return bluebird.Promise.fromCallback((cb: any) => stmt.run(rowIds.slice(index, index + chunkSize), cb)); - }) - .then(function() { - return bluebird.Promise.fromCallback((cb: any) => stmt.finalize(cb)); - }); - }); + const stmt = await this.prepare(preSql + chunkParams + postSql); + for (const index of _.range(0, numChunks * chunkSize, chunkSize)) { + debuglog("DocStorage.BulkRemoveRecord: chunk delete " + index + "-" + (index + chunkSize - 1)); + await stmt.run(rowIds.slice(index, index + chunkSize)); + } + await stmt.finalize(); } if (numLeftovers > 0) { @@ -1433,8 +1431,8 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { return this._markAsChanged(this._getDB().exec(sql)); } - public prepare(sql: string, ...args: any[]): Promise { - return this._getDB().prepare(sql, ...args); + public prepare(sql: string): Promise { + return this._getDB().prepare(sql); } public get(sql: string, ...args: any[]): Promise { @@ -1545,7 +1543,16 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { name LIKE 'sqlite_%' OR name LIKE '_gristsys_%' ); - `); + `).catch(e => { + if (String(e).match(/no such table: dbstat/)) { + // We are using a version of SQLite that doesn't have + // dbstat compiled in. But it would be sad to disable + // Grist entirely just because we can't track byte-count. + // So return NaN in this case. + return {totalSize: NaN}; + } + throw e; + }); return result!.totalSize; } @@ -1576,19 +1583,15 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { /** * Internal helper for applying Bulk Update or Add Record sql */ - private async _applyMaybeBulkUpdateOrAddSql(sql: string, sqlParams: any[]): Promise { + private async _applyMaybeBulkUpdateOrAddSql(sql: string, sqlParams: any[][]): Promise { if (sqlParams.length === 1) { - await this.run(sql, sqlParams[0]); + await this.run(sql, ...sqlParams[0]); } else { - return this.prepare(sql) - .then(function(stmt) { - return bluebird.Promise.each(sqlParams, function(param: string) { - return bluebird.Promise.fromCallback((cb: any) => stmt.run(param, cb)); - }) - .then(function() { - return bluebird.Promise.fromCallback((cb: any) => stmt.finalize(cb)); - }); - }); + const stmt = await this.prepare(sql); + for (const param of sqlParams) { + await stmt.run(...param); + } + await stmt.finalize(); } } @@ -1613,9 +1616,9 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { } const oldGristType = this._getGristType(tableId, colId); const oldSqlType = colInfo.type || 'BLOB'; - const oldDefault = colInfo.dflt_value; + const oldDefault = fixDefault(colInfo.dflt_value); const newSqlType = newColType ? DocStorage._getSqlType(newColType) : oldSqlType; - const newDefault = newColType ? DocStorage._formattedDefault(newColType) : oldDefault; + const newDefault = fixDefault(newColType ? DocStorage._formattedDefault(newColType) : oldDefault); const newInfo = {name: newColId, type: newSqlType, dflt_value: newDefault}; // Check if anything actually changed, and only rebuild the table then. if (Object.keys(newInfo).every(p => ((newInfo as any)[p] === colInfo[p]))) { @@ -1832,3 +1835,10 @@ export interface IndexInfo extends IndexColumns { export async function createAttachmentsIndex(db: ISQLiteDB) { await db.exec(`CREATE INDEX _grist_Attachments_fileIdent ON _grist_Attachments(fileIdent)`); } + +// Old docs may have incorrect quotes in their schema for default values +// that node-sqlite3 may tolerate but not other wrappers. Patch such +// material as we run into it. +function fixDefault(def: string) { + return (def === '""') ? "''" : def; +} diff --git a/app/server/lib/GranularAccess.ts b/app/server/lib/GranularAccess.ts index 9bbafc96..460a30d0 100644 --- a/app/server/lib/GranularAccess.ts +++ b/app/server/lib/GranularAccess.ts @@ -1879,11 +1879,10 @@ export class GranularAccess implements GranularAccessForBundle { * tables or examples. */ private async _getViewAsUser(linkParameters: Record): Promise { - // Look up user information in database. - if (!this._homeDbManager) { throw new Error('database required'); } + // Look up user information in database, if available const dbUser = linkParameters.aclAsUserId ? - (await this._homeDbManager.getUser(integerParam(linkParameters.aclAsUserId, 'aclAsUserId'))) : - (await this._homeDbManager.getExistingUserByLogin(linkParameters.aclAsUser)); + (await this._homeDbManager?.getUser(integerParam(linkParameters.aclAsUserId, 'aclAsUserId'))) : + (await this._homeDbManager?.getExistingUserByLogin(linkParameters.aclAsUser)); // If this is one of example users we will pretend that it doesn't exist, otherwise we would // end up using permissions of the real user. const isExampleUser = this.getExampleViewAsUsers().some(e => e.email === dbUser?.loginEmail); @@ -1905,13 +1904,13 @@ export class GranularAccess implements GranularAccessForBundle { }; } } - const docAuth = userExists ? await this._homeDbManager.getDocAuthCached({ + const docAuth = userExists ? await this._homeDbManager?.getDocAuthCached({ urlId: this._docId, userId: dbUser.id }) : null; const access = docAuth?.access || null; - const user = userExists ? this._homeDbManager.makeFullUser(dbUser) : null; - return { access, user }; + const user = userExists ? this._homeDbManager?.makeFullUser(dbUser) : null; + return { access, user: user || null }; } /** diff --git a/app/server/lib/ICreate.ts b/app/server/lib/ICreate.ts index db523b13..1072133f 100644 --- a/app/server/lib/ICreate.ts +++ b/app/server/lib/ICreate.ts @@ -7,7 +7,8 @@ import {IBilling} from 'app/server/lib/IBilling'; import {INotifier} from 'app/server/lib/INotifier'; import {ISandbox, ISandboxCreationOptions} from 'app/server/lib/ISandbox'; import {IShell} from 'app/server/lib/IShell'; -import {createSandbox} from 'app/server/lib/NSandbox'; +import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox'; +import {SqliteVariant} from 'app/server/lib/SqliteCommon'; export interface ICreate { @@ -31,6 +32,8 @@ export interface ICreate { // static page. getExtraHeadHtml?(): string; getStorageOptions?(name: string): ICreateStorageOptions|undefined; + getSqliteVariant?(): SqliteVariant; + getSandboxVariants?(): Record; } export interface ICreateActiveDocOptions { @@ -62,6 +65,8 @@ export function makeSimpleCreator(opts: { sandboxFlavor?: string, shell?: IShell, getExtraHeadHtml?: () => string, + getSqliteVariant?: () => SqliteVariant, + getSandboxVariants?: () => Record, }): ICreate { const {sessionSecret, storage, notifier, billing} = opts; return { @@ -121,6 +126,8 @@ export function makeSimpleCreator(opts: { }, getStorageOptions(name: string) { return storage?.find(s => s.name === name); - } + }, + getSqliteVariant: opts.getSqliteVariant, + getSandboxVariants: opts.getSandboxVariants, }; } diff --git a/app/server/lib/NSandbox.ts b/app/server/lib/NSandbox.ts index eb219d88..656e8216 100644 --- a/app/server/lib/NSandbox.ts +++ b/app/server/lib/NSandbox.ts @@ -3,6 +3,7 @@ */ import {arrayToString} from 'app/common/arrayToString'; import * as marshal from 'app/common/marshal'; +import {create} from 'app/server/lib/create'; import {ISandbox, ISandboxCreationOptions, ISandboxCreator} from 'app/server/lib/ISandbox'; import log from 'app/server/lib/log'; import {getAppRoot, getAppRootFor, getUnpackedAppRoot} from 'app/server/lib/places'; @@ -69,12 +70,18 @@ export interface ISandboxOptions { * We interact with sandboxes as a separate child process. Data engine work is done * across standard input and output streams from and to this process. We also monitor * and control resource utilization via a distinct control interface. + * + * More recently, a sandbox may not be a separate OS process, but (for + * example) a web worker. In this case, a pair of callbacks (getData and + * sendData) replace pipes. */ -interface SandboxProcess { - child: ChildProcess; +export interface SandboxProcess { + child?: ChildProcess; control: ISandboxControl; dataToSandboxDescriptor?: number; // override sandbox's 'stdin' for data dataFromSandboxDescriptor?: number; // override sandbox's 'stdout' for data + getData?: (cb: (data: any) => void) => void; // use a callback instead of a pipe to get data + sendData?: (data: any) => void; // use a callback instead of a pipe to send data } type ResolveRejectPair = [(value?: any) => void, (reason?: unknown) => void]; @@ -88,7 +95,7 @@ const recordBuffersRoot = process.env.RECORD_SANDBOX_BUFFERS_DIR; export class NSandbox implements ISandbox { - public readonly childProc: ChildProcess; + public readonly childProc?: ChildProcess; private _control: ISandboxControl; private _logTimes: boolean; private _exportedFunctions: {[name: string]: SandboxMethod}; @@ -101,8 +108,9 @@ export class NSandbox implements ISandbox { private _isWriteClosed = false; private _logMeta: log.ILogMeta; - private _streamToSandbox: Writable; + private _streamToSandbox?: Writable; private _streamFromSandbox: Stream; + private _dataToSandbox?: (data: any) => void; private _lastStderr: Uint8Array; // Record last error line seen. // Create a unique subdirectory for each sandbox process so they can be replayed separately @@ -129,52 +137,26 @@ export class NSandbox implements ISandbox { this._control = sandboxProcess.control; this.childProc = sandboxProcess.child; - this._logMeta = {sandboxPid: this.childProc.pid, ...options.logMeta}; + this._logMeta = {sandboxPid: this.childProc?.pid, ...options.logMeta}; - if (options.minimalPipeMode) { - log.rawDebug("3-pipe Sandbox started", this._logMeta); - if (sandboxProcess.dataToSandboxDescriptor) { - this._streamToSandbox = - (this.childProc.stdio as Stream[])[sandboxProcess.dataToSandboxDescriptor] as Writable; - } else { - this._streamToSandbox = this.childProc.stdin!; - } - if (sandboxProcess.dataFromSandboxDescriptor) { - this._streamFromSandbox = - (this.childProc.stdio as Stream[])[sandboxProcess.dataFromSandboxDescriptor]; + if (this.childProc) { + if (options.minimalPipeMode) { + this._initializeMinimalPipeMode(sandboxProcess); } else { - this._streamFromSandbox = this.childProc.stdout!; + this._initializeFivePipeMode(sandboxProcess); } } else { - log.rawDebug("5-pipe Sandbox started", this._logMeta); - if (sandboxProcess.dataFromSandboxDescriptor || sandboxProcess.dataToSandboxDescriptor) { - throw new Error('cannot override file descriptors in 5 pipe mode'); + // No child process. In this case, there should be a callback for + // receiving and sending data. + if (!sandboxProcess.getData) { + throw new Error('no way to get data from sandbox'); } - this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable; - this._streamFromSandbox = (this.childProc.stdio as Stream[])[4]; - this.childProc.stdout!.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta)); - } - const sandboxStderrLogger = sandboxUtil.makeLinePrefixer('Sandbox stderr: ', this._logMeta); - this.childProc.stderr!.on('data', data => { - this._lastStderr = data; - sandboxStderrLogger(data); - }); - - this.childProc.on('close', this._onExit.bind(this)); - this.childProc.on('error', this._onError.bind(this)); - - this._streamFromSandbox.on('data', (data) => this._onSandboxData(data)); - this._streamFromSandbox.on('end', () => this._onSandboxClose()); - this._streamFromSandbox.on('error', (err) => { - log.rawError(`Sandbox error reading: ${err}`, this._logMeta); - this._onSandboxClose(); - }); - - this._streamToSandbox.on('error', (err) => { - if (!this._isWriteClosed) { - log.rawError(`Sandbox error writing: ${err}`, this._logMeta); + if (!sandboxProcess.sendData) { + throw new Error('no way to send data to sandbox'); } - }); + sandboxProcess.getData((data) => this._onSandboxData(data)); + this._dataToSandbox = sandboxProcess.sendData; + } // On shutdown, shutdown the child process cleanly, and wait for it to exit. shutdown.addCleanupHandler(this, this.shutdown); @@ -203,9 +185,9 @@ export class NSandbox implements ISandbox { const result = await new Promise((resolve, reject) => { if (this._isWriteClosed) { resolve(); } - this.childProc.on('error', reject); - this.childProc.on('close', resolve); - this.childProc.on('exit', resolve); + this.childProc?.on('error', reject); + this.childProc?.on('close', resolve); + this.childProc?.on('exit', resolve); this._close(); }).finally(() => this._control.close()); @@ -244,6 +226,82 @@ export class NSandbox implements ISandbox { log.rawDebug('Sandbox memory', {memory, ...this._logMeta}); } + /** + * Get ready to communicate with a sandbox process using stdin, + * stdout, and stderr. + */ + private _initializeMinimalPipeMode(sandboxProcess: SandboxProcess) { + log.rawDebug("3-pipe Sandbox started", this._logMeta); + if (!this.childProc) { + throw new Error('child process required'); + } + if (sandboxProcess.dataToSandboxDescriptor) { + this._streamToSandbox = + (this.childProc.stdio as Stream[])[sandboxProcess.dataToSandboxDescriptor] as Writable; + } else { + this._streamToSandbox = this.childProc.stdin!; + } + if (sandboxProcess.dataFromSandboxDescriptor) { + this._streamFromSandbox = + (this.childProc.stdio as Stream[])[sandboxProcess.dataFromSandboxDescriptor]; + } else { + this._streamFromSandbox = this.childProc.stdout!; + } + this._initializeStreamEvents(); + } + + /** + * Get ready to communicate with a sandbox process using stdin, + * stdout, and stderr, and two extra FDs. This was a nice way + * to have a clean, separate data channel, when supported. + */ + private _initializeFivePipeMode(sandboxProcess: SandboxProcess) { + log.rawDebug("5-pipe Sandbox started", this._logMeta); + if (!this.childProc) { + throw new Error('child process required'); + } + if (sandboxProcess.dataFromSandboxDescriptor || sandboxProcess.dataToSandboxDescriptor) { + throw new Error('cannot override file descriptors in 5 pipe mode'); + } + this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable; + this._streamFromSandbox = (this.childProc.stdio as Stream[])[4]; + this.childProc.stdout!.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta)); + this._initializeStreamEvents(); + } + + /** + * Set up logging and events on streams to/from a sandbox. + */ + private _initializeStreamEvents() { + if (!this.childProc) { + throw new Error('child process required'); + } + if (!this._streamToSandbox) { + throw new Error('expected streamToSandbox to be configured'); + } + const sandboxStderrLogger = sandboxUtil.makeLinePrefixer('Sandbox stderr: ', this._logMeta); + this.childProc.stderr!.on('data', data => { + this._lastStderr = data; + sandboxStderrLogger(data); + }); + + this.childProc.on('close', this._onExit.bind(this)); + this.childProc.on('error', this._onError.bind(this)); + + this._streamFromSandbox.on('data', (data) => this._onSandboxData(data)); + this._streamFromSandbox.on('end', () => this._onSandboxClose()); + this._streamFromSandbox.on('error', (err) => { + log.rawError(`Sandbox error reading: ${err}`, this._logMeta); + this._onSandboxClose(); + }); + + this._streamToSandbox.on('error', (err) => { + if (!this._isWriteClosed) { + log.rawError(`Sandbox error writing: ${err}`, this._logMeta); + } + }); + } + private async _pyCallWait(funcName: string, startTime: number): Promise { try { return await new Promise((resolve, reject) => { @@ -263,7 +321,7 @@ export class NSandbox implements ISandbox { this._control.prepareToClose(); if (!this._isWriteClosed) { // Close the pipe to the sandbox, which should cause the sandbox to exit cleanly. - this._streamToSandbox.end(); + this._streamToSandbox?.end(); this._isWriteClosed = true; } } @@ -298,10 +356,17 @@ export class NSandbox implements ISandbox { if (this._recordBuffersDir) { fs.appendFileSync(path.resolve(this._recordBuffersDir, "input"), buf); } - return this._streamToSandbox.write(buf); + if (this._streamToSandbox) { + return this._streamToSandbox.write(buf); + } else { + if (!this._dataToSandbox) { + throw new Error('no way to send data to sandbox'); + } + this._dataToSandbox(buf); + return true; + } } - /** * Process a buffer of data received from the sandbox process. */ @@ -422,18 +487,26 @@ function isFlavor(flavor: string): flavor is keyof typeof spawners { * It is ignored by other flavors. */ export class NSandboxCreator implements ISandboxCreator { - private _flavor: keyof typeof spawners; + private _flavor: string; + private _spawner: SpawnFn; private _command?: string; private _preferredPythonVersion?: string; public constructor(options: { - defaultFlavor: keyof typeof spawners, + defaultFlavor: string, command?: string, preferredPythonVersion?: string, }) { const flavor = options.defaultFlavor; if (!isFlavor(flavor)) { - throw new Error(`Unrecognized sandbox flavor: ${flavor}`); + const variants = create.getSandboxVariants?.(); + if (!variants?.[flavor]) { + throw new Error(`Unrecognized sandbox flavor: ${flavor}`); + } else { + this._spawner = variants[flavor]; + } + } else { + this._spawner = spawners[flavor]; } this._flavor = flavor; this._command = options.command; @@ -463,12 +536,12 @@ export class NSandboxCreator implements ISandboxCreator { importDir: options.importMount, ...options.sandboxOptions, }; - return new NSandbox(translatedOptions, spawners[this._flavor]); + return new NSandbox(translatedOptions, this._spawner); } } // A function that takes sandbox options and starts a sandbox process. -type SpawnFn = (options: ISandboxOptions) => SandboxProcess; +export type SpawnFn = (options: ISandboxOptions) => SandboxProcess; /** * Helper function to run a nacl sandbox. It takes care of most arguments, similarly to @@ -750,7 +823,7 @@ function macSandboxExec(options: ISandboxOptions): SandboxProcess { ...getWrappingEnv(options), }; const command = findPython(options.command, options.preferredPythonVersion); - const realPath = fs.realpathSync(command); + const realPath = realpathSync(command); log.rawDebug("macSandboxExec found a python", {...options.logMeta, command: realPath}); // Prepare sandbox profile @@ -868,11 +941,11 @@ function getAbsolutePaths(options: ISandboxOptions) { // Get path to sandbox directory - this is a little idiosyncratic to work well // in grist-core. It is important to use real paths since we may be viewing // the file system through a narrow window in a container. - const sandboxDir = path.join(fs.realpathSync(path.join(process.cwd(), 'sandbox', 'grist')), + const sandboxDir = path.join(realpathSync(path.join(process.cwd(), 'sandbox', 'grist')), '..'); // Copy plugin options, and then make them absolute. if (options.importDir) { - options.importDir = fs.realpathSync(options.importDir); + options.importDir = realpathSync(options.importDir); } return { sandboxDir, @@ -976,9 +1049,6 @@ export function createSandbox(defaultFlavorSpec: string, options: ISandboxCreati const flavor = parts[parts.length - 1]; const version = parts.length === 2 ? parts[0] : '*'; if (preferredPythonVersion === version || version === '*' || !preferredPythonVersion) { - if (!isFlavor(flavor)) { - throw new Error(`Unrecognized sandbox flavor: ${flavor}`); - } const creator = new NSandboxCreator({ defaultFlavor: flavor, command: process.env['GRIST_SANDBOX' + (preferredPythonVersion||'')] || @@ -990,3 +1060,16 @@ export function createSandbox(defaultFlavorSpec: string, options: ISandboxCreati } throw new Error('Failed to create a sandbox'); } + +/** + * The realpath function may not be available, just return the + * path unchanged if it is not. Specifically, this happens when + * compiled for use in a browser environment. + */ +function realpathSync(src: string) { + try { + return fs.realpathSync(src); + } catch (e) { + return src; + } +} diff --git a/app/server/lib/SQLiteDB.ts b/app/server/lib/SQLiteDB.ts index 10e96a71..9551309f 100644 --- a/app/server/lib/SQLiteDB.ts +++ b/app/server/lib/SQLiteDB.ts @@ -69,23 +69,24 @@ import {ErrorWithCode} from 'app/common/ErrorWithCode'; import {timeFormat} from 'app/common/timeFormat'; +import {create} from 'app/server/lib/create'; import * as docUtils from 'app/server/lib/docUtils'; import log from 'app/server/lib/log'; -import {fromCallback} from 'app/server/lib/serverUtils'; - -import * as sqlite3 from '@gristlabs/sqlite3'; +import {MinDB, MinRunResult, PreparedStatement, ResultRow, + SqliteVariant, Statement} from 'app/server/lib/SqliteCommon'; +import {NodeSqliteVariant} from 'app/server/lib/SqliteNode'; import assert from 'assert'; -import {each} from 'bluebird'; import * as fse from 'fs-extra'; -import {RunResult} from 'sqlite3'; import fromPairs = require('lodash/fromPairs'); import isEqual = require('lodash/isEqual'); import noop = require('lodash/noop'); import range = require('lodash/range'); -// Describes the result of get() and all() database methods. -export interface ResultRow { - [column: string]: any; +export type {PreparedStatement, ResultRow, Statement}; +export type RunResult = MinRunResult; + +function getVariant(): SqliteVariant { + return create.getSqliteVariant?.() || new NodeSqliteVariant(); } // Describes how to create a new DB or migrate an old one. Any changes to the DB must be reflected @@ -136,7 +137,7 @@ export interface ISQLiteDB { run(sql: string, ...params: any[]): Promise; get(sql: string, ...params: any[]): Promise; all(sql: string, ...params: any[]): Promise; - prepare(sql: string, ...params: any[]): Promise; + prepare(sql: string, ...params: any[]): Promise; execTransaction(callback: () => Promise): Promise; runAndGetId(sql: string, ...params: any[]): Promise; requestVacuum(): Promise; @@ -196,18 +197,11 @@ export class SQLiteDB implements ISQLiteDB { */ public static async openDBRaw(dbPath: string, mode: OpenMode = OpenMode.OPEN_CREATE): Promise { - const sqliteMode: number = - // tslint:disable-next-line:no-bitwise - (mode === OpenMode.OPEN_READONLY ? sqlite3.OPEN_READONLY : sqlite3.OPEN_READWRITE) | - (mode === OpenMode.OPEN_CREATE || mode === OpenMode.CREATE_EXCL ? sqlite3.OPEN_CREATE : 0); - - let _db: sqlite3.Database; - await fromCallback(cb => { _db = new sqlite3.Database(dbPath, sqliteMode, cb); }); - limitAttach(_db!, 0); // Outside of VACUUM, we don't allow ATTACH. + const minDb: MinDB = await getVariant().opener(dbPath, mode); if (SQLiteDB._addOpens(dbPath, 1) > 1) { log.warn("SQLiteDB[%s] avoid opening same DB more than once", dbPath); } - return new SQLiteDB(_db!, dbPath); + return new SQLiteDB(minDb, dbPath); } /** @@ -261,12 +255,29 @@ export class SQLiteDB implements ISQLiteDB { private _migrationError: Error|null = null; private _needVacuum: boolean = false; - private constructor(private _db: sqlite3.Database, private _dbPath: string) { - // Default database to serialized execution. See https://github.com/mapbox/node-sqlite3/wiki/Control-Flow - // This isn't enough for transactions, which we serialize explicitly. - this._db.serialize(); + private constructor(protected _db: MinDB, private _dbPath: string) { + } + + public async all(sql: string, ...args: any[]): Promise { + const result = await this._db.all(sql, ...args); + return result; + } + + public run(sql: string, ...args: any[]): Promise { + return this._db.run(sql, ...args); } + public exec(sql: string): Promise { + return this._db.exec(sql); + } + + public prepare(sql: string): Promise { + return this._db.prepare(sql); + } + + public get(sql: string, ...args: any[]): Promise { + return this._db.get(sql, ...args); + } /** * If a DB was migrated on open, this will be set to the path of the pre-migration backup copy. @@ -285,40 +296,8 @@ export class SQLiteDB implements ISQLiteDB { // The following methods mirror https://github.com/mapbox/node-sqlite3/wiki/API, but return // Promises. We use fromCallback() rather than use promisify, to get better type-checking. - public exec(sql: string): Promise { - return fromCallback(cb => this._db.exec(sql, cb)); - } - - public run(sql: string, ...params: any[]): Promise { - return new Promise((resolve, reject) => { - function callback(this: RunResult, err: Error | null) { - if (err) { - reject(err); - } else { - resolve(this); - } - } - this._db.run(sql, ...params, callback); - }); - } - - public get(sql: string, ...params: any[]): Promise { - return fromCallback(cb => this._db.get(sql, ...params, cb)); - } - - public all(sql: string, ...params: any[]): Promise { - return fromCallback(cb => this._db.all(sql, ...params, cb)); - } - - public allMarshal(sql: string, ...params: any[]): Promise { - // allMarshal isn't in the typings, because it is our addition to our fork of sqlite3 JS lib. - return fromCallback(cb => (this._db as any).allMarshal(sql, ...params, cb)); - } - - public prepare(sql: string, ...params: any[]): Promise { - let stmt: sqlite3.Statement; - // The original interface is a little strange; we resolve to Statement if prepare() succeeded. - return fromCallback(cb => { stmt = this._db.prepare(sql, ...params, cb); }).then(() => stmt); + public async allMarshal(sql: string, ...params: any[]): Promise { + return this._db.allMarshal(sql, ...params); } /** @@ -336,11 +315,11 @@ export class SQLiteDB implements ISQLiteDB { } public async vacuum(): Promise { - limitAttach(this._db, 1); // VACUUM implementation uses ATTACH. + await this._db.limitAttach(1); // VACUUM implementation uses ATTACH. try { await this.exec("VACUUM"); } finally { - limitAttach(this._db, 0); // Outside of VACUUM, we don't allow ATTACH. + await this._db.limitAttach(0); // Outside of VACUUM, we don't allow ATTACH. } } @@ -348,25 +327,24 @@ export class SQLiteDB implements ISQLiteDB { * Run each of the statements in turn. Each statement is either a string, or an array of arguments * to db.run, e.g. [sqlString, [params...]]. */ - public runEach(...statements: Array): Promise { - return each(statements, - async (stmt: any) => { - try { - return await (Array.isArray(stmt) ? - this.run(stmt[0], ...stmt[1]) : - this.exec(stmt) - ); - } catch (err) { - log.warn(`SQLiteDB: Failed to run ${stmt}`); - throw err; + public async runEach(...statements: Array): Promise { + for (const stmt of statements) { + try { + if (Array.isArray(stmt)) { + await this.run(stmt[0], ...stmt[1]); + } else { + await this.exec(stmt); } + } catch (err) { + log.warn(`SQLiteDB: Failed to run ${stmt}`); + throw err; } - ); + } } - public close(): Promise { - return fromCallback(cb => this._db.close(cb)) - .then(() => { SQLiteDB._addOpens(this._dbPath, -1); }); + public async close(): Promise { + await this._db.close(); + SQLiteDB._addOpens(this._dbPath, -1); } /** @@ -375,8 +353,7 @@ export class SQLiteDB implements ISQLiteDB { * is only useful if the sql is actually an INSERT operation, but we don't check this. */ public async runAndGetId(sql: string, ...params: any[]): Promise { - const result = await this.run(sql, ...params); - return result.lastID; + return this._db.runAndGetId(sql, ...params); } /** @@ -567,12 +544,3 @@ export function quoteIdent(ident: string): string { assert(/^[\w.]+$/.test(ident), `SQL identifier is not valid: ${ident}`); return `"${ident}"`; } - -/** - * Limit the number of ATTACHed databases permitted. - */ -export function limitAttach(db: sqlite3.Database, maxAttach: number) { - // Pardon the casts, types are out of date. - const SQLITE_LIMIT_ATTACHED = (sqlite3 as any).LIMIT_ATTACHED; - (db as any).configure('limit', SQLITE_LIMIT_ATTACHED, maxAttach); -} diff --git a/app/server/lib/SqliteCommon.ts b/app/server/lib/SqliteCommon.ts new file mode 100644 index 00000000..4224a55b --- /dev/null +++ b/app/server/lib/SqliteCommon.ts @@ -0,0 +1,126 @@ +import { Marshaller } from 'app/common/marshal'; +import { OpenMode, quoteIdent } from 'app/server/lib/SQLiteDB'; + +/** + * Code common to SQLite wrappers. + */ + +/** + * It is important that Statement exists - but we don't expect + * anything of it. + */ +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface Statement {} + +export interface MinDB { + exec(sql: string): Promise; + run(sql: string, ...params: any[]): Promise; + get(sql: string, ...params: any[]): Promise; + all(sql: string, ...params: any[]): Promise; + prepare(sql: string, ...params: any[]): Promise; + runAndGetId(sql: string, ...params: any[]): Promise; + close(): Promise; + allMarshal(sql: string, ...params: any[]): Promise; + + /** + * Limit the number of ATTACHed databases permitted. + */ + limitAttach(maxAttach: number): Promise; +} + +export interface MinRunResult { + changes: number; +} + +// Describes the result of get() and all() database methods. +export interface ResultRow { + [column: string]: any; +} + +export interface PreparedStatement { + run(...params: any[]): Promise; + finalize(): Promise; + columns(): string[]; +} + +export interface SqliteVariant { + opener(dbPath: string, mode: OpenMode): Promise; +} + +/** + * A crude implementation of Grist marshalling. + * There is a fork of node-sqlite3 that has Grist + * marshalling built in, at: + * https://github.com/gristlabs/node-sqlite3 + * If using a version of SQLite without this built + * in, another option is to add custom functions + * to do it. This object has the initialize, step, + * and finalize callbacks typically needed to add + * a custom aggregration function. + */ +export const gristMarshal = { + initialize(): GristMarshalIntermediateValue { + return {}; + }, + step(accum: GristMarshalIntermediateValue, ...row: any[]) { + if (!accum.names || !accum.values) { + accum.names = row.map(value => String(value)); + accum.values = row.map(() => []); + } else { + for (const [i, v] of row.entries()) { + accum.values[i].push(v); + } + } + return accum; + }, + finalize(accum: GristMarshalIntermediateValue) { + const marshaller = new Marshaller({version: 2, keysAreBuffers: true}); + const result: Record> = {}; + if (accum.names && accum.values) { + for (const [i, name] of accum.names.entries()) { + result[name] = accum.values[i]; + } + } + marshaller.marshal(result); + return marshaller.dumpAsBuffer(); + } +}; + +/** + * An intermediate value used during an aggregation. + */ +interface GristMarshalIntermediateValue { + // The names of the columns, once known. + names?: string[]; + // Values stored in the columns. + // There is one element in the outermost array per column. + // That element contains a list of values stored in that column. + values?: Array>; +} + +/** + * Run Grist marshalling as a SQLite query, assuming + * a custom aggregation has been added as "grist_marshal". + * The marshalled result needs to contain the column + * identifiers embedded in it. This is a little awkward + * to organize - hence the hacky UNION here. This is + * for compatibility with the existing marshalling method, + * which could be replaced instead. + */ +export async function allMarshalQuery(db: MinDB, sql: string, ...params: any[]): Promise { + const statement = await db.prepare(sql); + const columns = statement.columns(); + const quotedColumnList = columns.map(quoteIdent).join(','); + const query = await db.all(`select grist_marshal(${quotedColumnList}) as buf FROM ` + + `(select ${quotedColumnList} UNION ALL select * from (` + sql + '))', ..._fixParameters(params)); + return query[0].buf; +} + +/** + * Booleans need to be cast to 1 or 0 for SQLite. + * The node-sqlite3 wrapper does this automatically, but other + * wrappers do not. + */ +function _fixParameters(params: any[]) { + return params.map(p => p === true ? 1 : (p === false ? 0 : p)); +} diff --git a/app/server/lib/SqliteNode.ts b/app/server/lib/SqliteNode.ts new file mode 100644 index 00000000..dd9858d8 --- /dev/null +++ b/app/server/lib/SqliteNode.ts @@ -0,0 +1,104 @@ +import * as sqlite3 from '@gristlabs/sqlite3'; +import { fromCallback } from 'app/server/lib/serverUtils'; +import { MinDB, PreparedStatement, ResultRow, SqliteVariant } from 'app/server/lib/SqliteCommon'; +import { OpenMode, RunResult } from 'app/server/lib/SQLiteDB'; + +export class NodeSqliteVariant implements SqliteVariant { + public opener(dbPath: string, mode: OpenMode): Promise { + return NodeSqlite3DatabaseAdapter.opener(dbPath, mode); + } +} + +export class NodeSqlite3PreparedStatement implements PreparedStatement { + public constructor(private _statement: sqlite3.Statement) { + } + + public async run(...params: any[]): Promise { + return fromCallback(cb => this._statement.run(...params, cb)); + } + + public async finalize() { + await fromCallback(cb => this._statement.finalize(cb)); + } + + public columns(): string[] { + // This method is only needed if marshalling is not built in - + // and node-sqlite3 has marshalling built in. + throw new Error('not available (but should not be needed)'); + } +} + +export class NodeSqlite3DatabaseAdapter implements MinDB { + public static async opener(dbPath: string, mode: OpenMode): Promise { + const sqliteMode: number = + // tslint:disable-next-line:no-bitwise + (mode === OpenMode.OPEN_READONLY ? sqlite3.OPEN_READONLY : sqlite3.OPEN_READWRITE) | + (mode === OpenMode.OPEN_CREATE || mode === OpenMode.CREATE_EXCL ? sqlite3.OPEN_CREATE : 0); + let _db: sqlite3.Database; + await fromCallback(cb => { _db = new sqlite3.Database(dbPath, sqliteMode, cb); }); + const result = new NodeSqlite3DatabaseAdapter(_db!); + await result.limitAttach(0); // Outside of VACUUM, we don't allow ATTACH. + return result; + } + + public constructor(protected _db: sqlite3.Database) { + // Default database to serialized execution. See https://github.com/mapbox/node-sqlite3/wiki/Control-Flow + // This isn't enough for transactions, which we serialize explicitly. + this._db.serialize(); + } + + public async exec(sql: string): Promise { + return fromCallback(cb => this._db.exec(sql, cb)); + } + + public async run(sql: string, ...params: any[]): Promise { + return new Promise((resolve, reject) => { + function callback(this: RunResult, err: Error | null) { + if (err) { + reject(err); + } else { + resolve(this); + } + } + this._db.run(sql, ...params, callback); + }); + } + + public async get(sql: string, ...params: any[]): Promise { + return fromCallback(cb => this._db.get(sql, ...params, cb)); + } + + public async all(sql: string, ...params: any[]): Promise { + return fromCallback(cb => this._db.all(sql, params, cb)); + } + + public async prepare(sql: string): Promise { + let stmt: sqlite3.Statement|undefined; + // The original interface is a little strange; we resolve to Statement if prepare() succeeded. + await fromCallback(cb => { stmt = this._db.prepare(sql, cb); }).then(() => stmt); + if (!stmt) { throw new Error('could not prepare statement'); } + return new NodeSqlite3PreparedStatement(stmt); + } + + public async close() { + this._db.close(); + } + + public async allMarshal(sql: string, ...params: any[]): Promise { + // allMarshal isn't in the typings, because it is our addition to our fork of sqlite3 JS lib. + return fromCallback(cb => (this._db as any).allMarshal(sql, ...params, cb)); + + } + + public async runAndGetId(sql: string, ...params: any[]): Promise { + const result = await this.run(sql, ...params); + return (result as any).lastID; + } + + public async limitAttach(maxAttach: number) { + const SQLITE_LIMIT_ATTACHED = (sqlite3 as any).LIMIT_ATTACHED; + // Cast because types out of date. + (this._db as any).configure('limit', SQLITE_LIMIT_ATTACHED, maxAttach); + } +} + diff --git a/app/server/lib/WorkCoordinator.ts b/app/server/lib/WorkCoordinator.ts index d0cefcde..fd22af3f 100644 --- a/app/server/lib/WorkCoordinator.ts +++ b/app/server/lib/WorkCoordinator.ts @@ -60,7 +60,12 @@ export class WorkCoordinator { private _maybeSchedule() { if (this._isStepScheduled && !this._isStepRunning) { - setImmediate(this._tryNextStepCB); + try { + setImmediate(this._tryNextStepCB); + } catch (e) { + // setImmediate may not be available outside node. + setTimeout(this._tryNextStepCB, 0); + } } } } diff --git a/app/server/lib/requestUtils.ts b/app/server/lib/requestUtils.ts index 66e26f1b..cee0def6 100644 --- a/app/server/lib/requestUtils.ts +++ b/app/server/lib/requestUtils.ts @@ -8,7 +8,6 @@ import {RequestWithGrist} from 'app/server/lib/GristServer'; import log from 'app/server/lib/log'; import {Permit} from 'app/server/lib/Permit'; import {Request, Response} from 'express'; -import {URL} from 'url'; import _ from 'lodash'; // log api details outside of dev environment (when GRIST_HOSTED_VERSION is set) diff --git a/app/server/lib/shutdown.js b/app/server/lib/shutdown.js index 5ad9a5d1..4133580b 100644 --- a/app/server/lib/shutdown.js +++ b/app/server/lib/shutdown.js @@ -3,8 +3,8 @@ */ +var log = require('app/server/lib/log'); var Promise = require('bluebird'); -var log = require('./log'); var cleanupHandlers = []; diff --git a/buildtools/build.sh b/buildtools/build.sh index 38c1b69d..0571eaa5 100755 --- a/buildtools/build.sh +++ b/buildtools/build.sh @@ -7,11 +7,17 @@ export GRIST_EXT=stubs if [[ -e ext/app ]]; then PROJECT="tsconfig-ext.json" fi +WEBPACK_CONFIG=buildtools/webpack.config.js +if [[ -e ext/buildtools/webpack.config.js ]]; then + # Allow webpack config file to be replaced (useful + # for grist-static) + WEBPACK_CONFIG=ext/buildtools/webpack.config.js +fi set -x tsc --build $PROJECT buildtools/update_type_info.sh app -webpack --config buildtools/webpack.config.js --mode production +webpack --config $WEBPACK_CONFIG --mode production webpack --config buildtools/webpack.check.js --mode production webpack --config buildtools/webpack.api.config.js --mode production cat app/client/*.css app/client/*/*.css > static/bundle.css diff --git a/sandbox/MANIFEST.in b/sandbox/MANIFEST.in new file mode 100644 index 00000000..5155b1f4 --- /dev/null +++ b/sandbox/MANIFEST.in @@ -0,0 +1,3 @@ +# see bundle_as_wheel.sh + +include grist/tzdata.data diff --git a/sandbox/bundle_as_wheel.sh b/sandbox/bundle_as_wheel.sh new file mode 100755 index 00000000..289f8163 --- /dev/null +++ b/sandbox/bundle_as_wheel.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Package up Grist code as a stand-alone wheel. +# This is useful for grist-static. +# It is the reason why MANIFEST.in and setup.py are present. + +set -e + +# Clean up any previous packaging. +rm -rf dist foo.egg-info grist.egg-info build + +# Go ahead and run packaging again. +python setup.py bdist_wheel + +echo "" +echo "Result is in the dist directory:" +ls dist diff --git a/sandbox/setup.py b/sandbox/setup.py new file mode 100644 index 00000000..935c9d43 --- /dev/null +++ b/sandbox/setup.py @@ -0,0 +1,15 @@ +# see bundle_as_wheel.sh + +from distutils.core import setup +import glob + +files = glob.glob('grist/*.py') + glob.glob('grist/**/*.py') +names = [f.split('.py')[0] for f in files] + +setup(name='grist', + version='1.0', + include_package_data=True, + packages=['grist', 'grist/functions', 'grist/imports'], + package_data={ + 'grist': ['grist/tzdata.data'], + }) diff --git a/sandbox/watch.sh b/sandbox/watch.sh index f314b537..d8b8d642 100755 --- a/sandbox/watch.sh +++ b/sandbox/watch.sh @@ -7,13 +7,17 @@ export GRIST_EXT=stubs if [[ -e ext/app ]]; then PROJECT="tsconfig-ext.json" fi +WEBPACK_CONFIG=buildtools/webpack.config.js +if [[ -e ext/buildtools/webpack.config.js ]]; then + WEBPACK_CONFIG=ext/buildtools/webpack.config.js +fi if [ ! -e _build ]; then buildtools/build.sh fi tsc --build -w --preserveWatchOutput $PROJECT & -catw app/client/*.css app/client/*/*.css -o static/bundle.css -v & webpack --config buildtools/webpack.config.js --mode development --watch & +catw app/client/*.css app/client/*/*.css -o static/bundle.css -v & webpack --config $WEBPACK_CONFIG --mode development --watch & NODE_PATH=_build:_build/stubs:_build/ext nodemon --delay 1 -w _build/app/server -w _build/app/common _build/stubs/app/server/server.js & wait diff --git a/test/server/lib/HostedStorageManager.ts b/test/server/lib/HostedStorageManager.ts index 5d1cf2d9..0c01d638 100644 --- a/test/server/lib/HostedStorageManager.ts +++ b/test/server/lib/HostedStorageManager.ts @@ -15,7 +15,6 @@ import { HostedStorageOptions } from 'app/server/lib/HostedStorageManager'; import log from 'app/server/lib/log'; -import {fromCallback} from 'app/server/lib/serverUtils'; import {SQLiteDB} from 'app/server/lib/SQLiteDB'; import * as bluebird from 'bluebird'; import {assert} from 'chai'; @@ -931,9 +930,9 @@ describe('backupSqliteDatabase', async function() { // Silly code to make a long random string to insert. // We can make a big db faster this way. const str = (new Array(100)).fill(1).map((_: any) => Math.random().toString(2)).join(); - stmt.run(str, str, str); + await stmt.run(str, str, str); } - await fromCallback(cb => stmt.finalize(cb)); + await stmt.finalize(); }); const stat = await fse.stat(src); assert(stat.size > 150 * 1000 * 1000);