diff --git a/app/gen-server/migration/1536634251710-Initial.ts b/app/gen-server/migration/1536634251710-Initial.ts index 7d88a87f..34c2bb42 100644 --- a/app/gen-server/migration/1536634251710-Initial.ts +++ b/app/gen-server/migration/1536634251710-Initial.ts @@ -1,3 +1,4 @@ +import * as sqlUtils from "app/gen-server/sqlUtils"; import {MigrationInterface, QueryRunner, Table} from "typeorm"; @@ -6,9 +7,9 @@ export class Initial1536634251710 implements MigrationInterface { // TypeORM doesn't currently help with types of created tables: // https://github.com/typeorm/typeorm/issues/305 // so we need to do a little smoothing over postgres and sqlite. - const sqlite = queryRunner.connection.driver.options.type === 'sqlite'; - const datetime = sqlite ? "datetime" : "timestamp with time zone"; - const now = "now()"; + const dbType = queryRunner.connection.driver.options.type; + const datetime = sqlUtils.datetime(dbType); + const now = sqlUtils.now(dbType); await queryRunner.createTable(new Table({ name: "users", diff --git a/app/gen-server/migration/1652273656610-Activations.ts b/app/gen-server/migration/1652273656610-Activations.ts index d2dded1c..66eea428 100644 --- a/app/gen-server/migration/1652273656610-Activations.ts +++ b/app/gen-server/migration/1652273656610-Activations.ts @@ -1,12 +1,13 @@ +import * as sqlUtils from "app/gen-server/sqlUtils"; import {MigrationInterface, QueryRunner, Table} from 'typeorm'; export class Activations1652273656610 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { // created_at and updated_at code is based on *-Initial.ts - const sqlite = queryRunner.connection.driver.options.type === 'sqlite'; - const datetime = sqlite ? "datetime" : "timestamp with time zone"; - const now = "now()"; + const dbType = queryRunner.connection.driver.options.type; + const datetime = sqlUtils.datetime(dbType); + const now = sqlUtils.now(dbType); await queryRunner.createTable(new Table({ name: 'activations', columns: [ diff --git a/app/server/companion.ts b/app/server/companion.ts new file mode 100644 index 00000000..0e82ced1 --- /dev/null +++ b/app/server/companion.ts @@ -0,0 +1,239 @@ +import { synchronizeProducts } from 'app/gen-server/entity/Product'; +import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager'; +import { applyPatch } from 'app/gen-server/lib/TypeORMPatches'; +import { getMigrations, getOrCreateConnection, undoLastMigration, updateDb } from 'app/server/lib/dbUtils'; +import { getDatabaseUrl } from 'app/server/lib/serverUtils'; +import { Gristifier } from 'app/server/utils/gristify'; +import { pruneActionHistory } from 'app/server/utils/pruneActionHistory'; +import * as commander from 'commander'; +import { Connection, getConnectionOptions } from 'typeorm'; + +/** + * Main entrypoint for a cli toolbox for configuring aspects of Grist + * and Grist documents. + */ +async function main() { + // Tweak TypeORM support of SQLite a little bit to support transactions. + applyPatch(); + const program = getProgram(); + await program.parseAsync(process.argv); +} + +if (require.main === module) { + main().then(() => process.exit(0)).catch(e => { + // tslint:disable-next-line:no-console + console.error(e); + process.exit(1); + }); +} + +/** + * Get the Grist companion client program as a commander object. + * To actually run it, call parseAsync(argv), optionally after + * adding any other commands that may be available. + */ +export function getProgram(): commander.Command { + const program = commander.program; + program + .name('grist-toolbox') // haven't really settled on a name yet. + // want to reserve "grist" for electron app? + .description('a toolbox of handy Grist-related utilities'); + + addDbCommand(program, {nested: true}); + addHistoryCommand(program, {nested: true}); + addSiteCommand(program, {nested: true}); + addSqliteCommand(program); + return program; +} + +// Add commands related to document history: +// history prune [N] +export function addHistoryCommand(program: commander.Command, options: CommandOptions) { + const sub = section(program, { + sectionName: 'history', + sectionDescription: 'fiddle with history of a Grist document', + ...options, + }); + sub('prune ') + .description('remove all but last N actions from doc') + .argument('[N]', 'number of actions to keep', parseIntForCommander, 1) + .action(pruneActionHistory); +} + +// Add commands related to sites: +// site create +export function addSiteCommand(program: commander.Command, + options: CommandOptions) { + const sub = section(program, { + sectionName: 'site', + sectionDescription: 'set up sites', + ...options + }); + sub('create ') + .description('create a site') + .action(async (domain, email) => { + console.log("create a site"); + const profile = {email, name: email}; + const db = await getHomeDBManager(); + const user = await db.getUserByLogin(email, {profile}); + if (!user) { + // This should not happen. + throw new Error('failed to create user'); + } + await db.addOrg(user, { + name: domain, + domain, + }, { + setUserAsOwner: false, + useNewPlan: true, + planType: 'teamFree' + }); + }); +} + +// Add commands related to home/landing database: +// db migrate +// db revert +// db check +// db url +export function addDbCommand(program: commander.Command, + options: CommandOptions, + reuseConnection?: Connection) { + function withConnection(op: (connection: Connection) => Promise) { + return async () => { + if (!process.env.TYPEORM_LOGGING) { + process.env.TYPEORM_LOGGING = 'true'; + } + const connection = reuseConnection || await getOrCreateConnection(); + const exitCode = await op(connection); + if (exitCode !== 0) { + program.error('db command failed', {exitCode}); + } + }; + } + const sub = section(program, { + sectionName: 'db', + sectionDescription: 'maintain the database of users, sites, workspaces, and docs', + ...options, + }); + + sub('migrate') + .description('run all pending migrations on database') + .action(withConnection(async (connection) => { + await updateDb(connection); + return 0; + })); + + sub('revert') + .description('revert last migration on database') + .action(withConnection(async (connection) => { + await undoLastMigration(connection); + return 0; + })); + + sub('check') + .description('check that there are no pending migrations on database') + .action(withConnection(dbCheck)); + + sub('url') + .description('construct a url for the database (for psql, catsql etc)') + .action(withConnection(async () => { + console.log(getDatabaseUrl(await getConnectionOptions(), true)); + return 0; + })); +} + +// Add command related to sqlite: +// sqlite gristify +// sqlite clean +export function addSqliteCommand(program: commander.Command) { + const sub = program.command('sqlite') + .description('commands for accessing sqlite files'); + + sub.command('gristify ') + .description('add grist metadata to an sqlite file') + .option('--add-sort', 'add a manualSort column, important for adding/removing rows') + .action((filename, options) => new Gristifier(filename).gristify(options)); + + sub.command('clean ') + .description('remove grist metadata from an sqlite file') + .action(filename => new Gristifier(filename).degristify()); +} + +// Report the status of the database. Migrations appied, migrations pending, +// product information applied, product changes pending. +export async function dbCheck(connection: Connection) { + const migrations = await getMigrations(connection); + const changingProducts = await synchronizeProducts(connection, false); + // eslint-disable-next-line @typescript-eslint/no-shadow + const log = process.env.TYPEORM_LOGGING === 'true' ? console.log : (...args: any[]) => null; + const options = await getConnectionOptions(); + log("database url:", getDatabaseUrl(options, false)); + log("migration files:", options.migrations); + log("migration directory:", (options.cli && options.cli.migrationsDir) || 'unspecified'); + log("migrations applied to db:", migrations.migrationsInDb); + log("migrations listed in code:", migrations.migrationsInCode); + let exitCode: number = 0; + if (migrations.pendingMigrations.length) { + log(`Migration(s) need to be applied: ${migrations.pendingMigrations}`); + exitCode = 1; + } else { + log("No migrations need to be applied"); + } + log(""); + if (changingProducts.length) { + log("Products need updating:", changingProducts); + log(` (to revert a product change, run an older version of the code)`); + log(` (db:revert will not undo product changes)`); + exitCode = 1; + } else { + log(`Products unchanged`); + } + return exitCode; +} + +// Get an interface to the home db. +export async function getHomeDBManager() { + const dbManager = new HomeDBManager(); + await dbManager.connect(); + await dbManager.initializeSpecialIds(); + return dbManager; +} + +// Get a function for adding a command to a section of related commands. +// There is a "nested" option that uses commander's nested command feature. +// Older cli code may use an older unnested style. +function section(program: commander.Command, options: { + sectionName: string, + sectionDescription: string, + nested: boolean +}) { + // If unnested, we'll return a function that adds commands directly to the + // program (section description is ignored in this case). If nested, we make + // a command to represent the section, and return a function that adds to that. + const sub = options.nested ? + program.command(options.sectionName).description(options.sectionDescription) : + program; + return (name: string) => { + if (options.nested) { + return sub.command(name); + } else { + return sub.command(`${options.sectionName}:${name}`); + } + }; +} + +// Options for command style. +export interface CommandOptions { + nested: boolean, + sectionName?: string, +} + +// This is based on the recommended way to parse integers for commander. +export function parseIntForCommander(value: string, prev: number) { + const pvalue = parseInt(value, 10); + if (isNaN(pvalue)) { + throw new Error('Not a number.'); + } + return pvalue; +} diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 22347b33..a2939d88 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -33,6 +33,7 @@ import { BulkUpdateRecord, CellValue, DocAction, + getTableId, TableDataAction, TableRecordValue, toTableDataAction, @@ -170,7 +171,7 @@ export class ActiveDoc extends EventEmitter { } public readonly docStorage: DocStorage; - public readonly docPluginManager: DocPluginManager; + public readonly docPluginManager: DocPluginManager|null; public readonly docClients: DocClients; // Only exposed for Sharing.ts public docData: DocData|null = null; @@ -204,6 +205,7 @@ export class ActiveDoc extends EventEmitter { private _product?: Product; private _gracePeriodStart: Date|null = null; private _isForkOrSnapshot: boolean = false; + private _onlyAllowMetaDataActionsOnDb: boolean = false; // Client watching for 'product changed' event published by Billing to update usage private _redisSubscriber?: RedisClient; @@ -276,8 +278,9 @@ export class ActiveDoc extends EventEmitter { this._triggers = new DocTriggers(this); this._requests = new DocRequests(this); this._actionHistory = new ActionHistoryImpl(this.docStorage); - this.docPluginManager = new DocPluginManager(docManager.pluginManager.getPlugins(), - docManager.pluginManager.appRoot!, this, this._docManager.gristServer); + this.docPluginManager = docManager.pluginManager ? + new DocPluginManager(docManager.pluginManager.getPlugins(), + docManager.pluginManager.appRoot!, this, this._docManager.gristServer) : null; this._tableMetadataLoader = new TableMetadataLoader({ decodeBuffer: this.docStorage.decodeMarshalledData.bind(this.docStorage), fetchTable: this.docStorage.fetchTable.bind(this.docStorage), @@ -529,7 +532,7 @@ export class ActiveDoc extends EventEmitter { } await Promise.all([ this.docStorage.shutdown(), - this.docPluginManager.shutdown(), + this.docPluginManager?.shutdown(), dataEngine?.shutdown() ]); // The this.waitForInitialization promise may not yet have resolved, but @@ -576,7 +579,7 @@ export class ActiveDoc extends EventEmitter { await this._initDoc(docSession); await this._tableMetadataLoader.clean(); // Makes sure docPluginManager is ready in case new doc is used to import new data - await this.docPluginManager.ready; + await this.docPluginManager?.ready; this._fullyLoaded = true; return this; } @@ -585,10 +588,13 @@ export class ActiveDoc extends EventEmitter { * Create a new blank document (no "Table1"), used as a stub when importing. */ @ActiveDoc.keepDocOpen - public async createEmptyDoc(docSession: OptDocSession): Promise { - await this.loadDoc(docSession, {forceNew: true, skipInitialTable: true}); + public async createEmptyDoc(docSession: OptDocSession, + options?: { useExisting?: boolean }): Promise { + await this.loadDoc(docSession, {forceNew: true, + skipInitialTable: true, + ...options}); // Makes sure docPluginManager is ready in case new doc is used to import new data - await this.docPluginManager.ready; + await this.docPluginManager?.ready; this._fullyLoaded = true; return this; } @@ -603,13 +609,18 @@ export class ActiveDoc extends EventEmitter { public async loadDoc(docSession: OptDocSession, options?: { forceNew?: boolean, // If set, document will be created. skipInitialTable?: boolean, // If set, and document is new, "Table1" will not be added. + useExisting?: boolean, // If set, document can be created as an overlay on + // an existing sqlite file. }): Promise { const startTime = Date.now(); this._log.debug(docSession, "loadDoc"); try { const isNew: boolean = options?.forceNew || await this._docManager.storageManager.prepareLocalDoc(this.docName); if (isNew) { - await this._createDocFile(docSession, {skipInitialTable: options?.skipInitialTable}); + await this._createDocFile(docSession, { + skipInitialTable: options?.skipInitialTable, + useExisting: options?.useExisting, + }); } else { await this.docStorage.openFile({ beforeMigration: async (currentVersion, newVersion) => { @@ -1253,6 +1264,7 @@ export class ActiveDoc extends EventEmitter { if (await this._granularAccess.hasNuancedAccess(docSession)) { throw new Error('cannot confirm access to plugin'); } + if (!this.docPluginManager) { throw new Error('no plugin manager available'); } const pluginRpc = this.docPluginManager.plugins[pluginId].rpc; switch (msg.mtype) { case MsgType.RpcCall: return pluginRpc.forwardCall(msg); @@ -1266,6 +1278,7 @@ export class ActiveDoc extends EventEmitter { */ public async reloadPlugins(docSession: DocSession) { // refresh the list plugins found on the system + if (!this._docManager.pluginManager || !this.docPluginManager) { return; } await this._docManager.pluginManager.reloadPlugins(); const plugins = this._docManager.pluginManager.getPlugins(); // reload found plugins @@ -1433,6 +1446,7 @@ export class ActiveDoc extends EventEmitter { } public getGristDocAPI(): GristDocAPI { + if (!this.docPluginManager) { throw new Error('no plugin manager available'); } return this.docPluginManager.gristDocAPI; } @@ -1573,6 +1587,27 @@ export class ActiveDoc extends EventEmitter { return this._docManager.makeAccessId(userId); } + /** + * Apply actions that have already occurred in the data engine to the + * database also. + */ + public async applyStoredActionsToDocStorage(docActions: DocAction[]): Promise { + // When "gristifying" an sqlite database, we may take create tables and + // columns in the data engine that already exist in the sqlite database. + // During that process, _onlyAllowMetaDataActionsOnDb will be turned on, + // and we silently swallow any non-metadata actions. + if (this._onlyAllowMetaDataActionsOnDb) { + docActions = docActions.filter(a => getTableId(a).startsWith('_grist')); + } + await this.docStorage.applyStoredActions(docActions); + } + + // Set a flag that controls whether user data can be changed in the database, + // or only grist-managed tables (those whose names start with _grist) + public onlyAllowMetaDataActionsOnDb(flag: boolean) { + this._onlyAllowMetaDataActionsOnDb = flag; + } + /** * Called by Sharing manager when working on modifying the document. * Called when DocActions have been produced from UserActions, but @@ -1720,10 +1755,12 @@ export class ActiveDoc extends EventEmitter { @ActiveDoc.keepDocOpen private async _createDocFile(docSession: OptDocSession, options?: { skipInitialTable?: boolean, // If set, "Table1" will not be added. + useExisting?: boolean, // If set, an existing sqlite db is permitted. + // Useful for "gristifying" an existing db. }): Promise { this._log.debug(docSession, "createDoc"); await this._docManager.storageManager.prepareToCreateDoc(this.docName); - await this.docStorage.createFile(); + await this.docStorage.createFile(options); const sql = options?.skipInitialTable ? GRIST_DOC_SQL : GRIST_DOC_WITH_TABLE1_SQL; await this.docStorage.exec(sql); const timezone = docSession.browserSettings?.timezone ?? DEFAULT_TIMEZONE; diff --git a/app/server/lib/ActiveDocImport.ts b/app/server/lib/ActiveDocImport.ts index b0f1392f..4329f007 100644 --- a/app/server/lib/ActiveDocImport.ts +++ b/app/server/lib/ActiveDocImport.ts @@ -242,6 +242,7 @@ export class ActiveDocImport { // The upload must be within the plugin-accessible directory. Once moved, subsequent calls to // moveUpload() will return without having to do anything. + if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); } await moveUpload(upload, this._activeDoc.docPluginManager.tmpDir()); const importResult: ImportResult = {options: parseOptions, tables: []}; @@ -287,6 +288,7 @@ export class ActiveDocImport { const {originalFilename, parseOptions, mergeOptionsMap, isHidden, uploadFileIndex, transformRuleMap} = importOptions; log.info("ActiveDoc._importFileAsNewTable(%s, %s)", tmpPath, originalFilename); + if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); } const optionsAndData: ParseFileResult = await this._activeDoc.docPluginManager.parseFile(tmpPath, originalFilename, parseOptions); const options = optionsAndData.parseOptions; diff --git a/app/server/lib/DocClients.ts b/app/server/lib/DocClients.ts index b66d5d5a..a6e039a5 100644 --- a/app/server/lib/DocClients.ts +++ b/app/server/lib/DocClients.ts @@ -95,7 +95,7 @@ export class DocClients { } if (type === "docUserAction" && messageData.docActions) { for (const action of messageData.docActions) { - this.activeDoc.docPluginManager.receiveAction(action); + this.activeDoc.docPluginManager?.receiveAction(action); } } } diff --git a/app/server/lib/DocManager.ts b/app/server/lib/DocManager.ts index a03d3c05..92a23190 100644 --- a/app/server/lib/DocManager.ts +++ b/app/server/lib/DocManager.ts @@ -54,7 +54,7 @@ export class DocManager extends EventEmitter { constructor( public readonly storageManager: IDocStorageManager, - public readonly pluginManager: PluginManager, + public readonly pluginManager: PluginManager|null, private _homeDbManager: HomeDBManager|null, public gristServer: GristServer ) { @@ -610,7 +610,7 @@ export class DocManager extends EventEmitter { return docUtils.createExclusive(this.storageManager.getPath(name)); }); log.debug('DocManager._createNewDoc picked name', docName); - await this.pluginManager.pluginsLoaded; + await this.pluginManager?.pluginsLoaded; return docName; } } diff --git a/app/server/lib/DocStorage.ts b/app/server/lib/DocStorage.ts index b6c06b7b..24291cb3 100644 --- a/app/server/lib/DocStorage.ts +++ b/app/server/lib/DocStorage.ts @@ -30,6 +30,7 @@ import uuidv4 from "uuid/v4"; import {OnDemandStorage} from './OnDemandActions'; import {ISQLiteDB, MigrationHooks, OpenMode, quoteIdent, ResultRow, SchemaInfo, SQLiteDB} from './SQLiteDB'; import chunk = require('lodash/chunk'); +import cloneDeep = require('lodash/cloneDeep'); import groupBy = require('lodash/groupBy'); @@ -666,10 +667,17 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { * After a database is created it should be initialized by applying the InitNewDoc action * or by executing the initialDocSql. */ - public createFile(): Promise { + public createFile(options?: { + useExisting?: boolean, // If set, it is ok if an sqlite file already exists + // where we would store the Grist document. Its content + // will not be touched. Useful when "gristifying" an + // existing SQLite DB. + }): Promise { // It turns out to be important to return a bluebird promise, a lot of code outside // of DocStorage ultimately depends on this. - return bluebird.Promise.resolve(this._openFile(OpenMode.CREATE_EXCL, {})) + return bluebird.Promise.resolve(this._openFile( + options?.useExisting ? OpenMode.OPEN_EXISTING : OpenMode.CREATE_EXCL, + {})) .then(() => this._initDB()); // Note that we don't call _updateMetadata() as there are no metadata tables yet anyway. } @@ -927,26 +935,50 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { public async applyStoredActions(docActions: DocAction[]): Promise { debuglog('DocStorage.applyStoredActions'); - await bluebird.Promise.each(docActions, (action: DocAction) => { - const actionType = action[0]; - const f = (this as any)["_process_" + actionType]; - if (!_.isFunction(f)) { - log.error("Unknown action: " + actionType); - } else { - return f.apply(this, action.slice(1)) - .then(() => { - const tableId = action[1]; // The first argument is always tableId; - if (DocStorage._isMetadataTable(tableId) && actionType !== 'AddTable') { - // We only need to update the metadata for actions that change - // the metadata. We don't update on AddTable actions - // because the additional of a table gives no additional data - // and if we tried to update when only _grist_Tables was added - // without _grist_Tables_column, we would get an error - return this._updateMetadata(); - } - }); + docActions = this._compressStoredActions(docActions); + for (const action of docActions) { + try { + await this.applyStoredAction(action); + } catch (e) { + // If the table doesn't have a manualSort column, we'll try + // again without setting manualSort. This should never happen + // for regular Grist documents, but could happen for a + // "gristified" Sqlite database where we are choosing to + // leave the user tables untouched. The manualSort column doesn't + // make much sense outside the context of spreadsheets. + // TODO: it could be useful to make Grist more inherently aware of + // and tolerant of tables without manual sorting. + if (String(e).match(/no column named manualSort/)) { + const modifiedAction = this._considerWithoutManualSort(action); + if (modifiedAction) { + await this.applyStoredAction(modifiedAction); + return; + } + } + throw e; } - }); + } + } + + // Apply a single stored action, dispatching to an appropriate + // _process_ handler. + public async applyStoredAction(action: DocAction): Promise { + const actionType = action[0]; + const f = (this as any)["_process_" + actionType]; + if (!_.isFunction(f)) { + log.error("Unknown action: " + actionType); + } else { + await f.apply(this, action.slice(1)); + const tableId = action[1]; // The first argument is always tableId; + if (DocStorage._isMetadataTable(tableId) && actionType !== 'AddTable') { + // We only need to update the metadata for actions that change + // the metadata. We don't update on AddTable actions + // because the additional of a table gives no additional data + // and if we tried to update when only _grist_Tables was added + // without _grist_Tables_column, we would get an error + await this._updateMetadata(); + } + } } /** @@ -1169,6 +1201,8 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { // Note that SQLite does not support easily dropping columns. To drop a column from a table, we // need to follow the instructions at https://sqlite.org/lang_altertable.html Since we don't use // indexes or triggers, we skip a few steps. + // TODO: SQLite has since added support for ALTER TABLE DROP COLUMN, should + // use that to be more efficient and less disruptive. // This returns rows with (at least) {name, type, dflt_value}. return this.all(`PRAGMA table_info(${quote(tableId)})`) @@ -1713,6 +1747,42 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { `${joinClauses} ${whereClause} ${limitClause}`; return sql; } + + // If we are being asked to add a record and then update several of its + // columns, compact that into a single action. For fully Grist-managed + // documents, this makes no difference. But if the underlying SQLite DB + // has extra constraints on columns, it can make a difference. + // TODO: consider dealing with other scenarios, especially a BulkAddRecord. + private _compressStoredActions(docActions: DocAction[]): DocAction[] { + if (docActions.length > 1) { + const first = docActions[0]; + if (first[0] === 'AddRecord' && + docActions.slice(1).every( + // Check other actions are UpdateRecords for the same table and row. + a => a[0] === 'UpdateRecord' && a[1] === first[1] && a[2] === first[2] + )) { + const merged = cloneDeep(first); + for (const a2 of docActions.slice(1)) { + Object.assign(merged[3], a2[3]); + } + docActions = [merged]; + } + } + return docActions; + } + + // If an action can have manualSort removed, go ahead and do it (after cloning), + // otherwise return null. + private _considerWithoutManualSort(act: DocAction): DocAction|null { + if (act[0] === 'AddRecord' || act[0] === 'UpdateRecord' || + act[0] === 'BulkAddRecord' || act[0] === 'BulkUpdateRecord' && + 'manualSort' in act[3]) { + act = cloneDeep(act); + delete act[3].manualSort; + return act; + } + return null; + } } interface RebuildResult { diff --git a/app/server/lib/GristServer.ts b/app/server/lib/GristServer.ts index 7e6cb8b3..1f28e7e3 100644 --- a/app/server/lib/GristServer.ts +++ b/app/server/lib/GristServer.ts @@ -6,6 +6,7 @@ import { Workspace } from 'app/gen-server/entity/Workspace'; import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager'; import { RequestWithLogin } from 'app/server/lib/Authorizer'; import { Comm } from 'app/server/lib/Comm'; +import { create } from 'app/server/lib/create'; import { Hosts } from 'app/server/lib/extractOrg'; import { ICreate } from 'app/server/lib/ICreate'; import { IDocStorageManager } from 'app/server/lib/IDocStorageManager'; @@ -88,3 +89,33 @@ export interface DocTemplate { page: string, tag: string, } + +/** + * A very minimal GristServer object that throws an error if its bluff is + * called. + */ +export function createDummyGristServer(): GristServer { + return { + create, + settings: {}, + getHost() { return 'localhost:4242'; }, + getHomeUrl() { return 'http://localhost:4242'; }, + getHomeUrlByDocId() { return Promise.resolve('http://localhost:4242'); }, + getMergedOrgUrl() { return 'http://localhost:4242'; }, + getOwnUrl() { return 'http://localhost:4242'; }, + getPermitStore() { throw new Error('no permit store'); }, + getExternalPermitStore() { throw new Error('no external permit store'); }, + getGristConfig() { return { homeUrl: '', timestampMs: 0 }; }, + getOrgUrl() { return Promise.resolve(''); }, + getResourceUrl() { return Promise.resolve(''); }, + getSessions() { throw new Error('no sessions'); }, + getComm() { throw new Error('no comms'); }, + getHosts() { throw new Error('no hosts'); }, + getHomeDBManager() { throw new Error('no db'); }, + getStorageManager() { throw new Error('no storage manager'); }, + getNotifier() { throw new Error('no notifier'); }, + getDocTemplate() { throw new Error('no doc template'); }, + getTag() { return 'tag'; }, + sendAppPage() { return Promise.resolve(); }, + }; +} diff --git a/app/server/lib/IDocStorageManager.ts b/app/server/lib/IDocStorageManager.ts index 9b03e272..54180a98 100644 --- a/app/server/lib/IDocStorageManager.ts +++ b/app/server/lib/IDocStorageManager.ts @@ -38,3 +38,33 @@ export interface IDocStorageManager { removeSnapshots(docName: string, snapshotIds: string[]): Promise; replace(docName: string, options: DocReplacementOptions): Promise; } + +/** + * A very minimal implementation of IDocStorageManager that is just + * enough to allow an ActiveDoc to open and get to work. + */ +export class TrivialDocStorageManager implements IDocStorageManager { + public getPath(docName: string): string { return docName; } + public getSampleDocPath() { return null; } + public async getCanonicalDocName(altDocName: string) { return altDocName; } + public async prepareLocalDoc() { return false; } + public async prepareToCreateDoc() { } + public async prepareFork(): Promise { throw new Error('no'); } + public async listDocs() { return []; } + public async deleteDoc(): Promise { throw new Error('no'); } + public async renameDoc(): Promise { throw new Error('no'); } + public async makeBackup(): Promise { throw new Error('no'); } + public async showItemInFolder(): Promise { throw new Error('no'); } + public async closeStorage() {} + public async closeDocument() {} + public markAsChanged() {} + public scheduleUsageUpdate() {} + public testReopenStorage() {} + public async addToStorage(): Promise { throw new Error('no'); } + public prepareToCloseStorage() {} + public async getCopy(): Promise { throw new Error('no'); } + public async flushDoc() {} + public async getSnapshots(): Promise { throw new Error('no'); } + public async removeSnapshots(): Promise { throw new Error('no'); } + public async replace(): Promise { throw new Error('no'); } +} diff --git a/app/server/lib/SQLiteDB.ts b/app/server/lib/SQLiteDB.ts index e509299d..10e96a71 100644 --- a/app/server/lib/SQLiteDB.ts +++ b/app/server/lib/SQLiteDB.ts @@ -166,7 +166,7 @@ export class SQLiteDB implements ISQLiteDB { // It's possible that userVersion is 0 for a non-empty DB if it was created without this // module. In that case, we apply migrations starting with the first one. - if (userVersion === 0 && (await isEmpty(db))) { + if (userVersion === 0 && (await isGristEmpty(db))) { await db._initNewDB(schemaInfo); } else if (mode === OpenMode.CREATE_EXCL) { await db.close(); @@ -537,15 +537,15 @@ export class SQLiteDB implements ISQLiteDB { // dummy DB, and we use it to do sanity checking, in particular after migrations. To avoid // creating dummy DBs multiple times, the result is cached, keyed by the "create" function itself. const dbMetadataCache: Map = new Map(); -interface DBMetadata { +export interface DBMetadata { [tableName: string]: { [colName: string]: string; // Maps column name to SQLite type, e.g. "TEXT". }; } -// Helper to see if a database is empty. -async function isEmpty(db: SQLiteDB): Promise { - return (await db.get("SELECT count(*) as count FROM sqlite_master"))!.count === 0; +// Helper to see if a database is empty of grist metadata tables. +async function isGristEmpty(db: SQLiteDB): Promise { + return (await db.get("SELECT count(*) as count FROM sqlite_master WHERE name LIKE '_grist%'"))!.count === 0; } /** diff --git a/app/server/lib/Sharing.ts b/app/server/lib/Sharing.ts index eb06ca32..460d7392 100644 --- a/app/server/lib/Sharing.ts +++ b/app/server/lib/Sharing.ts @@ -281,7 +281,7 @@ export class Sharing { // Apply the action to the database, and record in the action log. if (!trivial) { await this._activeDoc.docStorage.execTransaction(async () => { - await this._activeDoc.docStorage.applyStoredActions(getEnvContent(ownActionBundle.stored)); + await this._activeDoc.applyStoredActionsToDocStorage(getEnvContent(ownActionBundle.stored)); if (this.isShared() && branch === Branch.Local) { // this call will compute an actionHash for localActionBundle await this._actionHistory.recordNextLocalUnsent(localActionBundle); diff --git a/app/server/utils/gristify.ts b/app/server/utils/gristify.ts new file mode 100644 index 00000000..5a57a5b1 --- /dev/null +++ b/app/server/utils/gristify.ts @@ -0,0 +1,256 @@ +import { ColInfoWithId } from 'app/common/DocActions'; +import { ActiveDoc } from 'app/server/lib/ActiveDoc'; +import { DocManager } from 'app/server/lib/DocManager'; +import { makeExceptionalDocSession, OptDocSession } from 'app/server/lib/DocSession'; +import { createDummyGristServer } from 'app/server/lib/GristServer'; +import { TrivialDocStorageManager } from 'app/server/lib/IDocStorageManager'; +import { DBMetadata, quoteIdent, SQLiteDB } from 'app/server/lib/SQLiteDB'; + +/** + * A utility class for modifying a SQLite file to be viewed/edited with Grist. + */ +export class Gristifier { + public constructor(private _filename: string) { + } + + /** + * Add Grist metadata tables to a SQLite file. After this action, + * the file can be opened as a Grist document, with partial functionality. + * Level of functionality will depend on the nature of the tables in the + * SQLite file. + * + * The `user_version` slot of SQLite will be modified by this operation, + * losing whatever was in it previously. + * + * A "manualSort" column may be added to tables by specifying `addSort`, + * to support a notion of order that exists in spreadsheets. + * + * Grist is very finicky about primary keys, and tables that don't match + * its expectations cannot be viewed or edited directly at the moment. + * Instead, views are added supporting selects, updates, inserts, and + * deletes. Structure changes (e.g. adding/removing columns) are not + * supported unfortunately. + * + * This is very much an experiment, with plenty of limits and + * sharp edges. In general it isn't possible to treat an arbitrary + * SQLite file as a Grist document, but in particular cases it can + * work and be very useful. + */ + public async gristify(options: {addSort?: boolean}) { + // Remove any existing Grist material from the file. + await this.degristify(); + + // Enumerate user tables and columns. + const inventory = await this._getUserTables(); + + // Grist keeps a schema number in the SQLite "user_version" slot, + // so we need to zap it. This is the one destructive operation + // involved in gristification. + // TODO: consider moving schema information somewhere more neutral. + await this._zapUserVersion(); + + // Open the file as an empty Grist document, creating Grist metadata + // tables. + const docManager = new DocManager( + new TrivialDocStorageManager(), null, null, createDummyGristServer() + ); + const activeDoc = new ActiveDoc(docManager, this._filename); + const docSession = makeExceptionalDocSession('system'); + await activeDoc.createEmptyDoc(docSession, {useExisting: true}); + await activeDoc.waitForInitialization(); + + // Now "create" user tables and columns with Grist. The creation + // will be fictitious since the tables and columns already exist - + // they just don't have metadata describing them to Grist. + const outcomes: TableOutcome[] = []; + for (const [tableId, table] of Object.entries(inventory)) { + const columnDefs = this._collectColumnDefinitions(table); + if (!('id' in columnDefs)) { + // Can't handle this table in Grist directly at the moment, but + // we can do something via a view. + await this._createView(docSession, activeDoc, tableId, Object.keys(table), columnDefs); + outcomes.push({tableId, viewed: true, reason: 'id complications'}); + } else { + await this._registerTable(docSession, activeDoc, tableId, columnDefs); + if (options.addSort) { + await this._addManualSort(activeDoc, tableId); + outcomes.push({tableId, addManualSort: true}); + } else { + outcomes.push({tableId}); + } + } + } + await activeDoc.shutdown(); + + // Give a final readout of what happened for every table, since the + // conversion process is quite noisy. + for (const outcome of outcomes) { + console.log(JSON.stringify(outcome)); + } + } + + /** + * Remove all Grist metadata tables. Warning: attachments are considered metadata. + */ + public async degristify() { + const db = await SQLiteDB.openDBRaw(this._filename); + const tables = await db.all( + `SELECT name FROM sqlite_master WHERE type='table' ` + + ` AND name LIKE '_grist%'` + ); + for (const table of tables) { + console.log(`Removing ${table.name}`); + await db.exec(`DROP TABLE ${quoteIdent(table.name)}`); + } + const views = await db.all( + `SELECT name FROM sqlite_master WHERE type='view' ` + + ` AND name LIKE 'GristView%'` + ); + for (const view of views) { + console.log(`Removing ${view.name}`); + await db.exec(`DROP VIEW ${quoteIdent(view.name)}`); + } + await db.close(); + } + + /** + * Make definitions for the table's columns. This is very crude, it handles + * integers and leaves everything else as "Any". + */ + private _collectColumnDefinitions(table: DBMetadata[string]) { + const defs: Record = {}; + for (const [colId, info] of Object.entries(table)) { + if (colId.startsWith('manualSort')) { continue; } + const type = info.toLowerCase(); + const c: ColInfoWithId = { + id: colId, + type: 'Any', + isFormula: false, + formula: '', + }; + // see https://www.sqlite.org/datatype3.html#determination_of_column_affinity + if (type.includes('int')) { + c.type = 'Int'; + } + if (colId === 'id') { + if (c.type !== 'Int') { + // Grist can only support integer id columns. + // For now, just rename this column out of the way to id2, and use + // a view to map SQLite's built-in ROWID to the id column. + // TODO: could collide with a column called "id2". + c.id = 'id2'; + } + } + defs[c.id] = c; + } + return defs; + } + + /** + * Support tables that don't have an integer column called "id" through views. + * It would be better to enhance Grist to support a wider variety of scenarios, + * but this is helpful for now. + */ + private async _createView(docSession: OptDocSession, activeDoc: ActiveDoc, tableId: string, + cols: string[], columnDefs: Record) { + const newName = `GristView_${tableId}`; + function quote(name: string) { + return quoteIdent(name === 'id' ? 'id2' : name); + } + function quoteForSelect(name: string) { + if (name === 'id') { return 'id as id2'; } + return quoteIdent(name); + } + + // View table tableId via a view GristView_tableId, with id and manualSort supplied + // from ROWID. SQLite tables may not have a ROWID, but this is relatively rare. + await activeDoc.docStorage.exec(`CREATE VIEW ${quoteIdent(newName)} AS SELECT ` + + ['ROWID AS id', 'ROWID AS manualSort', ...cols.map(quoteForSelect)].join(', ') + + ` FROM ${quoteIdent(tableId)}`); + + // Make an INSTEAD OF UPDATE trigger, so that if someone tries to update the view, + // we instead update the underlying table. Updates of manualSort or id are just ignored. + // The trigger is a little awkward to write since we need to compare OLD and NEW + // to see what changed - updating unchanged material could needlessly run afoul of + // constraints. + const updateTrigger = `CREATE TRIGGER ${quoteIdent('trigger_update_' + newName)} ` + + `INSTEAD OF UPDATE ON ${quoteIdent(newName)} BEGIN ` + + cols.map(col => + `UPDATE ${quoteIdent(tableId)} SET ` + + `${quoteIdent(col)} = NEW.${quote(col)} ` + + ` WHERE OLD.${quote(col)} <> NEW.${quote(col)} ` + + ` AND ${quoteIdent(tableId)}.ROWID = NEW.ROWID` + ).join('; ') + + `; END`; + await activeDoc.docStorage.exec(updateTrigger); + + // Make an INSTEAD OF INSERT trigger. + const insertTrigger = `create trigger ${quoteIdent('trigger_insert_' + newName)} ` + + `INSTEAD OF INSERT ON ${quoteIdent(newName)} BEGIN ` + + `INSERT INTO ${quoteIdent(tableId)}` + + '(' + cols.map(quoteIdent).join(',') + ') VALUES(' + + cols.map(col => `NEW.${quote(col)}`).join(', ') + + `); END`; + await activeDoc.docStorage.exec(insertTrigger); + + // Make an INSTEAD OF DELETE trigger. + const deleteTrigger = `create trigger ${quoteIdent('trigger_delete_' + newName)} ` + + `INSTEAD OF DELETE ON ${quoteIdent(newName)} BEGIN ` + + `DELETE FROM ${quoteIdent(tableId)} WHERE ${quoteIdent(tableId)}.ROWID = OLD.ROWID` + + `; END`; + await activeDoc.docStorage.exec(deleteTrigger); + + const result = await this._registerTable(docSession, activeDoc, newName, columnDefs); + + // Now, tweak the Grist metadata to make the table name the expected one + // (the table id as far as Grist is concerned must remain that of the view) + const id = result.retValues[0].id; + await activeDoc.docStorage.run('update _grist_Views_section set title = ? ' + + 'where id in (select rawViewSectionRef from _grist_Tables where id = ?)', + [tableId, id]); + await activeDoc.docStorage.run('update _grist_Views set name = ? ' + + 'where id in (select primaryViewId from _grist_Tables where id = ?)', + [tableId, id]); + } + + private async _getUserTables(): Promise { + // Enumerate existing tables and columns. + const db = await SQLiteDB.openDBRaw(this._filename); + const inventory = await db.collectMetadata(); + await db.close(); + // We are not interested in the special "sqlite_sequence" table. + delete inventory.sqlite_sequence; + return inventory; + } + + private async _zapUserVersion(): Promise { + const db = await SQLiteDB.openDBRaw(this._filename); + await db.exec(`PRAGMA user_version = 0`); + await db.close(); + } + + private async _addManualSort(activeDoc: ActiveDoc, tableId: string) { + const db = activeDoc.docStorage; + await db.exec(`ALTER TABLE ${quoteIdent(tableId)} ADD COLUMN manualSort INTEGER`).catch(e => null); + await db.exec(`UPDATE ${quoteIdent(tableId)} SET manualSort = id`); + } + + private async _registerTable(docSession: OptDocSession, activeDoc: ActiveDoc, + tableId: string, args: Record) { + delete args.id; + activeDoc.onlyAllowMetaDataActionsOnDb(true); + const result = await activeDoc.applyUserActions(docSession, [ + ['AddTable', tableId, Object.values(args)], + ]); + activeDoc.onlyAllowMetaDataActionsOnDb(false); + return result; + } +} + +interface TableOutcome { + tableId: string; + skipped?: boolean; + viewed?: boolean; + addManualSort?: boolean; + reason?: string; +} diff --git a/app/server/utils/pruneActionHistory.ts b/app/server/utils/pruneActionHistory.ts new file mode 100644 index 00000000..818454e9 --- /dev/null +++ b/app/server/utils/pruneActionHistory.ts @@ -0,0 +1,63 @@ +import * as gutil from 'app/common/gutil'; +import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl'; +import {DocStorage} from 'app/server/lib/DocStorage'; +import {DocStorageManager} from 'app/server/lib/DocStorageManager'; +import * as docUtils from 'app/server/lib/docUtils'; +import log from 'app/server/lib/log'; + +/** + * A utility script for cleaning up the action log. + * + * @param {String} docPath - The path to the document from the current directory including + * the document name. + * @param {Int} keepN - The number of recent actions to keep. Must be at least 1. Defaults to 1 + * if not provided. + */ +export async function pruneActionHistory(docPath: string, keepN: number) { + if (!docPath || !gutil.endsWith(docPath, '.grist')) { + throw new Error('Invalid document: Document should be a valid .grist file'); + } + + const storageManager = new DocStorageManager(".", "."); + const docStorage = new DocStorage(storageManager, docPath); + const backupPath = gutil.removeSuffix(docPath, '.grist') + "-backup.grist"; + + // If the backup already exists, abort. Otherwise, create a backup copy and continue. + const exists = await docUtils.pathExists(backupPath); + if (exists) { throw new Error('Backup file already exists, aborting pruneActionHistory'); } + await docUtils.copyFile(docPath, backupPath); + await docStorage.openFile(); + try { + const history = new ActionHistoryImpl(docStorage); + await history.initialize(); + await history.deleteActions(keepN); + } finally { + await docStorage.shutdown(); + } +} + +/** + * Variant that accepts and parses command line arguments. + */ +export async function pruneActionHistoryFromConsole(argv: string[]): Promise { + if (argv.length === 0) { + log.error("Please supply document name, and optionally the number of actions to preserve (default=1)"); + return 1; + } + const docPath = argv[0]; + const keepN = parseInt(argv[1], 10) || 1; + try { + await pruneActionHistory(docPath, keepN); + } catch (e) { + log.error(e); + return 1; + } + return 0; +} + +if (require.main === module) { + pruneActionHistoryFromConsole(process.argv.slice(2)).catch((e) => { + log.error("pruneActionHistory failed: %s", e); + process.exit(1); + }); +} diff --git a/package.json b/package.json index acec6b94..46bb7692 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha -g ${GREP_TEST:-''} _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js", "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/server/**/*.js _build/test/gen-server/**/*.js", "test:smoke": "NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/nbrowser/Smoke.js", - "test:docker": "./test/test_under_docker.sh" + "test:docker": "./test/test_under_docker.sh", + "cli": "NODE_PATH=_build:_build/stubs:_build/ext node _build/app/server/companion.js" }, "keywords": [ "grist", @@ -96,6 +97,7 @@ "bowser": "2.7.0", "brace": "0.11.1", "collect-js-deps": "^0.1.1", + "commander": "9.3.0", "components-jqueryui": "1.12.1", "connect-redis": "3.4.0", "cookie": "0.5.0", diff --git a/test/server/docTools.ts b/test/server/docTools.ts index 24240a9a..bb569b5f 100644 --- a/test/server/docTools.ts +++ b/test/server/docTools.ts @@ -1,11 +1,10 @@ import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap'; import {ActiveDoc} from 'app/server/lib/ActiveDoc'; import {DummyAuthorizer} from 'app/server/lib/Authorizer'; -import {create} from 'app/server/lib/create'; import {DocManager} from 'app/server/lib/DocManager'; import {DocSession, makeExceptionalDocSession} from 'app/server/lib/DocSession'; import {DocStorageManager} from 'app/server/lib/DocStorageManager'; -import {GristServer} from 'app/server/lib/GristServer'; +import {createDummyGristServer, GristServer} from 'app/server/lib/GristServer'; import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; import {getAppRoot} from 'app/server/lib/places'; import {PluginManager} from 'app/server/lib/PluginManager'; @@ -151,32 +150,6 @@ export async function createDocManager( }); } -export function createDummyGristServer(): GristServer { - return { - create, - settings: {}, - getHost() { return 'localhost:4242'; }, - getHomeUrl() { return 'http://localhost:4242'; }, - getHomeUrlByDocId() { return Promise.resolve('http://localhost:4242'); }, - getMergedOrgUrl() { return 'http://localhost:4242'; }, - getOwnUrl() { return 'http://localhost:4242'; }, - getPermitStore() { throw new Error('no permit store'); }, - getExternalPermitStore() { throw new Error('no external permit store'); }, - getGristConfig() { return { homeUrl: '', timestampMs: 0 }; }, - getOrgUrl() { return Promise.resolve(''); }, - getResourceUrl() { return Promise.resolve(''); }, - getSessions() { throw new Error('no sessions'); }, - getComm() { throw new Error('no comms'); }, - getHosts() { throw new Error('no hosts'); }, - getHomeDBManager() { throw new Error('no db'); }, - getStorageManager() { throw new Error('no storage manager'); }, - getNotifier() { throw new Error('no notifier'); }, - getDocTemplate() { throw new Error('no doc template'); }, - getTag() { return 'tag'; }, - sendAppPage() { return Promise.resolve(); }, - }; -} - export async function createTmpDir(): Promise { const tmpRootDir = process.env.TESTDIR || tmpdir(); await fse.mkdirs(tmpRootDir); diff --git a/yarn.lock b/yarn.lock index 78925920..2af12b1c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1776,6 +1776,11 @@ commander@2.15.1: resolved "https://registry.yarnpkg.com/commander/-/commander-2.15.1.tgz#df46e867d0fc2aec66a34662b406a9ccafff5b0f" integrity sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag== +commander@9.3.0: + version "9.3.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-9.3.0.tgz#f619114a5a2d2054e0d9ff1b31d5ccf89255e26b" + integrity sha512-hv95iU5uXPbK83mjrJKuZyFM/LBAoCV/XhVGkS5Je6tl7sxr6A0ITMw5WoRV46/UaJ46Nllm3Xt7IaJhXTIkzw== + commander@^2.11.0, commander@^2.20.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"