diff --git a/app/server/lib/DocStorage.ts b/app/server/lib/DocStorage.ts index b188b380..8bbddb0b 100644 --- a/app/server/lib/DocStorage.ts +++ b/app/server/lib/DocStorage.ts @@ -11,6 +11,7 @@ import * as sqlite3 from '@gristlabs/sqlite3'; import {LocalActionBundle} from 'app/common/ActionBundle'; import {BulkColValues, DocAction, TableColValues, TableDataAction, toTableDataAction} from 'app/common/DocActions'; import * as gristTypes from 'app/common/gristTypes'; +import {isList} from 'app/common/gristTypes'; import * as marshal from 'app/common/marshal'; import * as schema from 'app/common/schema'; import {GristObjCode} from "app/plugin/GristData"; @@ -21,13 +22,13 @@ import * as log from 'app/server/lib/log'; import * as assert from 'assert'; import * as bluebird from 'bluebird'; import * as fse from 'fs-extra'; -import chunk = require('lodash/chunk'); -import groupBy = require('lodash/groupBy'); import * as _ from 'underscore'; import * as util from 'util'; import * as uuidv4 from "uuid/v4"; -import { ISQLiteDB, MigrationHooks, OpenMode, quoteIdent, ResultRow, SchemaInfo, SQLiteDB} from './SQLiteDB'; -import {isList} from "app/common/gristTypes"; +import {OnDemandStorage} from './OnDemandActions'; +import {ISQLiteDB, MigrationHooks, OpenMode, quoteIdent, ResultRow, SchemaInfo, SQLiteDB} from './SQLiteDB'; +import chunk = require('lodash/chunk'); +import groupBy = require('lodash/groupBy'); // Run with environment variable NODE_DEBUG=db (may include additional comma-separated sections) @@ -38,7 +39,7 @@ const maxSQLiteVariables = 500; // Actually could be 999, so this is playing const PENDING_VALUE = [GristObjCode.Pending]; -export class DocStorage implements ISQLiteDB { +export class DocStorage implements ISQLiteDB, OnDemandStorage { // ====================================================================== // Static fields @@ -623,6 +624,9 @@ export class DocStorage implements ISQLiteDB { // tables (obtained from auto-generated schema.js). private _docSchema: {[tableId: string]: {[colId: string]: string}}; + // The last time _logDataSize ran fully + private _lastLoggedDataSize: number = Date.now(); + public constructor(public storageManager: IDocStorageManager, public docName: string) { this.docPath = this.storageManager.getPath(docName); this._db = null; @@ -652,27 +656,6 @@ export class DocStorage implements ISQLiteDB { // Note that we don't call _updateMetadata() as there are no metadata tables yet anyway. } - /** - * Creates a backup and calls cb() within a transaction. Returns the backup path. In case of - * failure, adds .backupPath property to the error object (and transaction is rolled back). - */ - public execWithBackup(cb: (db: SQLiteDB) => Promise): Promise { - let backupPath: string; - return this.storageManager.makeBackup(this.docName, "migrate-db") - .then((_backupPath: string) => { - backupPath = _backupPath; - log.info(`DocStorage[${this.docName}]: backup made at ${backupPath}`); - return this.execTransaction(cb); - }) - .then(() => backupPath) - .catch((err: any) => { - // TODO: deal with typing for this kind of error (although nothing seems to depend - // on it yet. - err.backupPath = backupPath; - throw err; - }); - } - /** * Initializes the database with proper settings. */ @@ -915,7 +898,7 @@ export class DocStorage implements ISQLiteDB { public async applyStoredActions(docActions: DocAction[]): Promise { debuglog('DocStorage.applyStoredActions'); - return bluebird.Promise.each(docActions, (action: DocAction) => { + await bluebird.Promise.each(docActions, (action: DocAction) => { const actionType = action[0]; const f = (this as any)["_process_" + actionType]; if (!_.isFunction(f)) { @@ -935,6 +918,7 @@ export class DocStorage implements ISQLiteDB { }); } }); + this._logDataSize().catch(e => log.error(`Error in _logDataSize: ${e}`)); } /** @@ -1562,6 +1546,25 @@ export class DocStorage implements ISQLiteDB { `${joinClauses} ${whereClause} ${limitClause}`; return sql; } + + private async _logDataSize() { + // To reduce overhead, don't query and log data size more than once in 5 minutes + const now = Date.now(); + if (now - this._lastLoggedDataSize < 5 * 60 * 1000) { + return; + } + this._lastLoggedDataSize = now; + + const result = await this.get(` + SELECT SUM(pgsize) AS totalSize + FROM dbstat + WHERE NOT ( + name LIKE 'sqlite_%' OR + name LIKE '_gristsys_%' + ); + `); + log.rawInfo("Data size from dbstat...", {docId: this.docName, dataSize: result!.totalSize}); + } } interface RebuildResult { diff --git a/package.json b/package.json index c230937b..0a0820fc 100644 --- a/package.json +++ b/package.json @@ -76,11 +76,11 @@ "dependencies": { "@googleapis/drive": "0.3.1", "@googleapis/oauth2": "0.2.0", - "@gristlabs/connect-sqlite3": "0.9.11-grist.1", + "@gristlabs/connect-sqlite3": "0.9.11-grist.4", "@gristlabs/express-session": "1.17.0", "@gristlabs/moment-guess": "1.2.4-grist.1", "@gristlabs/pidusage": "2.0.17", - "@gristlabs/sqlite3": "4.1.1-grist.1", + "@gristlabs/sqlite3": "4.1.1-grist.4", "@popperjs/core": "2.3.3", "accept-language-parser": "1.5.0", "async-mutex": "0.2.4", diff --git a/yarn.lock b/yarn.lock index 1eaa8116..b8b5c036 100644 --- a/yarn.lock +++ b/yarn.lock @@ -41,12 +41,12 @@ dependencies: googleapis-common "^5.0.1" -"@gristlabs/connect-sqlite3@0.9.11-grist.1": - version "0.9.11-grist.1" - resolved "https://registry.yarnpkg.com/@gristlabs/connect-sqlite3/-/connect-sqlite3-0.9.11-grist.1.tgz#a9da7789786e1e32b94cdfb9749360f9eacd79da" - integrity sha512-AJr8y/hRPREM8YdyqV2aMw0yzsORjQyTiG0r0e8JZDJ6uPG/DQiIdTJqARF32dpfbTr6IN8/lXugNDLjqUizPQ== +"@gristlabs/connect-sqlite3@0.9.11-grist.4": + version "0.9.11-grist.4" + resolved "https://registry.yarnpkg.com/@gristlabs/connect-sqlite3/-/connect-sqlite3-0.9.11-grist.4.tgz#64dbd13adefa6830e1c2c8df1eb06ace13de8f47" + integrity sha512-65Oip7d7osR6wWoOJKi+L4duVFJTJCElpeyoYk3HozDokhk/scqmnoBA5zWFWoirwO84Gp9A41CYnSp+UKlDzw== dependencies: - "@gristlabs/sqlite3" "^4.1.1-grist.1" + "@gristlabs/sqlite3" "^4.1.1-grist.4" "@gristlabs/express-session@1.17.0": version "1.17.0" @@ -77,10 +77,10 @@ dependencies: safe-buffer "^5.1.2" -"@gristlabs/sqlite3@4.1.1-grist.1", "@gristlabs/sqlite3@^4.1.1-grist.1": - version "4.1.1-grist.1" - resolved "https://registry.yarnpkg.com/@gristlabs/sqlite3/-/sqlite3-4.1.1-grist.1.tgz#8dfefaec9a1014e73d4ff2f098acdd9676adf5a7" - integrity sha512-pRMoxhLCNKs3r5ltACPMO2hwMeufMq6tz5VRwnk1AJg6qkk01HSO8C2blTOlrxxZUvXJjZSttmO3DYvrZH5UsA== +"@gristlabs/sqlite3@4.1.1-grist.4", "@gristlabs/sqlite3@^4.1.1-grist.4": + version "4.1.1-grist.4" + resolved "https://registry.yarnpkg.com/@gristlabs/sqlite3/-/sqlite3-4.1.1-grist.4.tgz#b909983a33ac66f4a086a18318389ef34c779720" + integrity sha512-/PzqeZJm9bNaqP1hsj3bt0E5q1VFdZHDnqzLYxSJzDl0rxVdGkkOmnlW4cAwjyRCMyvpeXBVAY6U4BS2xehHSg== dependencies: node-addon-api "2.0.0" node-pre-gyp "^0.11.0"