mirror of
https://github.com/gristlabs/grist-core.git
synced 2025-06-13 20:53:59 +00:00
(core) Regularly log data size in DocStorage.applyStoredActions using sqlite dbstat
Summary: - Small cleanup: Make DocStorage implement OnDemandStorage, and remove unused execWithBackup - Upgrade to new versions (.3) of @gristlabs/sqlite3 and connect-sqlite3 to use dbstat - Add _logDataSize method which queries dbstat, adding up pgsize for tables loaded into the data engine - Only complete _logDataSize every 5 minutes using new field _lastLoggedDataSize Test Plan: Tested manually Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3277
This commit is contained in:
Alex Hall
parent
4c935e7fb3
commit
f1002c0e67
@ -11,6 +11,7 @@ import * as sqlite3 from '@gristlabs/sqlite3';
|
||||
import {LocalActionBundle} from 'app/common/ActionBundle';
|
||||
import {BulkColValues, DocAction, TableColValues, TableDataAction, toTableDataAction} from 'app/common/DocActions';
|
||||
import * as gristTypes from 'app/common/gristTypes';
|
||||
import {isList} from 'app/common/gristTypes';
|
||||
import * as marshal from 'app/common/marshal';
|
||||
import * as schema from 'app/common/schema';
|
||||
import {GristObjCode} from "app/plugin/GristData";
|
||||
@ -21,13 +22,13 @@ import * as log from 'app/server/lib/log';
|
||||
import * as assert from 'assert';
|
||||
import * as bluebird from 'bluebird';
|
||||
import * as fse from 'fs-extra';
|
||||
import chunk = require('lodash/chunk');
|
||||
import groupBy = require('lodash/groupBy');
|
||||
import * as _ from 'underscore';
|
||||
import * as util from 'util';
|
||||
import * as uuidv4 from "uuid/v4";
|
||||
import { ISQLiteDB, MigrationHooks, OpenMode, quoteIdent, ResultRow, SchemaInfo, SQLiteDB} from './SQLiteDB';
|
||||
import {isList} from "app/common/gristTypes";
|
||||
import {OnDemandStorage} from './OnDemandActions';
|
||||
import {ISQLiteDB, MigrationHooks, OpenMode, quoteIdent, ResultRow, SchemaInfo, SQLiteDB} from './SQLiteDB';
|
||||
import chunk = require('lodash/chunk');
|
||||
import groupBy = require('lodash/groupBy');
|
||||
|
||||
|
||||
// Run with environment variable NODE_DEBUG=db (may include additional comma-separated sections)
|
||||
@ -38,7 +39,7 @@ const maxSQLiteVariables = 500; // Actually could be 999, so this is playing
|
||||
|
||||
const PENDING_VALUE = [GristObjCode.Pending];
|
||||
|
||||
export class DocStorage implements ISQLiteDB {
|
||||
export class DocStorage implements ISQLiteDB, OnDemandStorage {
|
||||
|
||||
// ======================================================================
|
||||
// Static fields
|
||||
@ -623,6 +624,9 @@ export class DocStorage implements ISQLiteDB {
|
||||
// tables (obtained from auto-generated schema.js).
|
||||
private _docSchema: {[tableId: string]: {[colId: string]: string}};
|
||||
|
||||
// The last time _logDataSize ran fully
|
||||
private _lastLoggedDataSize: number = Date.now();
|
||||
|
||||
public constructor(public storageManager: IDocStorageManager, public docName: string) {
|
||||
this.docPath = this.storageManager.getPath(docName);
|
||||
this._db = null;
|
||||
@ -652,27 +656,6 @@ export class DocStorage implements ISQLiteDB {
|
||||
// Note that we don't call _updateMetadata() as there are no metadata tables yet anyway.
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a backup and calls cb() within a transaction. Returns the backup path. In case of
|
||||
* failure, adds .backupPath property to the error object (and transaction is rolled back).
|
||||
*/
|
||||
public execWithBackup(cb: (db: SQLiteDB) => Promise<void>): Promise<string> {
|
||||
let backupPath: string;
|
||||
return this.storageManager.makeBackup(this.docName, "migrate-db")
|
||||
.then((_backupPath: string) => {
|
||||
backupPath = _backupPath;
|
||||
log.info(`DocStorage[${this.docName}]: backup made at ${backupPath}`);
|
||||
return this.execTransaction(cb);
|
||||
})
|
||||
.then(() => backupPath)
|
||||
.catch((err: any) => {
|
||||
// TODO: deal with typing for this kind of error (although nothing seems to depend
|
||||
// on it yet.
|
||||
err.backupPath = backupPath;
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the database with proper settings.
|
||||
*/
|
||||
@ -915,7 +898,7 @@ export class DocStorage implements ISQLiteDB {
|
||||
public async applyStoredActions(docActions: DocAction[]): Promise<void> {
|
||||
debuglog('DocStorage.applyStoredActions');
|
||||
|
||||
return bluebird.Promise.each(docActions, (action: DocAction) => {
|
||||
await bluebird.Promise.each(docActions, (action: DocAction) => {
|
||||
const actionType = action[0];
|
||||
const f = (this as any)["_process_" + actionType];
|
||||
if (!_.isFunction(f)) {
|
||||
@ -935,6 +918,7 @@ export class DocStorage implements ISQLiteDB {
|
||||
});
|
||||
}
|
||||
});
|
||||
this._logDataSize().catch(e => log.error(`Error in _logDataSize: ${e}`));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -1562,6 +1546,25 @@ export class DocStorage implements ISQLiteDB {
|
||||
`${joinClauses} ${whereClause} ${limitClause}`;
|
||||
return sql;
|
||||
}
|
||||
|
||||
private async _logDataSize() {
|
||||
// To reduce overhead, don't query and log data size more than once in 5 minutes
|
||||
const now = Date.now();
|
||||
if (now - this._lastLoggedDataSize < 5 * 60 * 1000) {
|
||||
return;
|
||||
}
|
||||
this._lastLoggedDataSize = now;
|
||||
|
||||
const result = await this.get(`
|
||||
SELECT SUM(pgsize) AS totalSize
|
||||
FROM dbstat
|
||||
WHERE NOT (
|
||||
name LIKE 'sqlite_%' OR
|
||||
name LIKE '_gristsys_%'
|
||||
);
|
||||
`);
|
||||
log.rawInfo("Data size from dbstat...", {docId: this.docName, dataSize: result!.totalSize});
|
||||
}
|
||||
}
|
||||
|
||||
interface RebuildResult {
|
||||
|
||||
@ -76,11 +76,11 @@
|
||||
"dependencies": {
|
||||
"@googleapis/drive": "0.3.1",
|
||||
"@googleapis/oauth2": "0.2.0",
|
||||
"@gristlabs/connect-sqlite3": "0.9.11-grist.1",
|
||||
"@gristlabs/connect-sqlite3": "0.9.11-grist.4",
|
||||
"@gristlabs/express-session": "1.17.0",
|
||||
"@gristlabs/moment-guess": "1.2.4-grist.1",
|
||||
"@gristlabs/pidusage": "2.0.17",
|
||||
"@gristlabs/sqlite3": "4.1.1-grist.1",
|
||||
"@gristlabs/sqlite3": "4.1.1-grist.4",
|
||||
"@popperjs/core": "2.3.3",
|
||||
"accept-language-parser": "1.5.0",
|
||||
"async-mutex": "0.2.4",
|
||||
|
||||
18
yarn.lock
18
yarn.lock
@ -41,12 +41,12 @@
|
||||
dependencies:
|
||||
googleapis-common "^5.0.1"
|
||||
|
||||
"@gristlabs/connect-sqlite3@0.9.11-grist.1":
|
||||
version "0.9.11-grist.1"
|
||||
resolved "https://registry.yarnpkg.com/@gristlabs/connect-sqlite3/-/connect-sqlite3-0.9.11-grist.1.tgz#a9da7789786e1e32b94cdfb9749360f9eacd79da"
|
||||
integrity sha512-AJr8y/hRPREM8YdyqV2aMw0yzsORjQyTiG0r0e8JZDJ6uPG/DQiIdTJqARF32dpfbTr6IN8/lXugNDLjqUizPQ==
|
||||
"@gristlabs/connect-sqlite3@0.9.11-grist.4":
|
||||
version "0.9.11-grist.4"
|
||||
resolved "https://registry.yarnpkg.com/@gristlabs/connect-sqlite3/-/connect-sqlite3-0.9.11-grist.4.tgz#64dbd13adefa6830e1c2c8df1eb06ace13de8f47"
|
||||
integrity sha512-65Oip7d7osR6wWoOJKi+L4duVFJTJCElpeyoYk3HozDokhk/scqmnoBA5zWFWoirwO84Gp9A41CYnSp+UKlDzw==
|
||||
dependencies:
|
||||
"@gristlabs/sqlite3" "^4.1.1-grist.1"
|
||||
"@gristlabs/sqlite3" "^4.1.1-grist.4"
|
||||
|
||||
"@gristlabs/express-session@1.17.0":
|
||||
version "1.17.0"
|
||||
@ -77,10 +77,10 @@
|
||||
dependencies:
|
||||
safe-buffer "^5.1.2"
|
||||
|
||||
"@gristlabs/sqlite3@4.1.1-grist.1", "@gristlabs/sqlite3@^4.1.1-grist.1":
|
||||
version "4.1.1-grist.1"
|
||||
resolved "https://registry.yarnpkg.com/@gristlabs/sqlite3/-/sqlite3-4.1.1-grist.1.tgz#8dfefaec9a1014e73d4ff2f098acdd9676adf5a7"
|
||||
integrity sha512-pRMoxhLCNKs3r5ltACPMO2hwMeufMq6tz5VRwnk1AJg6qkk01HSO8C2blTOlrxxZUvXJjZSttmO3DYvrZH5UsA==
|
||||
"@gristlabs/sqlite3@4.1.1-grist.4", "@gristlabs/sqlite3@^4.1.1-grist.4":
|
||||
version "4.1.1-grist.4"
|
||||
resolved "https://registry.yarnpkg.com/@gristlabs/sqlite3/-/sqlite3-4.1.1-grist.4.tgz#b909983a33ac66f4a086a18318389ef34c779720"
|
||||
integrity sha512-/PzqeZJm9bNaqP1hsj3bt0E5q1VFdZHDnqzLYxSJzDl0rxVdGkkOmnlW4cAwjyRCMyvpeXBVAY6U4BS2xehHSg==
|
||||
dependencies:
|
||||
node-addon-api "2.0.0"
|
||||
node-pre-gyp "^0.11.0"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user