gristlabs_grist-core/app/server/lib/DocManager.ts
Alex Hall 02e69fb685 (core) Crudely show row count and limit in UI
Summary:
Add rowCount returned from sandbox when applying user actions to ActionGroup which is broadcast to clients.

Add rowCount to ActiveDoc and update it after applying user actions.

Add rowCount to OpenLocalDocResult using ActiveDoc value, to show when a client opens a doc before any user actions happen.

Add rowCount observable to DocPageModel which is set when the doc is opened and when action groups are received.

Add crude UI (commented out) in Tool.ts showing the row count and the limit in AppModel.currentFeatures. The actual UI doesn't have a place to go yet.

Followup tasks:

- Real, pretty UI
- Counts per table
- Keep count(s) secret from users with limited access?
- Data size indicator?
- Banner when close to or above limit
- Measure row counts outside of sandbox to avoid spoofing with formula
- Handle changes to the limit when the plan is changed or extra rows are purchased

Test Plan: Tested UI manually, including with free team site, opening a fresh doc, opening an initialised doc, adding rows, undoing, and changes from another tab. Automated tests seem like they should wait for a proper UI.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3318
2022-03-14 21:49:32 +02:00

577 lines
24 KiB
TypeScript

import * as pidusage from '@gristlabs/pidusage';
import * as bluebird from 'bluebird';
import {EventEmitter} from 'events';
import * as path from 'path';
import {ApiError} from 'app/common/ApiError';
import {mapSetOrClear} from 'app/common/AsyncCreate';
import {BrowserSettings} from 'app/common/BrowserSettings';
import {DocCreationInfo, DocEntry, DocListAPI, OpenDocMode, OpenLocalDocResult} from 'app/common/DocListAPI';
import {Invite} from 'app/common/sharing';
import {tbind} from 'app/common/tbind';
import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {assertAccess, Authorizer, DocAuthorizer, DummyAuthorizer, isSingleUserMode} from 'app/server/lib/Authorizer';
import {Client} from 'app/server/lib/Client';
import {
getDocSessionCachedDoc,
makeExceptionalDocSession,
makeOptDocSession,
OptDocSession
} from 'app/server/lib/DocSession';
import * as docUtils from 'app/server/lib/docUtils';
import {GristServer} from 'app/server/lib/GristServer';
import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
import {makeForkIds, makeId} from 'app/server/lib/idUtils';
import {checkAllegedGristDoc} from 'app/server/lib/serverUtils';
import * as log from 'app/server/lib/log';
import {ActiveDoc} from './ActiveDoc';
import {PluginManager} from './PluginManager';
import {getFileUploadInfo, globalUploadSet, makeAccessId, UploadInfo} from './uploads';
import noop = require('lodash/noop');
// A TTL in milliseconds to use for material that can easily be recomputed / refetched
// but is a bit of a burden under heavy traffic.
export const DEFAULT_CACHE_TTL = 10000;
/**
* DocManager keeps track of "active" Grist documents, i.e. those loaded
* in-memory, with clients connected to them.
*/
export class DocManager extends EventEmitter {
// Maps docName to promise for ActiveDoc object. Most of the time the promise
// will be long since resolved, with the resulting document cached.
private _activeDocs: Map<string, Promise<ActiveDoc>> = new Map();
constructor(
public readonly storageManager: IDocStorageManager,
public readonly pluginManager: PluginManager,
private _homeDbManager: HomeDBManager|null,
public gristServer: GristServer
) {
super();
}
// attach a home database to the DocManager. During some tests, it
// is awkward to have this set up at the point of construction.
public testSetHomeDbManager(dbManager: HomeDBManager) {
this._homeDbManager = dbManager;
}
public getHomeDbManager() {
return this._homeDbManager;
}
/**
* Returns an implementation of the DocListAPI for the given Client object.
*/
public getDocListAPIImpl(client: Client): DocListAPI {
return {
getDocList: tbind(this.listDocs, this, client),
createNewDoc: tbind(this.createNewDoc, this, client),
importSampleDoc: tbind(this.importSampleDoc, this, client),
importDoc: tbind(this.importDoc, this, client),
deleteDoc: tbind(this.deleteDoc, this, client),
renameDoc: tbind(this.renameDoc, this, client),
openDoc: tbind(this.openDoc, this, client),
};
}
/**
* Returns the number of currently open docs.
*/
public numOpenDocs(): number {
return this._activeDocs.size;
}
/**
* Returns a Map from docId to number of connected clients for each doc.
*/
public async getDocClientCounts(): Promise<Map<string, number>> {
const values = await Promise.all(Array.from(this._activeDocs.values(), async (adocPromise) => {
const adoc = await adocPromise;
return [adoc.docName, adoc.docClients.clientCount()] as [string, number];
}));
return new Map(values);
}
/**
* Returns a promise for all known Grist documents and document invites to show in the doc list.
*/
public async listDocs(client: Client): Promise<{docs: DocEntry[], docInvites: DocEntry[]}> {
const docs = await this.storageManager.listDocs();
return {docs, docInvites: []};
}
/**
* Returns a promise for invites to docs which have not been downloaded.
*/
public async getLocalInvites(client: Client): Promise<Invite[]> {
return [];
}
/**
* Creates a new document, fetches it, and adds a table to it.
* @returns {Promise:String} The name of the new document.
*/
public async createNewDoc(client: Client): Promise<string> {
log.debug('DocManager.createNewDoc');
const docSession = makeExceptionalDocSession('nascent', {client});
return this.createNamedDoc(docSession, 'Untitled');
}
public async createNamedDoc(docSession: OptDocSession, docId: string): Promise<string> {
const activeDoc: ActiveDoc = await this.createNewEmptyDoc(docSession, docId);
await activeDoc.addInitialTable(docSession);
return activeDoc.docName;
}
/**
* Creates a new document, fetches it, and adds a table to it.
* @param {String} sampleDocName: Doc name of a sample document.
* @returns {Promise:String} The name of the new document.
*/
public async importSampleDoc(client: Client, sampleDocName: string): Promise<string> {
const sourcePath = this.storageManager.getSampleDocPath(sampleDocName);
if (!sourcePath) {
throw new Error(`no path available to sample ${sampleDocName}`);
}
log.info('DocManager.importSampleDoc importing', sourcePath);
const basenameHint = path.basename(sampleDocName);
const targetName = await docUtils.createNumbered(basenameHint, '-',
(name: string) => docUtils.createExclusive(this.storageManager.getPath(name)));
const targetPath = this.storageManager.getPath(targetName);
log.info('DocManager.importSampleDoc saving as', targetPath);
await docUtils.copyFile(sourcePath, targetPath);
return targetName;
}
/**
* Processes an upload, containing possibly multiple files, to create a single new document, and
* returns the new document's name/id.
*/
public async importDoc(client: Client, uploadId: number): Promise<string> {
const userId = this._homeDbManager ? await client.requireUserId(this._homeDbManager) : null;
const result = await this._doImportDoc(makeOptDocSession(client),
globalUploadSet.getUploadInfo(uploadId, this.makeAccessId(userId)), {naming: 'classic'});
return result.id;
}
// Import a document, assigning it a unique id distinct from its title. Cleans up uploadId.
public importDocWithFreshId(docSession: OptDocSession, userId: number, uploadId: number): Promise<DocCreationInfo> {
const accessId = this.makeAccessId(userId);
return this._doImportDoc(docSession, globalUploadSet.getUploadInfo(uploadId, accessId),
{naming: 'saved'});
}
// Do an import targeted at a specific workspace. Cleans up uploadId.
// UserId should correspond to the user making the request.
// A workspaceId of null results in an import to an unsaved doc, not
// associated with a specific workspace.
public async importDocToWorkspace(
userId: number, uploadId: number, workspaceId: number|null, browserSettings?: BrowserSettings,
): Promise<DocCreationInfo> {
if (!this._homeDbManager) { throw new Error("HomeDbManager not available"); }
const accessId = this.makeAccessId(userId);
const docSession = makeExceptionalDocSession('nascent', {browserSettings});
const register = async (docId: string, docTitle: string) => {
if (!workspaceId || !this._homeDbManager) { return; }
const queryResult = await this._homeDbManager.addDocument({userId}, workspaceId,
{name: docTitle}, docId);
if (queryResult.status !== 200) {
// TODO The ready-to-add document is not yet in storageManager, but is in the filesystem. It
// should get cleaned up in case of error here.
throw new ApiError(queryResult.errMessage || 'unable to add imported document', queryResult.status);
}
};
return this._doImportDoc(docSession,
globalUploadSet.getUploadInfo(uploadId, accessId), {
naming: workspaceId ? 'saved' : 'unsaved',
register,
userId,
});
// The imported document is associated with the worker that did the import.
// We could break that association (see /api/docs/:docId/assign for how) if
// we start using dedicated import workers.
}
/**
* Imports file at filepath into the app by creating a new document and adding the file to
* the documents directory.
* @param {String} filepath - Path to the current location of the file on the server.
* @returns {Promise:String} The name of the new document.
*/
public async importNewDoc(filepath: string): Promise<DocCreationInfo> {
const uploadId = globalUploadSet.registerUpload([await getFileUploadInfo(filepath)], null, noop, null);
return await this._doImportDoc(makeOptDocSession(null), globalUploadSet.getUploadInfo(uploadId, null),
{naming: 'classic'});
}
/**
* Deletes the Grist files and directories for a given document name.
* @param {String} docName - The name of the Grist document to be deleted.
* @returns {Promise:String} The name of the deleted Grist document.
*
*/
public async deleteDoc(client: Client|null, docName: string, deletePermanently: boolean): Promise<string> {
log.debug('DocManager.deleteDoc starting for %s', docName);
const docPromise = this._activeDocs.get(docName);
if (docPromise) {
// Call activeDoc's shutdown method first, to remove the doc from internal structures.
const doc: ActiveDoc = await docPromise;
await doc.shutdown();
}
await this.storageManager.deleteDoc(docName, deletePermanently);
return docName;
}
/**
* Interrupt all clients, forcing them to reconnect. Handy when a document has changed
* status in some major way that affects access rights, such as being deleted.
*/
public async interruptDocClients(docName: string) {
const docPromise = this._activeDocs.get(docName);
if (docPromise) {
const doc: ActiveDoc = await docPromise;
doc.docClients.interruptAllClients();
}
}
/**
* Opens a document. Adds the client as a subscriber to the document, and fetches and returns the
* document's metadata.
* @returns {Promise:Object} An object with properties:
* `docFD` - the descriptor to use in further methods and messages about this document,
* `doc` - the object with metadata tables.
*/
public async openDoc(client: Client, docId: string,
mode: OpenDocMode = 'default',
linkParameters: Record<string, string> = {}): Promise<OpenLocalDocResult> {
let auth: Authorizer;
const dbManager = this._homeDbManager;
if (!isSingleUserMode()) {
if (!dbManager) { throw new Error("HomeDbManager not available"); }
// Sets up authorization of the document.
const org = client.getOrg();
if (!org) { throw new Error('Documents can only be opened in the context of a specific organization'); }
const userId = await client.getUserId(dbManager) || dbManager.getAnonymousUserId();
// We use docId in the key, and disallow urlId, so we can be sure that we are looking at the
// right doc when we re-query the DB over the life of the websocket.
const key = {urlId: docId, userId, org};
log.debug("DocManager.openDoc Authorizer key", key);
const docAuth = await dbManager.getDocAuthCached(key);
assertAccess('viewers', docAuth);
if (docAuth.docId !== docId) {
// The only plausible way to end up here is if we called openDoc with a urlId rather
// than a docId.
throw new Error(`openDoc expected docId ${docAuth.docId} not urlId ${docId}`);
}
auth = new DocAuthorizer(dbManager, key, mode, linkParameters, docAuth, client.getProfile() || undefined);
} else {
log.debug(`DocManager.openDoc not using authorization for ${docId} because GRIST_SINGLE_USER`);
auth = new DummyAuthorizer('owners', docId);
}
// Fetch the document, and continue when we have the ActiveDoc (which may be immediately).
const docSessionPrecursor = makeOptDocSession(client);
docSessionPrecursor.authorizer = auth;
return this._withUnmutedDoc(docSessionPrecursor, docId, async () => {
const activeDoc: ActiveDoc = await this.fetchDoc(docSessionPrecursor, docId);
// Get a fresh DocSession object.
const docSession = activeDoc.addClient(client, auth);
// If opening in (pre-)fork mode, check if it is appropriate to treat the user as
// an owner for granular access purposes.
if (mode === 'fork') {
if (await activeDoc.canForkAsOwner(docSession)) {
// Mark the session specially and flush any cached access
// information. It is easier to make this a property of the
// session than to try computing it later in the heat of
// battle, since it introduces a loop where a user property
// (user.Access) depends on evaluating rules, but rules need
// the user properties in order to be evaluated. It is also
// somewhat justifiable even if permissions change later on
// the theory that the fork is theoretically happening at this
// instance).
docSession.forkingAsOwner = true;
activeDoc.flushAccess(docSession);
} else {
// TODO: it would be kind to pass on a message to the client
// to let them know they won't be able to fork. They'll get
// an error when they make their first change. But currently
// we only have the blunt instrument of throwing an error,
// which would prevent access to the document entirely.
}
}
const [metaTables, recentActions, userOverride, rowCount] = await Promise.all([
activeDoc.fetchMetaTables(docSession),
activeDoc.getRecentMinimalActions(docSession),
activeDoc.getUserOverride(docSession),
activeDoc.getRowCount(docSession),
]);
const result = {
docFD: docSession.fd,
clientId: docSession.client.clientId,
doc: metaTables,
log: recentActions,
recoveryMode: activeDoc.recoveryMode,
userOverride,
rowCount,
} as OpenLocalDocResult;
if (!activeDoc.muted) {
this.emit('open-doc', this.storageManager.getPath(activeDoc.docName));
}
return {activeDoc, result};
});
}
/**
* Shut down all open docs. This is called, in particular, on server shutdown.
*/
public async shutdownAll() {
await Promise.all(Array.from(this._activeDocs.values(),
adocPromise => adocPromise.then(adoc => adoc.shutdown())));
try {
await this.storageManager.closeStorage();
} catch (err) {
log.error('DocManager had problem shutting down storage: %s', err.message);
}
// Clear the setInterval that the pidusage module sets up internally.
pidusage.clear();
}
// Access a document by name.
public getActiveDoc(docName: string): Promise<ActiveDoc>|undefined {
return this._activeDocs.get(docName);
}
public removeActiveDoc(activeDoc: ActiveDoc): void {
this._activeDocs.delete(activeDoc.docName);
}
public async renameDoc(client: Client, oldName: string, newName: string): Promise<void> {
log.debug('DocManager.renameDoc %s -> %s', oldName, newName);
const docPromise = this._activeDocs.get(oldName);
if (docPromise) {
const adoc: ActiveDoc = await docPromise;
await adoc.renameDocTo({client}, newName);
this._activeDocs.set(newName, docPromise);
this._activeDocs.delete(oldName);
} else {
await this.storageManager.renameDoc(oldName, newName);
}
}
public markAsChanged(activeDoc: ActiveDoc, reason?: 'edit') {
// Ignore changes if document is muted or in the middle of a migration.
if (!activeDoc.muted && !activeDoc.isMigrating()) {
this.storageManager.markAsChanged(activeDoc.docName, reason);
}
}
public async makeBackup(activeDoc: ActiveDoc, name: string): Promise<string> {
if (activeDoc.muted) { throw new Error('Document is disabled'); }
return this.storageManager.makeBackup(activeDoc.docName, name);
}
/**
* Helper function for creating a new empty document that also emits an event.
* @param docSession The client session.
* @param basenameHint Suggested base name to use (no directory, no extension).
*/
public async createNewEmptyDoc(docSession: OptDocSession, basenameHint: string): Promise<ActiveDoc> {
const docName = await this._createNewDoc(basenameHint);
return mapSetOrClear(this._activeDocs, docName,
this._createActiveDoc(docSession, docName)
.then(newDoc => newDoc.createEmptyDoc(docSession)));
}
/**
* Fetches an ActiveDoc object. Used by openDoc. If ActiveDoc is muted (for safe closing),
* wait for another.
*/
public async fetchDoc(docSession: OptDocSession, docName: string,
wantRecoveryMode?: boolean): Promise<ActiveDoc> {
log.debug('DocManager.fetchDoc', docName);
return this._withUnmutedDoc(docSession, docName, async () => {
const activeDoc = await this._fetchPossiblyMutedDoc(docSession, docName, wantRecoveryMode);
return {activeDoc, result: activeDoc};
});
}
public makeAccessId(userId: number|null): string|null {
return makeAccessId(this.gristServer, userId);
}
public isAnonymous(userId: number): boolean {
if (!this._homeDbManager) { throw new Error("HomeDbManager not available"); }
return userId === this._homeDbManager.getAnonymousUserId();
}
/**
* Perform the supplied operation and return its result - unless the activeDoc it returns
* is found to be muted, in which case we retry.
*/
private async _withUnmutedDoc<T>(docSession: OptDocSession, docName: string,
op: () => Promise<{ result: T, activeDoc: ActiveDoc }>): Promise<T> {
// Repeat until we acquire an ActiveDoc that is not muted (shutting down).
for (;;) {
const { result, activeDoc } = await op();
if (!activeDoc.muted) { return result; }
log.debug('DocManager._withUnmutedDoc waiting because doc is muted', docName);
await bluebird.delay(1000);
}
}
// Like fetchDoc(), but doesn't check if ActiveDoc returned is unmuted.
private async _fetchPossiblyMutedDoc(docSession: OptDocSession, docName: string,
wantRecoveryMode?: boolean): Promise<ActiveDoc> {
if (this._activeDocs.has(docName) && wantRecoveryMode !== undefined) {
const activeDoc = await this._activeDocs.get(docName);
if (activeDoc && activeDoc.recoveryMode !== wantRecoveryMode && await activeDoc.isOwner(docSession)) {
// shutting doc down to have a chance to re-open in the correct mode.
// TODO: there could be a battle with other users opening it in a different mode.
await activeDoc.shutdown();
}
}
let activeDoc: ActiveDoc;
if (!this._activeDocs.has(docName)) {
activeDoc = await mapSetOrClear(
this._activeDocs, docName,
this._createActiveDoc(docSession, docName, wantRecoveryMode)
.then(newDoc => {
// Propagate backupMade events from newly opened activeDocs (consolidate all to DocMan)
newDoc.on('backupMade', (bakPath: string) => {
this.emit('backupMade', bakPath);
});
return newDoc.loadDoc(docSession);
}));
} else {
activeDoc = await this._activeDocs.get(docName)!;
}
return activeDoc;
}
private async _createActiveDoc(docSession: OptDocSession, docName: string, safeMode?: boolean) {
// Get URL for document for use with SELF_HYPERLINK().
const cachedDoc = getDocSessionCachedDoc(docSession);
let docUrl: string|undefined;
try {
if (cachedDoc) {
docUrl = await this.gristServer.getResourceUrl(cachedDoc);
} else {
docUrl = await this.gristServer.getDocUrl(docName);
}
} catch (e) {
// If there is no home url, we cannot construct links. Accept this, for the benefit
// of legacy tests.
if (!String(e).match(/need APP_HOME_URL/)) {
throw e;
}
}
return this.gristServer.create.ActiveDoc(this, docName, {docUrl, safeMode});
}
/**
* Helper that implements doing the actual import of an uploaded set of files to create a new
* document.
*/
private async _doImportDoc(docSession: OptDocSession, uploadInfo: UploadInfo,
options: {
naming: 'classic'|'saved'|'unsaved',
register?: (docId: string, docTitle: string) => Promise<void>,
userId?: number,
}): Promise<DocCreationInfo> {
try {
const fileCount = uploadInfo.files.length;
const hasGristDoc = Boolean(uploadInfo.files.find(f => extname(f.origName) === '.grist'));
if (hasGristDoc && fileCount > 1) {
throw new Error('Grist docs must be uploaded individually');
}
const first = uploadInfo.files[0].origName;
const ext = extname(first);
const basename = path.basename(first, ext).trim() || "Untitled upload";
let id: string;
switch (options.naming) {
case 'saved':
id = makeId();
break;
case 'unsaved': {
const {userId} = options;
if (!userId) { throw new Error('unsaved import requires userId'); }
if (!this._homeDbManager) { throw new Error("HomeDbManager not available"); }
const isAnonymous = userId === this._homeDbManager.getAnonymousUserId();
id = makeForkIds({userId, isAnonymous, trunkDocId: NEW_DOCUMENT_CODE,
trunkUrlId: NEW_DOCUMENT_CODE}).docId;
break;
}
case 'classic':
id = basename;
break;
default:
throw new Error('naming mode not recognized');
}
await options.register?.(id, basename);
if (ext === '.grist') {
// If the import is a grist file, copy it to the docs directory.
// TODO: We should be skeptical of the upload file to close a possible
// security vulnerability. See https://phab.getgrist.com/T457.
const docName = await this._createNewDoc(id);
const docPath: string = this.storageManager.getPath(docName);
const srcDocPath = uploadInfo.files[0].absPath;
await checkAllegedGristDoc(docSession, srcDocPath);
await docUtils.copyFile(srcDocPath, docPath);
// Go ahead and claim this document. If we wanted to serve it
// from a potentially different worker, we'd call addToStorage(docName)
// instead (we used to do this). The upload should already be happening
// on a randomly assigned worker due to the special treatment of the
// 'import' assignmentId.
await this.storageManager.prepareLocalDoc(docName);
this.storageManager.markAsChanged(docName, 'edit');
return {title: basename, id: docName};
} else {
const doc = await this.createNewEmptyDoc(docSession, id);
await doc.oneStepImport(docSession, uploadInfo);
return {title: basename, id: doc.docName};
}
} catch (err) {
throw new ApiError(err.message, err.status || 400, {
tips: [{action: 'ask-for-help', message: 'Ask for help'}]
});
} finally {
await globalUploadSet.cleanup(uploadInfo.uploadId);
}
}
// Returns the name for a new doc, based on basenameHint.
private async _createNewDoc(basenameHint: string): Promise<string> {
const docName: string = await docUtils.createNumbered(basenameHint, '-', async (name: string) => {
if (this._activeDocs.has(name)) {
throw new Error("Existing entry in active docs for: " + name);
}
return docUtils.createExclusive(this.storageManager.getPath(name));
});
log.debug('DocManager._createNewDoc picked name', docName);
await this.pluginManager.pluginsLoaded;
return docName;
}
}
// Returns the extension of fpath (from last occurrence of "." to the end of the string), even
// when the basename is empty or starts with a period.
function extname(fpath: string): string {
return path.extname("X" + fpath);
}