gristlabs_grist-core/app/server/lib/DocManager.ts

484 lines
20 KiB
TypeScript
Raw Normal View History

import * as pidusage from '@gristlabs/pidusage';
import * as bluebird from 'bluebird';
import {EventEmitter} from 'events';
import noop = require('lodash/noop');
import * as path from 'path';
import {ApiError} from 'app/common/ApiError';
import {mapSetOrClear} from 'app/common/AsyncCreate';
import {BrowserSettings} from 'app/common/BrowserSettings';
import {DocCreationInfo, DocEntry, DocListAPI, OpenDocMode, OpenLocalDocResult} from 'app/common/DocListAPI';
import {EncActionBundleFromHub} from 'app/common/EncActionBundle';
import {Invite} from 'app/common/sharing';
import {tbind} from 'app/common/tbind';
import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {assertAccess, Authorizer, DocAuthorizer, DummyAuthorizer,
isSingleUserMode} from 'app/server/lib/Authorizer';
import {Client} from 'app/server/lib/Client';
import {makeExceptionalDocSession, makeOptDocSession, OptDocSession} from 'app/server/lib/DocSession';
import * as docUtils from 'app/server/lib/docUtils';
import {GristServer} from 'app/server/lib/GristServer';
import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
import {makeForkIds, makeId} from 'app/server/lib/idUtils';
import * as log from 'app/server/lib/log';
import {ActiveDoc} from './ActiveDoc';
import {PluginManager} from './PluginManager';
import {getFileUploadInfo, globalUploadSet, makeAccessId, UploadInfo} from './uploads';
// A TTL in milliseconds to use for material that can easily be recomputed / refetched
// but is a bit of a burden under heavy traffic.
export const DEFAULT_CACHE_TTL = 10000;
/**
* DocManager keeps track of "active" Grist documents, i.e. those loaded
* in-memory, with clients connected to them.
*/
export class DocManager extends EventEmitter {
// Maps docName to promise for ActiveDoc object. Most of the time the promise
// will be long since resolved, with the resulting document cached.
private _activeDocs: Map<string, Promise<ActiveDoc>> = new Map();
constructor(
public readonly storageManager: IDocStorageManager,
public readonly pluginManager: PluginManager,
private _homeDbManager: HomeDBManager|null,
public gristServer: GristServer
) {
super();
}
// attach a home database to the DocManager. During some tests, it
// is awkward to have this set up at the point of construction.
public testSetHomeDbManager(dbManager: HomeDBManager) {
this._homeDbManager = dbManager;
}
/**
* Returns an implementation of the DocListAPI for the given Client object.
*/
public getDocListAPIImpl(client: Client): DocListAPI {
return {
getDocList: tbind(this.listDocs, this, client),
createNewDoc: tbind(this.createNewDoc, this, client),
importSampleDoc: tbind(this.importSampleDoc, this, client),
importDoc: tbind(this.importDoc, this, client),
deleteDoc: tbind(this.deleteDoc, this, client),
renameDoc: tbind(this.renameDoc, this, client),
openDoc: tbind(this.openDoc, this, client),
};
}
/**
* Returns the number of currently open docs.
*/
public numOpenDocs(): number {
return this._activeDocs.size;
}
/**
* Returns a Map from docId to number of connected clients for each doc.
*/
public async getDocClientCounts(): Promise<Map<string, number>> {
const values = await Promise.all(Array.from(this._activeDocs.values(), async (adocPromise) => {
const adoc = await adocPromise;
return [adoc.docName, adoc.docClients.clientCount()] as [string, number];
}));
return new Map(values);
}
/**
* Returns a promise for all known Grist documents and document invites to show in the doc list.
*/
public async listDocs(client: Client): Promise<{docs: DocEntry[], docInvites: DocEntry[]}> {
const docs = await this.storageManager.listDocs();
return {docs, docInvites: []};
}
/**
* Returns a promise for invites to docs which have not been downloaded.
*/
public async getLocalInvites(client: Client): Promise<Invite[]> {
return [];
}
/**
* Creates a new document, fetches it, and adds a table to it.
* @returns {Promise:String} The name of the new document.
*/
public async createNewDoc(client: Client): Promise<string> {
log.debug('DocManager.createNewDoc');
const docSession = makeExceptionalDocSession('nascent', {client});
const activeDoc: ActiveDoc = await this.createNewEmptyDoc(docSession, 'Untitled');
await activeDoc.addInitialTable(docSession);
return activeDoc.docName;
}
/**
* Download a shared doc by creating a new doc and applying to it the shared doc snapshot actions.
* Also marks the invite to the doc as ignored, since it has already been accepted.
* @returns {Promise:String} The name of the new document.
*/
public async downloadSharedDoc(client: Client, docId: string, docName: string): Promise<string> {
throw new Error('downloadSharedDoc not implemented');
}
/**
* Creates a new document, fetches it, and adds a table to it.
* @param {String} sampleDocName: Doc name of a sample document.
* @returns {Promise:String} The name of the new document.
*/
public async importSampleDoc(client: Client, sampleDocName: string): Promise<string> {
const sourcePath = this.storageManager.getSampleDocPath(sampleDocName);
if (!sourcePath) {
throw new Error(`no path available to sample ${sampleDocName}`);
}
log.info('DocManager.importSampleDoc importing', sourcePath);
const basenameHint = path.basename(sampleDocName);
const targetName = await docUtils.createNumbered(basenameHint, '-',
(name: string) => docUtils.createExclusive(this.storageManager.getPath(name)));
const targetPath = this.storageManager.getPath(targetName);
log.info('DocManager.importSampleDoc saving as', targetPath);
await docUtils.copyFile(sourcePath, targetPath);
return targetName;
}
/**
* Processes an upload, containing possibly multiple files, to create a single new document, and
* returns the new document's name/id.
*/
public async importDoc(client: Client, uploadId: number): Promise<string> {
const userId = this._homeDbManager ? await client.requireUserId(this._homeDbManager) : null;
const result = await this._doImportDoc(makeOptDocSession(client),
globalUploadSet.getUploadInfo(uploadId, this.makeAccessId(userId)), {naming: 'classic'});
return result.id;
}
// Import a document, assigning it a unique id distinct from its title. Cleans up uploadId.
public importDocWithFreshId(docSession: OptDocSession, userId: number, uploadId: number): Promise<DocCreationInfo> {
const accessId = this.makeAccessId(userId);
return this._doImportDoc(docSession, globalUploadSet.getUploadInfo(uploadId, accessId),
{naming: 'saved'});
}
// Do an import targeted at a specific workspace. Cleans up uploadId.
// UserId should correspond to the user making the request.
// A workspaceId of null results in an import to an unsaved doc, not
// associated with a specific workspace.
public async importDocToWorkspace(
userId: number, uploadId: number, workspaceId: number|null, browserSettings?: BrowserSettings,
): Promise<DocCreationInfo> {
if (!this._homeDbManager) { throw new Error("HomeDbManager not available"); }
const accessId = this.makeAccessId(userId);
const docSession = makeExceptionalDocSession('nascent', {browserSettings});
const result = await this._doImportDoc(docSession,
globalUploadSet.getUploadInfo(uploadId, accessId), {
naming: workspaceId ? 'saved' : 'unsaved',
userId,
});
if (workspaceId) {
const queryResult = await this._homeDbManager.addDocument({userId}, workspaceId,
{name: result.title}, result.id);
if (queryResult.status !== 200) {
// TODO The ready-to-add document is not yet in storageManager, but is in the filesystem. It
// should get cleaned up in case of error here.
throw new ApiError(queryResult.errMessage || 'unable to add imported document', queryResult.status);
}
}
// Ship the import to S3, since it isn't associated with any particular worker at this time.
// We could associate it with the current worker, but that is not necessarily desirable.
await this.storageManager.addToStorage(result.id);
return result;
}
/**
* Imports file at filepath into the app by creating a new document and adding the file to
* the documents directory.
* @param {String} filepath - Path to the current location of the file on the server.
* @returns {Promise:String} The name of the new document.
*/
public async importNewDoc(filepath: string): Promise<DocCreationInfo> {
const uploadId = globalUploadSet.registerUpload([await getFileUploadInfo(filepath)], null, noop, null);
return await this._doImportDoc(makeOptDocSession(null), globalUploadSet.getUploadInfo(uploadId, null),
{naming: 'classic'});
}
/**
* Deletes the Grist files and directories for a given document name.
* @param {String} docName - The name of the Grist document to be deleted.
* @returns {Promise:String} The name of the deleted Grist document.
*
*/
public async deleteDoc(client: Client|null, docName: string, deletePermanently: boolean): Promise<string> {
log.debug('DocManager.deleteDoc starting for %s', docName);
const docPromise = this._activeDocs.get(docName);
if (docPromise) {
// Call activeDoc's shutdown method first, to remove the doc from internal structures.
const doc: ActiveDoc = await docPromise;
await doc.shutdown();
}
await this.storageManager.deleteDoc(docName, deletePermanently);
return docName;
}
/**
* Interrupt all clients, forcing them to reconnect. Handy when a document has changed
* status in some major way that affects access rights, such as being deleted.
*/
public async interruptDocClients(docName: string) {
const docPromise = this._activeDocs.get(docName);
if (docPromise) {
const doc: ActiveDoc = await docPromise;
doc.docClients.interruptAllClients();
}
}
/**
* Opens a document. Adds the client as a subscriber to the document, and fetches and returns the
* document's metadata.
* @returns {Promise:Object} An object with properties:
* `docFD` - the descriptor to use in further methods and messages about this document,
* `doc` - the object with metadata tables.
*/
public async openDoc(client: Client, docId: string,
mode: OpenDocMode = 'default'): Promise<OpenLocalDocResult> {
let auth: Authorizer;
const dbManager = this._homeDbManager;
if (!isSingleUserMode()) {
if (!dbManager) { throw new Error("HomeDbManager not available"); }
// Sets up authorization of the document.
const org = client.getOrg();
if (!org) { throw new Error('Documents can only be opened in the context of a specific organization'); }
const userId = await client.getUserId(dbManager) || dbManager.getAnonymousUserId();
// We use docId in the key, and disallow urlId, so we can be sure that we are looking at the
// right doc when we re-query the DB over the life of the websocket.
const key = {urlId: docId, userId, org};
log.debug("DocManager.openDoc Authorizer key", key);
const docAuth = await dbManager.getDocAuthCached(key);
assertAccess('viewers', docAuth);
if (docAuth.docId !== docId) {
// The only plausible way to end up here is if we called openDoc with a urlId rather
// than a docId.
throw new Error(`openDoc expected docId ${docAuth.docId} not urlId ${docId}`);
}
(core) support adding user characteristic tables for granular ACLs Summary: This is a prototype for expanding the conditions that can be used in granular ACLs. When processing ACLs, the following variables (called "characteristics") are now available in conditions: * UserID * Email * Name * Access (owners, editors, viewers) The set of variables can be expanded by adding a "characteristic" clause. This is a clause which specifies: * A tableId * The name of an existing characteristic * A colId The effect of the clause is to expand the available characteristics with all the columns in the table, with values taken from the record where there is a match between the specified characteristic and the specified column. Existing clauses are generalized somewhat to demonstrate and test the use these variables. That isn't the main point of this diff though, and I propose to leave generalizing+systematizing those clauses for a future diff. Issues I'm not dealing with here: * How clauses combine. (The scope on GranularAccessRowClause is a hack to save me worrying about that yet). * The full set of matching methods we'll allow. * Refreshing row access in clients when the tables mentioned in characteristic tables change. * Full CRUD permission control. * Default rules (part of combination). * Reporting errors in access rules. That said, with this diff it is possible to e.g. assign a City to editors by their email address or name, and have only rows for those Cities be visible in their client. Ability to modify those rows, and remain updates about them, remains under incomplete control. Test Plan: added tests Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2642
2020-10-19 14:25:21 +00:00
auth = new DocAuthorizer(dbManager, key, mode, docAuth, client.getProfile() || undefined);
} else {
log.debug(`DocManager.openDoc not using authorization for ${docId} because GRIST_SINGLE_USER`);
auth = new DummyAuthorizer('owners', docId);
}
// Fetch the document, and continue when we have the ActiveDoc (which may be immediately).
const docSessionPrecursor = makeOptDocSession(client);
docSessionPrecursor.authorizer = auth;
const activeDoc: ActiveDoc = await this.fetchDoc(docSessionPrecursor, docId);
if (activeDoc.muted) {
log.debug('DocManager.openDoc interrupting, called for a muted doc', docId);
client.interruptConnection();
throw new Error(`document ${docId} cannot be opened right now`);
}
const docSession = activeDoc.addClient(client, auth);
const [metaTables, recentActions] = await Promise.all([
activeDoc.fetchMetaTables(docSession),
activeDoc.getRecentActions(docSession, false)
]);
this.emit('open-doc', this.storageManager.getPath(activeDoc.docName));
return {
docFD: docSession.fd,
clientId: docSession.client.clientId,
doc: metaTables,
log: recentActions,
plugins: activeDoc.docPluginManager.getPlugins()
};
}
/**
* Shut down all open docs. This is called, in particular, on server shutdown.
*/
public async shutdownAll() {
await Promise.all(Array.from(this._activeDocs.values(),
adocPromise => adocPromise.then(adoc => adoc.shutdown())));
try {
await this.storageManager.closeStorage();
} catch (err) {
log.error('DocManager had problem shutting down storage: %s', err.message);
}
// Clear the setInterval that the pidusage module sets up internally.
pidusage.clear();
}
// Access a document by name.
public getActiveDoc(docName: string): Promise<ActiveDoc>|undefined {
return this._activeDocs.get(docName);
}
public async removeActiveDoc(activeDoc: ActiveDoc): Promise<void> {
this._activeDocs.delete(activeDoc.docName);
}
public async renameDoc(client: Client, oldName: string, newName: string): Promise<void> {
log.debug('DocManager.renameDoc %s -> %s', oldName, newName);
const docPromise = this._activeDocs.get(oldName);
if (docPromise) {
const adoc: ActiveDoc = await docPromise;
await adoc.renameDocTo({client}, newName);
this._activeDocs.set(newName, docPromise);
this._activeDocs.delete(oldName);
} else {
await this.storageManager.renameDoc(oldName, newName);
}
}
public markAsChanged(activeDoc: ActiveDoc, reason?: 'edit') {
// Ignore changes if document is muted or in the middle of a migration.
if (!activeDoc.muted && !activeDoc.isMigrating()) {
this.storageManager.markAsChanged(activeDoc.docName, reason);
}
}
public async makeBackup(activeDoc: ActiveDoc, name: string): Promise<string> {
if (activeDoc.muted) { throw new Error('Document is disabled'); }
return this.storageManager.makeBackup(activeDoc.docName, name);
}
/**
* Helper function for creating a new empty document that also emits an event.
* @param docSession The client session.
* @param basenameHint Suggested base name to use (no directory, no extension).
*/
public async createNewEmptyDoc(docSession: OptDocSession, basenameHint: string): Promise<ActiveDoc> {
const docName = await this._createNewDoc(basenameHint);
return mapSetOrClear(this._activeDocs, docName,
this.gristServer.create.ActiveDoc(this, docName).createDoc(docSession));
}
/**
* Fetches an ActiveDoc object. Used by openDoc.
*/
public async fetchDoc(docSession: OptDocSession, docName: string): Promise<ActiveDoc> {
log.debug('DocManager.fetchDoc', docName);
// Repeat until we acquire an ActiveDoc that is not muted (shutting down).
for (;;) {
if (!this._activeDocs.has(docName)) {
const newDoc = this.gristServer.create.ActiveDoc(this, docName);
// Propagate backupMade events from newly opened activeDocs (consolidate all to DocMan)
newDoc.on('backupMade', (bakPath: string) => {
this.emit('backupMade', bakPath);
});
return mapSetOrClear(this._activeDocs, docName, newDoc.loadDoc(docSession));
}
const activeDoc = await this._activeDocs.get(docName)!;
if (!activeDoc.muted) { return activeDoc; }
log.debug('DocManager.fetchDoc waiting because doc is muted', docName);
await bluebird.delay(1000);
}
}
public makeAccessId(userId: number|null): string|null {
return makeAccessId(this.gristServer, userId);
}
/**
* Helper function for creating a new shared document given the doc snapshot bundles received
* from the sharing hub.
* @param {String} basenameHint: Suggested base name to use (no directory, no extension).
* @param {String} docId: The docId of the doc received from the hub.
* @param {String} instanceId: The user instanceId creating the doc.
* @param {EncActionBundleFromHub[]} encBundles: The action bundles making up the doc snapshot.
* @returns {Promise:ActiveDoc} ActiveDoc for the newly created document.
*/
protected async _createNewSharedDoc(basenameHint: string, docId: string, instanceId: string,
encBundles: EncActionBundleFromHub[]): Promise<ActiveDoc> {
const docName = await this._createNewDoc(basenameHint);
return mapSetOrClear(this._activeDocs, docName,
this.gristServer.create.ActiveDoc(this, docName).downloadSharedDoc(docId, instanceId, encBundles));
}
/**
* Helper that implements doing the actual import of an uploaded set of files to create a new
* document.
*/
private async _doImportDoc(docSession: OptDocSession, uploadInfo: UploadInfo,
options: {
naming: 'classic'|'saved'|'unsaved',
userId?: number,
}): Promise<DocCreationInfo> {
try {
const fileCount = uploadInfo.files.length;
const hasGristDoc = Boolean(uploadInfo.files.find(f => extname(f.origName) === '.grist'));
if (hasGristDoc && fileCount > 1) {
throw new Error('Grist docs must be uploaded individually');
}
const first = uploadInfo.files[0].origName;
const ext = extname(first);
const basename = path.basename(first, ext).trim() || "Untitled upload";
let id: string;
switch (options.naming) {
case 'saved':
id = makeId();
break;
case 'unsaved': {
const {userId} = options;
if (!userId) { throw new Error('unsaved import requires userId'); }
if (!this._homeDbManager) { throw new Error("HomeDbManager not available"); }
const isAnonymous = userId === this._homeDbManager.getAnonymousUserId();
id = makeForkIds({userId, isAnonymous, trunkDocId: NEW_DOCUMENT_CODE,
trunkUrlId: NEW_DOCUMENT_CODE}).docId;
break;
}
case 'classic':
id = basename;
break;
default:
throw new Error('naming mode not recognized');
}
if (ext === '.grist') {
// If the import is a grist file, copy it to the docs directory.
// TODO: We should be skeptical of the upload file to close a possible
// security vulnerability. See https://phab.getgrist.com/T457.
const docName = await this._createNewDoc(id);
const docPath = await this.storageManager.getPath(docName);
await docUtils.copyFile(uploadInfo.files[0].absPath, docPath);
return {title: basename, id: docName};
} else {
const doc = await this.createNewEmptyDoc(docSession, id);
await doc.oneStepImport(docSession, uploadInfo);
return {title: basename, id: doc.docName};
}
} catch (err) {
throw new ApiError(err.message, err.status || 400, {
tips: [{action: 'ask-for-help', message: 'Ask for help'}]
});
} finally {
await globalUploadSet.cleanup(uploadInfo.uploadId);
}
}
// Returns the name for a new doc, based on basenameHint.
private async _createNewDoc(basenameHint: string): Promise<string> {
const docName: string = await docUtils.createNumbered(basenameHint, '-', async (name: string) => {
if (this._activeDocs.has(name)) {
throw new Error("Existing entry in active docs for: " + name);
}
return docUtils.createExclusive(this.storageManager.getPath(name));
});
log.debug('DocManager._createNewDoc picked name', docName);
await this.pluginManager.pluginsLoaded;
return docName;
}
}
// Returns the extension of fpath (from last occurrence of "." to the end of the string), even
// when the basename is empty or starts with a period.
function extname(fpath: string): string {
return path.extname("X" + fpath);
}