2021-07-21 08:46:03 +00:00
|
|
|
import {drive} from '@googleapis/drive';
|
|
|
|
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
|
|
|
|
import {RequestWithLogin} from 'app/server/lib/Authorizer';
|
|
|
|
import {makeXLSX} from 'app/server/lib/ExportXLSX';
|
|
|
|
import * as log from 'app/server/lib/log';
|
(core) support python3 in grist-core, and running engine via docker and/or gvisor
Summary:
* Moves essential plugins to grist-core, so that basic imports (e.g. csv) work.
* Adds support for a `GRIST_SANDBOX_FLAVOR` flag that can systematically override how the data engine is run.
- `GRIST_SANDBOX_FLAVOR=pynbox` is "classic" nacl-based sandbox.
- `GRIST_SANDBOX_FLAVOR=docker` runs engines in individual docker containers. It requires an image specified in `sandbox/docker` (alternative images can be named with `GRIST_SANDBOX` flag - need to contain python and engine requirements). It is a simple reference implementation for sandboxing.
- `GRIST_SANDBOX_FLAVOR=unsandboxed` runs whatever local version of python is specified by a `GRIST_SANDBOX` flag directly, with no sandboxing. Engine requirements must be installed, so an absolute path to a python executable in a virtualenv is easiest to manage.
- `GRIST_SANDBOX_FLAVOR=gvisor` runs the data engine via gvisor's runsc. Experimental, with implementation not included in grist-core. Since gvisor runs on Linux only, this flavor supports wrapping the sandboxes in a single shared docker container.
* Tweaks some recent express query parameter code to work in grist-core, which has a slightly different version of express (smoke test doesn't catch this since in Jenkins core is built within a workspace that has node_modules, and wires get crossed - in a dev environment the problem on master can be seen by doing `buildtools/build_core.sh /tmp/any_path_outside_grist`).
The new sandbox options do not have tests yet, nor does this they change the behavior of grist servers today. They are there to clean up and consolidate a collection of patches I've been using that were getting cumbersome, and make it easier to run experiments.
I haven't looked closely at imports beyond core.
Test Plan: tested manually against regular grist and grist-core, including imports
Reviewers: alexmojaki, dsagal
Reviewed By: alexmojaki
Differential Revision: https://phab.getgrist.com/D2942
2021-07-27 23:43:21 +00:00
|
|
|
import {optStringParam} from 'app/server/lib/requestUtils';
|
2021-07-21 08:46:03 +00:00
|
|
|
import {Request, Response} from 'express';
|
|
|
|
import {PassThrough} from 'stream';
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Endpoint logic for sending grist document to Google Drive. Grist document is first exported as an
|
|
|
|
* excel file and then pushed to Google Drive as a Google Spreadsheet.
|
|
|
|
*/
|
|
|
|
export async function exportToDrive(
|
|
|
|
activeDoc: ActiveDoc,
|
|
|
|
req: Request,
|
|
|
|
res: Response
|
|
|
|
) {
|
|
|
|
// Token should come from auth middleware
|
(core) support python3 in grist-core, and running engine via docker and/or gvisor
Summary:
* Moves essential plugins to grist-core, so that basic imports (e.g. csv) work.
* Adds support for a `GRIST_SANDBOX_FLAVOR` flag that can systematically override how the data engine is run.
- `GRIST_SANDBOX_FLAVOR=pynbox` is "classic" nacl-based sandbox.
- `GRIST_SANDBOX_FLAVOR=docker` runs engines in individual docker containers. It requires an image specified in `sandbox/docker` (alternative images can be named with `GRIST_SANDBOX` flag - need to contain python and engine requirements). It is a simple reference implementation for sandboxing.
- `GRIST_SANDBOX_FLAVOR=unsandboxed` runs whatever local version of python is specified by a `GRIST_SANDBOX` flag directly, with no sandboxing. Engine requirements must be installed, so an absolute path to a python executable in a virtualenv is easiest to manage.
- `GRIST_SANDBOX_FLAVOR=gvisor` runs the data engine via gvisor's runsc. Experimental, with implementation not included in grist-core. Since gvisor runs on Linux only, this flavor supports wrapping the sandboxes in a single shared docker container.
* Tweaks some recent express query parameter code to work in grist-core, which has a slightly different version of express (smoke test doesn't catch this since in Jenkins core is built within a workspace that has node_modules, and wires get crossed - in a dev environment the problem on master can be seen by doing `buildtools/build_core.sh /tmp/any_path_outside_grist`).
The new sandbox options do not have tests yet, nor does this they change the behavior of grist servers today. They are there to clean up and consolidate a collection of patches I've been using that were getting cumbersome, and make it easier to run experiments.
I haven't looked closely at imports beyond core.
Test Plan: tested manually against regular grist and grist-core, including imports
Reviewers: alexmojaki, dsagal
Reviewed By: alexmojaki
Differential Revision: https://phab.getgrist.com/D2942
2021-07-27 23:43:21 +00:00
|
|
|
const access_token = optStringParam(req.query.access_token);
|
2021-07-21 08:46:03 +00:00
|
|
|
if (!access_token) {
|
|
|
|
throw new Error("No access token - Can't send file to Google Drive");
|
|
|
|
}
|
|
|
|
|
2022-04-08 18:00:43 +00:00
|
|
|
const mreq = req as RequestWithLogin;
|
2021-07-21 08:46:03 +00:00
|
|
|
const meta = {
|
2022-04-08 18:00:43 +00:00
|
|
|
docId: activeDoc.docName,
|
|
|
|
userId: mreq.userId,
|
|
|
|
altSessionId: mreq.altSessionId,
|
2021-07-21 08:46:03 +00:00
|
|
|
};
|
|
|
|
// Prepare file for exporting.
|
|
|
|
log.debug(`Export to drive - Preparing file for export`, meta);
|
|
|
|
const { name, data } = await prepareFile(activeDoc, req);
|
|
|
|
try {
|
|
|
|
// Send file to GDrive and get the url for a preview.
|
|
|
|
const url = await sendFileToDrive(name, data, access_token);
|
|
|
|
log.debug(`Export to drive - File exported, redirecting to Google Spreadsheet ${url}`, meta);
|
|
|
|
res.json({ url });
|
|
|
|
} catch (err) {
|
|
|
|
log.error("Export to drive - Error while sending file to GDrive", meta, err);
|
|
|
|
// Test if google returned a valid error message.
|
|
|
|
if (err.errors && err.errors.length) {
|
|
|
|
throw new Error(err.errors[0].message);
|
|
|
|
} else {
|
|
|
|
throw err;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Creates spreadsheet file in a Google drive, by sending an excel and requesting for conversion.
|
|
|
|
async function sendFileToDrive(fileNameNoExt: string, data: ArrayBuffer, oauth_token: string): Promise<string> {
|
|
|
|
// Here we are asking google drive to convert excel file to a google spreadsheet
|
|
|
|
const requestBody = {
|
|
|
|
// name of the spreadsheet to create
|
|
|
|
name: fileNameNoExt,
|
|
|
|
// mime type of the google spreadsheet
|
|
|
|
mimeType: 'application/vnd.google-apps.spreadsheet'
|
|
|
|
};
|
|
|
|
// wrap buffer into a stream
|
|
|
|
const stream = new PassThrough();
|
|
|
|
stream.end(data);
|
|
|
|
// Define what gets send - excel file
|
|
|
|
const media = {
|
|
|
|
mimeType: 'application/vnd.ms-excel',
|
|
|
|
body: stream
|
|
|
|
};
|
|
|
|
const googleDrive = drive("v3");
|
|
|
|
const fileRes = await googleDrive.files.create({
|
|
|
|
requestBody, // what to do with file - convert to spreadsheet
|
|
|
|
oauth_token, // access token
|
|
|
|
media, // file
|
|
|
|
fields: "webViewLink" // return webViewLink after creating file
|
|
|
|
});
|
|
|
|
const url = fileRes.data.webViewLink;
|
|
|
|
if (!url) {
|
|
|
|
throw new Error("Google Api has not returned valid response");
|
|
|
|
}
|
|
|
|
return url;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Makes excel file the same way as export to excel works.
|
|
|
|
async function prepareFile(doc: ActiveDoc, req: Request) {
|
|
|
|
const data = await makeXLSX(doc, req);
|
(core) support python3 in grist-core, and running engine via docker and/or gvisor
Summary:
* Moves essential plugins to grist-core, so that basic imports (e.g. csv) work.
* Adds support for a `GRIST_SANDBOX_FLAVOR` flag that can systematically override how the data engine is run.
- `GRIST_SANDBOX_FLAVOR=pynbox` is "classic" nacl-based sandbox.
- `GRIST_SANDBOX_FLAVOR=docker` runs engines in individual docker containers. It requires an image specified in `sandbox/docker` (alternative images can be named with `GRIST_SANDBOX` flag - need to contain python and engine requirements). It is a simple reference implementation for sandboxing.
- `GRIST_SANDBOX_FLAVOR=unsandboxed` runs whatever local version of python is specified by a `GRIST_SANDBOX` flag directly, with no sandboxing. Engine requirements must be installed, so an absolute path to a python executable in a virtualenv is easiest to manage.
- `GRIST_SANDBOX_FLAVOR=gvisor` runs the data engine via gvisor's runsc. Experimental, with implementation not included in grist-core. Since gvisor runs on Linux only, this flavor supports wrapping the sandboxes in a single shared docker container.
* Tweaks some recent express query parameter code to work in grist-core, which has a slightly different version of express (smoke test doesn't catch this since in Jenkins core is built within a workspace that has node_modules, and wires get crossed - in a dev environment the problem on master can be seen by doing `buildtools/build_core.sh /tmp/any_path_outside_grist`).
The new sandbox options do not have tests yet, nor does this they change the behavior of grist servers today. They are there to clean up and consolidate a collection of patches I've been using that were getting cumbersome, and make it easier to run experiments.
I haven't looked closely at imports beyond core.
Test Plan: tested manually against regular grist and grist-core, including imports
Reviewers: alexmojaki, dsagal
Reviewed By: alexmojaki
Differential Revision: https://phab.getgrist.com/D2942
2021-07-27 23:43:21 +00:00
|
|
|
const name = (optStringParam(req.query.title) || doc.docName);
|
2021-07-21 08:46:03 +00:00
|
|
|
return { name, data };
|
|
|
|
}
|