(core) Extending Google Drive integration scope

Summary:
New environmental variable GOOGLE_DRIVE_SCOPE that modifies the scope
requested for Google Drive integration.
For prod it has value https://www.googleapis.com/auth/drive.file which leaves
current behavior (Grist is allowed only to access public files and for private
files - it fallbacks to Picker).
For staging it has value https://www.googleapis.com/auth/drive.readonly which
allows Grist to access all private files, and fallbacks to Picker only when the file is
neither public nor private).
Default value is https://www.googleapis.com/auth/drive.file

Test Plan: manual and existing tests

Reviewers: dsagal

Reviewed By: dsagal

Subscribers: dsagal

Differential Revision: https://phab.getgrist.com/D3038
This commit is contained in:
Jarosław Sadziński 2021-09-30 10:19:22 +02:00
parent a0c53f2b61
commit 42910cb8f7
13 changed files with 302 additions and 131 deletions

View File

@ -20,11 +20,12 @@ import {cssModalButtons, cssModalTitle} from 'app/client/ui2018/modals';
import {DataSourceTransformed, ImportResult, ImportTableResult, MergeOptions, import {DataSourceTransformed, ImportResult, ImportTableResult, MergeOptions,
MergeStrategy, TransformColumn, TransformRule, TransformRuleMap} from "app/common/ActiveDocAPI"; MergeStrategy, TransformColumn, TransformRule, TransformRuleMap} from "app/common/ActiveDocAPI";
import {byteString} from "app/common/gutil"; import {byteString} from "app/common/gutil";
import {UploadResult} from 'app/common/uploads'; import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
import {ParseOptions, ParseOptionSchema} from 'app/plugin/FileParserAPI'; import {ParseOptions, ParseOptionSchema} from 'app/plugin/FileParserAPI';
import {Computed, Disposable, dom, DomContents, IDisposable, MutableObsArray, obsArray, Observable, import {Computed, Disposable, dom, DomContents, IDisposable, MutableObsArray, obsArray, Observable,
styled} from 'grainjs'; styled} from 'grainjs';
import {labeledSquareCheckbox} from "app/client/ui2018/checkbox"; import {labeledSquareCheckbox} from "app/client/ui2018/checkbox";
import {ACCESS_DENIED, AUTH_INTERRUPTED, canReadPrivateFiles, getGoogleCodeForReading} from "app/client/ui/googleAuth";
// Special values for import destinations; null means "new table". // Special values for import destinations; null means "new table".
// TODO We should also support "skip table" (needs server support), so that one can open, say, // TODO We should also support "skip table" (needs server support), so that one can open, say,
@ -47,7 +48,7 @@ export interface SourceInfo {
transformSection: Observable<ViewSectionRec>; transformSection: Observable<ViewSectionRec>;
destTableId: Observable<DestId>; destTableId: Observable<DestId>;
} }
// UI state of selected merge options for each source table (from SourceInfo). // UI state of selected merge options for each source table (from SourceInfo).
interface MergeOptionsState { interface MergeOptionsState {
[srcTableId: string]: { [srcTableId: string]: {
updateExistingRecords: Observable<boolean>; updateExistingRecords: Observable<boolean>;
@ -182,14 +183,10 @@ export class Importer extends Disposable {
uploadResult = await uploadFiles(files, {docWorkerUrl: this._docComm.docWorkerUrl, uploadResult = await uploadFiles(files, {docWorkerUrl: this._docComm.docWorkerUrl,
sizeLimit: 'import'}); sizeLimit: 'import'});
} else if (item.kind === "url") { } else if (item.kind === "url") {
try {
uploadResult = await fetchURL(this._docComm, item.url);
} catch(err) {
if (isDriveUrl(item.url)) { if (isDriveUrl(item.url)) {
throw new GDriveUrlNotSupported(item.url); uploadResult = await this._fetchFromDrive(item.url);
} else { } else {
throw err; uploadResult = await fetchURL(this._docComm, item.url);
}
} }
} else { } else {
throw new Error(`Import source of kind ${(item as any).kind} are not yet supported!`); throw new Error(`Import source of kind ${(item as any).kind} are not yet supported!`);
@ -197,6 +194,10 @@ export class Importer extends Disposable {
} }
} }
} catch (err) { } catch (err) {
if (err instanceof CancelledError) {
await this._cancelImport();
return;
}
if (err instanceof GDriveUrlNotSupported) { if (err instanceof GDriveUrlNotSupported) {
await this._cancelImport(); await this._cancelImport();
throw err; throw err;
@ -488,15 +489,55 @@ export class Importer extends Disposable {
) )
]); ]);
} }
private async _fetchFromDrive(itemUrl: string) {
// First we will assume that this is public file, so no need to ask for permissions.
try {
return await fetchURL(this._docComm, itemUrl);
} catch(err) {
// It is not a public file or the file id in the url is wrong,
// but we have no way to check it, so we assume that it is private file
// and ask the user for the permission (if we are configured to do so)
if (canReadPrivateFiles()) {
const options: FetchUrlOptions = {};
try {
// Request for authorization code from Google.
const code = await getGoogleCodeForReading(this);
options.googleAuthorizationCode = code;
} catch(permError) {
if (permError?.message === ACCESS_DENIED) {
// User declined to give us full readonly permission, fallback to GoogleDrive plugin
// or cancel import if GoogleDrive plugin is not configured.
throw new GDriveUrlNotSupported(itemUrl);
} else if(permError?.message === AUTH_INTERRUPTED) {
// User closed the window - we assume he doesn't want to continue.
throw new CancelledError();
} else {
// Some other error happened during authentication, report to user.
throw err;
}
}
// Download file from private drive, if it fails, report the error to user.
return await fetchURL(this._docComm, itemUrl, options);
} else {
// We are not allowed to ask for full readonly permission, fallback to GoogleDrive plugin.
throw new GDriveUrlNotSupported(itemUrl);
}
}
}
} }
// Used for switching from URL plugin to Google drive plugin // Used for switching from URL plugin to Google drive plugin.
class GDriveUrlNotSupported extends Error { class GDriveUrlNotSupported extends Error {
constructor(public url: string) { constructor(public url: string) {
super(`This url ${url} is not supported`); super(`This url ${url} is not supported`);
} }
} }
// Used to cancel import (close the dialog without any error).
class CancelledError extends Error {
}
function getSourceDescription(sourceInfo: SourceInfo, upload: UploadResult) { function getSourceDescription(sourceInfo: SourceInfo, upload: UploadResult) {
const origName = upload.files[sourceInfo.uploadFileIndex].origName; const origName = upload.files[sourceInfo.uploadFileIndex].origName;
return sourceInfo.origTableName ? origName + ' - ' + sourceInfo.origTableName : origName; return sourceInfo.origTableName ? origName + ' - ' + sourceInfo.origTableName : origName;

View File

@ -13,7 +13,7 @@ import {FileDialogOptions, openFilePicker} from 'app/client/ui/FileDialog';
import {BaseAPI} from 'app/common/BaseAPI'; import {BaseAPI} from 'app/common/BaseAPI';
import {GristLoadConfig} from 'app/common/gristUrls'; import {GristLoadConfig} from 'app/common/gristUrls';
import {byteString, safeJsonParse} from 'app/common/gutil'; import {byteString, safeJsonParse} from 'app/common/gutil';
import {UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads'; import {FetchUrlOptions, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
import {docUrl} from 'app/common/urlUtils'; import {docUrl} from 'app/common/urlUtils';
import {OpenDialogOptions} from 'electron'; import {OpenDialogOptions} from 'electron';
import noop = require('lodash/noop'); import noop = require('lodash/noop');
@ -152,12 +152,12 @@ export async function uploadFiles(
* case, it guesses the name of the file based on the response's content-type and the url. * case, it guesses the name of the file based on the response's content-type and the url.
*/ */
export async function fetchURL( export async function fetchURL(
docComm: DocComm, url: string, onProgress: ProgressCB = noop docComm: DocComm, url: string, options?: FetchUrlOptions, onProgress: ProgressCB = noop
): Promise<UploadResult> { ): Promise<UploadResult> {
if (isDriveUrl(url)) { if (isDriveUrl(url)) {
// don't download from google drive, immediately fallback to server side. // don't download from google drive, immediately fallback to server side.
return docComm.fetchURL(url); return docComm.fetchURL(url, options);
} }
let response: Response; let response: Response;
@ -167,14 +167,14 @@ export async function fetchURL(
console.log( // tslint:disable-line:no-console console.log( // tslint:disable-line:no-console
`Could not fetch ${url} on the Client, falling back to server fetch: ${err.message}` `Could not fetch ${url} on the Client, falling back to server fetch: ${err.message}`
); );
return docComm.fetchURL(url); return docComm.fetchURL(url, options);
} }
// TODO: We should probably parse response.headers.get('content-disposition') when available // TODO: We should probably parse response.headers.get('content-disposition') when available
// (see content-disposition npm module). // (see content-disposition npm module).
const fileName = basename(url); const fileName = basename(url);
const mimeType = response.headers.get('content-type'); const mimeType = response.headers.get('content-type');
const options = mimeType ? { type: mimeType } : {}; const fileOptions = mimeType ? { type: mimeType } : {};
const fileObj = new File([await response.blob()], fileName, options); const fileObj = new File([await response.blob()], fileName, fileOptions);
const res = await uploadFiles([fileObj], {docWorkerUrl: docComm.docWorkerUrl}, onProgress); const res = await uploadFiles([fileObj], {docWorkerUrl: docComm.docWorkerUrl}, onProgress);
return res!; return res!;
} }

159
app/client/ui/googleAuth.ts Normal file
View File

@ -0,0 +1,159 @@
import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
import type {Disposable} from 'grainjs';
import { GristLoadConfig } from "app/common/gristUrls";
/**
* Functions to perform server side authentication with Google.
*
* The authentication flow is performed by server side (app/server/lib/GoogleAuth.ts). Here we will
* open up a popup with a stub html file (served by the server), that will redirect user to Google
* Auth Service. In return, we will get authorization_code (which will be delivered by a postMessage
* from the iframe), that when converted to authorization_token, can be used to access Google Drive
* API. Accessing Google Drive files is done by the server, here we only ask for the permissions.
*
* Exposed methods are:
* - getGoogleCodeForSending: asks google for a permission to create files on the drive (and read
* them)
* - getGoogleCodeForReading: asks google for a permission to read all files
* - canReadPrivateFiles: Grist by default won't ask for permission to read all files, but can be
* configured this way by an environmental variable.
*/
const G = getBrowserGlobals('window');
export const ACCESS_DENIED = "access_denied";
export const AUTH_INTERRUPTED = "auth_interrupted";
// https://developers.google.com/identity/protocols/oauth2/scopes#drive
// "View and manage Google Drive files and folders that you have opened or created with this app"
const APP_SCOPE = "https://www.googleapis.com/auth/drive.file";
// "See and download all your Google Drive files"
const READ_SCOPE = "https://www.googleapis.com/auth/drive.readonly";
export function getGoogleCodeForSending(owner: Disposable) {
return getGoogleAuthCode(owner, APP_SCOPE);
}
export function getGoogleCodeForReading(owner: Disposable) {
const gristConfig: GristLoadConfig = (window as any).gristConfig || {};
// Default scope allows as to manage files we created.
return getGoogleAuthCode(owner, gristConfig.googleDriveScope || APP_SCOPE);
}
/**
* Checks if default scope for Google Drive integration will allow to access all personal files
*/
export function canReadPrivateFiles() {
const gristConfig: GristLoadConfig = (window as any).gristConfig || {};
return gristConfig.googleDriveScope === READ_SCOPE;
}
/**
* Opens up a popup with server side Google Authentication. Returns a code that can be used
* by server side to retrieve access_token required in Google Api.
*/
function getGoogleAuthCode(owner: Disposable, scope: string) {
// Compute google auth server endpoint (grist endpoint for server side google authentication).
// This endpoint renders a page that redirects user to Google Consent screen and after Google
// sends a response, it will post this response back to us.
// Message will be an object { code, error }.
const authLink = getGoogleAuthEndpoint(scope);
const authWindow = openPopup(authLink);
return new Promise<string>((resolve, reject) => {
attachListener(owner, authWindow, async (event: MessageEvent|null) => {
// If the no message, or window was closed (user closed it intentionally).
if (!event || authWindow.closed) {
reject(new Error(AUTH_INTERRUPTED));
return;
}
// For the first message (we expect only a single message) close the window.
authWindow.close();
if (owner.isDisposed()) {
reject(new Error(AUTH_INTERRUPTED));
return;
}
// Check response from the popup
const response = (event.data || {}) as {code?: string, error?: string};
// - when user declined, report back, caller should stop current flow,
if (response.error === "access_denied") {
reject(new Error(ACCESS_DENIED));
return;
}
// - when there is no authorization, or error is different from what we expected - report to user.
if (!response.code) {
reject(new Error(response.error || "Missing authorization code"));
return;
}
resolve(response.code);
});
});
}
// Helper function that attaches a handler to message event from a popup window.
function attachListener(owner: Disposable, popup: Window, listener: (e: MessageEvent|null) => void) {
const wrapped = (e: MessageEvent) => {
// Listen to events only from our window.
if (e.source !== popup) { return; }
// In case when Grist was reloaded or user navigated away - do nothing.
if (owner.isDisposed()) { return; }
listener(e);
// Clear the listener, to avoid orphaned calls from closed event.
listener = () => {};
};
// Unfortunately there is no ease way to detect if user has closed the popup.
const closeHandler = onClose(popup, () => {
listener(null);
// Clear the listener, to avoid orphaned messages from window.
listener = () => {};
});
owner.onDispose(closeHandler);
G.window.addEventListener('message', wrapped);
owner.onDispose(() => {
G.window.removeEventListener('message', wrapped);
});
}
// Periodically checks if the window is closed.
// Returns a function that can be used to cancel the event.
function onClose(window: Window, clb: () => void) {
const interval = setInterval(() => {
if (window.closed) {
clearInterval(interval);
clb();
}
}, 1000);
return () => clearInterval(interval);
}
function openPopup(url: string): Window {
// Center window on desktop
// https://stackoverflow.com/questions/16363474/window-open-on-a-multi-monitor-dual-monitor-system-where-does-window-pop-up
const width = 600;
const height = 650;
const left = window.screenX + (screen.width - width) / 2;
const top = (screen.height - height) / 4;
let windowFeatures = `top=${top},left=${left},menubar=no,location=no,` +
`resizable=yes,scrollbars=yes,status=yes,height=${height},width=${width}`;
// If window will be too large (for example on mobile) - open as a new tab
if (screen.width <= width || screen.height <= height) {
windowFeatures = '';
}
const authWindow = G.window.open(url, "GoogleAuthPopup", windowFeatures);
if (!authWindow) {
// This method should be invoked by an user action.
throw new Error("This method should be invoked synchronously");
}
return authWindow;
}
/**
* Generates Google Auth endpoint (exposed by Grist) url. For example:
* https://docs.getgrist.com/auth/google
* @param scope Requested access scope for Google Services:
* https://developers.google.com/identity/protocols/oauth2/scopes
*/
function getGoogleAuthEndpoint(scope: string) {
return new URL(`auth/google?scope=${scope}`, window.location.origin).href;
}

View File

@ -1,30 +1,18 @@
import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals'; import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
import {reportError} from 'app/client/models/errors'; import {reportError} from 'app/client/models/errors';
import {spinnerModal} from 'app/client/ui2018/modals'; import {spinnerModal} from 'app/client/ui2018/modals';
import type { DocPageModel } from 'app/client/models/DocPageModel'; import type {DocPageModel} from 'app/client/models/DocPageModel';
import type { Document } from 'app/common/UserAPI'; import type {Document} from 'app/common/UserAPI';
import type { Disposable } from 'grainjs'; import { getGoogleCodeForSending } from "app/client/ui/googleAuth";
const G = getBrowserGlobals('window'); const G = getBrowserGlobals('window');
/** /**
* Generates Google Auth endpoint (exposed by Grist) url. For example: * Sends xlsx file to Google Drive. It first authenticates with Google to get encrypted access
* https://docs.getgrist.com/auth/google * token, then it calls "send-to-drive" api endpoint to upload xlsx file to drive and finally it
* @param scope Requested access scope for Google Services * redirects to the created spreadsheet. Code that is received from Google contains encrypted access
* https://developers.google.com/identity/protocols/oauth2/scopes * token, server is able to decrypt it using GOOGLE_CLIENT_SECRET key.
*/ */
function getGoogleAuthEndpoint(scope?: string) { export async function sendToDrive(doc: Document, pageModel: DocPageModel) {
return new URL(`auth/google?scope=${scope || ''}`, window.location.origin).href;
}
/**
* Sends xlsx file to Google Drive. It first authenticates with Google to get encrypted access token,
* then it calls "send-to-drive" api endpoint to upload xlsx file to drive and finally it redirects
* to the created spreadsheet.
* Code that is received from Google contains encrypted access token, server is able to decrypt it
* using GOOGLE_CLIENT_SECRET key.
*/
export function sendToDrive(doc: Document, pageModel: DocPageModel) {
// Get current document - it will be used to remove popup listener. // Get current document - it will be used to remove popup listener.
const gristDoc = pageModel.gristDoc.get(); const gristDoc = pageModel.gristDoc.get();
// Sanity check - gristDoc should be always present // Sanity check - gristDoc should be always present
@ -38,68 +26,11 @@ export function sendToDrive(doc: Document, pageModel: DocPageModel) {
.sendToDrive(code, pageModel.currentDocTitle.get()) .sendToDrive(code, pageModel.currentDocTitle.get())
); );
// Compute google auth server endpoint (grist endpoint for server side google authentication).
// This endpoint will redirect user to Google Consent screen and after Google sends a response,
// it will render a page (/static/message.html) that will post a message containing message
// from Google. Message will be an object { code, error }. We will use the code to invoke
// "send-to-drive" api endpoint - that will actually send the xlsx file to Google Drive.
const authLink = getGoogleAuthEndpoint();
const authWindow = openPopup(authLink);
attachListener(gristDoc, authWindow, async (event: MessageEvent) => {
// For the first message (we expect only a single message) close the window.
authWindow.close();
// Check response from the popup
const response = (event.data || {}) as { code?: string, error?: string };
// - when user declined, do nothing,
if (response.error === "access_denied") { return; }
// - when there is no response code or error code is different from what we expected - report to user.
if (!response.code) { reportError(response.error || "Unrecognized or empty error code"); return; }
// Send file to Google Drive.
try { try {
const { url } = await send(response.code); const token = await getGoogleCodeForSending(gristDoc);
const {url} = await send(token);
G.window.location.assign(url); G.window.location.assign(url);
} catch (err) { } catch (err) {
reportError(err); reportError(err);
} }
});
}
// Helper function that attaches a handler to message event from a popup window.
function attachListener(owner: Disposable, popup: Window, listener: (e: MessageEvent) => any) {
const wrapped = (e: MessageEvent) => {
// Listen to events only from our window.
if (e.source !== popup) { return; }
// In case when Grist was reloaded or user navigated away - do nothing.
if (owner.isDisposed()) { return; }
listener(e);
};
G.window.addEventListener('message', wrapped);
owner.onDispose(() => {
G.window.removeEventListener('message', wrapped);
});
}
function openPopup(url: string): Window {
// Center window on desktop
// https://stackoverflow.com/questions/16363474/window-open-on-a-multi-monitor-dual-monitor-system-where-does-window-pop-up
const width = 600;
const height = 650;
const left = window.screenX + (screen.width - width) / 2;
const top = (screen.height - height) / 4;
let windowFeatures = `top=${top},left=${left},menubar=no,location=no,` +
`resizable=yes,scrollbars=yes,status=yes,height=${height},width=${width}`;
// If window will be too large (for example on mobile) - open as a new tab
if (screen.width <= width || screen.height <= height) {
windowFeatures = '';
}
const authWindow = G.window.open(url, "GoogleAuthPopup", windowFeatures);
if (!authWindow) {
// This method should be invoked by an user action.
throw new Error("This method should be invoked synchronously");
}
return authWindow;
} }

View File

@ -1,7 +1,7 @@
import {ActionGroup} from 'app/common/ActionGroup'; import {ActionGroup} from 'app/common/ActionGroup';
import {CellValue, TableDataAction, UserAction} from 'app/common/DocActions'; import {CellValue, TableDataAction, UserAction} from 'app/common/DocActions';
import {FormulaProperties} from 'app/common/GranularAccessClause'; import {FormulaProperties} from 'app/common/GranularAccessClause';
import {UploadResult} from 'app/common/uploads'; import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
import {ParseOptions} from 'app/plugin/FileParserAPI'; import {ParseOptions} from 'app/plugin/FileParserAPI';
import {IMessage} from 'grain-rpc'; import {IMessage} from 'grain-rpc';
@ -207,7 +207,7 @@ export interface ActiveDocAPI {
/** /**
* Fetch content at a url. * Fetch content at a url.
*/ */
fetchURL(url: string): Promise<UploadResult>; fetchURL(url: string, options?: FetchUrlOptions): Promise<UploadResult>;
/** /**
* Find and return a list of auto-complete suggestions that start with `txt`, when editing a * Find and return a list of auto-complete suggestions that start with `txt`, when editing a

View File

@ -456,6 +456,12 @@ export interface GristLoadConfig {
// Google Client Id, used in Google integration (ex: Google Drive Plugin) // Google Client Id, used in Google integration (ex: Google Drive Plugin)
googleClientId?: string; googleClientId?: string;
// Max scope we can request for accessing files from Google Drive.
// Default used by Grist is https://www.googleapis.com/auth/drive.file:
// View and manage Google Drive files and folders that you have opened or created with this app.
// More on scopes: https://developers.google.com/identity/protocols/oauth2/scopes#drive
googleDriveScope?: string;
// List of registered plugins (used by HomePluginManager and DocPluginManager) // List of registered plugins (used by HomePluginManager and DocPluginManager)
plugins?: LocalPlugin[]; plugins?: LocalPlugin[];
} }

View File

@ -42,3 +42,12 @@ export interface FileUploadResult {
* the page's <base> will be respected. * the page's <base> will be respected.
*/ */
export const UPLOAD_URL_PATH = 'uploads'; export const UPLOAD_URL_PATH = 'uploads';
/**
* Additional options for fetching external resources.
*/
export interface FetchUrlOptions {
googleAuthorizationCode?: string; // The authorization code received from Google Auth Service.
fileName?: string; // The filename for external resource.
headers?: {[key: string]: string}; // Additional headers to use when accessing external resource.
}

View File

@ -44,7 +44,7 @@ import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccess
import {byteString, countIf, safeJsonParse} from 'app/common/gutil'; import {byteString, countIf, safeJsonParse} from 'app/common/gutil';
import {InactivityTimer} from 'app/common/InactivityTimer'; import {InactivityTimer} from 'app/common/InactivityTimer';
import * as marshal from 'app/common/marshal'; import * as marshal from 'app/common/marshal';
import {UploadResult} from 'app/common/uploads'; import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
import {DocReplacementOptions, DocState} from 'app/common/UserAPI'; import {DocReplacementOptions, DocState} from 'app/common/UserAPI';
import {ParseOptions} from 'app/plugin/FileParserAPI'; import {ParseOptions} from 'app/plugin/FileParserAPI';
import {GristDocAPI} from 'app/plugin/GristAPI'; import {GristDocAPI} from 'app/plugin/GristAPI';
@ -890,8 +890,8 @@ export class ActiveDoc extends EventEmitter {
return this._pyCall('autocomplete', txt, tableId, columnId, user.toJSON()); return this._pyCall('autocomplete', txt, tableId, columnId, user.toJSON());
} }
public fetchURL(docSession: DocSession, url: string): Promise<UploadResult> { public fetchURL(docSession: DocSession, url: string, options?: FetchUrlOptions): Promise<UploadResult> {
return fetchURL(url, this.makeAccessId(docSession.authorizer.getUserId())); return fetchURL(url, this.makeAccessId(docSession.authorizer.getUserId()), options);
} }
public async forwardPluginRpc(docSession: DocSession, pluginId: string, msg: IMessage): Promise<any> { public async forwardPluginRpc(docSession: DocSession, pluginId: string, msg: IMessage): Promise<any> {

View File

@ -16,6 +16,8 @@ import {URL} from 'url';
* the same client id is used in Google Drive Plugin * the same client id is used in Google Drive Plugin
* - GOOGLE_CLIENT_SECRET: secret key for Google Project, can't be shared - it is used to (d)encrypt * - GOOGLE_CLIENT_SECRET: secret key for Google Project, can't be shared - it is used to (d)encrypt
* data that we obtain from Google Auth Service (all done in the api) * data that we obtain from Google Auth Service (all done in the api)
* - GOOGLE_DRIVE_SCOPE: scope requested for the Google drive integration (defaults to drive.file which allows
* to create files and get files via Google Drive Picker)
* *
* High level description: * High level description:
* *
@ -55,13 +57,9 @@ import {URL} from 'url';
* in a query string. * in a query string.
*/ */
// Path for the auth endpoint. // Path for the auth endpoint.
const authHandlerPath = "/auth/google"; const authHandlerPath = "/auth/google";
// "View and manage Google Drive files and folders that you have opened or created with this app.""
export const DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.file';
// Redirect host after the Google Auth login form is completed. This reuses the same domain name // Redirect host after the Google Auth login form is completed. This reuses the same domain name
// as for Cognito login. // as for Cognito login.
const AUTH_SUBDOMAIN = process.env.GRIST_ID_PREFIX ? `docs-${process.env.GRIST_ID_PREFIX}` : 'docs'; const AUTH_SUBDOMAIN = process.env.GRIST_ID_PREFIX ? `docs-${process.env.GRIST_ID_PREFIX}` : 'docs';
@ -97,7 +95,7 @@ export async function googleAuthTokenMiddleware(
throw new ApiError("Google Auth endpoint requires a code parameter in the query string", 400); throw new ApiError("Google Auth endpoint requires a code parameter in the query string", 400);
} else { } else {
try { try {
const oAuth2Client = _googleAuthClient(); const oAuth2Client = getGoogleAuth();
// Decrypt code that was send back from Google Auth service. Uses GOOGLE_CLIENT_SECRET key. // Decrypt code that was send back from Google Auth service. Uses GOOGLE_CLIENT_SECRET key.
const tokenResponse = await oAuth2Client.getToken(stringParam(req.query.code)); const tokenResponse = await oAuth2Client.getToken(stringParam(req.query.code));
// Get the access token (access token will be present in a default request configuration). // Get the access token (access token will be present in a default request configuration).
@ -150,8 +148,8 @@ export function addGoogleAuthEndpoint(
throw new ApiError("Error authenticating with Google", 500); throw new ApiError("Error authenticating with Google", 500);
} }
} else { } else {
const oAuth2Client = _googleAuthClient(); const oAuth2Client = getGoogleAuth();
const scope = optStringParam(req.query.scope) || DRIVE_SCOPE; const scope = stringParam(req.query.scope);
// Create url for origin parameter for a popup window. // Create url for origin parameter for a popup window.
const origin = getOriginUrl(req); const origin = getOriginUrl(req);
const authUrl = oAuth2Client.generateAuthUrl({ const authUrl = oAuth2Client.generateAuthUrl({
@ -172,7 +170,7 @@ export function addGoogleAuthEndpoint(
/** /**
* Builds the OAuth2 Google client. * Builds the OAuth2 Google client.
*/ */
function _googleAuthClient() { export function getGoogleAuth() {
const CLIENT_ID = process.env.GOOGLE_CLIENT_ID; const CLIENT_ID = process.env.GOOGLE_CLIENT_ID;
const CLIENT_SECRET = process.env.GOOGLE_CLIENT_SECRET; const CLIENT_SECRET = process.env.GOOGLE_CLIENT_SECRET;
const oAuth2Client = new auth.OAuth2(CLIENT_ID, CLIENT_SECRET, getFullAuthEndpointUrl()); const oAuth2Client = new auth.OAuth2(CLIENT_ID, CLIENT_SECRET, getFullAuthEndpointUrl());

View File

@ -2,14 +2,15 @@ import {drive} from '@googleapis/drive';
import {Readable} from 'form-data'; import {Readable} from 'form-data';
import {GaxiosError, GaxiosPromise} from 'gaxios'; import {GaxiosError, GaxiosPromise} from 'gaxios';
import {FetchError, Response as FetchResponse, Headers} from 'node-fetch'; import {FetchError, Response as FetchResponse, Headers} from 'node-fetch';
import {getGoogleAuth} from "app/server/lib/GoogleAuth";
import * as contentDisposition from 'content-disposition';
const const
SPREADSHEETS_MIMETYPE = 'application/vnd.google-apps.spreadsheet', SPREADSHEETS_MIMETYPE = 'application/vnd.google-apps.spreadsheet',
XLSX_MIMETYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; XLSX_MIMETYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
export async function downloadFromGDrive(url: string) { export async function downloadFromGDrive(url: string, code?: string) {
const fileId = fileIdFromUrl(url); const fileId = fileIdFromUrl(url);
const googleDrive = drive("v3");
const key = process.env.GOOGLE_API_KEY; const key = process.env.GOOGLE_API_KEY;
if (!key) { if (!key) {
throw new Error("Can't download file from Google Drive. Api key is not configured"); throw new Error("Can't download file from Google Drive. Api key is not configured");
@ -17,29 +18,51 @@ export async function downloadFromGDrive(url: string) {
if (!fileId) { if (!fileId) {
throw new Error(`Can't download from ${url}. Url is not valid`); throw new Error(`Can't download from ${url}. Url is not valid`);
} }
const googleDrive = await initDriveApi(code);
const fileRes = await googleDrive.files.get({ const fileRes = await googleDrive.files.get({
key, key,
fileId fileId
}); });
if (fileRes.data.mimeType === SPREADSHEETS_MIMETYPE) { if (fileRes.data.mimeType === SPREADSHEETS_MIMETYPE) {
let filename = fileRes.data.name;
if (filename && !filename.includes(".")) {
filename = `${filename}.xlsx`;
}
return await asFetchResponse(googleDrive.files.export( return await asFetchResponse(googleDrive.files.export(
{ key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE }, {key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE},
{ responseType: 'stream' } {responseType: 'stream'}
)); ), filename);
} else { } else {
return await asFetchResponse(googleDrive.files.get( return await asFetchResponse(googleDrive.files.get(
{ key, fileId, alt: 'media' }, {key, fileId, alt: 'media'},
{ responseType: 'stream' } {responseType: 'stream'}
)); ), fileRes.data.name);
} }
} }
async function initDriveApi(code?: string) {
if (code) {
// Create drive with access token.
const auth = getGoogleAuth();
const token = await auth.getToken(code);
if (token.tokens) {
auth.setCredentials(token.tokens);
}
return drive({version: 'v3', auth: code ? auth : undefined});
}
// Create drive for public access.
return drive({version: 'v3'});
}
async function asFetchResponse(req: GaxiosPromise<Readable>) { async function asFetchResponse(req: GaxiosPromise<Readable>, filename?: string | null) {
try { try {
const res = await req; const res = await req;
const headers = new Headers(res.headers);
if (filename) {
headers.set("content-disposition", contentDisposition(filename));
}
return new FetchResponse(res.data, { return new FetchResponse(res.data, {
headers: new Headers(res.headers), headers,
status: res.status, status: res.status,
statusText: res.statusText statusText: res.statusText
}); });
@ -67,6 +90,6 @@ export function isDriveUrl(url: string) {
function fileIdFromUrl(url: string) { function fileIdFromUrl(url: string) {
if (!url) { return null; } if (!url) { return null; }
const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url); const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/?]*)/i.exec(url);
return match ? match[3] : null; return match ? match[3] : null;
} }

View File

@ -40,6 +40,7 @@ export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadCo
pluginUrl, pluginUrl,
stripeAPIKey: process.env.STRIPE_PUBLIC_API_KEY, stripeAPIKey: process.env.STRIPE_PUBLIC_API_KEY,
googleClientId: process.env.GOOGLE_CLIENT_ID, googleClientId: process.env.GOOGLE_CLIENT_ID,
googleDriveScope: process.env.GOOGLE_DRIVE_SCOPE,
helpScoutBeaconId: process.env.HELP_SCOUT_BEACON_ID, helpScoutBeaconId: process.env.HELP_SCOUT_BEACON_ID,
maxUploadSizeImport: (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || undefined, maxUploadSizeImport: (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || undefined,
maxUploadSizeAttachment: (Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024) || undefined, maxUploadSizeAttachment: (Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024) || undefined,

View File

@ -1,6 +1,6 @@
import {ApiError} from 'app/common/ApiError'; import {ApiError} from 'app/common/ApiError';
import {InactivityTimer} from 'app/common/InactivityTimer'; import {InactivityTimer} from 'app/common/InactivityTimer';
import {FileUploadResult, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads'; import {FetchUrlOptions, FileUploadResult, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode, import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode,
RequestWithLogin} from 'app/server/lib/Authorizer'; RequestWithLogin} from 'app/server/lib/Authorizer';
import {expressWrap} from 'app/server/lib/expressWrap'; import {expressWrap} from 'app/server/lib/expressWrap';
@ -328,20 +328,23 @@ export async function createTmpDir(options: tmp.Options): Promise<TmpDirResult>
/** /**
* Register a new upload with resource fetched from a public url. Returns corresponding UploadInfo. * Register a new upload with resource fetched from a public url. Returns corresponding UploadInfo.
*/ */
export async function fetchURL(url: string, accessId: string|null): Promise<UploadResult> { export async function fetchURL(url: string, accessId: string|null, options?: FetchUrlOptions): Promise<UploadResult> {
return _fetchURL(url, accessId, path.basename(url)); return _fetchURL(url, accessId, { fileName: path.basename(url), ...options});
} }
/** /**
* Register a new upload with resource fetched from a url, optionally including credentials in request. * Register a new upload with resource fetched from a url, optionally including credentials in request.
* Returns corresponding UploadInfo. * Returns corresponding UploadInfo.
*/ */
async function _fetchURL(url: string, accessId: string|null, fileName: string, async function _fetchURL(url: string, accessId: string|null, options?: FetchUrlOptions): Promise<UploadResult> {
headers?: {[key: string]: string}): Promise<UploadResult> {
try { try {
const code = options?.googleAuthorizationCode;
let fileName = options?.fileName ?? '';
const headers = options?.headers;
let response: FetchResponse; let response: FetchResponse;
if (isDriveUrl(url)) { if (isDriveUrl(url)) {
response = await downloadFromGDrive(url); response = await downloadFromGDrive(url, code);
fileName = ''; // Read the file name from headers.
} else { } else {
response = await Deps.fetch(url, { response = await Deps.fetch(url, {
redirect: 'follow', redirect: 'follow',
@ -402,7 +405,7 @@ async function fetchDoc(homeUrl: string, docId: string, req: Request, accessId:
// Download the document, in full or as a template. // Download the document, in full or as a template.
const url = `${docWorkerUrl}download?doc=${docId}&template=${Number(template)}`; const url = `${docWorkerUrl}download?doc=${docId}&template=${Number(template)}`;
return _fetchURL(url, accessId, '', headers); return _fetchURL(url, accessId, {headers});
} }
// Re-issue failures as exceptions. // Re-issue failures as exceptions.

View File

@ -638,7 +638,7 @@ export async function importFileDialog(filePath: string): Promise<void> {
await driver.findWait('.test-dp-add-new', 1000).doClick(); await driver.findWait('.test-dp-add-new', 1000).doClick();
await driver.findContent('.test-dp-import-option', /Import from file/i).doClick(); await driver.findContent('.test-dp-import-option', /Import from file/i).doClick();
}); });
await driver.findWait('.test-importer-dialog', 1000); await driver.findWait('.test-importer-dialog', 5000);
await waitForServer(); await waitForServer();
} }