From 42910cb8f7e96c93990e48c9d7002e57ffcff699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20Sadzi=C5=84ski?= Date: Thu, 30 Sep 2021 10:19:22 +0200 Subject: [PATCH] (core) Extending Google Drive integration scope Summary: New environmental variable GOOGLE_DRIVE_SCOPE that modifies the scope requested for Google Drive integration. For prod it has value https://www.googleapis.com/auth/drive.file which leaves current behavior (Grist is allowed only to access public files and for private files - it fallbacks to Picker). For staging it has value https://www.googleapis.com/auth/drive.readonly which allows Grist to access all private files, and fallbacks to Picker only when the file is neither public nor private). Default value is https://www.googleapis.com/auth/drive.file Test Plan: manual and existing tests Reviewers: dsagal Reviewed By: dsagal Subscribers: dsagal Differential Revision: https://phab.getgrist.com/D3038 --- app/client/components/Importer.ts | 61 ++++++++++-- app/client/lib/uploads.ts | 12 +-- app/client/ui/googleAuth.ts | 159 ++++++++++++++++++++++++++++++ app/client/ui/sendToDrive.ts | 97 +++--------------- app/common/ActiveDocAPI.ts | 4 +- app/common/gristUrls.ts | 6 ++ app/common/uploads.ts | 9 ++ app/server/lib/ActiveDoc.ts | 6 +- app/server/lib/GoogleAuth.ts | 14 ++- app/server/lib/GoogleImport.ts | 45 ++++++--- app/server/lib/sendAppPage.ts | 1 + app/server/lib/uploads.ts | 17 ++-- test/nbrowser/gristUtils.ts | 2 +- 13 files changed, 302 insertions(+), 131 deletions(-) create mode 100644 app/client/ui/googleAuth.ts diff --git a/app/client/components/Importer.ts b/app/client/components/Importer.ts index e32f8e18..45131782 100644 --- a/app/client/components/Importer.ts +++ b/app/client/components/Importer.ts @@ -20,11 +20,12 @@ import {cssModalButtons, cssModalTitle} from 'app/client/ui2018/modals'; import {DataSourceTransformed, ImportResult, ImportTableResult, MergeOptions, MergeStrategy, TransformColumn, TransformRule, TransformRuleMap} from "app/common/ActiveDocAPI"; import {byteString} from "app/common/gutil"; -import {UploadResult} from 'app/common/uploads'; +import {FetchUrlOptions, UploadResult} from 'app/common/uploads'; import {ParseOptions, ParseOptionSchema} from 'app/plugin/FileParserAPI'; import {Computed, Disposable, dom, DomContents, IDisposable, MutableObsArray, obsArray, Observable, styled} from 'grainjs'; import {labeledSquareCheckbox} from "app/client/ui2018/checkbox"; +import {ACCESS_DENIED, AUTH_INTERRUPTED, canReadPrivateFiles, getGoogleCodeForReading} from "app/client/ui/googleAuth"; // Special values for import destinations; null means "new table". // TODO We should also support "skip table" (needs server support), so that one can open, say, @@ -47,7 +48,7 @@ export interface SourceInfo { transformSection: Observable; destTableId: Observable; } - // UI state of selected merge options for each source table (from SourceInfo). +// UI state of selected merge options for each source table (from SourceInfo). interface MergeOptionsState { [srcTableId: string]: { updateExistingRecords: Observable; @@ -182,14 +183,10 @@ export class Importer extends Disposable { uploadResult = await uploadFiles(files, {docWorkerUrl: this._docComm.docWorkerUrl, sizeLimit: 'import'}); } else if (item.kind === "url") { - try { + if (isDriveUrl(item.url)) { + uploadResult = await this._fetchFromDrive(item.url); + } else { uploadResult = await fetchURL(this._docComm, item.url); - } catch(err) { - if (isDriveUrl(item.url)) { - throw new GDriveUrlNotSupported(item.url); - } else { - throw err; - } } } else { throw new Error(`Import source of kind ${(item as any).kind} are not yet supported!`); @@ -197,6 +194,10 @@ export class Importer extends Disposable { } } } catch (err) { + if (err instanceof CancelledError) { + await this._cancelImport(); + return; + } if (err instanceof GDriveUrlNotSupported) { await this._cancelImport(); throw err; @@ -488,15 +489,55 @@ export class Importer extends Disposable { ) ]); } + + private async _fetchFromDrive(itemUrl: string) { + // First we will assume that this is public file, so no need to ask for permissions. + try { + return await fetchURL(this._docComm, itemUrl); + } catch(err) { + // It is not a public file or the file id in the url is wrong, + // but we have no way to check it, so we assume that it is private file + // and ask the user for the permission (if we are configured to do so) + if (canReadPrivateFiles()) { + const options: FetchUrlOptions = {}; + try { + // Request for authorization code from Google. + const code = await getGoogleCodeForReading(this); + options.googleAuthorizationCode = code; + } catch(permError) { + if (permError?.message === ACCESS_DENIED) { + // User declined to give us full readonly permission, fallback to GoogleDrive plugin + // or cancel import if GoogleDrive plugin is not configured. + throw new GDriveUrlNotSupported(itemUrl); + } else if(permError?.message === AUTH_INTERRUPTED) { + // User closed the window - we assume he doesn't want to continue. + throw new CancelledError(); + } else { + // Some other error happened during authentication, report to user. + throw err; + } + } + // Download file from private drive, if it fails, report the error to user. + return await fetchURL(this._docComm, itemUrl, options); + } else { + // We are not allowed to ask for full readonly permission, fallback to GoogleDrive plugin. + throw new GDriveUrlNotSupported(itemUrl); + } + } + } } -// Used for switching from URL plugin to Google drive plugin +// Used for switching from URL plugin to Google drive plugin. class GDriveUrlNotSupported extends Error { constructor(public url: string) { super(`This url ${url} is not supported`); } } +// Used to cancel import (close the dialog without any error). +class CancelledError extends Error { +} + function getSourceDescription(sourceInfo: SourceInfo, upload: UploadResult) { const origName = upload.files[sourceInfo.uploadFileIndex].origName; return sourceInfo.origTableName ? origName + ' - ' + sourceInfo.origTableName : origName; diff --git a/app/client/lib/uploads.ts b/app/client/lib/uploads.ts index e934ecd8..bfce3cce 100644 --- a/app/client/lib/uploads.ts +++ b/app/client/lib/uploads.ts @@ -13,7 +13,7 @@ import {FileDialogOptions, openFilePicker} from 'app/client/ui/FileDialog'; import {BaseAPI} from 'app/common/BaseAPI'; import {GristLoadConfig} from 'app/common/gristUrls'; import {byteString, safeJsonParse} from 'app/common/gutil'; -import {UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads'; +import {FetchUrlOptions, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads'; import {docUrl} from 'app/common/urlUtils'; import {OpenDialogOptions} from 'electron'; import noop = require('lodash/noop'); @@ -152,12 +152,12 @@ export async function uploadFiles( * case, it guesses the name of the file based on the response's content-type and the url. */ export async function fetchURL( - docComm: DocComm, url: string, onProgress: ProgressCB = noop + docComm: DocComm, url: string, options?: FetchUrlOptions, onProgress: ProgressCB = noop ): Promise { if (isDriveUrl(url)) { // don't download from google drive, immediately fallback to server side. - return docComm.fetchURL(url); + return docComm.fetchURL(url, options); } let response: Response; @@ -167,14 +167,14 @@ export async function fetchURL( console.log( // tslint:disable-line:no-console `Could not fetch ${url} on the Client, falling back to server fetch: ${err.message}` ); - return docComm.fetchURL(url); + return docComm.fetchURL(url, options); } // TODO: We should probably parse response.headers.get('content-disposition') when available // (see content-disposition npm module). const fileName = basename(url); const mimeType = response.headers.get('content-type'); - const options = mimeType ? { type: mimeType } : {}; - const fileObj = new File([await response.blob()], fileName, options); + const fileOptions = mimeType ? { type: mimeType } : {}; + const fileObj = new File([await response.blob()], fileName, fileOptions); const res = await uploadFiles([fileObj], {docWorkerUrl: docComm.docWorkerUrl}, onProgress); return res!; } diff --git a/app/client/ui/googleAuth.ts b/app/client/ui/googleAuth.ts new file mode 100644 index 00000000..088ba405 --- /dev/null +++ b/app/client/ui/googleAuth.ts @@ -0,0 +1,159 @@ +import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals'; +import type {Disposable} from 'grainjs'; +import { GristLoadConfig } from "app/common/gristUrls"; + +/** + * Functions to perform server side authentication with Google. + * + * The authentication flow is performed by server side (app/server/lib/GoogleAuth.ts). Here we will + * open up a popup with a stub html file (served by the server), that will redirect user to Google + * Auth Service. In return, we will get authorization_code (which will be delivered by a postMessage + * from the iframe), that when converted to authorization_token, can be used to access Google Drive + * API. Accessing Google Drive files is done by the server, here we only ask for the permissions. + * + * Exposed methods are: + * - getGoogleCodeForSending: asks google for a permission to create files on the drive (and read + * them) + * - getGoogleCodeForReading: asks google for a permission to read all files + * - canReadPrivateFiles: Grist by default won't ask for permission to read all files, but can be + * configured this way by an environmental variable. + */ + +const G = getBrowserGlobals('window'); + +export const ACCESS_DENIED = "access_denied"; +export const AUTH_INTERRUPTED = "auth_interrupted"; + +// https://developers.google.com/identity/protocols/oauth2/scopes#drive +// "View and manage Google Drive files and folders that you have opened or created with this app" +const APP_SCOPE = "https://www.googleapis.com/auth/drive.file"; +// "See and download all your Google Drive files" +const READ_SCOPE = "https://www.googleapis.com/auth/drive.readonly"; + +export function getGoogleCodeForSending(owner: Disposable) { + return getGoogleAuthCode(owner, APP_SCOPE); +} + +export function getGoogleCodeForReading(owner: Disposable) { + const gristConfig: GristLoadConfig = (window as any).gristConfig || {}; + // Default scope allows as to manage files we created. + return getGoogleAuthCode(owner, gristConfig.googleDriveScope || APP_SCOPE); +} + +/** + * Checks if default scope for Google Drive integration will allow to access all personal files + */ +export function canReadPrivateFiles() { + const gristConfig: GristLoadConfig = (window as any).gristConfig || {}; + return gristConfig.googleDriveScope === READ_SCOPE; +} + +/** + * Opens up a popup with server side Google Authentication. Returns a code that can be used + * by server side to retrieve access_token required in Google Api. + */ +function getGoogleAuthCode(owner: Disposable, scope: string) { + // Compute google auth server endpoint (grist endpoint for server side google authentication). + // This endpoint renders a page that redirects user to Google Consent screen and after Google + // sends a response, it will post this response back to us. + // Message will be an object { code, error }. + const authLink = getGoogleAuthEndpoint(scope); + const authWindow = openPopup(authLink); + return new Promise((resolve, reject) => { + attachListener(owner, authWindow, async (event: MessageEvent|null) => { + // If the no message, or window was closed (user closed it intentionally). + if (!event || authWindow.closed) { + reject(new Error(AUTH_INTERRUPTED)); + return; + } + // For the first message (we expect only a single message) close the window. + authWindow.close(); + if (owner.isDisposed()) { + reject(new Error(AUTH_INTERRUPTED)); + return; + } + // Check response from the popup + const response = (event.data || {}) as {code?: string, error?: string}; + // - when user declined, report back, caller should stop current flow, + if (response.error === "access_denied") { + reject(new Error(ACCESS_DENIED)); + return; + } + // - when there is no authorization, or error is different from what we expected - report to user. + if (!response.code) { + reject(new Error(response.error || "Missing authorization code")); + return; + } + resolve(response.code); + }); + }); +} + +// Helper function that attaches a handler to message event from a popup window. +function attachListener(owner: Disposable, popup: Window, listener: (e: MessageEvent|null) => void) { + const wrapped = (e: MessageEvent) => { + // Listen to events only from our window. + if (e.source !== popup) { return; } + // In case when Grist was reloaded or user navigated away - do nothing. + if (owner.isDisposed()) { return; } + listener(e); + // Clear the listener, to avoid orphaned calls from closed event. + listener = () => {}; + }; + // Unfortunately there is no ease way to detect if user has closed the popup. + const closeHandler = onClose(popup, () => { + listener(null); + // Clear the listener, to avoid orphaned messages from window. + listener = () => {}; + }); + owner.onDispose(closeHandler); + G.window.addEventListener('message', wrapped); + owner.onDispose(() => { + G.window.removeEventListener('message', wrapped); + }); +} + +// Periodically checks if the window is closed. +// Returns a function that can be used to cancel the event. +function onClose(window: Window, clb: () => void) { + const interval = setInterval(() => { + if (window.closed) { + clearInterval(interval); + clb(); + } + }, 1000); + return () => clearInterval(interval); +} + +function openPopup(url: string): Window { + // Center window on desktop + // https://stackoverflow.com/questions/16363474/window-open-on-a-multi-monitor-dual-monitor-system-where-does-window-pop-up + const width = 600; + const height = 650; + const left = window.screenX + (screen.width - width) / 2; + const top = (screen.height - height) / 4; + let windowFeatures = `top=${top},left=${left},menubar=no,location=no,` + + `resizable=yes,scrollbars=yes,status=yes,height=${height},width=${width}`; + + // If window will be too large (for example on mobile) - open as a new tab + if (screen.width <= width || screen.height <= height) { + windowFeatures = ''; + } + + const authWindow = G.window.open(url, "GoogleAuthPopup", windowFeatures); + if (!authWindow) { + // This method should be invoked by an user action. + throw new Error("This method should be invoked synchronously"); + } + return authWindow; +} + +/** + * Generates Google Auth endpoint (exposed by Grist) url. For example: + * https://docs.getgrist.com/auth/google + * @param scope Requested access scope for Google Services: + * https://developers.google.com/identity/protocols/oauth2/scopes + */ +function getGoogleAuthEndpoint(scope: string) { + return new URL(`auth/google?scope=${scope}`, window.location.origin).href; +} diff --git a/app/client/ui/sendToDrive.ts b/app/client/ui/sendToDrive.ts index 79a9c35d..f5a78100 100644 --- a/app/client/ui/sendToDrive.ts +++ b/app/client/ui/sendToDrive.ts @@ -1,30 +1,18 @@ import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals'; import {reportError} from 'app/client/models/errors'; import {spinnerModal} from 'app/client/ui2018/modals'; -import type { DocPageModel } from 'app/client/models/DocPageModel'; -import type { Document } from 'app/common/UserAPI'; -import type { Disposable } from 'grainjs'; - +import type {DocPageModel} from 'app/client/models/DocPageModel'; +import type {Document} from 'app/common/UserAPI'; +import { getGoogleCodeForSending } from "app/client/ui/googleAuth"; const G = getBrowserGlobals('window'); /** - * Generates Google Auth endpoint (exposed by Grist) url. For example: - * https://docs.getgrist.com/auth/google - * @param scope Requested access scope for Google Services - * https://developers.google.com/identity/protocols/oauth2/scopes + * Sends xlsx file to Google Drive. It first authenticates with Google to get encrypted access + * token, then it calls "send-to-drive" api endpoint to upload xlsx file to drive and finally it + * redirects to the created spreadsheet. Code that is received from Google contains encrypted access + * token, server is able to decrypt it using GOOGLE_CLIENT_SECRET key. */ -function getGoogleAuthEndpoint(scope?: string) { - return new URL(`auth/google?scope=${scope || ''}`, window.location.origin).href; -} - -/** - * Sends xlsx file to Google Drive. It first authenticates with Google to get encrypted access token, - * then it calls "send-to-drive" api endpoint to upload xlsx file to drive and finally it redirects - * to the created spreadsheet. - * Code that is received from Google contains encrypted access token, server is able to decrypt it - * using GOOGLE_CLIENT_SECRET key. - */ -export function sendToDrive(doc: Document, pageModel: DocPageModel) { +export async function sendToDrive(doc: Document, pageModel: DocPageModel) { // Get current document - it will be used to remove popup listener. const gristDoc = pageModel.gristDoc.get(); // Sanity check - gristDoc should be always present @@ -38,68 +26,11 @@ export function sendToDrive(doc: Document, pageModel: DocPageModel) { .sendToDrive(code, pageModel.currentDocTitle.get()) ); - // Compute google auth server endpoint (grist endpoint for server side google authentication). - // This endpoint will redirect user to Google Consent screen and after Google sends a response, - // it will render a page (/static/message.html) that will post a message containing message - // from Google. Message will be an object { code, error }. We will use the code to invoke - // "send-to-drive" api endpoint - that will actually send the xlsx file to Google Drive. - const authLink = getGoogleAuthEndpoint(); - const authWindow = openPopup(authLink); - attachListener(gristDoc, authWindow, async (event: MessageEvent) => { - // For the first message (we expect only a single message) close the window. - authWindow.close(); - - // Check response from the popup - const response = (event.data || {}) as { code?: string, error?: string }; - // - when user declined, do nothing, - if (response.error === "access_denied") { return; } - // - when there is no response code or error code is different from what we expected - report to user. - if (!response.code) { reportError(response.error || "Unrecognized or empty error code"); return; } - - // Send file to Google Drive. - try { - const { url } = await send(response.code); - G.window.location.assign(url); - } catch (err) { - reportError(err); - } - }); -} - -// Helper function that attaches a handler to message event from a popup window. -function attachListener(owner: Disposable, popup: Window, listener: (e: MessageEvent) => any) { - const wrapped = (e: MessageEvent) => { - // Listen to events only from our window. - if (e.source !== popup) { return; } - // In case when Grist was reloaded or user navigated away - do nothing. - if (owner.isDisposed()) { return; } - listener(e); - }; - G.window.addEventListener('message', wrapped); - owner.onDispose(() => { - G.window.removeEventListener('message', wrapped); - }); -} - -function openPopup(url: string): Window { - // Center window on desktop - // https://stackoverflow.com/questions/16363474/window-open-on-a-multi-monitor-dual-monitor-system-where-does-window-pop-up - const width = 600; - const height = 650; - const left = window.screenX + (screen.width - width) / 2; - const top = (screen.height - height) / 4; - let windowFeatures = `top=${top},left=${left},menubar=no,location=no,` + - `resizable=yes,scrollbars=yes,status=yes,height=${height},width=${width}`; - - // If window will be too large (for example on mobile) - open as a new tab - if (screen.width <= width || screen.height <= height) { - windowFeatures = ''; + try { + const token = await getGoogleCodeForSending(gristDoc); + const {url} = await send(token); + G.window.location.assign(url); + } catch (err) { + reportError(err); } - - const authWindow = G.window.open(url, "GoogleAuthPopup", windowFeatures); - if (!authWindow) { - // This method should be invoked by an user action. - throw new Error("This method should be invoked synchronously"); - } - return authWindow; } diff --git a/app/common/ActiveDocAPI.ts b/app/common/ActiveDocAPI.ts index 6a29e8d7..98c7eac0 100644 --- a/app/common/ActiveDocAPI.ts +++ b/app/common/ActiveDocAPI.ts @@ -1,7 +1,7 @@ import {ActionGroup} from 'app/common/ActionGroup'; import {CellValue, TableDataAction, UserAction} from 'app/common/DocActions'; import {FormulaProperties} from 'app/common/GranularAccessClause'; -import {UploadResult} from 'app/common/uploads'; +import {FetchUrlOptions, UploadResult} from 'app/common/uploads'; import {ParseOptions} from 'app/plugin/FileParserAPI'; import {IMessage} from 'grain-rpc'; @@ -207,7 +207,7 @@ export interface ActiveDocAPI { /** * Fetch content at a url. */ - fetchURL(url: string): Promise; + fetchURL(url: string, options?: FetchUrlOptions): Promise; /** * Find and return a list of auto-complete suggestions that start with `txt`, when editing a diff --git a/app/common/gristUrls.ts b/app/common/gristUrls.ts index 58bb2caf..cf7a23c7 100644 --- a/app/common/gristUrls.ts +++ b/app/common/gristUrls.ts @@ -456,6 +456,12 @@ export interface GristLoadConfig { // Google Client Id, used in Google integration (ex: Google Drive Plugin) googleClientId?: string; + // Max scope we can request for accessing files from Google Drive. + // Default used by Grist is https://www.googleapis.com/auth/drive.file: + // View and manage Google Drive files and folders that you have opened or created with this app. + // More on scopes: https://developers.google.com/identity/protocols/oauth2/scopes#drive + googleDriveScope?: string; + // List of registered plugins (used by HomePluginManager and DocPluginManager) plugins?: LocalPlugin[]; } diff --git a/app/common/uploads.ts b/app/common/uploads.ts index a5479565..17028822 100644 --- a/app/common/uploads.ts +++ b/app/common/uploads.ts @@ -42,3 +42,12 @@ export interface FileUploadResult { * the page's will be respected. */ export const UPLOAD_URL_PATH = 'uploads'; + +/** + * Additional options for fetching external resources. + */ +export interface FetchUrlOptions { + googleAuthorizationCode?: string; // The authorization code received from Google Auth Service. + fileName?: string; // The filename for external resource. + headers?: {[key: string]: string}; // Additional headers to use when accessing external resource. +} diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 01fc6ec7..50f0babe 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -44,7 +44,7 @@ import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccess import {byteString, countIf, safeJsonParse} from 'app/common/gutil'; import {InactivityTimer} from 'app/common/InactivityTimer'; import * as marshal from 'app/common/marshal'; -import {UploadResult} from 'app/common/uploads'; +import {FetchUrlOptions, UploadResult} from 'app/common/uploads'; import {DocReplacementOptions, DocState} from 'app/common/UserAPI'; import {ParseOptions} from 'app/plugin/FileParserAPI'; import {GristDocAPI} from 'app/plugin/GristAPI'; @@ -890,8 +890,8 @@ export class ActiveDoc extends EventEmitter { return this._pyCall('autocomplete', txt, tableId, columnId, user.toJSON()); } - public fetchURL(docSession: DocSession, url: string): Promise { - return fetchURL(url, this.makeAccessId(docSession.authorizer.getUserId())); + public fetchURL(docSession: DocSession, url: string, options?: FetchUrlOptions): Promise { + return fetchURL(url, this.makeAccessId(docSession.authorizer.getUserId()), options); } public async forwardPluginRpc(docSession: DocSession, pluginId: string, msg: IMessage): Promise { diff --git a/app/server/lib/GoogleAuth.ts b/app/server/lib/GoogleAuth.ts index 0e94a213..d659504d 100644 --- a/app/server/lib/GoogleAuth.ts +++ b/app/server/lib/GoogleAuth.ts @@ -16,6 +16,8 @@ import {URL} from 'url'; * the same client id is used in Google Drive Plugin * - GOOGLE_CLIENT_SECRET: secret key for Google Project, can't be shared - it is used to (d)encrypt * data that we obtain from Google Auth Service (all done in the api) + * - GOOGLE_DRIVE_SCOPE: scope requested for the Google drive integration (defaults to drive.file which allows + * to create files and get files via Google Drive Picker) * * High level description: * @@ -55,13 +57,9 @@ import {URL} from 'url'; * in a query string. */ - // Path for the auth endpoint. const authHandlerPath = "/auth/google"; -// "View and manage Google Drive files and folders that you have opened or created with this app."" -export const DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.file'; - // Redirect host after the Google Auth login form is completed. This reuses the same domain name // as for Cognito login. const AUTH_SUBDOMAIN = process.env.GRIST_ID_PREFIX ? `docs-${process.env.GRIST_ID_PREFIX}` : 'docs'; @@ -97,7 +95,7 @@ export async function googleAuthTokenMiddleware( throw new ApiError("Google Auth endpoint requires a code parameter in the query string", 400); } else { try { - const oAuth2Client = _googleAuthClient(); + const oAuth2Client = getGoogleAuth(); // Decrypt code that was send back from Google Auth service. Uses GOOGLE_CLIENT_SECRET key. const tokenResponse = await oAuth2Client.getToken(stringParam(req.query.code)); // Get the access token (access token will be present in a default request configuration). @@ -150,8 +148,8 @@ export function addGoogleAuthEndpoint( throw new ApiError("Error authenticating with Google", 500); } } else { - const oAuth2Client = _googleAuthClient(); - const scope = optStringParam(req.query.scope) || DRIVE_SCOPE; + const oAuth2Client = getGoogleAuth(); + const scope = stringParam(req.query.scope); // Create url for origin parameter for a popup window. const origin = getOriginUrl(req); const authUrl = oAuth2Client.generateAuthUrl({ @@ -172,7 +170,7 @@ export function addGoogleAuthEndpoint( /** * Builds the OAuth2 Google client. */ -function _googleAuthClient() { +export function getGoogleAuth() { const CLIENT_ID = process.env.GOOGLE_CLIENT_ID; const CLIENT_SECRET = process.env.GOOGLE_CLIENT_SECRET; const oAuth2Client = new auth.OAuth2(CLIENT_ID, CLIENT_SECRET, getFullAuthEndpointUrl()); diff --git a/app/server/lib/GoogleImport.ts b/app/server/lib/GoogleImport.ts index f6c1fa6f..435f5698 100644 --- a/app/server/lib/GoogleImport.ts +++ b/app/server/lib/GoogleImport.ts @@ -2,14 +2,15 @@ import {drive} from '@googleapis/drive'; import {Readable} from 'form-data'; import {GaxiosError, GaxiosPromise} from 'gaxios'; import {FetchError, Response as FetchResponse, Headers} from 'node-fetch'; +import {getGoogleAuth} from "app/server/lib/GoogleAuth"; +import * as contentDisposition from 'content-disposition'; const SPREADSHEETS_MIMETYPE = 'application/vnd.google-apps.spreadsheet', XLSX_MIMETYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; -export async function downloadFromGDrive(url: string) { +export async function downloadFromGDrive(url: string, code?: string) { const fileId = fileIdFromUrl(url); - const googleDrive = drive("v3"); const key = process.env.GOOGLE_API_KEY; if (!key) { throw new Error("Can't download file from Google Drive. Api key is not configured"); @@ -17,29 +18,51 @@ export async function downloadFromGDrive(url: string) { if (!fileId) { throw new Error(`Can't download from ${url}. Url is not valid`); } + const googleDrive = await initDriveApi(code); const fileRes = await googleDrive.files.get({ key, fileId }); if (fileRes.data.mimeType === SPREADSHEETS_MIMETYPE) { + let filename = fileRes.data.name; + if (filename && !filename.includes(".")) { + filename = `${filename}.xlsx`; + } return await asFetchResponse(googleDrive.files.export( - { key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE }, - { responseType: 'stream' } - )); + {key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE}, + {responseType: 'stream'} + ), filename); } else { return await asFetchResponse(googleDrive.files.get( - { key, fileId, alt: 'media' }, - { responseType: 'stream' } - )); + {key, fileId, alt: 'media'}, + {responseType: 'stream'} + ), fileRes.data.name); } } +async function initDriveApi(code?: string) { + if (code) { + // Create drive with access token. + const auth = getGoogleAuth(); + const token = await auth.getToken(code); + if (token.tokens) { + auth.setCredentials(token.tokens); + } + return drive({version: 'v3', auth: code ? auth : undefined}); + } + // Create drive for public access. + return drive({version: 'v3'}); +} -async function asFetchResponse(req: GaxiosPromise) { +async function asFetchResponse(req: GaxiosPromise, filename?: string | null) { try { const res = await req; + const headers = new Headers(res.headers); + if (filename) { + headers.set("content-disposition", contentDisposition(filename)); + } return new FetchResponse(res.data, { - headers: new Headers(res.headers), + headers, status: res.status, statusText: res.statusText }); @@ -67,6 +90,6 @@ export function isDriveUrl(url: string) { function fileIdFromUrl(url: string) { if (!url) { return null; } - const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url); + const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/?]*)/i.exec(url); return match ? match[3] : null; } diff --git a/app/server/lib/sendAppPage.ts b/app/server/lib/sendAppPage.ts index 6c9ebf65..f6eeb32f 100644 --- a/app/server/lib/sendAppPage.ts +++ b/app/server/lib/sendAppPage.ts @@ -40,6 +40,7 @@ export function makeGristConfig(homeUrl: string|null, extra: Partial /** * Register a new upload with resource fetched from a public url. Returns corresponding UploadInfo. */ -export async function fetchURL(url: string, accessId: string|null): Promise { - return _fetchURL(url, accessId, path.basename(url)); +export async function fetchURL(url: string, accessId: string|null, options?: FetchUrlOptions): Promise { + return _fetchURL(url, accessId, { fileName: path.basename(url), ...options}); } /** * Register a new upload with resource fetched from a url, optionally including credentials in request. * Returns corresponding UploadInfo. */ -async function _fetchURL(url: string, accessId: string|null, fileName: string, - headers?: {[key: string]: string}): Promise { +async function _fetchURL(url: string, accessId: string|null, options?: FetchUrlOptions): Promise { try { + const code = options?.googleAuthorizationCode; + let fileName = options?.fileName ?? ''; + const headers = options?.headers; let response: FetchResponse; if (isDriveUrl(url)) { - response = await downloadFromGDrive(url); + response = await downloadFromGDrive(url, code); + fileName = ''; // Read the file name from headers. } else { response = await Deps.fetch(url, { redirect: 'follow', @@ -402,7 +405,7 @@ async function fetchDoc(homeUrl: string, docId: string, req: Request, accessId: // Download the document, in full or as a template. const url = `${docWorkerUrl}download?doc=${docId}&template=${Number(template)}`; - return _fetchURL(url, accessId, '', headers); + return _fetchURL(url, accessId, {headers}); } // Re-issue failures as exceptions. diff --git a/test/nbrowser/gristUtils.ts b/test/nbrowser/gristUtils.ts index 937e403f..41803879 100644 --- a/test/nbrowser/gristUtils.ts +++ b/test/nbrowser/gristUtils.ts @@ -638,7 +638,7 @@ export async function importFileDialog(filePath: string): Promise { await driver.findWait('.test-dp-add-new', 1000).doClick(); await driver.findContent('.test-dp-import-option', /Import from file/i).doClick(); }); - await driver.findWait('.test-importer-dialog', 1000); + await driver.findWait('.test-importer-dialog', 5000); await waitForServer(); }