From 6ed1d8dfeac249d8f2c56cd0d1ef0c6784e44faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20Sadzi=C5=84ski?= Date: Tue, 3 Aug 2021 12:34:05 +0200 Subject: [PATCH] (core) Adding google drive plugin as a fallback for url plugin Summary: When importing from url, user types a url for google spreadsheet, Grist will switch to Google Drive plugin to allow user to choose file manualy. Test Plan: Browser tests Reviewers: paulfitz, dsagal Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D2945 --- app/client/components/GristDoc.ts | 4 +- app/client/components/Importer.ts | 68 +++++++++++++++++++++++++---- app/client/lib/uploads.ts | 12 ++++++ app/server/devServerMain.ts | 8 ++-- app/server/lib/FlexServer.ts | 2 +- app/server/lib/GoogleImport.ts | 72 +++++++++++++++++++++++++++++++ app/server/lib/uploads.ts | 16 ++++--- 7 files changed, 162 insertions(+), 20 deletions(-) create mode 100644 app/server/lib/GoogleImport.ts diff --git a/app/client/components/GristDoc.ts b/app/client/components/GristDoc.ts index 5b85142c..f4d212f9 100644 --- a/app/client/components/GristDoc.ts +++ b/app/client/components/GristDoc.ts @@ -239,11 +239,11 @@ export class GristDoc extends DisposableWithEvents { const importMenuItems = [ { label: 'Import from file', - action: () => Importer.selectAndImport(this, null, createPreview), + action: () => Importer.selectAndImport(this, importSourceElems, null, createPreview), }, ...importSourceElems.map(importSourceElem => ({ label: importSourceElem.importSource.label, - action: () => Importer.selectAndImport(this, importSourceElem, createPreview) + action: () => Importer.selectAndImport(this, importSourceElems, importSourceElem, createPreview) })) ]; diff --git a/app/client/components/Importer.ts b/app/client/components/Importer.ts index c85cb875..82d7ddb7 100644 --- a/app/client/components/Importer.ts +++ b/app/client/components/Importer.ts @@ -7,7 +7,7 @@ import {GristDoc} from "app/client/components/GristDoc"; import {buildParseOptionsForm, ParseOptionValues} from 'app/client/components/ParseOptions'; import {ImportSourceElement} from 'app/client/lib/ImportSourceElement'; -import {fetchURL, selectFiles, uploadFiles} from 'app/client/lib/uploads'; +import {fetchURL, isDriveUrl, selectFiles, uploadFiles} from 'app/client/lib/uploads'; import {reportError} from 'app/client/models/AppModel'; import {ViewSectionRec} from 'app/client/models/DocModel'; import {openFilePicker} from "app/client/ui/FileDialog"; @@ -55,7 +55,10 @@ export class Importer extends Disposable { * Imports using the given plugin importer, or the built-in file-picker when null is passed in. */ public static async selectAndImport( - gristDoc: GristDoc, importSourceElem: ImportSourceElement|null, createPreview: CreatePreviewFunc + gristDoc: GristDoc, + imports: ImportSourceElement[], + importSourceElem: ImportSourceElement|null, + createPreview: CreatePreviewFunc ) { // In case of using built-in file picker we want to get upload result before instantiating Importer // because if the user dismisses the dialog without picking a file, @@ -84,9 +87,33 @@ export class Importer extends Disposable { } } } - // Importer disposes itself when its dialog is closed, so we do not take ownership of it. - Importer.create(null, gristDoc, importSourceElem, createPreview).pickAndUploadSource(uploadResult) - .catch((err) => reportError(err)); + // HACK: The url plugin does not support importing from google drive, and we don't want to + // ask a user for permission to access all his files (needed to download a single file from an URL). + // So to have a nice user experience, we will switch to the built-in google drive plugin and allow + // user to chose a file manually. + // Suggestion for the future is: + // (1) ask the user for the greater permission, + // (2) detect when the permission is not granted, and open the picker-based plugin in that case. + try { + // Importer disposes itself when its dialog is closed, so we do not take ownership of it. + await Importer.create(null, gristDoc, importSourceElem, createPreview).pickAndUploadSource(uploadResult); + } catch(err1) { + // If the url was a Google Drive Url, run the google drive plugin. + if (!(err1 instanceof GDriveUrlNotSupported)) { + reportError(err1); + } else { + const gdrivePlugin = imports.find((p) => p.plugin.definition.id === 'builtIn/gdrive' && p !== importSourceElem); + if (!gdrivePlugin) { + reportError(err1); + } else { + try { + await Importer.create(null, gristDoc, gdrivePlugin, createPreview).pickAndUploadSource(uploadResult); + } catch(err2) { + reportError(err2); + } + } + } + } } private _docComm = this._gristDoc.docComm; @@ -143,6 +170,11 @@ export class Importer extends Disposable { const importSource = await this._importSourceElem.importSourceStub.getImportSource(handle); plugin.removeRenderTarget(handle); + if (!this._openModalCtl) { + this._showImportDialog(); + } + this._renderSpinner(); + if (importSource) { // If data has been picked, upload it. const item = importSource.item; @@ -150,14 +182,26 @@ export class Importer extends Disposable { const files = item.files.map(({content, name}) => new File([content], name)); uploadResult = await uploadFiles(files, {docWorkerUrl: this._docComm.docWorkerUrl, sizeLimit: 'import'}); - } else if (item.kind === "url") { - uploadResult = await fetchURL(this._docComm, item.url); - } else { + } else if (item.kind === "url") { + try { + uploadResult = await fetchURL(this._docComm, item.url); + } catch(err) { + if (isDriveUrl(item.url)) { + throw new GDriveUrlNotSupported(item.url); + } else { + throw err; + } + } + } else { throw new Error(`Import source of kind ${(item as any).kind} are not yet supported!`); } } } } catch (err) { + if (err instanceof GDriveUrlNotSupported) { + await this._cancelImport(); + throw err; + } if (!this._openModalCtl) { this._showImportDialog(); } @@ -393,12 +437,18 @@ export class Importer extends Disposable { } } +// Used for switching from URL plugin to Google drive plugin +class GDriveUrlNotSupported extends Error { + constructor(public url: string) { + super(`This url ${url} is not supported`); + } +} + function getSourceDescription(sourceInfo: SourceInfo, upload: UploadResult) { const origName = upload.files[sourceInfo.uploadFileIndex].origName; return sourceInfo.origTableName ? origName + ' - ' + sourceInfo.origTableName : origName; } - const cssActionLink = styled('div', ` display: inline-flex; align-items: center; diff --git a/app/client/lib/uploads.ts b/app/client/lib/uploads.ts index ddd2735a..e934ecd8 100644 --- a/app/client/lib/uploads.ts +++ b/app/client/lib/uploads.ts @@ -154,6 +154,12 @@ export async function uploadFiles( export async function fetchURL( docComm: DocComm, url: string, onProgress: ProgressCB = noop ): Promise { + + if (isDriveUrl(url)) { + // don't download from google drive, immediately fallback to server side. + return docComm.fetchURL(url); + } + let response: Response; try { response = await window.fetch(url); @@ -173,6 +179,12 @@ export async function fetchURL( return res!; } +export function isDriveUrl(url: string) { + if (!url) { return null; } + const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url); + return !!match; +} + /** * Convert a form to a JSON-stringifiable object, ignoring any File fields. */ diff --git a/app/server/devServerMain.ts b/app/server/devServerMain.ts index c59c4842..dc901112 100644 --- a/app/server/devServerMain.ts +++ b/app/server/devServerMain.ts @@ -93,9 +93,11 @@ export async function main() { } if (!process.env.GOOGLE_CLIENT_ID) { - // those key is only for development purposes - // and is no secret as it is publicly visible in a plugin page - process.env.GOOGLE_CLIENT_ID = '632317221841-ce66sfp00rf92dip4548dn4hf2ga79us.apps.googleusercontent.com'; + log.warn('GOOGLE_CLIENT_ID is not defined, Google Drive Plugin will not work.'); + } + + if (!process.env.GOOGLE_API_KEY) { + log.warn('GOOGLE_API_KEY is not defined, Url plugin will not be able to access public files.'); } if (process.env.GRIST_SINGLE_PORT) { diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts index 90c7d984..f6fdf028 100644 --- a/app/server/lib/FlexServer.ts +++ b/app/server/lib/FlexServer.ts @@ -760,7 +760,7 @@ export class FlexServer implements GristServer { // Those keys are eventually visible by the client, but should be usable // only from Grist's domains. const secrets = { - googleClientId : config.googleClientId, + googleClientId: config.googleClientId, }; res.set('Content-Type', 'application/javascript'); res.status(200); diff --git a/app/server/lib/GoogleImport.ts b/app/server/lib/GoogleImport.ts new file mode 100644 index 00000000..f6c1fa6f --- /dev/null +++ b/app/server/lib/GoogleImport.ts @@ -0,0 +1,72 @@ +import {drive} from '@googleapis/drive'; +import {Readable} from 'form-data'; +import {GaxiosError, GaxiosPromise} from 'gaxios'; +import {FetchError, Response as FetchResponse, Headers} from 'node-fetch'; + +const + SPREADSHEETS_MIMETYPE = 'application/vnd.google-apps.spreadsheet', + XLSX_MIMETYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + +export async function downloadFromGDrive(url: string) { + const fileId = fileIdFromUrl(url); + const googleDrive = drive("v3"); + const key = process.env.GOOGLE_API_KEY; + if (!key) { + throw new Error("Can't download file from Google Drive. Api key is not configured"); + } + if (!fileId) { + throw new Error(`Can't download from ${url}. Url is not valid`); + } + const fileRes = await googleDrive.files.get({ + key, + fileId + }); + if (fileRes.data.mimeType === SPREADSHEETS_MIMETYPE) { + return await asFetchResponse(googleDrive.files.export( + { key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE }, + { responseType: 'stream' } + )); + } else { + return await asFetchResponse(googleDrive.files.get( + { key, fileId, alt: 'media' }, + { responseType: 'stream' } + )); + } +} + + +async function asFetchResponse(req: GaxiosPromise) { + try { + const res = await req; + return new FetchResponse(res.data, { + headers: new Headers(res.headers), + status: res.status, + statusText: res.statusText + }); + } catch (err) { + const error: GaxiosError = err; + if (!error.response) { + // Fetch throws exception on network error. + // https://github.com/node-fetch/node-fetch/blob/master/docs/ERROR-HANDLING.md + throw new FetchError(error.message, "system", error.code || "unknown"); + } else { + // Fetch returns failure response on http error + const resInit = error.response ? { + status: error.response.status, + headers: new Headers(error.response.headers), + statusText: error.response.statusText + } : undefined; + return new FetchResponse(error.response.data, resInit); + } + } +} + +export function isDriveUrl(url: string) { + return !!fileIdFromUrl(url); +} + +function fileIdFromUrl(url: string) { + if (!url) { return null; } + const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url); + return match ? match[3] : null; +} diff --git a/app/server/lib/uploads.ts b/app/server/lib/uploads.ts index f0585292..3efe2acb 100644 --- a/app/server/lib/uploads.ts +++ b/app/server/lib/uploads.ts @@ -5,6 +5,7 @@ import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode, RequestWithLogin} from 'app/server/lib/Authorizer'; import {expressWrap} from 'app/server/lib/expressWrap'; import {RequestWithGrist} from 'app/server/lib/FlexServer'; +import {downloadFromGDrive, isDriveUrl} from 'app/server/lib/GoogleImport'; import {GristServer} from 'app/server/lib/GristServer'; import {guessExt} from 'app/server/lib/guessExt'; import * as log from 'app/server/lib/log'; @@ -338,11 +339,16 @@ export async function fetchURL(url: string, accessId: string|null): Promise { try { - const response: FetchResponse = await Deps.fetch(url, { - redirect: 'follow', - follow: 10, - headers - }); + let response: FetchResponse; + if (isDriveUrl(url)) { + response = await downloadFromGDrive(url); + } else { + response = await Deps.fetch(url, { + redirect: 'follow', + follow: 10, + headers + }); + } await _checkForError(response); if (fileName === '') { const disposition = response.headers.get('content-disposition') || '';