(core) Adding google drive plugin as a fallback for url plugin

Summary:
When importing from url, user types a url for google spreadsheet,
Grist will switch to Google Drive plugin to allow user to choose file manualy.

Test Plan: Browser tests

Reviewers: paulfitz, dsagal

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2945

app/client/components/GristDoc.ts

@@ -239,11 +239,11 @@ export class GristDoc extends DisposableWithEvents {
     const importMenuItems = [
       {
         label: 'Import from file',
-        action: () => Importer.selectAndImport(this, null, createPreview),
+        action: () => Importer.selectAndImport(this, importSourceElems, null, createPreview),
       },
       ...importSourceElems.map(importSourceElem => ({
         label: importSourceElem.importSource.label,
-        action: () => Importer.selectAndImport(this, importSourceElem, createPreview)
+        action: () => Importer.selectAndImport(this, importSourceElems, importSourceElem, createPreview)
       }))
     ];
 

app/client/components/Importer.ts

@@ -7,7 +7,7 @@
 import {GristDoc} from "app/client/components/GristDoc";
 import {buildParseOptionsForm, ParseOptionValues} from 'app/client/components/ParseOptions';
 import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
-import {fetchURL, selectFiles, uploadFiles} from 'app/client/lib/uploads';
+import {fetchURL, isDriveUrl, selectFiles, uploadFiles} from 'app/client/lib/uploads';
 import {reportError} from 'app/client/models/AppModel';
 import {ViewSectionRec} from 'app/client/models/DocModel';
 import {openFilePicker} from "app/client/ui/FileDialog";
@@ -55,7 +55,10 @@ export class Importer extends Disposable {
    * Imports using the given plugin importer, or the built-in file-picker when null is passed in.
    */
   public static async selectAndImport(
-    gristDoc: GristDoc, importSourceElem: ImportSourceElement|null, createPreview: CreatePreviewFunc
+    gristDoc: GristDoc,
+    imports: ImportSourceElement[],
+    importSourceElem: ImportSourceElement|null,
+    createPreview: CreatePreviewFunc
   ) {
     // In case of using built-in file picker we want to get upload result before instantiating Importer
     // because if the user dismisses the dialog without picking a file,
@@ -84,9 +87,33 @@ export class Importer extends Disposable {
         }
       }
     }
-    // Importer disposes itself when its dialog is closed, so we do not take ownership of it.
+    // HACK: The url plugin does not support importing from google drive, and we don't want to
-    Importer.create(null, gristDoc, importSourceElem, createPreview).pickAndUploadSource(uploadResult)
+    // ask a user for permission to access all his files (needed to download a single file from an URL).
-    .catch((err) => reportError(err));
+    // So to have a nice user experience, we will switch to the built-in google drive plugin and allow
+    // user to chose a file manually.
+    // Suggestion for the future is:
+    // (1) ask the user for the greater permission,
+    // (2) detect when the permission is not granted, and open the picker-based plugin in that case.
+    try {
+      // Importer disposes itself when its dialog is closed, so we do not take ownership of it.
+      await Importer.create(null, gristDoc, importSourceElem, createPreview).pickAndUploadSource(uploadResult);
+    } catch(err1) {
+      // If the url was a Google Drive Url, run the google drive plugin.
+      if (!(err1 instanceof GDriveUrlNotSupported)) {
+        reportError(err1);
+      } else {
+        const gdrivePlugin = imports.find((p) => p.plugin.definition.id === 'builtIn/gdrive' && p !== importSourceElem);
+        if (!gdrivePlugin) {
+          reportError(err1);
+        } else {
+          try {
+            await Importer.create(null, gristDoc, gdrivePlugin, createPreview).pickAndUploadSource(uploadResult);
+          } catch(err2) {
+            reportError(err2);
+          }
+        }
+      }
+    }
   }
 
   private _docComm = this._gristDoc.docComm;
@@ -143,6 +170,11 @@ export class Importer extends Disposable {
         const importSource = await this._importSourceElem.importSourceStub.getImportSource(handle);
         plugin.removeRenderTarget(handle);
 
+        if (!this._openModalCtl) {
+          this._showImportDialog();
+        }
+        this._renderSpinner();
+
         if (importSource) {
           // If data has been picked, upload it.
           const item = importSource.item;
@@ -150,14 +182,26 @@ export class Importer extends Disposable {
             const files = item.files.map(({content, name}) => new File([content], name));
             uploadResult = await uploadFiles(files, {docWorkerUrl: this._docComm.docWorkerUrl,
                                                      sizeLimit: 'import'});
-           } else if (item.kind ===  "url") {
+          } else if (item.kind ===  "url") {
-            uploadResult = await fetchURL(this._docComm, item.url);
+            try {
-           } else {
+              uploadResult = await fetchURL(this._docComm, item.url);
+            } catch(err) {
+              if (isDriveUrl(item.url)) {
+                throw new GDriveUrlNotSupported(item.url);
+              } else {
+                throw err;
+              }
+            }
+          } else {
             throw new Error(`Import source of kind ${(item as any).kind} are not yet supported!`);
           }
         }
       }
     } catch (err) {
+      if (err instanceof GDriveUrlNotSupported) {
+        await this._cancelImport();
+        throw err;
+      }
       if (!this._openModalCtl) {
         this._showImportDialog();
       }
@@ -393,12 +437,18 @@ export class Importer extends Disposable {
   }
 }
 
+// Used for switching from URL plugin to Google drive plugin
+class GDriveUrlNotSupported extends Error {
+  constructor(public url: string) {
+    super(`This url ${url} is not supported`);
+  }
+}
+
 function getSourceDescription(sourceInfo: SourceInfo, upload: UploadResult) {
   const origName = upload.files[sourceInfo.uploadFileIndex].origName;
   return sourceInfo.origTableName ? origName + ' - ' + sourceInfo.origTableName : origName;
 }
 
-
 const cssActionLink = styled('div', `
   display: inline-flex;
   align-items: center;

app/client/lib/uploads.ts

@@ -154,6 +154,12 @@ export async function uploadFiles(
 export async function fetchURL(
   docComm: DocComm, url: string, onProgress: ProgressCB = noop
   ): Promise<UploadResult> {
+
+  if (isDriveUrl(url)) {
+    // don't download from google drive, immediately fallback to server side.
+    return docComm.fetchURL(url);
+  }
+
   let response: Response;
   try {
     response = await window.fetch(url);
@@ -173,6 +179,12 @@ export async function fetchURL(
   return res!;
 }
 
+export function isDriveUrl(url: string) {
+  if (!url) { return null; }
+  const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url);
+  return !!match;
+}
+
 /**
  * Convert a form to a JSON-stringifiable object, ignoring any File fields.
  */

app/server/devServerMain.ts

@@ -93,9 +93,11 @@ export async function main() {
   }
 
   if (!process.env.GOOGLE_CLIENT_ID) {
-    // those key is only for development purposes
+    log.warn('GOOGLE_CLIENT_ID is not defined, Google Drive Plugin will not work.');
-    // and is no secret as it is publicly visible in a plugin page
+  }
-    process.env.GOOGLE_CLIENT_ID  = '632317221841-ce66sfp00rf92dip4548dn4hf2ga79us.apps.googleusercontent.com';
+
+  if (!process.env.GOOGLE_API_KEY) {
+    log.warn('GOOGLE_API_KEY is not defined, Url plugin will not be able to access public files.');
   }
 
   if (process.env.GRIST_SINGLE_PORT) {

app/server/lib/FlexServer.ts

@@ -760,7 +760,7 @@ export class FlexServer implements GristServer {
       // Those keys are eventually visible by the client, but should be usable
       // only from Grist's domains.
       const secrets = {
-        googleClientId : config.googleClientId,
+        googleClientId: config.googleClientId,
       };
       res.set('Content-Type', 'application/javascript');
       res.status(200);

app/server/lib/GoogleImport.ts

@@ -0,0 +1,72 @@
+import {drive} from '@googleapis/drive';
+import {Readable} from 'form-data';
+import {GaxiosError, GaxiosPromise} from 'gaxios';
+import {FetchError, Response as FetchResponse, Headers} from 'node-fetch';
+
+const
+  SPREADSHEETS_MIMETYPE = 'application/vnd.google-apps.spreadsheet',
+  XLSX_MIMETYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+
+export async function downloadFromGDrive(url: string) {
+  const fileId = fileIdFromUrl(url);
+  const googleDrive = drive("v3");
+  const key = process.env.GOOGLE_API_KEY;
+  if (!key) {
+    throw new Error("Can't download file from Google Drive. Api key is not configured");
+  }
+  if (!fileId) {
+    throw new Error(`Can't download from ${url}. Url is not valid`);
+  }
+  const fileRes = await googleDrive.files.get({
+    key,
+    fileId
+  });
+  if (fileRes.data.mimeType === SPREADSHEETS_MIMETYPE) {
+    return await asFetchResponse(googleDrive.files.export(
+      { key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE },
+      { responseType: 'stream' }
+    ));
+  } else {
+    return await asFetchResponse(googleDrive.files.get(
+      { key, fileId, alt: 'media' },
+      { responseType: 'stream' }
+    ));
+  }
+}
+
+
+async function asFetchResponse(req: GaxiosPromise<Readable>) {
+  try {
+    const res = await req;
+    return new FetchResponse(res.data, {
+      headers: new Headers(res.headers),
+      status: res.status,
+      statusText: res.statusText
+    });
+  } catch (err) {
+    const error: GaxiosError<Readable> = err;
+    if (!error.response) {
+      // Fetch throws exception on network error.
+      // https://github.com/node-fetch/node-fetch/blob/master/docs/ERROR-HANDLING.md
+      throw new FetchError(error.message, "system", error.code || "unknown");
+    } else {
+      // Fetch returns failure response on http error
+      const resInit = error.response ? {
+        status: error.response.status,
+        headers: new Headers(error.response.headers),
+        statusText: error.response.statusText
+      } : undefined;
+      return new FetchResponse(error.response.data, resInit);
+    }
+  }
+}
+
+export function isDriveUrl(url: string) {
+  return !!fileIdFromUrl(url);
+}
+
+function fileIdFromUrl(url: string) {
+  if (!url) { return null; }
+  const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url);
+  return match ? match[3] : null;
+}

app/server/lib/uploads.ts

@@ -5,6 +5,7 @@ import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode,
         RequestWithLogin} from 'app/server/lib/Authorizer';
 import {expressWrap} from 'app/server/lib/expressWrap';
 import {RequestWithGrist} from 'app/server/lib/FlexServer';
+import {downloadFromGDrive, isDriveUrl} from 'app/server/lib/GoogleImport';
 import {GristServer} from 'app/server/lib/GristServer';
 import {guessExt} from 'app/server/lib/guessExt';
 import * as log from 'app/server/lib/log';
@@ -338,11 +339,16 @@ export async function fetchURL(url: string, accessId: string|null): Promise<Uplo
 async function _fetchURL(url: string, accessId: string|null, fileName: string,
                          headers?: {[key: string]: string}): Promise<UploadResult> {
   try {
-    const response: FetchResponse = await Deps.fetch(url, {
+    let response: FetchResponse;
-      redirect: 'follow',
+    if (isDriveUrl(url)) {
-      follow: 10,
+      response = await downloadFromGDrive(url);
-      headers
+    } else {
-    });
+      response = await Deps.fetch(url, {
+        redirect: 'follow',
+        follow: 10,
+        headers
+      });
+    }
     await _checkForError(response);
     if (fileName === '') {
       const disposition = response.headers.get('content-disposition') || '';