(core) Extending Google Drive integration scope

Summary:
New environmental variable GOOGLE_DRIVE_SCOPE that modifies the scope
requested for Google Drive integration.
For prod it has value https://www.googleapis.com/auth/drive.file which leaves
current behavior (Grist is allowed only to access public files and for private
files - it fallbacks to Picker).
For staging it has value https://www.googleapis.com/auth/drive.readonly which
allows Grist to access all private files, and fallbacks to Picker only when the file is
neither public nor private).
Default value is https://www.googleapis.com/auth/drive.file

Test Plan: manual and existing tests

Reviewers: dsagal

Reviewed By: dsagal

Subscribers: dsagal

Differential Revision: https://phab.getgrist.com/D3038

app/client/components/Importer.ts

@@ -20,11 +20,12 @@ import {cssModalButtons, cssModalTitle} from 'app/client/ui2018/modals';
 import {DataSourceTransformed, ImportResult, ImportTableResult, MergeOptions,
         MergeStrategy, TransformColumn, TransformRule, TransformRuleMap} from "app/common/ActiveDocAPI";
 import {byteString} from "app/common/gutil";
-import {UploadResult} from 'app/common/uploads';
+import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {ParseOptions, ParseOptionSchema} from 'app/plugin/FileParserAPI';
 import {Computed, Disposable, dom, DomContents, IDisposable, MutableObsArray, obsArray, Observable,
         styled} from 'grainjs';
 import {labeledSquareCheckbox} from "app/client/ui2018/checkbox";
+import {ACCESS_DENIED, AUTH_INTERRUPTED, canReadPrivateFiles, getGoogleCodeForReading} from "app/client/ui/googleAuth";
 
 // Special values for import destinations; null means "new table".
 // TODO We should also support "skip table" (needs server support), so that one can open, say,
@@ -47,7 +48,7 @@ export interface SourceInfo {
   transformSection: Observable<ViewSectionRec>;
   destTableId: Observable<DestId>;
 }
- // UI state of selected merge options for each source table (from SourceInfo).
+// UI state of selected merge options for each source table (from SourceInfo).
 interface MergeOptionsState {
   [srcTableId: string]: {
     updateExistingRecords: Observable<boolean>;
@@ -182,14 +183,10 @@ export class Importer extends Disposable {
             uploadResult = await uploadFiles(files, {docWorkerUrl: this._docComm.docWorkerUrl,
                                                      sizeLimit: 'import'});
           } else if (item.kind ===  "url") {
-            try {
+            if (isDriveUrl(item.url)) {
+              uploadResult = await this._fetchFromDrive(item.url);
+            } else {
               uploadResult = await fetchURL(this._docComm, item.url);
-            } catch(err) {
-              if (isDriveUrl(item.url)) {
-                throw new GDriveUrlNotSupported(item.url);
-              } else {
-                throw err;
-              }
             }
           } else {
             throw new Error(`Import source of kind ${(item as any).kind} are not yet supported!`);
@@ -197,6 +194,10 @@ export class Importer extends Disposable {
         }
       }
     } catch (err) {
+      if (err instanceof CancelledError) {
+        await this._cancelImport();
+        return;
+      }
       if (err instanceof GDriveUrlNotSupported) {
         await this._cancelImport();
         throw err;
@@ -488,15 +489,55 @@ export class Importer extends Disposable {
       )
     ]);
   }
+
+  private async _fetchFromDrive(itemUrl: string) {
+    // First we will assume that this is public file, so no need to ask for permissions.
+    try {
+      return await fetchURL(this._docComm, itemUrl);
+    } catch(err) {
+      // It is not a public file or the file id in the url is wrong,
+      // but we have no way to check it, so we assume that it is private file
+      // and ask the user for the permission (if we are configured to do so)
+      if (canReadPrivateFiles()) {
+        const options: FetchUrlOptions = {};
+        try {
+          // Request for authorization code from Google.
+          const code = await getGoogleCodeForReading(this);
+          options.googleAuthorizationCode = code;
+        } catch(permError) {
+          if (permError?.message === ACCESS_DENIED) {
+            // User declined to give us full readonly permission, fallback to GoogleDrive plugin
+            // or cancel import if GoogleDrive plugin is not configured.
+            throw new GDriveUrlNotSupported(itemUrl);
+          } else if(permError?.message === AUTH_INTERRUPTED) {
+            // User closed the window - we assume he doesn't want to continue.
+            throw new CancelledError();
+          } else {
+            // Some other error happened during authentication, report to user.
+            throw err;
+          }
+        }
+        // Download file from private drive, if it fails, report the error to user.
+        return await fetchURL(this._docComm, itemUrl, options);
+      } else {
+        // We are not allowed to ask for full readonly permission, fallback to GoogleDrive plugin.
+        throw new GDriveUrlNotSupported(itemUrl);
+      }
+    }
+  }
 }
 
-// Used for switching from URL plugin to Google drive plugin
+// Used for switching from URL plugin to Google drive plugin.
 class GDriveUrlNotSupported extends Error {
   constructor(public url: string) {
     super(`This url ${url} is not supported`);
   }
 }
 
+// Used to cancel import (close the dialog without any error).
+class CancelledError extends Error {
+}
+
 function getSourceDescription(sourceInfo: SourceInfo, upload: UploadResult) {
   const origName = upload.files[sourceInfo.uploadFileIndex].origName;
   return sourceInfo.origTableName ? origName + ' - ' + sourceInfo.origTableName : origName;

app/client/lib/uploads.ts

@@ -13,7 +13,7 @@ import {FileDialogOptions, openFilePicker} from 'app/client/ui/FileDialog';
 import {BaseAPI} from 'app/common/BaseAPI';
 import {GristLoadConfig} from 'app/common/gristUrls';
 import {byteString, safeJsonParse} from 'app/common/gutil';
-import {UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
+import {FetchUrlOptions, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
 import {docUrl} from 'app/common/urlUtils';
 import {OpenDialogOptions} from 'electron';
 import noop = require('lodash/noop');
@@ -152,12 +152,12 @@ export async function uploadFiles(
  * case, it guesses the name of the file based on the response's content-type and the url.
  */
 export async function fetchURL(
-  docComm: DocComm, url: string, onProgress: ProgressCB = noop
+  docComm: DocComm, url: string, options?: FetchUrlOptions, onProgress: ProgressCB = noop
   ): Promise<UploadResult> {
 
   if (isDriveUrl(url)) {
     // don't download from google drive, immediately fallback to server side.
-    return docComm.fetchURL(url);
+    return docComm.fetchURL(url, options);
   }
 
   let response: Response;
@@ -167,14 +167,14 @@ export async function fetchURL(
     console.log( // tslint:disable-line:no-console
       `Could not fetch ${url} on the Client, falling back to server fetch: ${err.message}`
     );
-    return docComm.fetchURL(url);
+    return docComm.fetchURL(url, options);
   }
   // TODO: We should probably parse response.headers.get('content-disposition') when available
   // (see content-disposition npm module).
   const fileName = basename(url);
   const mimeType = response.headers.get('content-type');
-  const options = mimeType ? { type: mimeType } : {};
+  const fileOptions = mimeType ? { type: mimeType } : {};
-  const fileObj = new File([await response.blob()], fileName, options);
+  const fileObj = new File([await response.blob()], fileName, fileOptions);
   const res = await uploadFiles([fileObj], {docWorkerUrl: docComm.docWorkerUrl}, onProgress);
   return res!;
 }

app/client/ui/googleAuth.ts

@@ -0,0 +1,159 @@
+import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
+import type {Disposable} from 'grainjs';
+import { GristLoadConfig } from "app/common/gristUrls";
+
+/**
+ * Functions to perform server side authentication with Google.
+ *
+ * The authentication flow is performed by server side (app/server/lib/GoogleAuth.ts). Here we will
+ * open up a popup with a stub html file (served by the server), that will redirect user to Google
+ * Auth Service. In return, we will get authorization_code (which will be delivered by a postMessage
+ * from the iframe), that when converted to authorization_token, can be used to access Google Drive
+ * API. Accessing Google Drive files is done by the server, here we only ask for the permissions.
+ *
+ * Exposed methods are:
+ * - getGoogleCodeForSending: asks google for a permission to create files on the drive (and read
+ *                            them)
+ * - getGoogleCodeForReading: asks google for a permission to read all files
+ * - canReadPrivateFiles:     Grist by default won't ask for permission to read all files, but can be
+ *                            configured this way by an environmental variable.
+ */
+
+const G = getBrowserGlobals('window');
+
+export const ACCESS_DENIED = "access_denied";
+export const AUTH_INTERRUPTED = "auth_interrupted";
+
+// https://developers.google.com/identity/protocols/oauth2/scopes#drive
+// "View and manage Google Drive files and folders that you have opened or created with this app"
+const APP_SCOPE = "https://www.googleapis.com/auth/drive.file";
+// "See and download all your Google Drive files"
+const READ_SCOPE = "https://www.googleapis.com/auth/drive.readonly";
+
+export function getGoogleCodeForSending(owner: Disposable) {
+  return getGoogleAuthCode(owner, APP_SCOPE);
+}
+
+export function getGoogleCodeForReading(owner: Disposable) {
+  const gristConfig: GristLoadConfig = (window as any).gristConfig || {};
+  // Default scope allows as to manage files we created.
+  return getGoogleAuthCode(owner, gristConfig.googleDriveScope || APP_SCOPE);
+}
+
+/**
+ * Checks if default scope for Google Drive integration will allow to access all personal files
+ */
+export function canReadPrivateFiles() {
+  const gristConfig: GristLoadConfig = (window as any).gristConfig || {};
+  return gristConfig.googleDriveScope === READ_SCOPE;
+}
+
+/**
+ * Opens up a popup with server side Google Authentication. Returns a code that can be used
+ * by server side to retrieve access_token required in Google Api.
+ */
+function getGoogleAuthCode(owner: Disposable, scope: string) {
+  // Compute google auth server endpoint (grist endpoint for server side google authentication).
+  // This endpoint renders a page that redirects user to Google Consent screen and after Google
+  // sends a response, it will post this response back to us.
+  // Message will be an object { code, error }.
+  const authLink = getGoogleAuthEndpoint(scope);
+  const authWindow = openPopup(authLink);
+  return new Promise<string>((resolve, reject) => {
+    attachListener(owner, authWindow, async (event: MessageEvent|null) => {
+      // If the no message, or window was closed (user closed it intentionally).
+      if (!event || authWindow.closed) {
+        reject(new Error(AUTH_INTERRUPTED));
+        return;
+      }
+      // For the first message (we expect only a single message) close the window.
+      authWindow.close();
+      if (owner.isDisposed()) {
+        reject(new Error(AUTH_INTERRUPTED));
+        return;
+      }
+      // Check response from the popup
+      const response = (event.data || {}) as {code?: string, error?: string};
+      // - when user declined, report back, caller should stop current flow,
+      if (response.error === "access_denied") {
+        reject(new Error(ACCESS_DENIED));
+        return;
+      }
+      // - when there is no authorization, or error is different from what we expected - report to user.
+      if (!response.code) {
+        reject(new Error(response.error || "Missing authorization code"));
+        return;
+      }
+      resolve(response.code);
+    });
+  });
+}
+
+// Helper function that attaches a handler to message event from a popup window.
+function attachListener(owner: Disposable, popup: Window, listener: (e: MessageEvent|null) => void) {
+  const wrapped = (e: MessageEvent) => {
+    // Listen to events only from our window.
+    if (e.source !== popup) { return; }
+    // In case when Grist was reloaded or user navigated away - do nothing.
+    if (owner.isDisposed()) { return; }
+    listener(e);
+    // Clear the listener, to avoid orphaned calls from closed event.
+    listener = () => {};
+  };
+  // Unfortunately there is no ease way to detect if user has closed the popup.
+  const closeHandler = onClose(popup, () => {
+    listener(null);
+    // Clear the listener, to avoid orphaned messages from window.
+    listener = () => {};
+  });
+  owner.onDispose(closeHandler);
+  G.window.addEventListener('message', wrapped);
+  owner.onDispose(() => {
+    G.window.removeEventListener('message', wrapped);
+  });
+}
+
+// Periodically checks if the window is closed.
+// Returns a function that can be used to cancel the event.
+function onClose(window: Window, clb: () => void) {
+  const interval = setInterval(() => {
+    if (window.closed) {
+      clearInterval(interval);
+      clb();
+    }
+  }, 1000);
+  return () => clearInterval(interval);
+}
+
+function openPopup(url: string): Window {
+  // Center window on desktop
+  // https://stackoverflow.com/questions/16363474/window-open-on-a-multi-monitor-dual-monitor-system-where-does-window-pop-up
+  const width = 600;
+  const height = 650;
+  const left = window.screenX + (screen.width - width) / 2;
+  const top = (screen.height - height) / 4;
+  let windowFeatures = `top=${top},left=${left},menubar=no,location=no,` +
+    `resizable=yes,scrollbars=yes,status=yes,height=${height},width=${width}`;
+
+  // If window will be too large (for example on mobile) - open as a new tab
+  if (screen.width <= width || screen.height <= height) {
+    windowFeatures = '';
+  }
+
+  const authWindow = G.window.open(url, "GoogleAuthPopup", windowFeatures);
+  if (!authWindow) {
+    // This method should be invoked by an user action.
+    throw new Error("This method should be invoked synchronously");
+  }
+  return authWindow;
+}
+
+/**
+ * Generates Google Auth endpoint (exposed by Grist) url. For example:
+ * https://docs.getgrist.com/auth/google
+ * @param scope Requested access scope for Google Services:
+ * https://developers.google.com/identity/protocols/oauth2/scopes
+ */
+function getGoogleAuthEndpoint(scope: string) {
+  return new URL(`auth/google?scope=${scope}`, window.location.origin).href;
+}

app/client/ui/sendToDrive.ts

@@ -1,30 +1,18 @@
 import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
 import {reportError} from 'app/client/models/errors';
 import {spinnerModal} from 'app/client/ui2018/modals';
-import type { DocPageModel } from 'app/client/models/DocPageModel';
+import type {DocPageModel} from 'app/client/models/DocPageModel';
-import type { Document } from 'app/common/UserAPI';
+import type {Document} from 'app/common/UserAPI';
-import type { Disposable } from 'grainjs';
+import { getGoogleCodeForSending } from "app/client/ui/googleAuth";
-
 const G = getBrowserGlobals('window');
 
 /**
- * Generates Google Auth endpoint (exposed by Grist) url. For example:
+ * Sends xlsx file to Google Drive. It first authenticates with Google to get encrypted access
- * https://docs.getgrist.com/auth/google
+ * token, then it calls "send-to-drive" api endpoint to upload xlsx file to drive and finally it
- * @param scope Requested access scope for Google Services
+ * redirects to the created spreadsheet. Code that is received from Google contains encrypted access
- * https://developers.google.com/identity/protocols/oauth2/scopes
+ * token, server is able to decrypt it using GOOGLE_CLIENT_SECRET key.
  */
-function getGoogleAuthEndpoint(scope?: string) {
+export async function sendToDrive(doc: Document, pageModel: DocPageModel) {
-  return new URL(`auth/google?scope=${scope || ''}`, window.location.origin).href;
-}
-
-/**
- * Sends xlsx file to Google Drive. It first authenticates with Google to get encrypted access token,
- * then it calls "send-to-drive" api endpoint to upload xlsx file to drive and finally it redirects
- * to the created spreadsheet.
- * Code that is received from Google contains encrypted access token, server is able to decrypt it
- * using GOOGLE_CLIENT_SECRET key.
- */
-export function sendToDrive(doc: Document, pageModel: DocPageModel) {
   // Get current document - it will be used to remove popup listener.
   const gristDoc = pageModel.gristDoc.get();
   // Sanity check - gristDoc should be always present
@@ -38,68 +26,11 @@ export function sendToDrive(doc: Document, pageModel: DocPageModel) {
         .sendToDrive(code, pageModel.currentDocTitle.get())
     );
 
-  // Compute google auth server endpoint (grist endpoint for server side google authentication).
+  try {
-  // This endpoint will redirect user to Google Consent screen and after Google sends a response,
+    const token = await getGoogleCodeForSending(gristDoc);
-  // it will render a page (/static/message.html) that will post a message containing message
+    const {url} = await send(token);
-  // from Google. Message will be an object { code, error }. We will use the code to invoke
+    G.window.location.assign(url);
-  // "send-to-drive" api endpoint - that will actually send the xlsx file to Google Drive.
+  } catch (err) {
-  const authLink = getGoogleAuthEndpoint();
+    reportError(err);
-  const authWindow = openPopup(authLink);
-  attachListener(gristDoc, authWindow, async (event: MessageEvent) => {
-    // For the first message (we expect only a single message) close the window.
-    authWindow.close();
-
-    // Check response from the popup
-    const response = (event.data || {}) as { code?: string, error?: string };
-    // - when user declined, do nothing,
-    if (response.error === "access_denied") { return; }
-    // - when there is no response code or error code is different from what we expected - report to user.
-    if (!response.code) { reportError(response.error || "Unrecognized or empty error code"); return; }
-
-    // Send file to Google Drive.
-    try {
-      const { url } = await send(response.code);
-      G.window.location.assign(url);
-    } catch (err) {
-      reportError(err);
-    }
-  });
-}
-
-// Helper function that attaches a handler to message event from a popup window.
-function attachListener(owner: Disposable, popup: Window, listener: (e: MessageEvent) => any) {
-  const wrapped = (e: MessageEvent) => {
-    // Listen to events only from our window.
-    if (e.source !== popup) { return; }
-    // In case when Grist was reloaded or user navigated away - do nothing.
-    if (owner.isDisposed()) { return; }
-    listener(e);
-  };
-  G.window.addEventListener('message', wrapped);
-  owner.onDispose(() => {
-    G.window.removeEventListener('message', wrapped);
-  });
-}
-
-function openPopup(url: string): Window {
-  // Center window on desktop
-  // https://stackoverflow.com/questions/16363474/window-open-on-a-multi-monitor-dual-monitor-system-where-does-window-pop-up
-  const width = 600;
-  const height = 650;
-  const left = window.screenX + (screen.width - width) / 2;
-  const top = (screen.height - height) / 4;
-  let windowFeatures = `top=${top},left=${left},menubar=no,location=no,` +
-    `resizable=yes,scrollbars=yes,status=yes,height=${height},width=${width}`;
-
-  // If window will be too large (for example on mobile) - open as a new tab
-  if (screen.width <= width || screen.height <= height) {
-    windowFeatures = '';
   }
-
-  const authWindow = G.window.open(url, "GoogleAuthPopup", windowFeatures);
-  if (!authWindow) {
-    // This method should be invoked by an user action.
-    throw new Error("This method should be invoked synchronously");
-  }
-  return authWindow;
 }

app/common/ActiveDocAPI.ts

@@ -1,7 +1,7 @@
 import {ActionGroup} from 'app/common/ActionGroup';
 import {CellValue, TableDataAction, UserAction} from 'app/common/DocActions';
 import {FormulaProperties} from 'app/common/GranularAccessClause';
-import {UploadResult} from 'app/common/uploads';
+import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
 import {IMessage} from 'grain-rpc';
 
@@ -207,7 +207,7 @@ export interface ActiveDocAPI {
   /**
    * Fetch content at a url.
    */
-  fetchURL(url: string): Promise<UploadResult>;
+  fetchURL(url: string, options?: FetchUrlOptions): Promise<UploadResult>;
 
   /**
    * Find and return a list of auto-complete suggestions that start with `txt`, when editing a

app/common/gristUrls.ts

@@ -456,6 +456,12 @@ export interface GristLoadConfig {
   // Google Client Id, used in Google integration (ex: Google Drive Plugin)
   googleClientId?: string;
 
+  // Max scope we can request for accessing files from Google Drive.
+  // Default used by Grist is https://www.googleapis.com/auth/drive.file:
+  // View and manage Google Drive files and folders that you have opened or created with this app.
+  // More on scopes: https://developers.google.com/identity/protocols/oauth2/scopes#drive
+  googleDriveScope?: string;
+
   // List of registered plugins (used by HomePluginManager and DocPluginManager)
   plugins?: LocalPlugin[];
 }

app/common/uploads.ts

@@ -42,3 +42,12 @@ export interface FileUploadResult {
  * the page's <base> will be respected.
  */
 export const UPLOAD_URL_PATH = 'uploads';
+
+/**
+ * Additional options for fetching external resources.
+ */
+export interface FetchUrlOptions {
+  googleAuthorizationCode?: string;   // The authorization code received from Google Auth Service.
+  fileName?: string;                  // The filename for external resource.
+  headers?: {[key: string]: string};  // Additional headers to use when accessing external resource.
+}

app/server/lib/ActiveDoc.ts

@@ -44,7 +44,7 @@ import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccess
 import {byteString, countIf, safeJsonParse} from 'app/common/gutil';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import * as marshal from 'app/common/marshal';
-import {UploadResult} from 'app/common/uploads';
+import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {DocReplacementOptions, DocState} from 'app/common/UserAPI';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
 import {GristDocAPI} from 'app/plugin/GristAPI';
@@ -890,8 +890,8 @@ export class ActiveDoc extends EventEmitter {
     return this._pyCall('autocomplete', txt, tableId, columnId, user.toJSON());
   }
 
-  public fetchURL(docSession: DocSession, url: string): Promise<UploadResult> {
+  public fetchURL(docSession: DocSession, url: string, options?: FetchUrlOptions): Promise<UploadResult> {
-    return fetchURL(url, this.makeAccessId(docSession.authorizer.getUserId()));
+    return fetchURL(url, this.makeAccessId(docSession.authorizer.getUserId()), options);
   }
 
   public async forwardPluginRpc(docSession: DocSession, pluginId: string, msg: IMessage): Promise<any> {

app/server/lib/GoogleAuth.ts

@@ -16,6 +16,8 @@ import {URL} from 'url';
  *                          the same client id is used in Google Drive Plugin
  * - GOOGLE_CLIENT_SECRET:  secret key for Google Project, can't be shared - it is used to (d)encrypt
  *                          data that we obtain from Google Auth Service (all done in the api)
+ * - GOOGLE_DRIVE_SCOPE:    scope requested for the Google drive integration (defaults to drive.file which allows
+ *                          to create files and get files via Google Drive Picker)
  *
  * High level description:
  *
@@ -55,13 +57,9 @@ import {URL} from 'url';
  * in a query string.
  */
 
-
 // Path for the auth endpoint.
 const authHandlerPath = "/auth/google";
 
-// "View and manage Google Drive files and folders that you have opened or created with this app.""
-export const DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.file';
-
 // Redirect host after the Google Auth login form is completed. This reuses the same domain name
 // as for Cognito login.
 const AUTH_SUBDOMAIN = process.env.GRIST_ID_PREFIX ? `docs-${process.env.GRIST_ID_PREFIX}` : 'docs';
@@ -97,7 +95,7 @@ export async function googleAuthTokenMiddleware(
     throw new ApiError("Google Auth endpoint requires a code parameter in the query string", 400);
   } else {
     try {
-      const oAuth2Client = _googleAuthClient();
+      const oAuth2Client = getGoogleAuth();
       // Decrypt code that was send back from Google Auth service. Uses GOOGLE_CLIENT_SECRET key.
       const tokenResponse = await oAuth2Client.getToken(stringParam(req.query.code));
       // Get the access token (access token will be present in a default request configuration).
@@ -150,8 +148,8 @@ export function addGoogleAuthEndpoint(
         throw new ApiError("Error authenticating with Google", 500);
       }
     } else {
-      const oAuth2Client = _googleAuthClient();
+      const oAuth2Client = getGoogleAuth();
-      const scope = optStringParam(req.query.scope) || DRIVE_SCOPE;
+      const scope = stringParam(req.query.scope);
       // Create url for origin parameter for a popup window.
       const origin = getOriginUrl(req);
       const authUrl = oAuth2Client.generateAuthUrl({
@@ -172,7 +170,7 @@ export function addGoogleAuthEndpoint(
 /**
  * Builds the OAuth2 Google client.
  */
-function _googleAuthClient() {
+export function getGoogleAuth() {
   const CLIENT_ID = process.env.GOOGLE_CLIENT_ID;
   const CLIENT_SECRET = process.env.GOOGLE_CLIENT_SECRET;
   const oAuth2Client = new auth.OAuth2(CLIENT_ID, CLIENT_SECRET, getFullAuthEndpointUrl());

app/server/lib/GoogleImport.ts

@@ -2,14 +2,15 @@ import {drive} from '@googleapis/drive';
 import {Readable} from 'form-data';
 import {GaxiosError, GaxiosPromise} from 'gaxios';
 import {FetchError, Response as FetchResponse, Headers} from 'node-fetch';
+import {getGoogleAuth} from "app/server/lib/GoogleAuth";
+import * as contentDisposition from 'content-disposition';
 
 const
   SPREADSHEETS_MIMETYPE = 'application/vnd.google-apps.spreadsheet',
   XLSX_MIMETYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
 
-export async function downloadFromGDrive(url: string) {
+export async function downloadFromGDrive(url: string, code?: string) {
   const fileId = fileIdFromUrl(url);
-  const googleDrive = drive("v3");
   const key = process.env.GOOGLE_API_KEY;
   if (!key) {
     throw new Error("Can't download file from Google Drive. Api key is not configured");
@@ -17,29 +18,51 @@ export async function downloadFromGDrive(url: string) {
   if (!fileId) {
     throw new Error(`Can't download from ${url}. Url is not valid`);
   }
+  const googleDrive = await initDriveApi(code);
   const fileRes = await googleDrive.files.get({
     key,
     fileId
   });
   if (fileRes.data.mimeType === SPREADSHEETS_MIMETYPE) {
+    let filename = fileRes.data.name;
+    if (filename && !filename.includes(".")) {
+      filename = `${filename}.xlsx`;
+    }
     return await asFetchResponse(googleDrive.files.export(
-      { key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE },
+      {key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE},
-      { responseType: 'stream' }
+      {responseType: 'stream'}
-    ));
+    ), filename);
   } else {
     return await asFetchResponse(googleDrive.files.get(
-      { key, fileId, alt: 'media' },
+      {key, fileId, alt: 'media'},
-      { responseType: 'stream' }
+      {responseType: 'stream'}
-    ));
+    ), fileRes.data.name);
   }
 }
 
+async function initDriveApi(code?: string) {
+  if (code) {
+    // Create drive with access token.
+    const auth = getGoogleAuth();
+    const token = await auth.getToken(code);
+    if (token.tokens) {
+      auth.setCredentials(token.tokens);
+    }
+    return drive({version: 'v3', auth: code ? auth : undefined});
+  }
+  // Create drive for public access.
+  return drive({version: 'v3'});
+}
 
-async function asFetchResponse(req: GaxiosPromise<Readable>) {
+async function asFetchResponse(req: GaxiosPromise<Readable>, filename?: string | null) {
   try {
     const res = await req;
+    const headers = new Headers(res.headers);
+    if (filename) {
+      headers.set("content-disposition", contentDisposition(filename));
+    }
     return new FetchResponse(res.data, {
-      headers: new Headers(res.headers),
+      headers,
       status: res.status,
       statusText: res.statusText
     });
@@ -67,6 +90,6 @@ export function isDriveUrl(url: string) {
 
 function fileIdFromUrl(url: string) {
   if (!url) { return null; }
-  const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url);
+  const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/?]*)/i.exec(url);
   return match ? match[3] : null;
 }

app/server/lib/sendAppPage.ts

@@ -40,6 +40,7 @@ export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadCo
     pluginUrl,
     stripeAPIKey: process.env.STRIPE_PUBLIC_API_KEY,
     googleClientId: process.env.GOOGLE_CLIENT_ID,
+    googleDriveScope: process.env.GOOGLE_DRIVE_SCOPE,
     helpScoutBeaconId: process.env.HELP_SCOUT_BEACON_ID,
     maxUploadSizeImport: (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || undefined,
     maxUploadSizeAttachment: (Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024) || undefined,

app/server/lib/uploads.ts

@@ -1,6 +1,6 @@
 import {ApiError} from 'app/common/ApiError';
 import {InactivityTimer} from 'app/common/InactivityTimer';
-import {FileUploadResult, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
+import {FetchUrlOptions, FileUploadResult, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
 import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode,
         RequestWithLogin} from 'app/server/lib/Authorizer';
 import {expressWrap} from 'app/server/lib/expressWrap';
@@ -328,20 +328,23 @@ export async function createTmpDir(options: tmp.Options): Promise<TmpDirResult>
 /**
  * Register a new upload with resource fetched from a public url. Returns corresponding UploadInfo.
  */
-export async function fetchURL(url: string, accessId: string|null): Promise<UploadResult> {
+export async function fetchURL(url: string, accessId: string|null, options?: FetchUrlOptions): Promise<UploadResult> {
-  return _fetchURL(url, accessId, path.basename(url));
+  return _fetchURL(url, accessId, { fileName: path.basename(url), ...options});
 }
 
 /**
  * Register a new upload with resource fetched from a url, optionally including credentials in request.
  * Returns corresponding UploadInfo.
  */
-async function _fetchURL(url: string, accessId: string|null, fileName: string,
+async function _fetchURL(url: string, accessId: string|null, options?: FetchUrlOptions): Promise<UploadResult> {
-                         headers?: {[key: string]: string}): Promise<UploadResult> {
   try {
+    const code = options?.googleAuthorizationCode;
+    let fileName = options?.fileName ?? '';
+    const headers = options?.headers;
     let response: FetchResponse;
     if (isDriveUrl(url)) {
-      response = await downloadFromGDrive(url);
+      response = await downloadFromGDrive(url, code);
+      fileName = ''; // Read the file name from headers.
     } else {
       response = await Deps.fetch(url, {
         redirect: 'follow',
@@ -402,7 +405,7 @@ async function fetchDoc(homeUrl: string, docId: string, req: Request, accessId:
 
   // Download the document, in full or as a template.
   const url = `${docWorkerUrl}download?doc=${docId}&template=${Number(template)}`;
-  return _fetchURL(url, accessId, '', headers);
+  return _fetchURL(url, accessId, {headers});
 }
 
 // Re-issue failures as exceptions.

test/nbrowser/gristUtils.ts

@@ -638,7 +638,7 @@ export async function importFileDialog(filePath: string): Promise<void> {
     await driver.findWait('.test-dp-add-new', 1000).doClick();
     await driver.findContent('.test-dp-import-option', /Import from file/i).doClick();
   });
-  await driver.findWait('.test-importer-dialog', 1000);
+  await driver.findWait('.test-importer-dialog', 5000);
   await waitForServer();
 }