(core) Extending Google Drive integration scope

Summary:
New environmental variable GOOGLE_DRIVE_SCOPE that modifies the scope
requested for Google Drive integration.
For prod it has value https://www.googleapis.com/auth/drive.file which leaves
current behavior (Grist is allowed only to access public files and for private
files - it fallbacks to Picker).
For staging it has value https://www.googleapis.com/auth/drive.readonly which
allows Grist to access all private files, and fallbacks to Picker only when the file is
neither public nor private).
Default value is https://www.googleapis.com/auth/drive.file

Test Plan: manual and existing tests

Reviewers: dsagal

Reviewed By: dsagal

Subscribers: dsagal

Differential Revision: https://phab.getgrist.com/D3038

app/server/lib/ActiveDoc.ts

@@ -44,7 +44,7 @@ import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccess
 import {byteString, countIf, safeJsonParse} from 'app/common/gutil';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import * as marshal from 'app/common/marshal';
-import {UploadResult} from 'app/common/uploads';
+import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {DocReplacementOptions, DocState} from 'app/common/UserAPI';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
 import {GristDocAPI} from 'app/plugin/GristAPI';
@@ -890,8 +890,8 @@ export class ActiveDoc extends EventEmitter {
     return this._pyCall('autocomplete', txt, tableId, columnId, user.toJSON());
   }
 
-  public fetchURL(docSession: DocSession, url: string): Promise<UploadResult> {
-    return fetchURL(url, this.makeAccessId(docSession.authorizer.getUserId()));
+  public fetchURL(docSession: DocSession, url: string, options?: FetchUrlOptions): Promise<UploadResult> {
+    return fetchURL(url, this.makeAccessId(docSession.authorizer.getUserId()), options);
   }
 
   public async forwardPluginRpc(docSession: DocSession, pluginId: string, msg: IMessage): Promise<any> {

app/server/lib/GoogleAuth.ts

@@ -16,6 +16,8 @@ import {URL} from 'url';
  *                          the same client id is used in Google Drive Plugin
  * - GOOGLE_CLIENT_SECRET:  secret key for Google Project, can't be shared - it is used to (d)encrypt
  *                          data that we obtain from Google Auth Service (all done in the api)
+ * - GOOGLE_DRIVE_SCOPE:    scope requested for the Google drive integration (defaults to drive.file which allows
+ *                          to create files and get files via Google Drive Picker)
  *
  * High level description:
  *
@@ -55,13 +57,9 @@ import {URL} from 'url';
  * in a query string.
  */
 
-
 // Path for the auth endpoint.
 const authHandlerPath = "/auth/google";
 
-// "View and manage Google Drive files and folders that you have opened or created with this app.""
-export const DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive.file';
-
 // Redirect host after the Google Auth login form is completed. This reuses the same domain name
 // as for Cognito login.
 const AUTH_SUBDOMAIN = process.env.GRIST_ID_PREFIX ? `docs-${process.env.GRIST_ID_PREFIX}` : 'docs';
@@ -97,7 +95,7 @@ export async function googleAuthTokenMiddleware(
     throw new ApiError("Google Auth endpoint requires a code parameter in the query string", 400);
   } else {
     try {
-      const oAuth2Client = _googleAuthClient();
+      const oAuth2Client = getGoogleAuth();
       // Decrypt code that was send back from Google Auth service. Uses GOOGLE_CLIENT_SECRET key.
       const tokenResponse = await oAuth2Client.getToken(stringParam(req.query.code));
       // Get the access token (access token will be present in a default request configuration).
@@ -150,8 +148,8 @@ export function addGoogleAuthEndpoint(
         throw new ApiError("Error authenticating with Google", 500);
       }
     } else {
-      const oAuth2Client = _googleAuthClient();
-      const scope = optStringParam(req.query.scope) || DRIVE_SCOPE;
+      const oAuth2Client = getGoogleAuth();
+      const scope = stringParam(req.query.scope);
       // Create url for origin parameter for a popup window.
       const origin = getOriginUrl(req);
       const authUrl = oAuth2Client.generateAuthUrl({
@@ -172,7 +170,7 @@ export function addGoogleAuthEndpoint(
 /**
  * Builds the OAuth2 Google client.
  */
-function _googleAuthClient() {
+export function getGoogleAuth() {
   const CLIENT_ID = process.env.GOOGLE_CLIENT_ID;
   const CLIENT_SECRET = process.env.GOOGLE_CLIENT_SECRET;
   const oAuth2Client = new auth.OAuth2(CLIENT_ID, CLIENT_SECRET, getFullAuthEndpointUrl());

app/server/lib/GoogleImport.ts

@@ -2,14 +2,15 @@ import {drive} from '@googleapis/drive';
 import {Readable} from 'form-data';
 import {GaxiosError, GaxiosPromise} from 'gaxios';
 import {FetchError, Response as FetchResponse, Headers} from 'node-fetch';
+import {getGoogleAuth} from "app/server/lib/GoogleAuth";
+import * as contentDisposition from 'content-disposition';
 
 const
   SPREADSHEETS_MIMETYPE = 'application/vnd.google-apps.spreadsheet',
   XLSX_MIMETYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
 
-export async function downloadFromGDrive(url: string) {
+export async function downloadFromGDrive(url: string, code?: string) {
   const fileId = fileIdFromUrl(url);
-  const googleDrive = drive("v3");
   const key = process.env.GOOGLE_API_KEY;
   if (!key) {
     throw new Error("Can't download file from Google Drive. Api key is not configured");
@@ -17,29 +18,51 @@ export async function downloadFromGDrive(url: string) {
   if (!fileId) {
     throw new Error(`Can't download from ${url}. Url is not valid`);
   }
+  const googleDrive = await initDriveApi(code);
   const fileRes = await googleDrive.files.get({
     key,
     fileId
   });
   if (fileRes.data.mimeType === SPREADSHEETS_MIMETYPE) {
+    let filename = fileRes.data.name;
+    if (filename && !filename.includes(".")) {
+      filename = `${filename}.xlsx`;
+    }
     return await asFetchResponse(googleDrive.files.export(
-      { key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE },
-      { responseType: 'stream' }
-    ));
+      {key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE},
+      {responseType: 'stream'}
+    ), filename);
   } else {
     return await asFetchResponse(googleDrive.files.get(
-      { key, fileId, alt: 'media' },
-      { responseType: 'stream' }
-    ));
+      {key, fileId, alt: 'media'},
+      {responseType: 'stream'}
+    ), fileRes.data.name);
   }
 }
 
+async function initDriveApi(code?: string) {
+  if (code) {
+    // Create drive with access token.
+    const auth = getGoogleAuth();
+    const token = await auth.getToken(code);
+    if (token.tokens) {
+      auth.setCredentials(token.tokens);
+    }
+    return drive({version: 'v3', auth: code ? auth : undefined});
+  }
+  // Create drive for public access.
+  return drive({version: 'v3'});
+}
 
-async function asFetchResponse(req: GaxiosPromise<Readable>) {
+async function asFetchResponse(req: GaxiosPromise<Readable>, filename?: string | null) {
   try {
     const res = await req;
+    const headers = new Headers(res.headers);
+    if (filename) {
+      headers.set("content-disposition", contentDisposition(filename));
+    }
     return new FetchResponse(res.data, {
-      headers: new Headers(res.headers),
+      headers,
       status: res.status,
       statusText: res.statusText
     });
@@ -67,6 +90,6 @@ export function isDriveUrl(url: string) {
 
 function fileIdFromUrl(url: string) {
   if (!url) { return null; }
-  const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url);
+  const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/?]*)/i.exec(url);
   return match ? match[3] : null;
 }

app/server/lib/sendAppPage.ts

@@ -40,6 +40,7 @@ export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadCo
     pluginUrl,
     stripeAPIKey: process.env.STRIPE_PUBLIC_API_KEY,
     googleClientId: process.env.GOOGLE_CLIENT_ID,
+    googleDriveScope: process.env.GOOGLE_DRIVE_SCOPE,
     helpScoutBeaconId: process.env.HELP_SCOUT_BEACON_ID,
     maxUploadSizeImport: (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || undefined,
     maxUploadSizeAttachment: (Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024) || undefined,

app/server/lib/uploads.ts

@@ -1,6 +1,6 @@
 import {ApiError} from 'app/common/ApiError';
 import {InactivityTimer} from 'app/common/InactivityTimer';
-import {FileUploadResult, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
+import {FetchUrlOptions, FileUploadResult, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
 import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode,
         RequestWithLogin} from 'app/server/lib/Authorizer';
 import {expressWrap} from 'app/server/lib/expressWrap';
@@ -328,20 +328,23 @@ export async function createTmpDir(options: tmp.Options): Promise<TmpDirResult>
 /**
  * Register a new upload with resource fetched from a public url. Returns corresponding UploadInfo.
  */
-export async function fetchURL(url: string, accessId: string|null): Promise<UploadResult> {
-  return _fetchURL(url, accessId, path.basename(url));
+export async function fetchURL(url: string, accessId: string|null, options?: FetchUrlOptions): Promise<UploadResult> {
+  return _fetchURL(url, accessId, { fileName: path.basename(url), ...options});
 }
 
 /**
  * Register a new upload with resource fetched from a url, optionally including credentials in request.
  * Returns corresponding UploadInfo.
  */
-async function _fetchURL(url: string, accessId: string|null, fileName: string,
-                         headers?: {[key: string]: string}): Promise<UploadResult> {
+async function _fetchURL(url: string, accessId: string|null, options?: FetchUrlOptions): Promise<UploadResult> {
   try {
+    const code = options?.googleAuthorizationCode;
+    let fileName = options?.fileName ?? '';
+    const headers = options?.headers;
     let response: FetchResponse;
     if (isDriveUrl(url)) {
-      response = await downloadFromGDrive(url);
+      response = await downloadFromGDrive(url, code);
+      fileName = ''; // Read the file name from headers.
     } else {
       response = await Deps.fetch(url, {
         redirect: 'follow',
@@ -402,7 +405,7 @@ async function fetchDoc(homeUrl: string, docId: string, req: Request, accessId:
 
   // Download the document, in full or as a template.
   const url = `${docWorkerUrl}download?doc=${docId}&template=${Number(template)}`;
-  return _fetchURL(url, accessId, '', headers);
+  return _fetchURL(url, accessId, {headers});
 }
 
 // Re-issue failures as exceptions.