(core) Adding google drive plugin as a fallback for url plugin

Summary:
When importing from url, user types a url for google spreadsheet,
Grist will switch to Google Drive plugin to allow user to choose file manualy.

Test Plan: Browser tests

Reviewers: paulfitz, dsagal

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D2945
This commit is contained in:
Jarosław Sadziński 2021-08-03 12:34:05 +02:00
parent 5aed22dc1e
commit 6ed1d8dfea
7 changed files with 162 additions and 20 deletions

View File

@ -239,11 +239,11 @@ export class GristDoc extends DisposableWithEvents {
const importMenuItems = [
{
label: 'Import from file',
action: () => Importer.selectAndImport(this, null, createPreview),
action: () => Importer.selectAndImport(this, importSourceElems, null, createPreview),
},
...importSourceElems.map(importSourceElem => ({
label: importSourceElem.importSource.label,
action: () => Importer.selectAndImport(this, importSourceElem, createPreview)
action: () => Importer.selectAndImport(this, importSourceElems, importSourceElem, createPreview)
}))
];

View File

@ -7,7 +7,7 @@
import {GristDoc} from "app/client/components/GristDoc";
import {buildParseOptionsForm, ParseOptionValues} from 'app/client/components/ParseOptions';
import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
import {fetchURL, selectFiles, uploadFiles} from 'app/client/lib/uploads';
import {fetchURL, isDriveUrl, selectFiles, uploadFiles} from 'app/client/lib/uploads';
import {reportError} from 'app/client/models/AppModel';
import {ViewSectionRec} from 'app/client/models/DocModel';
import {openFilePicker} from "app/client/ui/FileDialog";
@ -55,7 +55,10 @@ export class Importer extends Disposable {
* Imports using the given plugin importer, or the built-in file-picker when null is passed in.
*/
public static async selectAndImport(
gristDoc: GristDoc, importSourceElem: ImportSourceElement|null, createPreview: CreatePreviewFunc
gristDoc: GristDoc,
imports: ImportSourceElement[],
importSourceElem: ImportSourceElement|null,
createPreview: CreatePreviewFunc
) {
// In case of using built-in file picker we want to get upload result before instantiating Importer
// because if the user dismisses the dialog without picking a file,
@ -84,9 +87,33 @@ export class Importer extends Disposable {
}
}
}
// Importer disposes itself when its dialog is closed, so we do not take ownership of it.
Importer.create(null, gristDoc, importSourceElem, createPreview).pickAndUploadSource(uploadResult)
.catch((err) => reportError(err));
// HACK: The url plugin does not support importing from google drive, and we don't want to
// ask a user for permission to access all his files (needed to download a single file from an URL).
// So to have a nice user experience, we will switch to the built-in google drive plugin and allow
// user to chose a file manually.
// Suggestion for the future is:
// (1) ask the user for the greater permission,
// (2) detect when the permission is not granted, and open the picker-based plugin in that case.
try {
// Importer disposes itself when its dialog is closed, so we do not take ownership of it.
await Importer.create(null, gristDoc, importSourceElem, createPreview).pickAndUploadSource(uploadResult);
} catch(err1) {
// If the url was a Google Drive Url, run the google drive plugin.
if (!(err1 instanceof GDriveUrlNotSupported)) {
reportError(err1);
} else {
const gdrivePlugin = imports.find((p) => p.plugin.definition.id === 'builtIn/gdrive' && p !== importSourceElem);
if (!gdrivePlugin) {
reportError(err1);
} else {
try {
await Importer.create(null, gristDoc, gdrivePlugin, createPreview).pickAndUploadSource(uploadResult);
} catch(err2) {
reportError(err2);
}
}
}
}
}
private _docComm = this._gristDoc.docComm;
@ -143,6 +170,11 @@ export class Importer extends Disposable {
const importSource = await this._importSourceElem.importSourceStub.getImportSource(handle);
plugin.removeRenderTarget(handle);
if (!this._openModalCtl) {
this._showImportDialog();
}
this._renderSpinner();
if (importSource) {
// If data has been picked, upload it.
const item = importSource.item;
@ -150,14 +182,26 @@ export class Importer extends Disposable {
const files = item.files.map(({content, name}) => new File([content], name));
uploadResult = await uploadFiles(files, {docWorkerUrl: this._docComm.docWorkerUrl,
sizeLimit: 'import'});
} else if (item.kind === "url") {
uploadResult = await fetchURL(this._docComm, item.url);
} else {
} else if (item.kind === "url") {
try {
uploadResult = await fetchURL(this._docComm, item.url);
} catch(err) {
if (isDriveUrl(item.url)) {
throw new GDriveUrlNotSupported(item.url);
} else {
throw err;
}
}
} else {
throw new Error(`Import source of kind ${(item as any).kind} are not yet supported!`);
}
}
}
} catch (err) {
if (err instanceof GDriveUrlNotSupported) {
await this._cancelImport();
throw err;
}
if (!this._openModalCtl) {
this._showImportDialog();
}
@ -393,12 +437,18 @@ export class Importer extends Disposable {
}
}
// Used for switching from URL plugin to Google drive plugin
class GDriveUrlNotSupported extends Error {
constructor(public url: string) {
super(`This url ${url} is not supported`);
}
}
function getSourceDescription(sourceInfo: SourceInfo, upload: UploadResult) {
const origName = upload.files[sourceInfo.uploadFileIndex].origName;
return sourceInfo.origTableName ? origName + ' - ' + sourceInfo.origTableName : origName;
}
const cssActionLink = styled('div', `
display: inline-flex;
align-items: center;

View File

@ -154,6 +154,12 @@ export async function uploadFiles(
export async function fetchURL(
docComm: DocComm, url: string, onProgress: ProgressCB = noop
): Promise<UploadResult> {
if (isDriveUrl(url)) {
// don't download from google drive, immediately fallback to server side.
return docComm.fetchURL(url);
}
let response: Response;
try {
response = await window.fetch(url);
@ -173,6 +179,12 @@ export async function fetchURL(
return res!;
}
export function isDriveUrl(url: string) {
if (!url) { return null; }
const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url);
return !!match;
}
/**
* Convert a form to a JSON-stringifiable object, ignoring any File fields.
*/

View File

@ -93,9 +93,11 @@ export async function main() {
}
if (!process.env.GOOGLE_CLIENT_ID) {
// those key is only for development purposes
// and is no secret as it is publicly visible in a plugin page
process.env.GOOGLE_CLIENT_ID = '632317221841-ce66sfp00rf92dip4548dn4hf2ga79us.apps.googleusercontent.com';
log.warn('GOOGLE_CLIENT_ID is not defined, Google Drive Plugin will not work.');
}
if (!process.env.GOOGLE_API_KEY) {
log.warn('GOOGLE_API_KEY is not defined, Url plugin will not be able to access public files.');
}
if (process.env.GRIST_SINGLE_PORT) {

View File

@ -760,7 +760,7 @@ export class FlexServer implements GristServer {
// Those keys are eventually visible by the client, but should be usable
// only from Grist's domains.
const secrets = {
googleClientId : config.googleClientId,
googleClientId: config.googleClientId,
};
res.set('Content-Type', 'application/javascript');
res.status(200);

View File

@ -0,0 +1,72 @@
import {drive} from '@googleapis/drive';
import {Readable} from 'form-data';
import {GaxiosError, GaxiosPromise} from 'gaxios';
import {FetchError, Response as FetchResponse, Headers} from 'node-fetch';
const
SPREADSHEETS_MIMETYPE = 'application/vnd.google-apps.spreadsheet',
XLSX_MIMETYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
export async function downloadFromGDrive(url: string) {
const fileId = fileIdFromUrl(url);
const googleDrive = drive("v3");
const key = process.env.GOOGLE_API_KEY;
if (!key) {
throw new Error("Can't download file from Google Drive. Api key is not configured");
}
if (!fileId) {
throw new Error(`Can't download from ${url}. Url is not valid`);
}
const fileRes = await googleDrive.files.get({
key,
fileId
});
if (fileRes.data.mimeType === SPREADSHEETS_MIMETYPE) {
return await asFetchResponse(googleDrive.files.export(
{ key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE },
{ responseType: 'stream' }
));
} else {
return await asFetchResponse(googleDrive.files.get(
{ key, fileId, alt: 'media' },
{ responseType: 'stream' }
));
}
}
async function asFetchResponse(req: GaxiosPromise<Readable>) {
try {
const res = await req;
return new FetchResponse(res.data, {
headers: new Headers(res.headers),
status: res.status,
statusText: res.statusText
});
} catch (err) {
const error: GaxiosError<Readable> = err;
if (!error.response) {
// Fetch throws exception on network error.
// https://github.com/node-fetch/node-fetch/blob/master/docs/ERROR-HANDLING.md
throw new FetchError(error.message, "system", error.code || "unknown");
} else {
// Fetch returns failure response on http error
const resInit = error.response ? {
status: error.response.status,
headers: new Headers(error.response.headers),
statusText: error.response.statusText
} : undefined;
return new FetchResponse(error.response.data, resInit);
}
}
}
export function isDriveUrl(url: string) {
return !!fileIdFromUrl(url);
}
function fileIdFromUrl(url: string) {
if (!url) { return null; }
const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url);
return match ? match[3] : null;
}

View File

@ -5,6 +5,7 @@ import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode,
RequestWithLogin} from 'app/server/lib/Authorizer';
import {expressWrap} from 'app/server/lib/expressWrap';
import {RequestWithGrist} from 'app/server/lib/FlexServer';
import {downloadFromGDrive, isDriveUrl} from 'app/server/lib/GoogleImport';
import {GristServer} from 'app/server/lib/GristServer';
import {guessExt} from 'app/server/lib/guessExt';
import * as log from 'app/server/lib/log';
@ -338,11 +339,16 @@ export async function fetchURL(url: string, accessId: string|null): Promise<Uplo
async function _fetchURL(url: string, accessId: string|null, fileName: string,
headers?: {[key: string]: string}): Promise<UploadResult> {
try {
const response: FetchResponse = await Deps.fetch(url, {
redirect: 'follow',
follow: 10,
headers
});
let response: FetchResponse;
if (isDriveUrl(url)) {
response = await downloadFromGDrive(url);
} else {
response = await Deps.fetch(url, {
redirect: 'follow',
follow: 10,
headers
});
}
await _checkForError(response);
if (fileName === '') {
const disposition = response.headers.get('content-disposition') || '';