mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
(core) Adding google drive plugin as a fallback for url plugin
Summary: When importing from url, user types a url for google spreadsheet, Grist will switch to Google Drive plugin to allow user to choose file manualy. Test Plan: Browser tests Reviewers: paulfitz, dsagal Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D2945
This commit is contained in:
parent
5aed22dc1e
commit
6ed1d8dfea
@ -239,11 +239,11 @@ export class GristDoc extends DisposableWithEvents {
|
||||
const importMenuItems = [
|
||||
{
|
||||
label: 'Import from file',
|
||||
action: () => Importer.selectAndImport(this, null, createPreview),
|
||||
action: () => Importer.selectAndImport(this, importSourceElems, null, createPreview),
|
||||
},
|
||||
...importSourceElems.map(importSourceElem => ({
|
||||
label: importSourceElem.importSource.label,
|
||||
action: () => Importer.selectAndImport(this, importSourceElem, createPreview)
|
||||
action: () => Importer.selectAndImport(this, importSourceElems, importSourceElem, createPreview)
|
||||
}))
|
||||
];
|
||||
|
||||
|
@ -7,7 +7,7 @@
|
||||
import {GristDoc} from "app/client/components/GristDoc";
|
||||
import {buildParseOptionsForm, ParseOptionValues} from 'app/client/components/ParseOptions';
|
||||
import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
|
||||
import {fetchURL, selectFiles, uploadFiles} from 'app/client/lib/uploads';
|
||||
import {fetchURL, isDriveUrl, selectFiles, uploadFiles} from 'app/client/lib/uploads';
|
||||
import {reportError} from 'app/client/models/AppModel';
|
||||
import {ViewSectionRec} from 'app/client/models/DocModel';
|
||||
import {openFilePicker} from "app/client/ui/FileDialog";
|
||||
@ -55,7 +55,10 @@ export class Importer extends Disposable {
|
||||
* Imports using the given plugin importer, or the built-in file-picker when null is passed in.
|
||||
*/
|
||||
public static async selectAndImport(
|
||||
gristDoc: GristDoc, importSourceElem: ImportSourceElement|null, createPreview: CreatePreviewFunc
|
||||
gristDoc: GristDoc,
|
||||
imports: ImportSourceElement[],
|
||||
importSourceElem: ImportSourceElement|null,
|
||||
createPreview: CreatePreviewFunc
|
||||
) {
|
||||
// In case of using built-in file picker we want to get upload result before instantiating Importer
|
||||
// because if the user dismisses the dialog without picking a file,
|
||||
@ -84,9 +87,33 @@ export class Importer extends Disposable {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Importer disposes itself when its dialog is closed, so we do not take ownership of it.
|
||||
Importer.create(null, gristDoc, importSourceElem, createPreview).pickAndUploadSource(uploadResult)
|
||||
.catch((err) => reportError(err));
|
||||
// HACK: The url plugin does not support importing from google drive, and we don't want to
|
||||
// ask a user for permission to access all his files (needed to download a single file from an URL).
|
||||
// So to have a nice user experience, we will switch to the built-in google drive plugin and allow
|
||||
// user to chose a file manually.
|
||||
// Suggestion for the future is:
|
||||
// (1) ask the user for the greater permission,
|
||||
// (2) detect when the permission is not granted, and open the picker-based plugin in that case.
|
||||
try {
|
||||
// Importer disposes itself when its dialog is closed, so we do not take ownership of it.
|
||||
await Importer.create(null, gristDoc, importSourceElem, createPreview).pickAndUploadSource(uploadResult);
|
||||
} catch(err1) {
|
||||
// If the url was a Google Drive Url, run the google drive plugin.
|
||||
if (!(err1 instanceof GDriveUrlNotSupported)) {
|
||||
reportError(err1);
|
||||
} else {
|
||||
const gdrivePlugin = imports.find((p) => p.plugin.definition.id === 'builtIn/gdrive' && p !== importSourceElem);
|
||||
if (!gdrivePlugin) {
|
||||
reportError(err1);
|
||||
} else {
|
||||
try {
|
||||
await Importer.create(null, gristDoc, gdrivePlugin, createPreview).pickAndUploadSource(uploadResult);
|
||||
} catch(err2) {
|
||||
reportError(err2);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private _docComm = this._gristDoc.docComm;
|
||||
@ -143,6 +170,11 @@ export class Importer extends Disposable {
|
||||
const importSource = await this._importSourceElem.importSourceStub.getImportSource(handle);
|
||||
plugin.removeRenderTarget(handle);
|
||||
|
||||
if (!this._openModalCtl) {
|
||||
this._showImportDialog();
|
||||
}
|
||||
this._renderSpinner();
|
||||
|
||||
if (importSource) {
|
||||
// If data has been picked, upload it.
|
||||
const item = importSource.item;
|
||||
@ -150,14 +182,26 @@ export class Importer extends Disposable {
|
||||
const files = item.files.map(({content, name}) => new File([content], name));
|
||||
uploadResult = await uploadFiles(files, {docWorkerUrl: this._docComm.docWorkerUrl,
|
||||
sizeLimit: 'import'});
|
||||
} else if (item.kind === "url") {
|
||||
uploadResult = await fetchURL(this._docComm, item.url);
|
||||
} else {
|
||||
} else if (item.kind === "url") {
|
||||
try {
|
||||
uploadResult = await fetchURL(this._docComm, item.url);
|
||||
} catch(err) {
|
||||
if (isDriveUrl(item.url)) {
|
||||
throw new GDriveUrlNotSupported(item.url);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw new Error(`Import source of kind ${(item as any).kind} are not yet supported!`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof GDriveUrlNotSupported) {
|
||||
await this._cancelImport();
|
||||
throw err;
|
||||
}
|
||||
if (!this._openModalCtl) {
|
||||
this._showImportDialog();
|
||||
}
|
||||
@ -393,12 +437,18 @@ export class Importer extends Disposable {
|
||||
}
|
||||
}
|
||||
|
||||
// Used for switching from URL plugin to Google drive plugin
|
||||
class GDriveUrlNotSupported extends Error {
|
||||
constructor(public url: string) {
|
||||
super(`This url ${url} is not supported`);
|
||||
}
|
||||
}
|
||||
|
||||
function getSourceDescription(sourceInfo: SourceInfo, upload: UploadResult) {
|
||||
const origName = upload.files[sourceInfo.uploadFileIndex].origName;
|
||||
return sourceInfo.origTableName ? origName + ' - ' + sourceInfo.origTableName : origName;
|
||||
}
|
||||
|
||||
|
||||
const cssActionLink = styled('div', `
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
|
@ -154,6 +154,12 @@ export async function uploadFiles(
|
||||
export async function fetchURL(
|
||||
docComm: DocComm, url: string, onProgress: ProgressCB = noop
|
||||
): Promise<UploadResult> {
|
||||
|
||||
if (isDriveUrl(url)) {
|
||||
// don't download from google drive, immediately fallback to server side.
|
||||
return docComm.fetchURL(url);
|
||||
}
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await window.fetch(url);
|
||||
@ -173,6 +179,12 @@ export async function fetchURL(
|
||||
return res!;
|
||||
}
|
||||
|
||||
export function isDriveUrl(url: string) {
|
||||
if (!url) { return null; }
|
||||
const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url);
|
||||
return !!match;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a form to a JSON-stringifiable object, ignoring any File fields.
|
||||
*/
|
||||
|
@ -93,9 +93,11 @@ export async function main() {
|
||||
}
|
||||
|
||||
if (!process.env.GOOGLE_CLIENT_ID) {
|
||||
// those key is only for development purposes
|
||||
// and is no secret as it is publicly visible in a plugin page
|
||||
process.env.GOOGLE_CLIENT_ID = '632317221841-ce66sfp00rf92dip4548dn4hf2ga79us.apps.googleusercontent.com';
|
||||
log.warn('GOOGLE_CLIENT_ID is not defined, Google Drive Plugin will not work.');
|
||||
}
|
||||
|
||||
if (!process.env.GOOGLE_API_KEY) {
|
||||
log.warn('GOOGLE_API_KEY is not defined, Url plugin will not be able to access public files.');
|
||||
}
|
||||
|
||||
if (process.env.GRIST_SINGLE_PORT) {
|
||||
|
@ -760,7 +760,7 @@ export class FlexServer implements GristServer {
|
||||
// Those keys are eventually visible by the client, but should be usable
|
||||
// only from Grist's domains.
|
||||
const secrets = {
|
||||
googleClientId : config.googleClientId,
|
||||
googleClientId: config.googleClientId,
|
||||
};
|
||||
res.set('Content-Type', 'application/javascript');
|
||||
res.status(200);
|
||||
|
72
app/server/lib/GoogleImport.ts
Normal file
72
app/server/lib/GoogleImport.ts
Normal file
@ -0,0 +1,72 @@
|
||||
import {drive} from '@googleapis/drive';
|
||||
import {Readable} from 'form-data';
|
||||
import {GaxiosError, GaxiosPromise} from 'gaxios';
|
||||
import {FetchError, Response as FetchResponse, Headers} from 'node-fetch';
|
||||
|
||||
const
|
||||
SPREADSHEETS_MIMETYPE = 'application/vnd.google-apps.spreadsheet',
|
||||
XLSX_MIMETYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
||||
|
||||
export async function downloadFromGDrive(url: string) {
|
||||
const fileId = fileIdFromUrl(url);
|
||||
const googleDrive = drive("v3");
|
||||
const key = process.env.GOOGLE_API_KEY;
|
||||
if (!key) {
|
||||
throw new Error("Can't download file from Google Drive. Api key is not configured");
|
||||
}
|
||||
if (!fileId) {
|
||||
throw new Error(`Can't download from ${url}. Url is not valid`);
|
||||
}
|
||||
const fileRes = await googleDrive.files.get({
|
||||
key,
|
||||
fileId
|
||||
});
|
||||
if (fileRes.data.mimeType === SPREADSHEETS_MIMETYPE) {
|
||||
return await asFetchResponse(googleDrive.files.export(
|
||||
{ key, fileId, alt: 'media', mimeType: XLSX_MIMETYPE },
|
||||
{ responseType: 'stream' }
|
||||
));
|
||||
} else {
|
||||
return await asFetchResponse(googleDrive.files.get(
|
||||
{ key, fileId, alt: 'media' },
|
||||
{ responseType: 'stream' }
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async function asFetchResponse(req: GaxiosPromise<Readable>) {
|
||||
try {
|
||||
const res = await req;
|
||||
return new FetchResponse(res.data, {
|
||||
headers: new Headers(res.headers),
|
||||
status: res.status,
|
||||
statusText: res.statusText
|
||||
});
|
||||
} catch (err) {
|
||||
const error: GaxiosError<Readable> = err;
|
||||
if (!error.response) {
|
||||
// Fetch throws exception on network error.
|
||||
// https://github.com/node-fetch/node-fetch/blob/master/docs/ERROR-HANDLING.md
|
||||
throw new FetchError(error.message, "system", error.code || "unknown");
|
||||
} else {
|
||||
// Fetch returns failure response on http error
|
||||
const resInit = error.response ? {
|
||||
status: error.response.status,
|
||||
headers: new Headers(error.response.headers),
|
||||
statusText: error.response.statusText
|
||||
} : undefined;
|
||||
return new FetchResponse(error.response.data, resInit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function isDriveUrl(url: string) {
|
||||
return !!fileIdFromUrl(url);
|
||||
}
|
||||
|
||||
function fileIdFromUrl(url: string) {
|
||||
if (!url) { return null; }
|
||||
const match = /^https:\/\/(docs|drive).google.com\/(spreadsheets|file)\/d\/([^/]*)/i.exec(url);
|
||||
return match ? match[3] : null;
|
||||
}
|
@ -5,6 +5,7 @@ import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode,
|
||||
RequestWithLogin} from 'app/server/lib/Authorizer';
|
||||
import {expressWrap} from 'app/server/lib/expressWrap';
|
||||
import {RequestWithGrist} from 'app/server/lib/FlexServer';
|
||||
import {downloadFromGDrive, isDriveUrl} from 'app/server/lib/GoogleImport';
|
||||
import {GristServer} from 'app/server/lib/GristServer';
|
||||
import {guessExt} from 'app/server/lib/guessExt';
|
||||
import * as log from 'app/server/lib/log';
|
||||
@ -338,11 +339,16 @@ export async function fetchURL(url: string, accessId: string|null): Promise<Uplo
|
||||
async function _fetchURL(url: string, accessId: string|null, fileName: string,
|
||||
headers?: {[key: string]: string}): Promise<UploadResult> {
|
||||
try {
|
||||
const response: FetchResponse = await Deps.fetch(url, {
|
||||
redirect: 'follow',
|
||||
follow: 10,
|
||||
headers
|
||||
});
|
||||
let response: FetchResponse;
|
||||
if (isDriveUrl(url)) {
|
||||
response = await downloadFromGDrive(url);
|
||||
} else {
|
||||
response = await Deps.fetch(url, {
|
||||
redirect: 'follow',
|
||||
follow: 10,
|
||||
headers
|
||||
});
|
||||
}
|
||||
await _checkForError(response);
|
||||
if (fileName === '') {
|
||||
const disposition = response.headers.get('content-disposition') || '';
|
||||
|
Loading…
Reference in New Issue
Block a user