Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

(core) updates from grist-core

Browse Source
This commit is contained in:
Paul Fitzpatrick
2023-07-17 01:22:18 -04:00
parent e208f827af 61bd064f73
commit 450472f74c
29 changed files with 446 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
app/server/lib/ActiveDoc.ts
View File

@@ -80,7 +80,7 @@ import {guessColInfo} from 'app/common/ValueGuesser';
import {parseUserAction} from 'app/common/ValueParser';
import {TEMPLATES_ORG_DOMAIN} from 'app/gen-server/ApiServer';
import {Document} from 'app/gen-server/entity/Document';
import {ParseOptions} from 'app/plugin/FileParserAPI';
import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI';
import {AccessTokenOptions, AccessTokenResult, GristDocAPI} from 'app/plugin/GristAPI';
import {compileAclFormula} from 'app/server/lib/ACLFormula';
import {AssistanceSchemaPromptV1Context} from 'app/server/lib/Assistance';
@@ -113,7 +113,7 @@ import tmp from 'tmp';
import {ActionHistory} from './ActionHistory';
import {ActionHistoryImpl} from './ActionHistoryImpl';
import {ActiveDocImport} from './ActiveDocImport';
import {ActiveDocImport, FileImportOptions} from './ActiveDocImport';
import {DocClients} from './DocClients';
import {DocPluginManager} from './DocPluginManager';
import {
@@ -773,6 +773,17 @@ export class ActiveDoc extends EventEmitter {
await this._activeDocImport.oneStepImport(docSession, uploadInfo);
}
/**
* Import data resulting from parsing a file into a new table.
* In normal circumstances this is only used internally.
* It's exposed publicly for use by grist-static which doesn't use the plugin system.
*/
public async importParsedFileAsNewTable(
docSession: OptDocSession, optionsAndData: ParseFileResult, importOptions: FileImportOptions
): Promise<ImportResult> {
return this._activeDocImport.importParsedFileAsNewTable(docSession, optionsAndData, importOptions);
}
/**
* This function saves attachments from a given upload and creates an entry for them in the database.
* It returns the list of rowIds for the rows created in the _grist_Attachments table.

143
app/server/lib/ActiveDocImport.ts
View File

@@ -44,7 +44,7 @@ interface ReferenceDescription {
refTableId: string;
}
interface FileImportOptions {
export interface FileImportOptions {
// Suggested name of the import file. It is sometimes used as a suggested table name, e.g. for csv imports.
originalFilename: string;
// Containing parseOptions as serialized JSON to pass to the import plugin.
@@ -227,71 +227,14 @@ export class ActiveDocImport {
}
/**
* Imports all files as new tables, using the given transform rules and import options.
* The isHidden flag indicates whether to create temporary hidden tables, or final ones.
* Import data resulting from parsing a file into a new table.
* In normal circumstances this is only used internally.
* It's exposed publicly for use by grist-static which doesn't use the plugin system.
*/
private async _importFiles(docSession: OptDocSession, upload: UploadInfo, transforms: TransformRuleMap[],
{parseOptions = {}, mergeOptionMaps = []}: ImportOptions,
isHidden: boolean): Promise<ImportResult> {
// Check that upload size is within the configured limits.
const limit = (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || Infinity;
const totalSize = upload.files.reduce((acc, f) => acc + f.size, 0);
if (totalSize > limit) {
throw new ApiError(`Imported files must not exceed ${gutil.byteString(limit)}`, 413);
}
// The upload must be within the plugin-accessible directory. Once moved, subsequent calls to
// moveUpload() will return without having to do anything.
if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); }
await moveUpload(upload, this._activeDoc.docPluginManager.tmpDir());
const importResult: ImportResult = {options: parseOptions, tables: []};
for (const [index, file] of upload.files.entries()) {
// If we have a better guess for the file's extension, replace it in origName, to ensure
// that DocPluginManager has access to it to guess the best parser type.
let origName: string = file.origName;
if (file.ext) {
origName = path.basename(origName, path.extname(origName)) + file.ext;
}
const res = await this._importFileAsNewTable(docSession, file.absPath, {
parseOptions,
mergeOptionsMap: mergeOptionMaps[index] || {},
isHidden,
originalFilename: origName,
uploadFileIndex: index,
transformRuleMap: transforms[index] || {}
});
if (index === 0) {
// Returned parse options from the first file should be used for all files in one upload.
importResult.options = parseOptions = res.options;
}
importResult.tables.push(...res.tables);
}
return importResult;
}
/**
* Imports the data stored at tmpPath.
*
* Currently it starts a python parser as a child process
* outside the sandbox, and supports xlsx, csv, and perhaps some other formats. It may
* result in the import of multiple tables, in case of e.g. Excel formats.
* @param {OptDocSession} docSession: Session instance to use for importing.
* @param {String} tmpPath: The path from of the original file.
* @param {FileImportOptions} importOptions: File import options.
* @returns {Promise<ImportResult>} with `options` property containing parseOptions as serialized JSON as adjusted
* or guessed by the plugin, and `tables`, which is which is a list of objects with information about
* tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`.
*/
private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string,
importOptions: FileImportOptions): Promise<ImportResult> {
const {originalFilename, parseOptions, mergeOptionsMap, isHidden, uploadFileIndex,
transformRuleMap} = importOptions;
log.info("ActiveDoc._importFileAsNewTable(%s, %s)", tmpPath, originalFilename);
if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); }
const optionsAndData: ParseFileResult =
await this._activeDoc.docPluginManager.parseFile(tmpPath, originalFilename, parseOptions);
public async importParsedFileAsNewTable(
docSession: OptDocSession, optionsAndData: ParseFileResult, importOptions: FileImportOptions
): Promise<ImportResult> {
const {originalFilename, mergeOptionsMap, isHidden, uploadFileIndex, transformRuleMap} = importOptions;
const options = optionsAndData.parseOptions;
const parsedTables = optionsAndData.tables;
@@ -374,6 +317,76 @@ export class ActiveDocImport {
return ({options, tables});
}
/**
* Imports all files as new tables, using the given transform rules and import options.
* The isHidden flag indicates whether to create temporary hidden tables, or final ones.
*/
private async _importFiles(docSession: OptDocSession, upload: UploadInfo, transforms: TransformRuleMap[],
{parseOptions = {}, mergeOptionMaps = []}: ImportOptions,
isHidden: boolean): Promise<ImportResult> {
// Check that upload size is within the configured limits.
const limit = (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || Infinity;
const totalSize = upload.files.reduce((acc, f) => acc + f.size, 0);
if (totalSize > limit) {
throw new ApiError(`Imported files must not exceed ${gutil.byteString(limit)}`, 413);
}
// The upload must be within the plugin-accessible directory. Once moved, subsequent calls to
// moveUpload() will return without having to do anything.
if (!this._activeDoc.docPluginManager) { throw new Error('no plugin manager available'); }
await moveUpload(upload, this._activeDoc.docPluginManager.tmpDir());
const importResult: ImportResult = {options: parseOptions, tables: []};
for (const [index, file] of upload.files.entries()) {
// If we have a better guess for the file's extension, replace it in origName, to ensure
// that DocPluginManager has access to it to guess the best parser type.
let origName: string = file.origName;
if (file.ext) {
origName = path.basename(origName, path.extname(origName)) + file.ext;
}
const res = await this._importFileAsNewTable(docSession, file.absPath, {
parseOptions,
mergeOptionsMap: mergeOptionMaps[index] || {},
isHidden,
originalFilename: origName,
uploadFileIndex: index,
transformRuleMap: transforms[index] || {}
});
if (index === 0) {
// Returned parse options from the first file should be used for all files in one upload.
importResult.options = parseOptions = res.options;
}
importResult.tables.push(...res.tables);
}
return importResult;
}
/**
* Imports the data stored at tmpPath.
*
* Currently it starts a python parser as a child process
* outside the sandbox, and supports xlsx, csv, and perhaps some other formats. It may
* result in the import of multiple tables, in case of e.g. Excel formats.
* @param {OptDocSession} docSession: Session instance to use for importing.
* @param {String} tmpPath: The path from of the original file.
* @param {FileImportOptions} importOptions: File import options.
* @returns {Promise<ImportResult>} with `options` property containing parseOptions as serialized JSON as adjusted
* or guessed by the plugin, and `tables`, which is which is a list of objects with information about
* tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`.
*/
private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string,
importOptions: FileImportOptions): Promise<ImportResult> {
const {originalFilename, parseOptions} = importOptions;
log.info("ActiveDoc._importFileAsNewTable(%s, %s)", tmpPath, originalFilename);
if (!this._activeDoc.docPluginManager) {
throw new Error('no plugin manager available');
}
const optionsAndData: ParseFileResult =
await this._activeDoc.docPluginManager.parseFile(tmpPath, originalFilename, parseOptions);
return this.importParsedFileAsNewTable(docSession, optionsAndData, importOptions);
}
/**
* Imports records from `hiddenTableId` into `destTableId`, transforming the column
* values from `hiddenTableId` according to the `transformRule`. Finalizes import when done.

8
app/server/lib/DocStorage.ts
View File

@@ -378,7 +378,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
const colListSql = newCols.map(c => `${quoteIdent(c.colId)}=?`).join(', ');
const types = newCols.map(c => c.type);
const sqlParams = DocStorage._encodeColumnsToRows(types, newCols.map(c => [PENDING_VALUE]));
await db.run(`UPDATE ${quoteIdent(tableId)} SET ${colListSql}`, sqlParams[0]);
await db.run(`UPDATE ${quoteIdent(tableId)} SET ${colListSql}`, ...sqlParams[0]);
}
},
@@ -1093,7 +1093,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
public _process_RemoveRecord(tableId: string, rowId: string): Promise<RunResult> {
const sql = "DELETE FROM " + quoteIdent(tableId) + " WHERE id=?";
debuglog("RemoveRecord SQL: " + sql, [rowId]);
return this.run(sql, [rowId]);
return this.run(sql, rowId);
}
@@ -1130,7 +1130,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
const stmt = await this.prepare(preSql + chunkParams + postSql);
for (const index of _.range(0, numChunks * chunkSize, chunkSize)) {
debuglog("DocStorage.BulkRemoveRecord: chunk delete " + index + "-" + (index + chunkSize - 1));
await stmt.run(rowIds.slice(index, index + chunkSize));
await stmt.run(...rowIds.slice(index, index + chunkSize));
}
await stmt.finalize();
}
@@ -1139,7 +1139,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
debuglog("DocStorage.BulkRemoveRecord: leftover delete " + (numChunks * chunkSize) + "-" + (rowIds.length - 1));
const leftoverParams = _.range(numLeftovers).map(q).join(',');
await this.run(preSql + leftoverParams + postSql,
rowIds.slice(numChunks * chunkSize, rowIds.length));
...rowIds.slice(numChunks * chunkSize, rowIds.length));
}
}

5
app/server/lib/FlexServer.ts
View File

@@ -1132,9 +1132,6 @@ export class FlexServer implements GristServer {
await this.loadConfig();
this.addComm();
// Temporary duplication of external storage configuration.
// This may break https://github.com/gristlabs/grist-core/pull/546,
// but will revive other uses of external storage. TODO: reconcile.
await this.create.configure?.();
if (!isSingleUserMode()) {
@@ -1147,7 +1144,7 @@ export class FlexServer implements GristServer {
this._disableExternalStorage = true;
externalStorage.flag('active').set(false);
}
await this.create.configure?.();
await this.create.checkBackend?.();
const workers = this._docWorkerMap;
const docWorkerId = await this._addSelfAsWorker(workers);

9
app/server/lib/ICreate.ts
View File

@@ -32,6 +32,8 @@ export interface ICreate {
sessionSecret(): string;
// Check configuration of the app early enough to show on startup.
configure?(): Promise<void>;
// Optionally perform sanity checks on the configured storage, throwing a fatal error if it is not functional
checkBackend?(): Promise<void>;
// Return a string containing 1 or more HTML tags to insert into the head element of every
// static page.
getExtraHeadHtml?(): string;
@@ -119,6 +121,13 @@ export function makeSimpleCreator(opts: {
return secret;
},
async configure() {
for (const s of storage || []) {
if (s.check()) {
break;
}
}
},
async checkBackend() {
for (const s of storage || []) {
if (s.check()) {
await s.checkBackend?.();