mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
(core) open documents without blocking on data engine
Summary: With this diff, when a user opens a Grist document in a browser, they will be able to view its contents without waiting for the data engine to start up. Once the data engine starts, it will run a calculation and send any updates made. Changes to the document will be blocked until the engine is started and the initial calculation is complete. The increase in responsiveness is useful in its own right, and also reduces the impact of an extra startup time in a candidate next-generation sandbox. A small unrelated fix is included for `core/package.json`, to catch up with a recent change to `package.json`. A small `./build schema` convenience is added to just rebuild the typescript schema file. Test Plan: added test; existing tests pass - small fixes needed in some cases because of new timing Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D3036
This commit is contained in:
Paul Fitzpatrick
parent
42910cb8f7
commit
b3b7410ede
@ -55,6 +55,7 @@ export class DocComm extends Disposable implements ActiveDocAPI {
|
||||
public fork = this._wrapMethod("fork");
|
||||
public checkAclFormula = this._wrapMethod("checkAclFormula");
|
||||
public getAclResources = this._wrapMethod("getAclResources");
|
||||
public waitForInitialization = this._wrapMethod("waitForInitialization");
|
||||
|
||||
public changeUrlIdEmitter = this.autoDispose(new Emitter());
|
||||
|
||||
|
||||
@ -223,7 +223,7 @@ export class GristDoc extends DisposableWithEvents {
|
||||
if (!state.docTour) {
|
||||
startWelcomeTour(() => this._showGristTour.set(false));
|
||||
} else {
|
||||
await startDocTour(this.docData, () => null);
|
||||
await startDocTour(this.docData, this.docComm, () => null);
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import {DocComm} from "app/client/components/DocComm";
|
||||
import {IOnBoardingMsg, startOnBoarding} from "app/client/ui/OnBoardingPopups";
|
||||
import {DocData} from "../../common/DocData";
|
||||
import * as _ from "lodash";
|
||||
@ -9,8 +10,8 @@ import {IconList, IconName} from "../ui2018/IconList";
|
||||
import {cssButtons, cssLinkBtn, cssLinkIcon} from "./ExampleCard";
|
||||
|
||||
|
||||
export async function startDocTour(docData: DocData, onFinishCB: () => void) {
|
||||
const docTour: IOnBoardingMsg[] = await makeDocTour(docData) || invalidDocTour;
|
||||
export async function startDocTour(docData: DocData, docComm: DocComm, onFinishCB: () => void) {
|
||||
const docTour: IOnBoardingMsg[] = await makeDocTour(docData, docComm) || invalidDocTour;
|
||||
exposeDocTour(docTour);
|
||||
startOnBoarding(docTour, onFinishCB);
|
||||
}
|
||||
@ -23,11 +24,15 @@ const invalidDocTour: IOnBoardingMsg[] = [{
|
||||
showHasModal: true,
|
||||
}];
|
||||
|
||||
async function makeDocTour(docData: DocData): Promise<IOnBoardingMsg[] | null> {
|
||||
async function makeDocTour(docData: DocData, docComm: DocComm): Promise<IOnBoardingMsg[] | null> {
|
||||
const tableId = "GristDocTour";
|
||||
if (!docData.getTable(tableId)) {
|
||||
return null;
|
||||
}
|
||||
// Make sure any formulas in GristDocTour table have had time to evaluate. For example, for a
|
||||
// first time open of a new document copy, any use of SELF_HYPERLINK will be stale since the URL
|
||||
// of the document has changed.
|
||||
await docComm.waitForInitialization();
|
||||
await docData.fetchTable(tableId);
|
||||
const tableData = docData.getTable(tableId)!;
|
||||
const result = _.sortBy(tableData.getRowIds(), tableData.getRowPropFunc('manualSort') as any).map(rowId => {
|
||||
|
||||
@ -95,7 +95,7 @@ export function tools(owner: Disposable, gristDoc: GristDoc, leftPanelOpen: Obse
|
||||
automaticHelpTool(
|
||||
async ({markAsSeen}) => {
|
||||
const gristDocModule = await loadGristDoc();
|
||||
await gristDocModule.startDocTour(gristDoc.docData, markAsSeen);
|
||||
await gristDocModule.startDocTour(gristDoc.docData, gristDoc.docComm, markAsSeen);
|
||||
},
|
||||
gristDoc,
|
||||
"seenDocTours",
|
||||
|
||||
@ -267,4 +267,9 @@ export interface ActiveDocAPI {
|
||||
* regardless of rules that may block access to them.
|
||||
*/
|
||||
getAclResources(): Promise<{[tableId: string]: string[]}>;
|
||||
|
||||
/**
|
||||
* Wait for document to finish initializing.
|
||||
*/
|
||||
waitForInitialization(): Promise<void>;
|
||||
}
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
/*** THIS FILE IS AUTO-GENERATED BY core/sandbox/gen_js_schema.py ***/
|
||||
// tslint:disable:object-literal-key-quotes
|
||||
|
||||
export const SCHEMA_VERSION = 24;
|
||||
|
||||
export const schema = {
|
||||
|
||||
"_grist_DocInfo": {
|
||||
|
||||
63
app/server/generateInitialDocSql.ts
Normal file
63
app/server/generateInitialDocSql.ts
Normal file
@ -0,0 +1,63 @@
|
||||
import { ActiveDoc } from 'app/server/lib/ActiveDoc';
|
||||
import { create } from 'app/server/lib/create';
|
||||
import { DocManager } from 'app/server/lib/DocManager';
|
||||
import { makeExceptionalDocSession } from 'app/server/lib/DocSession';
|
||||
import { DocStorageManager } from 'app/server/lib/DocStorageManager';
|
||||
import { PluginManager } from 'app/server/lib/PluginManager';
|
||||
|
||||
import * as childProcess from 'child_process';
|
||||
import * as fse from 'fs-extra';
|
||||
import * as util from 'util';
|
||||
|
||||
const execFile = util.promisify(childProcess.execFile);
|
||||
|
||||
// tslint:disable:no-console
|
||||
|
||||
/**
|
||||
* Output to stdout typescript code containing SQL strings for creating an empty document.
|
||||
* The code is of the form:
|
||||
* export const GRIST_DOC_SQL = <sql code to create a completely empty document>;
|
||||
* export const GRIST_DOC_WITH_TABLE1_SQL = <sql code to create a document with Table1>;
|
||||
* Only tables managed by the data engine are included. Any _gristsys_ tables are excluded.
|
||||
*/
|
||||
export async function main(baseName: string) {
|
||||
console.log("/*** THIS FILE IS AUTO-GENERATED BY app/server/generateInitialDocSql.ts ***/");
|
||||
console.log("");
|
||||
console.log("/* eslint-disable max-len */");
|
||||
for (const version of ['DOC', 'DOC_WITH_TABLE1'] as const) {
|
||||
const storageManager = new DocStorageManager(process.cwd());
|
||||
const pluginManager = new PluginManager();
|
||||
const fname = storageManager.getPath(baseName);
|
||||
if (await fse.pathExists(fname)) {
|
||||
await fse.remove(fname);
|
||||
}
|
||||
const docManager = new DocManager(storageManager, pluginManager, null as any, {create} as any);
|
||||
const activeDoc = new ActiveDoc(docManager, baseName);
|
||||
const session = makeExceptionalDocSession('nascent');
|
||||
await activeDoc.createEmptyDocWithDataEngine(session);
|
||||
if (version === 'DOC_WITH_TABLE1') {
|
||||
await activeDoc.addInitialTable(session);
|
||||
}
|
||||
// Remove all _gristsys_ tables, since creation of these tables is handled by DocStorage,
|
||||
// not data engine.
|
||||
const tables = await activeDoc.docStorage.all("SELECT name FROM sqlite_master WHERE" +
|
||||
" type = 'table' AND" +
|
||||
" name LIKE '_gristsys_%'");
|
||||
for (const table of tables) {
|
||||
await activeDoc.docStorage.exec(`DROP TABLE ${table.name}`);
|
||||
}
|
||||
console.log("");
|
||||
console.log("export const GRIST_" + version + "_SQL = `");
|
||||
console.log((await execFile('sqlite3', [baseName + '.grist', '.dump'])).stdout.trim());
|
||||
console.log("`;");
|
||||
await activeDoc.shutdown();
|
||||
await docManager.shutdownAll();
|
||||
await storageManager.closeStorage();
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main(process.argv[2]).catch(e => {
|
||||
console.log(e);
|
||||
});
|
||||
}
|
||||
@ -27,9 +27,8 @@ import {
|
||||
ServerQuery
|
||||
} from 'app/common/ActiveDocAPI';
|
||||
import {ApiError} from 'app/common/ApiError';
|
||||
import {AsyncCreate, mapGetOrSet, MapWithTTL} from 'app/common/AsyncCreate';
|
||||
import {mapGetOrSet, MapWithTTL} from 'app/common/AsyncCreate';
|
||||
import {
|
||||
BulkColValues,
|
||||
CellValue,
|
||||
DocAction,
|
||||
RowRecord,
|
||||
@ -43,7 +42,7 @@ import {DocumentSettings} from 'app/common/DocumentSettings';
|
||||
import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause';
|
||||
import {byteString, countIf, safeJsonParse} from 'app/common/gutil';
|
||||
import {InactivityTimer} from 'app/common/InactivityTimer';
|
||||
import * as marshal from 'app/common/marshal';
|
||||
import {schema, SCHEMA_VERSION} from 'app/common/schema';
|
||||
import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
|
||||
import {DocReplacementOptions, DocState} from 'app/common/UserAPI';
|
||||
import {ParseOptions} from 'app/plugin/FileParserAPI';
|
||||
@ -54,9 +53,11 @@ import {checksumFile} from 'app/server/lib/checksumFile';
|
||||
import {Client} from 'app/server/lib/Client';
|
||||
import {DEFAULT_CACHE_TTL, DocManager} from 'app/server/lib/DocManager';
|
||||
import {makeForkIds} from 'app/server/lib/idUtils';
|
||||
import {GRIST_DOC_SQL, GRIST_DOC_WITH_TABLE1_SQL} from 'app/server/lib/initialDocSql';
|
||||
import {ISandbox} from 'app/server/lib/ISandbox';
|
||||
import * as log from 'app/server/lib/log';
|
||||
import {shortDesc} from 'app/server/lib/shortDesc';
|
||||
import {TableMetadataLoader} from 'app/server/lib/TableMetadataLoader';
|
||||
import {fetchURL, FileUploadInfo, globalUploadSet, UploadInfo} from 'app/server/lib/uploads';
|
||||
|
||||
import {ActionHistory} from './ActionHistory';
|
||||
@ -75,7 +76,7 @@ import {DocStorage} from './DocStorage';
|
||||
import {expandQuery} from './ExpandedQuery';
|
||||
import {GranularAccess, GranularAccessForBundle} from './GranularAccess';
|
||||
import {OnDemandActions} from './OnDemandActions';
|
||||
import { getLogMetaFromDocSession, supportsEngineChoices} from './serverUtils';
|
||||
import {getLogMetaFromDocSession, supportsEngineChoices, timeoutReached} from './serverUtils';
|
||||
import {findOrAddAllEnvelope, Sharing} from './Sharing';
|
||||
import cloneDeep = require('lodash/cloneDeep');
|
||||
import flatten = require('lodash/flatten');
|
||||
@ -102,7 +103,7 @@ export const Deps = {ACTIVEDOC_TIMEOUT};
|
||||
|
||||
/**
|
||||
* Represents an active document with the given name. The document isn't actually open until
|
||||
* either .loadDoc() or .createDoc() is called.
|
||||
* either .loadDoc() or .createEmptyDoc() is called.
|
||||
* @param {String} docName - The document's filename, without the '.grist' extension.
|
||||
*/
|
||||
export class ActiveDoc extends EventEmitter {
|
||||
@ -131,10 +132,11 @@ export class ActiveDoc extends EventEmitter {
|
||||
// result).
|
||||
protected _modificationLock: Mutex = new Mutex();
|
||||
|
||||
private _dataEngine: AsyncCreate<ISandbox>|undefined;
|
||||
private _dataEngine: Promise<ISandbox>|undefined;
|
||||
private _activeDocImport: ActiveDocImport;
|
||||
private _onDemandActions: OnDemandActions;
|
||||
private _granularAccess: GranularAccess;
|
||||
private _tableMetadataLoader: TableMetadataLoader;
|
||||
private _muted: boolean = false; // If set, changes to this document should not propagate
|
||||
// to outside world
|
||||
private _migrating: number = 0; // If positive, a migration is in progress
|
||||
@ -163,6 +165,12 @@ export class ActiveDoc extends EventEmitter {
|
||||
this._actionHistory = new ActionHistoryImpl(this.docStorage);
|
||||
this.docPluginManager = new DocPluginManager(docManager.pluginManager.getPlugins(),
|
||||
docManager.pluginManager.appRoot!, this, this._docManager.gristServer);
|
||||
this._tableMetadataLoader = new TableMetadataLoader({
|
||||
decodeBuffer: this.docStorage.decodeMarshalledData.bind(this.docStorage),
|
||||
fetchTable: this.docStorage.fetchTable.bind(this.docStorage),
|
||||
loadMetaTables: this._rawPyCall.bind(this, 'load_meta_tables'),
|
||||
loadTable: this._rawPyCall.bind(this, 'load_table'),
|
||||
});
|
||||
|
||||
// Our DataEngine is a separate sandboxed process (one per open document). The data engine runs
|
||||
// user-defined python code including formula calculations. It maintains all document data and
|
||||
@ -170,7 +178,8 @@ export class ActiveDoc extends EventEmitter {
|
||||
|
||||
// Creation of the data engine currently needs to be deferred if we support a choice of
|
||||
// engines, since we need to look at the document to see what kind of engine it needs.
|
||||
// If we don't offer a choice, go ahead and start creating the data engine now, so it
|
||||
// This doesn't delay loading the document, but does delay first calculation and modification.
|
||||
// So if we don't offer a choice, go ahead and start creating the data engine now, so it
|
||||
// is created in parallel to fetching the document from external storage (if needed).
|
||||
// TODO: cache engine requirement for doc in home db so we can retain this parallelism
|
||||
// when offering a choice of data engines.
|
||||
@ -317,6 +326,13 @@ export class ActiveDoc extends EventEmitter {
|
||||
try {
|
||||
const dataEngine = this._dataEngine ? await this._getEngine() : null;
|
||||
this._shuttingDown = true; // Block creation of engine if not yet in existence.
|
||||
if (dataEngine) {
|
||||
// Give a small grace period for finishing initialization if we are being shut
|
||||
// down while initialization is still in progress, and we don't have an easy
|
||||
// way yet to cancel it cleanly. This is mainly for the benefit of automated
|
||||
// tests.
|
||||
await timeoutReached(3000, this.waitForInitialization());
|
||||
}
|
||||
await Promise.all([
|
||||
this.docStorage.shutdown(),
|
||||
this.docPluginManager.shutdown(),
|
||||
@ -336,11 +352,12 @@ export class ActiveDoc extends EventEmitter {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new blank document. Returns a promise for the ActiveDoc itself.
|
||||
* Create a new blank document (no "Table1") using the data engine. This is used only
|
||||
* to generate the SQL saved to initialDocSql.ts
|
||||
*/
|
||||
@ActiveDoc.keepDocOpen
|
||||
public async createDoc(docSession: OptDocSession): Promise<ActiveDoc> {
|
||||
this.logDebug(docSession, "createDoc");
|
||||
public async createEmptyDocWithDataEngine(docSession: OptDocSession): Promise<ActiveDoc> {
|
||||
this.logDebug(docSession, "createEmptyDocWithDataEngine");
|
||||
await this._docManager.storageManager.prepareToCreateDoc(this.docName);
|
||||
await this.docStorage.createFile();
|
||||
await this._rawPyCall('load_empty');
|
||||
@ -353,6 +370,19 @@ export class ActiveDoc extends EventEmitter {
|
||||
this.docStorage.applyStoredActions(getEnvContent(initBundle.stored)));
|
||||
|
||||
await this._initDoc(docSession);
|
||||
await this._tableMetadataLoader.clean();
|
||||
// Makes sure docPluginManager is ready in case new doc is used to import new data
|
||||
await this.docPluginManager.ready;
|
||||
this._fullyLoaded = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new blank document (no "Table1"), used as a stub when importing.
|
||||
*/
|
||||
@ActiveDoc.keepDocOpen
|
||||
public async createEmptyDoc(docSession: OptDocSession): Promise<ActiveDoc> {
|
||||
await this.loadDoc(docSession, {forceNew: true, skipInitialTable: true});
|
||||
// Makes sure docPluginManager is ready in case new doc is used to import new data
|
||||
await this.docPluginManager.ready;
|
||||
this._fullyLoaded = true;
|
||||
@ -366,14 +396,16 @@ export class ActiveDoc extends EventEmitter {
|
||||
* @returns {Promise} Promise for this ActiveDoc itself.
|
||||
*/
|
||||
@ActiveDoc.keepDocOpen
|
||||
public async loadDoc(docSession: OptDocSession): Promise<ActiveDoc> {
|
||||
public async loadDoc(docSession: OptDocSession, options?: {
|
||||
forceNew?: boolean, // If set, document will be created.
|
||||
skipInitialTable?: boolean, // If set, and document is new, "Table1" will not be added.
|
||||
}): Promise<ActiveDoc> {
|
||||
const startTime = Date.now();
|
||||
this.logDebug(docSession, "loadDoc");
|
||||
try {
|
||||
const isNew: boolean = await this._docManager.storageManager.prepareLocalDoc(this.docName);
|
||||
const isNew: boolean = options?.forceNew || await this._docManager.storageManager.prepareLocalDoc(this.docName);
|
||||
if (isNew) {
|
||||
await this.createDoc(docSession);
|
||||
await this.addInitialTable(docSession);
|
||||
await this._createDocFile(docSession, {skipInitialTable: options?.skipInitialTable});
|
||||
} else {
|
||||
await this.docStorage.openFile({
|
||||
beforeMigration: async (currentVersion, newVersion) => {
|
||||
@ -383,13 +415,13 @@ export class ActiveDoc extends EventEmitter {
|
||||
return this._afterMigration(docSession, 'storage', newVersion, success);
|
||||
},
|
||||
});
|
||||
}
|
||||
const tableNames = await this._loadOpenDoc(docSession);
|
||||
const desiredTableNames = tableNames.filter(name => name.startsWith('_grist_'));
|
||||
await this._loadTables(docSession, desiredTableNames);
|
||||
this._startLoadingTables(docSession, desiredTableNames);
|
||||
const pendingTableNames = tableNames.filter(name => !name.startsWith('_grist_'));
|
||||
await this._initDoc(docSession);
|
||||
this._initializationPromise = this._finishInitialization(docSession, pendingTableNames, startTime);
|
||||
}
|
||||
} catch (err) {
|
||||
await this.shutdown();
|
||||
throw err;
|
||||
@ -435,8 +467,8 @@ export class ActiveDoc extends EventEmitter {
|
||||
/**
|
||||
* Finish initializing ActiveDoc, by initializing ActionHistory, Sharing, and docData.
|
||||
*/
|
||||
public async _initDoc(docSession: OptDocSession|null): Promise<void> {
|
||||
const metaTableData = await this._rawPyCall('fetch_meta_tables');
|
||||
public async _initDoc(docSession: OptDocSession): Promise<void> {
|
||||
const metaTableData = await this._tableMetadataLoader.fetchTablesAsActions();
|
||||
this.docData = new DocData(tableId => this.fetchTable(makeExceptionalDocSession('system'), tableId), metaTableData);
|
||||
this._onDemandActions = new OnDemandActions(this.docStorage, this.docData);
|
||||
|
||||
@ -446,6 +478,25 @@ export class ActiveDoc extends EventEmitter {
|
||||
}, this.recoveryMode, this.getHomeDbManager(), this.docName);
|
||||
await this._granularAccess.update();
|
||||
this._sharing = new Sharing(this, this._actionHistory, this._modificationLock);
|
||||
// Make sure there is at least one item in action history. The document will be perfectly
|
||||
// functional without it, but comparing documents would need updating if history can
|
||||
// be empty. For example, comparing an empty document immediately forked with the
|
||||
// original would fail. So far, we have treated documents without a common history
|
||||
// as incomparible, and we'd need to weaken that to allow comparisons with a document
|
||||
// with nothing in action history.
|
||||
if (this._actionHistory.getNextLocalActionNum() === 1) {
|
||||
await this._actionHistory.recordNextShared({
|
||||
userActions: [],
|
||||
undo: [],
|
||||
info: [0, this._makeInfo(makeExceptionalDocSession('system'))],
|
||||
actionNum: 1,
|
||||
actionHash: null, // set by ActionHistory
|
||||
parentActionHash: null,
|
||||
stored: [],
|
||||
calc: [],
|
||||
envelopes: [],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public getHomeDbManager() {
|
||||
@ -458,7 +509,7 @@ export class ActiveDoc extends EventEmitter {
|
||||
public addInitialTable(docSession: OptDocSession) {
|
||||
// Use a non-client-specific session, so that this action is not part of anyone's undo history.
|
||||
const newDocSession = makeExceptionalDocSession('nascent');
|
||||
return this._applyUserActions(newDocSession, [["AddEmptyTable"]]);
|
||||
return this.applyUserActions(newDocSession, [["AddEmptyTable"]]);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -523,7 +574,7 @@ export class ActiveDoc extends EventEmitter {
|
||||
try {
|
||||
const userActions: UserAction[] = await Promise.all(
|
||||
upload.files.map(file => this._prepAttachment(docSession, file)));
|
||||
const result = await this._applyUserActions(docSession, userActions);
|
||||
const result = await this.applyUserActions(docSession, userActions);
|
||||
return result.retValues;
|
||||
} finally {
|
||||
await globalUploadSet.cleanup(uploadId);
|
||||
@ -1128,16 +1179,13 @@ export class ActiveDoc extends EventEmitter {
|
||||
* Loads an open document from DocStorage. Returns a list of the tables it contains.
|
||||
*/
|
||||
protected async _loadOpenDoc(docSession: OptDocSession): Promise<string[]> {
|
||||
// Fetch the schema version of document and sandbox, and migrate if the sandbox is newer.
|
||||
const [schemaVersion, docInfoData] = await Promise.all([
|
||||
this._rawPyCall('get_version'),
|
||||
this.docStorage.fetchTable('_grist_DocInfo'),
|
||||
]);
|
||||
// Check the schema version of document and sandbox, and migrate if the sandbox is newer.
|
||||
const schemaVersion = SCHEMA_VERSION;
|
||||
|
||||
// Migrate the document if needed.
|
||||
const values = marshal.loads(docInfoData);
|
||||
const versionCol = values.schemaVersion;
|
||||
const docSchemaVersion = (versionCol && versionCol.length === 1 ? versionCol[0] : 0);
|
||||
const docInfo = await this._tableMetadataLoader.fetchBulkColValuesWithoutIds('_grist_DocInfo');
|
||||
const versionCol = docInfo.schemaVersion;
|
||||
const docSchemaVersion = (versionCol && versionCol.length === 1 ? versionCol[0] : 0) as number;
|
||||
if (docSchemaVersion < schemaVersion) {
|
||||
this.logInfo(docSession, "Doc needs migration from v%s to v%s", docSchemaVersion, schemaVersion);
|
||||
await this._beforeMigration(docSession, 'schema', docSchemaVersion, schemaVersion);
|
||||
@ -1147,6 +1195,7 @@ export class ActiveDoc extends EventEmitter {
|
||||
success = true;
|
||||
} finally {
|
||||
await this._afterMigration(docSession, 'schema', schemaVersion, success);
|
||||
await this._tableMetadataLoader.clean(); // _grist_DocInfo may have changed.
|
||||
}
|
||||
} else if (docSchemaVersion > schemaVersion) {
|
||||
// We do NOT attempt to down-migrate in this case. Migration code cannot down-migrate
|
||||
@ -1158,16 +1207,19 @@ export class ActiveDoc extends EventEmitter {
|
||||
"proceeding with fingers crossed", docSchemaVersion, schemaVersion);
|
||||
}
|
||||
|
||||
// Load the initial meta tables which determine the document schema.
|
||||
const [tablesData, columnsData] = await Promise.all([
|
||||
this.docStorage.fetchTable('_grist_Tables'),
|
||||
this.docStorage.fetchTable('_grist_Tables_column'),
|
||||
]);
|
||||
// Start loading the initial meta tables which determine the document schema.
|
||||
this._tableMetadataLoader.startStreamingToEngine();
|
||||
this._tableMetadataLoader.startFetchingTable('_grist_Tables');
|
||||
this._tableMetadataLoader.startFetchingTable('_grist_Tables_column');
|
||||
|
||||
const tableNames: string[] = await this._rawPyCall('load_meta_tables', tablesData, columnsData);
|
||||
// Get names of remaining tables.
|
||||
const tablesParsed = await this._tableMetadataLoader.fetchBulkColValuesWithoutIds('_grist_Tables');
|
||||
const tableNames = (tablesParsed.tableId as string[])
|
||||
.concat(Object.keys(schema))
|
||||
.filter(tableId => tableId !== '_grist_Tables' && tableId !== '_grist_Tables_column')
|
||||
.sort();
|
||||
|
||||
// Figure out which tables are on-demand.
|
||||
const tablesParsed: BulkColValues = marshal.loads(tablesData);
|
||||
const onDemandMap = zipObject(tablesParsed.tableId as string[], tablesParsed.onDemand);
|
||||
const onDemandNames = remove(tableNames, (t) => onDemandMap[t]);
|
||||
|
||||
@ -1209,19 +1261,9 @@ export class ActiveDoc extends EventEmitter {
|
||||
}
|
||||
await this._granularAccess.assertCanMaybeApplyUserActions(docSession, actions);
|
||||
|
||||
const user = docSession.mode === 'system' ? 'grist' :
|
||||
(client?.getProfile()?.email || '');
|
||||
|
||||
// Create the UserActionBundle.
|
||||
const action: UserActionBundle = {
|
||||
info: {
|
||||
time: Date.now(),
|
||||
user,
|
||||
inst: this._sharing.instanceId || "unset-inst",
|
||||
desc: options.desc,
|
||||
otherId: options.otherId || 0,
|
||||
linkId: options.linkId || 0,
|
||||
},
|
||||
info: this._makeInfo(docSession, options),
|
||||
userActions: actions,
|
||||
};
|
||||
|
||||
@ -1237,6 +1279,38 @@ export class ActiveDoc extends EventEmitter {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new document file without using or initializing the data engine.
|
||||
*/
|
||||
@ActiveDoc.keepDocOpen
|
||||
private async _createDocFile(docSession: OptDocSession, options?: {
|
||||
skipInitialTable?: boolean, // If set, "Table1" will not be added.
|
||||
}): Promise<void> {
|
||||
this.logDebug(docSession, "createDoc");
|
||||
await this._docManager.storageManager.prepareToCreateDoc(this.docName);
|
||||
await this.docStorage.createFile();
|
||||
const sql = options?.skipInitialTable ? GRIST_DOC_SQL : GRIST_DOC_WITH_TABLE1_SQL;
|
||||
await this.docStorage.exec(sql);
|
||||
const timezone = docSession.browserSettings?.timezone ?? DEFAULT_TIMEZONE;
|
||||
const locale = docSession.browserSettings?.locale ?? DEFAULT_LOCALE;
|
||||
await this.docStorage.run('UPDATE _grist_DocInfo SET timezone = ?, documentSettings = ?',
|
||||
[timezone, JSON.stringify({locale})]);
|
||||
}
|
||||
|
||||
private _makeInfo(docSession: OptDocSession, options: ApplyUAOptions = {}) {
|
||||
const client = docSession.client;
|
||||
const user = docSession.mode === 'system' ? 'grist' :
|
||||
(client?.getProfile()?.email || '');
|
||||
return {
|
||||
time: Date.now(),
|
||||
user,
|
||||
inst: this._sharing.instanceId || "unset-inst",
|
||||
desc: options.desc,
|
||||
otherId: options.otherId || 0,
|
||||
linkId: options.linkId || 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares a single attachment by adding it DocStorage and returns a UserAction to apply.
|
||||
*/
|
||||
@ -1339,6 +1413,18 @@ export class ActiveDoc extends EventEmitter {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start loading the specified tables from the db, without waiting for completion.
|
||||
* The loader can be directed to stream the tables on to the engine.
|
||||
*/
|
||||
private _startLoadingTables(docSession: OptDocSession, tableNames: string[]) {
|
||||
this.logDebug(docSession, "starting to load %s tables: %s", tableNames.length,
|
||||
tableNames.join(", "));
|
||||
for (const tableId of tableNames) {
|
||||
this._tableMetadataLoader.startFetchingTable(tableId);
|
||||
}
|
||||
}
|
||||
|
||||
// Fetches and returns the requested table, or null if it's missing. This allows documents to
|
||||
// load with missing metadata tables (should only matter if migrations are also broken).
|
||||
private async _fetchTableIfPresent(tableName: string): Promise<Buffer|null> {
|
||||
@ -1356,6 +1442,8 @@ export class ActiveDoc extends EventEmitter {
|
||||
@ActiveDoc.keepDocOpen
|
||||
private async _finishInitialization(docSession: OptDocSession, pendingTableNames: string[], startTime: number) {
|
||||
try {
|
||||
await this._tableMetadataLoader.wait();
|
||||
await this._tableMetadataLoader.clean();
|
||||
await this._loadTables(docSession, pendingTableNames);
|
||||
// Calculations are not associated specifically with the user opening the document.
|
||||
// TODO: be careful with which users can create formulas.
|
||||
@ -1457,8 +1545,11 @@ export class ActiveDoc extends EventEmitter {
|
||||
|
||||
private async _getEngine(): Promise<ISandbox> {
|
||||
if (this._shuttingDown) { throw new Error('shutting down, data engine unavailable'); }
|
||||
this._dataEngine = this._dataEngine || new AsyncCreate<ISandbox>(async () => {
|
||||
this._dataEngine = this._dataEngine || this._makeEngine();
|
||||
return this._dataEngine;
|
||||
}
|
||||
|
||||
private async _makeEngine(): Promise<ISandbox> {
|
||||
// Figure out what kind of engine we need for this document.
|
||||
let preferredPythonVersion: '2' | '3' = '2';
|
||||
|
||||
@ -1490,8 +1581,6 @@ export class ActiveDoc extends EventEmitter {
|
||||
docUrl: this._options?.docUrl,
|
||||
preferredPythonVersion,
|
||||
});
|
||||
});
|
||||
return this._dataEngine.get();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -391,7 +391,7 @@ export class DocManager extends EventEmitter {
|
||||
const docName = await this._createNewDoc(basenameHint);
|
||||
return mapSetOrClear(this._activeDocs, docName,
|
||||
this._createActiveDoc(docSession, docName)
|
||||
.then(newDoc => newDoc.createDoc(docSession)));
|
||||
.then(newDoc => newDoc.createEmptyDoc(docSession)));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -109,6 +109,7 @@ export class DocWorker {
|
||||
fork: activeDocMethod.bind(null, 'viewers', 'fork'),
|
||||
checkAclFormula: activeDocMethod.bind(null, 'viewers', 'checkAclFormula'),
|
||||
getAclResources: activeDocMethod.bind(null, 'viewers', 'getAclResources'),
|
||||
waitForInitialization: activeDocMethod.bind(null, 'viewers', 'waitForInitialization'),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
211
app/server/lib/TableMetadataLoader.ts
Normal file
211
app/server/lib/TableMetadataLoader.ts
Normal file
@ -0,0 +1,211 @@
|
||||
import { BulkColValues, TableColValues, TableDataAction, toTableDataAction } from 'app/common/DocActions';
|
||||
|
||||
import fromPairs = require('lodash/fromPairs');
|
||||
|
||||
/**
|
||||
*
|
||||
* Handle fetching tables from the database and pushing them to the data engine during
|
||||
* document load. The goal is to allow opening a document and viewing its contents
|
||||
* without needing to wait for the data engine.
|
||||
*
|
||||
* Fetches are done in parallel, but will be bottlenecked by node-sqlite3 and then
|
||||
* sqlite itself. Pushes are limited to concurrency of 3, and will be bottlenecked
|
||||
* by pipe to engine in any case.
|
||||
*
|
||||
* Historically, there is some tolerance for missing tables. TableMetadataLoader retains
|
||||
* that tolerance.
|
||||
*
|
||||
* The TableMetadataLoader doesn't play a role in document creation or migrations.
|
||||
*
|
||||
* This class is only used for loading metadata. There is no need to use it for
|
||||
* user tables, since the server never needs those tables, they should be passed
|
||||
* on to the data engine without caching. Everything the TableMetadataLoader loads persists
|
||||
* until clean() is called.
|
||||
*
|
||||
*/
|
||||
export class TableMetadataLoader {
|
||||
// Promises of buffers for tables being fetched from database, by tableId.
|
||||
private _fetches = new Map<string, Promise<Buffer|null>>();
|
||||
|
||||
// Set of all tableIds for tables that are fully fetched from database.
|
||||
private _fetched = new Set<string>();
|
||||
|
||||
// Operation promises for tables being loaded into the data engine.
|
||||
private _pushes = new Map<string, Promise<void>>();
|
||||
|
||||
// Set of all tableIds for tables fully loaded into the data engine.
|
||||
private _pushed = new Set<string>();
|
||||
|
||||
// Unpacked tables, for reading within node. Only done if requested.
|
||||
private _tables = new Map<string, TableDataAction>();
|
||||
|
||||
// Operation promise for loading core schema (table and column list) into the data engine.
|
||||
private _corePush: Promise<void>|undefined;
|
||||
|
||||
// True once core push is complete.
|
||||
private _corePushed: boolean = false;
|
||||
|
||||
// The number of promises currently pending.
|
||||
private _pending: number = 0;
|
||||
|
||||
// Buffers will only be pushed to data engine once startStreamingToEngine() is called.
|
||||
private _allowPushes: boolean = false;
|
||||
|
||||
// TableMetadataLoader requires access to database, and the ability to call the data engine.
|
||||
constructor(private _options: {
|
||||
decodeBuffer(buffer: Buffer, tableId: string): TableColValues,
|
||||
fetchTable(tableId: string): Promise<Buffer>,
|
||||
loadMetaTables(tables: Buffer, columns: Buffer): Promise<any>,
|
||||
loadTable(tableId: string, buffer: Buffer): Promise<any>,
|
||||
}) {
|
||||
}
|
||||
|
||||
// Start sending tables to data engine as they are fetched.
|
||||
public startStreamingToEngine() {
|
||||
this._allowPushes = true;
|
||||
this._update();
|
||||
}
|
||||
|
||||
// Start fetching a table from the database, if it isn't already on the way.
|
||||
public startFetchingTable(tableId: string): void {
|
||||
if (!this._fetches.has(tableId)) {
|
||||
this._fetches.set(tableId, this._counted(this.opFetch(tableId)));
|
||||
}
|
||||
}
|
||||
|
||||
// Read out a table as a Buffer.
|
||||
public async fetchTableAsBuffer(tableId: string): Promise<Buffer> {
|
||||
this.startFetchingTable(tableId);
|
||||
const buffer = await this._fetches.get(tableId);
|
||||
if (!buffer) {
|
||||
throw new Error(`required table not found: ${tableId}`);
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
// Read out a table as a TableDataAction. Table is cached in this._tables.
|
||||
public async fetchTableAsAction(tableId: string): Promise<TableDataAction> {
|
||||
let cachedTable = this._tables.get(tableId);
|
||||
if (cachedTable) { return cachedTable; }
|
||||
const buffer = await this.fetchTableAsBuffer(tableId);
|
||||
const values = this._options.decodeBuffer(buffer, tableId);
|
||||
cachedTable = toTableDataAction(tableId, values);
|
||||
this._tables.set(tableId, cachedTable);
|
||||
return cachedTable;
|
||||
}
|
||||
|
||||
// Read content of table as BulkColValues. Does not include row ids.
|
||||
public async fetchBulkColValuesWithoutIds(tableId: string): Promise<BulkColValues> {
|
||||
const table = await this.fetchTableAsAction(tableId);
|
||||
return table[3];
|
||||
}
|
||||
|
||||
// Read out all tables requested thus far as TableDataActions.
|
||||
public async fetchTablesAsActions(): Promise<Record<string, TableDataAction>> {
|
||||
for (const [tableId, opFetch] of this._fetches.entries()) {
|
||||
if (!await opFetch) {
|
||||
// Tolerate missing tables.
|
||||
continue;
|
||||
}
|
||||
await this.fetchTableAsAction(tableId);
|
||||
}
|
||||
return fromPairs([...this._tables.entries()]);
|
||||
}
|
||||
|
||||
// Wait for all operations to complete.
|
||||
public async wait() {
|
||||
while (this._pending > 0) {
|
||||
await Promise.all(this._fetches.values());
|
||||
await this._corePush;
|
||||
await Promise.all(this._pushes.values());
|
||||
}
|
||||
}
|
||||
|
||||
// Wipe all stored state.
|
||||
public async clean() {
|
||||
await this.wait();
|
||||
this._fetches.clear();
|
||||
this._fetched.clear();
|
||||
this._pushes.clear();
|
||||
this._pushed.clear();
|
||||
this._corePush = undefined;
|
||||
this._corePushed = false;
|
||||
this._tables.clear();
|
||||
this._pending = 0;
|
||||
}
|
||||
|
||||
// Core push operation. Before we can send arbitrary tables to engine, we must call
|
||||
// load_meta_tables with tables and columns.
|
||||
public async opCorePush() {
|
||||
const tables = await this.fetchTableAsBuffer('_grist_Tables');
|
||||
const columns = await this.fetchTableAsBuffer('_grist_Tables_column');
|
||||
await this._options.loadMetaTables(tables, columns);
|
||||
this._corePushed = true;
|
||||
// It appears to be bad and unnecessary to send tables and columns outside of core push.
|
||||
this._pushed.add('_grist_Tables');
|
||||
this._pushed.add('_grist_Tables_column');
|
||||
this._update();
|
||||
}
|
||||
|
||||
// Operation to fetch a single table from database.
|
||||
public async opFetch(tableId: string) {
|
||||
try {
|
||||
return await this._options.fetchTable(tableId);
|
||||
} catch (err) {
|
||||
if (/no such table/.test(err.message)) { return null; }
|
||||
throw err;
|
||||
} finally {
|
||||
this._fetched.add(tableId);
|
||||
this._update();
|
||||
}
|
||||
}
|
||||
|
||||
// Operation to push a single table to the data engine.
|
||||
public async opPush(tableId: string) {
|
||||
const buffer = await this._fetches.get(tableId);
|
||||
// Tolerate missing tables.
|
||||
if (buffer) {
|
||||
await this._options.loadTable(tableId, buffer);
|
||||
}
|
||||
this._pushed.add(tableId);
|
||||
this._update();
|
||||
}
|
||||
|
||||
// Called after any operation has completed, to see if there's any more work we can start
|
||||
// doing.
|
||||
private _update() {
|
||||
// If pushes are not allowed yet, there's no possibility of follow-on work.
|
||||
if (!this._allowPushes) { return; }
|
||||
|
||||
// Get a list of new pushes that will be needed.
|
||||
const newPushes = new Set([...this._fetched]
|
||||
.filter(tableId => !(this._pushes.has(tableId) ||
|
||||
this._pushed.has(tableId))));
|
||||
|
||||
// Be careful to do the core push first, once we can.
|
||||
if (!this._corePushed) {
|
||||
if (this._corePush === undefined && newPushes.has('_grist_Tables') && newPushes.has('_grist_Tables_column')) {
|
||||
this._corePush = this._counted(this.opCorePush());
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Start new pushes. Sort to give a bit more determinism, but the order depends on a lot
|
||||
// of low-level details (meaning DocRegressionTest is not on a very firm foundation).
|
||||
for (const tableId of [...newPushes].sort()) {
|
||||
// Put a limit on the number of outstanding pushes permitted.
|
||||
if (this._pushes.size >= this._pushed.size + 3) { break; }
|
||||
this._pushes.set(tableId, this._counted(this.opPush(tableId)));
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper to keep track of pending promises.
|
||||
private async _counted<T>(op: Promise<T>): Promise<T> {
|
||||
this._pending++;
|
||||
try {
|
||||
return await op;
|
||||
} finally {
|
||||
this._pending--;
|
||||
}
|
||||
}
|
||||
}
|
||||
84
app/server/lib/initialDocSql.ts
Normal file
84
app/server/lib/initialDocSql.ts
Normal file
@ -0,0 +1,84 @@
|
||||
/*** THIS FILE IS AUTO-GENERATED BY app/server/generateInitialDocSql.ts ***/
|
||||
|
||||
/* eslint-disable max-len */
|
||||
|
||||
export const GRIST_DOC_SQL = `
|
||||
PRAGMA foreign_keys=OFF;
|
||||
BEGIN TRANSACTION;
|
||||
CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT '');
|
||||
INSERT INTO _grist_DocInfo VALUES(1,'','','',24,'UTC','{"locale": "en-US"}');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Imports" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "origFileName" TEXT DEFAULT '', "parseFormula" TEXT DEFAULT '', "delimiter" TEXT DEFAULT '', "doublequote" BOOLEAN DEFAULT 0, "escapechar" TEXT DEFAULT '', "quotechar" TEXT DEFAULT '', "skipinitialspace" BOOLEAN DEFAULT 0, "encoding" TEXT DEFAULT '', "hasHeaders" BOOLEAN DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_External_database" (id INTEGER PRIMARY KEY, "host" TEXT DEFAULT '', "port" INTEGER DEFAULT 0, "username" TEXT DEFAULT '', "dialect" TEXT DEFAULT '', "database" TEXT DEFAULT '', "storage" TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_External_table" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "databaseRef" INTEGER DEFAULT 0, "tableName" TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_TableViews" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "viewRef" INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_TabItems" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "viewRef" INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_TabBar" (id INTEGER PRIMARY KEY, "viewRef" INTEGER DEFAULT 0, "tabPos" NUMERIC DEFAULT 1e999);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Pages" (id INTEGER PRIMARY KEY, "viewRef" INTEGER DEFAULT 0, "indentation" INTEGER DEFAULT 0, "pagePos" NUMERIC DEFAULT 1e999);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Views" (id INTEGER PRIMARY KEY, "name" TEXT DEFAULT '', "type" TEXT DEFAULT '', "layoutSpec" TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Views_section" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "parentId" INTEGER DEFAULT 0, "parentKey" TEXT DEFAULT '', "title" TEXT DEFAULT '', "defaultWidth" INTEGER DEFAULT 0, "borderWidth" INTEGER DEFAULT 0, "theme" TEXT DEFAULT '', "options" TEXT DEFAULT '', "chartType" TEXT DEFAULT '', "layoutSpec" TEXT DEFAULT '', "filterSpec" TEXT DEFAULT '', "sortColRefs" TEXT DEFAULT '', "linkSrcSectionRef" INTEGER DEFAULT 0, "linkSrcColRef" INTEGER DEFAULT 0, "linkTargetColRef" INTEGER DEFAULT 0, "embedId" TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Views_section_field" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colRef" INTEGER DEFAULT 0, "width" INTEGER DEFAULT 0, "widgetOptions" TEXT DEFAULT '', "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "filter" TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Validations" (id INTEGER PRIMARY KEY, "formula" TEXT DEFAULT '', "name" TEXT DEFAULT '', "tableRef" INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_REPL_Hist" (id INTEGER PRIMARY KEY, "code" TEXT DEFAULT '', "outputText" TEXT DEFAULT '', "errorText" TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeUploaded" DATETIME DEFAULT NULL);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Triggers" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "eventTypes" TEXT DEFAULT NULL, "isReadyColRef" INTEGER DEFAULT 0, "actions" TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_ACLRules" (id INTEGER PRIMARY KEY, "resource" INTEGER DEFAULT 0, "permissions" INTEGER DEFAULT 0, "principals" TEXT DEFAULT '', "aclFormula" TEXT DEFAULT '', "aclColumn" INTEGER DEFAULT 0, "aclFormulaParsed" TEXT DEFAULT '', "permissionsText" TEXT DEFAULT '', "rulePos" NUMERIC DEFAULT 1e999, "userAttributes" TEXT DEFAULT '');
|
||||
INSERT INTO _grist_ACLRules VALUES(1,1,63,'[1]','',0,'','',1e999,'');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_ACLResources" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "colIds" TEXT DEFAULT '');
|
||||
INSERT INTO _grist_ACLResources VALUES(1,'','');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_ACLPrincipals" (id INTEGER PRIMARY KEY, "type" TEXT DEFAULT '', "userEmail" TEXT DEFAULT '', "userName" TEXT DEFAULT '', "groupName" TEXT DEFAULT '', "instanceId" TEXT DEFAULT '');
|
||||
INSERT INTO _grist_ACLPrincipals VALUES(1,'group','','','Owners','');
|
||||
INSERT INTO _grist_ACLPrincipals VALUES(2,'group','','','Admins','');
|
||||
INSERT INTO _grist_ACLPrincipals VALUES(3,'group','','','Editors','');
|
||||
INSERT INTO _grist_ACLPrincipals VALUES(4,'group','','','Viewers','');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_ACLMemberships" (id INTEGER PRIMARY KEY, "parent" INTEGER DEFAULT 0, "child" INTEGER DEFAULT 0);
|
||||
COMMIT;
|
||||
`;
|
||||
|
||||
export const GRIST_DOC_WITH_TABLE1_SQL = `
|
||||
PRAGMA foreign_keys=OFF;
|
||||
BEGIN TRANSACTION;
|
||||
CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT '');
|
||||
INSERT INTO _grist_DocInfo VALUES(1,'','','',24,'UTC','{"locale": "en-US"}');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0);
|
||||
INSERT INTO _grist_Tables VALUES(1,'Table1',1,0,0);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
|
||||
INSERT INTO _grist_Tables_column VALUES(1,1,1,'manualSort','ManualSortPos','',0,'','manualSort',0,0,0,0,0,NULL);
|
||||
INSERT INTO _grist_Tables_column VALUES(2,1,2,'A','Any','',1,'','A',0,0,0,0,0,NULL);
|
||||
INSERT INTO _grist_Tables_column VALUES(3,1,3,'B','Any','',1,'','B',0,0,0,0,0,NULL);
|
||||
INSERT INTO _grist_Tables_column VALUES(4,1,4,'C','Any','',1,'','C',0,0,0,0,0,NULL);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Imports" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "origFileName" TEXT DEFAULT '', "parseFormula" TEXT DEFAULT '', "delimiter" TEXT DEFAULT '', "doublequote" BOOLEAN DEFAULT 0, "escapechar" TEXT DEFAULT '', "quotechar" TEXT DEFAULT '', "skipinitialspace" BOOLEAN DEFAULT 0, "encoding" TEXT DEFAULT '', "hasHeaders" BOOLEAN DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_External_database" (id INTEGER PRIMARY KEY, "host" TEXT DEFAULT '', "port" INTEGER DEFAULT 0, "username" TEXT DEFAULT '', "dialect" TEXT DEFAULT '', "database" TEXT DEFAULT '', "storage" TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_External_table" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "databaseRef" INTEGER DEFAULT 0, "tableName" TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_TableViews" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "viewRef" INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_TabItems" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "viewRef" INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_TabBar" (id INTEGER PRIMARY KEY, "viewRef" INTEGER DEFAULT 0, "tabPos" NUMERIC DEFAULT 1e999);
|
||||
INSERT INTO _grist_TabBar VALUES(1,1,1);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Pages" (id INTEGER PRIMARY KEY, "viewRef" INTEGER DEFAULT 0, "indentation" INTEGER DEFAULT 0, "pagePos" NUMERIC DEFAULT 1e999);
|
||||
INSERT INTO _grist_Pages VALUES(1,1,0,1);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Views" (id INTEGER PRIMARY KEY, "name" TEXT DEFAULT '', "type" TEXT DEFAULT '', "layoutSpec" TEXT DEFAULT '');
|
||||
INSERT INTO _grist_Views VALUES(1,'Table1','raw_data','');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Views_section" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "parentId" INTEGER DEFAULT 0, "parentKey" TEXT DEFAULT '', "title" TEXT DEFAULT '', "defaultWidth" INTEGER DEFAULT 0, "borderWidth" INTEGER DEFAULT 0, "theme" TEXT DEFAULT '', "options" TEXT DEFAULT '', "chartType" TEXT DEFAULT '', "layoutSpec" TEXT DEFAULT '', "filterSpec" TEXT DEFAULT '', "sortColRefs" TEXT DEFAULT '', "linkSrcSectionRef" INTEGER DEFAULT 0, "linkSrcColRef" INTEGER DEFAULT 0, "linkTargetColRef" INTEGER DEFAULT 0, "embedId" TEXT DEFAULT '');
|
||||
INSERT INTO _grist_Views_section VALUES(1,1,1,'record','',100,1,'','','','','','[]',0,0,0,'');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Views_section_field" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colRef" INTEGER DEFAULT 0, "width" INTEGER DEFAULT 0, "widgetOptions" TEXT DEFAULT '', "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "filter" TEXT DEFAULT '');
|
||||
INSERT INTO _grist_Views_section_field VALUES(1,1,1,2,0,'',0,0,'');
|
||||
INSERT INTO _grist_Views_section_field VALUES(2,1,2,3,0,'',0,0,'');
|
||||
INSERT INTO _grist_Views_section_field VALUES(3,1,3,4,0,'',0,0,'');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Validations" (id INTEGER PRIMARY KEY, "formula" TEXT DEFAULT '', "name" TEXT DEFAULT '', "tableRef" INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_REPL_Hist" (id INTEGER PRIMARY KEY, "code" TEXT DEFAULT '', "outputText" TEXT DEFAULT '', "errorText" TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeUploaded" DATETIME DEFAULT NULL);
|
||||
CREATE TABLE IF NOT EXISTS "_grist_Triggers" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "eventTypes" TEXT DEFAULT NULL, "isReadyColRef" INTEGER DEFAULT 0, "actions" TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_ACLRules" (id INTEGER PRIMARY KEY, "resource" INTEGER DEFAULT 0, "permissions" INTEGER DEFAULT 0, "principals" TEXT DEFAULT '', "aclFormula" TEXT DEFAULT '', "aclColumn" INTEGER DEFAULT 0, "aclFormulaParsed" TEXT DEFAULT '', "permissionsText" TEXT DEFAULT '', "rulePos" NUMERIC DEFAULT 1e999, "userAttributes" TEXT DEFAULT '');
|
||||
INSERT INTO _grist_ACLRules VALUES(1,1,63,'[1]','',0,'','',1e999,'');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_ACLResources" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "colIds" TEXT DEFAULT '');
|
||||
INSERT INTO _grist_ACLResources VALUES(1,'','');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_ACLPrincipals" (id INTEGER PRIMARY KEY, "type" TEXT DEFAULT '', "userEmail" TEXT DEFAULT '', "userName" TEXT DEFAULT '', "groupName" TEXT DEFAULT '', "instanceId" TEXT DEFAULT '');
|
||||
INSERT INTO _grist_ACLPrincipals VALUES(1,'group','','','Owners','');
|
||||
INSERT INTO _grist_ACLPrincipals VALUES(2,'group','','','Admins','');
|
||||
INSERT INTO _grist_ACLPrincipals VALUES(3,'group','','','Editors','');
|
||||
INSERT INTO _grist_ACLPrincipals VALUES(4,'group','','','Viewers','');
|
||||
CREATE TABLE IF NOT EXISTS "_grist_ACLMemberships" (id INTEGER PRIMARY KEY, "parent" INTEGER DEFAULT 0, "child" INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS "Table1" (id INTEGER PRIMARY KEY, "manualSort" NUMERIC DEFAULT 1e999, "A" BLOB DEFAULT NULL, "B" BLOB DEFAULT NULL, "C" BLOB DEFAULT NULL);
|
||||
COMMIT;
|
||||
`;
|
||||
@ -36,6 +36,7 @@
|
||||
"@types/image-size": "0.0.29",
|
||||
"@types/js-yaml": "3.11.2",
|
||||
"@types/lodash": "4.14.117",
|
||||
"@types/lru-cache": "5.1.1",
|
||||
"@types/mime-types": "2.1.0",
|
||||
"@types/mocha": "5.2.5",
|
||||
"@types/moment-timezone": "0.5.9",
|
||||
|
||||
@ -26,8 +26,10 @@ def main():
|
||||
/*** THIS FILE IS AUTO-GENERATED BY %s ***/
|
||||
// tslint:disable:object-literal-key-quotes
|
||||
|
||||
export const SCHEMA_VERSION = %d;
|
||||
|
||||
export const schema = {
|
||||
""" % __file__)
|
||||
""" % (__file__, schema.SCHEMA_VERSION))
|
||||
|
||||
for table in schema.schema_create_actions():
|
||||
print(' "%s": {' % table.table_id)
|
||||
|
||||
@ -240,6 +240,11 @@
|
||||
resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.117.tgz#695a7f514182771a1e0f4345d189052ee33c8778"
|
||||
integrity sha512-xyf2m6tRbz8qQKcxYZa7PA4SllYcay+eh25DN3jmNYY6gSTL7Htc/bttVdkqj2wfJGbeWlQiX8pIyJpKU+tubw==
|
||||
|
||||
"@types/lru-cache@5.1.1":
|
||||
version "5.1.1"
|
||||
resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.1.tgz#c48c2e27b65d2a153b19bfc1a317e30872e01eef"
|
||||
integrity sha512-ssE3Vlrys7sdIzs5LOxCzTVMsU7i9oa/IaW92wF32JFb3CVczqOkru2xspuKczHEbG3nvmPY7IFqVmGGHdNbYw==
|
||||
|
||||
"@types/mime-types@2.1.0":
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/@types/mime-types/-/mime-types-2.1.0.tgz#9ca52cda363f699c69466c2a6ccdaad913ea7a73"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user