Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

(core) updates from grist-core

Browse Source
This commit is contained in:
Paul Fitzpatrick
2024-04-15 09:24:09 -04:00
parent 6f3cd1ec2f fe9cc80ccc
commit 4567fad947
46 changed files with 2829 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
app/server/lib/DocApi.ts
View File

@@ -392,7 +392,7 @@ export class DocWorkerApi {
const tablesTable = activeDoc.docData!.getMetaTable("_grist_Tables");
const trigger = webhookId ? activeDoc.triggers.getWebhookTriggerRecord(webhookId) : undefined;
let currentTableId = trigger ? tablesTable.getValue(trigger.tableRef, 'tableId')! : undefined;
const {url, eventTypes, isReadyColumn, name} = webhook;
const {url, eventTypes, watchedColIds, isReadyColumn, name} = webhook;
const tableId = await getRealTableId(req.params.tableId || webhook.tableId, {metaTables});
const fields: Partial<SchemaTypes['_grist_Triggers']> = {};
@@ -409,6 +409,23 @@ export class DocWorkerApi {
}
if (tableId !== undefined) {
if (watchedColIds) {
if (tableId !== currentTableId && currentTableId) {
// if the tableId changed, we need to reset the watchedColIds
fields.watchedColRefList = [GristObjCode.List];
} else {
if (!tableId) {
throw new ApiError(`Cannot find columns "${watchedColIds}" because table is not known`, 404);
}
fields.watchedColRefList = [GristObjCode.List, ...watchedColIds
.filter(colId => colId.trim() !== "")
.map(
colId => { return colIdToReference(metaTables, tableId, colId.trim().replace(/^\$/, '')); }
)];
}
} else {
fields.watchedColRefList = [GristObjCode.List];
}
fields.tableRef = tableIdToRef(metaTables, tableId);
currentTableId = tableId;
}
@@ -910,7 +927,6 @@ export class DocWorkerApi {
const docId = activeDoc.docName;
const webhookId = req.params.webhookId;
const {fields, url} = await getWebhookSettings(activeDoc, req, webhookId, req.body);
if (fields.enabled === false) {
await activeDoc.triggers.clearSingleWebhookQueue(webhookId);
}

4
app/server/lib/DocWorkerMap.ts
View File

@@ -6,7 +6,7 @@
import { IChecksumStore } from 'app/server/lib/IChecksumStore';
import { IElectionStore } from 'app/server/lib/IElectionStore';
import { IPermitStores } from 'app/server/lib/Permit';
import {RedisClient} from 'redis';
import { RedisClient } from 'redis';
export interface DocWorkerInfo {
id: string;
@@ -57,6 +57,8 @@ export interface IDocWorkerMap extends IPermitStores, IElectionStore, IChecksumS
// release existing assignments.
setWorkerAvailability(workerId: string, available: boolean): Promise<void>;
isWorkerRegistered(workerInfo: DocWorkerInfo): Promise<boolean>;
// Releases doc from worker, freeing it to be assigned elsewhere.
// Assignments should only be released for workers that are now unavailable.
releaseAssignment(workerId: string, docId: string): Promise<void>;

17
app/server/lib/FlexServer.ts
View File

@@ -452,7 +452,8 @@ export class FlexServer implements GristServer {
// /status/hooks allows the tests to wait for them to be ready.
// If db=1 query parameter is included, status will include the status of DB connection.
// If redis=1 query parameter is included, status will include the status of the Redis connection.
// If ready=1 query parameter is included, status will include whether the server is fully ready.
// If docWorkerRegistered=1 query parameter is included, status will include the status of the
// doc worker registration in Redis.
this.app.get('/status(/hooks)?', async (req, res) => {
const checks = new Map<string, Promise<boolean>|boolean>();
const timeout = optIntegerParam(req.query.timeout, 'timeout') || 10_000;
@@ -474,6 +475,20 @@ export class FlexServer implements GristServer {
if (isParameterOn(req.query.redis)) {
checks.set('redis', asyncCheck(this._docWorkerMap.getRedisClient()?.pingAsync()));
}
if (isParameterOn(req.query.docWorkerRegistered) && this.worker) {
// Only check whether the doc worker is registered if we have a worker.
// The Redis client may not be connected, but in this case this has to
// be checked with the 'redis' parameter (the user may want to avoid
// removing workers when connection is unstable).
if (this._docWorkerMap.getRedisClient()?.connected) {
checks.set('docWorkerRegistered', asyncCheck(
this._docWorkerMap.isWorkerRegistered(this.worker).then(isRegistered => {
if (!isRegistered) { throw new Error('doc worker not registered'); }
return isRegistered;
})
));
}
}
if (isParameterOn(req.query.ready)) {
checks.set('ready', this._isReady);
}

31
app/server/lib/Triggers.ts
View File

@@ -277,6 +277,7 @@ export class DocTriggers {
// Webhook might have been deleted in the mean time.
continue;
}
const decodedWatchedColRefList = decodeObject(t.watchedColRefList) as number[] || [];
// Report some basic info and usage stats.
const entry: WebhookSummary = {
// Id of the webhook
@@ -288,6 +289,7 @@ export class DocTriggers {
// Other fields used to register this webhook.
eventTypes: decodeObject(t.eventTypes) as string[],
isReadyColumn: getColId(t.isReadyColRef) ?? null,
watchedColIds: decodedWatchedColRefList.map((columnRef) => getColId(columnRef)),
tableId: getTableId(t.tableRef) ?? null,
// For future use - for now every webhook is enabled.
enabled: t.enabled,
@@ -509,6 +511,21 @@ export class DocTriggers {
}
}
if (trigger.watchedColRefList) {
for (const colRef of trigger.watchedColRefList.slice(1)) {
if (!this._validateColId(colRef as number, trigger.tableRef)) {
// column does not belong to table, let's ignore trigger and log stats
for (const action of webhookActions) {
const colId = this._getColId(colRef as number); // no validation
const tableId = this._getTableId(trigger.tableRef);
const error = `column is not valid: colId ${colId} does not belong to ${tableId}`;
this._stats.logInvalid(action.id, error).catch(e => log.error("Webhook stats failed to log", e));
}
continue;
}
}
}
// TODO: would be worth checking that the trigger's fields are valid (ie: eventTypes, url,
// ...) as there's no guarantee that they are.
@@ -585,9 +602,21 @@ export class DocTriggers {
}
}
const colIdsToCheck: Array<string> = [];
if (trigger.watchedColRefList) {
for (const colRef of trigger.watchedColRefList.slice(1)) {
colIdsToCheck.push(this._getColId(colRef as number)!);
}
}
let eventType: EventType;
if (readyBefore) {
eventType = "update";
// check if any of the columns to check were changed to consider this an update
if (colIdsToCheck.length === 0 || colIdsToCheck.some(colId => tableDelta.columnDeltas[colId]?.[rowId])) {
eventType = "update";
} else {
return false;
}
// If we allow subscribing to deletion in the future
// if (recordDelta.existedAfter) {
// eventType = "update";

8
app/server/lib/initialDocSql.ts
View File

@@ -6,7 +6,7 @@ export const GRIST_DOC_SQL = `
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT '');
INSERT INTO _grist_DocInfo VALUES(1,'','','',41,'','');
INSERT INTO _grist_DocInfo VALUES(1,'','','',42,'','');
CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0, "rawViewSectionRef" INTEGER DEFAULT 0, "recordCardViewSectionRef" INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "description" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "rules" TEXT DEFAULT NULL, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "_grist_Imports" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "origFileName" TEXT DEFAULT '', "parseFormula" TEXT DEFAULT '', "delimiter" TEXT DEFAULT '', "doublequote" BOOLEAN DEFAULT 0, "escapechar" TEXT DEFAULT '', "quotechar" TEXT DEFAULT '', "skipinitialspace" BOOLEAN DEFAULT 0, "encoding" TEXT DEFAULT '', "hasHeaders" BOOLEAN DEFAULT 0);
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS "_grist_Views_section_field" (id INTEGER PRIMARY KEY,
CREATE TABLE IF NOT EXISTS "_grist_Validations" (id INTEGER PRIMARY KEY, "formula" TEXT DEFAULT '', "name" TEXT DEFAULT '', "tableRef" INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS "_grist_REPL_Hist" (id INTEGER PRIMARY KEY, "code" TEXT DEFAULT '', "outputText" TEXT DEFAULT '', "errorText" TEXT DEFAULT '');
CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "fileExt" TEXT DEFAULT '', "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeDeleted" DATETIME DEFAULT NULL, "timeUploaded" DATETIME DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "_grist_Triggers" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "eventTypes" TEXT DEFAULT NULL, "isReadyColRef" INTEGER DEFAULT 0, "actions" TEXT DEFAULT '', "label" TEXT DEFAULT '', "memo" TEXT DEFAULT '', "enabled" BOOLEAN DEFAULT 0);
CREATE TABLE IF NOT EXISTS "_grist_Triggers" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "eventTypes" TEXT DEFAULT NULL, "isReadyColRef" INTEGER DEFAULT 0, "actions" TEXT DEFAULT '', "label" TEXT DEFAULT '', "memo" TEXT DEFAULT '', "enabled" BOOLEAN DEFAULT 0, "watchedColRefList" TEXT DEFAULT NULL, "options" TEXT DEFAULT '');
CREATE TABLE IF NOT EXISTS "_grist_ACLRules" (id INTEGER PRIMARY KEY, "resource" INTEGER DEFAULT 0, "permissions" INTEGER DEFAULT 0, "principals" TEXT DEFAULT '', "aclFormula" TEXT DEFAULT '', "aclColumn" INTEGER DEFAULT 0, "aclFormulaParsed" TEXT DEFAULT '', "permissionsText" TEXT DEFAULT '', "rulePos" NUMERIC DEFAULT 1e999, "userAttributes" TEXT DEFAULT '', "memo" TEXT DEFAULT '');
INSERT INTO _grist_ACLRules VALUES(1,1,63,'[1]','',0,'','',1e999,'','');
CREATE TABLE IF NOT EXISTS "_grist_ACLResources" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "colIds" TEXT DEFAULT '');
@@ -44,7 +44,7 @@ export const GRIST_DOC_WITH_TABLE1_SQL = `
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT '');
INSERT INTO _grist_DocInfo VALUES(1,'','','',41,'','');
INSERT INTO _grist_DocInfo VALUES(1,'','','',42,'','');
CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0, "rawViewSectionRef" INTEGER DEFAULT 0, "recordCardViewSectionRef" INTEGER DEFAULT 0);
INSERT INTO _grist_Tables VALUES(1,'Table1',1,0,0,2,3);
CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "description" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "rules" TEXT DEFAULT NULL, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
@@ -80,7 +80,7 @@ INSERT INTO _grist_Views_section_field VALUES(9,3,9,4,0,'',0,0,'',NULL);
CREATE TABLE IF NOT EXISTS "_grist_Validations" (id INTEGER PRIMARY KEY, "formula" TEXT DEFAULT '', "name" TEXT DEFAULT '', "tableRef" INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS "_grist_REPL_Hist" (id INTEGER PRIMARY KEY, "code" TEXT DEFAULT '', "outputText" TEXT DEFAULT '', "errorText" TEXT DEFAULT '');
CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "fileExt" TEXT DEFAULT '', "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeDeleted" DATETIME DEFAULT NULL, "timeUploaded" DATETIME DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "_grist_Triggers" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "eventTypes" TEXT DEFAULT NULL, "isReadyColRef" INTEGER DEFAULT 0, "actions" TEXT DEFAULT '', "label" TEXT DEFAULT '', "memo" TEXT DEFAULT '', "enabled" BOOLEAN DEFAULT 0);
CREATE TABLE IF NOT EXISTS "_grist_Triggers" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "eventTypes" TEXT DEFAULT NULL, "isReadyColRef" INTEGER DEFAULT 0, "actions" TEXT DEFAULT '', "label" TEXT DEFAULT '', "memo" TEXT DEFAULT '', "enabled" BOOLEAN DEFAULT 0, "watchedColRefList" TEXT DEFAULT NULL, "options" TEXT DEFAULT '');
CREATE TABLE IF NOT EXISTS "_grist_ACLRules" (id INTEGER PRIMARY KEY, "resource" INTEGER DEFAULT 0, "permissions" INTEGER DEFAULT 0, "principals" TEXT DEFAULT '', "aclFormula" TEXT DEFAULT '', "aclColumn" INTEGER DEFAULT 0, "aclFormulaParsed" TEXT DEFAULT '', "permissionsText" TEXT DEFAULT '', "rulePos" NUMERIC DEFAULT 1e999, "userAttributes" TEXT DEFAULT '', "memo" TEXT DEFAULT '');
INSERT INTO _grist_ACLRules VALUES(1,1,63,'[1]','',0,'','',1e999,'','');
CREATE TABLE IF NOT EXISTS "_grist_ACLResources" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "colIds" TEXT DEFAULT '');

6
app/server/lib/requestUtils.ts
View File

@@ -103,6 +103,12 @@ export function allowHost(req: IncomingMessage, allowedHost: string|URL) {
const proto = getEndUserProtocol(req);
const actualUrl = new URL(getOriginUrl(req));
const allowedUrl = (typeof allowedHost === 'string') ? new URL(`${proto}://${allowedHost}`) : allowedHost;
log.rawDebug('allowHost: ', {
req: (new URL(req.url!, `http://${req.headers.host}`).href),
origin: req.headers.origin,
actualUrl: actualUrl.hostname,
allowedUrl: allowedUrl.hostname,
});
if ((req as RequestWithOrg).isCustomHost) {
// For a request to a custom domain, the full hostname must match.
return actualUrl.hostname === allowedUrl.hostname;

6
app/server/lib/uploads.ts
View File

@@ -409,6 +409,12 @@ export async function fetchDoc(server: GristServer, docId: string, req: Request,
// Prepare headers that preserve credentials of current user.
const headers = getTransitiveHeaders(req);
// Passing the Origin header would serve no purpose here, as we are
// constructing an internal request to fetch from our own doc worker
// URL. Indeed, it may interfere, as it could incur a CORS check in
// `trustOrigin`, which we do not need.
delete headers.Origin;
// Find the doc worker responsible for the document we wish to copy.
// The backend needs to be well configured for this to work.
const homeUrl = server.getHomeUrl(req);