(core) updates from grist-core

test/gen-server/apiUtils.ts

@@ -14,7 +14,6 @@ import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import * as docUtils from 'app/server/lib/docUtils';
 import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
-import log from 'app/server/lib/log';
 import {main as mergedServerMain, ServerType} from 'app/server/mergedServerMain';
 import axios from 'axios';
 import FormData from 'form-data';
@@ -287,7 +286,6 @@ export class TestSession {
       fetch: fetch as any,
       headers,
       newFormData: () => new FormData() as any,
-      logger: log,
     });
     // Make sure api is functioning, and create user if this is their first time to hit API.
     if (checkAccess) { await api.getOrg('current'); }

test/nbrowser/homeUtil.ts

@@ -13,7 +13,6 @@ import {UserProfile} from 'app/common/LoginSessionAPI';
 import {BehavioralPrompt, UserPrefs, WelcomePopup} from 'app/common/Prefs';
 import {DocWorkerAPI, UserAPI, UserAPIImpl} from 'app/common/UserAPI';
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
-import log from 'app/server/lib/log';
 import {TestingHooksClient} from 'app/server/lib/TestingHooks';
 import EventEmitter = require('events');
 
@@ -445,7 +444,7 @@ export class HomeUtil {
       headers,
       fetch: fetch as any,
       newFormData: () => new FormData() as any,  // form-data isn't quite type compatible
-      logger: log});
+    });
   }
 
   private async _toggleTips(enabled: boolean, email: string) {

test/server/lib/DocApi.ts

@@ -14,7 +14,6 @@ import {
   getDocApiUsageKeysToIncr,
   WebhookSubscription
 } from 'app/server/lib/DocApi';
-import log from 'app/server/lib/log';
 import {delayAbort} from 'app/server/lib/serverUtils';
 import axios, {AxiosRequestConfig, AxiosResponse} from 'axios';
 import {delay} from 'bluebird';
@@ -875,6 +874,15 @@ function testDocApi() {
     resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/_grist_Tables/data/delete`,
       [2, 3, 4, 5, 6], chimpy);
     assert.equal(resp.status, 200);
+
+    // Despite deleting tables (even in a more official way than above),
+    // there are rules lingering relating to them. TODO: look into this.
+    resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/_grist_ACLRules/data/delete`,
+      [2, 3], chimpy);
+    assert.equal(resp.status, 200);
+    resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/_grist_ACLResources/data/delete`,
+      [2, 3], chimpy);
+    assert.equal(resp.status, 200);
   });
 
   describe("/docs/{did}/tables/{tid}/columns", function () {
@@ -2485,7 +2493,6 @@ function testDocApi() {
       headers: {Authorization: 'Bearer api_key_for_kiwi'},
       fetch: fetch as any,
       newFormData: () => new FormData() as any,
-      logger: log
     });
     // upload something for Chimpy and something else for Kiwi.
     const worker1 = await userApi.getWorkerAPI('import');
@@ -2593,7 +2600,6 @@ function testDocApi() {
       headers: {Authorization: 'Bearer api_key_for_chimpy'},
       fetch: fetch as any,
       newFormData: () => new FormData() as any,
-      logger: log
     });
     const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id;
     const doc2 = await nasaApi.newDoc({name: 'testdoc2', urlId: 'urlid'}, ws2);
@@ -2625,7 +2631,6 @@ function testDocApi() {
       headers: {Authorization: 'Bearer api_key_for_chimpy'},
       fetch: fetch as any,
       newFormData: () => new FormData() as any,
-      logger: log
     });
     const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id;
     const doc2 = await nasaApi.newDoc({name: 'testdoc2'}, ws2);
@@ -3178,6 +3183,7 @@ function testDocApi() {
         users: {"kiwi@getgrist.com": 'editors' as string | null}
       };
       let accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
+      await flushAuth();
       assert.equal(accessResp.status, 200);
 
       const check = userCheck.bind(null, kiwi);
@@ -3202,6 +3208,7 @@ function testDocApi() {
       delta.users['kiwi@getgrist.com'] = null;
       accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
       assert.equal(accessResp.status, 200);
+      await flushAuth();
     });
 
     it("DELETE /docs/{did}/tables/webhooks should not be allowed for not-owner", async function () {
@@ -3214,6 +3221,7 @@ function testDocApi() {
       };
       let accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
       assert.equal(accessResp.status, 200);
+      await flushAuth();
 
       // Actually unsubscribe with the same unsubscribeKey that was returned by registration - it shouldn't be accepted
       await check(subscribeResponse.webhookId, 403, /No owner access/);
@@ -3223,6 +3231,7 @@ function testDocApi() {
       delta.users['kiwi@getgrist.com'] = null;
       accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
       assert.equal(accessResp.status, 200);
+      await flushAuth();
     });
   });
 
@@ -4506,6 +4515,160 @@ function testDocApi() {
 
   });
 
+
+  it ("GET /docs/{did}/sql is functional", async function () {
+    const query = 'select+*+from+Table1+order+by+id';
+    const resp = await axios.get(`${homeUrl}/api/docs/${docIds.Timesheets}/sql?q=${query}`, chimpy);
+    assert.equal(resp.status, 200);
+    assert.deepEqual(resp.data, {
+      statement: 'select * from Table1 order by id',
+      records: [
+        {
+          fields: {
+            id: 1,
+            manualSort: 1,
+            A: 'hello',
+            B: '',
+            C: '',
+            D: null,
+            E: 'HELLO'
+          },
+        },
+        {
+          fields: { id: 2, manualSort: 2, A: '', B: 'world', C: '', D: null, E: '' }
+        },
+        {
+          fields: { id: 3, manualSort: 3, A: '', B: '', C: '', D: null, E: '' }
+        },
+        {
+          fields: { id: 4, manualSort: 4, A: '', B: '', C: '', D: null, E: '' }
+        },
+      ]
+    });
+  });
+
+  it ("POST /docs/{did}/sql is functional", async function () {
+    let resp = await axios.post(
+      `${homeUrl}/api/docs/${docIds.Timesheets}/sql`,
+      { sql: "select A from Table1 where id = ?", args: [ 1 ] },
+      chimpy);
+    assert.equal(resp.status, 200);
+    assert.deepEqual(resp.data.records, [{
+      fields: {
+        A: 'hello',
+      }
+    }]);
+
+    resp = await axios.post(
+      `${homeUrl}/api/docs/${docIds.Timesheets}/sql`,
+      { nosql: "select A from Table1 where id = ?", args: [ 1 ] },
+      chimpy);
+    assert.equal(resp.status, 400);
+    assert.deepEqual(resp.data, {
+      error: 'Invalid payload',
+      details: { userError: 'Error: body.sql is missing' }
+    });
+  });
+
+  it ("POST /docs/{did}/sql has access control", async function () {
+    // Check non-viewer doesn't have access.
+    const url = `${homeUrl}/api/docs/${docIds.Timesheets}/sql`;
+    const query = { sql: "select A from Table1 where id = ?", args: [ 1 ] };
+    let resp = await axios.post(url, query, kiwi);
+    assert.equal(resp.status, 403);
+    assert.deepEqual(resp.data, {
+      error: 'No view access',
+    });
+
+    try {
+      // Check a viewer would have access.
+      const delta = {
+        users: { 'kiwi@getgrist.com': 'viewers' },
+      };
+      await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
+      await flushAuth();
+      resp = await axios.post(url, query, kiwi);
+      assert.equal(resp.status, 200);
+
+      // Check a viewer would not have access if there is some private material.
+      await axios.post(
+        `${homeUrl}/api/docs/${docIds.Timesheets}/apply`, [
+          ['AddTable', 'TablePrivate', [{id: 'A', type: 'Int'}]],
+          ['AddRecord', '_grist_ACLResources', -1, {tableId: 'TablePrivate', colIds: '*'}],
+          ['AddRecord', '_grist_ACLRules', null, {
+            resource: -1, aclFormula: '', permissionsText: 'none',
+          }],
+        ], chimpy);
+      resp = await axios.post(url, query, kiwi);
+      assert.equal(resp.status, 403);
+    } finally {
+      // Remove extra viewer; remove extra table.
+      const delta = {
+        users: { 'kiwi@getgrist.com': null },
+      };
+      await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
+      await flushAuth();
+      await axios.post(
+        `${homeUrl}/api/docs/${docIds.Timesheets}/apply`, [
+          ['RemoveTable', 'TablePrivate'],
+        ], chimpy);
+    }
+  });
+
+  it ("POST /docs/{did}/sql accepts only selects", async function () {
+    async function check(accept: boolean, sql: string, ...args: any[]) {
+      const resp = await axios.post(
+        `${homeUrl}/api/docs/${docIds.Timesheets}/sql`,
+        { sql, args },
+        chimpy);
+      if (accept) {
+        assert.equal(resp.status, 200);
+      } else {
+        assert.equal(resp.status, 400);
+      }
+      return resp.data;
+    }
+    await check(true, 'select * from Table1');
+    await check(true, '  SeLeCT * from Table1');
+    await check(true, 'with results as (select 1) select * from results');
+
+    // rejected quickly since no select
+    await check(false, 'delete from Table1');
+    await check(false, '');
+
+    // rejected because deletes/updates/... can't be nested within a select
+    await check(false, "delete from Table1 where id in (select id from Table1) and 'selecty' = 'selecty'");
+    await check(false, "update Table1 set A = ? where 'selecty' = 'selecty'", 'test');
+    await check(false, "pragma data_store_directory = 'selecty'");
+    await check(false, "create table selecty(x, y)");
+    await check(false, "attach database 'selecty' AS test");
+
+    // rejected because ";" can't be nested
+    await check(false, 'select * from Table1; delete from Table1');
+
+    // Of course, we can get out of the wrapping select, but we can't
+    // add on more statements. For example, the following runs with no
+    // trouble - but only the SELECT part. The DELETE is discarded.
+    // (node-sqlite3 doesn't expose enough to give an error message for
+    // this, though we could extend it).
+    await check(true, 'select * from Table1); delete from Table1 where id in (select id from Table1');
+    const {records} = await check(true, 'select * from Table1');
+    // Double-check the deletion didn't happen.
+    assert.lengthOf(records, 4);
+  });
+
+  it ("POST /docs/{did}/sql timeout is effective", async function () {
+    const slowQuery = 'WITH RECURSIVE r(i) AS (VALUES(0) ' +
+        'UNION ALL SELECT i FROM r  LIMIT 1000000) ' +
+        'SELECT i FROM r WHERE i = 1';
+    const resp = await axios.post(
+      `${homeUrl}/api/docs/${docIds.Timesheets}/sql`,
+      { sql: slowQuery, timeout: 10 },
+      chimpy);
+    assert.equal(resp.status, 400);
+    assert.match(resp.data.error, /database interrupt/);
+  });
+
   // PLEASE ADD MORE TESTS HERE
 }
 
@@ -4562,3 +4725,10 @@ async function setupDataDir(dir: string) {
     'ApiDataRecordsTest.grist',
     path.resolve(dir, docIds.ApiDataRecordsTest + '.grist'));
 }
+
+// The access control level of a user on a document may be cached for a
+// few seconds. This method flushes that cache.
+async function flushAuth() {
+  await home.testingHooks.flushAuthorizerCache();
+  await docs.testingHooks.flushAuthorizerCache();
+}

test/server/lib/helpers/TestServer.ts

@@ -137,7 +137,6 @@ export class TestServer {
       headers: {Authorization: `Bearer api_key_for_${user}`},
       fetch: fetch as unknown as typeof globalThis.fetch,
       newFormData: () => new FormData() as any,
-      logger: log
     });
   }