add an endpoint for doing SQL selects (#641)

* add an endpoint for doing SQL selects

This adds an endpoint for doing SQL selects directly on a Grist document. Other kinds of statements are not supported. There is a default timeout of a second on queries.

This follows loosely an API design by Alex Hall.

Co-authored-by: jarek <jaroslaw.sadzinski@gmail.com>

test/server/lib/DocApi.ts

@@ -874,6 +874,15 @@ function testDocApi() {
     resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/_grist_Tables/data/delete`,
       [2, 3, 4, 5, 6], chimpy);
     assert.equal(resp.status, 200);
+
+    // Despite deleting tables (even in a more official way than above),
+    // there are rules lingering relating to them. TODO: look into this.
+    resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/_grist_ACLRules/data/delete`,
+      [2, 3], chimpy);
+    assert.equal(resp.status, 200);
+    resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/_grist_ACLResources/data/delete`,
+      [2, 3], chimpy);
+    assert.equal(resp.status, 200);
   });
 
   describe("/docs/{did}/tables/{tid}/columns", function () {
@@ -3174,6 +3183,7 @@ function testDocApi() {
         users: {"kiwi@getgrist.com": 'editors' as string | null}
       };
       let accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
+      await flushAuth();
       assert.equal(accessResp.status, 200);
 
       const check = userCheck.bind(null, kiwi);
@@ -3198,6 +3208,7 @@ function testDocApi() {
       delta.users['kiwi@getgrist.com'] = null;
       accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
       assert.equal(accessResp.status, 200);
+      await flushAuth();
     });
 
     it("DELETE /docs/{did}/tables/webhooks should not be allowed for not-owner", async function () {
@@ -3210,6 +3221,7 @@ function testDocApi() {
       };
       let accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
       assert.equal(accessResp.status, 200);
+      await flushAuth();
 
       // Actually unsubscribe with the same unsubscribeKey that was returned by registration - it shouldn't be accepted
       await check(subscribeResponse.webhookId, 403, /No owner access/);
@@ -3219,6 +3231,7 @@ function testDocApi() {
       delta.users['kiwi@getgrist.com'] = null;
       accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
       assert.equal(accessResp.status, 200);
+      await flushAuth();
     });
   });
 
@@ -4502,6 +4515,160 @@ function testDocApi() {
 
   });
 
+
+  it ("GET /docs/{did}/sql is functional", async function () {
+    const query = 'select+*+from+Table1+order+by+id';
+    const resp = await axios.get(`${homeUrl}/api/docs/${docIds.Timesheets}/sql?q=${query}`, chimpy);
+    assert.equal(resp.status, 200);
+    assert.deepEqual(resp.data, {
+      statement: 'select * from Table1 order by id',
+      records: [
+        {
+          fields: {
+            id: 1,
+            manualSort: 1,
+            A: 'hello',
+            B: '',
+            C: '',
+            D: null,
+            E: 'HELLO'
+          },
+        },
+        {
+          fields: { id: 2, manualSort: 2, A: '', B: 'world', C: '', D: null, E: '' }
+        },
+        {
+          fields: { id: 3, manualSort: 3, A: '', B: '', C: '', D: null, E: '' }
+        },
+        {
+          fields: { id: 4, manualSort: 4, A: '', B: '', C: '', D: null, E: '' }
+        },
+      ]
+    });
+  });
+
+  it ("POST /docs/{did}/sql is functional", async function () {
+    let resp = await axios.post(
+      `${homeUrl}/api/docs/${docIds.Timesheets}/sql`,
+      { sql: "select A from Table1 where id = ?", args: [ 1 ] },
+      chimpy);
+    assert.equal(resp.status, 200);
+    assert.deepEqual(resp.data.records, [{
+      fields: {
+        A: 'hello',
+      }
+    }]);
+
+    resp = await axios.post(
+      `${homeUrl}/api/docs/${docIds.Timesheets}/sql`,
+      { nosql: "select A from Table1 where id = ?", args: [ 1 ] },
+      chimpy);
+    assert.equal(resp.status, 400);
+    assert.deepEqual(resp.data, {
+      error: 'Invalid payload',
+      details: { userError: 'Error: body.sql is missing' }
+    });
+  });
+
+  it ("POST /docs/{did}/sql has access control", async function () {
+    // Check non-viewer doesn't have access.
+    const url = `${homeUrl}/api/docs/${docIds.Timesheets}/sql`;
+    const query = { sql: "select A from Table1 where id = ?", args: [ 1 ] };
+    let resp = await axios.post(url, query, kiwi);
+    assert.equal(resp.status, 403);
+    assert.deepEqual(resp.data, {
+      error: 'No view access',
+    });
+
+    try {
+      // Check a viewer would have access.
+      const delta = {
+        users: { 'kiwi@getgrist.com': 'viewers' },
+      };
+      await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
+      await flushAuth();
+      resp = await axios.post(url, query, kiwi);
+      assert.equal(resp.status, 200);
+
+      // Check a viewer would not have access if there is some private material.
+      await axios.post(
+        `${homeUrl}/api/docs/${docIds.Timesheets}/apply`, [
+          ['AddTable', 'TablePrivate', [{id: 'A', type: 'Int'}]],
+          ['AddRecord', '_grist_ACLResources', -1, {tableId: 'TablePrivate', colIds: '*'}],
+          ['AddRecord', '_grist_ACLRules', null, {
+            resource: -1, aclFormula: '', permissionsText: 'none',
+          }],
+        ], chimpy);
+      resp = await axios.post(url, query, kiwi);
+      assert.equal(resp.status, 403);
+    } finally {
+      // Remove extra viewer; remove extra table.
+      const delta = {
+        users: { 'kiwi@getgrist.com': null },
+      };
+      await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
+      await flushAuth();
+      await axios.post(
+        `${homeUrl}/api/docs/${docIds.Timesheets}/apply`, [
+          ['RemoveTable', 'TablePrivate'],
+        ], chimpy);
+    }
+  });
+
+  it ("POST /docs/{did}/sql accepts only selects", async function () {
+    async function check(accept: boolean, sql: string, ...args: any[]) {
+      const resp = await axios.post(
+        `${homeUrl}/api/docs/${docIds.Timesheets}/sql`,
+        { sql, args },
+        chimpy);
+      if (accept) {
+        assert.equal(resp.status, 200);
+      } else {
+        assert.equal(resp.status, 400);
+      }
+      return resp.data;
+    }
+    await check(true, 'select * from Table1');
+    await check(true, '  SeLeCT * from Table1');
+    await check(true, 'with results as (select 1) select * from results');
+
+    // rejected quickly since no select
+    await check(false, 'delete from Table1');
+    await check(false, '');
+
+    // rejected because deletes/updates/... can't be nested within a select
+    await check(false, "delete from Table1 where id in (select id from Table1) and 'selecty' = 'selecty'");
+    await check(false, "update Table1 set A = ? where 'selecty' = 'selecty'", 'test');
+    await check(false, "pragma data_store_directory = 'selecty'");
+    await check(false, "create table selecty(x, y)");
+    await check(false, "attach database 'selecty' AS test");
+
+    // rejected because ";" can't be nested
+    await check(false, 'select * from Table1; delete from Table1');
+
+    // Of course, we can get out of the wrapping select, but we can't
+    // add on more statements. For example, the following runs with no
+    // trouble - but only the SELECT part. The DELETE is discarded.
+    // (node-sqlite3 doesn't expose enough to give an error message for
+    // this, though we could extend it).
+    await check(true, 'select * from Table1); delete from Table1 where id in (select id from Table1');
+    const {records} = await check(true, 'select * from Table1');
+    // Double-check the deletion didn't happen.
+    assert.lengthOf(records, 4);
+  });
+
+  it ("POST /docs/{did}/sql timeout is effective", async function () {
+    const slowQuery = 'WITH RECURSIVE r(i) AS (VALUES(0) ' +
+        'UNION ALL SELECT i FROM r  LIMIT 1000000) ' +
+        'SELECT i FROM r WHERE i = 1';
+    const resp = await axios.post(
+      `${homeUrl}/api/docs/${docIds.Timesheets}/sql`,
+      { sql: slowQuery, timeout: 10 },
+      chimpy);
+    assert.equal(resp.status, 400);
+    assert.match(resp.data.error, /database interrupt/);
+  });
+
   // PLEASE ADD MORE TESTS HERE
 }
 
@@ -4558,3 +4725,10 @@ async function setupDataDir(dir: string) {
     'ApiDataRecordsTest.grist',
     path.resolve(dir, docIds.ApiDataRecordsTest + '.grist'));
 }
+
+// The access control level of a user on a document may be cached for a
+// few seconds. This method flushes that cache.
+async function flushAuth() {
+  await home.testingHooks.flushAuthorizerCache();
+  await docs.testingHooks.flushAuthorizerCache();
+}