gristlabs_grist-core/app/server/lib/ExpandedQuery.ts

import { ServerQuery } from 'app/common/ActiveDocAPI';
import { ApiError } from 'app/common/ApiError';
import { CellValue } from 'app/common/DocActions';
import { DocData } from 'app/common/DocData';
import { parseFormula } from 'app/common/Formula';
import { removePrefix } from 'app/common/gutil';
import { GristObjCode } from 'app/plugin/GristData';
import { quoteIdent } from 'app/server/lib/SQLiteDB';

/**
 * Represents a query for Grist data with support for SQL-based
 * formulas.  Use of this representation should be limited to within a
 * trusted part of Grist since it assembles SQL strings.
 */
export interface ExpandedQuery extends ServerQuery {
  // Errors detected for given columns because of formula issues.  We
  // need to make sure the result of the query contains these error
  // objects.  It is awkward to write a sql selection that constructs
  // an error object, so instead we select 0 in the case of an error,
  // and substitute in the error object in javascript after the SQL
  // step.  That means we need to pass the error message along
  // explicitly.
  constants?: {
    [colId: string]: [GristObjCode.Exception, string] | [GristObjCode.Pending];
  };

  // A list of join clauses to bring in data from other tables.
  joins?: string[];

  // A list of selections for regular data and data computed via formulas.
  selects?: string[];

  // A list of conditions for filtering query results.
  wheres?: string[];
}

/**
 * Add JOINs and SELECTs to a query in order to implement formulas via SQL.
 *
 * Supports simple formulas that load a column via a reference.
 * The referenced column itself cannot (yet) be a formula.
 * Filtered columns cannot (yet) be a formula.
 *
 * If onDemandFormulas is set, ignore stored formula columns, and compute them using SQL.
 */
export function expandQuery(iquery: ServerQuery, docData: DocData, onDemandFormulas: boolean = true): ExpandedQuery {
  const query: ExpandedQuery = {
    tableId: iquery.tableId,
    filters: iquery.filters,
    limit: iquery.limit
  };

  // Start accumulating a set of joins and selects needed for the query.
  const joins = new Set<string>();
  const selects = new Set<string>();

  // Iterate through all formulas, adding joins and selects as we go.
  if (onDemandFormulas) {
    // Look up the main table for the query.
    const tables = docData.getMetaTable('_grist_Tables');
    const columns = docData.getMetaTable('_grist_Tables_column');
    const tableRef = tables.findRow('tableId', query.tableId);
    if (!tableRef) { throw new ApiError('table not found', 404); }

    // Find any references to other tables.
    const dataColumns = columns.filterRecords({parentId: tableRef, isFormula: false});
    const references = new Map<string, string>();
    for (const column of dataColumns) {
      const refTableId = removePrefix(column.type as string, 'Ref:');
      if (refTableId) { references.set(column.colId as string, refTableId); }
    }

    selects.add(`${quoteIdent(query.tableId)}.id`);
    for (const column of dataColumns) {
      selects.add(`${quoteIdent(query.tableId)}.${quoteIdent(column.colId as string)}`);
    }
    const formulaColumns = columns.filterRecords({parentId: tableRef, isFormula: true});
    for (const column of formulaColumns) {
      const formula = parseFormula(column.formula as string);
      const colId = column.colId as string;
      let sqlFormula = "";
      let error = "";
      if (formula.kind === 'foreignColumn') {
        const altTableId = references.get(formula.refColId);
        const altTableRef = tables.findRow('tableId', altTableId!);
        if (altTableId && altTableRef) {
          const altColumn = columns.filterRecords({parentId: altTableRef, isFormula: false, colId: formula.colId});
          // TODO: deal with a formula column in the other table.
          if (altColumn.length > 0) {
            const alias = `${query.tableId}_${formula.refColId}`;
            joins.add(`LEFT JOIN ${quoteIdent(altTableId)} AS ${quoteIdent(alias)} ` +
                      `ON ${quoteIdent(alias)}.id = ` +
                      `${quoteIdent(query.tableId)}.${quoteIdent(formula.refColId)}`);
            sqlFormula = `${quoteIdent(alias)}.${quoteIdent(formula.colId)}`;
          } else {
            error = "Cannot find column";
          }
        } else {
          error = "Cannot find table";
        }
      } else if (formula.kind === 'column') {
        const altColumn = columns.filterRecords({parentId: tableRef, isFormula: false, colId: formula.colId});
        // TODO: deal with a formula column.
        if (altColumn.length > 0) {
          sqlFormula = `${quoteIdent(query.tableId)}.${quoteIdent(formula.colId)}`;
        } else {
          error = "Cannot find column";
        }
      } else if (formula.kind === 'literalNumber') {
        sqlFormula = `${formula.value}`;
      } else if (formula.kind === 'error') {
        error = formula.msg;
      } else {
        throw new Error('Unrecognized type of formula');
      }
      if (error) {
        // We add a trivial selection, and store errors in the query for substitution later.
        sqlFormula = '0';
        if (!query.constants) { query.constants = {}; }
        query.constants[colId] = [GristObjCode.Exception, error];
      }
      if (sqlFormula) {
        selects.add(`${sqlFormula} as ${quoteIdent(colId)}`);
      }
    }
  } else {
    // Select all data and formula columns.
    selects.add(`${quoteIdent(query.tableId)}.*`);
  }

  // Copy decisions to the query object, and return.
  query.joins = [...joins];
  query.selects = [...selects];
  return query;
}

export function getFormulaErrorForExpandQuery(docData: DocData, tableId: string, colId: string): CellValue {
  // On-demand tables may produce several kinds of error messages, e.g. "Formula not supported" or
  // "Cannot find column". We construct the full query to get the basic message for the requested
  // column, then tack on the detail, which is fine to be the same for all of them.
  const iquery: ServerQuery = {tableId, filters: {}};
  const expanded = expandQuery(iquery, docData, true);
  const constantValue = expanded.constants?.[colId];
  if (constantValue?.length === 2) {
    return [GristObjCode.Exception, constantValue[1],
`Not supported in on-demand tables.

This table is marked as an on-demand table. Such tables don't support most formulas. \
For proper formula support, unmark it as on-demand.
`];
  }
  return null;
}

/**
 * Build a query that relates two homogeneous tables sharing a common set of columns,
 * returning rows that exist in both tables (if they have differences), and rows from
 * `leftTableId` that don't exist in `rightTableId`.
 *
 * In practice, this is currently only used for generating diffs and add/update actions
 * for incremental imports into existing tables. Specifically, `leftTableId` is the
 * source table, and `rightTableId` is the destination table.
 *
 * Columns from the query result are prefixed with the table id and a '.' separator.
 *
 * NOTE: Intended for internal use from trusted parts of Grist only.
 *
 * @param {string} leftTableId Name of the left table in the comparison.
 * @param {string} rightTableId Name of the right table in the comparison.
 * @param {Map<string, string[]>} selectColumns Map of left table column ids to their matching equivalent(s)
 * from the right table. A single left column can be compared against 2 or more right columns, so the
 * values of `selectColumns` are arrays. All of these columns will be included in the result, aliased by
 * table id.
 * @param {Map<string, string>} joinColumns Map of right table column ids to their matching equivalent
 * from the left table. These columns are used to join `leftTableId` to `rightTableId`.
 * @returns {ExpandedQuery} The constructed query.
 */
export function buildComparisonQuery(leftTableId: string, rightTableId: string, selectColumns: Map<string, string[]>,
                                     joinColumns: Map<string, string>): ExpandedQuery {
  const query: ExpandedQuery = { tableId: leftTableId, filters: {} };

  // Start accumulating the JOINS, SELECTS and WHERES needed for the query.
  const joins: string[] = [];
  const selects: string[] = [];
  const wheres: string[] = [];

  // Include the 'id' column from both tables.
  selects.push(
    `${quoteIdent(leftTableId)}.id AS ${quoteIdent(leftTableId + '.id')}`,
    `${quoteIdent(rightTableId)}.id AS ${quoteIdent(rightTableId + '.id')}`
  );

  // Select columns from both tables, using the table id as a prefix for each column name.
  selectColumns.forEach((rightTableColumns, leftTableColumn) => {
    const leftColumnAlias = `${leftTableId}.${leftTableColumn}`;
    selects.push(`${quoteIdent(leftTableId)}.${quoteIdent(leftTableColumn)} AS ${quoteIdent(leftColumnAlias)}`);

    rightTableColumns.forEach(colId => {
      const rightColumnAlias = `${rightTableId}.${colId}`;
      selects.push(`${quoteIdent(rightTableId)}.${quoteIdent(colId)} AS ${quoteIdent(rightColumnAlias)}`
      );
    });
  });

  /**
   * Performance can suffer when large (right) tables have many duplicates for their join columns.
   * Specifically, the number of rows returned by the query can be unreasonably large if each
   * row from the left table is joined against up to N rows from the right table.
   *
   * To work around this, we de-duplicate the right table before joining, returning the first row id
   * we find for a given group of join column values. In practice, this means that each row from
   * the left table can only be matched with at most 1 equivalent row from the right table.
   */
  const dedupedRightTableQuery =
    `SELECT MIN(id) AS id, ${[...joinColumns.keys()].map(v => quoteIdent(v)).join(', ')} ` +
    `FROM ${quoteIdent(rightTableId)} ` +
    `GROUP BY ${[...joinColumns.keys()].map(v => quoteIdent(v)).join(', ')}`;
  const dedupedRightTableAlias = quoteIdent('deduped_' + rightTableId);

  // Join the left table to the (de-duplicated) right table, and include unmatched left rows.
  const joinConditions: string[] = [];
  joinColumns.forEach((leftTableColumn, rightTableColumn) => {
    const leftExpression = `${quoteIdent(leftTableId)}.${quoteIdent(leftTableColumn)}`;
    const rightExpression = `${dedupedRightTableAlias}.${quoteIdent(rightTableColumn)}`;
    joinConditions.push(`${leftExpression} = ${rightExpression}`);
  });
  joins.push(
    `LEFT JOIN (${dedupedRightTableQuery}) AS ${dedupedRightTableAlias} ` +
    `ON ${joinConditions.join(' AND ')}`);

  // Finally, join the de-duplicated right table to the original right table to get all its columns.
  joins.push(
    `LEFT JOIN ${quoteIdent(rightTableId)} ` +
    `ON ${dedupedRightTableAlias}.id = ${quoteIdent(rightTableId)}.id`);

  // Filter out matching rows where all non-join columns from both tables are identical.
  const whereConditions: string[] = [];
  for (const [leftTableColumnId, rightTableColumnIds] of selectColumns.entries()) {
    const leftColumnAlias = quoteIdent(`${leftTableId}.${leftTableColumnId}`);

    for (const rightTableColId of rightTableColumnIds) {
      // If this left/right column id pair was already used for joining, skip it.
      if (joinColumns.get(rightTableColId) === leftTableColumnId) { continue; }

      // Only include rows that have differences in column values.
      const rightColumnAlias = quoteIdent(`${rightTableId}.${rightTableColId}`);
      whereConditions.push(`${leftColumnAlias} IS NOT ${rightColumnAlias}`);
    }
  }
  if (whereConditions.length > 0) {
    wheres.push(`(${whereConditions.join(' OR ')})`);
  }

  // Copy decisions to the query object, and return.
  query.joins = joins;
  query.selects = selects;
  query.wheres = wheres;
  return query;
}