gristlabs_grist-core/app/server/lib/ExpandedQuery.ts

import {ServerQuery} from 'app/common/ActiveDocAPI';
import {ApiError} from 'app/common/ApiError';
import {DocData} from 'app/common/DocData';
import {parseFormula} from 'app/common/Formula';
import {removePrefix} from 'app/common/gutil';
import {quoteIdent} from 'app/server/lib/SQLiteDB';

/**
 * Represents a query for Grist data with support for SQL-based
 * formulas.  Use of this representation should be limited to within a
 * trusted part of Grist since it assembles SQL strings.
 */
export interface ExpandedQuery extends ServerQuery {
  // Errors detected for given columns because of formula issues.  We
  // need to make sure the result of the query contains these error
  // objects.  It is awkward to write a sql selection that constructs
  // an error object, so instead we select 0 in the case of an error,
  // and substitute in the error object in javascript after the SQL
  // step.  That means we need to pass the error message along
  // explicitly.
  constants?: {
    [colId: string]: ['E', string] | ['P'];
  };

  // A list of join clauses to bring in data from other tables.
  joins?: string[];

  // A list of selections for regular data and data computed via formulas.
  selects?: string[];

  // A list of conditions for filtering query results.
  wheres?: string[];
}

/**
 * Add JOINs and SELECTs to a query in order to implement formulas via SQL.
 *
 * Supports simple formulas that load a column via a reference.
 * The referenced column itself cannot (yet) be a formula.
 * Filtered columns cannot (yet) be a formula.
 *
 * If onDemandFormulas is set, ignore stored formula columns, and compute them using SQL.
 */
export function expandQuery(iquery: ServerQuery, docData: DocData, onDemandFormulas: boolean = true): ExpandedQuery {
  const query: ExpandedQuery = {
    tableId: iquery.tableId,
    filters: iquery.filters,
    limit: iquery.limit
  };

  // Start accumulating a set of joins and selects needed for the query.
  const joins = new Set<string>();
  const selects = new Set<string>();

  // Iterate through all formulas, adding joins and selects as we go.
  if (onDemandFormulas) {
    // Look up the main table for the query.
    const tables = docData.getTable('_grist_Tables')!;
    const columns = docData.getTable('_grist_Tables_column')!;
    const tableRef = tables.findRow('tableId', query.tableId);
    if (!tableRef) { throw new ApiError('table not found', 404); }

    // Find any references to other tables.
    const dataColumns = columns.filterRecords({parentId: tableRef, isFormula: false});
    const references = new Map<string, string>();
    for (const column of dataColumns) {
      const refTableId = removePrefix(column.type as string, 'Ref:');
      if (refTableId) { references.set(column.colId as string, refTableId); }
    }

    selects.add(`${quoteIdent(query.tableId)}.id`);
    for (const column of dataColumns) {
      selects.add(`${quoteIdent(query.tableId)}.${quoteIdent(column.colId as string)}`);
    }
    const formulaColumns = columns.filterRecords({parentId: tableRef, isFormula: true});
    for (const column of formulaColumns) {
      const formula = parseFormula(column.formula as string);
      const colId = column.colId as string;
      let sqlFormula = "";
      let error = "";
      if (formula.kind === 'foreignColumn') {
        const altTableId = references.get(formula.refColId);
        const altTableRef = tables.findRow('tableId', altTableId);
        if (altTableId && altTableRef) {
          const altColumn = columns.filterRecords({parentId: altTableRef, isFormula: false, colId: formula.colId});
          // TODO: deal with a formula column in the other table.
          if (altColumn.length > 0) {
            const alias = `${query.tableId}_${formula.refColId}`;
            joins.add(`LEFT JOIN ${quoteIdent(altTableId)} AS ${quoteIdent(alias)} ` +
                      `ON ${quoteIdent(alias)}.id = ` +
                      `${quoteIdent(query.tableId)}.${quoteIdent(formula.refColId)}`);
            sqlFormula = `${quoteIdent(alias)}.${quoteIdent(formula.colId)}`;
          } else {
            error = "Cannot find column";
          }
        } else {
          error = "Cannot find table";
        }
      } else if (formula.kind === 'column') {
        const altColumn = columns.filterRecords({parentId: tableRef, isFormula: false, colId: formula.colId});
        // TODO: deal with a formula column.
        if (altColumn.length > 0) {
          sqlFormula = `${quoteIdent(query.tableId)}.${quoteIdent(formula.colId)}`;
        } else {
          error = "Cannot find column";
        }
      } else if (formula.kind === 'literalNumber') {
        sqlFormula = `${formula.value}`;
      } else if (formula.kind === 'error') {
        error = formula.msg;
      } else {
        throw new Error('Unrecognized type of formula');
      }
      if (error) {
        // We add a trivial selection, and store errors in the query for substitution later.
        sqlFormula = '0';
        if (!query.constants) { query.constants = {}; }
        query.constants[colId] = ['E', error];
      }
      if (sqlFormula) {
        selects.add(`${sqlFormula} as ${quoteIdent(colId)}`);
      }
    }
  } else {
    // Select all data and formula columns.
    selects.add(`${quoteIdent(query.tableId)}.*`);
  }

  // Copy decisions to the query object, and return.
  query.joins = [...joins];
  query.selects = [...selects];
  return query;
}

/**
 * Build a query that relates two homogenous tables sharing a common set of columns,
 * returning rows that exist in both tables (if they have differences), and rows from
 * `leftTableId` that don't exist in `rightTableId`.
 *
 * In practice, this is currently only used for generating diffs and add/update actions
 * for incremental imports into existing tables. Specifically, `leftTableId` is the
 * source table, and `rightTableId` is the destination table.
 *
 * Columns from the query result are prefixed with the table id and a '.' separator.
 *
 * NOTE: Intended for internal use from trusted parts of Grist only.
 *
 * @param {string} leftTableId Name of the left table in the comparison.
 * @param {string} rightTableId Name of the right table in the comparison.
 * @param {Map<string, string>} selectColumns Map of left table column ids to their matching equivalent
 * from the right table. All of these columns will be included in the result, aliased by table id.
 * @param {Map<string, string>} joinColumns Map of left table column ids to their matching equivalent
 * from the right table. These columns are used to join `leftTableID` to `rightTableId`.
 * @returns {ExpandedQuery} The constructed query.
 */
export function buildComparisonQuery(leftTableId: string, rightTableId: string, selectColumns: Map<string, string>,
                                     joinColumns: Map<string, string>): ExpandedQuery {
  const query: ExpandedQuery = { tableId: leftTableId, filters: {} };

  // Start accumulating the JOINS, SELECTS and WHERES needed for the query.
  const joins: string[] = [];
  const selects: string[] = [];
  const wheres: string[] = [];

  // Include the 'id' column from both tables.
  selects.push(
    `${quoteIdent(leftTableId)}.id AS ${quoteIdent(leftTableId + '.id')}`,
    `${quoteIdent(rightTableId)}.id AS ${quoteIdent(rightTableId + '.id')}`
  );

  // Select columns from both tables using the table id as a prefix for each column name.
  selectColumns.forEach((rightTableColumn, leftTableColumn) => {
    const leftColumnAlias = `${leftTableId}.${leftTableColumn}`;
    const rightColumnAlias = `${rightTableId}.${rightTableColumn}`;
    selects.push(
      `${quoteIdent(leftTableId)}.${quoteIdent(leftTableColumn)} AS ${quoteIdent(leftColumnAlias)}`,
      `${quoteIdent(rightTableId)}.${quoteIdent(rightTableColumn)} AS ${quoteIdent(rightColumnAlias)}`
    );
  });

  // Join both tables on `joinColumns`, including unmatched rows from `leftTableId`.
  const joinConditions: string[] = [];
  joinColumns.forEach((rightTableColumn, leftTableColumn) => {
    const leftExpression = `${quoteIdent(leftTableId)}.${quoteIdent(leftTableColumn)}`;
    const rightExpression = `${quoteIdent(rightTableId)}.${quoteIdent(rightTableColumn)}`;
    joinConditions.push(`${leftExpression} = ${rightExpression}`);
  });
  joins.push(`LEFT JOIN ${quoteIdent(rightTableId)} ON ${joinConditions.join(' AND ')}`);

  // Filter out matching rows where all non-join columns from both tables are identical.
  const whereConditions: string[] = [];
  for (const [leftTableColumn, rightTableColumn] of selectColumns.entries()) {
    if (joinColumns.has(leftTableColumn)) { continue; }

    const leftColumnAlias = quoteIdent(`${leftTableId}.${leftTableColumn}`);
    const rightColumnAlias = quoteIdent(`${rightTableId}.${rightTableColumn}`);

    // Only include rows that have differences in column values.
    whereConditions.push(`${leftColumnAlias} IS NOT ${rightColumnAlias}`);
  }
  wheres.push(`(${whereConditions.join(' OR ')})`);

  // Copy decisions to the query object, and return.
  query.joins = joins;
  query.selects = selects;
  query.wheres = wheres;
  return query;
}