from six.moves import zip import column import identifiers import logger log = logger.Logger(__name__, logger.INFO) # Prefix for transform columns created during imports. _import_transform_col_prefix = 'gristHelper_Import_' def _gen_colids(transform_rule): """ For a transform_rule with colIds = None, fills in colIds generated from labels and returns the updated transform_rule. """ dest_cols = transform_rule["destCols"] if any(dc["colId"] for dc in dest_cols): raise ValueError("transform_rule already has colIds in _gen_colids") col_labels = [dest_col["label"] for dest_col in dest_cols] col_ids = identifiers.pick_col_ident_list(col_labels, avoid={'id'}) for dest_col, col_id in zip(dest_cols, col_ids): dest_col["colId"] = col_id return transform_rule def _strip_prefixes(transform_rule): "If transform_rule has prefixed _col_ids, strips prefix" dest_cols = transform_rule["destCols"] for dest_col in dest_cols: colId = dest_col["colId"] if colId and colId.startswith(_import_transform_col_prefix): dest_col["colId"] = colId[len(_import_transform_col_prefix):] class ImportActions(object): def __init__(self, useractions, docmodel, engine): self._useractions = useractions self._docmodel = docmodel self._engine = engine ######################## ## NOTES # transform_rule is an object like this: { # destCols: [ { colId, label, type, formula }, ... ], # ..., # other params unused in sandbox # } # # colId is defined if into_new_table, otherwise is None # GenImporterView gets a hidden table with a preview of the import data (~100 rows) # It adds formula cols and viewsections to the hidden table for the user to # preview and edit import options. GenImporterView can start with a default transform_rule # from table columns, or use one that's passed in (for reimporting). # client/components/Importer.ts then puts together transform_rule, which # specifies destination column formulas, types, labels, and colIds. It only contains colIds # if importing into an existing table, and they are sometimes prefixed with # _import_transform_col_prefix (if transform_rule comes from client) def _MakeDefaultTransformRule(self, hidden_table_id, dest_table_id): """ Makes a basic transform_rule.dest_cols copying all the source cols hidden_table_id: table with src data dest_table_id: table data is going to If dst_table is null, copy all src columns If dst_table exists, copy all dst columns, and make copy formulas if any names match returns transform_rule with only destCols filled in """ tables = self._docmodel.tables hidden_table_rec = tables.lookupOne(tableId=hidden_table_id) # will use these to set default formulas (if column names match in src and dest table) src_cols = {c.colId: c for c in hidden_table_rec.columns} target_table = tables.lookupOne(tableId=dest_table_id) if dest_table_id else hidden_table_rec target_cols = target_table.columns # makes dest_cols for each column in target_cols (defaults to same columns as hidden_table) #loop through visible, non-formula target columns dest_cols = [] for c in target_cols: if column.is_visible_column(c.colId) and (not c.isFormula or c.formula == ""): source_col = src_cols.get(c.colId) dest_col = { "label": c.label, "colId": c.colId if dest_table_id else None, #should be None if into new table "type": c.type, "widgetOptions": getattr(c, "widgetOptions", ""), "formula": ("$" + source_col.colId) if source_col else '', } if source_col and c.type.startswith("Ref:"): ref_table_id = c.type.split(':')[1] visible_col = c.visibleCol if visible_col: dest_col["visibleCol"] = visible_col.id dest_col["formula"] = '{}.lookupOne({}=${c}) or (${c} and str(${c}))'.format( ref_table_id, visible_col.colId, c=source_col.colId) dest_cols.append(dest_col) return {"destCols": dest_cols} # doesnt generate other fields of transform_rule, but sandbox only used destCols def _MakeImportTransformColumns(self, hidden_table_id, transform_rule, gen_all): """ Makes prefixed columns in the grist hidden import table (hidden_table_id) hidden_table_id: id of temporary hidden table in which columns are made transform_rule: defines columns to make (colids must be filled in!) gen_all: If true, all columns will be generated If false, formulas that just copy will be skipped returns list of newly created colrefs (rowids into _grist_Tables_column) """ tables = self._docmodel.tables hidden_table_rec = tables.lookupOne(tableId=hidden_table_id) src_cols = {c.colId for c in hidden_table_rec.columns} log.debug("destCols:" + repr(transform_rule['destCols'])) dest_cols = transform_rule['destCols'] log.debug("_MakeImportTransformColumns: {}".format("gen_all" if gen_all else "optimize")) # Calling rebuild_usercode once per added column is wasteful and can be very slow. self._engine._should_rebuild_usercode = False #create prefixed formula column for each of dest_cols #take formula from transform_rule new_cols = [] try: for c in dest_cols: # skip copy columns (unless gen_all) formula = c["formula"].strip() isCopyFormula = (formula.startswith("$") and formula[1:] in src_cols) if gen_all or not isCopyFormula: # If colId specified, use that. Otherwise, use the (sanitized) label. col_id = c["colId"] or identifiers.pick_col_ident(c["label"]) visible_col_ref = c.get("visibleCol", 0) new_col_id = _import_transform_col_prefix + col_id new_col_spec = { "label": c["label"], "type": c["type"], "widgetOptions": c.get("widgetOptions", ""), "isFormula": True, "formula": c["formula"], "visibleCol": visible_col_ref, } result = self._useractions.doAddColumn(hidden_table_id, new_col_id, new_col_spec) new_col_id, new_col_ref = result["colId"], result["colRef"] if visible_col_ref: visible_col_id = self._docmodel.columns.table.get_record(visible_col_ref).colId self._useractions.SetDisplayFormula(hidden_table_id, None, new_col_ref, '${}.{}'.format(new_col_id, visible_col_id)) new_cols.append(new_col_ref) finally: self._engine._should_rebuild_usercode = True self._engine.rebuild_usercode() return new_cols def DoGenImporterView(self, source_table_id, dest_table_id, transform_rule, options): """ Generates formula columns for transformed importer columns, and optionally a new viewsection. source_table_id: id of temporary hidden table, containing source data and used for preview. dest_table_id: id of table to import to, or None for new table. transform_rule: transform_rule to reuse (if it still applies), if None will generate new one options: a dictionary with optional keys: createViewSection: defaults to True, in which case creates a new view-section to show the generated columns, for use in review, and remove any previous ones. genAll: defaults to True; if False, transform formulas that just copy will not be generated. refsAsInts: if set, treat Ref columns as type Int for a new dest_table. This is used when finishing imports from multi-table sources (e.g. from json) to avoid issues such as importing linked tables in the wrong order. Caller is expected to fix these up separately. Returns and object with: transformRule: updated (normalized) transform rule, or a newly generated one. viewSectionRef: rowId of the newly added section, present only if createViewSection is set. """ createViewSection = options.get("createViewSection", True) genAll = options.get("genAll", True) refsAsInts = options.get("refsAsInts", True) if createViewSection: src_table_rec = self._docmodel.tables.lookupOne(tableId=source_table_id) # ======== Cleanup old sections/columns # Transform columns are those that start with a special prefix. old_cols = [c for c in src_table_rec.columns if c.colId.startswith(_import_transform_col_prefix)] old_sections = {field.parentId for c in old_cols for field in c.viewFields} self._docmodel.remove(old_sections) self._docmodel.remove(old_cols) #======== Prepare/normalize transform_rule, Create new formula columns # Defaults to duplicating dest_table columns (or src_table columns for a new table) # If transform_rule provided, use that if transform_rule is None: transform_rule = self._MakeDefaultTransformRule(source_table_id, dest_table_id) # ensure prefixes, colIds are correct _strip_prefixes(transform_rule) if not dest_table_id: # into new table: 'colId's are undefined _gen_colids(transform_rule) # Treat destination Ref:* columns as Int instead, for new tables, to avoid issues when # importing linked tables in the wrong order. Caller is expected to fix up afterwards. if refsAsInts: for col in transform_rule["destCols"]: if col["type"].startswith("Ref:"): col["type"] = "Int" else: if None in (dc["colId"] for dc in transform_rule["destCols"]): errstr = "colIds must be defined in transform_rule for importing into existing table: " raise ValueError(errstr + repr(transform_rule)) new_cols = self._MakeImportTransformColumns(source_table_id, transform_rule, gen_all=genAll) result = {"transformRule": transform_rule} if createViewSection: #========= Create new transform view section. new_section = self._docmodel.add(self._docmodel.view_sections, tableRef=src_table_rec.id, parentKey='record', borderWidth=1, defaultWidth=100, sortColRefs='[]')[0] self._docmodel.add(new_section.fields, colRef=new_cols) result["viewSectionRef"] = new_section.id return result