poc works

sandbox/grist/column.py

@@ -140,6 +140,7 @@ class BaseColumn(object):
       raise Exception('Column already detached: ', self.table_id, self.col_id)
     self._data.set(row_id, value)
 
+
   def unset(self, row_id):
     """
     Sets the value for the given row_id to the default value.
@@ -149,6 +150,7 @@ class BaseColumn(object):
     self.set(row_id, self.getdefault())
     self._data.unset(row_id)
 
+
   def get_cell_value(self, row_id, restore=False):
     """
     Returns the "rich" value for the given row_id, i.e. the value that would be seen by formulas.
@@ -173,7 +175,7 @@ class BaseColumn(object):
     # Inline _convert_raw_value here because this is particularly hot code, called on every access
     # of any data field in a formula.
     if self._is_right_type(raw):
-      return self._make_rich_value(raw)
+      return self._make_rich_value(raw, row_id)
     return self._alt_text(raw)
 
   def _convert_raw_value(self, raw):
@@ -184,7 +186,7 @@ class BaseColumn(object):
   def _alt_text(self, raw):
     return usertypes.AltText(str(raw), self.type_obj.typename())
 
-  def _make_rich_value(self, typed_value):
+  def _make_rich_value(self, typed_value, row_id=None):
     """
     Called by get_cell_value() with a value of the right type for this column. Should be
     implemented by derived classes to produce a "rich" version of the value.
@@ -301,7 +303,7 @@ class DateColumn(NumericColumn):
   DateColumn contains numerical timestamps represented as seconds since epoch, in type float,
   to midnight of specific UTC dates. Accessing them yields date objects.
   """
-  def _make_rich_value(self, typed_value):
+  def _make_rich_value(self, typed_value, row_id=None):
     return typed_value and moment.ts_to_date(typed_value)
 
   def sample_value(self):
@@ -316,7 +318,7 @@ class DateTimeColumn(NumericColumn):
     super(DateTimeColumn, self).__init__(table, col_id, col_info)
     self._timezone = col_info.type_obj.timezone
 
-  def _make_rich_value(self, typed_value):
+  def _make_rich_value(self, typed_value, row_id=None):
     return typed_value and moment.ts_to_dt(typed_value, self._timezone)
 
   def sample_value(self):
@@ -420,7 +422,7 @@ class ChoiceListColumn(ChoiceColumn):
       value = tuple(value)
     super(ChoiceListColumn, self).set(row_id, value)
 
-  def _make_rich_value(self, typed_value):
+  def _make_rich_value(self, typed_value, row_id=None):
     return () if typed_value is None else typed_value
 
   def _rename_cell_choice(self, renames, value):
@@ -501,7 +503,7 @@ class ReferenceColumn(BaseReferenceColumn):
   ReferenceColumn contains IDs of rows in another table. Accessing them yields the records in the
   other table.
   """
-  def _make_rich_value(self, typed_value):
+  def _make_rich_value(self, typed_value, row_id=None):
     # If we refer to an invalid table, return integers rather than fail completely.
     if not self._target_table:
       return typed_value
@@ -545,12 +547,22 @@ class ReferenceListColumn(BaseReferenceColumn):
     for new_value in new_list or ():
       self._relation.add_reference(row_id, new_value)
 
-  def _make_rich_value(self, typed_value):
+  def _make_rich_value(self, typed_value, row_id):
     if typed_value is None:
       typed_value = []
+    elif isinstance(typed_value, six.string_types) and typed_value.startswith(u'['):
+      try:
+        typed_value = json.loads(typed_value)
+      except Exception:
+        pass
     # If we refer to an invalid table, return integers rather than fail completely.
     if not self._target_table:
       return typed_value
+    if isinstance(self.type_obj, usertypes.ChildReferenceList) and row_id:
+      typed_value = self._target_table.RecordSet(typed_value, self._relation)
+      typed_value._sort_by = 'parentPos'
+      typed_value._group_by = {"parentId": row_id}
+      return typed_value
     return self._target_table.RecordSet(typed_value, self._relation)
 
   def _raw_get_without(self, row_id, target_row_ids):

sandbox/grist/data.py

@@ -1,4 +1,4 @@
-class ColumnData(object):
+class MemoryColumn(object):
   def __init__(self, col):
     self.col = col
     self.data = []
@@ -46,3 +46,213 @@ class ColumnData(object):
 
   def unset(self, row_id):
     pass
+
+class MemoryDatabase(object):
+  __slots__ = ('engine', 'tables')
+
+  def __init__(self, engine):
+    self.engine = engine
+    self.tables = {}
+
+
+  def create_table(self, table):
+    if table.table_id in self.tables:
+      raise ValueError("Table %s already exists" % table.table_id)
+    print("Creating table %s" % table.table_id)
+    self.tables[table.table_id] = dict()
+
+
+  def drop_table(self, table):
+    if table.table_id not in self.tables:
+      raise ValueError("Table %s already exists" % table.table_id)
+    print("Deleting table %s" % table.table_id)
+    del self.tables[table.table_id]
+
+
+  def create_column(self, col):
+    if col.table_id not in self.tables:
+      self.tables[col.table_id] = dict()
+
+    if col.col_id in self.tables[col.table_id]:
+      old_one = self.tables[col.table_id][col.col_id]
+      col._data = old_one._data
+      col._data.col = col
+      old_one.detached = True
+      old_one._data = None
+    else:
+      col._data = MemoryColumn(col)
+      # print('Column {}.{} is detaching column {}.{}'.format(self.table_id, self.col_id, old_one.table_id, old_one.col_id))
+    # print('Creating column: ', self.table_id, self.col_id)
+    self.tables[col.table_id][col.col_id] = col
+    col.detached = False
+
+  def drop_column(self, col):
+    tables = self.tables
+
+    if col.table_id not in tables:
+      raise Exception('Table not found for column: ', col.table_id, col.col_id)
+    
+    if col.col_id not in tables[col.table_id]:
+      raise Exception('Column not found: ', col.table_id, col.col_id)
+
+    print('Destroying column: ', col.table_id, col.col_id)
+    col._data.drop()
+    del tables[col.table_id][col.col_id]
+
+import json
+import random
+import string
+import actions
+from sql import delete_column, open_connection
+
+
+class SqlColumn(object):
+  def __init__(self, db, col):
+    self.db = db
+    self.col = col
+    self.create_column()
+
+  def growto(self, size):
+    if self.size() < size:
+      for i in range(self.size(), size):
+        self.set(i, self.getdefault())
+
+
+  def iterate(self):
+    cursor = self.db.sql.cursor()
+    try:
+      for row in cursor.execute('SELECT id, "{}" FROM "{}" ORDER BY id'.format(self.col.col_id, self.col.table_id)):
+        yield row[0], row[1] if row[1] is not None else self.getdefault()
+    finally:
+      cursor.close()
+
+  def copy_from(self, other_column):
+    self.growto(other_column.size())
+    for i, value in other_column.iterate():
+      self.set(i, value)
+
+  def raw_get(self, row_id):
+    cursor = self.db.sql.cursor()
+    value = cursor.execute('SELECT "{}" FROM "{}" WHERE id = ?'.format(self.col.col_id, self.col.table_id), (row_id,)).fetchone()
+    cursor.close()
+    correct = value[0] if value else None
+    return correct if correct is not None else self.getdefault()
+
+  def set(self, row_id, value):
+    if self.col.col_id == "id" and not value:
+      return
+    # First check if we have this id in the table, using exists statmenet
+    cursor = self.db.sql.cursor()
+    value = value
+    if isinstance(value, list):
+      value = json.dumps(value)
+    exists = cursor.execute('SELECT EXISTS(SELECT 1 FROM "{}" WHERE id = ?)'.format(self.col.table_id), (row_id,)).fetchone()[0]
+    if not exists:
+      cursor.execute('INSERT INTO "{}" (id, "{}") VALUES (?, ?)'.format(self.col.table_id, self.col.col_id), (row_id, value))
+    else:
+      cursor.execute('UPDATE "{}" SET "{}" = ? WHERE id = ?'.format(self.col.table_id, self.col.col_id), (value, row_id))
+
+  def getdefault(self):
+    return self.col.type_obj.default
+
+  def size(self):
+    max_id = self.db.sql.execute('SELECT MAX(id) FROM "{}"'.format(self.col.table_id)).fetchone()[0]
+    max_id = max_id if max_id is not None else 0
+    return max_id + 1
+
+  def create_column(self):
+    cursor = self.db.sql.cursor()
+    col = self.col
+    if col.col_id == "id":
+      pass
+    else:
+      cursor.execute('ALTER TABLE "{}" ADD COLUMN "{}" {}'.format(self.col.table_id, self.col.col_id, self.col.type_obj.sql_type()))
+    cursor.close()
+
+  def clear(self):
+    pass
+
+  def drop(self):
+    delete_column(self.db.sql, self.col.table_id, self.col.col_id)
+
+  def unset(self, row_id):
+    if self.col.col_id != 'id':
+      return
+    print('Removing row {} from column {}.{}'.format(row_id, self.col.table_id, self.col.col_id))
+    cursor = self.db.sql.cursor()
+    cursor.execute('DELETE FROM "{}" WHERE id = ?'.format(self.col.table_id), (row_id,))
+    cursor.close()
+
+    
+
+
+class SqlDatabase(object):
+  def __init__(self, engine) -> None:
+    self.engine = engine
+    random_file = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) + '.grist'
+    self.sql = open_connection(random_file)
+    self.tables = {}
+
+  def read_table(self, table_id):
+    return read_table(self.sql, table_id)
+  
+
+  def create_table(self, table):
+    cursor = self.sql.cursor()
+    cursor.execute('CREATE TABLE ' + table.table_id + ' (id INTEGER PRIMARY KEY AUTOINCREMENT)')
+    self.tables[table.table_id] = {}
+
+
+  def create_column(self, col):
+    if col.table_id not in self.tables:
+      self.tables[col.table_id] = dict()
+
+    if col.col_id in self.tables[col.table_id]:
+      old_one = self.tables[col.table_id][col.col_id]
+      col._data = old_one._data
+      col._data.col = col
+      old_one.detached = True
+      old_one._data = None
+    else:
+      col._data = SqlColumn(self, col)
+      # print('Column {}.{} is detaching column {}.{}'.format(self.table_id, self.col_id, old_one.table_id, old_one.col_id))
+    # print('Creating column: ', self.table_id, self.col_id)
+    self.tables[col.table_id][col.col_id] = col
+    col.detached = False
+
+  def drop_column(self, col):
+    tables = self.tables
+
+    if col.table_id not in tables:
+      raise Exception('Table not found for column: ', col.table_id, col.col_id)
+    
+    if col.col_id not in tables[col.table_id]:
+      raise Exception('Column not found: ', col.table_id, col.col_id)
+
+    print('Destroying column: ', col.table_id, col.col_id)
+    col._data.drop()
+    del tables[col.table_id][col.col_id]
+
+
+  def drop_table(self, table):
+    if table.table_id not in self.tables:
+      raise Exception('Table not found: ', table.table_id)
+    cursor = self.sql.cursor()
+    cursor.execute('DROP TABLE ' + table.table_id)
+    del self.tables[table.table_id]
+
+
+def read_table(sql, tableId):
+  cursor = sql.cursor()
+  cursor.execute('SELECT * FROM ' + tableId)
+  data = cursor.fetchall()
+  cursor.close()
+  rowIds = [row['id'] for row in data]
+  columns = {}
+  for row in data:
+    for key in row.keys():
+      if key != 'id':
+        if key not in columns:
+          columns[key] = []
+        columns[key].append(row[key])
+  return actions.TableData(tableId, rowIds, columns)

sandbox/grist/docactions.py

@@ -45,6 +45,8 @@ class DocActions(object):
     # make sure we don't have stale values hanging around.
     undo_values = {}
     for column in six.itervalues(table.all_columns):
+      if column.col_id == "id":
+        continue
       if not column.is_private() and column.col_id != "id":
         col_values = [column.raw_get(r) for r in row_ids]
         default = column.getdefault()
@@ -54,6 +56,13 @@ class DocActions(object):
       for row_id in row_ids:
         column.unset(row_id)
     
+    # Remove id column as last one, as this also removes the row
+    for column in six.itervalues(table.all_columns):
+      if column.col_id != "id":
+        continue
+      for row_id in row_ids:
+        column.unset(row_id)
+
     # Generate the undo action.
     self._engine.out_actions.undo.append(
         actions.BulkAddRecord(table_id, row_ids, undo_values).simplify())

sandbox/grist/docmodel.py

@@ -20,7 +20,7 @@ from schema import RecalcWhen
 # pylint:disable=redefined-outer-name
 
 def _record_set(table_id, group_by, sort_by=None):
-  @usertypes.formulaType(usertypes.ReferenceList(table_id))
+  @usertypes.formulaType(usertypes.ChildReferenceList(table_id))
   def func(rec, table):
     lookup_table = table.docmodel.get_table(table_id)
     return lookup_table.lookupRecords(sort_by=sort_by, **{group_by: rec.id})

sandbox/grist/engine.py

@@ -15,7 +15,7 @@ import six
 from six.moves import zip
 from six.moves.collections_abc import Hashable  # pylint:disable-all
 from sortedcontainers import SortedSet
-from data import ColumnData
+from data import MemoryColumn, MemoryDatabase, SqlDatabase
 import acl
 import actions
 import action_obj
@@ -105,58 +105,6 @@ skipped_completions = re.compile(r'\.(_|lookupOrAddDerived|getSummarySourceGroup
 # column may refer to derived tables or independent tables. Derived tables would have an extra
 # property, marking them as derived, which would affect certain UI decisions.
 
-class Database(object):
-  __slots__ = ('engine', 'tables')
-
-  def __init__(self, engine):
-    self.engine = engine
-    self.tables = {}
-
-
-  def create_table(self, table):
-    if table.table_id in self.tables:
-      raise ValueError("Table %s already exists" % table.table_id)
-    print("Creating table %s" % table.table_id)
-    self.tables[table.table_id] = dict()
-
-
-  def drop_table(self, table):
-    if table.table_id not in self.tables:
-      raise ValueError("Table %s already exists" % table.table_id)
-    print("Deleting table %s" % table.table_id)
-    del self.tables[table.table_id]
-
-
-  def create_column(self, col):
-    if col.table_id not in self.tables:
-      self.tables[col.table_id] = dict()
-
-    if col.col_id in self.tables[col.table_id]:
-      old_one = self.tables[col.table_id][col.col_id]
-      col._data = old_one._data
-      col._data.col = col
-      old_one.detached = True
-      old_one._data = None
-    else:
-      col._data = ColumnData(col)
-      # print('Column {}.{} is detaching column {}.{}'.format(self.table_id, self.col_id, old_one.table_id, old_one.col_id))
-    # print('Creating column: ', self.table_id, self.col_id)
-    self.tables[col.table_id][col.col_id] = col
-    col.detached = False
-
-  def drop_column(self, col):
-    tables = self.tables
-
-    if col.table_id not in tables:
-      raise Exception('Table not found for column: ', col.table_id, col.col_id)
-    
-    if col.col_id not in tables[col.table_id]:
-      raise Exception('Column not found: ', col.table_id, col.col_id)
-
-    print('Destroying column: ', col.table_id, col.col_id)
-    col._data.drop()
-    del tables[col.table_id][col.col_id]
-
 class Engine(object):
   """
   The Engine is the core of the grist per-document logic. Some of its methods form the API exposed
@@ -191,7 +139,7 @@ class Engine(object):
   """
 
   def __init__(self):
-    self.data = Database(self)      # The document data, including logic (formulas), and metadata.
+    self.data = None      # The document data, including logic (formulas), and metadata.
 
     # The document data, including logic (formulas), and metadata (tables prefixed with "_grist_").
     self.tables = {}                # Maps table IDs (or names) to Table objects.
@@ -351,6 +299,7 @@ class Engine(object):
 
     return dict(result)
   
+
   def load_empty(self):
     """
     Initialize an empty document, e.g. a newly-created one.
@@ -365,6 +314,9 @@ class Engine(object):
     _grist_Tables and _grist_Tables_column tables, in the form of actions.TableData.
     Returns the list of all the other table names that data engine expects to be loaded.
     """
+
+    self.data = SqlDatabase(self)
+
     self.schema = schema.build_schema(meta_tables, meta_columns)
 
     # Compile the user-defined module code (containing all formulas in particular).
@@ -1351,6 +1303,7 @@ class Engine(object):
           self.assert_schema_consistent()
 
     except Exception as e:
+      raise e
       # Save full exception info, so that we can rethrow accurately even if undo also fails.
       exc_info = sys.exc_info()
       # If we get an exception, we should revert all changes applied so far, to keep things

sandbox/grist/objtypes.py

@@ -62,6 +62,8 @@ class AltText(object):
   with unexpected result.
   """
   def __init__(self, text, typename=None):
+    if text == "None":
+      raise  InvalidTypedValue(typename, text)
     self._text = text
     self._typename = typename
 

sandbox/grist/poc.py

@@ -31,11 +31,13 @@ def apply(actions):
 try:
   apply(['AddRawTable', 'Table1'])
   apply(['AddRecord', 'Table1', None, {'A': 1, 'B': 2, 'C': 3}])
-  # apply(['RenameColumn', 'Table1', 'A', 'NewA'])
+  apply(['AddColumn', 'Table1', 'D', {'type': 'Numeric', 'isFormula': True, 'formula': '$A + 3'}]),
+  apply(['RenameColumn', 'Table1', 'A', 'NewA'])
   apply(['RenameTable', 'Table1', 'Dwa'])
+  apply(['RemoveColumn', 'Dwa', 'B'])
+  apply(['RemoveTable', 'Dwa'])
 
   #  ['RemoveColumn', "Table1", 'A'],
-    # ['AddColumn', 'Table1', 'D', {'type': 'Numeric', 'isFormula': True, 'formula': '$A + 3'}],
     # ['AddColumn', 'Table1', 'D', {'type': 'Numeric', 'isFormula': True, 'formula': '$A + 3'}],
     # ['ModifyColumn', 'Table1', 'B', {'type': 'Numeric', 'isFormula': True, 'formula': '$A + 1'}],
   #])

sandbox/grist/sql.py

@@ -0,0 +1,298 @@
+import json
+import marshal
+import sqlite3
+import actions
+import six
+
+
+import sqlite3
+
+
+def change_id_to_primary_key(conn, table_name):
+  cursor = conn.cursor()
+  cursor.execute('PRAGMA table_info("{}");'.format(table_name))
+  columns = cursor.fetchall()
+  create_table_sql = 'CREATE TABLE "{}_temp" ('.format(table_name)
+  for column in columns:
+      column_name, column_type, _, _, _, _ = column
+      primary_key = "PRIMARY KEY" if column_name == "id" else ""
+      create_table_sql += '"{}" {} {}, '.format(column_name, column_type, primary_key)
+  create_table_sql = create_table_sql.rstrip(", ") + ");"
+  cursor.execute(create_table_sql)
+  cursor.execute('INSERT INTO "{}_temp" SELECT * FROM "{}";'.format(table_name, table_name))
+  cursor.execute('DROP TABLE "{}";'.format(table_name))
+  cursor.execute('ALTER TABLE "{}_temp" RENAME TO "{}";'.format(table_name, table_name))
+  cursor.close()
+
+
+def delete_column(conn, table_name, column_name):
+  cursor = conn.cursor()
+  cursor.execute('PRAGMA table_info("{}");'.format(table_name))
+  columns_info = cursor.fetchall()
+  new_columns = ", ".join(
+      '"{}" {}'.format(col[1], col[2])
+      for col in columns_info
+      if col[1] != column_name
+  )
+  if new_columns:
+    cursor.execute('CREATE TABLE "new_{}" ({})'.format(table_name, new_columns))
+    cursor.execute('INSERT INTO "new_{}" SELECT {} FROM "{}"'.format(table_name, new_columns, table_name))
+    cursor.execute('DROP TABLE "{}"'.format(table_name))
+    cursor.execute('ALTER TABLE "new_{}" RENAME TO "{}"'.format(table_name, table_name))
+  conn.commit()
+
+
+def rename_column(conn, table_name, old_column_name, new_column_name):
+  cursor = conn.cursor()
+  cursor.execute('PRAGMA table_info("{}");'.format(table_name))
+  columns_info = cursor.fetchall()
+
+  # Construct new column definitions string
+  new_columns = []
+  for col in columns_info:
+      if col[1] == old_column_name:
+          new_columns.append('"{}" {}'.format(new_column_name, col[2]))
+      else:
+          new_columns.append('"{}" {}'.format(col[1], col[2]))
+  new_columns_str = ", ".join(new_columns)
+
+  # Create new table with renamed column
+  cursor.execute('CREATE TABLE "new_{}" ({});'.format(table_name, new_columns_str))
+  cursor.execute('INSERT INTO "new_{}" SELECT {} FROM "{}";'.format(table_name, new_columns_str, table_name))
+
+  # Drop original table and rename new table to match original table name
+  cursor.execute('DROP TABLE "{}";'.format(table_name))
+  cursor.execute('ALTER TABLE "new_{}" RENAME TO "{}";'.format(table_name, table_name))
+
+  conn.commit()
+
+
+
+def change_column_type(conn, table_name, column_name, new_type):
+  cursor = conn.cursor()
+  cursor.execute('PRAGMA table_info("{}");'.format(table_name))
+  columns_info = cursor.fetchall()
+  old_type = new_type
+  for col in columns_info:
+    if col[1] == column_name:
+      old_type = col[2].upper()
+      break
+  if old_type == new_type:
+    return
+  new_columns = ", ".join(
+      '"{}" {}'.format(col[1], new_type if col[1] == column_name else col[2])
+      for col in columns_info
+  )
+  cursor.execute('CREATE TABLE "new_{}" ({});'.format(table_name, new_columns))
+  cursor.execute('INSERT INTO "new_{}" SELECT * FROM "{}";'.format(table_name, table_name))
+  cursor.execute('DROP TABLE "{}";'.format(table_name))
+  cursor.execute('ALTER TABLE "new_{}" RENAME TO "{}";'.format(table_name, table_name))
+  conn.commit()
+
+
+def is_primitive(value):
+  string_types = six.string_types if six.PY3 else (str,)
+  numeric_types = six.integer_types + (float,)
+  bool_type = (bool,)
+  return isinstance(value, string_types + numeric_types + bool_type)
+
+def size(sql: sqlite3.Connection, table):
+  cursor = sql.execute('SELECT MAX(id) FROM %s' % table)
+  value = (cursor.fetchone()[0] or 0)
+  return value
+
+def next_row_id(sql: sqlite3.Connection, table):
+  cursor = sql.execute('SELECT MAX(id) FROM %s' % table)
+  value = (cursor.fetchone()[0] or 0) + 1
+  return value
+
+def create_table(sql, table_id):
+  sql.execute("CREATE TABLE IF NOT EXISTS {} (id INTEGER PRIMARY KEY)".format(table_id))
+
+def column_raw_get(sql, table_id, col_id, row_id):
+  value = sql.execute('SELECT "{}" FROM {} WHERE id = ?'.format(col_id, table_id), (row_id,)).fetchone()
+  return value[col_id] if value else None
+
+def column_set(sql, table_id, col_id, row_id, value):
+  if col_id == 'id':
+    raise Exception('Cannot set id')
+
+  if isinstance(value, list):
+    value = json.dumps(value)
+
+  if not is_primitive(value) and value is not None:
+    value = repr(value)
+  
+  try:
+    id = column_raw_get(sql, table_id, 'id', row_id)
+    if id is None:
+      # print("Insert [{}][{}][{}] = {}".format(table_id, col_id, row_id, value))
+      sql.execute('INSERT INTO {} (id) VALUES (?)'.format(table_id), (row_id,))
+    else:
+      # print("Update [{}][{}][{}] = {}".format(table_id, col_id, row_id, value))
+      pass
+    sql.execute('UPDATE {} SET "{}" = ? WHERE id = ?'.format(table_id, col_id), (value, row_id))
+  except sqlite3.OperationalError:
+    raise
+
+def column_grow(sql, table_id, col_id):
+  sql.execute("INSERT INTO {} DEFAULT VALUES".format(table_id, col_id))
+
+def col_exists(sql, table_id, col_id):
+  cursor = sql.execute('PRAGMA table_info({})'.format(table_id))
+  for row in cursor:
+    if row[1] == col_id:
+      return True
+  return False
+
+def column_create(sql, table_id, col_id, col_type='BLOB'):
+  if col_exists(sql, table_id, col_id):
+    change_column_type(sql, table_id, col_id, col_type)
+    return
+  try:
+    sql.execute('ALTER TABLE {} ADD COLUMN "{}" {}'.format(table_id, col_id, col_type))
+  except sqlite3.OperationalError as e:
+    if str(e).startswith('duplicate column name'):
+      return
+    raise e
+
+class Column(object):
+  def __init__(self, sql, col):
+    self.sql = sql
+    self.col = col
+    self.col_id = col.col_id
+    self.table_id = col.table_id
+    create_table(self.sql, self.col.table_id)
+    column_create(self.sql, self.col.table_id, self.col.col_id, self.col.type_obj.sql_type())
+
+
+  def __iter__(self):
+    for i in range(0, len(self)):
+      if i == 0:
+        yield None
+      yield self[i]
+
+  def __len__(self):
+    len = size(self.sql, self.col.table_id)
+    return len + 1
+  
+  def __setitem__(self, row_id, value):
+    if self.col.col_id == 'id':
+      if value == 0:
+        # print('Deleting by setting id to 0')
+        self.__delitem__(row_id)
+      return
+    column_set(self.sql, self.col.table_id, self.col.col_id, row_id, value)
+
+  def __getitem__(self, key):
+    if self.col.col_id == 'id' and key == 0:
+      return key
+    value = column_raw_get(self.sql, self.col.table_id, self.col.col_id, key)
+    return value
+  
+  def __delitem__(self, row_id):
+    # print("Delete [{}][{}]".format(self.col.table_id, row_id))
+    self.sql.execute('DELETE FROM {} WHERE id = ?'.format(self.col.table_id), (row_id,))
+
+  def remove(self):
+    delete_column(self.sql, self.col.table_id, self.col.col_id)
+
+  def rename(self, new_name):
+    rename_column(self.sql, self.table_id, self.col_id, new_name)
+    self.col_id = new_name
+
+  def copy_from(self, other):
+    if self.col_id == other.col_id and self.table_id == other.table_id:
+      return
+    try:
+      if self.table_id == other.table_id:
+        query = ('''
+          UPDATE "{}" SET "{}" = "{}"
+        '''.format(self.table_id, self.col_id, other.col_id))
+        self.sql.execute(query)
+        return
+      query = ('''
+        INSERT INTO "{}" (id, "{}") SELECT id, "{}" FROM "{}" WHERE true
+        ON CONFLICT(id) DO UPDATE SET "{}" = excluded."{}"
+      '''.format(self.table_id, self.col_id, other.col_id, other.table_id, self.col_id, other.col_id))
+      self.sql.execute(query)
+    except sqlite3.OperationalError as e:
+      if str(e).startswith('no such table'):
+        return
+      raise e
+
+def column(sql, col):
+  return Column(sql, col)
+
+def create_schema(sql):
+  sql.executescript('''
+    PRAGMA foreign_keys=OFF;
+    BEGIN TRANSACTION;
+    CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT '');
+    INSERT INTO _grist_DocInfo VALUES(1,'','','',37,'','');
+    CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0, "rawViewSectionRef" INTEGER DEFAULT 0);
+    CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "description" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "rules" TEXT DEFAULT NULL, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
+    CREATE TABLE IF NOT EXISTS "_grist_Imports" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "origFileName" TEXT DEFAULT '', "parseFormula" TEXT DEFAULT '', "delimiter" TEXT DEFAULT '', "doublequote" BOOLEAN DEFAULT 0, "escapechar" TEXT DEFAULT '', "quotechar" TEXT DEFAULT '', "skipinitialspace" BOOLEAN DEFAULT 0, "encoding" TEXT DEFAULT '', "hasHeaders" BOOLEAN DEFAULT 0);
+    CREATE TABLE IF NOT EXISTS "_grist_External_database" (id INTEGER PRIMARY KEY, "host" TEXT DEFAULT '', "port" INTEGER DEFAULT 0, "username" TEXT DEFAULT '', "dialect" TEXT DEFAULT '', "database" TEXT DEFAULT '', "storage" TEXT DEFAULT '');
+    CREATE TABLE IF NOT EXISTS "_grist_External_table" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "databaseRef" INTEGER DEFAULT 0, "tableName" TEXT DEFAULT '');
+    CREATE TABLE IF NOT EXISTS "_grist_TableViews" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "viewRef" INTEGER DEFAULT 0);
+    CREATE TABLE IF NOT EXISTS "_grist_TabItems" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "viewRef" INTEGER DEFAULT 0);
+    CREATE TABLE IF NOT EXISTS "_grist_TabBar" (id INTEGER PRIMARY KEY, "viewRef" INTEGER DEFAULT 0, "tabPos" NUMERIC DEFAULT 1e999);
+    CREATE TABLE IF NOT EXISTS "_grist_Pages" (id INTEGER PRIMARY KEY, "viewRef" INTEGER DEFAULT 0, "indentation" INTEGER DEFAULT 0, "pagePos" NUMERIC DEFAULT 1e999);
+    CREATE TABLE IF NOT EXISTS "_grist_Views" (id INTEGER PRIMARY KEY, "name" TEXT DEFAULT '', "type" TEXT DEFAULT '', "layoutSpec" TEXT DEFAULT '');
+    CREATE TABLE IF NOT EXISTS "_grist_Views_section" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "parentId" INTEGER DEFAULT 0, "parentKey" TEXT DEFAULT '', "title" TEXT DEFAULT '', "defaultWidth" INTEGER DEFAULT 0, "borderWidth" INTEGER DEFAULT 0, "theme" TEXT DEFAULT '', "options" TEXT DEFAULT '', "chartType" TEXT DEFAULT '', "layoutSpec" TEXT DEFAULT '', "filterSpec" TEXT DEFAULT '', "sortColRefs" TEXT DEFAULT '', "linkSrcSectionRef" INTEGER DEFAULT 0, "linkSrcColRef" INTEGER DEFAULT 0, "linkTargetColRef" INTEGER DEFAULT 0, "embedId" TEXT DEFAULT '', "rules" TEXT DEFAULT NULL);
+    CREATE TABLE IF NOT EXISTS "_grist_Views_section_field" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colRef" INTEGER DEFAULT 0, "width" INTEGER DEFAULT 0, "widgetOptions" TEXT DEFAULT '', "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "filter" TEXT DEFAULT '', "rules" TEXT DEFAULT NULL);
+    CREATE TABLE IF NOT EXISTS "_grist_Validations" (id INTEGER PRIMARY KEY, "formula" TEXT DEFAULT '', "name" TEXT DEFAULT '', "tableRef" INTEGER DEFAULT 0);
+    CREATE TABLE IF NOT EXISTS "_grist_REPL_Hist" (id INTEGER PRIMARY KEY, "code" TEXT DEFAULT '', "outputText" TEXT DEFAULT '', "errorText" TEXT DEFAULT '');
+    CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "fileExt" TEXT DEFAULT '', "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeDeleted" DATETIME DEFAULT NULL, "timeUploaded" DATETIME DEFAULT NULL);
+    CREATE TABLE IF NOT EXISTS "_grist_Triggers" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "eventTypes" TEXT DEFAULT NULL, "isReadyColRef" INTEGER DEFAULT 0, "actions" TEXT DEFAULT '');
+    CREATE TABLE IF NOT EXISTS "_grist_ACLRules" (id INTEGER PRIMARY KEY, "resource" INTEGER DEFAULT 0, "permissions" INTEGER DEFAULT 0, "principals" TEXT DEFAULT '', "aclFormula" TEXT DEFAULT '', "aclColumn" INTEGER DEFAULT 0, "aclFormulaParsed" TEXT DEFAULT '', "permissionsText" TEXT DEFAULT '', "rulePos" NUMERIC DEFAULT 1e999, "userAttributes" TEXT DEFAULT '', "memo" TEXT DEFAULT '');
+    INSERT INTO _grist_ACLRules VALUES(1,1,63,'[1]','',0,'','',1e999,'','');
+    CREATE TABLE IF NOT EXISTS "_grist_ACLResources" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "colIds" TEXT DEFAULT '');
+    INSERT INTO _grist_ACLResources VALUES(1,'','');
+    CREATE TABLE IF NOT EXISTS "_grist_ACLPrincipals" (id INTEGER PRIMARY KEY, "type" TEXT DEFAULT '', "userEmail" TEXT DEFAULT '', "userName" TEXT DEFAULT '', "groupName" TEXT DEFAULT '', "instanceId" TEXT DEFAULT '');
+    INSERT INTO _grist_ACLPrincipals VALUES(1,'group','','','Owners','');
+    INSERT INTO _grist_ACLPrincipals VALUES(2,'group','','','Admins','');
+    INSERT INTO _grist_ACLPrincipals VALUES(3,'group','','','Editors','');
+    INSERT INTO _grist_ACLPrincipals VALUES(4,'group','','','Viewers','');
+    CREATE TABLE IF NOT EXISTS "_grist_ACLMemberships" (id INTEGER PRIMARY KEY, "parent" INTEGER DEFAULT 0, "child" INTEGER DEFAULT 0);
+    CREATE TABLE IF NOT EXISTS "_grist_Filters" (id INTEGER PRIMARY KEY, "viewSectionRef" INTEGER DEFAULT 0, "colRef" INTEGER DEFAULT 0, "filter" TEXT DEFAULT '', "pinned" BOOLEAN DEFAULT 0);
+    CREATE TABLE IF NOT EXISTS "_grist_Cells" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "colRef" INTEGER DEFAULT 0, "rowId" INTEGER DEFAULT 0, "root" BOOLEAN DEFAULT 0, "parentId" INTEGER DEFAULT 0, "type" INTEGER DEFAULT 0, "content" TEXT DEFAULT '', "userRef" TEXT DEFAULT '');
+    CREATE INDEX _grist_Attachments_fileIdent ON _grist_Attachments(fileIdent);
+
+    CREATE TABLE IF NOT EXISTS changes (colRef INTEGER, rowId INTEGER, value BLOB, PRIMARY KEY(colRef, rowId));
+    CREATE TABLE IF NOT EXISTS recalc (colRef Integer, rowId INTEGER, seq INTEGER DEFAULT NULL, PRIMARY KEY(colRef, rowId));
+    CREATE TABLE IF NOT EXISTS cell_graph (lColumn INTEGER, lRow INTEGER, rColumn INTEGER, rRow INTEGER, PRIMARY KEY(lColumn, lRow, rColumn, rRow));
+    CREATE TABLE IF NOT EXISTS filter_graph (lColumn INTEGER, lRow INTEGER, rColumn INTEGER, filter INTEGER, PRIMARY KEY(lColumn, lRow, rColumn, filter));
+
+    CREATE UNIQUE INDEX IF NOT EXISTS "changes_auto" ON "changes" ("colRef", "rowId");
+    CREATE UNIQUE INDEX IF NOT EXISTS recalc_auto ON "recalc" ("colRef", "rowId");
+    CREATE UNIQUE INDEX IF NOT EXISTS recalc_seq ON "recalc" ("seq", "colRef", "rowId");
+    CREATE UNIQUE INDEX IF NOT EXISTS "cell_graph_l" ON "cell_graph" ("lColumn", "lRow", "rColumn", "rRow");
+    CREATE INDEX IF NOT EXISTS "colId" ON "_grist_Tables_column" ("colId", "parentId");
+
+    CREATE INDEX IF NOT EXISTS "_grist_Tables_column_parent" ON "_grist_Tables_column" ("parentId", "type");
+    CREATE INDEX IF NOT EXISTS "cell_graph_i1" ON "cell_graph" ("lColumn", "lRow", "rColumn");
+    CREATE INDEX IF NOT EXISTS "cell_graph_i2" ON "cell_graph" ("rRow", "rColumn", "lColumn", "lRow");
+    CREATE INDEX IF NOT EXISTS "cell_graph_i3" ON "cell_graph" ("rColumn", "lRow", "lColumn");
+    CREATE INDEX IF NOT EXISTS "changes_value" ON "changes" ("colRef", "value");
+    CREATE INDEX IF NOT EXISTS "filter_1" ON "filter_graph" ("lColumn", "lRow", "rColumn", "filter");
+    CREATE INDEX IF NOT EXISTS "filter_2" ON "filter_graph" ("rColumn", "filter", "lColumn", "lRow");
+    CREATE INDEX IF NOT EXISTS "formulas" ON "_grist_Tables_column" ("parentId", "formula", "isFormula");
+    CREATE INDEX IF NOT EXISTS "graph_1" ON cell_graph ("lColumn", "rColumn");
+    CREATE INDEX IF NOT EXISTS "graph_2" ON filter_graph ("lColumn", "rColumn");
+    CREATE INDEX IF NOT EXISTS "graph_3" ON cell_graph ("rColumn", "rRow");
+    CREATE INDEX IF NOT EXISTS "tableId" ON "_grist_Tables" ("tableId");
+
+    COMMIT;
+  ''')
+
+def open_connection(file):
+  sql = sqlite3.connect(file, isolation_level=None)
+  sql.row_factory = sqlite3.Row
+  # sql.execute('PRAGMA encoding="UTF-8"')
+  # # sql.execute('PRAGMA journal_mode = DELETE;')
+  # # sql.execute('PRAGMA journal_mode = WAL;')
+  # sql.execute('PRAGMA synchronous = OFF;')
+  # sql.execute('PRAGMA trusted_schema = OFF;')
+  return sql

sandbox/grist/useractions.py

@@ -1790,6 +1790,7 @@ class UserActions(object):
     extra = {'summarySourceTable': summarySourceTableRef} if summarySourceTableRef else {}
     table_rec = self._docmodel.add(self._docmodel.tables, tableId=table_id, primaryViewId=0,
                                    **extra)[0]
+    
     self._docmodel.insert(
       table_rec.columns, None,
       colId         = col_ids,

sandbox/grist/usertypes.py

@@ -93,6 +93,7 @@ class BaseColumnType(object):
     self._creation_order = BaseColumnType._global_creation_order
     BaseColumnType._global_creation_order += 1
 
+
   @classmethod
   def typename(cls):
     """
@@ -100,6 +101,13 @@ class BaseColumnType(object):
     """
     return cls.__name__
   
+  @classmethod
+  def sql_type(cls):
+    """
+    Returns the SQL type for this column, e.g. "INTEGER", "TEXT", or "BLOB".
+    """
+    raise NotImplementedError
+
   @classmethod
   def is_right_type(cls, _value):
     """
@@ -143,6 +151,8 @@ class BaseColumnType(object):
 
 
 class Text(BaseColumnType):
+
+
   """
   Text is the type for a field holding string (text) data.
   """
@@ -173,6 +183,9 @@ class Text(BaseColumnType):
   def is_right_type(cls, value):
     return isinstance(value, (six.string_types, NoneType))
 
+  @classmethod
+  def sql_type(cls):
+    return "TEXT"
 
 class Blob(BaseColumnType):
   """
@@ -186,6 +199,9 @@ class Blob(BaseColumnType):
   def is_right_type(cls, value):
     return isinstance(value, (six.binary_type, NoneType))
 
+  @classmethod
+  def sql_type(cls):
+    return "BLOB"
 
 class Any(BaseColumnType):
   """
@@ -196,6 +212,9 @@ class Any(BaseColumnType):
     # Convert AltText values to plain text when assigning to type Any.
     return six.text_type(value) if isinstance(value, AltText) else value
   
+  @classmethod
+  def sql_type(cls):
+    return "BLOB"
 
 class Bool(BaseColumnType):
   """
@@ -223,6 +242,11 @@ class Bool(BaseColumnType):
     return isinstance(value, (bool, NoneType))
 
 
+  @classmethod
+  def sql_type(cls):
+    return "INTEGER"
+
+
 class Int(BaseColumnType):
   """
   Int is the type for a field holding integer data.
@@ -241,6 +265,10 @@ class Int(BaseColumnType):
   def is_right_type(cls, value):
     return value is None or (type(value) in integer_types and is_int_short(value))
 
+  @classmethod
+  def sql_type(cls):
+    return "INTEGER"
+
 
 class Numeric(BaseColumnType):
   """
@@ -257,6 +285,9 @@ class Numeric(BaseColumnType):
     # will have type 'int'.
     return type(value) in _numeric_or_none
 
+  @classmethod
+  def sql_type(cls):
+    return "REAL"
 
 class Date(Numeric):
   """
@@ -310,6 +341,11 @@ class DateTime(Date):
     else:
       raise objtypes.ConversionError('DateTime')
     
+  
+  @classmethod
+  def sql_type(cls):
+    return "DATE"
+
 class Choice(Text):
   """
   Choice is the type for a field holding one of a set of acceptable string (text) values.
@@ -355,6 +391,10 @@ class ChoiceList(BaseColumnType):
     return value
 
   
+  @classmethod
+  def sql_type(cls):
+    return "TEXT"
+
 class PositionNumber(BaseColumnType):
   """
   PositionNumber is the type for a position field used to order records in record lists.
@@ -370,6 +410,9 @@ class PositionNumber(BaseColumnType):
     # Same as Numeric, but does not support None.
     return type(value) in _numeric_types
   
+  @classmethod
+  def sql_type(cls):
+    return "INTEGER"
 
 class ManualSortPos(PositionNumber):
   pass
@@ -398,6 +441,10 @@ class Id(BaseColumnType):
   def is_right_type(cls, value):
     return (type(value) in integer_types and is_int_short(value))
   
+  @classmethod
+  def sql_type(cls):
+    return "INTEGER"
+
 
 class Reference(Id):
   """
@@ -415,6 +462,10 @@ class Reference(Id):
   def typename(cls):
     return "Ref"
   
+  @classmethod
+  def sql_type(cls):
+    return "INTEGER"
+
 
 class ReferenceList(BaseColumnType):
   """
@@ -429,13 +480,11 @@ class ReferenceList(BaseColumnType):
     return "RefList"
 
   def do_convert(self, value):
-    if isinstance(value, six.string_types):
+    if is_json_array(value):
-      # If it's a string that looks like JSON, try to parse it as such.
+      try:
-      if value.startswith('['):
+        value = json.loads(value)
-        try:
+      except Exception:
-          value = json.loads(value)
+        pass
-        except Exception:
-          pass
 
     if isinstance(value, RecordSet):
       assert value._table.table_id == self.table_id
@@ -445,11 +494,24 @@ class ReferenceList(BaseColumnType):
       return None
     return [Reference.do_convert(val) for val in value]
 
+
   @classmethod
   def is_right_type(cls, value):
-    return value is None or (isinstance(value, list) and
+    return value is None or is_json_array(value) or (isinstance(value, six.string_types + (list,)) and
                              all(Reference.is_right_type(val) for val in value))
   
+  @classmethod
+  def sql_type(cls):
+    return "TEXT"
+
+
+class ChildReferenceList(ReferenceList):
+  """
+  Chil genuis reference list type.
+  """
+  def __init__(self, table_id):
+    super(ChildReferenceList, self).__init__(table_id)
+
 
 class Attachments(ReferenceList):
   """
@@ -457,3 +519,7 @@ class Attachments(ReferenceList):
   """
   def __init__(self):
     super(Attachments, self).__init__('_grist_Attachments')
+
+
+def is_json_array(val):
+  return isinstance(val, six.string_types) and val.startswith('[')