(core) move data engine code to core

Summary:
this moves sandbox/grist to core, and adds a requirements.txt
file for reconstructing the content of sandbox/thirdparty.

Test Plan:
existing tests pass.
Tested core functionality manually.  Tested docker build manually.

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2563

sandbox/grist/imports/main.py

@@ -0,0 +1,60 @@
+"""This module loads a file_importer that implements the Grist import
+API, and calls its selected method passing argument received from
+PluginManager.sandboxImporter(). It returns an object formatted so
+that it can be used by Grist.
+
+"""
+import sys
+import argparse
+import logging
+import imp
+import json
+import marshal
+log = logging.getLogger(__name__)
+
+# Include /thirdparty into module search paths, in particular for messytables.
+# pylint: disable=wrong-import-position
+sys.path.append('/thirdparty')
+
+def marshal_data(export_list):
+  return marshal.dumps(export_list, 2)
+
+def main():
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument('-d', '--debug', action='store_true',
+                      help="Print debug instead of producing normal binary output")
+  parser.add_argument('-t', '--table',
+                      help="Suggested table name to use with CSV imports")
+  parser.add_argument('-n', '--plugin-name', required=True,
+                      help="Name of a python module implementing the import API.")
+  parser.add_argument('-p', '--plugin-path',
+                      help="Location of the module.")
+  parser.add_argument('--action-options',
+                      help="Options to pass to the action. See API documentation.")
+  parser.add_argument('action', help='Action to call',
+                      choices=['can_parse', 'parse_file'])
+  parser.add_argument('input', help='File to convert')
+  args = parser.parse_args()
+
+  s = logging.StreamHandler()
+  s.setFormatter(logging.Formatter(fmt='%(asctime)s.%(msecs)03d %(message)s',
+                                   datefmt='%Y-%m-%d %H:%M:%S'))
+  rootLogger = logging.getLogger()
+  rootLogger.addHandler(s)
+  rootLogger.setLevel(logging.DEBUG if args.debug else logging.INFO)
+  import_plugin = imp.load_compiled(
+    args.plugin_name,
+    args.plugin_path)
+  options = {}
+  if args.action_options:
+    options = json.loads(args.action_options)
+  parsed_data = getattr(import_plugin, args.action)(args.input, **options)
+  marshalled_data = marshal_data(parsed_data)
+  log.info("Marshalled data has %d bytes", len(marshalled_data))
+  if not args.debug:
+    sys.stdout.write(marshalled_data)
+
+
+if __name__ == "__main__":
+  main()

sandbox/grist/imports/test_messytables.py

@@ -0,0 +1,22 @@
+import unittest
+import messytables
+import os
+
+class TestMessyTables(unittest.TestCase):
+
+  # Just a skeleton test
+  def test_any_tableset(self):
+    path = os.path.join(os.path.dirname(__file__),
+                        "fixtures", "nyc_schools_progress_report_ec_2013.xlsx")
+    with open(path, "r") as f:
+      table_set = messytables.any.any_tableset(f, extension=os.path.splitext(path)[1])
+
+    self.assertIsInstance(table_set, messytables.XLSTableSet)
+    self.assertEqual([t.name for t in table_set.tables],
+                     ['Summary', 'Student Progress', 'Student Performance', 'School Environment',
+                      'Closing the Achievement Gap', 'Middle School Course Metrics',
+                      'All Information', 'Peer Groups'])
+
+
+if __name__ == "__main__":
+  unittest.main()