diff --git a/app/plugin/CustomSectionAPI-ti.ts b/app/plugin/CustomSectionAPI-ti.ts index 37f361f3..6d7b8853 100644 --- a/app/plugin/CustomSectionAPI-ti.ts +++ b/app/plugin/CustomSectionAPI-ti.ts @@ -7,6 +7,7 @@ import * as t from "ts-interface-checker"; export const ColumnToMap = t.iface([], { "name": "string", "title": t.opt(t.union("string", "null")), + "description": t.opt(t.union("string", "null")), "type": t.opt("string"), "optional": t.opt("boolean"), "allowMultiple": t.opt("boolean"), diff --git a/app/server/lib/SafePythonComponent.ts b/app/server/lib/SafePythonComponent.ts index 9d0eb87a..1518d8cc 100644 --- a/app/server/lib/SafePythonComponent.ts +++ b/app/server/lib/SafePythonComponent.ts @@ -41,6 +41,7 @@ export class SafePythonComponent extends BaseComponent { importMount: this._tmpDir, logTimes: true, logMeta: this._logMeta, + preferredPythonVersion: '3', }); } diff --git a/sandbox/grist/imports/fixtures/strange_dates.xlsx b/sandbox/grist/imports/fixtures/strange_dates.xlsx index 8219e1fe..4efdc848 100644 Binary files a/sandbox/grist/imports/fixtures/strange_dates.xlsx and b/sandbox/grist/imports/fixtures/strange_dates.xlsx differ diff --git a/sandbox/grist/imports/test_import_csv.py b/sandbox/grist/imports/import_csv_test.py similarity index 100% rename from sandbox/grist/imports/test_import_csv.py rename to sandbox/grist/imports/import_csv_test.py diff --git a/sandbox/grist/imports/test_import_json.py b/sandbox/grist/imports/import_json_test.py similarity index 100% rename from sandbox/grist/imports/test_import_json.py rename to sandbox/grist/imports/import_json_test.py diff --git a/sandbox/grist/imports/import_utils.py b/sandbox/grist/imports/import_utils.py index 07d83b77..37671568 100644 --- a/sandbox/grist/imports/import_utils.py +++ b/sandbox/grist/imports/import_utils.py @@ -1,17 +1,17 @@ """ Helper functions for import plugins """ -import sys import itertools import logging import os -# Include /thirdparty into module search paths, in particular for messytables. -sys.path.append('/thirdparty') - import six from six.moves import zip +if six.PY2: + raise RuntimeError("Imports should use a Python 3 environment") + + log = logging.getLogger(__name__) # Get path to an imported file. diff --git a/sandbox/grist/imports/import_xls.py b/sandbox/grist/imports/import_xls.py index ebd15150..2f3adfe3 100644 --- a/sandbox/grist/imports/import_xls.py +++ b/sandbox/grist/imports/import_xls.py @@ -5,8 +5,10 @@ and returns a object formatted so that it can be used by grist for a bulk add re import logging import messytables -import openpyxl import six +import openpyxl +from openpyxl.utils.datetime import from_excel +from openpyxl.worksheet import _reader from six.moves import zip import parse_data @@ -15,6 +17,19 @@ from imports import import_utils log = logging.getLogger(__name__) +# Some strange Excel files have values that are marked as dates but are invalid as dates. +# Normally, openpyxl converts these to `#VALUE!`, but keeping the original value is better. +# In the case where this was encountered, the original value is a large number: +# the 6281228502068 in test_excel_strange_dates. +# Here we monkeypatch openpyxl to keep the original value. +def new_from_excel(value, *args, **kwargs): + try: + return from_excel(value, *args, **kwargs) + except (ValueError, OverflowError): + return value +_reader.from_excel = new_from_excel + + def import_file(file_source): path = import_utils.get_path(file_source["path"]) parse_options, tables = parse_file(path) diff --git a/sandbox/grist/imports/test_import_xls.py b/sandbox/grist/imports/import_xls_test.py similarity index 98% rename from sandbox/grist/imports/test_import_xls.py rename to sandbox/grist/imports/import_xls_test.py index 86d5da1d..a905ef9a 100644 --- a/sandbox/grist/imports/test_import_xls.py +++ b/sandbox/grist/imports/import_xls_test.py @@ -133,7 +133,8 @@ class TestImportXLS(unittest.TestCase): {'id': 'f', 'type': 'Date'}, {'id': 'g', 'type': 'Any'}, {'id': 'h', 'type': 'Date'}, - {'id': 'i', 'type': 'Date'}, + {'id': 'i', 'type': 'Any'}, + {'id': 'j', 'type': 'Numeric'}, ], 'table_data': [ [u'21:14:00'], @@ -144,7 +145,8 @@ class TestImportXLS(unittest.TestCase): [-2207347200.0], [u'7/4/1776'], [205286400.0], - [-2209161600.0], + ['00:00:00'], + [6281228502068], ], }]) diff --git a/sandbox/grist/imports/test_messytables.py b/sandbox/grist/imports/messytables_test.py similarity index 100% rename from sandbox/grist/imports/test_messytables.py rename to sandbox/grist/imports/messytables_test.py diff --git a/sandbox/grist/imports/test_imports.py b/sandbox/grist/imports/test_imports.py new file mode 100644 index 00000000..25893b6c --- /dev/null +++ b/sandbox/grist/imports/test_imports.py @@ -0,0 +1,41 @@ +""" +Imports only run in Python 3 sandboxes, so the tests here only run in Python 3. +The test files in this directory have been renamed to look like 'import_xls_test.py' instead +of 'test_import_xls.py' so that they're not discovered automatically by default. +`load_tests` below then discovers that pattern directly, but only in Python 3. +This allows the tests to be skipped without having to specify a pattern when discovering tests. +The downside is that if you *do* want to specify a pattern, that probably won't work. + +The reason for this method is that there's a bug in Python 2's unittest module +regarding packages and directories that we must handle. +This means that in Python 2, I wasn't able to prevent files like 'test_import_xls.py' +from being discovered automatically regardless of what I did in `load_tests`. + +Compare https://docs.python.org/3.9/library/unittest.html#load-tests-protocol + vs https://docs.python.org/2/library/unittest.html#load-tests-protocol + +from "If discovery is started" on both pages. Note in particular: + + > Changed in version 3.5: Discovery no longer checks package names for matching pattern + due to the impossibility of package names matching the default pattern. + +The reason for skipping entire files from being discovered instead of skipping TestCase classes +is that just importing the test file will fail with an error, both because we manually raise +an exception and because dependencies are missing. +""" + +import os + +import six + + +def load_tests(loader, standard_tests, _pattern): + if six.PY2: + return standard_tests + + this_dir = os.path.join(os.path.dirname(__file__)) + package_tests = list(loader.discover(start_dir=this_dir, pattern='*_test.py')) + if len(package_tests) < 3: + raise Exception("Expected more import tests to be discovered") + standard_tests.addTests(package_tests) + return standard_tests diff --git a/sandbox/requirements.txt b/sandbox/requirements.txt index 49a32d01..d956f6c9 100644 --- a/sandbox/requirements.txt +++ b/sandbox/requirements.txt @@ -3,22 +3,12 @@ asttokens==2.0.5 backports.functools-lru-cache==1.6.4 chardet==4.0.0 enum34==1.1.10 -et-xmlfile==1.0.1 -html5lib==1.1 iso8601==0.1.12 -jdcal==1.4.1 -json_table_schema==0.2.1 lazy_object_proxy==1.6.0 -lxml==4.6.3 # used in csv plugin only? -messytables==0.15.2 -openpyxl==2.6.4 python_dateutil==2.8.2 -python_magic==0.4.12 roman==2.0.0 singledispatch==3.6.2 six==1.16.0 sortedcontainers==2.4.0 -webencodings==0.5 wrapt==1.12.1 -xlrd==1.2.0 unittest-xml-reporting==2.0.0 diff --git a/sandbox/requirements3.txt b/sandbox/requirements3.txt index f0e39250..7f016bfc 100644 --- a/sandbox/requirements3.txt +++ b/sandbox/requirements3.txt @@ -20,8 +20,8 @@ json_table_schema==0.2.1 lazy_object_proxy==1.6.0 lxml==4.6.3 # used in csv plugin only? messytables==0.15.2 -openpyxl==2.6.4 python_dateutil==2.8.2 +openpyxl==3.0.10 python_magic==0.4.12 roman==2.0.0 singledispatch==3.6.2