(core) Make Python tests pass in Python 3.11

Summary: Mostly just changes to tests to make them more flexible.

Test Plan: Python tests pass locally with 3.10 and 3.11. Making tests run in CI on these versions will happen in grist-core.

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: paulfitz

Differential Revision: https://phab.getgrist.com/D3978

sandbox/grist/autocomplete_context.py

@@ -52,7 +52,10 @@ class AutocompleteContext(object):
     }
     for key, value in six.iteritems(self._context):
       if value and callable(value):
-        argspec = inspect.formatargspec(*inspect.getargspec(value))
+        if six.PY2:
+          argspec = inspect.formatargspec(*inspect.getargspec(value))
+        else:
+          argspec = str(inspect.signature(value))  # pylint: disable=no-member
         self._functions[key] = Completion(key, argspec, is_grist_func(value))
 
     for key, value in self._context.copy().items():

sandbox/grist/test_completion.py

@@ -1,4 +1,5 @@
 import datetime
+import sys
 
 import test_engine
 import testsamples
@@ -69,16 +70,13 @@ class TestCompletion(test_engine.EngineTestCase):
     self.assertEqual(self.autocomplete("valu", "Schools", "lastModifier"),
                      ["value"])
     # Should have same type as column.
-    self.assertGreaterEqual(
-      set(self.autocomplete("value.", "Schools", "lastModifier")),
+    self.assert_autocomplete_includes("value.", "Schools", "lastModifier",
       {'value.startswith(', 'value.replace(', 'value.title('}
     )
-    self.assertGreaterEqual(
-      set(self.autocomplete("value.", "Schools", "lastModified")),
+    self.assert_autocomplete_includes("value.", "Schools", "lastModified",
       {'value.month', 'value.strftime(', 'value.replace('}
     )
-    self.assertGreaterEqual(
-      set(self.autocomplete("value.m", "Schools", "lastModified")),
+    self.assert_autocomplete_includes("value.m", "Schools", "lastModified",
       {'value.month', 'value.minute'}
     )
 
@@ -158,14 +156,14 @@ class TestCompletion(test_engine.EngineTestCase):
 
   def test_function(self):
     self.assertEqual(self.autocomplete("MEDI", "Address", "city"),
-        [('MEDIAN', '(value, *more_values)', True)])
-    self.assertEqual(self.autocomplete("ma", "Address", "city"), [
+                     [('MEDIAN', '(value, *more_values)', True)])
+    self.assert_autocomplete_includes("ma", "Address", "city", {
       ('MAX', '(value, *more_values)', True),
       ('MAXA', '(value, *more_values)', True),
       'map(',
       'math',
       'max(',
-    ])
+    })
 
   def test_member(self):
     self.assertEqual(self.autocomplete("datetime.tz", "Address", "city"),
@@ -294,34 +292,28 @@ class TestCompletion(test_engine.EngineTestCase):
 
   def test_suggest_column_type_methods(self):
     # Should treat columns as correct types.
-    self.assertGreaterEqual(
-      set(self.autocomplete("$firstName.", "Students", "firstName")),
+    self.assert_autocomplete_includes("$firstName.", "Students", "firstName",
       {'$firstName.startswith(', '$firstName.replace(', '$firstName.title('}
     )
-    self.assertGreaterEqual(
-      set(self.autocomplete("$birthDate.", "Students", "lastName")),
+    self.assert_autocomplete_includes("$birthDate.", "Students", "lastName",
       {'$birthDate.month', '$birthDate.strftime(', '$birthDate.replace('}
     )
-    self.assertGreaterEqual(
-      set(self.autocomplete("$lastVisit.m", "Students", "firstName")),
+    self.assert_autocomplete_includes("$lastVisit.m", "Students", "firstName",
       {'$lastVisit.month', '$lastVisit.minute'}
     )
-    self.assertGreaterEqual(
-      set(self.autocomplete("$school.", "Students", "firstName")),
+    self.assert_autocomplete_includes("$school.", "Students", "firstName",
       {'$school.address', '$school.name', '$school.yearFounded', '$school.budget'}
     )
     self.assertEqual(self.autocomplete("$school.year", "Students", "lastName"),
                      ['$school.yearFounded'])
-    self.assertGreaterEqual(
-      set(self.autocomplete("$yearFounded.", "Schools", "budget")),
+    self.assert_autocomplete_includes("$yearFounded.", "Schools", "budget",
       {
         '$yearFounded.denominator',    # Only integers have this
         '$yearFounded.bit_length(',    # and this
         '$yearFounded.real'
       }
     )
-    self.assertGreaterEqual(
-      set(self.autocomplete("$budget.", "Schools", "budget")),
+    self.assert_autocomplete_includes("$budget.", "Schools", "budget",
       {'$budget.is_integer(', '$budget.real'}    # Only floats have this
     )
 
@@ -331,8 +323,7 @@ class TestCompletion(test_engine.EngineTestCase):
       self.autocomplete("$school.name.st", "Students", "firstName"),
       ['$school.name.startswith(', '$school.name.strip(']
     )
-    self.assertGreaterEqual(
-      set(self.autocomplete("$school.yearFounded.","Students", "firstName")),
+    self.assert_autocomplete_includes("$school.yearFounded.","Students", "firstName",
       {
         '$school.yearFounded.denominator',
         '$school.yearFounded.bit_length(',
@@ -498,6 +489,20 @@ class TestCompletion(test_engine.EngineTestCase):
     else:
       return results
 
+  def assert_autocomplete_includes(self, formula, table, column, expected, user=None, row_id=None):
+    completions = self.autocomplete(formula, table, column, user=user, row_id=row_id)
+
+    def replace_completion(completion):
+      if isinstance(completion, str) and completion.endswith('()'):
+        # Python 3.10+ autocompletes the closing paren for methods with no arguments.
+        # This allows the test to check for `somestring.title(` and work across Python versions.
+        assert sys.version_info >= (3, 10)
+        return completion[:-1]
+      return completion
+
+    completions = set(replace_completion(completion) for completion in completions)
+    self.assertGreaterEqual(completions, expected)
+
   def test_example_values(self):
     self.assertEqual(
       self.autocomplete("$", "Schools", "name", row_id=1),

sandbox/grist/test_engine.py

@@ -2,6 +2,7 @@ import difflib
 import functools
 import json
 import logging
+import sys
 import unittest
 from collections import namedtuple
 from pprint import pprint
@@ -273,7 +274,16 @@ class EngineTestCase(unittest.TestCase):
     self.assertIsInstance(exc.error, type_)
     self.assertEqual(exc._message, message)
     if tracebackRegexp:
-      self.assertRegex(exc.details, tracebackRegexp)
+      traceback_string = exc.details
+      if sys.version_info >= (3, 11) and type_ != SyntaxError:
+        # Python 3.11+ adds lines with only spaces and ^ to indicate the location of the error.
+        # We remove those lines to make the test work with both old and new versions.
+        # This doesn't apply to SyntaxError, which has those lines in all versions.
+        traceback_string = "\n".join(
+          line for line in traceback_string.splitlines()
+          if set(line) != {" ", "^"}
+        )
+      self.assertRegex(traceback_string.strip(), tracebackRegexp.strip())
 
   def assertViews(self, list_of_views):
     """

sandbox/grist/test_gencode.py

@@ -37,7 +37,7 @@ schema_data = [
     [28, "country",     "Text",       False, "'US'", "country", ''],
     [29, "region",      "Any",        True,
           "{'US': 'North America', 'UK': 'Europe'}.get(rec.country, 'N/A')", "region", ''],
-    [30, "badSyntax",   "Any",        True, "for a in b\n10", "", ""],
+    [30, "badSyntax",   "Any",        True, "for a in\n10", "", ""],
   ]]
 ]
 
@@ -72,11 +72,10 @@ class TestGenCode(unittest.TestCase):
     generated = gcode.get_user_text()
     if six.PY3:
       saved_sample = saved_sample.replace(
-        "raise SyntaxError('invalid syntax', ('usercode', 1, 11, u'for a in b'))",
+        "raise SyntaxError('invalid syntax', ('usercode', 1, 9, u'for a in'))",
         "raise SyntaxError('invalid syntax\\n\\n"
-        "A `SyntaxError` occurs when Python cannot understand your code.\\n\\n"
-        "You wrote a `for` loop but\\nforgot to add a colon `:` at the end\\n\\n', "
-        "('usercode', 1, 11, 'for a in b'))"
+        "A `SyntaxError` occurs when Python cannot understand your code.\\n\\n', "
+        "('usercode', 1, 9, 'for a in'))"
       )
     self.assertEqual(generated, saved_sample, "Generated code doesn't match sample:\n" +
                      "".join(difflib.unified_diff(generated.splitlines(True),

sandbox/grist/usercode.py

@@ -61,8 +61,8 @@ class Address:
     return {'US': 'North America', 'UK': 'Europe'}.get(rec.country, 'N/A')
 
   def badSyntax(rec, table):
-    # for a in b
+    # for a in
     # 10
-    raise SyntaxError('invalid syntax', ('usercode', 1, 11, u'for a in b'))
+    raise SyntaxError('invalid syntax', ('usercode', 1, 9, u'for a in'))
 ======================================================================
 """