gristlabs_grist-core/sandbox/grist/test_codebuilder.py

# -*- coding: utf-8 -*-
import unittest

import six
from asttokens.util import fstring_positions_work

import codebuilder
import test_engine

unicode_prefix = 'u' if six.PY2 else ''

def make_body(formula, default=None):
  return codebuilder.make_formula_body(formula, default).get_text()

class TestCodeBuilder(test_engine.EngineTestCase):
  def test_make_formula_body(self):
    # Test simple usage.
    self.assertEqual(make_body(""), "return None")
    self.assertEqual(make_body("", 0.0), "return 0.0")
    self.assertEqual(make_body("", ""), "return ''")
    self.assertEqual(make_body("  "), "return None")
    self.assertEqual(make_body("  ", "-"), "return '-'")
    self.assertEqual(make_body("\n\t"), "return None")
    self.assertEqual(make_body("$foo"), "return rec.foo")
    self.assertEqual(make_body("rec.foo"), "return rec.foo")
    self.assertEqual(make_body("return $foo"), "return rec.foo")
    self.assertEqual(make_body("return $f123"), "return rec.f123")
    self.assertEqual(make_body("return rec.foo"), "return rec.foo")
    self.assertEqual(make_body("$foo if $bar else max($foo.bar.baz)"),
                     "return rec.foo if rec.bar else max(rec.foo.bar.baz)")

    # Check that we don't mistake our temporary representation of "$" for the real thing.
    self.assertEqual(make_body("return DOLLARfoo"), "return DOLLARfoo")

    # Test that we don't translate $foo inside string literals or comments.
    self.assertEqual(make_body("$foo or '$foo'"), "return rec.foo or '$foo'")
    self.assertEqual(make_body("$foo * 2 # $foo"), "return rec.foo * 2 # $foo")
    self.assertEqual(make_body("$foo * 2 # $foo\n$bar"), "rec.foo * 2 # $foo\nreturn rec.bar")
    self.assertEqual(make_body("$foo or '\\'$foo\\''"), "return rec.foo or '\\'$foo\\''")
    self.assertEqual(make_body('$foo or """$foo"""'), 'return rec.foo or """$foo"""')
    self.assertEqual(make_body('$foo or """Some "$foos" stay"""'),
                     'return rec.foo or """Some "$foos" stay"""')

    # Check that we only insert a return appropriately.
    self.assertEqual(make_body('if $foo:\n  return 1\nelse:\n  return 2\n'),
                                    'if rec.foo:\n  return 1\nelse:\n  return 2\n')
    self.assertEqual(make_body('a = $foo\nmax(a, a*2)'), 'a = rec.foo\nreturn max(a, a*2)')

    # Check that return gets inserted correctly when there is a multi-line expression.
    self.assertEqual(make_body('($foo or\n $bar)'), 'return (rec.foo or\n rec.bar)')
    self.assertEqual(make_body('return ($foo or\n  $bar)'), 'return (rec.foo or\n  rec.bar)')
    self.assertEqual(make_body('if $foo: return 17'), 'if rec.foo: return 17')
    self.assertEqual(make_body('$foo\n# return $bar'), 'return rec.foo\n# return $bar')

    # Test that formulas with a single string literal work, including multi-line string literals.
    self.assertEqual(make_body('"test"'), 'return "test"')
    self.assertEqual(make_body('("""test1\ntest2\ntest3""")'), 'return ("""test1\ntest2\ntest3""")')
    self.assertEqual(make_body('"""test1\ntest2\ntest3"""'), 'return """test1\ntest2\ntest3"""')
    self.assertEqual(make_body('"""test1\\ntest2\\ntest3"""'), 'return """test1\\ntest2\\ntest3"""')

    # Same, with single quotes.
    self.assertEqual(make_body("'test'"), "return 'test'")
    self.assertEqual(make_body("('''test1\ntest2\ntest3''')"), "return ('''test1\ntest2\ntest3''')")
    self.assertEqual(make_body("'''test1\ntest2\ntest3'''"), "return '''test1\ntest2\ntest3'''")
    self.assertEqual(make_body("'''test1\\ntest2\\ntest3'''"), "return '''test1\\ntest2\\ntest3'''")

    # And with mixing quotes
    self.assertEqual(make_body("'''test1\"\"\" +\\\n  \"\"\"test2'''"),
                     "return '''test1\"\"\" +\\\n  \"\"\"test2'''")
    self.assertEqual(make_body("'''test1''' +\\\n  \"\"\"test2\"\"\""),
                     "return '''test1''' +\\\n  \"\"\"test2\"\"\"")
    self.assertEqual(make_body("'''test1\"\"\"\n\"\"\"test2'''"),
                     "return '''test1\"\"\"\n\"\"\"test2'''")
    self.assertEqual(make_body("'''test1'''\n\"\"\"test2\"\"\""),
                     "'''test1'''\nreturn \"\"\"test2\"\"\"")

    if fstring_positions_work():
      self.assertEqual(
        make_body("f'{$foo + 1 + $bar} 2 {3 + $baz}' + $foo2 + f'{4 + $bar2}!'"),
        "return f'{rec.foo + 1 + rec.bar} 2 {3 + rec.baz}' + rec.foo2 + f'{4 + rec.bar2}!'"
      )

    # Test that we produce valid code when "$foo" occurs in invalid places.
    if six.PY2:
      raise_code = "raise SyntaxError('invalid syntax', ('usercode', 1, 5, u'foo($bar=1)'))"
    else:
      raise_code = ("raise SyntaxError('invalid syntax\\n\\n"
                    "A `SyntaxError` occurs when Python cannot understand your code.\\n\\n', "
                    "('usercode', 1, 5, 'foo($bar=1)'))")
    self.assertEqual(make_body('foo($bar=1)'),
                     "# foo($bar=1)\n" + raise_code)

    if six.PY2:
      raise_code = ("raise SyntaxError('invalid syntax', "
                    "('usercode', 1, 5, u'def $bar(): return 3'))")
    else:
      raise_code = ("raise SyntaxError('invalid syntax\\n\\n"
                    "A `SyntaxError` occurs when Python cannot understand your code.\\n\\n', "
                    "('usercode', 1, 5, 'def $bar(): return 3'))")
    self.assertEqual(make_body('def $bar(): return 3'),
                     "# def $bar(): return 3\n" + raise_code)

    # If $ is a syntax error, we don't want to turn it into a different syntax error.
    if six.PY2:
      raise_code = ("raise SyntaxError('invalid syntax', "
                    "('usercode', 1, 17, u'$foo + (\"$%.2f\" $ ($17.5))'))")
    else:
      raise_code = ("raise SyntaxError('invalid syntax\\n\\n"
                    "A `SyntaxError` occurs when Python cannot understand your code.\\n\\n', "
                    "('usercode', 1, 17, '$foo + (\"$%.2f\" $ ($17.5))'))")
    self.assertEqual(make_body('$foo + ("$%.2f" $ ($17.5))'),
                     '# $foo + ("$%.2f" $ ($17.5))\n' + raise_code)

    if six.PY2:
      raise_code = "raise SyntaxError('invalid syntax', ('usercode', 4, 10, u'  return $ bar'))"
    else:
      raise_code = ("raise SyntaxError('invalid syntax\\n\\n"
                    "A `SyntaxError` occurs when Python cannot understand your code.\\n\\n"
                    "I am guessing that you wrote `$` by mistake.\\n"
                    "Removing it and writing `return  bar` seems to fix the error.\\n\\n', "
                    "('usercode', 4, 10, '  return $ bar'))")
    self.assertEqual(make_body('if $foo:\n' +
                               '  return $foo\n' +
                               'else:\n' +
                               '  return $ bar\n'),
                     '# if $foo:\n' +
                     '#   return $foo\n' +
                     '# else:\n' +
                     '#   return $ bar\n' +
                     raise_code)

    # Check for reasonable behaviour with non-empty text and no statements.
    self.assertEqual(make_body('# comment'), '# comment\npass')

    self.assertEqual(make_body('rec = 1; rec'), "# rec = 1; rec\n" +
                     "raise SyntaxError('Grist disallows assignment " +
                     "to the special variable \"rec\"', ('usercode', 1, 1, %s'rec = 1; rec'))"
                     % unicode_prefix)
    self.assertEqual(make_body('for rec in []: return rec'), "# for rec in []: return rec\n" +
                     "raise SyntaxError('Grist disallows assignment " +
                     "to the special variable \"rec\"', "
                     "('usercode', 1, 4, %s'for rec in []: return rec'))"
                     % unicode_prefix)

    # some legitimates use of rec
    body = ("""
foo = rec
[rec for x in rec]
for a in rec:
  t = a
[rec for x in rec]
return rec
""")
    self.assertEqual(make_body(body), body)

    # mostly legitimate use of rec but one failing
    body = ("""
foo = rec
[1 for rec in []]
for a in rec:
  t = a
[rec for x in rec]
return rec
""")

    self.assertRegex(make_body(body),
                     r"raise SyntaxError\('Grist disallows assignment" +
                     r" to the special variable \"rec\"', "
                     r"\('usercode', 3, 7, %s'\[1 for rec in \[\]\]'\)\)"
                     % unicode_prefix)

    self.assertEqual(make_body('rec.foo = 1; rec'), "# rec.foo = 1; rec\n" +
                     "raise SyntaxError(\"You can't assign a value to a column with `=`. "
                     "If you mean to check for equality, use `==` instead.\", "
                     "('usercode', 1, 1, %s'rec.foo = 1; rec'))"
                     % unicode_prefix)

    self.assertEqual(make_body('$foo = 1; rec'), "# $foo = 1; rec\n" +
                     "raise SyntaxError(\"You can't assign a value to a column with `=`. "
                     "If you mean to check for equality, use `==` instead.\", "
                     "('usercode', 1, 1, %s'$foo = 1; rec'))"
                     % unicode_prefix)

    self.assertEqual(make_body('assert foo'), "# assert foo\n" +
                     'raise SyntaxError("No `return` statement, '
                     "and the last line isn't an expression.\", "
                     "('usercode', 1, 1, %s'assert foo'))"
                     % unicode_prefix)

    self.assertEqual(make_body('foo = 1'), "# foo = 1\n" +
                     'raise SyntaxError("No `return` statement, '
                     "and the last line isn't an expression."
                     " If you want to check for equality, use `==` instead of `=`.\", "
                     "('usercode', 1, 1, %s'foo = 1'))"
                     % unicode_prefix)

  def test_make_formula_body_unicode(self):
    # Test that we don't fail when strings include unicode characters
    self.assertEqual(make_body("'résumé' + $foo"), u"return 'résumé' + rec.foo")

    # Or when a unicode object is passed in, rather than a byte string
    self.assertEqual(make_body(u"'résumé' + $foo"), u"return 'résumé' + rec.foo")

    # Check the return type of make_body()
    self.assertEqual(type(make_body("foo")), six.text_type)
    self.assertEqual(type(make_body(u"foo")), six.text_type)

  @unittest.skipUnless(six.PY3, "Only Python 3 supports non-ascii variable names")
  def test_make_formula_body_unicode_token_bug(self):
    # Python < 3.12 has a bug in tokenizing certain unicode characters in variable names.
    # This was worked around in https://github.com/gristlabs/asttokens/pull/82
    # Surprisingly this test passes either way, but keeping it as a potentially tricky case.
    self.assertEqual(
      make_body(
        "℘℘··℘℘2=℘℘··℘℘2($foo+℘℘··℘℘2*℘℘··℘℘2+$bar)\n"
        "℘℘··℘℘2=1+a℘℘··℘℘b+a℘℘··℘℘2b\n"
        "℘℘··℘℘2==℘℘··℘℘2($foo+℘℘··℘℘2*℘℘··℘℘2+$bar)"
      ),
      (
        "℘℘··℘℘2=℘℘··℘℘2(rec.foo+℘℘··℘℘2*℘℘··℘℘2+rec.bar)\n"
        "℘℘··℘℘2=1+a℘℘··℘℘b+a℘℘··℘℘2b\n"
        "return ℘℘··℘℘2==℘℘··℘℘2(rec.foo+℘℘··℘℘2*℘℘··℘℘2+rec.bar)"
      ),
    )

  def test_wrap_logical(self):
    self.assertEqual(make_body("IF($foo, $bar, $baz)"),
        "return IF(rec.foo, lambda: (rec.bar), lambda: (rec.baz))")
    self.assertEqual(make_body("return IF(FOO(x,y), BAR(x,y) * 2, BAZ(x,y) + 5)"),
        "return IF(FOO(x,y), lambda: (BAR(x,y) * 2), lambda: (BAZ(x,y) + 5))")
    self.assertEqual(make_body("""
y = $Test
x = IF( FOO(x,y) or 6,
  BAR($x,y).blahh ,
  Foo.lookupRecords(foo=$foo.bar,
    bar=True
  ).baz
 )
return x or y
"""), """
y = rec.Test
x = IF( FOO(x,y) or 6,
  lambda: (BAR(rec.x,y).blahh) ,
  lambda: (Foo.lookupRecords(foo=rec.foo.bar,
    bar=True
  ).baz)
 )
return x or y
""")
    self.assertEqual(make_body("IF($A == 0, IF($B > 5, 'Test1'), IF($C < 10, 'Test2', 'Test3'))"),
        "return IF(rec.A == 0, " +
          "lambda: (IF(rec.B > 5, lambda: ('Test1'))), " +
          "lambda: (IF(rec.C < 10, lambda: ('Test2'), lambda: ('Test3'))))"
    )

  def test_wrap_error(self):
    self.assertEqual(make_body("ISERR($foo.bar)"), "return ISERR(lambda: (rec.foo.bar))")
    self.assertEqual(make_body("ISERROR(1 / 0)"), "return ISERROR(lambda: (1 / 0))")
    self.assertEqual(make_body("IFERROR($foo + #\n  1 / 0, 'XX')"),
        "return IFERROR(lambda: (rec.foo + #\n  1 / 0), 'XX')")

    # Check that extra parentheses are OK.
    self.assertEqual(make_body("IFERROR((($foo + 1) / 0))"),
        "return IFERROR((lambda: ((rec.foo + 1) / 0)))")

    # Check that missing arguments is OK
    self.assertEqual(make_body("ISERR()"), "return ISERR()")


  def test_leading_whitespace(self):
    self.assertEqual(make_body(" $A + 1"), "return rec.A + 1")

    self.assertEqual(make_body("""
  if $A:
    return $A

  $B
"""), """
if rec.A:
  return rec.A

return rec.B
""")