(core) Nicer conversion from numeric to text

Summary: Expanded Text.do_convert for float values

Test Plan: Add python unit test test_numeric_to_text_conversion

Reviewers: jarek

Reviewed By: jarek

Subscribers: dsagal

Differential Revision: https://phab.getgrist.com/D3049

sandbox/grist/test_summary.py

@@ -650,19 +650,19 @@ class Address:
     # Check the values in the summary tables: they should reflect the new formula.
     self.assertTableData('GristSummary_7_Address', cols="subset", data=[
       [ "id", "city",     "state", "count", "amount"  ],
-      [ 1,    "New York", "NY"   , 3,       str(100*(1.+6+11))],
+      [ 1,    "New York", "NY"   , 3,       str(100*(1+6+11))],
-      [ 2,    "Albany",   "NY"   , 1,       "200.0"        ],
+      [ 2,    "Albany",   "NY"   , 1,       "200"        ],
-      [ 3,    "Seattle",  "WA"   , 1,       "300.0"        ],
+      [ 3,    "Seattle",  "WA"   , 1,       "300"        ],
-      [ 4,    "Chicago",  "IL"   , 1,       "400.0"        ],
+      [ 4,    "Chicago",  "IL"   , 1,       "400"        ],
-      [ 5,    "Bedford",  "MA"   , 1,       "500.0"        ],
+      [ 5,    "Bedford",  "MA"   , 1,       "500"        ],
-      [ 6,    "Buffalo",  "NY"   , 1,       "700.0"        ],
+      [ 6,    "Buffalo",  "NY"   , 1,       "700"        ],
-      [ 7,    "Bedford",  "NY"   , 1,       "800.0"        ],
+      [ 7,    "Bedford",  "NY"   , 1,       "800"        ],
-      [ 8,    "Boston",   "MA"   , 1,       "900.0"        ],
+      [ 8,    "Boston",   "MA"   , 1,       "900"        ],
-      [ 9,    "Yonkers",  "NY"   , 1,       "1000.0"       ],
+      [ 9,    "Yonkers",  "NY"   , 1,       "1000"       ],
     ])
     self.assertTableData('GristSummary_7_Address2', cols="subset", data=[
       [ "id", "count",  "amount"],
-      [ 1,    11,       "6600.0"],
+      [ 1,    11,       "6600"],
     ])
 
     # Add a new summary table, and check that it gets the new formula.
@@ -698,10 +698,10 @@ class Address:
     # Verify the summarized data.
     self.assertTableData('GristSummary_7_Address3', cols="subset", data=[
       [ "id", "state", "count", "amount"                    ],
-      [ 1,    "NY",     7,      str(100*(1.+2+6+7+8+10+11)) ],
+      [ 1,    "NY",     7,      str(int(100*(1.+2+6+7+8+10+11))) ],
-      [ 2,    "WA",     1,      "300.0"                     ],
+      [ 2,    "WA",     1,      "300"                     ],
-      [ 3,    "IL",     1,      "400.0"                     ],
+      [ 3,    "IL",     1,      "400"                     ],
-      [ 4,    "MA",     2,      str(500.+900)               ],
+      [ 4,    "MA",     2,      str(500+900)               ],
     ])
 
   #----------------------------------------------------------------------
@@ -760,15 +760,15 @@ class Address:
       ])
     ])
     self.assertTableData('Table1', data=[
-      [ "id", "manualSort", "A",    "B",  "C"   ],
+      [ "id", "manualSort", "A",  "B",  "C"   ],
-      [ 1,    1.0,          "10.0", 1.0,    None  ],
+      [ 1,    1.0,          "10", 1.0,  None  ],
-      [ 2,    2.0,          "20.0", 2.0,    None  ],
+      [ 2,    2.0,          "20", 2.0,  None  ],
-      [ 3,    3.0,          "10.0", 3.0,    None  ],
+      [ 3,    3.0,          "10", 3.0,  None  ],
     ])
     self.assertTableData('GristSummary_6_Table1', data=[
-      [ "id", "A",    "group",  "count",  "B" ],
+      [ "id", "A",  "group",  "count",  "B" ],
-      [ 1,    "10.0", [1,3],    2,        4   ],
+      [ 1,    "10", [1,3],    2,        4   ],
-      [ 2,    "20.0", [2],      1,        2   ],
+      [ 2,    "20", [2],      1,        2   ],
     ])
 
   #----------------------------------------------------------------------

sandbox/grist/test_types.py

@@ -117,7 +117,7 @@ class TestTypes(test_engine.EngineTestCase):
       "stored": [
         ["ModifyColumn", "Types", "numeric", {"type": "Text"}],
         ["BulkUpdateRecord", "Types", [13, 14, 15, 16, 17, 18],
-          {"numeric": ["False", "True", "1509556595.0", "8.153", "0.0", "1.0"]}],
+          {"numeric": ["False", "True", "1509556595", "8.153", "0", "1"]}],
         ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Text"}],
         ["UpdateRecord", "Formulas", 1, {"division": ["E", "TypeError"]}],
       ],
@@ -170,7 +170,7 @@ class TestTypes(test_engine.EngineTestCase):
       "stored": [
         ["ModifyColumn", "Types", "date", {"type": "Text"}],
         ["BulkUpdateRecord", "Types", [13, 14, 15, 16, 17, 18],
-          {"date": ["False", "True", "1509556595.0", "8.153", "0.0", "1.0"]}],
+          {"date": ["False", "True", "1509556595", "8.153", "0", "1"]}],
         ["UpdateRecord", "_grist_Tables_column", 25, {"type": "Text"}]
       ],
       "undo": [
@@ -188,10 +188,10 @@ class TestTypes(test_engine.EngineTestCase):
       [12,   u"Chîcágö",   u"Chîcágö",   u"Chîcágö",   u"Chîcágö",   u"Chîcágö"],
       [13,   False,       "False",     "False",     "False",     "False"],
       [14,   True,        "True",      "True",      "True",      "True"],
-      [15,   1509556595,  "1509556595.0","1509556595","1509556595","1509556595.0"],
+      [15,   1509556595,  "1509556595","1509556595","1509556595","1509556595"],
       [16,   8.153,       "8.153",     "8.153",     "8.153",     "8.153"],
-      [17,   0,           "0.0",       "0",         "False",     "0.0"],
+      [17,   0,           "0",         "0",         "False",     "0"],
-      [18,   1,           "1.0",       "1",         "True",      "1.0"],
+      [18,   1,           "1",         "1",         "True",      "1"],
       [19,   "",          "",          "",          "",          ""],
       [20,   None,        None,        None,        None,        None]
     ])
@@ -295,6 +295,120 @@ class TestTypes(test_engine.EngineTestCase):
       [20,   None,       None,       None,       None,       None],
     ])
 
+  def test_numeric_to_text_conversion(self):
+    """
+    Tests text formatting of floats of different sizes.
+    """
+    sample = testutil.parse_test_sample({
+      "SCHEMA": [
+        [1, "Types", [
+          [22, "numeric", "Numeric", False, "", "", ""],
+          [23, "other", "Text", False, "", "", ""],
+        ]],
+      ],
+      "DATA": {
+        "Types": [["id", "numeric"]] + [[i+1, 1.23456789 * 10 ** (i-20)] for i in range(40)]
+      },
+    })
+    self.load_sample(sample)
+
+    out_actions = self.apply_user_action(["ModifyColumn", "Types", "numeric", { "type" : "Text" }])
+    self.assertPartialOutActions(out_actions, {
+      "stored": [
+        ["ModifyColumn", "Types", "numeric", {"type": "Text"}],
+        ["BulkUpdateRecord", "Types",
+         [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+          21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40],
+         {"numeric": ["1.23456789e-20",
+                      "1.23456789e-19",
+                      "1.23456789e-18",
+                      "1.23456789e-17",
+                      "1.23456789e-16",
+                      "1.23456789e-15",
+                      "1.23456789e-14",
+                      "1.23456789e-13",
+                      "1.23456789e-12",
+                      "1.23456789e-11",
+                      "1.23456789e-10",
+                      "1.23456789e-09",
+                      "1.23456789e-08",
+                      "1.23456789e-07",
+                      "1.23456789e-06",
+                      "1.23456789e-05",
+                      "0.000123456789",
+                      "0.00123456789",
+                      "0.0123456789",
+                      "0.123456789",
+                      "1.23456789",
+                      "12.3456789",
+                      "123.456789",
+                      "1234.56789",
+                      "12345.6789",
+                      "123456.789",
+                      "1234567.89",
+                      "12345678.9",
+                      "123456789",
+                      "1234567890",
+                      "12345678900",
+                      "123456789000",
+                      "1234567890000",
+                      "12345678900000",
+                      "123456789000000",
+                      "1234567890000000",
+                      "1.23456789e+16",
+                      "1.23456789e+17",
+                      "1.23456789e+18",
+                      "1.23456789e+19"]}],
+        ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Text"}],
+      ],
+      "undo": [
+        ["BulkUpdateRecord", "Types",
+         [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+          21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40],
+         {"numeric": [1.2345678899999998e-20,
+                      1.2345678899999999e-19,
+                      1.23456789e-18,
+                      1.23456789e-17,
+                      1.2345678899999998e-16,
+                      1.23456789e-15,
+                      1.23456789e-14,
+                      1.23456789e-13,
+                      1.2345678899999998e-12,
+                      1.2345678899999998e-11,
+                      1.2345678899999998e-10,
+                      1.23456789e-09,
+                      1.2345678899999999e-08,
+                      1.23456789e-07,
+                      1.2345678899999998e-06,
+                      1.23456789e-05,
+                      0.000123456789,
+                      0.00123456789,
+                      0.012345678899999999,
+                      0.123456789,
+                      1.23456789,
+                      12.3456789,
+                      123.45678899999999,
+                      1234.5678899999998,
+                      12345.678899999999,
+                      123456.78899999999,
+                      1234567.89,
+                      12345678.899999999,
+                      123456788.99999999,
+                      1234567890.0,
+                      12345678899.999998,
+                      123456788999.99998,
+                      1234567890000.0,
+                      12345678899999.998,
+                      123456788999999.98,
+                      1234567890000000.0,
+                      1.2345678899999998e+16,
+                      1.2345678899999998e+17,
+                      1.23456789e+18,
+                      1.2345678899999998e+19]}],
+        ["ModifyColumn", "Types", "numeric", {"type": "Numeric"}],
+        ["UpdateRecord", "_grist_Tables_column", 22, {"type": "Numeric"}],
+      ]
+    })
 
   def test_int_conversions(self):
     """

sandbox/grist/usertypes.py

@@ -14,6 +14,8 @@ the extra complexity.
 import csv
 import datetime
 import json
+import math
+
 import six
 import objtypes
 from objtypes import AltText
@@ -157,6 +159,20 @@ class Text(BaseColumnType):
       return value.decode('utf8')
     elif value is None:
       return None
+    elif isinstance(value, float) and not (math.isinf(value) or math.isnan(value)):
+      # Format as integer if possible to avoid scientific notation
+      # so that strings of digits that aren't meant to represent numbers convert correctly.
+      # https://stackoverflow.com/questions/1848700/biggest-integer-that-can-be-stored-in-a-double
+      # says that 2^53+1 is the first integer that isn't accurately stored in a float,
+      # and it looks like 2^53 so we can't trust that either ;)
+      if abs(value) < 2 ** 53:
+        as_int = int(value)
+        if value == as_int:
+          return six.text_type(as_int)
+
+      # More than 15 digits of precision can make large numbers (e.g. 2^53+1) look as if
+      # they're represented exactly when they're not
+      return u"%.15g" % value
     else:
       return six.text_type(value)