Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity

(core) When importing JSON, create columns of type Numeric rather than Int

Browse Source 
Summary:
JSON import logic was creating columns of type Int when JSON contained integral
values. This causes errors with large errors (e.g. millisecond timestamps), and
Numeric is generally the more convenient and common default.

Test Plan: TBD

Reviewers: jarek, alexmojaki

Reviewed By: jarek, alexmojaki

Subscribers: jarek, alexmojaki

Differential Revision: https://phab.getgrist.com/D3339
This commit is contained in:
Dmitry S 2022-03-27 01:18:19 -04:00
parent 06956f84a5
commit 8269c33d01
2 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
sandbox/grist/imports/import_json.py
View File

@ -7,7 +7,7 @@ example:
``` ```
[{'a': 1, 'b': 'tree'}, {'a': 4, 'b': 'flowers'}, ... ] [{'a': 1, 'b': 'tree'}, {'a': 4, 'b': 'flowers'}, ... ]
``` ```
is turned into a table with two columns 'a' of type 'Int' and 'b' of is turned into a table with two columns 'a' of type 'Numeric' and 'b' of
type 'Text'. type 'Text'.
Nested object are stored as references to a distinct table where the Nested object are stored as references to a distinct table where the
@ -17,7 +17,7 @@ nested object is stored. For example:
``` ```
is turned into a column 'a' of type 'Ref:my_import_name.a', and into is turned into a column 'a' of type 'Ref:my_import_name.a', and into
another table 'my_import_name.a' with a column 'b' of type another table 'my_import_name.a' with a column 'b' of type
'Int'. (Nested-nested objects are supported as well and the module 'Numeric'. (Nested-nested objects are supported as well and the module
assumes no limit to the number of level of nesting you can do.) assumes no limit to the number of level of nesting you can do.)
Each value which is not an object will be stored into a column with id Each value which is not an object will be stored into a column with id
@ -90,7 +90,7 @@ GRIST_TYPES={
} }
for typ in six.integer_types: for typ in six.integer_types:
GRIST_TYPES[typ] = "Int" GRIST_TYPES[typ] = "Numeric"
for typ in six.string_types: for typ in six.string_types:
GRIST_TYPES[typ] = "Text" GRIST_TYPES[typ] = "Text"

20
sandbox/grist/imports/test_import_json.py
View File

@ -9,7 +9,7 @@ class TestImportJSON(TestCase):
grist_tables = import_json.dumps([{'a': 1, 'b': 'baba'}, {'a': 4, 'b': 'abab'}], '') grist_tables = import_json.dumps([{'a': 1, 'b': 'baba'}, {'a': 4, 'b': 'abab'}], '')
self.assertEqual(grist_tables['tables'], [{ self.assertEqual(grist_tables['tables'], [{
'column_metadata': [ 'column_metadata': [
{'id': 'a', 'type': 'Int'}, {'id': 'b', 'type': 'Text'}], {'id': 'a', 'type': 'Numeric'}, {'id': 'b', 'type': 'Text'}],
'table_data': [[1, 4], ['baba', 'abab']], 'table_data': [[1, 4], ['baba', 'abab']],
'table_name': '' 'table_name': ''
}]) }])
@ -18,7 +18,7 @@ class TestImportJSON(TestCase):
grist_tables = import_json.dumps([{'a': 1}, {'b': 'abab'}, {'a': 4}]) grist_tables = import_json.dumps([{'a': 1}, {'b': 'abab'}, {'a': 4}])
self.assertEqual(grist_tables['tables'], [{ self.assertEqual(grist_tables['tables'], [{
'column_metadata': [ 'column_metadata': [
{'id': 'a', 'type': 'Int'}, {'id': 'b', 'type': 'Text'}], {'id': 'a', 'type': 'Numeric'}, {'id': 'b', 'type': 'Text'}],
'table_data': [[1, None, 4], [None, 'abab', None]], 'table_data': [[1, None, 4], [None, 'abab', None]],
'table_name': '' 'table_name': ''
}]) }])
@ -56,7 +56,7 @@ class TestImportJSON(TestCase):
'table_name': 'Hello' 'table_name': 'Hello'
}, { }, {
'column_metadata': [ 'column_metadata': [
{'id': 'b', 'type': 'Int'} {'id': 'b', 'type': 'Numeric'}
], ],
'table_data': [[2]], 'table_data': [[2]],
'table_name': 'Hello_a' 'table_name': 'Hello_a'
@ -73,7 +73,7 @@ class TestImportJSON(TestCase):
'table_name': 'Hello' 'table_name': 'Hello'
}, { }, {
'column_metadata': [ 'column_metadata': [
{'id': 'b', 'type': 'Int'}, {'id': 'd', 'type': 'Ref:Hello_a_d'} {'id': 'b', 'type': 'Numeric'}, {'id': 'd', 'type': 'Ref:Hello_a_d'}
], ],
'table_data': [[2], [1]], 'table_data': [[2], [1]],
'table_name': 'Hello_a' 'table_name': 'Hello_a'
@ -103,12 +103,12 @@ class TestImportJSON(TestCase):
self.assertEqual( self.assertEqual(
import_json.dumps([{'a': [{'b': 1}, {'b': 4}]}, {'c': 2}], 'Hello')['tables'], import_json.dumps([{'a': [{'b': 1}, {'b': 4}]}, {'c': 2}], 'Hello')['tables'],
[ { [ {
'column_metadata': [{'id': 'c', 'type': 'Int'}], 'column_metadata': [{'id': 'c', 'type': 'Numeric'}],
'table_data': [[None, 2]], 'table_data': [[None, 2]],
'table_name': 'Hello' 'table_name': 'Hello'
}, { }, {
'column_metadata': [ 'column_metadata': [
{'id': 'b', 'type': 'Int'}, {'id': 'b', 'type': 'Numeric'},
{'id': 'Hello', 'type': 'Ref:Hello'} {'id': 'Hello', 'type': 'Ref:Hello'}
], ],
'table_data': [[1, 4], [1, 1]], 'table_data': [[1, 4], [1, 1]],
@ -143,7 +143,7 @@ class TestImportJSON(TestCase):
}, { }, {
'table_name': 'Hello_bar', 'table_name': 'Hello_bar',
'column_metadata': [ 'column_metadata': [
{'id': 'c', 'type': 'Int'}, {'id': 'c', 'type': 'Numeric'},
{'id': 'd', 'type': 'Text'}, {'id': 'd', 'type': 'Text'},
{'id': 'Hello', 'type': 'Ref:Hello'} {'id': 'Hello', 'type': 'Ref:Hello'}
], ],
@ -151,7 +151,7 @@ class TestImportJSON(TestCase):
}, { }, {
'table_name': 'Hello_foo', 'table_name': 'Hello_foo',
'column_metadata': [ 'column_metadata': [
{'id': 'a', 'type': 'Int'}, {'id': 'a', 'type': 'Numeric'},
{'id': 'b', 'type': 'Text'}, {'id': 'b', 'type': 'Text'},
{'id': 'Hello', 'type': 'Ref:Hello'}], {'id': 'Hello', 'type': 'Ref:Hello'}],
'table_data': [[1, 4], ['santa', 'cats'], [1, 1]] 'table_data': [[1, 4], ['santa', 'cats'], [1, 1]]
@ -171,11 +171,11 @@ class TestImportJSON(TestCase):
[{ [{
'table_name': 'Hello', 'table_name': 'Hello',
'column_metadata': [ 'column_metadata': [
{'id': 'a', 'type': 'Int'}, {'id': 'a', 'type': 'Numeric'},
{'id': 'b', 'type': 'Numeric'}, {'id': 'b', 'type': 'Numeric'},
{'id': 'c', 'type': 'Bool'}, {'id': 'c', 'type': 'Bool'},
{'id': 'd', 'type': 'Text'}, {'id': 'd', 'type': 'Text'},
{'id': 'e', 'type': 'Int'}, {'id': 'e', 'type': 'Numeric'},
{'id': 'f', 'type': 'Text'}, {'id': 'f', 'type': 'Text'},
{'id': 'g', 'type': 'Text'} {'id': 'g', 'type': 'Text'}
], ],