(core) Converting big number (9 digits or more) to date directly

Summary:
Interpret huge numbers (>8 digits) as timestamps when converting numeric column to date.
Convert date/date time columns to timestamp when converted from numeric/int column.

Test Plan: Updated

Reviewers: georgegevoian

Reviewed By: georgegevoian

Subscribers: dsagal, alexmojaki

Differential Revision: https://phab.getgrist.com/D4030

app/client/components/TypeConversion.ts

@@ -11,6 +11,7 @@ import {csvDecodeRow} from 'app/common/csvFormat';
 import * as gristTypes from 'app/common/gristTypes';
 import {isFullReferencingType} from 'app/common/gristTypes';
 import * as gutil from 'app/common/gutil';
+import {isNonNullish} from 'app/common/gutil';
 import NumberParse from 'app/common/NumberParse';
 import {dateTimeWidgetOptions, guessDateFormat, timeFormatOptions} from 'app/common/parseDate';
 import {TableData} from 'app/common/TableData';
@@ -134,7 +135,8 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe
       if (!dateFormat) {
         // Guess date and time format if there aren't any already
         const colValues = tableData.getColValues(sourceCol.colId()) || [];
-        dateFormat = guessDateFormat(colValues.map(String));
+        const strValues = colValues.map(v => isNonNullish(v) ? String(v) : null);
+        dateFormat = guessDateFormat(strValues);
         widgetOptions = {...widgetOptions, ...(dateTimeWidgetOptions(dateFormat, true))};
       }
       if (toType === 'DateTime' && !widgetOptions.timeFormat) {

app/common/gutil.ts

@@ -928,6 +928,13 @@ export function isNonNullish<T>(value: T | null | undefined): value is T {
   return value !== null && value !== undefined;
 }
 
+/**
+ * Ensures that a value is truthy, with a type guard for the return type.
+ */
+export function truthy<T>(value: T | null | undefined): value is Exclude<T, false | "" | 0> {
+  return Boolean(value);
+}
+
 /**
  * Returns the value of both grainjs and knockout observable without creating a dependency.
  */

app/common/parseDate.ts

@@ -108,6 +108,13 @@ export function parseDate(date: string, options: ParseOptions = {}): number | nu
   if (!date) {
     return null;
   }
+
+  // If this looks like a timestamp (string with 9 or more digits), just return it.
+  const timestamp = parseTimeStamp(date);
+  if (timestamp !== null) {
+    return timestamp;
+  }
+
   const dateFormat = options.dateFormat || "YYYY-MM-DD";
   const dateFormats = [..._buildVariations(dateFormat, date), ...PARSER_FORMATS];
   const cleanDate = date.replace(SEPARATORS, ' ');
@@ -125,11 +132,11 @@ export function parseDate(date: string, options: ParseOptions = {}): number | nu
     datetime += ' ' + time + tzOffset;
     timeformat = ' HH:mm:ss' + (tzOffset ? 'Z' : '');
   }
-  for (const f of dateFormats) {
+  for (const format of dateFormats) {
-    const fullFormat = f + timeformat;
+    const fullFormat = format + timeformat;
     const m = moment.tz(datetime, fullFormat, true, options.timezone || 'UTC');
     if (m.isValid()) {
-      return m.valueOf() / 1000;
+      return m.unix();
     }
   }
   return null;
@@ -149,6 +156,11 @@ export function parseDateStrict(
   if (!date) {
     return;
   }
+  // If this looks like a timestamp (string with 9 or more digits), just return it.
+  const timestamp = parseTimeStamp(date);
+  if (timestamp !== null) {
+    return timestamp;
+  }
   dateFormat = dateFormat || "YYYY-MM-DD";
   const dateFormats = [..._buildVariations(dateFormat, date), ...UNAMBIGUOUS_FORMATS];
   const cleanDate = date.replace(SEPARATORS, ' ').trim();
@@ -419,3 +431,22 @@ export function dateTimeWidgetOptions(fullFormat: string, defaultTimeFormat: boo
     isCustomTimeFormat: !timeFormatOptions.includes(timeFormat),
   };
 }
+
+/**
+ * Attempts to parse a timestamp string. Returns the timestamp in seconds
+ * since epoch, or returns null on failure. Accepts only strings with 9 to 11 digits.
+ * Lowest 11 digit timestamp is 2286-11-20, so we don't consider them valid.
+ */
+export function parseTimeStamp(date: string): number | null {
+  // If this looks like a timestamp (number with 9 or more digits), just return it.
+  // This covers most of the cases leaving some time around the unix epoch not covered.
+  // So time before 100 000 000 (1974-04-26) is not covered. Also negative values
+  // are also not supported, as they overlap with the YYYYYY date format.
+  if (date && /^[1-9]\d{8,9}$/.test(date)) {
+    const parsedDate = moment(date, 'X');
+    if (parsedDate.isValid()) {
+      return parsedDate.unix();
+    }
+  }
+  return null;
+}

test/common/parseDate.ts

@@ -19,7 +19,7 @@ const month = String(today.getUTCMonth() + 1).padStart(2, '0');
  * Otherwise, parseDateStrict should return a result
  * unless no dateFormat is given in which case it may or may not.
  */
-function testParse(dateFormat: string|null, input: string, expectedDateStr: string, fallback: boolean = false) {
+function testParse(dateFormat: string|null, input: string, expectedDateStr: string|null, fallback: boolean = false) {
   assertDateEqual(parseDate(input, dateFormat ? {dateFormat} : {}), expectedDateStr);
 
   const strict = new Set<number>();
@@ -41,7 +41,7 @@ function testParse(dateFormat: string|null, input: string, expectedDateStr: stri
   }
 }
 
-function assertDateEqual(parsed: number|null, expectedDateStr: string) {
+function assertDateEqual(parsed: number|null, expectedDateStr: string|null) {
   const formatted = parsed === null ? null : new Date(parsed * 1000).toISOString().slice(0, 10);
   assert.equal(formatted, expectedDateStr);
 }
@@ -94,6 +94,7 @@ function testDateTimeStringParse(
 
 describe('parseDate', function() {
   this.timeout(5000);
+  this.slow(50);
 
   it('should allow parsing common date formats', function() {
     testParse(null, 'November 18th, 1994',  '1994-11-18');
@@ -411,6 +412,35 @@ describe('parseDate', function() {
     testParse('MM-DD-YY', `1/2/98`, `1998-01-02`);
   });
 
+  it('should parse timestamps as dates', function() {
+    testParse(null,   '123456789', '1973-11-29');
+    testParse(null,   '100000000', '1973-03-03');
+    testParse(null,  '1000000000', '2001-09-09');
+    testParse(null, '10000000000', null);
+
+    testParse(null,   '20230926', null);
+    testParse(null, '12345678', null);
+    testParse(null, '-1000000', null);
+    testParse(null, '-9999999', null);
+    testParse(null, '123456789.0', null);
+    testParse(null,  '-100000000', null);
+
+    testParse(null, '100000000000', null);
+    testParse(null, '1000000000000', null);
+
+    // Test exact times.
+    assert.equal(parseDate( '123456789'), 123456789);
+    assert.equal(parseDate( '100000000'), 100000000);
+
+    // Now those that don't fit into our format.
+    assert.isNull(parseDate('1234567'));
+    assert.isNull(parseDate('-999999'));
+    assert.isNull(parseDate('12345678.0'));
+    assert.isNull(parseDate('-100000000'));
+    assert.isNull(parseDate('100000000000'));
+    assert.isNull(parseDate('1000000000000'));
+  });
+
   describe('guessDateFormat', function() {
     it('should guess date formats', function() {
       // guessDateFormats with an *s* shows all the equally likely guesses.

test/nbrowser/gristUtils.ts

@@ -1191,7 +1191,7 @@ export async function renameTable(tableId: string, newName: string) {
 /**
  * Rename the given column.
  */
-export async function renameColumn(col: IColHeader, newName: string) {
+export async function renameColumn(col: IColHeader|string, newName: string) {
   const header = await getColumnHeader(col);
   await header.click();
   await header.click();   // Second click opens the label for editing.