From 0d8b7e80f6e8117ea26d838fafbb2b8494807eeb Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Wed, 11 Jan 2023 12:10:16 -0500 Subject: [PATCH 1/3] tweak translation collection to be weblate-friendly This tweaks the script for collecting translation keys so that it changes the existing file minimally, and matches the formatting output by weblate. I'm not sure what is the best way to handle keys that are no longer needed. I think it may be best to delete these within weblate, but will need to experiment to see (I'm a weblate newbie). I think I saw keys reappear from weblate if not deleted there. --- buildtools/generate_translation_keys.js | 19 ++++++++++++++++--- static/locales/en.client.json | 19 +++++++++++++++++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/buildtools/generate_translation_keys.js b/buildtools/generate_translation_keys.js index 81cad4bf..3b572a0c 100644 --- a/buildtools/generate_translation_keys.js +++ b/buildtools/generate_translation_keys.js @@ -11,7 +11,6 @@ const fs = require("fs"); const path = require("path"); const Parser = require("i18next-scanner").Parser; const englishKeys = require("../static/locales/en.client.json"); -const _ = require("lodash"); const parser = new Parser({ keySeparator: "/", @@ -65,6 +64,20 @@ const getKeysFromFile = (filePath, fileName) => { return keys; }; +// It is highly desirable to retain existing order, to not generate +// unnecessary merges/conflicts, so we do a specialized merge. +function merge(target, scanned) { + for (const key of Object.keys(scanned)) { + if (!(key in target)) { + target[key] = scanned[key]; + } else if (typeof target[key] === 'object') { + merge(target[key], scanned[key]); + } else { + scanned[key] = target[key]; + } + } +} + async function walkTranslation(dirs) { for await (const p of walk(dirs)) { const { name } = path.parse(p); @@ -72,10 +85,10 @@ async function walkTranslation(dirs) { getKeysFromFile(p, name); } const keys = parser.get({ sort: true }); - const newTranslations = _.merge(keys.en.translation, englishKeys); + merge(englishKeys, sort(keys.en.translation)); await fs.promises.writeFile( "static/locales/en.client.json", - JSON.stringify(sort(newTranslations), null, 2), + JSON.stringify(englishKeys, null, 4) + '\n', // match weblate's default "utf-8" ); return keys; diff --git a/static/locales/en.client.json b/static/locales/en.client.json index 4f97b652..c7e8bf20 100644 --- a/static/locales/en.client.json +++ b/static/locales/en.client.json @@ -36,7 +36,9 @@ "Special Rules": "Special Rules", "Type a message...": "Type a message...", "User Attributes": "User Attributes", - "View As": "View As" + "View As": "View As", + "Seed rules": "Seed rules", + "When adding table rules, automatically add a rule to grant OWNER full access.": "When adding table rules, automatically add a rule to grant OWNER full access." }, "AccountPage": { "API": "API", @@ -332,7 +334,8 @@ "Add Column": "Add Column" }, "FilterBar": { - "SearchColumns": "Search columns" + "SearchColumns": "Search columns", + "Search Columns": "Search Columns" }, "GridOptions": { "Grid Options": "Grid Options", @@ -761,5 +764,17 @@ "NTextBox": { "false": "false", "true": "true" + }, + "ACLUsers": { + "Example Users": "Example Users", + "Users from table": "Users from table", + "View As": "View As" + }, + "TypeTransform": { + "Apply": "Apply", + "Cancel": "Cancel", + "Preview": "Preview", + "Revise": "Revise", + "Update formula (Shift+Enter)": "Update formula (Shift+Enter)" } } From b8e964564cf76ee862d3155ddff133fbd1b1b5d6 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Thu, 12 Jan 2023 12:54:14 -0500 Subject: [PATCH 2/3] report unrecognized keys; be careful to not clobber English translation of single/plural forms --- buildtools/generate_translation_keys.js | 41 ++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/buildtools/generate_translation_keys.js b/buildtools/generate_translation_keys.js index 3b572a0c..1c013c54 100644 --- a/buildtools/generate_translation_keys.js +++ b/buildtools/generate_translation_keys.js @@ -11,6 +11,7 @@ const fs = require("fs"); const path = require("path"); const Parser = require("i18next-scanner").Parser; const englishKeys = require("../static/locales/en.client.json"); +const _ = require("lodash"); const parser = new Parser({ keySeparator: "/", @@ -67,31 +68,63 @@ const getKeysFromFile = (filePath, fileName) => { // It is highly desirable to retain existing order, to not generate // unnecessary merges/conflicts, so we do a specialized merge. function merge(target, scanned) { + let merges = 0; for (const key of Object.keys(scanned)) { if (!(key in target)) { + console.log("Merging key", {key}); target[key] = scanned[key]; + merges++; } else if (typeof target[key] === 'object') { - merge(target[key], scanned[key]); + merges += merge(target[key], scanned[key]); + } else if (scanned[key] !== target[key]) { + if (!key.endsWith('_one')) { + console.log("Value difference", {key, value: target[key]}); + } + } + } + return merges; +} + +// Look for keys that are listed in json file but not found in source +// code. These may be stale and need deleting in weblate. +function reportUnrecognizedKeys(originalKeys, foundKeys) { + let unknowns = 0; + for (const section of Object.keys(originalKeys)) { + if (!(section in foundKeys)) { + console.log("Unknown section found", {section}); + unknowns++; } else { - scanned[key] = target[key]; + for (const key of Object.keys(originalKeys[section])) { + if (!(key in foundKeys[section])) { + console.log("Unknown key found", {section, key}); + unknowns++; + } + } } } + return unknowns; } async function walkTranslation(dirs) { + const originalKeys = _.cloneDeep(englishKeys); for await (const p of walk(dirs)) { const { name } = path.parse(p); if (p.endsWith('.map')) { continue; } getKeysFromFile(p, name); } const keys = parser.get({ sort: true }); - merge(englishKeys, sort(keys.en.translation)); + const foundKeys = _.cloneDeep(keys.en.translation); + const mergeCount = merge(englishKeys, sort(keys.en.translation)); await fs.promises.writeFile( "static/locales/en.client.json", JSON.stringify(englishKeys, null, 4) + '\n', // match weblate's default "utf-8" ); - return keys; + // Now, print a report of unrecognized keys - candidates + // for deletion in weblate. + const unknownCount = reportUnrecognizedKeys(originalKeys, foundKeys); + console.log(`Found ${unknownCount} unknown key(s).`); + console.log(`Make ${mergeCount} merge(s).`); } walkTranslation(["_build/app/client", ...process.argv.slice(2)]); From cf096889db7839ec0d35711613158556d16afd0c Mon Sep 17 00:00:00 2001 From: CamilleLegeron Date: Fri, 13 Jan 2023 05:13:26 +0100 Subject: [PATCH 3/3] feat(TypeConversion): allow converting list of string with quotes to ChoiceList (#397) --- app/client/components/TypeConversion.ts | 3 ++- test/nbrowser/ChoiceList.ts | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/app/client/components/TypeConversion.ts b/app/client/components/TypeConversion.ts index 0fbfd399..cb97fa32 100644 --- a/app/client/components/TypeConversion.ts +++ b/app/client/components/TypeConversion.ts @@ -7,6 +7,7 @@ import {isString} from 'app/client/lib/sessionObs'; import {DocModel} from 'app/client/models/DocModel'; import {ColumnRec} from 'app/client/models/entities/ColumnRec'; +import {csvDecodeRow} from 'app/common/csvFormat'; import * as gristTypes from 'app/common/gristTypes'; import {isFullReferencingType} from 'app/common/gristTypes'; import * as gutil from 'app/common/gutil'; @@ -175,7 +176,7 @@ export async function prepTransformColInfo(docModel: DocModel, origCol: ColumnRe for (let value of tableData.getColValues(sourceCol.colId()) || []) { if (value === null) { continue; } value = String(decodeObject(value)).trim(); - const tags: unknown[] = (value.startsWith('[') && gutil.safeJsonParse(value, null)) || value.split(","); + const tags: unknown[] = (value.startsWith('[') && gutil.safeJsonParse(value, null)) || csvDecodeRow(value); for (const tag of tags) { choices.add(String(tag).trim()); if (choices.size > 100) { break; } // Don't suggest excessively many choices. diff --git a/test/nbrowser/ChoiceList.ts b/test/nbrowser/ChoiceList.ts index d528ecb9..901bcec5 100644 --- a/test/nbrowser/ChoiceList.ts +++ b/test/nbrowser/ChoiceList.ts @@ -447,7 +447,7 @@ describe('ChoiceList', function() { it('should allow reasonable conversions between ChoiceList and other types', async function() { await gu.enterGridRows({rowNum: 1, col: 'A'}, - [['Hello'], ['World'], [' Foo,Bar;Baz!,']]); + [['Hello'], ['World'], [' Foo,Bar;Baz!,"Qux, quux corge", "80\'s",']]); await testTextChoiceListConversions(); }); @@ -468,17 +468,17 @@ describe('ChoiceList', function() { // Check that choices got populated. await driver.find('.test-right-tab-field').click(); - assert.deepEqual(await getChoiceLabels(), ['Hello', 'World', 'Foo', 'Bar;Baz!']); + assert.deepEqual(await getChoiceLabels(), ['Hello', 'World', 'Foo', 'Bar;Baz!', 'Qux, quux corge', '80\'s']); assert.deepEqual( await getChoiceColors(), - [UNSET_FILL, UNSET_FILL, UNSET_FILL, UNSET_FILL] + [UNSET_FILL, UNSET_FILL, UNSET_FILL, UNSET_FILL, UNSET_FILL, UNSET_FILL] ); // Check that the result contains the right tags. assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3], cols: ['A']}), [ 'Hello', 'World', - 'Foo\nBar;Baz!' + 'Foo\nBar;Baz!\nQux, quux corge\n80\'s' ]); await gu.checkForErrors(); @@ -494,17 +494,21 @@ describe('ChoiceList', function() { [ {fillColor: DEFAULT_FILL, textColor: DEFAULT_TEXT, ...VALID_CHOICE}, {fillColor: DEFAULT_FILL, textColor: DEFAULT_TEXT, ...VALID_CHOICE}, + {fillColor: DEFAULT_FILL, textColor: DEFAULT_TEXT, ...VALID_CHOICE}, + {fillColor: DEFAULT_FILL, textColor: DEFAULT_TEXT, ...VALID_CHOICE}, ] ); // Open a cell to see the actual tags. await gu.getCell({rowNum: 3, col: 'A'}).click(); await driver.sendKeys(Key.ENTER); - assert.deepEqual(await getEditorTokens(), ['Foo', 'Bar;Baz!']); - assert.deepEqual(await getEditorTokensIsInvalid(), [false, false]); + assert.deepEqual(await getEditorTokens(), ['Foo', 'Bar;Baz!', 'Qux, quux corge', '80\'s']); + assert.deepEqual(await getEditorTokensIsInvalid(), [ false, false, false, false ]); assert.deepEqual( await getEditorTokenStyles(), [ + {fillColor: DEFAULT_FILL, textColor: DEFAULT_TEXT, ...VALID_CHOICE}, + {fillColor: DEFAULT_FILL, textColor: DEFAULT_TEXT, ...VALID_CHOICE}, {fillColor: DEFAULT_FILL, textColor: DEFAULT_TEXT, ...VALID_CHOICE}, {fillColor: DEFAULT_FILL, textColor: DEFAULT_TEXT, ...VALID_CHOICE} ] @@ -520,7 +524,7 @@ describe('ChoiceList', function() { assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3], cols: ['A']}), [ 'Hello', 'World', - 'Foo, Bar;Baz!, hooray' + 'Foo, Bar;Baz!, "Qux, quux corge", 80\'s, hooray' ]); // Undo the cell change and both conversions (back to ChoiceList, back to Text), and check @@ -529,7 +533,7 @@ describe('ChoiceList', function() { assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3], cols: ['A']}), [ 'Hello', 'World', - ' Foo,Bar;Baz!,', // That's the text originally entered into this Text cell. + ' Foo,Bar;Baz!,"Qux, quux corge", "80\'s",', // That's the text originally entered into this Text cell. ]); }