Xtn.Cldr: Check CldrNames for IsKnownLanguageTag [#388]

400_xowa/src/gplx/xowa/langs/names/Xol_name_mgr.java

@@ -38,7 +38,7 @@ public class Xol_name_mgr {
 	*/
 	public String fetchLanguageName(String code, String inLanguage, String include, byte[] page_url) {
 		code = String_.Lower(code);
-		if (include == null) include = "all";
+		if (include == null) include = Scope__str__all;
 		Ordered_hash array = fetchLanguageNames(inLanguage, include, page_url);
 		Keyval rv = (Keyval)array.Get_by(code);
 		return rv == null ? "" : rv.Val_to_str_or_null();
@@ -66,8 +66,8 @@ public class Xol_name_mgr {
 			languageNameCache = Ordered_hash_.New();
 		Ordered_hash ret = (Ordered_hash)languageNameCache.Get_by(cacheKey);
 		if (ret == null) {
-			Byte_obj_val include_byte = (Byte_obj_val)fetchLanguageNamesUncachedEnum.Get_by(include_str);
+			Byte_obj_val include_byte = (Byte_obj_val)Scope__hash.Get_by(include_str);
-			byte include = include_byte == null ? fetchLanguageNamesUncached__all : include_byte.Val();
+			byte include = include_byte == null ? Scope__int__all : include_byte.Val();
 
 			Cldr_name_file cldr_file = cldr_loader.Load_or_empty(inLanguage);
 			if (cldr_file == Cldr_name_file.Empty) {
@@ -142,19 +142,28 @@ public class Xol_name_mgr {
 //				lang_files.Add(code, Keyval_.new_(code, code));
 //			}
 //
-//			hash = fetchLanguageNamesUncached(host_lang, fetchLanguageNamesUncached__mw, cldr_names, lang_names, lang_files);
+//			hash = fetchLanguageNamesUncached(host_lang, Scope__int__mw, cldr_names, lang_names, lang_files);
 //			kvs = (Keyval[])hash.To_ary(typeof(Keyval));
 //		}
 
+	/*
+	Scope__mw__str
+	Scope__hash
+	*/
 	public static final byte 
-		  fetchLanguageNamesUncached__mw      = 0 // cldr + Names.php + *.json|*.php 
+		  Scope__int__mw      = 0 // cldr + Names.php + *.json|*.php 
-		, fetchLanguageNamesUncached__all     = 1 // cldr + Names.php
+		, Scope__int__all     = 1 // cldr + Names.php
-		, fetchLanguageNamesUncached__mwFile  = 2 // *.json|*.php
+		, Scope__int__mwFile  = 2 // *.json|*.php
 		;
-	private static final    Hash_adp fetchLanguageNamesUncachedEnum = Hash_adp_.New()
+	public static final    String
-		.Add_and_more("mw"    , Byte_obj_val.new_(fetchLanguageNamesUncached__mw))
+		  Scope__str__mw      = "mw"
-		.Add_and_more("all"   , Byte_obj_val.new_(fetchLanguageNamesUncached__all))
+		, Scope__str__all     = "all"
-		.Add_and_more("mwFile", Byte_obj_val.new_(fetchLanguageNamesUncached__mwFile))
+		, Scope__str__mwFile  = "mwFile"
+		;
+	private static final    Hash_adp Scope__hash = Hash_adp_.New()
+		.Add_and_more(Scope__str__mw    , Byte_obj_val.new_(Scope__int__mw))
+		.Add_and_more(Scope__str__all   , Byte_obj_val.new_(Scope__int__all))
+		.Add_and_more(Scope__str__mwFile, Byte_obj_val.new_(Scope__int__mwFile))
 		;
 	public static Ordered_hash fetchLanguageNamesUncached
 		( String inLanguage, byte include
@@ -196,7 +205,7 @@ public class Xol_name_mgr {
 			}
 		}
 
-		if (include == fetchLanguageNamesUncached__all) {
+		if (include == Scope__int__all) {
 			names.Sort_by(Hash_kv_sorter.Instance);
 			return names;
 		}
@@ -211,7 +220,7 @@ public class Xol_name_mgr {
 
 		// REF.MW: /languages/classes/i18n/*.json
 		// REF.MW: /languages/classes/messages/Messages*.php
-		if (include == fetchLanguageNamesUncached__mwFile) {
+		if (include == Scope__int__mwFile) {
 			Ordered_hash namesMwFile = Ordered_hash_.New();
 			// # We do this using a foreach over the codes instead of a directory
 			// # loop so that messages files in extensions will work correctly.

400_xowa/src/gplx/xowa/langs/names/Xol_name_mgr_tst.java

@@ -22,7 +22,7 @@ public class Xol_name_mgr_tst {
 
 	@Test   public void Cldr_only() {
 		fxt.Test__fetchLanguageNamesUncached
-			( "en", Xol_name_mgr.fetchLanguageNamesUncached__all
+			( "en", Xol_name_mgr.Scope__int__all
 			, fxt.Make__cldr_names("en", "de")
 			, fxt.Make__lang_names()
 			, fxt.Make__lang_files()
@@ -33,7 +33,7 @@ public class Xol_name_mgr_tst {
 	}
 	@Test   public void Lang_name__langs() {
 		fxt.Test__fetchLanguageNamesUncached
-			( "en", Xol_name_mgr.fetchLanguageNamesUncached__all
+			( "en", Xol_name_mgr.Scope__int__all
 			, fxt.Make__cldr_names("en", "de", "es")
 			, fxt.Make__lang_names("en", "de", "fr")
 			, fxt.Make__lang_files()
@@ -46,7 +46,7 @@ public class Xol_name_mgr_tst {
 	}
 	@Test   public void mwFile() {
 		fxt.Test__fetchLanguageNamesUncached
-			( "en", Xol_name_mgr.fetchLanguageNamesUncached__mwFile
+			( "en", Xol_name_mgr.Scope__int__mwFile
 			, fxt.Make__cldr_names("en", "de")
 			, fxt.Make__lang_names("en", "de", "fr")
 			, fxt.Make__lang_files("en", "de", "es")
@@ -57,7 +57,7 @@ public class Xol_name_mgr_tst {
 	}
 	@Test   public void mw() {
 		fxt.Test__fetchLanguageNamesUncached
-			( "en", Xol_name_mgr.fetchLanguageNamesUncached__mw
+			( "en", Xol_name_mgr.Scope__int__mw
 			, fxt.Make__cldr_names("en", "de")
 			, fxt.Make__lang_names("en", "de", "fr")
 			, fxt.Make__lang_files("en", "de", "es")

400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_name_loader_fxt.java

@@ -0,0 +1,49 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012-2017 gnosygnu@gmail.com
+
+XOWA is licensed under the terms of the General Public License (GPL) Version 3,
+or alternatively under the terms of the Apache License Version 2.0.
+
+You may use XOWA according to either of these licenses as is most appropriate
+for your project on a case-by-case basis.
+
+The terms of each license can be found in the source code repository:
+
+GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
+Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
+*/
+package gplx.xowa.xtns.cldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
+import gplx.core.tests.*;
+import gplx.langs.jsons.*;
+public class Cldr_name_loader_fxt {
+	private static final    String Dir_name = "mem/CldrNames/";
+	private final    Cldr_name_loader name_loader = new Cldr_name_loader(Io_url_.mem_dir_(Dir_name));
+
+	public void Clear() {name_loader.Clear();}
+	public void Init__file(String fil_name, String txt) {
+		Io_mgr.Instance.SaveFilStr(Io_url_.new_fil_(Dir_name + fil_name), txt);
+	}
+	public void Test__load_file_is_empty(boolean expd, String lang_key) {
+		Cldr_name_file name_file = name_loader.Load_or_empty(lang_key);
+		Gftest.Eq__bool(expd, name_file == Cldr_name_file.Empty);
+	}
+	public static void Create_file_w_langs(String lang_name, Keyval... lang_ary) {
+		String root_dir = "mem/xowa/bin/any/xowa/xtns/cldr/CldrNames/";
+		String file = String_.Concat(root_dir, "CldrNames", lang_name, ".json");
+		
+		Json_doc_wtr wtr = new Json_doc_wtr();
+		wtr.Nde_bgn();
+		wtr.Key(Bool_.N, "languageNames");
+		wtr.Nde_bgn();
+		for (int i = 0; i < lang_ary.length; i++) {
+			Keyval lang = lang_ary[i];
+			wtr.Kv(i != 0, Bry_.new_u8(lang.Key()), Bry_.new_u8(lang.Val_to_str_or_null()));
+		}
+		wtr.Nde_end();
+		wtr.Nde_end();
+		String json = wtr.Bld_as_str();
+
+		Io_mgr.Instance.SaveFilStr(file, json);
+	}
+}

400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_name_loader_tst.java

@@ -29,16 +29,3 @@ public class Cldr_name_loader_tst {
 		fxt.Test__load_file_is_empty(Bool_.Y, "en_gb");
 	}
 }
-class Cldr_name_loader_fxt {
-	private static final    String Dir_name = "mem/CldrNames/";
-	private final    Cldr_name_loader name_loader = new Cldr_name_loader(Io_url_.mem_dir_(Dir_name));
-
-	public void Clear() {name_loader.Clear();}
-	public void Init__file(String fil_name, String txt) {
-		Io_mgr.Instance.SaveFilStr(Io_url_.new_fil_(Dir_name + fil_name), txt);
-	}
-	public void Test__load_file_is_empty(boolean expd, String lang_key) {
-		Cldr_name_file name_file = name_loader.Load_or_empty(lang_key);
-		Gftest.Eq__bool(expd, name_file == Cldr_name_file.Empty);
-	}
-}

400_xowa/src/gplx/xowa/xtns/cldrs/Language_name.java

@@ -28,4 +28,6 @@ public class Language_name implements gplx.core.brys.Bry_bfr_able {
 		bfr.Add(name).Add_byte_pipe();
 		bfr.Add(note);
 	}
+
+	public static final    Language_name[] Ary_empty = new Language_name[0];
 }

400_xowa/src/gplx/xowa/xtns/cldrs/Language_name_loader.java

@@ -43,6 +43,7 @@ public class Language_name_loader {
 		List_adp list = List_adp_.New();
 
 		Json_doc jdoc = parser.Parse(json);
+		if (jdoc == null) return Language_name.Ary_empty;
 		Json_ary root = jdoc.Root_ary();
 		int len = root.Len();
 		for (int i = 0; i < len; i++) {

400_xowa/src/gplx/xowa/xtns/cldrs/Language_names_converter_tst.java

@@ -53,12 +53,12 @@ public class Language_names_converter_tst {
 		, "]"
 		));
 	}
-	@Test   public void Convert() {
+//		@Test   public void Convert() {
-		Language_names_converter converter = new Language_names_converter();
+//			Language_names_converter converter = new Language_names_converter();
-		Language_name[] names = converter.Parse_fil(Io_url_.new_dir_("C:\\000\\100_bin\\200_server\\200_http\\100_apache\\100_v2.4\\htdocs\\mediawiki\\v1.29.1\\languages\\data\\Names.php"));
+//			Language_name[] names = converter.Parse_fil(Io_url_.new_dir_("C:\\000\\100_bin\\200_server\\200_http\\100_apache\\100_v2.4\\htdocs\\mediawiki\\v1.29.1\\languages\\data\\Names.php"));
-		String json = converter.To_json(names);
+//			String json = converter.To_json(names);
-		Io_mgr.Instance.SaveFilStr(Io_url_.new_fil_("C:\\xowa\\bin\\any\\xowa\\cfg\\lang\\data\\names.json"), json);
+//			Io_mgr.Instance.SaveFilStr(Io_url_.new_fil_("C:\\xowa\\bin\\any\\xowa\\cfg\\lang\\data\\names.json"), json);
-	}
+//		}
 }
 class Language_names_converter_fxt {
 	private final    Language_names_converter converter = new Language_names_converter();

400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_language.java

@@ -14,7 +14,7 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
 Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
 */
 package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*;
-import gplx.xowa.langs.*;
+import gplx.xowa.langs.*; import gplx.xowa.langs.names.*;
 import gplx.xowa.mediawiki.*;
 import gplx.xowa.xtns.pfuncs.times.*; import gplx.xowa.langs.numbers.*; import gplx.xowa.xtns.pfuncs.numbers.*; import gplx.xowa.langs.durations.*;
 import gplx.xowa.xtns.scribunto.procs.*;
@@ -82,14 +82,26 @@ public class Scrib_lib_language implements Scrib_lib {
 	public void Notify_lang_changed() {if (notify_lang_changed_fnc != null) core.Interpreter().CallFunction(notify_lang_changed_fnc.Id(), Keyval_.Ary_empty);}
 	public boolean GetContLangCode(Scrib_proc_args args, Scrib_proc_rslt rslt)		{return rslt.Init_obj(core.Ctx().Lang().Key_str());}
 	public boolean IsSupportedLanguage(Scrib_proc_args args, Scrib_proc_rslt rslt)	{return IsKnownLanguageTag(args, rslt);}// NOTE: checks if "MessagesXX.php" exists; note that xowa has all "MessagesXX.php"; for now, assume same functionality as IsKnownLanguageTag (worst case is that a small wiki depends on a lang not being there; will need to put in a "wiki.Langs()" then)
-	public boolean IsKnownLanguageTag(Scrib_proc_args args, Scrib_proc_rslt rslt) {	// NOTE: checks if in languages/Names.php; TODO: support foreign translations; EX: Englische is en in de.w
+	public boolean IsKnownLanguageTag(Scrib_proc_args args, Scrib_proc_rslt rslt) {	// NOTE: checks if in languages/data/Names.php; TODO: support foreign translations; EX: Englische is en in de.w
 		String lang_code = args.Cast_str_or_null(0);
 		boolean exists = false;
+
+		// TODO:create literal equivalent of Language.php|isValidBuiltInCode; WHEN:refactor
+		// * REF: https://github.com/wikimedia/mediawiki-extensions-Scribunto/blob/master/includes/engines/LuaCommon/LanguageLibrary.php#L86
+		// * REF: https://github.com/wikimedia/mediawiki/blob/master/languages/Language.php#L427
+
+		// check lang stubs which correlates roughly to languages/data/Names.php;
 		if (	lang_code != null									// null check; protecting against Module passing in nil from lua
 			&&	String_.Eq(lang_code, String_.Lower(lang_code))		// must be lower-case; REF.MW: $code === strtolower( $code )
 			&&	Xol_lang_stub_.Exists(Bry_.new_a7(lang_code))
 			)
 			exists = true;
+
+		// check cldr names; ISSUE#:388; EX:goh in CldrNamesEn.json; DATE:2019-06-11
+		if (!exists) {
+			String lang_name = core.App().Lang_mgr().Name_mgr().fetchLanguageName(lang_code, core.Lang().Key_str(), Xol_name_mgr.Scope__str__all, core.Page_url());
+			exists = String_.Len_gt_0(lang_name); // NOTE: missing langs will come back as empty String
+		}
 		return rslt.Init_obj(exists);
 	}
 	public boolean IsValidCode(Scrib_proc_args args, Scrib_proc_rslt rslt) {	// REF.MW: Language.php!isValidCode

400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_language_tst.java

@@ -17,6 +17,7 @@ package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import
 import org.junit.*;
 import gplx.langs.jsons.*;
 import gplx.xowa.langs.*; import gplx.xowa.langs.msgs.*; import gplx.xowa.langs.numbers.*;
+import gplx.xowa.xtns.cldrs.*;
 public class Scrib_lib_language_tst {
 	@Before public void init() {
 		fxt.Clear_for_lib();
@@ -34,6 +35,11 @@ public class Scrib_lib_language_tst {
 		fxt.Test_scrib_proc_bool(lib, Scrib_lib_language.Invk_isKnownLanguageTag, Object_.Ary("fr"), true);
 		fxt.Test_scrib_proc_bool(lib, Scrib_lib_language.Invk_isKnownLanguageTag, Object_.Ary("qq"), false);
 	}
+	@Test  public void IsKnownLanguageTag_cldr() {
+		Io_mgr.Instance.InitEngine_mem();
+		Cldr_name_loader_fxt.Create_file_w_langs(fxt.Core().Lang().Key_str(), Keyval_.Ary(Keyval_.new_("goh", "goh_name")));
+		fxt.Test_scrib_proc_bool(lib, Scrib_lib_language.Invk_isKnownLanguageTag, Object_.Ary("goh"), true);
+	}
 	@Test  public void IsValidCode() {
 		fxt.Test_scrib_proc_bool(lib, Scrib_lib_language.Invk_isValidCode, Object_.Ary("a,b"), true);
 		fxt.Test_scrib_proc_bool(lib, Scrib_lib_language.Invk_isValidCode, Object_.Ary("a'b"), false);