JsonConfig: Add Localize function for tabular data

400_xowa/src/gplx/xowa/xtns/gallery/Gallery_mgr_wtr.java

@@ -175,7 +175,7 @@ public class Gallery_mgr_wtr {
 			int len = xatr_list.Count();
 			for (int i = 0; i < len; i++) {
 				Mwh_atr_itm xatr = (Mwh_atr_itm)xatr_list.Get_at(i);
-				if (!whitelist_mgr.Chk(Xop_xnde_tag_.Tid__ul, src, xatr)) continue;
+				if (!whitelist_mgr.Chk(Xop_xnde_tag_.Tid__ul, xatr)) continue;
 				byte[] key = xatr.Key_bry();
 				byte[] val = xatr.Val_as_bry();
 				Gfh_wtr.Write_atr_bry(bfr, key, val);

400_xowa/src/gplx/xowa/xtns/jsonConfigs/scribunto/Jscfg_localizer.java

@@ -0,0 +1,151 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012-2017 gnosygnu@gmail.com
+
+XOWA is licensed under the terms of the General Public License (GPL) Version 3,
+or alternatively under the terms of the Apache License Version 2.0.
+
+You may use XOWA according to either of these licenses as is most appropriate
+for your project on a case-by-case basis.
+
+The terms of each license can be found in the source code repository:
+
+GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
+Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
+*/
+package gplx.xowa.xtns.jsonConfigs.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.jsonConfigs.*;
+import gplx.xowa.langs.*;
+class Jscfg_localizer {
+	public Keyval[] Localize(Xol_lang_itm lang, byte[] page, Keyval[] root) {
+		if (lang == null) return root; // if no lang, return original
+
+		int len = root.length;
+		for (int i = 0; i < len; i++) {
+			Keyval nde = root[i];
+			String nde_key = nde.Key();
+			if      (String_.Eq(nde_key, Id__root__license)) {
+			}
+			else if (String_.Eq(nde_key, Id__root__description)) {
+				root[i] = pickLocalizedString(lang, page, nde);
+			}
+			else if (String_.Eq(nde_key, Id__root__schema)) {
+				nde = Localize_schema(lang, page, nde);
+			}
+			else if (String_.Eq(nde_key, Id__root__data)) {
+				nde = Localize_data(lang, page, nde);
+			}
+		}
+
+		return root;
+	}
+	private Keyval Localize_schema(Xol_lang_itm lang, byte[] page, Keyval schema) {
+		Keyval[] schemas = Cast_to_kvary_or_null(page, schema) ; if (schemas == null) return schema;
+		Keyval[] fields  = Cast_to_kvary_or_null(page, schemas[0]); if (fields == null) return schema;
+		for (Keyval field : fields) {
+			Keyval[] atrs = (Keyval[])field.Val();
+			int atrs_len = atrs.length;
+                for (int i = 0; i < atrs_len; i++) {
+				Keyval atr = atrs[i];
+				if (String_.Eq(atr.Key(), Id__fld__title)) {
+					atrs[i] = pickLocalizedString(lang, page, atr);
+				}
+			}
+		}
+		return schema;
+	}
+	private Keyval Localize_data(Xol_lang_itm lang, byte[] page, Keyval data) {
+		Keyval[] rows = Cast_to_kvary_or_null(page, data); if (rows == null) return data;
+		for (Keyval row : rows) {
+			Object[] vals = (Object[])row.Val();
+			int len = vals.length;
+                for (int i = 0; i < len; i++) {
+				Object val = vals[i];
+				if (Type_.Eq_by_obj(val, Keyval[].class)) {
+					Keyval val_as_kv = pickLocalizedString(lang, Int_.To_str(i), (Keyval[])val);
+					vals[i] = val_as_kv.Val();
+				}
+			}
+		}
+		return data;
+	}
+	private static Keyval pickLocalizedString(Xol_lang_itm lang, byte[] page, Keyval kv) {
+		Keyval[] kvs = Cast_to_kvary_or_null(page, kv.Key(), kv.Val());
+		Keyval rv = pickLocalizedString(lang, kv.Key(), kvs);
+		return rv == null ? kv : rv;
+	}
+	public static Keyval pickLocalizedString(Xol_lang_itm lang, String key, Keyval[] ary) {
+		// local vars for conditional logic
+		Object val_lang = null, val_en = null, val_1st = null;
+		Object[] val_fallbacks = null;
+
+		// local vars for lang
+		String langCode = lang.Key_str();
+		Ordered_hash fallback_hash = lang.Fallback_hash();
+
+		// loop ary to populate local vars
+		for (Keyval itm : ary) {
+			String itm_key = itm.Key();
+			Object itm_val = itm.Val();
+			if (val_1st == null) {
+				val_1st = itm_val;
+			}
+			if (String_.Eq(itm_key, langCode)) {
+				val_lang = itm_val;
+			}
+			else if (fallback_hash.Has(itm_key)) {
+				if (val_fallbacks == null) {
+					val_fallbacks = new Object[fallback_hash.Len()];
+				}
+				int idx = fallback_hash.Idx_of(itm_key);
+				val_fallbacks[idx] = itm_val;
+			}
+			else if (String_.Eq(itm_key, "en")) {
+				val_en = itm_val;
+			}
+
+		}
+
+		if (val_lang != null) {
+			return Keyval_.new_(key, val_lang);
+		}
+
+		if (val_fallbacks != null) {
+			for (Object v : val_fallbacks) {
+				if (v != null)
+					return Keyval_.new_(key, v);
+			}
+		}
+
+		// If fallbacks fail, check if english is defined
+		if (val_en != null) {
+			return Keyval_.new_(key, val_en);
+		}
+
+		// We have a custom default, return that
+		// if (defaultValue != null) {
+		// 	return null;
+		// }
+
+		// Return first available value, or an empty String
+		// There might be a better way to get the first value from an Object
+		Object val = val_1st == null ? "" : val_1st;
+		return Keyval_.new_(key, val);
+	}
+	private static Keyval[] Cast_to_kvary_or_null(byte[] page, Keyval kv) {return Cast_to_kvary_or_null(page, kv.Key(), kv.Val());}
+	private static Keyval[] Cast_to_kvary_or_null(byte[] page, String key, Object val) {
+		if (Type_.Eq_by_obj(val, Keyval[].class)) {
+			return (Keyval[])val;
+		}
+		else {
+			Gfo_usr_dlg_.Instance.Warn_many("", "", "could not cast to kvary; page=~{0} key=~{1}", key);
+			return null;
+		}
+	} 
+	private static final String
+	  Id__root__schema        = "schema"
+	, Id__root__data          = "data"
+	, Id__root__description   = "description"
+	, Id__root__license       = "license"
+	, Id__fld__title          = "title"
+	;
+}

400_xowa/src/gplx/xowa/xtns/jsonConfigs/scribunto/Jscfg_scrib_lib.java

@@ -18,6 +18,7 @@ import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.scribunto.libs.*; impor
 import gplx.xowa.wikis.domains.*;
 public class Jscfg_scrib_lib implements Scrib_lib {
 	private final    Scrib_lib_text__json_util json_util = new Scrib_lib_text__json_util();
+	private final    Jscfg_localizer localizer = new Jscfg_localizer();
 	private Scrib_core core;
 	public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod;
 	public Scrib_lib Init() {procs.Init_by_lib(this, Proc_names); return this;}
@@ -57,6 +58,8 @@ public class Jscfg_scrib_lib implements Scrib_lib {
 			throw Err_.new_wo_type("bad argument #1 to 'get' (not a valid title) " + String_.new_u8(ttl_bry));
 		}
 
-		return Scrib_lib_text.JsonDecodeStatic(args, rslt, core, json_util, page, Scrib_lib_text__json_util.Opt__force_assoc, Scrib_lib_text__json_util.Flag__none);
+		Keyval[] rv = Scrib_lib_text.JsonDecodeStatic(args, core, json_util, page, Scrib_lib_text__json_util.Opt__force_assoc, Scrib_lib_text__json_util.Flag__none);
+		rv = localizer.Localize(core.Wiki().Lang(), page, rv);
+		return rslt.Init_obj(rv);
 	}
 }

400_xowa/src/gplx/xowa/xtns/jsonConfigs/scribunto/Jscfg_scrib_lib_tst.java

@@ -14,18 +14,14 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
 Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
 */
 package gplx.xowa.xtns.jsonConfigs.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.jsonConfigs.*;
-import org.junit.*;
+import org.junit.*; import gplx.core.tests.*;
 import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.scribunto.libs.*;
+import gplx.xowa.langs.*;
+import gplx.langs.jsons.*;
 public class Jscfg_scrib_lib_tst {
-	@Before public void init() {
-		fxt.Clear_for_lib();
-		lib = new Jscfg_scrib_lib();
-		lib.Init();
-		lib.Core_(fxt.Core());
-	}	private Scrib_invoke_func_fxt fxt = new Scrib_invoke_func_fxt(); private Jscfg_scrib_lib lib;
+	private final    Jscfg_scrib_lib_fxt fxt = new Jscfg_scrib_lib_fxt();
 	@Test   public void Get() {
-		Xowe_wiki commons_wiki = fxt.Parser_fxt().Wiki().Appe().Wiki_mgr().Get_by_or_make(gplx.xowa.wikis.domains.Xow_domain_itm_.Bry__commons).Init_assert();
-		fxt.Parser_fxt().Init_page_create(commons_wiki, "Data:Test.tab", gplx.langs.jsons.Json_doc.Make_str_by_apos
+		fxt.Init__page("Data:Test.tab", Json_doc.Make_str_by_apos
 		( "{"
 		, "  'data':"
 		, "  ["
@@ -41,7 +37,7 @@ public class Jscfg_scrib_lib_tst {
 		, "  ]"
 		, "}"
 		));
-		fxt.Test_scrib_proc_str_ary(lib, Jscfg_scrib_lib.Invk_get, Keyval_.Ary(Keyval_.int_(1, "Test.tab")), String_.Concat_lines_nl_skip_last
+		fxt.Test__get("Test.tab", String_.Concat_lines_nl_skip_last
 		( "1="
 		, "  data="
 		, "    1="
@@ -52,4 +48,126 @@ public class Jscfg_scrib_lib_tst {
 		, "      2=Data:Q2"
 		));
 	}
+	@Test   public void Get_localize() {
+		fxt.Init__page("Data:Test_localize.tab", Json_doc.Make_str_by_apos
+		( "{"
+		, "    'license': 'CC0-1.0',"
+		, "    'description': {"
+		, "        'de': 'Objekttabelle',"
+		, "        'en': 'Object table'"
+		, "    },"
+		, "    'sources': 'Objects in Data:Data.tab completed by [https://www.wikidata.org Wikidata]',"
+		, "    'schema': {"
+		, "        'fields': ["
+		, "            {"
+		, "                'name': 'wikidataID',"
+		, "                'type': 'String',"
+		, "                'title': {"
+		, "                    'de': 'Wikidata-Item',"
+		, "                    'en': 'Wikidata item'"
+		, "                }"
+		, "            },"
+		, "            {"
+		, "                'name': 'wikidataLabel',"
+		, "                'type': 'localized',"
+		, "                'title': {"
+		, "                    'de': 'Wikidata-Label',"
+		, "                    'en': 'Wikidata label'"
+		, "                }"
+		, "            }"
+		, "        ]"
+		, "    },"
+		, "    'data': ["
+		, "        ["
+		, "            'Q183',"
+		, "            {"
+		, "                'de': 'Deutschland',"
+		, "                'en': 'Germany'"
+		, "            }"
+		, "        ],"
+		, "        ["
+		, "            'Q61912',"
+		, "            {"
+		, "                'de': 'Wertheim',"
+		, "                'en': 'Wertheim am Main'"
+		, "            }"
+		, "        ]"
+		, "    ]"
+		, "}"
+		));
+		fxt.Test__get( "Test_localize.tab", String_.Concat_lines_nl_skip_last
+		( "1="
+		, "  license=CC0-1.0"
+		, "  description=Object table"
+		, "  sources=Objects in Data:Data.tab completed by [https://www.wikidata.org Wikidata]"
+		, "  schema="
+		, "    fields="
+		, "      1="
+		, "        name=wikidataID"
+		, "        type=String"
+		, "        title=Wikidata item"
+		, "      2="
+		, "        name=wikidataLabel"
+		, "        type=localized"
+		, "        title=Wikidata label"
+		, "  data="
+		, "    1="
+		, "      1=Q183"
+		, "      2=Germany"
+		, "    2="
+		, "      1=Q61912"
+		, "      2=Wertheim am Main"
+		));
+	}
+	@Test   public void pickLocalizedString() {
+		Xol_lang_itm lang = fxt.Init__lang("zh-cn", "zh1,zh0");
+
+		// match key
+		fxt.Test__pickLocalizedString(lang, fxt.Init__picklocalizedStringKvs("fr", "zh-cn"), "zh-cn");
+
+		// match fallback; note that zh1 is higher in fallback list, but lower in kvs
+		fxt.Test__pickLocalizedString(lang, fxt.Init__picklocalizedStringKvs("zh0", "zh1"), "zh1");
+
+		// match en if no key or fallbacks
+		fxt.Test__pickLocalizedString(lang, fxt.Init__picklocalizedStringKvs("fr", "en"), "en");
+
+		// pick 1st if no match
+		fxt.Test__pickLocalizedString(lang, fxt.Init__picklocalizedStringKvs("fr", "de"), "fr");
+	}
 }	
+class Jscfg_scrib_lib_fxt {
+	private final    Scrib_invoke_func_fxt fxt = new Scrib_invoke_func_fxt();
+	private final    Jscfg_scrib_lib lib;
+	private final    Xowe_wiki commons_wiki;
+	public Jscfg_scrib_lib_fxt() {
+		fxt.Clear_for_lib();
+		lib = new Jscfg_scrib_lib();
+		lib.Init();
+		lib.Core_(fxt.Core());
+		this.commons_wiki = fxt.Parser_fxt().Wiki().Appe().Wiki_mgr().Get_by_or_make(gplx.xowa.wikis.domains.Xow_domain_itm_.Bry__commons).Init_assert();
+	}
+	public void Init__page(String page, String text) {
+		fxt.Parser_fxt().Init_page_create(commons_wiki, page, text);
+	}
+	public Xol_lang_itm Init__lang(String key, String fallbacks) {
+		Xol_lang_itm lang = new Xol_lang_itm(fxt.Core().App().Lang_mgr(), Bry_.new_u8(key));
+		lang.Fallback_bry_(Bry_.new_a7(fallbacks));
+		return lang;
+	}
+	public Keyval[] Init__picklocalizedStringKvs(String... vals) {
+		int len = vals.length;
+		Keyval[] rv = new Keyval[len];
+		for (int i = 0; i < len; i++) {
+			String val = vals[i];
+			rv[i] = Keyval_.new_(val, val);
+		}
+		return rv;
+	}
+	public void Test__get(String page, String expd) {
+		fxt.Test_scrib_proc_str_ary(lib, Jscfg_scrib_lib.Invk_get, Keyval_.Ary(Keyval_.int_(1, page)), expd);
+	}
+	public void Test__pickLocalizedString(Xol_lang_itm lang, Keyval[] kv_ary, String expd) {
+		Keyval actl_kv = Jscfg_localizer.pickLocalizedString(lang, "key", kv_ary);
+		Gftest.Eq__obj_or_null(expd, actl_kv.Val());
+	}
+}

400_xowa/src/gplx/xowa/xtns/jsonConfigs/scribunto/Jscfg_xtn_mgr.java

@@ -15,11 +15,23 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
 */
 package gplx.xowa.xtns.jsonConfigs.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.jsonConfigs.*;
 import gplx.xowa.xtns.scribunto.*;
+import gplx.xowa.mediawiki.*;
 public class Jscfg_xtn_mgr extends Xox_mgr_base {
 	@Override public byte[] Xtn_key() {return XTN_KEY;} public static final    byte[] XTN_KEY = Bry_.new_a7("JsonConfig");
 	@Override public void Xtn_init_by_wiki(Xowe_wiki wiki) {
 		Scrib_xtn_mgr scrib_xtn = (Scrib_xtn_mgr)wiki.Xtn_mgr().Get_or_fail(Scrib_xtn_mgr.XTN_KEY);
 		scrib_xtn.Lib_mgr().Add(new Jscfg_scrib_lib());
 	}
+	/*
+	@Override public void Xtn_ctor_by_app(Xoae_app app) {
+		Init_xtn();
+	}
+	public void Init_xtn() {
+		JCSingleton singleton = new JCSingleton();
+		singleton.ConfigModels().Add(JCTabularContent.Model_id, JCTabularContent.Model_id);
+		XophpEnv.Instance.Singletons().Add(JCSingleton.Singleton_Id, singleton);
+		XophpEnv.Instance.ClassBldrs().Add(JCTabularContent.Model_id, new JCTabularContentFactory());
+	}
+	*/
 	@Override public Xox_mgr Xtn_clone_new() {return new Jscfg_xtn_mgr();}
 }

400_xowa/src/gplx/xowa/xtns/math/Xomath_html_wtr.java

@@ -67,7 +67,7 @@ class Xomath_html_wtr {
 		}
 
 		// write html: <span>math_expr</math>
-		byte[] unique_bry = wiki.Parser_mgr().Uniq_mgr().Add(Bry__math, math_bry);
+		byte[] unique_bry = wiki.Parser_mgr().Uniq_mgr().Add(Bool_.Y, Bry__math, math_bry);
 		Bry_fmt fmt = is_latex ? fmt__latex : fmt__mathjax;
 		fmt.Bld_many(tmp_bfr, uid, unique_bry);
 		bfr.Add_bfr_and_clear(tmp_bfr);

400_xowa/src/gplx/xowa/xtns/pfuncs/strings/Pfunc_tag.java

@@ -52,7 +52,7 @@ public class Pfunc_tag extends Pf_func_base {// REF:CoreParserFunctions.php
 
 			// add to UNIQ hash; DATE:2017-03-31
 			byte[] val = tmp_bfr.To_bry_and_clear();
-			byte[] key = ctx.Wiki().Parser_mgr().Uniq_mgr().Add(tag_name, val);
+			byte[] key = ctx.Wiki().Parser_mgr().Uniq_mgr().Add(Bool_.Y, tag_name, val);
 			bfr.Add(key);
 		}
 		finally {tmp_bfr.Mkr_rls();}

400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_text.java

@@ -14,12 +14,18 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
 Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
 */
 package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*;
-import gplx.core.bits.*;
+import gplx.core.bits.*; import gplx.core.btries.*;
 import gplx.xowa.langs.msgs.*;
 import gplx.xowa.xtns.scribunto.procs.*;
 public class Scrib_lib_text implements Scrib_lib {
 	private final    Scrib_lib_text__json_util json_util = new Scrib_lib_text__json_util();
-	public Scrib_lib_text(Scrib_core core) {this.core = core;} private Scrib_core core;
+	private final    Scrib_lib_text__nowiki_util nowiki_util = new Scrib_lib_text__nowiki_util();
+	private final    Scrib_core core;
+	private final    Btrie_slim_mgr trie;
+	public Scrib_lib_text(Scrib_core core) {
+		this.core = core;
+		this.trie = nowiki_util.Make_trie(gplx.xowa.parsers.xndes.Xop_xnde_tag_.Tag__nowiki.Name_bry());
+	}
 	public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod;
 	public Scrib_lib Init() {procs.Init_by_lib(this, Proc_names); return this;}
 	public Scrib_lib Clone_lib(Scrib_core core) {return new Scrib_lib_text(core);}
@@ -47,7 +53,11 @@ public class Scrib_lib_text implements Scrib_lib {
 	, Invk_init_text_for_wiki = "init_text_for_wiki", Invk_jsonEncode = "jsonEncode", Invk_jsonDecode = "jsonDecode";
 	private static final    String[] Proc_names = String_.Ary(Invk_unstrip, Invk_unstripNoWiki, Invk_killMarkers, Invk_getEntityTable, Invk_init_text_for_wiki, Invk_jsonEncode, Invk_jsonDecode);
 	public boolean Unstrip(Scrib_proc_args args, Scrib_proc_rslt rslt)			{return rslt.Init_obj(args.Pull_str(0));}	// NOTE: XOWA does not use MediaWiki strip markers; just return original; DATE:2015-01-20
-	public boolean UnstripNoWiki(Scrib_proc_args args, Scrib_proc_rslt rslt)	{return rslt.Init_obj(args.Pull_str(0));}	// NOTE: XOWA does not use MediaWiki strip markers; just return original; DATE:2015-01-20
+	public boolean UnstripNoWiki(Scrib_proc_args args, Scrib_proc_rslt rslt)	{
+		// NOTE: XOWA does not use MediaWiki strip markers; just return original; DATE:2015-01-20
+		byte[] src = args.Pull_bry(0);
+		return rslt.Init_obj(nowiki_util.Strip_tag(core.Page().Url_bry_safe(), src, trie));
+	}
 	public boolean KillMarkers(Scrib_proc_args args, Scrib_proc_rslt rslt)		{return rslt.Init_obj(args.Pull_str(0));}	// NOTE: XOWA does not use MediaWiki strip markers; just return original; DATE:2015-01-20
 	public boolean GetEntityTable(Scrib_proc_args args, Scrib_proc_rslt rslt) {
 		if (html_entities == null) html_entities = Scrib_lib_text_html_entities.new_();
@@ -108,10 +118,11 @@ public class Scrib_lib_text implements Scrib_lib {
 		if (Bitmask_.Has_int(flags, Scrib_lib_text__json_util.Flag__try_fixing))
 			opts = Bitmask_.Add_int(opts, Scrib_lib_text__json_util.Flag__try_fixing);
 
-		return JsonDecodeStatic(args, rslt, core, json_util, json, opts, flags);
+		Keyval[] rv = JsonDecodeStatic(args, core, json_util, json, opts, flags);
+		return rslt.Init_obj(rv);
 	}
-	public static boolean JsonDecodeStatic
-		( Scrib_proc_args args, Scrib_proc_rslt rslt, Scrib_core core, Scrib_lib_text__json_util json_util
+	public static Keyval[] JsonDecodeStatic
+		( Scrib_proc_args args, Scrib_core core, Scrib_lib_text__json_util json_util
 		, byte[] json, int opts, int flags) {
 		// decode json to Object; note that Bool_.Y means ary and Bool_.N means ary
 		byte rv_tid = json_util.Decode(core.App().Utl__json_parser(), json, opts);
@@ -125,10 +136,10 @@ public class Scrib_lib_text implements Scrib_lib {
 				json_util.Reindex_arrays(reindex_data, rv_as_kvy, false);
 				rv_as_kvy = reindex_data.Rv_is_kvy() ? (Keyval[])reindex_data.Rv_as_kvy() : (Keyval[])reindex_data.Rv_as_ary();
 			}					
-			return rslt.Init_obj(rv_as_kvy);
+			return rv_as_kvy;
 		}
 		else
-			return rslt.Init_obj(json_util.Decode_rslt_as_ary());
+			return json_util.Decode_rslt_as_ary();
 	}
 
 	public void Notify_wiki_changed() {if (notify_wiki_changed_fnc != null) core.Interpreter().CallFunction(notify_wiki_changed_fnc.Id(), Keyval_.Ary_empty);}

400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_text__nowiki_util.java

@@ -0,0 +1,90 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012-2017 gnosygnu@gmail.com
+
+XOWA is licensed under the terms of the General Public License (GPL) Version 3,
+or alternatively under the terms of the Apache License Version 2.0.
+
+You may use XOWA according to either of these licenses as is most appropriate
+for your project on a case-by-case basis.
+
+The terms of each license can be found in the source code repository:
+
+GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
+Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
+*/
+package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*;
+import gplx.core.btries.*;
+import gplx.core.primitives.*;
+public class Scrib_lib_text__nowiki_util {
+	public Btrie_slim_mgr Make_trie(byte[] tag) {
+		Bry_bfr tmp = Bry_bfr_.New();
+		Btrie_slim_mgr rv = Btrie_slim_mgr.ci_u8();
+		byte[] lhs_bry = tmp.Add_bry_many(Byte_ascii.Angle_bgn_bry, tag, Byte_ascii.Angle_end_bry).To_bry_and_clear();
+		byte[] rhs_bry = tmp.Add_bry_many(Byte_ascii.Angle_bgn_bry, Byte_ascii.Slash_bry, tag, Byte_ascii.Angle_end_bry).To_bry_and_clear();
+		rv.Add_obj(lhs_bry, Bool_obj_val.True);
+		rv.Add_obj(rhs_bry, Bool_obj_val.False);
+		return rv;
+	}
+	public byte[] Strip_tag(byte[] page, byte[] src, Btrie_slim_mgr trie) {
+		Btrie_rv trv = new Btrie_rv();
+		Bry_bfr tmp = null;
+		int bgn = 0;
+		int end = src.length;
+
+		// main loop
+		boolean lhs_found = false;
+		int pos = bgn;
+		int rhs_end = pos, lhs_bgn = pos, lhs_end = pos;
+		while (pos < end) {
+			// check byte against trie
+			Object o = trie.Match_at_w_b0(trv, src[pos], src, pos, end);
+
+			// no match; increment and continue;
+			if (o == null) {
+				pos++;
+				continue;
+			}
+
+			// match found
+			Bool_obj_val tag_marker = (Bool_obj_val)o;
+
+			// match is open tag; EX: <tag>
+			if (tag_marker.Val()) {
+				// set lhs_bgn and lhs_end; note that if there are multiple open tags, it will only keep the first
+				if (!lhs_found) {
+					lhs_found = true;
+					lhs_bgn = pos;
+					lhs_end = trv.Pos();
+				}
+			}
+			// match is close tag; EX: </tag>
+			else {
+				// only splice if open tag exists; avoids dangling rhs; EX: "a</tag>b"
+				if (lhs_found) {
+					lhs_found = false;
+					if (tmp == null) tmp = Bry_bfr_.New();
+
+					// add text from previous </tag> to current <tag>;
+					tmp.Add_mid(src, rhs_end, lhs_bgn);
+
+					// add text between <tag> and </tag>;
+					tmp.Add_mid(src, lhs_end, pos);
+
+					// update </tag> pos
+					rhs_end = trv.Pos();
+				}
+			}
+
+			// update pos to after match
+			pos = trv.Pos();
+		}
+
+		// add remaining text to bfr
+		if (tmp != null) {
+			tmp.Add_mid(src, rhs_end, end);
+		}
+
+		return tmp == null ? src : tmp.To_bry_and_clear();
+	}
+}

400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_text__nowiki_util_tst.java

@@ -0,0 +1,55 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012-2017 gnosygnu@gmail.com
+
+XOWA is licensed under the terms of the General Public License (GPL) Version 3,
+or alternatively under the terms of the Apache License Version 2.0.
+
+You may use XOWA according to either of these licenses as is most appropriate
+for your project on a case-by-case basis.
+
+The terms of each license can be found in the source code repository:
+
+GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
+Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
+*/
+package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*;
+import org.junit.*; import gplx.core.tests.*;
+import gplx.core.btries.*;
+import gplx.xowa.parsers.xndes.*;
+public class Scrib_lib_text__nowiki_util_tst {
+	private final    Scrib_lib_text__nowiki_util_fxt fxt = new Scrib_lib_text__nowiki_util_fxt();
+	@Test  public void Basic() {
+		// noop
+		fxt.Test__Strip_tag("abc", "abc");
+
+		// one
+		fxt.Test__Strip_tag("a<nowiki>b</nowiki>c", "abc");
+
+		// mixed case
+		fxt.Test__Strip_tag("a<NOwiki>b</noWIKI>c", "abc");
+
+		// multiple: consecutive
+		fxt.Test__Strip_tag("a<nowiki>b</nowiki>c<nowiki>d</nowiki>e", "abcde");
+
+		// dangling: left
+		fxt.Test__Strip_tag("a<nowiki>b<nowiki>c</nowiki>d", "ab<nowiki>cd");
+
+		// dangling: right
+		fxt.Test__Strip_tag( "a<nowiki>b</nowiki>c</nowiki>d", "abc</nowiki>d");
+
+		// nested
+		fxt.Test__Strip_tag("a<nowiki>b<nowiki>c</nowiki>d</nowiki>e", "ab<nowiki>cd</nowiki>e");
+	}
+}
+class Scrib_lib_text__nowiki_util_fxt {
+	private final    Scrib_lib_text__nowiki_util util = new Scrib_lib_text__nowiki_util();
+	private final    Btrie_slim_mgr trie;
+	public Scrib_lib_text__nowiki_util_fxt() {
+		this.trie = util.Make_trie(Xop_xnde_tag_.Tag__nowiki.Name_bry());
+	}
+	public void Test__Strip_tag(String src, String expd) {
+		byte[] actl = util.Strip_tag(Bry_.new_a7("Page"), Bry_.new_u8(src), trie);
+		Gftest.Eq__str(expd, actl);
+	} 
+}

400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_text_tst.java

@@ -25,6 +25,9 @@ public class Scrib_lib_text_tst {
 	@Test  public void Unstrip() {
 		fxt.Test_scrib_proc_str(lib, Scrib_lib_text.Invk_unstrip, Object_.Ary("a"), "a");
 	}
+	@Test  public void UnstripNoWiki() {
+		fxt.Test_scrib_proc_str(lib, Scrib_lib_text.Invk_unstripNoWiki, Object_.Ary("a<nowiki>b</nowiki>c"), "abc");
+	}
 	@Test  public void GetEntityTable() {
 		Keyval[] actl = fxt.Test_scrib_proc_rv_as_kv_ary(lib, Scrib_lib_text.Invk_getEntityTable, Object_.Ary());
 		Tfds.Eq(1510, actl.length);	// large result; only test # of entries