Source: Restore broken commit

400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_itm__base.java

@@ -0,0 +1,37 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.apps.site_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
+import gplx.langs.jsons.*;
+import gplx.xowa.bldrs.wms.sites.*;
+public abstract class Xoa_site_cfg_itm__base {
+	public void Ctor(String key_str) {
+		this.key_str = key_str; this.key_bry = Bry_.new_u8(key_str);
+	}
+	public String Key_str() {return key_str;} private String key_str;
+	public byte[] Key_bry() {return key_bry;} private byte[] key_bry;
+	public byte[] Parse_json(Xow_wiki wiki, Json_itm js_itm) {
+		Json_ary ary = Json_ary.cast(js_itm);
+		Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b512();
+		int len = ary.Len();
+		for (int i = 0; i < len; ++i)
+			Parse_json_ary_itm(bfr, wiki, i, ary.Get_at(i));
+		return bfr.To_bry_and_rls();
+	}
+	public abstract void Parse_json_ary_itm(Bry_bfr bfr, Xow_wiki wiki, int i, Json_itm itm);
+	public abstract void Exec_csv(Xow_wiki wiki, int loader_tid, byte[] dsv_bry);
+}

400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_itm__extensiontags.java

@@ -0,0 +1,48 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.apps.site_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
+import gplx.langs.jsons.*;
+class Xoa_site_cfg_itm__extensiontags extends Xoa_site_cfg_itm__base {
+	public Xoa_site_cfg_itm__extensiontags() {
+		this.Ctor(Xoa_site_cfg_loader__inet.Qarg__extensiontags);
+	}
+	@Override public void Parse_json_ary_itm(Bry_bfr bfr, Xow_wiki wiki, int i, Json_itm itm) {
+		byte[] tag = itm.Data_bry();
+		if (i != 0) bfr.Add_byte_nl();
+		int idx_last = tag.length - 1;
+		if (	tag.length < 3
+			||	tag[0]			!= Byte_ascii.Angle_bgn
+			||	tag[idx_last]	!= Byte_ascii.Angle_end
+			)
+			throw Err_.new_("site_meta", "invalid extensiontag", "tag", tag);
+		bfr.Add_mid(tag, 1, idx_last);	// convert "<pre>" to "pre"
+	}
+	@Override public void Exec_csv(Xow_wiki wiki, int loader_tid, byte[] dsv_bry) {
+		Hash_adp_bry hash = null;
+		if (loader_tid != Xoa_site_cfg_loader_.Tid__fallback) {	// fallback will result in null hash which will result in loading all extensions
+			hash = Hash_adp_bry.ci_a7();						// NOTE: must be case-insensitive; EX: <imageMap> vs <imagemap>
+			byte[][] lines = Bry_split_.Split_lines(dsv_bry);
+			int lines_len = lines.length;
+			for (int i = 0; i < lines_len; ++i) {
+				byte[] line = lines[i]; if (Bry_.Len_eq_0(line)) continue;	// ignore blank lines
+				hash.Add(line, line);
+			}
+		}
+		wiki.Mw_parser_mgr().Xnde_tag_regy().Init_by_meta(hash);
+	}
+}

400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_itm__interwikimap.java

@@ -0,0 +1,118 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.apps.site_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
+import gplx.core.net.*; import gplx.langs.jsons.*; import gplx.xowa.apps.gfs.*;
+import gplx.xowa.langs.*;
+import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.xwikis.*;
+class Xoa_site_cfg_itm__interwikimap extends Xoa_site_cfg_itm__base {
+	private final    Bry_bfr tmp_bfr = Bry_bfr_.New();
+	private final    Gfo_url_parser url_parser = new Gfo_url_parser();
+	public Xoa_site_cfg_itm__interwikimap() {this.Ctor(Xoa_site_cfg_loader__inet.Qarg__interwikimap);}
+	@Override public void Parse_json_ary_itm(Bry_bfr bfr, Xow_wiki wiki, int i, Json_itm itm) {
+		Json_nde nde = Json_nde.cast(itm);
+		if (i != 0) bfr.Add_byte_nl();
+		byte[] iw_key = nde.Get_bry_or_null("prefix");	if (iw_key == null) throw Err_.new_("site_meta", "invalid interwiki", "key", iw_key);
+		byte[] iw_url = nde.Get_bry_or_null("url");		if (iw_url == null) throw Err_.new_("site_meta", "invalid interwiki", "url", iw_key);
+		bfr.Add(iw_key).Add_byte_pipe().Add(Gfs_php_converter.To_gfs(tmp_bfr, iw_url));
+	}
+	@Override public void Exec_csv(Xow_wiki wiki, int loader_tid, byte[] dsv_bry) {
+		if (loader_tid == Xoa_site_cfg_loader_.Tid__fallback)
+			Exec_csv__fallback(wiki);
+		else {
+			byte[][] lines = Bry_split_.Split_lines(dsv_bry);
+			int lines_len = lines.length;
+			for (int i = 0; i < lines_len; ++i) {
+				byte[] line = lines[i]; if (Bry_.Len_eq_0(line)) continue;	// ignore blank lines
+				byte[][] flds = Bry_split_.Split(line, Byte_ascii.Pipe);
+				byte[] url_fmt = flds[1];
+				byte[] domain_bry = Xow_xwiki_mgr.Get_domain_from_url(url_parser, url_fmt);
+				wiki.Xwiki_mgr().Add_by_atrs(flds[0], domain_bry, url_fmt);
+			}
+		}
+	}
+	private void Exec_csv__fallback(Xow_wiki wiki) {
+		Xow_xwiki_mgr xwiki_mgr = wiki.Xwiki_mgr();
+		int domain_tid = wiki.Domain_tid();
+		xwiki_mgr.Add_by_csv(Csv__manual);	// adds manual xwikis that should exist in all wikis; EX: 'commons', 'wikidata', 'oldwikisource', 'wmf'
+		switch (domain_tid) {
+			case Xow_domain_tid_.Tid__wikipedia: case Xow_domain_tid_.Tid__wiktionary:	case Xow_domain_tid_.Tid__wikisource:	case Xow_domain_tid_.Tid__wikivoyage:
+			case Xow_domain_tid_.Tid__wikiquote: case Xow_domain_tid_.Tid__wikibooks:	case Xow_domain_tid_.Tid__wikiversity:	case Xow_domain_tid_.Tid__wikinews:
+				xwiki_mgr.Add_by_sitelink_mgr();									// lang: EX: [[fr:]]  -> fr.wikipedia.org
+				xwiki_mgr.Add_by_csv(Csv__peers__lang);								// peer: EX: [[wikt]] -> en.wiktionary.org
+				break;
+			case Xow_domain_tid_.Tid__commons:		case Xow_domain_tid_.Tid__wikidata:
+			case Xow_domain_tid_.Tid__wikimedia:	case Xow_domain_tid_.Tid__species:	case Xow_domain_tid_.Tid__meta: case Xow_domain_tid_.Tid__incubator:
+			case Xow_domain_tid_.Tid__mediawiki:	case Xow_domain_tid_.Tid__wmfblog:
+			case Xow_domain_tid_.Tid__home:			case Xow_domain_tid_.Tid__other:
+				xwiki_mgr.Add_by_sitelink_mgr(Xow_domain_tid_.Tid__wikipedia);		// lang: hardcode to wikipedia; EX: "[[en:]] -> "en.wikipedia.org"
+				xwiki_mgr.Add_by_csv(Csv__peers__english);							// peer: hardcode to english
+				break;
+		}
+
+		// wikivoyage
+		switch (domain_tid) {
+			case Xow_domain_tid_.Tid__wikivoyage: case Xow_domain_tid_.Tid__home:	// NOTE: home needed for diagnostic; DATE:2015-10-13
+				xwiki_mgr.Add_by_csv(Csv__wikivoyage);
+				break;
+		}
+
+		// if simplewiki, add "w" -> "enwiki"
+		if	(Bry_.Eq(wiki.Domain_bry(), Xow_domain_itm_.Bry__simplewiki))	
+			xwiki_mgr.Add_by_csv(Csv__enwiki);
+	}
+	private static final    byte[] 
+	  Csv__manual = Bry_.new_a7(String_.Concat_lines_nl_skip_last
+	( "1|commons;c|commons.wikimedia.org|Commons"
+	, "1|m;metawikipedia|meta.wikipedia.org"
+	, "1|species;wikispecies|species.wikimedia.org"
+	, "1|d;wikidata|www.wikidata.org"
+	, "1|mw;mediawikiwiki|www.mediawiki.org"
+	, "1|wmf;wikimedia;foundation|wikimediafoundation.org"
+	, "1|incubator|incubator.wikimedia.org"
+	, "0|oldwikisource|https://wikisource.org/wiki/~{0}|Old Wikisoure"
+	, "0|mail|https://lists.wikimedia.org/mailman/listinfo/~{0}|Wikitech Mailing List"
+	))
+	, Csv__peers__lang = Bry_.new_a7(String_.Concat_lines_nl_skip_last
+	( "2|w;wikipedia|wikipedia"
+	, "2|wikt;wiktionary|wiktionary"
+	, "2|s;wikisource|wikisource"
+	, "2|b;wikibooks|wikibooks"
+	, "2|v;wikiversity|wikiversity"
+	, "2|q;wikiquote|wikiquote"
+	, "2|n;wikinews|wikinews"
+	, "2|voy;wikivoyage|wikivoyage"
+	))
+	, Csv__peers__english = Bry_.new_a7(String_.Concat_lines_nl_skip_last
+	( "1|w|en.wikipedia.org"
+	, "1|wikt|en.wiktionary.org"
+	, "1|s|en.wikisource.org"
+	, "1|b|en.wikibooks.org"
+	, "1|v|en.wikiversity.org"
+	, "1|q|en.wikiquote.org"
+	, "1|n|en.wikinews.org"
+	, "1|voy|en.wikivoyage.org"
+	))
+	, Csv__wikivoyage = Bry_.new_a7(String_.Concat_lines_nl_skip_last
+	( "0|commons|commons.wikimedia.org|Wikimedia Commons"
+	, "2|wikipedia|wikipedia|Wikipedia"
+	, "0|dmoz|http://www.dmoz.org/~{0}|DMOZ"
+	))
+	, Csv__enwiki = Bry_.new_a7("2|w|wikipedia")
+	;
+}
+

400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_loader.java

@@ -0,0 +1,36 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.apps.site_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
+public interface Xoa_site_cfg_loader {
+	int			Tid();
+	void		Load_csv__bgn(Xoa_site_cfg_mgr mgr, Xow_wiki wiki);
+	byte[]		Load_csv(Xoa_site_cfg_mgr mgr, Xow_wiki wiki, Xoa_site_cfg_itm__base itm);
+}
+class Xoa_site_cfg_loader_ {
+	public static final int Tid__null = -1, Tid__cfg = 0, Tid__fsys = 1, Tid__inet = 2, Tid__fallback = 3;
+	private static final String Key__cfg = "cfg", Key__fsys = "fsys", Key__inet = "inet", Key__fallback = "itm";
+	public static String Get_key(int tid) {
+		switch (tid) {
+			case Tid__cfg:			return Key__cfg;
+			case Tid__fsys:			return Key__fsys;
+			case Tid__inet:			return Key__inet;
+			case Tid__fallback:		return Key__fallback;
+			default:				throw Err_.new_unhandled(tid);
+		}
+	}
+}

400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_loader__cfg.java

@@ -0,0 +1,44 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.apps.site_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
+import gplx.dbs.cfgs.*;
+class Xoa_site_cfg_loader__db implements Xoa_site_cfg_loader {
+	private Db_cfg_tbl cfg_tbl;
+	public int Tid() {return Xoa_site_cfg_loader_.Tid__cfg;}
+	public void Load_csv__bgn(Xoa_site_cfg_mgr mgr, Xow_wiki wiki) {
+		this.cfg_tbl = wiki.Data__core_mgr().Tbl__cfg();
+		cfg_tbl.Select_as_hash_bry(mgr.Data_hash(), Grp__xowa_wm_api);
+	}
+	public byte[] Load_csv(Xoa_site_cfg_mgr mgr, Xow_wiki wiki, Xoa_site_cfg_itm__base itm) {
+		byte[] rv = (byte[])mgr.Data_hash().Get_by_bry(itm.Key_bry()); if (rv == null) return null;
+		int meta_end = Bry_find_.Find_fwd(rv, Byte_ascii.Nl);
+		if (meta_end == Bry_find_.Not_found) {// fallback will only log one line; ignore; EX: //#xowa|fallback
+			return null;
+		}
+		return Bry_.Mid(rv, meta_end + 1);
+	}
+	public void Save_bry(int loader_tid, String db_key, byte[] val) {
+		byte[] meta = Bry_.new_a7(Bld_meta(loader_tid));
+		byte[] data = Bry_.Len_eq_0(val) ? meta : Bry_.Add(meta, Byte_ascii.Nl_bry, val);
+		cfg_tbl.Upsert_bry(Grp__xowa_wm_api, db_key, data);
+	}
+	public static String Bld_meta(int loader_tid) {
+		return String_.Format("//#xowa|{0}|{1}|{2}", Xoa_app_.Version, Xoa_site_cfg_loader_.Get_key(loader_tid), Datetime_now.Get().XtoUtc().XtoStr_fmt_yyyyMMdd_HHmmss());
+	}
+	public static final String Grp__xowa_wm_api = "xowa.site_cfg";
+}

400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_loader__fallback.java

@@ -0,0 +1,23 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.apps.site_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
+class Xoa_site_cfg_loader__fallback implements Xoa_site_cfg_loader {
+	public int			Tid() {return Xoa_site_cfg_loader_.Tid__fallback;}
+	public void			Load_csv__bgn(Xoa_site_cfg_mgr mgr, Xow_wiki wiki) {}
+	public byte[]		Load_csv(Xoa_site_cfg_mgr mgr, Xow_wiki wiki, Xoa_site_cfg_itm__base itm) {return Bry_.Empty;}	// return non-null so "data != null" check works
+}

400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_loader__fsys.java

@@ -0,0 +1,30 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.apps.site_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
+import gplx.xowa.apps.fsys.*;
+class Xoa_site_cfg_loader__fsys implements Xoa_site_cfg_loader {
+	private final Io_url site_cfg_dir;
+	public Xoa_site_cfg_loader__fsys(Io_url site_cfg_dir) {this.site_cfg_dir = site_cfg_dir;}
+	public int Tid() {return Xoa_site_cfg_loader_.Tid__fsys;}
+	public void Load_csv__bgn(Xoa_site_cfg_mgr mgr, Xow_wiki wiki) {}
+	public byte[] Load_csv(Xoa_site_cfg_mgr mgr, Xow_wiki wiki, Xoa_site_cfg_itm__base itm) {
+		return Io_mgr.Instance.LoadFilBryOrNull(Make_load_url(wiki.Domain_str(), itm.Key_str()));
+	}
+	public Io_url Make_load_url(String wiki, String key) {return site_cfg_dir.GenSubFil_nest(key, key + "-" + wiki + ".csv");}
+	public static Xoa_site_cfg_loader__fsys new_(Xoa_app app) {return new Xoa_site_cfg_loader__fsys(app.Fsys_mgr().Bin_xowa_dir().GenSubDir_nest("cfg", "wiki", "api"));}
+}

400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_loader__inet.java

@@ -0,0 +1,66 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.apps.site_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
+import gplx.core.net.*; import gplx.xowa.bldrs.wms.*;
+import gplx.langs.jsons.*;
+import gplx.xowa.wikis.domains.*;
+public class Xoa_site_cfg_loader__inet implements Xoa_site_cfg_loader {
+	private final    Gfo_inet_conn inet_conn; private final    Json_parser json_parser;
+	private String api_url; private boolean call_api = true; private Json_doc jdoc;
+	private final    Bry_bfr tmp_bfr = Bry_bfr_.New();
+	public Xoa_site_cfg_loader__inet(Gfo_inet_conn inet_conn, Json_parser json_parser) {this.inet_conn = inet_conn; this.json_parser = json_parser;}
+	public int Tid() {return Xoa_site_cfg_loader_.Tid__inet;}
+	public void Load_csv__bgn(Xoa_site_cfg_mgr mgr, Xow_wiki wiki) {
+		this.call_api = true;
+		this.jdoc = null;
+		this.api_url = Bld_url(tmp_bfr, wiki.Domain_str(), mgr.Data_hash(), mgr.Itm_ary());
+	}
+	public String Api_url() {return api_url;}
+	public byte[] Load_csv(Xoa_site_cfg_mgr mgr, Xow_wiki wiki, Xoa_site_cfg_itm__base itm) {
+		if (Int_.In(wiki.Domain_tid(), Xow_domain_tid_.Tid__home, Xow_domain_tid_.Tid__other)) return null;	// do not call api if "home", "other"; for "wikia" and other non-wm wikis
+		if (call_api) {
+			call_api = false;
+			byte[] json_bry = Xowm_api_mgr.Call_by_url(Xoa_app_.Usr_dlg(), inet_conn, wiki.Domain_str(), api_url);
+			if (json_bry != null) jdoc = json_parser.Parse(json_bry);
+		}
+		if (jdoc == null) return null;
+		Json_itm js_itm = jdoc.Get_grp_many(Bry__query, itm.Key_bry()); if (js_itm == null) return null;
+		return itm.Parse_json(wiki, js_itm);
+	}
+	public static String Bld_url(Bry_bfr tmp_bfr, String domain_str, Hash_adp_bry db_hash, Xoa_site_cfg_itm__base[] itm_ary) {
+		boolean first = true;
+		int len = itm_ary.length;
+		for (int i = 0; i < len; ++i) {
+			Xoa_site_cfg_itm__base itm = itm_ary[i];
+			// if (db_hash.Has(itm_key)) continue;	// TOMBSTONE: always add itm to url, even if in db; note that fallback gets saved to db; DATE:2016-04-13
+			if (first)
+				first = false;
+			else
+				tmp_bfr.Add_byte_pipe();
+			tmp_bfr.Add(itm.Key_bry());
+		}			
+		return first ? null : Xowm_api_mgr.Bld_api_url(domain_str, Qarg__bgn + tmp_bfr.To_str_and_clear());
+	}
+	private static final    byte[] Bry__query = Bry_.new_a7("query");
+	public static final String
+	  Qarg__all							= "action=query&format=json&rawcontinue=&meta=siteinfo&siprop=general|namespaces|statistics|interwikimap|namespacealiases|specialpagealiases|libraries|extensions|skins|magicwords|functionhooks|showhooks|extensiontags|protocols|defaultoptions|languages"
+	, Qarg__bgn							= "action=query&format=json&rawcontinue=&meta=siteinfo&siprop="
+	, Qarg__extensiontags				= "extensiontags"
+	, Qarg__interwikimap				= "interwikimap"
+	;
+}

400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_mgr.java

@@ -0,0 +1,69 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.apps.site_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
+import gplx.xowa.bldrs.wms.sites.*;
+public class Xoa_site_cfg_mgr {
+	private final    Xoa_site_cfg_loader__db loader__db = new Xoa_site_cfg_loader__db();		
+	public Xoa_site_cfg_mgr(Xoa_app app) {
+		this.itm_ary = new Xoa_site_cfg_itm__base[]
+		{ new Xoa_site_cfg_itm__extensiontags()
+		, new Xoa_site_cfg_itm__interwikimap()
+		};
+		this.loader_ary = new Xoa_site_cfg_loader[] 
+		{ loader__db
+		, Xoa_site_cfg_loader__fsys.new_(app)
+		, new Xoa_site_cfg_loader__inet(app.Utl__inet_conn(), app.Utl__json_parser())
+		, new Xoa_site_cfg_loader__fallback()
+		};
+	}
+	public Xoa_site_cfg_loader[]		Loader_ary() {return loader_ary;} private final    Xoa_site_cfg_loader[] loader_ary;
+	public Xoa_site_cfg_itm__base[]		Itm_ary() {return itm_ary;} private final    Xoa_site_cfg_itm__base[] itm_ary;
+	public Hash_adp_bry					Data_hash() {return data_hash;} private final    Hash_adp_bry data_hash = Hash_adp_bry.cs();
+	public void Init_loader_bgn(Xow_wiki wiki) {
+		data_hash.Clear();	
+		int loader_len = loader_ary.length;
+		for (int i = 0; i < loader_len; ++i)
+			loader_ary[i].Load_csv__bgn(this, wiki);
+	}
+	public void Load(Xow_wiki wiki) {
+		if (wiki.Data__core_mgr() == null) return;	// TEST:
+		synchronized (loader__db) {	// THREAD: data_hash; loader_ary
+			this.Init_loader_bgn(wiki);
+			int len = itm_ary.length;
+			for (int i = 0; i < len; ++i) {
+				Xoa_site_cfg_itm__base itm = itm_ary[i];
+				Load_by_loader(wiki, itm);
+			}
+		}
+	}
+	private void Load_by_loader(Xow_wiki wiki, Xoa_site_cfg_itm__base itm) {
+		int len = loader_ary.length;
+		for (int i = 0; i < len; ++i) {
+			Xoa_site_cfg_loader loader = loader_ary[i];
+			try {// guard against individual loader failing, particularly inet
+				byte[] data = loader.Load_csv(this, wiki, itm);
+				if (data != null) {
+					if (loader.Tid() != Xoa_site_cfg_loader_.Tid__cfg) loader__db.Save_bry(loader.Tid(), itm.Key_str(), data);
+					Xoa_app_.Usr_dlg().Log_many("", "", "site_cfg loaded; wiki=~{0} itm=~{1} tid=~{2}", wiki.Domain_str(), itm.Key_bry(), Xoa_site_cfg_loader_.Get_key(loader.Tid()));
+					itm.Exec_csv(wiki, loader.Tid(), data);
+					break;
+				}
+			} catch (Exception e) {Xoa_app_.Usr_dlg().Warn_many("", "", "error while loading site_cfg; wiki=~{0} itm=~{1} err=~{2}", wiki.Domain_str(), itm.Key_bry(), Err_.Message_gplx_log(e));}
+		}
+	}
+}

400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_mgr_tst.java

@@ -0,0 +1,180 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.apps.site_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
+import org.junit.*;
+import gplx.core.btries.*;
+import gplx.dbs.cfgs.*;
+import gplx.langs.jsons.*; import gplx.xowa.wikis.nss.*;
+import gplx.xowa.parsers.*; import gplx.xowa.bldrs.wms.*;
+public class Xoa_site_cfg_mgr_tst {
+	private final    Xoa_site_cfg_mgr_fxt fxt = new Xoa_site_cfg_mgr_fxt();
+	@Before		public void init() {fxt.Init();}
+	@After  public void term() {fxt.Term();}
+	@Test  public void Extensiontags__cfg() {
+		fxt.Init_db(Xoa_site_cfg_loader__inet.Qarg__extensiontags, fxt.Make_data(Xoa_site_cfg_loader_.Tid__inet, 1, "math", "source"));
+		fxt.Exec_load();
+		fxt.Test_extensiontags_y("math"	, "source");
+		fxt.Test_extensiontags_n("ref"	, "graph");
+		fxt.Test_extensiontags_y("translate", "languages");	// always whitelist <translate>,<languages> DATE:2015-10-13
+	}
+	@Test  public void Extensiontags__fsys() {
+		fxt.Init_fsys(Xoa_site_cfg_loader__inet.Qarg__extensiontags, fxt.Make_data(Xoa_site_cfg_loader_.Tid__null, 1, "math", "source"));
+		fxt.Exec_load();
+		fxt.Test_extensiontags_y("math", "source");
+		fxt.Test_extensiontags_n("ref"	, "graph");
+		fxt.Test_db(Xoa_site_cfg_loader__inet.Qarg__extensiontags, fxt.Make_data(Xoa_site_cfg_loader_.Tid__fsys, 1, "math", "source"));
+	}
+	@Test  public void Extensiontags__inet() {
+		fxt.Init_inet(fxt.Make_api(fxt.Make_api_extensiontags("math", "source")));
+		fxt.Exec_load();
+		fxt.Test_extensiontags_y("math", "source");
+		fxt.Test_extensiontags_n("ref"	, "graph");
+		fxt.Test_db(Xoa_site_cfg_loader__inet.Qarg__extensiontags, fxt.Make_data(Xoa_site_cfg_loader_.Tid__inet, 1, "math", "source"));
+	}
+	@Test  public void Extensiontags__fallback() {
+		fxt.Exec_load();
+		fxt.Test_db(Xoa_site_cfg_loader__inet.Qarg__extensiontags, fxt.Make_data(Xoa_site_cfg_loader_.Tid__fallback, 1));
+		fxt.Test_extensiontags_y("math", "source", "ref", "graph");
+	}
+	@Test  public void Interwikimap__inet() {
+		fxt.Init_inet(fxt.Make_api(fxt.Make_api_interwikimap("w", "https://en.wikipedia.org", "c", "https://commons.wikimedia.org")));
+		fxt.Exec_load();
+		fxt.Test_db(Xoa_site_cfg_loader__inet.Qarg__interwikimap, fxt.Make_data(Xoa_site_cfg_loader_.Tid__inet, 2, "w", "https://en.wikipedia.org", "c", "https://commons.wikimedia.org"));
+	}
+//		@Test   public void Print() {
+//			String s = fxt.Make_api(fxt.Make_api_interwikimap("k1", "v1", "k2", "v2"), fxt.Make_api_extensiontags2("k3", "v3", "k4", "v4"));
+//            Tfds.Dbg(s);
+//		}
+}
+class Xoa_site_cfg_mgr_fxt {
+	private final    Xoae_app app; private final    Xowe_wiki wiki;
+	private final    Xoa_site_cfg_mgr site_cfg_mgr;
+	private final    Db_cfg_tbl cfg_tbl;
+	private final    Json_printer printer = new Json_printer();
+	private final    Bry_bfr tmp_bfr = Bry_bfr_.New();
+	public Xoa_site_cfg_mgr_fxt() {
+		// Xoa_app_.Usr_dlg_(Xoa_app_.New__usr_dlg__console());
+		gplx.core.ios.IoEngine_system.Web_access_enabled = true;	// HACK: must manually enable web_access else above tests will fail due to some other test disabling singleton; DATE:2016-12-15
+		Xoa_test_.Inet__init();
+		this.app = Xoa_app_fxt.Make__app__edit();
+		this.wiki = Xoa_app_fxt.Make__wiki__edit(app);
+		Xoa_test_.Init__db__edit(wiki);
+		this.cfg_tbl = wiki.Data__core_mgr().Tbl__cfg();
+		this.site_cfg_mgr = app.Site_cfg_mgr();
+	}
+	public void Init() {
+		Datetime_now.Manual_y_(); Datetime_now.Autoincrement_n_();
+		Io_mgr.Instance.InitEngine_mem();
+		cfg_tbl.Delete_grp(Xoa_site_cfg_loader__db.Grp__xowa_wm_api);
+		site_cfg_mgr.Init_loader_bgn(wiki);
+		app.Utl__inet_conn().Clear();
+	}
+	public void Term() {
+		Datetime_now.Manual_n_();
+	}
+	public void Init_db(String key, String data) {
+		cfg_tbl.Assert_bry(Xoa_site_cfg_loader__db.Grp__xowa_wm_api, key, Bry_.new_u8(data));
+	}
+	public void Test_db(String key, String expd) {
+		byte[] actl = cfg_tbl.Select_bry_or(Xoa_site_cfg_loader__db.Grp__xowa_wm_api, key, null);
+		Tfds.Eq_str_lines(expd, String_.new_u8(actl));
+	}
+	public void Init_inet(String data) {
+		String url = Xoa_site_cfg_loader__inet.Bld_url(tmp_bfr, wiki.Domain_str(), site_cfg_mgr.Data_hash(), site_cfg_mgr.Itm_ary());
+		app.Utl__inet_conn().Upload_by_bytes(url, Bry_.new_u8(data));
+	}
+	public void Init_fsys(String key, String data) {
+		Xoa_site_cfg_loader__fsys loader = Xoa_site_cfg_loader__fsys.new_(app);
+		Io_url url = loader.Make_load_url(wiki.Domain_str(), key);
+		Io_mgr.Instance.SaveFilStr(url, data);
+	}
+	public Xoa_site_cfg_mgr_fxt Exec_load() {
+		site_cfg_mgr.Load(wiki);
+		return this;
+	}
+	public void Test_extensiontags_y(String... ary) {Test_extensiontags(Bool_.Y, ary);}
+	public void Test_extensiontags_n(String... ary) {Test_extensiontags(Bool_.N, ary);}
+	public void Test_extensiontags(boolean expd_exists, String... ary) {
+		Btrie_slim_mgr trie = wiki.Mw_parser_mgr().Xnde_tag_regy().Get_trie(Xop_parser_tid_.Tid__defn);
+		int len = ary.length;
+		for (int i = 0; i < len; ++i) {
+			String str = ary[i];
+			byte[] bry = Bry_.new_u8(str);
+			boolean actl_exists = trie.Match_exact(bry, 0, bry.length) != null;
+			Tfds.Eq_bool(expd_exists, actl_exists, str);
+		}
+	}
+	public void Test_inet_qarg(String expd) {
+		Xoa_site_cfg_loader__inet loader__inet = (Xoa_site_cfg_loader__inet)site_cfg_mgr.Loader_ary()[2];
+		String api_url = loader__inet.Api_url();
+		Tfds.Eq(expd, String_.Mid(api_url, String_.FindBwd(api_url, "=") + 1));
+	}
+	public String Make_api(byte[]... sections) {
+		Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b512();
+		bfr.Add_str_a7("{'query':");
+		int len = sections.length;
+		bfr.Add_str_a7("{");
+		for (int i = 0; i < len; ++i) {
+			if (i != 0) bfr.Add_str_a7(",");
+			bfr.Add(sections[i]);
+		}
+		bfr.Add_str_a7("}");
+		bfr.Add_str_a7("}");
+		return printer.Print_by_bry(Bry_.new_u8(Json_doc.Make_str_by_apos(bfr.To_str_and_rls()))).To_str();
+	}
+	public byte[] Make_api_interwikimap(String... ary) {
+		Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b512();
+		bfr.Add_str_a7("'interwikimap':");
+		int len = ary.length;
+		bfr.Add_str_a7("[");
+		for (int i = 0; i < len; i += 2) {
+			if (i != 0) bfr.Add_str_a7(",");
+			bfr.Add_str_a7("{'prefix':'"	+ ary[i] + "'");
+			bfr.Add_str_a7(",'url':'"	+ ary[i + 1] + "'");
+			bfr.Add_str_a7("}");
+		}
+		bfr.Add_str_a7("]");
+		return bfr.To_bry_and_clear();
+	}
+	public byte[] Make_api_extensiontags(String... ary) {
+		Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b512();
+		bfr.Add_str_a7("'extensiontags':");
+		int len = ary.length;
+		bfr.Add_str_a7("[");
+		for (int i = 0; i < len; ++i) {
+			if (i != 0) bfr.Add_str_a7(",");
+			bfr.Add_str_a7("'<"	+ ary[i] + ">'");
+		}
+		bfr.Add_str_a7("]");
+		return bfr.To_bry_and_clear();
+	}
+	public String Make_data(int loader_tid, int flds, String... ary) {
+		Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b512();
+		if (loader_tid != Xoa_site_cfg_loader_.Tid__null)	// null when constructing data for fsys
+			bfr.Add_str_u8(Xoa_site_cfg_loader__db.Bld_meta(loader_tid)).Add_byte_nl();
+		int len = ary.length;
+		for (int i = 0; i < len; i += flds) {
+			if (i != 0) bfr.Add_byte_nl();
+			for (int j = 0; j < flds; ++j) {
+				if (j != 0) bfr.Add_byte_pipe();
+				bfr.Add_str_u8(ary[i + j]);
+			}
+		}
+		return bfr.To_str_and_rls();
+	}
+}