v2.7.2.1

400_xowa/src/gplx/xowa/bldrs/Db_idx_mode.java

@@ -0,0 +1,38 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
+public class Db_idx_mode {
+	private int tid;
+	Db_idx_mode(int tid) {this.tid = tid;}
+	public boolean Tid_is_bgn()	{return tid == Tid_bgn;}
+	public boolean Tid_is_end()	{return tid == Tid_end;}
+	public boolean Tid_is_skip()	{return tid == Tid_skip;}
+	private static final int Tid_skip = 0, Tid_bgn = 1, Tid_end = 2;
+	private static final String Key_skip = "skip", Key_bgn = "bgn", Key_end = "end";
+	public static final Db_idx_mode
+	  Itm_skip	= new Db_idx_mode(Tid_skip)
+	, Itm_bgn	= new Db_idx_mode(Tid_bgn)
+	, Itm_end	= new Db_idx_mode(Tid_end)
+	;
+	public static Db_idx_mode Xto_itm(String key) {
+		if		(String_.Eq(key, Key_skip))		return Itm_skip;
+		else if	(String_.Eq(key, Key_bgn))		return Itm_bgn;
+		else if	(String_.Eq(key, Key_end))		return Itm_end;
+		else									throw Exc_.new_unhandled(key);
+	}
+}

400_xowa/src/gplx/xowa/bldrs/Db_mgr_fxt.java

@@ -0,0 +1,138 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
+import gplx.core.primitives.*; import gplx.core.strings.*;
+import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.dbs.*; import gplx.xowa.specials.search.*; import gplx.xowa.ctgs.*;
+import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.infos.*;
+public class Db_mgr_fxt {
+	public Db_mgr_fxt Ctor_fsys()	{bldr_fxt = new Xob_fxt().Ctor(Xoa_test_.Url_root().GenSubDir("root")); return this;} 
+	public Db_mgr_fxt Ctor_mem()	{bldr_fxt = new Xob_fxt().Ctor_mem(); return this;} private Xob_fxt bldr_fxt;
+	public Xowd_page_itm page_(int id, String modified_on, boolean type_redirect, int text_len) {return new Xowd_page_itm().Id_(id).Modified_on_(DateAdp_.parse_gplx(modified_on)).Redirected_(type_redirect).Text_len_(text_len);}
+	public Xowe_wiki Wiki() {return bldr_fxt.Wiki();}
+	public Xob_bldr Bldr() {return bldr_fxt.Bldr();}
+	public Db_mgr_fxt doc_ary_(Xowd_page_itm... v) {bldr_fxt.doc_ary_(v); return this;}
+	public Xowd_page_itm doc_(int id, String date, String title, String text) {return bldr_fxt.doc_(id, date, title, text);}
+	public Xowd_page_itm doc_wo_date_(int id, String title, String text) {return bldr_fxt.doc_(id, "2012-01-02 03:04", title, text);}
+	public Xowd_page_itm doc_ttl_(int id, String title) {return bldr_fxt.doc_(id, "2012-01-02 03:04", title, "IGNORE");}
+	public Db_mgr_fxt Init_fil(String url, String raw) {return Init_fil(Io_url_.new_fil_(url), raw);}
+	public Db_mgr_fxt Init_fil(Io_url url, String raw) {Io_mgr.I.SaveFilStr(url, raw); return this;}
+	public Db_mgr_fxt Exec_run(Xobd_wkr wkr)		{bldr_fxt.Run(wkr); return this;}
+	public Db_mgr_fxt Exec_run(Xob_cmd cmd)			{bldr_fxt.Run_cmds(cmd); return this;}
+	public Db_mgr_fxt Exec_run(Xobd_parser_wkr wkr) {bldr_fxt.Run(wkr); return this;}
+	public void Init_page_insert(Int_obj_ref page_id_next, int ns_id, String[] ttls) {
+		Xowe_wiki wiki = this.Wiki();
+		int len = ttls.length;
+		DateAdp modified_on = Tfds.Now_time0_add_min(0);
+		Xowd_page_tbl tbl_page = wiki.Db_mgr_as_sql().Core_data_mgr().Tbl__page();
+		tbl_page.Insert_bgn();
+		for (int i = 0; i < len; i++) {
+			String ttl = ttls[i];
+			int page_id = page_id_next.Val();
+			tbl_page.Insert_cmd_by_batch(page_id, ns_id, Bry_.new_u8(ttl), false, modified_on, 0, page_id, 0, 0);
+			page_id_next.Val_add(1);
+		}
+		tbl_page.Insert_end();
+	}
+	public void Test_load_ttl(int ns_id, String ttl_str, Xowd_page_itm expd) {
+		Xowe_wiki wiki = bldr_fxt.Wiki();
+		Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id);
+		byte[] ttl_bry = Bry_.new_a7(ttl_str);
+		wiki.Db_mgr_as_sql().Load_mgr().Load_by_ttl(actl, ns, ttl_bry);
+		Tfds.Eq(expd.Id(), actl.Id());
+		Tfds.Eq_date(expd.Modified_on(), actl.Modified_on());
+		Tfds.Eq(expd.Redirected(), actl.Redirected());
+		Tfds.Eq(expd.Text_len(), actl.Text_len());
+	}	private Xowd_page_itm actl = new Xowd_page_itm();
+	public void Test_load_page(int ns_id, int page_id, String expd) {
+		Xowe_wiki wiki = bldr_fxt.Wiki();
+		Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id);
+		wiki.Db_mgr_as_sql().Load_mgr().Load_page(actl.Id_(page_id), ns, false);
+		Tfds.Eq(expd, String_.new_a7(actl.Text()));
+	}
+	public void Test_search(String search_word_str, int... expd) {
+		Xowe_wiki wiki = bldr_fxt.Wiki();
+		List_adp rv = List_adp_.new_();
+		byte[] search_word_bry = Bry_.new_a7(search_word_str);
+		wiki.Db_mgr_as_sql().Load_mgr().Load_search(Cancelable_.Never, rv, search_word_bry, 100);
+		Tfds.Eq_ary(expd, Xto_int_ary(rv));
+	}
+	int[] Xto_int_ary(List_adp rslts) {
+		int len = rslts.Count();
+		int[] rv = new int[len];
+		for (int i = 0; i < len; i++) {
+			Xowd_page_itm page = (Xowd_page_itm)rslts.Get_at(i);
+			rv[i] = page.Id();
+		}
+		return rv;
+	}
+	public void Test_category_v1(String ctg_name_str, int... expd) {
+		Xowe_wiki wiki = bldr_fxt.Wiki();
+		byte[] ctg_name_bry = Bry_.new_a7(ctg_name_str);
+		Xoctg_view_ctg ctg = new Xoctg_view_ctg();
+		wiki.Db_mgr_as_sql().Load_mgr().Load_ctg_v1(ctg, ctg_name_bry);
+		Tfds.Eq_ary(expd, Xto_int_ary(ctg));
+	}
+	int[] Xto_int_ary(Xoctg_view_ctg ctg) {
+		List_adp list = List_adp_.new_();
+		byte tid_max = Xoa_ctg_mgr.Tid__max;
+		for (byte tid = 0; tid < tid_max; tid++) {
+			Xoctg_view_grp grp = ctg.Grp_by_tid(tid); if (grp == null) continue;
+			int len = grp.Itms_list().Count();
+			for (int i = 0; i < len; i++) {
+				Xoctg_view_itm itm = (Xoctg_view_itm)grp.Itms_list().Get_at(i);
+				list.Add(itm.Id());
+			}
+		}
+		return (int[])list.To_ary_and_clear(int.class);
+	}
+	public void Test_category_v2(String ctg_name_str, int... expd) {
+		Xowe_wiki wiki = bldr_fxt.Wiki();
+		byte[] ctg_name_bry = Bry_.new_a7(ctg_name_str);
+		Xoctg_data_ctg ctg = new Xoctg_data_ctg(ctg_name_bry);
+		wiki.Db_mgr_as_sql().Load_mgr().Load_ctg_v2(ctg, ctg_name_bry);
+		Tfds.Eq_ary(expd, Xto_int_ary(ctg));
+	}
+	public void Test_file(String url, String expd) {
+		String actl = Io_mgr.I.LoadFilStr(url);
+		Tfds.Eq_str_lines(expd, actl);
+	}
+	int[] Xto_int_ary(Xoctg_data_ctg ctg) {
+		List_adp list = List_adp_.new_();
+		byte tid_max = Xoa_ctg_mgr.Tid__max;
+		for (byte tid = 0; tid < tid_max; tid++) {
+			Xoctg_idx_mgr grp = ctg.Grp_by_tid(tid); if (grp == null) continue;
+			int len = grp.Itms_len();
+			for (int i = 0; i < len; i++) {
+				Xoctg_idx_itm itm = grp.Itms_get_at(i);
+				list.Add(itm.Id());
+			}
+		}
+		return (int[])list.To_ary_and_clear(int.class);
+	}
+	public void Init_db_sqlite() {
+		Xowe_wiki wiki = this.Wiki();
+		Db_conn_pool.I.Clear();
+		Db_conn_bldr.I.Reg_default_sqlite();
+		Io_mgr.I.DeleteDir_cmd(wiki.Fsys_mgr().Root_dir()).MissingIgnored_().Exec();
+		wiki.Db_mgr_create_as_sql().Core_data_mgr().Init_by_make(Xowd_core_db_props.Test, Xob_info_session.Test);
+		Io_mgr.I.SaveFilStr(wiki.Import_cfg().Src_dir().GenSubFil("a.xml"), "<test/>");
+	}
+	public void Rls() {
+		this.Wiki().Db_mgr_as_sql().Core_data_mgr().Rls();
+	}
+}

400_xowa/src/gplx/xowa/bldrs/Xob_base_fxt.java

@@ -0,0 +1,78 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
+import gplx.ios.*; import gplx.xowa.bldrs.*; import gplx.xowa.wikis.data.tbls.*;
+public class Xob_base_fxt {
+	public Xob_base_fxt Clear() {
+		if (app == null) {
+			app = Xoa_app_fxt.app_();
+			wiki = Xoa_app_fxt.wiki_tst_(app);
+			bldr = Xoa_app_fxt.bldr_(app);
+		}
+		this.Init_(bldr, wiki);
+		Clear_hook();
+		return this;
+	}
+	@gplx.Virtual public void Clear_hook() {}
+	public Xob_base_fxt Init_(Xob_bldr bldr, Xowe_wiki wiki) {this.bldr = bldr; this.wiki = wiki; return this;}
+	public Xoae_app App() {return app;} private Xoae_app app;
+	public Xob_bldr Bldr() {return bldr;} private Xob_bldr bldr;
+	public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki;
+	public GfoInvkAble Bldr_itm() {return bldr_itm;} GfoInvkAble bldr_itm;
+	public Xowd_page_itm page_(String ttl) {return page_(ttl, "");}
+	public Xowd_page_itm page_(String ttl, String text) {return new Xowd_page_itm().Ttl_(Bry_.new_u8(ttl), wiki.Ns_mgr()).Text_(Bry_.new_u8(text));}
+	public Io_fil_chkr meta_(String url, String data) {return new Io_fil_chkr(Io_url_.mem_fil_(url), data);}
+	public void Init_fxts(Xob_bldr bldr, Xowe_wiki wiki, Xob_base_fxt... fxt_ary) {
+		int fxt_ary_len = fxt_ary.length;
+		for (int i = 0; i < fxt_ary_len; i++)
+			fxt_ary[i].Init_(bldr, wiki);
+	}
+	public Xob_base_fxt Init_fil(String url, String raw) {return Init_fil(Io_url_.new_fil_(url), raw);}
+	public Xob_base_fxt Init_fil(Io_url url, String raw) {Io_mgr.I.SaveFilStr(url, raw); return this;}
+	public Xob_base_fxt Exec_cmd(String cmd_key, GfoMsg... msgs) {
+		Xob_cmd cmd = (Xob_cmd)bldr.Cmd_mgr().Add_cmd(wiki, cmd_key);
+		this.bldr_itm = cmd;
+		int len = msgs.length;
+		GfsCtx ctx = GfsCtx.new_();
+		for (int i = 0; i < len; i++) {
+			GfoMsg msg = msgs[i];
+			cmd.Invk(ctx, GfsCtx.Ikey_null, msg.Key(), msg);
+		}
+		Run_cmd(bldr, cmd);
+		return this;
+	}
+	public Xob_base_fxt Test_fil(String url, String expd) {return Test_fil(Io_url_.new_fil_(url), expd);}
+	public Xob_base_fxt Test_fil(Io_url url, String expd) {
+		Tfds.Eq_str_lines(expd, Io_mgr.I.LoadFilStr(url));
+		return this;
+	}
+	public static void Run_cmd(Xob_bldr bldr, Xob_cmd cmd) {
+		cmd.Cmd_bgn(bldr);
+		cmd.Cmd_run();
+		cmd.Cmd_end();
+	}
+	public static void Run_wkr(Xob_bldr bldr, Xobd_wkr wkr, Xowd_page_itm[] page_ary) {
+		wkr.Wkr_bgn(bldr);
+		int page_ary_len = page_ary.length;
+		for (int i = 0; i < page_ary_len; i++) {
+			Xowd_page_itm page = page_ary[i];
+			wkr.Wkr_run(page);
+		}
+		wkr.Wkr_end();		
+	}
+}

400_xowa/src/gplx/xowa/bldrs/Xob_cmd.java

@@ -0,0 +1,26 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
+public interface Xob_cmd extends GfoInvkAble {
+	String Cmd_key();
+	void Cmd_init(Xob_bldr bldr);
+	void Cmd_bgn(Xob_bldr bldr);
+	void Cmd_run();
+	void Cmd_end();
+	void Cmd_term();
+}

400_xowa/src/gplx/xowa/bldrs/Xob_cmd_keys.java

@@ -0,0 +1,77 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
+public class Xob_cmd_keys {
+	public static final String
+	  Key_text_init					= "text.init"				// "import.sql.init"
+	, Key_text_page					= "text.page"				// "import.sql.page"
+	, Key_text_css					= "text.css"
+	, Key_text_search_cmd			= "text.search.cmd"			// "import.sql.search_title.cmd"
+	, Key_text_search_wkr			= "text.search"				// "import.sql.search_title.wkr"
+	, Key_text_cat_core_v1			= "text.cat.core.v1"		// "import.sql.category_v1"
+	, Key_text_cat_core				= "text.cat.core"			// "import.sql.category_registry"
+	, Key_text_cat_link				= "text.cat.link"			// "import.sql.categorylinks"
+	, Key_text_cat_hidden			= "text.cat.hidden"			// "import.sql.hiddencat"
+	, Key_text_term					= "text.term"				// "import.sql.term"
+	, Key_wiki_redirect				= "wiki.redirect"			// "wiki.redirect"
+	, Key_wiki_image				= "wiki.image"				// "wiki.image"
+	, Key_wiki_page_dump_make		= "wiki.page_dump.make"		// "file.page_dump"
+	, Key_wiki_page_dump_drop		= "wiki.page_dump.drop"
+	, Key_file_lnki_temp			= "file.lnki_temp"
+	, Key_file_lnki_regy			= "file.lnki_regy"
+	, Key_file_page_regy			= "file.page_regy"
+	, Key_file_orig_regy			= "file.orig_regy"
+	, Key_file_xfer_temp_thumb		= "file.xfer_temp.thumb"
+	, Key_file_xfer_temp_orig		= "file.xfer_temp.orig"
+	, Key_file_xfer_regy			= "file.xfer_regy"
+	, Key_file_xfer_regy_update		= "file.xfer_regy_update"
+	, Key_file_fsdb_make			= "file.fsdb_make"
+	, Key_file_fsdb_reduce			= "file.fsdb_reduce"
+	, Key_file_orig_reg				= "file.orig_reg"
+	, Key_file_xfer_update			= "file.xfer_update"
+	, Key_html_redlinks				= "html.redlinks"
+	, Key_util_cleanup				= "util.cleanup"			// "core.cleanup"
+	, Key_util_download				= "util.download"			// "file.download"
+	, Key_util_xml_dump				= "util.xml_dump"
+	, Key_wbase_json_dump			= "wbase.json_dump"
+	, Key_wbase_qid					= "wbase.qid"				// "text.wdata.qid"
+	, Key_wbase_pid					= "wbase.pid"				// "text.wdata.pid"
+	, Key_wbase_db					= "wbase.db"				// "wiki.wdata_db"
+	, Key_wbase_ns					= "wbase.ns"
+	, Key_tdb_text_init				= "tdb.text.init"			// "core.init"
+	, Key_tdb_make_page				= "tdb.text.page"			// "core.make_page"
+	, Key_tdb_make_id				= "core.make_id"
+	, Key_tdb_make_search_title		= "core.make_search_title"
+	, Key_tdb_make_category			= "core.make_category"
+	, Key_tdb_calc_stats			= "core.calc_stats"
+	, Key_tdb_core_term				= "tdb.text.term"			// "core.term"
+	, Key_tdb_text_cat_link			= "ctg.link_sql"
+	, Key_tdb_ctg_link_idx			= "ctg.link_idx"
+	, Key_tdb_cat_hidden_sql		= "ctg.hiddencat_sql"
+	, Key_tdb_cat_hidden_ttl		= "ctg.hiddencat_ttl"
+	, Key_tdb_text_wdata_qid		= "tdb.text.wdata.qid"
+	, Key_tdb_text_wdata_pid		= "tdb.text.wdata.pid"
+	, Key_diff_regy_exec			= "file.diff_regy.exec"
+	, Key_diff_regy_make			= "file.diff_regy.make"
+	, Key_exec_sql					= "import.sql.exec_sql"
+	, Key_deploy_zip				= "deploy.zip"
+	, Key_deploy_copy				= "deploy.copy"
+	, Key_decompress_bz2			= "core.decompress_bz2"
+	;
+}
+ 

400_xowa/src/gplx/xowa/bldrs/Xob_cmd_mgr.java

@@ -0,0 +1,149 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
+import gplx.core.primitives.*;
+import gplx.xowa.wikis.*; import gplx.xowa.xtns.wdatas.imports.*;
+import gplx.xowa.bldrs.cmds.texts.*; import gplx.xowa.bldrs.cmds.texts.sqls.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*; import gplx.xowa.bldrs.cmds.files.*; import gplx.xowa.bldrs.cmds.ctgs.*; import gplx.xowa.bldrs.cmds.utils.*; import gplx.xowa.bldrs.cmds.wikis.*;
+import gplx.xowa.files.origs.*; import gplx.xowa.html.hdumps.bldrs.*;
+public class Xob_cmd_mgr implements GfoInvkAble {
+	public Xob_cmd_mgr(Xob_bldr bldr) {this.bldr = bldr;} private Xob_bldr bldr;
+	public void Clear() {list.Clear(); dump_rdrs.Clear();}
+	public int Len() {return list.Count();} List_adp list = List_adp_.new_();
+	public Xob_cmd Get_at(int i) {return (Xob_cmd)list.Get_at(i);} 
+	public Xob_cmd Add(Xob_cmd cmd) {list.Add(cmd); return cmd;}
+	public GfoInvkAble Add_cmd(Xowe_wiki wiki, String cmd_key) {
+		if		(String_.Eq(cmd_key, Xob_cmd_keys.Key_text_init))					return Add(new Xob_init_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_text_page))					return Xml_rdr_direct_add(wiki, new Xob_page_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_text_css))					return Add(new Xob_css_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_text_search_wkr))				return Xml_rdr_direct_add(wiki, new Xob_search_sql_wkr(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_text_search_cmd))				return Add(new Xob_search_sql_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_text_cat_core_v1))			return Xml_rdr_parser_add(wiki, new Xob_ctg_v1_sql().Ctor(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_text_cat_core))				return Add(new Xob_category_registry_sql(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_text_cat_link))				return Add(new Xob_categorylinks_sql(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_text_cat_hidden))				return Add(new Xoctg_hiddencat_parser_sql(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_text_term))					return Add(new Xob_term_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_wiki_page_dump_make))			return Add(new Xob_page_dump_cmd_make(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_wiki_page_dump_drop))			return Add(new Xob_page_dump_cmd_drop(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_wiki_redirect))				return Add(new Xob_redirect_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_wiki_image))					return Add(new Xob_image_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_file_lnki_temp))				return Add(new Xob_lnki_temp_wkr(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_file_lnki_regy))				return Add(new Xob_lnki_regy_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_file_page_regy))				return Add(new Xob_page_regy_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_file_orig_regy))				return Add(new Xob_orig_regy_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_file_xfer_temp_thumb))		return Add(new Xob_xfer_temp_cmd_thumb(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_file_xfer_temp_orig))			return Add(new Xob_xfer_temp_cmd_orig(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_file_xfer_regy))				return Add(new Xob_xfer_regy_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_file_xfer_regy_update))		return Add(new Xob_xfer_regy_update_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_file_fsdb_make))				return Add(new Xob_fsdb_make_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_file_orig_reg))				return Add(new Xob_orig_tbl_bldr(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_file_xfer_update))			return Add(new Xob_xfer_update_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_html_redlinks))				return Add(new Xob_redlink_mkr_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_util_cleanup))				return Add(new Xob_cleanup_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_util_download))				return Add(new Xob_download_wkr(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_util_xml_dump))				return Add(new Xob_xml_dumper_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_json_dump))				return Add(new Xob_wbase_json_dump_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_qid))					return Xml_rdr_direct_add(wiki, new Xob_wdata_qid_sql().Ctor(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_pid))					return Xml_rdr_direct_add(wiki, new Xob_wdata_pid_sql().Ctor(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_db))					return Add(new Xob_wdata_db_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_ns))					return Add(new Xob_site_ns_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_init))				return Add(new Xob_init_tdb(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_page))				return Xml_rdr_direct_add(wiki, new Xob_page_txt(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_id))					return Xml_rdr_direct_add(wiki, new Xob_make_id_wkr(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_search_title))		return Xml_rdr_direct_add(wiki, new Xob_search_tdb(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_category))			return Xml_rdr_parser_add(wiki, new Xob_ctg_v1_txt().Ctor(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_calc_stats))				return Add(new Xob_calc_stats_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_cat_link))			return Add(new Xob_categorylinks_txt(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_ctg_link_idx))			return Add(new Xoctg_link_idx_wkr(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_cat_hidden_sql))			return Add(new Xoctg_hiddencat_parser_txt(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_cat_hidden_ttl))			return Add(new Xoctg_hiddencat_ttl_wkr(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_core_term))				return Add(new Xob_term_txt(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_wdata_qid))			return Xml_rdr_direct_add(wiki, new Xob_wdata_qid_txt().Ctor(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_wdata_pid))			return Xml_rdr_direct_add(wiki, new Xob_wdata_pid_txt().Ctor(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_diff_regy_exec))				return Add(new Xob_diff_regy_exec_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_diff_regy_make))				return Add(new Xob_diff_regy_make_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_exec_sql))					return Add(new Xob_exec_sql_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_decompress_bz2))				return Add(new Xob_decompress_bz2_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_deploy_zip))					return Add(new Xob_deploy_zip_cmd(bldr, wiki));
+		else if	(String_.Eq(cmd_key, Xob_cmd_keys.Key_deploy_copy))					return Add(new Xob_deploy_copy_cmd(bldr, wiki));
+		else 																		throw Exc_.new_unhandled(cmd_key);
+	}
+	private Xobd_wkr Xml_rdr_direct_add(Xowe_wiki wiki, Xobd_wkr wkr) {
+		Xobd_rdr dump_rdr = Xml_rdr_get(wiki);
+		dump_rdr.Wkr_add(wkr);
+		return wkr;
+	}
+	private Xobd_parser_wkr Xml_rdr_parser_add(Xowe_wiki wiki, Xobd_parser_wkr wkr) {
+		Xobd_rdr dump_rdr = Xml_rdr_get(wiki);
+		dump_rdr.Page_parser_assert().Wkr_add(wkr);
+		return wkr;
+	}
+	private Xobd_rdr Xml_rdr_get(Xowe_wiki wiki) {
+		byte[] wiki_key = wiki.Domain_bry();
+		Xobd_rdr rv = (Xobd_rdr)dump_rdrs.Get_by(dump_rdrs_ref.Val_(wiki_key));
+		if (rv == null) {
+			rv = new Xobd_rdr(bldr, wiki);
+			dump_rdrs.Add(Bry_obj_ref.new_(wiki_key), rv);
+			this.Add(rv);
+		}
+		return rv;
+	}
+	private Hash_adp dump_rdrs = Hash_adp_.new_(); private Bry_obj_ref dump_rdrs_ref = Bry_obj_ref.null_();
+	public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if			(ctx.Match(k, Invk_add))				return Add_cmd(Wiki_get_or_make(m), m.ReadStr("v"));
+		else if		(ctx.Match(k, Invk_add_many))			return Add_many(m);
+		else if		(ctx.Match(k, Invk_get_first))			return Get_first(m);
+		else if		(ctx.Match(k, Invk_new_batch))			return new Xob_core_batch_utl(bldr, m.ReadBry("v"));
+		else	return GfoInvkAble_.Rv_unhandled;
+	}
+	private static final String Invk_add = "add", Invk_add_many = "add_many", Invk_new_batch = "new_batch", Invk_get_first = "get_first";
+	private Object Get_first(GfoMsg m) {
+		String cmd_key = m.ReadStr("v");
+		int cmds_len = list.Count();
+		for (int i = 0;i < cmds_len; i++) {
+			Xob_cmd cmd = (Xob_cmd)list.Get_at(i);
+			if (String_.Eq(cmd.Cmd_key(), cmd_key)) return cmd;
+		}
+		throw Exc_.new_("cmd not found", "key", cmd_key);
+	}
+	private Object Add_many(GfoMsg m) {
+		Xowe_wiki wiki = Wiki_get_or_make(m);
+		wiki.Lang().Init_by_load_assert();	// NOTE: must check that lang is loaded; else case_mgr will not initialize; DATE:2013-05-11
+		int args_len = m.Args_count();
+		String[] cmds = new String[args_len - 1];	// -1 b/c 1st arg is wiki
+		for (int i = 1; i < args_len; i++) {
+			KeyVal kv = m.Args_getAt(i);
+			cmds[i - 1] = kv.Val_to_str_or_empty();
+		}
+		return Add_many(wiki, cmds);
+	}
+	public Object Add_many(Xowe_wiki wiki, String... cmds) {
+		int len = cmds.length; if (len == 0) throw Exc_.new_("add_many cannot have 0 cmds");
+		Object rv = null;
+		for (int i = 0; i < len; i++)
+			rv = Add_cmd(wiki, cmds[i]);
+		return rv;
+	}
+	private Xowe_wiki Wiki_get_or_make(GfoMsg m) {
+		byte[] wiki_key = m.ReadBry("v");
+		Xoae_wiki_mgr wiki_mgr = bldr.App().Wiki_mgr();
+		Xowe_wiki rv = wiki_mgr.Get_by_key_or_make(wiki_key);
+		rv.Lang().Init_by_load();
+		return rv;
+	}
+	public static final String GRP_KEY = "xowa.bldr.cmds";
+}

400_xowa/src/gplx/xowa/bldrs/Xob_db_file.java

@@ -0,0 +1,49 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
+import gplx.dbs.*; import gplx.dbs.cfgs.*;
+public class Xob_db_file {
+	Xob_db_file(Io_url url, Db_conn conn) {
+		this.url = url; this.conn = conn;
+		this.tbl__cfg = new Db_cfg_tbl(conn, "xowa_cfg");
+	}
+	public Io_url			Url()		{return url;} private final Io_url url;
+	public Db_conn			Conn()		{return conn;} private final Db_conn conn;
+	public Db_cfg_tbl		Tbl__cfg()	{return tbl__cfg;} private final Db_cfg_tbl tbl__cfg;
+
+	public static Xob_db_file new__file_make(Io_url dir)			{return new_(dir, Name__file_make);}
+	public static Xob_db_file new__page_regy(Io_url dir)			{return new_(dir, Name__page_regy);}
+	public static Xob_db_file new__wiki_image(Io_url dir)			{return new_(dir, Name__wiki_image);}
+	public static Xob_db_file new__wiki_redirect(Io_url dir)		{return new_(dir, Name__wiki_redirect);}
+	public static Xob_db_file new__temp_log(Io_url dir)				{return new_(dir, Name__temp_log);}
+	public static Xob_db_file new__redlink(Io_url dir)				{return new_(dir, Name__redlink);}
+	public static Xob_db_file new_(Io_url dir, String name) {
+		Io_url url = dir.GenSubFil(name);
+		Db_conn_bldr_data conn_data = Db_conn_bldr.I.Get_or_new(url);
+		Db_conn conn = conn_data.Conn();
+		Xob_db_file rv = new Xob_db_file(url, conn);
+		if (conn_data.Created())
+			rv.Tbl__cfg().Create_tbl();
+		return rv;
+	}
+	public static final String 
+	  Name__wiki_image = "xowa.wiki.image.sqlite3", Name__wiki_redirect = "xowa.wiki.redirect.sqlite3"
+	, Name__file_make = "xowa.file.make.sqlite3", Name__temp_log = "xowa.temp.log.sqlite3"
+	, Name__page_regy = "xowa.file.page_regy.sqlite3", Name__redlink = "xowa.temp.redlink.sqlite3"
+	;
+}

400_xowa/src/gplx/xowa/bldrs/Xob_fxt.java

@@ -0,0 +1,161 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
+import gplx.ios.*; import gplx.dbs.*; import gplx.xowa.tdbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*;
+public class Xob_fxt {
+	public Xob_fxt Ctor_mem() {
+		Io_mgr.I.InitEngine_mem();
+		return Ctor(Io_url_.mem_dir_("mem/xowa/"));
+	}
+	public Xob_fxt Ctor(Io_url root_dir) {
+		Db_conn_bldr.I.Reg_default_sqlite();
+		app = Xoa_app_fxt.app_("linux", root_dir);
+		wiki = Xoa_app_fxt.wiki_tst_(app);
+		bldr = Xoa_app_fxt.bldr_(app);
+		return this;
+	}
+	public Xoae_app App() {return app;} private Xoae_app app;
+	public Xob_bldr Bldr() {return bldr;} private Xob_bldr bldr;
+	public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki;
+	public Io_url fil_ns_title(int ns_id, int idx)	{return wiki.Tdb_fsys_mgr().Url_ns_fil(Xotdb_dir_info_.Tid_ttl, ns_id, idx);}
+	public Io_url fil_ns_page(int ns_id, int idx)	{return wiki.Tdb_fsys_mgr().Url_ns_fil(Xotdb_dir_info_.Tid_page, ns_id, idx);}
+	public Io_url fil_ns_sttl(int ns_id, int idx)	{return wiki.Tdb_fsys_mgr().Url_ns_fil(Xotdb_dir_info_.Tid_search_ttl, ns_id, idx);}
+	public Io_url fil_site(byte tid, int idx)		{return wiki.Tdb_fsys_mgr().Url_site_fil(tid, idx);}
+	public Io_url fil_site_ctg(int idx)				{return wiki.Tdb_fsys_mgr().Url_site_fil(Xotdb_dir_info_.Tid_category, idx);}
+	public Io_url fil_site_id(int idx)				{return wiki.Tdb_fsys_mgr().Url_site_fil(Xotdb_dir_info_.Tid_id, idx);}
+	public Io_url fil_reg(byte tid) 				{return wiki.Tdb_fsys_mgr().Url_site_reg(tid);}
+	public Io_url fil_reg(int ns_id, byte tid) 		{return wiki.Tdb_fsys_mgr().Url_ns_reg(Int_.Xto_str_pad_bgn_zero(ns_id, 3), tid);}
+	public Xob_fxt Fil_expd(Io_url url, String... expd) {
+		String text = String_.Concat_lines_nl_skip_last(expd);	// skipLast b/c if trailing line wanted, easier to pass in extra argument for ""
+		expd_list.Add(new Io_fil_chkr(url, text));
+		return this;
+	} List_adp expd_list = List_adp_.new_();
+	public Xob_fxt Fil_skip(Io_url... urls) {
+		for (int i = 0; i < urls.length; i++)
+			skip_list.Add(urls[i]);
+		return this;
+	} 	List_adp skip_list = List_adp_.new_();
+	public Xob_fxt doc_ary_(Xowd_page_itm... v) {doc_ary = v; return this;} private Xowd_page_itm[] doc_ary;
+	public Xowd_page_itm doc_wo_date_(int id, String title, String text) {return doc_(id, "2012-01-02 13:14", title, text);}
+	public Xowd_page_itm doc_(int id, String date, String title, String text) {
+		Xowd_page_itm rv = new Xowd_page_itm().Id_(id).Ttl_(Bry_.new_u8(title), wiki.Ns_mgr()).Text_(Bry_.new_u8(text));
+		int[] modified_on = new int[7];
+		dateParser.Parse_iso8651_like(modified_on, date);
+		rv.Modified_on_(DateAdp_.seg_(modified_on));
+		return rv;
+	}
+	public Xob_fxt Run_ctg() {
+		Xobd_parser parser = new Xobd_parser();
+		gplx.xowa.bldrs.cmds.ctgs.Xob_ctg_v1_base ctg_wkr = new gplx.xowa.bldrs.cmds.ctgs.Xob_ctg_v1_txt().Ctor(bldr, wiki);
+		byte[] bry = Bry_.new_u8("[[Category:");
+		ctg_wkr.Wkr_hooks().Add(bry, bry);
+		parser.Wkr_add(ctg_wkr);
+		return Run(parser);
+	}
+	public Xob_fxt Run_id() {
+		Xob_make_id_wkr wkr = new Xob_make_id_wkr(bldr, wiki);
+		Run(wkr);
+		return this;
+	}
+	private void Run_wkr(Xobd_wkr wkr) {
+		wkr.Wkr_bgn(bldr);
+		for (int i = 0; i < doc_ary.length; i++) {
+			Xowd_page_itm page = doc_ary[i];
+			wkr.Wkr_run(page);
+		}
+		wkr.Wkr_end();		
+	}
+	private void tst_fils(Io_url[] ary) {
+		Io_fil[] actls = Get_actl(ary);
+		Io_fil_chkr[] expds = (Io_fil_chkr[])expd_list.To_ary(Io_fil_chkr.class);
+		tst_mgr.Tst_ary("all", expds, actls);		
+	}
+	Io_fil[] Get_actl(Io_url[] ary) {
+		int len = ary.length;
+		Io_fil[] rv = new Io_fil[len];
+		for (int i = 0; i < len; i++) {
+			Io_url url = ary[i];
+			String data = Io_mgr.I.LoadFilStr(url);
+			rv[i] = new Io_fil(url, data);
+		}
+		return rv;
+	}
+	public Xob_fxt Run_tmpl_dump() {
+		Xob_parse_dump_templates_cmd wkr = new Xob_parse_dump_templates_cmd(bldr, wiki);
+		Run_wkr(wkr);
+		tst_fils(wkr.Dump_url_gen().Prv_urls());
+		return this;
+	}
+	public Xob_fxt Run_page_title() {return Run(new gplx.xowa.bldrs.cmds.texts.tdbs.Xob_page_txt(bldr, wiki));}
+	public Xob_fxt Run(Xobd_parser_wkr... wkrs) {
+		Xobd_parser parser_wkr = new Xobd_parser();
+		int len = wkrs.length;
+		for (int i = 0; i < len; i++)
+			parser_wkr.Wkr_add(wkrs[i]);
+		Run(parser_wkr);
+		return this;
+	}
+	public Xob_fxt Run(Xobd_wkr... wkrs) {
+		int doc_ary_len = doc_ary.length;
+		for (int j = 0; j < wkrs.length; j++) {
+			Xobd_wkr wkr = wkrs[j];
+			wkr.Wkr_bgn(bldr);
+			for (int i = 0; i < doc_ary_len; i++) {
+				Xowd_page_itm page = doc_ary[i];
+				wkr.Wkr_run(page);
+			}
+			wkr.Wkr_end();
+		}
+		Test_expd_files();
+		return this;
+	}
+	public Xob_fxt Run_cmds(Xob_cmd... cmds) {
+		for (int j = 0; j < cmds.length; j++) {
+			Xob_cmd cmd = cmds[j];
+			cmd.Cmd_bgn(bldr);
+			cmd.Cmd_run();
+			cmd.Cmd_end();
+		}
+		Test_expd_files();
+		return this;
+	}
+	private void Test_expd_files() {
+		if (expd_list.Count() > 0) {
+			Io_fil_chkr[] expd = (Io_fil_chkr[])expd_list.To_ary(Io_fil_chkr.class);
+			Io_fil[] actl = wiki_();
+			tst_mgr.Tst_ary("all", expd, actl);
+		}
+	}
+	Io_fil[] wiki_() {
+		List_adp rv = List_adp_.new_();
+		wiki_fil_add(rv, wiki.Tdb_fsys_mgr().Ns_dir());
+		wiki_fil_add(rv, wiki.Tdb_fsys_mgr().Site_dir());
+		rv.Sort();
+		return (Io_fil[])rv.To_ary(Io_fil.class);
+	}
+	private void wiki_fil_add(List_adp list, Io_url root_dir) {
+		Io_url[] ary = Io_mgr.I.QueryDir_args(root_dir).Recur_().ExecAsUrlAry();
+		for (int i = 0; i < ary.length; i++) {
+			Io_url url = ary[i]; 
+			Io_fil fil = new Io_fil(url, Io_mgr.I.LoadFilStr_args(url).MissingIgnored_().Exec());
+			list.Add(fil);
+		}		
+	}
+	Tst_mgr tst_mgr = new Tst_mgr();
+	DateAdp_parser dateParser = DateAdp_parser.new_();
+} 

400_xowa/src/gplx/xowa/bldrs/Xob_ns_to_db_mgr.java

@@ -0,0 +1,88 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
+import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.cmds.*;
+public class Xob_ns_to_db_mgr {
+	private final Xob_ns_to_db_wkr wkr; private final Xowd_db_mgr db_mgr; private final long db_max; private boolean one_file_conn_init = true;
+	private final Ordered_hash db_list = Ordered_hash_.new_();
+	public Xob_ns_to_db_mgr(Xob_ns_to_db_wkr wkr, Xowd_db_mgr db_mgr, long db_max) {
+		this.wkr = wkr; this.db_mgr = db_mgr; this.db_max = db_max;
+	}
+	public Xowd_db_file Get_by_ns(Xob_ns_file_itm ns_file_itm, int data_len) {
+		Xowd_db_file rv = null;
+		if		(wkr.Db_tid() == Xowd_db_file_.Tid_text			&& db_mgr.Props().Layout_text().Tid_is_all_or_few()) {
+			rv = db_mgr.Db__core();
+			if (one_file_conn_init) {
+				one_file_conn_init = false;
+				Init_tbl(rv);
+			}
+		}
+		else if (wkr.Db_tid() == Xowd_db_file_.Tid_html_data	&& db_mgr.Props().Layout_html().Tid_is_all_or_few()) {
+			if (one_file_conn_init) {
+				one_file_conn_init = false;
+				rv = db_mgr.Dbs__make_by_tid(wkr.Db_tid());
+				Init_tbl(rv);
+			}
+			else
+				rv = db_mgr.Db__html();
+		}
+		else {
+			int db_id = ns_file_itm.Nth_db_id();
+			if (db_id == Xob_ns_file_itm.Nth_db_id_null)	// ns not assigned yet to db
+				rv = Init_db(ns_file_itm);
+			else
+				rv = db_mgr.Dbs__get_at(db_id);
+			long file_len = rv.File_len();
+			if (file_len + data_len > db_max) {				// file is "full"
+				Term_tbl(rv);
+				rv = Init_db(ns_file_itm);
+			}
+		}
+		rv.File_len_add(data_len);
+		return rv;
+	}
+	private Xowd_db_file Init_db(Xob_ns_file_itm ns_file_itm) {
+		Xowd_db_file rv = db_mgr.Dbs__make_by_tid(ns_file_itm.Db_file_tid(), Int_.Xto_str(ns_file_itm.Ns_ids(), "|"), ns_file_itm.Nth_db_idx(), ns_file_itm.Make_file_name());
+		ns_file_itm.Nth_db_id_(rv.Id());
+		Init_tbl(rv);
+		return rv;
+	}
+	private void Init_tbl(Xowd_db_file db) {
+		wkr.Tbl_init(db);
+		db_list.Add(db.Id(), db);
+	}
+	private void Term_tbl(Xowd_db_file db) {
+		wkr.Tbl_term(db);
+		db_list.Del(db.Id());
+	}
+	public void Rls_all() {
+		Xowd_db_file[] ary = (Xowd_db_file[])db_list.To_ary(Xowd_db_file.class);
+		int len = ary.length;
+		for (int i = 0; i < len; ++i) {
+			Xowd_db_file db = (Xowd_db_file)ary[i];
+			Term_tbl(db); // SQLITE:1_TXN; may call close on db where txn is already closed
+		}
+	}
+	public void Commit() {
+		int len = db_list.Count();
+		for (int i = 0; i < len; ++i) {
+			Xowd_db_file db = (Xowd_db_file)db_list.Get_at(i);
+			db.Conn().Txn_sav();
+		}
+	}
+}

400_xowa/src/gplx/xowa/bldrs/Xob_ns_to_db_wkr.java

@@ -0,0 +1,24 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
+import gplx.xowa.wikis.data.*;
+public interface Xob_ns_to_db_wkr {
+	byte Db_tid();
+	void Tbl_init(Xowd_db_file db);
+	void Tbl_term(Xowd_db_file db);
+}

400_xowa/src/gplx/xowa/bldrs/aria2/Aria2_lib_mgr.java

@@ -0,0 +1,50 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.aria2; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
+import gplx.xowa.apps.fsys.*;
+public class Aria2_lib_mgr implements GfoInvkAble {
+	public ProcessAdp Lib() {return lib;} private ProcessAdp lib = new ProcessAdp();
+	public void Init_by_app(Xoae_app app) {
+		Xoa_fsys_eval cmd_eval = app.Url_cmd_eval();
+		ProcessAdp.ini_(this, app.Usr_dlg(), lib, cmd_eval, ProcessAdp.Run_mode_sync_block, Int_.MaxValue
+		, "~{<>bin_plat_dir<>}aria2" + Op_sys.Cur().Fsys_dir_spr_str() +  "aria2c"
+		, Lib_args_fmt
+		, "wiki_abrv", "wiki_date", "wiki_type");
+	}
+	// private Bry_bfr tmp_bfr = Bry_bfr.reset_(255);
+	public void Exec(Xob_dump_file dump_file) {			
+		// byte[] args_bry = lib.Args_fmtr().Bld_bry_many(tmp_bfr, dump_file.Wiki_alias(), dump_file.Dump_date(), dump_file.Dump_file_type());
+		// ProcessAdp process = new ProcessAdp().Exe_url_(lib.Exe_url()).Args_str_(String_.new_u8(args_bry));
+		// process.Run_wait();			
+	}
+	public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_lib))				return lib;
+		else	return GfoInvkAble_.Rv_unhandled;
+	}
+	private static final String Invk_lib = "lib";
+	private static final String Lib_args_fmt = String_.Concat
+	( "--max-connection-per-server=2"
+	, " --max-concurrent-downloads=20"
+	, " --split=4"
+	, " --file-allocation=prealloc"
+	, " --remote-time=true"
+	, " --server-stat-of=serverstats.txt"
+	, " ftp://ftpmirror.your.org/pub/wikimedia/dumps/~{wiki_abrv}/~{wiki_date}/~{wiki_abrv}-~{wiki_date}-~{wiki_type}.bz2"
+	, " https://dumps.wikimedia.org/~{wiki_abrv}/~{wiki_date}/~{wiki_abrv}-~{wiki_date}-~{wiki_type}.xml.bz2"
+	);
+}

400_xowa/src/gplx/xowa/bldrs/aria2/Gfui_process_win.java

@@ -0,0 +1,40 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.aria2; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
+import gplx.gfui.*;
+class Gfui_process_win {
+	public void Exec_async(String process, String args, GfoInvkAbleCmd done_cbk) {
+		// Gfo_process process = new Gfo_process().Init_process_(process, args).Init_async_(done_cbk).Init_strm_out_err_(output_box).Exec();			
+	}
+}
+class Gfo_process {
+//		private GfoInvkAbleCmd done_cbk;
+//		private Gfo_process_wtr out_wtr, err_wtr;
+	public String Cmd_path() {return cmd_path;} private String cmd_path;
+	public String Cmd_args() {return cmd_args;} private String cmd_args;
+	public byte Mode() {return mode;} private byte mode;
+	public Gfo_process Init_cmd_(String cmd_path, String cmd_args) {this.cmd_path = cmd_path; this.cmd_args = cmd_args; return this;}
+	public Gfo_process Init_mode_async_() {mode = Gfo_process_.Mode_async; return this;}
+//		public Gfo_process Init_mode_async_(GfoInvkAbleCmd done_cbk) {this.done_cbk = done_cbk; return this.Init_mode_async_();}
+//		public Gfo_process Init_wtr_out_err_(Gfo_process_wtr wtr) {out_wtr = err_wtr = wtr; return this;}
+}
+class Gfo_process_wtr {}
+class Gfo_process_rdr {}
+class Gfo_process_ {
+	public static final byte Mode_async = 0, Mode_sync = 1, Mode_sync_timeout = 2; 
+}

400_xowa/src/gplx/xowa/bldrs/cfgs/Xob_wiki_cfg_bldr.java

@@ -0,0 +1,56 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
+import gplx.core.strings.*;
+public class Xob_wiki_cfg_bldr implements GfoInvkAble {
+	public Xob_wiki_cfg_bldr(Xob_bldr bldr) {this.app = bldr.App();} private Xoae_app app;
+	public void Exec() {
+		int len = hash.Count();
+		for (int i = 0; i < len; i++) {
+			Xoac_wiki_cfg_bldr_fil fil = (Xoac_wiki_cfg_bldr_fil)hash.Get_at(i);
+			Exec_fil(fil);
+		}
+	}
+	private void Exec_fil(Xoac_wiki_cfg_bldr_fil fil) {
+		String wiki_key = fil.Wiki();
+		Io_url cfg_file = app.Fsys_mgr().Cfg_wiki_core_dir().GenSubFil(wiki_key + ".gfs");
+		String cfg_text = Io_mgr.I.LoadFilStr_args(cfg_file).MissingIgnored_().Exec();
+		int len = fil.Itms_count();
+		String_bldr sb = String_bldr_.new_();
+		for (int i = 0; i < len; i++) {
+			Xoac_wiki_cfg_bldr_cmd cmd = fil.Itms_get_at(i);
+			cfg_text = cmd.Exec(sb, wiki_key, cfg_text);
+		}
+		Io_mgr.I.SaveFilStr(cfg_file, cfg_text);
+	}
+	public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_get))		 		return Itms_get_or_new(m.ReadStr("v"));
+		else if	(ctx.Match(k, Invk_run))		 		Exec();
+		else	return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}	private static final String Invk_get = "get", Invk_run = "run";
+	public void Clear() {hash.Clear();}
+	public Xoac_wiki_cfg_bldr_fil Itms_get_or_new(String wiki) {
+		Xoac_wiki_cfg_bldr_fil rv = (Xoac_wiki_cfg_bldr_fil)hash.Get_by(wiki);
+		if (rv == null) {
+			rv = new Xoac_wiki_cfg_bldr_fil(wiki);
+			hash.Add(wiki, rv);
+		}
+		return rv;
+	}	private Ordered_hash hash = Ordered_hash_.new_();
+}

400_xowa/src/gplx/xowa/bldrs/cfgs/Xob_wiki_cfg_bldr_tst.java

@@ -0,0 +1,179 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
+import org.junit.*; import gplx.core.strings.*;
+public class Xob_wiki_cfg_bldr_tst {
+	Xob_wiki_cfg_bldr_fxt fxt = new Xob_wiki_cfg_bldr_fxt();
+	@Before public void init() {fxt.Clear();}
+	@Test  public void Exec() {
+		fxt	.Init_cmd("en.wikipedia.org", "key0", "en.val0")
+			.Init_cmd("en.wikipedia.org", "key1", "en.val1")
+			.Init_cmd("fr.wikipedia.org", "key0", "fr.val0")
+			.Init_cmd("fr.wikipedia.org", "key1", "fr.val1")
+			.Expd_txt("en.wikipedia.org", String_.Concat_lines_nl
+			(	"// key0.bgn"
+			,	"en.val0"
+			,	"// key0.end"
+			,	"// key1.bgn"
+			,	"en.val1"
+			,	"// key1.end"
+			))
+			.Expd_txt("fr.wikipedia.org", String_.Concat_lines_nl
+			(	"// key0.bgn"
+			,	"fr.val0"
+			,	"// key0.end"
+			,	"// key1.bgn"
+			,	"fr.val1"
+			,	"// key1.end"
+			))
+			.Test()
+			;
+		fxt	.Clear()
+			.Init_cmd("en.wikipedia.org", "key2", "en.val2")
+			.Expd_txt("en.wikipedia.org", String_.Concat_lines_nl
+			(	"// key0.bgn"
+			,	"en.val0"
+			,	"// key0.end"
+			,	"// key1.bgn"
+			,	"en.val1"
+			,	"// key1.end"
+			,	"// key2.bgn"
+			,	"en.val2"
+			,	"// key2.end"
+			));
+	}
+//		@Test  public void Lang_names_run() {
+//			Io_url dir = Io_url_.new_dir_("/var/www/mediawiki/languages/messages/");
+//			Io_url[] fils = Io_mgr.I.QueryDir_args(dir).ExecAsUrlAry();
+//			int fils_len = fils.length;
+//			String_bldr sb = String_bldr_.new_();
+//			for (int i = 0; i < fils_len; i++) {
+//				Io_url fil = fils[i];
+//				String lang_code = String_.Lower(String_.Replace(fil.NameOnly(), "Messages", ""));
+//				String txt = Io_mgr.I.LoadFilStr(fil);
+//				String[] lines = String_.Split(txt, '\n');
+//				String line = lines[1];
+//				line = String_.Replace(line, "/** ", "");
+//				int pos = String_.FindBwd(line, " (");
+//				if (pos == String_.Find_none) continue;	// en; en_rtl have no "language" line
+//				if (	String_.Has_at_bgn(lang_code, "be_")
+//					|| 	String_.Has_at_bgn(lang_code, "crh_")
+//					|| 	String_.Has_at_bgn(lang_code, "kk_")
+//					|| 	String_.Has_at_bgn(lang_code, "ku_")
+//					|| 	String_.Has_at_bgn(lang_code, "sr_")
+//					|| 	String_.In(lang_code, "de_formal", "nb", "nl_informal", "nn", "no")
+//					) {
+//					int new_pos = String_.FindBwd(line, " (", pos - 1);
+//					if (new_pos != String_.Find_none)
+//						pos = new_pos;
+//				}
+//				line = Replace_by_pos(line, pos, pos + 2, "|");
+//				int line_len = String_.Len(line);
+//				if (line_len > 0)
+//					line = String_.MidByLen(line, 0, line_len - 1);
+//				String[] terms = String_.Split(line, '|');
+//				sb.Add(lang_code).Add("|").Add(String_.Trim(terms[0])).Add("|").Add(String_.Trim(terms[1])).Add("\n");
+//			}
+//			Tfds.Write(sb.Xto_str_and_clear());
+//		}
+	@Test  public void Ns_aliases() {
+		Io_mgr.I.InitEngine_mem();
+		Io_mgr.I.SaveFilStr("mem/en.wikipedia.org/w/api.php?action=query&format=xml&meta=siteinfo&siprop=namespacealiases", String_.Concat_lines_nl
+		(	"<api>"
+		,	"<query>"
+		,	"<namespacealiases>"
+		,	"<ns id=\"4\" xml:space=\"preserve\">WP"
+		,	"</ns>"
+		,	"<ns id=\"5\" xml:space=\"preserve\">WT"
+		,	"</ns>"
+		,	"<ns id=\"6\" xml:space=\"preserve\">Image"
+		,	"</ns>"
+		,	"<ns id=\"7\" xml:space=\"preserve\">Image talk"
+		,	"</ns>"
+		,	"</namespacealiases>"
+		,	"</query>"
+		,	"</api>"
+		));
+		String[] wikis = new String[] {"en.wikipedia.org"}; String protocol = "mem/";
+		Tfds.Eq_str_lines(Query_ns(protocol, gplx.ios.IoEngine_.MemKey, wikis), String_.Concat_lines_nl
+		(	"app.bldr.wiki_cfg_bldr.get('en.wikipedia.org').new_cmd_('wiki.ns_mgr.aliases', 'ns_mgr.add_alias_bulk(\""
+		,	"4|WP"
+		,	"5|WT"
+		,	"6|Image"
+		,	"7|Image_talk"
+		,	"\");');"
+		));
+	}
+	String Query_ns(String protocol, String trg_engine_key, String[] wikis) {
+		String_bldr sb = String_bldr_.new_();
+		int wikis_len = wikis.length;
+		for (int i = 0; i < wikis_len; i++) {
+			String wiki = wikis[i];
+			if (String_.Len_eq_0(wiki)) continue;
+			try {
+			String api = protocol + wiki + "/w/api.php?action=query&format=xml&meta=siteinfo&siprop=namespacealiases";
+			String xml = String_.new_u8(Io_mgr.I.DownloadFil_args("", null).Trg_engine_key_(trg_engine_key).Exec_as_bry(api));
+			if (xml == null) continue;	// not found
+			gplx.xmls.XmlDoc xdoc = gplx.xmls.XmlDoc_.parse_(xml);
+			gplx.xmls.XmlNde xnde = gplx.xmls.Xpath_.SelectFirst(xdoc.Root(), "query/namespacealiases");
+			sb.Add("app.bldr.wiki_cfg_bldr.get('").Add(wiki).Add("').new_cmd_('wiki.ns_mgr.aliases', 'ns_mgr.add_alias_bulk(\"\n");
+			int xndes_len = xnde.SubNdes().Count();
+			for (int j = 0; j < xndes_len; j++) {
+				gplx.xmls.XmlNde ns_nde = xnde.SubNdes().Get_at(j);
+				if (!String_.Eq(ns_nde.Name(), "ns")) continue;
+				int id = Int_.parse_(ns_nde.Atrs().FetchValOr("id", "-1"));
+				String name = String_.Replace(String_.Replace(ns_nde.Text_inner(), " ", "_"), "'", "''");
+				sb.Add(Int_.Xto_str(id)).Add("|").Add(String_.Trim(name)).Add_char_nl();
+			}
+			sb.Add("\");');\n");
+			}
+			catch(Exception e) {sb.Add("// fail: " + wiki + " " + Err_.Message_gplx_brief(e)).Add_char_nl();}
+		}
+		return sb.Xto_str_and_clear();
+	}
+}
+class Xob_wiki_cfg_bldr_fxt {
+	public Xob_wiki_cfg_bldr_fxt Clear() {
+		if (app == null) {
+			app = Xoa_app_fxt.app_();
+			wiki_cfg_bldr = app.Bldr().Wiki_cfg_bldr();
+		}
+		wiki_cfg_bldr.Clear();
+		hash.Clear();
+		return this;
+	}	private Xoae_app app; Xob_wiki_cfg_bldr wiki_cfg_bldr; Ordered_hash hash = Ordered_hash_.new_();
+	public Xob_wiki_cfg_bldr_fxt Init_cmd(String wiki, String key, String text) {
+		wiki_cfg_bldr.Itms_get_or_new(wiki).Itms_add(key, text);
+		return this;
+	}
+	public Xob_wiki_cfg_bldr_fxt Expd_txt(String wiki, String text) {
+		hash.Add(wiki, KeyVal_.new_(wiki, text));
+		return this;
+	}
+	public void Test() {
+		wiki_cfg_bldr.Exec();
+		int len = hash.Count();
+		for (int i = 0; i < len; i++) {
+			KeyVal kv = (KeyVal)hash.Get_at(i);
+			String wiki = kv.Key();
+			String expd = (String)kv.Val();
+			String actl = Io_mgr.I.LoadFilStr(app.Fsys_mgr().Cfg_wiki_core_dir().GenSubFil(wiki + ".gfs"));
+			Tfds.Eq_str_lines(expd, actl);
+		}
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/Xob_dump_mgr_base.java

@@ -0,0 +1,315 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
+import gplx.dbs.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.bldrs.cmds.files.*; import gplx.xowa.files.origs.*;
+import gplx.xowa.wikis.data.*; import gplx.xowa.dbs.*; import gplx.xowa.wikis.data.tbls.*;
+public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xob_cmd, GfoInvkAble {
+	private Xob_dump_src_id page_src;
+	private Xowd_db_mgr db_fsys_mgr; protected Xop_parser parser; protected Xop_ctx ctx; protected Xop_root_tkn root;
+	private int[] ns_ary; private Xowd_db_file[] db_ary;
+	private int ns_bgn = -1, db_bgn = -1, pg_bgn = -1;
+	private int ns_end = -1, db_end = -1, pg_end = Int_.MaxValue;
+	private int commit_interval = 1000, progress_interval = 250, cleanup_interval = 2500, select_size = 10 * Io_mgr.Len_mb;
+	private int exec_count, exec_count_max = Int_.MaxValue;
+	private boolean reset_db = false, exit_after_commit = false, exit_now = false;
+	private boolean load_tmpls;
+	private Xob_dump_bmk_mgr bmk_mgr = new Xob_dump_bmk_mgr();
+	private Xobu_poll_mgr poll_mgr; private int poll_interval = 5000;
+	private Xob_rate_mgr rate_mgr = new Xob_rate_mgr();
+	public abstract String Cmd_key();
+	@Override protected void Cmd_ctor_end(Xob_bldr bldr, Xowe_wiki wiki) {
+		poll_mgr = new Xobu_poll_mgr(bldr.App());	// init in ctor so gfs can invoke methods
+	}
+	public void Cmd_bgn(Xob_bldr bldr) {
+		parser = wiki.Parser();
+		ctx = wiki.Ctx();
+		root = ctx.Tkn_mkr().Root(Bry_.Empty);			
+		wiki.Init_assert();	// NOTE: must init wiki for db_mgr_as_sql		
+		wiki.Db_mgr_as_sql().Core_data_mgr().Init_by_load(gplx.xowa.wikis.Xow_fsys_mgr.Find_core_fil(wiki));	// NOTE: must reinit providers as previous steps may have rls'd (and left member variable conn which is closed)
+		wiki.File__orig_mgr().Wkrs_del(Xof_orig_wkr_.Tid_wmf_api);
+		db_fsys_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
+		db_ary = Xob_dump_mgr_base_.Init_text_files_ary(db_fsys_mgr);
+		poll_interval = poll_mgr.Poll_interval();
+
+		page_src = new Xob_dump_src_id().Init(wiki, this.Init_redirect(), select_size);
+		ns_ary = Init_ns_ary();						
+		Db_conn conn = Init_db_file();
+		Io_url wiki_dir = wiki.Fsys_mgr().Root_dir();
+		bmk_mgr.Cfg_url_(wiki_dir.GenSubFil("xowa.file.make.cfg.gfs"));
+		rate_mgr.Log_file_(wiki_dir.GenSubFil("xowa.file.make.log.csv"));
+		if (reset_db) {
+			bmk_mgr.Reset();
+			Init_reset(conn);
+		}
+		bmk_mgr.Load(wiki.Appe(), this);
+		Cmd_bgn_end();
+	}
+	protected abstract void Cmd_bgn_end();
+	public abstract byte Init_redirect();
+	public abstract int[] Init_ns_ary();
+	protected abstract void Init_reset(Db_conn p);
+	protected abstract Db_conn Init_db_file();
+	private long time_bgn;
+	public void Cmd_run() {Exec_ns_ary();}
+	private void Exec_ns_ary() {
+		if (pg_bgn == Int_.MaxValue) return;
+		if (load_tmpls) Xob_dump_mgr_base_.Load_all_tmpls(usr_dlg, wiki, page_src);
+		time_bgn = Env_.TickCount();
+		Xob_dump_bmk dump_bmk = new Xob_dump_bmk();
+		rate_mgr.Init();
+		int ns_ary_len = ns_ary.length;
+		for (int i = 0; i < ns_ary_len; i++) {
+			int ns_id = ns_ary[i];
+			if (ns_bgn != -1) {							// ns_bgn set
+				if (ns_id == ns_bgn)					// ns_id is ns_bgn; null out ns_bgn and continue
+					ns_bgn = -1;
+				else									// ns_id is not ns_bgn; keep looking
+					continue;
+			}
+			dump_bmk.Ns_id_(ns_id);
+			Exec_db_ary(i, dump_bmk, ns_id);
+			if (ns_id == ns_end) exit_now = true;		// ns_end set; exit
+			if (exit_now) break;						// exit_now b/c of pg_bgn, db_bgn or something else
+		}
+		Exec_commit(dump_bmk.Ns_id(), dump_bmk.Db_id(), dump_bmk.Pg_id(), Bry_.Empty);
+	}
+	private void Exec_db_ary(int ns_ord, Xob_dump_bmk dump_bmk, int ns_id) {
+		int db_ary_len = db_ary.length;
+		for (int i = 0; i < db_ary_len; i++) {
+			int db_id = db_ary[i].Id();
+			if (db_bgn != -1) {							// db_bgn set
+				if (db_id == db_bgn)					// db_id is db_bgn; null out db_bgn and continue
+					db_bgn = -1;
+				else									// db_id is not db_bgn; keep looking
+					continue;
+			}
+			dump_bmk.Db_id_(db_id);
+			Exec_db_itm(dump_bmk, ns_ord, ns_id, db_id);
+			if (db_id == db_end) exit_now = true;		// db_end set; exit;
+			if (exit_now) return;						// exit_now b/c of pg_bgn, db_bgn or something else
+		}
+	}
+	private void Exec_db_itm(Xob_dump_bmk dump_bmk, int ns_ord, int ns_id, int db_id) {
+		List_adp pages = List_adp_.new_();
+		Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id);
+		int pg_id = pg_bgn;
+		while (true) {
+			page_src.Get_pages(pages, db_id, ns_id, pg_id);
+			int pages_len = pages.Count();
+			if (pages_len == 0) {	// no more pages in db;
+				if (pg_id > pg_bgn)	// reset pg_bgn to 0 only if pg_bgn seen;
+					pg_bgn = 0;
+				return;	
+			}
+			usr_dlg.Prog_many("", "", "fetched pages: ~{0}", pages_len);
+			for (int i = 0; i < pages_len; i++) {
+				Xowd_page_itm page = (Xowd_page_itm)pages.Get_at(i);
+				dump_bmk.Pg_id_(pg_id);
+				Exec_pg_itm(ns_ord, ns, db_id, page);
+				if (	pg_id		>= pg_end
+					||	exec_count	>= exec_count_max) {
+					exit_now = true;
+				}
+				if (exit_now) return;
+				pg_id = page.Id();
+			}
+		}
+	}
+	private void Exec_pg_itm(int ns_ord, Xow_ns ns, int db_id, Xowd_page_itm page) {
+		try {
+			if ((exec_count % progress_interval) == 0)
+				usr_dlg.Prog_many("", "", "parsing: ns=~{0} db=~{1} pg=~{2} count=~{3} time=~{4} rate=~{5} ttl=~{6}"
+					, ns.Id(), db_id, page.Id(), exec_count
+					, Env_.TickCount_elapsed_in_sec(time_bgn), rate_mgr.Rate_as_str(), String_.new_u8(page.Ttl_page_db()));
+			ctx.Clear();
+			Exec_pg_itm_hook(ns_ord, ns, page, page.Text());
+			ctx.App().Utl__bfr_mkr().Clear_fail_check();	// make sure all bfrs are released
+			if (ctx.Wiki().Cache_mgr().Tmpl_result_cache().Count() > 50000) 
+				ctx.Wiki().Cache_mgr().Tmpl_result_cache().Clear();
+			++exec_count;
+			rate_mgr.Increment();
+			if ((exec_count % poll_interval) == 0)
+				poll_mgr.Poll();
+			if	((exec_count % commit_interval) == 0)
+				Exec_commit(ns.Id(), db_id, page.Id(), page.Ttl_page_db());
+			if ((exec_count % cleanup_interval) == 0)
+				Free();
+		}
+		catch (Exception exc) {
+			bldr.Usr_dlg().Warn_many(GRP_KEY, "parse", "failed to parse ~{0} error ~{1}", String_.new_u8(page.Ttl_page_db()), Err_.Message_lang(exc));
+			ctx.App().Utl__bfr_mkr().Clear();
+			this.Free();
+		}
+	}
+	public abstract void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm page, byte[] page_text);
+	private void Exec_commit(int ns_id, int db_id, int pg_id, byte[] ttl) {
+		usr_dlg.Prog_many("", "", "committing: ns=~{0} db=~{1} pg=~{2} count=~{3} ttl=~{4}", ns_id, db_id, pg_id, exec_count, String_.new_u8(ttl));
+		Exec_commit_hook();
+		bmk_mgr.Save(ns_id, db_id, pg_id);
+		if (exit_after_commit) exit_now = true;
+	}
+	public abstract void Exec_commit_hook();
+	public abstract void Exec_end_hook();
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_term() {}		
+	public void Cmd_end() {
+		if (!exit_now)
+			pg_bgn = Int_.MaxValue;
+		Exec_commit(-1, -1, -1, Bry_.Empty);
+		Exec_end_hook();
+		Free();
+		usr_dlg.Note_many("", "", "done: ~{0} ~{1}", exec_count, DecimalAdp_.divide_safe_(exec_count, Env_.TickCount_elapsed_in_sec(time_bgn)).Xto_str("#,###.000"));
+	}
+	private void Free() {
+		ctx.App().Free_mem(true);
+		gplx.xowa.xtns.scribunto.Scrib_core.Core_invalidate();
+		wiki.Cache_mgr().Free_mem_all();
+	}
+	protected void Reset_db_y_() {this.reset_db = true;}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_commit_interval_))		commit_interval = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_progress_interval_))		progress_interval = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_rate_interval_))			rate_mgr.Reset_interval_(m.ReadInt("v"));
+		else if	(ctx.Match(k, Invk_cleanup_interval_))		cleanup_interval = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_select_size_))			select_size = m.ReadInt("v") * Io_mgr.Len_mb;
+		else if	(ctx.Match(k, Invk_ns_bgn_))				{ns_bgn = m.ReadInt("v"); Notify_restoring("ns", ns_bgn);}
+		else if	(ctx.Match(k, Invk_db_bgn_))				{db_bgn = m.ReadInt("v"); Notify_restoring("db", db_bgn);}
+		else if	(ctx.Match(k, Invk_pg_bgn_))				{pg_bgn = m.ReadInt("v"); Notify_restoring("pg", pg_bgn);}
+		else if	(ctx.Match(k, Invk_ns_end_))				ns_end = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_db_end_))				db_end = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_pg_end_))				pg_end = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_load_tmpls_))			load_tmpls = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_poll_mgr))				return poll_mgr;
+		else if	(ctx.Match(k, Invk_reset_db_))				reset_db = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_exec_count_max_))		exec_count_max = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_exit_now_))				exit_now = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_exit_after_commit_))		exit_after_commit = m.ReadYn("v");
+		else	return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}
+	private void Notify_restoring(String itm, int val) {
+		usr_dlg.Note_many("", "", "restoring: itm=~{0} val=~{1}", itm, val);
+	}
+	public static final String 
+	  Invk_progress_interval_ = "progress_interval_", Invk_commit_interval_ = "commit_interval_", Invk_cleanup_interval_ = "cleanup_interval_", Invk_rate_interval_ = "rate_interval_"
+	, Invk_select_size_ = "select_size_"
+	, Invk_ns_bgn_ = "ns_bgn_", Invk_db_bgn_ = "db_bgn_", Invk_pg_bgn_ = "pg_bgn_"
+	, Invk_ns_end_ = "ns_end_", Invk_db_end_ = "db_end_", Invk_pg_end_ = "pg_end_"
+	, Invk_load_tmpls_ = "load_tmpls_"
+	, Invk_poll_mgr = "poll_mgr", Invk_reset_db_ = "reset_db_"
+	, Invk_exec_count_max_ = "exec_count_max_", Invk_exit_now_ = "exit_now_", Invk_exit_after_commit_ = "exit_after_commit_"
+	;
+	private static final String GRP_KEY = "xowa.bldr.parse";
+}
+class Xob_dump_mgr_base_ {
+	public static void Load_all_tmpls(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, Xob_dump_src_id page_src) {
+		List_adp pages = List_adp_.new_();
+		Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template();
+		Xow_defn_cache defn_cache = wiki.Cache_mgr().Defn_cache();
+		int cur_page_id = -1;
+		int load_count = 0;
+		usr_dlg.Note_many("", "", "tmpl_load init");
+		while (true) {
+			page_src.Get_pages(pages, 0, Xow_ns_.Id_template, cur_page_id);	// 0 is always template db
+			int page_count = pages.Count();
+			if (page_count == 0) break;	// no more pages in db;
+			Xowd_page_itm page = null;
+			for (int i = 0; i < page_count; i++) {
+				page = (Xowd_page_itm)pages.Get_at(i);
+				Xot_defn_tmpl defn = new Xot_defn_tmpl();
+				defn.Init_by_new(ns_tmpl, ns_tmpl.Gen_ttl(page.Ttl_page_db()), page.Text(), null, false);	// NOTE: passing null, false; will be overriden later when Parse is called
+				defn_cache.Add(defn, ns_tmpl.Case_match());
+				++load_count;
+				if ((load_count % 10000) == 0) usr_dlg.Prog_many("", "", "tmpl_loading: ~{0}", load_count);
+			}
+			cur_page_id = page.Id();
+		}
+		usr_dlg.Note_many("", "", "tmpl_load done: ~{0}", load_count);
+	}
+	public static Xowd_db_file[] Init_text_files_ary(Xowd_db_mgr core_data_mgr) {
+		List_adp text_files_list = List_adp_.new_();
+		int len = core_data_mgr.Dbs__len();
+		if (len == 1) return new Xowd_db_file[] {core_data_mgr.Dbs__get_at(0)};	// single file: return core; note that there are no Tid = Text
+		for (int i = 0; i < len; i++) {
+			Xowd_db_file file = core_data_mgr.Dbs__get_at(i);
+			switch (file.Tid()) {
+				case Xowd_db_file_.Tid_text:
+				case Xowd_db_file_.Tid_text_solo:
+					text_files_list.Add(file);
+					break;
+			}
+		}
+		return (Xowd_db_file[])text_files_list.To_ary_and_clear(Xowd_db_file.class);
+	}
+}
+class Xob_dump_bmk_mgr {
+	private Bry_bfr save_bfr = Bry_bfr.reset_(1024);
+	public Io_url Cfg_url() {return cfg_url;} public Xob_dump_bmk_mgr Cfg_url_(Io_url v) {cfg_url = v; return this;} private Io_url cfg_url;
+	public void Reset() {Io_mgr.I.DeleteFil(cfg_url);}
+	public void Load(Xoae_app app, Xob_dump_mgr_base dump_mgr) {
+		app.Gfs_mgr().Run_url_for(dump_mgr, cfg_url);
+	}
+	public void Save(int ns_id, int db_id, int pg_id) {
+		Save_itm(save_bfr, Xob_dump_mgr_base.Invk_ns_bgn_, ns_id);
+		Save_itm(save_bfr, Xob_dump_mgr_base.Invk_db_bgn_, db_id);
+		Save_itm(save_bfr, Xob_dump_mgr_base.Invk_pg_bgn_, pg_id);
+		Io_mgr.I.SaveFilBfr(cfg_url, save_bfr);
+	}
+	private void Save_itm(Bry_bfr save_bfr, String key, int val) {
+		String fmt = "{0}('{1}');\n";
+		String str = String_.Format(fmt, key, val);
+		save_bfr.Add_str(str);
+	}
+}
+class Xob_rate_mgr {
+	private long time_bgn;
+	private int item_len;
+	private Bry_bfr save_bfr = Bry_bfr.reset_(255);
+	public int Reset_interval() {return reset_interval;} public Xob_rate_mgr Reset_interval_(int v) {reset_interval = v; return this;} private int reset_interval = 10000;
+	public Io_url Log_file() {return log_file;} public Xob_rate_mgr Log_file_(Io_url v) {log_file = v; return this;} private Io_url log_file;
+	public void Init() {time_bgn = Env_.TickCount();}
+	public void Increment() {
+		++item_len;
+		if (item_len % reset_interval == 0) {
+			long time_end = Env_.TickCount();
+			Save(item_len, time_bgn, time_end);
+			time_bgn = time_end;
+			item_len = 0;
+		}
+	}
+	private void Save(int count, long bgn, long end) {
+		int dif = (int)(end - bgn) / 1000;
+		DecimalAdp rate = DecimalAdp_.divide_safe_(count, dif);
+		save_bfr
+			.Add_str(rate.Xto_str("#,##0.000")).Add_byte_pipe()
+			.Add_int_variable(count).Add_byte_pipe()
+			.Add_int_variable(dif).Add_byte_nl()
+			;
+		Io_mgr.I.AppendFilByt(log_file, save_bfr.Xto_bry_and_clear());
+	}
+	public String Rate_as_str() {return Int_.Xto_str(Rate());}
+	public int Rate() {
+		int elapsed = Env_.TickCount_elapsed_in_sec(time_bgn);
+		return Math_.Div_safe_as_int(item_len, elapsed);
+	}
+}
+class Xob_dump_bmk {
+	public int Ns_id() {return ns_id;} public Xob_dump_bmk Ns_id_(int v) {ns_id = v; return this;} private int ns_id;
+	public int Db_id() {return db_id;} public Xob_dump_bmk Db_id_(int v) {db_id = v; return this;} private int db_id;
+	public int Pg_id() {return pg_id;} public Xob_dump_bmk Pg_id_(int v) {pg_id = v; return this;} private int pg_id;
+}

400_xowa/src/gplx/xowa/bldrs/cmds/Xob_ns_file_itm.java

@@ -0,0 +1,63 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
+import gplx.xowa.wikis.data.*;
+public class Xob_ns_file_itm {
+	public Xob_ns_file_itm(byte db_file_tid, String file_name, int[] ns_ids) {
+		this.db_file_tid = db_file_tid; this.file_name = file_name; this.ns_ids = ns_ids;
+		this.nth_db_id = Nth_db_id_null; this.nth_db_idx = 1;			
+	}
+	public byte		Db_file_tid() {return db_file_tid;} private final byte db_file_tid;
+	public String	File_name() {return file_name;} private final String file_name;
+	public int[]	Ns_ids() {return ns_ids;} private final int[] ns_ids;
+	public int		Nth_db_id() {return nth_db_id;} public void Nth_db_id_(int v) {nth_db_id = v;} private int nth_db_id;
+	public int		Nth_db_idx() {return nth_db_idx;} private int nth_db_idx;
+	public String Make_file_name() {								// EX: en.wikipedia.org-text-ns.000-001.xowa
+		String rv = String_.Format("-{0}{1}{2}.xowa"				// EX: -text-ns.000-db.001.xowa
+			, Xowd_db_file_.To_key(db_file_tid)						// text
+			, String_.Len_eq_0(file_name) ? "" : "-" + file_name	// if empty, don't add "ns.000" segment; produces en.wikipedia.org-text-001.xowa
+			, nth_db_idx == 1 ? "" : "-db." + Int_.Xto_str_pad_bgn_zero(nth_db_idx, 3)			// "-db.001"
+			);
+		++nth_db_idx;
+		return rv;
+	}
+	public static final int Nth_db_id_null = -1;
+
+	public static void Init_ns_bldr_data(byte db_file_tid, Xow_ns_mgr ns_mgr, byte[] ns_file_map) {
+		int ns_len = ns_mgr.Ords_len();
+		Xob_ns_file_itm ns_file_itm_default = new Xob_ns_file_itm(db_file_tid, "", null);
+		for (int i = 0; i < ns_len; ++i) {
+			Xow_ns ns = ns_mgr.Ords_get_at(i);
+			ns.Bldr_data_(ns_file_itm_default);
+		}
+		Xob_ns_file_itm_parser ns_itm_parser = new Xob_ns_file_itm_parser();
+		ns_itm_parser.Ctor(db_file_tid, ns_mgr);
+		Xob_ns_file_itm[] ns_itm_ary = ns_itm_parser.To_ary(ns_file_map);
+		int ns_itm_ary_len = ns_itm_ary.length;
+		for (int i = 0; i < ns_itm_ary_len; ++i) {
+			Xob_ns_file_itm itm = ns_itm_ary[i];
+			int[] ns_ids = itm.Ns_ids();
+			int ns_ids_len = ns_ids.length;
+			for (int j = 0; j < ns_ids_len; j++) {
+				int ns_id = ns_ids[j];
+				Xow_ns ns = ns_mgr.Ids_get_or_null(ns_id); if (ns == null) continue; // some dumps may not have ns; for example, pre-2013 dumps won't have Module (828)
+				ns.Bldr_data_(itm);
+			}
+		}
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/Xob_ns_file_itm_parser.java

@@ -0,0 +1,79 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
+import gplx.srls.dsvs.*;
+public class Xob_ns_file_itm_parser extends Dsv_wkr_base {
+	private byte[] ns_ids_bry; private String name; private final List_adp rslts = List_adp_.new_();
+	private Xow_ns_mgr ns_mgr; private byte db_file_tid; private boolean mode_each = false;
+	public void Ctor(byte db_file_tid, Xow_ns_mgr ns_mgr) {
+		this.db_file_tid = db_file_tid; this.ns_mgr = ns_mgr;
+		this.mode_each = false; rslts.Clear();
+	}
+	@Override public Dsv_fld_parser[] Fld_parsers() {return new Dsv_fld_parser[] {Dsv_fld_parser_.Bry_parser, Dsv_fld_parser_.Bry_parser};}
+	@Override public boolean Write_bry(Dsv_tbl_parser parser, int fld_idx, byte[] src, int bgn, int end) {
+		switch (fld_idx) {
+			case 0: ns_ids_bry	= Bry_.Mid(src, bgn, end); return true;
+			case 1: name		= String_.new_u8(src, bgn, end); return true;
+			default: return false;
+		}
+	}		
+	@Override public void Commit_itm(Dsv_tbl_parser parser, int pos) {
+		if (ns_ids_bry == null)		throw parser.Err_row_bgn("ns_itm missing ns_ids", pos);
+		if (mode_each) return;
+		if (Bry_.Eq(ns_ids_bry, ns_ids_bry_each)) {
+			mode_each = true;
+			int len = ns_mgr.Ords_len();
+			for (int i = 0; i < len; ++i) {
+				Xow_ns ns = ns_mgr.Ords_get_at(i);
+				int ns_id = ns.Id();
+				rslts.Add(new Xob_ns_file_itm(db_file_tid, "ns." + Int_.Xto_str_pad_bgn_zero(ns_id, 3), Int_.Ary(ns_id)));
+			}
+			return;
+		}
+		int[] ns_ids = null;
+		if (ns_ids_bry.length == 1 && ns_ids_bry[0] == Byte_ascii.Star) {	// "*"
+			int len = ns_mgr.Ords_len();
+			ns_ids = new int[len];
+			for (int i = 0; i < len; ++i)
+				ns_ids[i] = ns_mgr.Ords_get_at(i).Id();
+		}
+		else
+			ns_ids = Int_.Ary_parse(String_.new_u8(ns_ids_bry), ",");
+		if (ns_ids.length == 0) throw Exc_.new_("map.invalid.ns_missing", "src", this.Src());
+		if (String_.Len_eq_0(name)) {	// no name; auto-generate
+			int ns_id_1st = ns_ids[0];	// take 1st ns_id
+			name = "ns." + Int_.Xto_str_pad_bgn_zero(ns_id_1st, 3);	// EX: ns.000
+		}
+		Xob_ns_file_itm ns_itm = new Xob_ns_file_itm(db_file_tid, name, ns_ids);
+		rslts.Add(ns_itm);
+		ns_itm.toString();
+		ns_ids = null; name = null;
+	}
+	public Xob_ns_file_itm[] To_ary(byte[] bry) {
+		this.Load_by_bry(bry);
+		return (Xob_ns_file_itm[])rslts.To_ary(Xob_ns_file_itm.class);
+	}
+	private static final byte[] ns_ids_bry_each = Bry_.new_a7("<each>");
+	/*
+"" -> no rules; return "default"; generates "text-001" and lumps all ns into it
+"*|<id>|3700|2" -> auto-generate per ns
+<single-file>
+<all>||gzip
+<each>||gzip
+	*/
+}

400_xowa/src/gplx/xowa/bldrs/cmds/Xob_parse_all_src_sql.java

@@ -0,0 +1,92 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
+import gplx.xowa.wikis.data.*; import gplx.xowa.dbs.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.data.tbls.*;
+class Xob_dump_src_id {
+	private Xodb_mgr_sql db_mgr; private byte redirect;
+	private String page_db_url; private int size_max;
+	private Db_stmt text_stmt; int cur_text_db_idx = -1;
+	public Xob_dump_src_id Init(Xowe_wiki wiki, byte redirect, int size_max) {
+		this.db_mgr = wiki.Db_mgr_as_sql(); this.redirect = redirect;
+		this.size_max = size_max;
+		this.page_db_url = db_mgr.Core_data_mgr().Db__core().Url().Raw();
+		return this;
+	}
+	public void Get_pages(List_adp list, int text_db_idx, int cur_ns, int prv_id) {
+		DataRdr rdr = DataRdr_.Null;
+		int size_len = 0;
+		list.Clear();
+		try {
+			rdr = New_rdr(db_mgr, page_db_url, text_db_idx, cur_ns, prv_id, redirect);
+			while (rdr.MoveNextPeer()) {
+				Xowd_page_itm page = New_page(db_mgr, cur_ns, rdr);
+				list.Add(page);
+				size_len += page.Text_len();
+				if (size_len > size_max)
+					break;
+			}
+		}
+		finally {rdr.Rls();}
+	}
+	private DataRdr New_rdr(Xodb_mgr_sql db_mgr, String page_db_url, int text_db_idx, int cur_ns, int prv_id, byte redirect) {
+		if (cur_text_db_idx != text_db_idx) {
+			cur_text_db_idx = text_db_idx;
+			Xowd_db_file text_db = db_mgr.Core_data_mgr().Dbs__get_at(text_db_idx);
+			Db_conn conn = text_db.Conn();
+			String sql = String_.Format(Sql_select, New_rdr__redirect_clause(redirect));
+			text_stmt = conn.Stmt_new(Db_qry_sql.rdr_(sql));
+		}
+		return text_stmt.Clear().Val_int(prv_id).Val_int(cur_ns).Exec_select();
+	}
+	private static Xowd_page_itm New_page(Xodb_mgr_sql db_mgr, int ns_id, DataRdr rdr) {
+		Xowd_page_tbl page_core_tbl = db_mgr.Core_data_mgr().Tbl__page();
+		Xowd_page_itm rv = new Xowd_page_itm();
+		rv.Id_(rdr.ReadInt(page_core_tbl.Fld_page_id()));
+		rv.Ns_id_(ns_id);
+		rv.Ttl_page_db_(rdr.ReadBryByStr(page_core_tbl.Fld_page_title()));
+		
+		String text_data_name = db_mgr.Core_data_mgr().Db__core().Tbl__text().Fld_text_data();
+		byte[] text_data = rdr.ReadBry(text_data_name);
+		text_data = db_mgr.Wiki().Appe().Zip_mgr().Unzip(db_mgr.Core_data_mgr().Props().Zip_tid_text(), text_data);
+		rv.Text_(text_data);
+		return rv;
+	}
+	private static String New_rdr__redirect_clause(byte redirect) {
+		switch (redirect) {
+			case Bool_.Y_byte:	return Sql_select__redirect_y;
+			case Bool_.N_byte:	return Sql_select__redirect_n;
+			case Bool_.__byte:	return Sql_select__redirect__;
+			default:			throw Exc_.new_unhandled(redirect);
+		}
+	}
+	private static final String Sql_select = String_.Concat_lines_nl
+	( "SELECT  p.page_id"
+	, ",       p.page_title"
+	, ",       t.text_data"
+	, "FROM    page_dump p"
+	, "        JOIN text t ON t.page_id = p.page_id"
+	, "WHERE   p.page_id > ?"
+	, "AND     p.page_namespace = ?{0}" 
+	, "ORDER BY p.page_id"
+	);
+	private static final String
+	  Sql_select__redirect_y = "\nAND     p.page_is_redirect = 1"
+	, Sql_select__redirect_n = "\nAND     p.page_is_redirect = 0"
+	, Sql_select__redirect__ = ""
+	;
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Uca_trie_tst.java

@@ -0,0 +1,45 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*; import gplx.xowa.bldrs.*;
+public class Uca_trie_tst {		
+	@Before public void init() {fxt.Clear();} private Xob_base_fxt fxt = new Xob_base_fxt();
+	@Test  public void Basic() {
+		Uca_trie_fxt fxt = new Uca_trie_fxt();
+		fxt.Clear();
+		fxt.Init_trie_itm("a", Bry_.ints_(10, 11));
+		fxt.Init_trie_itm("b", Bry_.ints_(20, 21));
+		fxt.Init_trie_itm("c", Bry_.ints_(30, 31));
+		fxt.Test_decode(Bry_.ints_(10, 11), "a");
+		fxt.Test_decode(Bry_.ints_(10, 11, 20, 21, 30, 31), "abc");
+	}
+}
+class Uca_trie_fxt {
+	public void Clear() {
+		if (trie == null) {
+			trie = new Uca_trie();
+			bfr = Bry_bfr.new_();
+		}
+		trie.Clear();
+	}	Uca_trie trie; Bry_bfr bfr;
+	public void Init_trie_itm(String charAsStr, byte[] uca) {trie.Init_itm(gplx.intl.Utf16_.Decode_to_int(Bry_.new_u8(charAsStr), 0), uca);}
+	public void Test_decode(byte[] bry, String expd) {
+		trie.Decode(bfr, bry, 0, bry.length);
+		Tfds.Eq(expd, bfr.Xto_str_and_clear());
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_category_registry_sql.java

@@ -0,0 +1,54 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.ios.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.dbs.*; import gplx.xowa.bldrs.*;
+public class Xob_category_registry_sql implements Xob_cmd {
+	public Xob_category_registry_sql(Xob_bldr bldr, Xowe_wiki wiki) {this.wiki = wiki;} private Xowe_wiki wiki;
+	public String Cmd_key() {return Xob_cmd_keys.Key_text_cat_core;}
+	public void Cmd_end() {	// NOTE: placing in end, b/c must run *after* page_sql
+		// setup
+		wiki.Html_mgr().Importing_ctgs_(Bool_.Y);
+		Io_url rslt_dir = Xob_category_registry_sql.Tmp_dir(wiki);
+		Io_mgr.I.DeleteDirDeep(rslt_dir);
+		Xob_tmp_wtr rslt_wtr = Xob_tmp_wtr.new_wo_ns_(Io_url_gen_.dir_(rslt_dir), Io_mgr.Len_mb);
+		// read data
+		Gfo_usr_dlg usr_dlg = wiki.Appe().Usr_dlg();
+		wiki.Init_db_mgr();
+		Xowd_page_tbl page_core_tbl = wiki.Db_mgr_as_sql().Core_data_mgr().Tbl__page();
+		Db_rdr rdr = page_core_tbl.Conn().Stmt_select_order(page_core_tbl.Tbl_name(), String_.Ary(page_core_tbl.Fld_page_title(), page_core_tbl.Fld_page_id()), String_.Ary(page_core_tbl.Fld_page_ns()), page_core_tbl.Fld_page_title())
+			.Crt_int(page_core_tbl.Fld_page_ns(), Xow_ns_.Id_category)
+			.Exec_select__rls_auto();
+		try {
+			while (rdr.Move_next()) {
+				byte[] page_ttl = rdr.Read_bry_by_str(page_core_tbl.Fld_page_title());
+				int page_id = rdr.Read_int(page_core_tbl.Fld_page_id());
+				if (rslt_wtr.FlushNeeded(page_ttl.length + 2 + 5)) rslt_wtr.Flush(usr_dlg);
+				rslt_wtr.Bfr().Add(page_ttl).Add_byte_pipe().Add_base85_len_5(page_id).Add_byte_nl();
+			}
+		}	finally {rdr.Rls();}
+		// cleanup
+		rslt_wtr.Flush(usr_dlg);
+		wiki.Html_mgr().Importing_ctgs_(Bool_.N);
+	}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run()	{}
+	public void Cmd_term()	{}
+	public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return GfoInvkAble_.Rv_unhandled;}
+	public static Io_url Tmp_dir(Xowe_wiki wiki) {return wiki.Fsys_mgr().Tmp_dir().GenSubDir(Xob_cmd_keys.Key_text_cat_core);}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_category_registry_sql_tst.java

@@ -0,0 +1,66 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*; import gplx.core.primitives.*; import gplx.xowa.bldrs.*;
+public class Xob_category_registry_sql_tst {
+	@Before	public void init() {if (Xoa_test_.Db_skip()) return; fxt.Clear();} private Xob_category_registry_sql_fxt fxt = new Xob_category_registry_sql_fxt();
+	@After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();}
+	@Test   public void Basic() {
+		if (Xoa_test_.Db_skip()) return;
+		fxt.Init_page_insert(String_.Ary("Ctg3", "Ctg2", "Ctg1"));
+		fxt.Exec_category_registry_cmd();
+		fxt.Test_ids(Int_.Ary(3, 2, 1));	// note that Ctg1 is page_id 3
+	}
+}
+class Xob_category_registry_sql_fxt {
+	Db_mgr_fxt fxt; Xowe_wiki wiki; Xoae_app app; Int_obj_ref page_id_next = Int_obj_ref.new_(1);
+	public void Clear() {
+		if (fxt == null) {
+			fxt = new Db_mgr_fxt().Ctor_fsys();
+			fxt.Init_db_sqlite();
+			wiki = fxt.Wiki();
+			app = wiki.Appe();
+		}
+	}
+	public void Rls() {fxt.Rls();}
+	public void Init_page_insert(String[] ttls) {
+		fxt.Init_page_insert(page_id_next, Xow_ns_.Id_category, ttls);
+	}
+	public void Exec_category_registry_cmd() {
+		app.Bldr().Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_core);
+		app.Bldr().Run();
+	}
+	public void Test_ids(int[] expd) {
+		Io_url rslts_dir = Xob_category_registry_sql.Tmp_dir(wiki);
+		String rslts_txt = Io_mgr.I.LoadFilStr(Io_mgr.I.QueryDir_fils(rslts_dir)[0]);
+		int[] actl = Parse_rslts_txt(rslts_txt);
+		Tfds.Eq_ary(expd, actl);
+	}
+	int[] Parse_rslts_txt(String rslts_txt) {
+		String[] lines = String_.SplitLines_nl(rslts_txt);
+		List_adp list = List_adp_.new_();
+		int len = lines.length;
+		for (int i = 0; i < len; i++) {
+			String line = lines[i];
+			if (String_.Len_eq_0(line)) break;	// ignore blank line
+			String[] flds = String_.Split(line, '|');
+			list.Add(Base85_utl.XtoIntByStr(flds[1]));
+		}
+		return (int[])list.To_ary_and_clear(int.class);
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_base.java

@@ -0,0 +1,89 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.core.brys.*; import gplx.ios.*; import gplx.xowa.ctgs.*;
+public abstract class Xob_categorylinks_base extends Xob_sql_dump_base implements Sql_file_parser_cmd {
+	private DateAdp_parser date_parser = DateAdp_parser.new_(); private Sql_file_parser sql_parser; Uca_trie trie; private Bry_bfr uca_bfr = Bry_bfr.reset_(255);		
+	public abstract Io_sort_cmd Make_sort_cmd(Sql_file_parser sql_parser);
+	@Override public String Sql_file_name() {return "categorylinks";}
+	@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
+		this.sql_parser = parser;
+		wiki.Html_mgr().Importing_ctgs_(Bool_.Y);
+		parser.Fld_cmd_(this).Flds_req_(Fld_cl_from, Fld_cl_to, Fld_cl_timestamp, Fld_cl_collation, Fld_cl_sortkey, Fld_cl_type);
+	}
+	public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) {
+		if		(Bry_.Eq(fld_key, Fld_cl_from))			cur_id = Bry_.Xto_int_or(src, fld_bgn, fld_end, -1);
+		else if (Bry_.Eq(fld_key, Fld_cl_to))			cur_ctg = Bry_.Mid(src, fld_bgn, fld_end);
+		else if (Bry_.Eq(fld_key, Fld_cl_collation))	cur_collation_is_uca = Bry_.Has_at_bgn(src, Collation_uca, fld_bgn, fld_end);
+		else if (Bry_.Eq(fld_key, Fld_cl_timestamp)) {
+			date_parser.Parse_iso8651_like(cur_modified_on, src, fld_bgn, fld_end);
+			cur_date = fld_end - fld_bgn == 0	// ignore null dates added by ctg_v1
+				? 0 : Bit_.Xto_int_date_short(cur_modified_on);
+		}
+		else if (Bry_.Eq(fld_key, Fld_cl_sortkey)) {
+			int nl_pos = Bry_finder.Find_fwd(src, Byte_ascii.Nl, fld_bgn, fld_end);
+			if (nl_pos != Bry_.NotFound)	// sortkey sometimes has format of "sortkey\ntitle"; EX: "WALES, JIMMY\nJIMMY WALES"; discard 2nd to conserve hard-disk space
+				fld_end = nl_pos;
+			cur_sortkey = Bry_.Mid(src, fld_bgn, fld_end);
+		}
+		else if (Bry_.Eq(fld_key, Fld_cl_type)) {
+			byte char_0 = src[fld_bgn];
+			switch (char_0) {
+				case Byte_ascii.Ltr_f:
+				case Byte_ascii.Ltr_p:	cur_tid = char_0; break;
+				case Byte_ascii.Ltr_s:	cur_tid = Byte_ascii.Ltr_c; break; 
+				default:				throw Exc_.new_unhandled(char_0);
+			}
+			if (cur_collation_is_uca) {
+				if (trie == null) {
+					trie = new Uca_trie();
+					trie.Init();
+				}
+				trie.Decode(uca_bfr, cur_sortkey, 0, cur_sortkey.length);
+				cur_sortkey = uca_bfr.Len() == 0 ? Sortkey_space : uca_bfr.Xto_bry_and_clear();
+			}
+			fld_wtr.Bfr_(file_bfr);
+			fld_wtr.Write_bry_escape_fld(cur_ctg);
+			fld_wtr.Write_byte_fld(cur_tid);
+			fld_wtr.Write_bry_escape_fld(cur_sortkey);
+			fld_wtr.Write_int_base85_len5_fld(cur_id);
+			fld_wtr.Write_int_base85_len5_fld(cur_date);
+		}
+	}	int cur_id = -1, cur_date = -1; byte[] cur_ctg = null, cur_sortkey = null; byte cur_tid = Byte_.Max_value_127; boolean cur_collation_is_uca; int[] cur_modified_on = new int[7];
+	@Override public void Cmd_end() {
+		Xobdc_merger.Basic(bldr.Usr_dlg(), dump_url_gen, temp_dir.GenSubDir("sort"), sort_mem_len, Xoctg_link_sql_sorter._, Io_line_rdr_key_gen_.noop, Make_sort_cmd(sql_parser));
+		wiki.Html_mgr().Importing_ctgs_(Bool_.N);
+	}		
+	private static final byte[] Fld_cl_from = Bry_.new_a7("cl_from"), Fld_cl_to = Bry_.new_a7("cl_to"), Fld_cl_timestamp = Bry_.new_a7("cl_timestamp"), Fld_cl_collation = Bry_.new_a7("cl_collation"), Fld_cl_sortkey = Bry_.new_a7("cl_sortkey"), Fld_cl_type = Bry_.new_a7("cl_type");
+	private static final byte[] Collation_uca = Bry_.new_u8("uca"), Sortkey_space = new byte[] {Byte_ascii.Space};
+}
+class Xoctg_link_sql_sorter implements gplx.lists.ComparerAble {
+	public int compare(Object lhsObj, Object rhsObj) {
+		Io_sort_split_itm lhs = (Io_sort_split_itm)lhsObj, rhs = (Io_sort_split_itm)rhsObj;
+		byte[] lhs_bry = lhs.Bfr(); int lhs_bgn = 0, lhs_end = lhs.Row_bgn() - 1;
+		byte[] rhs_bry = rhs.Bfr(); int rhs_bgn = 0, rhs_end = rhs.Row_bgn() - 1;
+		for (int i = 0; i < 3; i++) {
+			lhs_bgn = lhs_end + 1; lhs_end = Bry_finder.Find_fwd(lhs_bry, Byte_ascii.Pipe, lhs_bgn);
+			rhs_bgn = rhs_end + 1; rhs_end = Bry_finder.Find_fwd(rhs_bry, Byte_ascii.Pipe, rhs_bgn);
+			int comp = Bry_.Compare(lhs_bry, lhs_bgn, lhs_end, rhs_bry, rhs_bgn, rhs_end);
+			if (comp != CompareAble_.Same) return comp;
+		}
+		return CompareAble_.Same;
+	}
+	public static final Xoctg_link_sql_sorter _ = new Xoctg_link_sql_sorter(); Xoctg_link_sql_sorter() {}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_base_tst.java

@@ -0,0 +1,70 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*; import gplx.xowa.bldrs.*;
+public class Xob_categorylinks_base_tst {
+	@Before public void init() {fxt.Clear();} private Xob_base_fxt fxt = new Xob_base_fxt();
+	@Test   public void Basic() {
+		Io_url src_fil = Io_url_.new_fil_("mem/temp/sql_dump.sql");
+		fxt	.Init_fil(src_fil, String_.Concat
+		(	Xob_categorylinks_sql.Sql_categorylinks
+		,	"INSERT INTO `categorylinks` VALUES"
+		,	" (1,'Ctg_2','PAGE_2a','2013-04-15 01:02:03','','uppercase','page')"
+		,	",(2,'Ctg_1','PAGE_1b','2013-04-15 01:02:03','','uppercase','page')"
+		,	",(3,'Ctg_1','PAGE_1a','2013-04-15 01:02:03','','uppercase','page')"
+		,	";"
+		))
+		.Exec_cmd(Xob_cmd_keys.Key_tdb_text_cat_link, GfoMsg_.basic_(Xob_categorylinks_base.Invk_src_fil_, src_fil))
+		.Test_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.link_sql/make/0000000000.csv", String_.Concat_lines_nl
+		(	"Ctg_1|p|PAGE_1a|!!!!$|#8DDL|"
+		,	"Ctg_1|p|PAGE_1b|!!!!#|#8DDL|"
+		,	"Ctg_2|p|PAGE_2a|!!!!\"|#8DDL|"
+		))
+		;
+	}
+	@Test   public void Super_earths() {	// PURPOSE: handle multi-field sort
+		Io_url src_fil = Io_url_.new_fil_("mem/temp/sql_dump.sql");
+		fxt	.Init_fil(src_fil, String_.Concat
+		(	Xob_categorylinks_sql.Sql_categorylinks
+		,	"INSERT INTO `categorylinks` VALUES"
+		,	" (1,'Super-Earths','PAGE_1a','2013-04-15 01:02:03','','uppercase','page')"
+		,	",(2,'Super-Earths_in_the_habitable_zone','PAGE_1b','2013-04-15 01:02:03','','uppercase','page')"
+		,	";"
+		))
+		.Exec_cmd(Xob_cmd_keys.Key_tdb_text_cat_link, GfoMsg_.basic_(Xob_categorylinks_base.Invk_src_fil_, src_fil))
+		.Test_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.link_sql/make/0000000000.csv", String_.Concat_lines_nl
+		(	"Super-Earths|p|PAGE_1a|!!!!\"|#8DDL|"
+		,	"Super-Earths_in_the_habitable_zone|p|PAGE_1b|!!!!#|#8DDL|"
+		))
+		;
+	}
+	@Test   public void Sortkey_has_newline() {	// PURPOSE: sortkey sometimes has format of "sortkey\ntitle"; EX: "WALES, JIMMY\nJIMMY WALES"; discard 2nd for hard-disk space savings
+		Io_url src_fil = Io_url_.new_fil_("mem/temp/sql_dump.sql");
+		fxt	.Init_fil(src_fil, String_.Concat
+		(	Xob_categorylinks_sql.Sql_categorylinks
+		,	"INSERT INTO `categorylinks` VALUES"
+		,	" (1,'Ctg_1','LAST,FIRST\\nFIRST LAST','2013-04-15 01:02:03','','uppercase','page')"
+		,	";"
+		))
+		.Exec_cmd(Xob_cmd_keys.Key_tdb_text_cat_link, GfoMsg_.basic_(Xob_categorylinks_base.Invk_src_fil_, src_fil))
+		.Test_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.link_sql/make/0000000000.csv", String_.Concat_lines_nl
+		(	"Ctg_1|p|LAST,FIRST|!!!!\"|#8DDL|"
+		))
+		;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_sql.java

@@ -0,0 +1,43 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.ios.*; import gplx.xowa.bldrs.*;
+public class Xob_categorylinks_sql extends Xob_categorylinks_base {
+	private Db_idx_mode idx_mode = Db_idx_mode.Itm_end;
+	public Xob_categorylinks_sql(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}
+	@Override public String Cmd_key() {return Xob_cmd_keys.Key_text_cat_link;}
+	@Override public Io_sort_cmd Make_sort_cmd(Sql_file_parser sql_parser) {return new Xob_categorylinks_sql_make(sql_parser, wiki, idx_mode);}
+	public static final String Sql_categorylinks = String_.Concat_lines_nl
+	( "CREATE TABLE `categorylinks` ("
+	, "  `cl_from` int(10) unsigned NOT NULL DEFAULT '0',"
+	, "  `cl_to` varbinary(255) NOT NULL DEFAULT '',"
+	, "  `cl_sortkey` varbinary(230) NOT NULL DEFAULT '',"
+	, "  `cl_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,"
+	, "  `cl_sortkey_prefix` varbinary(255) NOT NULL DEFAULT '',"
+	, "  `cl_collation` varbinary(32) NOT NULL DEFAULT '',"
+	, "  `cl_type` enum('page','subcat','file') NOT NULL DEFAULT 'page',"
+	, "  UNIQUE KEY `cl_from` (`cl_from`,`cl_to`),"
+	, ");"
+	);
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_idx_mode_))					idx_mode = Db_idx_mode.Xto_itm(m.ReadStr("v"));
+		else													return super.Invk(ctx, ikey, k, m);
+		return this;
+	}
+	private static final String Invk_idx_mode_ = "idx_mode_";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_sql_make.java

@@ -0,0 +1,153 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.core.flds.*; import gplx.ios.*; import gplx.dbs.*; import gplx.xowa.dbs.*; import gplx.xowa.ctgs.*; 
+import gplx.xowa.bldrs.*;
+import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
+public class Xob_categorylinks_sql_make implements Io_make_cmd {
+	private Gfo_usr_dlg usr_dlg; private final Xowe_wiki wiki; private final Db_idx_mode idx_mode; private Xowd_db_mgr core_db_mgr;
+	private final Sql_file_parser sql_parser; private final Gfo_fld_rdr fld_rdr = Gfo_fld_rdr.xowa_(); private Io_line_rdr name_id_rdr;
+	private Xowd_cat_core_tbl cat_core_tbl; private Xowd_cat_link_tbl cat_link_tbl;
+	private long cat_db_size, cat_db_max; private int cur_row_count;
+	private int[] cur_cat_counts = new int[Xoa_ctg_mgr.Tid__max]; private byte[] cur_cat_ttl = Ttl_first; private int cur_cat_id; private int cur_cat_file_idx;
+	public Xob_categorylinks_sql_make(Sql_file_parser sql_parser, Xowe_wiki wiki, Db_idx_mode idx_mode) {
+		this.wiki = wiki; this.sql_parser = sql_parser; this.idx_mode = idx_mode; usr_dlg = wiki.Appe().Usr_dlg();
+	}
+	public void Sort_bgn() {			
+		this.core_db_mgr	= wiki.Db_mgr_as_sql().Core_data_mgr();
+		this.name_id_rdr	= New_registry_rdr(wiki, usr_dlg);
+		this.cat_db_max		= wiki.Appe().Api_root().Bldr().Wiki().Import().Cat_link_db_max();
+		boolean one_file		= core_db_mgr.Props().Layout_text().Tid_is_all_or_few();
+		if (!one_file)		// cat is in its own dbs: delete dbs
+			core_db_mgr.Dbs__delete_by_tid(Xowd_db_file_.Tid_cat, Xowd_db_file_.Tid_cat_core, Xowd_db_file_.Tid_cat_link);	// delete existing category files else upgrade won't work
+		Db_make(Bool_.Y);
+		if (one_file) {		// cat is in single db; delete tbls;
+			cat_core_tbl.Delete_all();
+			cat_link_tbl.Delete_all();
+		}
+	}
+	public void Sort_do(Io_line_rdr rdr) {
+		byte[] new_cat_ttl = Bry_.Empty;
+		try {
+			fld_rdr.Ini(rdr.Bfr(), rdr.Itm_pos_bgn());
+			new_cat_ttl = fld_rdr.Read_bry_escape();
+			if (!Bry_.Eq(new_cat_ttl, cur_cat_ttl))		// ttl changed
+				cur_cat_id = Save_ctg(new_cat_ttl);
+			if (cur_cat_id == Cur_cat_id_null) return;
+			byte cat_tid = Convert_cat_tid_to_byte(fld_rdr.Read_byte());
+			byte[] cat_sortkey = fld_rdr.Read_bry_escape();
+			int page_id = fld_rdr.Read_int_base85_len5();
+			int cat_timestamp = fld_rdr.Read_int_base85_len5();			
+			cat_link_tbl.Insert_cmd_by_batch(page_id, cur_cat_id, cat_tid, cat_sortkey, cat_timestamp);
+			cat_db_size += 34 + 20 + 12 + (cat_sortkey.length * 2);	// NOTE_1: categorylinks row size
+			++cur_cat_counts[cat_tid];
+			++cur_row_count;
+			if (cur_row_count % 100000  == 0) usr_dlg.Prog_one("", "", "inserting category row: ~{0}", cur_row_count);
+			if (cur_row_count % 1000000 == 0) {cat_core_tbl.Conn().Txn_sav(); cat_link_tbl.Conn().Txn_sav();}
+		}	catch (Exception e) {usr_dlg.Warn_many("", "", "ctg_links.insert failed: name=~{0} err=~{1}", String_.new_u8(new_cat_ttl), Err_.Message_gplx_brief(e));}
+	}
+	public void Sort_end() {
+		Save_ctg(Ttl_last);
+		Db_close();			
+		Xodb_mgr_sql db_mgr_sql = wiki.Db_mgr_as_sql();
+		if (db_mgr_sql.Category_version() == Xoa_ctg_mgr.Version_null)	// NOTE: ctg_v1 wkr will set this to v1; only set to v2 if null  
+			db_mgr_sql.Category_version_update(false);
+		usr_dlg.Log_many("", "", "import.category.v2: insert done; committing; rows=~{0}", cur_row_count);
+		name_id_rdr.Rls();
+		if (String_.Eq(sql_parser.Src_fil().NameAndExt(), Xob_ctg_v1_sql_make.Url_sql))	// delete temp xowa_categorylinks.sql file created by cat_v1
+			Io_mgr.I.DeleteFil(sql_parser.Src_fil());
+	}
+	private int Save_ctg(byte[] new_ctg_ttl) {
+		if (cur_cat_ttl != Bry_.Empty && cur_cat_id != -1)
+			cat_core_tbl.Insert_cmd_by_batch(cur_cat_id, cur_cat_counts[Xoa_ctg_mgr.Tid_page], cur_cat_counts[Xoa_ctg_mgr.Tid_subc], cur_cat_counts[Xoa_ctg_mgr.Tid_file], Xoa_ctg_mgr.Hidden_n, cur_cat_file_idx);
+		if (new_ctg_ttl == Ttl_last) return Cur_cat_id_null;	// last ttl; called by this.End(); exit early else will fail in Cur_cat_id_find()
+		if (cat_db_size > cat_db_max) {Db_close(); Db_make(Bool_.N);}
+		cur_cat_id = Cur_cat_id_find(new_ctg_ttl);
+		for (int i = 0; i < Xoa_ctg_mgr.Tid__max; i++)
+			cur_cat_counts[i] = 0;
+		cur_cat_ttl = new_ctg_ttl;
+		return cur_cat_id;
+	}
+	private void Db_make(boolean first) {
+		boolean one_file = core_db_mgr.Props().Layout_text().Tid_is_all_or_few();
+		if (first) {	// create cat_core
+			Xowd_db_file cat_core_db = one_file ? core_db_mgr.Db__cat_core() : core_db_mgr.Dbs__make_by_tid(Xowd_db_file_.Tid_cat_core);
+			this.cat_core_tbl = cat_core_db.Tbl__cat_core().Create_tbl();
+		}
+		Xowd_db_file cat_link_db = one_file ? core_db_mgr.Db__core() : core_db_mgr.Dbs__make_by_tid(Xowd_db_file_.Tid_cat_link);
+		this.cat_link_tbl = cat_link_db.Tbl__cat_link();
+		if (	(one_file && first)
+			||	!one_file)
+			cat_link_tbl.Create_tbl();
+		this.cur_cat_file_idx = cat_link_db.Id();
+		cat_core_tbl.Insert_bgn();
+		cat_link_tbl.Insert_bgn();
+		if (idx_mode.Tid_is_bgn()) cat_link_tbl.Create_idx();
+	}
+	private void Db_close() {
+		cat_core_tbl.Insert_end();
+		cat_link_tbl.Insert_end();
+		cat_db_size = 0;
+		if (idx_mode.Tid_is_end()) cat_link_tbl.Create_idx();
+	}
+	private static byte Convert_cat_tid_to_byte(byte ltr) {
+		switch (ltr) {
+			case Byte_ascii.Ltr_f:	return Xoa_ctg_mgr.Tid_file;
+			case Byte_ascii.Ltr_p:	return Xoa_ctg_mgr.Tid_page;
+			case Byte_ascii.Ltr_c:	return Xoa_ctg_mgr.Tid_subc;
+			default:				throw Exc_.new_unhandled(ltr);
+		}
+	}
+	public Io_sort_cmd Make_dir_(Io_url v) {return this;}
+	private int Cur_cat_id_find(byte[] ttl) {
+		while (true) {
+			int compare = Bry_.Compare(ttl, 0, ttl.length, name_id_rdr.Bfr(), name_id_rdr.Key_pos_bgn(), name_id_rdr.Key_pos_end());
+			switch (compare) {
+				case CompareAble_.Same: return Base85_utl.XtoIntByAry(name_id_rdr.Bfr(), name_id_rdr.Key_pos_end() + 1, name_id_rdr.Itm_pos_end() - 2);
+				case CompareAble_.More:
+					boolean reading = name_id_rdr.Read_next();
+					if (!reading) return Cur_cat_id_null; // eof return
+					break;
+				case CompareAble_.Less: return Cur_cat_id_null;	// stop 
+			}
+		}
+	}
+	private static final int Cur_cat_id_null = -1;
+	private static Io_line_rdr New_registry_rdr(Xowe_wiki wiki, Gfo_usr_dlg usr_dlg) {
+		Io_url make_dir = Xob_category_registry_sql.Tmp_dir(wiki);
+		usr_dlg.Prog_many("", "", "loading category_registry files: ~{0}", make_dir.Raw());
+		Io_url[] urls = Io_mgr.I.QueryDir_args(make_dir).ExecAsUrlAry();
+		return new Io_line_rdr(usr_dlg, urls).Key_gen_(Io_line_rdr_key_gen_.first_pipe);
+	}
+	private static final byte[] Ttl_last = null, Ttl_first = Bry_.Empty;
+}
+/*
+NOTE_1: categorylinks row size: 34 + 20 + 12 + (cat_sortkey.length * 2)
+row length (data)		: 34=8+4+4+14+4				ROWID, cl_from, cl_to_id, cl_timestamp, cl_type_id
+cl_main length (idx)	: 20=8+4+4+4				ROWID, cl_from, cl_to_id, cl_type_id
+cl_from length (idx)	: 12=8+4					ROWID, cl_from
+variable_data length	: cat_sortkey.length * 2	sortkey is used for row and cl_main
+
+Note the following
+. ints are 4 bytes
+. tinyint is assumed to be 4 bytes (should be 1, but sqlite only has one numeric datatype, so import all 4?)
+. varchar(14) is assumed to be 14 bytes (should be 15? +1 for length of varchar?)
+. calculations work out "too well". comparing 4 databases gets +/- .25 bytes per row. however
+.. - bytes should not be possible
+.. +.25 bytes is too low (18 MB out of 5.5 GB).*; there must be other bytes used for page breaks / fragmentation
+*/

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_sql_tst.java

@@ -0,0 +1,140 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*; import gplx.core.primitives.*; import gplx.dbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.ctgs.*; import gplx.xowa.bldrs.*;
+public class Xob_categorylinks_sql_tst {
+	@Before public void init() {if (Xoa_test_.Db_skip()) return; fxt.Ctor_fsys();} Db_mgr_fxt fxt = new Db_mgr_fxt();
+	@After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();} 
+	@Test   public void Basic() {
+		if (Xoa_test_.Db_skip()) return;
+		fxt.Init_db_sqlite();
+		fxt.Init_page_insert(Int_obj_ref.new_(1), Xow_ns_.Id_category, String_.Ary("Ctg_1", "Ctg_2"));
+		fxt.Init_fil(Xoa_test_.Url_wiki_enwiki().GenSubFil("xowa_categorylinks.sql"), String_.Concat
+		(	Xob_categorylinks_sql.Sql_categorylinks
+		,	"INSERT INTO `categorylinks` VALUES"
+		,	" (3,'Ctg_2','File:2a','2013-04-15 01:02:03','','uppercase','file')"
+		,	",(4,'Ctg_1','1b','2013-04-15 01:02:03','','uppercase','page')"
+		,	",(5,'Ctg_1','1a','2013-04-15 01:02:03','','uppercase','page')"
+		,	";"
+		));
+		fxt.Exec_run(new Xob_category_registry_sql(fxt.Bldr(), fxt.Wiki()));
+		fxt.Exec_run(new Xob_categorylinks_sql(fxt.Bldr(), fxt.Wiki()));
+		Db_conn conn = fxt.Wiki().Db_mgr_as_sql().Core_data_mgr().Db__cat_core().Conn();
+		Db_tst_qry.tbl_("cat_core", "cat_id")
+			.Cols_("cat_id", "cat_subcats", "cat_files", "cat_pages")
+			.Rows_add_vals(1, 0, 0, 2)
+			.Rows_add_vals(2, 0, 1, 0)
+			.Test(conn);
+		Db_tst_qry.tbl_("cat_link", "cl_from")
+			.Cols_("cl_from", "cl_to_id", "cl_sortkey", "cl_type_id")
+			.Rows_add_vals(3, 2, "File:2a"	, Xoa_ctg_mgr.Tid_file)
+			.Rows_add_vals(4, 1, "1b"		, Xoa_ctg_mgr.Tid_page)
+			.Rows_add_vals(5, 1, "1a"		, Xoa_ctg_mgr.Tid_page)
+			.Test(conn);		
+	}
+}
+class Db_tst_val {
+	public Db_tst_val(int idx, String key, Object val) {this.idx = idx; this.key = key; this.val = val;}
+	public int Idx() {return idx;} public Db_tst_val Idx_(int val) {idx = val; return this;} private int idx = -1;
+	public String Key() {return key;} public Db_tst_val Key_(String v) {this.key = v; return this;} private String key;
+	public Object Val() {return val;} public Db_tst_val Val_(Object v) {this.val = v; return this;} Object val;
+	public static final int Idx_null = -1;
+	public static final String Key_null = null;
+}
+class Db_tst_row {
+	public int Idx() {return idx;} public Db_tst_row Idx_(int val) {idx = val; return this;} private int idx = -1;
+	public Db_tst_val[] Vals_ary() {return vals_ary;} Db_tst_val[] vals_ary = null;
+	public static Db_tst_row vals_(Object... ary) {
+		Db_tst_row rv = new Db_tst_row();
+		int len = ary.length;
+		Db_tst_val[] vals_ary = new Db_tst_val[len];
+		for (int i = 0; i < len; i++)
+			vals_ary[i] = new Db_tst_val(Db_tst_val.Idx_null, Db_tst_val.Key_null, ary[i]);
+		rv.vals_ary = vals_ary;
+		return rv;
+	}
+	public static Db_tst_row kvs_(String[] cols, Object[] vals) {
+		int cols_len = cols.length;
+		int vals_len = vals.length;
+		if (cols_len != vals_len) throw Exc_.new_("mismatch in cols / vals");
+		Db_tst_row rv = new Db_tst_row();
+		Db_tst_val[] vals_ary = new Db_tst_val[cols_len];
+		for (int i = 0; i < cols_len; i++) {
+			String col = cols[i];
+			Object val = vals[i];
+			vals_ary[i] = new Db_tst_val(Db_tst_val.Idx_null, col, val);
+		}
+		rv.vals_ary = vals_ary;
+		return rv;
+	}
+}
+class Db_tst_qry {
+	public Db_qry Qry() {return qry;} Db_qry qry;
+	public String[] Cols() {return cols;} public Db_tst_qry Cols_(String... v) {this.cols = v; return this;} private String[] cols;
+	public List_adp Rows() {return rows;} List_adp rows = List_adp_.new_();
+	public Db_tst_qry Rows_add_vals(Object... ary) {
+		Db_tst_row row = Db_tst_row.kvs_(cols, ary);
+		rows.Add(row);
+		return this;
+	}
+	public void Test(Db_conn conn) {
+		DataRdr rdr = DataRdr_.Null;
+		Bry_bfr bfr = Bry_bfr.new_();
+		try {
+			rdr = conn.Exec_qry_as_rdr(qry);
+			int expd_row_idx = 0, expd_row_max = rows.Count();
+			while (rdr.MoveNextPeer()) {
+				if (expd_row_idx == expd_row_max) break;
+				Db_tst_row expd_row = (Db_tst_row)rows.Get_at(expd_row_idx);
+				Test_row(bfr, expd_row_idx, expd_row, rdr);
+				++expd_row_idx;
+			}
+		}	finally {rdr.Rls();}
+	}
+	private void Test_row(Bry_bfr bfr, int expd_row_idx, Db_tst_row expd_row, DataRdr rdr) {
+		Db_tst_val[] expd_vals = expd_row.Vals_ary();
+		int len = expd_vals.length;
+		int pad_max = 16;
+		boolean pass = true;
+		bfr.Clear().Add_byte_nl();	// add prefix nl for JUnit formatting 
+		for (int i = 0; i < len; i++) {
+			Db_tst_val expd_val = expd_vals[i];
+			String expd_key = expd_val.Key();
+			String expd_str = Object_.Xto_str_strict_or_empty(expd_val.Val());
+			String actl_str = Object_.Xto_str_strict_or_empty(rdr.Read(expd_key));
+			boolean eq = String_.Eq(expd_str, actl_str);	// NOTE: always compare strings, not objs; problems with comparing byte to int
+			bfr.Add_str_pad_space_end(expd_key, pad_max);
+			bfr.Add_str_pad_space_bgn(expd_str, pad_max);
+			bfr.Add(eq ? Lbl_eq_y : Lbl_eq_n);
+			bfr.Add_str_pad_space_end(actl_str, pad_max);
+			bfr.Add_byte_nl();
+			if (!eq) pass = false;
+		}
+		if (!pass) {
+			bfr.Add(Lbl_row_hdr).Add_int_variable(expd_row_idx).Add_byte_nl();
+			bfr.Add_str(qry.Xto_sql()).Add_byte(Byte_ascii.Semic);
+			throw Exc_.new_(bfr.Xto_str_and_clear());
+		}
+	}	static final byte[] Lbl_row_hdr = Bry_.new_a7("row: "), Lbl_eq_y = Bry_.new_a7(" == "), Lbl_eq_n = Bry_.new_a7(" != ");
+	public static Db_tst_qry tbl_(String tbl_name, String order_by) {return new_(Db_qry_.select_tbl_(tbl_name).OrderBy_asc_(order_by));}
+	public static Db_tst_qry new_(Db_qry qry) {
+		Db_tst_qry rv = new Db_tst_qry();
+		rv.qry = qry;
+		return rv;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_txt.java

@@ -0,0 +1,24 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.ios.*;
+public class Xob_categorylinks_txt extends Xob_categorylinks_base {
+	public Xob_categorylinks_txt(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}
+	@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_text_cat_link;}
+	@Override public Io_sort_cmd Make_sort_cmd(Sql_file_parser sql_parser) {return new Io_sort_fil_basic(bldr.Usr_dlg(), this.Make_url_gen(), make_fil_len);}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_ctg_v1_base.java

@@ -0,0 +1,141 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.core.primitives.*; import gplx.core.btries.*; import gplx.core.flds.*; import gplx.ios.*; import gplx.xowa.tdbs.*;
+import gplx.xowa.wikis.data.tbls.*;
+public abstract class Xob_ctg_v1_base extends Xob_itm_dump_base implements Xobd_parser_wkr, GfoInvkAble {
+	protected Xob_ctg_v1_base() {}	// TEST:needed for fxt
+	public Xob_ctg_v1_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); return this;}
+	public abstract String Wkr_key();
+	public abstract Io_sort_cmd Make_sort_cmd();
+	public Ordered_hash Wkr_hooks() {return wkr_hooks;} private Ordered_hash wkr_hooks = Ordered_hash_.new_bry_();
+	public void Wkr_bgn(Xob_bldr bldr) {
+		this.Init_dump(this.Wkr_key(), wiki.Tdb_fsys_mgr().Site_dir().GenSubDir(Xotdb_dir_info_.Name_category));
+		Bry_bfr tmp_bfr = bldr.App().Utl__bfr_mkr().Get_b512();
+		Xol_lang lang = wiki.Lang();
+		wkr_hooks_add(tmp_bfr, lang.Ns_names());
+		wkr_hooks_add(tmp_bfr, lang.Ns_aliases());
+		wkr_hooks_add(tmp_bfr, Xow_ns_.Canonical);
+		tmp_bfr.Mkr_rls();
+		fld_wtr.Bfr_(dump_bfr);
+	}
+	public int Wkr_run(Xowd_page_itm page, byte[] src, int src_len, int bgn, int end) {
+		int ttl_bgn = end, ttl_end = -1;
+		int pos = end;
+		while (true) {
+			if (pos == src_len) {
+				Log(Tid_eos, page, src, bgn);
+				return end;
+			}
+			Object o = trie.Match_bgn(src, pos, src_len);
+			if (o != null) {
+				Btrie_itm_stub stub = (Btrie_itm_stub)o;
+				byte[] bry = stub.Val();
+				switch (stub.Tid()) {
+					case Tid_brack_end: case Tid_pipe:
+						ttl_bgn = Move_fwd_while_space(src, src_len, ttl_bgn);
+						ttl_end = Move_bwd_while_space(src, src_len, pos - 1);
+						if (ttl_end > ttl_bgn)	// NOTE: ignore examples like [[Category: ]]
+							Process_ctg(page, src, src_len, ttl_bgn, ttl_end);
+						break;
+					case Tid_brack_bgn:
+						Log(Tid_brack_bgn, page, src, bgn);
+						return pos;
+					case Tid_nl:
+						Log(Tid_nl, page, src, bgn);
+						return pos;
+				}
+				return pos + bry.length;
+			}
+			++pos;
+		}
+	}
+	@gplx.Virtual public void Log(byte err_tid, Xowd_page_itm page, byte[] src, int ctg_bgn) {
+		String title = String_.new_u8(page.Ttl_full_db());
+		int ctg_end = ctg_bgn + 40; if (ctg_end > src.length) ctg_end = src.length;
+		String ctg_str = String_.Replace(String_.new_u8(src, ctg_bgn, ctg_end), "\n", "");
+		String err = "";
+		switch (err_tid) {
+			case Tid_eos:		err = "eos"; break;
+			case Tid_nl:		err = "bad \\n"; break;
+			case Tid_brack_bgn:	err = "bad [["; break;
+		}
+		bldr.Usr_dlg().Log_many(GRP_KEY, "ctg_fail", "~{0}\n>> ~{1}\n~{2}\n~{3}\n\n", LogErr_hdr, err + " " + ctg_str, "http://" + wiki.Domain_str() + "/wiki/" + title, Bry_.MidByLenToStr(src, ctg_bgn, 100));
+		log_idx++;
+	}	int log_idx = 0; final String LogErr_hdr = String_.Repeat("-", 80);
+	@gplx.Virtual public void Process_ctg(Xowd_page_itm page, byte[] src, int src_len, int bgn, int end) {
+		Process_ctg_row(fld_wtr, dump_fil_len, dump_url_gen, page.Id(), src, src_len, bgn, end);
+	}
+	public static void Process_ctg_row(Gfo_fld_wtr fld_wtr, int dump_fil_len, Io_url_gen dump_url_gen, int page_id, byte[] src, int src_len, int bgn, int end) {
+		int len = end - bgn;
+		Bry_bfr dump_bfr = fld_wtr.Bfr();
+		if (dump_bfr.Len() + row_fixed_len + len > dump_fil_len) Io_mgr.I.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
+		byte[] ttl = Bry_.Mid(src, bgn, end);
+		Bry_.Replace_reuse(ttl, Byte_ascii.Space, Byte_ascii.Underline);
+		fld_wtr.Write_bry_escape_fld(ttl).Write_int_base85_len5_row(page_id);
+	}
+	public void Wkr_end() {
+		this.Term_dump(this.Make_sort_cmd());
+		if (delete_temp) Io_mgr.I.DeleteDirDeep(temp_dir);
+	}
+	private Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
+	Btrie_fast_mgr trie = Btrie_fast_mgr.cs_().Add_stub(Tid_brack_end, "]]").Add_stub(Tid_pipe, "|").Add_stub(Tid_nl, "\n").Add_stub(Tid_brack_bgn, "[[");
+	static final int row_fixed_len = 5 + 1 + 1;	// 5=rowId; 1=|; 1=\n
+	List_adp category_list = List_adp_.new_(); Int_obj_ref cur_pos = Int_obj_ref.zero_();
+	static final byte Tid_eos = 0, Tid_brack_end = 1, Tid_pipe = 2, Tid_nl = 3, Tid_brack_bgn = 4;
+	private static int Move_fwd_while_space(byte[] src, int src_len, int pos) {
+		while (true) {
+			if (pos == src_len) return pos;
+			switch (src[pos]) {
+				case Byte_ascii.Space:	break;
+				default:				return pos;
+			}
+			++pos;
+		}
+	}
+	private static int Move_bwd_while_space(byte[] src, int src_len, int pos) {
+		while (true) {
+			if (pos == -1) return 0;
+			switch (src[pos]) {
+				case Byte_ascii.Space:	break;
+				default:				return pos + 1;
+			}
+			--pos;
+		}
+	}
+	private void wkr_hooks_add(Bry_bfr tmp_bfr, Xow_ns[] ary) {
+		int len = ary.length;
+		for (int i = 0; i < len; i++) {
+			Xow_ns ns = ary[i];
+			if (ns.Id_ctg()) wkr_hooks_add(tmp_bfr, ns.Name_bry());
+		}
+	}
+	private void wkr_hooks_add(Bry_bfr tmp_bfr, Xol_ns_grp ns_grp) {
+		int len = ns_grp.Len();
+		for (int i = 0; i < len; i++) {
+			Xow_ns ns = ns_grp.Get_at(i);
+			if (ns.Id_ctg()) wkr_hooks_add(tmp_bfr, ns.Name_bry());
+		}
+	}
+	private void wkr_hooks_add(Bry_bfr tmp_bfr, byte[] word) {
+		tmp_bfr.Add_byte(Byte_ascii.Brack_bgn).Add_byte(Byte_ascii.Brack_bgn).Add(word).Add_byte(Byte_ascii.Colon);
+		byte[] key = tmp_bfr.Xto_bry_and_clear();
+		if (!wkr_hooks.Has(key)) wkr_hooks.Add(key, key);
+	}	
+	static final String GRP_KEY = "xowa.bldr.make_ctg";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_ctg_v1_base_tst.java

@@ -0,0 +1,61 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*; import gplx.ios.*; import gplx.xowa.wikis.data.tbls.*;
+public class Xob_ctg_v1_base_tst {
+	Xowd_page_wkr_ctg_fxt fxt = new Xowd_page_wkr_ctg_fxt();
+	@Test  public void One() 		{fxt.ini_("[[Category:A]]").tst_("A");}
+	@Test  public void Many() 		{fxt.ini_("[[Category:A]] [[Category:B]] [[Category:C]]").tst_("A", "B", "C");}
+	@Test  public void Pipe() 		{fxt.ini_("[[Category:A|B]]").tst_("A");}
+	@Test  public void Nl() 		{fxt.ini_("[[Category:A\nB]]").tst_();}
+	@Test  public void Brack_bgn() 	{fxt.ini_("[[Category:A[[Category:B]]").tst_("B");}
+	@Test  public void Mix() 		{fxt.ini_("[[Category:A]] [[Category:B\nc]][[Category:D|E]]").tst_("A", "D");}
+	@Test  public void Ws() 		{fxt.ini_("[[Category: A ]]").tst_("A");}
+	@Test  public void Eos() 		{fxt.ini_("[[Category:abcdef").tst_();}
+}
+class Xowd_page_wkr_ctg_fxt {
+	byte[] src;
+	public Xowd_page_wkr_ctg_fxt ini_(String s) {src = Bry_.new_u8(s); return this;}
+	public Xowd_page_wkr_ctg_fxt tst_(String... expd) {
+		Xobd_parser mgr = new Xobd_parser();
+		Xoae_app app = Xoa_app_fxt.app_();
+		Xowe_wiki wiki = Xoa_app_fxt.wiki_tst_(app);
+		Xob_bldr bldr = Xoa_app_fxt.bldr_(app);
+		Xobd_parser_wkr_ctg_tstr wkr = (Xobd_parser_wkr_ctg_tstr)new Xobd_parser_wkr_ctg_tstr().Ctor(bldr, wiki);
+		byte[] bry = Bry_.new_u8("[[Category:");
+		wkr.Wkr_hooks().Add(bry, bry);
+		mgr.Wkr_add(wkr);
+		Xowd_page_itm page = new Xowd_page_itm().Text_(src);//.Ttl_(Bry_.new_u8("Test"), new Xow_ns_mgr());
+		mgr.Wkr_bgn(bldr);
+		mgr.Wkr_run(page);
+		byte[][] ttl = (byte[][])wkr.Found().To_ary(byte[].class);
+		String[] actl = new String[ttl.length];
+		for (int i = 0; i < actl.length; i++) {
+			actl[i] = String_.new_u8(ttl[i]);
+		}
+		Tfds.Eq_ary_str(expd, actl);
+		return this;
+	}
+}
+class Xobd_parser_wkr_ctg_tstr extends Xob_ctg_v1_txt {	public List_adp Found() {return found;} List_adp found = List_adp_.new_();
+	@Override public void Process_ctg(Xowd_page_itm page, byte[] src, int src_len, int bgn, int end) {
+		found.Add(Bry_.Mid(src, bgn, end));
+	}
+	@Override public void Log(byte err_tid, Xowd_page_itm page, byte[] src, int ctg_bgn) {
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_ctg_v1_sql.java

@@ -0,0 +1,77 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.core.flds.*; import gplx.ios.*; import gplx.dbs.*; import gplx.xowa.dbs.*; import gplx.xowa.ctgs.*;
+public class Xob_ctg_v1_sql extends Xob_ctg_v1_base {
+	@Override public String Wkr_key() {return Xob_cmd_keys.Key_text_cat_core_v1;}
+	@Override public Io_sort_cmd Make_sort_cmd() {return new Xob_ctg_v1_sql_make(wiki);}
+}
+class Xob_ctg_v1_sql_make implements Io_make_cmd {
+	private Gfo_fld_rdr fld_rdr = Gfo_fld_rdr.xowa_(); private Xob_tmp_wtr sql_wtr; private Gfo_usr_dlg usr_dlg; private boolean is_first = true;
+	private byte[] prv_ctg_name = Bry_.Empty; private int prv_page_id = 0;
+	private Xowe_wiki wiki; private Xodb_mgr_sql db_mgr; private int page_count = 0; private int progress_interval = 10000;
+	private final Bry_fmtr fmtr = Bry_fmtr.new_("(~{page_id},'~{cat_name}','','','','','~{cat_type}')\n", "page_id", "cat_name", "cat_type");
+	public Xob_ctg_v1_sql_make(Xowe_wiki wiki) {this.wiki = wiki; db_mgr = wiki.Db_mgr_as_sql();}
+	public Io_sort_cmd Make_dir_(Io_url v) {return this;}	// ignore
+	public void Sort_bgn() {
+		usr_dlg = wiki.Appe().Usr_dlg();
+		Io_url sql_url = wiki.Fsys_mgr().Root_dir().GenSubFil(Url_sql);
+		Io_mgr.I.DeleteFil_args(sql_url).MissingFails_off().Exec();
+		sql_wtr = Xob_tmp_wtr.new_wo_ns_(Io_url_gen_.fil_(sql_url), Io_mgr.Len_mb);
+		sql_wtr.Bfr().Add_str(Xob_categorylinks_sql.Sql_categorylinks).Add(Sql_hdr);
+	}
+	public void Sort_do(Io_line_rdr rdr) {
+		if (page_count++ % progress_interval == 0) usr_dlg.Prog_many("", "", "building category; count=~{0}", page_count);
+		fld_rdr.Ini(rdr.Bfr(), rdr.Itm_pos_bgn());
+		byte[] ctg_name = fld_rdr.Read_bry_escape();
+		ctg_name = Escape_for_sql(wiki, ctg_name);
+		int page_id = fld_rdr.Read_int_base85_len5();
+		if (Bry_.Eq(prv_ctg_name, ctg_name) && page_id == prv_page_id) return; 
+		if (sql_wtr.FlushNeeded(5 + 2 + ctg_name.length)) sql_wtr.Flush(usr_dlg);	// 5=base85; 2=dlms
+		byte row_dlm = is_first ? Byte_ascii.Space : Byte_ascii.Comma;	// handle " (" or ",("
+		is_first = false;
+		sql_wtr.Bfr().Add_byte(row_dlm); 
+		fmtr.Bld_bfr_many(sql_wtr.Bfr(), page_id, ctg_name, Byte_ascii.Ltr_p);	// 'p' b/c page_ttl is not available; limitation of ctg_v1
+		prv_page_id = page_id;
+		prv_ctg_name = ctg_name;
+	}
+	public void Sort_end() {
+		sql_wtr.Bfr().Add_byte(Byte_ascii.Semic);	// close INSERT with ";"
+		sql_wtr.Flush(usr_dlg);
+		db_mgr.Category_version_update(true);
+	}
+	private static final byte[] Sql_hdr = Bry_.new_a7("INSERT INTO 'categorylinks' VALUES");	
+	public static final String Url_sql = "xowa_categorylinks.sql";
+	private static byte[] Escape_for_sql(Xowe_wiki wiki, byte[] bry) {
+		Bry_bfr bfr = wiki.Appe().Utl__bfr_mkr().Get_b512();
+		int len = bry.length;
+		boolean dirty = false;
+		for (int i = 0; i < len; i++) {
+			byte b = bry[i];
+			switch (b) {
+				case Byte_ascii.Apos:		if (!dirty) {bfr.Add_mid(bry, 0, i); dirty = true;} bfr.Add_byte(Byte_ascii.Backslash).Add_byte(Byte_ascii.Apos); break;
+				case Byte_ascii.Backslash:	if (!dirty) {bfr.Add_mid(bry, 0, i); dirty = true;} bfr.Add_byte(Byte_ascii.Backslash).Add_byte(Byte_ascii.Backslash); break;
+				default:
+					if (dirty) bfr.Add_byte(b);
+					break;
+			}
+		}
+		bfr.Mkr_rls();
+		return dirty ? bfr.Xto_bry_and_clear() : bry;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_ctg_v1_sql_tst.java

@@ -0,0 +1,84 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*; import gplx.dbs.*; import gplx.xowa.specials.search.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.xowa.bldrs.cmds.texts.sqls.*;
+public class Xob_ctg_v1_sql_tst {
+	@Before public void init() {if (Xoa_test_.Db_skip()) return; fxt.Ctor_fsys();} Db_mgr_fxt fxt = new Db_mgr_fxt();
+	@After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();} 
+	@Test   public void Basic() {
+		if (Xoa_test_.Db_skip()) return; 
+		fxt.Init_db_sqlite();
+		fxt.doc_ary_
+		(	fxt.doc_wo_date_(1, "A"				, "[[Category:Ctg_0]] [[Category:Ctg_1]]")
+		,	fxt.doc_wo_date_(2, "B"				, "[[Category:Ctg_1]] [[Category:Ctg_2]]")
+		,	fxt.doc_wo_date_(3, "File:C"		, "[[Category:Ctg_0]]")
+		,	fxt.doc_wo_date_(4, "Category:D"	, "[[Category:Ctg_0]]")
+		,	fxt.doc_wo_date_(5, "E"				, "[[Category:Ctg_a'b]]")
+		,	fxt.doc_wo_date_(6, "F"				, "[[Category:Ctg_a\\]]")
+		)
+		.Exec_run(new Xob_page_cmd(fxt.Bldr(), fxt.Wiki()))
+		.Exec_run(new Xob_ctg_v1_sql().Ctor(fxt.Bldr(), fxt.Wiki()))
+		;
+		fxt.Test_file(Xoa_test_.Url_root().GenSubFil_nest("root", "wiki", "en.wikipedia.org", "xowa_categorylinks.sql").Raw(), String_.Concat_lines_nl
+		(	  "CREATE TABLE `categorylinks` ("
+		,	"  `cl_from` int(10) unsigned NOT NULL DEFAULT '0',"
+		,	"  `cl_to` varbinary(255) NOT NULL DEFAULT '',"
+		,	"  `cl_sortkey` varbinary(230) NOT NULL DEFAULT '',"
+		,	"  `cl_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,"
+		,	"  `cl_sortkey_prefix` varbinary(255) NOT NULL DEFAULT '',"
+		,	"  `cl_collation` varbinary(32) NOT NULL DEFAULT '',"
+		,	"  `cl_type` enum('page','subcat','file') NOT NULL DEFAULT 'page',"
+		,	"  UNIQUE KEY `cl_from` (`cl_from`,`cl_to`),"
+		,	");"
+		,	"INSERT INTO 'categorylinks' VALUES (1,'Ctg_0','','','','','p')"
+		,	",(3,'Ctg_0','','','','','p')"
+		,	",(4,'Ctg_0','','','','','p')"
+		,	",(1,'Ctg_1','','','','','p')"
+		,	",(2,'Ctg_1','','','','','p')"
+		,	",(2,'Ctg_2','','','','','p')"
+		,	",(5,'Ctg_a\\'b','','','','','p')"
+		,	",(6,'Ctg_a\\\\','','','','','p')"
+		,	";"
+		));
+	}
+	@Test   public void Ignore_dupes() {	// PURPOSE: ignore dupe ctgs
+		if (Xoa_test_.Db_skip()) return;
+		fxt.Init_db_sqlite();
+		fxt.doc_ary_
+		(	fxt.doc_wo_date_(1, "A"				, "[[Category:Ctg_0]] [[Category:Ctg_0]]")
+		)
+		.Exec_run(new Xob_page_cmd(fxt.Bldr(), fxt.Wiki()))
+		.Exec_run(new Xob_ctg_v1_sql().Ctor(fxt.Bldr(), fxt.Wiki()))
+		;
+		fxt.Test_file(Xoa_test_.Url_root().GenSubFil_nest("root", "wiki", "en.wikipedia.org", "xowa_categorylinks.sql").Raw(), String_.Concat_lines_nl
+		(	  "CREATE TABLE `categorylinks` ("
+		,	"  `cl_from` int(10) unsigned NOT NULL DEFAULT '0',"
+		,	"  `cl_to` varbinary(255) NOT NULL DEFAULT '',"
+		,	"  `cl_sortkey` varbinary(230) NOT NULL DEFAULT '',"
+		,	"  `cl_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,"
+		,	"  `cl_sortkey_prefix` varbinary(255) NOT NULL DEFAULT '',"
+		,	"  `cl_collation` varbinary(32) NOT NULL DEFAULT '',"
+		,	"  `cl_type` enum('page','subcat','file') NOT NULL DEFAULT 'page',"
+		,	"  UNIQUE KEY `cl_from` (`cl_from`,`cl_to`),"
+		,	");"
+		,	"INSERT INTO 'categorylinks' VALUES (1,'Ctg_0','','','','','p')"
+		,	";"
+		));
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_ctg_v1_txt.java

@@ -0,0 +1,23 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.ios.*;
+public class Xob_ctg_v1_txt extends Xob_ctg_v1_base {
+	@Override public String Wkr_key() {return Xob_cmd_keys.Key_tdb_make_category;}
+	@Override public Io_sort_cmd Make_sort_cmd() {return new Xob_make_cmd_site(bldr.Usr_dlg(), make_dir, make_fil_len);}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_sql_join_mgr.java

@@ -0,0 +1,36 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.ios.*;
+interface Xob_sql_join_wkr {
+	Io_line_rdr New_main_rdr();
+	Io_line_rdr New_join_rdr();
+	void Process_match(Io_line_rdr main, Io_line_rdr join, byte[] key_bry);
+}
+class Xob_sql_join_mgr {
+	public static void Exec_join(Xob_sql_join_wkr wkr) {
+		Io_line_rdr main_rdr = wkr.New_main_rdr();
+		Io_line_rdr join_rdr = wkr.New_join_rdr();
+		while (main_rdr.Read_next()) {
+			byte[] key_bry = Bry_.Mid(main_rdr.Bfr(), main_rdr.Key_pos_bgn(), main_rdr.Key_pos_end());
+			if (join_rdr.Match(key_bry)) {
+				wkr.Process_match(main_rdr, join_rdr, key_bry);
+			}
+		}
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_parser_base.java

@@ -0,0 +1,43 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.ios.*;
+public abstract class Xoctg_hiddencat_parser_base extends Xob_sql_dump_base implements Sql_file_parser_cmd {
+	public Xoctg_hiddencat_parser_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb; return this;}
+	@Override public String Sql_file_name() {return "page_props";}
+	private static final byte Fld_id = 0, Fld_key = 1, Fld_val = 2;
+	private int cur_id = -1;
+	private boolean cur_is_hiddencat = false;
+	private int rows = 0;
+	@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
+		parser.Fld_cmd_(this).Flds_req_idx_(4, 0, 1, 2);	// NOTE: 4 b/c MW added fld_3:pp_sortkey; DATE:2014-04-28
+	}
+	public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) {
+		switch (fld_idx) {
+			case Fld_id:			cur_id = Bry_.Xto_int_or(src, fld_bgn, fld_end, -1); break;
+			case Fld_key:			cur_is_hiddencat = Bry_.Eq(Key_hiddencat, src, fld_bgn, fld_end); break;
+			case Fld_val:
+				if (!cur_is_hiddencat) {data.Cancel_row_y_(); return;}
+				Exec_hook(file_bfr, cur_id, cur_is_hiddencat);
+				if (++rows % 100000 == 0) usr_dlg.Prog_many("", "", "reading row ~{0}", Int_.Xto_str_fmt(rows, "#,##0"));
+				break;
+		}
+	}
+	public abstract void Exec_hook(Bry_bfr file_bfr, int cur_id, boolean cur_is_hiddencat);
+	public static final byte[] Key_hiddencat = Bry_.new_a7("hiddencat");
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_parser_sql.java

@@ -0,0 +1,47 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.xowa.dbs.*; import gplx.xowa.wikis.data.tbls.*;
+import gplx.xowa.wikis.data.*;
+public class Xoctg_hiddencat_parser_sql extends Xoctg_hiddencat_parser_base {
+	private Xowd_cat_core_tbl tbl;
+	public Xoctg_hiddencat_parser_sql(Xob_bldr bldr, Xowe_wiki wiki) {this.Ctor(bldr, wiki);}
+	@Override public String Cmd_key() {return Xob_cmd_keys.Key_text_cat_hidden;}
+	@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
+		super.Cmd_bgn_hook(bldr, parser);
+		Xodb_mgr_sql db_mgr = wiki.Db_mgr_as_sql();
+		tbl = db_mgr.Core_data_mgr().Db__cat_core().Tbl__cat_core();
+		tbl.Update_bgn();
+	}
+	@Override public void Exec_hook(Bry_bfr file_bfr, int cur_id, boolean cur_is_hiddencat) {
+		if (cur_is_hiddencat)
+			tbl.Update_by_batch(cur_id, cur_is_hiddencat ? Bool_.Y_byte : Bool_.N_byte);
+	}
+	@Override public void Cmd_end() {
+		tbl.Update_end();
+		if (!Env_.Mode_testing())	// NOTE: do not delete when testing
+			Io_mgr.I.DeleteDirDeep(wiki.Fsys_mgr().Tmp_dir());	// delete /wiki/wiki_name/tmp
+		Io_url[] sql_files = Io_mgr.I.QueryDir_args(wiki.Fsys_mgr().Root_dir()).FilPath_("*.sql.gz").ExecAsUrlAry();
+		int len = sql_files.length;
+		for (int i = 0; i < len; i++) {
+			Io_url sql_file = sql_files[i];
+			Io_mgr.I.DeleteFil(sql_file);
+		}
+		Io_mgr.I.DeleteFil_args(wiki.Fsys_mgr().Root_dir().GenSubFil("xowa_categorylinks.sql")).MissingFails_off().Exec();
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_parser_sql_tst.java

@@ -0,0 +1,61 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*; import gplx.dbs.*; import gplx.xowa.dbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.*;
+public class Xoctg_hiddencat_parser_sql_tst {
+	@Before public void init() {if (Xoa_test_.Db_skip()) return; fxt.Ctor_fsys();} Db_mgr_fxt fxt = new Db_mgr_fxt();
+	@After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();} 
+	@Test   public void Basic() {
+		if (Xoa_test_.Db_skip()) return; 
+		fxt.Init_db_sqlite();
+		fxt.Wiki().Db_mgr_as_sql().Core_data_mgr().Db__cat_core().Tbl__cat_core().Create_tbl();
+		Init_ctgs(1, 2, 3);
+		Io_url page_props_url = Xoa_test_.Url_root().GenSubFil_nest("root", "wiki", "en.wikipedia.org", "page_props.sql");		
+		fxt	.Init_fil(page_props_url, String_.Concat
+		(	"INSERT INTO `page_props` VALUES"
+		,	" (1,'hiddencat','')"
+		,	",(2,'pageimage','A.png')"
+		,	",(3,'hiddencat','')"
+		,	";"
+		))
+		.Exec_run(new Xoctg_hiddencat_parser_sql(fxt.Bldr(), fxt.Wiki()))
+		;
+		Tst_ctg_hidden(Bool_.Y, 1, 3);
+		Tst_ctg_hidden(Bool_.N, 2);
+	}
+	private void Init_ctgs(int... ctgs) {
+		int len = ctgs.length;
+		Xowd_cat_core_tbl tbl = fxt.Wiki().Db_mgr_as_sql().Core_data_mgr().Db__cat_core().Tbl__cat_core().Create_tbl();
+		tbl.Insert_bgn();
+		try {
+			for (int i = 0; i < len; i++) {
+				int ctg_id = ctgs[i];
+				tbl.Insert_cmd_by_batch(ctg_id, 0, 0, 0, Bool_.N_byte, 0);
+			}		
+		} finally {tbl.Insert_end();}
+	}
+	private void Tst_ctg_hidden(boolean expd_hidden, int... ctgs) {
+		int len = ctgs.length;
+		Xowd_cat_core_tbl tbl = fxt.Wiki().Db_mgr_as_sql().Core_data_mgr().Db__cat_core().Tbl__cat_core();
+		for (int i = 0; i < len; i++) {
+			int ctg_id = ctgs[i];
+			Xowd_category_itm ctg_itm = tbl.Select(ctg_id);
+			Tfds.Eq(expd_hidden, ctg_itm.Hidden(), Int_.Xto_str(ctg_id));
+		}
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_parser_txt.java

@@ -0,0 +1,33 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.ios.*;
+public class Xoctg_hiddencat_parser_txt extends Xoctg_hiddencat_parser_base {
+	public Xoctg_hiddencat_parser_txt(Xob_bldr bldr, Xowe_wiki wiki) {this.Ctor(bldr, wiki);}
+	@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_cat_hidden_sql;}
+	@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
+		super.Cmd_bgn_hook(bldr, parser);
+	}
+	@Override public void Exec_hook(Bry_bfr file_bfr, int cur_id, boolean cur_is_hiddencat) {
+		fld_wtr.Bfr_(file_bfr);
+		fld_wtr.Write_int_base85_len5_fld(cur_id);
+	}
+	@Override public void Cmd_end() {
+		Xobdc_merger.Basic(bldr.Usr_dlg(), dump_url_gen, temp_dir.GenSubDir("sort"), sort_mem_len, Xoctg_link_sql_sorter._, Io_line_rdr_key_gen_.noop, new Io_sort_fil_basic(bldr.Usr_dlg(), this.Make_url_gen(), make_fil_len));
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_ttl_wkr.java

@@ -0,0 +1,68 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.ios.*; import gplx.xowa.tdbs.*; import gplx.xowa.bldrs.*;
+public class Xoctg_hiddencat_ttl_wkr extends Xob_itm_dump_base implements Xob_cmd, GfoInvkAble {
+	public Xoctg_hiddencat_ttl_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;} private Xob_sql_join_wkr_ctg_hidden join_wkr;
+	public String Cmd_key() {return Xob_cmd_keys.Key_tdb_cat_hidden_ttl;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {
+		this.Init_dump(Xob_cmd_keys.Key_tdb_cat_hidden_ttl);
+		src_sql_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest(Xob_cmd_keys.Key_tdb_cat_hidden_sql, "make");
+		join_wkr = new Xob_sql_join_wkr_ctg_hidden(bldr.App(), wiki, temp_dir, src_sql_dir);
+	}	Io_url src_sql_dir;
+	public void Cmd_run() {
+		Xob_sql_join_mgr.Exec_join(join_wkr);		
+	}
+	public void Cmd_end() {
+		join_wkr.Flush();
+		Io_url_gen make_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("make"));
+		Xobdc_merger.Basic(bldr.Usr_dlg(), join_wkr.Dump_url_gen(), temp_dir.GenSubDir("sort"), sort_mem_len, Io_sort_split_itm_sorter._, Io_line_rdr_key_gen_.first_pipe, new Io_sort_fil_basic(bldr.Usr_dlg(), make_url_gen, make_fil_len));
+		if (delete_temp) Io_mgr.I.DeleteDirDeep(src_sql_dir);
+	}
+	public void Cmd_term() {}
+}
+class Xob_sql_join_wkr_ctg_hidden implements Xob_sql_join_wkr {
+	public Xob_sql_join_wkr_ctg_hidden(Xoae_app app, Xowe_wiki wiki, Io_url temp_dir, Io_url src_sql_dir) {
+		this.app = app; this.wiki = wiki;
+		this.dump_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("dump"));
+		this.src_sql_dir = src_sql_dir;
+	}	private Xoae_app app = null; Xowe_wiki wiki = null; Io_url src_sql_dir;
+	public Io_url_gen Dump_url_gen() {return dump_url_gen;} Io_url_gen dump_url_gen;
+	public Io_line_rdr New_main_rdr() {
+		Io_url[] urls = Io_mgr.I.QueryDir_fils(src_sql_dir);
+		return new Io_line_rdr(app.Usr_dlg(), urls).Key_gen_(Io_line_rdr_key_gen_.first_pipe);
+	} 
+	public Io_line_rdr New_join_rdr() {
+		Io_url make_dir = wiki.Tdb_fsys_mgr().Url_site_dir(Xotdb_dir_info_.Tid_id);
+		app.Usr_dlg().Prog_many("", "", "getting id files: ~{0}", make_dir.Raw());
+		Io_url[] urls = Io_mgr.I.QueryDir_args(make_dir).Recur_().FilPath_("*.xdat").ExecAsUrlAry();
+		return new Io_line_rdr(app.Usr_dlg(), urls).Key_gen_(Io_line_rdr_key_gen_.first_pipe).File_skip_line0_(true);
+	} 
+	public void Process_match(Io_line_rdr main, Io_line_rdr join, byte[] key_bry) {
+		byte[] src = join.Bfr();
+		int itm_end = join.Itm_pos_end();
+		int pipe_pos = Bry_finder.Find_bwd(src, Byte_ascii.Pipe, itm_end);
+		if (pipe_pos == Bry_.NotFound) throw Exc_.new_("failed to find pipe for name", "excerpt", String_.new_u8(src, join.Itm_pos_bgn(), join.Itm_pos_end()));
+		file_bfr.Add_mid(src, pipe_pos + 1, itm_end - 1).Add_byte_pipe();
+		file_bfr.Add(key_bry).Add_byte_nl();
+	}	private Bry_bfr file_bfr = Bry_bfr.new_();
+	public void Flush() {
+		Io_mgr.I.SaveFilBfr(dump_url_gen.Nxt_url(), file_bfr);
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_ttl_wkr_tst.java

@@ -0,0 +1,45 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*; import gplx.xowa.bldrs.*;
+public class Xoctg_hiddencat_ttl_wkr_tst {
+	@Before public void init() {fxt.Clear();} private Xob_base_fxt fxt = new Xob_base_fxt();
+	@Test   public void Basic() {
+		fxt
+		.Init_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.hiddencat_sql/make/0000000000.csv", String_.Concat_lines_nl
+		(	"!!!!#|"
+		,	"!!!!$|"
+		,	"!!!!%|"
+		))
+		.Init_fil("mem/xowa/wiki/en.wikipedia.org/site/id/00/00/00/00/0000000000.xdat", String_.Concat_lines_nl
+		(	"header_line"
+		,	"!!!!#|C"
+		,	"!!!!%|A"
+		))
+		.Exec_cmd(Xob_cmd_keys.Key_tdb_cat_hidden_ttl)
+		.Test_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.hiddencat_ttl/dump/0000000000.csv", String_.Concat_lines_nl
+		(	"C|!!!!#"
+		,	"A|!!!!%"
+		))
+		.Test_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.hiddencat_ttl/make/0000000000.csv", String_.Concat_lines_nl
+		(	"A|!!!!%"
+		,	"C|!!!!#"
+		))
+		;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_link_idx_wkr.java

@@ -0,0 +1,185 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.core.flds.*; import gplx.ios.*; import gplx.xowa.ctgs.*; import gplx.xowa.tdbs.*;
+public class Xoctg_link_idx_wkr extends Xob_idx_base {	// NOTE: similar functionality to Xob_make_cmd_site, but more complicated due to p,f,s; not inheriting
+	Io_url src_link_dir; int make_fil_max = Int_.MinValue;
+	public Xoctg_link_idx_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_ctg_link_idx;}
+	@Override public void Cmd_bgn_hook() {
+		this.fld_rdr = this.Fld_rdr();
+		if (src_link_dir == null) src_link_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest(Xob_cmd_keys.Key_tdb_text_cat_link, "make");
+		if (make_fil_max == Int_.MinValue) make_fil_max = Io_mgr.Len_mb;
+		make_link_mgr = new Xoctg_make_link_mgr(usr_dlg, make_fil_max, wiki.Tdb_fsys_mgr()); 
+		make_main_mgr = new Xoctg_make_main_mgr(usr_dlg, make_fil_max, wiki.Tdb_fsys_mgr());
+		Io_mgr.I.DeleteDirDeep_ary(make_link_mgr.Make_dir(), make_main_mgr.Make_dir());
+	}	Gfo_fld_rdr fld_rdr; Xoctg_make_link_mgr make_link_mgr;
+	@Override public void Cmd_run() {
+		Xoctg_make_link_grp cur_grp = null;
+		Io_line_rdr link_rdr = rdr_(src_link_dir);
+		byte[] prv_name = Bry_.Empty; byte prv_tid = Xoa_ctg_mgr.Tid_null;
+		make_link_mgr.Write_bgn();
+		make_main_mgr.Write_bgn();
+		while (link_rdr.Read_next()) {
+			link_data.Parse_from_sql_dump(fld_rdr, link_rdr);
+			byte[] cur_name = link_data.Name();
+			if (!Bry_.Eq(cur_name, prv_name)) {	// grp changed
+				if (prv_name != Bry_.Empty)		// ignore change from 1st row
+					Write_grp(prv_name, make_link_mgr);
+				prv_name = cur_name;
+			}
+			byte cur_tid = link_data.Tid();
+			if (cur_tid != prv_tid) {				// tid changed
+				cur_grp = make_link_mgr.Grp_by_tid(cur_tid);
+				prv_tid = cur_tid;
+			}
+			cur_grp.Add_itm(link_data);
+		}
+		Write_grp(prv_name, make_link_mgr);
+		make_link_mgr.Flush();
+		make_main_mgr.Flush();
+	}	private Xoctg_idx_data_link link_data = new Xoctg_idx_data_link();
+	@Override public void Cmd_end() {
+		if (delete_temp) Io_mgr.I.DeleteDirDeep_ary(src_link_dir, make_main_mgr.Src_dir());
+	}	boolean delete_temp = true;
+	private void Write_grp(byte[] cur_name, Xoctg_make_link_mgr make_link_mgr) {
+		make_main_mgr.Grps_write(cur_name, make_link_mgr.Subc_grp().Count(), make_link_mgr.File_grp().Count(), make_link_mgr.Page_grp().Count());
+		make_link_mgr.Grps_write(cur_name);
+	}	private Xoctg_make_main_mgr make_main_mgr;
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_src_link_dir_))				src_link_dir = m.ReadIoUrl("v");
+		else if	(ctx.Match(k, Invk_make_fil_max_))				make_fil_max = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_make_fil_max_))				make_fil_max = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_delete_temp_))				delete_temp = m.ReadBool("v");
+		else													return super.Invk(ctx, ikey, k, m);
+		return this;
+	}
+	public static final String Invk_src_link_dir_ = "src_link_dir_", Invk_make_fil_max_ = "make_fil_max_", Invk_delete_temp_ = "delete_temp_";
+}
+class Xoctg_make_link_mgr {
+	public Xoctg_make_link_mgr(Gfo_usr_dlg usr_dlg, int make_fil_max, Xotdb_fsys_mgr fsys_mgr) {
+		this.make_fil_max = make_fil_max;
+		subc_grp = new Xoctg_make_link_grp(Xoa_ctg_mgr.Tid_subc, make_fil_max);
+		file_grp = new Xoctg_make_link_grp(Xoa_ctg_mgr.Tid_file, make_fil_max);
+		page_grp = new Xoctg_make_link_grp(Xoa_ctg_mgr.Tid_page, make_fil_max);
+		make_fil_bfr = Bry_bfr.reset_(make_fil_max);
+		make_fld_wtr = Gfo_fld_wtr.xowa_().Bfr_(make_fil_bfr); 
+		make_dir = fsys_mgr.Url_site_dir(Xotdb_dir_info_.Tid_category2_link);
+		make_cmd = new Xob_make_cmd_site(usr_dlg, make_dir, make_fil_max);
+	}	Gfo_fld_wtr make_fld_wtr; Bry_bfr make_fil_bfr; int make_fil_max; Xob_make_cmd_site make_cmd;
+	public Io_url Make_dir() {return make_dir;} Io_url make_dir;
+	public Xoctg_make_link_grp Subc_grp() {return subc_grp;} private Xoctg_make_link_grp subc_grp;
+	public Xoctg_make_link_grp File_grp() {return file_grp;} private Xoctg_make_link_grp file_grp;
+	public Xoctg_make_link_grp Page_grp() {return page_grp;} private Xoctg_make_link_grp page_grp;
+	public Xoctg_make_link_grp Grp_by_tid(byte tid) {
+		switch (tid) {
+			case Byte_ascii.Ltr_c: return subc_grp;
+			case Byte_ascii.Ltr_f: return file_grp;
+			case Byte_ascii.Ltr_p: return page_grp;
+			default: throw Exc_.new_unhandled(tid);
+		}
+	}
+	public int Grps_bfr_len() {return subc_grp.Bfr_len() + file_grp.Bfr_len() + page_grp.Bfr_len();}
+	public void Grps_reset() {subc_grp.Reset(); file_grp.Reset(); page_grp.Reset();}
+	public void Grps_write(byte[] cur_name) {
+		make_fld_wtr.Write_bry_escape_fld(cur_name);		// write name
+		make_fld_wtr.Write_int_base85_len5_fld(subc_grp.Bfr_len()).Write_int_base85_len5_fld(file_grp.Bfr_len()).Write_int_base85_len5_fld(page_grp.Bfr_len());
+		Write_grp(make_fil_bfr, subc_grp); Write_grp(make_fil_bfr, file_grp); Write_grp(make_fil_bfr, page_grp);
+		make_fil_bfr.Add_byte_nl();
+		byte[] bry = make_fld_wtr.Bfr().Bfr();
+		make_cmd.Do_bry(bry, 0, cur_name.length, 0, make_fld_wtr.Bfr().Len());
+		make_fil_bfr.Clear();
+		this.Grps_reset();
+	}
+	public void Write_bgn() {
+		make_cmd.Sort_bgn();
+		make_cmd.Line_dlm_(Byte_ascii.Nl);
+	}
+	public void Flush() {
+		make_cmd.Sort_end();
+	}
+	private void Write_grp(Bry_bfr make_fil_bfr, Xoctg_make_link_grp grp) {
+		if (grp.Bfr_len() == 0) return;
+		make_fil_bfr.Add_bfr_and_clear(grp.Bfr());		// NOTE: should have trailing pipe
+	}
+	static final int Itm_len_fixed 
+		= 1 				// name: 1 pipe
+		+ (4 + (5 * 3))		// meta: 3 semis; 1 pipe; 3 base85 ints
+		+ (2 * 3)			// sect: c| f| p|
+		+ 1					// nl
+		;
+}
+class Xoctg_make_link_grp {
+	public Xoctg_make_link_grp(byte tid, int bfr_max) {this.tid = tid; bfr = Bry_bfr.reset_(bfr_max);} Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
+	public byte Tid() {return tid;} private byte tid;
+	public int Bfr_len() {return bfr.Len();}
+	public Bry_bfr Bfr() {return bfr;} Bry_bfr bfr;
+	public int Count() {return count;} private int count;
+	public void Reset() {count = 0;}
+	public void Add_itm(Xoctg_idx_data_link link_data) {
+		bfr	.Add_base85_len_5(link_data.Id()).Add_byte(Byte_ascii.Semic)			// add id;			"!!!!;"
+			.Add_base85_len_5(link_data.Timestamp()).Add_byte(Byte_ascii.Semic);	// add timestamp;	"!!!!;"
+		fld_wtr.Bfr_(bfr).Write_bry_escape_fld(link_data.Sortkey());				// add sortkey;		"A|"
+		++count;
+	}
+}
+class Xoctg_idx_data_link {
+	public byte[] Name() {return name;} private byte[] name;
+	public byte Tid() {return tid;} private byte tid;
+	public int Id() {return id;} private int id;
+	public int Timestamp() {return timestamp;} private int timestamp;
+	public byte[] Sortkey() {return sortkey;} private byte[] sortkey;
+	public void Parse_from_sql_dump(Gfo_fld_rdr fld_rdr, Io_line_rdr rdr) {
+		fld_rdr.Ini(rdr.Bfr(), rdr.Itm_pos_bgn());
+		name 			= fld_rdr.Read_bry_escape();
+		tid 			= (byte)(fld_rdr.Read_byte());// - Byte_ascii.Num_0);
+		sortkey			= fld_rdr.Read_bry_escape();
+		id				= fld_rdr.Read_int_base85_len5();
+		timestamp		= fld_rdr.Read_int_base85_len5();
+	}
+}
+class Xoctg_make_main_mgr {
+	public Xoctg_make_main_mgr(Gfo_usr_dlg usr_dlg, int make_fil_max, Xotdb_fsys_mgr fsys_mgr) {
+		this.make_fil_max = make_fil_max;
+		make_fil_bfr = Bry_bfr.reset_(make_fil_max);
+		make_fld_wtr = Gfo_fld_wtr.xowa_().Bfr_(make_fil_bfr); 
+		make_dir = fsys_mgr.Url_site_dir(Xotdb_dir_info_.Tid_category2_main);
+		make_cmd = new Xob_make_cmd_site(usr_dlg, make_dir, make_fil_max);
+		src_dir = fsys_mgr.Tmp_dir().GenSubDir_nest(Xob_cmd_keys.Key_tdb_cat_hidden_ttl, "make");
+		hidden_rdr = new Io_line_rdr(usr_dlg, Io_mgr.I.QueryDir_fils(src_dir));
+	}	Gfo_fld_wtr make_fld_wtr; Bry_bfr make_fil_bfr; int make_fil_max; Xob_make_cmd_site make_cmd; Io_line_rdr hidden_rdr;
+	public Io_url Src_dir() {return src_dir;} Io_url src_dir;
+	public Io_url Make_dir() {return make_dir;} Io_url make_dir;
+	public void Grps_write(byte[] cur_name, int count_subc, int count_file, int count_page) {
+		boolean hidden = hidden_rdr.Match(cur_name);
+		make_fld_wtr.Write_bry_escape_fld(cur_name);		// write name
+		make_fld_wtr.Write_byte_fld(hidden ? Byte_ascii.Ltr_y : Byte_ascii.Ltr_n);
+		make_fld_wtr.Write_int_base85_len5_fld(count_subc).Write_int_base85_len5_fld(count_file).Write_int_base85_len5_fld(count_page);
+		make_fil_bfr.Add_byte_nl();
+		byte[] bry = make_fld_wtr.Bfr().Bfr();
+		make_cmd.Do_bry(bry, 0, cur_name.length, 0, make_fld_wtr.Bfr().Len());
+		make_fil_bfr.Clear();
+	}
+	public void Write_bgn() {
+		make_cmd.Sort_bgn();
+		make_cmd.Line_dlm_(Byte_ascii.Nl);
+	}
+	public void Flush() {
+		make_cmd.Sort_end();
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_link_idx_wkr_tst.java

@@ -0,0 +1,63 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*; import gplx.xowa.ctgs.*; import gplx.xowa.bldrs.*;
+public class Xoctg_link_idx_wkr_tst {		
+	@Before public void init() {fxt.Clear();} private Xob_base_fxt fxt = new Xob_base_fxt();
+	@Test   public void Basic() {
+		fxt
+		.Init_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.link_sql/make/0000000000.csv", String_.Concat_lines_nl
+		(	"Ctg_1|c|Ctg_1a|!!!!@|#8DDL|"
+		,	"Ctg_1|f|A.png|!!!!#|#8DDL|"
+		,	"Ctg_1|p|A0|!!!!$|#8DDL|"
+		,	"Ctg_1|p|A1|!!!!%|#8DDL|"
+		,	"Ctg_2|p|B1|!!!!%|#8DDL|"
+		))
+		.Init_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.hiddencat_ttl/make/0000000000.csv", String_.Concat_lines_nl
+		(	"Ctg_2|!!!!%|"
+		))
+		.Exec_cmd(Xob_cmd_keys.Key_tdb_ctg_link_idx, GfoMsg_.basic_(Xoctg_link_idx_wkr.Invk_make_fil_max_, 72))
+		.Test_fil("mem/xowa/wiki/en.wikipedia.org/site/category2/link/00/00/00/00/0000000000.xdat", String_.Concat_lines_nl
+		(	"!!!\"(|"
+		,	"Ctg_1|!!!!4|!!!!3|!!!!?|!!!!@;#8DDL;Ctg_1a|!!!!#;#8DDL;A.png|!!!!$;#8DDL;A0|!!!!%;#8DDL;A1|"
+		))
+		.Test_fil("mem/xowa/wiki/en.wikipedia.org/site/category2/link/00/00/00/00/0000000001.xdat", String_.Concat_lines_nl
+		(	"!!!!I|"
+		,	"Ctg_2|!!!!!|!!!!!|!!!!0|!!!!%;#8DDL;B1|"
+		))
+		.Test_fil("mem/xowa/wiki/en.wikipedia.org/site/category2/link/reg.csv", String_.Concat_lines_nl
+		(	"0|Ctg_1|Ctg_1|1"
+		,	"1|Ctg_2|Ctg_2|1"
+		))
+		.Test_fil("mem/xowa/wiki/en.wikipedia.org/site/category2/main/00/00/00/00/0000000000.xdat", String_.Concat_lines_nl
+		(	"!!!!<|!!!!<|"
+		,	"Ctg_1|n|!!!!\"|!!!!\"|!!!!#|"
+		,	"Ctg_2|y|!!!!!|!!!!!|!!!!\"|"
+		))
+		.Test_fil("mem/xowa/wiki/en.wikipedia.org/site/category2/main/reg.csv", String_.Concat_lines_nl
+		(	"0|Ctg_1|Ctg_2|2"
+		))
+		;
+		byte[] ctg_name = Bry_.new_a7("Ctg_1");
+		Xoctg_data_ctg main = new Xoctg_data_ctg(ctg_name);
+		fxt.Wiki().Db_mgr().Load_mgr().Load_ctg_v2(main, ctg_name);
+		Tfds.Eq(1, main.Grp_by_tid(Xoa_ctg_mgr.Tid_subc).Total());
+		Tfds.Eq(1, main.Grp_by_tid(Xoa_ctg_mgr.Tid_file).Total());
+		Tfds.Eq(2, main.Grp_by_tid(Xoa_ctg_mgr.Tid_page).Total());
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_bin_db_itm.java

@@ -0,0 +1,57 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.fsdb.meta.*;
+class Xob_bin_db_itm {
+	public Xob_bin_db_itm(int id, Io_url db_url, int ns_id, int pt_id) {this.id = id; this.db_url = db_url; this.ns_id = ns_id; this.pt_id = pt_id;}
+	public int Id() {return id;} private int id;
+	public int Ns_id() {return ns_id;} private final int ns_id;
+	public int Pt_id() {return pt_id;} private int pt_id;
+	public long Db_len() {return db_len;} public void Db_len_(long v) {this.db_len = v;} private long db_len;
+	public Io_url Db_url() {return db_url;} public void Db_url_(Io_url v) {db_url = v;} private Io_url db_url;
+	public void Set(int id, int pt_id, Io_url db_url) {
+		this.id = id; this.pt_id = pt_id; this.db_url = db_url;
+	}
+	public static String Gen_name_v1(int pt_id) {
+		return String_.Format("fsdb.bin.{0}.sqlite3", Int_.Xto_str_pad_bgn_zero(pt_id, 4));
+	}
+	public static String Gen_name_v2(String domain_str, int ns_id, int pt_id) {
+		String ns_id_str = Int_.Xto_str_pad_bgn_zero(ns_id, 3);
+		String pt_id_str = Int_.Xto_str_pad_bgn_zero(pt_id, 3);
+		return String_.Format("{0}-file-ns.{1}-db.{2}.xowa", domain_str, ns_id_str, pt_id_str);
+	}
+	public static Xob_bin_db_itm new_v1(Fsm_bin_fil fil) {
+		byte[] name = Bry_.new_u8(fil.Url_rel());	// EX: "fsdb.bin.0000.sqlite3"
+		int ns_id = 0; // assume v1 dbs are all in main ns
+		int pt_id = Bry_.Xto_int_or(name, 9 , 13, Int_.MinValue);			if (pt_id == Int_.MinValue) throw Exc_.new_("bin_db_itm.parse: invalid pt_id", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Xto_raw());
+		return new Xob_bin_db_itm(fil.Id(), fil.Url(), ns_id, pt_id);
+	}
+	public static Xob_bin_db_itm new_v2(Fsm_bin_fil fil) {
+		byte[] ns_bgn_tkn = Bry_.new_a7("file-ns."), ns_end_tkn = Bry_.new_a7("-db."), pt_end_tkn = Bry_.new_a7(".xowa");
+		int ns_bgn_tkn_len = ns_bgn_tkn.length, ns_end_tkn_len = ns_end_tkn.length; 
+		byte[] name = Bry_.new_u8(fil.Url_rel());	// EX: en.wikipedia.org-file-ns.000-db.001.xowa
+		int ns_bgn = Bry_finder.Find_fwd(name, ns_bgn_tkn, 0);				if (ns_bgn == Bry_finder.Not_found) throw Exc_.new_("bin_db_itm.parse: invalid ns_bgn", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Xto_raw());
+		ns_bgn += ns_bgn_tkn_len;
+		int ns_end = Bry_finder.Find_fwd(name, ns_end_tkn, ns_bgn);			if (ns_end == Bry_finder.Not_found) throw Exc_.new_("bin_db_itm.parse: invalid ns_end", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Xto_raw());
+		int pt_bgn = ns_end + ns_end_tkn_len;
+		int pt_end = Bry_finder.Find_fwd(name, pt_end_tkn, pt_bgn);			if (pt_end == Bry_finder.Not_found) throw Exc_.new_("bin_db_itm.parse: invalid pt_end", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Xto_raw());
+		int ns_id = Bry_.Xto_int_or(name, ns_bgn, ns_end, Int_.MinValue);	if (ns_id == Int_.MinValue) throw Exc_.new_("bin_db_itm.parse: invalid ns_id", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Xto_raw());
+		int pt_id = Bry_.Xto_int_or(name, pt_bgn, pt_end, Int_.MinValue);	if (pt_id == Int_.MinValue) throw Exc_.new_("bin_db_itm.parse: invalid pt_id", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Xto_raw());
+		return new Xob_bin_db_itm(fil.Id(), fil.Url(), ns_id, pt_id);
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_bin_db_mgr.java

@@ -0,0 +1,71 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.core.primitives.*; import gplx.ios.*;
+import gplx.fsdb.meta.*;
+class Xob_bin_db_mgr {
+	private final int[] ns_ids; private final int ns_ids_len;
+	private final Ordered_hash nth_hash = Ordered_hash_.new_(); private final Int_obj_ref tier_key = Int_obj_ref.neg1_();
+	public Xob_bin_db_mgr(int[] ns_ids) {
+		this.ns_ids = ns_ids; this.ns_ids_len = ns_ids.length;
+	}
+	public boolean Schema_is_1() {return schema_is_1;} private boolean schema_is_1;
+	public void Init_by_mnt_mgr(Fsm_mnt_mgr trg_mnt_mgr) {
+		Fsm_mnt_itm trg_mnt_itm = trg_mnt_mgr.Mnts__get_main(); 
+		this.schema_is_1 = trg_mnt_itm.Db_mgr().File__schema_is_1();
+		Fsm_bin_mgr bin_db_mgr = trg_mnt_itm.Bin_mgr();
+		int len = ns_ids_len;
+		for (int i = 0; i < len; ++i) {	// iterate ns_ids and add default nth
+			int ns_id = ns_ids[i];
+			Xob_bin_db_itm nth = new Xob_bin_db_itm(-1, null, ns_id, 0);
+			nth_hash.Add(Int_obj_ref.new_(ns_ids[i]), nth);
+		}
+		len = bin_db_mgr.Dbs__len();
+		for (int i = 0; i < len; ++i) {	// iterate bin_dbs to find max pt_id for each ns
+			Fsm_bin_fil fil = bin_db_mgr.Dbs__get_at(i);
+			Xob_bin_db_itm itm = schema_is_1 ? Xob_bin_db_itm.new_v1(fil) : Xob_bin_db_itm.new_v2(fil);
+			int ns_id = itm.Ns_id();
+			Xob_bin_db_itm nth = (Xob_bin_db_itm)nth_hash.Get_by(tier_key.Val_(ns_id));
+			if (itm.Pt_id() > nth.Pt_id())	// update max pt_id
+				nth.Set(itm.Id(), itm.Pt_id(), itm.Db_url()); // note that ns_id is same
+		}
+		len = nth_hash.Count();
+		for (int i = 0; i < len; ++i) {	// iterated tiers to calculate max_size
+			Xob_bin_db_itm nth = (Xob_bin_db_itm)nth_hash.Get_at(i);
+			if (nth.Id() == -1) continue;	// ignore default nth
+			IoItmFil nth_itm = Io_mgr.I.QueryFil(nth.Db_url());
+			nth.Db_len_(nth_itm.Size());
+		}
+	}
+	public boolean Tier_id_is_last(int tier_id) {return tier_id >= ns_ids_len;}	// assumes tier_id is 0 based; EX: 0,1,2 for 
+	public int Get_ns_id(int tier_id) {return ns_ids[tier_id];}
+	public int Increment_pt_id(Xob_bin_db_itm itm) {
+		itm.Set(-1, itm.Pt_id() + 1, null);
+		itm.Db_len_(0);
+		return itm.Pt_id();
+	}
+	public String Gen_name(String domain_str, int ns_id, int pt_id) {
+		return schema_is_1 ? Xob_bin_db_itm.Gen_name_v1(pt_id) : Xob_bin_db_itm.Gen_name_v2(domain_str, ns_id, pt_id);
+	}
+	public Xob_bin_db_itm Get_nth_by_tier(int tier_id) {
+		if (schema_is_1) return (Xob_bin_db_itm)nth_hash.Get_by(tier_key.Val_(0)); // v1 is always in ns_0
+		if (tier_id >= ns_ids_len) throw Exc_.new_("tier out of range", "tier_id", tier_id, "len", ns_ids_len);
+		int ns_id = ns_ids[tier_id];
+		return (Xob_bin_db_itm)nth_hash.Get_by(tier_key.Val_(ns_id));
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_diff_regy_exec_cmd.java

@@ -0,0 +1,101 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.fsdb.*; import gplx.fsdb.meta.*;
+import gplx.xowa.bldrs.*;
+public class Xob_diff_regy_exec_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	private Io_url sql_dir;
+	public Xob_diff_regy_exec_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_diff_regy_exec;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run() {Exec_main();}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	private void Exec_main() {
+		if (sql_dir == null)
+			sql_dir = wiki.Ctx().App().Fsys_mgr().File_dir().GenSubDir_nest(wiki.Domain_str(), "tmp_sql");
+		Xob_diff_regy_sql_runner runner = new Xob_diff_regy_sql_runner();
+		Io_url[] urls = Io_mgr.I.QueryDir_fils(sql_dir);
+		int urls_len = urls.length;
+		for (int i = 0; i < urls_len; ++i)
+			runner.Exec(app, urls[i]);
+		Xob_diff_regy_sql_runner.Get_conn(wiki, 0, Fsdb_db_tid_.Tid_atr).Exec_sql("VACUUM;");
+	}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_sql_dir_))		sql_dir = m.ReadIoUrl("v");
+		else	return super.Invk(ctx, ikey, k, m);
+		return this;
+	}	private static final String Invk_sql_dir_ = "sql_dir_";
+}
+class Xob_diff_regy_sql_runner {
+	public Io_url Url() {return url;} private Io_url url;
+	public String Wiki_domain() {return wiki_domain;} private String wiki_domain;
+	public int Fsdb_db_id() {return fsdb_db_id;} private int fsdb_db_id;
+	public byte Fsdb_db_tid() {return fsdb_db_tid;} private byte fsdb_db_tid;
+	public void Exec(Xoae_app app, Io_url url) {
+		Parse_url(url);
+		Run_sql(app);
+	}
+	public void Parse_url(Io_url url) {
+		this.url = url;
+		String[] parts = String_.Split(url.NameOnly(), "-");
+		wiki_domain = parts[0];
+		fsdb_db_id = Int_.parse_(parts[1]);
+		fsdb_db_tid = Fsdb_db_tid_.Xto_tid(parts[2]);
+	}
+	public void Run_sql(Xoae_app app) {
+		Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_null(Bry_.new_u8(wiki_domain));
+		app.Usr_dlg().Prog_many("", "", "running sql: url=~{0}", url.NameAndExt());
+		Db_conn conn = Get_conn(wiki, fsdb_db_id, fsdb_db_tid);
+		conn.Exec_sql(Io_mgr.I.LoadFilStr(url));
+		if (fsdb_db_tid == Fsdb_db_tid_.Tid_bin)
+			conn.Exec_sql("VACUUM;");
+	}
+	public static Db_conn Get_conn(Xowe_wiki wiki, int fsdb_db_id, byte fsdb_db_tid) {
+		wiki.File_mgr().Init_file_mgr_by_load(wiki);
+		Fsm_mnt_itm abc_mgr = wiki.File_mgr().Fsdb_mgr().Mnt_mgr().Mnts__get_main();
+		if		(fsdb_db_tid == Fsdb_db_tid_.Tid_bin)
+			return abc_mgr.Bin_mgr().Dbs__get_at(fsdb_db_id).Conn();
+		else if (fsdb_db_tid == Fsdb_db_tid_.Tid_atr)
+			return abc_mgr.Atr_mgr().Db__core().Conn();
+		else
+			throw Exc_.new_unhandled(fsdb_db_tid);
+	}
+	public static String Build_url(String wiki_domain, int fsdb_db_id, String fsdb_db_type) {
+		return String_.Format("{0}-{1}-{2}.sql", wiki_domain, Int_.Xto_str_pad_bgn_zero(fsdb_db_id, 3), fsdb_db_type);
+	}
+}
+class Fsdb_db_tid_ {
+	public static final byte Tid_cfg = 0, Tid_atr = 1, Tid_bin = 2;
+	public static final String Key_cfg = "cfg", Key_atr = "atr", Key_bin = "bin";
+	public static byte Xto_tid(String s) {
+		if		(String_.Eq(s, Key_cfg))		return Tid_cfg;
+		else if	(String_.Eq(s, Key_atr))		return Tid_atr;
+		else if	(String_.Eq(s, Key_bin))		return Tid_bin;
+		else									throw Exc_.new_unhandled(s);
+	}
+	public static String Xto_key(byte v) {
+		switch (v) {
+			case Tid_cfg:	return Key_cfg;
+			case Tid_atr:	return Key_atr;
+			case Tid_bin:	return Key_bin;
+			default:		throw Exc_.new_unhandled(v);
+		}
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_diff_regy_exec_cmd_tst.java

@@ -0,0 +1,45 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*;
+import gplx.fsdb.*;
+public class Xob_diff_regy_exec_cmd_tst {
+	private Xob_diff_regy_exec_cmd_fxt fxt = new Xob_diff_regy_exec_cmd_fxt();
+	@Test   public void Xto_commons() {
+		fxt.Test_build_url("enwiki", 1, "atr", "enwiki-001-atr.sql");
+		fxt.Test_parse_url("/file/enwiki-001-atr.sql", "enwiki", 1, Fsdb_db_tid_.Tid_atr);
+	}
+//		@Test  public void Smoke() {
+//			Xoae_app app = Xoa_app_fxt.app_(Io_url_.new_dir_("C:\\xowa\\"), "wnt");
+//			Xowe_wiki wiki = Xoa_app_fxt.wiki_tst_(app);
+//			Xob_diff_regy_make_cmd cmd = new Xob_diff_regy_make_cmd(app.Bldr(), wiki);
+//			cmd.Cmd_run();
+//		}
+}
+class Xob_diff_regy_exec_cmd_fxt {
+	public void Test_build_url(String wiki_domain, int fsdb_db_id, String fsdb_db_type, String expd) {
+		Tfds.Eq(expd, Xob_diff_regy_sql_runner.Build_url(wiki_domain, fsdb_db_id, fsdb_db_type));
+	}
+	public void Test_parse_url(String raw, String expd_wiki_domain, int expd_fsdb_db_id, byte expd_fsdb_db_tid) {
+		Xob_diff_regy_sql_runner runner = new Xob_diff_regy_sql_runner();
+		runner.Parse_url(Io_url_.new_any_(raw));
+		Tfds.Eq(expd_wiki_domain, runner.Wiki_domain());
+		Tfds.Eq(expd_fsdb_db_id, runner.Fsdb_db_id());
+		Tfds.Eq(expd_fsdb_db_tid, runner.Fsdb_db_tid());
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_diff_regy_make_cmd.java

@@ -0,0 +1,137 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.fsdb.*; import gplx.dbs.engines.sqlite.*;
+public class Xob_diff_regy_make_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	public Xob_diff_regy_make_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_diff_regy_make;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run() {Exec_main();}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	private void Exec_main() {
+		Db_conn make_db_provider = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
+		this.Make_join_indexes(make_db_provider);
+		this.Make_diff_regy(make_db_provider);
+		this.Make_delete_sql(make_db_provider);
+	}
+	private void Make_join_indexes(Db_conn make_db_provider) {
+		try {
+			Sqlite_engine_.Idx_create(make_db_provider, Xob_diff_regy_tbl.Idx_fsdb_regy__join);
+			Sqlite_engine_.Idx_create(make_db_provider, Xob_diff_regy_tbl.Idx_xfer_regy__join);
+		}
+		catch (Exception exc) {
+			app.Usr_dlg().Warn_many("", "", "error while making indexes: err=~{0}", Err_.Message_gplx(exc));
+		}
+	}
+	private void Make_diff_regy(Db_conn make_db_provider) {
+		Sqlite_engine_.Tbl_create_and_delete(make_db_provider, Xob_diff_regy_tbl.Tbl_name, Xob_diff_regy_tbl.Tbl_sql);
+		make_db_provider.Exec_sql(Xob_diff_regy_tbl.Make_diff_regy);
+		Sqlite_engine_.Idx_create(make_db_provider, Xob_diff_regy_tbl.Idx_diff_regy__load);
+	}
+	private void Make_delete_sql(Db_conn make_db_provider) {
+		DataRdr rdr = make_db_provider.Exec_sql_as_rdr(Xob_diff_regy_tbl.Make_deletes);
+		int cur_db_id = -1, cur_count = 0;
+		Bry_bfr atr_bfr = Bry_bfr.new_(), bin_bfr = Bry_bfr.new_();
+		Io_url sql_tmp_dir = wiki.Ctx().App().Fsys_mgr().File_dir().GenSubDir_nest(wiki.Domain_str(), "tmp_sql");
+		while (rdr.MoveNextPeer()) {
+			byte	diff_is_orig	= rdr.ReadByte("diff_is_orig");
+			int		diff_db_id		= rdr.ReadInt("diff_db_id");
+			int		diff_fil_id		= rdr.ReadInt("diff_fil_id");
+			int		diff_thm_id		= rdr.ReadInt("diff_thm_id");
+			if (cur_db_id != diff_db_id) {
+				Make_delete_sql_file(atr_bfr, bin_bfr, sql_tmp_dir, cur_db_id, cur_count);
+				cur_db_id = diff_db_id;
+				cur_count = 0;
+			}
+			Make_delete_sql_item(atr_bfr, bin_bfr, diff_is_orig, diff_db_id, diff_fil_id, diff_thm_id);
+			++cur_count;
+		}
+	}
+	private void Make_delete_sql_file(Bry_bfr atr_bfr, Bry_bfr bin_bfr, Io_url sql_tmp_dir, int cur_db_id, int cur_count) {
+		Make_delete_sql_file(atr_bfr, sql_tmp_dir, cur_db_id, cur_count, Fsdb_db_tid_.Tid_atr);
+		Make_delete_sql_file(bin_bfr, sql_tmp_dir, cur_db_id, cur_count, Fsdb_db_tid_.Tid_bin);
+		app.Usr_dlg().Note_many("", "", "file.diff:sql generated: db_id=~{0} count=~{1}", Int_.Xto_str_pad_bgn_space(cur_db_id, 3), Int_.Xto_str_pad_bgn_space(cur_count, 7));
+	}
+	private void Make_delete_sql_file(Bry_bfr bfr, Io_url sql_dir, int db_id, int cur_count, byte db_tid) {
+		if (db_id != -1 && cur_count > 0) { // do not write 1st bfr
+			bfr.Add_str_a7("COMMIT;\n");
+			String sql_url_name = String_.Format("{0}-{1}-{2}.sql", wiki.Domain_str(), Int_.Xto_str_pad_bgn_zero(db_id, 3), Fsdb_db_tid_.Xto_key(db_tid));
+			Io_url sql_url = sql_dir.GenSubFil(sql_url_name);
+			Io_mgr.I.SaveFilBfr(sql_url, bfr);
+		}
+		bfr.Clear();	// clear bfr if cur_count == 0
+		bfr.Add_str_a7("BEGIN TRANSACTION;\n");
+	}
+	private void Make_delete_sql_item(Bry_bfr atr_bfr, Bry_bfr bin_bfr, byte diff_is_orig, int diff_db_id, int diff_fil_id, int diff_thm_id) {
+		if (diff_is_orig == Byte_.Zero) {
+			atr_bfr.Add_str_a7("DELETE FROM fsdb_xtn_thm WHERE thm_id = " + Int_.Xto_str(diff_thm_id) + ";\n");
+			bin_bfr.Add_str_a7("DELETE FROM fsdb_bin WHERE bin_owner_id = " + Int_.Xto_str(diff_thm_id) + ";\n");
+		}
+		else {
+			atr_bfr.Add_str_a7("UPDATE fsdb_fil SET fil_bin_db_id = -1 WHERE fil_id = " + Int_.Xto_str(diff_fil_id) + ";\n");
+			bin_bfr.Add_str_a7("DELETE FROM fsdb_bin WHERE bin_owner_id = " + Int_.Xto_str(diff_fil_id) + ";\n");
+		}
+	}
+}
+class Xob_diff_regy_tbl {
+	public static final String Tbl_name = "diff_regy";
+	public static final String Tbl_sql = String_.Concat_lines_nl
+	(  "CREATE TABLE diff_regy"
+	, "( diff_id             integer             NOT NULL PRIMARY KEY"
+	, ", diff_is_orig        tinyint             NOT NULL "
+	, ", diff_status         integer             NOT NULL"
+	, ", diff_db_id          integer             NOT NULL"
+	, ", diff_fil_id         integer             NOT NULL"
+	, ", diff_thm_id         integer             NOT NULL"
+	, ", diff_name           varchar(255)        NOT NULL"
+	, ", diff_repo           tinyint             NOT NULL"
+	, ", diff_w              integer             NOT NULL"
+	, ", diff_time           double              NOT NULL"
+	, ", diff_page           integer             NOT NULL"
+	, ", diff_size           bigint              NOT NULL"
+	, ");"
+	);
+	public static final Db_idx_itm Idx_fsdb_regy__join = Db_idx_itm.sql_("CREATE INDEX fsdb_regy__join ON fsdb_regy (fsdb_name, fsdb_is_orig, fsdb_repo, fsdb_w, fsdb_time, fsdb_page);");
+	public static final Db_idx_itm Idx_xfer_regy__join = Db_idx_itm.sql_("CREATE INDEX xfer_regy__join ON xfer_regy (lnki_ttl , file_is_orig, orig_repo, file_w, lnki_time, lnki_page);");
+	public static final Db_idx_itm Idx_diff_regy__load = Db_idx_itm.sql_("CREATE INDEX diff_regy__load ON diff_regy (diff_status, diff_db_id, diff_is_orig, diff_fil_id, diff_thm_id);");
+	public static final String
+	  Make_diff_regy = String_.Concat_lines_nl
+	( "INSERT INTO diff_regy "
+	, "        (diff_id, diff_is_orig, diff_db_id, diff_fil_id, diff_thm_id, diff_name, diff_repo, diff_w, diff_time, diff_page, diff_size, diff_status)"
+	, "SELECT "
+	, "        f.fsdb_id, f.fsdb_is_orig, f.fsdb_db_id, f.fsdb_fil_id, f.fsdb_thm_id, f.fsdb_name, f.fsdb_repo, f.fsdb_w, f.fsdb_time, f.fsdb_page, f.fsdb_size, CASE WHEN x.lnki_ttl IS NULL THEN 0 ELSE 1 END"
+	, "FROM    fsdb_regy f"
+	, "        LEFT JOIN xfer_regy x "
+	, "            ON  f.fsdb_name     = x.lnki_ttl"
+	, "            AND f.fsdb_is_orig  = x.file_is_orig"
+	, "            AND f.fsdb_repo     = x.orig_repo"
+	, "            AND f.fsdb_w        = x.file_w"
+	, "            AND f.fsdb_time     = x.lnki_time"
+	, "            AND f.fsdb_page     = x.lnki_page"
+	, ";"
+	)
+	, Make_deletes = String_.Concat_lines_nl
+	( "SELECT  diff_db_id, diff_is_orig, diff_fil_id, diff_thm_id"
+	, "FROM    diff_regy"
+	, "WHERE   diff_status = 0"
+	, "ORDER BY diff_db_id, diff_is_orig, diff_fil_id, diff_thm_id"
+	)
+	;
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_fsdb_make_cmd.java

@@ -0,0 +1,378 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.dbs.engines.sqlite.*;
+import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.dbs.*; import gplx.fsdb.*; import gplx.ios.*; import gplx.xowa.wikis.data.tbls.*;
+import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.bins.*; import gplx.xowa.files.fsdb.*;	
+import gplx.fsdb.data.*; import gplx.fsdb.meta.*;
+public class Xob_fsdb_make_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	private Db_conn bldr_conn; private Db_cfg_tbl bldr_cfg_tbl;
+	private Xof_bin_mgr src_bin_mgr; private Xof_bin_wkr__fsdb_sql src_fsdb_wkr; private boolean src_bin_mgr__cache_enabled = Bool_.N; private String src_bin_mgr__fsdb_version; private String[] src_bin_mgr__fsdb_skip_wkrs; private boolean src_bin_mgr__wmf_enabled;
+	private Fsm_mnt_itm trg_mnt_itm; private Fsm_cfg_mgr trg_cfg_mgr; private Fsm_atr_fil trg_atr_fil; private Fsm_bin_fil trg_bin_fil; private long trg_bin_db_max;
+	private final Xof_bin_updater trg_bin_updater = new Xof_bin_updater(); private Xob_bin_db_mgr bin_db_mgr; private int[] ns_ids; private int prv_lnki_tier_id = -1;
+	private long download_size_max = Io_mgr.Len_mb_long * 5; private int[] download_keep_tier_ids = Int_.Ary(0);
+	private Xobu_poll_mgr poll_mgr; private int poll_interval; private long time_bgn;
+	private int select_interval = 2500, progress_interval = 1, commit_interval = 1, delete_interval = 5000;
+	private boolean exec_done, resume_enabled; private int exec_count, exec_count_max = Int_.MaxValue, exec_fail, exec_fail_max = 10000; // 115 errors over 900k images		
+	private int tier_id_bmk = -1, tier_id_val = -1; private int page_id_bmk = -1, page_id_val = -1, page_id_end = Int_.MaxValue; private int lnki_id_bmk = -1, lnki_id_val = -1;
+	private boolean exit_after_commit, exit_now;
+	public Xob_fsdb_make_cmd(Xob_bldr bldr, Xowe_wiki wiki) {
+		this.Cmd_ctor(bldr, wiki);
+		this.poll_mgr = new Xobu_poll_mgr(bldr.App());
+		wiki.File__fsdb_mode().Tid_v2_bld_y_();
+		this.src_bin_mgr = new Xof_bin_mgr(new Fsm_mnt_mgr(), wiki.File__repo_mgr(), app.File__img_mgr().Wkr_resize_img(), app.Wmf_mgr().Download_wkr().Download_xrg().Download_fmt());
+	}
+	public String Cmd_key() {return Xob_cmd_keys.Key_file_fsdb_make;}
+	public void Cmd_bgn(Xob_bldr bldr) {			
+		wiki.Init_assert();
+		this.poll_interval = poll_mgr.Poll_interval();
+		this.bin_db_mgr = new Xob_bin_db_mgr(ns_ids);
+		// src_bin_mgr
+		if (src_bin_mgr__fsdb_version != null) {
+			this.src_fsdb_wkr = Xof_bin_wkr__fsdb_sql.new_(wiki.File__mnt_mgr());
+			src_bin_mgr.Wkrs__add(src_fsdb_wkr);
+			src_fsdb_wkr.Mnt_mgr().Ctor_by_load(new_src_bin_db_mgr(wiki, src_bin_mgr__fsdb_version));
+			src_fsdb_wkr.Mnt_mgr().Mnts__get_main().Txn_bgn();			// NOTE: txn on atr speeds up from 50 -> 300; DATE:2015-03-21				
+			if (src_bin_mgr__fsdb_skip_wkrs != null) {
+				src_fsdb_wkr.Skip_mgr_init(src_fsdb_wkr.Mnt_mgr().Mnts__get_main().Cfg_mgr(), src_bin_mgr__fsdb_skip_wkrs);
+			}
+			if (src_bin_mgr__cache_enabled) {
+				usr_dlg.Prog_many("", "", "src_bin_mgr.cache.bgn");
+				src_fsdb_wkr.Mnt_mgr().Mnts__get_main().Atr_mgr().Db__core().Fil_cache_enabled_y_();
+				usr_dlg.Prog_many("", "", "src_bin_mgr.cache.end");
+			}
+		}
+		if (src_bin_mgr__wmf_enabled) {
+			Xof_bin_wkr__http_wmf wmf_wkr = Xof_bin_wkr__http_wmf.new_(wiki);
+			src_bin_mgr.Wkrs__add(wmf_wkr);
+			wmf_wkr.Fail_timeout_(0);	// 1000; NOTE: set Fail_timeout here; DATE:2014-06-21; NOTE: do not put in ctor, or else will be 1st wkr; DATE:2014-06-28
+		}
+		// trg_mnt_itm
+		this.trg_bin_db_max = app.Api_root().Bldr().Wiki().Import().File_db_max();
+		Fsdb_db_mgr trg_db_mgr = Fsdb_db_mgr_.new_detect(wiki, wiki.Fsys_mgr().Root_dir(), wiki.Fsys_mgr().File_dir());
+		if (trg_db_mgr == null) trg_db_mgr = Fsdb_db_mgr__v2_bldr.I.Get_or_make(wiki, Bool_.Y);
+		Fsm_mnt_mgr trg_mnt_mgr = new Fsm_mnt_mgr(); trg_mnt_mgr.Ctor_by_load(trg_db_mgr);
+		trg_mnt_mgr.Mnts__get_insert_idx_(Fsm_mnt_mgr.Mnt_idx_main);		// NOTE: do not delete; mnt_mgr default to Mnt_idx_user; DATE:2014-04-25
+		this.trg_mnt_itm = trg_mnt_mgr.Mnts__get_insert();
+		Fsm_mnt_mgr.Patch(trg_mnt_itm.Cfg_mgr().Tbl());						// NOTE: always patch again; fsdb_make may be run separately without lnki_temp; DATE:2014-04-26
+		this.trg_atr_fil = trg_mnt_itm.Atr_mgr().Db__core();
+		this.trg_cfg_mgr = trg_mnt_itm.Cfg_mgr();
+		bin_db_mgr.Init_by_mnt_mgr(trg_mnt_mgr);
+		trg_atr_fil.Conn().Txn_bgn();			
+		if (!trg_atr_fil.Conn().Eq(trg_cfg_mgr.Tbl().Conn()))	// need to create txn for v1; DATE:2015-07-04
+			trg_cfg_mgr.Tbl().Conn().Txn_bgn();
+		// bldr_db
+		Xob_db_file bldr_db = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir());
+		this.bldr_conn = bldr_db.Conn();
+		this.bldr_cfg_tbl = bldr_db.Tbl__cfg();	// NOTE: cfg and atr is in same db; use it
+		bldr_cfg_tbl.Conn().Txn_bgn();
+	}
+	public void Cmd_run() {
+		Init_bldr_bmks();
+		this.time_bgn = Env_.TickCount();
+		int total_pending = Xob_xfer_regy_tbl.Select_total_pending(bldr_conn);
+		// if (total_pending > 250000 && src_bin_mgr__fsdb_version == null) 
+		usr_dlg.Note_many("", "", "total pending: ~{0}", total_pending);
+		List_adp list = List_adp_.new_();
+		boolean loop = true;
+		while (loop) {
+			byte rslt = Select_fsdb_itms(list);
+			switch (rslt) {
+				case Select_rv_stop:
+					if (bin_db_mgr.Tier_id_is_last(tier_id_val))
+						loop = false;
+					else {
+						++tier_id_val;
+						page_id_val = 0;
+						continue;
+					}
+					break;
+				case Select_rv_next_page:	++page_id_val; lnki_id_val = 0; continue;
+				case Select_rv_process:		break;
+			}
+			if (!loop) break; // no more ttls found
+			int len = list.Count();
+			usr_dlg.Prog_many("", "", "fetched pages: ~{0}", len);
+			for (int i = 0; i < len; ++i) {
+				Xodb_tbl_oimg_xfer_itm fsdb = (Xodb_tbl_oimg_xfer_itm)list.Get_at(i);
+				Download_itm(fsdb);
+				if (	exit_now
+					||	exec_count	>= exec_count_max
+					||	exec_fail	>= exec_fail_max
+					||	page_id_val >= page_id_end
+					) {
+					this.Txn_sav();
+					return;
+				}
+			}
+		}
+		exec_done = true;
+	}
+	private void Init_bldr_bmks() {
+		if (!resume_enabled) // clear cfg entries if resume disabled; note that disabled by default; DATE:2014-10-24
+			bldr_cfg_tbl.Delete_grp(Cfg_fsdb_make);
+		Db_cfg_hash bmk_hash = bldr_cfg_tbl.Select_as_hash(Cfg_fsdb_make);
+		String tier_id_str = bmk_hash.Get(Cfg_tier_id_bmk).To_str_or(null);
+		if (tier_id_str == null) {	// bmks not found; new db;
+			bldr_conn.Txn_bgn();
+			bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_tier_id_bmk	, tier_id_bmk);
+			bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_page_id_bmk	, page_id_bmk);
+			bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_lnki_id_bmk	, lnki_id_bmk);
+			bldr_conn.Txn_end();
+			if (tier_id_bmk == -1) tier_id_bmk = 0;
+			if (page_id_bmk == -1) page_id_bmk = 0;
+			if (lnki_id_bmk == -1) lnki_id_bmk = 0;
+		}
+		else {
+			if (tier_id_bmk == -1) {
+				tier_id_bmk = Int_.parse_(tier_id_str);
+				usr_dlg.Note_many("", "", "restoring from bmk: tier_id=~{0}", tier_id_bmk);
+			}
+			if (page_id_bmk == -1) {
+				page_id_bmk = bmk_hash.Get(Cfg_page_id_bmk).To_int();
+				usr_dlg.Note_many("", "", "restoring from bmk: page_id=~{0}", page_id_bmk);
+			}
+			if (lnki_id_bmk == -1) {
+				lnki_id_bmk = bmk_hash.Get(Cfg_lnki_id_bmk).To_int();
+				usr_dlg.Note_many("", "", "restoring from bmk: lnki_id=~{0}", lnki_id_bmk);
+			}
+		}
+		tier_id_val = tier_id_bmk;
+		page_id_val = page_id_bmk;
+		lnki_id_val = lnki_id_bmk;
+	}
+	private byte Select_fsdb_itms(List_adp list) {
+		list.Clear();
+		boolean pages_found = false, links_found = false;
+		DataRdr rdr = Xob_xfer_regy_tbl.Select_by_tier_page(bldr_conn, tier_id_val, page_id_val, select_interval);
+		try {
+			while (rdr.MoveNextPeer()) {
+				pages_found = true;								// at least one page found; set true
+				Xodb_tbl_oimg_xfer_itm itm = Xodb_tbl_oimg_xfer_itm.new_rdr_( rdr);
+				if (	itm.Lnki_page_id() == page_id_val		// same page_id 
+					&&	itm.Lnki_id() <= lnki_id_val			// ... but lnki_id < last
+					)
+					continue;									// ... ignore; note that select is by page_id, not page_id + link_id; needed else restarts would not resume exactly at same point;
+				links_found = true;
+				list.Add(itm);
+			}
+		}	finally {rdr.Rls();}
+		if		(pages_found && !links_found)	return Select_rv_next_page;		// pages found, but all links processed
+		else if (!pages_found)					return Select_rv_stop;			// no more pages found
+		else									return Select_rv_process;		// pages and links found
+	}
+	private void Download_itm(Xodb_tbl_oimg_xfer_itm fsdb) {
+		try {
+			tier_id_val = fsdb.Lnki_tier_id();
+			page_id_val = fsdb.Lnki_page_id();
+			lnki_id_val = fsdb.Lnki_id();
+			fsdb.Orig_repo_name_(fsdb.Orig_repo_id() == Xof_repo_itm_.Repo_local ? wiki.Domain_bry() : Xow_domain_.Domain_bry_commons);
+			Download_exec(fsdb);
+			++exec_count;
+			if (exec_count % progress_interval	== 0) Print_progress(fsdb);
+			if (exec_count % poll_interval		== 0) poll_mgr.Poll();
+			if (exec_count % commit_interval	== 0) Txn_sav();
+			if (exec_count % delete_interval	== 0) Delete_files();
+		}
+		catch (Exception exc) {
+			++exec_fail;
+			usr_dlg.Warn_many("", "", "download error; ttl=~{0} w=~{1} err=~{2}", fsdb.Orig_ttl(), fsdb.Lnki_w(), Err_.Message_gplx(exc));
+		}
+	}
+	private void Download_exec(Xodb_tbl_oimg_xfer_itm fsdb) {
+		Io_stream_rdr src_rdr = src_bin_mgr.Find_as_rdr(Xof_exec_tid.Tid_wiki_page, fsdb);
+		try {
+			if (src_rdr == Io_stream_rdr_.Noop) {	// download failed
+				++exec_fail;
+				usr_dlg.Warn_many("", "", "failed: ttl=~{0}", String_.Format("[[File:{0}|{1}px]]", fsdb.Orig_ttl(), fsdb.Html_w()));
+				Print_progress(fsdb);
+			}
+			else {									// download passed
+				long src_rdr_len = src_rdr.Len();
+				int lnki_tier_id = fsdb.Lnki_tier_id();
+				if (	src_rdr_len > download_size_max
+					&&	!Int_.In(lnki_tier_id, download_keep_tier_ids)) {
+					usr_dlg.Warn_many("", "", "skipped; ttl=~{0} w=~{1} size=~{2} tier=~{3}", fsdb.Orig_ttl(), fsdb.Lnki_w(), src_rdr_len, lnki_tier_id);
+					return;
+				}
+				if		(trg_bin_fil == null)									// no trg_bin_fil
+					Make_trg_bin_file(Bool_.Y, fsdb, src_rdr_len);
+				else if (trg_bin_fil.Bin_len() + src_rdr_len > trg_bin_db_max) 	// or trg_bin_fil is out of space
+					Make_trg_bin_file(Bool_.N, fsdb, src_rdr_len);
+				else if (prv_lnki_tier_id != lnki_tier_id) {					// or tier has changed
+					if (	prv_lnki_tier_id != -1
+						&&	!bin_db_mgr.Schema_is_1())		// do not increment dbs for v1
+						Make_trg_bin_file(Bool_.Y, fsdb, src_rdr_len);
+					prv_lnki_tier_id = lnki_tier_id;
+				}
+				trg_bin_updater.Save_bin(trg_mnt_itm, trg_atr_fil, trg_bin_fil, fsdb, src_rdr, src_rdr_len);
+			}
+		}
+		finally {src_rdr.Rls();}
+	}
+	private void Make_trg_bin_file(boolean try_nth, Xodb_tbl_oimg_xfer_itm fsdb, long src_rdr_len) {
+		boolean make = true, txn_bgn = true;
+		int tier_id = fsdb.Lnki_tier_id();
+		Xob_bin_db_itm nth_bin_db = bin_db_mgr.Get_nth_by_tier(tier_id);
+		if (try_nth) {						// try_nth is true; occurs for new runs or changed tier
+			if (	nth_bin_db.Id() != -1	// nth exists; 
+				&&	nth_bin_db.Db_len() + src_rdr_len < trg_bin_db_max) { // if src_rdr_len exceeds
+				make = false;				// do not make; use existing
+				txn_bgn = false;
+			}				
+		}
+		if (make) {							// no nth; make it;
+			if (trg_bin_fil != null) {			// pre-existing bin_file; 
+				trg_bin_fil.Conn().Txn_end();	// close txn before making new db
+				if (trg_mnt_itm.Db_mgr().File__solo_file())
+					txn_bgn = false;
+			}
+			int ns_id = bin_db_mgr.Get_ns_id(tier_id);
+			int pt_id = bin_db_mgr.Increment_pt_id(nth_bin_db);
+			String new_bin_db_name = bin_db_mgr.Gen_name(wiki.Domain_str(), ns_id, pt_id);
+			this.trg_bin_fil = trg_mnt_itm.Bin_mgr().Dbs__make(new_bin_db_name);
+			if (!trg_mnt_itm.Db_mgr().File__solo_file()) {
+				Fsdb_db_file trg_bin_db = trg_mnt_itm.Db_mgr().File__bin_file__at(trg_mnt_itm.Id(), trg_bin_fil.Id(), new_bin_db_name);
+				if (!bin_db_mgr.Schema_is_1())
+					Fsdb_db_mgr__v2_bldr.Make_cfg_data(wiki, trg_atr_fil.Url_rel(), trg_bin_db, Xowd_db_file_.Tid_file_data, trg_bin_fil.Id() + List_adp_.Base1);
+			}
+		}
+		else {	// nth available; use it
+			this.trg_bin_fil = trg_mnt_itm.Bin_mgr().Dbs__get_at(nth_bin_db.Id());
+			trg_bin_fil.Bin_len_(nth_bin_db.Db_len());
+		}
+		if (txn_bgn)
+			trg_bin_fil.Conn().Txn_bgn();
+	}
+	private void Txn_sav() {
+		usr_dlg.Prog_many("", "", "committing data: count=~{0} failed=~{1}", exec_count, exec_fail);
+		bldr_cfg_tbl.Update_int(Cfg_fsdb_make, Cfg_page_id_bmk, page_id_val);
+		bldr_cfg_tbl.Update_int(Cfg_fsdb_make, Cfg_lnki_id_bmk, lnki_id_val);
+		bldr_cfg_tbl.Conn().Txn_sav();
+		trg_cfg_mgr.Next_id_commit();
+		trg_atr_fil.Conn().Txn_sav();
+		if (!trg_atr_fil.Conn().Eq(trg_cfg_mgr.Tbl().Conn()))	// need to create txn for v1
+			trg_cfg_mgr.Tbl().Conn().Txn_sav();
+		if (src_bin_mgr__fsdb_version != null && src_bin_mgr__fsdb_skip_wkrs != null) {
+			src_fsdb_wkr.Skip_mgr().Skip_term(src_fsdb_wkr.Mnt_mgr().Mnts__get_main().Cfg_mgr());
+		}
+		if (!trg_mnt_itm.Db_mgr().File__solo_file())
+			trg_bin_fil.Conn().Txn_sav();
+		if (exit_after_commit) exit_now = true;
+	}
+	public void Cmd_end() {
+		usr_dlg.Note_many("", "", "fsdb_make.done: count=~{0} rate=~{1}", exec_count, DecimalAdp_.divide_safe_(exec_count, Env_.TickCount_elapsed_in_sec(time_bgn)).Xto_str("#,###.000"));
+		if (src_fsdb_wkr != null) {
+			src_fsdb_wkr.Mnt_mgr().Mnts__get_main().Txn_end();	// NOTE: src_fsdb_wkr will be null if no src db defined
+		}
+		trg_atr_fil.Conn().Txn_end(); trg_atr_fil.Conn().Rls_conn();
+		if (!trg_atr_fil.Conn().Eq(trg_cfg_mgr.Tbl().Conn()))	// need to create txn for v1
+			trg_cfg_mgr.Tbl().Conn().Txn_end();
+		trg_cfg_mgr.Tbl().Conn().Rls_conn();
+		if (!trg_mnt_itm.Db_mgr().File__solo_file()) {
+			trg_bin_fil.Conn().Txn_end(); trg_bin_fil.Conn().Rls_conn();
+		}
+		if (exec_done) {
+			bldr_cfg_tbl.Delete_grp(Cfg_fsdb_make);	// delete bmks for future reruns; DATE:2014-08-20
+			Io_mgr.I.DeleteFil_args(wiki.Fsys_mgr().Root_dir().GenSubFil("xowa.file.make.cfg.gfs")).MissingFails_off().Exec();
+		}
+		bldr_conn.Rls_conn();
+	}
+	private void Print_progress(Xodb_tbl_oimg_xfer_itm itm) {
+		int time_elapsed = Env_.TickCount_elapsed_in_sec(time_bgn);
+		usr_dlg.Prog_many("", "", "prog: num=~{0} err=~{1} time=~{2} rate=~{3} page=~{4} lnki=~{5} ttl=~{6}", exec_count, exec_fail, time_elapsed, Math_.Div_safe_as_int(exec_count, time_elapsed), page_id_val, lnki_id_val, itm.Orig_ttl());
+	}
+	private void Delete_files() {}// TODO: purge /xowa/file/ dir to free up hard disk space
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_term() {}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_tier_id_bmk_))					tier_id_bmk = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_page_id_bmk_))					page_id_bmk = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_lnki_id_bmk_))					lnki_id_bmk = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_select_interval_))				select_interval = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_commit_interval_))				commit_interval = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_progress_interval_))				progress_interval = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_delete_interval_))				delete_interval = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_exec_count_max_))				exec_count_max = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_exec_fail_max_))					exec_fail_max = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_exit_after_commit_))				exit_after_commit = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_exit_now_))						exit_now = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_resume_enabled_))				resume_enabled = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_ns_ids_))						ns_ids = Int_.Ary_parse(m.ReadStr("v"), "|");
+		else if	(ctx.Match(k, Invk_src_bin_mgr__fsdb_version_))		src_bin_mgr__fsdb_version = m.ReadStr("v");
+		else if	(ctx.Match(k, Invk_src_bin_mgr__fsdb_skip_wkrs_))	src_bin_mgr__fsdb_skip_wkrs = m.ReadStrAry("v", "|");
+		else if	(ctx.Match(k, Invk_src_bin_mgr__wmf_enabled_))		src_bin_mgr__wmf_enabled = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_src_bin_mgr__cache_enabled_))	src_bin_mgr__cache_enabled = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_poll_mgr))						return poll_mgr;
+		else if	(ctx.Match(k, Invk_download_keep_tier_ids))			download_keep_tier_ids = Int_.Ary_parse(m.ReadStr("v"), "|");
+		else if	(ctx.Match(k, Invk_download_size_max))				download_size_max = Io_size_.To_long_by_msg_mb(m, download_size_max);
+		else	return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}
+	private static final String
+	  Invk_tier_id_bmk_ = "tier_id_bmk_", Invk_page_id_bmk_ = "page_id_bmk_", Invk_lnki_id_bmk_ = "lnki_id_bmk_"
+	, Invk_select_interval_ = "select_interval_", Invk_commit_interval_ = "commit_interval_", Invk_progress_interval_ = "progress_interval_", Invk_delete_interval_ = "delete_interval_"
+	, Invk_exec_count_max_ = "exec_count_max_", Invk_exec_fail_max_ = "exec_fail_max_", Invk_exit_now_ = "exit_now_", Invk_exit_after_commit_ = "exit_after_commit_"
+	, Invk_resume_enabled_ = "resume_enabled_", Invk_poll_mgr = "poll_mgr"
+	, Invk_src_bin_mgr__fsdb_version_ = "src_bin_mgr__fsdb_version_", Invk_src_bin_mgr__fsdb_skip_wkrs_ = "src_bin_mgr__fsdb_skip_wkrs_"
+	, Invk_src_bin_mgr__wmf_enabled_ = "src_bin_mgr__wmf_enabled_"
+	, Invk_src_bin_mgr__cache_enabled_ = "src_bin_mgr__cache_enabled_", Invk_ns_ids_ = "ns_ids_"
+	, Invk_download_size_max = "download_size_max", Invk_download_keep_tier_ids = "download_keep_tier_ids"
+	;
+	public static Fsdb_db_mgr new_src_bin_db_mgr(Xow_wiki wiki, String version) {
+		String domain_str = wiki.Domain_str();
+		Fsdb_db_mgr rv = null; Io_url url = null;
+		if		(String_.Eq(version, "v1")) {
+			url = wiki.Fsys_mgr().File_dir().OwnerDir().GenSubDir(domain_str + "-prv");	// v1: EX: /xowa/file/en.wikipedia.org-prv/
+			rv = new Fsdb_db_mgr__v1(url);
+		}
+		else if (String_.Eq(version, "v2")) {
+			url = wiki.Fsys_mgr().Root_dir().GenSubDir("prv");		// v2: EX: /xowa/wiki/en.wikipedia.org/prv/
+			rv = Fsdb_db_mgr_.new_detect(wiki, url, url);			// note that v2 is prioritized over v1
+		}
+		else throw Exc_.new_("fsdb.make:unknown fsdb_type", "version", version);
+		if (rv == null) throw Exc_.new_("fsdb.make:source fsdb not found", "version", version, "url", url.Raw());
+		return rv;
+	}
+	private static final byte Select_rv_stop = 0, Select_rv_process = 1, Select_rv_next_page = 2;
+	private static final String Cfg_fsdb_make = "bldr.fsdb_make", Cfg_tier_id_bmk = "tier_id_bmk", Cfg_page_id_bmk = "page_id_bmk", Cfg_lnki_id_bmk = "lnki_id_bmk";
+	public static byte Status_null = 0, Status_pass = 1, Status_fail = 2;
+}
+class Xodb_tbl_oimg_xfer_itm extends Xof_fsdb_itm {	public int 			Lnki_id()		{return lnki_id;} private int lnki_id;
+	public int			Lnki_tier_id()	{return lnki_tier_id;} private int lnki_tier_id;
+	public int 			Lnki_page_id()	{return lnki_page_id;} private int lnki_page_id;
+	public static Xodb_tbl_oimg_xfer_itm new_rdr_(DataRdr rdr) {
+		Xodb_tbl_oimg_xfer_itm rv = new Xodb_tbl_oimg_xfer_itm();
+		rv.lnki_id			= rdr.ReadInt(Xob_xfer_regy_tbl.Fld_lnki_id);
+		rv.lnki_page_id		= rdr.ReadInt(Xob_xfer_regy_tbl.Fld_lnki_page_id);
+		rv.lnki_tier_id		= rdr.ReadInt(Xob_xfer_regy_tbl.Fld_lnki_tier_id);
+		rv.Init_at_fsdb_make
+		( rdr.ReadBryByStr(Xob_xfer_regy_tbl.Fld_lnki_ttl)
+		, rdr.ReadInt(Xob_xfer_regy_tbl.Fld_lnki_ext)
+		, rdr.ReadInt(Xob_xfer_regy_tbl.Fld_file_w), rdr.ReadInt(Xob_xfer_regy_tbl.Fld_file_h)	// set lnki_size; Xof_bin_mgr uses lnki_size
+		, Xof_lnki_time.Db_load_double(rdr, Xob_xfer_regy_tbl.Fld_lnki_time)
+		, Xof_lnki_page.Db_load_int(rdr, Xob_xfer_regy_tbl.Fld_lnki_page)
+		, rdr.ReadByte(Xob_xfer_regy_tbl.Fld_orig_repo)
+		, rdr.ReadInt(Xob_xfer_regy_tbl.Fld_orig_w)
+		, rdr.ReadInt(Xob_xfer_regy_tbl.Fld_orig_h)
+		, Bry_.Empty
+		, rdr.ReadByte(Xob_xfer_regy_tbl.Fld_file_is_orig) == Bool_.Y_byte
+		);
+		return rv;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_fsdb_reduce_cmd.java

@@ -0,0 +1,62 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.fsdb.meta.*;
+public class Xob_fsdb_reduce_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	public Xob_fsdb_reduce_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_file_fsdb_reduce;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run() {Exec_main();}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	private void Exec_main() {
+		/*
+		Open_bin_dir
+		Iterate_files
+			Convert_bin
+			Mark_meta_record				
+		*/
+	}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		return this;
+	}
+}
+class Fsdb_reduce_wkr {
+	private final ProcessAdp convert_process;
+	private final Io_url src_url, trg_url;
+	private final int dpi, quality;
+	public Fsdb_reduce_wkr(ProcessAdp convert_process, Io_url tmp_dir, int dpi, int quality) {
+		this.convert_process = convert_process;
+		this.src_url = tmp_dir.GenSubFil("fsdb_reduce.src.bin");
+		this.trg_url = tmp_dir.GenSubFil("fsdb_reduce.trg.bin");
+		this.dpi = dpi;
+		this.quality = quality;
+	}
+	public byte[] Reduce(byte[] orig_bry) {
+		Io_mgr.I.SaveFilBry(src_url, orig_bry);
+		convert_process.Run(src_url, trg_url, dpi, quality); // -strip -quality 50% -density 72 -resample 72
+		if (!convert_process.Exit_code_pass()) {
+			// throw err with convert_process.Rslt_out();
+		}
+		byte[] rv = Io_mgr.I.LoadFilBry(trg_url);
+		// fail if 0; fail if greater than;
+		// warn if not between 50% - 70% of size
+		return rv;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_regy_cmd.java

@@ -0,0 +1,32 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.xowa.dbs.*;
+public class Xob_lnki_regy_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	public Xob_lnki_regy_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_file_lnki_regy;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {
+		Db_conn conn = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
+		Xob_lnki_regy_tbl.Create_table(conn);
+		Xob_lnki_regy_tbl.Create_data(usr_dlg, conn, Xob_lnki_temp_wkr.Ns_file_is_case_match_all(wiki));
+	}
+	public void Cmd_run() {}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_regy_tbl.java

@@ -0,0 +1,115 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.dbs.engines.sqlite.*;
+class Xob_lnki_regy_tbl {
+	public static void Create_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
+	public static void Create_data(Gfo_usr_dlg usr_dlg, Db_conn p, boolean wiki_ns_for_file_is_case_match_all) {
+		p.Exec_sql(Sql_create_data);
+		Sqlite_engine_.Idx_create(usr_dlg, p, "lnki_regy", Idx_ttl);
+		if (wiki_ns_for_file_is_case_match_all)
+			Sqlite_engine_.Idx_create(usr_dlg, p, "lnki_regy_commons", Idx_ttl_commons);
+	}
+	public static final String Tbl_name = "lnki_regy"
+	, Fld_lnki_id = "lnki_id", Fld_lnki_tier_id = "lnki_tier_id", Fld_lnki_page_id = "lnki_page_id", Fld_lnki_page_ns = "lnki_page_ns"
+	, Fld_lnki_ttl = "lnki_ttl", Fld_lnki_commons_ttl = "lnki_commons_ttl"
+	, Fld_lnki_ext = "lnki_ext", Fld_lnki_type = "lnki_type", Fld_lnki_src_tid = "lnki_src_tid"
+	, Fld_lnki_w = "lnki_w", Fld_lnki_h = "lnki_h", Fld_lnki_upright = "lnki_upright", Fld_lnki_time = "lnki_time", Fld_lnki_page = "lnki_page"
+	, Fld_lnki_count = "lnki_count"
+	;
+	private static final String Tbl_sql = String_.Concat_lines_nl
+	(	"CREATE TABLE IF NOT EXISTS lnki_regy"
+	,	"( lnki_id                 integer             NOT NULL			    PRIMARY KEY"
+	,	", lnki_tier_id            integer             NOT NULL"
+	,	", lnki_page_id            integer             NOT NULL"
+	,	", lnki_ttl                varchar(255)        NOT NULL"
+	,	", lnki_commons_ttl        varchar(255)        NULL"
+	,	", lnki_commons_flag       integer             NULL"
+	,	", lnki_ext                integer             NOT NULL"
+	,	", lnki_type               integer             NOT NULL"
+	,	", lnki_src_tid            integer             NOT NULL"
+	,	", lnki_w                  integer             NOT NULL"
+	,	", lnki_h                  integer             NOT NULL"
+	,	", lnki_upright            double              NOT NULL"
+	,	", lnki_time               double              NOT NULL"
+	,	", lnki_page               integer             NOT NULL"
+	,	", lnki_count              integer             NOT NULL"
+	,	");"
+	);
+	public static final String Sql_create_data = String_.Concat_lines_nl
+	(	"INSERT INTO lnki_regy (lnki_id, lnki_tier_id, lnki_page_id, lnki_ttl, lnki_commons_ttl, lnki_ext, lnki_type, lnki_src_tid, lnki_w, lnki_h, lnki_upright, lnki_time, lnki_page, lnki_count)"
+	,	"SELECT  Min(lnki_id)"
+	,	",       Min(lnki_tier_id)"
+	,	",       Min(lnki_page_id)"
+	,	",       lnki_ttl"
+	,	",       lnki_commons_ttl"
+	,	",       lnki_ext"
+	,	",       lnki_type"
+	,	",       lnki_src_tid"
+	,	",       lnki_w"
+	,	",       lnki_h"
+	,	",       lnki_upright"
+	,	",       lnki_time"
+	,	",       lnki_page"
+	,	",       Count(lnki_ttl)"
+	,	"FROM    lnki_temp"
+	,	"GROUP BY"
+	,	"        lnki_ttl"
+	,	",       lnki_commons_ttl"
+	,	",       lnki_ext"
+	,	",       lnki_type"
+	,	",       lnki_src_tid"
+	,	",       lnki_w"
+	,	",       lnki_h"
+	,	",       lnki_upright"
+	,	",       lnki_time"
+	,	",       lnki_page"
+	,	";"
+	)
+	,	Sql_cs_mark_changed = String_.Concat_lines_nl
+	(	"REPLACE INTO lnki_regy"
+	,	"SELECT  l.lnki_id"
+	,	",       l.lnki_tier_id"
+	,	",       l.lnki_page_id"
+	,	",       l.lnki_ttl"
+	,	",       l.lnki_commons_ttl"
+	,	",       CASE WHEN o.lnki_ttl IS NULL THEN NULL ELSE 1 END"
+	,	",       l.lnki_ext"
+	,	",       l.lnki_type"
+	,	",       l.lnki_src_tid"
+	,	",       l.lnki_w"
+	,	",       l.lnki_h"
+	,	",       l.lnki_upright"
+	,	",       l.lnki_time"
+	,	",       l.lnki_page"
+	,	",       l.lnki_count"
+	,	"FROM    lnki_regy l"
+	,	"        LEFT JOIN orig_regy o ON o.lnki_ttl = l.lnki_commons_ttl AND o.orig_commons_flag = 2"
+	,	";"
+	)
+	,	Sql_cs_update_ttls = String_.Concat_lines_nl
+	(	"UPDATE  lnki_regy"
+	,	"SET     lnki_ttl = lnki_commons_ttl"
+	,	"WHERE   lnki_commons_flag = 1"
+	,	";"
+	);
+	private static final Db_idx_itm 
+	  Idx_ttl			= Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS lnki_regy__ttl ON lnki_regy (lnki_ttl, lnki_ext, lnki_id, lnki_page_id);")
+	, Idx_ttl_commons   = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS lnki_regy__ttl ON lnki_regy (lnki_commons_ttl, lnki_ext, lnki_id, lnki_page_id);")
+	;
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_src_tid.java

@@ -0,0 +1,21 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+public class Xob_lnki_src_tid {
+	public static final byte Tid_file = 0, Tid_media = 1, Tid_gallery = 2, Tid_imageMap = 3;
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_temp_tbl.java

@@ -0,0 +1,62 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.xowa.files.*;
+class Xob_lnki_temp_tbl {
+	private static final Db_meta_fld_list flds = Db_meta_fld_list.new_();
+	private static final String Tbl_name = "lnki_temp";
+	public static final String 
+	  Fld_lnki_id				= flds.Add_int_pkey_autonum("lnki_id");	// NOTE: insertion order index; public b/c not used and want to bypass warning
+	private static final String 
+	  Fld_lnki_tier_id			= flds.Add_int("lnki_tier_id")
+	, Fld_lnki_page_id			= flds.Add_int("lnki_page_id")
+	, Fld_lnki_ttl				= flds.Add_str("lnki_ttl", 255)
+	, Fld_lnki_commons_ttl		= flds.Add_str_null("lnki_commons_ttl", 255)
+	, Fld_lnki_ext				= flds.Add_int("lnki_ext")
+	, Fld_lnki_type				= flds.Add_int("lnki_type")
+	, Fld_lnki_src_tid			= flds.Add_int("lnki_src_tid")
+	, Fld_lnki_w				= flds.Add_int("lnki_w")
+	, Fld_lnki_h				= flds.Add_int("lnki_h")
+	, Fld_lnki_upright			= flds.Add_double("lnki_upright")
+	, Fld_lnki_time				= flds.Add_double("lnki_time")	// NOTE: thumbtime is float; using double b/c upright is double and would like to keep datatypes same; https://bugzilla.wikimedia.org/show_bug.cgi?id=39014
+	, Fld_lnki_page				= flds.Add_int("lnki_page")
+	;
+	private Db_stmt stmt_insert;
+	public Xob_lnki_temp_tbl(Db_conn conn) {this.conn = conn;}
+	public Db_conn Conn()		{return conn;} private final Db_conn conn;
+	public void Create_tbl()	{conn.Ddl_create_tbl(Db_meta_tbl.new_(Tbl_name, flds));}
+	public void Insert_bgn()	{conn.Txn_bgn(); stmt_insert = conn.Stmt_insert(Tbl_name, flds);}
+	public void Insert_commit()	{conn.Txn_sav();}
+	public void Insert_end()	{conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
+	public void Insert_cmd_by_batch(int tier_id, int page_id, byte[] ttl, byte[] ttl_commons, byte ext_id, byte img_type, byte lnki_src_tid, int w, int h, double upright, double time, int page) {
+		stmt_insert.Clear()
+		.Val_int		(Fld_lnki_tier_id		, tier_id)
+		.Val_int		(Fld_lnki_page_id		, page_id)
+		.Val_bry_as_str	(Fld_lnki_ttl			, ttl)
+		.Val_bry_as_str	(Fld_lnki_commons_ttl	, ttl_commons)
+		.Val_byte		(Fld_lnki_ext			, ext_id)
+		.Val_byte		(Fld_lnki_type			, img_type)
+		.Val_int		(Fld_lnki_src_tid		, lnki_src_tid)
+		.Val_int		(Fld_lnki_w				, w)
+		.Val_int		(Fld_lnki_h				, h)
+		.Val_double		(Fld_lnki_upright		, upright)
+		.Val_double		(Fld_lnki_time			, Xof_lnki_time.Db_save_double(time))
+		.Val_int		(Fld_lnki_page			, page)
+		.Exec_insert();
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_temp_wkr.java

@@ -0,0 +1,218 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.pages.*;
+import gplx.xowa.files.*;
+import gplx.xowa.wikis.*;
+import gplx.xowa.parsers.lnkis.redlinks.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.html.hdumps.bldrs.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.wdatas.*;
+import gplx.fsdb.meta.*; import gplx.xowa.files.fsdb.*; import gplx.fsdb.*;
+public class Xob_lnki_temp_wkr extends Xob_dump_mgr_base implements Xopg_redlink_logger {
+	private Xob_lnki_temp_tbl tbl; private boolean wdata_enabled = true, xtn_ref_enabled = true, gen_html, gen_hdump;
+	private Xop_log_invoke_wkr invoke_wkr; private Xop_log_property_wkr property_wkr;		
+	private boolean ns_file_is_case_match_all = true; private Xowe_wiki commons_wiki;
+	private Xob_hdump_bldr hdump_bldr; private Xob_link_dump_cmd link_dump_cmd; private boolean hzip_enabled = true;
+	public Xob_lnki_temp_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	@Override public String Cmd_key() {return Xob_cmd_keys.Key_file_lnki_temp;}
+	@Override public byte Init_redirect()	{return Bool_.N_byte;}	// lnki_temp does not look at redirect pages
+	@Override public int[] Init_ns_ary()		{return ns_ids;} private int[] ns_ids = Int_.Ary(Xow_ns_.Id_main);
+	@Override protected void Init_reset(Db_conn conn) {
+		Db_cfg_tbl cfg_tbl = new Db_cfg_tbl(conn, "xowa_cfg");
+		cfg_tbl.Delete_all();
+		invoke_wkr.Init_reset();
+		property_wkr.Init_reset();
+	}
+	@Override protected Db_conn Init_db_file() {
+		ctx.Lnki().File_wkr_(this);
+		Xob_db_file make_db_file = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir());
+		Db_conn conn = make_db_file.Conn();
+		this.tbl = new Xob_lnki_temp_tbl(conn);
+		tbl.Create_tbl();
+		return conn;
+	}
+	@Override protected void Cmd_bgn_end() {
+		ns_file_is_case_match_all = Ns_file_is_case_match_all(wiki);							// NOTE: must call after wiki.init
+		wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode.Tid_read).Ctgs_enabled_(false);		// disable categories else progress messages written (also for PERF)
+		if (wiki.File__bin_mgr() != null)
+			wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf);	// remove wmf wkr, else will try to download images during parsing
+		commons_wiki = app.Wiki_mgr().Get_by_key_or_make(Xow_domain_.Domain_bry_commons);
+		Xop_log_mgr log_mgr = ctx.App().Log_mgr();
+		log_mgr.Log_dir_(wiki.Fsys_mgr().Root_dir());	// put log in wiki dir, instead of user.temp
+		invoke_wkr = this.Invoke_wkr();					// set member reference
+		invoke_wkr = log_mgr.Make_wkr_invoke();
+		property_wkr = this.Property_wkr();				// set member reference
+		property_wkr = log_mgr.Make_wkr_property();
+		wiki.Appe().Wiki_mgr().Wdata_mgr().Enabled_(wdata_enabled);
+		if (!xtn_ref_enabled) gplx.xowa.xtns.cite.References_nde.Enabled = false;
+		gplx.xowa.xtns.gallery.Gallery_xnde.Log_wkr = log_mgr.Make_wkr().Save_src_str_(Bool_.Y);
+		gplx.xowa.xtns.imaps.Imap_xnde.Log_wkr = log_mgr.Make_wkr();
+		gplx.xowa.Xop_xnde_wkr.Timeline_log_wkr = log_mgr.Make_wkr();
+		gplx.xowa.xtns.scores.Score_xnde.Log_wkr = log_mgr.Make_wkr();
+		gplx.xowa.xtns.hieros.Hiero_xnde.Log_wkr = log_mgr.Make_wkr();
+		Xof_fsdb_mgr__sql trg_fsdb_mgr = new Xof_fsdb_mgr__sql();
+		wiki.File__fsdb_mode().Tid_v2_bld_y_();
+		Fsdb_db_mgr__v2 fsdb_core = Fsdb_db_mgr__v2_bldr.I.Get_or_make(wiki, Bool_.Y);
+		trg_fsdb_mgr.Init_by_wiki(wiki);
+		Fsm_mnt_mgr trg_mnt_mgr = trg_fsdb_mgr.Mnt_mgr();
+		wiki.File_mgr().Init_file_mgr_by_load(wiki);										// must happen after fsdb.make
+		wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf);	// must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
+		wiki.File__orig_mgr().Wkrs_del(gplx.xowa.files.origs.Xof_orig_wkr_.Tid_wmf_api);
+		trg_mnt_mgr = new Fsm_mnt_mgr(); trg_mnt_mgr.Ctor_by_load(fsdb_core);
+		trg_mnt_mgr.Mnts__get_insert_idx_(Fsm_mnt_mgr.Mnt_idx_main);
+		Fsm_mnt_mgr.Patch(trg_mnt_mgr.Mnts__get_main().Cfg_mgr().Tbl()); // NOTE: see fsdb_make; DATE:2014-04-26
+		if (gen_hdump) {
+			gplx.xowa.apis.xowa.bldrs.imports.Xoapi_import import_cfg = wiki.Appe().Api_root().Bldr().Wiki().Import();
+			hdump_bldr = new Xob_hdump_bldr(wiki.Ns_mgr(), wiki.Db_mgr_as_sql(), tbl.Conn(), import_cfg.Html_db_max(), hzip_enabled);
+			link_dump_cmd = new Xob_link_dump_cmd();
+			link_dump_cmd.Init_by_wiki(wiki);
+		}
+		tbl.Insert_bgn();
+		log_mgr.Txn_bgn();
+	}
+	@Override public void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm db_page, byte[] page_src) {
+		Xoa_ttl ttl = Xoa_ttl.parse_(wiki, ns.Gen_ttl(db_page.Ttl_page_db()));
+		byte[] ttl_bry = ttl.Page_db();
+		byte page_tid = Xow_page_tid.Identify(wiki.Domain_tid(), ns.Id(), ttl_bry);
+		if (page_tid != Xow_page_tid.Tid_wikitext) return; // ignore js, css, lua, json
+		Xoae_page page = ctx.Cur_page();
+		page.Clear();
+		page.Bldr__ns_ord_(ns_ord);
+		page.Ttl_(ttl).Revision_data().Id_(db_page.Id());
+		page.Redlink_lnki_list().Clear();
+		if (ns.Id_tmpl())
+			parser.Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), wiki.Ns_mgr().Ns_template(), ttl_bry, page_src);
+		else {
+			parser.Parse_page_all_clear(root, ctx, ctx.Tkn_mkr(), page_src);
+			if (gen_html) {
+				page.Root_(root);
+				if (!page.Redirected())
+					wiki.Html_mgr().Page_wtr_mgr().Gen(ctx.Cur_page(), Xopg_view_mode.Tid_read);
+			}
+			if (gen_hdump) {
+				page.Root_(root);
+				hdump_bldr.Insert_page(page);
+				link_dump_cmd.Page_bgn(page.Revision_data().Id());
+				List_adp lnki_list = page.Redlink_lnki_list().Lnki_list();
+				int len = lnki_list.Count();
+				for (int i = 0; i < len; ++i) {
+					Xop_lnki_tkn lnki = (Xop_lnki_tkn)lnki_list.Get_at(i);
+					Xoa_ttl trg_ttl = lnki.Ttl();
+					link_dump_cmd.Add(lnki.Html_uid(), trg_ttl.Ns().Id(), trg_ttl.Page_db());
+				}
+			}
+			root.Clear();
+		}
+	}
+	@Override public void Exec_commit_hook() {
+		tbl.Conn().Txn_sav();
+		if (gen_hdump) {
+			hdump_bldr.Commit();
+			link_dump_cmd.Wkr_commit();
+		}
+	}
+	@Override public void Exec_end_hook() {
+		if (gen_hdump) {
+			hdump_bldr.Bld_term();
+			link_dump_cmd.Wkr_end();
+		}
+		String err_filter_mgr = invoke_wkr.Err_filter_mgr().Print();
+		if (String_.Len_gt_0(err_filter_mgr)) usr_dlg.Warn_many("", "", err_filter_mgr);
+		wiki.Appe().Log_mgr().Txn_end();
+		tbl.Insert_end();
+	}
+	public void Wkr_exec(Xop_ctx ctx, byte[] src, Xop_lnki_tkn lnki, byte lnki_src_tid) {
+		if (lnki.Ttl().ForceLiteralLink()) return; // ignore literal links which creat a link to file, but do not show the image; EX: [[:File:A.png|thumb|120px]] creates a link to File:A.png, regardless of other display-oriented args
+		byte[] ttl = lnki.Ttl().Page_db();
+		Xof_ext ext = Xof_ext_.new_by_ttl_(ttl);
+		double lnki_time = lnki.Time();
+		int lnki_page = lnki.Page();
+		byte[] ttl_commons = Xto_commons(ns_file_is_case_match_all, commons_wiki, ttl);
+		if (	Xof_lnki_page.Null_n(lnki_page) 				// page set
+			&&	Xof_lnki_time.Null_n(lnki_time))			// thumbtime set
+				usr_dlg.Warn_many("", "", "page and thumbtime both set; this may be an issue with fsdb: page=~{0} ttl=~{1}", ctx.Cur_page().Ttl().Page_db_as_str(), String_.new_u8(ttl));
+		if (lnki.Ns_id() == Xow_ns_.Id_media)
+			lnki_src_tid = Xob_lnki_src_tid.Tid_media;
+		tbl.Insert_cmd_by_batch(ctx.Cur_page().Bldr__ns_ord(), ctx.Cur_page().Revision_data().Id(), ttl, ttl_commons, Byte_.By_int(ext.Id()), lnki.Lnki_type(), lnki_src_tid, lnki.W(), lnki.H(), lnki.Upright(), lnki_time, lnki_page);
+	}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_wdata_enabled_))				wdata_enabled = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_xtn_ref_enabled_))			xtn_ref_enabled = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_gen_html_))					gen_html = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_gen_hdump_))					gen_hdump = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_hzip_enabled_))				hzip_enabled = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_ns_ids_))					ns_ids = Int_.Ary_parse(m.ReadStr("v"), "|");
+		else if	(ctx.Match(k, Invk_ns_ids_by_aliases))			ns_ids = Xob_lnki_temp_wkr_.Ns_ids_by_aliases(wiki, m.ReadStrAry("v", "|"));
+		else if	(ctx.Match(k, Invk_property_wkr))				return this.Property_wkr();
+		else if	(ctx.Match(k, Invk_invoke_wkr))					return this.Invoke_wkr();
+		else	return super.Invk(ctx, ikey, k, m);
+		return this;
+	}
+	private static final String Invk_wdata_enabled_ = "wdata_enabled_", Invk_xtn_ref_enabled_ = "xtn_ref_enabled_"
+	, Invk_ns_ids_ = "ns_ids_", Invk_ns_ids_by_aliases = "ns_ids_by_aliases"
+	, Invk_invoke_wkr = "invoke_wkr", Invk_property_wkr = "property_wkr"
+	, Invk_gen_html_ = "gen_html_", Invk_gen_hdump_ = "gen_hdump_", Invk_hzip_enabled_ = "hzip_enabled_"
+	;
+	private Xop_log_invoke_wkr Invoke_wkr() {
+		if (invoke_wkr == null) invoke_wkr = ((Scrib_xtn_mgr)bldr.App().Xtn_mgr().Get_or_fail(Scrib_xtn_mgr.XTN_KEY)).Invoke_wkr_or_new();
+		return invoke_wkr;
+	}
+	private Xop_log_property_wkr Property_wkr() {
+		if (property_wkr == null) property_wkr = bldr.App().Wiki_mgr().Wdata_mgr().Property_wkr_or_new();
+		return property_wkr;
+	}
+	public static byte[] Xto_commons(boolean ns_file_is_case_match_all, Xowe_wiki commons_wiki, byte[] ttl_bry) {
+		if (!ns_file_is_case_match_all) return null;	// return "" if wiki matches common
+		Xoa_ttl ttl = Xoa_ttl.parse_(commons_wiki, Xow_ns_.Id_file, ttl_bry);
+		byte[] rv = ttl.Page_db();
+		return Bry_.Eq(rv, ttl_bry) ? null : rv;
+	}
+	public static boolean Ns_file_is_case_match_all(Xow_wiki wiki) {return wiki.Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Id_all;}
+}
+class Xob_lnki_temp_wkr_ {
+	public static int[] Ns_ids_by_aliases(Xowe_wiki wiki, String[] aliases) {
+		int[] rv = Xob_lnki_temp_wkr_.Ids_by_aliases(wiki.Ns_mgr(), aliases);
+		int aliases_len = aliases.length;
+		int ids_len = rv.length;
+		for (int i = 0; i < aliases_len; i++) {
+			String alias = aliases[i];
+			int id = i < ids_len ? rv[i] : -1;
+			wiki.Appe().Usr_dlg().Note_many("", "", "ns: ~{0} <- ~{1}", Int_.Xto_str_fmt(id, "0000"), alias);
+		}
+		if (aliases_len != ids_len) throw Exc_.new_("mismatch in aliases and ids", "aliases", aliases_len, "ids", ids_len);
+		return rv;
+	}
+	private static int[] Ids_by_aliases(Xow_ns_mgr ns_mgr, String[] aliases) {
+		List_adp list = List_adp_.new_();
+		int len = aliases.length;
+		for (int i = 0; i < len; i++) {
+			String alias = aliases[i];
+			if (String_.Eq(alias, Xow_ns_.Key_main))
+				list.Add(ns_mgr.Ns_main());
+			else {
+				Xow_ns ns = ns_mgr.Names_get_or_null(Bry_.new_u8(alias));
+				if (ns != null)
+					list.Add(ns);
+			}
+		}
+		len = list.Count();
+		int[] rv = new int[len];
+		for (int i = 0; i < len; i++) {
+			rv[i] = ((Xow_ns)list.Get_at(i)).Id();
+		}
+		return rv;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_temp_wkr_tst.java

@@ -0,0 +1,45 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*;
+public class Xob_lnki_temp_wkr_tst {
+	private Xob_lnki_temp_wkr_fxt fxt = new Xob_lnki_temp_wkr_fxt();
+	@Test   public void Xto_commons() {
+		fxt.Init_Xto_commons(true);
+		fxt.Test_Xto_commons("a", "A");
+		fxt.Test_Xto_commons("A", null);
+
+		fxt.Init_Xto_commons(false);
+		fxt.Test_Xto_commons("a", null);
+		fxt.Test_Xto_commons("A", null);
+	}
+}
+class Xob_lnki_temp_wkr_fxt {
+	private boolean wiki_ns_file_is_case_match_all;
+	private Xowe_wiki commons_wiki;
+	public Xob_lnki_temp_wkr_fxt Init_Xto_commons(boolean wiki_ns_file_is_case_match_all) {
+		Xoae_app app = Xoa_app_fxt.app_();
+		this.wiki_ns_file_is_case_match_all = wiki_ns_file_is_case_match_all;
+		this.commons_wiki = Xoa_app_fxt.wiki_tst_(app);	// commons_wiki will default to Xow_ns.Id_file of case_match_1st
+		return this;
+	}
+
+	public void Test_Xto_commons(String ttl, String expd) {
+		Tfds.Eq(expd, String_.new_u8(Xob_lnki_temp_wkr.Xto_commons(wiki_ns_file_is_case_match_all, commons_wiki, Bry_.new_u8(ttl))));
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_orig_regy_cmd.java

@@ -0,0 +1,46 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.xowa.wikis.*;
+public class Xob_orig_regy_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	private boolean repo_0_is_remote = false;
+	public Xob_orig_regy_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_file_orig_regy;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {
+		Db_conn conn = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
+		Xob_orig_regy_tbl.Create_table(conn);
+		Xowe_wiki commons_wiki = bldr.App().Wiki_mgr().Get_by_key_or_make(Xow_domain_.Domain_bry_commons).Init_assert();
+		Xowe_wiki repo_0 = wiki, repo_1 = commons_wiki;
+		if (repo_0_is_remote) {	// NOTE: default is false; local_wiki will be preferred over commons_wiki
+			repo_0 = commons_wiki;
+			repo_1 = wiki;
+		}
+		repo_0.Init_assert(); repo_1.Init_assert();
+		Xob_db_file file_registry_db = Xob_db_file.new__page_regy(commons_wiki.Fsys_mgr().Root_dir());
+		Xob_orig_regy_tbl.Create_data(bldr.Usr_dlg(), conn, file_registry_db, repo_0_is_remote, repo_0, repo_1, Xob_lnki_temp_wkr.Ns_file_is_case_match_all(wiki));
+	}
+	public void Cmd_run() {}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_repo_0_is_remote_))				repo_0_is_remote = m.ReadYn("v");
+		else														return super.Invk(ctx, ikey, k, m);
+		return this;
+	}	private static final String Invk_repo_0_is_remote_ = "repo_0_is_remote_";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_orig_regy_tbl.java

@@ -0,0 +1,221 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.xowa.wikis.*; import gplx.xowa.files.repos.*;
+import gplx.dbs.engines.sqlite.*;
+class Xob_orig_regy_tbl {
+	public static void Create_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
+	public static void Create_data(Gfo_usr_dlg usr_dlg, Db_conn p, Xob_db_file file_registry_db, boolean repo_0_is_remote, Xowe_wiki repo_0_wiki, Xowe_wiki repo_1_wiki, boolean wiki_ns_for_file_is_case_match_all) {
+		usr_dlg.Prog_many("", "", "inserting lnki_regy");
+		p.Exec_sql(Sql_create_data);
+		Sqlite_engine_.Idx_create(usr_dlg, p, "orig_regy", Idx_ttl_local);
+		Sqlite_engine_.Db_attach(p, "page_db", file_registry_db.Url().Raw());
+		Io_url repo_0_dir = repo_0_wiki.Fsys_mgr().Root_dir(), repo_1_dir = repo_1_wiki.Fsys_mgr().Root_dir();
+		byte repo_0_tid = Xof_repo_itm_.Repo_local, repo_1_tid = Xof_repo_itm_.Repo_remote;
+		boolean local_is_remote = Bry_.Eq(repo_0_wiki.Domain_bry(), repo_1_wiki.Domain_bry());
+		Xowe_wiki local_wiki = repo_0_wiki;
+		if (	repo_0_is_remote														// .gfs manually marked specifes repo_0 as remote
+			||	(	Bry_.Eq(repo_0_wiki.Domain_bry(), Xow_domain_.Domain_bry_commons)		// repo_0 = commons; force repo_0 to be remote; else all orig_repo will be 1; DATE:2014-02-01
+				&&	local_is_remote														// repo_0 = repo_1
+				)
+			) {
+			repo_0_tid = Xof_repo_itm_.Repo_remote;
+			repo_1_tid = Xof_repo_itm_.Repo_local;
+			local_wiki = repo_1_wiki;
+		}
+		Create_data_for_repo(usr_dlg, p, local_wiki, Byte_.By_int(repo_0_tid), repo_0_dir.GenSubFil(Xob_db_file.Name__wiki_image));
+		if (!local_is_remote) {	// only run for repo_1 if local != remote; only affects commons
+			Create_data_for_repo(usr_dlg, p, local_wiki, Byte_.By_int(repo_1_tid), repo_1_dir.GenSubFil(Xob_db_file.Name__wiki_image));
+			if (wiki_ns_for_file_is_case_match_all) {
+				Io_url repo_remote_dir = repo_0_is_remote ? repo_0_dir : repo_1_dir;
+				Create_data_for_cs(usr_dlg, p, local_wiki, repo_remote_dir);
+			}
+		}
+		Sqlite_engine_.Db_detach(p, "page_db");
+		Sqlite_engine_.Idx_create(usr_dlg, p, "orig_regy", Idx_xfer_temp);
+	}
+	private static void Create_data_for_repo(Gfo_usr_dlg usr_dlg, Db_conn conn, Xowe_wiki local_wiki, byte repo_tid, Io_url join) {
+		usr_dlg.Note_many("", "", "inserting page for xowa.wiki.image: ~{0}", join.OwnerDir().NameOnly());
+		boolean wiki_has_cs_file = repo_tid == Xof_repo_itm_.Repo_remote && local_wiki.Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Id_all;
+		String lnki_ttl_fld = wiki_has_cs_file ? "Coalesce(o.lnki_commons_ttl, o.lnki_ttl)" : "o.lnki_ttl";	// NOTE: use lnki_commons_ttl if [[File]] is cs PAGE:en.d:water EX:[[image:wikiquote-logo.png|50px|none|alt=]]; DATE:2014-09-05
+		if (wiki_has_cs_file)
+			Sqlite_engine_.Idx_create(usr_dlg, conn, "orig_regy", Idx_ttl_remote);
+		Db_attach_cmd.new_(conn, "image_db", join)
+			.Add_fmt("orig_regy:updating page"		, Sql_update_repo_page, repo_tid, lnki_ttl_fld)
+			.Add_fmt("orig_regy:updating redirect"	, Sql_update_repo_redirect, repo_tid, lnki_ttl_fld)
+			.Exec()
+			;
+	}
+	private static void Create_data_for_cs(Gfo_usr_dlg usr_dlg, Db_conn p, Xowe_wiki local_wiki, Io_url repo_remote_dir) {
+		p.Exec_sql(Xob_orig_regy_tbl.Sql_cs_mark_dupes);	// orig_regy: find dupes; see note in SQL
+		p.Exec_sql(Xob_orig_regy_tbl.Sql_cs_update_ttls);	// orig_regy: update lnki_ttl with lnki_commons_ttl
+		Create_data_for_repo(usr_dlg, p, local_wiki, Xof_repo_itm_.Repo_remote, repo_remote_dir.GenSubFil(Xob_db_file.Name__wiki_image));
+		p.Exec_sql(Xob_lnki_regy_tbl.Sql_cs_mark_changed);	// lnki_regy: update lnki_commons_flag
+		p.Exec_sql(Xob_lnki_regy_tbl.Sql_cs_update_ttls);	// lnki_regy: update cs
+	}
+	public static final String Tbl_name = "orig_regy"
+	, Fld_lnki_id = "lnki_id", Fld_lnki_ttl = "lnki_ttl", Fld_lnki_ext = "lnki_ext", Fld_lnki_count = "lnki_count"
+	, Fld_orig_repo = "orig_repo", Fld_orig_page_id = "orig_page_id"
+	, Fld_orig_redirect_id = "orig_redirect_id", Fld_orig_redirect_ttl = "orig_redirect_ttl", Fld_orig_file_id = "orig_file_id", Fld_orig_file_ttl = "orig_file_ttl"
+	, Fld_orig_size = "orig_size", Fld_orig_w = "orig_w", Fld_orig_h = "orig_h", Fld_orig_bits = "orig_bits"
+	, Fld_orig_media_type = "orig_media_type", Fld_orig_minor_mime = "orig_minor_mime", Fld_orig_file_ext = "orig_file_ext", Fld_orig_timestamp = "orig_timestamp"
+	;
+	private static final Db_idx_itm
+	  Idx_ttl_local     	= Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS orig_regy__ttl_local  ON orig_regy (lnki_ttl);")
+	, Idx_ttl_remote     	= Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS orig_regy__ttl_remote ON orig_regy (lnki_commons_ttl, lnki_ttl);")
+	, Idx_xfer_temp			= Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS orig_regy__xfer_temp  ON orig_regy (lnki_ttl, orig_file_ttl, orig_repo, orig_timestamp);")
+	;
+	private static final String
+		Tbl_sql = String_.Concat_lines_nl
+	(	"CREATE TABLE IF NOT EXISTS orig_regy"
+	,	"( lnki_id                     integer             NOT NULL			PRIMARY KEY"	// NOTE: must be PRIMARY KEY, else later REPLACE INTO will create dupe rows
+	,	", lnki_ttl                    varchar(256)        NOT NULL"
+	,	", lnki_commons_ttl            varchar(256)        NULL"
+	,	", orig_commons_flag           integer             NULL"
+	,	", lnki_ext                    integer             NOT NULL"
+	,	", lnki_count                  integer             NOT NULL"
+	,	", orig_repo                   integer             NULL"
+	,	", orig_page_id                integer             NULL"
+	,	", orig_redirect_id            integer             NULL"
+	,	", orig_redirect_ttl           varchar(256)        NULL"
+	,	", orig_file_id                integer             NULL"
+	,	", orig_file_ttl               varchar(256)        NULL"
+	,	", orig_file_ext               integer             NULL"
+	,	", orig_size                   integer             NULL"
+	,	", orig_w                      integer             NULL"
+	,	", orig_h                      integer             NULL"
+	,	", orig_bits                   smallint            NULL"
+	,	", orig_media_type             varchar(64)         NULL"
+	,	", orig_minor_mime             varchar(32)         NULL"
+	,	", orig_timestamp              varchar(14)         NULL"
+	,	");"
+	)
+	,	Sql_create_data = String_.Concat_lines_nl
+	(	"INSERT INTO orig_regy (lnki_id, lnki_ttl, lnki_commons_ttl, orig_commons_flag, lnki_ext, lnki_count, orig_timestamp)"
+	,	"SELECT  Min(lnki_id)"
+	,	",       lnki_ttl"
+	,	",       lnki_commons_ttl"
+	,	",       NULL"
+	,	",       lnki_ext"
+	,	",       Sum(lnki_count)"
+	,	",       ''"
+	,	"FROM    lnki_regy"
+	,	"GROUP BY lnki_ttl"
+	,	",       lnki_ext"
+	,	"ORDER BY 1"	// must order by lnki_id since it is PRIMARY KEY, else sqlite will spend hours shuffling rows in table
+	,	";"
+	)
+	,	Sql_update_repo_page = String_.Concat_lines_nl
+	(	"REPLACE INTO orig_regy"
+	,	"SELECT  o.lnki_id"
+	,	",       o.lnki_ttl"
+	,	",       o.lnki_commons_ttl"
+	,	",       o.orig_commons_flag"
+	,	",       o.lnki_ext"
+	,	",       o.lnki_count"
+	,	",       {0}"
+	,	",       m.src_id"
+	,	",       NULL"
+	,	",       NULL"
+	,	",       m.src_id"
+	,	",       m.src_ttl"
+	,	",       i.img_ext_id"
+	,	",       i.img_size"
+	,	",       i.img_width"
+	,	",       i.img_height"
+	,	",       i.img_bits"
+	,	",       i.img_media_type"
+	,	",       i.img_minor_mime"
+	,	",       i.img_timestamp"
+	,	"FROM    orig_regy o"
+	,	"        JOIN <attach_db>image i ON {1} = i.img_name"
+	,	"        JOIN page_db.page_regy m ON m.repo_id = {0} AND m.itm_tid = 0 AND {1} = m.src_ttl"
+	,	"WHERE   o.orig_file_ttl IS NULL"					// NOTE: only insert if file doesn't exist; changed from timestamp b/c old images may exist in both wikis; EX:ar.n:File:Facebook.png; DATE:2014-08-20
+	// ,	"WHERE   i.img_timestamp > o.orig_timestamp"	// NOTE: this handles an image in local and remote by taking later version; DATE:2014-07-22
+	,	"ORDER BY 1"	// must order by lnki_id since it is PRIMARY KEY, else sqlite will spend hours shuffling rows in table
+	,	";"
+	)
+	,	Sql_update_repo_redirect = String_.Concat_lines_nl
+	(	"REPLACE INTO orig_regy"
+	,	"SELECT  o.lnki_id"
+	,	",       o.lnki_ttl"
+	,	",       o.lnki_commons_ttl"
+	,	",       o.orig_commons_flag"
+	,	",       o.lnki_ext"
+	,	",       o.lnki_count"
+	,	",       {0}"
+	,	",       m.src_id"
+	,	",       m.trg_id"
+	,	",       m.trg_ttl"
+	,	",       m.trg_id"
+	,	",       m.trg_ttl"
+	,	",       i.img_ext_id"
+	,	",       i.img_size"
+	,	",       i.img_width"
+	,	",       i.img_height"
+	,	",       i.img_bits"
+	,	",       i.img_media_type"
+	,	",       i.img_minor_mime"
+	,	",       i.img_timestamp"
+	,	"FROM    orig_regy o"
+	,	"        JOIN page_db.page_regy m ON m.repo_id = {0} AND m.itm_tid = 1 AND {1} = m.src_ttl"
+	,	"            JOIN <attach_db>image i ON m.trg_ttl = i.img_name"
+	,	"WHERE   o.orig_file_ttl IS NULL"						// NOTE: only insert if file doesn't exist; changed from timestamp b/c old images may exist in both wikis; EX:ar.n:File:Facebook.png; DATE:2014-08-20
+	// ,	"WHERE   i.img_timestamp > o.orig_timestamp"	// NOTE: this handles an image in local and remote by taking later version; DATE:2014-07-22
+	,	"ORDER BY 1"	// must order by lnki_id since it is PRIMARY KEY, else sqlite will spend hours shuffling rows in table
+	,	";"
+	)
+	,	Sql_cs_mark_dupes = String_.Concat_lines_nl
+	(	"REPLACE INTO orig_regy"
+	,	"SELECT  o.lnki_id"
+	,	",       o.lnki_ttl"
+	,	",       o.lnki_commons_ttl"
+	,	",       1"
+	,	",       o.lnki_ext"
+	,	",       o.lnki_count"
+	,	",       o.orig_repo"
+	,	",       o.orig_page_id"
+	,	",       o.orig_redirect_id"
+	,	",       o.orig_redirect_ttl"
+	,	",       o.orig_file_id"
+	,	",       o.orig_file_ttl"
+	,	",       o.orig_file_ext"
+	,	",       o.orig_size"
+	,	",       o.orig_w"
+	,	",       o.orig_h"
+	,	",       o.orig_bits"
+	,	",       o.orig_media_type"
+	,	",       o.orig_minor_mime"
+	,	",       o.orig_timestamp"
+	,	"FROM    orig_regy o"
+	,	"        JOIN orig_regy o2 ON o.lnki_commons_ttl = o2.lnki_ttl"	// EX: 2 rows in table (1) A.jpg; and (2) "a.jpg,A.jpg"; do not insert row (2) b/c row (1) exists;
+	,	"WHERE   o.orig_file_ttl IS NULL"	// NOTE: don't use timestamp logic here
+	,	"ORDER BY 1"	// must order by lnki_id since it is PRIMARY KEY, else sqlite will spend hours shuffling rows in table
+	,	";"
+	)
+	,	Sql_cs_update_ttls = String_.Concat_lines_nl
+	(	"UPDATE  orig_regy"
+	,	"SET     lnki_ttl = lnki_commons_ttl"
+	,	",       orig_commons_flag = 2"
+	,	"WHERE   orig_file_ttl IS NULL"			// orig not found
+	,	"AND     lnki_commons_ttl IS NOT NULL"	// orig commons ttl exists
+	,	"AND     orig_commons_flag IS NULL"		// orig is not dupe
+	,	";"
+	)
+	;
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_orig_regy_update_bmk_mgr.java

@@ -0,0 +1,100 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.xowa.wikis.data.tbls.*;
+public class Xob_orig_regy_update_bmk_mgr implements GfoInvkAble {
+	private Db_cfg_tbl cfg_tbl;
+	private String cfg_grp;
+	public byte Repo_prv() {return repo_prv;} private byte repo_prv;
+	public int Ns_prv() {return ns_prv;} private int ns_prv;
+	public byte[] Ttl_prv() {return ttl_prv;} private byte[] ttl_prv;
+	private boolean repo_enable, ns_enable, ttl_enable;
+	private boolean repo_dirty, ns_dirty, ttl_dirty;
+	public Xob_orig_regy_update_bmk_mgr Init(Db_conn p, String grp, boolean repo_enable, boolean ns_enable, boolean ttl_enable) {
+		this.cfg_grp = grp;
+		cfg_tbl = new Db_cfg_tbl(p, "xowa_cfg");
+		this.repo_enable = repo_enable;
+		this.ns_enable = ns_enable;
+		this.ttl_enable = ttl_enable;
+		return this;
+	}
+	public void Term() {
+		this.Save();	// flush existing values
+		cfg_tbl.Rls();
+	}
+	public void Update_repo_ttl(byte repo, byte[] ttl) {Update(repo, Null_ns, ttl);}
+	public void Update(byte repo, int ns, byte[] ttl) {
+		if (repo_enable) {
+			if (repo_prv != repo) {
+				repo_dirty = true;
+				this.repo_prv = repo;
+			}
+		}
+		if (ns_enable) {
+			if (ns_prv != ns) {
+				ns_dirty = true;
+				this.ns_prv = ns;
+			}
+		}
+		if (ttl_enable) {
+			if (!Bry_.Eq(ttl_prv, ttl)) {
+				ttl_dirty = true;
+				this.ttl_prv = ttl;
+			}
+		}
+	}
+	public Xob_orig_regy_update_bmk_mgr Load() {
+		if (repo_enable)
+			repo_prv	= cfg_tbl.Assert_byte(cfg_grp, Cfg_repo_prv, Byte_.Zero);
+		if (ns_enable)
+			ns_prv		= cfg_tbl.Assert_int(cfg_grp, Cfg_ns_prv, 0);
+		if (ttl_enable)
+			ttl_prv		= cfg_tbl.Assert_bry(cfg_grp, Cfg_ttl_prv, Bry_.Empty);
+		repo_dirty = ns_dirty = ttl_dirty = false;
+		return this;
+	}
+	public void Save() {
+		if (repo_enable && repo_dirty) {
+			Save(Cfg_repo_prv, Byte_.Xto_str(repo_prv));
+			repo_dirty = false;
+		}
+		if (ns_enable && ns_dirty) {
+			Save(Cfg_ns_prv, Int_.Xto_str(ns_prv));
+			ns_dirty = false;
+		}
+		if (ttl_enable && ttl_dirty) {
+			Save(Cfg_ttl_prv, String_.new_u8(ttl_prv));
+			ttl_dirty = false;
+		}
+	}
+	private void Save(String cfg_key, String cfg_val) {
+		cfg_tbl.Update_str(cfg_grp, cfg_key, cfg_val);
+	}
+	public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_repo_prv_))				repo_prv = m.ReadByte("v");
+		else if	(ctx.Match(k, Invk_ns_prv_))				ns_prv = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_ttl_prv_))				ttl_prv = m.ReadBry("v");
+		else	return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}
+	private static final String Invk_repo_prv_ = "repo_prv_", Invk_ns_prv_ = "ns_prv_", Invk_ttl_prv_ = "ttl_prv_";
+	private static final String Cfg_repo_prv = "repo_prv", Cfg_ns_prv = "ns_prv", Cfg_ttl_prv = "ttl_prv";
+	private static final int Null_ns = -1;
+//		public static final byte Null_repo = Byte_.Max_value_127;
+//		public static final byte[ Null_ttl = null;
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_orig_regy_update_cmd.java

@@ -0,0 +1,64 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.dbs.*; import gplx.xowa.files.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.files.fsdb.*; import gplx.xowa.files.origs.*;
+public class Xob_orig_regy_update_cmd extends Xob_itm_basic_base implements Xob_cmd {	// downloads latest orig data
+	public Xob_orig_regy_update_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return KEY_oimg;} public static final String KEY_oimg = "oimg.orig_qry";
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {
+//			Xof_orig_mgr qry_mgr = new Xof_orig_mgr();
+		Db_conn conn = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
+		Xob_orig_regy_update_bmk_mgr bmk = new Xob_orig_regy_update_bmk_mgr();
+		bmk.Init(conn, this.Cmd_key(), true, false, true);
+		bmk.Load();
+//			Xof_fsdb_itm itm = new Xof_fsdb_itm();
+		DataRdr rdr = DataRdr_.Null;
+		try {
+			// rdr = Select(conn, bmk.Repo_prv(), bmk.Ttl_prv());
+			while (rdr.MoveNextPeer()) {
+				// Load_itm(itm, rdr);
+				// QueryItm
+			}
+		}
+		finally {
+			rdr.Rls();
+		}
+		/*		
+		DataRdr rdr = Get(repo_prv, ttl_prv);
+		while (rdr.MoveNextPeer()) {
+			Itm itm = Itm.load_(rdr);
+			qry.Call(itm);
+		}
+		*/
+	}
+	public DataRdr Select(Db_conn p, byte prv_repo_id, byte[] prv_ttl) {
+		String sql = String_.Concat_lines_nl_skip_last
+		(	"SELECT	lnki_ttl"
+		,	"FROM	orig_regy"	
+		,	"WHERE	lnki_repo >= '" + Byte_.Xto_str(prv_repo_id) + "'"
+		,	"AND    lnki_ttl > '" + prv_ttl + "'"
+		,	"AND	oimg_orig_page_id = -1;"
+		);
+		Db_qry select_qry = Db_qry_sql.rdr_(sql);
+		return p.Exec_qry_as_rdr(select_qry);
+	}
+	public void Cmd_run() {}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_page_regy_cmd.java

@@ -0,0 +1,50 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.xowa.wikis.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.files.repos.*;
+public class Xob_page_regy_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	public Xob_page_regy_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	private boolean build_commons = false;
+	public String Cmd_key() {return Xob_cmd_keys.Key_file_page_regy;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {
+		Xowe_wiki commons_wiki = bldr.App().Wiki_mgr().Get_by_key_or_make(Xow_domain_.Domain_bry_commons).Init_assert();
+		Db_conn page_regy_provider = Xob_db_file.new__page_regy(commons_wiki.Fsys_mgr().Root_dir()).Conn();
+		commons_wiki.Init_assert();
+		if (build_commons) {
+			Xob_page_regy_tbl.Reset_table(page_regy_provider);
+			Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_itm_.Repo_remote, commons_wiki);
+			Sqlite_engine_.Idx_create(usr_dlg, page_regy_provider, "repo_page", Xob_page_regy_tbl.Idx_main);
+		}
+		else {
+			if (!Bry_.Eq(commons_wiki.Domain_bry(), wiki.Domain_bry())) {	// skip local wiki if cur wiki is commons
+				wiki.Init_assert();
+				Xob_page_regy_tbl.Delete_local(page_regy_provider);
+				Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_itm_.Repo_local, wiki);
+			}
+		}
+	}
+	public void Cmd_run() {}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_build_commons_))				build_commons = m.ReadYn("v");
+		else	return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}	private static final String Invk_build_commons_ = "build_commons_";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_page_regy_tbl.java

@@ -0,0 +1,86 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.xowa.files.repos.*; import gplx.dbs.engines.sqlite.*;
+class Xob_page_regy_tbl {
+	public static void Reset_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
+	public static void Create_data(Gfo_usr_dlg usr_dlg, Db_conn p, byte repo_tid, Xowe_wiki wiki) {
+		Xowd_db_file db_core = wiki.Db_mgr_as_sql().Core_data_mgr().Db__core();
+		Create_data__insert_page(usr_dlg, p, repo_tid, db_core.Url());
+		Create_data__insert_redirect(usr_dlg, p, repo_tid, wiki.Fsys_mgr().Root_dir().GenSubFil(Xob_db_file.Name__wiki_redirect));
+	}
+	public static void Delete_local(Db_conn p) {
+		p.Exec_sql("DELETE FROM page_regy WHERE repo_id = " + Xof_repo_itm_.Repo_local);
+	}
+	private static void Create_data__insert_page(Gfo_usr_dlg usr_dlg, Db_conn cur, byte repo_tid, Io_url join) {
+		usr_dlg.Note_many("", "", "inserting page: ~{0}", join.NameOnly());
+		Sqlite_engine_.Db_attach(cur, "page_db", join.Raw());
+		cur.Exec_sql(String_.Format(Sql_create_page, repo_tid));
+		Sqlite_engine_.Db_detach(cur, "page_db");
+	}
+	private static void Create_data__insert_redirect(Gfo_usr_dlg usr_dlg, Db_conn cur, byte repo_tid, Io_url join) {
+		if (!Io_mgr.I.ExistsFil(join)) return;	// redirect_db will not exist when commons.wikimedia.org is set up on new machine
+		usr_dlg.Note_many("", "", "inserting redirect: ~{0}", join.OwnerDir().NameOnly());
+		Sqlite_engine_.Db_attach(cur, "redirect_db", join.Raw());
+		cur.Exec_sql(String_.Format(Sql_create_redirect, repo_tid));
+		Sqlite_engine_.Db_detach(cur, "redirect_db");
+	}
+	public static final String Tbl_name = "page_regy"
+	, Fld_uid = "uid", Fld_repo_id = "repo_id", Fld_itm_tid = "itm_tid"
+	, Fld_src_id = "src_id", Fld_src_ttl = "src_ttl"
+	, Fld_trg_id = "trg_id", Fld_trg_ttl = "trg_ttl"
+	;
+	public static final Db_idx_itm
+		Idx_main     		= Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS page_regy__main           ON page_regy (repo_id, itm_tid, src_ttl, src_id, trg_id, trg_ttl);")
+	;
+	private static final String
+		Tbl_sql = String_.Concat_lines_nl
+	(	"CREATE TABLE IF NOT EXISTS page_regy"
+	,	"( uid                     integer             NOT NULL			PRIMARY KEY           AUTOINCREMENT"	// NOTE: must be PRIMARY KEY, else later REPLACE INTO will create dupe rows
+	,	", repo_id                 integer             NOT NULL"
+	,	", itm_tid                 tinyint             NOT NULL"
+	,	", src_id                  integer             NOT NULL"
+	,	", src_ttl                 varchar(255)        NOT NULL"
+	,	", trg_id                  integer             NOT NULL"
+	,	", trg_ttl                 varchar(255)        NOT NULL"
+	,	");"
+	)
+	,	Sql_create_page = String_.Concat_lines_nl
+	(	"INSERT INTO page_regy (repo_id, itm_tid, src_id, src_ttl, trg_id, trg_ttl)"
+	,	"SELECT  {0}"
+	,	",       0"		// 0=page
+	,	",       p.page_id"
+	,	",       p.page_title"
+	,	",       -1"
+	,	",       ''"
+	,	"FROM    page_db.page p"
+	,	";"
+	)
+	,	Sql_create_redirect = String_.Concat_lines_nl
+	(	"INSERT INTO page_regy (repo_id, itm_tid, src_id, src_ttl, trg_id, trg_ttl)"
+	,	"SELECT  {0}"
+	,	",       1"		// 1=redirect
+	,	",       r.src_id"
+	,	",       r.src_ttl"
+	,	",       r.trg_id"
+	,	",       r.trg_ttl"
+	,	"FROM    redirect_db.redirect r"
+	,	";"
+	)
+	;
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_regy_cmd.java

@@ -0,0 +1,36 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*;
+public class Xob_xfer_regy_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	public Xob_xfer_regy_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_file_xfer_regy;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run() {
+		Db_conn conn = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
+		conn.Txn_bgn();
+		Xob_xfer_regy_tbl.Create_table(conn);
+		Xob_xfer_regy_tbl.Create_data(usr_dlg, conn);
+		Xob_xfer_regy_tbl.Create_index(usr_dlg, conn);
+		Xob_xfer_regy_log_tbl.Create_table(conn);
+		conn.Txn_end();
+	}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_regy_tbl.java

@@ -0,0 +1,157 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.dbs.engines.sqlite.*;
+public class Xob_xfer_regy_tbl {
+	public static final String Tbl_name = "xfer_regy"
+	, Fld_lnki_id = "lnki_id", Fld_lnki_tier_id = "lnki_tier_id", Fld_lnki_page_id = "lnki_page_id", Fld_lnki_ttl = "lnki_ttl", Fld_lnki_ext = "lnki_ext"
+	, Fld_lnki_time = "lnki_time", Fld_lnki_page = "lnki_page", Fld_lnki_count = "lnki_count"
+	, Fld_orig_repo = "orig_repo", Fld_orig_page_id = "orig_page_id", Fld_orig_redirect_src = "orig_redirect_src", Fld_orig_media_type = "orig_media_type"
+	, Fld_orig_w = "orig_w", Fld_orig_h = "orig_h"
+	, Fld_file_w = "file_w", Fld_file_h = "file_h", Fld_file_is_orig = "file_is_orig"		
+	, Fld_xfer_status = "xfer_status"
+	;
+	public static void Create_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
+	public static void Create_data(Gfo_usr_dlg usr_dlg, Db_conn p) {
+		p.Exec_sql(Sql_create_data_orig);
+		p.Exec_sql(Sql_create_data_thumb);
+	}
+	public static void Create_index(Gfo_usr_dlg usr_dlg, Db_conn p)	{Sqlite_engine_.Idx_create(usr_dlg, p, Xob_db_file.Name__file_make, Idx_lnki_page_id, Idx_lnki_ttl);}
+	public static DataRdr Select(Db_conn p, byte repo_id, byte[] ttl, int limit) {
+		Db_qry qry = Db_qry_.select_().Cols_all_()
+			.From_(Tbl_name)
+			.Where_(gplx.core.criterias.Criteria_.And_many(Db_crt_.mte_(Fld_orig_repo, repo_id), Db_crt_.mt_(Fld_lnki_ttl, String_.new_u8(ttl)), Db_crt_.eq_(Fld_xfer_status, 0)))
+			.OrderBy_many_(Fld_xfer_status, Fld_orig_repo, Fld_lnki_ttl, Fld_file_w)
+			.Limit_(limit)
+			;
+		return p.Exec_qry_as_rdr(qry);
+	}
+	public static Db_stmt Select_by_page_id_stmt(Db_conn p) {return p.Stmt_new(Db_qry_sql.rdr_(Sql_select));}
+	public static DataRdr Select_by_page_id(Db_stmt stmt, int page_id, int limit) {return stmt.Val_int(page_id).Val_int(limit).Exec_select();}
+	private static final String
+	  Sql_select = String_.Concat_lines_nl
+		( "SELECT   *"
+		, "FROM     xfer_regy"
+		, "WHERE    xfer_status  =  0"
+		, "AND      lnki_page_id >= ?"
+		, "ORDER BY lnki_tier_id, lnki_page_id, lnki_id"
+		, "LIMIT    ?"
+		)
+	, Sql_select_total_pending = String_.Concat_lines_nl
+		( "SELECT   Count(*) AS CountAll"
+		, "FROM     xfer_regy"
+		, "WHERE    xfer_status  =  0"
+		)
+	;
+	public static DataRdr Select_by_tier_page(Db_conn p, int tier_id, int page_id, int select_interval) {
+		Db_qry qry = Db_qry_.select_().Cols_all_()
+			.From_(Tbl_name)
+			.Where_(gplx.core.criterias.Criteria_.And_many(Db_crt_.eq_(Fld_xfer_status, 0), Db_crt_.eq_(Fld_lnki_tier_id, tier_id), Db_crt_.mte_(Fld_lnki_page_id, page_id)))
+			.OrderBy_many_(Fld_lnki_tier_id, Fld_lnki_page_id, Fld_lnki_id)
+			.Limit_(select_interval)
+			;
+		return p.Exec_qry_as_rdr(qry);
+	}
+	public static int Select_total_pending(Db_conn p) {
+		DataRdr rdr = p.Exec_sql_as_rdr(Sql_select_total_pending);
+		int rv = 0;
+		if (rdr.MoveNextPeer())
+			rv = rdr.ReadInt("CountAll");
+		rdr.Rls();
+		return rv;
+	}
+	private static final String Tbl_sql = String_.Concat_lines_nl
+	(	"CREATE TABLE IF NOT EXISTS xfer_regy"
+	,	"( lnki_id             integer             NOT NULL			    PRIMARY KEY"
+	,	", lnki_tier_id        integer             NOT NULL"
+	,	", lnki_page_id        integer             NOT NULL"
+	,	", lnki_ttl            varchar(255)        NOT NULL"
+	,	", lnki_ext            integer             NOT NULL"
+	,	", lnki_time           double              NOT NULL"
+	,	", lnki_page           integer             NOT NULL"
+	,	", lnki_count          integer             NOT NULL"
+	,	", orig_repo           tinyint             NOT NULL"
+	,	", orig_page_id        integer             NOT NULL"
+	,	", orig_redirect_src   varchar(255)        NOT NULL"
+	,	", orig_media_type     varchar(64)         NOT NULL"
+	,	", orig_w              integer             NOT NULL"
+	,	", orig_h              integer             NOT NULL"
+	,	", file_is_orig        byte                NOT NULL"
+	,	", file_w              integer             NOT NULL"
+	,	", file_h              integer             NOT NULL"
+	,	", xfer_status         integer             NOT NULL"
+	,	");"
+	);
+	private static final String Sql_create_data_orig = String_.Concat_lines_nl
+	( "INSERT INTO xfer_regy "
+	, "( lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
+	, ", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
+	, ", xfer_status"
+	, ")"
+	, "SELECT "
+	, "  Min(lnki_id), Min(lnki_tier_id), Min(lnki_page_id), Min(orig_page_id), orig_repo, lnki_ttl, Max(orig_redirect_src), lnki_ext, orig_media_type"	// NOTE: Max(orig_redirect_src) not Min (else would get '')
+	, ", file_is_orig, orig_w, orig_h, -1, -1, lnki_time, lnki_page, Sum(lnki_count)"
+	, ", 0"
+	, "FROM    xfer_temp x"
+	, "WHERE   file_is_orig = 1"
+	, "GROUP BY orig_repo, lnki_ttl, lnki_ext, orig_media_type, file_is_orig, orig_w, orig_h, lnki_time, lnki_page"
+	);
+	private static final String Sql_create_data_thumb = String_.Concat_lines_nl
+	( "INSERT INTO xfer_regy "
+	, "( lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
+	, ", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
+	, ", xfer_status"
+	, ")"
+	, "SELECT "
+	, "  Min(lnki_id), Min(lnki_tier_id), Min(lnki_page_id), Min(orig_page_id), orig_repo, lnki_ttl, Max(orig_redirect_src), lnki_ext, orig_media_type"
+	, ", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, Sum(lnki_count)"
+	, ", 0"
+	, "FROM    xfer_temp x"
+	, "WHERE   file_is_orig = 0"
+	, "GROUP BY orig_repo, lnki_ttl, lnki_ext, orig_media_type, file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page"
+	);
+	private static final Db_idx_itm
+//		  Idx_select		= Db_idx_itm.sql_("CREATE        INDEX IF NOT EXISTS xfer_regy__select        ON xfer_regy (xfer_status, orig_repo, lnki_ttl, file_w);")
+	  Idx_lnki_page_id	= Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS xfer_regy__lnki_page_id  ON xfer_regy (xfer_status, lnki_tier_id, lnki_page_id, lnki_id, orig_repo, file_is_orig, lnki_ttl, lnki_ext, lnki_time, lnki_page, file_w, file_h);")
+	, Idx_lnki_ttl		= Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS xfer_regy__lnki_ttl ON xfer_regy (lnki_ttl);")	// needed for troubleshooting
+	;
+	public static byte Status_todo = 0, Status_pass = 1, Status_fail = 2, Status_ignore_processed = 3;
+}
+class Xob_xfer_regy_log_tbl {
+	public static void Create_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
+	public static Db_stmt Insert_stmt(Db_conn p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_lnki_id, Fld_xfer_status, Fld_xfer_bin_tid, Fld_xfer_bin_msg);}
+	public static void Insert(Db_stmt stmt, byte status, int id, byte wkr_tid, String wkr_msg) {
+		stmt.Clear()
+		.Val_int(id)
+		.Val_byte(status)
+		.Val_byte(wkr_tid)
+		.Val_str(wkr_msg)
+		.Exec_insert();
+	}
+	private static final String Tbl_sql = String_.Concat_lines_nl
+	(	"CREATE TABLE IF NOT EXISTS xfer_regy_log"
+	,	"( lnki_id             integer             NOT NULL"
+	,	", xfer_status         tinyint             NOT NULL"	// 0=todo; 1=fail; 2=pass; 3=done
+	,	", xfer_bin_tid        tinyint             NOT NULL"
+	,	", xfer_bin_msg        varchar(255)        NOT NULL"
+	,	");"
+	);
+	public static final String Tbl_name = "xfer_regy_log"
+	, Fld_lnki_id = "lnki_id", Fld_xfer_status = "xfer_status", Fld_xfer_bin_tid = "xfer_bin_tid", Fld_xfer_bin_msg = "xfer_bin_msg"
+	;
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_regy_update_cmd.java

@@ -0,0 +1,172 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.fsdb.meta.*;
+public class Xob_xfer_regy_update_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	public Xob_xfer_regy_update_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_file_xfer_regy_update;} 
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run() {Exec_main();}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	private void Exec_main() {
+		Db_conn make_db_provider = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
+		this.Copy_atrs_to_make_db(make_db_provider);
+		this.Update_status(make_db_provider);
+	}
+	private void Copy_atrs_to_make_db(Db_conn make_db_provider) {
+		wiki.File_mgr().Init_file_mgr_by_load(wiki);
+		Fsm_mnt_itm fsdb_abc_mgr = wiki.File_mgr().Fsdb_mgr().Mnt_mgr().Mnts__get_main();		// 0 = fsdb.main
+		Db_conn conn = fsdb_abc_mgr.Atr_mgr().Db__core().Conn();								// 0 = fsdb.atr.00
+		Io_url fsdb_atr_url = ((gplx.dbs.engines.sqlite.Sqlite_conn_info)conn.Conn_info()).Url();
+		Sqlite_engine_.Tbl_create_and_delete(make_db_provider, Xob_fsdb_regy_tbl.Tbl_name, Xob_fsdb_regy_tbl.Tbl_sql);
+		Sqlite_engine_.Db_attach(make_db_provider, "fsdb_db", fsdb_atr_url.Raw());
+		make_db_provider.Txn_bgn();
+		make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Update_regy_nil);
+		make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Insert_fsdb_fil);
+		String fsdb_thm_tbl = fsdb_abc_mgr.Db_mgr().File__schema_is_1() ? "fsdb_xtn_thm" : "fsdb_thm";
+		String insert_sql_fsdb_thm = wiki.File_mgr().Fsdb_mgr().Mnt_mgr().Mnts__get_main().Cfg_mgr().Schema_thm_page()	// Cfg_get(Fsm_cfg_mgr.Grp_core).Get_yn_or_n(Fsm_cfg_mgr.Key_schema_thm_page)
+			? String_.Format(Xob_fsdb_regy_tbl.Insert_fsdb_thm, fsdb_thm_tbl)
+			: Xob_fsdb_regy_tbl.Insert_fsdb_thm_v0
+			;
+		make_db_provider.Exec_sql(insert_sql_fsdb_thm);
+		make_db_provider.Txn_end();
+		Sqlite_engine_.Idx_create(make_db_provider, Xob_fsdb_regy_tbl.Idx_main);
+		Sqlite_engine_.Db_detach(make_db_provider, "fsdb_db");
+	}
+	private void Update_status(Db_conn make_db_provider) {
+		make_db_provider.Txn_bgn();
+		make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Update_regy_fil);
+		make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Update_regy_thm);
+		make_db_provider.Txn_end();
+	}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		return this;
+	}
+}
+class Xob_fsdb_regy_tbl {
+	public static final String Tbl_name = "fsdb_regy";
+	public static final String Tbl_sql = String_.Concat_lines_nl
+	(	"CREATE TABLE fsdb_regy "
+	,	"( fsdb_id               integer             NOT NULL       PRIMARY KEY        AUTOINCREMENT"
+	,	", fsdb_name             varchar(255)        NOT NULL"
+	,	", fsdb_is_orig          tinyint             NOT NULL"
+	,	", fsdb_repo             tinyint             NOT NULL"
+	,	", fsdb_w                integer             NOT NULL"
+	,	", fsdb_time		     double              NOT NULL"
+	,	", fsdb_page		     integer             NOT NULL"
+	,	", fsdb_db_id		     integer             NOT NULL"
+	,	", fsdb_size		     bigint              NOT NULL"
+	,	", fsdb_status		     tinyint             NOT NULL"
+	,	", fsdb_fil_id           integer             NOT NULL"
+	,	", fsdb_thm_id           integer             NOT NULL"
+	,	");"
+	);
+	public static final Db_idx_itm Idx_main = Db_idx_itm.sql_("CREATE INDEX fsdb_regy__main ON fsdb_regy (fsdb_name, fsdb_is_orig, fsdb_repo, fsdb_w, fsdb_time, fsdb_page);");
+	public static final String 
+		Insert_fsdb_fil = String_.Concat_lines_nl
+	(	"INSERT INTO fsdb_regy (fsdb_name, fsdb_is_orig, fsdb_repo, fsdb_w, fsdb_time, fsdb_page, fsdb_db_id, fsdb_size, fsdb_status, fsdb_fil_id, fsdb_thm_id)"
+	,	"SELECT  f.fil_name"
+	,	",       1"
+	,	",       CASE WHEN d.dir_name = 'commons.wikimedia.org' THEN 0 ELSE 1 END"
+	,	",       -1"
+	,	",       -1"
+	,	",       -1"
+	,	",       f.fil_bin_db_id"
+	,	",       f.fil_size"
+	,	",       0"
+	,	",       f.fil_id"
+	,	",       -1"
+	,	"FROM    fsdb_db.fsdb_fil f"
+	,	"        JOIN fsdb_db.fsdb_dir d ON f.fil_owner_id = d.dir_id"
+	,	"WHERE   f.fil_bin_db_id != -1"
+	,	";"
+	)
+	,	Insert_fsdb_thm = String_.Concat_lines_nl
+	(	"INSERT INTO fsdb_regy (fsdb_name, fsdb_is_orig, fsdb_repo, fsdb_w, fsdb_time, fsdb_page, fsdb_db_id, fsdb_size, fsdb_status, fsdb_fil_id, fsdb_thm_id)"
+	,	"SELECT  f.fil_name"
+	,	",       0"
+	,	",       CASE WHEN d.dir_name = 'commons.wikimedia.org' THEN 0 ELSE 1 END"
+	,	",       t.thm_w"
+	,	",       t.thm_time"
+	,	",       t.thm_page"
+	,	",       t.thm_bin_db_id"
+	,	",       t.thm_size"
+	,	",       0"
+	,	",       f.fil_id"
+	,	",       t.thm_id"
+	,	"FROM    fsdb_db.fsdb_fil f"
+	,	"        JOIN fsdb_db.{0} t ON f.fil_id = t.thm_owner_id"
+	,	"        JOIN fsdb_db.fsdb_dir d ON f.fil_owner_id = d.dir_id"
+	,	";"
+	)
+	,	Insert_fsdb_thm_v0 = String_.Concat_lines_nl
+	(	"INSERT INTO fsdb_regy (fsdb_name, fsdb_is_orig, fsdb_repo, fsdb_w, fsdb_time, fsdb_page, fsdb_db_id, fsdb_size, fsdb_status, fsdb_fil_id, fsdb_thm_id)"
+	,	"SELECT  f.fil_name"
+	,	",       0"
+	,	",       CASE WHEN d.dir_name = 'commons.wikimedia.org' THEN 0 ELSE 1 END"
+	,	",       t.thm_w"
+	,	",       t.thm_thumbtime"
+	,	",       -1"
+	,	",       t.thm_bin_db_id"
+	,	",       t.thm_size"
+	,	",       0"
+	,	",       f.fil_id"
+	,	",       t.thm_id"
+	,	"FROM    fsdb_db.fsdb_fil f"
+	,	"        JOIN fsdb_db.{0} t ON f.fil_id = t.thm_owner_id"
+	,	"        JOIN fsdb_db.fsdb_dir d ON f.fil_owner_id = d.dir_id"
+	,	";"
+	)
+	,	Update_regy_nil = "UPDATE xfer_regy SET xfer_status = 0;"
+	,	Update_regy_fil = String_.Concat_lines_nl
+	(	"REPLACE INTO xfer_regy "
+	,	"( lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
+	,	", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
+	,	", xfer_status"
+	,	")"
+	,	"SELECT "
+	,	"  lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
+	,	", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
+	,	", CASE WHEN f.fsdb_name IS NOT NULL THEN 1 ELSE 0 END"
+	,	"FROM    xfer_regy x"
+	,	"        LEFT JOIN fsdb_regy f ON x.lnki_ttl = f.fsdb_name"
+	,	"WHERE   x.file_is_orig = 1 AND f.fsdb_is_orig  = 1"
+	,	"AND     x.orig_repo = f.fsdb_repo"
+	,	";"
+	)
+	,	Update_regy_thm = String_.Concat_lines_nl
+	(	"REPLACE INTO xfer_regy "
+	,	"( lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
+	,	", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
+	,	", xfer_status"
+	,	")"
+	,	"SELECT "
+	,	"  lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
+	,	", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
+	,	", CASE WHEN f.fsdb_name IS NOT NULL THEN 1 ELSE 0 END"
+	,	"FROM    xfer_regy x"
+	,	"        LEFT JOIN fsdb_regy f ON x.lnki_ttl = f.fsdb_name AND x.file_w = f.fsdb_w"
+	,   "          AND x.lnki_time = f.fsdb_time AND x.lnki_page = f.fsdb_page"
+	,	"WHERE   x.file_is_orig = 0 AND f.fsdb_is_orig  = 0"
+	,	"AND     x.orig_repo = f.fsdb_repo"
+	,	";"
+	)
+	;
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_cmd_orig.java

@@ -0,0 +1,103 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.xowa.dbs.*; import gplx.xowa.files.*; import gplx.xowa.files.exts.*;	
+public class Xob_xfer_temp_cmd_orig extends Xob_itm_basic_base implements Xob_cmd {
+	private byte[] ext_rules_key = Bry_.Empty;
+	public Xob_xfer_temp_cmd_orig(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_file_xfer_temp_orig;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {
+		Db_conn conn = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
+		Xob_xfer_temp_tbl.Create_table(conn);
+		Db_stmt trg_stmt = Xob_xfer_temp_tbl.Insert_stmt(conn);
+		conn.Txn_bgn();
+		DataRdr rdr = conn.Exec_sql_as_rdr(Sql_select);
+		long[] ext_maxs = Calc_ext_max();
+		while (rdr.MoveNextPeer()) {
+			int lnki_ext = rdr.ReadByte(Xob_lnki_regy_tbl.Fld_lnki_ext);
+			String orig_media_type = rdr.ReadStrOr(Xob_orig_regy_tbl.Fld_orig_media_type, "");	// convert nulls to ""
+			lnki_ext = rdr.ReadInt(Xob_orig_regy_tbl.Fld_orig_file_ext);
+			int lnki_id = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_id);
+			int lnki_tier_id = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_tier_id);
+			byte orig_repo = rdr.ReadByte(Xob_orig_regy_tbl.Fld_orig_repo);
+			int orig_page_id = rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_page_id, -1);
+			if (orig_page_id == -1) continue;	// no orig found; ignore
+			String join_ttl = rdr.ReadStr(Xob_orig_regy_tbl.Fld_orig_file_ttl);
+			String redirect_src = rdr.ReadStr(Xob_orig_regy_tbl.Fld_lnki_ttl);
+			if (String_.Eq(join_ttl, redirect_src))	// lnki_ttl is same as redirect_src; not a redirect
+				redirect_src = "";
+			int orig_w = rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_w, -1);
+			int orig_h = rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_h, -1);
+			int orig_size = rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_size, -1);
+			if (orig_size > ext_maxs[lnki_ext]) continue;
+			int lnki_page_id = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_page_id);
+			Xob_xfer_temp_tbl.Insert(trg_stmt, lnki_id, lnki_tier_id, lnki_page_id, orig_repo, orig_page_id, join_ttl, redirect_src, lnki_ext, Xop_lnki_type.Id_none, orig_media_type
+			, Bool_.Y									// orig is y
+			, orig_w, orig_h
+			, orig_w, orig_h							// file_w, file_h is same as orig_w,orig_h; i.e.: make same file_w as orig_w
+			, Xof_img_size.Null, Xof_img_size.Null		// html_w, html_h is -1; i.e.: will not be displayed in page at specific size (this matches logic in Xob_xfer_temp_cmd_thumb)
+			, Xof_lnki_time.Null
+			, Xof_lnki_page.Null
+			, 0);
+		}
+		conn.Txn_end();
+	}
+	private long[] Calc_ext_max() {
+		Xof_rule_grp ext_rules = wiki.Appe().File_mgr().Ext_rules().Get_or_new(ext_rules_key);
+		long[] rv = new long[Xof_ext_.Id__max];
+		for (int i = 0; i < Xof_ext_.Id__max; i++) {
+			byte[] ext = Xof_ext_.Get_ext_by_id_(i);
+			Xof_rule_itm ext_rule = ext_rules.Get_or_null(ext);
+			long max = ext_rule == null ? 0 : ext_rule.Make_max();
+			rv[i] = max;
+		}
+		return rv;
+	}
+	public void Cmd_run() {}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	private static final String
+		Sql_select = String_.Concat_lines_nl
+	(	"SELECT  DISTINCT"
+	,   "        l.lnki_id"
+//		,	",       lnki_ttl"
+	,	",       l.lnki_ext"
+	,	",       l.lnki_page_id"
+	,	",       o.orig_repo"
+	,	",       o.orig_page_id"
+//		,	",       orig_file_id"
+	,	",       o.orig_file_ttl"
+	,	",       o.orig_file_ext"
+	,	",       o.lnki_ttl"
+	,	",       o.orig_size"
+	,	",       o.orig_w"
+	,	",       o.orig_h"
+	,	",       o.orig_media_type"
+//		,	",       orig_bits"
+	,	"FROM    lnki_regy l"
+	,	"        JOIN orig_regy o ON o.lnki_ttl = l.lnki_ttl"
+	,	"WHERE   o.orig_file_ttl IS NOT NULL"
+	,	"ORDER BY o.orig_file_ttl DESC"
+	);
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_ext_rules_))			ext_rules_key = m.ReadBry("v");
+		else	return super.Invk (ctx, ikey, k, m);
+		return this;
+	}	private static final String Invk_ext_rules_ = "ext_rules_";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_cmd_thumb.java

@@ -0,0 +1,81 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.xowa.files.*;
+public class Xob_xfer_temp_cmd_thumb extends Xob_itm_basic_base implements Xob_cmd {
+	public Xob_xfer_temp_cmd_thumb(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_file_xfer_temp_thumb;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {
+		Db_conn conn = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
+		Xob_xfer_temp_tbl.Create_table(conn);
+		Db_stmt trg_stmt = Xob_xfer_temp_tbl.Insert_stmt(conn);
+		conn.Txn_bgn();
+		DataRdr rdr = conn.Exec_sql_as_rdr(Sql_select);
+		Xob_xfer_temp_itm temp_itm = new Xob_xfer_temp_itm();
+		Xof_img_size img_size = new Xof_img_size();
+		byte[] cur_ttl = Bry_.Empty; byte cur_repo = Byte_.Max_value_127;
+		while (rdr.MoveNextPeer()) {
+			temp_itm.Clear();
+			temp_itm.Load(rdr);
+			if (Bry_.Eq(cur_ttl, temp_itm.Orig_file_ttl())) {	// same ttl; DATE:2015-03-22
+				if (temp_itm.Orig_repo() != cur_repo)			// if repo is different, ignore 2nd; handles images in both repos; take 1st only (which should be local)
+					continue;
+			}
+			else {												// new ttl; update ttl, repo
+				cur_ttl = temp_itm.Orig_file_ttl();
+				cur_repo = temp_itm.Orig_repo();
+			}
+			if (temp_itm.Chk(img_size))
+				temp_itm.Insert(trg_stmt, img_size);
+		}
+		conn.Txn_end();
+	}
+	public void Cmd_run() {}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	private static final String
+		Sql_select = String_.Concat_lines_nl
+	(	"SELECT  l.lnki_id"
+	,	",       l.lnki_tier_id"
+	,	",       l.lnki_page_id"
+	,	",       l.lnki_ext"
+	,	",       l.lnki_type"
+	,	",       l.lnki_src_tid"
+	,	",       l.lnki_w"
+	,	",       l.lnki_h"
+	,	",       l.lnki_upright"
+	,	",       l.lnki_time"
+	,	",       l.lnki_page"
+	,	",       l.lnki_count"
+	,	",       o.orig_repo"
+	,	",       o.orig_page_id"
+	,	",       o.orig_file_ttl"
+	,	",       o.orig_file_ext"
+	,	",       o.orig_file_id"
+	,	",       o.lnki_ttl"
+	,	",       o.orig_w"
+	,	",       o.orig_h"
+	,	",       o.orig_media_type"
+	,	",       o.orig_minor_mime"
+	,	"FROM    lnki_regy l"
+	,	"        JOIN orig_regy o ON o.lnki_ttl = l.lnki_ttl"
+	,	"WHERE   o.orig_file_ttl IS NOT NULL"
+	,	"ORDER BY o.orig_file_ttl, o.orig_repo DESC, l.lnki_w DESC"	// NOTE: local=1,common=0; DATE:2015-03-22
+	);
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_itm.java

@@ -0,0 +1,131 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.xowa.files.*;
+class Xob_xfer_temp_itm {
+	public int Lnki_id() {return lnki_id;} private int lnki_id;
+	public int Lnki_tier_id() {return lnki_tier_id;} private int lnki_tier_id;
+	public byte Orig_repo() {return orig_repo;} private byte orig_repo;
+	public int Lnki_ext() {return lnki_ext;} private int lnki_ext;
+	public byte[] Orig_file_ttl() {return orig_file_ttl;} private byte[] orig_file_ttl;
+	public String Orig_media_type() {return orig_media_type;} private String orig_media_type;
+	public String Orig_minor_mime() {return orig_minor_mime;} private String orig_minor_mime;
+	public byte Orig_media_type_tid() {return orig_media_type_tid;} private byte orig_media_type_tid;
+	public int Orig_page_id() {return orig_page_id;} private int orig_page_id;
+	public String Join_ttl() {return join_ttl;} private String join_ttl;
+	public String Redirect_src() {return redirect_src;} private String redirect_src;
+	public byte Lnki_type() {return lnki_type;} private byte lnki_type;
+	public byte Lnki_src_tid() {return lnki_src_tid;} private byte lnki_src_tid;
+	public int Lnki_w() {return lnki_w;} private int lnki_w;
+	public int Lnki_h() {return lnki_h;} private int lnki_h;
+	public int Lnki_count() {return lnki_count;} private int lnki_count;
+	public int Lnki_page_id() {return lnki_page_id;} private int lnki_page_id;
+	public int Orig_w() {return orig_w;} private int orig_w;
+	public int Orig_h() {return orig_h;} private int orig_h;
+	public int Orig_ext_id() {return orig_ext_id;} private int orig_ext_id;
+	public double Lnki_upright() {return lnki_upright;} private double lnki_upright;
+	public double Lnki_thumbtime() {return lnki_thumbtime;} private double lnki_thumbtime;
+	public int Lnki_page() {return lnki_page;} private int lnki_page;
+	public void Clear() {
+		orig_file_ttl = null;
+		lnki_ext = lnki_type = lnki_src_tid
+				= orig_repo = orig_media_type_tid = Byte_.Max_value_127;
+		chk_tid = Chk_tid_none;
+		lnki_id = lnki_tier_id = lnki_w = lnki_h = lnki_count =  lnki_page_id
+				= orig_w = orig_h = orig_page_id = Int_.Neg1;
+		join_ttl =  redirect_src = orig_media_type = null;
+		lnki_upright = Xop_lnki_tkn.Upright_null;
+		lnki_thumbtime = Xof_lnki_time.Null;
+		lnki_page = Xof_lnki_page.Null;
+	}
+	public void Load(DataRdr rdr) {
+		lnki_id			= rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_id);
+		lnki_tier_id	= rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_tier_id);
+		lnki_page_id	= rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_page_id);
+		lnki_ext		= rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_ext);
+		lnki_type		= rdr.ReadByte(Xob_lnki_regy_tbl.Fld_lnki_type);
+		lnki_src_tid	= rdr.ReadByte(Xob_lnki_regy_tbl.Fld_lnki_src_tid);
+		lnki_w			= rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_w);
+		lnki_h			= rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_h);
+		lnki_upright	= rdr.ReadDouble(Xob_lnki_regy_tbl.Fld_lnki_upright);
+		lnki_thumbtime	= Xof_lnki_time.Db_load_double(rdr, Xob_lnki_regy_tbl.Fld_lnki_time);
+		lnki_page		= rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_page);
+		lnki_count		= rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_count);
+		orig_file_ttl	= rdr.ReadBryByStr(Xob_lnki_regy_tbl.Fld_lnki_ttl);
+		orig_repo		= rdr.ReadByte(Xob_orig_regy_tbl.Fld_orig_repo);
+		orig_page_id	= rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_page_id, -1);
+		join_ttl		= rdr.ReadStr(Xob_orig_regy_tbl.Fld_orig_file_ttl);
+		redirect_src	= rdr.ReadStr(Xob_orig_regy_tbl.Fld_lnki_ttl);
+		orig_w			= rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_w, -1);
+		orig_h			= rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_h, -1);
+		orig_media_type = rdr.ReadStrOr(Xob_orig_regy_tbl.Fld_orig_media_type, "");	// convert nulls to ""
+		orig_minor_mime = rdr.ReadStrOr(Xob_orig_regy_tbl.Fld_orig_minor_mime, "");	// convert nulls to ""
+		orig_ext_id		= rdr.ReadInt(Xob_orig_regy_tbl.Fld_orig_file_ext);
+	}
+	public static final byte 
+	  Chk_tid_none = 0
+	, Chk_tid_orig_page_id_is_null = 1
+	, Chk_tid_orig_media_type_is_audio = 2
+	, Chk_tid_ns_is_media = 3
+	, Chk_tid_orig_w_is_0 = 4
+	;
+	public byte Chk_tid() {return chk_tid;} private byte chk_tid;
+	public boolean Chk(Xof_img_size img_size) {
+		if (String_.Eq(join_ttl, redirect_src)) // join_ttl is same as redirect_src; not a redirect; EX:(direct) join="A.png";redirect_src="A.png"; (redirect) join="A.png";redirect_src="B.png" (i.e.: B redirects to A)
+			redirect_src = "";
+//			else {	// redirect; make sure extension matches; EX: A.png redirects to B.png; lnki_ext will be .png (the lnki's ext); should be .png (the actual file's ext)
+//				Xof_ext join_ext = Xof_ext_.new_by_ttl_(Bry_.new_u8(join_ttl));
+//				lnki_ext = join_ext.Id();
+//			}
+		lnki_ext = orig_ext_id;
+		orig_media_type_tid = Xof_media_type.Xto_byte(orig_media_type);
+		if (	Xof_lnki_time.Null_n(lnki_thumbtime)				// thumbtime defined
+			&&	orig_media_type_tid != Xof_media_type.Tid_video 	// video can have thumbtime
+			)
+			lnki_thumbtime = Xof_lnki_time.Null;					// set thumbtime to NULL; actually occurs for one file: [[File:Crash.arp.600pix.jpg|thumb|thumbtime=2]]
+		if (	Xof_lnki_page.Null_n(lnki_page)
+			&&	!Xof_ext_.Id_supports_page(orig_ext_id))			// djvu / pdf can have page parameters, which are currently being stored in thumbtime; DATE:2014-01-18
+			lnki_page = Xof_lnki_page.Null;
+		if (orig_page_id == -1) {	// no orig found (i.e.: not in local's / remote's image.sql);
+			chk_tid = Chk_tid_orig_page_id_is_null;
+			return false;
+		}
+		if (orig_media_type_tid == Xof_media_type.Tid_audio) {		// ignore: audio will never have thumbs
+			chk_tid = Chk_tid_orig_media_type_is_audio;
+			return false;
+		}
+		if (orig_w <= 0) {	// ignore files that have an orig_w of 0; note that ogg files that are sometimes flagged as VIDEO; EX:2009_10_08_Marc_Randazza_interview.ogg; DATE:2014-08-20
+			chk_tid = Chk_tid_orig_w_is_0;
+			return false;
+		}
+		if (lnki_ext == Xof_ext_.Id_mid) {	// NOTE: .mid does not have orig_media_type of "AUDIO"
+			chk_tid = Chk_tid_orig_media_type_is_audio;
+			return false;
+		}
+		if (lnki_src_tid == Xob_lnki_src_tid.Tid_media) {
+			chk_tid = Chk_tid_ns_is_media;
+			return false;
+		}
+		int upright_patch = Xof_patch_upright_tid_.Tid_all;	// all future blds will have upright_patch
+		img_size.Html_size_calc(Xof_exec_tid.Tid_wiki_page, lnki_w, lnki_h, lnki_type, upright_patch, lnki_upright, lnki_ext, orig_w, orig_h, Xof_img_size.Thumb_width_img);
+		return true;
+	}
+	public void Insert(Db_stmt stmt, Xof_img_size img_size) {
+		Xob_xfer_temp_tbl.Insert(stmt, lnki_id, lnki_tier_id, lnki_page_id, orig_repo, orig_page_id, join_ttl, redirect_src, lnki_ext, lnki_type, orig_media_type, img_size.File_is_orig(), orig_w, orig_h, img_size.File_w(), img_size.File_h(), img_size.Html_w(), img_size.Html_h(), lnki_thumbtime, lnki_page, lnki_count);
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_itm_tst.java

@@ -0,0 +1,192 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import org.junit.*;
+import gplx.stores.*; import gplx.xowa.files.*; import gplx.xowa.files.repos.*;
+public class Xob_xfer_temp_itm_tst {
+	private Xob_xfer_temp_itm_fxt fxt = new Xob_xfer_temp_itm_fxt();
+	@Before public void init() {fxt.Reset();}
+	@Test   public void Pass()				{fxt.Test_pass().Test_itm_chk_fail_id_none();}
+	@Test   public void Missing_orig()		{fxt.Test_fail(Xob_xfer_temp_itm.Chk_tid_orig_page_id_is_null		, KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_page_id, null));}
+	@Test   public void File_is_audio()		{fxt.Test_fail(Xob_xfer_temp_itm.Chk_tid_orig_media_type_is_audio	, KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_media_type, Xof_media_type.Name_audio));}
+	@Test   public void File_is_mid()		{
+		fxt.Test_fail(Xob_xfer_temp_itm.Chk_tid_orig_media_type_is_audio	, KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext, Xof_ext_.Id_mid));
+	}
+	@Test   public void Redirect_src_is_empty() {	// orig_cmd sets all direct files to have "orig_join" == "lnki_ttl"
+		fxt.Test_bgn
+		(	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_file_ttl	, "A.png")
+		,	KeyVal_.new_(Xob_orig_regy_tbl.Fld_lnki_ttl			, "A.png")
+		);
+		fxt.Test_lnki_redirect_src("");								// confirm redirect_src set to ""
+	}
+	@Test   public void Redirect_src_has_val() {	// orig_cmd sets all redirect files to have "orig_join" = redirect and "lnki_ttl" as orig; EX: A.png redirects to B.png; orig_join will be B.png (the actual image) and redirect_src will be A.png (the original lnki)
+		fxt.Test_bgn
+		(	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_file_ttl	, "B.png")
+		,	KeyVal_.new_(Xob_orig_regy_tbl.Fld_lnki_ttl			, "A.png")
+		);
+		fxt.Test_lnki_redirect_src("A.png");							// confirm redirect_src set to ""
+	}
+	@Test   public void Redirect_should_take_trg_ext() {// if "A.png" redirects to "B.jpg", ext_id should be ".jpg" (the actual file) not ".png (lnki_ext_id)
+		fxt.Test_bgn
+		(	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_file_ttl	, "B.jpg")
+		,	KeyVal_.new_(Xob_orig_regy_tbl.Fld_lnki_ttl			, "A.png")
+		,	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext	, Xof_ext_.Id_jpg)	// .png b/c B.jpg
+		);
+		fxt.Test_lnki_ext_id(Xof_ext_.Id_jpg);						// confirm ext changed to .jpg
+	}
+	@Test   public void Thumbtime_check() {// PURPOSE: one image actually had a thumbtime defined; EX: General_Dynamics_F-16_Fighting_Falcon; [[File:Crash.arp.600pix.jpg|thumb|thumbtime=2]]
+		fxt.Test_bgn
+		(	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext	, Xof_ext_.Id_jpg)
+		,	KeyVal_.new_(Xob_lnki_regy_tbl.Fld_lnki_time		, (double)3)
+		);
+		fxt.Test_lnki_thumbtime(Xof_lnki_time.Null);
+
+		fxt.Reset().Test_bgn
+		(	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_media_type	, Xof_media_type.Name_video)
+		,	KeyVal_.new_(Xob_lnki_regy_tbl.Fld_lnki_time		, (double)3)
+		);
+		fxt.Test_lnki_thumbtime(3);
+	}
+	@Test   public void Page_check() {
+		fxt.Test_bgn
+		(	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext	, Xof_ext_.Id_jpg)
+		,	KeyVal_.new_(Xob_lnki_regy_tbl.Fld_lnki_page		, 3)
+		);
+		fxt.Test_lnki_page(Xof_lnki_page.Null);
+
+		fxt.Reset().Test_bgn
+		(	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext	, Xof_ext_.Id_pdf)
+		,	KeyVal_.new_(Xob_lnki_regy_tbl.Fld_lnki_page		, 3)
+		);
+		fxt.Test_lnki_page(3);
+
+		fxt.Reset().Test_bgn
+		(	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext	, Xof_ext_.Id_djvu)
+		,	KeyVal_.new_(Xob_lnki_regy_tbl.Fld_lnki_page		, 3)
+		);
+		fxt.Test_lnki_page(3);
+	}
+	@Test   public void Media_should_be_ignored() {// ignore [[Media:]] for xfer_thumb (needed for xfer_orig)
+		fxt.Test_bgn
+		(	KeyVal_.new_(Xob_orig_regy_tbl.Fld_lnki_ttl			, "A.png")
+		,	KeyVal_.new_(Xob_lnki_regy_tbl.Fld_lnki_src_tid		, Xob_lnki_src_tid.Tid_media)
+		);
+		fxt.Test_itm_chk_fail_id(Xob_xfer_temp_itm.Chk_tid_ns_is_media);
+	}
+	@Test   public void Orig_width_is_0() {// PURPOSE: ignore files with an orig width of 0; note that ogg files that are sometimes flagged as VIDEO; EX:2009_10_08_Marc_Randazza_interview.ogg; DATE:2014-08-20
+		fxt.Test_bgn
+		(	KeyVal_.new_(Xob_orig_regy_tbl.Fld_lnki_ttl			, "A.ogg")
+		,	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_media_type	, Xof_media_type.Name_video)	// VIDEO
+		,	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_w			, 0)							// no width defined in image table
+		,	KeyVal_.new_(Xob_orig_regy_tbl.Fld_orig_h			, 0)
+		);
+		fxt.Test_itm_chk_fail_id(Xob_xfer_temp_itm.Chk_tid_orig_w_is_0);
+	}
+}
+class Xob_xfer_temp_itm_fxt {
+	private Xob_xfer_temp_itm itm = new Xob_xfer_temp_itm();
+	private Xof_img_size img_size = new Xof_img_size();
+	private DataRdr_mem rdr;
+	private GfoNde nde;
+	public static String[] Flds = new String[] 
+	{ Xob_lnki_regy_tbl.Fld_lnki_ext
+	, Xob_lnki_regy_tbl.Fld_lnki_id
+	, Xob_lnki_regy_tbl.Fld_lnki_tier_id
+	, Xob_orig_regy_tbl.Fld_orig_repo
+	, Xob_orig_regy_tbl.Fld_orig_file_ttl
+	, Xob_orig_regy_tbl.Fld_orig_file_ext
+	, Xob_orig_regy_tbl.Fld_lnki_ttl
+	, Xob_lnki_regy_tbl.Fld_lnki_type
+	, Xob_lnki_regy_tbl.Fld_lnki_src_tid
+	, Xob_lnki_regy_tbl.Fld_lnki_w
+	, Xob_lnki_regy_tbl.Fld_lnki_h
+	, Xob_lnki_regy_tbl.Fld_lnki_count
+	, Xob_lnki_regy_tbl.Fld_lnki_page_id
+	, Xob_orig_regy_tbl.Fld_orig_w
+	, Xob_orig_regy_tbl.Fld_orig_h
+	, Xob_orig_regy_tbl.Fld_orig_page_id
+	, Xob_lnki_regy_tbl.Fld_lnki_upright
+	, Xob_lnki_regy_tbl.Fld_lnki_time
+	, Xob_lnki_regy_tbl.Fld_lnki_page
+	, Xob_orig_regy_tbl.Fld_orig_media_type
+	, Xob_orig_regy_tbl.Fld_orig_minor_mime
+	}
+	;
+	public Xob_xfer_temp_itm_fxt Reset() {
+		itm.Clear();
+		return this;
+	}
+	public Xob_xfer_temp_itm_fxt Init_rdr_image() {
+		GfoFldList flds = GfoFldList_.str_(Flds);
+		nde = GfoNde_.vals_(flds, Object_.Ary
+		( Xof_ext_.Id_png, 1, 1, Xof_repo_itm_.Repo_remote
+		, "A.png", Xof_ext_.Id_png, "A.png", Xop_lnki_type.Id_thumb, Xob_lnki_src_tid.Tid_file
+		, 220, 200, 1, 2, 440, 400, 3
+		, Xop_lnki_tkn.Upright_null, Xof_lnki_time.Null, Xof_lnki_page.Null
+		, Xof_media_type.Name_bitmap, "png"
+		));
+		GfoNdeList subs = GfoNdeList_.new_();
+		subs.Add(nde);
+		GfoNde root = GfoNde_.root_(nde);
+		rdr = DataRdr_mem.new_(root, flds, subs);
+		rdr.MoveNextPeer();
+		return this;
+	}
+	public Xob_xfer_temp_itm_fxt Init_rdr(String key, Object val) {
+		nde.Write(key, val);
+		return this;
+	}
+	public Xob_xfer_temp_itm_fxt Test_pass() {return Test_bgn();}
+	public Xob_xfer_temp_itm_fxt Test_fail(byte fail_tid, KeyVal... kvs) {
+		Test_bgn(kvs);
+		this.Test_itm_chk_fail_id(fail_tid);
+		return this;
+	}
+	public Xob_xfer_temp_itm_fxt Test_bgn(KeyVal... kvs) {
+		Init_rdr_image();
+		int len = kvs.length;
+		for (int i = 0; i < len; i++) {
+			KeyVal kv = kvs[i];
+			Init_rdr(kv.Key(), kv.Val());
+		}
+		this.Exec_load();
+		this.Exec_chk();
+		return this;
+	}
+	public Xob_xfer_temp_itm_fxt Test_atr(String key, Object val) {
+		Tfds.Eq(rdr.Read(key), val);
+		return this;
+	}
+	public Xob_xfer_temp_itm_fxt Exec_load() {
+		itm.Load(rdr);
+		return this;
+	}
+	public Xob_xfer_temp_itm_fxt Exec_chk() {
+		itm.Chk(img_size);
+		return this;
+	}
+	public Xob_xfer_temp_itm_fxt Test_lnki_ext_id(int expd) {Tfds.Eq(expd, itm.Lnki_ext()); return this;}
+	public Xob_xfer_temp_itm_fxt Test_lnki_thumbtime(double expd) {Tfds.Eq(expd, itm.Lnki_thumbtime()); return this;}
+	public Xob_xfer_temp_itm_fxt Test_lnki_page(int expd) {Tfds.Eq(expd, itm.Lnki_page()); return this;}
+	public Xob_xfer_temp_itm_fxt Test_lnki_redirect_src(String expd) {Tfds.Eq(expd, itm.Redirect_src()); return this;}
+	public Xob_xfer_temp_itm_fxt Test_itm_chk_fail_id_none() {return Test_itm_chk_fail_id(Xob_xfer_temp_itm.Chk_tid_none);}
+	public Xob_xfer_temp_itm_fxt Test_itm_chk_fail_id(byte expd) {
+		Tfds.Eq(expd, itm.Chk_tid());
+		return this;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_tbl.java

@@ -0,0 +1,79 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.dbs.*; import gplx.xowa.files.*;
+class Xob_xfer_temp_tbl {
+	public static void Create_table(Db_conn p)		{Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
+	public static Db_stmt Insert_stmt(Db_conn p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_lnki_id, Fld_lnki_tier_id, Fld_lnki_page_id, Fld_orig_repo, Fld_orig_page_id, Fld_lnki_ttl, Fld_orig_redirect_src, Fld_lnki_ext, Fld_lnki_type, Fld_orig_media_type, Fld_file_is_orig, Fld_orig_w, Fld_orig_h, Fld_file_w, Fld_file_h, Fld_html_w, Fld_html_h, Fld_lnki_time, Fld_lnki_page, Fld_lnki_count);}
+	public static void Insert(Db_stmt stmt, int lnki_id, int lnki_tier_id, int lnki_page_id, byte repo_id, int page_id, String ttl, String redirect_src, int ext_id, byte lnki_type, String orig_media_type, boolean file_is_orig, int orig_w, int orig_h, int html_w, int html_h, int file_w, int file_h, double thumbtime, int page, int count) {
+		stmt.Clear()
+		.Val_int(lnki_id)
+		.Val_int(lnki_tier_id)
+		.Val_int(lnki_page_id)
+		.Val_byte(repo_id)
+		.Val_int(page_id)
+		.Val_str(ttl)
+		.Val_str(redirect_src)
+		.Val_int(ext_id)
+		.Val_byte(lnki_type)
+		.Val_str(orig_media_type)
+		.Val_bool_as_byte(file_is_orig)
+		.Val_int(orig_w)
+		.Val_int(orig_h)
+		.Val_int(file_w)
+		.Val_int(file_h)
+		.Val_int(html_w)
+		.Val_int(html_h)
+		.Val_double(Xof_lnki_time.Db_save_double(thumbtime))
+		.Val_int(page)
+		.Val_int(count)
+		.Exec_insert();
+	}
+	public static final String Tbl_name = "xfer_temp"		
+	, Fld_lnki_id = "lnki_id", Fld_lnki_tier_id = "lnki_tier_id", Fld_lnki_page_id = "lnki_page_id", Fld_lnki_ttl = "lnki_ttl", Fld_lnki_ext = "lnki_ext", Fld_lnki_type = "lnki_type"
+	, Fld_lnki_time = "lnki_time", Fld_lnki_page = "lnki_page", Fld_lnki_count = "lnki_count"
+	, Fld_orig_repo = "orig_repo", Fld_orig_page_id = "orig_page_id", Fld_orig_redirect_src = "orig_redirect_src", Fld_orig_media_type = "orig_media_type"
+	, Fld_orig_w = "orig_w", Fld_orig_h = "orig_h"
+	, Fld_file_w = "file_w", Fld_file_h = "file_h", Fld_file_is_orig = "file_is_orig"
+	, Fld_html_w = "html_w", Fld_html_h = "html_h"
+	;
+	private static final String Tbl_sql = String_.Concat_lines_nl
+	(	"CREATE TABLE IF NOT EXISTS xfer_temp"
+	,	"( lnki_id             integer             NOT NULL			    PRIMARY KEY"
+	,	", lnki_tier_id         integer             NOT NULL"
+	,	", lnki_page_id        integer             NOT NULL"
+	,	", lnki_ttl            varchar(255)        NOT NULL"
+	,	", lnki_ext            integer             NOT NULL"
+	,	", lnki_type           integer             NOT NULL"
+	,	", lnki_time           double              NOT NULL"
+	,	", lnki_page           integer             NOT NULL"
+	,	", lnki_count          integer             NOT NULL"
+	,	", orig_repo           integer             NOT NULL"
+	,	", orig_page_id        integer             NOT NULL"
+	,	", orig_redirect_src   varchar(255)        NOT NULL"
+	,	", orig_media_type     varchar(64)         NOT NULL"
+	,	", orig_w              integer             NOT NULL"
+	,	", orig_h              integer             NOT NULL"
+	,	", file_is_orig        tinyint             NOT NULL"
+	,	", file_w              integer             NOT NULL"
+	,	", file_h              integer             NOT NULL"
+	,	", html_w              integer             NOT NULL"
+	,	", html_h              integer             NOT NULL"
+	,	");"
+	);
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_update_cmd.java

@@ -0,0 +1,83 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.dbs.*; import gplx.xowa.files.*; import gplx.xowa.bldrs.cmds.files.*;
+public class Xob_xfer_update_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	private Io_url prv_url;
+	public Xob_xfer_update_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_file_xfer_update;}
+	public void Cmd_run() {
+		// init vars
+		Xob_db_file cur_file = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir());
+		Db_conn conn = cur_file.Conn();
+		if (prv_url == null) {
+			prv_url = wiki.Appe().Fsys_mgr().File_dir().GenSubFil_nest(wiki.Domain_str(), "bldr", Xob_db_file.Name__file_make);
+		}
+
+		// run sql
+		Sqlite_engine_.Tbl_rename(conn, "xfer_regy", "xfer_regy_old");
+		Xob_xfer_regy_tbl.Create_table(conn);
+		Sqlite_engine_.Db_attach(conn, "old_db", prv_url.Raw());
+		conn.Exec_sql(Sql_update);
+		Sqlite_engine_.Db_detach(conn, "old_db");
+		Sqlite_engine_.Tbl_delete(conn, "xfer_regy_old");
+		Xob_xfer_regy_tbl.Create_index(usr_dlg, conn);
+
+//			// rotate db
+//			DateAdp wiki_date = wiki.Db_mgr().Dump_date_query();
+//			Io_url archive_url = prv_url.GenNewNameOnly("oimg_lnki_" + wiki_date.XtoStr_fmt("yyyyMMdd"));
+//			Io_mgr.I.CopyFil(cur_file.Url(), archive_url, true);
+//			Io_mgr.I.CopyFil(cur_file.Url(), prv_url, true);
+	}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	public static final String Sql_update = String_.Concat_lines_nl
+	( "INSERT INTO xfer_regy"
+	, "SELECT  cur.lnki_id"
+	, ",       cur.orig_page_id"
+	, ",       cur.orig_repo"
+	, ",       cur.lnki_ttl"
+	, ",       cur.orig_redirect_src"
+	, ",       cur.lnki_ext"
+	, ",       cur.orig_media_type"
+	, ",       cur.file_is_orig"
+	, ",       cur.orig_w"
+	, ",       cur.orig_h"
+	, ",       cur.file_w"
+	, ",       cur.file_h"
+	, ",       cur.lnki_time"
+	, ",       cur.lnki_count"
+	, ",       CASE"
+	, "          WHEN old.lnki_ttl IS NULL THEN"	// not in old table; mark todo
+	, "            " + Byte_.Xto_str(Xob_xfer_regy_tbl.Status_todo)
+	, "          ELSE"									// in old table; mark processed
+	, "            " + Byte_.Xto_str(Xob_xfer_regy_tbl.Status_ignore_processed)
+	, "        END"
+	, ",       cur.xfer_bin_tid"
+	, ",       cur.xfer_bin_msg"
+	, "FROM    xfer_regy_old cur"
+	, "        LEFT JOIN old_db.xfer_regy old ON cur.lnki_ttl = old.lnki_ttl AND cur.file_w = old.file_w"
+	);
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_previous_url_))				prv_url = m.ReadIoUrl("v");
+		else	return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}	private static final String Invk_previous_url_ = "previous_url_";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/files/Xobu_poll_mgr.java

@@ -0,0 +1,38 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+public class Xobu_poll_mgr implements GfoInvkAble {
+	public Xobu_poll_mgr(Xoae_app app) {this.app = app;} private Xoae_app app;
+	public int Poll_interval() {return poll_interval;} private int poll_interval = 1000;
+	private Io_url poll_file;
+	public void Poll() {
+		if (poll_file == null) poll_file = app.Fsys_mgr().Root_dir().GenSubFil("bldr_poll.gfs");
+		if (!Io_mgr.I.ExistsFil(poll_file)) return; // file doesn't exist
+		String poll_text = Io_mgr.I.LoadFilStr(poll_file);
+		Io_mgr.I.DeleteFil(poll_file);
+		app.Usr_dlg().Note_many("", "", "poll file found: ~{0}", poll_file.Raw());
+		app.Gfs_mgr().Run_str(poll_text);
+	}
+	public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_poll_interval_))		poll_interval = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_poll_file_))			poll_file = m.ReadIoUrl("v");
+		else	return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}
+	private static final String Invk_poll_interval_ = "poll_interval_", Invk_poll_file_ = "poll_file_";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/Xob_init_base.java

@@ -0,0 +1,59 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.xowa.wikis.*; import gplx.xowa.xtns.wdatas.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.xmls.*;
+import gplx.xowa.bldrs.css.*;
+public abstract class Xob_init_base implements Xob_cmd, GfoInvkAble {
+	private Xob_bldr bldr; private Xowe_wiki wiki; private Gfo_usr_dlg usr_dlg;
+	private byte wbase_enabled = Bool_.__byte;
+	public Xob_init_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.bldr = bldr; this.wiki = wiki; this.usr_dlg = wiki.Appe().Usr_dlg(); return this;}
+	public abstract String Cmd_key();
+	public abstract void Cmd_ini_wdata(Xob_bldr bldr, Xowe_wiki wiki);
+	public abstract void Cmd_run_end(Xowe_wiki wiki);
+	public void Cmd_init(Xob_bldr bldr) {		// add other cmds; EX: wikidata
+		bldr.Import_marker().Bgn(wiki);
+		if (wbase_enabled == Bool_.__byte) wbase_enabled = wiki.Domain_tid() == Xow_domain_type_.Tid_wikidata ? Bool_.Y_byte : Bool_.N_byte;	// if wbase_enabled not explicitly set, set it to y if wiki is "www.wikidata.org"
+		if (wbase_enabled == Bool_.Y_byte)		// if wbase_enabled, auto-add wdata_wkrs bldr
+			this.Cmd_ini_wdata(bldr, wiki);
+	}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run() {						// parse site_info
+		gplx.ios.Io_stream_rdr src_rdr = wiki.Import_cfg().Src_rdr();
+		usr_dlg.Plog_many("", "", "reading dump header: ~{0}", src_rdr.Url().Raw());
+		byte[] siteinfo_xml = Xob_siteinfo_parser.Siteinfo_extract(src_rdr);
+		Xob_siteinfo_parser.Siteinfo_parse(wiki, usr_dlg, String_.new_u8(siteinfo_xml));
+		this.Cmd_run_end(wiki);					// save site info
+	}
+	public void Cmd_end() {
+		wiki.Appe().Gui_mgr().Html_mgr().Portal_mgr().Wikis().Itms_reset();	// dirty wiki list so that next refresh will load itm
+		if (wiki.Appe().Setup_mgr().Dump_mgr().Css_wiki_update()) {
+			Io_url url = wiki.Appe().Fsys_mgr().Wiki_css_dir(wiki.Domain_str()).GenSubFil(Xoa_css_extractor.Css_wiki_name);
+			usr_dlg.Log_many("", "", "deleting css: ~{0}", url.Raw());
+			Io_mgr.I.DeleteFil_args(url).MissingFails_off().Exec();
+		}
+	}
+	public void Cmd_term() {}
+	@gplx.Virtual public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_src_xml_fil_))				wiki.Import_cfg().Src_fil_xml_(m.ReadIoUrl("v"));
+		else if	(ctx.Match(k, Invk_src_bz2_fil_))				wiki.Import_cfg().Src_fil_bz2_(m.ReadIoUrl("v"));
+		else if	(ctx.Match(k, Invk_wdata_enabled_))				wbase_enabled = m.ReadYn("v") ? Bool_.Y_byte : Bool_.N_byte;
+		else if	(ctx.Match(k, Invk_owner))						return bldr.Cmd_mgr();
+		else return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}	private static final String Invk_src_xml_fil_ = "src_xml_fil_", Invk_src_bz2_fil_ = "src_bz2_fil_", Invk_owner = "owner", Invk_wdata_enabled_ = "wdata_enabled_";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/Xob_search_base.java

@@ -0,0 +1,109 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.core.primitives.*;
+import gplx.ios.*;
+import gplx.xowa.wikis.data.*; import gplx.xowa.dbs.*; import gplx.xowa.tdbs.*; import gplx.xowa.wikis.data.tbls.*;
+public abstract class Xob_search_base extends Xob_itm_dump_base implements Xobd_wkr, GfoInvkAble {
+	private final Ordered_hash list = Ordered_hash_.new_(); private Xol_lang lang;
+	public abstract String Wkr_key();
+	public abstract Io_make_cmd Make_cmd_site();
+	public void Wkr_ini(Xob_bldr bldr) {}
+	public void Wkr_bgn(Xob_bldr bldr) {
+		make_dir = wiki.Tdb_fsys_mgr().Ns_dir();
+		this.Init_dump(this.Wkr_key(), make_dir);
+		lang = wiki.Lang(); // wiki.Appe().Lang_mgr().Lang_en();	// NOTE: was .Lang_en which is wrong (should match lang of wiki); DATE:2013-05-11
+		tmp_wtr_mgr = new Xob_tmp_wtr_mgr(new Xob_tmp_wtr_wkr__ttl(temp_dir, dump_fil_len));
+		if (wiki.Db_mgr().Tid() == Xodb_mgr_sql.Tid_sql)	// if sqlite, hard-code to ns_main; aggregates all ns into one
+			ns_main = wiki.Ns_mgr().Ns_main();
+	}	private Xob_tmp_wtr_mgr tmp_wtr_mgr; private Xow_ns ns_main;
+	public void Wkr_run(Xowd_page_itm page) {
+//			if (page.Ns_id() != Xow_ns_.Id_main) return; // limit to main ns for now
+		try {
+			byte[] ttl = page.Ttl_page_db();
+			byte[][] words = Split_ttl_into_words(lang, list, dump_bfr, ttl);
+			Xob_tmp_wtr wtr = tmp_wtr_mgr.Get_or_new(ns_main == null ? page.Ns() : ns_main);
+			int words_len = words.length;
+			int row_len = 0;
+			for (int i = 0; i < words_len; i++) {
+				byte[] word = words[i];	
+				row_len += word.length + 13;	// 13=5(id) + 5(page_len) + 3(dlms)
+			}
+			if (wtr.FlushNeeded(row_len)) wtr.Flush(bldr.Usr_dlg());
+			for (int i = 0; i < words_len; i++) {
+				byte[] word = words[i];				
+				wtr.Bfr()	.Add(word)								.Add_byte(Byte_ascii.Pipe)
+							.Add_base85_len_5(page.Id())			.Add_byte(Byte_ascii.Semic)
+							.Add_base85_len_5(page.Text().length)	.Add_byte(Byte_ascii.Nl);
+			}
+		} catch (Exception e) {bldr.Usr_dlg().Warn_many("", "", "search_index:fatal error: err=~{0}", Err_.Message_gplx_brief(e));}	// never let single page crash entire import
+	}
+	public void Wkr_end() {
+		tmp_wtr_mgr.Flush_all(bldr.Usr_dlg());
+		dump_bfr.ClearAndReset();
+		Xobdc_merger.Ns(bldr.Usr_dlg(), tmp_wtr_mgr.Regy(), Xotdb_dir_info_.Name_search_ttl, temp_dir, make_dir, sort_mem_len, Io_line_rdr_key_gen_.first_pipe, this.Make_cmd_site());
+		tmp_wtr_mgr.Rls_all();
+		if (delete_temp) Io_mgr.I.DeleteDirDeep(temp_dir);
+	}
+	public void Wkr_print() {}
+	// private static final int row_fixed_len = 5 + 1 + 1 + 1;	// 5=rowId; 1=|; 1=NmsOrd; 1=|		
+	public static byte[][] Split_ttl_into_words(Xol_lang lang, Ordered_hash list, Bry_bfr bfr, byte[] ttl) {
+		if (lang != null)	// null lang passed in by searcher
+			ttl = lang.Case_mgr().Case_build_lower(ttl);
+		int ttl_len = ttl.length; Bry_obj_ref word_ref = Bry_obj_ref.new_(Bry_.Empty);
+		int i = 0; boolean word_done = false;
+		while (true) {
+			if (word_done || i == ttl_len) {
+				if (bfr.Len() > 0) {
+					byte[] word = bfr.Xto_bry_and_clear();
+					word_ref.Val_(word);
+					if (!list.Has(word_ref)) list.Add(word_ref, word);	// don't add same word twice; EX: Title of "Can Can" should only have "Can" in index
+				}
+				if (i == ttl_len) break;
+				word_done = false;
+			}
+			byte b = ttl[i];
+			switch (b) {
+				case Byte_ascii.Underline:	// underline is word-breaking; EX: A_B -> A, B
+				case Byte_ascii.Space:		// should not occur, but just in case (only underscores)
+				case Byte_ascii.Tab: case Byte_ascii.Nl: case Byte_ascii.Cr:	// should not occur in titles, but just in case
+
+				case Byte_ascii.Dash:	// treat hypenated words separately
+				case Byte_ascii.Dot:	// treat abbreviations as separate words; EX: A.B.C.
+				case Byte_ascii.Bang: case Byte_ascii.Hash: case Byte_ascii.Dollar: case Byte_ascii.Percent:
+				case Byte_ascii.Amp: case Byte_ascii.Paren_bgn: case Byte_ascii.Paren_end: case Byte_ascii.Star:
+				case Byte_ascii.Comma: case Byte_ascii.Slash:
+				case Byte_ascii.Colon: case Byte_ascii.Semic: case Byte_ascii.Gt:
+				case Byte_ascii.Question: case Byte_ascii.At: case Byte_ascii.Brack_bgn: case Byte_ascii.Brack_end:
+				case Byte_ascii.Pow: case Byte_ascii.Tick:
+				case Byte_ascii.Curly_bgn: case Byte_ascii.Pipe: case Byte_ascii.Curly_end: case Byte_ascii.Tilde:
+				case Byte_ascii.Quote:	case Byte_ascii.Apos: // FUTURE: apos will split "Earth's" to Earth and s; should remove latter
+					++i;
+					word_done = true;
+					break;
+				default:
+					bfr.Add_byte(b);
+					++i;
+					break;
+			}
+		}
+		byte[][] rv = (byte[][])list.To_ary(byte[].class);
+		list.Clear(); list.Resize_bounds(16);
+		return rv;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/Xob_term_base.java

@@ -0,0 +1,41 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.xowa.bldrs.xmls.*; import gplx.xowa.xtns.wdatas.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.dbs.*;	
+public abstract class Xob_term_base implements Xob_cmd, GfoInvkAble {
+	public Xob_term_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.wiki = wiki; return this;} private Xowe_wiki wiki;
+	public abstract String Cmd_key();
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run() {}
+	public void Cmd_end() {
+		Xoae_app app = wiki.Appe();
+		app.Gui_mgr().Html_mgr().Portal_mgr().Wikis().Itms_reset();	// NOTE: dirty wiki list so that next refresh will load itm			
+		app.Free_mem(false);	// clear cache, else import will load new page with old items from cache; DATE:2013-11-21
+		wiki.Props().Main_page_update(wiki);
+		app.Bldr().Import_marker().End(wiki);
+		wiki.Init_needed_(true);// flag init_needed prior to show; dir_info will show page_txt instead of page_gz;
+		wiki.Init_assert();	// force load; needed to pick up MediaWiki ns for MediaWiki:mainpage
+		Cmd_end_hook();
+	}
+	public abstract void Cmd_end_hook();
+	public void Cmd_term() {}
+	public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		return this;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_css_cmd.java

@@ -0,0 +1,52 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.xowa.bldrs.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.html.css.*;
+public class Xob_css_cmd implements Xob_cmd {
+	private final Xob_bldr bldr; private final Xowe_wiki wiki; private final Gfo_usr_dlg usr_dlg;
+	private Io_url css_dir; private String css_key;
+	public Xob_css_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.bldr = bldr; this.wiki = wiki; this.usr_dlg = wiki.Appe().Usr_dlg();}
+	public String Cmd_key() {return Xob_cmd_keys.Key_text_css;}
+	public void Cmd_init(Xob_bldr bldr) {
+		if (css_dir == null) css_dir = wiki.App().Fsys_mgr().Wiki_css_dir(wiki.Domain_str());	// EX: /xowa/user/anonymous/wiki/en.wikipedia.org
+		if (css_key == null) css_key = Xowd_css_core_mgr.Key_default;
+	}
+	public void Cmd_run() {
+		usr_dlg.Plog_many("", "", Cmd_key() + ":bgn;");			
+		bldr.App().Html__css_installer().Install(wiki, null);				// download from wmf
+		usr_dlg.Plog_many("", "", Cmd_key() + ":css_dir; dir=~{0}", css_dir.Raw());
+		wiki.Init_db_mgr();	// NOTE: must follow Install b/c Init_assert also calls Install; else will download any css from db
+		Xowd_db_file core_db = wiki.Db_mgr_as_sql().Core_data_mgr().Db__core();
+		core_db.Conn().Txn_bgn();
+		core_db.Tbl__css_core().Create_tbl();
+		core_db.Tbl__css_file().Create_tbl();
+		gplx.xowa.html.css.Xowd_css_core_mgr.Set(core_db.Tbl__css_core(), core_db.Tbl__css_file(), css_dir, css_key);
+		core_db.Tbl__cfg().Upsert_yn(Xow_cfg_consts.Grp__wiki_schema, Xowd_db_file_schema_props.Key__tbl_css_core, Bool_.Y);
+		core_db.Conn().Txn_end();
+		usr_dlg.Plog_many("", "", Cmd_key() + ":end;");
+	}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_css_dir_))		css_dir = m.ReadIoUrl("v");
+		else if	(ctx.Match(k, Invk_css_key_))		css_key = m.ReadStr("v");
+		else	return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}	private static final String Invk_css_dir_ = "css_dir_", Invk_css_key_ = "css_key_";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_init_cmd.java

@@ -0,0 +1,33 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.xowa.bldrs.*; import gplx.xowa.apis.xowa.bldrs.imports.*;
+import gplx.xowa.xtns.wdatas.imports.*;
+public class Xob_init_cmd extends Xob_init_base {
+	public Xob_init_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Ctor(bldr, wiki);}
+	@Override public String Cmd_key() {return Xob_cmd_keys.Key_text_init;}
+	@Override public void Cmd_ini_wdata(Xob_bldr bldr, Xowe_wiki wiki) {
+		bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_wbase_qid);
+		bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_wbase_pid);
+	}
+	@Override public void Cmd_run_end(Xowe_wiki wiki) {
+		if (gplx.xowa.wikis.Xow_fsys_mgr.Find_core_fil(wiki) != null)
+			throw wiki.Appe().Bldr().Usr_dlg().Fail_many("", "", "directory must not contain any .xowa or .sqlite3 files: dir=~{0}", wiki.Fsys_mgr().Root_dir().Raw());
+		Xowe_wiki_bldr.Create(wiki, wiki.Import_cfg().Src_rdr_len(), wiki.Import_cfg().Src_fil().NameOnly());
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_ns_to_db_wkr__text.java

@@ -0,0 +1,30 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
+public class Xob_ns_to_db_wkr__text implements Xob_ns_to_db_wkr {
+	public byte Db_tid() {return Xowd_db_file_.Tid_text;}
+	public void Tbl_init(Xowd_db_file db) {
+		Xowd_text_tbl tbl = db.Tbl__text();
+		tbl.Create_tbl();
+		tbl.Insert_bgn();
+	}
+	public void Tbl_term(Xowd_db_file db) {
+		db.Tbl__text().Insert_end(); 
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_page_cmd.java

@@ -0,0 +1,103 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.dbs.*; import gplx.ios.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.wikis.*;
+import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.dbs.*; 
+import gplx.xowa.wikis.*; import gplx.xowa.bldrs.filters.dansguardians.*; import gplx.xowa.apis.xowa.bldrs.imports.*;	
+public class Xob_page_cmd extends Xob_itm_basic_base implements Xobd_wkr, GfoInvkAble {
+	private Xowd_db_mgr db_mgr; private Db_idx_mode idx_mode = Db_idx_mode.Itm_end; private Xowd_page_tbl page_core_tbl; private Io_stream_zip_mgr text_zip_mgr; private byte text_zip_tid;
+	private Xop_redirect_mgr redirect_mgr; private Xob_redirect_tbl redirect_tbl; private boolean redirect_id_enabled;
+	private DateAdp modified_latest = DateAdp_.MinValue; private int page_count_all, page_count_main = 0; private int commit_interval = 100000;	// 100 k				
+	private Dg_match_mgr dg_match_mgr; private final Xow_page_mgr page_mgr; private Xob_ns_to_db_mgr ns_to_db_mgr; 
+	public Xob_page_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.page_mgr = wiki.Page_mgr();}
+	public String Wkr_key() {return Xob_cmd_keys.Key_text_page;}
+	public void Wkr_bgn(Xob_bldr bldr) {
+		Xoae_app app = wiki.Appe();
+		Xoapi_import import_cfg = app.Api_root().Bldr().Wiki().Import();
+		this.redirect_mgr = wiki.Redirect_mgr(); 
+		this.db_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
+		this.page_core_tbl = db_mgr.Tbl__page();
+		this.text_zip_mgr = Xoa_app_.Utl__zip_mgr(); text_zip_tid = import_cfg.Zip_tid_text();
+		this.ns_to_db_mgr = new Xob_ns_to_db_mgr(new Xob_ns_to_db_wkr__text(), db_mgr, import_cfg.Text_db_max());
+		if (redirect_id_enabled) {
+			this.redirect_tbl = new Xob_redirect_tbl(wiki.Fsys_mgr().Root_dir(), Xoa_app_.Utl__encoder_mgr().Http_url_ttl()).Create_table();
+			redirect_tbl.Conn().Txn_bgn();
+		}
+		this.dg_match_mgr = app.Api_root().Bldr().Wiki().Filter().Dansguardian().New_mgr(wiki.Domain_str(), wiki.Fsys_mgr().Root_dir());
+		app.Bldr().Dump_parser().Trie_tab_del_();	// disable swapping &#09; for \t
+		byte[] ns_file_map = import_cfg.New_ns_file_map(wiki.Import_cfg().Src_rdr_len());
+		Xob_ns_file_itm.Init_ns_bldr_data(Xowd_db_file_.Tid_text, wiki.Ns_mgr(), ns_file_map);
+		if (idx_mode.Tid_is_bgn()) page_core_tbl.Create_index();
+		page_core_tbl.Insert_bgn();
+		usr_dlg.Prog_many("", "", "import.page.bgn");
+	}
+	public void Wkr_run(Xowd_page_itm page) {
+		int id = page.Id();
+		DateAdp modified = page.Modified_on(); if (modified.compareTo(modified_latest) == CompareAble_.More) modified_latest = modified;
+		byte[] text_raw = page.Text(); int text_raw_len = page.Text_len();
+		Xoa_ttl redirect_ttl = redirect_mgr.Extract_redirect(text_raw, text_raw_len); boolean redirect = redirect_ttl != null;
+		page.Redirected_(redirect);
+		Xow_ns ns = page.Ns();
+		int random_int = ns.Count() + 1; ns.Count_(random_int);
+		if (dg_match_mgr != null) {
+			if (dg_match_mgr.Match(1, id, ns.Id(), page.Ttl_page_db(), page.Ttl_full_db(), wiki.Lang(), text_raw)) return;
+		}
+		byte[] text_zip = text_zip_mgr.Zip(text_zip_tid, text_raw);
+		Xowd_db_file text_db = ns_to_db_mgr.Get_by_ns(ns.Bldr_data(), text_zip.length);
+		try {page_mgr.Create(page_core_tbl, text_db.Tbl__text(), id, page.Ns_id(), page.Ttl_page_db(), redirect, modified, text_zip, text_raw_len, random_int, text_db.Id(), -1);}
+		catch (Exception e) {
+			throw Exc_.new_exc(e, "bldr", "create page in db failed", "id", id, "ns", page.Ns_id(), "name", page.Ttl_page_db(), "redirect", redirect, "modified", modified, "text_len", text_raw_len, "text_db_id", text_db.Id());
+		}
+		if (redirect && redirect_id_enabled)
+			redirect_tbl.Insert(id, page.Ttl_page_db(), redirect_ttl);
+		++page_count_all;
+		if (ns.Id_main() && !page.Redirected()) ++page_count_main;
+		if (page_count_all % commit_interval == 0) {
+			page_core_tbl.Conn().Txn_sav(); text_db.Conn().Txn_sav();
+			if (redirect_id_enabled) redirect_tbl.Conn().Txn_sav();
+			if (dg_match_mgr != null) dg_match_mgr.Commit();
+		}
+	}
+	public void Wkr_end() {
+		usr_dlg.Log_many("", "", "import.page: insert done; committing pages; pages=~{0}", page_count_all);
+		page_core_tbl.Insert_end(); ns_to_db_mgr.Rls_all();
+		if (dg_match_mgr != null) dg_match_mgr.Rls();
+		usr_dlg.Log_many("", "", "import.page: updating core stats");
+		Xow_ns_mgr ns_mgr = wiki.Ns_mgr();
+		Xowd_db_file db_core = db_mgr.Db__core();
+		db_core.Tbl__site_stats().Update(page_count_main, page_count_all, ns_mgr.Ns_file().Count());	// save page stats
+		db_core.Tbl__ns().Insert(ns_mgr);															// save ns
+		db_mgr.Tbl__cfg().Insert_str(Xow_cfg_consts.Grp__wiki_init, "props.modified_latest", modified_latest.XtoStr_fmt(DateAdp_.Fmt_iso8561_date_time));
+		if (idx_mode.Tid_is_end()) page_core_tbl.Create_index();
+		if (redirect_id_enabled) {
+			redirect_tbl.Conn().Txn_end();
+			redirect_tbl.Update_trg_redirect_id(db_core.Url(), 1);
+			redirect_tbl.Update_src_redirect_id(db_core.Url(), page_core_tbl.Conn());
+		}
+	}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_commit_interval_))			commit_interval = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_idx_mode_))					idx_mode = Db_idx_mode.Xto_itm(m.ReadStr("v"));
+		else if	(ctx.Match(k, Invk_redirect_id_enabled_))		redirect_id_enabled = m.ReadYn("v");
+		else													return super.Invk(ctx, ikey, k, m);
+		return this;
+	}
+	private static final String Invk_commit_interval_ = "commit_interval_", Invk_idx_mode_ = "idx_mode_", Invk_redirect_id_enabled_ = "redirect_id_enabled_";
+	public void Wkr_ini(Xob_bldr bldr) {}
+	public void Wkr_print() {}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_page_cmd_tst.java

@@ -0,0 +1,37 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import org.junit.*; import gplx.dbs.*; import gplx.xowa.specials.search.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.ctgs.*;
+public class Xob_page_cmd_tst {
+	@Before public void init() {if (Xoa_test_.Db_skip()) return; fxt.Ctor_fsys();} Db_mgr_fxt fxt = new Db_mgr_fxt();
+	@After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();} 
+	@Test   public void Basic() {
+		if (Xoa_test_.Db_skip()) return;
+		fxt.Init_db_sqlite();
+		fxt.Bldr().App().Api_root().Bldr().Wiki().Import().Zip_tid_text_raw_();
+		fxt.doc_ary_
+		(	fxt.doc_(2, "2013-06-03 01:23", "A", "text_a")
+		,	fxt.doc_(1, "2013-06-03 12:34", "B", "#REDIRECT [[A]]")
+		)
+		.Exec_run(new Xob_page_cmd(fxt.Bldr(), fxt.Wiki()))
+		;
+		fxt.Test_load_ttl(Xow_ns_.Id_main, "A", fxt.page_(2, "2013-06-03 01:23", false, 6));
+		fxt.Test_load_page(Xow_ns_.Id_main, 2, "text_a");
+		fxt.Test_load_ttl(Xow_ns_.Id_main, "B", fxt.page_(1, "2013-06-03 12:34", true, 15));
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_search_sql_cmd.java

@@ -0,0 +1,86 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.dbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.*;
+public class Xob_search_sql_cmd extends Xob_itm_basic_base implements Xob_cmd {	// search version 2; upgrade
+	private int commit_interval = 100000, progress_interval = 10000;
+	public Xob_search_sql_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_text_search_cmd;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run() {this.Exec(wiki);}
+	public void Cmd_end() {}
+	public void Cmd_term() {}		
+	public void Exec(Xowe_wiki wiki) {
+		if (!Env_.Mode_testing()) wiki.Init_assert();
+		Xowd_db_mgr db_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
+		db_mgr.Dbs__delete_by_tid(Xowd_db_file_.Tid_search_core);
+		Xowd_db_file search_db = Xob_search_sql_cmd.Dbs__get_or_make(db_mgr);
+		Db_conn search_conn = search_db.Conn();
+		Xowd_search_temp_tbl search_temp_tbl = new Xowd_search_temp_tbl(search_conn, db_mgr.Props().Schema_is_1());
+		search_temp_tbl.Create_tbl();
+		search_temp_tbl.Insert_bgn();
+		Xowd_page_tbl page_tbl = db_mgr.Tbl__page();
+		Db_rdr page_rdr = page_tbl.Select_all();
+		try {
+			Xol_lang lang = wiki.Lang();
+			Bry_bfr bfr = Bry_bfr.reset_(1024);
+			Ordered_hash hash = Ordered_hash_.new_();
+			int page_count = 0;
+			String fld_page_id = page_tbl.Fld_page_id(), fld_page_ttl = page_tbl.Fld_page_title();
+			while (page_rdr.Move_next()) {
+				int page_id = page_rdr.Read_int(fld_page_id);
+				byte[] ttl = page_rdr.Read_bry_by_str(fld_page_ttl);
+				byte[][] words = Xob_search_base.Split_ttl_into_words(lang, hash, bfr, ttl);
+				int words_len = words.length;
+				for (int i = 0; i < words_len; i++) {
+					byte[] word = words[i];
+					search_temp_tbl.Insert_cmd_by_batch(page_id, word);
+				}
+				++page_count;
+				if		((page_count % commit_interval) == 0)
+					Commit(search_conn);
+				else if ((page_count % progress_interval) == 0)
+					usr_dlg.Prog_many("", "", "parse progress: count=~{0} last=~{1}", page_count, String_.new_u8(ttl));
+			}
+			this.Commit(search_conn);
+		} 
+		finally {page_rdr.Rls();}
+		search_conn.Txn_end();
+		search_temp_tbl.Make_data(usr_dlg, db_mgr.Db__search().Tbl__search_link(), db_mgr.Db__search().Tbl__search_word());
+	}
+	private void Commit(Db_conn search_conn) {
+		search_conn.Txn_sav();
+	}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_commit_interval_))		commit_interval = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_progress_interval_))		progress_interval = m.ReadInt("v");
+		else	return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}	private static final String Invk_progress_interval_ = "progress_interval_", Invk_commit_interval_ = "commit_interval_";
+	public static Xowd_db_file Dbs__get_or_make(Xowd_db_mgr db_mgr) {
+		Xowd_db_file db = db_mgr.Props().Layout_text().Tid_is_all_or_few()
+			? db_mgr.Db__core()
+			: db_mgr.Dbs__make_by_tid(Xowd_db_file_.Tid_search_core)
+			;
+		db.Tbl__search_word().Ddl__page_count_y_();
+		db.Tbl__search_word().Create_tbl();
+		db.Tbl__search_link().Create_tbl();
+		return db;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_search_sql_cmd_tst.java

@@ -0,0 +1,52 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import org.junit.*; import gplx.dbs.*; import gplx.xowa.specials.search.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.texts.*; import gplx.xowa.bldrs.cmds.ctgs.*;
+import gplx.xowa.wikis.data.tbls.*;
+public class Xob_search_sql_cmd_tst {
+	@Before public void init() {if (Xoa_test_.Db_skip()) return; fxt.Ctor_fsys();} private final Db_mgr_fxt fxt = new Db_mgr_fxt();
+	@After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();} 
+	@Test   public void Basic() {
+		if (Xoa_test_.Db_skip()) return;
+		fxt.Init_db_sqlite();
+		fxt.doc_ary_
+		(	fxt.doc_ttl_(2, "A b")
+		,	fxt.doc_ttl_(1, "B c")
+		)
+		.Exec_run(new Xob_page_cmd(fxt.Bldr(), fxt.Wiki()))
+		.Exec_run(new Xob_search_sql_cmd(fxt.Bldr(), fxt.Wiki()))
+		;
+		fxt.Test_search("a", 2);
+		fxt.Test_search("b", 1, 2);
+		fxt.Test_search("c", 1);
+		fxt.Test_search("d");
+	}
+	@Test   public void Wildcard() {
+		if (Xoa_test_.Db_skip()) return;
+		fxt.Init_db_sqlite();
+		fxt.doc_ary_
+		(	fxt.doc_ttl_(1, "A")
+		,	fxt.doc_ttl_(2, "A2")
+		,	fxt.doc_ttl_(3, "A3")
+		)
+		.Exec_run(new Xob_page_cmd(fxt.Bldr(), fxt.Wiki()))
+		.Exec_run(new Xob_search_sql_cmd(fxt.Bldr(), fxt.Wiki()))
+		;
+		fxt.Test_search("a*", 1, 2, 3);
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_search_sql_wkr.java

@@ -0,0 +1,93 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.ios.*;
+import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.dbs.*; import gplx.xowa.wikis.data.tbls.*;
+public class Xob_search_sql_wkr extends Xob_search_base implements Io_make_cmd {			// search version 2
+	private Xowd_db_mgr db_mgr; private Xowd_search_link_tbl search_page_tbl; private Xowd_search_word_tbl search_word_tbl;
+	private int search_id = 0; private byte[] prv_word = Bry_.Empty; private int page_count;
+	public Xob_search_sql_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	@Override public String Wkr_key() {return Xob_cmd_keys.Key_text_search_wkr;}
+	@Override public gplx.ios.Io_make_cmd Make_cmd_site() {return this;}
+	public Io_sort_cmd Make_dir_(Io_url v) {return this;}	// noop	
+	public void Sort_bgn() {
+		this.usr_dlg = Xoa_app_.Usr_dlg();
+		this.db_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
+		Xob_search_sql_cmd.Dbs__get_or_make(db_mgr);
+		this.search_page_tbl = db_mgr.Db__search().Tbl__search_link();
+		this.search_word_tbl = db_mgr.Db__search().Tbl__search_word(); 
+		search_page_tbl.Insert_bgn(); search_word_tbl.Insert_bgn();
+	}
+	public void Sort_do(Io_line_rdr rdr) {
+		byte[] bry = rdr.Bfr();
+		byte[] cur_word = Bry_.Mid(bry, rdr.Key_pos_bgn(), rdr.Key_pos_end());
+		if (!Bry_.Eq(cur_word, prv_word)) {
+			search_word_tbl.Insert_cmd_by_batch(++search_id, cur_word, page_count);
+			prv_word = cur_word;
+			page_count = 0;
+		}
+		search_page_tbl.Insert_cmd_by_batch(search_id, Base85_utl.XtoIntByAry(bry, rdr.Key_pos_end() + 1, rdr.Key_pos_end() +  5)); // -1: ignore rdr_dlm
+		++page_count;
+		if (search_id % 10000 == 0)
+			usr_dlg.Prog_many("", "", "creating title index: count=~{0}", search_id);
+	}
+	public void Sort_end() {
+		search_page_tbl.Insert_end(); search_word_tbl.Insert_end();
+		Xowd_db_file search_db = db_mgr.Db__search();
+		Xowd_search_temp_tbl search_temp_tbl = new Xowd_search_temp_tbl(search_db.Conn(), db_mgr.Props().Schema_is_1());
+		search_temp_tbl.Create_idx(usr_dlg, search_page_tbl, search_word_tbl);
+		search_word_tbl.Ddl__page_count__cfg(search_db.Tbl__cfg());
+	}
+}
+class Xob_search_wkr extends Xob_itm_basic_base implements Xobd_wkr {
+	private Xowd_db_file search_db; private Xowd_search_temp_tbl search_temp_tbl;
+	private Xol_lang lang; private final Bry_bfr tmp_bfr = Bry_bfr.new_(255); private final Ordered_hash list = Ordered_hash_.new_bry_();
+	public String Wkr_key() {return Xob_cmd_keys.Key_text_search_wkr;}
+	public void Wkr_ini(Xob_bldr bldr) {}
+	public void Wkr_bgn(Xob_bldr bldr) {
+		if (!Env_.Mode_testing()) wiki.Init_assert();
+		this.lang = wiki.Lang();
+		Xowd_db_mgr db_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
+		this.search_db = Xob_search_sql_cmd.Dbs__get_or_make(db_mgr);
+		this.search_temp_tbl = new Xowd_search_temp_tbl(search_db.Conn(), db_mgr.Props().Schema_is_1());
+		search_temp_tbl.Create_tbl();
+		search_temp_tbl.Insert_bgn();
+	}
+	public void Wkr_run(Xowd_page_itm page) {
+		try {
+			int page_id = page.Id();
+			byte[] ttl = page.Ttl_page_db();
+			byte[][] words = Xob_search_base.Split_ttl_into_words(lang, list, tmp_bfr, ttl);
+			int len = words.length;
+			for (int i = 0; i < len; ++i) {
+				byte[] word = words[i];
+				search_temp_tbl.Insert_cmd_by_batch(page_id, word);
+			}
+		}	catch (Exception e) {bldr.Usr_dlg().Warn_many("", "", "search_index:fatal error: err=~{0}", Err_.Message_gplx_brief(e));}	// never let single page crash entire import
+	}
+	public void Wkr_end() {
+		search_temp_tbl.Make_data(usr_dlg, search_db.Tbl__search_link(), search_db.Tbl__search_word());
+		search_temp_tbl.Insert_end();
+	}
+	public void Wkr_print() {}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_set)) {}
+		else	return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}	private static final String Invk_set = "set";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_term_cmd.java

@@ -0,0 +1,33 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.dbs.cfgs.*; import gplx.xowa.dbs.*; import gplx.xowa.wikis.*;
+public class Xob_term_cmd extends Xob_term_base {
+	public Xob_term_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Ctor(bldr, wiki); this.wiki = wiki;} private Xowe_wiki wiki;
+	@Override public String Cmd_key() {return KEY;} public static final String KEY = "text.term";
+	@Override public void Cmd_end_hook() {
+		Io_mgr.I.DeleteDirDeep(wiki.Fsys_mgr().Tmp_dir());
+		Db_cfg_tbl cfg_tbl = wiki.Data__core_mgr().Tbl__cfg();
+		cfg_tbl.Insert_bry(Xow_cfg_consts.Grp__wiki_init, "props.bldr_version", wiki.Props().Bldr_version());
+		cfg_tbl.Insert_bry(Xow_cfg_consts.Grp__wiki_init, Xow_cfg_consts.Key__init__main_page, wiki.Props().Main_page());
+		cfg_tbl.Insert_bry(Xow_cfg_consts.Grp__wiki_init, "props.siteinfo_misc", wiki.Props().Siteinfo_misc());
+		cfg_tbl.Insert_bry(Xow_cfg_consts.Grp__wiki_init, "props.siteinfo_mainpage", wiki.Props().Siteinfo_mainpage());
+		gplx.fsdb.Fsdb_db_mgr__v2_bldr.I.Get_or_make(wiki, false);// always build file.user db; DATE:2015-05-12
+		wiki.Data__core_mgr().Rls();
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_calc_stats_cmd.java

@@ -0,0 +1,112 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.xowa.tdbs.*; import gplx.xowa.wikis.data.tbls.*;
+public class Xob_calc_stats_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	public Xob_calc_stats_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_tdb_calc_stats;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run() {Exec();}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	private void Exec() {
+		int ns_len = wiki.Ns_mgr().Ords_len();
+		int total = 0;
+		for (int i = 0; i < ns_len; i++) {
+			Xow_ns ns = wiki.Ns_mgr().Ords_ary()[i];
+			int ns_count = Calc_counts(ns);
+			ns.Count_(ns_count);
+			total += ns_count;
+		}
+		int count_main = Calc_count_articles(wiki.Ns_mgr().Ns_main());
+		int count_file = Calc_count_articles(wiki.Ns_mgr().Ns_file());
+		Bry_bfr bfr = Bry_bfr.new_();
+		Gen_call(Bool_.Y, bfr, Xowe_wiki.Invk_stats);
+		Gen_call(Bool_.N, bfr, Xow_wiki_stats.Invk_number_of_articles_, count_main);
+		Gen_call(Bool_.N, bfr, Xow_wiki_stats.Invk_number_of_files_, count_file);
+		Gen_call(Bool_.N, bfr, Xow_wiki_stats.Invk_number_of_pages_, total);
+		for (int i = 0; i < ns_len; i++) {
+			Xow_ns ns = wiki.Ns_mgr().Ords_ary()[i];
+			if (ns.Id() < 0) continue;
+			bfr.Add_byte_nl();
+			Gen_call(Bool_.N, bfr, Xow_wiki_stats.Invk_number_of_articles_in_ns_, ns.Num_str(), Int_.Xto_str_pad_bgn_zero(ns.Count(), 10));
+		}
+		bfr.Add_byte_nl().Add_byte(Byte_ascii.Semic).Add_byte_nl();
+		Io_url wiki_gfs = Wiki_gfs_url(wiki);
+		Io_mgr.I.SaveFilBfr(wiki_gfs, bfr);
+	}
+	private void Gen_call(boolean first, Bry_bfr bfr, String key, Object... vals) {
+		if (!first) bfr.Add_byte(Byte_ascii.Dot);
+		bfr.Add_str(key);
+		int len = vals.length;
+		if (len > 0) {
+			bfr.Add_byte(Byte_ascii.Paren_bgn);
+			for (int i = 0; i < len; i++) {
+				if (i != 0) bfr.Add_byte(Byte_ascii.Comma).Add_byte(Byte_ascii.Space);
+				Object val = vals[i];
+				bfr.Add_str(Object_.Xto_str_strict_or_null_mark(val));
+			}
+			bfr.Add_byte(Byte_ascii.Paren_end);
+		}
+	}
+	int Calc_counts(Xow_ns ns) {
+		Io_url reg_url = wiki.Tdb_fsys_mgr().Url_ns_reg(ns.Num_str(), Xotdb_dir_info_.Tid_ttl);
+		Xowd_regy_mgr reg_mgr = new Xowd_regy_mgr(reg_url);
+		int files_ary_len = reg_mgr.Files_ary().length;
+		int count = 0;
+		for (int i = 0; i < files_ary_len; i++) {
+			count += reg_mgr.Files_ary()[i].Count();
+		}
+		return count;
+	}
+	int Calc_count_articles(Xow_ns ns) {
+		Io_url hive_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest(Xotdb_dir_info_.Name_ns, ns.Num_str(), Xotdb_dir_info_.Name_title);
+		return Calc_count_articles_dir(ns, hive_dir);
+	}
+	int Calc_count_articles_dir(Xow_ns ns, Io_url dir) {
+		Io_url[] subs = Io_mgr.I.QueryDir_args(dir).DirInclude_().ExecAsUrlAry();
+		int count = 0;
+		int subs_len = subs.length;
+		bldr.Usr_dlg().Prog_one(GRP_KEY, "count", "calculating: ~{0}", dir.Raw());
+		for (int i = 0; i < subs_len; i++) {
+			Io_url sub = subs[i];
+			if (sub.Type_dir())
+				count += Calc_count_articles_dir(ns, sub);
+			else
+				count += Calc_count_articles_fil(ns, sub);
+		}
+		return count;
+	}
+	int Calc_count_articles_fil(Xow_ns ns, Io_url fil) {
+		if (String_.Eq(fil.NameAndExt(), Xotdb_dir_info_.Name_reg_fil)) return 0;
+		int rv = 0;
+		byte[] bry = Io_mgr.I.LoadFilBry(fil);
+		Xob_xdat_file xdat_file = new Xob_xdat_file().Parse(bry, bry.length, fil);
+		Xowd_page_itm page = Xowd_page_itm.new_tmp();
+		int count = xdat_file.Count();
+		for (int i = 0; i < count; i++) {
+			byte[] ttl_bry = xdat_file.Get_bry(i);
+			Xotdb_page_itm_.Txt_ttl_load(page, ttl_bry);
+			rv += page.Redirected() ? 0 : 1;
+		}
+		return rv;
+	}
+	static final String GRP_KEY = "xowa.bldr.calc_stats";
+	public static Io_url Wiki_gfs_url(Xowe_wiki wiki) {return wiki.Fsys_mgr().Root_dir().GenSubFil_nest("cfg", "wiki_stats.gfs");}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_init_base_tst.java

@@ -0,0 +1,43 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import org.junit.*; import gplx.xowa.html.portal.*; import gplx.xowa.wikis.xwikis.*;
+public class Xob_init_base_tst {
+	@Before public void init() {fxt.Clear();} private Xob_init_base_fxt fxt = new Xob_init_base_fxt();
+	@Test  public void Dirty_wiki_itms() {
+		Xoae_app app = fxt.App(); Xowe_wiki wiki = fxt.Wiki();
+		Xoa_available_wikis_mgr wikis_list = fxt.App().Gui_mgr().Html_mgr().Portal_mgr().Wikis();
+		Tfds.Eq("", wikis_list.Itms_as_html());			// assert
+		Xow_xwiki_itm xwiki_itm = app.Usere().Wiki().Xwiki_mgr().Add_full("en.wikipedia.org", "en.wikipedia.org");
+		xwiki_itm.Offline_(Bool_.Y);	// simulate add via Available_from_fsys; DATE:2014-09-21
+		Tfds.Eq("", wikis_list.Itms_as_html());			// still empty
+		new Xob_init_tdb(app.Bldr(), wiki).Cmd_end();	// mock "init" task
+		Tfds.Eq("\n        <li><a href=\"/site/en.wikipedia.org/\" class='xowa-hover-off'>en.wikipedia.org</a></li>", wikis_list.Itms_as_html());	// no longer empty
+	}
+}
+class Xob_init_base_fxt {		
+	public void Clear() {
+		if (app == null) {
+			app = Xoa_app_fxt.app_();
+			wiki = Xoa_app_fxt.wiki_tst_(app);
+		}
+		Io_mgr.I.InitEngine_mem();
+	}
+	public Xoae_app App() {return app;} private Xoae_app app;
+	public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki;
+} 

400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_init_tdb.java

@@ -0,0 +1,28 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.xowa.xtns.wdatas.imports.*;
+public class Xob_init_tdb extends Xob_init_base {
+	public Xob_init_tdb(Xob_bldr bldr, Xowe_wiki wiki) {this.Ctor(bldr, wiki);}
+	@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_text_init;}
+	@Override public void Cmd_ini_wdata(Xob_bldr bldr, Xowe_wiki wiki) {
+		bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_wdata_qid);
+		bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_wdata_pid);
+	}
+	@Override public void Cmd_run_end(Xowe_wiki wiki) {}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_make_id_wkr.java

@@ -0,0 +1,38 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.ios.*; import gplx.xowa.tdbs.*; import gplx.xowa.wikis.data.tbls.*;
+public class Xob_make_id_wkr extends Xob_itm_dump_base implements Xobd_wkr, GfoInvkAble {
+	public Xob_make_id_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Wkr_key() {return KEY;} public static final String KEY = "core.make_id";
+	public void Wkr_ini(Xob_bldr bldr) {}
+	public void Wkr_bgn(Xob_bldr bldr) {
+		this.Init_dump(KEY, wiki.Tdb_fsys_mgr().Site_dir().GenSubDir(Xotdb_dir_info_.Name_id));
+	}
+	public void Wkr_run(Xowd_page_itm page) {
+		byte[] ttl = page.Ttl_page_db();
+		if (dump_bfr.Len() + row_fixed_len + ttl.length > dump_fil_len) Io_mgr.I.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
+		Xotdb_page_itm_.Txt_id_save(dump_bfr, page);
+	}
+	public void Wkr_end() {
+		this.Term_dump(new Xob_make_cmd_site(bldr.Usr_dlg(), make_dir, make_fil_len));
+		if (delete_temp) Io_mgr.I.DeleteDirDeep(temp_dir);
+	}
+	public void Wkr_print() {}
+	static final int row_fixed_len = 25 + 1 + 7;	// 25=5 base_85 flds; 1=Redirect; 7=dlm
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_page_txt.java

@@ -0,0 +1,90 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.ios.*; import gplx.xowa.tdbs.*; import gplx.xowa.wikis.data.tbls.*;
+public class Xob_page_txt extends Xob_itm_dump_base implements Xobd_wkr, GfoInvkAble {
+	public Xob_page_txt(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Wkr_key() {return Xob_cmd_keys.Key_tdb_make_page;}
+	public void Wkr_ini(Xob_bldr bldr) {}
+	public void Wkr_bgn(Xob_bldr bldr) {
+		redirect_mgr = wiki.Redirect_mgr(); page_storage_type = wiki.Appe().Setup_mgr().Dump_mgr().Data_storage_format();
+		fsys_mgr = wiki.Tdb_fsys_mgr();			
+		make_dir = fsys_mgr.Ns_dir();
+		if (Io_mgr.I.QueryDir_args(make_dir).DirOnly_().ExecAsUrlAry().length > 0) throw bldr.Usr_dlg().Fail_many("xowa.bldr.itm", "dir_empty", "dir_must_be_empty: ~{0}", make_dir.Raw());
+		this.Init_dump(Xob_cmd_keys.Key_tdb_make_page, make_dir);
+		this.data_rpt_typ = stat_mgr.GetOrNew(Xotdb_dir_info_.Tid_page);
+		ttl_wtr_mgr = new Xob_tmp_wtr_mgr(new Xob_tmp_wtr_wkr__ttl(temp_dir, dump_fil_len));			
+	}	private Xotdb_fsys_mgr fsys_mgr; Xop_redirect_mgr redirect_mgr;
+	int page_file_len = 512 * Io_mgr.Len_kb, title_file_len = 64 * Io_mgr.Len_kb; Xob_tmp_wtr_mgr ttl_wtr_mgr;
+	Xob_xdat_file_wtr[] page_wtr_regy = new Xob_xdat_file_wtr[Ns_ordinal_max]; static final int Ns_ordinal_max = Xow_ns_mgr_.Ordinal_max;	// ASSUME: no more than 128 ns in a wiki
+	Xob_stat_type data_rpt_typ; Xob_stat_mgr stat_mgr = new Xob_stat_mgr(); byte page_storage_type;		
+	public void Wkr_run(Xowd_page_itm page) {
+		int id = page.Id(); byte[] ttl_wo_ns = page.Ttl_page_db(), text = page.Text(); int ttl_len = ttl_wo_ns.length, text_len = text.length; Xow_ns ns = page.Ns();
+		boolean redirect = redirect_mgr.Is_redirect(text, text_len);
+		page.Redirected_(redirect);
+
+		// page: EX: \t123\t2012-06-09\ttitle\ttext\n; NOTE: 512k * ~20 ns = 10 MB max memory; no need for intermediary flushing
+		Xob_xdat_file_wtr page_wtr = Page_wtr_get(ns);
+		if (page_wtr.FlushNeeded(Xotdb_page_itm_.Txt_page_len__fixed + ttl_len + text_len)) page_wtr.Flush(bldr.Usr_dlg());
+		Xotdb_page_itm_.Txt_page_save(page_wtr.Bfr(), id, page.Modified_on(), ttl_wo_ns, text, false);
+		page_wtr.Add_idx(Byte_ascii.Nl);
+		
+		// idx: EX: 00100|aB64|Ttl;
+		Xob_tmp_wtr ttl_wtr = ttl_wtr_mgr.Get_or_new(ns);
+		int file_idx = page_wtr.Fil_idx(), row_idx = page_wtr.Idx_pos() - List_adp_.LastIdxOffset;
+		page.Text_db_id_(file_idx).Tdb_row_idx_(row_idx);
+		if (ttl_wtr.FlushNeeded(Xotdb_page_itm_.Txt_ttl_len__fixed + ttl_len)) ttl_wtr.Flush(bldr.Usr_dlg());
+		Xotdb_page_itm_.Txt_ttl_save(ttl_wtr.Bfr(), id, file_idx, row_idx, redirect, text_len, ttl_wo_ns);
+	}
+	public void Wkr_end() {
+		Flush_page(page_wtr_regy);
+		ttl_wtr_mgr.Flush_all(bldr.Usr_dlg());
+		Xobdc_merger.Ns(bldr.Usr_dlg(), ttl_wtr_mgr.Regy(), Xotdb_dir_info_.Name_title, temp_dir, make_dir, sort_mem_len, Io_line_rdr_key_gen_.last_pipe, new Io_sort_cmd_ns(bldr.Usr_dlg()));
+		ttl_wtr_mgr.Rls_all();
+		if (delete_temp) Io_mgr.I.DeleteDirDeep(temp_dir);
+	}
+	public void Wkr_print() {bldr.Usr_dlg().Note_many(GRP_KEY, "print", "~{0}", stat_mgr.Print(wiki.Ns_mgr()));}
+	Xob_xdat_file_wtr Page_wtr_get(Xow_ns ns) {
+		Xob_xdat_file_wtr rv = page_wtr_regy[ns.Ord()];
+		if (rv == null) {
+			rv = Xob_xdat_file_wtr.new_by_tid_(page_file_len, fsys_mgr.Ns_dir().GenSubDir_nest(ns.Num_str()), Xotdb_dir_info_.Tid_page, page_storage_type).Ns_ord_idx_(ns.Ord());
+			page_wtr_regy[ns.Ord()] = rv;
+		}
+		return rv;
+	}
+	private void Flush_page(Xob_xdat_file_wtr[] regy) {
+		for (int i = 0; i < Ns_ordinal_max; i++) {
+			Xob_xdat_file_wtr wtr = regy[i];
+			if (wtr != null) {
+				Xow_ns ns_itm = wiki.Ns_mgr().Ords_get_at(wtr.Ns_ord_idx());
+				Xob_stat_itm datRptItm = data_rpt_typ.GetOrNew(ns_itm.Name_str());
+				datRptItm.Tally(wtr.Fil_len(), wtr.Fil_idx());
+				wtr.Flush(bldr.Usr_dlg());
+				wtr.Rls();
+				regy[i] = null;
+			}
+		}
+	}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_page_file_len_))		page_file_len = gplx.ios.Io_size_.Load_int_(m);
+		else if (ctx.Match(k, Invk_title_file_len_))	title_file_len = gplx.ios.Io_size_.Load_int_(m);
+		else	return super.Invk(ctx, ikey, k, m);
+		return this;
+	}	private static final String Invk_page_file_len_ = "page_file_len_", Invk_title_file_len_ = "title_file_len_";
+	static final String GRP_KEY = "xowa.bldr.make_page";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_parse_dump_templates_cmd.java

@@ -0,0 +1,36 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.ios.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.tdbs.*;
+public class Xob_parse_dump_templates_cmd extends Xob_itm_dump_base implements Xobd_wkr, GfoInvkAble {
+	public Xob_parse_dump_templates_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Wkr_key() {return KEY;} public static final String KEY = "parse.dump_templates";
+	public static final int FixedLen_page = 1 + 5 + 1 + 5 + 1 + 1 + 1;	// \tid|date|title|text\n
+	public void Wkr_ini(Xob_bldr bldr) {}
+	public void Wkr_bgn(Xob_bldr bldr) {
+		Init_dump(KEY);
+	}
+	public void Wkr_run(Xowd_page_itm page) {
+		if (page.Ns_id() != Xow_ns_.Id_template) return;
+		int id = page.Id(); byte[] title = page.Ttl_page_db(), text = page.Text(); int title_len = title.length, text_len = text.length;
+		if (FixedLen_page + title_len + text_len + dump_bfr.Len() > dump_fil_len) super.Flush_dump();
+		Xotdb_page_itm_.Txt_page_save(dump_bfr, id, page.Modified_on(), title, text, true);
+	}
+	public void Wkr_end() {super.Flush_dump();}
+	public void Wkr_print() {}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_search_base_tst.java

@@ -0,0 +1,56 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import org.junit.*; import gplx.xowa.tdbs.*; import gplx.xowa.bldrs.*;
+public class Xob_search_base_tst {
+	@Test  public void Split() {
+		tst_Split("a", "a");
+		tst_Split("a b", "a", "b");
+		tst_Split("a b c", "a", "b", "c");
+		tst_Split("a-b.c", "a", "b", "c");
+		tst_Split("a A", "a");
+		tst_Split("a_b", "a", "b");
+		tst_Split("a (b)", "a", "b");
+	}
+	@Test  public void Title_word() {
+		fxt.doc_ary_
+		(	fxt.doc_wo_date_(2, "A b",  "text")
+		,	fxt.doc_wo_date_(3, "B a",  "text")
+		)
+		.Fil_expd(fxt.fil_ns_sttl(Xow_ns_.Id_main, 0)
+		,	"!!!!;|!!!!;|"
+		,	"a|!!!!#;!!!!%|!!!!$;!!!!%"
+		,	"b|!!!!#;!!!!%|!!!!$;!!!!%"
+		,	""
+		)
+		.Fil_expd(fxt.fil_reg(Xow_ns_.Id_main, Xotdb_dir_info_.Tid_search_ttl)
+		,	"0|a|b|2"
+		,	""
+		)
+		.Run(new Xob_search_tdb(fxt.Bldr(), this.fxt.Wiki()))
+		;
+	}	private Xob_fxt fxt = new Xob_fxt().Ctor_mem();
+	private void tst_Split(String raw, String... expd) {
+		Ordered_hash list = Ordered_hash_.new_(); Bry_bfr bfr = Bry_bfr.new_();
+		byte[][] actl_bry = Xob_search_base.Split_ttl_into_words(fxt.App().Lang_mgr().Lang_en(), list, bfr, Bry_.new_u8(raw));
+		String[] actl = new String[actl_bry.length];
+		for (int i = 0; i < actl_bry.length; i++)
+			actl[i] = String_.new_u8(actl_bry[i]);
+		Tfds.Eq_ary(expd, actl);
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_search_tdb.java

@@ -0,0 +1,25 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+public class Xob_search_tdb extends Xob_search_base {
+	public Xob_search_tdb(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	@Override public String Wkr_key() {return Xob_cmd_keys.Key_tdb_make_search_title;}
+	@Override public gplx.ios.Io_make_cmd Make_cmd_site() {
+		return new Xob_make_cmd_site(bldr.Usr_dlg(), this.make_dir, this.make_fil_len);
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_term_txt.java

@@ -0,0 +1,26 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import gplx.xowa.bldrs.cmds.texts.*;
+public class Xob_term_txt extends Xob_term_base {
+	public Xob_term_txt(Xob_bldr bldr, Xowe_wiki wiki) {this.Ctor(bldr, wiki); this.wiki = wiki;} private Xowe_wiki wiki;
+	@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_core_term;}
+	@Override public void Cmd_end_hook() {
+		Io_mgr.I.SaveFilBry(wiki.Tdb_fsys_mgr().Cfg_wiki_core_fil(), wiki.Cfg_wiki_core().Build_gfs());
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_tst.java

@@ -0,0 +1,204 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
+import org.junit.*;
+import gplx.ios.*; import gplx.xowa.tdbs.*; import gplx.xowa.wikis.data.tbls.*;
+public class Xob_tst {		
+	@Before public void init() {fxt = new Xob_fxt().Ctor_mem();} private Xob_fxt fxt;
+	// @After public void term() {fxt.Wiki().Ctx().Sys_load_tmpls_(true);} // commented during wiki.Ctx() removal; DATE:2014-04-22
+	@Test  public void Basic() {
+		fxt.doc_ary_
+		(	fxt.doc_(3, "2012-01-02 13:15", "Title 2a", "text2a\ny")
+		,	fxt.doc_(2, "2012-01-02 13:14", "Title 1",  "text1\nz")
+		)
+		.Fil_expd(fxt.fil_ns_page(Xow_ns_.Id_main, 0)
+		,	"!!!!@|!!!!>|"
+		,	"!!!!$\t#6>K6\tTitle 2a\ttext2a"
+		,	"y\t"
+		,	"!!!!#\t#6>K5\tTitle 1\ttext1"
+		,	"z\t"
+		,	""
+		)
+		.Fil_expd(fxt.fil_ns_title(Xow_ns_.Id_main, 0)
+		,	"!!!!C|!!!!D|"
+		,	"!!!!#|!!!!!|!!!!\"|0|!!!!(|Title 1"
+		,	"!!!!$|!!!!!|!!!!!|0|!!!!)|Title 2a"
+		,	""
+		)
+		.Fil_expd(fxt.fil_reg(Xow_ns_.Id_main, Xotdb_dir_info_.Tid_ttl)
+		,	"0|Title 1|Title 2a|2"
+		,	""
+		)
+		.Run_page_title()
+		;
+	}
+	@Test  public void Utf8() {
+		fxt.doc_ary_
+		(	fxt.doc_(3, "2012-01-02 13:15", "↑", "t2")
+		,	fxt.doc_(2, "2012-01-02 13:14", "!", "t1")
+		)
+		.Fil_expd(fxt.fil_ns_page(Xow_ns_.Id_main, 0)
+		,	"!!!!5|!!!!3|"
+		,	"!!!!$\t#6>K6\t↑\tt2\t"
+		,	"!!!!#\t#6>K5\t!\tt1\t"
+		,	""
+		)
+		.Fil_expd(fxt.fil_ns_title(Xow_ns_.Id_main, 0)
+		,	"!!!!=|!!!!?|"
+		,	"!!!!#|!!!!!|!!!!\"|0|!!!!#|!"
+		,	"!!!!$|!!!!!|!!!!!|0|!!!!#|↑"
+		,	""
+		)
+		.Fil_expd(fxt.fil_reg(Xow_ns_.Id_main, Xotdb_dir_info_.Tid_ttl)
+		,	"0|!|↑|2"
+		,	""
+		)
+		.Run_page_title()
+		;
+	}
+	@Test  public void Ns() {
+		fxt.doc_ary_
+		(	fxt.doc_(2, "2012-01-02 13:14", "Template:A",  "test a")
+		)
+		.Fil_expd(fxt.fil_ns_page(Xow_ns_.Id_template, 0)
+		,	"!!!!7|"
+		,	"!!!!#\t#6>K5\tA\ttest a\t"
+		,	""
+		)
+		.Fil_expd(fxt.fil_ns_title(Xow_ns_.Id_template, 0)
+		,	"!!!!=|"
+		,	"!!!!#|!!!!!|!!!!!|0|!!!!'|A"
+		,	""
+		)
+		.Fil_expd(fxt.fil_reg(Xow_ns_.Id_template, Xotdb_dir_info_.Tid_ttl)
+		,	"0|A|A|1"
+		,	""
+		)
+		.Run_page_title()
+		;
+	}
+	@Test  public void Id() {
+		fxt.doc_ary_
+		(	fxt.doc_wo_date_(2, "A",  "a")
+		,	fxt.doc_wo_date_(3, "Template:B",  "b")
+		)
+		.Fil_expd(fxt.fil_site_id(0)
+		,	"!!!!C|!!!!C|"
+		,	"!!!!#|!!!!!|!!!!!|0|!!!!\"|!!!!!|A"
+		,	"!!!!$|!!!!!|!!!!!|0|!!!!\"|!!!!+|B"
+		,	""
+		)
+		.Fil_expd(fxt.fil_reg(Xotdb_dir_info_.Tid_id)
+		,	"0|!!!!#|!!!!$|2"
+		,	""
+		)
+		.Run_id()
+		;
+	}
+	@Test  public void Category() {
+		fxt.doc_ary_
+		(	fxt.doc_wo_date_(2, "A",  "[[Category:Y]] [[Category:Z]]")
+		,	fxt.doc_wo_date_(3, "B",  "[[Category:Y]]")
+		)
+		.Fil_expd(fxt.fil_site_ctg(0)
+		,	"!!!!/|!!!!)|"
+		,	"Y|!!!!#|!!!!$"
+		,	"Z|!!!!#"
+		,	""
+		)
+		.Fil_expd(fxt.fil_reg(Xotdb_dir_info_.Tid_category)
+		,	"0|Y|Z|2"
+		,	""
+		)
+		.Run_ctg()
+		;
+	}
+	@Test  public void Category2() {
+		fxt.doc_ary_
+		(	fxt.doc_wo_date_(2, "A",  "[[Category:X]] [[Category:Y]]")
+		,	fxt.doc_wo_date_(3, "B",  "[[Category:Y]]")
+		)
+		.Fil_expd(fxt.fil_site_ctg(0)
+		,	"!!!!)|!!!!/|"
+		,	"X|!!!!#"
+		,	"Y|!!!!#|!!!!$"
+		,	""
+		)
+		.Fil_expd(fxt.fil_reg(Xotdb_dir_info_.Tid_category)
+		,	"0|X|Y|2"
+		,	""
+		)
+		.Run_ctg()
+		;
+	}
+	@Test  public void Tmpl_dump() {
+		fxt.doc_ary_
+		(	fxt.doc_wo_date_(2, "Template:A",  "a")
+		,	fxt.doc_wo_date_(3, "B",  "b")
+		,	fxt.doc_wo_date_(4, "Template:C",  "c")
+		)
+		.Fil_expd(null
+		,	"!!!!#\t#6>K5\tA\ta\t"
+		,	"!!!!%\t#6>K5\tC\tc\t"
+		,	""
+		)
+		.Run_tmpl_dump()
+		;
+	}
+	@Test  public void Parse() {
+		tst_Parse(String_.Concat
+			(	"!!!!#\t#6>K5\tA\ta\t\n"
+			,	"!!!!$\t#6>K5\tB\tb\t\n"	
+			,	"!!!!%\t#6>K5\tC\tc\t\n"
+			)
+			,	"a", "b", "c");
+	}
+	private void tst_Parse(String raw, String... expd) {
+		Xoae_app app = Xoa_app_fxt.app_();	// NOTE: resets mem file system, so must happen first
+		Io_url url = Io_url_.mem_fil_("mem/raw_page.csv");
+		Io_mgr.I.SaveFilStr(url, raw);
+		Xotdb_page_raw_parser parser = new Xotdb_page_raw_parser();
+		Xowe_wiki wiki = Xoa_app_fxt.wiki_tst_(app);
+		parser.Load(Gfo_usr_dlg_.Test(), wiki, new Xow_ns(Xow_ns_.Id_template, Xow_ns_case_.Id_1st, Bry_.new_u8("Template"), false), new Io_url[] {url}, 1 * Io_mgr.Len_kb);
+		List_adp actl = List_adp_.new_();
+		Xowd_page_itm page = new Xowd_page_itm();
+		while (parser.Read(page)) {
+			actl.Add(String_.new_u8(page.Text()));
+		}
+		Tfds.Eq_ary(expd, actl.To_str_ary());
+	}
+	@Test  public void Img_gen() {
+		/*
+		fxt.Src_(String_.Concat_lines_nl
+		( 	"A.png|||1234|90|80|8|1||0,-1,-1,-1"
+		,	"B.png|||1234|90|80|8|1||0,-1,-1,-1"
+		,	"C.png|||1234|90|80|8|1||0,-1,-1,-1"
+		)
+		.Expd_len_(3)
+		.Expd_files_
+		(	Io_fil.new_("mem/dump/70a"
+			,  "A.png|||1234|90|80|8|1||0,-1,-1,-1|70a")
+		,	Io_fil.new_("mem/dump/70b"
+			,  "B.png|||1234|90|80|8|1||0,-1,-1,-1|71b")
+		,	Io_fil.new_("mem/dump/70c"
+			,  "C.png|||1234|90|80|8|1||0,-1,-1,-1|72c")
+		)
+		.tst_()
+		*/;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_cleanup_cmd.java

@@ -0,0 +1,132 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.core.criterias.*;
+public class Xob_cleanup_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	private String bz2_cmd;
+	private boolean delete_all, delete_tmp;
+	private Criteria_ioMatch[] delete_by_match_ary;
+	public Xob_cleanup_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_util_cleanup;}
+	public Xob_cleanup_cmd Delete_sqlite3_(boolean v){delete_sqlite3 = v; return this;} private boolean delete_sqlite3;
+	public Xob_cleanup_cmd Delete_xml_(boolean v)	{delete_xml = v; return this;} private boolean delete_xml;
+	public Xob_cleanup_cmd Delete_tdb_(boolean v)	{delete_tdb = v; return this;} private boolean delete_tdb;
+	public void Bz2_fil_(Io_url v)		{bz2_fil = v;} private Io_url bz2_fil;
+	public void Cmd_run() {
+		Io_url wiki_root_dir = wiki.Fsys_mgr().Root_dir();
+		if (bz2_fil != null) {
+			if		(String_.Eq(bz2_cmd, "delete"))
+				Io_mgr.I.DeleteFil(bz2_fil);
+			else if (String_.Eq(bz2_cmd, "move"))
+				Io_mgr.I.MoveFil(bz2_fil, bz2_fil.OwnerDir().OwnerDir().GenSubFil_nest("done", bz2_fil.NameAndExt()));
+		}
+		if (delete_xml)						Io_mgr.I.DeleteFil(Xobd_rdr.Find_fil_by(wiki_root_dir, "*.xml"));
+		if (delete_tdb) {
+			usr_dlg.Note_many("", "", "bldr.wiki:deleting tdb wiki");
+			Delete_tdb(wiki_root_dir);
+		}
+		if (delete_sqlite3)
+			Delete_wiki_sql(wiki);			
+		if (delete_all) {
+			Io_mgr.I.DeleteDir_cmd(wiki_root_dir).Exec();	// do not delete subdirs; needed to support "/prv" for fsdb; DATE:2015-04-01
+			Io_mgr.I.DeleteDirDeep(app.Usere().Fsys_mgr().Wiki_root_dir()); // delete css dir; DATE:2015-07-06
+		}
+		if (delete_by_match_ary != null)
+			Delete_by_match(wiki_root_dir, delete_by_match_ary);
+		if (delete_tmp)
+			Io_mgr.I.DeleteDirDeep(wiki_root_dir.GenSubDir("tmp"));
+	}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_bz2_cmd_))				bz2_cmd = m.ReadStr("v");
+		else if	(ctx.Match(k, Invk_delete_xml_))			delete_xml = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_delete_wiki_))			delete_tdb = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_delete_sqlite3_))		delete_sqlite3 = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_delete_all_))			delete_all = m.ReadYn("v");
+		else if	(ctx.Match(k, Invk_bz2_fil_))				bz2_fil = m.ReadIoUrl("v");
+		else if	(ctx.Match(k, Invk_delete_by_match_))		delete_by_match_ary = Delete_by_match_parse(m.ReadStr("v"));
+		else if	(ctx.Match(k, Invk_delete_tmp_))			delete_tmp = m.ReadYn("v");
+		else	return super.Invk(ctx, ikey, k, m);
+		return this;
+	}
+	private static final String Invk_bz2_cmd_ = "bz2_cmd_", Invk_bz2_fil_ = "bz2_fil_"
+	, Invk_delete_xml_ = "delete_xml_", Invk_delete_wiki_ = "delete_wiki_", Invk_delete_sqlite3_ = "delete_sqlite3_"
+	, Invk_delete_all_ = "delete_all_"
+	, Invk_delete_tmp_ = "delete_tmp_"
+	, Invk_delete_by_match_ = "delete_by_match"
+	;
+	private static Criteria_ioMatch[] Delete_by_match_parse(String raw) {
+		String[] match_ary = String_.Split(raw, '|');
+		int match_ary_len = match_ary.length;
+		Criteria_ioMatch[] rv = new Criteria_ioMatch[match_ary_len];
+		for (int i = 0; i < rv.length; i++) {
+			String match = match_ary[i];
+			rv[i] = Criteria_ioMatch.parse_(true, match, false);
+		}
+		return rv;
+	}
+	private static void Delete_by_match(Io_url dir, Criteria_ioMatch[] match_ary) {
+		int match_len = match_ary.length;
+		Io_url[] subs = Io_mgr.I.QueryDir_fils(dir);
+		int subs_len = subs.length;
+		for (int i = 0; i < subs_len; i++) {
+			Io_url sub = subs[i];
+			for (int j = 0; j < match_len; j++) {
+				Criteria_ioMatch match = match_ary[j];
+				if (match.Matches(sub)) {
+					if (sub.Type_fil())
+						Io_mgr.I.DeleteFil(sub);
+				}
+			}
+		}
+	}
+	private static void Delete_tdb(Io_url wiki_root_dir) {
+		Io_url[] dirs = Io_mgr.I.QueryDir_args(wiki_root_dir).DirOnly_().DirInclude_().ExecAsUrlAry();
+		int dirs_len = dirs.length;
+		for (int i = 0; i < dirs_len; i++) {
+			Io_url dir = dirs[i];
+			if (gplx.xowa.tdbs.Xotdb_dir_info_.Dir_name_is_tdb(dir.NameOnly()))
+				Io_mgr.I.DeleteDirDeep(dir);
+		}
+	}
+	public static void Delete_wiki_sql(Xowe_wiki wiki) {
+		Gfo_usr_dlg usr_dlg = wiki.Appe().Usr_dlg(); Io_url wiki_root_dir = wiki.Fsys_mgr().Root_dir();
+		if (wiki.Db_mgr().Tid() == gplx.xowa.dbs.Xodb_mgr_sql.Tid_sql)		// NOTE: must check; if empty dir (or text db) than db_mgr will be txt
+			wiki.Db_mgr_as_sql().Core_data_mgr().Rls();						// NOTE: if sqlite files, must rls;
+		Io_url[] files = Io_mgr.I.QueryDir_fils(wiki_root_dir);
+		int files_len = files.length;
+		int deleted = 0;
+		String file_prefix = wiki.Domain_str() + "-file";					// NOTE: skip anything with "-file"; EX: "en.wikipedia.org-file.xowa"
+		String html_prefix = wiki.Domain_str() + "-html";					// NOTE: skip anything with "-html"; EX: "en.wikipedia.org-html-ns.000-db.002.xowa"
+		for (int i = 0; i < files_len; i++) {
+			Io_url url = files[i];
+			if (	!String_.Eq(url.Ext(), ".xowa")
+				&&	!String_.Eq(url.Ext(), ".sqlite3"))
+				continue;
+			if (	String_.Has_at_bgn(url.NameAndExt(), file_prefix)
+				||	String_.Has_at_bgn(url.NameAndExt(), html_prefix)
+				) continue;	// skip
+			Io_mgr.I.DeleteFil(url);
+			deleted++;
+		}
+		usr_dlg.Note_many("", "delete_wiki", "deleting sqlite3 files: ~{0} ~{1}", deleted, wiki_root_dir.Raw());
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_core_batch_utl.java

@@ -0,0 +1,42 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.xowa.wikis.*;
+public class Xob_core_batch_utl implements GfoInvkAble {
+	public Xob_core_batch_utl(Xob_bldr bldr, byte[] raw) {this.bldr = bldr; fmtr.Fmt_(raw);} private Xob_bldr bldr;
+	Bry_fmtr fmtr = Bry_fmtr.keys_("bz2_fil", "wiki_key");
+	private void Run() {
+		Io_url[] bz2_fils = Io_mgr.I.QueryDir_fils(bldr.App().Fsys_mgr().Wiki_dir().GenSubDir_nest(Dir_dump, "todo"));
+		Bry_bfr bfr = Bry_bfr.reset_(Io_mgr.Len_kb);
+		int bz2_fils_len = bz2_fils.length;
+		Xow_wiki_alias bz2_fil = new Xow_wiki_alias();
+		for (int i = 0; i < bz2_fils_len; i++) {
+			Io_url bz2_fil_url = bz2_fils[i];
+			bz2_fil.Fil_(bz2_fil_url).Parse(bz2_fil_url.NameOnly());
+			fmtr.Bld_bfr_many(bfr, bz2_fil_url.Raw(), bz2_fil.Domain());
+			bldr.Usr_dlg().Note_many("", "", "starting script for ~{0}", String_.new_u8(bz2_fil.Domain()));
+			bldr.App().Gfs_mgr().Run_str(bfr.Xto_str_and_clear());
+		}
+	}
+	public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_owner))			return bldr.Cmd_mgr();
+		else if	(ctx.Match(k, Invk_run))			Run();
+		return this;
+	}	private static final String Invk_owner = "owner", Invk_run = "run";
+	public static String Dir_dump = "#dump";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_decompress_bz2_cmd.java

@@ -0,0 +1,54 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.ios.*; import gplx.core.threads.*; import gplx.xowa.bldrs.*; 
+public class Xob_decompress_bz2_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	public Xob_decompress_bz2_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}		
+	public String Cmd_key() {return Xob_cmd_keys.Key_decompress_bz2;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run() {
+		if (Io_mgr.I.ExistsFil(trg)) return; // file already exists; don't decompress again
+		usr_dlg.Note_many(GRP_KEY, "bgn", "decompressing ~{0}", src.Raw(), trg.Raw());
+		Decompress(bldr.App(), src.Raw(), trg);
+	}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_src_))				this.Src_(m.ReadIoUrl("v"));
+		else	return super.Invk(ctx, ikey, k, m);
+		return this;
+	}	private static final String Invk_src_ = "src_";
+	private void Src_(Io_url v) {
+		src = v;
+		trg = bldr.App().Fsys_mgr().Wiki_dir().GenSubFil_nest(wiki.Domain_str(), v.NameOnly());	// NOTE: NameOnly() will take "enwiki.xml.bz2" and make it "enwiki.xml"
+	}	Io_url src, trg;
+	static final String GRP_KEY = "xowa.bldr.cmd.decompress_bz2";
+	public static boolean Decompress(Xoae_app app, String src_fil, Io_url trg_fil) {
+		Io_mgr.I.CreateDirIfAbsent(trg_fil.OwnerDir());	// 7zip will fail if dir does not exist
+		ProcessAdp decompress = app.Prog_mgr().App_decompress_bz2();
+		decompress.Prog_dlg_(app.Usr_dlg()).Run_mode_(ProcessAdp.Run_mode_async);
+		decompress.Run(src_fil, trg_fil, trg_fil.OwnerDir().Xto_api());
+		while (decompress.Exit_code() == ProcessAdp.Exit_init) {
+			String size = gplx.ios.Io_size_.To_str(Io_mgr.I.QueryFil(trg_fil).Size());
+			app.Usr_dlg().Prog_many(GRP_KEY, "decompress", "decompressing: ~{0}", size);
+			Thread_adp_.Sleep(1000);
+		}
+		return true;
+	}
+}

400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_deploy_copy_cmd.java

@@ -0,0 +1,70 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.xowa.bldrs.*; import gplx.xowa.tdbs.*;
+public class Xob_deploy_copy_cmd extends Xob_itm_basic_base implements Xob_cmd, GfoInvkAble {
+	public Xob_deploy_copy_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_deploy_copy;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_end() {}
+	public void Cmd_run() {
+		Io_url src_root_dir = wiki.Fsys_mgr().Root_dir();
+		Xotdb_fsys_mgr url_mgr = wiki.Tdb_fsys_mgr();
+		Copy_dir_root(src_root_dir, Xotdb_dir_info_.Name_cfg);
+		Copy_dir_ns(url_mgr.Ns_dir(), Xotdb_dir_info_.Name_page);
+		Copy_dir_ns(url_mgr.Ns_dir(), Xotdb_dir_info_.Name_title);
+		Copy_dir_root(src_root_dir, Xotdb_dir_info_.Name_site, Xotdb_dir_info_.Name_category);
+		Copy_dir_root(src_root_dir, Xotdb_dir_info_.Name_site, Xotdb_dir_info_.Name_id);
+		Copy_dir_root(src_root_dir, Xotdb_dir_info_.Name_site, Xotdb_dir_info_.Name_search_ttl);
+	}
+	public void Cmd_term() {}
+	private void Copy_dir_ns(Io_url root, String name) {
+		Io_url[] ns_dirs = Io_mgr.I.QueryDir_args(root).DirOnly_().ExecAsUrlAry();
+		for (int i = 0; i < ns_dirs.length; i++) {
+			Io_url ns_dir = ns_dirs[i];
+			Io_url src_sub_dir = ns_dir.GenSubDir(name);
+			String dir_name = name;
+			if (zip) {
+				Io_url src_zip_dir = ns_dir.GenSubDir(name + Xob_deploy_zip_cmd.Dir_zip_suffix);
+				if (Io_mgr.I.ExistsDir(src_zip_dir)) {
+					src_sub_dir = src_zip_dir;
+					dir_name = name + Xob_deploy_zip_cmd.Dir_zip_suffix;
+				}
+			}
+			Copy_dir(src_sub_dir, trg_root_dir.GenSubDir_nest(Xotdb_dir_info_.Name_ns, ns_dir.NameOnly(), dir_name));
+		}
+	}
+	private void Copy_dir_root(Io_url src_root_dir, String... sub_dirs) {
+		Io_url src = src_root_dir.GenSubDir_nest(sub_dirs);
+		Io_url trg = trg_root_dir.GenSubDir_nest(sub_dirs);
+		Copy_dir(src, trg);
+	}
+	private void Copy_dir(Io_url src, Io_url trg) {
+		bldr.Usr_dlg().Prog_many(GRP_KEY, "copy", "copying to ~{1}", src.Xto_api(), trg.Xto_api());
+		Io_mgr.I.CopyDirDeep(src, trg);
+	}
+	boolean zip; Io_url trg_root_dir;
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_trg_dir_))	{trg_root_dir = m.ReadIoUrl("val");}
+		else if	(ctx.Match(k, Invk_zip_))		{zip = m.ReadBoolOrFalse("val");}
+		else	return super.Invk(ctx, ikey, k, m);
+		return this;
+	}	private static final String Invk_trg_dir_ = "trg_dir_", Invk_zip_ = "zip_";
+	static final String GRP_KEY = "xowa.bldr.copy";
+}

400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_deploy_zip_cmd.java

@@ -0,0 +1,83 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.xowa.bldrs.*; import gplx.xowa.tdbs.*;
+public class Xob_deploy_zip_cmd extends Xob_itm_basic_base implements Xob_cmd {
+	public Xob_deploy_zip_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_deploy_zip;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_run() {Exec(bldr, Xotdb_dir_info_.Name_page);}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	private void Exec(Xob_bldr bldr, String type_name) {
+		Log("initing wiki");
+		try {wiki.Init_assert();}
+		catch (Exception exc) {Log("failed to init wiki: ~{0}", Err_.Message_gplx_brief(exc));}
+		Log("probing ns_dirs: ~{0}", wiki.Tdb_fsys_mgr().Ns_dir().Raw());
+		Io_url[] ns_dirs = Io_mgr.I.QueryDir_args(wiki.Tdb_fsys_mgr().Ns_dir()).DirOnly_().ExecAsUrlAry();
+		for (Io_url ns_dir : ns_dirs) {
+			Log("zipping dir: ~{0}", ns_dir.Raw());
+			String ns_num = ns_dir.NameOnly();
+			Xow_ns ns_itm = wiki.Ns_mgr().Ids_get_or_null(Int_.parse_(ns_num));
+			Zip_ns(bldr, ns_dir, type_name, ns_itm.Name_str());
+		}
+	}
+	private void Zip_ns(Xob_bldr bldr, Io_url root_dir, String type_name, String ns_name) {
+		bldr.Usr_dlg().Prog_one(GRP_KEY, "scan", "scanning dir ~{0}", type_name);
+		Io_url[] fils = Io_mgr.I.QueryDir_args(root_dir.GenSubDir(type_name)).Recur_().ExecAsUrlAry();
+		int fils_len = fils.length;
+		String fils_len_str = Int_.Xto_str_pad_bgn_zero(fils_len, 6);
+		for (int i = 0; i < fils_len; i++) {
+			Io_url fil = fils[i];
+			bldr.Print_prog_msg(i, fils.length, -1, "zipping ~{0} ~{1} ~{2} of ~{3}", type_name, ns_name, Int_.Xto_str_pad_bgn_zero(i, 6), fils_len_str);
+			Io_url trg_fil = Gen_trg(root_dir, fil, type_name);
+			if (String_.Eq(fil.NameAndExt(), Xotdb_dir_info_.Name_reg_fil))	// do not zip reg.csv
+				Io_mgr.I.CopyFil(fil, trg_fil, true);
+			else
+				zip_mgr.Zip_fil(fil, trg_fil.GenNewExt(Xotdb_dir_info_.Ext_zip));
+		}
+		if (delete_dirs_page) Io_mgr.I.DeleteDirDeep(root_dir.GenSubDir(type_name));
+	}
+	public boolean Delete_dirs_page() {return delete_dirs_page;} public Xob_deploy_zip_cmd Delete_dirs_page_(boolean v) {delete_dirs_page = v; return this;} private boolean delete_dirs_page = true;
+	Io_url Gen_trg(Io_url root_dir, Io_url fil, String type_name) {
+		String src_fil = fil.Xto_api();
+		int pos = String_.FindBwd(src_fil, type_name);	// SEE:NOTE_1
+		pos += String_.Len(type_name);					// get rest of String after type; EX: in "/page/000000" start with "/000000"
+		return root_dir.GenSubFil(String_.Concat(type_name, Dir_zip_suffix, String_.Mid(src_fil, pos)));
+	}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.MatchIn(k, Invk_delete_temp_, Invk_delete_dirs_page_))			delete_dirs_page = m.ReadYn("v");
+		else	return super.Invk(ctx, ikey, k, m);
+		return this;
+	}	private static final String Invk_delete_temp_ = "delete_temp_", Invk_delete_dirs_page_ = "delete_dirs_page_";
+	private void Log(String fmt, Object... args) {
+		bldr.App().Usr_dlg().Log_many(GRP_KEY, "deploy.zip.msg", fmt, args);
+	}
+	Io_zip_mgr zip_mgr = Io_zip_mgr_base._;
+	public static final String Dir_zip_suffix = "_zip";
+	static final String GRP_KEY = "xowa.bldr.zip";
+}
+/*
+NOTE_1:generate trg url
+find "type_name" to replace with "type_name_zip";
+EX:
+old: /xowa/ns/000/page/000000/000100/000123.xdat; 
+new: /xowa/ns/000/page_zip/000000/000100/000123.xdat;
+NOTE: FindBwd (and not Replace) b/c root_dir could be something like /page/xowa
+*/

400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_download_wkr.java

@@ -0,0 +1,69 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.dbs.*; import gplx.ios.*; import gplx.xowa.bldrs.*;
+public class Xob_download_wkr extends Xob_itm_basic_base implements Xob_cmd {
+	private String dump_date = "latest";
+	private String dump_type = null;
+	private String dump_src = null;
+	private Io_url dump_trg_zip = null, dump_trg_bin = null;
+	private boolean unzip = true;
+	public Xob_download_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
+	public String Cmd_key() {return Xob_cmd_keys.Key_util_download;}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {
+		if (dump_type == null) throw Exc_.new_("dump_type must be specified");
+		Xob_dump_file dump_file = Xob_dump_file.new_(wiki.Domain_str(), dump_date, dump_type);
+		if (dump_src == null) {
+			dump_file.Server_url_(app.Setup_mgr().Dump_mgr().Server_urls()[0]);
+			dump_src = dump_file.File_url();
+		}
+		if (dump_trg_zip == null)
+			dump_trg_zip = wiki.Fsys_mgr().Root_dir().GenSubFil(dump_file.File_name());
+		if (dump_trg_bin == null && unzip)
+			dump_trg_bin = dump_trg_zip.GenNewNameAndExt(dump_trg_zip.NameOnly());	// convert a.sql.gz -> a.sql
+	}
+	public void Cmd_run() {
+		usr_dlg.Note_many("", "", "downloading file: now=~{0} src=~{1} trg=~{2}", DateAdp_.Now().XtoStr_fmt_yyyyMMdd_HHmmss(), dump_src, dump_trg_zip.OwnerDir());
+		IoEngine_xrg_downloadFil download_wkr = app.Wmf_mgr().Download_wkr().Download_xrg();
+		download_wkr.Src_last_modified_query_(false).Init(dump_src, dump_trg_zip);
+		if (!download_wkr.Exec())
+			usr_dlg.Warn_many("", "", "download failed: src=~{0} trg=~{1} err=~{2}", dump_src, dump_trg_zip.Raw(), Err_.Message_gplx_brief(download_wkr.Rslt_err()));
+		if (unzip) {
+			usr_dlg.Note_many("", "", "unzipping file: now=~{0} trg=~{1}", DateAdp_.Now().XtoStr_fmt_yyyyMMdd_HHmmss(), dump_trg_bin.Raw());
+			Xob_unzip_wkr unzip_wkr = new Xob_unzip_wkr().Init(app).Process_run_mode_(ProcessAdp.Run_mode_sync_block);
+			unzip_wkr.Decompress(dump_trg_zip, dump_trg_bin);
+		}
+	}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(String_.Eq(k, Invk_dump_date_))			dump_date = m.ReadStr("v");
+		else if	(String_.Eq(k, Invk_dump_type_))			dump_type = m.ReadStr("v");
+		else if	(String_.Eq(k, Invk_unzip_))				unzip = m.ReadYn("v");
+		else if	(String_.Eq(k, Invk_dump_src_))				dump_src = m.ReadStr("v");
+		else if	(String_.Eq(k, Invk_dump_trg_zip_))			dump_trg_zip = m.ReadIoUrl("v");
+		else if	(String_.Eq(k, Invk_dump_trg_bin_))			dump_trg_bin = m.ReadIoUrl("v");
+		else												return GfoInvkAble_.Rv_unhandled;
+		return this;
+	}
+	private static final String
+	  Invk_dump_date_ = "dump_date_", Invk_dump_type_ = "dump_type_", Invk_unzip_ = "unzip_"
+	, Invk_dump_src_ = "dump_src_", Invk_dump_trg_zip_ = "dump_trg_zip_", Invk_dump_trg_bin_ = "dump_trg_bin_"
+	;
+}

400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_exec_sql_cmd.java

@@ -0,0 +1,43 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
+import gplx.xowa.wikis.data.*; import gplx.xowa.dbs.*;
+public class Xob_exec_sql_cmd implements Xob_cmd {
+	private Xowe_wiki wiki; private int file_idx = -1; private String sql;
+	public Xob_exec_sql_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.wiki = wiki;}
+	public String Cmd_key() {return Xob_cmd_keys.Key_exec_sql;}
+	public void Cmd_run() {
+		Xoae_app app = wiki.Appe();
+		wiki.Init_assert();	// force load; needed to pick up MediaWiki ns for MediaWiki:mainpage
+		Xodb_mgr_sql db_mgr = wiki.Db_mgr_as_sql();
+		Xowd_db_mgr fsys_mgr = db_mgr.Core_data_mgr();
+		Xowd_db_file file = fsys_mgr.Dbs__get_at(file_idx);
+		app.Usr_dlg().Plog_many("", "", "exec_sql: running sql; file_idx=~{0} sql=~{1}", file_idx, sql);
+		file.Conn().Exec_sql(sql);
+	}
+	public void Cmd_init(Xob_bldr bldr) {}
+	public void Cmd_bgn(Xob_bldr bldr) {}
+	public void Cmd_end() {}
+	public void Cmd_term() {}
+	public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
+		if		(ctx.Match(k, Invk_file_idx_))				file_idx = m.ReadInt("v");
+		else if	(ctx.Match(k, Invk_sql_))					sql = m.ReadStr("v");
+		return this;
+	}
+	private static final String Invk_file_idx_ = "file_idx_", Invk_sql_ = "sql_";
+}