gnosygnu_xowa/400_xowa/src/gplx/xowa/bldrs/Xob_bldr.java

/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.consoles.*; import gplx.core.envs.*;
import gplx.xowa.apps.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.xmls.*; import gplx.xowa.bldrs.cfgs.*; import gplx.xowa.langs.bldrs.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.langs.jsons.*;
public class Xob_bldr implements Gfo_invk {
	private boolean pause_at_end = false; private long prv_prog_time; private Xob_xml_parser dump_parser;
	public Xob_bldr(Xoae_app app) {
		this.app = app;
		this.cmd_mgr = new Xob_cmd_mgr(this, cmd_regy);
		this.import_marker = new Xob_import_marker();
		this.wiki_cfg_bldr = new Xob_wiki_cfg_bldr(this);
	}
	public Xoae_app				App() {return app;} private final    Xoae_app app;
	public Xob_cmd_regy			Cmd_regy() {return cmd_regy;} private final    Xob_cmd_regy cmd_regy = new Xob_cmd_regy();
	public Xob_cmd_mgr			Cmd_mgr() {return cmd_mgr;} private final    Xob_cmd_mgr cmd_mgr;
	public Gfo_usr_dlg			Usr_dlg() {return app.Usr_dlg();}
	public int					Sort_mem_len() {return sort_mem_len;} public Xob_bldr Sort_mem_len_(int v) {sort_mem_len = v; return this;} private int sort_mem_len = 16 * Io_mgr.Len_mb;
	public int					Dump_fil_len() {return dump_fil_len;} public Xob_bldr Dump_fil_len_(int v) {dump_fil_len = v; return this;} private int dump_fil_len =  1 * Io_mgr.Len_mb;
	public int					Make_fil_len() {return make_fil_len;} public Xob_bldr Make_fil_len_(int v) {make_fil_len = v; return this;} private int make_fil_len = 64 * Io_mgr.Len_kb;
	public Xob_xml_parser		Dump_parser() {if (dump_parser == null) this.dump_parser = new Xob_xml_parser(); return dump_parser;}
	public Xob_import_marker	Import_marker() {return import_marker;} private Xob_import_marker import_marker;
	public Xob_wiki_cfg_bldr	Wiki_cfg_bldr() {return wiki_cfg_bldr;} private Xob_wiki_cfg_bldr wiki_cfg_bldr;
	public void					Pause_at_end_(boolean v) {this.pause_at_end = v;}
	public void					Print_prog_msg(long cur, long end, int pct_idx, String fmt, Object... ary) {
		long now = System_.Ticks(); if (now - prv_prog_time < 100) return;
		this.prv_prog_time = now;
		if (pct_idx > -1) ary[pct_idx] = Decimal_adp_.CalcPctStr(cur, end, "00.00");
		app.Usr_dlg().Prog_many("", "", fmt, ary);
	}
	public Xob_bldr Exec_json(String script) {
		try {
			this.cmd_mgr.Clear();
			Json_parser jdoc_parser = new Json_parser();
			Json_doc jdoc = jdoc_parser.Parse(script);
			Json_ary cmds = jdoc.Root_ary();
			int cmds_len = cmds.Len();
			for (int i = 0; i < cmds_len; ++i) {
				Json_nde cmd = cmds.Get_at_as_nde(i);
				byte[] key = cmd.Get_bry_or_null("key");
				Xob_cmd prime = cmd_regy.Get_or_null(String_.new_u8(key));
				if (prime == null) throw Err_.new_("bldr", "bldr.cmd does not exists: cmd={0}", key);
				byte[] wiki_key = cmd.Get_bry_or_null("wiki");
				Xowe_wiki wiki = wiki_key == null ? app.Usere().Wiki() : app.Wiki_mgr().Get_by_or_make(wiki_key);
				Xob_cmd clone = prime.Cmd_clone(this, wiki);
				int atrs_len = cmd.Len();
				for (int j = 0; j < atrs_len; ++j) {
					Json_kv atr_kv = cmd.Get_at_as_kv(j);
					String atr_key = atr_kv.Key_as_str();
					if	(	String_.Eq(atr_key, "key")
						||	String_.Eq(atr_key, "wiki"))	continue;
					byte[] atr_val = atr_kv.Val_as_bry();
					Gfo_invk_.Invk_by_val(clone, atr_key + Gfo_invk_.Mutator_suffix, String_.new_u8(atr_val));
				}
				cmd_mgr.Add(clone);
			}
			gplx.core.threads.Thread_adp_.Start_by_key("bldr_by_json", this, Invk_run_by_kit);
		} catch (Exception e) {
			app.Gui_mgr().Kit().Ask_ok("", "", "error: ~{0}", Err_.Message_gplx_log(e));
		}
		return this;
	}
	private void Run_by_kit() {	// same as Run, but shows exception; don't want to change backward compatibility on Run
		try {this.Run();}
		catch (Exception e) {
			String log_msg = Err_.Message_gplx_log(e);
			Xoa_app_.Usr_dlg().Log_many("", "", log_msg);
			app.Gui_mgr().Kit().Ask_ok("", "", "error: ~{0}", Err_.Message_gplx_full(e));
		}
	}
	public void Run() {
		try {
			app.Bldr__running_(true);
			app.Launch();	// HACK: bldr will be called by a gfs file which embeds "bldr.run" inside it; need to call Launch though before Run; DATE:2013-03-23
			long time_bgn = System_.Ticks();
			int cmd_mgr_len = cmd_mgr.Len();
			for (int i = 0; i < cmd_mgr_len; i++) {
				Xob_cmd cmd = cmd_mgr.Get_at(i);
				cmd.Cmd_init(this);
			}
			cmd_mgr_len = cmd_mgr.Len(); // NOTE: refresh len b/c other cmds may have added new ones in Cmd_init
			for (int i = 0; i < cmd_mgr_len; i++) {
				Xob_cmd cmd = cmd_mgr.Get_at(i);
				app.Usr_dlg().Note_many("", "", "cmd bgn: ~{0}", cmd.Cmd_key());
				long time_cur = System_.Ticks();
				try {
					cmd.Cmd_bgn(this);
					cmd.Cmd_run();
					cmd.Cmd_end();
				} catch (Exception e) {
					throw Err_.new_exc(e, "bldr", "unknown error", "key", cmd.Cmd_key());
				}
				System_.Garbage_collect();
				app.Usr_dlg().Note_many("", "", "cmd end: ~{0} ~{1}", cmd.Cmd_key(), Time_span_.from_(time_cur).XtoStrUiAbbrv());
			}
			for (int i = 0; i < cmd_mgr_len; i++) {
				Xob_cmd cmd = cmd_mgr.Get_at(i);
				cmd.Cmd_term();
			}
			app.Usr_dlg().Note_many("", "", "bldr done: ~{0}", Time_span_.from_(time_bgn).XtoStrUiAbbrv());
			cmd_mgr.Clear();
			if (pause_at_end && !Env_.Mode_testing()) {Console_adp__sys.Instance.Read_line("press enter to continue");}
		}
		catch (Exception e) {
			app.Bldr__running_(false);
			throw Err_.new_exc(e, "bldr", "unknown error");
		}
	}
	private void Cancel() {
		int cmd_mgr_len = cmd_mgr.Len();
		for (int i = 0; i < cmd_mgr_len; i++) {
			Xob_cmd cmd = cmd_mgr.Get_at(i);
			cmd.Cmd_end();
		}
	}
	public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
		if		(ctx.Match(k, Invk_pause_at_end_))			pause_at_end = m.ReadBoolOrTrue("val");
		else if	(ctx.Match(k, Invk_cmds))					return cmd_mgr;
		else if	(ctx.Match(k, Invk_wiki_cfg_bldr))			return wiki_cfg_bldr;
		else if	(ctx.Match(k, Invk_sort_mem_len_)) 			sort_mem_len = gplx.core.ios.Io_size_.Load_int_(m);
		else if	(ctx.Match(k, Invk_dump_fil_len_)) 			dump_fil_len = gplx.core.ios.Io_size_.Load_int_(m);
		else if	(ctx.Match(k, Invk_make_fil_len_)) 			make_fil_len = gplx.core.ios.Io_size_.Load_int_(m);
		else if	(ctx.Match(k, Invk_run)) 					Run();
		else if	(ctx.Match(k, Invk_run_by_kit)) 			Run_by_kit();
		else if	(ctx.Match(k, Invk_cancel)) 				Cancel();
		else	return Gfo_invk_.Rv_unhandled;
		return this;
	}
	private static final String 
	  Invk_cmds = "cmds", Invk_wiki_cfg_bldr = "wiki_cfg_bldr"
	, Invk_pause_at_end_ = "pause_at_end_", Invk_sort_mem_len_ = "sort_mem_len_", Invk_dump_fil_len_ = "dump_fil_len_", Invk_make_fil_len_ = "make_fil_len_"
	, Invk_cancel = "cancel"
	, Invk_run_by_kit = "run_by_kit"
	;
	public static final String Invk_run = "run";
}
/*
. make_fil_len: max size of made file; EX: /id/..../0000000001.csv will have max len of 64 KB
. dump_fil_len: max size of temp file; EX: /tmp/.../0000000001.csv will have max len of 1 MB
. sort_mem_len: max size of memory for external merge process; note the following
.. a continguous range of memory of that size will be needed: "Bry_bfr_.New(sort_mem_len)" will be called
.. large sort_mem_len will result in smaller number of merge files
... EX: 16 MB will take en.wikipedia.org's 640 MB title files and generate 40 temp files of 8 MB each
.. number of merge files is number of open file channels during merge process
... 40 is a "reasonable" number; the 1st max is 512 (for older windows OS's) and 2048 for Windows XP; Linux seems to be about 7000
.. small sort_mem_len will use smaller buffer; 16 MB / 40 files -> 400 kb buffer for each file
... do not go under max page size for a given row
... for example, a 100 b buffer will fail if a given row is > 100 b (the entire row won't be loaded in memory)
.. smaller buffer will mean more refills which will require more I/O
... EX: 400 kb buffer will require at least 20 refills to read the entire 8 MB file
*/