From 5c3e4287f0dbf48e78fc418a83def85adf125703 Mon Sep 17 00:00:00 2001 From: gnosygnu Date: Fri, 30 Dec 2016 12:22:43 -0500 Subject: [PATCH] Mass_parse: Refactor make cmd to merge other wkr tables such as log_basic_temp --- .../mass_parses/makes/Xomp_make_cmd.java | 2 +- .../mass_parses/makes/Xomp_make_cmd_cfg.java | 10 +- .../mass_parses/makes/Xomp_make_lnki.java | 100 ++++++------------ .../mass_parses/makes/Xomp_make_merger.java | 25 +++++ .../makes/Xomp_make_merger__base.java | 100 ++++++++++++++++++ .../makes/Xomp_make_merger__lnki_temp.java | 46 ++++++++ .../makes/Xomp_make_merger__xnde.java | 67 ++++++++++++ .../parses/mgrs/Xomp_parse_mgr_cfg.java | 3 + .../parses/wkrs/Xomp_parse_wkr.java | 6 +- .../xowa/parsers/logs/Xop_log_basic_tbl.java | 15 +-- .../xowa/parsers/logs/Xop_log_basic_wkr.java | 5 +- .../xowa/parsers/logs/Xop_log_invoke_wkr.java | 5 +- .../gplx/xowa/parsers/logs/Xop_log_mgr.java | 6 +- .../parsers/logs/Xop_log_property_wkr.java | 6 +- .../parsers/logs/Xop_log_wkr_factory.java | 29 +++++ .../src/gplx/xowa/xtns/math/Xomath_core.java | 7 +- 16 files changed, 338 insertions(+), 94 deletions(-) create mode 100644 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger.java create mode 100644 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__base.java create mode 100644 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__lnki_temp.java create mode 100644 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__xnde.java create mode 100644 400_xowa/src/gplx/xowa/parsers/logs/Xop_log_wkr_factory.java diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java index 7e6374ead..7b491e809 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java @@ -23,7 +23,7 @@ public class Xomp_make_cmd extends Xob_cmd__base { @Override public void Cmd_run() { wiki.Init_assert(); new Xomp_make_html().Exec(wiki, cfg); - new Xomp_make_lnki().Exec(wiki, 10000); + new Xomp_make_lnki().Exec(wiki, cfg, 10000); } @Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { if (ctx.Match(k, Invk__cfg)) return cfg; diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd_cfg.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd_cfg.java index fe697b1db..32386974d 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd_cfg.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd_cfg.java @@ -18,12 +18,16 @@ along with this program. If not, see . package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; public class Xomp_make_cmd_cfg implements Gfo_invk { public boolean Delete_html_dbs() {return delete_html_dbs;} private boolean delete_html_dbs = true; + public Ordered_hash Merger_wkrs() {return merger_wkrs;} private final Ordered_hash merger_wkrs = Ordered_hash_.New(); public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { if (ctx.Match(k, Invk__delete_html_dbs_)) delete_html_dbs = m.ReadYn("v"); + else if (ctx.Match(k, Invk__merger_wkrs_)) { + String[] ary = m.ReadStrAry("k", "|"); + for (String itm : ary) + merger_wkrs.Add(itm, itm); + } else return Gfo_invk_.Rv_unhandled; return this; } - private static final String - Invk__delete_html_dbs_ = "delete_html_dbs_" - ; + private static final String Invk__delete_html_dbs_ = "delete_html_dbs_", Invk__merger_wkrs_ = "merger_wkrs_"; } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java index 184229cc0..63cff8829 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java @@ -16,94 +16,60 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; -import gplx.core.brys.*; import gplx.core.primitives.*; -import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*; import gplx.xowa.addons.bldrs.files.dbs.*; -import gplx.xowa.bldrs.*; +import gplx.xowa.addons.bldrs.mass_parses.dbs.*; class Xomp_make_lnki { - public void Exec(Xowe_wiki wiki, int uid_count) { + public void Exec(Xowe_wiki wiki, Xomp_make_cmd_cfg cfg, int uid_count) { // init - Xomp_mgr_db xomp_db = Xomp_mgr_db.New__load(wiki); + Xomp_mgr_db src_mgr_db = Xomp_mgr_db.New__load(wiki); - Xob_db_file make_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()); - Xob_lnki_temp_tbl lnki_temp_tbl = new Xob_lnki_temp_tbl(make_db.Conn()); - make_db.Conn().Meta_tbl_remake(lnki_temp_tbl); - lnki_temp_tbl.Insert_bgn(); + // make mergers; always add lnki_temp + List_adp merger_list = List_adp_.New(); + merger_list.Add(new Xomp_make_merger__lnki_temp()); + if (cfg.Merger_wkrs().Has("xnde")) merger_list.Add(new Xomp_make_merger__xnde()); + Xomp_make_merger[] merger_ary = (Xomp_make_merger[])merger_list.To_ary_and_clear(Xomp_make_merger.class); // create ary; add index - int wkr_count = xomp_db.Tbl__wkr().Select_count(); - Xomp_wkr_db[] db_ary = new Xomp_wkr_db[wkr_count]; + int wkr_count = src_mgr_db.Tbl__wkr().Select_count(); + Xomp_wkr_db[] src_mgr_dbs = new Xomp_wkr_db[wkr_count]; for (int i = 0; i < wkr_count; ++i) { - Xomp_wkr_db wkr_db = Xomp_wkr_db.New(xomp_db.Dir(), i); - db_ary[i] = wkr_db; - wkr_db.Conn().Meta_idx_assert("lnki_temp", "lnki_page_id", "lnki_page_id"); + Xomp_wkr_db src_wkr_db = Xomp_wkr_db.New(src_mgr_db.Dir(), i); + src_mgr_dbs[i] = src_wkr_db; } + // run init + for (Xomp_make_merger merger : merger_ary) + merger.Merger__init(wiki, src_mgr_dbs); + // get max xomp_uid; note that xomp_uid is (a) per page; (b) ordered by page_ns, page_id; (c) starts from 1 - int max_xomp_uid = xomp_db.Conn().Exec_select_max_as_int("xomp_page", "xomp_uid", -1); + int max_xomp_uid = src_mgr_db.Conn().Exec_select_max_as_int("xomp_page", "xomp_uid", -1); // loop over wkrs using range of xomp_uid - List_adp rows = List_adp_.New(); - int cur_xomp_uid = -1; Int_obj_ref lnki_id_ref = Int_obj_ref.New_zero(); + int cur_xomp_uid = -1; while (true) { + // load rows + Gfo_usr_dlg_.Instance.Prog_many("", "", "merging rows; bgn_uid=~{0} end_uid=~{1}", cur_xomp_uid, cur_xomp_uid + uid_count); int tmp_xomp_uid_max = -1; // maximum uid for a grp of wkrs; EX: looping over 8 wkrs with xomp_uid range of 1 - 1000; max xomp_uid may only be 990 b/c pages are missing / failed for (int i = 0; i < wkr_count; ++i) { - Xomp_wkr_db wkr_db = db_ary[i]; - int wkr_uid_max = Load_rows(rows, xomp_db, wkr_db, cur_xomp_uid, cur_xomp_uid + uid_count, lnki_id_ref); + Xomp_wkr_db src_wkr_db = src_mgr_dbs[i]; + int wkr_uid_max = tmp_xomp_uid_max; + for (Xomp_make_merger merger : merger_ary) + wkr_uid_max = merger.Merger__load(src_mgr_db, src_wkr_db, cur_xomp_uid, cur_xomp_uid + uid_count); if (wkr_uid_max > tmp_xomp_uid_max) tmp_xomp_uid_max = wkr_uid_max; } + + // save rows + for (Xomp_make_merger merger : merger_ary) + merger.Merger__save(); + // NOTE: not ">=" else small wikis will fail with 0 images; EX:cs.q; DATE:2016-09-04 if (tmp_xomp_uid_max > max_xomp_uid || tmp_xomp_uid_max == -1) break; // if max_xomp_uid seen, break; note that ">" necessary because max_xomp_uid may not be in set of wkrs; cur_xomp_uid += uid_count; // note that this sequentially counts up by uid_count (1000), so inevitable that cur_xomp_uid will exceed wkr_uid_max - Gfo_usr_dlg_.Instance.Prog_many("", "", "building lnki_temp; cur_xomp_uid=~{0}", cur_xomp_uid); - Save_rows(rows, lnki_temp_tbl); } - // term - lnki_temp_tbl.Insert_end(); - xomp_db.Conn().Rls_conn(); - make_db.Conn().Rls_conn(); - } - private int Load_rows(List_adp rows, Xomp_mgr_db xomp_db, Xomp_wkr_db wkr_db, int uid_bgn, int uid_end, Int_obj_ref lnki_id) { - // build sql - Db_attach_mgr attach_mgr = new Db_attach_mgr(xomp_db.Conn()); - attach_mgr.Conn_links_(new Db_attach_itm("wkr_db", wkr_db.Conn())); - String sql = String_.Format(String_.Concat_lines_nl_skip_last - ( "SELECT mgr.xomp_uid" - , ", wkr.*" - , ", wkr.lnki_tier_id, wkr.lnki_page_id, wkr.lnki_ttl, wkr.lnki_commons_ttl, wkr.lnki_ext, wkr.lnki_type, wkr.lnki_src_tid, wkr.lnki_w, wkr.lnki_h, wkr.lnki_upright, wkr.lnki_time, wkr.lnki_page" - , "FROM lnki_temp wkr" - , " JOIN xomp_page mgr ON wkr.lnki_page_id = mgr.page_id" - , "WHERE mgr.xomp_uid > {0} AND mgr.xomp_uid <= {1}" // mgr.xomp_uid will sort pages by ns_id, page_id - ) - , uid_bgn - , uid_end - ); - sql = attach_mgr.Resolve_sql(sql); - - attach_mgr.Attach(); - Db_rdr rdr = xomp_db.Conn().Stmt_sql(sql).Exec_select__rls_auto(); // ANSI.Y - int rv = -1; - try { - while (rdr.Move_next()) { - rv = rdr.Read_int("xomp_uid"); - Xob_lnki_temp_row row = new Xob_lnki_temp_row(); - row.Load(rdr, lnki_id.Val_add_pre()); - rows.Add(row); - } - } finally {rdr.Rls();} - attach_mgr.Detach(); - return rv; - } - - private void Save_rows(List_adp rows, Xob_lnki_temp_tbl lnki_temp_tbl) { - int len = rows.Len(); - for (int i = 0; i < len; ++i) { - Xob_lnki_temp_row row = (Xob_lnki_temp_row)rows.Get_at(i); - lnki_temp_tbl.Insert_cmd_by_batch(row.Lnki_tier_id(), row.Lnki_page_id(), row.Lnki_ttl(), row.Lnki_commons_ttl() - , row.Lnki_ext(), row.Lnki_type(), row.Lnki_src_tid(), row.Lnki_w(), row.Lnki_h(), row.Lnki_upright() - , row.Lnki_time(), row.Lnki_page()); - } - rows.Clear(); + // save rows + for (Xomp_make_merger merger : merger_ary) + merger.Merger__term(); + src_mgr_db.Conn().Rls_conn(); } } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger.java new file mode 100644 index 000000000..481048b22 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger.java @@ -0,0 +1,25 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; +import gplx.xowa.addons.bldrs.mass_parses.dbs.*; +interface Xomp_make_merger { + void Merger__init(Xowe_wiki wiki, Xomp_wkr_db[] src_dbs); + int Merger__load(Xomp_mgr_db src_mgr_db, Xomp_wkr_db src_wkr_db, int uid_bgn, int uid_end); + void Merger__save(); + void Merger__term(); +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__base.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__base.java new file mode 100644 index 000000000..f43fbb7d4 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__base.java @@ -0,0 +1,100 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; +import gplx.dbs.*; import gplx.xowa.bldrs.*; +import gplx.xowa.addons.bldrs.mass_parses.dbs.*; +// merges xomp.wkr_dbs into xowa.file.make.sqlite3 +// NOTE: can't do INSERT b/c (a) autonum id is same in diff worker dbs; (b): want autonum to match ns_id, page_id order +abstract class Xomp_make_merger__base implements Xomp_make_merger, gplx.core.lists.ComparerAble { + private Xob_db_file trg_db; + protected Db_tbl trg_tbl; + private List_adp rows = List_adp_.New(); + private String src_tbl__name, src_fld__page_id; + public void Merger__init(Xowe_wiki wiki, Xomp_wkr_db[] src_dbs) { + // get trg.db + this.trg_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()); + + // make trg.tbl + this.trg_tbl = Init__trg_tbl(trg_db); + trg_db.Conn().Meta_tbl_remake(trg_tbl); + + // make idxs on src.tbls + this.src_tbl__name = this.trg_tbl.Tbl_name(); + this.src_fld__page_id = this.Init__src_fld__page_id(); + int len = src_dbs.length; + for (int i = 0; i < len; ++i) { + src_dbs[i].Conn().Meta_idx_assert(src_tbl__name, src_fld__page_id, src_fld__page_id); + } + + // do any other init, such as init'ing insert stmt + this.Init__trg_bgn(); + this.trg_db.Conn().Txn_bgn("merger__" + src_tbl__name); + } + protected abstract Db_tbl Init__trg_tbl(Xob_db_file trg_db); + protected abstract String Init__src_fld__page_id(); + @gplx.Virtual protected void Init__trg_bgn() {} + + public int Merger__load(Xomp_mgr_db src_mgr_db, Xomp_wkr_db src_wkr_db, int uid_bgn, int uid_end) { + // build sql + Db_attach_mgr attach_mgr = new Db_attach_mgr(src_mgr_db.Conn()); + attach_mgr.Conn_links_(new Db_attach_itm("src_wkr_db", src_wkr_db.Conn())); + String sql = Db_sql_.Make_by_fmt(String_.Ary + ( "SELECT src_mgr.xomp_uid" + , ", src_wkr.*" + , "FROM {0} src_wkr" + , " JOIN xomp_page src_mgr ON src_wkr.{1} = src_mgr.page_id" + , "WHERE src_mgr.xomp_uid > {2} AND src_mgr.xomp_uid <= {3}" // NOTE: mgr.xomp_uid will sort pages by ns_id, page_id + ) + , src_tbl__name, src_fld__page_id + , uid_bgn, uid_end + ); + sql = attach_mgr.Resolve_sql(sql); + + // load rows + attach_mgr.Attach(); + Db_rdr rdr = src_mgr_db.Conn().Stmt_sql(sql).Exec_select__rls_auto(); + int rv = -1; + try { + while (rdr.Move_next()) { + rv = rdr.Read_int("xomp_uid"); + Object row = Load__src_row(rdr); + rows.Add(row); + } + } finally {rdr.Rls();} + attach_mgr.Detach(); + return rv; + } + protected abstract Object Load__src_row(Db_rdr rdr); + + public void Merger__save() { + rows.Sort_by(this); + int len = rows.Len(); + for (int i = 0; i < len; ++i) { + Save__trg_row(rows.Get_at(i)); + } + rows.Clear(); + } + protected abstract void Save__trg_row(Object row_obj); + public void Merger__term() { + this.trg_db.Conn().Txn_end(); + } + public int compare(Object lhsObj, Object rhsObj) { + return Compare__hook(lhsObj, rhsObj); + } + protected abstract int Compare__hook(Object lhsObj, Object rhsObj); +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__lnki_temp.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__lnki_temp.java new file mode 100644 index 000000000..e532a4a80 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__lnki_temp.java @@ -0,0 +1,46 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; +import gplx.dbs.*; import gplx.xowa.bldrs.*; +import gplx.xowa.addons.bldrs.files.dbs.*; +class Xomp_make_merger__lnki_temp extends Xomp_make_merger__base { + private int lnki_id; + private Xob_lnki_temp_tbl trg_tbl__lnki_temp; + @Override protected Db_tbl Init__trg_tbl(Xob_db_file trg_db) { + this.trg_tbl__lnki_temp = new Xob_lnki_temp_tbl(trg_db.Conn()); + return trg_tbl__lnki_temp; + } + @Override protected String Init__src_fld__page_id() {return "lnki_page_id";} + @Override protected void Init__trg_bgn() {trg_tbl__lnki_temp.Insert_stmt_make();} + @Override protected Object Load__src_row(Db_rdr rdr) { + Xob_lnki_temp_row rv = new Xob_lnki_temp_row(); + rv.Load(rdr, ++lnki_id); + return rv; + } + @Override protected void Save__trg_row(Object row_obj) { + Xob_lnki_temp_row row = (Xob_lnki_temp_row)row_obj; + trg_tbl__lnki_temp.Insert_cmd_by_batch(row.Lnki_tier_id(), row.Lnki_page_id(), row.Lnki_ttl(), row.Lnki_commons_ttl() + , row.Lnki_ext(), row.Lnki_type(), row.Lnki_src_tid(), row.Lnki_w(), row.Lnki_h(), row.Lnki_upright() + , row.Lnki_time(), row.Lnki_page()); + } + @Override protected int Compare__hook(Object lhsObj, Object rhsObj) { + Xob_lnki_temp_row lhs = (Xob_lnki_temp_row)lhsObj; + Xob_lnki_temp_row rhs = (Xob_lnki_temp_row)rhsObj; + return Int_.Compare(lhs.Lnki_page_id(), rhs.Lnki_page_id()); + } +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__xnde.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__xnde.java new file mode 100644 index 000000000..2ac8123a9 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__xnde.java @@ -0,0 +1,67 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; +import gplx.dbs.*; import gplx.xowa.bldrs.*; +import gplx.xowa.addons.bldrs.mass_parses.dbs.*; +import gplx.xowa.addons.bldrs.files.dbs.*; +import gplx.xowa.parsers.logs.*; +class Xomp_make_merger__xnde extends Xomp_make_merger__base { + private Xop_log_basic_tbl trg_tbl__log_basic_temp; + @Override protected Db_tbl Init__trg_tbl(Xob_db_file trg_db) { + this.trg_tbl__log_basic_temp = new Xop_log_basic_tbl(trg_db.Conn()); + return trg_tbl__log_basic_temp; + } + @Override protected String Init__src_fld__page_id() {return "page_id";} + @Override protected void Init__trg_bgn() {} // NOTE: trg_tbl has implicit insert_stmt creation + @Override protected Object Load__src_row(Db_rdr rdr) { + Xop_log_basic_row rv = new Xop_log_basic_row(); + rv.Load(rdr); + return rv; + } + @Override protected void Save__trg_row(Object row_obj) { + Xop_log_basic_row row = (Xop_log_basic_row)row_obj; + trg_tbl__log_basic_temp.Insert(row.Log_tid, row.Log_msg, row.Log_time, row.Page_id, row.Page_ttl, row.Args_len, row.Args_str, row.Src_len, row.Src_str); + } + @Override protected int Compare__hook(Object lhsObj, Object rhsObj) { + Xop_log_basic_row lhs = (Xop_log_basic_row)lhsObj; + Xop_log_basic_row rhs = (Xop_log_basic_row)rhsObj; + return Int_.Compare(lhs.Page_id, rhs.Page_id); + } +} +class Xop_log_basic_row { + public int Log_tid; + public String Log_msg; + public int Log_time; + public int Page_id; + public String Page_ttl; + public int Args_len; + public String Args_str; + public int Src_len; + public String Src_str; + public void Load(Db_rdr rdr) { + this.Log_tid = rdr.Read_int("log_tid"); + this.Log_msg = rdr.Read_str("log_msg"); + this.Log_time = rdr.Read_int("log_time"); + this.Page_id = rdr.Read_int("page_id"); + this.Page_ttl = rdr.Read_str("page_ttl"); + this.Args_len = rdr.Read_int("args_len"); + this.Args_str = rdr.Read_str("args_str"); + this.Src_len = rdr.Read_int("src_len"); + this.Src_str = rdr.Read_str("src_str"); + } +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java index 554cacf58..b10ae4e65 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java @@ -32,6 +32,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk { public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true; public boolean Load_all_imglinks() {return load_all_imglinks;} private boolean load_all_imglinks = true; public String Load_ifexists_ns() {return load_ifexists_ns;} private String load_ifexists_ns = null; + public boolean Log_math() {return log_math;} private boolean log_math = false; public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_tid_.Tid__gzip; public Io_url Mgr_url() {return mgr_url;} private Io_url mgr_url; public String Wkr_machine_name() {return wkr_machine_name;} private String wkr_machine_name; @@ -62,6 +63,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk { else if (ctx.Match(k, Invk__wkr_machine_name_)) wkr_machine_name = m.ReadStr("v"); else if (ctx.Match(k, Invk__show_msg__fetched_pool_)) show_msg__fetched_pool = m.ReadYn("v"); else if (ctx.Match(k, Invk__hdump_catboxes_)) hdump_catboxs = m.ReadYn("v"); + else if (ctx.Match(k, Invk__log_math_)) log_math = m.ReadYn("v"); else return Gfo_invk_.Rv_unhandled; return this; } @@ -71,6 +73,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk { , Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_", Invk__zip_tid_ = "zip_tid_" , Invk__load_all_templates_ = "load_all_templates_", Invk__load_all_imglinks_ = "load_all_imglinks_", Invk__load_ifexists_ns_ = "load_ifexists_ns_", Invk__manual_now_ = "manual_now_" , Invk__hdump_catboxes_ = "hdump_catboxes_" + , Invk__log_math_ = "log_math_" , Invk__mgr_url_ = "mgr_url_", Invk__wkr_machine_name_ = "wkr_machine_name_" , Invk__show_msg__fetched_pool_ = "show_msg__fetched_pool_" ; diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java index 5e22ac9e8..fc3771185 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java @@ -19,7 +19,7 @@ package gplx.xowa.addons.bldrs.mass_parses.parses.wkrs; import gplx.*; import gp import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*; import gplx.xowa.files.origs.*; import gplx.xowa.htmls.core.bldrs.*; -import gplx.xowa.parsers.*; +import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.pools.*; public class Xomp_parse_wkr implements Gfo_invk { // mgr vars @@ -80,6 +80,10 @@ public class Xomp_parse_wkr implements Gfo_invk { logger.Bgn(); } + // init log_mgr / property_wkr + Xop_log_wkr_factory wkr_factory = new Xop_log_wkr_factory(wkr_db.Conn()); + if (cfg.Log_math()) wiki.Parser_mgr().Math__core().Log_wkr_(wkr_factory); + // enable hdump hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled()).Zip_tid_(cfg.Zip_tid()); hdump_bldr.Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl())); diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_tbl.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_tbl.java index 55f4e1003..8bf10b8b7 100644 --- a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_tbl.java +++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_tbl.java @@ -17,14 +17,15 @@ along with this program. If not, see . */ package gplx.xowa.parsers.logs; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.dbs.engines.sqlite.*; -public class Xop_log_basic_tbl { +public class Xop_log_basic_tbl implements Db_tbl { private Db_stmt stmt_insert; - public Xop_log_basic_tbl(Db_conn conn){this.conn = conn; this.Create_table();} + public Xop_log_basic_tbl(Db_conn conn){this.conn = conn; this.Create_tbl();} public Db_conn Conn() {return conn;} private Db_conn conn; - private void Create_table() {Sqlite_engine_.Tbl_create(conn, Tbl_name, Tbl_sql);} - public void Delete() {conn.Exec_qry(Db_qry_delete.new_all_(Tbl_name));} + public String Tbl_name() {return TBL_NAME;} + public void Create_tbl() {Sqlite_engine_.Tbl_create(conn, TBL_NAME, Tbl_sql);} + public void Delete() {conn.Exec_qry(Db_qry_delete.new_all_(TBL_NAME));} public void Insert(int log_tid, String log_msg, int log_time, int page_id, String page_ttl, int args_len, String args_str, int src_len, String src_str) { - if (stmt_insert == null) stmt_insert = Db_stmt_.new_insert_(conn, Tbl_name, Fld_log_tid, Fld_log_msg, Fld_log_time, Fld_page_id, Fld_page_ttl, Fld_args_len, Fld_args_str, Fld_src_len, Fld_src_str); + if (stmt_insert == null) stmt_insert = Db_stmt_.new_insert_(conn, TBL_NAME, Fld_log_tid, Fld_log_msg, Fld_log_time, Fld_page_id, Fld_page_ttl, Fld_args_len, Fld_args_str, Fld_src_len, Fld_src_str); stmt_insert.Clear() .Val_int(log_tid) .Val_str(log_msg) @@ -40,13 +41,13 @@ public class Xop_log_basic_tbl { public void Rls() { stmt_insert.Rls(); } - public static final String Tbl_name = "log_basic_temp" + public static final String TBL_NAME = "log_basic_temp" , Fld_log_tid = "log_tid", Fld_log_msg = "log_msg", Fld_log_time = "log_time" , Fld_page_id = "page_id", Fld_page_ttl = "page_ttl" , Fld_args_len = "args_len", Fld_args_str = "args_str" , Fld_src_len = "src_len", Fld_src_str = "src_str" ; - private static final String Tbl_sql = String_.Concat_lines_nl + private static final String Tbl_sql = String_.Concat_lines_nl ( "CREATE TABLE IF NOT EXISTS log_basic_temp" , "( log_id integer NOT NULL PRIMARY KEY AUTOINCREMENT" , ", log_tid integer NOT NULL" diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_wkr.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_wkr.java index 58bf47127..89d2f4cce 100644 --- a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_wkr.java +++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_wkr.java @@ -20,10 +20,10 @@ import gplx.core.envs.*; import gplx.dbs.*; import gplx.xowa.parsers.xndes.*; import gplx.xowa.parsers.htmls.*; public class Xop_log_basic_wkr implements Gfo_invk { - private Xop_log_mgr log_mgr; private Xop_log_basic_tbl log_tbl; + private Xop_log_basic_tbl log_tbl; private boolean save_page_ttl, save_log_time, save_args_len, save_args_str; public boolean Save_src_str() {return save_src_str;} public Xop_log_basic_wkr Save_src_str_(boolean v) {save_src_str = v; return this;} private boolean save_src_str; - public Xop_log_basic_wkr(Xop_log_mgr log_mgr, Xop_log_basic_tbl log_tbl) {this.log_mgr = log_mgr; this.log_tbl = log_tbl;} + public Xop_log_basic_wkr(Xop_log_basic_tbl log_tbl) {this.log_tbl = log_tbl;} public boolean Log_bgn(Xoae_page page, byte[] src, Xop_xnde_tkn xnde) {return true;} public void Log_end_xnde(Xoae_page page, int log_tid, byte[] src, Xop_xnde_tkn xnde_tkn) { Mwh_atr_itm[] atrs_ary = xnde_tkn.Atrs_ary(); @@ -45,7 +45,6 @@ public class Xop_log_basic_wkr implements Gfo_invk { , src_end - src_bgn , save_src_str ? String_.new_u8(src, src_bgn, src_end) : Xop_log_basic_wkr.Null_src_str ); - log_mgr.Commit_chk(); } public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { if (ctx.Match(k, Invk_save_page_ttl_)) save_page_ttl = m.ReadYn("v"); diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_invoke_wkr.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_invoke_wkr.java index 3e9603867..f10a2e26c 100644 --- a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_invoke_wkr.java +++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_invoke_wkr.java @@ -20,13 +20,11 @@ import gplx.core.envs.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.xtns.scribunto.*; public class Xop_log_invoke_wkr implements Gfo_invk { - private Xop_log_mgr log_mgr; private Db_conn conn; private Db_stmt stmt; private boolean log_enabled = true; private Hash_adp_bry exclude_mod_names = Hash_adp_bry.cs(); public Scrib_err_filter_mgr Err_filter_mgr() {return err_filter_mgr;} private final Scrib_err_filter_mgr err_filter_mgr = new Scrib_err_filter_mgr(); - public Xop_log_invoke_wkr(Xop_log_mgr log_mgr, Db_conn conn) { - this.log_mgr = log_mgr; + public Xop_log_invoke_wkr(Db_conn conn) { this.conn = conn; if (log_enabled) { Xop_log_invoke_tbl.Create_table(conn); @@ -39,7 +37,6 @@ public class Xop_log_invoke_wkr implements Gfo_invk { if (log_enabled && stmt != null) { int eval_time = (int)(System_.Ticks() - invoke_time_bgn); Xop_log_invoke_tbl.Insert(stmt, page.Ttl().Rest_txt(), mod_name, fnc_name, eval_time); - log_mgr.Commit_chk(); } } private void Exclude_mod_names_add(String[] v) { diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_mgr.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_mgr.java index 8a8de453b..d725efc09 100644 --- a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_mgr.java +++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_mgr.java @@ -38,12 +38,12 @@ public class Xop_log_mgr implements Gfo_invk { } return conn; } - public Xop_log_invoke_wkr Make_wkr_invoke() {return new Xop_log_invoke_wkr(this, this.Conn());} - public Xop_log_property_wkr Make_wkr_property() {return new Xop_log_property_wkr(this, this.Conn());} + public Xop_log_invoke_wkr Make_wkr_invoke() {return new Xop_log_invoke_wkr(this.Conn());} + public Xop_log_property_wkr Make_wkr_property() {return new Xop_log_property_wkr(this.Conn());} public Xop_log_basic_wkr Make_wkr() { if (log_tbl == null) log_tbl = new Xop_log_basic_tbl(this.Conn()); - return new Xop_log_basic_wkr(this, log_tbl); + return new Xop_log_basic_wkr(log_tbl); } public void Commit_chk() { ++exec_count; diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_property_wkr.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_property_wkr.java index a7658157b..ccfcaeb5e 100644 --- a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_property_wkr.java +++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_property_wkr.java @@ -19,12 +19,11 @@ package gplx.xowa.parsers.logs; import gplx.*; import gplx.xowa.*; import gplx.x import gplx.core.envs.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.dbs.engines.sqlite.*; public class Xop_log_property_wkr implements Gfo_invk { - private Xop_log_mgr log_mgr; private Db_conn conn; private Db_stmt stmt; + private Db_conn conn; private Db_stmt stmt; private boolean log_enabled = true; private boolean include_all = true; private Hash_adp_bry include_props = Hash_adp_bry.cs(); - public Xop_log_property_wkr(Xop_log_mgr log_mgr, Db_conn conn) { - this.log_mgr = log_mgr; + public Xop_log_property_wkr(Db_conn conn) { this.conn = conn; if (log_enabled) { Xob_log_property_temp_tbl.Create_table(conn); @@ -37,7 +36,6 @@ public class Xop_log_property_wkr implements Gfo_invk { if (log_enabled && stmt != null) { int eval_time = (int)(System_.Ticks() - invoke_time_bgn); Xob_log_property_temp_tbl.Insert(stmt, page.Ttl().Rest_txt(), prop, eval_time); - log_mgr.Commit_chk(); } } private void Include_props_add(String[] v) { diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_wkr_factory.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_wkr_factory.java new file mode 100644 index 000000000..4f4b4614f --- /dev/null +++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_wkr_factory.java @@ -0,0 +1,29 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.parsers.logs; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*; +import gplx.dbs.*; import gplx.xowa.bldrs.*; +public class Xop_log_wkr_factory { + private final Db_conn conn; private Xop_log_basic_tbl log_tbl; + public Xop_log_wkr_factory(Db_conn conn) {this.conn = conn;} + public Xop_log_invoke_wkr Make__invoke() {return new Xop_log_invoke_wkr(conn);} + public Xop_log_property_wkr Make__property() {return new Xop_log_property_wkr(conn);} + public Xop_log_basic_wkr Make__generic() { + if (log_tbl == null) log_tbl = new Xop_log_basic_tbl(conn); + return new Xop_log_basic_wkr(log_tbl); + } +} diff --git a/400_xowa/src/gplx/xowa/xtns/math/Xomath_core.java b/400_xowa/src/gplx/xowa/xtns/math/Xomath_core.java index 38f22dd12..53d2edac6 100644 --- a/400_xowa/src/gplx/xowa/xtns/math/Xomath_core.java +++ b/400_xowa/src/gplx/xowa/xtns/math/Xomath_core.java @@ -16,16 +16,21 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.xtns.math; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; -import gplx.xowa.parsers.*; import gplx.xowa.parsers.xndes.*; +import gplx.xowa.parsers.*; import gplx.xowa.parsers.xndes.*; import gplx.xowa.parsers.logs.*; public class Xomath_core implements Gfo_invk { private final Xomath_html_wtr html_wtr = new Xomath_html_wtr(); + private Xop_log_basic_wkr log_wkr; public boolean Enabled() {return enabled;} private boolean enabled = true; public boolean Renderer_is_mathjax() {return renderer_is_mathjax;} private boolean renderer_is_mathjax = true; public void Renderer_is_mathjax_(boolean v) {renderer_is_mathjax = v;} // TEST: public void Init_by_wiki(Xow_wiki wiki) { wiki.App().Cfg().Bind_many_wiki(this, wiki, Cfg__enabled, Cfg__renderer); } + public void Log_wkr_(Xop_log_wkr_factory factory) { + this.log_wkr = factory.Make__generic().Save_src_str_(Bool_.Y); + } public void Write(Bry_bfr bfr, Xop_ctx ctx, Xop_xnde_tkn xnde, byte[] src) { + if (log_wkr != null) log_wkr.Log_end_xnde(ctx.Page(), Xop_log_basic_wkr.Tid_math, src, xnde); html_wtr.Write(bfr, ctx, xnde, src, !renderer_is_mathjax, enabled); } public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {