diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java
index 7e6374ead..7b491e809 100644
--- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java
+++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java
@@ -23,7 +23,7 @@ public class Xomp_make_cmd extends Xob_cmd__base {
@Override public void Cmd_run() {
wiki.Init_assert();
new Xomp_make_html().Exec(wiki, cfg);
- new Xomp_make_lnki().Exec(wiki, 10000);
+ new Xomp_make_lnki().Exec(wiki, cfg, 10000);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__cfg)) return cfg;
diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd_cfg.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd_cfg.java
index fe697b1db..32386974d 100644
--- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd_cfg.java
+++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd_cfg.java
@@ -18,12 +18,16 @@ along with this program. If not, see .
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
public class Xomp_make_cmd_cfg implements Gfo_invk {
public boolean Delete_html_dbs() {return delete_html_dbs;} private boolean delete_html_dbs = true;
+ public Ordered_hash Merger_wkrs() {return merger_wkrs;} private final Ordered_hash merger_wkrs = Ordered_hash_.New();
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__delete_html_dbs_)) delete_html_dbs = m.ReadYn("v");
+ else if (ctx.Match(k, Invk__merger_wkrs_)) {
+ String[] ary = m.ReadStrAry("k", "|");
+ for (String itm : ary)
+ merger_wkrs.Add(itm, itm);
+ }
else return Gfo_invk_.Rv_unhandled;
return this;
}
- private static final String
- Invk__delete_html_dbs_ = "delete_html_dbs_"
- ;
+ private static final String Invk__delete_html_dbs_ = "delete_html_dbs_", Invk__merger_wkrs_ = "merger_wkrs_";
}
diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java
index 184229cc0..63cff8829 100644
--- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java
+++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java
@@ -16,94 +16,60 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see .
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
-import gplx.core.brys.*; import gplx.core.primitives.*;
-import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*; import gplx.xowa.addons.bldrs.files.dbs.*;
-import gplx.xowa.bldrs.*;
+import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
class Xomp_make_lnki {
- public void Exec(Xowe_wiki wiki, int uid_count) {
+ public void Exec(Xowe_wiki wiki, Xomp_make_cmd_cfg cfg, int uid_count) {
// init
- Xomp_mgr_db xomp_db = Xomp_mgr_db.New__load(wiki);
+ Xomp_mgr_db src_mgr_db = Xomp_mgr_db.New__load(wiki);
- Xob_db_file make_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
- Xob_lnki_temp_tbl lnki_temp_tbl = new Xob_lnki_temp_tbl(make_db.Conn());
- make_db.Conn().Meta_tbl_remake(lnki_temp_tbl);
- lnki_temp_tbl.Insert_bgn();
+ // make mergers; always add lnki_temp
+ List_adp merger_list = List_adp_.New();
+ merger_list.Add(new Xomp_make_merger__lnki_temp());
+ if (cfg.Merger_wkrs().Has("xnde")) merger_list.Add(new Xomp_make_merger__xnde());
+ Xomp_make_merger[] merger_ary = (Xomp_make_merger[])merger_list.To_ary_and_clear(Xomp_make_merger.class);
// create ary; add index
- int wkr_count = xomp_db.Tbl__wkr().Select_count();
- Xomp_wkr_db[] db_ary = new Xomp_wkr_db[wkr_count];
+ int wkr_count = src_mgr_db.Tbl__wkr().Select_count();
+ Xomp_wkr_db[] src_mgr_dbs = new Xomp_wkr_db[wkr_count];
for (int i = 0; i < wkr_count; ++i) {
- Xomp_wkr_db wkr_db = Xomp_wkr_db.New(xomp_db.Dir(), i);
- db_ary[i] = wkr_db;
- wkr_db.Conn().Meta_idx_assert("lnki_temp", "lnki_page_id", "lnki_page_id");
+ Xomp_wkr_db src_wkr_db = Xomp_wkr_db.New(src_mgr_db.Dir(), i);
+ src_mgr_dbs[i] = src_wkr_db;
}
+ // run init
+ for (Xomp_make_merger merger : merger_ary)
+ merger.Merger__init(wiki, src_mgr_dbs);
+
// get max xomp_uid; note that xomp_uid is (a) per page; (b) ordered by page_ns, page_id; (c) starts from 1
- int max_xomp_uid = xomp_db.Conn().Exec_select_max_as_int("xomp_page", "xomp_uid", -1);
+ int max_xomp_uid = src_mgr_db.Conn().Exec_select_max_as_int("xomp_page", "xomp_uid", -1);
// loop over wkrs using range of xomp_uid
- List_adp rows = List_adp_.New();
- int cur_xomp_uid = -1; Int_obj_ref lnki_id_ref = Int_obj_ref.New_zero();
+ int cur_xomp_uid = -1;
while (true) {
+ // load rows
+ Gfo_usr_dlg_.Instance.Prog_many("", "", "merging rows; bgn_uid=~{0} end_uid=~{1}", cur_xomp_uid, cur_xomp_uid + uid_count);
int tmp_xomp_uid_max = -1; // maximum uid for a grp of wkrs; EX: looping over 8 wkrs with xomp_uid range of 1 - 1000; max xomp_uid may only be 990 b/c pages are missing / failed
for (int i = 0; i < wkr_count; ++i) {
- Xomp_wkr_db wkr_db = db_ary[i];
- int wkr_uid_max = Load_rows(rows, xomp_db, wkr_db, cur_xomp_uid, cur_xomp_uid + uid_count, lnki_id_ref);
+ Xomp_wkr_db src_wkr_db = src_mgr_dbs[i];
+ int wkr_uid_max = tmp_xomp_uid_max;
+ for (Xomp_make_merger merger : merger_ary)
+ wkr_uid_max = merger.Merger__load(src_mgr_db, src_wkr_db, cur_xomp_uid, cur_xomp_uid + uid_count);
if (wkr_uid_max > tmp_xomp_uid_max)
tmp_xomp_uid_max = wkr_uid_max;
}
+
+ // save rows
+ for (Xomp_make_merger merger : merger_ary)
+ merger.Merger__save();
+
// NOTE: not ">=" else small wikis will fail with 0 images; EX:cs.q; DATE:2016-09-04
if (tmp_xomp_uid_max > max_xomp_uid || tmp_xomp_uid_max == -1) break; // if max_xomp_uid seen, break; note that ">" necessary because max_xomp_uid may not be in set of wkrs;
cur_xomp_uid += uid_count; // note that this sequentially counts up by uid_count (1000), so inevitable that cur_xomp_uid will exceed wkr_uid_max
- Gfo_usr_dlg_.Instance.Prog_many("", "", "building lnki_temp; cur_xomp_uid=~{0}", cur_xomp_uid);
- Save_rows(rows, lnki_temp_tbl);
}
- // term
- lnki_temp_tbl.Insert_end();
- xomp_db.Conn().Rls_conn();
- make_db.Conn().Rls_conn();
- }
- private int Load_rows(List_adp rows, Xomp_mgr_db xomp_db, Xomp_wkr_db wkr_db, int uid_bgn, int uid_end, Int_obj_ref lnki_id) {
- // build sql
- Db_attach_mgr attach_mgr = new Db_attach_mgr(xomp_db.Conn());
- attach_mgr.Conn_links_(new Db_attach_itm("wkr_db", wkr_db.Conn()));
- String sql = String_.Format(String_.Concat_lines_nl_skip_last
- ( "SELECT mgr.xomp_uid"
- , ", wkr.*"
- , ", wkr.lnki_tier_id, wkr.lnki_page_id, wkr.lnki_ttl, wkr.lnki_commons_ttl, wkr.lnki_ext, wkr.lnki_type, wkr.lnki_src_tid, wkr.lnki_w, wkr.lnki_h, wkr.lnki_upright, wkr.lnki_time, wkr.lnki_page"
- , "FROM lnki_temp wkr"
- , " JOIN xomp_page mgr ON wkr.lnki_page_id = mgr.page_id"
- , "WHERE mgr.xomp_uid > {0} AND mgr.xomp_uid <= {1}" // mgr.xomp_uid will sort pages by ns_id, page_id
- )
- , uid_bgn
- , uid_end
- );
- sql = attach_mgr.Resolve_sql(sql);
-
- attach_mgr.Attach();
- Db_rdr rdr = xomp_db.Conn().Stmt_sql(sql).Exec_select__rls_auto(); // ANSI.Y
- int rv = -1;
- try {
- while (rdr.Move_next()) {
- rv = rdr.Read_int("xomp_uid");
- Xob_lnki_temp_row row = new Xob_lnki_temp_row();
- row.Load(rdr, lnki_id.Val_add_pre());
- rows.Add(row);
- }
- } finally {rdr.Rls();}
- attach_mgr.Detach();
- return rv;
- }
-
- private void Save_rows(List_adp rows, Xob_lnki_temp_tbl lnki_temp_tbl) {
- int len = rows.Len();
- for (int i = 0; i < len; ++i) {
- Xob_lnki_temp_row row = (Xob_lnki_temp_row)rows.Get_at(i);
- lnki_temp_tbl.Insert_cmd_by_batch(row.Lnki_tier_id(), row.Lnki_page_id(), row.Lnki_ttl(), row.Lnki_commons_ttl()
- , row.Lnki_ext(), row.Lnki_type(), row.Lnki_src_tid(), row.Lnki_w(), row.Lnki_h(), row.Lnki_upright()
- , row.Lnki_time(), row.Lnki_page());
- }
- rows.Clear();
+ // save rows
+ for (Xomp_make_merger merger : merger_ary)
+ merger.Merger__term();
+ src_mgr_db.Conn().Rls_conn();
}
}
diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger.java
new file mode 100644
index 000000000..481048b22
--- /dev/null
+++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger.java
@@ -0,0 +1,25 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+*/
+package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
+import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
+interface Xomp_make_merger {
+ void Merger__init(Xowe_wiki wiki, Xomp_wkr_db[] src_dbs);
+ int Merger__load(Xomp_mgr_db src_mgr_db, Xomp_wkr_db src_wkr_db, int uid_bgn, int uid_end);
+ void Merger__save();
+ void Merger__term();
+}
diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__base.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__base.java
new file mode 100644
index 000000000..f43fbb7d4
--- /dev/null
+++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__base.java
@@ -0,0 +1,100 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+*/
+package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
+import gplx.dbs.*; import gplx.xowa.bldrs.*;
+import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
+// merges xomp.wkr_dbs into xowa.file.make.sqlite3
+// NOTE: can't do INSERT b/c (a) autonum id is same in diff worker dbs; (b): want autonum to match ns_id, page_id order
+abstract class Xomp_make_merger__base implements Xomp_make_merger, gplx.core.lists.ComparerAble {
+ private Xob_db_file trg_db;
+ protected Db_tbl trg_tbl;
+ private List_adp rows = List_adp_.New();
+ private String src_tbl__name, src_fld__page_id;
+ public void Merger__init(Xowe_wiki wiki, Xomp_wkr_db[] src_dbs) {
+ // get trg.db
+ this.trg_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
+
+ // make trg.tbl
+ this.trg_tbl = Init__trg_tbl(trg_db);
+ trg_db.Conn().Meta_tbl_remake(trg_tbl);
+
+ // make idxs on src.tbls
+ this.src_tbl__name = this.trg_tbl.Tbl_name();
+ this.src_fld__page_id = this.Init__src_fld__page_id();
+ int len = src_dbs.length;
+ for (int i = 0; i < len; ++i) {
+ src_dbs[i].Conn().Meta_idx_assert(src_tbl__name, src_fld__page_id, src_fld__page_id);
+ }
+
+ // do any other init, such as init'ing insert stmt
+ this.Init__trg_bgn();
+ this.trg_db.Conn().Txn_bgn("merger__" + src_tbl__name);
+ }
+ protected abstract Db_tbl Init__trg_tbl(Xob_db_file trg_db);
+ protected abstract String Init__src_fld__page_id();
+ @gplx.Virtual protected void Init__trg_bgn() {}
+
+ public int Merger__load(Xomp_mgr_db src_mgr_db, Xomp_wkr_db src_wkr_db, int uid_bgn, int uid_end) {
+ // build sql
+ Db_attach_mgr attach_mgr = new Db_attach_mgr(src_mgr_db.Conn());
+ attach_mgr.Conn_links_(new Db_attach_itm("src_wkr_db", src_wkr_db.Conn()));
+ String sql = Db_sql_.Make_by_fmt(String_.Ary
+ ( "SELECT src_mgr.xomp_uid"
+ , ", src_wkr.*"
+ , "FROM {0} src_wkr"
+ , " JOIN xomp_page src_mgr ON src_wkr.{1} = src_mgr.page_id"
+ , "WHERE src_mgr.xomp_uid > {2} AND src_mgr.xomp_uid <= {3}" // NOTE: mgr.xomp_uid will sort pages by ns_id, page_id
+ )
+ , src_tbl__name, src_fld__page_id
+ , uid_bgn, uid_end
+ );
+ sql = attach_mgr.Resolve_sql(sql);
+
+ // load rows
+ attach_mgr.Attach();
+ Db_rdr rdr = src_mgr_db.Conn().Stmt_sql(sql).Exec_select__rls_auto();
+ int rv = -1;
+ try {
+ while (rdr.Move_next()) {
+ rv = rdr.Read_int("xomp_uid");
+ Object row = Load__src_row(rdr);
+ rows.Add(row);
+ }
+ } finally {rdr.Rls();}
+ attach_mgr.Detach();
+ return rv;
+ }
+ protected abstract Object Load__src_row(Db_rdr rdr);
+
+ public void Merger__save() {
+ rows.Sort_by(this);
+ int len = rows.Len();
+ for (int i = 0; i < len; ++i) {
+ Save__trg_row(rows.Get_at(i));
+ }
+ rows.Clear();
+ }
+ protected abstract void Save__trg_row(Object row_obj);
+ public void Merger__term() {
+ this.trg_db.Conn().Txn_end();
+ }
+ public int compare(Object lhsObj, Object rhsObj) {
+ return Compare__hook(lhsObj, rhsObj);
+ }
+ protected abstract int Compare__hook(Object lhsObj, Object rhsObj);
+}
diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__lnki_temp.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__lnki_temp.java
new file mode 100644
index 000000000..e532a4a80
--- /dev/null
+++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__lnki_temp.java
@@ -0,0 +1,46 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+*/
+package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
+import gplx.dbs.*; import gplx.xowa.bldrs.*;
+import gplx.xowa.addons.bldrs.files.dbs.*;
+class Xomp_make_merger__lnki_temp extends Xomp_make_merger__base {
+ private int lnki_id;
+ private Xob_lnki_temp_tbl trg_tbl__lnki_temp;
+ @Override protected Db_tbl Init__trg_tbl(Xob_db_file trg_db) {
+ this.trg_tbl__lnki_temp = new Xob_lnki_temp_tbl(trg_db.Conn());
+ return trg_tbl__lnki_temp;
+ }
+ @Override protected String Init__src_fld__page_id() {return "lnki_page_id";}
+ @Override protected void Init__trg_bgn() {trg_tbl__lnki_temp.Insert_stmt_make();}
+ @Override protected Object Load__src_row(Db_rdr rdr) {
+ Xob_lnki_temp_row rv = new Xob_lnki_temp_row();
+ rv.Load(rdr, ++lnki_id);
+ return rv;
+ }
+ @Override protected void Save__trg_row(Object row_obj) {
+ Xob_lnki_temp_row row = (Xob_lnki_temp_row)row_obj;
+ trg_tbl__lnki_temp.Insert_cmd_by_batch(row.Lnki_tier_id(), row.Lnki_page_id(), row.Lnki_ttl(), row.Lnki_commons_ttl()
+ , row.Lnki_ext(), row.Lnki_type(), row.Lnki_src_tid(), row.Lnki_w(), row.Lnki_h(), row.Lnki_upright()
+ , row.Lnki_time(), row.Lnki_page());
+ }
+ @Override protected int Compare__hook(Object lhsObj, Object rhsObj) {
+ Xob_lnki_temp_row lhs = (Xob_lnki_temp_row)lhsObj;
+ Xob_lnki_temp_row rhs = (Xob_lnki_temp_row)rhsObj;
+ return Int_.Compare(lhs.Lnki_page_id(), rhs.Lnki_page_id());
+ }
+}
diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__xnde.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__xnde.java
new file mode 100644
index 000000000..2ac8123a9
--- /dev/null
+++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_merger__xnde.java
@@ -0,0 +1,67 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+*/
+package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
+import gplx.dbs.*; import gplx.xowa.bldrs.*;
+import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
+import gplx.xowa.addons.bldrs.files.dbs.*;
+import gplx.xowa.parsers.logs.*;
+class Xomp_make_merger__xnde extends Xomp_make_merger__base {
+ private Xop_log_basic_tbl trg_tbl__log_basic_temp;
+ @Override protected Db_tbl Init__trg_tbl(Xob_db_file trg_db) {
+ this.trg_tbl__log_basic_temp = new Xop_log_basic_tbl(trg_db.Conn());
+ return trg_tbl__log_basic_temp;
+ }
+ @Override protected String Init__src_fld__page_id() {return "page_id";}
+ @Override protected void Init__trg_bgn() {} // NOTE: trg_tbl has implicit insert_stmt creation
+ @Override protected Object Load__src_row(Db_rdr rdr) {
+ Xop_log_basic_row rv = new Xop_log_basic_row();
+ rv.Load(rdr);
+ return rv;
+ }
+ @Override protected void Save__trg_row(Object row_obj) {
+ Xop_log_basic_row row = (Xop_log_basic_row)row_obj;
+ trg_tbl__log_basic_temp.Insert(row.Log_tid, row.Log_msg, row.Log_time, row.Page_id, row.Page_ttl, row.Args_len, row.Args_str, row.Src_len, row.Src_str);
+ }
+ @Override protected int Compare__hook(Object lhsObj, Object rhsObj) {
+ Xop_log_basic_row lhs = (Xop_log_basic_row)lhsObj;
+ Xop_log_basic_row rhs = (Xop_log_basic_row)rhsObj;
+ return Int_.Compare(lhs.Page_id, rhs.Page_id);
+ }
+}
+class Xop_log_basic_row {
+ public int Log_tid;
+ public String Log_msg;
+ public int Log_time;
+ public int Page_id;
+ public String Page_ttl;
+ public int Args_len;
+ public String Args_str;
+ public int Src_len;
+ public String Src_str;
+ public void Load(Db_rdr rdr) {
+ this.Log_tid = rdr.Read_int("log_tid");
+ this.Log_msg = rdr.Read_str("log_msg");
+ this.Log_time = rdr.Read_int("log_time");
+ this.Page_id = rdr.Read_int("page_id");
+ this.Page_ttl = rdr.Read_str("page_ttl");
+ this.Args_len = rdr.Read_int("args_len");
+ this.Args_str = rdr.Read_str("args_str");
+ this.Src_len = rdr.Read_int("src_len");
+ this.Src_str = rdr.Read_str("src_str");
+ }
+}
diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
index 554cacf58..b10ae4e65 100644
--- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
+++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
@@ -32,6 +32,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true;
public boolean Load_all_imglinks() {return load_all_imglinks;} private boolean load_all_imglinks = true;
public String Load_ifexists_ns() {return load_ifexists_ns;} private String load_ifexists_ns = null;
+ public boolean Log_math() {return log_math;} private boolean log_math = false;
public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_tid_.Tid__gzip;
public Io_url Mgr_url() {return mgr_url;} private Io_url mgr_url;
public String Wkr_machine_name() {return wkr_machine_name;} private String wkr_machine_name;
@@ -62,6 +63,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, Invk__wkr_machine_name_)) wkr_machine_name = m.ReadStr("v");
else if (ctx.Match(k, Invk__show_msg__fetched_pool_)) show_msg__fetched_pool = m.ReadYn("v");
else if (ctx.Match(k, Invk__hdump_catboxes_)) hdump_catboxs = m.ReadYn("v");
+ else if (ctx.Match(k, Invk__log_math_)) log_math = m.ReadYn("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}
@@ -71,6 +73,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
, Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_", Invk__zip_tid_ = "zip_tid_"
, Invk__load_all_templates_ = "load_all_templates_", Invk__load_all_imglinks_ = "load_all_imglinks_", Invk__load_ifexists_ns_ = "load_ifexists_ns_", Invk__manual_now_ = "manual_now_"
, Invk__hdump_catboxes_ = "hdump_catboxes_"
+ , Invk__log_math_ = "log_math_"
, Invk__mgr_url_ = "mgr_url_", Invk__wkr_machine_name_ = "wkr_machine_name_"
, Invk__show_msg__fetched_pool_ = "show_msg__fetched_pool_"
;
diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java
index 5e22ac9e8..fc3771185 100644
--- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java
+++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java
@@ -19,7 +19,7 @@ package gplx.xowa.addons.bldrs.mass_parses.parses.wkrs; import gplx.*; import gp
import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.files.origs.*;
import gplx.xowa.htmls.core.bldrs.*;
-import gplx.xowa.parsers.*;
+import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.pools.*;
public class Xomp_parse_wkr implements Gfo_invk {
// mgr vars
@@ -80,6 +80,10 @@ public class Xomp_parse_wkr implements Gfo_invk {
logger.Bgn();
}
+ // init log_mgr / property_wkr
+ Xop_log_wkr_factory wkr_factory = new Xop_log_wkr_factory(wkr_db.Conn());
+ if (cfg.Log_math()) wiki.Parser_mgr().Math__core().Log_wkr_(wkr_factory);
+
// enable hdump
hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled()).Zip_tid_(cfg.Zip_tid());
hdump_bldr.Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_tbl.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_tbl.java
index 55f4e1003..8bf10b8b7 100644
--- a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_tbl.java
+++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_tbl.java
@@ -17,14 +17,15 @@ along with this program. If not, see .
*/
package gplx.xowa.parsers.logs; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.dbs.engines.sqlite.*;
-public class Xop_log_basic_tbl {
+public class Xop_log_basic_tbl implements Db_tbl {
private Db_stmt stmt_insert;
- public Xop_log_basic_tbl(Db_conn conn){this.conn = conn; this.Create_table();}
+ public Xop_log_basic_tbl(Db_conn conn){this.conn = conn; this.Create_tbl();}
public Db_conn Conn() {return conn;} private Db_conn conn;
- private void Create_table() {Sqlite_engine_.Tbl_create(conn, Tbl_name, Tbl_sql);}
- public void Delete() {conn.Exec_qry(Db_qry_delete.new_all_(Tbl_name));}
+ public String Tbl_name() {return TBL_NAME;}
+ public void Create_tbl() {Sqlite_engine_.Tbl_create(conn, TBL_NAME, Tbl_sql);}
+ public void Delete() {conn.Exec_qry(Db_qry_delete.new_all_(TBL_NAME));}
public void Insert(int log_tid, String log_msg, int log_time, int page_id, String page_ttl, int args_len, String args_str, int src_len, String src_str) {
- if (stmt_insert == null) stmt_insert = Db_stmt_.new_insert_(conn, Tbl_name, Fld_log_tid, Fld_log_msg, Fld_log_time, Fld_page_id, Fld_page_ttl, Fld_args_len, Fld_args_str, Fld_src_len, Fld_src_str);
+ if (stmt_insert == null) stmt_insert = Db_stmt_.new_insert_(conn, TBL_NAME, Fld_log_tid, Fld_log_msg, Fld_log_time, Fld_page_id, Fld_page_ttl, Fld_args_len, Fld_args_str, Fld_src_len, Fld_src_str);
stmt_insert.Clear()
.Val_int(log_tid)
.Val_str(log_msg)
@@ -40,13 +41,13 @@ public class Xop_log_basic_tbl {
public void Rls() {
stmt_insert.Rls();
}
- public static final String Tbl_name = "log_basic_temp"
+ public static final String TBL_NAME = "log_basic_temp"
, Fld_log_tid = "log_tid", Fld_log_msg = "log_msg", Fld_log_time = "log_time"
, Fld_page_id = "page_id", Fld_page_ttl = "page_ttl"
, Fld_args_len = "args_len", Fld_args_str = "args_str"
, Fld_src_len = "src_len", Fld_src_str = "src_str"
;
- private static final String Tbl_sql = String_.Concat_lines_nl
+ private static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS log_basic_temp"
, "( log_id integer NOT NULL PRIMARY KEY AUTOINCREMENT"
, ", log_tid integer NOT NULL"
diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_wkr.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_wkr.java
index 58bf47127..89d2f4cce 100644
--- a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_wkr.java
+++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_basic_wkr.java
@@ -20,10 +20,10 @@ import gplx.core.envs.*;
import gplx.dbs.*;
import gplx.xowa.parsers.xndes.*; import gplx.xowa.parsers.htmls.*;
public class Xop_log_basic_wkr implements Gfo_invk {
- private Xop_log_mgr log_mgr; private Xop_log_basic_tbl log_tbl;
+ private Xop_log_basic_tbl log_tbl;
private boolean save_page_ttl, save_log_time, save_args_len, save_args_str;
public boolean Save_src_str() {return save_src_str;} public Xop_log_basic_wkr Save_src_str_(boolean v) {save_src_str = v; return this;} private boolean save_src_str;
- public Xop_log_basic_wkr(Xop_log_mgr log_mgr, Xop_log_basic_tbl log_tbl) {this.log_mgr = log_mgr; this.log_tbl = log_tbl;}
+ public Xop_log_basic_wkr(Xop_log_basic_tbl log_tbl) {this.log_tbl = log_tbl;}
public boolean Log_bgn(Xoae_page page, byte[] src, Xop_xnde_tkn xnde) {return true;}
public void Log_end_xnde(Xoae_page page, int log_tid, byte[] src, Xop_xnde_tkn xnde_tkn) {
Mwh_atr_itm[] atrs_ary = xnde_tkn.Atrs_ary();
@@ -45,7 +45,6 @@ public class Xop_log_basic_wkr implements Gfo_invk {
, src_end - src_bgn
, save_src_str ? String_.new_u8(src, src_bgn, src_end) : Xop_log_basic_wkr.Null_src_str
);
- log_mgr.Commit_chk();
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_save_page_ttl_)) save_page_ttl = m.ReadYn("v");
diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_invoke_wkr.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_invoke_wkr.java
index 3e9603867..f10a2e26c 100644
--- a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_invoke_wkr.java
+++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_invoke_wkr.java
@@ -20,13 +20,11 @@ import gplx.core.envs.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.parsers.logs.*;
import gplx.xowa.xtns.scribunto.*;
public class Xop_log_invoke_wkr implements Gfo_invk {
- private Xop_log_mgr log_mgr;
private Db_conn conn; private Db_stmt stmt;
private boolean log_enabled = true;
private Hash_adp_bry exclude_mod_names = Hash_adp_bry.cs();
public Scrib_err_filter_mgr Err_filter_mgr() {return err_filter_mgr;} private final Scrib_err_filter_mgr err_filter_mgr = new Scrib_err_filter_mgr();
- public Xop_log_invoke_wkr(Xop_log_mgr log_mgr, Db_conn conn) {
- this.log_mgr = log_mgr;
+ public Xop_log_invoke_wkr(Db_conn conn) {
this.conn = conn;
if (log_enabled) {
Xop_log_invoke_tbl.Create_table(conn);
@@ -39,7 +37,6 @@ public class Xop_log_invoke_wkr implements Gfo_invk {
if (log_enabled && stmt != null) {
int eval_time = (int)(System_.Ticks() - invoke_time_bgn);
Xop_log_invoke_tbl.Insert(stmt, page.Ttl().Rest_txt(), mod_name, fnc_name, eval_time);
- log_mgr.Commit_chk();
}
}
private void Exclude_mod_names_add(String[] v) {
diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_mgr.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_mgr.java
index 8a8de453b..d725efc09 100644
--- a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_mgr.java
+++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_mgr.java
@@ -38,12 +38,12 @@ public class Xop_log_mgr implements Gfo_invk {
}
return conn;
}
- public Xop_log_invoke_wkr Make_wkr_invoke() {return new Xop_log_invoke_wkr(this, this.Conn());}
- public Xop_log_property_wkr Make_wkr_property() {return new Xop_log_property_wkr(this, this.Conn());}
+ public Xop_log_invoke_wkr Make_wkr_invoke() {return new Xop_log_invoke_wkr(this.Conn());}
+ public Xop_log_property_wkr Make_wkr_property() {return new Xop_log_property_wkr(this.Conn());}
public Xop_log_basic_wkr Make_wkr() {
if (log_tbl == null)
log_tbl = new Xop_log_basic_tbl(this.Conn());
- return new Xop_log_basic_wkr(this, log_tbl);
+ return new Xop_log_basic_wkr(log_tbl);
}
public void Commit_chk() {
++exec_count;
diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_property_wkr.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_property_wkr.java
index a7658157b..ccfcaeb5e 100644
--- a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_property_wkr.java
+++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_property_wkr.java
@@ -19,12 +19,11 @@ package gplx.xowa.parsers.logs; import gplx.*; import gplx.xowa.*; import gplx.x
import gplx.core.envs.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.dbs.engines.sqlite.*;
public class Xop_log_property_wkr implements Gfo_invk {
- private Xop_log_mgr log_mgr; private Db_conn conn; private Db_stmt stmt;
+ private Db_conn conn; private Db_stmt stmt;
private boolean log_enabled = true;
private boolean include_all = true;
private Hash_adp_bry include_props = Hash_adp_bry.cs();
- public Xop_log_property_wkr(Xop_log_mgr log_mgr, Db_conn conn) {
- this.log_mgr = log_mgr;
+ public Xop_log_property_wkr(Db_conn conn) {
this.conn = conn;
if (log_enabled) {
Xob_log_property_temp_tbl.Create_table(conn);
@@ -37,7 +36,6 @@ public class Xop_log_property_wkr implements Gfo_invk {
if (log_enabled && stmt != null) {
int eval_time = (int)(System_.Ticks() - invoke_time_bgn);
Xob_log_property_temp_tbl.Insert(stmt, page.Ttl().Rest_txt(), prop, eval_time);
- log_mgr.Commit_chk();
}
}
private void Include_props_add(String[] v) {
diff --git a/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_wkr_factory.java b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_wkr_factory.java
new file mode 100644
index 000000000..4f4b4614f
--- /dev/null
+++ b/400_xowa/src/gplx/xowa/parsers/logs/Xop_log_wkr_factory.java
@@ -0,0 +1,29 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012 gnosygnu@gmail.com
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+*/
+package gplx.xowa.parsers.logs; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
+import gplx.dbs.*; import gplx.xowa.bldrs.*;
+public class Xop_log_wkr_factory {
+ private final Db_conn conn; private Xop_log_basic_tbl log_tbl;
+ public Xop_log_wkr_factory(Db_conn conn) {this.conn = conn;}
+ public Xop_log_invoke_wkr Make__invoke() {return new Xop_log_invoke_wkr(conn);}
+ public Xop_log_property_wkr Make__property() {return new Xop_log_property_wkr(conn);}
+ public Xop_log_basic_wkr Make__generic() {
+ if (log_tbl == null) log_tbl = new Xop_log_basic_tbl(conn);
+ return new Xop_log_basic_wkr(log_tbl);
+ }
+}
diff --git a/400_xowa/src/gplx/xowa/xtns/math/Xomath_core.java b/400_xowa/src/gplx/xowa/xtns/math/Xomath_core.java
index 38f22dd12..53d2edac6 100644
--- a/400_xowa/src/gplx/xowa/xtns/math/Xomath_core.java
+++ b/400_xowa/src/gplx/xowa/xtns/math/Xomath_core.java
@@ -16,16 +16,21 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see .
*/
package gplx.xowa.xtns.math; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
-import gplx.xowa.parsers.*; import gplx.xowa.parsers.xndes.*;
+import gplx.xowa.parsers.*; import gplx.xowa.parsers.xndes.*; import gplx.xowa.parsers.logs.*;
public class Xomath_core implements Gfo_invk {
private final Xomath_html_wtr html_wtr = new Xomath_html_wtr();
+ private Xop_log_basic_wkr log_wkr;
public boolean Enabled() {return enabled;} private boolean enabled = true;
public boolean Renderer_is_mathjax() {return renderer_is_mathjax;} private boolean renderer_is_mathjax = true;
public void Renderer_is_mathjax_(boolean v) {renderer_is_mathjax = v;} // TEST:
public void Init_by_wiki(Xow_wiki wiki) {
wiki.App().Cfg().Bind_many_wiki(this, wiki, Cfg__enabled, Cfg__renderer);
}
+ public void Log_wkr_(Xop_log_wkr_factory factory) {
+ this.log_wkr = factory.Make__generic().Save_src_str_(Bool_.Y);
+ }
public void Write(Bry_bfr bfr, Xop_ctx ctx, Xop_xnde_tkn xnde, byte[] src) {
+ if (log_wkr != null) log_wkr.Log_end_xnde(ctx.Page(), Xop_log_basic_wkr.Tid_math, src, xnde);
html_wtr.Write(bfr, ctx, xnde, src, !renderer_is_mathjax, enabled);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {