Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2024-10-27 20:34:16 +00:00
Code Issues Projects Releases Wiki Activity

Mass_parse: Add error_filter to ignore logging certain errors [#793]

Browse Source
This commit is contained in:
gnosygnu 2020-09-01 09:02:47 -04:00
parent 1b6324938c
commit f19228c886
3 changed files with 378 additions and 328 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
View File

@ -1,6 +1,6 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
@ -13,8 +13,20 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs;
import gplx.core.ios.streams.*;
import gplx.Datetime_now;
import gplx.GfoMsg;
import gplx.Gfo_invk;
import gplx.Gfo_invk_;
import gplx.GfsCtx;
import gplx.Io_mgr;
import gplx.Io_url;
import gplx.core.ios.streams.Io_stream_tid_;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.parsers.logs.Xop_log_invoke_wkr;
import gplx.xowa.xtns.scribunto.Scrib_err_filter_mgr;
public class Xomp_parse_mgr_cfg implements Gfo_invk { public class Xomp_parse_mgr_cfg implements Gfo_invk {
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1; public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1;
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = -1; public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = -1;
@ -80,9 +92,11 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v"); else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v");
else if (ctx.Match(k, "page_cache_min_")) page_cache_min = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_min); else if (ctx.Match(k, "page_cache_min_")) page_cache_min = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_min);
else if (ctx.Match(k, "page_cache_max_")) page_cache_max = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_max); else if (ctx.Match(k, "page_cache_max_")) page_cache_max = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_max);
else if (ctx.Match(k, Invk__err_filter)) return Scrib_err_filter_mgr.INSTANCE;
else return Gfo_invk_.Rv_unhandled; else return Gfo_invk_.Rv_unhandled;
return this; return this;
} }
private Xop_log_invoke_wkr invoke_wkr;
private static final String private static final String
Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_" Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
, Invk__progress_interval_ = "progress_interval_", Invk__commit_interval_ = "commit_interval_", Invk__cleanup_interval_ = "cleanup_interval_" , Invk__progress_interval_ = "progress_interval_", Invk__commit_interval_ = "commit_interval_", Invk__cleanup_interval_ = "cleanup_interval_"
@ -92,5 +106,6 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
, Invk__log_math_ = "log_math_" , Invk__log_math_ = "log_math_"
, Invk__mgr_url_ = "mgr_url_", Invk__wkr_machine_name_ = "wkr_machine_name_" , Invk__mgr_url_ = "mgr_url_", Invk__wkr_machine_name_ = "wkr_machine_name_"
, Invk__show_msg__fetched_pool_ = "show_msg__fetched_pool_" , Invk__show_msg__fetched_pool_ = "show_msg__fetched_pool_"
, Invk__err_filter = "err_filter"
; ;
} }

26
400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_err_filter_mgr.java
View File

@ -1,6 +1,6 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
@ -13,11 +13,25 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.xtns.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; package gplx.xowa.xtns.scribunto;
import gplx.Bry_bfr;
import gplx.Bry_bfr_;
import gplx.GfoMsg;
import gplx.Gfo_invk;
import gplx.Gfo_invk_;
import gplx.GfsCtx;
import gplx.List_adp;
import gplx.List_adp_;
import gplx.Ordered_hash;
import gplx.Ordered_hash_;
import gplx.String_;
public class Scrib_err_filter_mgr implements Gfo_invk { public class Scrib_err_filter_mgr implements Gfo_invk {
private final Object thread_lock = new Object();
private final Ordered_hash hash_by_mod = Ordered_hash_.New(); private final Ordered_hash hash_by_mod = Ordered_hash_.New();
public void Clear() {hash_by_mod.Clear();} public void Clear() {hash_by_mod.Clear();}
public boolean Count_eq_0() {return hash_by_mod.Count() == 0;} public boolean Empty() {return empty;} private boolean empty = true;
public boolean Match(String mod, String fnc, String err) { public boolean Match(String mod, String fnc, String err) {
List_adp itms = Get_itms_or_null(mod, fnc); if (itms == null) return false; List_adp itms = Get_itms_or_null(mod, fnc); if (itms == null) return false;
int itms_len = itms.Count(); int itms_len = itms.Count();
@ -33,10 +47,13 @@ public class Scrib_err_filter_mgr implements Gfo_invk {
return match; return match;
} }
public void Add(int count_expd, String mod, String fnc, String err, String comment) { public void Add(int count_expd, String mod, String fnc, String err, String comment) {
synchronized (thread_lock) {
empty = false;
List_adp itms = Get_itms_or_null(mod, fnc); List_adp itms = Get_itms_or_null(mod, fnc);
if (itms == null) itms = New_itms(mod, fnc); if (itms == null) itms = New_itms(mod, fnc);
itms.Add(new Scrib_err_filter_itm(count_expd, mod, fnc, err, comment)); itms.Add(new Scrib_err_filter_itm(count_expd, mod, fnc, err, comment));
} }
}
public String Print() { public String Print() {
Bry_bfr bfr = Bry_bfr_.New_w_size(8); Bry_bfr bfr = Bry_bfr_.New_w_size(8);
int i_len = hash_by_mod.Count(); int i_len = hash_by_mod.Count();
@ -79,6 +96,9 @@ public class Scrib_err_filter_mgr implements Gfo_invk {
else return Gfo_invk_.Rv_unhandled; else return Gfo_invk_.Rv_unhandled;
return this; return this;
} private static final String Invk_add = "add"; } private static final String Invk_add = "add";
// 2020-09-01: singleton b/c xomp instantiates multiple wikis; previous implementation was `((Scrib_xtn_mgr)(wiki.Xtn_mgr().Get_or_fail(Scrib_xtn_mgr.XTN_KEY))).Invoke_wkr();` which doesn't multi-thread
public static final Scrib_err_filter_mgr INSTANCE = new Scrib_err_filter_mgr();
} }
class Scrib_err_filter_itm { class Scrib_err_filter_itm {
public Scrib_err_filter_itm(int count_expd, String mod, String fnc, String err, String comment) {this.count_expd = count_expd; this.mod = mod; this.err = err; this.fnc = fnc; this.comment = comment;} public Scrib_err_filter_itm(int count_expd, String mod, String fnc, String err, String comment) {this.count_expd = count_expd; this.mod = mod; this.err = err; this.fnc = fnc; this.comment = comment;}

41
400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func.java
View File

@ -1,6 +1,6 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
@ -13,17 +13,33 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.xtns.scribunto; import gplx.*; package gplx.xowa.xtns.scribunto;
import gplx.Bry_;
import gplx.Bry_bfr;
import gplx.Err;
import gplx.Err_;
import gplx.GfoMsg;
import gplx.Gfo_invk;
import gplx.GfsCtx;
import gplx.String_;
import gplx.core.brys.fmtrs.Bry_fmtr;
import gplx.core.envs.System_;
import gplx.core.threads.Thread_adp; import gplx.core.threads.Thread_adp;
import gplx.core.threads.Thread_adp_; import gplx.core.threads.Thread_adp_;
import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.Xoa_ttl;
import gplx.core.brys.fmtrs.*; import gplx.core.envs.*; import gplx.xowa.Xowe_wiki;
import gplx.langs.htmls.*; import gplx.xowa.langs.kwds.Xol_kwd_grp_;
import gplx.xowa.langs.kwds.*; import gplx.xowa.langs.msgs.*; import gplx.xowa.langs.msgs.Xol_msg_itm_;
import gplx.xowa.wikis.nss.*; import gplx.xowa.langs.msgs.Xow_msg_mgr;
import gplx.xowa.htmls.*; import gplx.xowa.parsers.Xop_ctx;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.parsers.tmpls.*; import gplx.xowa.parsers.logs.Xop_log_invoke_wkr;
import gplx.xowa.xtns.pfuncs.*; import gplx.xowa.parsers.tmpls.Xot_invk;
import gplx.xowa.wikis.nss.Xow_ns;
import gplx.xowa.wikis.nss.Xow_ns_;
import gplx.xowa.xtns.pfuncs.Pf_func;
import gplx.xowa.xtns.pfuncs.Pf_func_;
import gplx.xowa.xtns.pfuncs.Pf_func_base;
import gplx.xowa.xtns.scribunto.cfgs.ScribCfg; import gplx.xowa.xtns.scribunto.cfgs.ScribCfg;
import gplx.xowa.xtns.scribunto.cfgs.ScribCfgResolver; import gplx.xowa.xtns.scribunto.cfgs.ScribCfgResolver;
@ -106,9 +122,8 @@ public class Scrib_invoke_func extends Pf_func_base {
catch (Throwable e) { catch (Throwable e) {
Err err = Err_.Cast_or_make(e); Err err = Err_.Cast_or_make(e);
Error(bfr, wiki.Msg_mgr(), err); Error(bfr, wiki.Msg_mgr(), err);
Scrib_err_filter_mgr err_filter_mgr = invoke_wkr == null ? null : invoke_wkr.Err_filter_mgr(); Scrib_err_filter_mgr err_filter_mgr = Scrib_err_filter_mgr.INSTANCE;
if ( err_filter_mgr == null // no err_filter_mgr defined; if ( err_filter_mgr.Empty() // err_filter_mgr exists, but no definitions
|| err_filter_mgr.Count_eq_0() // err_filter_mgr exists, but no definitions
|| !err_filter_mgr.Match(String_.new_u8(mod_name), String_.new_u8(fnc_name), err.To_str__msg_only())) // NOTE: must be To_str__msg_only; err_filter_mgr has defintion and it doesn't match current; print warn; DATE:2015-07-24 || !err_filter_mgr.Match(String_.new_u8(mod_name), String_.new_u8(fnc_name), err.To_str__msg_only())) // NOTE: must be To_str__msg_only; err_filter_mgr has defintion and it doesn't match current; print warn; DATE:2015-07-24
ctx.App().Usr_dlg().Warn_many("", "", "invoke failed: ~{0} ~{1} ~{2}", ctx.Page().Ttl().Raw(), Bry_.Replace_nl_w_tab(src, self.Src_bgn(), self.Src_end()), err.To_str__log()); ctx.App().Usr_dlg().Warn_many("", "", "invoke failed: ~{0} ~{1} ~{2}", ctx.Page().Ttl().Raw(), Bry_.Replace_nl_w_tab(src, self.Src_bgn(), self.Src_end()), err.To_str__log());
wiki.Parser_mgr().Scrib().Terminate_when_page_changes_y_(); // NOTE: terminate core when page changes; not terminating now, else page with many errors will be very slow due to multiple remakes of core; PAGE:th.d:all; DATE:2014-10-03 wiki.Parser_mgr().Scrib().Terminate_when_page_changes_y_(); // NOTE: terminate core when page changes; not terminating now, else page with many errors will be very slow due to multiple remakes of core; PAGE:th.d:all; DATE:2014-10-03