Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2025-06-13 12:54:14 +00:00
Code Issues Projects Releases Wiki Activity

Mass_parse: Add error_filter to ignore logging certain errors [#793]

Browse Source
This commit is contained in:
gnosygnu 2020-09-01 09:02:47 -04:00
parent 1b6324938c
commit f19228c886
3 changed files with 378 additions and 328 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

179
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
View File

@ -1,6 +1,6 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
@ -13,84 +13,99 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs;
import gplx.core.ios.streams.*;
public class Xomp_parse_mgr_cfg implements Gfo_invk { import gplx.Datetime_now;
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1; import gplx.GfoMsg;
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = -1; import gplx.Gfo_invk;
public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000; import gplx.Gfo_invk_;
public int Progress_interval() {return progress_interval;} private int progress_interval = 1000; import gplx.GfsCtx;
public int Perf_interval() {return perf_interval;} private int perf_interval = 10000; import gplx.Io_mgr;
public int Commit_interval() {return commit_interval;} private int commit_interval = 10000; import gplx.Io_url;
public int Cleanup_interval() {return cleanup_interval;} private int cleanup_interval = 50; // setting at 1000 uses lots of memory import gplx.core.ios.streams.Io_stream_tid_;
public boolean Hdump_enabled() {return hdump_enabled;} private boolean hdump_enabled = true; import gplx.xowa.Xowe_wiki;
public boolean Hdump_catboxs() {return hdump_catboxs;} private boolean hdump_catboxs = false; import gplx.xowa.parsers.logs.Xop_log_invoke_wkr;
public boolean Hzip_enabled() {return hzip_enabled;} private boolean hzip_enabled = true; import gplx.xowa.xtns.scribunto.Scrib_err_filter_mgr;
public boolean Hdiff_enabled() {return hdiff_enabled;} private boolean hdiff_enabled = true;
public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true; public class Xomp_parse_mgr_cfg implements Gfo_invk {
public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true; public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1;
public boolean Load_all_imglinks() {return load_all_imglinks;} private boolean load_all_imglinks = true; public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = -1;
public String Load_ifexists_ns() {return load_ifexists_ns;} private String load_ifexists_ns = null; public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000;
public boolean Log_math() {return log_math;} private boolean log_math = false; public int Progress_interval() {return progress_interval;} private int progress_interval = 1000;
public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_tid_.Tid__gzip; public int Perf_interval() {return perf_interval;} private int perf_interval = 10000;
public Io_url Mgr_url() {return mgr_url;} private Io_url mgr_url; public int Commit_interval() {return commit_interval;} private int commit_interval = 10000;
public String Wkr_machine_name() {return wkr_machine_name;} private String wkr_machine_name; public int Cleanup_interval() {return cleanup_interval;} private int cleanup_interval = 50; // setting at 1000 uses lots of memory
public boolean Show_msg__fetched_pool() {return show_msg__fetched_pool;} private boolean show_msg__fetched_pool; public boolean Hdump_enabled() {return hdump_enabled;} private boolean hdump_enabled = true;
public boolean Indexer_enabled() {return indexer_enabled;} private boolean indexer_enabled; public boolean Hdump_catboxs() {return hdump_catboxs;} private boolean hdump_catboxs = false;
public String Indexer_opt() {return indexer_opt;} private String indexer_opt = gplx.gflucene.indexers.Gflucene_idx_opt.Docs_and_freqs.Key(); public boolean Hzip_enabled() {return hzip_enabled;} private boolean hzip_enabled = true;
public String Wbase_cache_mru_type() {return wbase_cache_mru_type;} private String wbase_cache_mru_type = "mru"; public boolean Hdiff_enabled() {return hdiff_enabled;} private boolean hdiff_enabled = true;
public long Wbase_cache_mru_size() {return wbase_cache_mru_size;} private long wbase_cache_mru_size = 100; public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true;
public long Wbase_cache_mru_weight() {return wbase_cache_mru_weight;} private long wbase_cache_mru_weight = 10; public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true;
public long Wbase_cache_mru_compress_size() {return wbase_cache_mru_compress_size;} private long wbase_cache_mru_compress_size = 70; public boolean Load_all_imglinks() {return load_all_imglinks;} private boolean load_all_imglinks = true;
public long Page_cache_min() {return page_cache_min;} private long page_cache_min = 1500 * Io_mgr.Len_mb_long; public String Load_ifexists_ns() {return load_ifexists_ns;} private String load_ifexists_ns = null;
public long Page_cache_max() {return page_cache_max;} private long page_cache_max = 2000 * Io_mgr.Len_mb_long; public boolean Log_math() {return log_math;} private boolean log_math = false;
public void Init(Xowe_wiki wiki) { public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_tid_.Tid__gzip;
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count(); public Io_url Mgr_url() {return mgr_url;} private Io_url mgr_url;
if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000; public String Wkr_machine_name() {return wkr_machine_name;} private String wkr_machine_name;
if (mgr_url == null) mgr_url = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp"); public boolean Show_msg__fetched_pool() {return show_msg__fetched_pool;} private boolean show_msg__fetched_pool;
if (wkr_machine_name == null) wkr_machine_name = gplx.core.envs.System_.Env__machine_name(); public boolean Indexer_enabled() {return indexer_enabled;} private boolean indexer_enabled;
} public String Indexer_opt() {return indexer_opt;} private String indexer_opt = gplx.gflucene.indexers.Gflucene_idx_opt.Docs_and_freqs.Key();
public String Wbase_cache_mru_type() {return wbase_cache_mru_type;} private String wbase_cache_mru_type = "mru";
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { public long Wbase_cache_mru_size() {return wbase_cache_mru_size;} private long wbase_cache_mru_size = 100;
if (ctx.Match(k, Invk__num_wkrs_)) num_wkrs = m.ReadInt("v"); public long Wbase_cache_mru_weight() {return wbase_cache_mru_weight;} private long wbase_cache_mru_weight = 10;
else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v"); public long Wbase_cache_mru_compress_size() {return wbase_cache_mru_compress_size;} private long wbase_cache_mru_compress_size = 70;
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v"); public long Page_cache_min() {return page_cache_min;} private long page_cache_min = 1500 * Io_mgr.Len_mb_long;
else if (ctx.Match(k, Invk__progress_interval_)) progress_interval = m.ReadInt("v"); public long Page_cache_max() {return page_cache_max;} private long page_cache_max = 2000 * Io_mgr.Len_mb_long;
else if (ctx.Match(k, "perf_interval_")) perf_interval = m.ReadInt("v"); public void Init(Xowe_wiki wiki) {
else if (ctx.Match(k, Invk__commit_interval_)) commit_interval = m.ReadInt("v"); if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count();
else if (ctx.Match(k, Invk__cleanup_interval_)) cleanup_interval = m.ReadInt("v"); if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000;
else if (ctx.Match(k, Invk__hdump_enabled_)) hdump_enabled = m.ReadYn("v"); if (mgr_url == null) mgr_url = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
else if (ctx.Match(k, Invk__hzip_enabled_)) hzip_enabled = m.ReadYn("v"); if (wkr_machine_name == null) wkr_machine_name = gplx.core.envs.System_.Env__machine_name();
else if (ctx.Match(k, Invk__hdiff_enabled_)) hdiff_enabled = m.ReadYn("v"); }
else if (ctx.Match(k, Invk__zip_tid_)) zip_tid = m.ReadByte("v");
else if (ctx.Match(k, Invk__load_all_templates_)) load_all_templates = m.ReadYn("v"); public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
else if (ctx.Match(k, Invk__load_all_imglinks_)) load_all_imglinks = m.ReadYn("v"); if (ctx.Match(k, Invk__num_wkrs_)) num_wkrs = m.ReadInt("v");
else if (ctx.Match(k, Invk__load_ifexists_ns_)) load_ifexists_ns = m.ReadStr("v"); else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v");
else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v")); else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
else if (ctx.Match(k, Invk__mgr_url_)) mgr_url = m.ReadIoUrl("v"); else if (ctx.Match(k, Invk__progress_interval_)) progress_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__wkr_machine_name_)) wkr_machine_name = m.ReadStr("v"); else if (ctx.Match(k, "perf_interval_")) perf_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__show_msg__fetched_pool_)) show_msg__fetched_pool = m.ReadYn("v"); else if (ctx.Match(k, Invk__commit_interval_)) commit_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__hdump_catboxes_)) hdump_catboxs = m.ReadYn("v"); else if (ctx.Match(k, Invk__cleanup_interval_)) cleanup_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__log_math_)) log_math = m.ReadYn("v"); else if (ctx.Match(k, Invk__hdump_enabled_)) hdump_enabled = m.ReadYn("v");
else if (ctx.Match(k, "indexer_enabled_")) indexer_enabled = m.ReadYn("v"); else if (ctx.Match(k, Invk__hzip_enabled_)) hzip_enabled = m.ReadYn("v");
else if (ctx.Match(k, "indexer_opt_")) indexer_opt = m.ReadStr("v"); else if (ctx.Match(k, Invk__hdiff_enabled_)) hdiff_enabled = m.ReadYn("v");
else if (ctx.Match(k, "wbase_cache_mru_type_")) wbase_cache_mru_type = m.ReadStr("v"); else if (ctx.Match(k, Invk__zip_tid_)) zip_tid = m.ReadByte("v");
else if (ctx.Match(k, "wbase_cache_mru_size_")) wbase_cache_mru_size = m.ReadLong("v"); else if (ctx.Match(k, Invk__load_all_templates_)) load_all_templates = m.ReadYn("v");
else if (ctx.Match(k, "wbase_cache_mru_weight_")) wbase_cache_mru_weight = m.ReadLong("v"); else if (ctx.Match(k, Invk__load_all_imglinks_)) load_all_imglinks = m.ReadYn("v");
else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v"); else if (ctx.Match(k, Invk__load_ifexists_ns_)) load_ifexists_ns = m.ReadStr("v");
else if (ctx.Match(k, "page_cache_min_")) page_cache_min = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_min); else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v"));
else if (ctx.Match(k, "page_cache_max_")) page_cache_max = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_max); else if (ctx.Match(k, Invk__mgr_url_)) mgr_url = m.ReadIoUrl("v");
else return Gfo_invk_.Rv_unhandled; else if (ctx.Match(k, Invk__wkr_machine_name_)) wkr_machine_name = m.ReadStr("v");
return this; else if (ctx.Match(k, Invk__show_msg__fetched_pool_)) show_msg__fetched_pool = m.ReadYn("v");
} else if (ctx.Match(k, Invk__hdump_catboxes_)) hdump_catboxs = m.ReadYn("v");
private static final String else if (ctx.Match(k, Invk__log_math_)) log_math = m.ReadYn("v");
Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_" else if (ctx.Match(k, "indexer_enabled_")) indexer_enabled = m.ReadYn("v");
, Invk__progress_interval_ = "progress_interval_", Invk__commit_interval_ = "commit_interval_", Invk__cleanup_interval_ = "cleanup_interval_" else if (ctx.Match(k, "indexer_opt_")) indexer_opt = m.ReadStr("v");
, Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_", Invk__zip_tid_ = "zip_tid_" else if (ctx.Match(k, "wbase_cache_mru_type_")) wbase_cache_mru_type = m.ReadStr("v");
, Invk__load_all_templates_ = "load_all_templates_", Invk__load_all_imglinks_ = "load_all_imglinks_", Invk__load_ifexists_ns_ = "load_ifexists_ns_", Invk__manual_now_ = "manual_now_" else if (ctx.Match(k, "wbase_cache_mru_size_")) wbase_cache_mru_size = m.ReadLong("v");
, Invk__hdump_catboxes_ = "hdump_catboxes_" else if (ctx.Match(k, "wbase_cache_mru_weight_")) wbase_cache_mru_weight = m.ReadLong("v");
, Invk__log_math_ = "log_math_" else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v");
, Invk__mgr_url_ = "mgr_url_", Invk__wkr_machine_name_ = "wkr_machine_name_" else if (ctx.Match(k, "page_cache_min_")) page_cache_min = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_min);
, Invk__show_msg__fetched_pool_ = "show_msg__fetched_pool_" else if (ctx.Match(k, "page_cache_max_")) page_cache_max = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_max);
; else if (ctx.Match(k, Invk__err_filter)) return Scrib_err_filter_mgr.INSTANCE;
} else return Gfo_invk_.Rv_unhandled;
return this;
}
private Xop_log_invoke_wkr invoke_wkr;
private static final String
Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
, Invk__progress_interval_ = "progress_interval_", Invk__commit_interval_ = "commit_interval_", Invk__cleanup_interval_ = "cleanup_interval_"
, Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_", Invk__zip_tid_ = "zip_tid_"
, Invk__load_all_templates_ = "load_all_templates_", Invk__load_all_imglinks_ = "load_all_imglinks_", Invk__load_ifexists_ns_ = "load_ifexists_ns_", Invk__manual_now_ = "manual_now_"
, Invk__hdump_catboxes_ = "hdump_catboxes_"
, Invk__log_math_ = "log_math_"
, Invk__mgr_url_ = "mgr_url_", Invk__wkr_machine_name_ = "wkr_machine_name_"
, Invk__show_msg__fetched_pool_ = "show_msg__fetched_pool_"
, Invk__err_filter = "err_filter"
;
}

176
400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_err_filter_mgr.java
View File

@ -1,6 +1,6 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
@ -13,80 +13,100 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.xtns.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; package gplx.xowa.xtns.scribunto;
public class Scrib_err_filter_mgr implements Gfo_invk {
private final Ordered_hash hash_by_mod = Ordered_hash_.New(); import gplx.Bry_bfr;
public void Clear() {hash_by_mod.Clear();} import gplx.Bry_bfr_;
public boolean Count_eq_0() {return hash_by_mod.Count() == 0;} import gplx.GfoMsg;
public boolean Match(String mod, String fnc, String err) { import gplx.Gfo_invk;
List_adp itms = Get_itms_or_null(mod, fnc); if (itms == null) return false; import gplx.Gfo_invk_;
int itms_len = itms.Count(); import gplx.GfsCtx;
boolean match = false; import gplx.List_adp;
for (int i = 0; i < itms_len; ++i) { import gplx.List_adp_;
Scrib_err_filter_itm itm = (Scrib_err_filter_itm)itms.Get_at(i); import gplx.Ordered_hash;
if (String_.Has(err, itm.Err())) { import gplx.Ordered_hash_;
match = true; import gplx.String_;
itm.Count_actl_add_1();
break; public class Scrib_err_filter_mgr implements Gfo_invk {
} private final Object thread_lock = new Object();
} private final Ordered_hash hash_by_mod = Ordered_hash_.New();
return match; public void Clear() {hash_by_mod.Clear();}
} public boolean Empty() {return empty;} private boolean empty = true;
public void Add(int count_expd, String mod, String fnc, String err, String comment) { public boolean Match(String mod, String fnc, String err) {
List_adp itms = Get_itms_or_null(mod, fnc); List_adp itms = Get_itms_or_null(mod, fnc); if (itms == null) return false;
if (itms == null) itms = New_itms(mod, fnc); int itms_len = itms.Count();
itms.Add(new Scrib_err_filter_itm(count_expd, mod, fnc, err, comment)); boolean match = false;
} for (int i = 0; i < itms_len; ++i) {
public String Print() { Scrib_err_filter_itm itm = (Scrib_err_filter_itm)itms.Get_at(i);
Bry_bfr bfr = Bry_bfr_.New_w_size(8); if (String_.Has(err, itm.Err())) {
int i_len = hash_by_mod.Count(); match = true;
for (int i = 0; i < i_len; ++i) { itm.Count_actl_add_1();
Ordered_hash fncs = (Ordered_hash)hash_by_mod.Get_at(i); break;
int j_len = fncs.Count(); }
for (int j = 0; j < j_len; ++j) { }
List_adp errs = (List_adp)fncs.Get_at(j); return match;
int k_len = errs.Count(); }
for (int k = 0; k < k_len; ++k) { public void Add(int count_expd, String mod, String fnc, String err, String comment) {
Scrib_err_filter_itm err = (Scrib_err_filter_itm)errs.Get_at(k); synchronized (thread_lock) {
bfr.Add_int_variable(err.Count_actl()).Add_byte_pipe().Add_int_variable(err.Count_expd()) empty = false;
.Add_byte_pipe().Add_str_u8(err.Mod()).Add_byte_pipe().Add_str_u8(err.Fnc()).Add_byte_pipe().Add_str_u8(err.Err()) List_adp itms = Get_itms_or_null(mod, fnc);
.Add_byte_pipe().Add_str_u8(err.Comment()) if (itms == null) itms = New_itms(mod, fnc);
.Add_byte_nl(); itms.Add(new Scrib_err_filter_itm(count_expd, mod, fnc, err, comment));
} }
} }
} public String Print() {
return bfr.To_str_and_clear(); Bry_bfr bfr = Bry_bfr_.New_w_size(8);
} int i_len = hash_by_mod.Count();
private List_adp Get_itms_or_null(String mod, String fnc) { for (int i = 0; i < i_len; ++i) {
Ordered_hash hash_by_fnc = (Ordered_hash)hash_by_mod.Get_by(mod); if (hash_by_fnc == null) return null; Ordered_hash fncs = (Ordered_hash)hash_by_mod.Get_at(i);
return (List_adp)hash_by_fnc.Get_by(fnc); int j_len = fncs.Count();
} for (int j = 0; j < j_len; ++j) {
private List_adp New_itms(String mod, String fnc) { List_adp errs = (List_adp)fncs.Get_at(j);
Ordered_hash hash_by_fnc = (Ordered_hash)hash_by_mod.Get_by(mod); int k_len = errs.Count();
if (hash_by_fnc == null) { for (int k = 0; k < k_len; ++k) {
hash_by_fnc = Ordered_hash_.New(); Scrib_err_filter_itm err = (Scrib_err_filter_itm)errs.Get_at(k);
hash_by_mod.Add(mod, hash_by_fnc); bfr.Add_int_variable(err.Count_actl()).Add_byte_pipe().Add_int_variable(err.Count_expd())
} .Add_byte_pipe().Add_str_u8(err.Mod()).Add_byte_pipe().Add_str_u8(err.Fnc()).Add_byte_pipe().Add_str_u8(err.Err())
List_adp list_of_err = (List_adp)hash_by_fnc.Get_by(fnc); .Add_byte_pipe().Add_str_u8(err.Comment())
if (list_of_err == null) { .Add_byte_nl();
list_of_err = List_adp_.New(); }
hash_by_fnc.Add(fnc, list_of_err); }
} }
return list_of_err; return bfr.To_str_and_clear();
} }
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { private List_adp Get_itms_or_null(String mod, String fnc) {
if (ctx.Match(k, Invk_add)) Add(m.ReadInt("count_expd"), m.ReadStr("mod"), m.ReadStr("fnc"), m.ReadStr("err"), m.ReadStr("comment")); Ordered_hash hash_by_fnc = (Ordered_hash)hash_by_mod.Get_by(mod); if (hash_by_fnc == null) return null;
else return Gfo_invk_.Rv_unhandled; return (List_adp)hash_by_fnc.Get_by(fnc);
return this; }
} private static final String Invk_add = "add"; private List_adp New_itms(String mod, String fnc) {
} Ordered_hash hash_by_fnc = (Ordered_hash)hash_by_mod.Get_by(mod);
class Scrib_err_filter_itm { if (hash_by_fnc == null) {
public Scrib_err_filter_itm(int count_expd, String mod, String fnc, String err, String comment) {this.count_expd = count_expd; this.mod = mod; this.err = err; this.fnc = fnc; this.comment = comment;} hash_by_fnc = Ordered_hash_.New();
public String Mod() {return mod;} private final String mod; hash_by_mod.Add(mod, hash_by_fnc);
public String Fnc() {return fnc;} private final String fnc; }
public String Err() {return err;} private final String err; List_adp list_of_err = (List_adp)hash_by_fnc.Get_by(fnc);
public String Comment() {return comment;} private final String comment; if (list_of_err == null) {
public int Count_expd() {return count_expd;} private final int count_expd; list_of_err = List_adp_.New();
public int Count_actl() {return count_actl;} private int count_actl; hash_by_fnc.Add(fnc, list_of_err);
public void Count_actl_add_1() {++count_actl;} }
} return list_of_err;
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_add)) Add(m.ReadInt("count_expd"), m.ReadStr("mod"), m.ReadStr("fnc"), m.ReadStr("err"), m.ReadStr("comment"));
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk_add = "add";
// 2020-09-01: singleton b/c xomp instantiates multiple wikis; previous implementation was `((Scrib_xtn_mgr)(wiki.Xtn_mgr().Get_or_fail(Scrib_xtn_mgr.XTN_KEY))).Invoke_wkr();` which doesn't multi-thread
public static final Scrib_err_filter_mgr INSTANCE = new Scrib_err_filter_mgr();
}
class Scrib_err_filter_itm {
public Scrib_err_filter_itm(int count_expd, String mod, String fnc, String err, String comment) {this.count_expd = count_expd; this.mod = mod; this.err = err; this.fnc = fnc; this.comment = comment;}
public String Mod() {return mod;} private final String mod;
public String Fnc() {return fnc;} private final String fnc;
public String Err() {return err;} private final String err;
public String Comment() {return comment;} private final String comment;
public int Count_expd() {return count_expd;} private final int count_expd;
public int Count_actl() {return count_actl;} private int count_actl;
public void Count_actl_add_1() {++count_actl;}
}

351
400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func.java
View File

@ -1,168 +1,183 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis. for your project on a case-by-case basis.
The terms of each license can be found in the source code repository: The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.xtns.scribunto; import gplx.*; package gplx.xowa.xtns.scribunto;
import gplx.core.threads.Thread_adp;
import gplx.core.threads.Thread_adp_; import gplx.Bry_;
import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.Bry_bfr;
import gplx.core.brys.fmtrs.*; import gplx.core.envs.*; import gplx.Err;
import gplx.langs.htmls.*; import gplx.Err_;
import gplx.xowa.langs.kwds.*; import gplx.xowa.langs.msgs.*; import gplx.GfoMsg;
import gplx.xowa.wikis.nss.*; import gplx.Gfo_invk;
import gplx.xowa.htmls.*; import gplx.GfsCtx;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.parsers.tmpls.*; import gplx.String_;
import gplx.xowa.xtns.pfuncs.*; import gplx.core.brys.fmtrs.Bry_fmtr;
import gplx.xowa.xtns.scribunto.cfgs.ScribCfg; import gplx.core.envs.System_;
import gplx.xowa.xtns.scribunto.cfgs.ScribCfgResolver; import gplx.core.threads.Thread_adp;
import gplx.core.threads.Thread_adp_;
public class Scrib_invoke_func extends Pf_func_base { import gplx.xowa.Xoa_ttl;
@Override public int Id() {return Xol_kwd_grp_.Id_invoke;} import gplx.xowa.Xowe_wiki;
@Override public Pf_func New(int id, byte[] name) {return new Scrib_invoke_func().Name_(name);} import gplx.xowa.langs.kwds.Xol_kwd_grp_;
@Override public void Func_evaluate(Bry_bfr bfr, Xop_ctx ctx, Xot_invk caller, Xot_invk self, byte[] src) {// {{#invoke:mod_name|prc_name|prc_args...}} import gplx.xowa.langs.msgs.Xol_msg_itm_;
boolean stat_enabled = ctx.Page().Stat_itm().Enabled(); import gplx.xowa.langs.msgs.Xow_msg_mgr;
if (stat_enabled) ctx.Page().Stat_itm().Scrib().Bgn(); import gplx.xowa.parsers.Xop_ctx;
Xowe_wiki wiki = ctx.Wiki(); import gplx.xowa.parsers.logs.Xop_log_invoke_wkr;
byte[] mod_name = Eval_argx(ctx, src, caller, self); import gplx.xowa.parsers.tmpls.Xot_invk;
if (Bry_.Len_eq_0(mod_name)) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:}}" import gplx.xowa.wikis.nss.Xow_ns;
int args_len = self.Args_len(); import gplx.xowa.wikis.nss.Xow_ns_;
byte[] fnc_name = Pf_func_.Eval_arg_or(ctx, src, caller, self, args_len, 0, null); import gplx.xowa.xtns.pfuncs.Pf_func;
Xop_log_invoke_wkr invoke_wkr = ctx.Xtn__scribunto__invoke_wkr(); import gplx.xowa.xtns.pfuncs.Pf_func_;
long log_time_bgn = 0; import gplx.xowa.xtns.pfuncs.Pf_func_base;
if (invoke_wkr != null) { import gplx.xowa.xtns.scribunto.cfgs.ScribCfg;
log_time_bgn = System_.Ticks(); import gplx.xowa.xtns.scribunto.cfgs.ScribCfgResolver;
if (!invoke_wkr.Eval_bgn(ctx.Page(), mod_name, fnc_name)) return;
} public class Scrib_invoke_func extends Pf_func_base {
Scrib_core core = wiki.Parser_mgr().Scrib().Core(); @Override public int Id() {return Xol_kwd_grp_.Id_invoke;}
if (core == null) { @Override public Pf_func New(int id, byte[] name) {return new Scrib_invoke_func().Name_(name);}
synchronized (this) { @Override public void Func_evaluate(Bry_bfr bfr, Xop_ctx ctx, Xot_invk caller, Xot_invk self, byte[] src) {// {{#invoke:mod_name|prc_name|prc_args...}}
core = wiki.Parser_mgr().Scrib().Core_init(ctx); boolean stat_enabled = ctx.Page().Stat_itm().Enabled();
core.Init(); if (stat_enabled) ctx.Page().Stat_itm().Scrib().Bgn();
core.When_page_changed(ctx.Page()); Xowe_wiki wiki = ctx.Wiki();
} byte[] mod_name = Eval_argx(ctx, src, caller, self);
} if (Bry_.Len_eq_0(mod_name)) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:}}"
byte[] mod_raw = null; int args_len = self.Args_len();
Scrib_lua_mod mod = core.Mods_get(mod_name); byte[] fnc_name = Pf_func_.Eval_arg_or(ctx, src, caller, self, args_len, 0, null);
if (mod == null) { Xop_log_invoke_wkr invoke_wkr = ctx.Xtn__scribunto__invoke_wkr();
Xow_ns module_ns = wiki.Ns_mgr().Ids_get_or_null(Xow_ns_.Tid__module); long log_time_bgn = 0;
Xoa_ttl mod_ttl = Xoa_ttl.Parse(wiki, Bry_.Add(module_ns.Name_db_w_colon(), mod_name)); if (invoke_wkr != null) {
mod_raw = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(mod_ttl); log_time_bgn = System_.Ticks();
if (mod_raw == null) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:missing_mod}}" if (!invoke_wkr.Eval_bgn(ctx.Page(), mod_name, fnc_name)) return;
} }
else Scrib_core core = wiki.Parser_mgr().Scrib().Core();
mod_raw = mod.Text_bry(); if (core == null) {
if (!core.Enabled()) {bfr.Add_mid(src, self.Src_bgn(), self.Src_end()); return;} synchronized (this) {
core = wiki.Parser_mgr().Scrib().Core_init(ctx);
try { core.Init();
// DBG: test code; ISSUE#:737 core.When_page_changed(ctx.Page());
// if (String_.Eq(String_.new_u8(mod_name), "Authority control")) { }
// Tfds.Write(String_.new_u8(ctx.Page().Ttl().Page_db()), String_.new_u8(mod_name), String_.new_u8(fnc_name)); }
// } byte[] mod_raw = null;
Scrib_lua_mod mod = core.Mods_get(mod_name);
// check if configured for threaded execution if (mod == null) {
boolean exec = true; Xow_ns module_ns = wiki.Ns_mgr().Ids_get_or_null(Xow_ns_.Tid__module);
ScribCfgResolver resolver = wiki.Parser_mgr().Scrib().CfgResolver(); Xoa_ttl mod_ttl = Xoa_ttl.Parse(wiki, Bry_.Add(module_ns.Name_db_w_colon(), mod_name));
if (resolver != null) { mod_raw = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(mod_ttl);
ScribCfg cfg = resolver.Resolve(ctx.Page().Ttl().Page_db(), Xoa_ttl.Replace_spaces(mod_name), fnc_name); if (mod_raw == null) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:missing_mod}}"
if (cfg != null) { }
if (cfg.TimeoutInMs() != 0) { else
exec = false; mod_raw = mod.Text_bry();
int timeoutInMs = cfg.TimeoutInMs(); if (!core.Enabled()) {bfr.Add_mid(src, self.Src_bgn(), self.Src_end()); return;}
long timeBgn = System_.Ticks();
try {
InvokeInvoker invoker = new InvokeInvoker(core, wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name); // DBG: test code; ISSUE#:737
Thread_adp thread = Thread_adp_.Start_by_key("scribunto", invoker, "default"); // if (String_.Eq(String_.new_u8(mod_name), "Authority control")) {
while (thread.Thread__is_alive()) { // Tfds.Write(String_.new_u8(ctx.Page().Ttl().Page_db()), String_.new_u8(mod_name), String_.new_u8(fnc_name));
Thread_adp_.Sleep(cfg.SleepInMs()); // }
if (System_.Ticks__elapsed_in_frac(timeBgn) > timeoutInMs) {
thread.Thread__stop(); // check if configured for threaded execution
invoker.Exc = Err_.new_wo_type(String_.Format("scribunto timeout: page={0} mod={1} func={2} time={3}", ctx.Page_url_str(), mod_name, fnc_name, timeoutInMs)); boolean exec = true;
} ScribCfgResolver resolver = wiki.Parser_mgr().Scrib().CfgResolver();
} if (resolver != null) {
if (invoker.Exc != null) { ScribCfg cfg = resolver.Resolve(ctx.Page().Ttl().Page_db(), Xoa_ttl.Replace_spaces(mod_name), fnc_name);
throw invoker.Exc; if (cfg != null) {
} if (cfg.TimeoutInMs() != 0) {
} exec = false;
} int timeoutInMs = cfg.TimeoutInMs();
} long timeBgn = System_.Ticks();
// no threaded execution; run sequentially
if (exec) { InvokeInvoker invoker = new InvokeInvoker(core, wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name);
core.Invoke(wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name); Thread_adp thread = Thread_adp_.Start_by_key("scribunto", invoker, "default");
} while (thread.Thread__is_alive()) {
if (invoke_wkr != null) Thread_adp_.Sleep(cfg.SleepInMs());
invoke_wkr.Eval_end(ctx.Page(), mod_name, fnc_name, log_time_bgn); if (System_.Ticks__elapsed_in_frac(timeBgn) > timeoutInMs) {
} thread.Thread__stop();
catch (Throwable e) { invoker.Exc = Err_.new_wo_type(String_.Format("scribunto timeout: page={0} mod={1} func={2} time={3}", ctx.Page_url_str(), mod_name, fnc_name, timeoutInMs));
Err err = Err_.Cast_or_make(e); }
Error(bfr, wiki.Msg_mgr(), err); }
Scrib_err_filter_mgr err_filter_mgr = invoke_wkr == null ? null : invoke_wkr.Err_filter_mgr(); if (invoker.Exc != null) {
if ( err_filter_mgr == null // no err_filter_mgr defined; throw invoker.Exc;
|| err_filter_mgr.Count_eq_0() // err_filter_mgr exists, but no definitions }
|| !err_filter_mgr.Match(String_.new_u8(mod_name), String_.new_u8(fnc_name), err.To_str__msg_only())) // NOTE: must be To_str__msg_only; err_filter_mgr has defintion and it doesn't match current; print warn; DATE:2015-07-24 }
ctx.App().Usr_dlg().Warn_many("", "", "invoke failed: ~{0} ~{1} ~{2}", ctx.Page().Ttl().Raw(), Bry_.Replace_nl_w_tab(src, self.Src_bgn(), self.Src_end()), err.To_str__log()); }
wiki.Parser_mgr().Scrib().Terminate_when_page_changes_y_(); // NOTE: terminate core when page changes; not terminating now, else page with many errors will be very slow due to multiple remakes of core; PAGE:th.d:all; DATE:2014-10-03 }
} // no threaded execution; run sequentially
if (stat_enabled) ctx.Page().Stat_itm().Scrib().End(); if (exec) {
} core.Invoke(wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name);
public static void Error(Bry_bfr bfr, Xow_msg_mgr msg_mgr, Err err) {Error(bfr, msg_mgr, Err_.Cast_or_make(err).To_str__top_wo_args());}// NOTE: must use "short" error message to show in wikitext; DATE:2015-07-27 }
public static void Error(Bry_bfr bfr, Xow_msg_mgr msg_mgr, String error) { if (invoke_wkr != null)
// for Luaj, msg combines both err; split out traceback else error message will be very long; note that Warn_many will still log traceback; DATE:2016-09-09 invoke_wkr.Eval_end(ctx.Page(), mod_name, fnc_name, log_time_bgn);
String error_visible = error; }
int traceback_pos = String_.FindFwd(error, "\nstack traceback:\n"); // NOTE: produced by LuaError.getMessage() catch (Throwable e) {
if (traceback_pos != String_.Find_none) Err err = Err_.Cast_or_make(e);
error_visible = String_.Mid(error_visible, 0, traceback_pos); Error(bfr, wiki.Msg_mgr(), err);
Scrib_err_filter_mgr err_filter_mgr = Scrib_err_filter_mgr.INSTANCE;
// write "Script error: some error" if ( err_filter_mgr.Empty() // err_filter_mgr exists, but no definitions
byte[] script_error_msg = msg_mgr.Val_by_id(Xol_msg_itm_.Id_scribunto_parser_error); || !err_filter_mgr.Match(String_.new_u8(mod_name), String_.new_u8(fnc_name), err.To_str__msg_only())) // NOTE: must be To_str__msg_only; err_filter_mgr has defintion and it doesn't match current; print warn; DATE:2015-07-24
error_fmtr.Bld_bfr_many(bfr, script_error_msg, error_visible); ctx.App().Usr_dlg().Warn_many("", "", "invoke failed: ~{0} ~{1} ~{2}", ctx.Page().Ttl().Raw(), Bry_.Replace_nl_w_tab(src, self.Src_bgn(), self.Src_end()), err.To_str__log());
} wiki.Parser_mgr().Scrib().Terminate_when_page_changes_y_(); // NOTE: terminate core when page changes; not terminating now, else page with many errors will be very slow due to multiple remakes of core; PAGE:th.d:all; DATE:2014-10-03
private static final Bry_fmtr error_fmtr = Bry_fmtr.new_("<strong class=\"error\"><span class=\"scribunto-error\" id=\"mw-scribunto-error-0\">~{0}: ~{1}</span></strong>"); // NOTE: must be "error" not 'error'; iferror checks for quote not apos; DATE:2015-09-17 }
public static final String Err_mod_missing = "No such module"; if (stat_enabled) ctx.Page().Stat_itm().Scrib().End();
}
class InvokeInvoker implements Gfo_invk { public static void Error(Bry_bfr bfr, Xow_msg_mgr msg_mgr, Err err) {Error(bfr, msg_mgr, Err_.Cast_or_make(err).To_str__top_wo_args());}// NOTE: must use "short" error message to show in wikitext; DATE:2015-07-27
private final Scrib_core core; public static void Error(Bry_bfr bfr, Xow_msg_mgr msg_mgr, String error) {
private final Xowe_wiki wiki; // for Luaj, msg combines both err; split out traceback else error message will be very long; note that Warn_many will still log traceback; DATE:2016-09-09
private final Xop_ctx ctx; String error_visible = error;
private final byte[] src; int traceback_pos = String_.FindFwd(error, "\nstack traceback:\n"); // NOTE: produced by LuaError.getMessage()
private final Xot_invk caller; if (traceback_pos != String_.Find_none)
private final Xot_invk self; error_visible = String_.Mid(error_visible, 0, traceback_pos);
private final Bry_bfr bfr;
private final byte[] mod_name; // write "Script error: some error"
private final byte[] mod_raw; byte[] script_error_msg = msg_mgr.Val_by_id(Xol_msg_itm_.Id_scribunto_parser_error);
private final byte[] fnc_name; error_fmtr.Bld_bfr_many(bfr, script_error_msg, error_visible);
}
public InvokeInvoker(Scrib_core core, Xowe_wiki wiki, Xop_ctx ctx, byte[] src, Xot_invk caller, Xot_invk self, Bry_bfr bfr, byte[] mod_name, byte[] mod_raw, byte[] fnc_name) { private static final Bry_fmtr error_fmtr = Bry_fmtr.new_("<strong class=\"error\"><span class=\"scribunto-error\" id=\"mw-scribunto-error-0\">~{0}: ~{1}</span></strong>"); // NOTE: must be "error" not 'error'; iferror checks for quote not apos; DATE:2015-09-17
this.core = core; public static final String Err_mod_missing = "No such module";
this.wiki = wiki;
this.ctx = ctx; class InvokeInvoker implements Gfo_invk {
this.src = src; private final Scrib_core core;
this.caller = caller; private final Xowe_wiki wiki;
this.self = self; private final Xop_ctx ctx;
this.bfr = bfr; private final byte[] src;
this.mod_name = mod_name; private final Xot_invk caller;
this.mod_raw = mod_raw; private final Xot_invk self;
this.fnc_name = fnc_name; private final Bry_bfr bfr;
} private final byte[] mod_name;
public Exception Exc; private final byte[] mod_raw;
public Object Invk(GfsCtx gctx, int ikey, String k, GfoMsg m) { private final byte[] fnc_name;
try {
core.Invoke(wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name); public InvokeInvoker(Scrib_core core, Xowe_wiki wiki, Xop_ctx ctx, byte[] src, Xot_invk caller, Xot_invk self, Bry_bfr bfr, byte[] mod_name, byte[] mod_raw, byte[] fnc_name) {
} this.core = core;
catch (Exception exc) { this.wiki = wiki;
this.Exc = exc; this.ctx = ctx;
} this.src = src;
return null; this.caller = caller;
} this.self = self;
} this.bfr = bfr;
} this.mod_name = mod_name;
this.mod_raw = mod_raw;
this.fnc_name = fnc_name;
}
public Exception Exc;
public Object Invk(GfsCtx gctx, int ikey, String k, GfoMsg m) {
try {
core.Invoke(wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name);
}
catch (Exception exc) {
this.Exc = exc;
}
return null;
}
}
}