diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java index f72f87f39..b6a1f28da 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java @@ -1,6 +1,6 @@ /* XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012-2017 gnosygnu@gmail.com +Copyright (C) 2012-2020 gnosygnu@gmail.com XOWA is licensed under the terms of the General Public License (GPL) Version 3, or alternatively under the terms of the Apache License Version 2.0. @@ -13,84 +13,99 @@ The terms of each license can be found in the source code repository: GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ -package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; -import gplx.core.ios.streams.*; -public class Xomp_parse_mgr_cfg implements Gfo_invk { - public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1; - public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = -1; - public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000; - public int Progress_interval() {return progress_interval;} private int progress_interval = 1000; - public int Perf_interval() {return perf_interval;} private int perf_interval = 10000; - public int Commit_interval() {return commit_interval;} private int commit_interval = 10000; - public int Cleanup_interval() {return cleanup_interval;} private int cleanup_interval = 50; // setting at 1000 uses lots of memory - public boolean Hdump_enabled() {return hdump_enabled;} private boolean hdump_enabled = true; - public boolean Hdump_catboxs() {return hdump_catboxs;} private boolean hdump_catboxs = false; - public boolean Hzip_enabled() {return hzip_enabled;} private boolean hzip_enabled = true; - public boolean Hdiff_enabled() {return hdiff_enabled;} private boolean hdiff_enabled = true; - public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true; - public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true; - public boolean Load_all_imglinks() {return load_all_imglinks;} private boolean load_all_imglinks = true; - public String Load_ifexists_ns() {return load_ifexists_ns;} private String load_ifexists_ns = null; - public boolean Log_math() {return log_math;} private boolean log_math = false; - public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_tid_.Tid__gzip; - public Io_url Mgr_url() {return mgr_url;} private Io_url mgr_url; - public String Wkr_machine_name() {return wkr_machine_name;} private String wkr_machine_name; - public boolean Show_msg__fetched_pool() {return show_msg__fetched_pool;} private boolean show_msg__fetched_pool; - public boolean Indexer_enabled() {return indexer_enabled;} private boolean indexer_enabled; - public String Indexer_opt() {return indexer_opt;} private String indexer_opt = gplx.gflucene.indexers.Gflucene_idx_opt.Docs_and_freqs.Key(); - public String Wbase_cache_mru_type() {return wbase_cache_mru_type;} private String wbase_cache_mru_type = "mru"; - public long Wbase_cache_mru_size() {return wbase_cache_mru_size;} private long wbase_cache_mru_size = 100; - public long Wbase_cache_mru_weight() {return wbase_cache_mru_weight;} private long wbase_cache_mru_weight = 10; - public long Wbase_cache_mru_compress_size() {return wbase_cache_mru_compress_size;} private long wbase_cache_mru_compress_size = 70; - public long Page_cache_min() {return page_cache_min;} private long page_cache_min = 1500 * Io_mgr.Len_mb_long; - public long Page_cache_max() {return page_cache_max;} private long page_cache_max = 2000 * Io_mgr.Len_mb_long; - public void Init(Xowe_wiki wiki) { - if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count(); - if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000; - if (mgr_url == null) mgr_url = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp"); - if (wkr_machine_name == null) wkr_machine_name = gplx.core.envs.System_.Env__machine_name(); - } - - public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { - if (ctx.Match(k, Invk__num_wkrs_)) num_wkrs = m.ReadInt("v"); - else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v"); - else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v"); - else if (ctx.Match(k, Invk__progress_interval_)) progress_interval = m.ReadInt("v"); - else if (ctx.Match(k, "perf_interval_")) perf_interval = m.ReadInt("v"); - else if (ctx.Match(k, Invk__commit_interval_)) commit_interval = m.ReadInt("v"); - else if (ctx.Match(k, Invk__cleanup_interval_)) cleanup_interval = m.ReadInt("v"); - else if (ctx.Match(k, Invk__hdump_enabled_)) hdump_enabled = m.ReadYn("v"); - else if (ctx.Match(k, Invk__hzip_enabled_)) hzip_enabled = m.ReadYn("v"); - else if (ctx.Match(k, Invk__hdiff_enabled_)) hdiff_enabled = m.ReadYn("v"); - else if (ctx.Match(k, Invk__zip_tid_)) zip_tid = m.ReadByte("v"); - else if (ctx.Match(k, Invk__load_all_templates_)) load_all_templates = m.ReadYn("v"); - else if (ctx.Match(k, Invk__load_all_imglinks_)) load_all_imglinks = m.ReadYn("v"); - else if (ctx.Match(k, Invk__load_ifexists_ns_)) load_ifexists_ns = m.ReadStr("v"); - else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v")); - else if (ctx.Match(k, Invk__mgr_url_)) mgr_url = m.ReadIoUrl("v"); - else if (ctx.Match(k, Invk__wkr_machine_name_)) wkr_machine_name = m.ReadStr("v"); - else if (ctx.Match(k, Invk__show_msg__fetched_pool_)) show_msg__fetched_pool = m.ReadYn("v"); - else if (ctx.Match(k, Invk__hdump_catboxes_)) hdump_catboxs = m.ReadYn("v"); - else if (ctx.Match(k, Invk__log_math_)) log_math = m.ReadYn("v"); - else if (ctx.Match(k, "indexer_enabled_")) indexer_enabled = m.ReadYn("v"); - else if (ctx.Match(k, "indexer_opt_")) indexer_opt = m.ReadStr("v"); - else if (ctx.Match(k, "wbase_cache_mru_type_")) wbase_cache_mru_type = m.ReadStr("v"); - else if (ctx.Match(k, "wbase_cache_mru_size_")) wbase_cache_mru_size = m.ReadLong("v"); - else if (ctx.Match(k, "wbase_cache_mru_weight_")) wbase_cache_mru_weight = m.ReadLong("v"); - else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v"); - else if (ctx.Match(k, "page_cache_min_")) page_cache_min = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_min); - else if (ctx.Match(k, "page_cache_max_")) page_cache_max = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_max); - else return Gfo_invk_.Rv_unhandled; - return this; - } - private static final String - Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_" - , Invk__progress_interval_ = "progress_interval_", Invk__commit_interval_ = "commit_interval_", Invk__cleanup_interval_ = "cleanup_interval_" - , Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_", Invk__zip_tid_ = "zip_tid_" - , Invk__load_all_templates_ = "load_all_templates_", Invk__load_all_imglinks_ = "load_all_imglinks_", Invk__load_ifexists_ns_ = "load_ifexists_ns_", Invk__manual_now_ = "manual_now_" - , Invk__hdump_catboxes_ = "hdump_catboxes_" - , Invk__log_math_ = "log_math_" - , Invk__mgr_url_ = "mgr_url_", Invk__wkr_machine_name_ = "wkr_machine_name_" - , Invk__show_msg__fetched_pool_ = "show_msg__fetched_pool_" - ; -} +package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; + +import gplx.Datetime_now; +import gplx.GfoMsg; +import gplx.Gfo_invk; +import gplx.Gfo_invk_; +import gplx.GfsCtx; +import gplx.Io_mgr; +import gplx.Io_url; +import gplx.core.ios.streams.Io_stream_tid_; +import gplx.xowa.Xowe_wiki; +import gplx.xowa.parsers.logs.Xop_log_invoke_wkr; +import gplx.xowa.xtns.scribunto.Scrib_err_filter_mgr; + +public class Xomp_parse_mgr_cfg implements Gfo_invk { + public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1; + public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = -1; + public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000; + public int Progress_interval() {return progress_interval;} private int progress_interval = 1000; + public int Perf_interval() {return perf_interval;} private int perf_interval = 10000; + public int Commit_interval() {return commit_interval;} private int commit_interval = 10000; + public int Cleanup_interval() {return cleanup_interval;} private int cleanup_interval = 50; // setting at 1000 uses lots of memory + public boolean Hdump_enabled() {return hdump_enabled;} private boolean hdump_enabled = true; + public boolean Hdump_catboxs() {return hdump_catboxs;} private boolean hdump_catboxs = false; + public boolean Hzip_enabled() {return hzip_enabled;} private boolean hzip_enabled = true; + public boolean Hdiff_enabled() {return hdiff_enabled;} private boolean hdiff_enabled = true; + public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true; + public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true; + public boolean Load_all_imglinks() {return load_all_imglinks;} private boolean load_all_imglinks = true; + public String Load_ifexists_ns() {return load_ifexists_ns;} private String load_ifexists_ns = null; + public boolean Log_math() {return log_math;} private boolean log_math = false; + public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_tid_.Tid__gzip; + public Io_url Mgr_url() {return mgr_url;} private Io_url mgr_url; + public String Wkr_machine_name() {return wkr_machine_name;} private String wkr_machine_name; + public boolean Show_msg__fetched_pool() {return show_msg__fetched_pool;} private boolean show_msg__fetched_pool; + public boolean Indexer_enabled() {return indexer_enabled;} private boolean indexer_enabled; + public String Indexer_opt() {return indexer_opt;} private String indexer_opt = gplx.gflucene.indexers.Gflucene_idx_opt.Docs_and_freqs.Key(); + public String Wbase_cache_mru_type() {return wbase_cache_mru_type;} private String wbase_cache_mru_type = "mru"; + public long Wbase_cache_mru_size() {return wbase_cache_mru_size;} private long wbase_cache_mru_size = 100; + public long Wbase_cache_mru_weight() {return wbase_cache_mru_weight;} private long wbase_cache_mru_weight = 10; + public long Wbase_cache_mru_compress_size() {return wbase_cache_mru_compress_size;} private long wbase_cache_mru_compress_size = 70; + public long Page_cache_min() {return page_cache_min;} private long page_cache_min = 1500 * Io_mgr.Len_mb_long; + public long Page_cache_max() {return page_cache_max;} private long page_cache_max = 2000 * Io_mgr.Len_mb_long; + public void Init(Xowe_wiki wiki) { + if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count(); + if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000; + if (mgr_url == null) mgr_url = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp"); + if (wkr_machine_name == null) wkr_machine_name = gplx.core.envs.System_.Env__machine_name(); + } + + public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { + if (ctx.Match(k, Invk__num_wkrs_)) num_wkrs = m.ReadInt("v"); + else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v"); + else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v"); + else if (ctx.Match(k, Invk__progress_interval_)) progress_interval = m.ReadInt("v"); + else if (ctx.Match(k, "perf_interval_")) perf_interval = m.ReadInt("v"); + else if (ctx.Match(k, Invk__commit_interval_)) commit_interval = m.ReadInt("v"); + else if (ctx.Match(k, Invk__cleanup_interval_)) cleanup_interval = m.ReadInt("v"); + else if (ctx.Match(k, Invk__hdump_enabled_)) hdump_enabled = m.ReadYn("v"); + else if (ctx.Match(k, Invk__hzip_enabled_)) hzip_enabled = m.ReadYn("v"); + else if (ctx.Match(k, Invk__hdiff_enabled_)) hdiff_enabled = m.ReadYn("v"); + else if (ctx.Match(k, Invk__zip_tid_)) zip_tid = m.ReadByte("v"); + else if (ctx.Match(k, Invk__load_all_templates_)) load_all_templates = m.ReadYn("v"); + else if (ctx.Match(k, Invk__load_all_imglinks_)) load_all_imglinks = m.ReadYn("v"); + else if (ctx.Match(k, Invk__load_ifexists_ns_)) load_ifexists_ns = m.ReadStr("v"); + else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v")); + else if (ctx.Match(k, Invk__mgr_url_)) mgr_url = m.ReadIoUrl("v"); + else if (ctx.Match(k, Invk__wkr_machine_name_)) wkr_machine_name = m.ReadStr("v"); + else if (ctx.Match(k, Invk__show_msg__fetched_pool_)) show_msg__fetched_pool = m.ReadYn("v"); + else if (ctx.Match(k, Invk__hdump_catboxes_)) hdump_catboxs = m.ReadYn("v"); + else if (ctx.Match(k, Invk__log_math_)) log_math = m.ReadYn("v"); + else if (ctx.Match(k, "indexer_enabled_")) indexer_enabled = m.ReadYn("v"); + else if (ctx.Match(k, "indexer_opt_")) indexer_opt = m.ReadStr("v"); + else if (ctx.Match(k, "wbase_cache_mru_type_")) wbase_cache_mru_type = m.ReadStr("v"); + else if (ctx.Match(k, "wbase_cache_mru_size_")) wbase_cache_mru_size = m.ReadLong("v"); + else if (ctx.Match(k, "wbase_cache_mru_weight_")) wbase_cache_mru_weight = m.ReadLong("v"); + else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v"); + else if (ctx.Match(k, "page_cache_min_")) page_cache_min = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_min); + else if (ctx.Match(k, "page_cache_max_")) page_cache_max = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_max); + else if (ctx.Match(k, Invk__err_filter)) return Scrib_err_filter_mgr.INSTANCE; + else return Gfo_invk_.Rv_unhandled; + return this; + } + private Xop_log_invoke_wkr invoke_wkr; + private static final String + Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_" + , Invk__progress_interval_ = "progress_interval_", Invk__commit_interval_ = "commit_interval_", Invk__cleanup_interval_ = "cleanup_interval_" + , Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_", Invk__zip_tid_ = "zip_tid_" + , Invk__load_all_templates_ = "load_all_templates_", Invk__load_all_imglinks_ = "load_all_imglinks_", Invk__load_ifexists_ns_ = "load_ifexists_ns_", Invk__manual_now_ = "manual_now_" + , Invk__hdump_catboxes_ = "hdump_catboxes_" + , Invk__log_math_ = "log_math_" + , Invk__mgr_url_ = "mgr_url_", Invk__wkr_machine_name_ = "wkr_machine_name_" + , Invk__show_msg__fetched_pool_ = "show_msg__fetched_pool_" + , Invk__err_filter = "err_filter" + ; +} diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_err_filter_mgr.java b/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_err_filter_mgr.java index 2aa5b7ccd..0361b8936 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_err_filter_mgr.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_err_filter_mgr.java @@ -1,6 +1,6 @@ /* XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012-2017 gnosygnu@gmail.com +Copyright (C) 2012-2020 gnosygnu@gmail.com XOWA is licensed under the terms of the General Public License (GPL) Version 3, or alternatively under the terms of the Apache License Version 2.0. @@ -13,80 +13,100 @@ The terms of each license can be found in the source code repository: GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ -package gplx.xowa.xtns.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; -public class Scrib_err_filter_mgr implements Gfo_invk { - private final Ordered_hash hash_by_mod = Ordered_hash_.New(); - public void Clear() {hash_by_mod.Clear();} - public boolean Count_eq_0() {return hash_by_mod.Count() == 0;} - public boolean Match(String mod, String fnc, String err) { - List_adp itms = Get_itms_or_null(mod, fnc); if (itms == null) return false; - int itms_len = itms.Count(); - boolean match = false; - for (int i = 0; i < itms_len; ++i) { - Scrib_err_filter_itm itm = (Scrib_err_filter_itm)itms.Get_at(i); - if (String_.Has(err, itm.Err())) { - match = true; - itm.Count_actl_add_1(); - break; - } - } - return match; - } - public void Add(int count_expd, String mod, String fnc, String err, String comment) { - List_adp itms = Get_itms_or_null(mod, fnc); - if (itms == null) itms = New_itms(mod, fnc); - itms.Add(new Scrib_err_filter_itm(count_expd, mod, fnc, err, comment)); - } - public String Print() { - Bry_bfr bfr = Bry_bfr_.New_w_size(8); - int i_len = hash_by_mod.Count(); - for (int i = 0; i < i_len; ++i) { - Ordered_hash fncs = (Ordered_hash)hash_by_mod.Get_at(i); - int j_len = fncs.Count(); - for (int j = 0; j < j_len; ++j) { - List_adp errs = (List_adp)fncs.Get_at(j); - int k_len = errs.Count(); - for (int k = 0; k < k_len; ++k) { - Scrib_err_filter_itm err = (Scrib_err_filter_itm)errs.Get_at(k); - bfr.Add_int_variable(err.Count_actl()).Add_byte_pipe().Add_int_variable(err.Count_expd()) - .Add_byte_pipe().Add_str_u8(err.Mod()).Add_byte_pipe().Add_str_u8(err.Fnc()).Add_byte_pipe().Add_str_u8(err.Err()) - .Add_byte_pipe().Add_str_u8(err.Comment()) - .Add_byte_nl(); - } - } - } - return bfr.To_str_and_clear(); - } - private List_adp Get_itms_or_null(String mod, String fnc) { - Ordered_hash hash_by_fnc = (Ordered_hash)hash_by_mod.Get_by(mod); if (hash_by_fnc == null) return null; - return (List_adp)hash_by_fnc.Get_by(fnc); - } - private List_adp New_itms(String mod, String fnc) { - Ordered_hash hash_by_fnc = (Ordered_hash)hash_by_mod.Get_by(mod); - if (hash_by_fnc == null) { - hash_by_fnc = Ordered_hash_.New(); - hash_by_mod.Add(mod, hash_by_fnc); - } - List_adp list_of_err = (List_adp)hash_by_fnc.Get_by(fnc); - if (list_of_err == null) { - list_of_err = List_adp_.New(); - hash_by_fnc.Add(fnc, list_of_err); - } - return list_of_err; - } - public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { - if (ctx.Match(k, Invk_add)) Add(m.ReadInt("count_expd"), m.ReadStr("mod"), m.ReadStr("fnc"), m.ReadStr("err"), m.ReadStr("comment")); - else return Gfo_invk_.Rv_unhandled; - return this; - } private static final String Invk_add = "add"; -} -class Scrib_err_filter_itm { - public Scrib_err_filter_itm(int count_expd, String mod, String fnc, String err, String comment) {this.count_expd = count_expd; this.mod = mod; this.err = err; this.fnc = fnc; this.comment = comment;} - public String Mod() {return mod;} private final String mod; - public String Fnc() {return fnc;} private final String fnc; - public String Err() {return err;} private final String err; - public String Comment() {return comment;} private final String comment; - public int Count_expd() {return count_expd;} private final int count_expd; - public int Count_actl() {return count_actl;} private int count_actl; - public void Count_actl_add_1() {++count_actl;} -} +package gplx.xowa.xtns.scribunto; + +import gplx.Bry_bfr; +import gplx.Bry_bfr_; +import gplx.GfoMsg; +import gplx.Gfo_invk; +import gplx.Gfo_invk_; +import gplx.GfsCtx; +import gplx.List_adp; +import gplx.List_adp_; +import gplx.Ordered_hash; +import gplx.Ordered_hash_; +import gplx.String_; + +public class Scrib_err_filter_mgr implements Gfo_invk { + private final Object thread_lock = new Object(); + private final Ordered_hash hash_by_mod = Ordered_hash_.New(); + public void Clear() {hash_by_mod.Clear();} + public boolean Empty() {return empty;} private boolean empty = true; + public boolean Match(String mod, String fnc, String err) { + List_adp itms = Get_itms_or_null(mod, fnc); if (itms == null) return false; + int itms_len = itms.Count(); + boolean match = false; + for (int i = 0; i < itms_len; ++i) { + Scrib_err_filter_itm itm = (Scrib_err_filter_itm)itms.Get_at(i); + if (String_.Has(err, itm.Err())) { + match = true; + itm.Count_actl_add_1(); + break; + } + } + return match; + } + public void Add(int count_expd, String mod, String fnc, String err, String comment) { + synchronized (thread_lock) { + empty = false; + List_adp itms = Get_itms_or_null(mod, fnc); + if (itms == null) itms = New_itms(mod, fnc); + itms.Add(new Scrib_err_filter_itm(count_expd, mod, fnc, err, comment)); + } + } + public String Print() { + Bry_bfr bfr = Bry_bfr_.New_w_size(8); + int i_len = hash_by_mod.Count(); + for (int i = 0; i < i_len; ++i) { + Ordered_hash fncs = (Ordered_hash)hash_by_mod.Get_at(i); + int j_len = fncs.Count(); + for (int j = 0; j < j_len; ++j) { + List_adp errs = (List_adp)fncs.Get_at(j); + int k_len = errs.Count(); + for (int k = 0; k < k_len; ++k) { + Scrib_err_filter_itm err = (Scrib_err_filter_itm)errs.Get_at(k); + bfr.Add_int_variable(err.Count_actl()).Add_byte_pipe().Add_int_variable(err.Count_expd()) + .Add_byte_pipe().Add_str_u8(err.Mod()).Add_byte_pipe().Add_str_u8(err.Fnc()).Add_byte_pipe().Add_str_u8(err.Err()) + .Add_byte_pipe().Add_str_u8(err.Comment()) + .Add_byte_nl(); + } + } + } + return bfr.To_str_and_clear(); + } + private List_adp Get_itms_or_null(String mod, String fnc) { + Ordered_hash hash_by_fnc = (Ordered_hash)hash_by_mod.Get_by(mod); if (hash_by_fnc == null) return null; + return (List_adp)hash_by_fnc.Get_by(fnc); + } + private List_adp New_itms(String mod, String fnc) { + Ordered_hash hash_by_fnc = (Ordered_hash)hash_by_mod.Get_by(mod); + if (hash_by_fnc == null) { + hash_by_fnc = Ordered_hash_.New(); + hash_by_mod.Add(mod, hash_by_fnc); + } + List_adp list_of_err = (List_adp)hash_by_fnc.Get_by(fnc); + if (list_of_err == null) { + list_of_err = List_adp_.New(); + hash_by_fnc.Add(fnc, list_of_err); + } + return list_of_err; + } + public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { + if (ctx.Match(k, Invk_add)) Add(m.ReadInt("count_expd"), m.ReadStr("mod"), m.ReadStr("fnc"), m.ReadStr("err"), m.ReadStr("comment")); + else return Gfo_invk_.Rv_unhandled; + return this; + } private static final String Invk_add = "add"; + + // 2020-09-01: singleton b/c xomp instantiates multiple wikis; previous implementation was `((Scrib_xtn_mgr)(wiki.Xtn_mgr().Get_or_fail(Scrib_xtn_mgr.XTN_KEY))).Invoke_wkr();` which doesn't multi-thread + public static final Scrib_err_filter_mgr INSTANCE = new Scrib_err_filter_mgr(); +} +class Scrib_err_filter_itm { + public Scrib_err_filter_itm(int count_expd, String mod, String fnc, String err, String comment) {this.count_expd = count_expd; this.mod = mod; this.err = err; this.fnc = fnc; this.comment = comment;} + public String Mod() {return mod;} private final String mod; + public String Fnc() {return fnc;} private final String fnc; + public String Err() {return err;} private final String err; + public String Comment() {return comment;} private final String comment; + public int Count_expd() {return count_expd;} private final int count_expd; + public int Count_actl() {return count_actl;} private int count_actl; + public void Count_actl_add_1() {++count_actl;} +} diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func.java b/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func.java index 86fffa62a..64c64ca9a 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func.java @@ -1,168 +1,183 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012-2017 gnosygnu@gmail.com - -XOWA is licensed under the terms of the General Public License (GPL) Version 3, -or alternatively under the terms of the Apache License Version 2.0. - -You may use XOWA according to either of these licenses as is most appropriate -for your project on a case-by-case basis. - -The terms of each license can be found in the source code repository: - -GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt -Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt -*/ -package gplx.xowa.xtns.scribunto; import gplx.*; -import gplx.core.threads.Thread_adp; -import gplx.core.threads.Thread_adp_; -import gplx.xowa.*; import gplx.xowa.xtns.*; -import gplx.core.brys.fmtrs.*; import gplx.core.envs.*; -import gplx.langs.htmls.*; -import gplx.xowa.langs.kwds.*; import gplx.xowa.langs.msgs.*; -import gplx.xowa.wikis.nss.*; -import gplx.xowa.htmls.*; -import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.parsers.tmpls.*; -import gplx.xowa.xtns.pfuncs.*; -import gplx.xowa.xtns.scribunto.cfgs.ScribCfg; -import gplx.xowa.xtns.scribunto.cfgs.ScribCfgResolver; - -public class Scrib_invoke_func extends Pf_func_base { - @Override public int Id() {return Xol_kwd_grp_.Id_invoke;} - @Override public Pf_func New(int id, byte[] name) {return new Scrib_invoke_func().Name_(name);} - @Override public void Func_evaluate(Bry_bfr bfr, Xop_ctx ctx, Xot_invk caller, Xot_invk self, byte[] src) {// {{#invoke:mod_name|prc_name|prc_args...}} - boolean stat_enabled = ctx.Page().Stat_itm().Enabled(); - if (stat_enabled) ctx.Page().Stat_itm().Scrib().Bgn(); - Xowe_wiki wiki = ctx.Wiki(); - byte[] mod_name = Eval_argx(ctx, src, caller, self); - if (Bry_.Len_eq_0(mod_name)) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:}}" - int args_len = self.Args_len(); - byte[] fnc_name = Pf_func_.Eval_arg_or(ctx, src, caller, self, args_len, 0, null); - Xop_log_invoke_wkr invoke_wkr = ctx.Xtn__scribunto__invoke_wkr(); - long log_time_bgn = 0; - if (invoke_wkr != null) { - log_time_bgn = System_.Ticks(); - if (!invoke_wkr.Eval_bgn(ctx.Page(), mod_name, fnc_name)) return; - } - Scrib_core core = wiki.Parser_mgr().Scrib().Core(); - if (core == null) { - synchronized (this) { - core = wiki.Parser_mgr().Scrib().Core_init(ctx); - core.Init(); - core.When_page_changed(ctx.Page()); - } - } - byte[] mod_raw = null; - Scrib_lua_mod mod = core.Mods_get(mod_name); - if (mod == null) { - Xow_ns module_ns = wiki.Ns_mgr().Ids_get_or_null(Xow_ns_.Tid__module); - Xoa_ttl mod_ttl = Xoa_ttl.Parse(wiki, Bry_.Add(module_ns.Name_db_w_colon(), mod_name)); - mod_raw = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(mod_ttl); - if (mod_raw == null) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:missing_mod}}" - } - else - mod_raw = mod.Text_bry(); - if (!core.Enabled()) {bfr.Add_mid(src, self.Src_bgn(), self.Src_end()); return;} - - try { - // DBG: test code; ISSUE#:737 - // if (String_.Eq(String_.new_u8(mod_name), "Authority control")) { - // Tfds.Write(String_.new_u8(ctx.Page().Ttl().Page_db()), String_.new_u8(mod_name), String_.new_u8(fnc_name)); - // } - - // check if configured for threaded execution - boolean exec = true; - ScribCfgResolver resolver = wiki.Parser_mgr().Scrib().CfgResolver(); - if (resolver != null) { - ScribCfg cfg = resolver.Resolve(ctx.Page().Ttl().Page_db(), Xoa_ttl.Replace_spaces(mod_name), fnc_name); - if (cfg != null) { - if (cfg.TimeoutInMs() != 0) { - exec = false; - int timeoutInMs = cfg.TimeoutInMs(); - long timeBgn = System_.Ticks(); - - InvokeInvoker invoker = new InvokeInvoker(core, wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name); - Thread_adp thread = Thread_adp_.Start_by_key("scribunto", invoker, "default"); - while (thread.Thread__is_alive()) { - Thread_adp_.Sleep(cfg.SleepInMs()); - if (System_.Ticks__elapsed_in_frac(timeBgn) > timeoutInMs) { - thread.Thread__stop(); - invoker.Exc = Err_.new_wo_type(String_.Format("scribunto timeout: page={0} mod={1} func={2} time={3}", ctx.Page_url_str(), mod_name, fnc_name, timeoutInMs)); - } - } - if (invoker.Exc != null) { - throw invoker.Exc; - } - } - } - } - // no threaded execution; run sequentially - if (exec) { - core.Invoke(wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name); - } - if (invoke_wkr != null) - invoke_wkr.Eval_end(ctx.Page(), mod_name, fnc_name, log_time_bgn); - } - catch (Throwable e) { - Err err = Err_.Cast_or_make(e); - Error(bfr, wiki.Msg_mgr(), err); - Scrib_err_filter_mgr err_filter_mgr = invoke_wkr == null ? null : invoke_wkr.Err_filter_mgr(); - if ( err_filter_mgr == null // no err_filter_mgr defined; - || err_filter_mgr.Count_eq_0() // err_filter_mgr exists, but no definitions - || !err_filter_mgr.Match(String_.new_u8(mod_name), String_.new_u8(fnc_name), err.To_str__msg_only())) // NOTE: must be To_str__msg_only; err_filter_mgr has defintion and it doesn't match current; print warn; DATE:2015-07-24 - ctx.App().Usr_dlg().Warn_many("", "", "invoke failed: ~{0} ~{1} ~{2}", ctx.Page().Ttl().Raw(), Bry_.Replace_nl_w_tab(src, self.Src_bgn(), self.Src_end()), err.To_str__log()); - wiki.Parser_mgr().Scrib().Terminate_when_page_changes_y_(); // NOTE: terminate core when page changes; not terminating now, else page with many errors will be very slow due to multiple remakes of core; PAGE:th.d:all; DATE:2014-10-03 - } - if (stat_enabled) ctx.Page().Stat_itm().Scrib().End(); - } - public static void Error(Bry_bfr bfr, Xow_msg_mgr msg_mgr, Err err) {Error(bfr, msg_mgr, Err_.Cast_or_make(err).To_str__top_wo_args());}// NOTE: must use "short" error message to show in wikitext; DATE:2015-07-27 - public static void Error(Bry_bfr bfr, Xow_msg_mgr msg_mgr, String error) { - // for Luaj, msg combines both err; split out traceback else error message will be very long; note that Warn_many will still log traceback; DATE:2016-09-09 - String error_visible = error; - int traceback_pos = String_.FindFwd(error, "\nstack traceback:\n"); // NOTE: produced by LuaError.getMessage() - if (traceback_pos != String_.Find_none) - error_visible = String_.Mid(error_visible, 0, traceback_pos); - - // write "Script error: some error" - byte[] script_error_msg = msg_mgr.Val_by_id(Xol_msg_itm_.Id_scribunto_parser_error); - error_fmtr.Bld_bfr_many(bfr, script_error_msg, error_visible); - } - private static final Bry_fmtr error_fmtr = Bry_fmtr.new_("~{0}: ~{1}"); // NOTE: must be "error" not 'error'; iferror checks for quote not apos; DATE:2015-09-17 - public static final String Err_mod_missing = "No such module"; - - class InvokeInvoker implements Gfo_invk { - private final Scrib_core core; - private final Xowe_wiki wiki; - private final Xop_ctx ctx; - private final byte[] src; - private final Xot_invk caller; - private final Xot_invk self; - private final Bry_bfr bfr; - private final byte[] mod_name; - private final byte[] mod_raw; - private final byte[] fnc_name; - - public InvokeInvoker(Scrib_core core, Xowe_wiki wiki, Xop_ctx ctx, byte[] src, Xot_invk caller, Xot_invk self, Bry_bfr bfr, byte[] mod_name, byte[] mod_raw, byte[] fnc_name) { - this.core = core; - this.wiki = wiki; - this.ctx = ctx; - this.src = src; - this.caller = caller; - this.self = self; - this.bfr = bfr; - this.mod_name = mod_name; - this.mod_raw = mod_raw; - this.fnc_name = fnc_name; - } - public Exception Exc; - public Object Invk(GfsCtx gctx, int ikey, String k, GfoMsg m) { - try { - core.Invoke(wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name); - } - catch (Exception exc) { - this.Exc = exc; - } - return null; - } - } -} +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012-2020 gnosygnu@gmail.com + +XOWA is licensed under the terms of the General Public License (GPL) Version 3, +or alternatively under the terms of the Apache License Version 2.0. + +You may use XOWA according to either of these licenses as is most appropriate +for your project on a case-by-case basis. + +The terms of each license can be found in the source code repository: + +GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt +Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt +*/ +package gplx.xowa.xtns.scribunto; + +import gplx.Bry_; +import gplx.Bry_bfr; +import gplx.Err; +import gplx.Err_; +import gplx.GfoMsg; +import gplx.Gfo_invk; +import gplx.GfsCtx; +import gplx.String_; +import gplx.core.brys.fmtrs.Bry_fmtr; +import gplx.core.envs.System_; +import gplx.core.threads.Thread_adp; +import gplx.core.threads.Thread_adp_; +import gplx.xowa.Xoa_ttl; +import gplx.xowa.Xowe_wiki; +import gplx.xowa.langs.kwds.Xol_kwd_grp_; +import gplx.xowa.langs.msgs.Xol_msg_itm_; +import gplx.xowa.langs.msgs.Xow_msg_mgr; +import gplx.xowa.parsers.Xop_ctx; +import gplx.xowa.parsers.logs.Xop_log_invoke_wkr; +import gplx.xowa.parsers.tmpls.Xot_invk; +import gplx.xowa.wikis.nss.Xow_ns; +import gplx.xowa.wikis.nss.Xow_ns_; +import gplx.xowa.xtns.pfuncs.Pf_func; +import gplx.xowa.xtns.pfuncs.Pf_func_; +import gplx.xowa.xtns.pfuncs.Pf_func_base; +import gplx.xowa.xtns.scribunto.cfgs.ScribCfg; +import gplx.xowa.xtns.scribunto.cfgs.ScribCfgResolver; + +public class Scrib_invoke_func extends Pf_func_base { + @Override public int Id() {return Xol_kwd_grp_.Id_invoke;} + @Override public Pf_func New(int id, byte[] name) {return new Scrib_invoke_func().Name_(name);} + @Override public void Func_evaluate(Bry_bfr bfr, Xop_ctx ctx, Xot_invk caller, Xot_invk self, byte[] src) {// {{#invoke:mod_name|prc_name|prc_args...}} + boolean stat_enabled = ctx.Page().Stat_itm().Enabled(); + if (stat_enabled) ctx.Page().Stat_itm().Scrib().Bgn(); + Xowe_wiki wiki = ctx.Wiki(); + byte[] mod_name = Eval_argx(ctx, src, caller, self); + if (Bry_.Len_eq_0(mod_name)) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:}}" + int args_len = self.Args_len(); + byte[] fnc_name = Pf_func_.Eval_arg_or(ctx, src, caller, self, args_len, 0, null); + Xop_log_invoke_wkr invoke_wkr = ctx.Xtn__scribunto__invoke_wkr(); + long log_time_bgn = 0; + if (invoke_wkr != null) { + log_time_bgn = System_.Ticks(); + if (!invoke_wkr.Eval_bgn(ctx.Page(), mod_name, fnc_name)) return; + } + Scrib_core core = wiki.Parser_mgr().Scrib().Core(); + if (core == null) { + synchronized (this) { + core = wiki.Parser_mgr().Scrib().Core_init(ctx); + core.Init(); + core.When_page_changed(ctx.Page()); + } + } + byte[] mod_raw = null; + Scrib_lua_mod mod = core.Mods_get(mod_name); + if (mod == null) { + Xow_ns module_ns = wiki.Ns_mgr().Ids_get_or_null(Xow_ns_.Tid__module); + Xoa_ttl mod_ttl = Xoa_ttl.Parse(wiki, Bry_.Add(module_ns.Name_db_w_colon(), mod_name)); + mod_raw = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(mod_ttl); + if (mod_raw == null) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:missing_mod}}" + } + else + mod_raw = mod.Text_bry(); + if (!core.Enabled()) {bfr.Add_mid(src, self.Src_bgn(), self.Src_end()); return;} + + try { + // DBG: test code; ISSUE#:737 + // if (String_.Eq(String_.new_u8(mod_name), "Authority control")) { + // Tfds.Write(String_.new_u8(ctx.Page().Ttl().Page_db()), String_.new_u8(mod_name), String_.new_u8(fnc_name)); + // } + + // check if configured for threaded execution + boolean exec = true; + ScribCfgResolver resolver = wiki.Parser_mgr().Scrib().CfgResolver(); + if (resolver != null) { + ScribCfg cfg = resolver.Resolve(ctx.Page().Ttl().Page_db(), Xoa_ttl.Replace_spaces(mod_name), fnc_name); + if (cfg != null) { + if (cfg.TimeoutInMs() != 0) { + exec = false; + int timeoutInMs = cfg.TimeoutInMs(); + long timeBgn = System_.Ticks(); + + InvokeInvoker invoker = new InvokeInvoker(core, wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name); + Thread_adp thread = Thread_adp_.Start_by_key("scribunto", invoker, "default"); + while (thread.Thread__is_alive()) { + Thread_adp_.Sleep(cfg.SleepInMs()); + if (System_.Ticks__elapsed_in_frac(timeBgn) > timeoutInMs) { + thread.Thread__stop(); + invoker.Exc = Err_.new_wo_type(String_.Format("scribunto timeout: page={0} mod={1} func={2} time={3}", ctx.Page_url_str(), mod_name, fnc_name, timeoutInMs)); + } + } + if (invoker.Exc != null) { + throw invoker.Exc; + } + } + } + } + // no threaded execution; run sequentially + if (exec) { + core.Invoke(wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name); + } + if (invoke_wkr != null) + invoke_wkr.Eval_end(ctx.Page(), mod_name, fnc_name, log_time_bgn); + } + catch (Throwable e) { + Err err = Err_.Cast_or_make(e); + Error(bfr, wiki.Msg_mgr(), err); + Scrib_err_filter_mgr err_filter_mgr = Scrib_err_filter_mgr.INSTANCE; + if ( err_filter_mgr.Empty() // err_filter_mgr exists, but no definitions + || !err_filter_mgr.Match(String_.new_u8(mod_name), String_.new_u8(fnc_name), err.To_str__msg_only())) // NOTE: must be To_str__msg_only; err_filter_mgr has defintion and it doesn't match current; print warn; DATE:2015-07-24 + ctx.App().Usr_dlg().Warn_many("", "", "invoke failed: ~{0} ~{1} ~{2}", ctx.Page().Ttl().Raw(), Bry_.Replace_nl_w_tab(src, self.Src_bgn(), self.Src_end()), err.To_str__log()); + wiki.Parser_mgr().Scrib().Terminate_when_page_changes_y_(); // NOTE: terminate core when page changes; not terminating now, else page with many errors will be very slow due to multiple remakes of core; PAGE:th.d:all; DATE:2014-10-03 + } + if (stat_enabled) ctx.Page().Stat_itm().Scrib().End(); + } + public static void Error(Bry_bfr bfr, Xow_msg_mgr msg_mgr, Err err) {Error(bfr, msg_mgr, Err_.Cast_or_make(err).To_str__top_wo_args());}// NOTE: must use "short" error message to show in wikitext; DATE:2015-07-27 + public static void Error(Bry_bfr bfr, Xow_msg_mgr msg_mgr, String error) { + // for Luaj, msg combines both err; split out traceback else error message will be very long; note that Warn_many will still log traceback; DATE:2016-09-09 + String error_visible = error; + int traceback_pos = String_.FindFwd(error, "\nstack traceback:\n"); // NOTE: produced by LuaError.getMessage() + if (traceback_pos != String_.Find_none) + error_visible = String_.Mid(error_visible, 0, traceback_pos); + + // write "Script error: some error" + byte[] script_error_msg = msg_mgr.Val_by_id(Xol_msg_itm_.Id_scribunto_parser_error); + error_fmtr.Bld_bfr_many(bfr, script_error_msg, error_visible); + } + private static final Bry_fmtr error_fmtr = Bry_fmtr.new_("~{0}: ~{1}"); // NOTE: must be "error" not 'error'; iferror checks for quote not apos; DATE:2015-09-17 + public static final String Err_mod_missing = "No such module"; + + class InvokeInvoker implements Gfo_invk { + private final Scrib_core core; + private final Xowe_wiki wiki; + private final Xop_ctx ctx; + private final byte[] src; + private final Xot_invk caller; + private final Xot_invk self; + private final Bry_bfr bfr; + private final byte[] mod_name; + private final byte[] mod_raw; + private final byte[] fnc_name; + + public InvokeInvoker(Scrib_core core, Xowe_wiki wiki, Xop_ctx ctx, byte[] src, Xot_invk caller, Xot_invk self, Bry_bfr bfr, byte[] mod_name, byte[] mod_raw, byte[] fnc_name) { + this.core = core; + this.wiki = wiki; + this.ctx = ctx; + this.src = src; + this.caller = caller; + this.self = self; + this.bfr = bfr; + this.mod_name = mod_name; + this.mod_raw = mod_raw; + this.fnc_name = fnc_name; + } + public Exception Exc; + public Object Invk(GfsCtx gctx, int ikey, String k, GfoMsg m) { + try { + core.Invoke(wiki, ctx, src, caller, self, bfr, mod_name, mod_raw, fnc_name); + } + catch (Exception exc) { + this.Exc = exc; + } + return null; + } + } +}