diff --git a/400_xowa/src/gplx/xowa/Xop_fxt.java b/400_xowa/src/gplx/xowa/Xop_fxt.java index 9f54f599a..5015af6c7 100644 --- a/400_xowa/src/gplx/xowa/Xop_fxt.java +++ b/400_xowa/src/gplx/xowa/Xop_fxt.java @@ -13,521 +13,520 @@ The terms of each license can be found in the source code repository: GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ -package gplx.xowa; import gplx.*; -import gplx.core.tests.*; import gplx.core.log_msgs.*; -import gplx.xowa.apps.cfgs.*; -import gplx.xowa.langs.*; import gplx.xowa.langs.kwds.*; import gplx.xowa.langs.msgs.*; -import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.htmls.*; -import gplx.xowa.parsers.*; import gplx.xowa.parsers.amps.*; import gplx.xowa.parsers.apos.*; import gplx.xowa.parsers.hdrs.*; import gplx.xowa.parsers.lists.*; import gplx.xowa.parsers.paras.*; import gplx.xowa.parsers.xndes.*; import gplx.xowa.parsers.tmpls.*; import gplx.xowa.parsers.miscs.*; import gplx.xowa.parsers.tblws.*; import gplx.xowa.parsers.lnkes.*; import gplx.xowa.parsers.lnkis.*; -import gplx.xowa.files.exts.*; import gplx.xowa.files.repos.*; import gplx.xowa.htmls.core.wkrs.lnkis.htmls.*; -import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.domains.*; -import gplx.xowa.wikis.tdbs.hives.*; -import gplx.xowa.wikis.pages.*; -public class Xop_fxt { - public Xop_fxt() { - Xoae_app app = Xoa_app_fxt.Make__app__edit(); - wiki = Xoa_app_fxt.Make__wiki__edit(app); - ctor(app, wiki); - } - public Xop_fxt(Xoae_app app, Xowe_wiki wiki) { - this.ctor(app, wiki); - } - public Xop_fxt(Xowe_wiki wiki) { - this.ctor(Xoa_app_fxt.Make__app__edit(), wiki); - } - private void ctor(Xoae_app app, Xowe_wiki wiki) { - this.app = app; - this.wiki = wiki; - - // NOTE: Xop_fxt does not call Init_by_app / Init_by_wiki b/c of test issues with DB; DATE:2019-09-15 - // call these methods manually - app.Addon_mgr().Add_dflts_by_app(app).Run_by_app(app); - wiki.Init_once(); - - app.Wiki_mgr().Add(wiki); - app.File_mgr().Repo_mgr().Set("src:wiki", "mem/wiki/repo/src/", wiki.Domain_str()).Ext_rules_(Xof_rule_grp.Grp_app_default).Dir_depth_(2); - app.File_mgr().Repo_mgr().Set("trg:wiki", "mem/wiki/repo/trg/", wiki.Domain_str()).Ext_rules_(Xof_rule_grp.Grp_app_default).Dir_depth_(2).Primary_(true); - wiki.File_mgr().Repo_mgr().Add_repo(Bry_.new_a7("src:wiki"), Bry_.new_a7("trg:wiki")); - ctx = wiki.Parser_mgr().Ctx(); - mock_wkr.Clear_commons(); // assume all files are in repo 0 - wiki.File_mgr().Repo_mgr().Page_finder_(mock_wkr); - parser = wiki.Parser_mgr().Main(); - this.tkn_mkr = app.Parser_mgr().Tkn_mkr(); - ctx.Para().Enabled_n_(); - hdom_wtr = wiki.Html_mgr().Html_wtr(); - wiki.Html_mgr().Img_suppress_missing_src_(false); - wiki.Xtn_mgr().Init_by_wiki(wiki); - Page_ttl_(Ttl_str); - Xot_invk_tkn.Cache_enabled = false;// always disable cache for tests; can cause strange behavior when running entire suite and lnki_temp test turns on; - } - private Xofw_wiki_wkr_mock mock_wkr = new Xofw_wiki_wkr_mock(); - public Xoae_app App() {return app;} private Xoae_app app; - public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki; - public Xop_ctx Ctx() {return ctx;} private Xop_ctx ctx; - public Xop_parser Parser() {return parser;} private Xop_parser parser; - public Xoae_page Page() {return ctx.Page();} - public void Lang_by_id_(int id) {ctx.Page().Lang_(wiki.Appe().Lang_mgr().Get_by_or_new(Xol_lang_stub_.Get_by_id(id).Key()));} - public Xoh_html_wtr_cfg Wtr_cfg() {return hdom_wtr.Cfg();} private Xoh_html_wtr hdom_wtr; - public Xop_fxt Reset() { - ctx.Clear_all(); - ctx.App().Free_mem(false); - ctx.Page().Clear_all(); - wiki.File_mgr().Clear_for_tests(); - wiki.Db_mgr().Load_mgr().Clear(); - wiki.Db_mgr().Save_mgr().Clear(); - app.Wiki_mgr().Clear(); - Io_mgr.Instance.InitEngine_mem(); // clear created pages - wiki.Cfg_parser().Display_title_restrict_(false); // default to false, as a small number of tests assume restrict = false; - return this; - } - public Xop_fxt Reset_for_msgs() { - Io_mgr.Instance.InitEngine_mem(); - wiki.Lang().Msg_mgr().Clear(); // need to clear out lang - wiki.Msg_mgr().Clear(); // need to clear out wiki.Msgs - this.Reset(); - return this; - } - public Xoa_ttl Page_ttl_(String txt) { - Xoa_ttl rv = Xoa_ttl.Parse(wiki, Bry_.new_u8(txt)); - ctx.Page().Ttl_(rv); - return rv; - } - - public Xop_tkn_chkr_base tkn_bry_(int bgn, int end) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_bry).Src_rng_(bgn, end);} - public Xop_tkn_chkr_base tkn_txt_() {return tkn_txt_(String_.Pos_neg1, String_.Pos_neg1);} - public Xop_tkn_chkr_base tkn_txt_(int bgn, int end) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_txt).Src_rng_(bgn, end);} - public Xop_tkn_chkr_base tkn_space_() {return tkn_space_(String_.Pos_neg1, String_.Pos_neg1);} - public Xop_tkn_chkr_base tkn_space_(int bgn, int end) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_space).Src_rng_(bgn, end);} - public Xop_tkn_chkr_base tkn_eq_(int bgn) {return tkn_eq_(bgn, bgn + 1);} - public Xop_tkn_chkr_base tkn_eq_(int bgn, int end) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_eq).Src_rng_(bgn, end);} - public Xop_tkn_chkr_base tkn_colon_(int bgn) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_colon).Src_rng_(bgn, bgn + 1);} - public Xop_tkn_chkr_base tkn_pipe_(int bgn) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_pipe).Src_rng_(bgn, bgn + 1);} - public Xop_tkn_chkr_base tkn_tab_(int bgn) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_tab).Src_rng_(bgn, bgn + 1);} - public Xop_apos_tkn_chkr tkn_apos_(int cmd) {return new Xop_apos_tkn_chkr().Apos_cmd_(cmd);} - public Xop_ignore_tkn_chkr tkn_comment_(int bgn, int end) {return tkn_ignore_(bgn, end, Xop_ignore_tkn.Ignore_tid_comment);} - public Xop_ignore_tkn_chkr tkn_ignore_(int bgn, int end, byte t){return (Xop_ignore_tkn_chkr)new Xop_ignore_tkn_chkr().Ignore_tid_(t).Src_rng_(bgn, end);} - public Xop_tkn_chkr_hr tkn_hr_(int bgn, int end) {return new Xop_tkn_chkr_hr(bgn, end).Hr_len_(Xop_hr_lxr.Hr_len);} - public Xop_tblw_tb_tkn_chkr tkn_tblw_tb_(int bgn, int end) {return (Xop_tblw_tb_tkn_chkr)new Xop_tblw_tb_tkn_chkr().Src_rng_(bgn, end);} - public Xop_tblw_tc_tkn_chkr tkn_tblw_tc_(int bgn, int end) {return (Xop_tblw_tc_tkn_chkr)new Xop_tblw_tc_tkn_chkr().Src_rng_(bgn, end);} - public Xop_tblw_td_tkn_chkr tkn_tblw_td_(int bgn, int end) {return (Xop_tblw_td_tkn_chkr)new Xop_tblw_td_tkn_chkr().Src_rng_(bgn, end);} - public Xop_tblw_th_tkn_chkr tkn_tblw_th_(int bgn, int end) {return (Xop_tblw_th_tkn_chkr)new Xop_tblw_th_tkn_chkr().Src_rng_(bgn, end);} - public Xop_tblw_tr_tkn_chkr tkn_tblw_tr_(int bgn, int end) {return (Xop_tblw_tr_tkn_chkr)new Xop_tblw_tr_tkn_chkr().Src_rng_(bgn, end);} - public Xop_xnde_tkn_chkr tkn_xnde_br_(int pos) {return tkn_xnde_(pos, pos).Xnde_tagId_(Xop_xnde_tag_.Tid__br);} - public Xop_xnde_tkn_chkr tkn_xnde_() {return tkn_xnde_(String_.Pos_neg1, String_.Pos_neg1);} - public Xop_xnde_tkn_chkr tkn_xnde_(int bgn, int end) {return (Xop_xnde_tkn_chkr)new Xop_xnde_tkn_chkr().Src_rng_(bgn, end);} - public Xop_tkn_chkr_base tkn_curly_bgn_(int bgn) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_tmpl_curly_bgn).Src_rng_(bgn, bgn + 2);} - public Xop_tkn_chkr_base tkn_para_blank_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_none, Xop_para_tkn.Tid_none);} - public Xop_tkn_chkr_base tkn_para_bgn_pre_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_none, Xop_para_tkn.Tid_pre);} - public Xop_tkn_chkr_base tkn_para_bgn_para_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_none, Xop_para_tkn.Tid_para);} - public Xop_tkn_chkr_base tkn_para_mid_para_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_para, Xop_para_tkn.Tid_para);} - public Xop_tkn_chkr_base tkn_para_end_para_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_para, Xop_para_tkn.Tid_none);} - public Xop_tkn_chkr_base tkn_para_end_pre_bgn_para_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_pre , Xop_para_tkn.Tid_para);} - public Xop_tkn_chkr_base tkn_para_end_para_bgn_pre_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_para, Xop_para_tkn.Tid_pre);} - public Xop_tkn_chkr_base tkn_para_end_pre_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_pre , Xop_para_tkn.Tid_none);} - public Xop_tkn_chkr_base tkn_para_(int pos, byte end, byte bgn) {return new Xop_para_tkn_chkr().Para_end_(end).Para_bgn_(bgn).Src_rng_(pos, pos);} - public Xop_tkn_chkr_base tkn_nl_char_(int bgn, int end) {return tkn_nl_(bgn, end, Xop_nl_tkn.Tid_char);} - public Xop_tkn_chkr_base tkn_nl_char_len1_(int bgn) {return tkn_nl_(bgn, bgn + 1, Xop_nl_tkn.Tid_char);} - public Xop_tkn_chkr_base tkn_nl_char_len0_(int pos) {return tkn_nl_(pos, pos, Xop_nl_tkn.Tid_char);} - public Xop_tkn_chkr_base tkn_nl_(int bgn, int end, byte tid) {return new Xop_nl_tkn_chkr().Nl_tid_(tid).Src_rng_(bgn, end);} - public Xop_list_tkn_chkr tkn_list_bgn_(int bgn, int end, byte listType) {return (Xop_list_tkn_chkr)new Xop_list_tkn_chkr().List_itmTyp_(listType).Src_rng_(bgn, end);} - public Xop_list_tkn_chkr tkn_list_end_(int pos) {return (Xop_list_tkn_chkr)new Xop_list_tkn_chkr().Src_rng_(pos, pos);} - public Xop_tkn_chkr_lnke tkn_lnke_(int bgn, int end) {return new Xop_tkn_chkr_lnke(bgn, end);} - public Xop_lnki_tkn_chkr tkn_lnki_() {return tkn_lnki_(-1, -1);} - public Xop_lnki_tkn_chkr tkn_lnki_(int bgn, int end) {return (Xop_lnki_tkn_chkr)new Xop_lnki_tkn_chkr().Src_rng_(bgn, end);} - public Xop_arg_itm_tkn_chkr tkn_arg_itm_(Xop_tkn_chkr_base... subs) {return (Xop_arg_itm_tkn_chkr)new Xop_arg_itm_tkn_chkr().Subs_(subs);} - public Xop_arg_nde_tkn_chkr tkn_arg_nde_() {return tkn_arg_nde_(String_.Pos_neg1, String_.Pos_neg1);} - public Xop_arg_nde_tkn_chkr tkn_arg_nde_(int bgn, int end) {return (Xop_arg_nde_tkn_chkr)new Xop_arg_nde_tkn_chkr().Src_rng_(bgn, end);} - public Xop_arg_nde_tkn_chkr tkn_arg_val_(Xop_tkn_chkr_base... subs) { - Xop_arg_nde_tkn_chkr rv = new Xop_arg_nde_tkn_chkr(); - Xop_arg_itm_tkn_chkr val = new Xop_arg_itm_tkn_chkr(); - val.Subs_(subs); - rv.Val_tkn_(val); - return rv; - } - public Xop_arg_nde_tkn_chkr tkn_arg_val_txt_(int bgn, int end) { - Xop_arg_nde_tkn_chkr rv = new Xop_arg_nde_tkn_chkr(); - Xop_arg_itm_tkn_chkr itm = new Xop_arg_itm_tkn_chkr(); - rv.Val_tkn_(itm); - itm.Src_rng_(bgn, end).Subs_(tkn_txt_(bgn, end)); - return rv; - } - Xop_arg_nde_tkn_chkr tkn_arg_key_txt_(int bgn, int end) { - Xop_arg_nde_tkn_chkr rv = new Xop_arg_nde_tkn_chkr(); - Xop_arg_itm_tkn_chkr itm = new Xop_arg_itm_tkn_chkr(); - rv.Key_tkn_(itm); - itm.Src_rng_(bgn, end).Subs_(tkn_txt_(bgn, end)); - return rv; - } - public Xot_invk_tkn_chkr tkn_tmpl_invk_(int bgn, int end) {return (Xot_invk_tkn_chkr)new Xot_invk_tkn_chkr().Src_rng_(bgn, end);} - public Xot_invk_tkn_chkr tkn_tmpl_invk_w_name(int bgn, int end, int name_bgn, int name_end) { - Xot_invk_tkn_chkr rv = new Xot_invk_tkn_chkr(); - rv.Src_rng_(bgn, end); - rv.Name_tkn_(tkn_arg_key_txt_(name_bgn, name_end)); - return rv; - } - public Xot_prm_chkr tkn_tmpl_prm_find_(Xop_tkn_chkr_base find) { - Xot_prm_chkr rv = new Xot_prm_chkr(); - rv.Find_tkn_(tkn_arg_itm_(find)); - return rv; - } - public void Init__file_find_mgr() { - Xowe_wiki commons_wiki = this.app.Wiki_mgr().Get_by_or_make(Xow_domain_itm_.Bry__commons); - Xof_repo_itm repo_itm = app.File_mgr().Repo_mgr().Get_at(0); - commons_wiki.Html_mgr().Html_wtr().Lnki_wtr().File_wtr().Find_file_mgr_(new Xow_find_file_mgr(repo_itm, commons_wiki.Domain_str())); - this.wiki.Html_mgr().Html_wtr().Lnki_wtr().File_wtr().Find_file_mgr_(new Xow_find_file_mgr(repo_itm, this.wiki.Domain_str())); - } - public void Init__orig(boolean wiki_is_commons, String orig_ttl, int orig_w, int orig_h) { - Xowe_wiki orig_wiki = wiki_is_commons ? (Xowe_wiki)this.wiki : (Xowe_wiki)app.Wiki_mgr().Get_by_or_null(gplx.xowa.wikis.domains.Xow_domain_itm_.Bry__commons); - orig_wiki.Html_mgr().Html_wtr().Lnki_wtr().File_wtr().Find_file_mgr().Init_file(orig_ttl, orig_w, orig_h); - } - public void Init__msg(String key, String val) { - Xol_msg_itm msg_itm = wiki.Msg_mgr().Get_or_make(Bry_.new_u8(key)); - msg_itm.Atrs_set(Bry_.new_u8(val), false, false); - } - public Xop_fxt Init_para_y_() {ctx.Para().Enabled_y_(); return this;} - public Xop_fxt Init_para_n_() {ctx.Para().Enabled_n_(); return this;} - public Xop_fxt Init_log_(Gfo_msg_itm... itms) {for (Gfo_msg_itm itm : itms) log_itms.Add(itm); return this;} List_adp log_itms = List_adp_.New(); - public void Init_defn_add(String name, String text) {Init_defn_add(name, text, Xow_ns_case_.Tid__all);} - public void Init_defn_add(String name, String text, byte case_match) {Init_defn_add(wiki, name, text, case_match);} - public void Init_defn_add(Xowe_wiki w, String name, String text, byte case_match) { - Xot_defn_tmpl itm = run_Parse_tmpl(w, Bry_.new_a7(name), Bry_.new_u8(text)); - w.Cache_mgr().Defn_cache().Add(itm, case_match); - } - public void Init_defn_clear() {wiki.Cache_mgr().Defn_cache().Free_mem_all();} - public Xop_fxt Init_id_create(int id, int fil_idx, int row_idx, boolean type_redirect, int itm_len, int ns_id, String ttl) {Xow_hive_mgr_fxt.Create_id(app, wiki.Hive_mgr(), id, fil_idx, row_idx, type_redirect, itm_len, ns_id, ttl); return this;} - public Xop_fxt Init_page_create(String ttl) {return Init_page_create(wiki, ttl, "");} - public Xop_fxt Init_page_create(String ttl, String txt) {return Init_page_create(wiki, ttl, txt);} - public Xop_fxt Init_page_create(Xowe_wiki wiki, String ttl, String txt) {Init_page_create_static(wiki, ttl, txt);return this;} - public static void Init_page_create_static(Xowe_wiki wiki, String ttl_str, String text_str) { - Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.new_u8(ttl_str)); - byte[] text = Bry_.new_u8(text_str); - wiki.Db_mgr().Save_mgr().Data_create(wiki, ttl, text); - } - public static void Init_msg(Xowe_wiki wiki, String key, String val) { - wiki.Lang().Msg_mgr().Itm_by_key_or_new(key, val); - } - public Xop_fxt Init_page_update(String ttl, String txt) {return Init_page_update(wiki, ttl, txt);} - public Xop_fxt Init_page_update(Xowe_wiki wiki, String ttl, String txt) { - Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, Bry_.new_u8(ttl)); - byte[] page_raw = Bry_.new_u8(txt); - Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(page_ttl); - wiki.Db_mgr().Save_mgr().Data_update(page, page_raw); - return this; - } - public Xop_fxt Init_xwiki_clear() { - wiki.Xwiki_mgr().Clear(); - app.Usere().Wiki().Xwiki_mgr().Clear(); - return this; - } - public Xop_fxt Init_xwiki_add_wiki_and_user_(String alias, String domain) { - wiki.Xwiki_mgr().Add_by_atrs(alias, domain); - app.Usere().Wiki().Xwiki_mgr().Add_by_atrs(domain, domain); - return this; - } - public Xop_fxt Init_xwiki_add_user_(String domain) {return Init_xwiki_add_user_(domain, domain);} - public Xop_fxt Init_xwiki_add_user_(String alias, String domain) { - app.Usere().Wiki().Xwiki_mgr().Add_by_atrs(alias, domain); - return this; - } - public void Test_parse_template(String tmpl_raw, String expd) {Test_parse_tmpl_str_test(tmpl_raw, "{{test}}", expd);} - public void Test_parse_tmpl_str_test(String tmpl_raw, String page_raw, String expd) {Test_parse_tmpl_str_test(wiki, tmpl_raw, page_raw, expd);} - public void Test_parse_tmpl_str_test(Xowe_wiki w, String tmpl_raw, String page_raw, String expd) { - Init_defn_add(w, "test", tmpl_raw, Xow_ns_case_.Tid__all); - Test_parse_tmpl_str(w, page_raw, expd); - } - public void Test_parse_tmpl_str(String raw, String expd) {Test_parse_tmpl_str(wiki, raw, expd);} - public void Test_parse_tmpl_str(Xowe_wiki w, String raw, String expd) { - byte[] actl = Test_parse_tmpl_str_rv(w, raw); - Tfds.Eq_str_lines(expd, String_.new_u8(actl)); - tst_Log_check(); - } - public byte[] Test_parse_tmpl_str_rv(String raw) {return Test_parse_tmpl_str_rv(wiki, raw);} - public byte[] Test_parse_tmpl_str_rv(Xowe_wiki w, String raw) { - byte[] raw_bry = Bry_.new_u8(raw); - Xop_root_tkn root = tkn_mkr.Root(raw_bry); - Xop_ctx c = w.Parser_mgr().Ctx(); - c.Page().Root_(root); - c.Page().Db().Text().Text_bry_(raw_bry); - return w.Parser_mgr().Main().Expand_tmpl(root, c, tkn_mkr, raw_bry); - } - public Xot_defn_tmpl run_Parse_tmpl(byte[] name, byte[] raw) {return run_Parse_tmpl(wiki, name, raw);} - public Xot_defn_tmpl run_Parse_tmpl(Xowe_wiki w, byte[] name, byte[] raw) { - Xop_ctx c = w.Parser_mgr().Ctx(); - return w.Parser_mgr().Main().Parse_text_to_defn_obj(c, c.Tkn_mkr(), w.Ns_mgr().Ns_template(), name, raw); - } - public void Test_parse_tmpl(String raw, Tst_chkr... expd) { - byte[] raw_bry = Bry_.new_u8(raw); - Xot_defn_tmpl itm = run_Parse_tmpl(Bry_.Empty, raw_bry); - Parse_chk(raw_bry, itm.Root(), expd); - } - public void Test_parse_page_tmpl_str(String raw, String expd) { - byte[] raw_bry = Bry_.new_u8(raw); - Xop_root_tkn root = tkn_mkr.Root(raw_bry); - byte[] actl = parser.Expand_tmpl(root, ctx, tkn_mkr, raw_bry); - Tfds.Eq(expd, String_.new_u8(actl)); - tst_Log_check(); - } - public Xop_root_tkn Test_parse_page_tmpl_tkn(String raw) { - byte[] raw_bry = Bry_.new_u8(raw); - Xop_root_tkn root = tkn_mkr.Root(raw_bry); - parser.Expand_tmpl(root, ctx, tkn_mkr, raw_bry); - return root; - } - public void Test_parse_page_tmpl(String raw, Tst_chkr... expd_ary) { - byte[] raw_bry = Bry_.new_u8(raw); - Xop_root_tkn root = tkn_mkr.Root(raw_bry); - parser.Expand_tmpl(root, ctx, tkn_mkr, raw_bry); - Parse_chk(raw_bry, root, expd_ary); - } - public void Test_parse_page_wiki(String raw, Tst_chkr... expd_ary) { - byte[] raw_bry = Bry_.new_u8(raw); - Xop_root_tkn root = Test_parse_page_wiki_root(raw_bry); - Parse_chk(raw_bry, root, expd_ary); - } - public Xop_root_tkn Test_parse_page_wiki_root(String raw) {return Test_parse_page_wiki_root(Bry_.new_u8(raw));} - Xop_root_tkn Test_parse_page_wiki_root(byte[] raw_bry) { - Xop_root_tkn root = tkn_mkr.Root(raw_bry); - parser.Parse_wtxt_to_wdom(root, ctx, tkn_mkr, raw_bry, Xop_parser_.Doc_bgn_bos); - return root; - } - public void Test_parse_page_all(String raw, Tst_chkr... expd_ary) { - byte[] raw_bry = Bry_.new_u8(raw); - Xop_root_tkn root = Exec_parse_page_all_as_root(Bry_.new_u8(raw)); - Parse_chk(raw_bry, root, expd_ary); - } - public void Data_create(String ttl_str, String text_str) {Init_page_create(wiki, ttl_str, text_str);} - public void Test_parse_page_all_str__esc(String raw, String expd) {Test_parse_page_all_str(raw, Xoh_consts.Escape_apos(expd));} - public void Test_parse_page_all_str(String raw, String expd) { - String actl = Exec_parse_page_all_as_str(raw); - Tfds.Eq_ary_str(String_.SplitLines_nl(expd), String_.SplitLines_nl(actl), raw); - } - public void Test_parse_page_all_str_and_chk(String raw, String expd, Gfo_msg_itm... ary) { - this.Init_log_(ary); - Test_parse_page_all_str(raw, expd); - this.tst_Log_check(); - } - public Xop_root_tkn Exec_parse_page_all_as_root(byte[] raw_bry) { - Xop_root_tkn root = tkn_mkr.Root(raw_bry); - parser.Parse_page_all_clear(root, ctx, tkn_mkr, raw_bry); - return root; - } - public String Exec_parse_page_all_as_str(String raw) { - Xop_root_tkn root = Exec_parse_page_all_as_root(Bry_.new_u8(raw)); - Bry_bfr actl_bfr = Bry_bfr_.New(); - hdom_wtr.Write_doc(actl_bfr, ctx, hctx, root.Root_src(), root); - return actl_bfr.To_str_and_clear(); - } - public Xoh_wtr_ctx Hctx() {return hctx;} private Xoh_wtr_ctx hctx = Xoh_wtr_ctx.Basic; - public void Hctx_(Xoh_wtr_ctx v) {hctx = v;} - public String Exec_parse_page_wiki_as_str(String raw) { - byte[] raw_bry = Bry_.new_u8(raw); - Xop_root_tkn root = tkn_mkr.Root(raw_bry); - parser.Parse_wtxt_to_wdom(root, ctx, tkn_mkr, raw_bry, Xop_parser_.Doc_bgn_bos); - Bry_bfr actl_bfr = Bry_bfr_.New(); - hdom_wtr.Write_doc(actl_bfr, ctx, hctx, raw_bry, root); - return actl_bfr.To_str_and_clear(); - } - private void Parse_chk(byte[] raw_bry, Xop_root_tkn root, Tst_chkr[] expd_ary) { - int subs_len = root.Subs_len(); - Object[] actl_ary = new Object[subs_len]; - for (int i = 0; i < subs_len; i++) - actl_ary[i] = root.Subs_get(i); - tst_mgr.Vars().Clear().Add("raw_bry", raw_bry); - tst_mgr.Tst_ary("tkns:", expd_ary, actl_ary); - tst_Log_check(); - } - public Xop_fxt Test_parse_page_wiki_str__esc(String raw, String expd) {return Test_parse_page_wiki_str(raw, Xoh_consts.Escape_apos(expd));} - public Xop_fxt Test_parse_page_wiki_str(String raw, String expd) { - String actl = Exec_parse_page_wiki_as_str(raw); - Tfds.Eq_str_lines(expd, actl, raw); - return this; - } - public void Log_clear() {ctx.App().Msg_log().Clear();} - public String[] Log_xtoAry() { - Gfo_msg_log msg_log = app.Msg_log(); - int len = msg_log.Ary_len(); - List_adp actl_list = List_adp_.New(); - for (int i = 0; i < len; i++) { - Gfo_msg_data eny = msg_log.Ary_get(i); - if (eny.Item().Cmd() > Gfo_msg_itm_.Cmd_note) { - actl_list.Add(String_.new_u8(eny.Item().Path_bry())); - } - } - String[] actl = actl_list.To_str_ary(); - msg_log.Clear(); - return actl; - } - public Xop_fxt tst_Log_check() { - int len = log_itms.Count(); - String[] expd = new String[len]; - for (int i = 0; i < len; i++) { - Gfo_msg_itm itm = (Gfo_msg_itm)log_itms.Get_at(i); - expd[i] = itm.Path_str(); - } - log_itms.Clear(); - String[] actl = Log_xtoAry(); - Tfds.Eq_ary_str(expd, actl); - return this; - } - public void tst_Warn(String... expd) { - Gfo_usr_dlg usr_dlg = app.Usr_dlg(); - Gfo_usr_dlg__gui_mock ui_wkr = (Gfo_usr_dlg__gui_mock)usr_dlg.Gui_wkr(); - String[] actl = ui_wkr.Warns().To_str_ary(); - Tfds.Eq_ary_str(expd, actl); - } - public void Test_parse_page(String ttl, String expd) { - byte[] actl = Load_page(wiki, ttl); - Tfds.Eq(expd, String_.new_u8(actl)); - } - public static byte[] Load_page(Xowe_wiki wiki, String ttl_str) { - byte[] ttl_bry = Bry_.new_u8(ttl_str); - Xoa_url page_url = Xoa_url.New(wiki.Domain_bry(), ttl_bry); - Xoa_ttl ttl = Xoa_ttl.Parse(wiki, ttl_bry); - return wiki.Data_mgr().Load_page_and_parse(page_url, ttl).Db().Text().Text_bry(); - } - public static void Reg_xwiki_alias(Xowe_wiki wiki, String alias, String domain) { - byte[] domain_bry = Bry_.new_a7(domain); - wiki.Xwiki_mgr().Add_by_atrs(Bry_.new_a7(alias), domain_bry, Bry_.Add(domain_bry, Bry_.new_a7("/wiki/~{0}"))); - wiki.Appe().Usere().Wiki().Xwiki_mgr().Add_by_atrs(domain_bry, domain_bry); - } - public static String html_img_none(String trg, String alt, String src, String ttl) { - return String_.Format(String_.Concat_lines_nl_skip_last("\"{1}\""), trg, alt, src, ttl); - } - private String Exec_html_full(String raw) {return this.Exec_parse_page_all_as_str(raw);} - private String Exec_html_wiki(String raw) {return this.Exec_parse_page_wiki_as_str(raw);} - public void Test_html_wiki_str(String raw, String expd) {Test_str_full(raw, expd, Exec_html_wiki(raw));} - public void Test_html_full_str(String raw, String expd) {Test_str_full(raw, expd, Exec_html_full(raw));} - public void Test_html_wiki_frag(String raw, String... expd_frags) {Test_str_part_y(Exec_html_wiki(raw), expd_frags);} - public void Test_html_full_frag(String raw, String... expd_frags) {Test_str_part_y(Exec_html_full(raw), expd_frags);} - public void Test_html_full_frag_n(String raw, String... expd_frags) {Test_str_part_n(Exec_html_full(raw), expd_frags);} - public void Test__parse__tmpl_to_html(String raw, String expd) {Test_str_full(raw, gplx.langs.htmls.Gfh_utl.Replace_apos(expd), Exec_html_full(raw));} - public void Test__parse__wtxt_to_html(String raw, String expd) { - String actl = Exec_html_wiki(raw); - Tfds.Eq_str_lines(gplx.langs.htmls.Gfh_utl.Replace_apos(expd), actl, raw); - } - - public void Test_str_full(String raw, String expd, String actl) { - Tfds.Eq_str_lines(expd, actl, (hctx.Mode_is_hdump() ? "hsave" : "hview") + " \n" + raw); - } - public void Test_str_part_y(String actl, String... expd_parts) { - int expd_parts_len = expd_parts.length; - for (int i = 0; i < expd_parts_len; i++) { - String expd_part = expd_parts[i]; - boolean pass = String_.Has(actl, expd_part); - if (!pass) - Tfds.Eq_true(false, expd_part + "\n" + actl); - } - } - private void Test_str_part_n(String actl, String... expd_parts) { - int expd_parts_len = expd_parts.length; - for (int i = 0; i < expd_parts_len; i++) { - String expd_part = expd_parts[i]; - boolean has = String_.Has(actl, expd_part); - if (has) - Tfds.Eq_true(false, expd_part + "\n" + actl); - } - } - public void Test_html_modules_js(String expd) { - Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_k004(); - this.Page().Html_data().Head_mgr().Init(app, wiki, this.Page()); - this.Page().Html_data().Head_mgr().Bfr_arg__add(bfr); - bfr.Mkr_rls(); - Tfds.Eq_str_lines(expd, bfr.To_str_and_clear()); - } - - private Tst_mgr tst_mgr = new Tst_mgr(); private Xop_tkn_mkr tkn_mkr; - public static final String Ttl_str = "Test page"; - public Xop_fxt Init_lang_numbers_separators_en() {return Init_lang_numbers_separators(",", ".");} - public Xop_fxt Init_lang_numbers_separators(String grp_spr, String dec_spr) {return Init_lang_numbers_separators(wiki.Lang(), grp_spr, dec_spr);} - public Xop_fxt Init_lang_numbers_separators(Xol_lang_itm lang, String grp_spr, String dec_spr) { - gplx.xowa.langs.numbers.Xol_transform_mgr separator_mgr = lang.Num_mgr().Separators_mgr(); - separator_mgr.Clear(); - separator_mgr.Set(gplx.xowa.langs.numbers.Xol_num_mgr.Separators_key__grp, Bry_.new_u8(grp_spr)); - separator_mgr.Set(gplx.xowa.langs.numbers.Xol_num_mgr.Separators_key__dec, Bry_.new_u8(dec_spr)); - return this; - } - public void Init_lang_kwds(int kwd_id, boolean case_match, String... kwds) {Init_lang_kwds(wiki.Lang(), kwd_id, case_match, kwds);} - public void Init_lang_kwds(Xol_lang_itm lang, int kwd_id, boolean case_match, String... kwds) { - Xol_kwd_mgr kwd_mgr = lang.Kwd_mgr(); - Xol_kwd_grp kwd_grp = kwd_mgr.Get_or_new(kwd_id); - kwd_grp.Srl_load(case_match, Bry_.Ary(kwds)); - } - public void Init_lang_vnts(String... vnts) { - wiki.Lang().Vnt_mgr().Enabled_(true); - gplx.xowa.langs.vnts.Xol_vnt_regy vnt_regy = wiki.Lang().Vnt_mgr().Regy(); - for (int i = 0; i < vnts.length; i++) { - byte[] vnt = Bry_.new_u8(vnts[i]); - vnt_regy.Add(vnt, vnt); - if (i == 0) { - wiki.Lang().Vnt_mgr().Cur_itm_(vnt); - } - } - wiki.Lang().Vnt_mgr().Init_end(); - } - public void Init_xtn_pages() { - Io_mgr.Instance.InitEngine_mem(); - wiki.Xtn_mgr().Xtn_proofread().Enabled_y_(); - wiki.Db_mgr().Load_mgr().Clear(); // must clear; otherwise fails b/c files get deleted, but wiki.data_mgr caches the Xowd_regy_mgr (the .reg file) in memory; - wiki.Ns_mgr().Add_new(Xowc_xtn_pages.Ns_page_id_default, "Page").Add_new(Xowc_xtn_pages.Ns_index_id_default, "Index").Init(); - } - public void Clear_ref_mgr() {this.Page().Ref_mgr().Grps_clear();} // clear to reset count - public static Xop_fxt new_nonwmf() { - Xoae_app app = Xoa_app_fxt.Make__app__edit(); - return new Xop_fxt(app, Xoa_app_fxt.Make__wiki__edit__nonwmf(app, "nethackwiki")); - } - private final Bry_bfr tmp_bfr = Bry_bfr_.New_w_size(255); - public String Exec__parse_to_hdump(String src_str) { - byte[] src_bry = Bry_.new_u8(src_str); - Xop_root_tkn root = Exec_parse_page_all_as_root(src_bry); - Xoh_wtr_ctx hctx = Xoh_wtr_ctx.Hdump; - Xoh_html_wtr html_wtr = wiki.Html_mgr().Html_wtr(); - html_wtr.Cfg().Toc__show_(Bool_.Y); // needed for hdr to show - ctx.Page().Html_data().Redlink_list().Clear(); - html_wtr.Write_doc(tmp_bfr, ctx, hctx, src_bry, root); - // Tfds.Dbg(tmp_bfr.To_str()); - return tmp_bfr.To_str_and_clear(); - } - public void Test__parse_to_html_mgr(String src_str, String expd) { - byte[] src_bry = Bry_.new_u8(src_str); - Xop_root_tkn root = Exec_parse_page_all_as_root(src_bry); - Xoae_page page = this.Page(); - page.Root_(root); - byte[] actl = wiki.Html_mgr().Page_wtr_mgr().Gen(page, Xopg_view_mode_.Tid__read); - Tfds.Eq_str_lines(expd, String_.new_u8(actl)); - } - public String Exec__parse_to_html_w_skin(String raw) { - Bry_bfr bfr = Bry_bfr_.New(); - Xow_html_mgr html_mgr = wiki.Html_mgr(); - this.Wiki().Html__wtr_mgr().Page_read_fmtr().Fmt_("~{page_data}"); - - byte[] raw_bry = Bry_.new_u8(raw); - Xop_root_tkn root = this.Exec_parse_page_all_as_root(raw_bry); - this.Page().Root_(root); - - html_mgr.Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Write_page(bfr, this.Page(), this.Ctx(), Xoh_page_html_source_.Wtr); - return bfr.To_str_and_clear(); - } - public void Test__parse_to_html_w_skin(String raw, String expd) { - Tfds.Eq_str_lines(expd, Exec__parse_to_html_w_skin(raw)); - } - public String Make__test_string(String expr, String expd) { - /* - {| class=wikitable - ! rslt !! expd !! actl !! code - |} - */ - Bry_bfr bfr = Bry_bfr_.New(); - bfr.Add_str_a7("|-\n"); - bfr.Add_str_u8("| {{#ifeq:" + String_.Replace(expd, "|", "{{!}}") + "|" + String_.Replace(expr, "|", "{{!}}") + "|pass|fail}}\n"); - bfr.Add_str_u8("| " + expd + "\n"); - bfr.Add_str_u8("| " + expr + "\n"); - bfr.Add_str_u8("| " + expr + "\n"); - return bfr.To_str(); - } - - public static Xop_fxt New_app_html() { - Xop_fxt fxt = new Xop_fxt(); - fxt.Wiki().Html_mgr().Page_wtr_mgr().Page_read_fmtr().Fmt_("~{page_data}"); - return fxt; - } -} +package gplx.xowa; import gplx.*; +import gplx.core.tests.*; import gplx.core.log_msgs.*; +import gplx.xowa.apps.cfgs.*; +import gplx.xowa.langs.*; import gplx.xowa.langs.kwds.*; import gplx.xowa.langs.msgs.*; +import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.htmls.*; +import gplx.xowa.parsers.*; import gplx.xowa.parsers.amps.*; import gplx.xowa.parsers.apos.*; import gplx.xowa.parsers.hdrs.*; import gplx.xowa.parsers.lists.*; import gplx.xowa.parsers.paras.*; import gplx.xowa.parsers.xndes.*; import gplx.xowa.parsers.tmpls.*; import gplx.xowa.parsers.miscs.*; import gplx.xowa.parsers.tblws.*; import gplx.xowa.parsers.lnkes.*; import gplx.xowa.parsers.lnkis.*; +import gplx.xowa.files.exts.*; import gplx.xowa.files.repos.*; import gplx.xowa.htmls.core.wkrs.lnkis.htmls.*; +import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.domains.*; +import gplx.xowa.wikis.tdbs.hives.*; +import gplx.xowa.wikis.pages.*; +public class Xop_fxt { + public Xop_fxt() { + Xoae_app app = Xoa_app_fxt.Make__app__edit(); + wiki = Xoa_app_fxt.Make__wiki__edit(app); + ctor(app, wiki); + } + public Xop_fxt(Xoae_app app, Xowe_wiki wiki) { + this.ctor(app, wiki); + } + public Xop_fxt(Xowe_wiki wiki) { + this.ctor(Xoa_app_fxt.Make__app__edit(), wiki); + } + private void ctor(Xoae_app app, Xowe_wiki wiki) { + this.app = app; + this.wiki = wiki; + + // NOTE: Xop_fxt does not call Init_by_app / Init_by_wiki b/c of test issues with DB; DATE:2019-09-15 + // call these methods manually + app.Addon_mgr().Add_dflts_by_app(app).Run_by_app(app); + wiki.Init_once(); + + app.Wiki_mgr().Add(wiki); + app.File_mgr().Repo_mgr().Set("src:wiki", "mem/wiki/repo/src/", wiki.Domain_str()).Ext_rules_(Xof_rule_grp.Grp_app_default).Dir_depth_(2); + app.File_mgr().Repo_mgr().Set("trg:wiki", "mem/wiki/repo/trg/", wiki.Domain_str()).Ext_rules_(Xof_rule_grp.Grp_app_default).Dir_depth_(2).Primary_(true); + wiki.File_mgr().Repo_mgr().Add_repo(Bry_.new_a7("src:wiki"), Bry_.new_a7("trg:wiki")); + ctx = wiki.Parser_mgr().Ctx(); + mock_wkr.Clear_commons(); // assume all files are in repo 0 + wiki.File_mgr().Repo_mgr().Page_finder_(mock_wkr); + parser = wiki.Parser_mgr().Main(); + this.tkn_mkr = app.Parser_mgr().Tkn_mkr(); + ctx.Para().Enabled_n_(); + hdom_wtr = wiki.Html_mgr().Html_wtr(); + wiki.Html_mgr().Img_suppress_missing_src_(false); + wiki.Xtn_mgr().Init_by_wiki(wiki); + Page_ttl_(Ttl_str); + } + private Xofw_wiki_wkr_mock mock_wkr = new Xofw_wiki_wkr_mock(); + public Xoae_app App() {return app;} private Xoae_app app; + public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki; + public Xop_ctx Ctx() {return ctx;} private Xop_ctx ctx; + public Xop_parser Parser() {return parser;} private Xop_parser parser; + public Xoae_page Page() {return ctx.Page();} + public void Lang_by_id_(int id) {ctx.Page().Lang_(wiki.Appe().Lang_mgr().Get_by_or_new(Xol_lang_stub_.Get_by_id(id).Key()));} + public Xoh_html_wtr_cfg Wtr_cfg() {return hdom_wtr.Cfg();} private Xoh_html_wtr hdom_wtr; + public Xop_fxt Reset() { + ctx.Clear_all(); + ctx.App().Free_mem(false); + ctx.Page().Clear_all(); + wiki.File_mgr().Clear_for_tests(); + wiki.Db_mgr().Load_mgr().Clear(); + wiki.Db_mgr().Save_mgr().Clear(); + app.Wiki_mgr().Clear(); + Io_mgr.Instance.InitEngine_mem(); // clear created pages + wiki.Cfg_parser().Display_title_restrict_(false); // default to false, as a small number of tests assume restrict = false; + return this; + } + public Xop_fxt Reset_for_msgs() { + Io_mgr.Instance.InitEngine_mem(); + wiki.Lang().Msg_mgr().Clear(); // need to clear out lang + wiki.Msg_mgr().Clear(); // need to clear out wiki.Msgs + this.Reset(); + return this; + } + public Xoa_ttl Page_ttl_(String txt) { + Xoa_ttl rv = Xoa_ttl.Parse(wiki, Bry_.new_u8(txt)); + ctx.Page().Ttl_(rv); + return rv; + } + + public Xop_tkn_chkr_base tkn_bry_(int bgn, int end) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_bry).Src_rng_(bgn, end);} + public Xop_tkn_chkr_base tkn_txt_() {return tkn_txt_(String_.Pos_neg1, String_.Pos_neg1);} + public Xop_tkn_chkr_base tkn_txt_(int bgn, int end) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_txt).Src_rng_(bgn, end);} + public Xop_tkn_chkr_base tkn_space_() {return tkn_space_(String_.Pos_neg1, String_.Pos_neg1);} + public Xop_tkn_chkr_base tkn_space_(int bgn, int end) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_space).Src_rng_(bgn, end);} + public Xop_tkn_chkr_base tkn_eq_(int bgn) {return tkn_eq_(bgn, bgn + 1);} + public Xop_tkn_chkr_base tkn_eq_(int bgn, int end) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_eq).Src_rng_(bgn, end);} + public Xop_tkn_chkr_base tkn_colon_(int bgn) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_colon).Src_rng_(bgn, bgn + 1);} + public Xop_tkn_chkr_base tkn_pipe_(int bgn) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_pipe).Src_rng_(bgn, bgn + 1);} + public Xop_tkn_chkr_base tkn_tab_(int bgn) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_tab).Src_rng_(bgn, bgn + 1);} + public Xop_apos_tkn_chkr tkn_apos_(int cmd) {return new Xop_apos_tkn_chkr().Apos_cmd_(cmd);} + public Xop_ignore_tkn_chkr tkn_comment_(int bgn, int end) {return tkn_ignore_(bgn, end, Xop_ignore_tkn.Ignore_tid_comment);} + public Xop_ignore_tkn_chkr tkn_ignore_(int bgn, int end, byte t){return (Xop_ignore_tkn_chkr)new Xop_ignore_tkn_chkr().Ignore_tid_(t).Src_rng_(bgn, end);} + public Xop_tkn_chkr_hr tkn_hr_(int bgn, int end) {return new Xop_tkn_chkr_hr(bgn, end).Hr_len_(Xop_hr_lxr.Hr_len);} + public Xop_tblw_tb_tkn_chkr tkn_tblw_tb_(int bgn, int end) {return (Xop_tblw_tb_tkn_chkr)new Xop_tblw_tb_tkn_chkr().Src_rng_(bgn, end);} + public Xop_tblw_tc_tkn_chkr tkn_tblw_tc_(int bgn, int end) {return (Xop_tblw_tc_tkn_chkr)new Xop_tblw_tc_tkn_chkr().Src_rng_(bgn, end);} + public Xop_tblw_td_tkn_chkr tkn_tblw_td_(int bgn, int end) {return (Xop_tblw_td_tkn_chkr)new Xop_tblw_td_tkn_chkr().Src_rng_(bgn, end);} + public Xop_tblw_th_tkn_chkr tkn_tblw_th_(int bgn, int end) {return (Xop_tblw_th_tkn_chkr)new Xop_tblw_th_tkn_chkr().Src_rng_(bgn, end);} + public Xop_tblw_tr_tkn_chkr tkn_tblw_tr_(int bgn, int end) {return (Xop_tblw_tr_tkn_chkr)new Xop_tblw_tr_tkn_chkr().Src_rng_(bgn, end);} + public Xop_xnde_tkn_chkr tkn_xnde_br_(int pos) {return tkn_xnde_(pos, pos).Xnde_tagId_(Xop_xnde_tag_.Tid__br);} + public Xop_xnde_tkn_chkr tkn_xnde_() {return tkn_xnde_(String_.Pos_neg1, String_.Pos_neg1);} + public Xop_xnde_tkn_chkr tkn_xnde_(int bgn, int end) {return (Xop_xnde_tkn_chkr)new Xop_xnde_tkn_chkr().Src_rng_(bgn, end);} + public Xop_tkn_chkr_base tkn_curly_bgn_(int bgn) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_tmpl_curly_bgn).Src_rng_(bgn, bgn + 2);} + public Xop_tkn_chkr_base tkn_para_blank_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_none, Xop_para_tkn.Tid_none);} + public Xop_tkn_chkr_base tkn_para_bgn_pre_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_none, Xop_para_tkn.Tid_pre);} + public Xop_tkn_chkr_base tkn_para_bgn_para_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_none, Xop_para_tkn.Tid_para);} + public Xop_tkn_chkr_base tkn_para_mid_para_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_para, Xop_para_tkn.Tid_para);} + public Xop_tkn_chkr_base tkn_para_end_para_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_para, Xop_para_tkn.Tid_none);} + public Xop_tkn_chkr_base tkn_para_end_pre_bgn_para_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_pre , Xop_para_tkn.Tid_para);} + public Xop_tkn_chkr_base tkn_para_end_para_bgn_pre_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_para, Xop_para_tkn.Tid_pre);} + public Xop_tkn_chkr_base tkn_para_end_pre_(int pos) {return tkn_para_(pos, Xop_para_tkn.Tid_pre , Xop_para_tkn.Tid_none);} + public Xop_tkn_chkr_base tkn_para_(int pos, byte end, byte bgn) {return new Xop_para_tkn_chkr().Para_end_(end).Para_bgn_(bgn).Src_rng_(pos, pos);} + public Xop_tkn_chkr_base tkn_nl_char_(int bgn, int end) {return tkn_nl_(bgn, end, Xop_nl_tkn.Tid_char);} + public Xop_tkn_chkr_base tkn_nl_char_len1_(int bgn) {return tkn_nl_(bgn, bgn + 1, Xop_nl_tkn.Tid_char);} + public Xop_tkn_chkr_base tkn_nl_char_len0_(int pos) {return tkn_nl_(pos, pos, Xop_nl_tkn.Tid_char);} + public Xop_tkn_chkr_base tkn_nl_(int bgn, int end, byte tid) {return new Xop_nl_tkn_chkr().Nl_tid_(tid).Src_rng_(bgn, end);} + public Xop_list_tkn_chkr tkn_list_bgn_(int bgn, int end, byte listType) {return (Xop_list_tkn_chkr)new Xop_list_tkn_chkr().List_itmTyp_(listType).Src_rng_(bgn, end);} + public Xop_list_tkn_chkr tkn_list_end_(int pos) {return (Xop_list_tkn_chkr)new Xop_list_tkn_chkr().Src_rng_(pos, pos);} + public Xop_tkn_chkr_lnke tkn_lnke_(int bgn, int end) {return new Xop_tkn_chkr_lnke(bgn, end);} + public Xop_lnki_tkn_chkr tkn_lnki_() {return tkn_lnki_(-1, -1);} + public Xop_lnki_tkn_chkr tkn_lnki_(int bgn, int end) {return (Xop_lnki_tkn_chkr)new Xop_lnki_tkn_chkr().Src_rng_(bgn, end);} + public Xop_arg_itm_tkn_chkr tkn_arg_itm_(Xop_tkn_chkr_base... subs) {return (Xop_arg_itm_tkn_chkr)new Xop_arg_itm_tkn_chkr().Subs_(subs);} + public Xop_arg_nde_tkn_chkr tkn_arg_nde_() {return tkn_arg_nde_(String_.Pos_neg1, String_.Pos_neg1);} + public Xop_arg_nde_tkn_chkr tkn_arg_nde_(int bgn, int end) {return (Xop_arg_nde_tkn_chkr)new Xop_arg_nde_tkn_chkr().Src_rng_(bgn, end);} + public Xop_arg_nde_tkn_chkr tkn_arg_val_(Xop_tkn_chkr_base... subs) { + Xop_arg_nde_tkn_chkr rv = new Xop_arg_nde_tkn_chkr(); + Xop_arg_itm_tkn_chkr val = new Xop_arg_itm_tkn_chkr(); + val.Subs_(subs); + rv.Val_tkn_(val); + return rv; + } + public Xop_arg_nde_tkn_chkr tkn_arg_val_txt_(int bgn, int end) { + Xop_arg_nde_tkn_chkr rv = new Xop_arg_nde_tkn_chkr(); + Xop_arg_itm_tkn_chkr itm = new Xop_arg_itm_tkn_chkr(); + rv.Val_tkn_(itm); + itm.Src_rng_(bgn, end).Subs_(tkn_txt_(bgn, end)); + return rv; + } + Xop_arg_nde_tkn_chkr tkn_arg_key_txt_(int bgn, int end) { + Xop_arg_nde_tkn_chkr rv = new Xop_arg_nde_tkn_chkr(); + Xop_arg_itm_tkn_chkr itm = new Xop_arg_itm_tkn_chkr(); + rv.Key_tkn_(itm); + itm.Src_rng_(bgn, end).Subs_(tkn_txt_(bgn, end)); + return rv; + } + public Xot_invk_tkn_chkr tkn_tmpl_invk_(int bgn, int end) {return (Xot_invk_tkn_chkr)new Xot_invk_tkn_chkr().Src_rng_(bgn, end);} + public Xot_invk_tkn_chkr tkn_tmpl_invk_w_name(int bgn, int end, int name_bgn, int name_end) { + Xot_invk_tkn_chkr rv = new Xot_invk_tkn_chkr(); + rv.Src_rng_(bgn, end); + rv.Name_tkn_(tkn_arg_key_txt_(name_bgn, name_end)); + return rv; + } + public Xot_prm_chkr tkn_tmpl_prm_find_(Xop_tkn_chkr_base find) { + Xot_prm_chkr rv = new Xot_prm_chkr(); + rv.Find_tkn_(tkn_arg_itm_(find)); + return rv; + } + public void Init__file_find_mgr() { + Xowe_wiki commons_wiki = this.app.Wiki_mgr().Get_by_or_make(Xow_domain_itm_.Bry__commons); + Xof_repo_itm repo_itm = app.File_mgr().Repo_mgr().Get_at(0); + commons_wiki.Html_mgr().Html_wtr().Lnki_wtr().File_wtr().Find_file_mgr_(new Xow_find_file_mgr(repo_itm, commons_wiki.Domain_str())); + this.wiki.Html_mgr().Html_wtr().Lnki_wtr().File_wtr().Find_file_mgr_(new Xow_find_file_mgr(repo_itm, this.wiki.Domain_str())); + } + public void Init__orig(boolean wiki_is_commons, String orig_ttl, int orig_w, int orig_h) { + Xowe_wiki orig_wiki = wiki_is_commons ? (Xowe_wiki)this.wiki : (Xowe_wiki)app.Wiki_mgr().Get_by_or_null(gplx.xowa.wikis.domains.Xow_domain_itm_.Bry__commons); + orig_wiki.Html_mgr().Html_wtr().Lnki_wtr().File_wtr().Find_file_mgr().Init_file(orig_ttl, orig_w, orig_h); + } + public void Init__msg(String key, String val) { + Xol_msg_itm msg_itm = wiki.Msg_mgr().Get_or_make(Bry_.new_u8(key)); + msg_itm.Atrs_set(Bry_.new_u8(val), false, false); + } + public Xop_fxt Init_para_y_() {ctx.Para().Enabled_y_(); return this;} + public Xop_fxt Init_para_n_() {ctx.Para().Enabled_n_(); return this;} + public Xop_fxt Init_log_(Gfo_msg_itm... itms) {for (Gfo_msg_itm itm : itms) log_itms.Add(itm); return this;} List_adp log_itms = List_adp_.New(); + public void Init_defn_add(String name, String text) {Init_defn_add(name, text, Xow_ns_case_.Tid__all);} + public void Init_defn_add(String name, String text, byte case_match) {Init_defn_add(wiki, name, text, case_match);} + public void Init_defn_add(Xowe_wiki w, String name, String text, byte case_match) { + Xot_defn_tmpl itm = run_Parse_tmpl(w, Bry_.new_a7(name), Bry_.new_u8(text)); + w.Cache_mgr().Defn_cache().Add(itm, case_match); + } + public void Init_defn_clear() {wiki.Cache_mgr().Defn_cache().Free_mem_all();} + public Xop_fxt Init_id_create(int id, int fil_idx, int row_idx, boolean type_redirect, int itm_len, int ns_id, String ttl) {Xow_hive_mgr_fxt.Create_id(app, wiki.Hive_mgr(), id, fil_idx, row_idx, type_redirect, itm_len, ns_id, ttl); return this;} + public Xop_fxt Init_page_create(String ttl) {return Init_page_create(wiki, ttl, "");} + public Xop_fxt Init_page_create(String ttl, String txt) {return Init_page_create(wiki, ttl, txt);} + public Xop_fxt Init_page_create(Xowe_wiki wiki, String ttl, String txt) {Init_page_create_static(wiki, ttl, txt);return this;} + public static void Init_page_create_static(Xowe_wiki wiki, String ttl_str, String text_str) { + Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.new_u8(ttl_str)); + byte[] text = Bry_.new_u8(text_str); + wiki.Db_mgr().Save_mgr().Data_create(wiki, ttl, text); + } + public static void Init_msg(Xowe_wiki wiki, String key, String val) { + wiki.Lang().Msg_mgr().Itm_by_key_or_new(key, val); + } + public Xop_fxt Init_page_update(String ttl, String txt) {return Init_page_update(wiki, ttl, txt);} + public Xop_fxt Init_page_update(Xowe_wiki wiki, String ttl, String txt) { + Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, Bry_.new_u8(ttl)); + byte[] page_raw = Bry_.new_u8(txt); + Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(page_ttl); + wiki.Db_mgr().Save_mgr().Data_update(page, page_raw); + return this; + } + public Xop_fxt Init_xwiki_clear() { + wiki.Xwiki_mgr().Clear(); + app.Usere().Wiki().Xwiki_mgr().Clear(); + return this; + } + public Xop_fxt Init_xwiki_add_wiki_and_user_(String alias, String domain) { + wiki.Xwiki_mgr().Add_by_atrs(alias, domain); + app.Usere().Wiki().Xwiki_mgr().Add_by_atrs(domain, domain); + return this; + } + public Xop_fxt Init_xwiki_add_user_(String domain) {return Init_xwiki_add_user_(domain, domain);} + public Xop_fxt Init_xwiki_add_user_(String alias, String domain) { + app.Usere().Wiki().Xwiki_mgr().Add_by_atrs(alias, domain); + return this; + } + public void Test_parse_template(String tmpl_raw, String expd) {Test_parse_tmpl_str_test(tmpl_raw, "{{test}}", expd);} + public void Test_parse_tmpl_str_test(String tmpl_raw, String page_raw, String expd) {Test_parse_tmpl_str_test(wiki, tmpl_raw, page_raw, expd);} + public void Test_parse_tmpl_str_test(Xowe_wiki w, String tmpl_raw, String page_raw, String expd) { + Init_defn_add(w, "test", tmpl_raw, Xow_ns_case_.Tid__all); + Test_parse_tmpl_str(w, page_raw, expd); + } + public void Test_parse_tmpl_str(String raw, String expd) {Test_parse_tmpl_str(wiki, raw, expd);} + public void Test_parse_tmpl_str(Xowe_wiki w, String raw, String expd) { + byte[] actl = Test_parse_tmpl_str_rv(w, raw); + Tfds.Eq_str_lines(expd, String_.new_u8(actl)); + tst_Log_check(); + } + public byte[] Test_parse_tmpl_str_rv(String raw) {return Test_parse_tmpl_str_rv(wiki, raw);} + public byte[] Test_parse_tmpl_str_rv(Xowe_wiki w, String raw) { + byte[] raw_bry = Bry_.new_u8(raw); + Xop_root_tkn root = tkn_mkr.Root(raw_bry); + Xop_ctx c = w.Parser_mgr().Ctx(); + c.Page().Root_(root); + c.Page().Db().Text().Text_bry_(raw_bry); + return w.Parser_mgr().Main().Expand_tmpl(root, c, tkn_mkr, raw_bry); + } + public Xot_defn_tmpl run_Parse_tmpl(byte[] name, byte[] raw) {return run_Parse_tmpl(wiki, name, raw);} + public Xot_defn_tmpl run_Parse_tmpl(Xowe_wiki w, byte[] name, byte[] raw) { + Xop_ctx c = w.Parser_mgr().Ctx(); + return w.Parser_mgr().Main().Parse_text_to_defn_obj(c, c.Tkn_mkr(), w.Ns_mgr().Ns_template(), name, raw); + } + public void Test_parse_tmpl(String raw, Tst_chkr... expd) { + byte[] raw_bry = Bry_.new_u8(raw); + Xot_defn_tmpl itm = run_Parse_tmpl(Bry_.Empty, raw_bry); + Parse_chk(raw_bry, itm.Root(), expd); + } + public void Test_parse_page_tmpl_str(String raw, String expd) { + byte[] raw_bry = Bry_.new_u8(raw); + Xop_root_tkn root = tkn_mkr.Root(raw_bry); + byte[] actl = parser.Expand_tmpl(root, ctx, tkn_mkr, raw_bry); + Tfds.Eq(expd, String_.new_u8(actl)); + tst_Log_check(); + } + public Xop_root_tkn Test_parse_page_tmpl_tkn(String raw) { + byte[] raw_bry = Bry_.new_u8(raw); + Xop_root_tkn root = tkn_mkr.Root(raw_bry); + parser.Expand_tmpl(root, ctx, tkn_mkr, raw_bry); + return root; + } + public void Test_parse_page_tmpl(String raw, Tst_chkr... expd_ary) { + byte[] raw_bry = Bry_.new_u8(raw); + Xop_root_tkn root = tkn_mkr.Root(raw_bry); + parser.Expand_tmpl(root, ctx, tkn_mkr, raw_bry); + Parse_chk(raw_bry, root, expd_ary); + } + public void Test_parse_page_wiki(String raw, Tst_chkr... expd_ary) { + byte[] raw_bry = Bry_.new_u8(raw); + Xop_root_tkn root = Test_parse_page_wiki_root(raw_bry); + Parse_chk(raw_bry, root, expd_ary); + } + public Xop_root_tkn Test_parse_page_wiki_root(String raw) {return Test_parse_page_wiki_root(Bry_.new_u8(raw));} + Xop_root_tkn Test_parse_page_wiki_root(byte[] raw_bry) { + Xop_root_tkn root = tkn_mkr.Root(raw_bry); + parser.Parse_wtxt_to_wdom(root, ctx, tkn_mkr, raw_bry, Xop_parser_.Doc_bgn_bos); + return root; + } + public void Test_parse_page_all(String raw, Tst_chkr... expd_ary) { + byte[] raw_bry = Bry_.new_u8(raw); + Xop_root_tkn root = Exec_parse_page_all_as_root(Bry_.new_u8(raw)); + Parse_chk(raw_bry, root, expd_ary); + } + public void Data_create(String ttl_str, String text_str) {Init_page_create(wiki, ttl_str, text_str);} + public void Test_parse_page_all_str__esc(String raw, String expd) {Test_parse_page_all_str(raw, Xoh_consts.Escape_apos(expd));} + public void Test_parse_page_all_str(String raw, String expd) { + String actl = Exec_parse_page_all_as_str(raw); + Tfds.Eq_ary_str(String_.SplitLines_nl(expd), String_.SplitLines_nl(actl), raw); + } + public void Test_parse_page_all_str_and_chk(String raw, String expd, Gfo_msg_itm... ary) { + this.Init_log_(ary); + Test_parse_page_all_str(raw, expd); + this.tst_Log_check(); + } + public Xop_root_tkn Exec_parse_page_all_as_root(byte[] raw_bry) { + Xop_root_tkn root = tkn_mkr.Root(raw_bry); + parser.Parse_page_all_clear(root, ctx, tkn_mkr, raw_bry); + return root; + } + public String Exec_parse_page_all_as_str(String raw) { + Xop_root_tkn root = Exec_parse_page_all_as_root(Bry_.new_u8(raw)); + Bry_bfr actl_bfr = Bry_bfr_.New(); + hdom_wtr.Write_doc(actl_bfr, ctx, hctx, root.Root_src(), root); + return actl_bfr.To_str_and_clear(); + } + public Xoh_wtr_ctx Hctx() {return hctx;} private Xoh_wtr_ctx hctx = Xoh_wtr_ctx.Basic; + public void Hctx_(Xoh_wtr_ctx v) {hctx = v;} + public String Exec_parse_page_wiki_as_str(String raw) { + byte[] raw_bry = Bry_.new_u8(raw); + Xop_root_tkn root = tkn_mkr.Root(raw_bry); + parser.Parse_wtxt_to_wdom(root, ctx, tkn_mkr, raw_bry, Xop_parser_.Doc_bgn_bos); + Bry_bfr actl_bfr = Bry_bfr_.New(); + hdom_wtr.Write_doc(actl_bfr, ctx, hctx, raw_bry, root); + return actl_bfr.To_str_and_clear(); + } + private void Parse_chk(byte[] raw_bry, Xop_root_tkn root, Tst_chkr[] expd_ary) { + int subs_len = root.Subs_len(); + Object[] actl_ary = new Object[subs_len]; + for (int i = 0; i < subs_len; i++) + actl_ary[i] = root.Subs_get(i); + tst_mgr.Vars().Clear().Add("raw_bry", raw_bry); + tst_mgr.Tst_ary("tkns:", expd_ary, actl_ary); + tst_Log_check(); + } + public Xop_fxt Test_parse_page_wiki_str__esc(String raw, String expd) {return Test_parse_page_wiki_str(raw, Xoh_consts.Escape_apos(expd));} + public Xop_fxt Test_parse_page_wiki_str(String raw, String expd) { + String actl = Exec_parse_page_wiki_as_str(raw); + Tfds.Eq_str_lines(expd, actl, raw); + return this; + } + public void Log_clear() {ctx.App().Msg_log().Clear();} + public String[] Log_xtoAry() { + Gfo_msg_log msg_log = app.Msg_log(); + int len = msg_log.Ary_len(); + List_adp actl_list = List_adp_.New(); + for (int i = 0; i < len; i++) { + Gfo_msg_data eny = msg_log.Ary_get(i); + if (eny.Item().Cmd() > Gfo_msg_itm_.Cmd_note) { + actl_list.Add(String_.new_u8(eny.Item().Path_bry())); + } + } + String[] actl = actl_list.To_str_ary(); + msg_log.Clear(); + return actl; + } + public Xop_fxt tst_Log_check() { + int len = log_itms.Count(); + String[] expd = new String[len]; + for (int i = 0; i < len; i++) { + Gfo_msg_itm itm = (Gfo_msg_itm)log_itms.Get_at(i); + expd[i] = itm.Path_str(); + } + log_itms.Clear(); + String[] actl = Log_xtoAry(); + Tfds.Eq_ary_str(expd, actl); + return this; + } + public void tst_Warn(String... expd) { + Gfo_usr_dlg usr_dlg = app.Usr_dlg(); + Gfo_usr_dlg__gui_mock ui_wkr = (Gfo_usr_dlg__gui_mock)usr_dlg.Gui_wkr(); + String[] actl = ui_wkr.Warns().To_str_ary(); + Tfds.Eq_ary_str(expd, actl); + } + public void Test_parse_page(String ttl, String expd) { + byte[] actl = Load_page(wiki, ttl); + Tfds.Eq(expd, String_.new_u8(actl)); + } + public static byte[] Load_page(Xowe_wiki wiki, String ttl_str) { + byte[] ttl_bry = Bry_.new_u8(ttl_str); + Xoa_url page_url = Xoa_url.New(wiki.Domain_bry(), ttl_bry); + Xoa_ttl ttl = Xoa_ttl.Parse(wiki, ttl_bry); + return wiki.Data_mgr().Load_page_and_parse(page_url, ttl).Db().Text().Text_bry(); + } + public static void Reg_xwiki_alias(Xowe_wiki wiki, String alias, String domain) { + byte[] domain_bry = Bry_.new_a7(domain); + wiki.Xwiki_mgr().Add_by_atrs(Bry_.new_a7(alias), domain_bry, Bry_.Add(domain_bry, Bry_.new_a7("/wiki/~{0}"))); + wiki.Appe().Usere().Wiki().Xwiki_mgr().Add_by_atrs(domain_bry, domain_bry); + } + public static String html_img_none(String trg, String alt, String src, String ttl) { + return String_.Format(String_.Concat_lines_nl_skip_last("\"{1}\""), trg, alt, src, ttl); + } + private String Exec_html_full(String raw) {return this.Exec_parse_page_all_as_str(raw);} + private String Exec_html_wiki(String raw) {return this.Exec_parse_page_wiki_as_str(raw);} + public void Test_html_wiki_str(String raw, String expd) {Test_str_full(raw, expd, Exec_html_wiki(raw));} + public void Test_html_full_str(String raw, String expd) {Test_str_full(raw, expd, Exec_html_full(raw));} + public void Test_html_wiki_frag(String raw, String... expd_frags) {Test_str_part_y(Exec_html_wiki(raw), expd_frags);} + public void Test_html_full_frag(String raw, String... expd_frags) {Test_str_part_y(Exec_html_full(raw), expd_frags);} + public void Test_html_full_frag_n(String raw, String... expd_frags) {Test_str_part_n(Exec_html_full(raw), expd_frags);} + public void Test__parse__tmpl_to_html(String raw, String expd) {Test_str_full(raw, gplx.langs.htmls.Gfh_utl.Replace_apos(expd), Exec_html_full(raw));} + public void Test__parse__wtxt_to_html(String raw, String expd) { + String actl = Exec_html_wiki(raw); + Tfds.Eq_str_lines(gplx.langs.htmls.Gfh_utl.Replace_apos(expd), actl, raw); + } + + public void Test_str_full(String raw, String expd, String actl) { + Tfds.Eq_str_lines(expd, actl, (hctx.Mode_is_hdump() ? "hsave" : "hview") + " \n" + raw); + } + public void Test_str_part_y(String actl, String... expd_parts) { + int expd_parts_len = expd_parts.length; + for (int i = 0; i < expd_parts_len; i++) { + String expd_part = expd_parts[i]; + boolean pass = String_.Has(actl, expd_part); + if (!pass) + Tfds.Eq_true(false, expd_part + "\n" + actl); + } + } + private void Test_str_part_n(String actl, String... expd_parts) { + int expd_parts_len = expd_parts.length; + for (int i = 0; i < expd_parts_len; i++) { + String expd_part = expd_parts[i]; + boolean has = String_.Has(actl, expd_part); + if (has) + Tfds.Eq_true(false, expd_part + "\n" + actl); + } + } + public void Test_html_modules_js(String expd) { + Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_k004(); + this.Page().Html_data().Head_mgr().Init(app, wiki, this.Page()); + this.Page().Html_data().Head_mgr().Bfr_arg__add(bfr); + bfr.Mkr_rls(); + Tfds.Eq_str_lines(expd, bfr.To_str_and_clear()); + } + + private Tst_mgr tst_mgr = new Tst_mgr(); private Xop_tkn_mkr tkn_mkr; + public static final String Ttl_str = "Test page"; + public Xop_fxt Init_lang_numbers_separators_en() {return Init_lang_numbers_separators(",", ".");} + public Xop_fxt Init_lang_numbers_separators(String grp_spr, String dec_spr) {return Init_lang_numbers_separators(wiki.Lang(), grp_spr, dec_spr);} + public Xop_fxt Init_lang_numbers_separators(Xol_lang_itm lang, String grp_spr, String dec_spr) { + gplx.xowa.langs.numbers.Xol_transform_mgr separator_mgr = lang.Num_mgr().Separators_mgr(); + separator_mgr.Clear(); + separator_mgr.Set(gplx.xowa.langs.numbers.Xol_num_mgr.Separators_key__grp, Bry_.new_u8(grp_spr)); + separator_mgr.Set(gplx.xowa.langs.numbers.Xol_num_mgr.Separators_key__dec, Bry_.new_u8(dec_spr)); + return this; + } + public void Init_lang_kwds(int kwd_id, boolean case_match, String... kwds) {Init_lang_kwds(wiki.Lang(), kwd_id, case_match, kwds);} + public void Init_lang_kwds(Xol_lang_itm lang, int kwd_id, boolean case_match, String... kwds) { + Xol_kwd_mgr kwd_mgr = lang.Kwd_mgr(); + Xol_kwd_grp kwd_grp = kwd_mgr.Get_or_new(kwd_id); + kwd_grp.Srl_load(case_match, Bry_.Ary(kwds)); + } + public void Init_lang_vnts(String... vnts) { + wiki.Lang().Vnt_mgr().Enabled_(true); + gplx.xowa.langs.vnts.Xol_vnt_regy vnt_regy = wiki.Lang().Vnt_mgr().Regy(); + for (int i = 0; i < vnts.length; i++) { + byte[] vnt = Bry_.new_u8(vnts[i]); + vnt_regy.Add(vnt, vnt); + if (i == 0) { + wiki.Lang().Vnt_mgr().Cur_itm_(vnt); + } + } + wiki.Lang().Vnt_mgr().Init_end(); + } + public void Init_xtn_pages() { + Io_mgr.Instance.InitEngine_mem(); + wiki.Xtn_mgr().Xtn_proofread().Enabled_y_(); + wiki.Db_mgr().Load_mgr().Clear(); // must clear; otherwise fails b/c files get deleted, but wiki.data_mgr caches the Xowd_regy_mgr (the .reg file) in memory; + wiki.Ns_mgr().Add_new(Xowc_xtn_pages.Ns_page_id_default, "Page").Add_new(Xowc_xtn_pages.Ns_index_id_default, "Index").Init(); + } + public void Clear_ref_mgr() {this.Page().Ref_mgr().Grps_clear();} // clear to reset count + public static Xop_fxt new_nonwmf() { + Xoae_app app = Xoa_app_fxt.Make__app__edit(); + return new Xop_fxt(app, Xoa_app_fxt.Make__wiki__edit__nonwmf(app, "nethackwiki")); + } + private final Bry_bfr tmp_bfr = Bry_bfr_.New_w_size(255); + public String Exec__parse_to_hdump(String src_str) { + byte[] src_bry = Bry_.new_u8(src_str); + Xop_root_tkn root = Exec_parse_page_all_as_root(src_bry); + Xoh_wtr_ctx hctx = Xoh_wtr_ctx.Hdump; + Xoh_html_wtr html_wtr = wiki.Html_mgr().Html_wtr(); + html_wtr.Cfg().Toc__show_(Bool_.Y); // needed for hdr to show + ctx.Page().Html_data().Redlink_list().Clear(); + html_wtr.Write_doc(tmp_bfr, ctx, hctx, src_bry, root); + // Tfds.Dbg(tmp_bfr.To_str()); + return tmp_bfr.To_str_and_clear(); + } + public void Test__parse_to_html_mgr(String src_str, String expd) { + byte[] src_bry = Bry_.new_u8(src_str); + Xop_root_tkn root = Exec_parse_page_all_as_root(src_bry); + Xoae_page page = this.Page(); + page.Root_(root); + byte[] actl = wiki.Html_mgr().Page_wtr_mgr().Gen(page, Xopg_view_mode_.Tid__read); + Tfds.Eq_str_lines(expd, String_.new_u8(actl)); + } + public String Exec__parse_to_html_w_skin(String raw) { + Bry_bfr bfr = Bry_bfr_.New(); + Xow_html_mgr html_mgr = wiki.Html_mgr(); + this.Wiki().Html__wtr_mgr().Page_read_fmtr().Fmt_("~{page_data}"); + + byte[] raw_bry = Bry_.new_u8(raw); + Xop_root_tkn root = this.Exec_parse_page_all_as_root(raw_bry); + this.Page().Root_(root); + + html_mgr.Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Write_page(bfr, this.Page(), this.Ctx(), Xoh_page_html_source_.Wtr); + return bfr.To_str_and_clear(); + } + public void Test__parse_to_html_w_skin(String raw, String expd) { + Tfds.Eq_str_lines(expd, Exec__parse_to_html_w_skin(raw)); + } + public String Make__test_string(String expr, String expd) { + /* + {| class=wikitable + ! rslt !! expd !! actl !! code + |} + */ + Bry_bfr bfr = Bry_bfr_.New(); + bfr.Add_str_a7("|-\n"); + bfr.Add_str_u8("| {{#ifeq:" + String_.Replace(expd, "|", "{{!}}") + "|" + String_.Replace(expr, "|", "{{!}}") + "|pass|fail}}\n"); + bfr.Add_str_u8("| " + expd + "\n"); + bfr.Add_str_u8("| " + expr + "\n"); + bfr.Add_str_u8("| " + expr + "\n"); + return bfr.To_str(); + } + + public static Xop_fxt New_app_html() { + Xop_fxt fxt = new Xop_fxt(); + fxt.Wiki().Html_mgr().Page_wtr_mgr().Page_read_fmtr().Fmt_("~{page_data}"); + return fxt; + } +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java index c8daaf622..ee9276708 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java @@ -1,6 +1,6 @@ /* XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012-2017 gnosygnu@gmail.com +Copyright (C) 2012-2020 gnosygnu@gmail.com XOWA is licensed under the terms of the General Public License (GPL) Version 3, or alternatively under the terms of the Apache License Version 2.0. @@ -13,208 +13,235 @@ The terms of each license can be found in the source code repository: GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ -package gplx.xowa.addons.bldrs.mass_parses.parses.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; -import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*; -import gplx.xowa.files.origs.*; -import gplx.xowa.htmls.core.htmls.*; import gplx.xowa.htmls.core.bldrs.*; import gplx.xowa.htmls.hxtns.pages.*; import gplx.xowa.htmls.core.hzips.*; -import gplx.xowa.wikis.pages.*; -import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; -import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.pools.*; -import gplx.xowa.addons.wikis.fulltexts.indexers.bldrs.*; -public class Xomp_parse_wkr implements Gfo_invk { - // mgr vars - private final Xomp_parse_mgr mgr; - private final Xomp_mgr_db mgr_db; - private final Xomp_prog_mgr prog_mgr; - private final Xomp_page_pool page_pool; - private final Xof_orig_wkr file_orig_wkr; - private final Xomp_ns_ord_mgr ns_ord_mgr; - - // cfg vars - private final Xomp_parse_mgr_cfg cfg; - private int cleanup_interval, commit_interval; - private boolean log_file_lnkis; - - // wkr vars - private final Xowe_wiki wiki; - private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr(); - private final int uid; - private Xomp_wkr_db wkr_db; - private Xomp_stat_tbl stat_tbl; - private Hxtn_page_mgr hxtn_mgr; - - // indexer vars - private final Xofulltext_indexer_wkr indexer; - - private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0; - private int done_count; private long done_time; - public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xomp_parse_mgr_cfg cfg - , Xomp_mgr_db mgr_db, Xomp_page_pool page_pool - , Xomp_prog_mgr prog_mgr, Xof_orig_wkr file_orig_wkr, Xomp_ns_ord_mgr ns_ord_mgr - , Xowe_wiki wiki, Xofulltext_indexer_wkr indexer, int uid) { - // mgr vars - this.mgr = mgr; this.mgr_db = mgr_db; - this.page_pool = page_pool; this.prog_mgr = prog_mgr; this.file_orig_wkr = file_orig_wkr; - this.ns_ord_mgr = ns_ord_mgr; - this.indexer = indexer; - - // cfg vars - this.cfg = cfg; - this.cleanup_interval = cfg.Cleanup_interval(); - this.commit_interval = cfg.Commit_interval(); - this.log_file_lnkis = cfg.Log_file_lnkis(); - - // wkr-specific vars - this.wiki = wiki; this.uid = uid; - this.wkr_db = Xomp_wkr_db.New(Xomp_mgr_db.New__url(wiki), uid); - this.stat_tbl = new Xomp_stat_tbl(wkr_db.Conn()); - this.hxtn_mgr = wiki.Hxtn_mgr(); - this.hxtn_mgr.Init_by_xomp_wkr(wkr_db.Conn(), cfg.Zip_tid()); - } - public void Exec() { - Xow_parser_mgr parser_mgr = wiki.Parser_mgr(); - - // disable file download - wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make - wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing - wiki.File__orig_mgr().Wkrs__set(file_orig_wkr); - wiki.File_mgr().Fsdb_mode().Tid__v2__mp__y_(); - - // enable disable categories according to flag - wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Ctgs_enabled_(cfg.Hdump_catboxs()); - - // enable lnki_temp - Xomp_lnki_temp_wkr logger = null; - if (log_file_lnkis) { - logger = new Xomp_lnki_temp_wkr(wiki, wkr_db.Conn()); - parser_mgr.Ctx().Lnki().File_logger_(logger); - logger.Bgn(); - } - - // init log_mgr / property_wkr / stats - Xop_log_wkr_factory wkr_factory = new Xop_log_wkr_factory(wkr_db.Conn()); - if (cfg.Log_math()) wiki.Parser_mgr().Math__core().Log_wkr_(wkr_factory); - - // enable hdump - hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled()).Zip_tid_(cfg.Zip_tid()); - hdump_bldr.Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl())); - wkr_db.Conn().Txn_bgn("xomp"); - stat_tbl.Stmt_new(); - hxtn_mgr.Insert_bgn(false); - Xoh_wtr_ctx hctx = Xoh_wtr_ctx.Hdump_by_hzip_tid(cfg.Hzip_enabled() ? Xoh_hzip_dict_.Hdb__hzip : Xoh_hzip_dict_.Hdb__htxt); // ISSUE#:553; DATE:2019-09-25 - - // set status to running - mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__running); - - // main loop - int prv_ns = -1; - while (true) { - // get page from page pool - Xomp_page_itm ppg = Get_next(); - if (ppg == Xomp_page_itm.Null) { - mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__sleeping); - break; // no more pages - } - if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05 - - try { - long done_bgn = gplx.core.envs.System_.Ticks(); - - // get ns / ttl - int cur_ns = ppg.Ns_id(); - Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry()); - // if ns changed and prv_ns is main - if (cur_ns != prv_ns) { - prv_ns = cur_ns; - } - - // init page - Xoae_page wpg = Xoae_page.New(wiki, ttl); - wpg.Bldr__ns_ord_(ns_ord_mgr.Get_ord_by_ns_id(cur_ns)); // NOTE: must set ns_id for tier_id in lnki_temp; DATE:2016-09-19 - wpg.Db().Text().Text_bry_(ppg.Text()); - wpg.Db().Page().Init_by_mp(ppg.Id(), ppg.Page_score()); - wpg.Stat_itm().Init(uid); - - // parse page - Xop_ctx pctx = parser_mgr.Ctx(); - pctx.Clear_all(); - parser_mgr.Parse(wpg, true); - - // gen_html - hdump_bldr.Insert(pctx, wpg, hctx); - - // index - long fulltext_time = 0; - if (indexer != null) { - fulltext_time = gplx.core.envs.System_.Ticks(); - indexer.Index(wpg); - fulltext_time = gplx.core.envs.System_.Ticks__elapsed_in_frac(fulltext_time); - } - - // mark done for sake of progress - prog_mgr.Mark_done(ppg.Id()); - - // update stats - long time_cur = gplx.core.envs.System_.Ticks(); - long page_time = time_cur - done_bgn; - done_time += page_time; - ++done_count; - stat_tbl.Insert(wpg, hdump_bldr.Tmp_hpg(), uid, page_time, fulltext_time); - - // cleanup - // ctx.App().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released - if (wiki.Cache_mgr().Tmpl_result_cache().Count() > 50000) - wiki.Cache_mgr().Tmpl_result_cache().Clear(); - if (done_count % cleanup_interval == 0) { - wiki.Cache_mgr().Free_mem__page(); - wiki.Parser_mgr().Scrib().Core_term(); - wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); - } - if (done_count % commit_interval == 0) { - wkr_db.Conn().Txn_sav(); - } - } catch (Exception e) { - Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e)); - } - } - - // cleanup - try { - if (logger != null) logger.End(); - wkr_db.Conn().Txn_end(); - wkr_db.Conn().Rls_conn(); - stat_tbl.Stmt_rls(); - hxtn_mgr.Insert_end(false); - mgr.Wkrs_done_add_1(); // NOTE: must release latch last else thread errors - } - catch (Exception e) { - Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail_end; err=~{0}", Err_.Message_gplx_log(e)); - } - } - public void Bld_stats(Bry_bfr bfr) { - int done_time_in_sec = (int)(done_time / 1000); if (done_time_in_sec == 0) done_time_in_sec = 1; - bfr.Add_int_pad_bgn(Byte_ascii.Space, 4, uid ); - bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, (int)(done_count / done_time_in_sec)); - bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_count); - bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_time_in_sec); - bfr.Add_byte_nl(); - } - private Xomp_page_itm Get_next() { - if (list_idx == list_len) { - mgr_db.Tbl__wkr().Update_exec(uid, done_count, done_time); - list.Clear(); - page_pool.Get_next(mgr_db, cfg.Wkr_machine_name(), list); - list_len = list.Len(); - if (list_len == 0) return Xomp_page_itm.Null; - list_idx = 0; - } - return (Xomp_page_itm)list.Get_at(list_idx++); - } - public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { - if (ctx.Match(k, Invk__exec)) this.Exec(); - else return Gfo_invk_.Rv_unhandled; - return this; - } - public static final String Invk__exec = "exec"; - public static final String Cfg__ns_ids = "xomp.ns_ids"; -} +package gplx.xowa.addons.bldrs.mass_parses.parses.wkrs; + +import gplx.Bry_bfr; +import gplx.Byte_ascii; +import gplx.Err_; +import gplx.GfoMsg; +import gplx.Gfo_invk; +import gplx.Gfo_invk_; +import gplx.Gfo_usr_dlg_; +import gplx.GfsCtx; +import gplx.List_adp; +import gplx.List_adp_; +import gplx.xowa.Xoa_ttl; +import gplx.xowa.Xoae_page; +import gplx.xowa.Xowe_wiki; +import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_mgr_db; +import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_stat_tbl; +import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_wkr_db; +import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_wkr_tbl; +import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.Xomp_parse_mgr; +import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.Xomp_parse_mgr_cfg; +import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.Xomp_prog_mgr; +import gplx.xowa.addons.bldrs.mass_parses.parses.pools.Xomp_page_itm; +import gplx.xowa.addons.bldrs.mass_parses.parses.pools.Xomp_page_pool; +import gplx.xowa.addons.bldrs.mass_parses.parses.utls.Xomp_lnki_temp_wkr; +import gplx.xowa.addons.bldrs.mass_parses.parses.utls.Xomp_ns_ord_mgr; +import gplx.xowa.addons.wikis.fulltexts.indexers.bldrs.Xofulltext_indexer_wkr; +import gplx.xowa.files.origs.Xof_orig_wkr; +import gplx.xowa.htmls.core.bldrs.Xob_hdump_bldr; +import gplx.xowa.htmls.core.htmls.Xoh_wtr_ctx; +import gplx.xowa.htmls.core.hzips.Xoh_hzip_dict_; +import gplx.xowa.htmls.hxtns.pages.Hxtn_page_mgr; +import gplx.xowa.parsers.Xop_ctx; +import gplx.xowa.parsers.Xow_parser_mgr; +import gplx.xowa.parsers.logs.Xop_log_wkr_factory; +import gplx.xowa.wikis.pages.Xopg_view_mode_; + +public class Xomp_parse_wkr implements Gfo_invk { + // mgr vars + private final Xomp_parse_mgr mgr; + private final Xomp_mgr_db mgr_db; + private final Xomp_prog_mgr prog_mgr; + private final Xomp_page_pool page_pool; + private final Xof_orig_wkr file_orig_wkr; + private final Xomp_ns_ord_mgr ns_ord_mgr; + + // cfg vars + private final Xomp_parse_mgr_cfg cfg; + private int cleanup_interval, commit_interval; + private boolean log_file_lnkis; + + // wkr vars + private final Xowe_wiki wiki; + private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr(); + private final int uid; + private Xomp_wkr_db wkr_db; + private Xomp_stat_tbl stat_tbl; + private Hxtn_page_mgr hxtn_mgr; + + // indexer vars + private final Xofulltext_indexer_wkr indexer; + + private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0; + private int done_count; private long done_time; + public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xomp_parse_mgr_cfg cfg + , Xomp_mgr_db mgr_db, Xomp_page_pool page_pool + , Xomp_prog_mgr prog_mgr, Xof_orig_wkr file_orig_wkr, Xomp_ns_ord_mgr ns_ord_mgr + , Xowe_wiki wiki, Xofulltext_indexer_wkr indexer, int uid) { + // mgr vars + this.mgr = mgr; this.mgr_db = mgr_db; + this.page_pool = page_pool; this.prog_mgr = prog_mgr; this.file_orig_wkr = file_orig_wkr; + this.ns_ord_mgr = ns_ord_mgr; + this.indexer = indexer; + + // cfg vars + this.cfg = cfg; + this.cleanup_interval = cfg.Cleanup_interval(); + this.commit_interval = cfg.Commit_interval(); + this.log_file_lnkis = cfg.Log_file_lnkis(); + + // wkr-specific vars + this.wiki = wiki; this.uid = uid; + this.wkr_db = Xomp_wkr_db.New(Xomp_mgr_db.New__url(wiki), uid); + this.stat_tbl = new Xomp_stat_tbl(wkr_db.Conn()); + this.hxtn_mgr = wiki.Hxtn_mgr(); + this.hxtn_mgr.Init_by_xomp_wkr(wkr_db.Conn(), cfg.Zip_tid()); + } + public void Exec() { + Xow_parser_mgr parser_mgr = wiki.Parser_mgr(); + + // disable file download + wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make + wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing + wiki.File__orig_mgr().Wkrs__set(file_orig_wkr); + wiki.File_mgr().Fsdb_mode().Tid__v2__mp__y_(); + + // enable disable categories according to flag + wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Ctgs_enabled_(cfg.Hdump_catboxs()); + + // enable lnki_temp + Xomp_lnki_temp_wkr logger = null; + if (log_file_lnkis) { + logger = new Xomp_lnki_temp_wkr(wiki, wkr_db.Conn()); + parser_mgr.Ctx().Lnki().File_logger_(logger); + logger.Bgn(); + } + + // init log_mgr / property_wkr / stats + Xop_log_wkr_factory wkr_factory = new Xop_log_wkr_factory(wkr_db.Conn()); + if (cfg.Log_math()) wiki.Parser_mgr().Math__core().Log_wkr_(wkr_factory); + + // enable hdump + hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled()).Zip_tid_(cfg.Zip_tid()); + hdump_bldr.Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl())); + wkr_db.Conn().Txn_bgn("xomp"); + stat_tbl.Stmt_new(); + hxtn_mgr.Insert_bgn(false); + Xoh_wtr_ctx hctx = Xoh_wtr_ctx.Hdump_by_hzip_tid(cfg.Hzip_enabled() ? Xoh_hzip_dict_.Hdb__hzip : Xoh_hzip_dict_.Hdb__htxt); // ISSUE#:553; DATE:2019-09-25 + + // set status to running + mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__running); + + // main loop + int prv_ns = -1; + while (true) { + // get page from page pool + Xomp_page_itm ppg = Get_next(); + if (ppg == Xomp_page_itm.Null) { + mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__sleeping); + break; // no more pages + } + if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05 + + try { + long done_bgn = gplx.core.envs.System_.Ticks(); + + // get ns / ttl + int cur_ns = ppg.Ns_id(); + Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry()); + // if ns changed and prv_ns is main + if (cur_ns != prv_ns) { + prv_ns = cur_ns; + } + + // init page + Xoae_page wpg = Xoae_page.New(wiki, ttl); + wpg.Bldr__ns_ord_(ns_ord_mgr.Get_ord_by_ns_id(cur_ns)); // NOTE: must set ns_id for tier_id in lnki_temp; DATE:2016-09-19 + wpg.Db().Text().Text_bry_(ppg.Text()); + wpg.Db().Page().Init_by_mp(ppg.Id(), ppg.Page_score()); + wpg.Stat_itm().Init(uid); + + // parse page + Xop_ctx pctx = parser_mgr.Ctx(); + pctx.Clear_all(); + parser_mgr.Parse(wpg, true); + + // gen_html + hdump_bldr.Insert(pctx, wpg, hctx); + + // index + long fulltext_time = 0; + if (indexer != null) { + fulltext_time = gplx.core.envs.System_.Ticks(); + indexer.Index(wpg); + fulltext_time = gplx.core.envs.System_.Ticks__elapsed_in_frac(fulltext_time); + } + + // mark done for sake of progress + prog_mgr.Mark_done(ppg.Id()); + + // update stats + long time_cur = gplx.core.envs.System_.Ticks(); + long page_time = time_cur - done_bgn; + done_time += page_time; + ++done_count; + stat_tbl.Insert(wpg, hdump_bldr.Tmp_hpg(), uid, page_time, fulltext_time); + + // cleanup + // ctx.App().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released + if (done_count % cleanup_interval == 0) { + wiki.Cache_mgr().Free_mem__page(); + wiki.Parser_mgr().Scrib().Core_term(); + wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); + } + if (done_count % commit_interval == 0) { + wkr_db.Conn().Txn_sav(); + } + } catch (Exception e) { + Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e)); + } + } + + // cleanup + try { + if (logger != null) logger.End(); + wkr_db.Conn().Txn_end(); + wkr_db.Conn().Rls_conn(); + stat_tbl.Stmt_rls(); + hxtn_mgr.Insert_end(false); + mgr.Wkrs_done_add_1(); // NOTE: must release latch last else thread errors + } + catch (Exception e) { + Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail_end; err=~{0}", Err_.Message_gplx_log(e)); + } + } + public void Bld_stats(Bry_bfr bfr) { + int done_time_in_sec = (int)(done_time / 1000); if (done_time_in_sec == 0) done_time_in_sec = 1; + bfr.Add_int_pad_bgn(Byte_ascii.Space, 4, uid ); + bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, (int)(done_count / done_time_in_sec)); + bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_count); + bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_time_in_sec); + bfr.Add_byte_nl(); + } + private Xomp_page_itm Get_next() { + if (list_idx == list_len) { + mgr_db.Tbl__wkr().Update_exec(uid, done_count, done_time); + list.Clear(); + page_pool.Get_next(mgr_db, cfg.Wkr_machine_name(), list); + list_len = list.Len(); + if (list_len == 0) return Xomp_page_itm.Null; + list_idx = 0; + } + return (Xomp_page_itm)list.Get_at(list_idx++); + } + public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { + if (ctx.Match(k, Invk__exec)) this.Exec(); + else return Gfo_invk_.Rv_unhandled; + return this; + } + public static final String Invk__exec = "exec"; + public static final String Cfg__ns_ids = "xomp.ns_ids"; +} diff --git a/400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java b/400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java index 2a4c8cf8f..71ea678e2 100644 --- a/400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java +++ b/400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java @@ -1,86 +1,85 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012-2020 gnosygnu@gmail.com - -XOWA is licensed under the terms of the General Public License (GPL) Version 3, -or alternatively under the terms of the Apache License Version 2.0. - -You may use XOWA according to either of these licenses as is most appropriate -for your project on a case-by-case basis. - -The terms of each license can be found in the source code repository: - -GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt -Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt -*/ -package gplx.xowa.addons.parsers.mediawikis; - -import gplx.Bry_; -import gplx.Bry_bfr; -import gplx.Bry_bfr_; -import gplx.String_; -import gplx.xowa.Xoa_ttl; -import gplx.xowa.Xoae_page; -import gplx.xowa.Xowe_wiki; -import gplx.xowa.addons.wikis.ctgs.htmls.pageboxs.Xoctg_pagebox_itm; -import gplx.xowa.htmls.core.htmls.Xoh_wtr_ctx; -import gplx.xowa.parsers.Xop_ctx; -import gplx.xowa.parsers.Xow_parser_mgr; -import gplx.xowa.wikis.Xow_page_tid; -import gplx.xowa.wikis.pages.Xopg_view_mode_; - -public class Xop_mediawiki_wkr { - private final Xowe_wiki wiki; - private final Bry_bfr tmp_bfr = Bry_bfr_.New(); - public Xop_mediawiki_wkr(Xowe_wiki wiki, Xop_mediawiki_loader loader) { - this.wiki = wiki; - if (loader != null) - wiki.Cache_mgr().Load_wkr_(new Xow_page_cache_wkr__embeddable(wiki, loader)); - } - public void Free_memory() { - wiki.Cache_mgr().Tmpl_result_cache().Clear(); - wiki.Cache_mgr().Free_mem__page(); - wiki.Parser_mgr().Scrib().Core_term(); - wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); - } - public String Parse(String page, String wikitext) { - Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page)); - - byte[] wtxt = Bry_.new_u8(wikitext); - Xoae_page wpg = Xoae_page.New(wiki, ttl); - wpg.Db().Text().Text_bry_(wtxt); - - Xow_parser_mgr parser_mgr = wiki.Parser_mgr(); - - // parse page - Xop_ctx pctx = parser_mgr.Ctx(); - pctx.Clear_all(); - parser_mgr.Parse(wpg, true); - - // write to html - boolean is_wikitext = Xow_page_tid.Identify(wpg.Wiki().Domain_tid(), ttl.Ns().Id(), ttl.Page_db()) == Xow_page_tid.Tid_wikitext; - byte[] orig_bry = Bry_.Empty; - if (is_wikitext) { - wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Write_hdump(tmp_bfr, pctx, Xoh_wtr_ctx.Embeddable, wpg); - - // write categories - int ctgs_len = wpg.Wtxt().Ctgs__len(); - if ( ctgs_len > 0 // skip if no categories found while parsing wikitext - ) { - Xoctg_pagebox_itm[] pagebox_itms = new Xoctg_pagebox_itm[ctgs_len]; - for (int i = 0; i < ctgs_len; i++) { - pagebox_itms[i] = new Xoctg_pagebox_itm(wpg.Wtxt().Ctgs__get_at(i)); - } - wiki.Ctg__pagebox_wtr().Write_pagebox(tmp_bfr, wpg, pagebox_itms); - } - - orig_bry = tmp_bfr.To_bry_and_clear(); - wpg.Db().Html().Html_bry_(orig_bry); - } - else { // not wikitext; EX: pages in MediaWiki: ns; DATE:2016-09-12 - wpg.Db().Html().Html_bry_(wpg.Db().Text().Text_bry()); - } - - return String_.new_u8(orig_bry); - } -} +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012-2020 gnosygnu@gmail.com + +XOWA is licensed under the terms of the General Public License (GPL) Version 3, +or alternatively under the terms of the Apache License Version 2.0. + +You may use XOWA according to either of these licenses as is most appropriate +for your project on a case-by-case basis. + +The terms of each license can be found in the source code repository: + +GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt +Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt +*/ +package gplx.xowa.addons.parsers.mediawikis; + +import gplx.Bry_; +import gplx.Bry_bfr; +import gplx.Bry_bfr_; +import gplx.String_; +import gplx.xowa.Xoa_ttl; +import gplx.xowa.Xoae_page; +import gplx.xowa.Xowe_wiki; +import gplx.xowa.addons.wikis.ctgs.htmls.pageboxs.Xoctg_pagebox_itm; +import gplx.xowa.htmls.core.htmls.Xoh_wtr_ctx; +import gplx.xowa.parsers.Xop_ctx; +import gplx.xowa.parsers.Xow_parser_mgr; +import gplx.xowa.wikis.Xow_page_tid; +import gplx.xowa.wikis.pages.Xopg_view_mode_; + +public class Xop_mediawiki_wkr { + private final Xowe_wiki wiki; + private final Bry_bfr tmp_bfr = Bry_bfr_.New(); + public Xop_mediawiki_wkr(Xowe_wiki wiki, Xop_mediawiki_loader loader) { + this.wiki = wiki; + if (loader != null) + wiki.Cache_mgr().Load_wkr_(new Xow_page_cache_wkr__embeddable(wiki, loader)); + } + public void Free_memory() { + wiki.Cache_mgr().Free_mem__page(); + wiki.Parser_mgr().Scrib().Core_term(); + wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); + } + public String Parse(String page, String wikitext) { + Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page)); + + byte[] wtxt = Bry_.new_u8(wikitext); + Xoae_page wpg = Xoae_page.New(wiki, ttl); + wpg.Db().Text().Text_bry_(wtxt); + + Xow_parser_mgr parser_mgr = wiki.Parser_mgr(); + + // parse page + Xop_ctx pctx = parser_mgr.Ctx(); + pctx.Clear_all(); + parser_mgr.Parse(wpg, true); + + // write to html + boolean is_wikitext = Xow_page_tid.Identify(wpg.Wiki().Domain_tid(), ttl.Ns().Id(), ttl.Page_db()) == Xow_page_tid.Tid_wikitext; + byte[] orig_bry = Bry_.Empty; + if (is_wikitext) { + wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Write_hdump(tmp_bfr, pctx, Xoh_wtr_ctx.Embeddable, wpg); + + // write categories + int ctgs_len = wpg.Wtxt().Ctgs__len(); + if ( ctgs_len > 0 // skip if no categories found while parsing wikitext + ) { + Xoctg_pagebox_itm[] pagebox_itms = new Xoctg_pagebox_itm[ctgs_len]; + for (int i = 0; i < ctgs_len; i++) { + pagebox_itms[i] = new Xoctg_pagebox_itm(wpg.Wtxt().Ctgs__get_at(i)); + } + wiki.Ctg__pagebox_wtr().Write_pagebox(tmp_bfr, wpg, pagebox_itms); + } + + orig_bry = tmp_bfr.To_bry_and_clear(); + wpg.Db().Html().Html_bry_(orig_bry); + } + else { // not wikitext; EX: pages in MediaWiki: ns; DATE:2016-09-12 + wpg.Db().Html().Html_bry_(wpg.Db().Text().Text_bry()); + } + + return String_.new_u8(orig_bry); + } +} diff --git a/400_xowa/src/gplx/xowa/bldrs/cmds/Xob_dump_mgr_base.java b/400_xowa/src/gplx/xowa/bldrs/cmds/Xob_dump_mgr_base.java index 133b5ea92..3bff3a6c1 100644 --- a/400_xowa/src/gplx/xowa/bldrs/cmds/Xob_dump_mgr_base.java +++ b/400_xowa/src/gplx/xowa/bldrs/cmds/Xob_dump_mgr_base.java @@ -1,6 +1,6 @@ /* XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012-2017 gnosygnu@gmail.com +Copyright (C) 2012-2020 gnosygnu@gmail.com XOWA is licensed under the terms of the General Public License (GPL) Version 3, or alternatively under the terms of the Apache License Version 2.0. @@ -13,314 +13,347 @@ The terms of each license can be found in the source code repository: GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ -package gplx.xowa.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; -import gplx.core.envs.*; -import gplx.dbs.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.files.origs.*; -import gplx.xowa.bldrs.wkrs.*; -import gplx.xowa.wikis.nss.*; -import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*; -import gplx.xowa.addons.bldrs.files.utls.*; -import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*; -public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xob_cmd, Gfo_invk { - private Xob_dump_src_id page_src; - private Xow_db_mgr db_fsys_mgr; protected Xop_parser parser; protected Xop_ctx ctx; protected Xop_root_tkn root; - private int[] ns_ary; private Xow_db_file[] db_ary; - private int ns_bgn = -1, db_bgn = -1, pg_bgn = -1; - private int ns_end = -1, db_end = -1, pg_end = Int_.Max_value; - private int commit_interval = 1000, progress_interval = 250, cleanup_interval = 2500, select_size = 10 * Io_mgr.Len_mb; - private int exec_count, exec_count_max = Int_.Max_value; - private boolean reset_db = false, exit_after_commit = false, exit_now = false; - private boolean load_tmpls; - private Xob_dump_bmk_mgr bmk_mgr = new Xob_dump_bmk_mgr(); - private Xobu_poll_mgr poll_mgr; private int poll_interval = 5000; - private Xob_rate_mgr rate_mgr = new Xob_rate_mgr(); - public abstract String Cmd_key(); - @Override protected void Cmd_ctor_end(Xob_bldr bldr, Xowe_wiki wiki) { - poll_mgr = new Xobu_poll_mgr(bldr.App()); // init in ctor so gfs can invoke methods - } - public void Cmd_bgn(Xob_bldr bldr) { - parser = wiki.Parser_mgr().Main(); - ctx = wiki.Parser_mgr().Ctx(); - root = ctx.Tkn_mkr().Root(Bry_.Empty); - wiki.Init_assert(); // NOTE: must init wiki for db_mgr_as_sql - - // assert by calling Db_mgr_as_sql - wiki.Db_mgr_as_sql().Core_data_mgr(); - - // load db_mgr - Xow_db_mgr.Init_by_load(wiki, gplx.xowa.wikis.data.Xow_db_file__core_.Find_core_fil_or_null(wiki)); // NOTE: must reinit providers as previous steps may have rls'd (and left member variable conn which is closed) - - wiki.File__orig_mgr().Wkrs__del(Xof_orig_wkr_.Tid_wmf_api); - db_fsys_mgr = wiki.Db_mgr_as_sql().Core_data_mgr(); - db_ary = Xob_dump_mgr_base_.Init_text_files_ary(db_fsys_mgr); - poll_interval = poll_mgr.Poll_interval(); - - page_src = new Xob_dump_src_id().Init(wiki, this.Init_redirect(), select_size); - ns_ary = Init_ns_ary(); - Db_conn conn = Init_db_file(); - Io_url wiki_dir = wiki.Fsys_mgr().Root_dir(); - bmk_mgr.Cfg_url_(wiki_dir.GenSubFil("xowa.file.make.cfg.gfs")); - rate_mgr.Log_file_(wiki_dir.GenSubFil("xowa.file.make.log.csv")); - if (reset_db) { - bmk_mgr.Reset(); - Init_reset(conn); - } - bmk_mgr.Load(wiki.Appe(), this); - - Cmd_bgn_end(); - } - protected abstract void Cmd_bgn_end(); - public abstract byte Init_redirect(); - public abstract int[] Init_ns_ary(); - protected abstract void Init_reset(Db_conn p); - protected abstract Db_conn Init_db_file(); - private long time_bgn; - public void Cmd_run() {Exec_ns_ary();} - private void Exec_ns_ary() { - if (pg_bgn == Int_.Max_value) return; - if (load_tmpls) Xob_dump_mgr_base_.Load_all_tmpls(usr_dlg, wiki, page_src); - time_bgn = System_.Ticks(); - Xob_dump_bmk dump_bmk = new Xob_dump_bmk(); - rate_mgr.Init(); - int ns_ary_len = ns_ary.length; - for (int i = 0; i < ns_ary_len; i++) { - int ns_id = ns_ary[i]; - if (ns_bgn != -1) { // ns_bgn set - if (ns_id == ns_bgn) // ns_id is ns_bgn; null out ns_bgn and continue - ns_bgn = -1; - else // ns_id is not ns_bgn; keep looking - continue; - } - dump_bmk.Ns_id_(ns_id); - Exec_db_ary(i, dump_bmk, ns_id); - if (ns_id == ns_end) exit_now = true; // ns_end set; exit - if (exit_now) break; // exit_now b/c of pg_bgn, db_bgn or something else - } - Exec_commit(dump_bmk.Ns_id(), dump_bmk.Db_id(), dump_bmk.Pg_id(), Bry_.Empty); - } - private void Exec_db_ary(int ns_ord, Xob_dump_bmk dump_bmk, int ns_id) { - int db_ary_len = db_ary.length; - for (int i = 0; i < db_ary_len; i++) { - int db_id = db_ary[i].Id(); - if (db_bgn != -1) { // db_bgn set - if (db_id == db_bgn) // db_id is db_bgn; null out db_bgn and continue - db_bgn = -1; - else // db_id is not db_bgn; keep looking - continue; - } - dump_bmk.Db_id_(db_id); - Exec_db_itm(dump_bmk, ns_ord, ns_id, db_id); - if (db_id == db_end) exit_now = true; // db_end set; exit; - if (exit_now) return; // exit_now b/c of pg_bgn, db_bgn or something else - } - } - private void Exec_db_itm(Xob_dump_bmk dump_bmk, int ns_ord, int ns_id, int db_id) { - List_adp pages = List_adp_.New(); - Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id); - int pg_id = pg_bgn; - while (true) { - page_src.Get_pages(wiki.Appe(), pages, db_id, ns_id, pg_id); - int pages_len = pages.Count(); - if (pages_len == 0) { // no more pages in db; - if (pg_id > pg_bgn) // reset pg_bgn to 0 only if pg_bgn seen; - pg_bgn = 0; - return; - } - usr_dlg.Prog_many("", "", "fetched pages: ~{0}", pages_len); - for (int i = 0; i < pages_len; i++) { - Xowd_page_itm page = (Xowd_page_itm)pages.Get_at(i); - dump_bmk.Pg_id_(pg_id); - Exec_pg_itm(ns_ord, ns, db_id, page); - if ( pg_id >= pg_end - || exec_count >= exec_count_max) { - exit_now = true; - } - if (exit_now) return; - pg_id = page.Id(); - } - } - } - private void Exec_pg_itm(int ns_ord, Xow_ns ns, int db_id, Xowd_page_itm page) { - try { - if ((exec_count % progress_interval) == 0) - usr_dlg.Prog_many("", "", "parsing: ns=~{0} db=~{1} pg=~{2} count=~{3} time=~{4} rate=~{5} ttl=~{6}" - , ns.Id(), db_id, page.Id(), exec_count - , System_.Ticks__elapsed_in_sec(time_bgn), rate_mgr.Rate_as_str(), String_.new_u8(page.Ttl_page_db())); - ctx.Clear_all(); - byte[] page_src = page.Text(); - if (page_src != null) // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05 - Exec_pg_itm_hook(ns_ord, ns, page, page_src); - ctx.Wiki().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released - if (ctx.Wiki().Cache_mgr().Tmpl_result_cache().Count() > 50000) - ctx.Wiki().Cache_mgr().Tmpl_result_cache().Clear(); - ++exec_count; - rate_mgr.Increment(); - if ((exec_count % poll_interval) == 0) - poll_mgr.Poll(); - if ((exec_count % commit_interval) == 0) - Exec_commit(ns.Id(), db_id, page.Id(), page.Ttl_page_db()); - if ((exec_count % cleanup_interval) == 0) - Free(); - } - catch (Exception exc) { - bldr.Usr_dlg().Warn_many("", "", "parse failed: wiki=~{0} ttl=~{1} err=~{2}", wiki.Domain_str(), page.Ttl_full_db(), Err_.Message_gplx_log(exc)); - ctx.Wiki().Utl__bfr_mkr().Clear(); - this.Free(); - } - } - public abstract void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm page, byte[] page_text); - private void Exec_commit(int ns_id, int db_id, int pg_id, byte[] ttl) { - usr_dlg.Prog_many("", "", "committing: ns=~{0} db=~{1} pg=~{2} count=~{3} ttl=~{4}", ns_id, db_id, pg_id, exec_count, String_.new_u8(ttl)); - Exec_commit_hook(); - bmk_mgr.Save(ns_id, db_id, pg_id); - if (exit_after_commit) exit_now = true; - } - public abstract void Exec_commit_hook(); - public abstract void Exec_end_hook(); - public void Cmd_init(Xob_bldr bldr) {} - public void Cmd_term() {} - public void Cmd_end() { - if (!exit_now) - pg_bgn = Int_.Max_value; - Exec_commit(-1, -1, -1, Bry_.Empty); - Exec_end_hook(); - Free(); - usr_dlg.Note_many("", "", "done: ~{0} ~{1}", exec_count, Decimal_adp_.divide_safe_(exec_count, System_.Ticks__elapsed_in_sec(time_bgn)).To_str("#,###.000")); - } - private void Free() { - Xowe_wiki_.Rls_mem(wiki, true); - } - protected void Reset_db_y_() {this.reset_db = true;} - @Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { - if (ctx.Match(k, Invk_commit_interval_)) commit_interval = m.ReadInt("v"); - else if (ctx.Match(k, Invk_progress_interval_)) progress_interval = m.ReadInt("v"); - else if (ctx.Match(k, Invk_rate_interval_)) rate_mgr.Reset_interval_(m.ReadInt("v")); - else if (ctx.Match(k, Invk_cleanup_interval_)) cleanup_interval = m.ReadInt("v"); - else if (ctx.Match(k, Invk_select_size_)) select_size = m.ReadInt("v") * Io_mgr.Len_mb; - else if (ctx.Match(k, Invk_ns_bgn_)) {ns_bgn = m.ReadInt("v"); Notify_restoring("ns", ns_bgn);} - else if (ctx.Match(k, Invk_db_bgn_)) {db_bgn = m.ReadInt("v"); Notify_restoring("db", db_bgn);} - else if (ctx.Match(k, Invk_pg_bgn_)) {pg_bgn = m.ReadInt("v"); Notify_restoring("pg", pg_bgn);} - else if (ctx.Match(k, Invk_ns_end_)) ns_end = m.ReadInt("v"); - else if (ctx.Match(k, Invk_db_end_)) db_end = m.ReadInt("v"); - else if (ctx.Match(k, Invk_pg_end_)) pg_end = m.ReadInt("v"); - else if (ctx.Match(k, Invk_load_tmpls_)) load_tmpls = m.ReadYn("v"); - else if (ctx.Match(k, Invk_poll_mgr)) return poll_mgr; - else if (ctx.Match(k, Invk_reset_db_)) reset_db = m.ReadYn("v"); - else if (ctx.Match(k, Invk_exec_count_max_)) exec_count_max = m.ReadInt("v"); - else if (ctx.Match(k, Invk_exit_now_)) exit_now = m.ReadYn("v"); - else if (ctx.Match(k, Invk_exit_after_commit_)) exit_after_commit = m.ReadYn("v"); - else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v")); - else return Gfo_invk_.Rv_unhandled; - return this; - } - private void Notify_restoring(String itm, int val) { - usr_dlg.Note_many("", "", "restoring: itm=~{0} val=~{1}", itm, val); - } - public static final String - Invk_progress_interval_ = "progress_interval_", Invk_commit_interval_ = "commit_interval_", Invk_cleanup_interval_ = "cleanup_interval_", Invk_rate_interval_ = "rate_interval_" - , Invk_select_size_ = "select_size_" - , Invk_ns_bgn_ = "ns_bgn_", Invk_db_bgn_ = "db_bgn_", Invk_pg_bgn_ = "pg_bgn_" - , Invk_ns_end_ = "ns_end_", Invk_db_end_ = "db_end_", Invk_pg_end_ = "pg_end_" - , Invk_load_tmpls_ = "load_tmpls_" - , Invk_poll_mgr = "poll_mgr", Invk_reset_db_ = "reset_db_" - , Invk_exec_count_max_ = "exec_count_max_", Invk_exit_now_ = "exit_now_", Invk_exit_after_commit_ = "exit_after_commit_" - , Invk__manual_now_ = "manual_now_" - ; -} -class Xob_dump_mgr_base_ { - public static void Load_all_tmpls(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, Xob_dump_src_id page_src) { - List_adp pages = List_adp_.New(); - Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template(); - Xow_defn_cache defn_cache = wiki.Cache_mgr().Defn_cache(); - int cur_page_id = -1; - int load_count = 0; - usr_dlg.Note_many("", "", "tmpl_load init"); - while (true) { - page_src.Get_pages(wiki.Appe(), pages, 0, Xow_ns_.Tid__template, cur_page_id); // 0 is always template db - int page_count = pages.Count(); - if (page_count == 0) break; // no more pages in db; - Xowd_page_itm page = null; - for (int i = 0; i < page_count; i++) { - page = (Xowd_page_itm)pages.Get_at(i); - Xot_defn_tmpl defn = new Xot_defn_tmpl(); - defn.Init_by_new(ns_tmpl, ns_tmpl.Gen_ttl(page.Ttl_page_db()), page.Text(), null, false); // NOTE: passing null, false; will be overriden later when Parse is called - defn_cache.Add(defn, ns_tmpl.Case_match()); - ++load_count; - if ((load_count % 10000) == 0) usr_dlg.Prog_many("", "", "tmpl_loading: ~{0}", load_count); - } - cur_page_id = page.Id(); - } - usr_dlg.Note_many("", "", "tmpl_load done: ~{0}", load_count); - } - public static Xow_db_file[] Init_text_files_ary(Xow_db_mgr core_data_mgr) { - List_adp text_files_list = List_adp_.New(); - int len = core_data_mgr.Dbs__len(); - if (len == 1) return new Xow_db_file[] {core_data_mgr.Dbs__get_at(0)}; // single file: return core; note that there are no Tid = Text - for (int i = 0; i < len; i++) { - Xow_db_file file = core_data_mgr.Dbs__get_at(i); - switch (file.Tid()) { - case Xow_db_file_.Tid__text: - case Xow_db_file_.Tid__text_solo: - text_files_list.Add(file); - break; - } - } - return (Xow_db_file[])text_files_list.To_ary_and_clear(Xow_db_file.class); - } -} -class Xob_dump_bmk_mgr { - private Bry_bfr save_bfr = Bry_bfr_.Reset(1024); - public Io_url Cfg_url() {return cfg_url;} public Xob_dump_bmk_mgr Cfg_url_(Io_url v) {cfg_url = v; return this;} private Io_url cfg_url; - public void Reset() {Io_mgr.Instance.DeleteFil(cfg_url);} - public void Load(Xoae_app app, Xob_dump_mgr_base dump_mgr) { - app.Gfs_mgr().Run_url_for(dump_mgr, cfg_url); - } - public void Save(int ns_id, int db_id, int pg_id) { - Save_itm(save_bfr, Xob_dump_mgr_base.Invk_ns_bgn_, ns_id); - Save_itm(save_bfr, Xob_dump_mgr_base.Invk_db_bgn_, db_id); - Save_itm(save_bfr, Xob_dump_mgr_base.Invk_pg_bgn_, pg_id); - Io_mgr.Instance.SaveFilBfr(cfg_url, save_bfr); - } - private void Save_itm(Bry_bfr save_bfr, String key, int val) { - String fmt = "{0}('{1}');\n"; - String str = String_.Format(fmt, key, val); - save_bfr.Add_str_u8(str); - } -} -class Xob_rate_mgr { - private long time_bgn; - private int item_len; - private Bry_bfr save_bfr = Bry_bfr_.Reset(255); - public int Reset_interval() {return reset_interval;} public Xob_rate_mgr Reset_interval_(int v) {reset_interval = v; return this;} private int reset_interval = 10000; - public Io_url Log_file_url() {return log_file;} public Xob_rate_mgr Log_file_(Io_url v) {log_file = v; return this;} private Io_url log_file; - public void Init() {time_bgn = System_.Ticks();} - public void Increment() { - ++item_len; - if (item_len % reset_interval == 0) { - long time_end = System_.Ticks(); - Save(item_len, time_bgn, time_end); - time_bgn = time_end; - item_len = 0; - } - } - private void Save(int count, long bgn, long end) { - int dif = (int)(end - bgn) / 1000; - Decimal_adp rate = Decimal_adp_.divide_safe_(count, dif); - save_bfr - .Add_str_a7(rate.To_str("#,##0.000")).Add_byte_pipe() - .Add_int_variable(count).Add_byte_pipe() - .Add_int_variable(dif).Add_byte_nl() - ; - Io_mgr.Instance.AppendFilByt(log_file, save_bfr.To_bry_and_clear()); - } - public String Rate_as_str() {return Int_.To_str(Rate());} - public int Rate() { - int elapsed = System_.Ticks__elapsed_in_sec(time_bgn); - return Math_.Div_safe_as_int(item_len, elapsed); - } -} -class Xob_dump_bmk { - public int Ns_id() {return ns_id;} public Xob_dump_bmk Ns_id_(int v) {ns_id = v; return this;} private int ns_id; - public int Db_id() {return db_id;} public Xob_dump_bmk Db_id_(int v) {db_id = v; return this;} private int db_id; - public int Pg_id() {return pg_id;} public Xob_dump_bmk Pg_id_(int v) {pg_id = v; return this;} private int pg_id; -} +package gplx.xowa.bldrs.cmds; + +import gplx.Bry_; +import gplx.Bry_bfr; +import gplx.Bry_bfr_; +import gplx.Datetime_now; +import gplx.Decimal_adp; +import gplx.Decimal_adp_; +import gplx.Err_; +import gplx.GfoMsg; +import gplx.Gfo_invk; +import gplx.Gfo_invk_; +import gplx.Gfo_usr_dlg; +import gplx.GfsCtx; +import gplx.Int_; +import gplx.Io_mgr; +import gplx.Io_url; +import gplx.List_adp; +import gplx.List_adp_; +import gplx.Math_; +import gplx.String_; +import gplx.core.envs.System_; +import gplx.dbs.Db_conn; +import gplx.xowa.Xoae_app; +import gplx.xowa.Xowe_wiki; +import gplx.xowa.Xowe_wiki_; +import gplx.xowa.addons.bldrs.files.utls.Xobu_poll_mgr; +import gplx.xowa.bldrs.Xob_bldr; +import gplx.xowa.bldrs.wkrs.Xob_cmd; +import gplx.xowa.bldrs.wkrs.Xob_itm_basic_base; +import gplx.xowa.files.origs.Xof_orig_wkr_; +import gplx.xowa.parsers.Xop_ctx; +import gplx.xowa.parsers.Xop_parser; +import gplx.xowa.parsers.Xop_root_tkn; +import gplx.xowa.parsers.tmpls.Xot_defn_tmpl; +import gplx.xowa.wikis.caches.Xow_defn_cache; +import gplx.xowa.wikis.data.Xow_db_file; +import gplx.xowa.wikis.data.Xow_db_file_; +import gplx.xowa.wikis.data.Xow_db_mgr; +import gplx.xowa.wikis.data.tbls.Xowd_page_itm; +import gplx.xowa.wikis.nss.Xow_ns; +import gplx.xowa.wikis.nss.Xow_ns_; + +public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xob_cmd, Gfo_invk { + private Xob_dump_src_id page_src; + private Xow_db_mgr db_fsys_mgr; protected Xop_parser parser; protected Xop_ctx ctx; protected Xop_root_tkn root; + private int[] ns_ary; private Xow_db_file[] db_ary; + private int ns_bgn = -1, db_bgn = -1, pg_bgn = -1; + private int ns_end = -1, db_end = -1, pg_end = Int_.Max_value; + private int commit_interval = 1000, progress_interval = 250, cleanup_interval = 2500, select_size = 10 * Io_mgr.Len_mb; + private int exec_count, exec_count_max = Int_.Max_value; + private boolean reset_db = false, exit_after_commit = false, exit_now = false; + private boolean load_tmpls; + private Xob_dump_bmk_mgr bmk_mgr = new Xob_dump_bmk_mgr(); + private Xobu_poll_mgr poll_mgr; private int poll_interval = 5000; + private Xob_rate_mgr rate_mgr = new Xob_rate_mgr(); + public abstract String Cmd_key(); + @Override protected void Cmd_ctor_end(Xob_bldr bldr, Xowe_wiki wiki) { + poll_mgr = new Xobu_poll_mgr(bldr.App()); // init in ctor so gfs can invoke methods + } + public void Cmd_bgn(Xob_bldr bldr) { + parser = wiki.Parser_mgr().Main(); + ctx = wiki.Parser_mgr().Ctx(); + root = ctx.Tkn_mkr().Root(Bry_.Empty); + wiki.Init_assert(); // NOTE: must init wiki for db_mgr_as_sql + + // assert by calling Db_mgr_as_sql + wiki.Db_mgr_as_sql().Core_data_mgr(); + + // load db_mgr + Xow_db_mgr.Init_by_load(wiki, gplx.xowa.wikis.data.Xow_db_file__core_.Find_core_fil_or_null(wiki)); // NOTE: must reinit providers as previous steps may have rls'd (and left member variable conn which is closed) + + wiki.File__orig_mgr().Wkrs__del(Xof_orig_wkr_.Tid_wmf_api); + db_fsys_mgr = wiki.Db_mgr_as_sql().Core_data_mgr(); + db_ary = Xob_dump_mgr_base_.Init_text_files_ary(db_fsys_mgr); + poll_interval = poll_mgr.Poll_interval(); + + page_src = new Xob_dump_src_id().Init(wiki, this.Init_redirect(), select_size); + ns_ary = Init_ns_ary(); + Db_conn conn = Init_db_file(); + Io_url wiki_dir = wiki.Fsys_mgr().Root_dir(); + bmk_mgr.Cfg_url_(wiki_dir.GenSubFil("xowa.file.make.cfg.gfs")); + rate_mgr.Log_file_(wiki_dir.GenSubFil("xowa.file.make.log.csv")); + if (reset_db) { + bmk_mgr.Reset(); + Init_reset(conn); + } + bmk_mgr.Load(wiki.Appe(), this); + + Cmd_bgn_end(); + } + protected abstract void Cmd_bgn_end(); + public abstract byte Init_redirect(); + public abstract int[] Init_ns_ary(); + protected abstract void Init_reset(Db_conn p); + protected abstract Db_conn Init_db_file(); + private long time_bgn; + public void Cmd_run() {Exec_ns_ary();} + private void Exec_ns_ary() { + if (pg_bgn == Int_.Max_value) return; + if (load_tmpls) Xob_dump_mgr_base_.Load_all_tmpls(usr_dlg, wiki, page_src); + time_bgn = System_.Ticks(); + Xob_dump_bmk dump_bmk = new Xob_dump_bmk(); + rate_mgr.Init(); + int ns_ary_len = ns_ary.length; + for (int i = 0; i < ns_ary_len; i++) { + int ns_id = ns_ary[i]; + if (ns_bgn != -1) { // ns_bgn set + if (ns_id == ns_bgn) // ns_id is ns_bgn; null out ns_bgn and continue + ns_bgn = -1; + else // ns_id is not ns_bgn; keep looking + continue; + } + dump_bmk.Ns_id_(ns_id); + Exec_db_ary(i, dump_bmk, ns_id); + if (ns_id == ns_end) exit_now = true; // ns_end set; exit + if (exit_now) break; // exit_now b/c of pg_bgn, db_bgn or something else + } + Exec_commit(dump_bmk.Ns_id(), dump_bmk.Db_id(), dump_bmk.Pg_id(), Bry_.Empty); + } + private void Exec_db_ary(int ns_ord, Xob_dump_bmk dump_bmk, int ns_id) { + int db_ary_len = db_ary.length; + for (int i = 0; i < db_ary_len; i++) { + int db_id = db_ary[i].Id(); + if (db_bgn != -1) { // db_bgn set + if (db_id == db_bgn) // db_id is db_bgn; null out db_bgn and continue + db_bgn = -1; + else // db_id is not db_bgn; keep looking + continue; + } + dump_bmk.Db_id_(db_id); + Exec_db_itm(dump_bmk, ns_ord, ns_id, db_id); + if (db_id == db_end) exit_now = true; // db_end set; exit; + if (exit_now) return; // exit_now b/c of pg_bgn, db_bgn or something else + } + } + private void Exec_db_itm(Xob_dump_bmk dump_bmk, int ns_ord, int ns_id, int db_id) { + List_adp pages = List_adp_.New(); + Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id); + int pg_id = pg_bgn; + while (true) { + page_src.Get_pages(wiki.Appe(), pages, db_id, ns_id, pg_id); + int pages_len = pages.Count(); + if (pages_len == 0) { // no more pages in db; + if (pg_id > pg_bgn) // reset pg_bgn to 0 only if pg_bgn seen; + pg_bgn = 0; + return; + } + usr_dlg.Prog_many("", "", "fetched pages: ~{0}", pages_len); + for (int i = 0; i < pages_len; i++) { + Xowd_page_itm page = (Xowd_page_itm)pages.Get_at(i); + dump_bmk.Pg_id_(pg_id); + Exec_pg_itm(ns_ord, ns, db_id, page); + if ( pg_id >= pg_end + || exec_count >= exec_count_max) { + exit_now = true; + } + if (exit_now) return; + pg_id = page.Id(); + } + } + } + private void Exec_pg_itm(int ns_ord, Xow_ns ns, int db_id, Xowd_page_itm page) { + try { + if ((exec_count % progress_interval) == 0) + usr_dlg.Prog_many("", "", "parsing: ns=~{0} db=~{1} pg=~{2} count=~{3} time=~{4} rate=~{5} ttl=~{6}" + , ns.Id(), db_id, page.Id(), exec_count + , System_.Ticks__elapsed_in_sec(time_bgn), rate_mgr.Rate_as_str(), String_.new_u8(page.Ttl_page_db())); + ctx.Clear_all(); + byte[] page_src = page.Text(); + if (page_src != null) // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05 + Exec_pg_itm_hook(ns_ord, ns, page, page_src); + ctx.Wiki().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released + ++exec_count; + rate_mgr.Increment(); + if ((exec_count % poll_interval) == 0) + poll_mgr.Poll(); + if ((exec_count % commit_interval) == 0) + Exec_commit(ns.Id(), db_id, page.Id(), page.Ttl_page_db()); + if ((exec_count % cleanup_interval) == 0) + Free(); + } + catch (Exception exc) { + bldr.Usr_dlg().Warn_many("", "", "parse failed: wiki=~{0} ttl=~{1} err=~{2}", wiki.Domain_str(), page.Ttl_full_db(), Err_.Message_gplx_log(exc)); + ctx.Wiki().Utl__bfr_mkr().Clear(); + this.Free(); + } + } + public abstract void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm page, byte[] page_text); + private void Exec_commit(int ns_id, int db_id, int pg_id, byte[] ttl) { + usr_dlg.Prog_many("", "", "committing: ns=~{0} db=~{1} pg=~{2} count=~{3} ttl=~{4}", ns_id, db_id, pg_id, exec_count, String_.new_u8(ttl)); + Exec_commit_hook(); + bmk_mgr.Save(ns_id, db_id, pg_id); + if (exit_after_commit) exit_now = true; + } + public abstract void Exec_commit_hook(); + public abstract void Exec_end_hook(); + public void Cmd_init(Xob_bldr bldr) {} + public void Cmd_term() {} + public void Cmd_end() { + if (!exit_now) + pg_bgn = Int_.Max_value; + Exec_commit(-1, -1, -1, Bry_.Empty); + Exec_end_hook(); + Free(); + usr_dlg.Note_many("", "", "done: ~{0} ~{1}", exec_count, Decimal_adp_.divide_safe_(exec_count, System_.Ticks__elapsed_in_sec(time_bgn)).To_str("#,###.000")); + } + private void Free() { + Xowe_wiki_.Rls_mem(wiki, true); + } + protected void Reset_db_y_() {this.reset_db = true;} + @Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { + if (ctx.Match(k, Invk_commit_interval_)) commit_interval = m.ReadInt("v"); + else if (ctx.Match(k, Invk_progress_interval_)) progress_interval = m.ReadInt("v"); + else if (ctx.Match(k, Invk_rate_interval_)) rate_mgr.Reset_interval_(m.ReadInt("v")); + else if (ctx.Match(k, Invk_cleanup_interval_)) cleanup_interval = m.ReadInt("v"); + else if (ctx.Match(k, Invk_select_size_)) select_size = m.ReadInt("v") * Io_mgr.Len_mb; + else if (ctx.Match(k, Invk_ns_bgn_)) {ns_bgn = m.ReadInt("v"); Notify_restoring("ns", ns_bgn);} + else if (ctx.Match(k, Invk_db_bgn_)) {db_bgn = m.ReadInt("v"); Notify_restoring("db", db_bgn);} + else if (ctx.Match(k, Invk_pg_bgn_)) {pg_bgn = m.ReadInt("v"); Notify_restoring("pg", pg_bgn);} + else if (ctx.Match(k, Invk_ns_end_)) ns_end = m.ReadInt("v"); + else if (ctx.Match(k, Invk_db_end_)) db_end = m.ReadInt("v"); + else if (ctx.Match(k, Invk_pg_end_)) pg_end = m.ReadInt("v"); + else if (ctx.Match(k, Invk_load_tmpls_)) load_tmpls = m.ReadYn("v"); + else if (ctx.Match(k, Invk_poll_mgr)) return poll_mgr; + else if (ctx.Match(k, Invk_reset_db_)) reset_db = m.ReadYn("v"); + else if (ctx.Match(k, Invk_exec_count_max_)) exec_count_max = m.ReadInt("v"); + else if (ctx.Match(k, Invk_exit_now_)) exit_now = m.ReadYn("v"); + else if (ctx.Match(k, Invk_exit_after_commit_)) exit_after_commit = m.ReadYn("v"); + else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v")); + else return Gfo_invk_.Rv_unhandled; + return this; + } + private void Notify_restoring(String itm, int val) { + usr_dlg.Note_many("", "", "restoring: itm=~{0} val=~{1}", itm, val); + } + public static final String + Invk_progress_interval_ = "progress_interval_", Invk_commit_interval_ = "commit_interval_", Invk_cleanup_interval_ = "cleanup_interval_", Invk_rate_interval_ = "rate_interval_" + , Invk_select_size_ = "select_size_" + , Invk_ns_bgn_ = "ns_bgn_", Invk_db_bgn_ = "db_bgn_", Invk_pg_bgn_ = "pg_bgn_" + , Invk_ns_end_ = "ns_end_", Invk_db_end_ = "db_end_", Invk_pg_end_ = "pg_end_" + , Invk_load_tmpls_ = "load_tmpls_" + , Invk_poll_mgr = "poll_mgr", Invk_reset_db_ = "reset_db_" + , Invk_exec_count_max_ = "exec_count_max_", Invk_exit_now_ = "exit_now_", Invk_exit_after_commit_ = "exit_after_commit_" + , Invk__manual_now_ = "manual_now_" + ; +} +class Xob_dump_mgr_base_ { + public static void Load_all_tmpls(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, Xob_dump_src_id page_src) { + List_adp pages = List_adp_.New(); + Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template(); + Xow_defn_cache defn_cache = wiki.Cache_mgr().Defn_cache(); + int cur_page_id = -1; + int load_count = 0; + usr_dlg.Note_many("", "", "tmpl_load init"); + while (true) { + page_src.Get_pages(wiki.Appe(), pages, 0, Xow_ns_.Tid__template, cur_page_id); // 0 is always template db + int page_count = pages.Count(); + if (page_count == 0) break; // no more pages in db; + Xowd_page_itm page = null; + for (int i = 0; i < page_count; i++) { + page = (Xowd_page_itm)pages.Get_at(i); + Xot_defn_tmpl defn = new Xot_defn_tmpl(); + defn.Init_by_new(ns_tmpl, ns_tmpl.Gen_ttl(page.Ttl_page_db()), page.Text(), null, false); // NOTE: passing null, false; will be overriden later when Parse is called + defn_cache.Add(defn, ns_tmpl.Case_match()); + ++load_count; + if ((load_count % 10000) == 0) usr_dlg.Prog_many("", "", "tmpl_loading: ~{0}", load_count); + } + cur_page_id = page.Id(); + } + usr_dlg.Note_many("", "", "tmpl_load done: ~{0}", load_count); + } + public static Xow_db_file[] Init_text_files_ary(Xow_db_mgr core_data_mgr) { + List_adp text_files_list = List_adp_.New(); + int len = core_data_mgr.Dbs__len(); + if (len == 1) return new Xow_db_file[] {core_data_mgr.Dbs__get_at(0)}; // single file: return core; note that there are no Tid = Text + for (int i = 0; i < len; i++) { + Xow_db_file file = core_data_mgr.Dbs__get_at(i); + switch (file.Tid()) { + case Xow_db_file_.Tid__text: + case Xow_db_file_.Tid__text_solo: + text_files_list.Add(file); + break; + } + } + return (Xow_db_file[])text_files_list.To_ary_and_clear(Xow_db_file.class); + } +} +class Xob_dump_bmk_mgr { + private Bry_bfr save_bfr = Bry_bfr_.Reset(1024); + public Io_url Cfg_url() {return cfg_url;} public Xob_dump_bmk_mgr Cfg_url_(Io_url v) {cfg_url = v; return this;} private Io_url cfg_url; + public void Reset() {Io_mgr.Instance.DeleteFil(cfg_url);} + public void Load(Xoae_app app, Xob_dump_mgr_base dump_mgr) { + app.Gfs_mgr().Run_url_for(dump_mgr, cfg_url); + } + public void Save(int ns_id, int db_id, int pg_id) { + Save_itm(save_bfr, Xob_dump_mgr_base.Invk_ns_bgn_, ns_id); + Save_itm(save_bfr, Xob_dump_mgr_base.Invk_db_bgn_, db_id); + Save_itm(save_bfr, Xob_dump_mgr_base.Invk_pg_bgn_, pg_id); + Io_mgr.Instance.SaveFilBfr(cfg_url, save_bfr); + } + private void Save_itm(Bry_bfr save_bfr, String key, int val) { + String fmt = "{0}('{1}');\n"; + String str = String_.Format(fmt, key, val); + save_bfr.Add_str_u8(str); + } +} +class Xob_rate_mgr { + private long time_bgn; + private int item_len; + private Bry_bfr save_bfr = Bry_bfr_.Reset(255); + public int Reset_interval() {return reset_interval;} public Xob_rate_mgr Reset_interval_(int v) {reset_interval = v; return this;} private int reset_interval = 10000; + public Io_url Log_file_url() {return log_file;} public Xob_rate_mgr Log_file_(Io_url v) {log_file = v; return this;} private Io_url log_file; + public void Init() {time_bgn = System_.Ticks();} + public void Increment() { + ++item_len; + if (item_len % reset_interval == 0) { + long time_end = System_.Ticks(); + Save(item_len, time_bgn, time_end); + time_bgn = time_end; + item_len = 0; + } + } + private void Save(int count, long bgn, long end) { + int dif = (int)(end - bgn) / 1000; + Decimal_adp rate = Decimal_adp_.divide_safe_(count, dif); + save_bfr + .Add_str_a7(rate.To_str("#,##0.000")).Add_byte_pipe() + .Add_int_variable(count).Add_byte_pipe() + .Add_int_variable(dif).Add_byte_nl() + ; + Io_mgr.Instance.AppendFilByt(log_file, save_bfr.To_bry_and_clear()); + } + public String Rate_as_str() {return Int_.To_str(Rate());} + public int Rate() { + int elapsed = System_.Ticks__elapsed_in_sec(time_bgn); + return Math_.Div_safe_as_int(item_len, elapsed); + } +} +class Xob_dump_bmk { + public int Ns_id() {return ns_id;} public Xob_dump_bmk Ns_id_(int v) {ns_id = v; return this;} private int ns_id; + public int Db_id() {return db_id;} public Xob_dump_bmk Db_id_(int v) {db_id = v; return this;} private int db_id; + public int Pg_id() {return pg_id;} public Xob_dump_bmk Pg_id_(int v) {pg_id = v; return this;} private int pg_id; +} diff --git a/400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn.java b/400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn.java index 36280fbb5..bffb95b21 100644 --- a/400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn.java +++ b/400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn.java @@ -1,6 +1,6 @@ /* XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012-2017 gnosygnu@gmail.com +Copyright (C) 2012-2020 gnosygnu@gmail.com XOWA is licensed under the terms of the General Public License (GPL) Version 3, or alternatively under the terms of the Apache License Version 2.0. @@ -13,437 +13,448 @@ The terms of each license can be found in the source code repository: GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ -package gplx.xowa.parsers.tmpls; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*; -import gplx.core.envs.*; -import gplx.xowa.langs.*; import gplx.xowa.langs.kwds.*; import gplx.xowa.langs.funcs.*; -import gplx.xowa.xtns.pfuncs.*; import gplx.xowa.xtns.pfuncs.ttls.*; -import gplx.xowa.wikis.pages.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.wikis.data.tbls.*; -public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk { - public Xot_invk_tkn(int bgn, int end) {this.Tkn_ini_pos(false, bgn, end);} - @Override public byte Tkn_tid() {return typeId;} private byte typeId = Xop_tkn_itm_.Tid_tmpl_invk; - public void Tkn_tid_to_txt() {typeId = Xop_tkn_itm_.Tid_txt;} - public Arg_nde_tkn Name_tkn() {return name_tkn;} public Xot_invk_tkn Name_tkn_(Arg_nde_tkn v) {name_tkn = v; return this;} Arg_nde_tkn name_tkn = Arg_nde_tkn.Null; - public byte Defn_tid() {return defn_tid;} private byte defn_tid = Xot_defn_.Tid_null; - public int Tmpl_subst_bgn() {return tmpl_subst_bgn;} private int tmpl_subst_bgn; - public int Tmpl_subst_end() {return tmpl_subst_end;} private int tmpl_subst_end; - public Xot_invk_tkn Tmpl_subst_props_(byte tid, int bgn, int end) {defn_tid = tid; tmpl_subst_bgn = bgn; tmpl_subst_end = end; return this;} - public Xot_defn Tmpl_defn() {return tmpl_defn;} public Xot_invk_tkn Tmpl_defn_(Xot_defn v) {tmpl_defn = v; return this;} private Xot_defn tmpl_defn = Xot_defn_.Null; - public boolean Frame_is_root() {return false;} - public byte Frame_tid() {return scrib_tid;} public void Frame_tid_(byte v) {scrib_tid = v;} private byte scrib_tid; - public byte[] Frame_ttl() {return frame_ttl;} public void Frame_ttl_(byte[] v) {frame_ttl = v;} private byte[] frame_ttl; - public int Frame_lifetime() {return frame_lifetime;} public void Frame_lifetime_(int v) {frame_lifetime = v;} private int frame_lifetime; - public boolean Rslt_is_redirect() {return rslt_is_redirect;} public void Rslt_is_redirect_(boolean v) {rslt_is_redirect = v;} private boolean rslt_is_redirect; - @Override public void Tmpl_fmt(Xop_ctx ctx, byte[] src, Xot_fmtr fmtr) {fmtr.Reg_tmpl(ctx, src, name_tkn, args_len, args);} - @Override public void Tmpl_compile(Xop_ctx ctx, byte[] src, Xot_compile_data prep_data) { - name_tkn.Tmpl_compile(ctx, src, prep_data); - int args_len = this.Args_len(); - for (int i = 0; i < args_len; i++) { - Arg_nde_tkn nde = args[i]; - Xop_tkn_itm key = nde.Key_tkn(); int key_subs_len = key.Subs_len(); - for (int j = 0; j < key_subs_len; j++) - key.Subs_get(j).Tmpl_compile(ctx, src, prep_data); - Xop_tkn_itm val = nde.Val_tkn(); int val_subs_len = val.Subs_len(); - for (int j = 0; j < val_subs_len; j++) - val.Subs_get(j).Tmpl_compile(ctx, src, prep_data); - } - } - @Override public boolean Tmpl_evaluate(Xop_ctx ctx, byte[] src, Xot_invk caller, Bry_bfr bfr) { // this="{{t|{{{0}}}}}" caller="{{t|1}}" - // init common - boolean rv = false; - Xowe_wiki wiki = ctx.Wiki(); - Xol_lang_itm lang = wiki.Lang(); - - // init defn / name - Xot_defn defn = tmpl_defn; - byte[] name_ary = defn.Name(); - byte[] name_ary_orig = Bry_.Empty; - int name_bgn = 0, name_ary_len = 0; - Arg_itm_tkn name_key_tkn = name_tkn.Key_tkn(); - - // init more - byte[] argx_ary = Bry_.Empty; - boolean subst_found = false; - boolean name_had_subst = false; - boolean template_prefix_found = false; - - // tmpl_name does not exist in db; may be dynamic, subst, transclusion, etc.. - if (defn == Xot_defn_.Null) { - // dynamic tmpl; EX:{{{{{1}}}|a}} - if (name_key_tkn.Itm_static() == Bool_.N_byte) { - Bry_bfr name_tkn_bfr = Bry_bfr_.New_w_size(name_tkn.Src_end() - name_tkn.Src_bgn()); - if (defn_tid == Xot_defn_.Tid_subst) - name_tkn_bfr.Add(Get_first_subst_itm(lang.Kwd_mgr())); - name_tkn.Tmpl_evaluate(ctx, src, caller, name_tkn_bfr); - name_ary = name_tkn_bfr.To_bry_and_clear(); - } - // tmpl is static; note that dat_ary is still valid but rest of name may not be; EX: {{subst:name{{{1}}}}} - else - name_ary = Bry_.Mid(src, name_key_tkn.Dat_bgn(), name_key_tkn.Dat_end()); - name_had_subst = name_key_tkn.Dat_ary_had_subst(); - name_ary_orig = name_ary; // cache name_ary_orig - name_ary_len = name_ary.length; - name_bgn = Bry_find_.Find_fwd_while_not_ws(name_ary, 0, name_ary_len); - if ( name_ary_len == 0 // name is blank; can occur with failed inner tmpl; EX: {{ {{does not exist}} }} - || name_bgn == name_ary_len // name is ws; EX: {{test| }} -> {{{{{1}}}}}is whitespace String; PAGE:en.d:wear_one's_heart_on_one's_sleeve; EX:{{t+|fr|avoir le cœur sur la main| }} - ) { - Gfo_usr_dlg_.Instance.Log_many("", "", "parser.tmpl:dynamic is blank; page=~{0}", ctx.Page().Url_bry_safe()); // downgraded from warning to note; PAGE:de.d:país DATE:2016-09-07 - return false; - } - if (name_ary[name_bgn] == Byte_ascii.Colon) { // check 1st letter for transclusion - return Transclude(ctx, wiki, bfr, Bool_.N, name_ary, caller, src); // transclusion; EX: {{:Name of page}} - } - - // ignore "{{Template:"; EX: {{Template:a}} is the same thing as {{a}} - int tmpl_ns_len = wiki.Ns_mgr().Tmpls_get_w_colon(name_ary, name_bgn, name_ary_len); - if (tmpl_ns_len != Bry_find_.Not_found) { - name_ary = Bry_.Mid(name_ary, name_bgn + tmpl_ns_len, name_ary_len); - name_ary_len = name_ary.length; - name_bgn = 0; - template_prefix_found = true; - } - byte[] ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); int ns_template_prefix_len = ns_template_prefix.length; - if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) { - name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len); - name_ary_len = name_ary.length; - name_bgn = 0; - } - - Xow_ns_mgr_name_itm ns_eval = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia - if (ns_eval != null && !template_prefix_found) // do not transclude ns if Template prefix seen earlier; EX: {{Template:Wikipedia:A}} should not transclude "Wikipedia:A"; DATE:2013-04-03 - return SubEval(ctx, wiki, bfr, name_ary, caller, src); - - Xol_func_itm finder = new Xol_func_itm(); // TS.MEM: DATE:2016-07-12 - lang.Func_regy().Find_defn(finder, name_ary, name_bgn, name_ary_len); - defn = finder.Func(); - int finder_tid = finder.Tid(); - int finder_colon_pos = finder.Colon_pos(); - int finder_subst_end = finder.Subst_end(); - - int colon_pos = -1; - switch (finder_tid) { - case Xot_defn_.Tid_subst: // subst is added verbatim; EX: {{subst:!}} -> {{subst:!}}; logic below is to handle printing of arg which could be standardized if src[] was available for tmpl - bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary); - for (int i = 0; i < args_len; i++) { - Arg_nde_tkn nde = args[i]; - bfr.Add_byte(Byte_ascii.Pipe); // add | - bfr.Add_mid(src, nde.Src_bgn(), nde.Src_end()); // add entire arg; "k=v"; note that src must be added, not evaluated, else may be dropped and cause stack overflow; PAGE:ru.w:Близкие_друзья_(Сезон_2) DATE:2014-10-21 - } - Xot_fmtr_prm.Instance.Print(bfr); - bfr.Add(Xop_curly_end_lxr.Hook); - return true; // NOTE: nothing else to do; return - case Xot_defn_.Tid_safesubst: - name_ary = Bry_.Mid(name_ary, finder_subst_end, name_ary_len); // chop off "safesubst:" - name_ary_len = name_ary.length; - if (defn != Xot_defn_.Null) { // func found - if (finder_colon_pos != -1) colon_pos = defn.Name().length; // set colon_pos; SEE NOTE_1 - } - subst_found = true; - break; - case Xot_defn_.Tid_func: - if (defn.Defn_require_colon_arg()) { - colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon); - if (colon_pos == Bry_find_.Not_found) - defn = Xot_defn_.Null; - } - else { - colon_pos = finder_colon_pos; - } - break; - case Xot_defn_.Tid_raw: - case Xot_defn_.Tid_msg: - int raw_colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon); - if (raw_colon_pos == Bry_find_.Not_found) {} // colon missing; EX: {{raw}}; noop and assume template name; DATE:2014-02-11 - else { // colon present; - name_ary = Bry_.Mid(name_ary, finder_subst_end + 1, name_ary_len); // chop off "raw"; +1 is for ":"; note that +1 is in bounds b/c raw_colon was found - name_ary_len = name_ary.length; - Xow_ns_mgr_name_itm ns_eval2 = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia - if (ns_eval2 != null) { - Xow_ns ns_eval_itm = ns_eval2.Ns(); - int pre_len = ns_eval_itm.Name_enc().length; - if (pre_len < name_ary_len && name_ary[pre_len] == Byte_ascii.Colon) - return SubEval(ctx, wiki, bfr, name_ary, caller, src); - } - } - switch (finder_tid) { - case Xot_defn_.Tid_msg: - defn = Xot_defn_.Null; // null out defn to force template load below; DATE:2014-07-10 - break; - } - break; - } - if (subst_found) {// subst found; remove Template: if it exists; EX: {{safesubst:Template:A}} -> {{A}} not {{Template:A}}; EX:en.d:Kazakhstan; DATE:2014-03-25 - ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); ns_template_prefix_len = ns_template_prefix.length; - if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) { - name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len); - name_ary_len = name_ary.length; - name_bgn = 0; - template_prefix_found = true; - } - } - if (colon_pos != -1) { // func; separate name_ary into name_ary and arg_x - argx_ary = Bry_.Trim(name_ary, colon_pos + 1, name_ary_len); // trim bgn ws; needed for "{{formatnum:\n{{#expr:2}}\n}}" - name_ary = Bry_.Mid(name_ary, 0, colon_pos); - } - if (defn == Xot_defn_.Null) { - if (ctx.Tid_is_popup()) { // popup && cur_tmpl > tmpl_max - if (Popup_skip(ctx, name_ary, bfr)) return false; - } - defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary); - if (defn == null) { - if (name_ary_len != 0 ) { // name_ary_len != 0 for direct template inclusions; PAGE:en.w:Human evolution and {{:Human evolution/Species chart}}; && ctx.Tmpl_whitelist().Has(name_ary) - Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary)); - if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args ); - if (subst_found || template_prefix_found) { // if "subst:" or "Template:" found, use orig name; DATE:2014-03-31 - bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary_orig).Add(Xop_curly_end_lxr.Hook); - return false; - } - else {// output entire tmpl_src WITH args; used to output name only which broke pages; PAGE:en.w:Flag_of_Greenland; DATE:2016-06-21 - bfr.Add(Xop_curly_bgn_lxr.Hook); - bfr.Add(name_ary); - for (int i = 0; i < args_len; ++i) { - Arg_nde_tkn nde = this.Args_get_by_idx(i); - bfr.Add_byte(Byte_ascii.Pipe); - // must evaluate args; "size={{{size|}}}" must become "size="; PAGE:en.w:Europe; en.w:Template:Country_data_Guernsey DATE:2016-10-13 - nde.Tmpl_compile(ctx, src, Xot_compile_data.Noop); - nde.Tmpl_evaluate(ctx, src, caller, bfr); - } - bfr.Add(Xop_curly_end_lxr.Hook); - return false; - } - } - else { // some templates produce null ttls; EX: "Citation needed{{subst" - defn = wiki.Cache_mgr().Defn_cache().Get_by_key(ttl.Page_db()); - if (defn == null && ctx.Tmpl_load_enabled()) - defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary); - } - } - } - if (defn == null) defn = Xot_defn_.Null; - } - } - if ( defn.Defn_tid() == Xot_defn_.Tid_null // name is not a known defn - && lang.Vnt_mgr().Enabled()) { // lang has vnts - Xowd_page_itm page = lang.Vnt_mgr().Convert_mgr().Convert_ttl(wiki, wiki.Ns_mgr().Ns_template(), name_ary); - if (page != Xowd_page_itm.Null) { - name_ary = page.Ttl_page_db(); - Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary)); - if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args ); - bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary).Add(Xop_curly_end_lxr.Hook); - return false; - } - defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary); - if (defn == null && ctx.Tmpl_load_enabled()) - defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary); - if (defn == null) defn = Xot_defn_.Null; - } - } - Xot_defn_trace trace = ctx.Defn_trace(); int trg_bgn = bfr.Len(); - switch (defn.Defn_tid()) { - case Xot_defn_.Tid_null: // defn is unknown - if (ignore_hash.Get_by_bry(name_ary) == null) { - if (Pfunc_rel2abs.Rel2abs_ttl(name_ary, name_bgn, name_ary_len)) {// rel_path; EX: {{/../Peer page}}; DATE:2013-03-27 - Bry_bfr tmp_bfr = ctx.Wiki().Utl__bfr_mkr().Get_b512(); - name_ary = Pfunc_rel2abs.Rel2abs(tmp_bfr, wiki.Parser_mgr().Rel2abs_ary(), Bry_.Mid(name_ary, name_bgn, name_ary_len), ctx.Page().Ttl().Raw()); - tmp_bfr.Mkr_rls(); - return SubEval(ctx, wiki, bfr, name_ary, caller, src); - } - if (subst_found) - return Transclude(ctx, wiki, bfr, template_prefix_found, name_ary, caller, src); - Xot_invk_tkn_.Print_not_found__w_template(bfr, wiki.Ns_mgr(), name_ary); - return false; - } - break; - case Xot_defn_.Tid_func: - try { - Xot_invk_tkn_.Eval_func(ctx, src, caller, this, bfr, defn, argx_ary); - rv = true; - } catch (Exception e) { - if (Env_.Mode_testing()) - throw Err_.new_exc(e, "xo", "failed to evaluate function", "page", ctx.Page().Ttl().Full_txt(), "defn", defn.Name(), "src", String_.new_u8(src, this.Src_bgn(), this.Src_end())); - else { - wiki.Appe().Usr_dlg().Warn_many("", "", "failed to evaluate function: page=~{0} defn=~{1} src=~{2} err=~{3}", ctx.Page().Ttl().Full_txt(), defn.Name(), Bry_.Replace_nl_w_tab(src, this.Src_bgn(), this.Src_end()), Err_.Message_gplx_log(e)); - rv = false; - } - } - break; - default: - Xot_defn_tmpl defn_tmpl = (Xot_defn_tmpl)defn; - if (defn_tmpl.Root() == null) defn_tmpl.Parse_tmpl(ctx); // NOTE: defn_tmpl.Root can be null after clearing out cache; must be non-null else will fail in trace; DATE:2013-02-01 - Xot_invk invk_tmpl = Xot_defn_tmpl_.CopyNew(ctx, defn_tmpl, this, caller, src, Xow_ns_.Tid__template, name_ary); - invk_tmpl.Frame_ttl_(defn_tmpl.Frame_ttl()); // set frame_ttl; needed for redirects; PAGE:en.w:Statutory_city; DATE:2014-08-22 - trace.Trace_bgn(ctx, src, name_ary, caller, invk_tmpl, defn); - - Bry_bfr rslt_bfr = wiki.Utl__bfr_mkr().Get_k004(); - try { - Xot_invk_tkn_.Bld_key(invk_tmpl, name_ary, rslt_bfr); - byte[] rslt_key = rslt_bfr.To_bry_and_clear(); - Object o = wiki.Cache_mgr().Tmpl_result_cache().Get_by(rslt_key); - Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(bfr); - if (o != null) { - byte[] rslt = (byte[])o; - prepend_mgr.End(ctx, bfr, rslt, rslt.length, Bool_.Y); - bfr.Add(rslt); - } - else { - rv = defn_tmpl.Tmpl_evaluate(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), invk_tmpl, rslt_bfr); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17 - prepend_mgr.End(ctx, bfr, rslt_bfr.Bfr(), rslt_bfr.Len(), Bool_.Y); - if (name_had_subst) { // current invk had "subst:"; parse incoming invk again to remove effects of subst; PAGE:pt.w:Argentina DATE:2014-09-24 - byte[] tmp_src = rslt_bfr.To_bry_and_clear(); - rslt_bfr.Add(wiki.Parser_mgr().Main().Expand_tmpl(tmp_src)); // this could be cleaner / more optimized - } - if (Cache_enabled) { - byte[] rslt_val = rslt_bfr.To_bry_and_clear(); - bfr.Add(rslt_val); - Hash_adp cache = wiki.Cache_mgr().Tmpl_result_cache(); - cache.Del(rslt_key); - cache.Add(rslt_key, rslt_val); - } - else - bfr.Add_bfr_and_clear(rslt_bfr); - } - trace.Trace_end(trg_bgn, bfr); - } finally {rslt_bfr.Mkr_rls();} - break; - } - return rv; - } - private boolean Popup_skip(Xop_ctx ctx, byte[] ttl, Bry_bfr bfr) { - boolean skip = false; - skip = this.Src_end() - this.Src_bgn() > ctx.Tmpl_tkn_max(); - if (!skip) { - gplx.xowa.htmls.modules.popups.keeplists.Xop_keeplist_wiki tmpl_keeplist = ctx.Tmpl_keeplist(); - if (tmpl_keeplist != null && tmpl_keeplist.Enabled()) { - byte[] ttl_lower = Xoa_ttl.Replace_spaces(ctx.Wiki().Lang().Case_mgr().Case_build_lower(ttl)); - skip = !tmpl_keeplist.Match(ttl_lower); - // if (skip) Tfds.Write_bry(ttl_lower); - } - } - if (skip) { - bfr.Add(gplx.xowa.parsers.miscs.Xop_comm_lxr.Xowa_skip_comment_bry); // add comment tkn; need something to separate ''{{lang|la|Ragusa}}'' else will become ''''; PAGE:en.w:Republic_of_Ragusa; DATE:2014-06-28 - return true; - } - else - return false; - } - private boolean Transclude(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, boolean template_prefix_found, byte[] name_ary, Xot_invk caller, byte[] src) { - Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}} - byte[] transclude_src = null; - if (page_ttl.Ns().Id_is_tmpl()) { // ttl is template; check tmpl_regy first before going to data_mgr - Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db()); - if (tmpl != null) transclude_src = tmpl.Data_raw(); - } - if (transclude_src == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load - Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl); - if ( cache_itm != null -// && Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Page_ttl().Full_db()) // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10 - ) { - transclude_src = cache_itm.Wtxt__direct(); - page_ttl = cache_itm.Ttl(); - } - } - if (transclude_src != null) { - // NOTE: must use new page, not current, else transcluded page can cause inconsistent TOC state; PAGE:de.w:Game_of_Thrones DATE:2016-11-21 - Xot_defn_tmpl transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, Xoae_page.New(wiki, page_ttl)), ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), transclude_src); - return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src, bfr); - } - else { - Xot_invk_tkn_.Print_not_found__by_transclude(bfr, wiki.Ns_mgr(), template_prefix_found, name_ary); - return false; - } - } - private boolean Eval_sub(Xop_ctx ctx, Xoa_ttl transclude_ttl, Xot_defn_tmpl transclude_tmpl, Xot_invk caller, byte[] src, Bry_bfr doc) { - boolean rv = false; - Xot_invk tmp_tmpl = Xot_defn_tmpl_.CopyNew(ctx, transclude_tmpl, this, caller, src, transclude_ttl.Ns().Id(), transclude_tmpl.Name()); - Bry_bfr tmp_bfr = Bry_bfr_.New(); - Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(doc); - rv = transclude_tmpl.Tmpl_evaluate(ctx, tmp_tmpl, tmp_bfr); - prepend_mgr.End(ctx, doc, tmp_bfr.Bfr(), tmp_bfr.Len(), Bool_.Y); - doc.Add_bfr_and_clear(tmp_bfr); - return rv; - } - private boolean SubEval(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, byte[] name_ary, Xot_invk caller, byte[] src_for_tkn) { - Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}} - Xot_defn_tmpl transclude_tmpl = null; - switch (page_ttl.Ns().Id()) { - case Xow_ns_.Tid__template: // ttl is template not in cache, or some other ns; do load - Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db()); - if (tmpl != null) { - if (tmpl.Root() == null) tmpl.Parse_tmpl(ctx); - transclude_tmpl = tmpl; - } - break; - case Xow_ns_.Tid__special: - bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon).Add(name_ary).Add(Xop_tkn_.Lnki_end); - return true; - } - if (transclude_tmpl == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load - Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl); - if ( cache_itm != null) { - if (!Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Ttl().Full_db())) { // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10 - transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), cache_itm.Wtxt__direct()); - page_ttl = cache_itm.Ttl(); - } - } - } - if (transclude_tmpl == null) { - bfr.Add(Xop_tkn_.Lnki_bgn).Add(name_ary).Add(Xop_tkn_.Lnki_end); // indicate template was not found; DATE:2014-02-12 - return false; - } - return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src_for_tkn, bfr); - } - public int Args_len() {return args_len;} private int args_len = 0; - public Arg_nde_tkn Args_get_by_idx(int idx) {return args[idx];} - public Arg_nde_tkn Args_eval_by_idx(byte[] src, int idx) { - int cur = 0; - for (int i = 0; i < args_len; i++) { - Arg_nde_tkn nde = args[i]; - if (nde.KeyTkn_exists()) continue; - if (cur++ == idx) return nde; - } - return null; - } - public Arg_nde_tkn Args_get_by_key(byte[] src, byte[] key) { - for (int i = 0; i < args_len; i++) { - Arg_nde_tkn nde = args[i]; - if (!nde.KeyTkn_exists()) continue; - if (Bry_.Match(src, nde.Key_tkn().Dat_bgn(), nde.Key_tkn().Dat_end(), key)) return nde; // NOTE: dat_ary is guaranteed to exist - } - return null; - } - public void Args_add(Xop_ctx ctx, Arg_nde_tkn arg) { - int newLen = args_len + 1; - if (newLen > args_max) { - args_max = newLen * 2; - args = Resize(args, args_len, args_max); - } - args[args_len] = arg; - arg.Tkn_grp_(this, args_len); - args_len = newLen; - } Arg_nde_tkn[] args = Arg_nde_tkn.Ary_empty; int args_max; - private Arg_nde_tkn[] Resize(Arg_nde_tkn[] src, int cur_len, int new_len) { - Arg_nde_tkn[] rv = new Arg_nde_tkn[new_len]; - Array_.Copy_to(src, 0, rv, 0, cur_len); - return rv; - } - private byte[] Get_first_subst_itm(Xol_kwd_mgr kwd_mgr) { - Xol_kwd_grp grp = kwd_mgr.Get_at(Xol_kwd_grp_.Id_subst); if (grp == null) return Bry_.Empty; - Xol_kwd_itm[] itms = grp.Itms(); - return itms.length == 0 ? Bry_.Empty : itms[0].Val(); - } - private static final Hash_adp_bry ignore_hash = Hash_adp_bry.ci_a7().Add_str_obj("Citation needed{{subst", "").Add_str_obj("Clarify{{subst", ""); // ignore SafeSubst templates - public static boolean Cache_enabled = false; -} -/* -NOTE_1: if (finder.Colon_pos() != -1) colon_pos = finder.Func().Name().length; -Two issues here: -1) "if (finder.Colon_pos() != -1)" - Colon_pos can either be -1 or >0 - EX: -1: safesubst:NAMESPACE - EX: >0: safesubst:#expr:0 - if -1, don't do anything; this will leave colon_pos as -1 -2) "finder.Func().Name().length" - Colon_pos is >0, but refers to String before it was chopped - EX: "safesubst:#expr:0" has Colon_pos of 15 but String is now "#expr:0" - so, get colon_pos by taking the finder.Func().Name().length - NOTE: whitespace does not matter b/c "safesubst: #expr:0" would never be a func; +package gplx.xowa.parsers.tmpls; + +import gplx.Array_; +import gplx.Bool_; +import gplx.Bry_; +import gplx.Bry_bfr; +import gplx.Bry_bfr_; +import gplx.Bry_find_; +import gplx.Byte_ascii; +import gplx.Err_; +import gplx.Gfo_usr_dlg_; +import gplx.Hash_adp_bry; +import gplx.String_; +import gplx.core.envs.Env_; +import gplx.xowa.Xoa_ttl; +import gplx.xowa.Xoae_page; +import gplx.xowa.Xowe_wiki; +import gplx.xowa.langs.Xol_lang_itm; +import gplx.xowa.langs.funcs.Xol_func_itm; +import gplx.xowa.langs.kwds.Xol_kwd_grp; +import gplx.xowa.langs.kwds.Xol_kwd_grp_; +import gplx.xowa.langs.kwds.Xol_kwd_itm; +import gplx.xowa.langs.kwds.Xol_kwd_mgr; +import gplx.xowa.parsers.Xop_ctx; +import gplx.xowa.parsers.Xop_tkn_itm; +import gplx.xowa.parsers.Xop_tkn_itm_; +import gplx.xowa.parsers.Xop_tkn_itm_base; +import gplx.xowa.wikis.caches.Xow_page_cache_itm; +import gplx.xowa.wikis.data.tbls.Xowd_page_itm; +import gplx.xowa.wikis.nss.Xow_ns; +import gplx.xowa.wikis.nss.Xow_ns_; +import gplx.xowa.wikis.nss.Xow_ns_mgr_name_itm; +import gplx.xowa.wikis.pages.Xopg_tmpl_prepend_mgr; +import gplx.xowa.xtns.pfuncs.ttls.Pfunc_rel2abs; + +public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk { + public Xot_invk_tkn(int bgn, int end) {this.Tkn_ini_pos(false, bgn, end);} + @Override public byte Tkn_tid() {return typeId;} private byte typeId = Xop_tkn_itm_.Tid_tmpl_invk; + public void Tkn_tid_to_txt() {typeId = Xop_tkn_itm_.Tid_txt;} + public Arg_nde_tkn Name_tkn() {return name_tkn;} public Xot_invk_tkn Name_tkn_(Arg_nde_tkn v) {name_tkn = v; return this;} Arg_nde_tkn name_tkn = Arg_nde_tkn.Null; + public byte Defn_tid() {return defn_tid;} private byte defn_tid = Xot_defn_.Tid_null; + public int Tmpl_subst_bgn() {return tmpl_subst_bgn;} private int tmpl_subst_bgn; + public int Tmpl_subst_end() {return tmpl_subst_end;} private int tmpl_subst_end; + public Xot_invk_tkn Tmpl_subst_props_(byte tid, int bgn, int end) {defn_tid = tid; tmpl_subst_bgn = bgn; tmpl_subst_end = end; return this;} + public Xot_defn Tmpl_defn() {return tmpl_defn;} public Xot_invk_tkn Tmpl_defn_(Xot_defn v) {tmpl_defn = v; return this;} private Xot_defn tmpl_defn = Xot_defn_.Null; + public boolean Frame_is_root() {return false;} + public byte Frame_tid() {return scrib_tid;} public void Frame_tid_(byte v) {scrib_tid = v;} private byte scrib_tid; + public byte[] Frame_ttl() {return frame_ttl;} public void Frame_ttl_(byte[] v) {frame_ttl = v;} private byte[] frame_ttl; + public int Frame_lifetime() {return frame_lifetime;} public void Frame_lifetime_(int v) {frame_lifetime = v;} private int frame_lifetime; + public boolean Rslt_is_redirect() {return rslt_is_redirect;} public void Rslt_is_redirect_(boolean v) {rslt_is_redirect = v;} private boolean rslt_is_redirect; + @Override public void Tmpl_fmt(Xop_ctx ctx, byte[] src, Xot_fmtr fmtr) {fmtr.Reg_tmpl(ctx, src, name_tkn, args_len, args);} + @Override public void Tmpl_compile(Xop_ctx ctx, byte[] src, Xot_compile_data prep_data) { + name_tkn.Tmpl_compile(ctx, src, prep_data); + int args_len = this.Args_len(); + for (int i = 0; i < args_len; i++) { + Arg_nde_tkn nde = args[i]; + Xop_tkn_itm key = nde.Key_tkn(); int key_subs_len = key.Subs_len(); + for (int j = 0; j < key_subs_len; j++) + key.Subs_get(j).Tmpl_compile(ctx, src, prep_data); + Xop_tkn_itm val = nde.Val_tkn(); int val_subs_len = val.Subs_len(); + for (int j = 0; j < val_subs_len; j++) + val.Subs_get(j).Tmpl_compile(ctx, src, prep_data); + } + } + @Override public boolean Tmpl_evaluate(Xop_ctx ctx, byte[] src, Xot_invk caller, Bry_bfr bfr) { // this="{{t|{{{0}}}}}" caller="{{t|1}}" + // init common + boolean rv = false; + Xowe_wiki wiki = ctx.Wiki(); + Xol_lang_itm lang = wiki.Lang(); + + // init defn / name + Xot_defn defn = tmpl_defn; + byte[] name_ary = defn.Name(); + byte[] name_ary_orig = Bry_.Empty; + int name_bgn = 0, name_ary_len = 0; + Arg_itm_tkn name_key_tkn = name_tkn.Key_tkn(); + + // init more + byte[] argx_ary = Bry_.Empty; + boolean subst_found = false; + boolean name_had_subst = false; + boolean template_prefix_found = false; + + // tmpl_name does not exist in db; may be dynamic, subst, transclusion, etc.. + if (defn == Xot_defn_.Null) { + // dynamic tmpl; EX:{{{{{1}}}|a}} + if (name_key_tkn.Itm_static() == Bool_.N_byte) { + Bry_bfr name_tkn_bfr = Bry_bfr_.New_w_size(name_tkn.Src_end() - name_tkn.Src_bgn()); + if (defn_tid == Xot_defn_.Tid_subst) + name_tkn_bfr.Add(Get_first_subst_itm(lang.Kwd_mgr())); + name_tkn.Tmpl_evaluate(ctx, src, caller, name_tkn_bfr); + name_ary = name_tkn_bfr.To_bry_and_clear(); + } + // tmpl is static; note that dat_ary is still valid but rest of name may not be; EX: {{subst:name{{{1}}}}} + else + name_ary = Bry_.Mid(src, name_key_tkn.Dat_bgn(), name_key_tkn.Dat_end()); + name_had_subst = name_key_tkn.Dat_ary_had_subst(); + name_ary_orig = name_ary; // cache name_ary_orig + name_ary_len = name_ary.length; + name_bgn = Bry_find_.Find_fwd_while_not_ws(name_ary, 0, name_ary_len); + if ( name_ary_len == 0 // name is blank; can occur with failed inner tmpl; EX: {{ {{does not exist}} }} + || name_bgn == name_ary_len // name is ws; EX: {{test| }} -> {{{{{1}}}}}is whitespace String; PAGE:en.d:wear_one's_heart_on_one's_sleeve; EX:{{t+|fr|avoir le cœur sur la main| }} + ) { + Gfo_usr_dlg_.Instance.Log_many("", "", "parser.tmpl:dynamic is blank; page=~{0}", ctx.Page().Url_bry_safe()); // downgraded from warning to note; PAGE:de.d:país DATE:2016-09-07 + return false; + } + if (name_ary[name_bgn] == Byte_ascii.Colon) { // check 1st letter for transclusion + return Transclude(ctx, wiki, bfr, Bool_.N, name_ary, caller, src); // transclusion; EX: {{:Name of page}} + } + + // ignore "{{Template:"; EX: {{Template:a}} is the same thing as {{a}} + int tmpl_ns_len = wiki.Ns_mgr().Tmpls_get_w_colon(name_ary, name_bgn, name_ary_len); + if (tmpl_ns_len != Bry_find_.Not_found) { + name_ary = Bry_.Mid(name_ary, name_bgn + tmpl_ns_len, name_ary_len); + name_ary_len = name_ary.length; + name_bgn = 0; + template_prefix_found = true; + } + byte[] ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); int ns_template_prefix_len = ns_template_prefix.length; + if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) { + name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len); + name_ary_len = name_ary.length; + name_bgn = 0; + } + + Xow_ns_mgr_name_itm ns_eval = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia + if (ns_eval != null && !template_prefix_found) // do not transclude ns if Template prefix seen earlier; EX: {{Template:Wikipedia:A}} should not transclude "Wikipedia:A"; DATE:2013-04-03 + return SubEval(ctx, wiki, bfr, name_ary, caller, src); + + Xol_func_itm finder = new Xol_func_itm(); // TS.MEM: DATE:2016-07-12 + lang.Func_regy().Find_defn(finder, name_ary, name_bgn, name_ary_len); + defn = finder.Func(); + int finder_tid = finder.Tid(); + int finder_colon_pos = finder.Colon_pos(); + int finder_subst_end = finder.Subst_end(); + + int colon_pos = -1; + switch (finder_tid) { + case Xot_defn_.Tid_subst: // subst is added verbatim; EX: {{subst:!}} -> {{subst:!}}; logic below is to handle printing of arg which could be standardized if src[] was available for tmpl + bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary); + for (int i = 0; i < args_len; i++) { + Arg_nde_tkn nde = args[i]; + bfr.Add_byte(Byte_ascii.Pipe); // add | + bfr.Add_mid(src, nde.Src_bgn(), nde.Src_end()); // add entire arg; "k=v"; note that src must be added, not evaluated, else may be dropped and cause stack overflow; PAGE:ru.w:Близкие_друзья_(Сезон_2) DATE:2014-10-21 + } + Xot_fmtr_prm.Instance.Print(bfr); + bfr.Add(Xop_curly_end_lxr.Hook); + return true; // NOTE: nothing else to do; return + case Xot_defn_.Tid_safesubst: + name_ary = Bry_.Mid(name_ary, finder_subst_end, name_ary_len); // chop off "safesubst:" + name_ary_len = name_ary.length; + if (defn != Xot_defn_.Null) { // func found + if (finder_colon_pos != -1) colon_pos = defn.Name().length; // set colon_pos; SEE NOTE_1 + } + subst_found = true; + break; + case Xot_defn_.Tid_func: + if (defn.Defn_require_colon_arg()) { + colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon); + if (colon_pos == Bry_find_.Not_found) + defn = Xot_defn_.Null; + } + else { + colon_pos = finder_colon_pos; + } + break; + case Xot_defn_.Tid_raw: + case Xot_defn_.Tid_msg: + int raw_colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon); + if (raw_colon_pos == Bry_find_.Not_found) {} // colon missing; EX: {{raw}}; noop and assume template name; DATE:2014-02-11 + else { // colon present; + name_ary = Bry_.Mid(name_ary, finder_subst_end + 1, name_ary_len); // chop off "raw"; +1 is for ":"; note that +1 is in bounds b/c raw_colon was found + name_ary_len = name_ary.length; + Xow_ns_mgr_name_itm ns_eval2 = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia + if (ns_eval2 != null) { + Xow_ns ns_eval_itm = ns_eval2.Ns(); + int pre_len = ns_eval_itm.Name_enc().length; + if (pre_len < name_ary_len && name_ary[pre_len] == Byte_ascii.Colon) + return SubEval(ctx, wiki, bfr, name_ary, caller, src); + } + } + switch (finder_tid) { + case Xot_defn_.Tid_msg: + defn = Xot_defn_.Null; // null out defn to force template load below; DATE:2014-07-10 + break; + } + break; + } + if (subst_found) {// subst found; remove Template: if it exists; EX: {{safesubst:Template:A}} -> {{A}} not {{Template:A}}; EX:en.d:Kazakhstan; DATE:2014-03-25 + ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); ns_template_prefix_len = ns_template_prefix.length; + if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) { + name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len); + name_ary_len = name_ary.length; + name_bgn = 0; + template_prefix_found = true; + } + } + if (colon_pos != -1) { // func; separate name_ary into name_ary and arg_x + argx_ary = Bry_.Trim(name_ary, colon_pos + 1, name_ary_len); // trim bgn ws; needed for "{{formatnum:\n{{#expr:2}}\n}}" + name_ary = Bry_.Mid(name_ary, 0, colon_pos); + } + if (defn == Xot_defn_.Null) { + if (ctx.Tid_is_popup()) { // popup && cur_tmpl > tmpl_max + if (Popup_skip(ctx, name_ary, bfr)) return false; + } + defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary); + if (defn == null) { + if (name_ary_len != 0 ) { // name_ary_len != 0 for direct template inclusions; PAGE:en.w:Human evolution and {{:Human evolution/Species chart}}; && ctx.Tmpl_whitelist().Has(name_ary) + Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary)); + if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args ); + if (subst_found || template_prefix_found) { // if "subst:" or "Template:" found, use orig name; DATE:2014-03-31 + bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary_orig).Add(Xop_curly_end_lxr.Hook); + return false; + } + else {// output entire tmpl_src WITH args; used to output name only which broke pages; PAGE:en.w:Flag_of_Greenland; DATE:2016-06-21 + bfr.Add(Xop_curly_bgn_lxr.Hook); + bfr.Add(name_ary); + for (int i = 0; i < args_len; ++i) { + Arg_nde_tkn nde = this.Args_get_by_idx(i); + bfr.Add_byte(Byte_ascii.Pipe); + // must evaluate args; "size={{{size|}}}" must become "size="; PAGE:en.w:Europe; en.w:Template:Country_data_Guernsey DATE:2016-10-13 + nde.Tmpl_compile(ctx, src, Xot_compile_data.Noop); + nde.Tmpl_evaluate(ctx, src, caller, bfr); + } + bfr.Add(Xop_curly_end_lxr.Hook); + return false; + } + } + else { // some templates produce null ttls; EX: "Citation needed{{subst" + defn = wiki.Cache_mgr().Defn_cache().Get_by_key(ttl.Page_db()); + if (defn == null && ctx.Tmpl_load_enabled()) + defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary); + } + } + } + if (defn == null) defn = Xot_defn_.Null; + } + } + if ( defn.Defn_tid() == Xot_defn_.Tid_null // name is not a known defn + && lang.Vnt_mgr().Enabled()) { // lang has vnts + Xowd_page_itm page = lang.Vnt_mgr().Convert_mgr().Convert_ttl(wiki, wiki.Ns_mgr().Ns_template(), name_ary); + if (page != Xowd_page_itm.Null) { + name_ary = page.Ttl_page_db(); + Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary)); + if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args ); + bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary).Add(Xop_curly_end_lxr.Hook); + return false; + } + defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary); + if (defn == null && ctx.Tmpl_load_enabled()) + defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary); + if (defn == null) defn = Xot_defn_.Null; + } + } + Xot_defn_trace trace = ctx.Defn_trace(); int trg_bgn = bfr.Len(); + switch (defn.Defn_tid()) { + case Xot_defn_.Tid_null: // defn is unknown + if (ignore_hash.Get_by_bry(name_ary) == null) { + if (Pfunc_rel2abs.Rel2abs_ttl(name_ary, name_bgn, name_ary_len)) {// rel_path; EX: {{/../Peer page}}; DATE:2013-03-27 + Bry_bfr tmp_bfr = ctx.Wiki().Utl__bfr_mkr().Get_b512(); + name_ary = Pfunc_rel2abs.Rel2abs(tmp_bfr, wiki.Parser_mgr().Rel2abs_ary(), Bry_.Mid(name_ary, name_bgn, name_ary_len), ctx.Page().Ttl().Raw()); + tmp_bfr.Mkr_rls(); + return SubEval(ctx, wiki, bfr, name_ary, caller, src); + } + if (subst_found) + return Transclude(ctx, wiki, bfr, template_prefix_found, name_ary, caller, src); + Xot_invk_tkn_.Print_not_found__w_template(bfr, wiki.Ns_mgr(), name_ary); + return false; + } + break; + case Xot_defn_.Tid_func: + try { + Xot_invk_tkn_.Eval_func(ctx, src, caller, this, bfr, defn, argx_ary); + rv = true; + } catch (Exception e) { + if (Env_.Mode_testing()) + throw Err_.new_exc(e, "xo", "failed to evaluate function", "page", ctx.Page().Ttl().Full_txt(), "defn", defn.Name(), "src", String_.new_u8(src, this.Src_bgn(), this.Src_end())); + else { + wiki.Appe().Usr_dlg().Warn_many("", "", "failed to evaluate function: page=~{0} defn=~{1} src=~{2} err=~{3}", ctx.Page().Ttl().Full_txt(), defn.Name(), Bry_.Replace_nl_w_tab(src, this.Src_bgn(), this.Src_end()), Err_.Message_gplx_log(e)); + rv = false; + } + } + break; + default: + Xot_defn_tmpl defn_tmpl = (Xot_defn_tmpl)defn; + if (defn_tmpl.Root() == null) defn_tmpl.Parse_tmpl(ctx); // NOTE: defn_tmpl.Root can be null after clearing out cache; must be non-null else will fail in trace; DATE:2013-02-01 + Xot_invk invk_tmpl = Xot_defn_tmpl_.CopyNew(ctx, defn_tmpl, this, caller, src, Xow_ns_.Tid__template, name_ary); + invk_tmpl.Frame_ttl_(defn_tmpl.Frame_ttl()); // set frame_ttl; needed for redirects; PAGE:en.w:Statutory_city; DATE:2014-08-22 + trace.Trace_bgn(ctx, src, name_ary, caller, invk_tmpl, defn); + + Bry_bfr rslt_bfr = wiki.Utl__bfr_mkr().Get_k004(); + try { + Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(bfr); + rv = defn_tmpl.Tmpl_evaluate(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), invk_tmpl, rslt_bfr); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17 + prepend_mgr.End(ctx, bfr, rslt_bfr.Bfr(), rslt_bfr.Len(), Bool_.Y); + if (name_had_subst) { // current invk had "subst:"; parse incoming invk again to remove effects of subst; PAGE:pt.w:Argentina DATE:2014-09-24 + byte[] tmp_src = rslt_bfr.To_bry_and_clear(); + rslt_bfr.Add(wiki.Parser_mgr().Main().Expand_tmpl(tmp_src)); // this could be cleaner / more optimized + } + bfr.Add_bfr_and_clear(rslt_bfr); + trace.Trace_end(trg_bgn, bfr); + } finally {rslt_bfr.Mkr_rls();} + break; + } + return rv; + } + private boolean Popup_skip(Xop_ctx ctx, byte[] ttl, Bry_bfr bfr) { + boolean skip = false; + skip = this.Src_end() - this.Src_bgn() > ctx.Tmpl_tkn_max(); + if (!skip) { + gplx.xowa.htmls.modules.popups.keeplists.Xop_keeplist_wiki tmpl_keeplist = ctx.Tmpl_keeplist(); + if (tmpl_keeplist != null && tmpl_keeplist.Enabled()) { + byte[] ttl_lower = Xoa_ttl.Replace_spaces(ctx.Wiki().Lang().Case_mgr().Case_build_lower(ttl)); + skip = !tmpl_keeplist.Match(ttl_lower); + // if (skip) Tfds.Write_bry(ttl_lower); + } + } + if (skip) { + bfr.Add(gplx.xowa.parsers.miscs.Xop_comm_lxr.Xowa_skip_comment_bry); // add comment tkn; need something to separate ''{{lang|la|Ragusa}}'' else will become ''''; PAGE:en.w:Republic_of_Ragusa; DATE:2014-06-28 + return true; + } + else + return false; + } + private boolean Transclude(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, boolean template_prefix_found, byte[] name_ary, Xot_invk caller, byte[] src) { + Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}} + byte[] transclude_src = null; + if (page_ttl.Ns().Id_is_tmpl()) { // ttl is template; check tmpl_regy first before going to data_mgr + Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db()); + if (tmpl != null) transclude_src = tmpl.Data_raw(); + } + if (transclude_src == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load + Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl); + if ( cache_itm != null +// && Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Page_ttl().Full_db()) // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10 + ) { + transclude_src = cache_itm.Wtxt__direct(); + page_ttl = cache_itm.Ttl(); + } + } + if (transclude_src != null) { + // NOTE: must use new page, not current, else transcluded page can cause inconsistent TOC state; PAGE:de.w:Game_of_Thrones DATE:2016-11-21 + Xot_defn_tmpl transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, Xoae_page.New(wiki, page_ttl)), ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), transclude_src); + return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src, bfr); + } + else { + Xot_invk_tkn_.Print_not_found__by_transclude(bfr, wiki.Ns_mgr(), template_prefix_found, name_ary); + return false; + } + } + private boolean Eval_sub(Xop_ctx ctx, Xoa_ttl transclude_ttl, Xot_defn_tmpl transclude_tmpl, Xot_invk caller, byte[] src, Bry_bfr doc) { + boolean rv = false; + Xot_invk tmp_tmpl = Xot_defn_tmpl_.CopyNew(ctx, transclude_tmpl, this, caller, src, transclude_ttl.Ns().Id(), transclude_tmpl.Name()); + Bry_bfr tmp_bfr = Bry_bfr_.New(); + Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(doc); + rv = transclude_tmpl.Tmpl_evaluate(ctx, tmp_tmpl, tmp_bfr); + prepend_mgr.End(ctx, doc, tmp_bfr.Bfr(), tmp_bfr.Len(), Bool_.Y); + doc.Add_bfr_and_clear(tmp_bfr); + return rv; + } + private boolean SubEval(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, byte[] name_ary, Xot_invk caller, byte[] src_for_tkn) { + Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}} + Xot_defn_tmpl transclude_tmpl = null; + switch (page_ttl.Ns().Id()) { + case Xow_ns_.Tid__template: // ttl is template not in cache, or some other ns; do load + Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db()); + if (tmpl != null) { + if (tmpl.Root() == null) tmpl.Parse_tmpl(ctx); + transclude_tmpl = tmpl; + } + break; + case Xow_ns_.Tid__special: + bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon).Add(name_ary).Add(Xop_tkn_.Lnki_end); + return true; + } + if (transclude_tmpl == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load + Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl); + if ( cache_itm != null) { + if (!Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Ttl().Full_db())) { // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10 + transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), cache_itm.Wtxt__direct()); + page_ttl = cache_itm.Ttl(); + } + } + } + if (transclude_tmpl == null) { + bfr.Add(Xop_tkn_.Lnki_bgn).Add(name_ary).Add(Xop_tkn_.Lnki_end); // indicate template was not found; DATE:2014-02-12 + return false; + } + return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src_for_tkn, bfr); + } + public int Args_len() {return args_len;} private int args_len = 0; + public Arg_nde_tkn Args_get_by_idx(int idx) {return args[idx];} + public Arg_nde_tkn Args_eval_by_idx(byte[] src, int idx) { + int cur = 0; + for (int i = 0; i < args_len; i++) { + Arg_nde_tkn nde = args[i]; + if (nde.KeyTkn_exists()) continue; + if (cur++ == idx) return nde; + } + return null; + } + public Arg_nde_tkn Args_get_by_key(byte[] src, byte[] key) { + for (int i = 0; i < args_len; i++) { + Arg_nde_tkn nde = args[i]; + if (!nde.KeyTkn_exists()) continue; + if (Bry_.Match(src, nde.Key_tkn().Dat_bgn(), nde.Key_tkn().Dat_end(), key)) return nde; // NOTE: dat_ary is guaranteed to exist + } + return null; + } + public void Args_add(Xop_ctx ctx, Arg_nde_tkn arg) { + int newLen = args_len + 1; + if (newLen > args_max) { + args_max = newLen * 2; + args = Resize(args, args_len, args_max); + } + args[args_len] = arg; + arg.Tkn_grp_(this, args_len); + args_len = newLen; + } Arg_nde_tkn[] args = Arg_nde_tkn.Ary_empty; int args_max; + private Arg_nde_tkn[] Resize(Arg_nde_tkn[] src, int cur_len, int new_len) { + Arg_nde_tkn[] rv = new Arg_nde_tkn[new_len]; + Array_.Copy_to(src, 0, rv, 0, cur_len); + return rv; + } + private byte[] Get_first_subst_itm(Xol_kwd_mgr kwd_mgr) { + Xol_kwd_grp grp = kwd_mgr.Get_at(Xol_kwd_grp_.Id_subst); if (grp == null) return Bry_.Empty; + Xol_kwd_itm[] itms = grp.Itms(); + return itms.length == 0 ? Bry_.Empty : itms[0].Val(); + } + private static final Hash_adp_bry ignore_hash = Hash_adp_bry.ci_a7().Add_str_obj("Citation needed{{subst", "").Add_str_obj("Clarify{{subst", ""); // ignore SafeSubst templates +} +/* +NOTE_1: if (finder.Colon_pos() != -1) colon_pos = finder.Func().Name().length; +Two issues here: +1) "if (finder.Colon_pos() != -1)" + Colon_pos can either be -1 or >0 + EX: -1: safesubst:NAMESPACE + EX: >0: safesubst:#expr:0 + if -1, don't do anything; this will leave colon_pos as -1 +2) "finder.Func().Name().length" + Colon_pos is >0, but refers to String before it was chopped + EX: "safesubst:#expr:0" has Colon_pos of 15 but String is now "#expr:0" + so, get colon_pos by taking the finder.Func().Name().length + NOTE: whitespace does not matter b/c "safesubst: #expr:0" would never be a func; */ \ No newline at end of file diff --git a/400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn_.java b/400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn_.java index f6a6de780..524cec2fe 100644 --- a/400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn_.java +++ b/400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn_.java @@ -1,6 +1,6 @@ /* XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012-2017 gnosygnu@gmail.com +Copyright (C) 2012-2020 gnosygnu@gmail.com XOWA is licensed under the terms of the General Public License (GPL) Version 3, or alternatively under the terms of the Apache License Version 2.0. @@ -13,67 +13,66 @@ The terms of each license can be found in the source code repository: GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ -package gplx.xowa.parsers.tmpls; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*; -import gplx.xowa.langs.kwds.*; -import gplx.xowa.xtns.pfuncs.*; -import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.caches.*; -public class Xot_invk_tkn_ { - public static void Eval_func(Xop_ctx ctx, byte[] src, Xot_invk caller, Xot_invk invk, Bry_bfr bfr, Xot_defn defn, byte[] argx_ary) { - Pf_func_base defn_func = (Pf_func_base)defn; - int defn_func_id = defn_func.Id(); - defn_func = (Pf_func_base)defn_func.New(defn_func_id, defn_func.Name()); // NOTE: always make copy b/c argx_ary may be dynamic - if (argx_ary != Bry_.Empty) defn_func.Argx_dat_(argx_ary); - defn_func.Eval_argx(ctx, src, caller, invk); - if (defn_func_id == Xol_kwd_grp_.Id_invoke) // NOTE: if #invoke, set frame_ttl to argx, not name; EX:{{#invoke:A}} - invk.Frame_ttl_(Bry_.Add(Xow_ns_.Bry__module_w_colon, Xoa_ttl.Replace_unders(defn_func.Argx_dat()))); // NOTE: always prepend "Module:" to frame_ttl; DATE:2014-06-13; NOTE: always use spaces; DATE:2014-08-14; always use canonical English "Module"; DATE:2015-11-09 - Bry_bfr bfr_func = Bry_bfr_.New(); - defn_func.Func_evaluate(bfr_func, ctx, caller, invk, src); - if (caller.Rslt_is_redirect()) // do not prepend if page is redirect; EX:"#REDIRECT" x> "\n#REDIRECT" DATE:2014-07-11 - caller.Rslt_is_redirect_(false); // reset flag; needed for TEST; kludgy, but Rslt_is_redirect is intended for single use - else - ctx.Page().Tmpl_prepend_mgr().End(ctx, bfr, bfr_func.Bfr(), bfr_func.Len(), Bool_.N); - bfr.Add_bfr_and_clear(bfr_func); - } - public static void Bld_key(Xot_invk invk, byte[] name_ary, Bry_bfr bfr) { - bfr.Clear(); - bfr.Add(name_ary); - int args_len = invk.Args_len(); - for (int i = 0; i < args_len; i++) { - Arg_nde_tkn nde = invk.Args_get_by_idx(i); - bfr.Add_byte(Byte_ascii.Pipe); - if (nde.KeyTkn_exists()) { - bfr.Add(nde.Key_tkn().Dat_ary()); - bfr.Add_byte(Byte_ascii.Eq); - } - bfr.Add(nde.Val_tkn().Dat_ary()); - } - } - public static Xot_defn_tmpl Load_defn(Xowe_wiki wiki, Xop_ctx ctx, Xot_invk_tkn invk_tkn, Xoa_ttl ttl, byte[] name_ary) { // recursive loading of templates - Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl); - byte[] tmpl_page_bry = tmpl_page_itm == null ? null : tmpl_page_itm.Wtxt__direct(); - Xot_defn_tmpl rv = null; - if (tmpl_page_bry != null) { - byte old_parse_tid = ctx.Parse_tid(); // NOTE: reusing ctxs is a bad idea; will change Parse_tid and cause strange errors; however, keeping for PERF reasons - Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template(); - rv = wiki.Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), ctx.Tkn_mkr(), ns_tmpl, name_ary, tmpl_page_bry); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17 - Xoa_ttl tmpl_page_ttl = tmpl_page_itm.Ttl(); - byte[] frame_ttl = Bry_.Add(tmpl_page_ttl.Ns().Name_db(), Byte_ascii.Colon_bry, tmpl_page_ttl.Page_txt()); // NOTE: (1) must have ns (Full); (2) must be txt (space, not underscore); EX:Template:Location map+; DATE:2014-08-22; (3) must be local language; Russian "Шаблон" not English "Template"; PAGE:ru.w:Королевство_Нидерландов DATE:2016-11-23 - rv.Frame_ttl_(frame_ttl); // set defn's frame_ttl; needed for redirect_trg; PAGE:en.w:Statutory_city; DATE:2014-08-22 - ctx.Parse_tid_(old_parse_tid); - wiki.Cache_mgr().Defn_cache().Add(rv, ns_tmpl.Case_match()); - } - return rv; - } - public static void Print_not_found__by_transclude(Bry_bfr bfr, Xow_ns_mgr ns_mgr, boolean name_has_template, byte[] name_ary) {// print missing as [[Missing]]; PAGE:en.d:a DATE:2016-06-24 - bfr.Add(Xop_tkn_.Lnki_bgn); - if (name_has_template) - bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon); - bfr.Add(name_ary); - bfr.Add(Xop_tkn_.Lnki_end); - } - public static void Print_not_found__w_template(Bry_bfr bfr, Xow_ns_mgr ns_mgr, byte[] name_ary) { // print missing as [[:Template:Missing]]; REF:MW: Parser.php|braceSubstitution|$text = "[[:$titleText]]"; EX:en.d:Kazakhstan; DATE:2014-03-25 - bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon); - bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon); - bfr.Add(name_ary).Add(Xop_tkn_.Lnki_end); - } -} +package gplx.xowa.parsers.tmpls; + +import gplx.Bool_; +import gplx.Bry_; +import gplx.Bry_bfr; +import gplx.Bry_bfr_; +import gplx.Byte_ascii; +import gplx.xowa.Xoa_ttl; +import gplx.xowa.Xowe_wiki; +import gplx.xowa.langs.kwds.Xol_kwd_grp_; +import gplx.xowa.parsers.Xop_ctx; +import gplx.xowa.wikis.caches.Xow_page_cache_itm; +import gplx.xowa.wikis.nss.Xow_ns; +import gplx.xowa.wikis.nss.Xow_ns_; +import gplx.xowa.wikis.nss.Xow_ns_mgr; +import gplx.xowa.xtns.pfuncs.Pf_func_base; + +public class Xot_invk_tkn_ { + public static void Eval_func(Xop_ctx ctx, byte[] src, Xot_invk caller, Xot_invk invk, Bry_bfr bfr, Xot_defn defn, byte[] argx_ary) { + Pf_func_base defn_func = (Pf_func_base)defn; + int defn_func_id = defn_func.Id(); + defn_func = (Pf_func_base)defn_func.New(defn_func_id, defn_func.Name()); // NOTE: always make copy b/c argx_ary may be dynamic + if (argx_ary != Bry_.Empty) defn_func.Argx_dat_(argx_ary); + defn_func.Eval_argx(ctx, src, caller, invk); + if (defn_func_id == Xol_kwd_grp_.Id_invoke) // NOTE: if #invoke, set frame_ttl to argx, not name; EX:{{#invoke:A}} + invk.Frame_ttl_(Bry_.Add(Xow_ns_.Bry__module_w_colon, Xoa_ttl.Replace_unders(defn_func.Argx_dat()))); // NOTE: always prepend "Module:" to frame_ttl; DATE:2014-06-13; NOTE: always use spaces; DATE:2014-08-14; always use canonical English "Module"; DATE:2015-11-09 + Bry_bfr bfr_func = Bry_bfr_.New(); + defn_func.Func_evaluate(bfr_func, ctx, caller, invk, src); + if (caller.Rslt_is_redirect()) // do not prepend if page is redirect; EX:"#REDIRECT" x> "\n#REDIRECT" DATE:2014-07-11 + caller.Rslt_is_redirect_(false); // reset flag; needed for TEST; kludgy, but Rslt_is_redirect is intended for single use + else + ctx.Page().Tmpl_prepend_mgr().End(ctx, bfr, bfr_func.Bfr(), bfr_func.Len(), Bool_.N); + bfr.Add_bfr_and_clear(bfr_func); + } + public static Xot_defn_tmpl Load_defn(Xowe_wiki wiki, Xop_ctx ctx, Xot_invk_tkn invk_tkn, Xoa_ttl ttl, byte[] name_ary) { // recursive loading of templates + Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl); + byte[] tmpl_page_bry = tmpl_page_itm == null ? null : tmpl_page_itm.Wtxt__direct(); + Xot_defn_tmpl rv = null; + if (tmpl_page_bry != null) { + byte old_parse_tid = ctx.Parse_tid(); // NOTE: reusing ctxs is a bad idea; will change Parse_tid and cause strange errors; however, keeping for PERF reasons + Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template(); + rv = wiki.Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), ctx.Tkn_mkr(), ns_tmpl, name_ary, tmpl_page_bry); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17 + Xoa_ttl tmpl_page_ttl = tmpl_page_itm.Ttl(); + byte[] frame_ttl = Bry_.Add(tmpl_page_ttl.Ns().Name_db(), Byte_ascii.Colon_bry, tmpl_page_ttl.Page_txt()); // NOTE: (1) must have ns (Full); (2) must be txt (space, not underscore); EX:Template:Location map+; DATE:2014-08-22; (3) must be local language; Russian "Шаблон" not English "Template"; PAGE:ru.w:Королевство_Нидерландов DATE:2016-11-23 + rv.Frame_ttl_(frame_ttl); // set defn's frame_ttl; needed for redirect_trg; PAGE:en.w:Statutory_city; DATE:2014-08-22 + ctx.Parse_tid_(old_parse_tid); + wiki.Cache_mgr().Defn_cache().Add(rv, ns_tmpl.Case_match()); + } + return rv; + } + public static void Print_not_found__by_transclude(Bry_bfr bfr, Xow_ns_mgr ns_mgr, boolean name_has_template, byte[] name_ary) {// print missing as [[Missing]]; PAGE:en.d:a DATE:2016-06-24 + bfr.Add(Xop_tkn_.Lnki_bgn); + if (name_has_template) + bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon); + bfr.Add(name_ary); + bfr.Add(Xop_tkn_.Lnki_end); + } + public static void Print_not_found__w_template(Bry_bfr bfr, Xow_ns_mgr ns_mgr, byte[] name_ary) { // print missing as [[:Template:Missing]]; REF:MW: Parser.php|braceSubstitution|$text = "[[:$titleText]]"; EX:en.d:Kazakhstan; DATE:2014-03-25 + bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon); + bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon); + bfr.Add(name_ary).Add(Xop_tkn_.Lnki_end); + } +} diff --git a/400_xowa/src/gplx/xowa/wikis/Xoae_wiki_mgr.java b/400_xowa/src/gplx/xowa/wikis/Xoae_wiki_mgr.java index 2582afa0c..d4bf0bf9b 100644 --- a/400_xowa/src/gplx/xowa/wikis/Xoae_wiki_mgr.java +++ b/400_xowa/src/gplx/xowa/wikis/Xoae_wiki_mgr.java @@ -1,6 +1,6 @@ /* XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012-2017 gnosygnu@gmail.com +Copyright (C) 2012-2020 gnosygnu@gmail.com XOWA is licensed under the terms of the General Public License (GPL) Version 3, or alternatively under the terms of the Apache License Version 2.0. @@ -13,118 +13,142 @@ The terms of each license can be found in the source code repository: GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ -package gplx.xowa.wikis; import gplx.*; import gplx.xowa.*; -import gplx.xowa.langs.*; import gplx.xowa.xtns.wbases.*; -import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.domains.crts.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.metas.*; import gplx.xowa.langs.names.*; -import gplx.xowa.addons.wikis.directorys.dbs.*; -public class Xoae_wiki_mgr implements Xoa_wiki_mgr, Gfo_invk { - private final Xoae_app app; - private final List_adp list = List_adp_.New(); private final Hash_adp_bry hash = Hash_adp_bry.ci_a7(); // ASCII:url_domain; EX:en.wikipedia.org - private Xowdir_db_mgr db_mgr; - public Xoae_wiki_mgr(Xoae_app app) { - this.app = app; - this.wiki_regy = new Xoa_wiki_regy(app); - this.wdata_mgr = new Wdata_wiki_mgr(app); - } - public Xoa_wiki_regy Wiki_regy() {return wiki_regy;} private final Xoa_wiki_regy wiki_regy; - public Xow_script_mgr Scripts() {return scripts;} private final Xow_script_mgr scripts = new Xow_script_mgr(); - public Wdata_wiki_mgr Wdata_mgr() {return wdata_mgr;} private final Wdata_wiki_mgr wdata_mgr; - public Xowe_wiki Wiki_commons() { - synchronized (this) { // LOCK:app-level; DATE:2016-07-06 - Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(Xow_domain_itm_.Bry__commons); - if (rv != null) rv.Init_assert(); - return rv; - } - } - public void Init_by_app() { - this.db_mgr = new Xowdir_db_mgr(app.User().User_db_mgr().Conn()); - wdata_mgr.Init_by_app(); - } - public int Count() {return list.Count();} - public boolean Has(byte[] key) {return hash.Has(key);} - public Xow_wiki Get_at(int idx) {return (Xow_wiki)list.Get_at(idx);} - public Xowe_wiki Get_at_or_null(int i) {return Int_.Between(i, 0, this.Count() - 1) ? (Xowe_wiki)list.Get_at(i) : null;} - public Xow_wiki Get_by_or_null(byte[] key) {return Bry_.Len_eq_0(key) ? null : (Xowe_wiki)hash.Get_by(key);} - public Xow_wiki Get_by_or_make_init_y(byte[] key) { - synchronized (this) { // LOCK:app-level; DATE:2016-07-06 - Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key); - rv.Init_assert(); - return rv; - } - } - public Xow_wiki Get_by_or_make_init_n(byte[] key) {return Get_by_or_make(key);} - public Xowe_wiki Get_by_or_make(byte[] key) { - Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key); - return rv; - } - public void Add(Xow_wiki wiki) { - if (hash.Get_by_bry(wiki.Domain_bry()) != null) return; // if already there, don't add again; basically, Add_if_dupe_use_1st - hash.Add(wiki.Domain_bry(), wiki); - list.Add(wiki); - } - public Xowe_wiki Make_and_add(byte[] domain_bry) { - // get wiki_root_url from either user_wiki or /xowa/wiki/ - Xowdir_wiki_itm user_wiki_itm = db_mgr == null - ? null // TEST: - : db_mgr.Tbl__wiki().Select_by_key_or_null(String_.new_u8(domain_bry)); - - Xowe_wiki rv = null; - if (user_wiki_itm == null) { - rv = (Xowe_wiki)Make(domain_bry, app.Fsys_mgr().Wiki_dir().GenSubDir(String_.new_a7(domain_bry))); - Add(rv); - } - else { - rv = gplx.xowa.addons.wikis.directorys.specials.items.bldrs.Xow_wiki_factory.Load_personal(app, domain_bry, user_wiki_itm.Url().OwnerDir()); - } - return rv; - } - public Xow_wiki Make(byte[] domain_bry, Io_url wiki_root_dir) { - // init domain - Xow_domain_itm domain_itm = Xow_domain_itm_.parse(domain_bry); - - // get lang from domain; if not wmf, default to en - byte[] lang_key = domain_itm.Lang_actl_key(); - if (lang_key == Xol_lang_stub_.Key__unknown) lang_key = Xol_lang_itm_.Key_en; // unknown langs default to english; note that this makes nonwmf english by default - Xol_lang_itm lang = app.Lang_mgr().Get_by_or_new(lang_key); - if (domain_itm.Domain_type_id() == Xow_domain_tid_.Tid__other) { - lang = Xol_lang_itm.New(app.Lang_mgr(), Xol_lang_itm_.Key_en).Kwd_mgr__strx_(true); // create a new english lang, but enable strx functions; DATE:2015-08-23 - Xol_lang_itm_.Lang_init(lang); - } - - // load ns from site_meta - Xow_ns_mgr ns_mgr = app.Dbmeta_mgr().Ns__get_or_load(domain_bry); - if (ns_mgr.Ids_len() == 0) ns_mgr = Xow_ns_mgr_.default_(lang.Case_mgr()); // non-wmf wikis will use default ns_mgr - - return new Xowe_wiki(app, lang, ns_mgr, domain_itm, wiki_root_dir); - } - public Xow_wiki Import_by_url(Io_url url) {return Xoa_wiki_mgr_.Import_by_url(app, this, url);} - public void Del(byte[] key) {hash.Del(key);} - public void Clear() {hash.Clear(); list.Clear();} - public void Free_mem(boolean clear_ctx) { - int list_len = list.Count(); - for (int i = 0; i < list_len; i++) { - Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i); -// wiki.Defn_cache().ReduceCache(); - if (clear_ctx) wiki.Parser_mgr().Ctx().Clear_all(); // NOTE: clear_ctx will reset toc and refs - wiki.Cache_mgr().Page_cache().Free_mem(true); - wiki.Cache_mgr().Tmpl_result_cache().Clear(); - } - } - public void Rls() { - int len = list.Count(); - for (int i = 0; i < len; i++) { - Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i); - wiki.Rls(); - } - } - public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { - if (ctx.Match(k, Invk_get)) return Get_by_or_make(m.ReadBry("v")); - else if (ctx.Match(k, Invk_scripts)) return scripts; - else if (ctx.Match(k, Invk_wdata)) return wdata_mgr; - else if (ctx.Match(k, Invk_len)) return this.Count(); - else if (ctx.Match(k, Invk_get_at)) return this.Get_at_or_null(m.ReadInt("v")); - else if (ctx.Match(k, Xoa_wiki_mgr_.Invk__import_by_url)) return this.Import_by_url(m.ReadIoUrl("v")); - else return Gfo_invk_.Rv_unhandled; - } private static final String Invk_get = "get", Invk_scripts = "scripts", Invk_wdata = "wdata"; - private static final String Invk_len = "len", Invk_get_at = "get_at"; -} +package gplx.xowa.wikis; + +import gplx.Bry_; +import gplx.GfoMsg; +import gplx.Gfo_invk; +import gplx.Gfo_invk_; +import gplx.GfsCtx; +import gplx.Hash_adp_bry; +import gplx.Int_; +import gplx.Io_url; +import gplx.List_adp; +import gplx.List_adp_; +import gplx.String_; +import gplx.xowa.Xoae_app; +import gplx.xowa.Xow_wiki; +import gplx.xowa.Xowe_wiki; +import gplx.xowa.addons.wikis.directorys.dbs.Xowdir_db_mgr; +import gplx.xowa.addons.wikis.directorys.dbs.Xowdir_wiki_itm; +import gplx.xowa.langs.Xol_lang_itm; +import gplx.xowa.langs.Xol_lang_itm_; +import gplx.xowa.langs.Xol_lang_stub_; +import gplx.xowa.wikis.domains.Xow_domain_itm; +import gplx.xowa.wikis.domains.Xow_domain_itm_; +import gplx.xowa.wikis.domains.Xow_domain_tid_; +import gplx.xowa.wikis.metas.Xow_script_mgr; +import gplx.xowa.wikis.nss.Xow_ns_mgr; +import gplx.xowa.wikis.nss.Xow_ns_mgr_; +import gplx.xowa.xtns.wbases.Wdata_wiki_mgr; + +public class Xoae_wiki_mgr implements Xoa_wiki_mgr, Gfo_invk { + private final Xoae_app app; + private final List_adp list = List_adp_.New(); private final Hash_adp_bry hash = Hash_adp_bry.ci_a7(); // ASCII:url_domain; EX:en.wikipedia.org + private Xowdir_db_mgr db_mgr; + public Xoae_wiki_mgr(Xoae_app app) { + this.app = app; + this.wiki_regy = new Xoa_wiki_regy(app); + this.wdata_mgr = new Wdata_wiki_mgr(app); + } + public Xoa_wiki_regy Wiki_regy() {return wiki_regy;} private final Xoa_wiki_regy wiki_regy; + public Xow_script_mgr Scripts() {return scripts;} private final Xow_script_mgr scripts = new Xow_script_mgr(); + public Wdata_wiki_mgr Wdata_mgr() {return wdata_mgr;} private final Wdata_wiki_mgr wdata_mgr; + public Xowe_wiki Wiki_commons() { + synchronized (this) { // LOCK:app-level; DATE:2016-07-06 + Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(Xow_domain_itm_.Bry__commons); + if (rv != null) rv.Init_assert(); + return rv; + } + } + public void Init_by_app() { + this.db_mgr = new Xowdir_db_mgr(app.User().User_db_mgr().Conn()); + wdata_mgr.Init_by_app(); + } + public int Count() {return list.Count();} + public boolean Has(byte[] key) {return hash.Has(key);} + public Xow_wiki Get_at(int idx) {return (Xow_wiki)list.Get_at(idx);} + public Xowe_wiki Get_at_or_null(int i) {return Int_.Between(i, 0, this.Count() - 1) ? (Xowe_wiki)list.Get_at(i) : null;} + public Xow_wiki Get_by_or_null(byte[] key) {return Bry_.Len_eq_0(key) ? null : (Xowe_wiki)hash.Get_by(key);} + public Xow_wiki Get_by_or_make_init_y(byte[] key) { + synchronized (this) { // LOCK:app-level; DATE:2016-07-06 + Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key); + rv.Init_assert(); + return rv; + } + } + public Xow_wiki Get_by_or_make_init_n(byte[] key) {return Get_by_or_make(key);} + public Xowe_wiki Get_by_or_make(byte[] key) { + Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key); + return rv; + } + public void Add(Xow_wiki wiki) { + if (hash.Get_by_bry(wiki.Domain_bry()) != null) return; // if already there, don't add again; basically, Add_if_dupe_use_1st + hash.Add(wiki.Domain_bry(), wiki); + list.Add(wiki); + } + public Xowe_wiki Make_and_add(byte[] domain_bry) { + // get wiki_root_url from either user_wiki or /xowa/wiki/ + Xowdir_wiki_itm user_wiki_itm = db_mgr == null + ? null // TEST: + : db_mgr.Tbl__wiki().Select_by_key_or_null(String_.new_u8(domain_bry)); + + Xowe_wiki rv = null; + if (user_wiki_itm == null) { + rv = (Xowe_wiki)Make(domain_bry, app.Fsys_mgr().Wiki_dir().GenSubDir(String_.new_a7(domain_bry))); + Add(rv); + } + else { + rv = gplx.xowa.addons.wikis.directorys.specials.items.bldrs.Xow_wiki_factory.Load_personal(app, domain_bry, user_wiki_itm.Url().OwnerDir()); + } + return rv; + } + public Xow_wiki Make(byte[] domain_bry, Io_url wiki_root_dir) { + // init domain + Xow_domain_itm domain_itm = Xow_domain_itm_.parse(domain_bry); + + // get lang from domain; if not wmf, default to en + byte[] lang_key = domain_itm.Lang_actl_key(); + if (lang_key == Xol_lang_stub_.Key__unknown) lang_key = Xol_lang_itm_.Key_en; // unknown langs default to english; note that this makes nonwmf english by default + Xol_lang_itm lang = app.Lang_mgr().Get_by_or_new(lang_key); + if (domain_itm.Domain_type_id() == Xow_domain_tid_.Tid__other) { + lang = Xol_lang_itm.New(app.Lang_mgr(), Xol_lang_itm_.Key_en).Kwd_mgr__strx_(true); // create a new english lang, but enable strx functions; DATE:2015-08-23 + Xol_lang_itm_.Lang_init(lang); + } + + // load ns from site_meta + Xow_ns_mgr ns_mgr = app.Dbmeta_mgr().Ns__get_or_load(domain_bry); + if (ns_mgr.Ids_len() == 0) ns_mgr = Xow_ns_mgr_.default_(lang.Case_mgr()); // non-wmf wikis will use default ns_mgr + + return new Xowe_wiki(app, lang, ns_mgr, domain_itm, wiki_root_dir); + } + public Xow_wiki Import_by_url(Io_url url) {return Xoa_wiki_mgr_.Import_by_url(app, this, url);} + public void Del(byte[] key) {hash.Del(key);} + public void Clear() {hash.Clear(); list.Clear();} + public void Free_mem(boolean clear_ctx) { + int list_len = list.Count(); + for (int i = 0; i < list_len; i++) { + Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i); +// wiki.Defn_cache().ReduceCache(); + if (clear_ctx) wiki.Parser_mgr().Ctx().Clear_all(); // NOTE: clear_ctx will reset toc and refs + wiki.Cache_mgr().Page_cache().Free_mem(true); + } + } + public void Rls() { + int len = list.Count(); + for (int i = 0; i < len; i++) { + Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i); + wiki.Rls(); + } + } + public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { + if (ctx.Match(k, Invk_get)) return Get_by_or_make(m.ReadBry("v")); + else if (ctx.Match(k, Invk_scripts)) return scripts; + else if (ctx.Match(k, Invk_wdata)) return wdata_mgr; + else if (ctx.Match(k, Invk_len)) return this.Count(); + else if (ctx.Match(k, Invk_get_at)) return this.Get_at_or_null(m.ReadInt("v")); + else if (ctx.Match(k, Xoa_wiki_mgr_.Invk__import_by_url)) return this.Import_by_url(m.ReadIoUrl("v")); + else return Gfo_invk_.Rv_unhandled; + } private static final String Invk_get = "get", Invk_scripts = "scripts", Invk_wdata = "wdata"; + private static final String Invk_len = "len", Invk_get_at = "get_at"; +} diff --git a/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java b/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java index ae7a8b9c8..5cdf8c38b 100644 --- a/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java +++ b/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java @@ -1,6 +1,6 @@ /* XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012-2017 gnosygnu@gmail.com +Copyright (C) 2012-2020 gnosygnu@gmail.com XOWA is licensed under the terms of the General Public License (GPL) Version 3, or alternatively under the terms of the Apache License Version 2.0. @@ -13,72 +13,83 @@ The terms of each license can be found in the source code repository: GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ -package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; -import gplx.core.caches.*; -import gplx.xowa.wikis.xwikis.sitelinks.*; -public class Xow_cache_mgr { - private final Xowe_wiki wiki; - public Xow_cache_mgr(Xowe_wiki wiki) { - this.wiki = wiki; - this.page_cache = new Xow_page_cache(wiki); - this.defn_cache = new Xow_defn_cache(wiki.Lang()); - this.ifexist_cache = new Xow_ifexist_cache(wiki, page_cache); - } - public Hash_adp Tmpl_result_cache() {return tmpl_result_cache;} private final Hash_adp tmpl_result_cache = Hash_adp_bry.cs(); - public Xow_defn_cache Defn_cache() {return defn_cache;} private final Xow_defn_cache defn_cache; - public Hash_adp_bry Lst_cache() {return lst_cache;} private final Hash_adp_bry lst_cache = Hash_adp_bry.cs(); - public Hash_adp Misc_cache() {return misc_cache;} private final Hash_adp misc_cache = Hash_adp_.New(); - public Xow_page_cache Page_cache() {return page_cache;} private Xow_page_cache page_cache; - public Gfo_cache_mgr Commons_cache() {return commons_cache;} private Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb); - public Xow_ifexist_cache Ifexist_cache() {return ifexist_cache;} private Xow_ifexist_cache ifexist_cache; - - public Xow_cache_mgr Page_cache_(Xow_page_cache v) {this.page_cache = v; return this;} - public Xow_cache_mgr Commons_cache_(Gfo_cache_mgr v) {this.commons_cache = v; return this;} - public Xow_cache_mgr Ifexist_cache_(Xow_ifexist_cache v) {this.ifexist_cache = v; return this;} - public void Load_wkr_(Xow_page_cache_wkr v) { - page_cache.Load_wkr_(v); - ifexist_cache.Load_wkr_(v); - } - public Keyval[] Scrib_lang_names() { - if (scrib_lang_names == null) { - List_adp list = List_adp_.New(); - Xoa_sitelink_itm_mgr itm_mgr = wiki.App().Xwiki_mgr__sitelink_mgr().Itm_mgr(); - int len = itm_mgr.Len(); - for (int i = 0; i < len; ++i) { - Xoa_sitelink_itm itm = itm_mgr.Get_at(i); - Keyval kv = Keyval_.new_(String_.new_u8(itm.Key()), String_.new_u8(itm.Name())); - list.Add(kv); - } - scrib_lang_names = (Keyval[])list.To_ary(Keyval.class); - } - return scrib_lang_names; - } - public void Free_mem__page() {this.Free_mem(Free_mem__page_tid);} - public void Free_mem__wbase() {this.Free_mem(Free_mem__wbase_tid);} - public void Free_mem__all() {this.Free_mem(Free_mem__all_tid);} - private void Free_mem(int level) { - switch (level) { - case Free_mem__page_tid: - page_cache.Free_mem(false); - break; - case Free_mem__wbase_tid: - page_cache.Free_mem(false); - wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); - break; - case Free_mem__all_tid: - page_cache.Free_mem(true); - wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); - commons_cache.Clear(); - ifexist_cache.Clear(); - break; - } - wiki.Ctg__catpage_mgr().Free_mem_all(); - tmpl_result_cache.Clear(); - defn_cache.Free_mem_all(); - lst_cache.Clear(); - misc_cache.Clear(); - // scrib_lang_names = null; - } - private static final int Free_mem__page_tid = 0, Free_mem__wbase_tid = 1, Free_mem__all_tid = 2; - private static Keyval[] scrib_lang_names; -} +package gplx.xowa.wikis.caches; + +import gplx.Hash_adp; +import gplx.Hash_adp_; +import gplx.Hash_adp_bry; +import gplx.Io_mgr; +import gplx.Keyval; +import gplx.Keyval_; +import gplx.List_adp; +import gplx.List_adp_; +import gplx.String_; +import gplx.core.caches.Gfo_cache_mgr; +import gplx.xowa.Xowe_wiki; +import gplx.xowa.wikis.xwikis.sitelinks.Xoa_sitelink_itm; +import gplx.xowa.wikis.xwikis.sitelinks.Xoa_sitelink_itm_mgr; + +public class Xow_cache_mgr { + private final Xowe_wiki wiki; + public Xow_cache_mgr(Xowe_wiki wiki) { + this.wiki = wiki; + this.page_cache = new Xow_page_cache(wiki); + this.defn_cache = new Xow_defn_cache(wiki.Lang()); + this.ifexist_cache = new Xow_ifexist_cache(wiki, page_cache); + } + public Xow_defn_cache Defn_cache() {return defn_cache;} private final Xow_defn_cache defn_cache; + public Hash_adp_bry Lst_cache() {return lst_cache;} private final Hash_adp_bry lst_cache = Hash_adp_bry.cs(); + public Hash_adp Misc_cache() {return misc_cache;} private final Hash_adp misc_cache = Hash_adp_.New(); + public Xow_page_cache Page_cache() {return page_cache;} private Xow_page_cache page_cache; + public Gfo_cache_mgr Commons_cache() {return commons_cache;} private Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb); + public Xow_ifexist_cache Ifexist_cache() {return ifexist_cache;} private Xow_ifexist_cache ifexist_cache; + + public Xow_cache_mgr Page_cache_(Xow_page_cache v) {this.page_cache = v; return this;} + public Xow_cache_mgr Commons_cache_(Gfo_cache_mgr v) {this.commons_cache = v; return this;} + public Xow_cache_mgr Ifexist_cache_(Xow_ifexist_cache v) {this.ifexist_cache = v; return this;} + public void Load_wkr_(Xow_page_cache_wkr v) { + page_cache.Load_wkr_(v); + ifexist_cache.Load_wkr_(v); + } + public Keyval[] Scrib_lang_names() { + if (scrib_lang_names == null) { + List_adp list = List_adp_.New(); + Xoa_sitelink_itm_mgr itm_mgr = wiki.App().Xwiki_mgr__sitelink_mgr().Itm_mgr(); + int len = itm_mgr.Len(); + for (int i = 0; i < len; ++i) { + Xoa_sitelink_itm itm = itm_mgr.Get_at(i); + Keyval kv = Keyval_.new_(String_.new_u8(itm.Key()), String_.new_u8(itm.Name())); + list.Add(kv); + } + scrib_lang_names = (Keyval[])list.To_ary(Keyval.class); + } + return scrib_lang_names; + } + public void Free_mem__page() {this.Free_mem(Free_mem__page_tid);} + public void Free_mem__wbase() {this.Free_mem(Free_mem__wbase_tid);} + public void Free_mem__all() {this.Free_mem(Free_mem__all_tid);} + private void Free_mem(int level) { + switch (level) { + case Free_mem__page_tid: + page_cache.Free_mem(false); + break; + case Free_mem__wbase_tid: + page_cache.Free_mem(false); + wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); + break; + case Free_mem__all_tid: + page_cache.Free_mem(true); + wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); + commons_cache.Clear(); + ifexist_cache.Clear(); + break; + } + wiki.Ctg__catpage_mgr().Free_mem_all(); + defn_cache.Free_mem_all(); + lst_cache.Clear(); + misc_cache.Clear(); + // scrib_lang_names = null; + } + private static final int Free_mem__page_tid = 0, Free_mem__wbase_tid = 1, Free_mem__all_tid = 2; + private static Keyval[] scrib_lang_names; +} diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_mw__invoke_tst.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_mw__invoke_tst.java index 61c1e5489..cba5e01f3 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_mw__invoke_tst.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_mw__invoke_tst.java @@ -1,135 +1,132 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012-2017 gnosygnu@gmail.com +package gplx.xowa.xtns.scribunto.libs; -XOWA is licensed under the terms of the General Public License (GPL) Version 3, -or alternatively under the terms of the Apache License Version 2.0. +import gplx.Err_; +import gplx.Keyval_; +import gplx.Object_; +import gplx.String_; +import gplx.Tfds; +import gplx.xowa.xtns.scribunto.Scrib_core; +import gplx.xowa.xtns.scribunto.Scrib_invoke_func; +import gplx.xowa.xtns.scribunto.Scrib_invoke_func_fxt; +import gplx.xowa.xtns.scribunto.Scrib_kv_utl_; +import gplx.xowa.xtns.scribunto.Scrib_lib; +import org.junit.Before; +import org.junit.Test; -You may use XOWA according to either of these licenses as is most appropriate -for your project on a case-by-case basis. - -The terms of each license can be found in the source code repository: - -GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt -Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt -*/ -package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; -import org.junit.*; -public class Scrib_lib_mw__invoke_tst { - @Before public void init() { - fxt.Clear_for_invoke(); - lib = fxt.Core().Lib_mw().Init(); - } private Scrib_invoke_func_fxt fxt = new Scrib_invoke_func_fxt(); private Scrib_lib lib; - @Test public void GetAllExpandedArguments_ws_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}") - fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key={{{1}}}}}"); - fxt.Init_page("{{test| a }}"); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a"); // " a " -> "a" - } - @Test public void GetAllExpandedArguments_ws_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}") - fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}"); - fxt.Init_page("{{test| a }}"); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a "); // " a " -> " a " - } - @Test public void GetAllExpandedArguments__ignore_empty_key() {// PURPOSE: ignore arguents that have an empty key (|=8|); EX:w:Fool's_mate; DATE:2014-03-05 - fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0}}"); - fxt.Init_page("{{test|a1||a2|=a3|a4}}"); - fxt.Init_server_print_key_y_(); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("parent"), "\n 1:a1;2:;3:a2;4:a4"); // NOTE: || is not ignored but |=a3| is - fxt.Init_server_print_key_n_(); - } - @Test public void GetExpandedArgument_ws_key_exists() { // PURPOSE: trim val if key exists; literal value - fxt.Init_page("{{#invoke:Mod_0|Prc_0| key1 = val1 }}"); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "val1"); // "key1" -> "key1" - } - @Test public void GetExpandedArgument_ws_key_missing() { // PURPOSE: do not trim val if key missing; literal value - fxt.Init_page("{{#invoke:Mod_0|Prc_0| a }}"); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a " - } - @Test public void GetExpandedArgument_ws_key_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}") - fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key1={{{1}}}}}"); - fxt.Init_page("{{test| a }}"); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "a"); // " a " -> "a" - } - @Test public void GetExpandedArgument_ws_key_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}") - fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}"); - fxt.Init_page("{{test| a }}"); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a " - } - @Test public void Preprocess() { - this.Init_preprocess(); - this.Exec_preprocess(Scrib_core.Frame_key_module , "1", "c"); - this.Exec_preprocess(Scrib_core.Frame_key_module , "2", "d"); - this.Exec_preprocess(Scrib_core.Frame_key_template , "1", "a"); - this.Exec_preprocess(Scrib_core.Frame_key_template , "2", "b"); - } - @Test public void Preprocess_duplicate_key() { - fxt.Init_page("{{#invoke:Mod_0|Prc_0|key1=a|key2=b|key1=c}}"); // add key1 twice - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_preprocess, Object_.Ary("current", "{{#ifeq:1|1|{{{key1}}}|{{{key2}}}}}"), "c"); - } - @Test public void CallParserFunction() { - fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); - fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#expr", "1") , "1"); // named: args is scalar - fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // named: args is table - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", "", "y", "n") , "n"); // list: args is ary - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // list: args is table - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if:", "y", "n") , "n"); // colon_in_name - } - @Test public void CallParserFunction_tag() { - fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); - fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Scrib_kv_utl_.flat_many_("3", "id=1", "2", "text", "1", "pre")), "
2=text
");// named: sort args; NOTE: keys should probably be stripped; // ISSUE#:462; DATE:2019-05-12 - } - @Test public void CallParserFunction__no_args() { // PURPOSE.fix: 0 args should not fail - fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); - fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Keyval_.Ary_empty), "");// failed with "Script error: index is out of bounds" - } - @Test public void CallParserFunction_displayTitle() { // PURPOSE: DISPLAYTITLE not being set when called through CallParserFunction; DATE:2013-08-05 - fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); - fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "DISPLAYTITLE", "''a''"), ""); - Tfds.Eq("a", String_.new_a7(fxt.Parser_fxt().Ctx().Page().Html_data().Display_ttl())); - } - @Test public void CallParserFunction__null() { // PURPOSE.fix: null arg should not fail; PAGE:en.w:Abziri DATE:2017-11-29 - fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); - fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#coordinates", 3, Keyval_.Ary(Keyval_.int_(1, "a"), Keyval_.int_(3, "b"), null)), "");// failed with NullPointerException - } - @Test public void ExpandTemplate_tmpl() { - fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); - fxt.Parser_fxt().Data_create("Template:A", "b{{{key1}}}c"); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary - } - @Test public void ExpandTemplate() { - fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); - fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}"); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Format", Scrib_kv_utl_.base1_many_("a")), "a"); - } - @Test public void ExpandTemplate_ns_specified() { - fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); - fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}"); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Template:Format", Scrib_kv_utl_.base1_many_("a")), "a"); // NOTE: "Template:" specified - } - @Test public void ExpandTemplate_tmpl_bool() { - fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); - fxt.Parser_fxt().Data_create("Template:Scribunto_bool", "bool_true={{{bool_true}}};bool_false={{{bool_false}}};"); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Scribunto_bool", Scrib_kv_utl_.flat_many_("bool_true", true, "bool_false", false)), "bool_true=1;bool_false={{{bool_false}}};"); - } - @Test public void ExpandTemplate_page() { - fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); - fxt.Parser_fxt().Data_create("A", "b{{{key1}}}c"); - fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", ":A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary - } - @Test public void Err_mod_blank() {fxt.Test_parse_err("{{#invoke:}}" , Scrib_invoke_func.Err_mod_missing);} - @Test public void Err_mod_missing() {fxt.Test_parse_err("{{#invoke:Missing}}" , Scrib_invoke_func.Err_mod_missing);} - @Test public void Err_mod_custom() { - fxt.Test_error(Err_.new_("err_type", "fail", "key0", "val0"), "Script error: fail"); - } - private void Init_preprocess() { - fxt.Init_tmpl("{{#invoke:Mod_0|Func_0|1|c|d}}"); // current - fxt.Init_page("{{test|1|a|b|c}}"); // parent - fxt.Init_cbk(fxt.Core().Lib_mw(), Scrib_lib_mw.Invk_preprocess); - } - private void Exec_preprocess(String frame, String arg_idx, String expd) { - fxt.Parser_fxt().Wiki().Cache_mgr().Tmpl_result_cache().Clear(); - fxt.Init_lua_module(); - fxt.Init_lua_rcvd_preprocess(frame, "{{#ifeq:" + arg_idx + "|{{{1}}}|{{{2}}}|{{{3}}}}}"); - fxt.Test_invoke(expd); - } -} +public class Scrib_lib_mw__invoke_tst { + @Before public void init() { + fxt.Clear_for_invoke(); + lib = fxt.Core().Lib_mw().Init(); + } private Scrib_invoke_func_fxt fxt = new Scrib_invoke_func_fxt(); private Scrib_lib lib; + @Test public void GetAllExpandedArguments_ws_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}") + fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key={{{1}}}}}"); + fxt.Init_page("{{test| a }}"); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a"); // " a " -> "a" + } + @Test public void GetAllExpandedArguments_ws_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}") + fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}"); + fxt.Init_page("{{test| a }}"); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a "); // " a " -> " a " + } + @Test public void GetAllExpandedArguments__ignore_empty_key() {// PURPOSE: ignore arguents that have an empty key (|=8|); EX:w:Fool's_mate; DATE:2014-03-05 + fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0}}"); + fxt.Init_page("{{test|a1||a2|=a3|a4}}"); + fxt.Init_server_print_key_y_(); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("parent"), "\n 1:a1;2:;3:a2;4:a4"); // NOTE: || is not ignored but |=a3| is + fxt.Init_server_print_key_n_(); + } + @Test public void GetExpandedArgument_ws_key_exists() { // PURPOSE: trim val if key exists; literal value + fxt.Init_page("{{#invoke:Mod_0|Prc_0| key1 = val1 }}"); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "val1"); // "key1" -> "key1" + } + @Test public void GetExpandedArgument_ws_key_missing() { // PURPOSE: do not trim val if key missing; literal value + fxt.Init_page("{{#invoke:Mod_0|Prc_0| a }}"); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a " + } + @Test public void GetExpandedArgument_ws_key_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}") + fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key1={{{1}}}}}"); + fxt.Init_page("{{test| a }}"); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "a"); // " a " -> "a" + } + @Test public void GetExpandedArgument_ws_key_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}") + fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}"); + fxt.Init_page("{{test| a }}"); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a " + } + @Test public void Preprocess() { + this.Init_preprocess(); + this.Exec_preprocess(Scrib_core.Frame_key_module , "1", "c"); + this.Exec_preprocess(Scrib_core.Frame_key_module , "2", "d"); + this.Exec_preprocess(Scrib_core.Frame_key_template , "1", "a"); + this.Exec_preprocess(Scrib_core.Frame_key_template , "2", "b"); + } + @Test public void Preprocess_duplicate_key() { + fxt.Init_page("{{#invoke:Mod_0|Prc_0|key1=a|key2=b|key1=c}}"); // add key1 twice + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_preprocess, Object_.Ary("current", "{{#ifeq:1|1|{{{key1}}}|{{{key2}}}}}"), "c"); + } + @Test public void CallParserFunction() { + fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); + fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#expr", "1") , "1"); // named: args is scalar + fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // named: args is table + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", "", "y", "n") , "n"); // list: args is ary + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // list: args is table + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if:", "y", "n") , "n"); // colon_in_name + } + @Test public void CallParserFunction_tag() { + fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); + fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Scrib_kv_utl_.flat_many_("3", "id=1", "2", "text", "1", "pre")), "
2=text
");// named: sort args; NOTE: keys should probably be stripped; // ISSUE#:462; DATE:2019-05-12 + } + @Test public void CallParserFunction__no_args() { // PURPOSE.fix: 0 args should not fail + fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); + fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Keyval_.Ary_empty), "");// failed with "Script error: index is out of bounds" + } + @Test public void CallParserFunction_displayTitle() { // PURPOSE: DISPLAYTITLE not being set when called through CallParserFunction; DATE:2013-08-05 + fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); + fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "DISPLAYTITLE", "''a''"), ""); + Tfds.Eq("a", String_.new_a7(fxt.Parser_fxt().Ctx().Page().Html_data().Display_ttl())); + } + @Test public void CallParserFunction__null() { // PURPOSE.fix: null arg should not fail; PAGE:en.w:Abziri DATE:2017-11-29 + fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); + fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#coordinates", 3, Keyval_.Ary(Keyval_.int_(1, "a"), Keyval_.int_(3, "b"), null)), "");// failed with NullPointerException + } + @Test public void ExpandTemplate_tmpl() { + fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); + fxt.Parser_fxt().Data_create("Template:A", "b{{{key1}}}c"); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary + } + @Test public void ExpandTemplate() { + fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); + fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}"); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Format", Scrib_kv_utl_.base1_many_("a")), "a"); + } + @Test public void ExpandTemplate_ns_specified() { + fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); + fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}"); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Template:Format", Scrib_kv_utl_.base1_many_("a")), "a"); // NOTE: "Template:" specified + } + @Test public void ExpandTemplate_tmpl_bool() { + fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); + fxt.Parser_fxt().Data_create("Template:Scribunto_bool", "bool_true={{{bool_true}}};bool_false={{{bool_false}}};"); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Scribunto_bool", Scrib_kv_utl_.flat_many_("bool_true", true, "bool_false", false)), "bool_true=1;bool_false={{{bool_false}}};"); + } + @Test public void ExpandTemplate_page() { + fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); + fxt.Parser_fxt().Data_create("A", "b{{{key1}}}c"); + fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", ":A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary + } + @Test public void Err_mod_blank() {fxt.Test_parse_err("{{#invoke:}}" , Scrib_invoke_func.Err_mod_missing);} + @Test public void Err_mod_missing() {fxt.Test_parse_err("{{#invoke:Missing}}" , Scrib_invoke_func.Err_mod_missing);} + @Test public void Err_mod_custom() { + fxt.Test_error(Err_.new_("err_type", "fail", "key0", "val0"), "Script error: fail"); + } + private void Init_preprocess() { + fxt.Init_tmpl("{{#invoke:Mod_0|Func_0|1|c|d}}"); // current + fxt.Init_page("{{test|1|a|b|c}}"); // parent + fxt.Init_cbk(fxt.Core().Lib_mw(), Scrib_lib_mw.Invk_preprocess); + } + private void Exec_preprocess(String frame, String arg_idx, String expd) { + fxt.Init_lua_module(); + fxt.Init_lua_rcvd_preprocess(frame, "{{#ifeq:" + arg_idx + "|{{{1}}}|{{{2}}}|{{{3}}}}}"); + fxt.Test_invoke(expd); + } +}