Parser: Remove `Tmpl_result_cache`, `Bld_key`, and corresponding code in `Xot_invk_tkn` [#793]

staging
gnosygnu 4 years ago
parent 316c6c6a58
commit d3896bf547

File diff suppressed because it is too large Load Diff

@ -1,6 +1,6 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
@ -13,208 +13,235 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.bldrs.mass_parses.parses.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; package gplx.xowa.addons.bldrs.mass_parses.parses.wkrs;
import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.files.origs.*; import gplx.Bry_bfr;
import gplx.xowa.htmls.core.htmls.*; import gplx.xowa.htmls.core.bldrs.*; import gplx.xowa.htmls.hxtns.pages.*; import gplx.xowa.htmls.core.hzips.*; import gplx.Byte_ascii;
import gplx.xowa.wikis.pages.*; import gplx.Err_;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; import gplx.GfoMsg;
import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.pools.*; import gplx.Gfo_invk;
import gplx.xowa.addons.wikis.fulltexts.indexers.bldrs.*; import gplx.Gfo_invk_;
public class Xomp_parse_wkr implements Gfo_invk { import gplx.Gfo_usr_dlg_;
// mgr vars import gplx.GfsCtx;
private final Xomp_parse_mgr mgr; import gplx.List_adp;
private final Xomp_mgr_db mgr_db; import gplx.List_adp_;
private final Xomp_prog_mgr prog_mgr; import gplx.xowa.Xoa_ttl;
private final Xomp_page_pool page_pool; import gplx.xowa.Xoae_page;
private final Xof_orig_wkr file_orig_wkr; import gplx.xowa.Xowe_wiki;
private final Xomp_ns_ord_mgr ns_ord_mgr; import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_mgr_db;
import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_stat_tbl;
// cfg vars import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_wkr_db;
private final Xomp_parse_mgr_cfg cfg; import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_wkr_tbl;
private int cleanup_interval, commit_interval; import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.Xomp_parse_mgr;
private boolean log_file_lnkis; import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.Xomp_parse_mgr_cfg;
import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.Xomp_prog_mgr;
// wkr vars import gplx.xowa.addons.bldrs.mass_parses.parses.pools.Xomp_page_itm;
private final Xowe_wiki wiki; import gplx.xowa.addons.bldrs.mass_parses.parses.pools.Xomp_page_pool;
private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr(); import gplx.xowa.addons.bldrs.mass_parses.parses.utls.Xomp_lnki_temp_wkr;
private final int uid; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.Xomp_ns_ord_mgr;
private Xomp_wkr_db wkr_db; import gplx.xowa.addons.wikis.fulltexts.indexers.bldrs.Xofulltext_indexer_wkr;
private Xomp_stat_tbl stat_tbl; import gplx.xowa.files.origs.Xof_orig_wkr;
private Hxtn_page_mgr hxtn_mgr; import gplx.xowa.htmls.core.bldrs.Xob_hdump_bldr;
import gplx.xowa.htmls.core.htmls.Xoh_wtr_ctx;
// indexer vars import gplx.xowa.htmls.core.hzips.Xoh_hzip_dict_;
private final Xofulltext_indexer_wkr indexer; import gplx.xowa.htmls.hxtns.pages.Hxtn_page_mgr;
import gplx.xowa.parsers.Xop_ctx;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0; import gplx.xowa.parsers.Xow_parser_mgr;
private int done_count; private long done_time; import gplx.xowa.parsers.logs.Xop_log_wkr_factory;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xomp_parse_mgr_cfg cfg import gplx.xowa.wikis.pages.Xopg_view_mode_;
, Xomp_mgr_db mgr_db, Xomp_page_pool page_pool
, Xomp_prog_mgr prog_mgr, Xof_orig_wkr file_orig_wkr, Xomp_ns_ord_mgr ns_ord_mgr public class Xomp_parse_wkr implements Gfo_invk {
, Xowe_wiki wiki, Xofulltext_indexer_wkr indexer, int uid) { // mgr vars
// mgr vars private final Xomp_parse_mgr mgr;
this.mgr = mgr; this.mgr_db = mgr_db; private final Xomp_mgr_db mgr_db;
this.page_pool = page_pool; this.prog_mgr = prog_mgr; this.file_orig_wkr = file_orig_wkr; private final Xomp_prog_mgr prog_mgr;
this.ns_ord_mgr = ns_ord_mgr; private final Xomp_page_pool page_pool;
this.indexer = indexer; private final Xof_orig_wkr file_orig_wkr;
private final Xomp_ns_ord_mgr ns_ord_mgr;
// cfg vars
this.cfg = cfg; // cfg vars
this.cleanup_interval = cfg.Cleanup_interval(); private final Xomp_parse_mgr_cfg cfg;
this.commit_interval = cfg.Commit_interval(); private int cleanup_interval, commit_interval;
this.log_file_lnkis = cfg.Log_file_lnkis(); private boolean log_file_lnkis;
// wkr-specific vars // wkr vars
this.wiki = wiki; this.uid = uid; private final Xowe_wiki wiki;
this.wkr_db = Xomp_wkr_db.New(Xomp_mgr_db.New__url(wiki), uid); private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
this.stat_tbl = new Xomp_stat_tbl(wkr_db.Conn()); private final int uid;
this.hxtn_mgr = wiki.Hxtn_mgr(); private Xomp_wkr_db wkr_db;
this.hxtn_mgr.Init_by_xomp_wkr(wkr_db.Conn(), cfg.Zip_tid()); private Xomp_stat_tbl stat_tbl;
} private Hxtn_page_mgr hxtn_mgr;
public void Exec() {
Xow_parser_mgr parser_mgr = wiki.Parser_mgr(); // indexer vars
private final Xofulltext_indexer_wkr indexer;
// disable file download
wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing private int done_count; private long done_time;
wiki.File__orig_mgr().Wkrs__set(file_orig_wkr); public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xomp_parse_mgr_cfg cfg
wiki.File_mgr().Fsdb_mode().Tid__v2__mp__y_(); , Xomp_mgr_db mgr_db, Xomp_page_pool page_pool
, Xomp_prog_mgr prog_mgr, Xof_orig_wkr file_orig_wkr, Xomp_ns_ord_mgr ns_ord_mgr
// enable disable categories according to flag , Xowe_wiki wiki, Xofulltext_indexer_wkr indexer, int uid) {
wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Ctgs_enabled_(cfg.Hdump_catboxs()); // mgr vars
this.mgr = mgr; this.mgr_db = mgr_db;
// enable lnki_temp this.page_pool = page_pool; this.prog_mgr = prog_mgr; this.file_orig_wkr = file_orig_wkr;
Xomp_lnki_temp_wkr logger = null; this.ns_ord_mgr = ns_ord_mgr;
if (log_file_lnkis) { this.indexer = indexer;
logger = new Xomp_lnki_temp_wkr(wiki, wkr_db.Conn());
parser_mgr.Ctx().Lnki().File_logger_(logger); // cfg vars
logger.Bgn(); this.cfg = cfg;
} this.cleanup_interval = cfg.Cleanup_interval();
this.commit_interval = cfg.Commit_interval();
// init log_mgr / property_wkr / stats this.log_file_lnkis = cfg.Log_file_lnkis();
Xop_log_wkr_factory wkr_factory = new Xop_log_wkr_factory(wkr_db.Conn());
if (cfg.Log_math()) wiki.Parser_mgr().Math__core().Log_wkr_(wkr_factory); // wkr-specific vars
this.wiki = wiki; this.uid = uid;
// enable hdump this.wkr_db = Xomp_wkr_db.New(Xomp_mgr_db.New__url(wiki), uid);
hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled()).Zip_tid_(cfg.Zip_tid()); this.stat_tbl = new Xomp_stat_tbl(wkr_db.Conn());
hdump_bldr.Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl())); this.hxtn_mgr = wiki.Hxtn_mgr();
wkr_db.Conn().Txn_bgn("xomp"); this.hxtn_mgr.Init_by_xomp_wkr(wkr_db.Conn(), cfg.Zip_tid());
stat_tbl.Stmt_new(); }
hxtn_mgr.Insert_bgn(false); public void Exec() {
Xoh_wtr_ctx hctx = Xoh_wtr_ctx.Hdump_by_hzip_tid(cfg.Hzip_enabled() ? Xoh_hzip_dict_.Hdb__hzip : Xoh_hzip_dict_.Hdb__htxt); // ISSUE#:553; DATE:2019-09-25 Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
// set status to running // disable file download
mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__running); wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
// main loop wiki.File__orig_mgr().Wkrs__set(file_orig_wkr);
int prv_ns = -1; wiki.File_mgr().Fsdb_mode().Tid__v2__mp__y_();
while (true) {
// get page from page pool // enable disable categories according to flag
Xomp_page_itm ppg = Get_next(); wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Ctgs_enabled_(cfg.Hdump_catboxs());
if (ppg == Xomp_page_itm.Null) {
mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__sleeping); // enable lnki_temp
break; // no more pages Xomp_lnki_temp_wkr logger = null;
} if (log_file_lnkis) {
if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05 logger = new Xomp_lnki_temp_wkr(wiki, wkr_db.Conn());
parser_mgr.Ctx().Lnki().File_logger_(logger);
try { logger.Bgn();
long done_bgn = gplx.core.envs.System_.Ticks(); }
// get ns / ttl // init log_mgr / property_wkr / stats
int cur_ns = ppg.Ns_id(); Xop_log_wkr_factory wkr_factory = new Xop_log_wkr_factory(wkr_db.Conn());
Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry()); if (cfg.Log_math()) wiki.Parser_mgr().Math__core().Log_wkr_(wkr_factory);
// if ns changed and prv_ns is main
if (cur_ns != prv_ns) { // enable hdump
prv_ns = cur_ns; hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled()).Zip_tid_(cfg.Zip_tid());
} hdump_bldr.Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
wkr_db.Conn().Txn_bgn("xomp");
// init page stat_tbl.Stmt_new();
Xoae_page wpg = Xoae_page.New(wiki, ttl); hxtn_mgr.Insert_bgn(false);
wpg.Bldr__ns_ord_(ns_ord_mgr.Get_ord_by_ns_id(cur_ns)); // NOTE: must set ns_id for tier_id in lnki_temp; DATE:2016-09-19 Xoh_wtr_ctx hctx = Xoh_wtr_ctx.Hdump_by_hzip_tid(cfg.Hzip_enabled() ? Xoh_hzip_dict_.Hdb__hzip : Xoh_hzip_dict_.Hdb__htxt); // ISSUE#:553; DATE:2019-09-25
wpg.Db().Text().Text_bry_(ppg.Text());
wpg.Db().Page().Init_by_mp(ppg.Id(), ppg.Page_score()); // set status to running
wpg.Stat_itm().Init(uid); mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__running);
// parse page // main loop
Xop_ctx pctx = parser_mgr.Ctx(); int prv_ns = -1;
pctx.Clear_all(); while (true) {
parser_mgr.Parse(wpg, true); // get page from page pool
Xomp_page_itm ppg = Get_next();
// gen_html if (ppg == Xomp_page_itm.Null) {
hdump_bldr.Insert(pctx, wpg, hctx); mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__sleeping);
break; // no more pages
// index }
long fulltext_time = 0; if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05
if (indexer != null) {
fulltext_time = gplx.core.envs.System_.Ticks(); try {
indexer.Index(wpg); long done_bgn = gplx.core.envs.System_.Ticks();
fulltext_time = gplx.core.envs.System_.Ticks__elapsed_in_frac(fulltext_time);
} // get ns / ttl
int cur_ns = ppg.Ns_id();
// mark done for sake of progress Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry());
prog_mgr.Mark_done(ppg.Id()); // if ns changed and prv_ns is main
if (cur_ns != prv_ns) {
// update stats prv_ns = cur_ns;
long time_cur = gplx.core.envs.System_.Ticks(); }
long page_time = time_cur - done_bgn;
done_time += page_time; // init page
++done_count; Xoae_page wpg = Xoae_page.New(wiki, ttl);
stat_tbl.Insert(wpg, hdump_bldr.Tmp_hpg(), uid, page_time, fulltext_time); wpg.Bldr__ns_ord_(ns_ord_mgr.Get_ord_by_ns_id(cur_ns)); // NOTE: must set ns_id for tier_id in lnki_temp; DATE:2016-09-19
wpg.Db().Text().Text_bry_(ppg.Text());
// cleanup wpg.Db().Page().Init_by_mp(ppg.Id(), ppg.Page_score());
// ctx.App().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released wpg.Stat_itm().Init(uid);
if (wiki.Cache_mgr().Tmpl_result_cache().Count() > 50000)
wiki.Cache_mgr().Tmpl_result_cache().Clear(); // parse page
if (done_count % cleanup_interval == 0) { Xop_ctx pctx = parser_mgr.Ctx();
wiki.Cache_mgr().Free_mem__page(); pctx.Clear_all();
wiki.Parser_mgr().Scrib().Core_term(); parser_mgr.Parse(wpg, true);
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
} // gen_html
if (done_count % commit_interval == 0) { hdump_bldr.Insert(pctx, wpg, hctx);
wkr_db.Conn().Txn_sav();
} // index
} catch (Exception e) { long fulltext_time = 0;
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e)); if (indexer != null) {
} fulltext_time = gplx.core.envs.System_.Ticks();
} indexer.Index(wpg);
fulltext_time = gplx.core.envs.System_.Ticks__elapsed_in_frac(fulltext_time);
// cleanup }
try {
if (logger != null) logger.End(); // mark done for sake of progress
wkr_db.Conn().Txn_end(); prog_mgr.Mark_done(ppg.Id());
wkr_db.Conn().Rls_conn();
stat_tbl.Stmt_rls(); // update stats
hxtn_mgr.Insert_end(false); long time_cur = gplx.core.envs.System_.Ticks();
mgr.Wkrs_done_add_1(); // NOTE: must release latch last else thread errors long page_time = time_cur - done_bgn;
} done_time += page_time;
catch (Exception e) { ++done_count;
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail_end; err=~{0}", Err_.Message_gplx_log(e)); stat_tbl.Insert(wpg, hdump_bldr.Tmp_hpg(), uid, page_time, fulltext_time);
}
} // cleanup
public void Bld_stats(Bry_bfr bfr) { // ctx.App().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released
int done_time_in_sec = (int)(done_time / 1000); if (done_time_in_sec == 0) done_time_in_sec = 1; if (done_count % cleanup_interval == 0) {
bfr.Add_int_pad_bgn(Byte_ascii.Space, 4, uid ); wiki.Cache_mgr().Free_mem__page();
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, (int)(done_count / done_time_in_sec)); wiki.Parser_mgr().Scrib().Core_term();
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_count); wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_time_in_sec); }
bfr.Add_byte_nl(); if (done_count % commit_interval == 0) {
} wkr_db.Conn().Txn_sav();
private Xomp_page_itm Get_next() { }
if (list_idx == list_len) { } catch (Exception e) {
mgr_db.Tbl__wkr().Update_exec(uid, done_count, done_time); Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e));
list.Clear(); }
page_pool.Get_next(mgr_db, cfg.Wkr_machine_name(), list); }
list_len = list.Len();
if (list_len == 0) return Xomp_page_itm.Null; // cleanup
list_idx = 0; try {
} if (logger != null) logger.End();
return (Xomp_page_itm)list.Get_at(list_idx++); wkr_db.Conn().Txn_end();
} wkr_db.Conn().Rls_conn();
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { stat_tbl.Stmt_rls();
if (ctx.Match(k, Invk__exec)) this.Exec(); hxtn_mgr.Insert_end(false);
else return Gfo_invk_.Rv_unhandled; mgr.Wkrs_done_add_1(); // NOTE: must release latch last else thread errors
return this; }
} catch (Exception e) {
public static final String Invk__exec = "exec"; Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail_end; err=~{0}", Err_.Message_gplx_log(e));
public static final String Cfg__ns_ids = "xomp.ns_ids"; }
} }
public void Bld_stats(Bry_bfr bfr) {
int done_time_in_sec = (int)(done_time / 1000); if (done_time_in_sec == 0) done_time_in_sec = 1;
bfr.Add_int_pad_bgn(Byte_ascii.Space, 4, uid );
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, (int)(done_count / done_time_in_sec));
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_count);
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_time_in_sec);
bfr.Add_byte_nl();
}
private Xomp_page_itm Get_next() {
if (list_idx == list_len) {
mgr_db.Tbl__wkr().Update_exec(uid, done_count, done_time);
list.Clear();
page_pool.Get_next(mgr_db, cfg.Wkr_machine_name(), list);
list_len = list.Len();
if (list_len == 0) return Xomp_page_itm.Null;
list_idx = 0;
}
return (Xomp_page_itm)list.Get_at(list_idx++);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__exec)) this.Exec();
else return Gfo_invk_.Rv_unhandled;
return this;
}
public static final String Invk__exec = "exec";
public static final String Cfg__ns_ids = "xomp.ns_ids";
}

@ -1,86 +1,85 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2020 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis. for your project on a case-by-case basis.
The terms of each license can be found in the source code repository: The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.parsers.mediawikis; package gplx.xowa.addons.parsers.mediawikis;
import gplx.Bry_; import gplx.Bry_;
import gplx.Bry_bfr; import gplx.Bry_bfr;
import gplx.Bry_bfr_; import gplx.Bry_bfr_;
import gplx.String_; import gplx.String_;
import gplx.xowa.Xoa_ttl; import gplx.xowa.Xoa_ttl;
import gplx.xowa.Xoae_page; import gplx.xowa.Xoae_page;
import gplx.xowa.Xowe_wiki; import gplx.xowa.Xowe_wiki;
import gplx.xowa.addons.wikis.ctgs.htmls.pageboxs.Xoctg_pagebox_itm; import gplx.xowa.addons.wikis.ctgs.htmls.pageboxs.Xoctg_pagebox_itm;
import gplx.xowa.htmls.core.htmls.Xoh_wtr_ctx; import gplx.xowa.htmls.core.htmls.Xoh_wtr_ctx;
import gplx.xowa.parsers.Xop_ctx; import gplx.xowa.parsers.Xop_ctx;
import gplx.xowa.parsers.Xow_parser_mgr; import gplx.xowa.parsers.Xow_parser_mgr;
import gplx.xowa.wikis.Xow_page_tid; import gplx.xowa.wikis.Xow_page_tid;
import gplx.xowa.wikis.pages.Xopg_view_mode_; import gplx.xowa.wikis.pages.Xopg_view_mode_;
public class Xop_mediawiki_wkr { public class Xop_mediawiki_wkr {
private final Xowe_wiki wiki; private final Xowe_wiki wiki;
private final Bry_bfr tmp_bfr = Bry_bfr_.New(); private final Bry_bfr tmp_bfr = Bry_bfr_.New();
public Xop_mediawiki_wkr(Xowe_wiki wiki, Xop_mediawiki_loader loader) { public Xop_mediawiki_wkr(Xowe_wiki wiki, Xop_mediawiki_loader loader) {
this.wiki = wiki; this.wiki = wiki;
if (loader != null) if (loader != null)
wiki.Cache_mgr().Load_wkr_(new Xow_page_cache_wkr__embeddable(wiki, loader)); wiki.Cache_mgr().Load_wkr_(new Xow_page_cache_wkr__embeddable(wiki, loader));
} }
public void Free_memory() { public void Free_memory() {
wiki.Cache_mgr().Tmpl_result_cache().Clear(); wiki.Cache_mgr().Free_mem__page();
wiki.Cache_mgr().Free_mem__page(); wiki.Parser_mgr().Scrib().Core_term();
wiki.Parser_mgr().Scrib().Core_term(); wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); }
} public String Parse(String page, String wikitext) {
public String Parse(String page, String wikitext) { Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));
Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));
byte[] wtxt = Bry_.new_u8(wikitext);
byte[] wtxt = Bry_.new_u8(wikitext); Xoae_page wpg = Xoae_page.New(wiki, ttl);
Xoae_page wpg = Xoae_page.New(wiki, ttl); wpg.Db().Text().Text_bry_(wtxt);
wpg.Db().Text().Text_bry_(wtxt);
Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
// parse page
// parse page Xop_ctx pctx = parser_mgr.Ctx();
Xop_ctx pctx = parser_mgr.Ctx(); pctx.Clear_all();
pctx.Clear_all(); parser_mgr.Parse(wpg, true);
parser_mgr.Parse(wpg, true);
// write to html
// write to html boolean is_wikitext = Xow_page_tid.Identify(wpg.Wiki().Domain_tid(), ttl.Ns().Id(), ttl.Page_db()) == Xow_page_tid.Tid_wikitext;
boolean is_wikitext = Xow_page_tid.Identify(wpg.Wiki().Domain_tid(), ttl.Ns().Id(), ttl.Page_db()) == Xow_page_tid.Tid_wikitext; byte[] orig_bry = Bry_.Empty;
byte[] orig_bry = Bry_.Empty; if (is_wikitext) {
if (is_wikitext) { wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Write_hdump(tmp_bfr, pctx, Xoh_wtr_ctx.Embeddable, wpg);
wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Write_hdump(tmp_bfr, pctx, Xoh_wtr_ctx.Embeddable, wpg);
// write categories
// write categories int ctgs_len = wpg.Wtxt().Ctgs__len();
int ctgs_len = wpg.Wtxt().Ctgs__len(); if ( ctgs_len > 0 // skip if no categories found while parsing wikitext
if ( ctgs_len > 0 // skip if no categories found while parsing wikitext ) {
) { Xoctg_pagebox_itm[] pagebox_itms = new Xoctg_pagebox_itm[ctgs_len];
Xoctg_pagebox_itm[] pagebox_itms = new Xoctg_pagebox_itm[ctgs_len]; for (int i = 0; i < ctgs_len; i++) {
for (int i = 0; i < ctgs_len; i++) { pagebox_itms[i] = new Xoctg_pagebox_itm(wpg.Wtxt().Ctgs__get_at(i));
pagebox_itms[i] = new Xoctg_pagebox_itm(wpg.Wtxt().Ctgs__get_at(i)); }
} wiki.Ctg__pagebox_wtr().Write_pagebox(tmp_bfr, wpg, pagebox_itms);
wiki.Ctg__pagebox_wtr().Write_pagebox(tmp_bfr, wpg, pagebox_itms); }
}
orig_bry = tmp_bfr.To_bry_and_clear();
orig_bry = tmp_bfr.To_bry_and_clear(); wpg.Db().Html().Html_bry_(orig_bry);
wpg.Db().Html().Html_bry_(orig_bry); }
} else { // not wikitext; EX: pages in MediaWiki: ns; DATE:2016-09-12
else { // not wikitext; EX: pages in MediaWiki: ns; DATE:2016-09-12 wpg.Db().Html().Html_bry_(wpg.Db().Text().Text_bry());
wpg.Db().Html().Html_bry_(wpg.Db().Text().Text_bry()); }
}
return String_.new_u8(orig_bry);
return String_.new_u8(orig_bry); }
} }
}

@ -1,6 +1,6 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
@ -13,314 +13,347 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; package gplx.xowa.bldrs.cmds;
import gplx.core.envs.*;
import gplx.dbs.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.files.origs.*; import gplx.Bry_;
import gplx.xowa.bldrs.wkrs.*; import gplx.Bry_bfr;
import gplx.xowa.wikis.nss.*; import gplx.Bry_bfr_;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.Datetime_now;
import gplx.xowa.addons.bldrs.files.utls.*; import gplx.Decimal_adp;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*; import gplx.Decimal_adp_;
public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xob_cmd, Gfo_invk { import gplx.Err_;
private Xob_dump_src_id page_src; import gplx.GfoMsg;
private Xow_db_mgr db_fsys_mgr; protected Xop_parser parser; protected Xop_ctx ctx; protected Xop_root_tkn root; import gplx.Gfo_invk;
private int[] ns_ary; private Xow_db_file[] db_ary; import gplx.Gfo_invk_;
private int ns_bgn = -1, db_bgn = -1, pg_bgn = -1; import gplx.Gfo_usr_dlg;
private int ns_end = -1, db_end = -1, pg_end = Int_.Max_value; import gplx.GfsCtx;
private int commit_interval = 1000, progress_interval = 250, cleanup_interval = 2500, select_size = 10 * Io_mgr.Len_mb; import gplx.Int_;
private int exec_count, exec_count_max = Int_.Max_value; import gplx.Io_mgr;
private boolean reset_db = false, exit_after_commit = false, exit_now = false; import gplx.Io_url;
private boolean load_tmpls; import gplx.List_adp;
private Xob_dump_bmk_mgr bmk_mgr = new Xob_dump_bmk_mgr(); import gplx.List_adp_;
private Xobu_poll_mgr poll_mgr; private int poll_interval = 5000; import gplx.Math_;
private Xob_rate_mgr rate_mgr = new Xob_rate_mgr(); import gplx.String_;
public abstract String Cmd_key(); import gplx.core.envs.System_;
@Override protected void Cmd_ctor_end(Xob_bldr bldr, Xowe_wiki wiki) { import gplx.dbs.Db_conn;
poll_mgr = new Xobu_poll_mgr(bldr.App()); // init in ctor so gfs can invoke methods import gplx.xowa.Xoae_app;
} import gplx.xowa.Xowe_wiki;
public void Cmd_bgn(Xob_bldr bldr) { import gplx.xowa.Xowe_wiki_;
parser = wiki.Parser_mgr().Main(); import gplx.xowa.addons.bldrs.files.utls.Xobu_poll_mgr;
ctx = wiki.Parser_mgr().Ctx(); import gplx.xowa.bldrs.Xob_bldr;
root = ctx.Tkn_mkr().Root(Bry_.Empty); import gplx.xowa.bldrs.wkrs.Xob_cmd;
wiki.Init_assert(); // NOTE: must init wiki for db_mgr_as_sql import gplx.xowa.bldrs.wkrs.Xob_itm_basic_base;
import gplx.xowa.files.origs.Xof_orig_wkr_;
// assert by calling Db_mgr_as_sql import gplx.xowa.parsers.Xop_ctx;
wiki.Db_mgr_as_sql().Core_data_mgr(); import gplx.xowa.parsers.Xop_parser;
import gplx.xowa.parsers.Xop_root_tkn;
// load db_mgr import gplx.xowa.parsers.tmpls.Xot_defn_tmpl;
Xow_db_mgr.Init_by_load(wiki, gplx.xowa.wikis.data.Xow_db_file__core_.Find_core_fil_or_null(wiki)); // NOTE: must reinit providers as previous steps may have rls'd (and left member variable conn which is closed) import gplx.xowa.wikis.caches.Xow_defn_cache;
import gplx.xowa.wikis.data.Xow_db_file;
wiki.File__orig_mgr().Wkrs__del(Xof_orig_wkr_.Tid_wmf_api); import gplx.xowa.wikis.data.Xow_db_file_;
db_fsys_mgr = wiki.Db_mgr_as_sql().Core_data_mgr(); import gplx.xowa.wikis.data.Xow_db_mgr;
db_ary = Xob_dump_mgr_base_.Init_text_files_ary(db_fsys_mgr); import gplx.xowa.wikis.data.tbls.Xowd_page_itm;
poll_interval = poll_mgr.Poll_interval(); import gplx.xowa.wikis.nss.Xow_ns;
import gplx.xowa.wikis.nss.Xow_ns_;
page_src = new Xob_dump_src_id().Init(wiki, this.Init_redirect(), select_size);
ns_ary = Init_ns_ary(); public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xob_cmd, Gfo_invk {
Db_conn conn = Init_db_file(); private Xob_dump_src_id page_src;
Io_url wiki_dir = wiki.Fsys_mgr().Root_dir(); private Xow_db_mgr db_fsys_mgr; protected Xop_parser parser; protected Xop_ctx ctx; protected Xop_root_tkn root;
bmk_mgr.Cfg_url_(wiki_dir.GenSubFil("xowa.file.make.cfg.gfs")); private int[] ns_ary; private Xow_db_file[] db_ary;
rate_mgr.Log_file_(wiki_dir.GenSubFil("xowa.file.make.log.csv")); private int ns_bgn = -1, db_bgn = -1, pg_bgn = -1;
if (reset_db) { private int ns_end = -1, db_end = -1, pg_end = Int_.Max_value;
bmk_mgr.Reset(); private int commit_interval = 1000, progress_interval = 250, cleanup_interval = 2500, select_size = 10 * Io_mgr.Len_mb;
Init_reset(conn); private int exec_count, exec_count_max = Int_.Max_value;
} private boolean reset_db = false, exit_after_commit = false, exit_now = false;
bmk_mgr.Load(wiki.Appe(), this); private boolean load_tmpls;
private Xob_dump_bmk_mgr bmk_mgr = new Xob_dump_bmk_mgr();
Cmd_bgn_end(); private Xobu_poll_mgr poll_mgr; private int poll_interval = 5000;
} private Xob_rate_mgr rate_mgr = new Xob_rate_mgr();
protected abstract void Cmd_bgn_end(); public abstract String Cmd_key();
public abstract byte Init_redirect(); @Override protected void Cmd_ctor_end(Xob_bldr bldr, Xowe_wiki wiki) {
public abstract int[] Init_ns_ary(); poll_mgr = new Xobu_poll_mgr(bldr.App()); // init in ctor so gfs can invoke methods
protected abstract void Init_reset(Db_conn p); }
protected abstract Db_conn Init_db_file(); public void Cmd_bgn(Xob_bldr bldr) {
private long time_bgn; parser = wiki.Parser_mgr().Main();
public void Cmd_run() {Exec_ns_ary();} ctx = wiki.Parser_mgr().Ctx();
private void Exec_ns_ary() { root = ctx.Tkn_mkr().Root(Bry_.Empty);
if (pg_bgn == Int_.Max_value) return; wiki.Init_assert(); // NOTE: must init wiki for db_mgr_as_sql
if (load_tmpls) Xob_dump_mgr_base_.Load_all_tmpls(usr_dlg, wiki, page_src);
time_bgn = System_.Ticks(); // assert by calling Db_mgr_as_sql
Xob_dump_bmk dump_bmk = new Xob_dump_bmk(); wiki.Db_mgr_as_sql().Core_data_mgr();
rate_mgr.Init();
int ns_ary_len = ns_ary.length; // load db_mgr
for (int i = 0; i < ns_ary_len; i++) { Xow_db_mgr.Init_by_load(wiki, gplx.xowa.wikis.data.Xow_db_file__core_.Find_core_fil_or_null(wiki)); // NOTE: must reinit providers as previous steps may have rls'd (and left member variable conn which is closed)
int ns_id = ns_ary[i];
if (ns_bgn != -1) { // ns_bgn set wiki.File__orig_mgr().Wkrs__del(Xof_orig_wkr_.Tid_wmf_api);
if (ns_id == ns_bgn) // ns_id is ns_bgn; null out ns_bgn and continue db_fsys_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
ns_bgn = -1; db_ary = Xob_dump_mgr_base_.Init_text_files_ary(db_fsys_mgr);
else // ns_id is not ns_bgn; keep looking poll_interval = poll_mgr.Poll_interval();
continue;
} page_src = new Xob_dump_src_id().Init(wiki, this.Init_redirect(), select_size);
dump_bmk.Ns_id_(ns_id); ns_ary = Init_ns_ary();
Exec_db_ary(i, dump_bmk, ns_id); Db_conn conn = Init_db_file();
if (ns_id == ns_end) exit_now = true; // ns_end set; exit Io_url wiki_dir = wiki.Fsys_mgr().Root_dir();
if (exit_now) break; // exit_now b/c of pg_bgn, db_bgn or something else bmk_mgr.Cfg_url_(wiki_dir.GenSubFil("xowa.file.make.cfg.gfs"));
} rate_mgr.Log_file_(wiki_dir.GenSubFil("xowa.file.make.log.csv"));
Exec_commit(dump_bmk.Ns_id(), dump_bmk.Db_id(), dump_bmk.Pg_id(), Bry_.Empty); if (reset_db) {
} bmk_mgr.Reset();
private void Exec_db_ary(int ns_ord, Xob_dump_bmk dump_bmk, int ns_id) { Init_reset(conn);
int db_ary_len = db_ary.length; }
for (int i = 0; i < db_ary_len; i++) { bmk_mgr.Load(wiki.Appe(), this);
int db_id = db_ary[i].Id();
if (db_bgn != -1) { // db_bgn set Cmd_bgn_end();
if (db_id == db_bgn) // db_id is db_bgn; null out db_bgn and continue }
db_bgn = -1; protected abstract void Cmd_bgn_end();
else // db_id is not db_bgn; keep looking public abstract byte Init_redirect();
continue; public abstract int[] Init_ns_ary();
} protected abstract void Init_reset(Db_conn p);
dump_bmk.Db_id_(db_id); protected abstract Db_conn Init_db_file();
Exec_db_itm(dump_bmk, ns_ord, ns_id, db_id); private long time_bgn;
if (db_id == db_end) exit_now = true; // db_end set; exit; public void Cmd_run() {Exec_ns_ary();}
if (exit_now) return; // exit_now b/c of pg_bgn, db_bgn or something else private void Exec_ns_ary() {
} if (pg_bgn == Int_.Max_value) return;
} if (load_tmpls) Xob_dump_mgr_base_.Load_all_tmpls(usr_dlg, wiki, page_src);
private void Exec_db_itm(Xob_dump_bmk dump_bmk, int ns_ord, int ns_id, int db_id) { time_bgn = System_.Ticks();
List_adp pages = List_adp_.New(); Xob_dump_bmk dump_bmk = new Xob_dump_bmk();
Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id); rate_mgr.Init();
int pg_id = pg_bgn; int ns_ary_len = ns_ary.length;
while (true) { for (int i = 0; i < ns_ary_len; i++) {
page_src.Get_pages(wiki.Appe(), pages, db_id, ns_id, pg_id); int ns_id = ns_ary[i];
int pages_len = pages.Count(); if (ns_bgn != -1) { // ns_bgn set
if (pages_len == 0) { // no more pages in db; if (ns_id == ns_bgn) // ns_id is ns_bgn; null out ns_bgn and continue
if (pg_id > pg_bgn) // reset pg_bgn to 0 only if pg_bgn seen; ns_bgn = -1;
pg_bgn = 0; else // ns_id is not ns_bgn; keep looking
return; continue;
} }
usr_dlg.Prog_many("", "", "fetched pages: ~{0}", pages_len); dump_bmk.Ns_id_(ns_id);
for (int i = 0; i < pages_len; i++) { Exec_db_ary(i, dump_bmk, ns_id);
Xowd_page_itm page = (Xowd_page_itm)pages.Get_at(i); if (ns_id == ns_end) exit_now = true; // ns_end set; exit
dump_bmk.Pg_id_(pg_id); if (exit_now) break; // exit_now b/c of pg_bgn, db_bgn or something else
Exec_pg_itm(ns_ord, ns, db_id, page); }
if ( pg_id >= pg_end Exec_commit(dump_bmk.Ns_id(), dump_bmk.Db_id(), dump_bmk.Pg_id(), Bry_.Empty);
|| exec_count >= exec_count_max) { }
exit_now = true; private void Exec_db_ary(int ns_ord, Xob_dump_bmk dump_bmk, int ns_id) {
} int db_ary_len = db_ary.length;
if (exit_now) return; for (int i = 0; i < db_ary_len; i++) {
pg_id = page.Id(); int db_id = db_ary[i].Id();
} if (db_bgn != -1) { // db_bgn set
} if (db_id == db_bgn) // db_id is db_bgn; null out db_bgn and continue
} db_bgn = -1;
private void Exec_pg_itm(int ns_ord, Xow_ns ns, int db_id, Xowd_page_itm page) { else // db_id is not db_bgn; keep looking
try { continue;
if ((exec_count % progress_interval) == 0) }
usr_dlg.Prog_many("", "", "parsing: ns=~{0} db=~{1} pg=~{2} count=~{3} time=~{4} rate=~{5} ttl=~{6}" dump_bmk.Db_id_(db_id);
, ns.Id(), db_id, page.Id(), exec_count Exec_db_itm(dump_bmk, ns_ord, ns_id, db_id);
, System_.Ticks__elapsed_in_sec(time_bgn), rate_mgr.Rate_as_str(), String_.new_u8(page.Ttl_page_db())); if (db_id == db_end) exit_now = true; // db_end set; exit;
ctx.Clear_all(); if (exit_now) return; // exit_now b/c of pg_bgn, db_bgn or something else
byte[] page_src = page.Text(); }
if (page_src != null) // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05 }
Exec_pg_itm_hook(ns_ord, ns, page, page_src); private void Exec_db_itm(Xob_dump_bmk dump_bmk, int ns_ord, int ns_id, int db_id) {
ctx.Wiki().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released List_adp pages = List_adp_.New();
if (ctx.Wiki().Cache_mgr().Tmpl_result_cache().Count() > 50000) Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id);
ctx.Wiki().Cache_mgr().Tmpl_result_cache().Clear(); int pg_id = pg_bgn;
++exec_count; while (true) {
rate_mgr.Increment(); page_src.Get_pages(wiki.Appe(), pages, db_id, ns_id, pg_id);
if ((exec_count % poll_interval) == 0) int pages_len = pages.Count();
poll_mgr.Poll(); if (pages_len == 0) { // no more pages in db;
if ((exec_count % commit_interval) == 0) if (pg_id > pg_bgn) // reset pg_bgn to 0 only if pg_bgn seen;
Exec_commit(ns.Id(), db_id, page.Id(), page.Ttl_page_db()); pg_bgn = 0;
if ((exec_count % cleanup_interval) == 0) return;
Free(); }
} usr_dlg.Prog_many("", "", "fetched pages: ~{0}", pages_len);
catch (Exception exc) { for (int i = 0; i < pages_len; i++) {
bldr.Usr_dlg().Warn_many("", "", "parse failed: wiki=~{0} ttl=~{1} err=~{2}", wiki.Domain_str(), page.Ttl_full_db(), Err_.Message_gplx_log(exc)); Xowd_page_itm page = (Xowd_page_itm)pages.Get_at(i);
ctx.Wiki().Utl__bfr_mkr().Clear(); dump_bmk.Pg_id_(pg_id);
this.Free(); Exec_pg_itm(ns_ord, ns, db_id, page);
} if ( pg_id >= pg_end
} || exec_count >= exec_count_max) {
public abstract void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm page, byte[] page_text); exit_now = true;
private void Exec_commit(int ns_id, int db_id, int pg_id, byte[] ttl) { }
usr_dlg.Prog_many("", "", "committing: ns=~{0} db=~{1} pg=~{2} count=~{3} ttl=~{4}", ns_id, db_id, pg_id, exec_count, String_.new_u8(ttl)); if (exit_now) return;
Exec_commit_hook(); pg_id = page.Id();
bmk_mgr.Save(ns_id, db_id, pg_id); }
if (exit_after_commit) exit_now = true; }
} }
public abstract void Exec_commit_hook(); private void Exec_pg_itm(int ns_ord, Xow_ns ns, int db_id, Xowd_page_itm page) {
public abstract void Exec_end_hook(); try {
public void Cmd_init(Xob_bldr bldr) {} if ((exec_count % progress_interval) == 0)
public void Cmd_term() {} usr_dlg.Prog_many("", "", "parsing: ns=~{0} db=~{1} pg=~{2} count=~{3} time=~{4} rate=~{5} ttl=~{6}"
public void Cmd_end() { , ns.Id(), db_id, page.Id(), exec_count
if (!exit_now) , System_.Ticks__elapsed_in_sec(time_bgn), rate_mgr.Rate_as_str(), String_.new_u8(page.Ttl_page_db()));
pg_bgn = Int_.Max_value; ctx.Clear_all();
Exec_commit(-1, -1, -1, Bry_.Empty); byte[] page_src = page.Text();
Exec_end_hook(); if (page_src != null) // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05
Free(); Exec_pg_itm_hook(ns_ord, ns, page, page_src);
usr_dlg.Note_many("", "", "done: ~{0} ~{1}", exec_count, Decimal_adp_.divide_safe_(exec_count, System_.Ticks__elapsed_in_sec(time_bgn)).To_str("#,###.000")); ctx.Wiki().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released
} ++exec_count;
private void Free() { rate_mgr.Increment();
Xowe_wiki_.Rls_mem(wiki, true); if ((exec_count % poll_interval) == 0)
} poll_mgr.Poll();
protected void Reset_db_y_() {this.reset_db = true;} if ((exec_count % commit_interval) == 0)
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { Exec_commit(ns.Id(), db_id, page.Id(), page.Ttl_page_db());
if (ctx.Match(k, Invk_commit_interval_)) commit_interval = m.ReadInt("v"); if ((exec_count % cleanup_interval) == 0)
else if (ctx.Match(k, Invk_progress_interval_)) progress_interval = m.ReadInt("v"); Free();
else if (ctx.Match(k, Invk_rate_interval_)) rate_mgr.Reset_interval_(m.ReadInt("v")); }
else if (ctx.Match(k, Invk_cleanup_interval_)) cleanup_interval = m.ReadInt("v"); catch (Exception exc) {
else if (ctx.Match(k, Invk_select_size_)) select_size = m.ReadInt("v") * Io_mgr.Len_mb; bldr.Usr_dlg().Warn_many("", "", "parse failed: wiki=~{0} ttl=~{1} err=~{2}", wiki.Domain_str(), page.Ttl_full_db(), Err_.Message_gplx_log(exc));
else if (ctx.Match(k, Invk_ns_bgn_)) {ns_bgn = m.ReadInt("v"); Notify_restoring("ns", ns_bgn);} ctx.Wiki().Utl__bfr_mkr().Clear();
else if (ctx.Match(k, Invk_db_bgn_)) {db_bgn = m.ReadInt("v"); Notify_restoring("db", db_bgn);} this.Free();
else if (ctx.Match(k, Invk_pg_bgn_)) {pg_bgn = m.ReadInt("v"); Notify_restoring("pg", pg_bgn);} }
else if (ctx.Match(k, Invk_ns_end_)) ns_end = m.ReadInt("v"); }
else if (ctx.Match(k, Invk_db_end_)) db_end = m.ReadInt("v"); public abstract void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm page, byte[] page_text);
else if (ctx.Match(k, Invk_pg_end_)) pg_end = m.ReadInt("v"); private void Exec_commit(int ns_id, int db_id, int pg_id, byte[] ttl) {
else if (ctx.Match(k, Invk_load_tmpls_)) load_tmpls = m.ReadYn("v"); usr_dlg.Prog_many("", "", "committing: ns=~{0} db=~{1} pg=~{2} count=~{3} ttl=~{4}", ns_id, db_id, pg_id, exec_count, String_.new_u8(ttl));
else if (ctx.Match(k, Invk_poll_mgr)) return poll_mgr; Exec_commit_hook();
else if (ctx.Match(k, Invk_reset_db_)) reset_db = m.ReadYn("v"); bmk_mgr.Save(ns_id, db_id, pg_id);
else if (ctx.Match(k, Invk_exec_count_max_)) exec_count_max = m.ReadInt("v"); if (exit_after_commit) exit_now = true;
else if (ctx.Match(k, Invk_exit_now_)) exit_now = m.ReadYn("v"); }
else if (ctx.Match(k, Invk_exit_after_commit_)) exit_after_commit = m.ReadYn("v"); public abstract void Exec_commit_hook();
else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v")); public abstract void Exec_end_hook();
else return Gfo_invk_.Rv_unhandled; public void Cmd_init(Xob_bldr bldr) {}
return this; public void Cmd_term() {}
} public void Cmd_end() {
private void Notify_restoring(String itm, int val) { if (!exit_now)
usr_dlg.Note_many("", "", "restoring: itm=~{0} val=~{1}", itm, val); pg_bgn = Int_.Max_value;
} Exec_commit(-1, -1, -1, Bry_.Empty);
public static final String Exec_end_hook();
Invk_progress_interval_ = "progress_interval_", Invk_commit_interval_ = "commit_interval_", Invk_cleanup_interval_ = "cleanup_interval_", Invk_rate_interval_ = "rate_interval_" Free();
, Invk_select_size_ = "select_size_" usr_dlg.Note_many("", "", "done: ~{0} ~{1}", exec_count, Decimal_adp_.divide_safe_(exec_count, System_.Ticks__elapsed_in_sec(time_bgn)).To_str("#,###.000"));
, Invk_ns_bgn_ = "ns_bgn_", Invk_db_bgn_ = "db_bgn_", Invk_pg_bgn_ = "pg_bgn_" }
, Invk_ns_end_ = "ns_end_", Invk_db_end_ = "db_end_", Invk_pg_end_ = "pg_end_" private void Free() {
, Invk_load_tmpls_ = "load_tmpls_" Xowe_wiki_.Rls_mem(wiki, true);
, Invk_poll_mgr = "poll_mgr", Invk_reset_db_ = "reset_db_" }
, Invk_exec_count_max_ = "exec_count_max_", Invk_exit_now_ = "exit_now_", Invk_exit_after_commit_ = "exit_after_commit_" protected void Reset_db_y_() {this.reset_db = true;}
, Invk__manual_now_ = "manual_now_" @Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
; if (ctx.Match(k, Invk_commit_interval_)) commit_interval = m.ReadInt("v");
} else if (ctx.Match(k, Invk_progress_interval_)) progress_interval = m.ReadInt("v");
class Xob_dump_mgr_base_ { else if (ctx.Match(k, Invk_rate_interval_)) rate_mgr.Reset_interval_(m.ReadInt("v"));
public static void Load_all_tmpls(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, Xob_dump_src_id page_src) { else if (ctx.Match(k, Invk_cleanup_interval_)) cleanup_interval = m.ReadInt("v");
List_adp pages = List_adp_.New(); else if (ctx.Match(k, Invk_select_size_)) select_size = m.ReadInt("v") * Io_mgr.Len_mb;
Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template(); else if (ctx.Match(k, Invk_ns_bgn_)) {ns_bgn = m.ReadInt("v"); Notify_restoring("ns", ns_bgn);}
Xow_defn_cache defn_cache = wiki.Cache_mgr().Defn_cache(); else if (ctx.Match(k, Invk_db_bgn_)) {db_bgn = m.ReadInt("v"); Notify_restoring("db", db_bgn);}
int cur_page_id = -1; else if (ctx.Match(k, Invk_pg_bgn_)) {pg_bgn = m.ReadInt("v"); Notify_restoring("pg", pg_bgn);}
int load_count = 0; else if (ctx.Match(k, Invk_ns_end_)) ns_end = m.ReadInt("v");
usr_dlg.Note_many("", "", "tmpl_load init"); else if (ctx.Match(k, Invk_db_end_)) db_end = m.ReadInt("v");
while (true) { else if (ctx.Match(k, Invk_pg_end_)) pg_end = m.ReadInt("v");
page_src.Get_pages(wiki.Appe(), pages, 0, Xow_ns_.Tid__template, cur_page_id); // 0 is always template db else if (ctx.Match(k, Invk_load_tmpls_)) load_tmpls = m.ReadYn("v");
int page_count = pages.Count(); else if (ctx.Match(k, Invk_poll_mgr)) return poll_mgr;
if (page_count == 0) break; // no more pages in db; else if (ctx.Match(k, Invk_reset_db_)) reset_db = m.ReadYn("v");
Xowd_page_itm page = null; else if (ctx.Match(k, Invk_exec_count_max_)) exec_count_max = m.ReadInt("v");
for (int i = 0; i < page_count; i++) { else if (ctx.Match(k, Invk_exit_now_)) exit_now = m.ReadYn("v");
page = (Xowd_page_itm)pages.Get_at(i); else if (ctx.Match(k, Invk_exit_after_commit_)) exit_after_commit = m.ReadYn("v");
Xot_defn_tmpl defn = new Xot_defn_tmpl(); else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v"));
defn.Init_by_new(ns_tmpl, ns_tmpl.Gen_ttl(page.Ttl_page_db()), page.Text(), null, false); // NOTE: passing null, false; will be overriden later when Parse is called else return Gfo_invk_.Rv_unhandled;
defn_cache.Add(defn, ns_tmpl.Case_match()); return this;
++load_count; }
if ((load_count % 10000) == 0) usr_dlg.Prog_many("", "", "tmpl_loading: ~{0}", load_count); private void Notify_restoring(String itm, int val) {
} usr_dlg.Note_many("", "", "restoring: itm=~{0} val=~{1}", itm, val);
cur_page_id = page.Id(); }
} public static final String
usr_dlg.Note_many("", "", "tmpl_load done: ~{0}", load_count); Invk_progress_interval_ = "progress_interval_", Invk_commit_interval_ = "commit_interval_", Invk_cleanup_interval_ = "cleanup_interval_", Invk_rate_interval_ = "rate_interval_"
} , Invk_select_size_ = "select_size_"
public static Xow_db_file[] Init_text_files_ary(Xow_db_mgr core_data_mgr) { , Invk_ns_bgn_ = "ns_bgn_", Invk_db_bgn_ = "db_bgn_", Invk_pg_bgn_ = "pg_bgn_"
List_adp text_files_list = List_adp_.New(); , Invk_ns_end_ = "ns_end_", Invk_db_end_ = "db_end_", Invk_pg_end_ = "pg_end_"
int len = core_data_mgr.Dbs__len(); , Invk_load_tmpls_ = "load_tmpls_"
if (len == 1) return new Xow_db_file[] {core_data_mgr.Dbs__get_at(0)}; // single file: return core; note that there are no Tid = Text , Invk_poll_mgr = "poll_mgr", Invk_reset_db_ = "reset_db_"
for (int i = 0; i < len; i++) { , Invk_exec_count_max_ = "exec_count_max_", Invk_exit_now_ = "exit_now_", Invk_exit_after_commit_ = "exit_after_commit_"
Xow_db_file file = core_data_mgr.Dbs__get_at(i); , Invk__manual_now_ = "manual_now_"
switch (file.Tid()) { ;
case Xow_db_file_.Tid__text: }
case Xow_db_file_.Tid__text_solo: class Xob_dump_mgr_base_ {
text_files_list.Add(file); public static void Load_all_tmpls(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, Xob_dump_src_id page_src) {
break; List_adp pages = List_adp_.New();
} Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template();
} Xow_defn_cache defn_cache = wiki.Cache_mgr().Defn_cache();
return (Xow_db_file[])text_files_list.To_ary_and_clear(Xow_db_file.class); int cur_page_id = -1;
} int load_count = 0;
} usr_dlg.Note_many("", "", "tmpl_load init");
class Xob_dump_bmk_mgr { while (true) {
private Bry_bfr save_bfr = Bry_bfr_.Reset(1024); page_src.Get_pages(wiki.Appe(), pages, 0, Xow_ns_.Tid__template, cur_page_id); // 0 is always template db
public Io_url Cfg_url() {return cfg_url;} public Xob_dump_bmk_mgr Cfg_url_(Io_url v) {cfg_url = v; return this;} private Io_url cfg_url; int page_count = pages.Count();
public void Reset() {Io_mgr.Instance.DeleteFil(cfg_url);} if (page_count == 0) break; // no more pages in db;
public void Load(Xoae_app app, Xob_dump_mgr_base dump_mgr) { Xowd_page_itm page = null;
app.Gfs_mgr().Run_url_for(dump_mgr, cfg_url); for (int i = 0; i < page_count; i++) {
} page = (Xowd_page_itm)pages.Get_at(i);
public void Save(int ns_id, int db_id, int pg_id) { Xot_defn_tmpl defn = new Xot_defn_tmpl();
Save_itm(save_bfr, Xob_dump_mgr_base.Invk_ns_bgn_, ns_id); defn.Init_by_new(ns_tmpl, ns_tmpl.Gen_ttl(page.Ttl_page_db()), page.Text(), null, false); // NOTE: passing null, false; will be overriden later when Parse is called
Save_itm(save_bfr, Xob_dump_mgr_base.Invk_db_bgn_, db_id); defn_cache.Add(defn, ns_tmpl.Case_match());
Save_itm(save_bfr, Xob_dump_mgr_base.Invk_pg_bgn_, pg_id); ++load_count;
Io_mgr.Instance.SaveFilBfr(cfg_url, save_bfr); if ((load_count % 10000) == 0) usr_dlg.Prog_many("", "", "tmpl_loading: ~{0}", load_count);
} }
private void Save_itm(Bry_bfr save_bfr, String key, int val) { cur_page_id = page.Id();
String fmt = "{0}('{1}');\n"; }
String str = String_.Format(fmt, key, val); usr_dlg.Note_many("", "", "tmpl_load done: ~{0}", load_count);
save_bfr.Add_str_u8(str); }
} public static Xow_db_file[] Init_text_files_ary(Xow_db_mgr core_data_mgr) {
} List_adp text_files_list = List_adp_.New();
class Xob_rate_mgr { int len = core_data_mgr.Dbs__len();
private long time_bgn; if (len == 1) return new Xow_db_file[] {core_data_mgr.Dbs__get_at(0)}; // single file: return core; note that there are no Tid = Text
private int item_len; for (int i = 0; i < len; i++) {
private Bry_bfr save_bfr = Bry_bfr_.Reset(255); Xow_db_file file = core_data_mgr.Dbs__get_at(i);
public int Reset_interval() {return reset_interval;} public Xob_rate_mgr Reset_interval_(int v) {reset_interval = v; return this;} private int reset_interval = 10000; switch (file.Tid()) {
public Io_url Log_file_url() {return log_file;} public Xob_rate_mgr Log_file_(Io_url v) {log_file = v; return this;} private Io_url log_file; case Xow_db_file_.Tid__text:
public void Init() {time_bgn = System_.Ticks();} case Xow_db_file_.Tid__text_solo:
public void Increment() { text_files_list.Add(file);
++item_len; break;
if (item_len % reset_interval == 0) { }
long time_end = System_.Ticks(); }
Save(item_len, time_bgn, time_end); return (Xow_db_file[])text_files_list.To_ary_and_clear(Xow_db_file.class);
time_bgn = time_end; }
item_len = 0; }
} class Xob_dump_bmk_mgr {
} private Bry_bfr save_bfr = Bry_bfr_.Reset(1024);
private void Save(int count, long bgn, long end) { public Io_url Cfg_url() {return cfg_url;} public Xob_dump_bmk_mgr Cfg_url_(Io_url v) {cfg_url = v; return this;} private Io_url cfg_url;
int dif = (int)(end - bgn) / 1000; public void Reset() {Io_mgr.Instance.DeleteFil(cfg_url);}
Decimal_adp rate = Decimal_adp_.divide_safe_(count, dif); public void Load(Xoae_app app, Xob_dump_mgr_base dump_mgr) {
save_bfr app.Gfs_mgr().Run_url_for(dump_mgr, cfg_url);
.Add_str_a7(rate.To_str("#,##0.000")).Add_byte_pipe() }
.Add_int_variable(count).Add_byte_pipe() public void Save(int ns_id, int db_id, int pg_id) {
.Add_int_variable(dif).Add_byte_nl() Save_itm(save_bfr, Xob_dump_mgr_base.Invk_ns_bgn_, ns_id);
; Save_itm(save_bfr, Xob_dump_mgr_base.Invk_db_bgn_, db_id);
Io_mgr.Instance.AppendFilByt(log_file, save_bfr.To_bry_and_clear()); Save_itm(save_bfr, Xob_dump_mgr_base.Invk_pg_bgn_, pg_id);
} Io_mgr.Instance.SaveFilBfr(cfg_url, save_bfr);
public String Rate_as_str() {return Int_.To_str(Rate());} }
public int Rate() { private void Save_itm(Bry_bfr save_bfr, String key, int val) {
int elapsed = System_.Ticks__elapsed_in_sec(time_bgn); String fmt = "{0}('{1}');\n";
return Math_.Div_safe_as_int(item_len, elapsed); String str = String_.Format(fmt, key, val);
} save_bfr.Add_str_u8(str);
} }
class Xob_dump_bmk { }
public int Ns_id() {return ns_id;} public Xob_dump_bmk Ns_id_(int v) {ns_id = v; return this;} private int ns_id; class Xob_rate_mgr {
public int Db_id() {return db_id;} public Xob_dump_bmk Db_id_(int v) {db_id = v; return this;} private int db_id; private long time_bgn;
public int Pg_id() {return pg_id;} public Xob_dump_bmk Pg_id_(int v) {pg_id = v; return this;} private int pg_id; private int item_len;
} private Bry_bfr save_bfr = Bry_bfr_.Reset(255);
public int Reset_interval() {return reset_interval;} public Xob_rate_mgr Reset_interval_(int v) {reset_interval = v; return this;} private int reset_interval = 10000;
public Io_url Log_file_url() {return log_file;} public Xob_rate_mgr Log_file_(Io_url v) {log_file = v; return this;} private Io_url log_file;
public void Init() {time_bgn = System_.Ticks();}
public void Increment() {
++item_len;
if (item_len % reset_interval == 0) {
long time_end = System_.Ticks();
Save(item_len, time_bgn, time_end);
time_bgn = time_end;
item_len = 0;
}
}
private void Save(int count, long bgn, long end) {
int dif = (int)(end - bgn) / 1000;
Decimal_adp rate = Decimal_adp_.divide_safe_(count, dif);
save_bfr
.Add_str_a7(rate.To_str("#,##0.000")).Add_byte_pipe()
.Add_int_variable(count).Add_byte_pipe()
.Add_int_variable(dif).Add_byte_nl()
;
Io_mgr.Instance.AppendFilByt(log_file, save_bfr.To_bry_and_clear());
}
public String Rate_as_str() {return Int_.To_str(Rate());}
public int Rate() {
int elapsed = System_.Ticks__elapsed_in_sec(time_bgn);
return Math_.Div_safe_as_int(item_len, elapsed);
}
}
class Xob_dump_bmk {
public int Ns_id() {return ns_id;} public Xob_dump_bmk Ns_id_(int v) {ns_id = v; return this;} private int ns_id;
public int Db_id() {return db_id;} public Xob_dump_bmk Db_id_(int v) {db_id = v; return this;} private int db_id;
public int Pg_id() {return pg_id;} public Xob_dump_bmk Pg_id_(int v) {pg_id = v; return this;} private int pg_id;
}

@ -1,6 +1,6 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
@ -13,437 +13,448 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.parsers.tmpls; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*; package gplx.xowa.parsers.tmpls;
import gplx.core.envs.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.kwds.*; import gplx.xowa.langs.funcs.*; import gplx.Array_;
import gplx.xowa.xtns.pfuncs.*; import gplx.xowa.xtns.pfuncs.ttls.*; import gplx.Bool_;
import gplx.xowa.wikis.pages.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.wikis.data.tbls.*; import gplx.Bry_;
public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk { import gplx.Bry_bfr;
public Xot_invk_tkn(int bgn, int end) {this.Tkn_ini_pos(false, bgn, end);} import gplx.Bry_bfr_;
@Override public byte Tkn_tid() {return typeId;} private byte typeId = Xop_tkn_itm_.Tid_tmpl_invk; import gplx.Bry_find_;
public void Tkn_tid_to_txt() {typeId = Xop_tkn_itm_.Tid_txt;} import gplx.Byte_ascii;
public Arg_nde_tkn Name_tkn() {return name_tkn;} public Xot_invk_tkn Name_tkn_(Arg_nde_tkn v) {name_tkn = v; return this;} Arg_nde_tkn name_tkn = Arg_nde_tkn.Null; import gplx.Err_;
public byte Defn_tid() {return defn_tid;} private byte defn_tid = Xot_defn_.Tid_null; import gplx.Gfo_usr_dlg_;
public int Tmpl_subst_bgn() {return tmpl_subst_bgn;} private int tmpl_subst_bgn; import gplx.Hash_adp_bry;
public int Tmpl_subst_end() {return tmpl_subst_end;} private int tmpl_subst_end; import gplx.String_;
public Xot_invk_tkn Tmpl_subst_props_(byte tid, int bgn, int end) {defn_tid = tid; tmpl_subst_bgn = bgn; tmpl_subst_end = end; return this;} import gplx.core.envs.Env_;
public Xot_defn Tmpl_defn() {return tmpl_defn;} public Xot_invk_tkn Tmpl_defn_(Xot_defn v) {tmpl_defn = v; return this;} private Xot_defn tmpl_defn = Xot_defn_.Null; import gplx.xowa.Xoa_ttl;
public boolean Frame_is_root() {return false;} import gplx.xowa.Xoae_page;
public byte Frame_tid() {return scrib_tid;} public void Frame_tid_(byte v) {scrib_tid = v;} private byte scrib_tid; import gplx.xowa.Xowe_wiki;
public byte[] Frame_ttl() {return frame_ttl;} public void Frame_ttl_(byte[] v) {frame_ttl = v;} private byte[] frame_ttl; import gplx.xowa.langs.Xol_lang_itm;
public int Frame_lifetime() {return frame_lifetime;} public void Frame_lifetime_(int v) {frame_lifetime = v;} private int frame_lifetime; import gplx.xowa.langs.funcs.Xol_func_itm;
public boolean Rslt_is_redirect() {return rslt_is_redirect;} public void Rslt_is_redirect_(boolean v) {rslt_is_redirect = v;} private boolean rslt_is_redirect; import gplx.xowa.langs.kwds.Xol_kwd_grp;
@Override public void Tmpl_fmt(Xop_ctx ctx, byte[] src, Xot_fmtr fmtr) {fmtr.Reg_tmpl(ctx, src, name_tkn, args_len, args);} import gplx.xowa.langs.kwds.Xol_kwd_grp_;
@Override public void Tmpl_compile(Xop_ctx ctx, byte[] src, Xot_compile_data prep_data) { import gplx.xowa.langs.kwds.Xol_kwd_itm;
name_tkn.Tmpl_compile(ctx, src, prep_data); import gplx.xowa.langs.kwds.Xol_kwd_mgr;
int args_len = this.Args_len(); import gplx.xowa.parsers.Xop_ctx;
for (int i = 0; i < args_len; i++) { import gplx.xowa.parsers.Xop_tkn_itm;
Arg_nde_tkn nde = args[i]; import gplx.xowa.parsers.Xop_tkn_itm_;
Xop_tkn_itm key = nde.Key_tkn(); int key_subs_len = key.Subs_len(); import gplx.xowa.parsers.Xop_tkn_itm_base;
for (int j = 0; j < key_subs_len; j++) import gplx.xowa.wikis.caches.Xow_page_cache_itm;
key.Subs_get(j).Tmpl_compile(ctx, src, prep_data); import gplx.xowa.wikis.data.tbls.Xowd_page_itm;
Xop_tkn_itm val = nde.Val_tkn(); int val_subs_len = val.Subs_len(); import gplx.xowa.wikis.nss.Xow_ns;
for (int j = 0; j < val_subs_len; j++) import gplx.xowa.wikis.nss.Xow_ns_;
val.Subs_get(j).Tmpl_compile(ctx, src, prep_data); import gplx.xowa.wikis.nss.Xow_ns_mgr_name_itm;
} import gplx.xowa.wikis.pages.Xopg_tmpl_prepend_mgr;
} import gplx.xowa.xtns.pfuncs.ttls.Pfunc_rel2abs;
@Override public boolean Tmpl_evaluate(Xop_ctx ctx, byte[] src, Xot_invk caller, Bry_bfr bfr) { // this="{{t|{{{0}}}}}" caller="{{t|1}}"
// init common public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk {
boolean rv = false; public Xot_invk_tkn(int bgn, int end) {this.Tkn_ini_pos(false, bgn, end);}
Xowe_wiki wiki = ctx.Wiki(); @Override public byte Tkn_tid() {return typeId;} private byte typeId = Xop_tkn_itm_.Tid_tmpl_invk;
Xol_lang_itm lang = wiki.Lang(); public void Tkn_tid_to_txt() {typeId = Xop_tkn_itm_.Tid_txt;}
public Arg_nde_tkn Name_tkn() {return name_tkn;} public Xot_invk_tkn Name_tkn_(Arg_nde_tkn v) {name_tkn = v; return this;} Arg_nde_tkn name_tkn = Arg_nde_tkn.Null;
// init defn / name public byte Defn_tid() {return defn_tid;} private byte defn_tid = Xot_defn_.Tid_null;
Xot_defn defn = tmpl_defn; public int Tmpl_subst_bgn() {return tmpl_subst_bgn;} private int tmpl_subst_bgn;
byte[] name_ary = defn.Name(); public int Tmpl_subst_end() {return tmpl_subst_end;} private int tmpl_subst_end;
byte[] name_ary_orig = Bry_.Empty; public Xot_invk_tkn Tmpl_subst_props_(byte tid, int bgn, int end) {defn_tid = tid; tmpl_subst_bgn = bgn; tmpl_subst_end = end; return this;}
int name_bgn = 0, name_ary_len = 0; public Xot_defn Tmpl_defn() {return tmpl_defn;} public Xot_invk_tkn Tmpl_defn_(Xot_defn v) {tmpl_defn = v; return this;} private Xot_defn tmpl_defn = Xot_defn_.Null;
Arg_itm_tkn name_key_tkn = name_tkn.Key_tkn(); public boolean Frame_is_root() {return false;}
public byte Frame_tid() {return scrib_tid;} public void Frame_tid_(byte v) {scrib_tid = v;} private byte scrib_tid;
// init more public byte[] Frame_ttl() {return frame_ttl;} public void Frame_ttl_(byte[] v) {frame_ttl = v;} private byte[] frame_ttl;
byte[] argx_ary = Bry_.Empty; public int Frame_lifetime() {return frame_lifetime;} public void Frame_lifetime_(int v) {frame_lifetime = v;} private int frame_lifetime;
boolean subst_found = false; public boolean Rslt_is_redirect() {return rslt_is_redirect;} public void Rslt_is_redirect_(boolean v) {rslt_is_redirect = v;} private boolean rslt_is_redirect;
boolean name_had_subst = false; @Override public void Tmpl_fmt(Xop_ctx ctx, byte[] src, Xot_fmtr fmtr) {fmtr.Reg_tmpl(ctx, src, name_tkn, args_len, args);}
boolean template_prefix_found = false; @Override public void Tmpl_compile(Xop_ctx ctx, byte[] src, Xot_compile_data prep_data) {
name_tkn.Tmpl_compile(ctx, src, prep_data);
// tmpl_name does not exist in db; may be dynamic, subst, transclusion, etc.. int args_len = this.Args_len();
if (defn == Xot_defn_.Null) { for (int i = 0; i < args_len; i++) {
// dynamic tmpl; EX:{{{{{1}}}|a}} Arg_nde_tkn nde = args[i];
if (name_key_tkn.Itm_static() == Bool_.N_byte) { Xop_tkn_itm key = nde.Key_tkn(); int key_subs_len = key.Subs_len();
Bry_bfr name_tkn_bfr = Bry_bfr_.New_w_size(name_tkn.Src_end() - name_tkn.Src_bgn()); for (int j = 0; j < key_subs_len; j++)
if (defn_tid == Xot_defn_.Tid_subst) key.Subs_get(j).Tmpl_compile(ctx, src, prep_data);
name_tkn_bfr.Add(Get_first_subst_itm(lang.Kwd_mgr())); Xop_tkn_itm val = nde.Val_tkn(); int val_subs_len = val.Subs_len();
name_tkn.Tmpl_evaluate(ctx, src, caller, name_tkn_bfr); for (int j = 0; j < val_subs_len; j++)
name_ary = name_tkn_bfr.To_bry_and_clear(); val.Subs_get(j).Tmpl_compile(ctx, src, prep_data);
} }
// tmpl is static; note that dat_ary is still valid but rest of name may not be; EX: {{subst:name{{{1}}}}} }
else @Override public boolean Tmpl_evaluate(Xop_ctx ctx, byte[] src, Xot_invk caller, Bry_bfr bfr) { // this="{{t|{{{0}}}}}" caller="{{t|1}}"
name_ary = Bry_.Mid(src, name_key_tkn.Dat_bgn(), name_key_tkn.Dat_end()); // init common
name_had_subst = name_key_tkn.Dat_ary_had_subst(); boolean rv = false;
name_ary_orig = name_ary; // cache name_ary_orig Xowe_wiki wiki = ctx.Wiki();
name_ary_len = name_ary.length; Xol_lang_itm lang = wiki.Lang();
name_bgn = Bry_find_.Find_fwd_while_not_ws(name_ary, 0, name_ary_len);
if ( name_ary_len == 0 // name is blank; can occur with failed inner tmpl; EX: {{ {{does not exist}} }} // init defn / name
|| name_bgn == name_ary_len // name is ws; EX: {{test| }} -> {{{{{1}}}}}is whitespace String; PAGE:en.d:wear_one's_heart_on_one's_sleeve; EX:{{t+|fr|avoir le cœur sur la main| }} Xot_defn defn = tmpl_defn;
) { byte[] name_ary = defn.Name();
Gfo_usr_dlg_.Instance.Log_many("", "", "parser.tmpl:dynamic is blank; page=~{0}", ctx.Page().Url_bry_safe()); // downgraded from warning to note; PAGE:de.d:país DATE:2016-09-07 byte[] name_ary_orig = Bry_.Empty;
return false; int name_bgn = 0, name_ary_len = 0;
} Arg_itm_tkn name_key_tkn = name_tkn.Key_tkn();
if (name_ary[name_bgn] == Byte_ascii.Colon) { // check 1st letter for transclusion
return Transclude(ctx, wiki, bfr, Bool_.N, name_ary, caller, src); // transclusion; EX: {{:Name of page}} // init more
} byte[] argx_ary = Bry_.Empty;
boolean subst_found = false;
// ignore "{{Template:"; EX: {{Template:a}} is the same thing as {{a}} boolean name_had_subst = false;
int tmpl_ns_len = wiki.Ns_mgr().Tmpls_get_w_colon(name_ary, name_bgn, name_ary_len); boolean template_prefix_found = false;
if (tmpl_ns_len != Bry_find_.Not_found) {
name_ary = Bry_.Mid(name_ary, name_bgn + tmpl_ns_len, name_ary_len); // tmpl_name does not exist in db; may be dynamic, subst, transclusion, etc..
name_ary_len = name_ary.length; if (defn == Xot_defn_.Null) {
name_bgn = 0; // dynamic tmpl; EX:{{{{{1}}}|a}}
template_prefix_found = true; if (name_key_tkn.Itm_static() == Bool_.N_byte) {
} Bry_bfr name_tkn_bfr = Bry_bfr_.New_w_size(name_tkn.Src_end() - name_tkn.Src_bgn());
byte[] ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); int ns_template_prefix_len = ns_template_prefix.length; if (defn_tid == Xot_defn_.Tid_subst)
if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) { name_tkn_bfr.Add(Get_first_subst_itm(lang.Kwd_mgr()));
name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len); name_tkn.Tmpl_evaluate(ctx, src, caller, name_tkn_bfr);
name_ary_len = name_ary.length; name_ary = name_tkn_bfr.To_bry_and_clear();
name_bgn = 0; }
} // tmpl is static; note that dat_ary is still valid but rest of name may not be; EX: {{subst:name{{{1}}}}}
else
Xow_ns_mgr_name_itm ns_eval = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia name_ary = Bry_.Mid(src, name_key_tkn.Dat_bgn(), name_key_tkn.Dat_end());
if (ns_eval != null && !template_prefix_found) // do not transclude ns if Template prefix seen earlier; EX: {{Template:Wikipedia:A}} should not transclude "Wikipedia:A"; DATE:2013-04-03 name_had_subst = name_key_tkn.Dat_ary_had_subst();
return SubEval(ctx, wiki, bfr, name_ary, caller, src); name_ary_orig = name_ary; // cache name_ary_orig
name_ary_len = name_ary.length;
Xol_func_itm finder = new Xol_func_itm(); // TS.MEM: DATE:2016-07-12 name_bgn = Bry_find_.Find_fwd_while_not_ws(name_ary, 0, name_ary_len);
lang.Func_regy().Find_defn(finder, name_ary, name_bgn, name_ary_len); if ( name_ary_len == 0 // name is blank; can occur with failed inner tmpl; EX: {{ {{does not exist}} }}
defn = finder.Func(); || name_bgn == name_ary_len // name is ws; EX: {{test| }} -> {{{{{1}}}}}is whitespace String; PAGE:en.d:wear_one's_heart_on_one's_sleeve; EX:{{t+|fr|avoir le cœur sur la main| }}
int finder_tid = finder.Tid(); ) {
int finder_colon_pos = finder.Colon_pos(); Gfo_usr_dlg_.Instance.Log_many("", "", "parser.tmpl:dynamic is blank; page=~{0}", ctx.Page().Url_bry_safe()); // downgraded from warning to note; PAGE:de.d:país DATE:2016-09-07
int finder_subst_end = finder.Subst_end(); return false;
}
int colon_pos = -1; if (name_ary[name_bgn] == Byte_ascii.Colon) { // check 1st letter for transclusion
switch (finder_tid) { return Transclude(ctx, wiki, bfr, Bool_.N, name_ary, caller, src); // transclusion; EX: {{:Name of page}}
case Xot_defn_.Tid_subst: // subst is added verbatim; EX: {{subst:!}} -> {{subst:!}}; logic below is to handle printing of arg which could be standardized if src[] was available for tmpl }
bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary);
for (int i = 0; i < args_len; i++) { // ignore "{{Template:"; EX: {{Template:a}} is the same thing as {{a}}
Arg_nde_tkn nde = args[i]; int tmpl_ns_len = wiki.Ns_mgr().Tmpls_get_w_colon(name_ary, name_bgn, name_ary_len);
bfr.Add_byte(Byte_ascii.Pipe); // add | if (tmpl_ns_len != Bry_find_.Not_found) {
bfr.Add_mid(src, nde.Src_bgn(), nde.Src_end()); // add entire arg; "k=v"; note that src must be added, not evaluated, else <nowiki> may be dropped and cause stack overflow; PAGE:ru.w:Близкиерузья_(Сезон_2) DATE:2014-10-21 name_ary = Bry_.Mid(name_ary, name_bgn + tmpl_ns_len, name_ary_len);
} name_ary_len = name_ary.length;
Xot_fmtr_prm.Instance.Print(bfr); name_bgn = 0;
bfr.Add(Xop_curly_end_lxr.Hook); template_prefix_found = true;
return true; // NOTE: nothing else to do; return }
case Xot_defn_.Tid_safesubst: byte[] ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); int ns_template_prefix_len = ns_template_prefix.length;
name_ary = Bry_.Mid(name_ary, finder_subst_end, name_ary_len); // chop off "safesubst:" if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) {
name_ary_len = name_ary.length; name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len);
if (defn != Xot_defn_.Null) { // func found name_ary_len = name_ary.length;
if (finder_colon_pos != -1) colon_pos = defn.Name().length; // set colon_pos; SEE NOTE_1 name_bgn = 0;
} }
subst_found = true;
break; Xow_ns_mgr_name_itm ns_eval = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia
case Xot_defn_.Tid_func: if (ns_eval != null && !template_prefix_found) // do not transclude ns if Template prefix seen earlier; EX: {{Template:Wikipedia:A}} should not transclude "Wikipedia:A"; DATE:2013-04-03
if (defn.Defn_require_colon_arg()) { return SubEval(ctx, wiki, bfr, name_ary, caller, src);
colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon);
if (colon_pos == Bry_find_.Not_found) Xol_func_itm finder = new Xol_func_itm(); // TS.MEM: DATE:2016-07-12
defn = Xot_defn_.Null; lang.Func_regy().Find_defn(finder, name_ary, name_bgn, name_ary_len);
} defn = finder.Func();
else { int finder_tid = finder.Tid();
colon_pos = finder_colon_pos; int finder_colon_pos = finder.Colon_pos();
} int finder_subst_end = finder.Subst_end();
break;
case Xot_defn_.Tid_raw: int colon_pos = -1;
case Xot_defn_.Tid_msg: switch (finder_tid) {
int raw_colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon); case Xot_defn_.Tid_subst: // subst is added verbatim; EX: {{subst:!}} -> {{subst:!}}; logic below is to handle printing of arg which could be standardized if src[] was available for tmpl
if (raw_colon_pos == Bry_find_.Not_found) {} // colon missing; EX: {{raw}}; noop and assume template name; DATE:2014-02-11 bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary);
else { // colon present; for (int i = 0; i < args_len; i++) {
name_ary = Bry_.Mid(name_ary, finder_subst_end + 1, name_ary_len); // chop off "raw"; +1 is for ":"; note that +1 is in bounds b/c raw_colon was found Arg_nde_tkn nde = args[i];
name_ary_len = name_ary.length; bfr.Add_byte(Byte_ascii.Pipe); // add |
Xow_ns_mgr_name_itm ns_eval2 = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia bfr.Add_mid(src, nde.Src_bgn(), nde.Src_end()); // add entire arg; "k=v"; note that src must be added, not evaluated, else <nowiki> may be dropped and cause stack overflow; PAGE:ru.w:Близкиерузья_(Сезон_2) DATE:2014-10-21
if (ns_eval2 != null) { }
Xow_ns ns_eval_itm = ns_eval2.Ns(); Xot_fmtr_prm.Instance.Print(bfr);
int pre_len = ns_eval_itm.Name_enc().length; bfr.Add(Xop_curly_end_lxr.Hook);
if (pre_len < name_ary_len && name_ary[pre_len] == Byte_ascii.Colon) return true; // NOTE: nothing else to do; return
return SubEval(ctx, wiki, bfr, name_ary, caller, src); case Xot_defn_.Tid_safesubst:
} name_ary = Bry_.Mid(name_ary, finder_subst_end, name_ary_len); // chop off "safesubst:"
} name_ary_len = name_ary.length;
switch (finder_tid) { if (defn != Xot_defn_.Null) { // func found
case Xot_defn_.Tid_msg: if (finder_colon_pos != -1) colon_pos = defn.Name().length; // set colon_pos; SEE NOTE_1
defn = Xot_defn_.Null; // null out defn to force template load below; DATE:2014-07-10 }
break; subst_found = true;
} break;
break; case Xot_defn_.Tid_func:
} if (defn.Defn_require_colon_arg()) {
if (subst_found) {// subst found; remove Template: if it exists; EX: {{safesubst:Template:A}} -> {{A}} not {{Template:A}}; EX:en.d:Kazakhstan; DATE:2014-03-25 colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon);
ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); ns_template_prefix_len = ns_template_prefix.length; if (colon_pos == Bry_find_.Not_found)
if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) { defn = Xot_defn_.Null;
name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len); }
name_ary_len = name_ary.length; else {
name_bgn = 0; colon_pos = finder_colon_pos;
template_prefix_found = true; }
} break;
} case Xot_defn_.Tid_raw:
if (colon_pos != -1) { // func; separate name_ary into name_ary and arg_x case Xot_defn_.Tid_msg:
argx_ary = Bry_.Trim(name_ary, colon_pos + 1, name_ary_len); // trim bgn ws; needed for "{{formatnum:\n{{#expr:2}}\n}}" int raw_colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon);
name_ary = Bry_.Mid(name_ary, 0, colon_pos); if (raw_colon_pos == Bry_find_.Not_found) {} // colon missing; EX: {{raw}}; noop and assume template name; DATE:2014-02-11
} else { // colon present;
if (defn == Xot_defn_.Null) { name_ary = Bry_.Mid(name_ary, finder_subst_end + 1, name_ary_len); // chop off "raw"; +1 is for ":"; note that +1 is in bounds b/c raw_colon was found
if (ctx.Tid_is_popup()) { // popup && cur_tmpl > tmpl_max name_ary_len = name_ary.length;
if (Popup_skip(ctx, name_ary, bfr)) return false; Xow_ns_mgr_name_itm ns_eval2 = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia
} if (ns_eval2 != null) {
defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary); Xow_ns ns_eval_itm = ns_eval2.Ns();
if (defn == null) { int pre_len = ns_eval_itm.Name_enc().length;
if (name_ary_len != 0 ) { // name_ary_len != 0 for direct template inclusions; PAGE:en.w:Human evolution and {{:Human evolution/Species chart}}; && ctx.Tmpl_whitelist().Has(name_ary) if (pre_len < name_ary_len && name_ary[pre_len] == Byte_ascii.Colon)
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary)); return SubEval(ctx, wiki, bfr, name_ary, caller, src);
if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args ); }
if (subst_found || template_prefix_found) { // if "subst:" or "Template:" found, use orig name; DATE:2014-03-31 }
bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary_orig).Add(Xop_curly_end_lxr.Hook); switch (finder_tid) {
return false; case Xot_defn_.Tid_msg:
} defn = Xot_defn_.Null; // null out defn to force template load below; DATE:2014-07-10
else {// output entire tmpl_src WITH args; used to output name only which broke pages; PAGE:en.w:Flag_of_Greenland; DATE:2016-06-21 break;
bfr.Add(Xop_curly_bgn_lxr.Hook); }
bfr.Add(name_ary); break;
for (int i = 0; i < args_len; ++i) { }
Arg_nde_tkn nde = this.Args_get_by_idx(i); if (subst_found) {// subst found; remove Template: if it exists; EX: {{safesubst:Template:A}} -> {{A}} not {{Template:A}}; EX:en.d:Kazakhstan; DATE:2014-03-25
bfr.Add_byte(Byte_ascii.Pipe); ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); ns_template_prefix_len = ns_template_prefix.length;
// must evaluate args; "size={{{size|}}}" must become "size="; PAGE:en.w:Europe; en.w:Template:Country_data_Guernsey DATE:2016-10-13 if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) {
nde.Tmpl_compile(ctx, src, Xot_compile_data.Noop); name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len);
nde.Tmpl_evaluate(ctx, src, caller, bfr); name_ary_len = name_ary.length;
} name_bgn = 0;
bfr.Add(Xop_curly_end_lxr.Hook); template_prefix_found = true;
return false; }
} }
} if (colon_pos != -1) { // func; separate name_ary into name_ary and arg_x
else { // some templates produce null ttls; EX: "Citation needed{{subst" argx_ary = Bry_.Trim(name_ary, colon_pos + 1, name_ary_len); // trim bgn ws; needed for "{{formatnum:\n{{#expr:2}}\n}}"
defn = wiki.Cache_mgr().Defn_cache().Get_by_key(ttl.Page_db()); name_ary = Bry_.Mid(name_ary, 0, colon_pos);
if (defn == null && ctx.Tmpl_load_enabled()) }
defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary); if (defn == Xot_defn_.Null) {
} if (ctx.Tid_is_popup()) { // popup && cur_tmpl > tmpl_max
} if (Popup_skip(ctx, name_ary, bfr)) return false;
} }
if (defn == null) defn = Xot_defn_.Null; defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary);
} if (defn == null) {
} if (name_ary_len != 0 ) { // name_ary_len != 0 for direct template inclusions; PAGE:en.w:Human evolution and {{:Human evolution/Species chart}}; && ctx.Tmpl_whitelist().Has(name_ary)
if ( defn.Defn_tid() == Xot_defn_.Tid_null // name is not a known defn Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary));
&& lang.Vnt_mgr().Enabled()) { // lang has vnts if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args );
Xowd_page_itm page = lang.Vnt_mgr().Convert_mgr().Convert_ttl(wiki, wiki.Ns_mgr().Ns_template(), name_ary); if (subst_found || template_prefix_found) { // if "subst:" or "Template:" found, use orig name; DATE:2014-03-31
if (page != Xowd_page_itm.Null) { bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary_orig).Add(Xop_curly_end_lxr.Hook);
name_ary = page.Ttl_page_db(); return false;
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary)); }
if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args ); else {// output entire tmpl_src WITH args; used to output name only which broke pages; PAGE:en.w:Flag_of_Greenland; DATE:2016-06-21
bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary).Add(Xop_curly_end_lxr.Hook); bfr.Add(Xop_curly_bgn_lxr.Hook);
return false; bfr.Add(name_ary);
} for (int i = 0; i < args_len; ++i) {
defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary); Arg_nde_tkn nde = this.Args_get_by_idx(i);
if (defn == null && ctx.Tmpl_load_enabled()) bfr.Add_byte(Byte_ascii.Pipe);
defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary); // must evaluate args; "size={{{size|}}}" must become "size="; PAGE:en.w:Europe; en.w:Template:Country_data_Guernsey DATE:2016-10-13
if (defn == null) defn = Xot_defn_.Null; nde.Tmpl_compile(ctx, src, Xot_compile_data.Noop);
} nde.Tmpl_evaluate(ctx, src, caller, bfr);
} }
Xot_defn_trace trace = ctx.Defn_trace(); int trg_bgn = bfr.Len(); bfr.Add(Xop_curly_end_lxr.Hook);
switch (defn.Defn_tid()) { return false;
case Xot_defn_.Tid_null: // defn is unknown }
if (ignore_hash.Get_by_bry(name_ary) == null) { }
if (Pfunc_rel2abs.Rel2abs_ttl(name_ary, name_bgn, name_ary_len)) {// rel_path; EX: {{/../Peer page}}; DATE:2013-03-27 else { // some templates produce null ttls; EX: "Citation needed{{subst"
Bry_bfr tmp_bfr = ctx.Wiki().Utl__bfr_mkr().Get_b512(); defn = wiki.Cache_mgr().Defn_cache().Get_by_key(ttl.Page_db());
name_ary = Pfunc_rel2abs.Rel2abs(tmp_bfr, wiki.Parser_mgr().Rel2abs_ary(), Bry_.Mid(name_ary, name_bgn, name_ary_len), ctx.Page().Ttl().Raw()); if (defn == null && ctx.Tmpl_load_enabled())
tmp_bfr.Mkr_rls(); defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary);
return SubEval(ctx, wiki, bfr, name_ary, caller, src); }
} }
if (subst_found) }
return Transclude(ctx, wiki, bfr, template_prefix_found, name_ary, caller, src); if (defn == null) defn = Xot_defn_.Null;
Xot_invk_tkn_.Print_not_found__w_template(bfr, wiki.Ns_mgr(), name_ary); }
return false; }
} if ( defn.Defn_tid() == Xot_defn_.Tid_null // name is not a known defn
break; && lang.Vnt_mgr().Enabled()) { // lang has vnts
case Xot_defn_.Tid_func: Xowd_page_itm page = lang.Vnt_mgr().Convert_mgr().Convert_ttl(wiki, wiki.Ns_mgr().Ns_template(), name_ary);
try { if (page != Xowd_page_itm.Null) {
Xot_invk_tkn_.Eval_func(ctx, src, caller, this, bfr, defn, argx_ary); name_ary = page.Ttl_page_db();
rv = true; Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary));
} catch (Exception e) { if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args );
if (Env_.Mode_testing()) bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary).Add(Xop_curly_end_lxr.Hook);
throw Err_.new_exc(e, "xo", "failed to evaluate function", "page", ctx.Page().Ttl().Full_txt(), "defn", defn.Name(), "src", String_.new_u8(src, this.Src_bgn(), this.Src_end())); return false;
else { }
wiki.Appe().Usr_dlg().Warn_many("", "", "failed to evaluate function: page=~{0} defn=~{1} src=~{2} err=~{3}", ctx.Page().Ttl().Full_txt(), defn.Name(), Bry_.Replace_nl_w_tab(src, this.Src_bgn(), this.Src_end()), Err_.Message_gplx_log(e)); defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary);
rv = false; if (defn == null && ctx.Tmpl_load_enabled())
} defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary);
} if (defn == null) defn = Xot_defn_.Null;
break; }
default: }
Xot_defn_tmpl defn_tmpl = (Xot_defn_tmpl)defn; Xot_defn_trace trace = ctx.Defn_trace(); int trg_bgn = bfr.Len();
if (defn_tmpl.Root() == null) defn_tmpl.Parse_tmpl(ctx); // NOTE: defn_tmpl.Root can be null after clearing out cache; must be non-null else will fail in trace; DATE:2013-02-01 switch (defn.Defn_tid()) {
Xot_invk invk_tmpl = Xot_defn_tmpl_.CopyNew(ctx, defn_tmpl, this, caller, src, Xow_ns_.Tid__template, name_ary); case Xot_defn_.Tid_null: // defn is unknown
invk_tmpl.Frame_ttl_(defn_tmpl.Frame_ttl()); // set frame_ttl; needed for redirects; PAGE:en.w:Statutory_city; DATE:2014-08-22 if (ignore_hash.Get_by_bry(name_ary) == null) {
trace.Trace_bgn(ctx, src, name_ary, caller, invk_tmpl, defn); if (Pfunc_rel2abs.Rel2abs_ttl(name_ary, name_bgn, name_ary_len)) {// rel_path; EX: {{/../Peer page}}; DATE:2013-03-27
Bry_bfr tmp_bfr = ctx.Wiki().Utl__bfr_mkr().Get_b512();
Bry_bfr rslt_bfr = wiki.Utl__bfr_mkr().Get_k004(); name_ary = Pfunc_rel2abs.Rel2abs(tmp_bfr, wiki.Parser_mgr().Rel2abs_ary(), Bry_.Mid(name_ary, name_bgn, name_ary_len), ctx.Page().Ttl().Raw());
try { tmp_bfr.Mkr_rls();
Xot_invk_tkn_.Bld_key(invk_tmpl, name_ary, rslt_bfr); return SubEval(ctx, wiki, bfr, name_ary, caller, src);
byte[] rslt_key = rslt_bfr.To_bry_and_clear(); }
Object o = wiki.Cache_mgr().Tmpl_result_cache().Get_by(rslt_key); if (subst_found)
Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(bfr); return Transclude(ctx, wiki, bfr, template_prefix_found, name_ary, caller, src);
if (o != null) { Xot_invk_tkn_.Print_not_found__w_template(bfr, wiki.Ns_mgr(), name_ary);
byte[] rslt = (byte[])o; return false;
prepend_mgr.End(ctx, bfr, rslt, rslt.length, Bool_.Y); }
bfr.Add(rslt); break;
} case Xot_defn_.Tid_func:
else { try {
rv = defn_tmpl.Tmpl_evaluate(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), invk_tmpl, rslt_bfr); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17 Xot_invk_tkn_.Eval_func(ctx, src, caller, this, bfr, defn, argx_ary);
prepend_mgr.End(ctx, bfr, rslt_bfr.Bfr(), rslt_bfr.Len(), Bool_.Y); rv = true;
if (name_had_subst) { // current invk had "subst:"; parse incoming invk again to remove effects of subst; PAGE:pt.w:Argentina DATE:2014-09-24 } catch (Exception e) {
byte[] tmp_src = rslt_bfr.To_bry_and_clear(); if (Env_.Mode_testing())
rslt_bfr.Add(wiki.Parser_mgr().Main().Expand_tmpl(tmp_src)); // this could be cleaner / more optimized throw Err_.new_exc(e, "xo", "failed to evaluate function", "page", ctx.Page().Ttl().Full_txt(), "defn", defn.Name(), "src", String_.new_u8(src, this.Src_bgn(), this.Src_end()));
} else {
if (Cache_enabled) { wiki.Appe().Usr_dlg().Warn_many("", "", "failed to evaluate function: page=~{0} defn=~{1} src=~{2} err=~{3}", ctx.Page().Ttl().Full_txt(), defn.Name(), Bry_.Replace_nl_w_tab(src, this.Src_bgn(), this.Src_end()), Err_.Message_gplx_log(e));
byte[] rslt_val = rslt_bfr.To_bry_and_clear(); rv = false;
bfr.Add(rslt_val); }
Hash_adp cache = wiki.Cache_mgr().Tmpl_result_cache(); }
cache.Del(rslt_key); break;
cache.Add(rslt_key, rslt_val); default:
} Xot_defn_tmpl defn_tmpl = (Xot_defn_tmpl)defn;
else if (defn_tmpl.Root() == null) defn_tmpl.Parse_tmpl(ctx); // NOTE: defn_tmpl.Root can be null after clearing out cache; must be non-null else will fail in trace; DATE:2013-02-01
bfr.Add_bfr_and_clear(rslt_bfr); Xot_invk invk_tmpl = Xot_defn_tmpl_.CopyNew(ctx, defn_tmpl, this, caller, src, Xow_ns_.Tid__template, name_ary);
} invk_tmpl.Frame_ttl_(defn_tmpl.Frame_ttl()); // set frame_ttl; needed for redirects; PAGE:en.w:Statutory_city; DATE:2014-08-22
trace.Trace_end(trg_bgn, bfr); trace.Trace_bgn(ctx, src, name_ary, caller, invk_tmpl, defn);
} finally {rslt_bfr.Mkr_rls();}
break; Bry_bfr rslt_bfr = wiki.Utl__bfr_mkr().Get_k004();
} try {
return rv; Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(bfr);
} rv = defn_tmpl.Tmpl_evaluate(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), invk_tmpl, rslt_bfr); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17
private boolean Popup_skip(Xop_ctx ctx, byte[] ttl, Bry_bfr bfr) { prepend_mgr.End(ctx, bfr, rslt_bfr.Bfr(), rslt_bfr.Len(), Bool_.Y);
boolean skip = false; if (name_had_subst) { // current invk had "subst:"; parse incoming invk again to remove effects of subst; PAGE:pt.w:Argentina DATE:2014-09-24
skip = this.Src_end() - this.Src_bgn() > ctx.Tmpl_tkn_max(); byte[] tmp_src = rslt_bfr.To_bry_and_clear();
if (!skip) { rslt_bfr.Add(wiki.Parser_mgr().Main().Expand_tmpl(tmp_src)); // this could be cleaner / more optimized
gplx.xowa.htmls.modules.popups.keeplists.Xop_keeplist_wiki tmpl_keeplist = ctx.Tmpl_keeplist(); }
if (tmpl_keeplist != null && tmpl_keeplist.Enabled()) { bfr.Add_bfr_and_clear(rslt_bfr);
byte[] ttl_lower = Xoa_ttl.Replace_spaces(ctx.Wiki().Lang().Case_mgr().Case_build_lower(ttl)); trace.Trace_end(trg_bgn, bfr);
skip = !tmpl_keeplist.Match(ttl_lower); } finally {rslt_bfr.Mkr_rls();}
// if (skip) Tfds.Write_bry(ttl_lower); break;
} }
} return rv;
if (skip) { }
bfr.Add(gplx.xowa.parsers.miscs.Xop_comm_lxr.Xowa_skip_comment_bry); // add comment tkn; need something to separate ''{{lang|la|Ragusa}}'' else will become ''''; PAGE:en.w:Republic_of_Ragusa; DATE:2014-06-28 private boolean Popup_skip(Xop_ctx ctx, byte[] ttl, Bry_bfr bfr) {
return true; boolean skip = false;
} skip = this.Src_end() - this.Src_bgn() > ctx.Tmpl_tkn_max();
else if (!skip) {
return false; gplx.xowa.htmls.modules.popups.keeplists.Xop_keeplist_wiki tmpl_keeplist = ctx.Tmpl_keeplist();
} if (tmpl_keeplist != null && tmpl_keeplist.Enabled()) {
private boolean Transclude(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, boolean template_prefix_found, byte[] name_ary, Xot_invk caller, byte[] src) { byte[] ttl_lower = Xoa_ttl.Replace_spaces(ctx.Wiki().Lang().Case_mgr().Case_build_lower(ttl));
Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}} skip = !tmpl_keeplist.Match(ttl_lower);
byte[] transclude_src = null; // if (skip) Tfds.Write_bry(ttl_lower);
if (page_ttl.Ns().Id_is_tmpl()) { // ttl is template; check tmpl_regy first before going to data_mgr }
Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db()); }
if (tmpl != null) transclude_src = tmpl.Data_raw(); if (skip) {
} bfr.Add(gplx.xowa.parsers.miscs.Xop_comm_lxr.Xowa_skip_comment_bry); // add comment tkn; need something to separate ''{{lang|la|Ragusa}}'' else will become ''''; PAGE:en.w:Republic_of_Ragusa; DATE:2014-06-28
if (transclude_src == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load return true;
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl); }
if ( cache_itm != null else
// && Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Page_ttl().Full_db()) // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10 return false;
) { }
transclude_src = cache_itm.Wtxt__direct(); private boolean Transclude(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, boolean template_prefix_found, byte[] name_ary, Xot_invk caller, byte[] src) {
page_ttl = cache_itm.Ttl(); Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}}
} byte[] transclude_src = null;
} if (page_ttl.Ns().Id_is_tmpl()) { // ttl is template; check tmpl_regy first before going to data_mgr
if (transclude_src != null) { Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db());
// NOTE: must use new page, not current, else transcluded page can cause inconsistent TOC state; PAGE:de.w:Game_of_Thrones DATE:2016-11-21 if (tmpl != null) transclude_src = tmpl.Data_raw();
Xot_defn_tmpl transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, Xoae_page.New(wiki, page_ttl)), ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), transclude_src); }
return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src, bfr); if (transclude_src == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load
} Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
else { if ( cache_itm != null
Xot_invk_tkn_.Print_not_found__by_transclude(bfr, wiki.Ns_mgr(), template_prefix_found, name_ary); // && Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Page_ttl().Full_db()) // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
return false; ) {
} transclude_src = cache_itm.Wtxt__direct();
} page_ttl = cache_itm.Ttl();
private boolean Eval_sub(Xop_ctx ctx, Xoa_ttl transclude_ttl, Xot_defn_tmpl transclude_tmpl, Xot_invk caller, byte[] src, Bry_bfr doc) { }
boolean rv = false; }
Xot_invk tmp_tmpl = Xot_defn_tmpl_.CopyNew(ctx, transclude_tmpl, this, caller, src, transclude_ttl.Ns().Id(), transclude_tmpl.Name()); if (transclude_src != null) {
Bry_bfr tmp_bfr = Bry_bfr_.New(); // NOTE: must use new page, not current, else transcluded page can cause inconsistent TOC state; PAGE:de.w:Game_of_Thrones DATE:2016-11-21
Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(doc); Xot_defn_tmpl transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, Xoae_page.New(wiki, page_ttl)), ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), transclude_src);
rv = transclude_tmpl.Tmpl_evaluate(ctx, tmp_tmpl, tmp_bfr); return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src, bfr);
prepend_mgr.End(ctx, doc, tmp_bfr.Bfr(), tmp_bfr.Len(), Bool_.Y); }
doc.Add_bfr_and_clear(tmp_bfr); else {
return rv; Xot_invk_tkn_.Print_not_found__by_transclude(bfr, wiki.Ns_mgr(), template_prefix_found, name_ary);
} return false;
private boolean SubEval(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, byte[] name_ary, Xot_invk caller, byte[] src_for_tkn) { }
Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}} }
Xot_defn_tmpl transclude_tmpl = null; private boolean Eval_sub(Xop_ctx ctx, Xoa_ttl transclude_ttl, Xot_defn_tmpl transclude_tmpl, Xot_invk caller, byte[] src, Bry_bfr doc) {
switch (page_ttl.Ns().Id()) { boolean rv = false;
case Xow_ns_.Tid__template: // ttl is template not in cache, or some other ns; do load Xot_invk tmp_tmpl = Xot_defn_tmpl_.CopyNew(ctx, transclude_tmpl, this, caller, src, transclude_ttl.Ns().Id(), transclude_tmpl.Name());
Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db()); Bry_bfr tmp_bfr = Bry_bfr_.New();
if (tmpl != null) { Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(doc);
if (tmpl.Root() == null) tmpl.Parse_tmpl(ctx); rv = transclude_tmpl.Tmpl_evaluate(ctx, tmp_tmpl, tmp_bfr);
transclude_tmpl = tmpl; prepend_mgr.End(ctx, doc, tmp_bfr.Bfr(), tmp_bfr.Len(), Bool_.Y);
} doc.Add_bfr_and_clear(tmp_bfr);
break; return rv;
case Xow_ns_.Tid__special: }
bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon).Add(name_ary).Add(Xop_tkn_.Lnki_end); private boolean SubEval(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, byte[] name_ary, Xot_invk caller, byte[] src_for_tkn) {
return true; Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}}
} Xot_defn_tmpl transclude_tmpl = null;
if (transclude_tmpl == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load switch (page_ttl.Ns().Id()) {
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl); case Xow_ns_.Tid__template: // ttl is template not in cache, or some other ns; do load
if ( cache_itm != null) { Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db());
if (!Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Ttl().Full_db())) { // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10 if (tmpl != null) {
transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), cache_itm.Wtxt__direct()); if (tmpl.Root() == null) tmpl.Parse_tmpl(ctx);
page_ttl = cache_itm.Ttl(); transclude_tmpl = tmpl;
} }
} break;
} case Xow_ns_.Tid__special:
if (transclude_tmpl == null) { bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon).Add(name_ary).Add(Xop_tkn_.Lnki_end);
bfr.Add(Xop_tkn_.Lnki_bgn).Add(name_ary).Add(Xop_tkn_.Lnki_end); // indicate template was not found; DATE:2014-02-12 return true;
return false; }
} if (transclude_tmpl == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load
return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src_for_tkn, bfr); Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
} if ( cache_itm != null) {
public int Args_len() {return args_len;} private int args_len = 0; if (!Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Ttl().Full_db())) { // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
public Arg_nde_tkn Args_get_by_idx(int idx) {return args[idx];} transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), cache_itm.Wtxt__direct());
public Arg_nde_tkn Args_eval_by_idx(byte[] src, int idx) { page_ttl = cache_itm.Ttl();
int cur = 0; }
for (int i = 0; i < args_len; i++) { }
Arg_nde_tkn nde = args[i]; }
if (nde.KeyTkn_exists()) continue; if (transclude_tmpl == null) {
if (cur++ == idx) return nde; bfr.Add(Xop_tkn_.Lnki_bgn).Add(name_ary).Add(Xop_tkn_.Lnki_end); // indicate template was not found; DATE:2014-02-12
} return false;
return null; }
} return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src_for_tkn, bfr);
public Arg_nde_tkn Args_get_by_key(byte[] src, byte[] key) { }
for (int i = 0; i < args_len; i++) { public int Args_len() {return args_len;} private int args_len = 0;
Arg_nde_tkn nde = args[i]; public Arg_nde_tkn Args_get_by_idx(int idx) {return args[idx];}
if (!nde.KeyTkn_exists()) continue; public Arg_nde_tkn Args_eval_by_idx(byte[] src, int idx) {
if (Bry_.Match(src, nde.Key_tkn().Dat_bgn(), nde.Key_tkn().Dat_end(), key)) return nde; // NOTE: dat_ary is guaranteed to exist int cur = 0;
} for (int i = 0; i < args_len; i++) {
return null; Arg_nde_tkn nde = args[i];
} if (nde.KeyTkn_exists()) continue;
public void Args_add(Xop_ctx ctx, Arg_nde_tkn arg) { if (cur++ == idx) return nde;
int newLen = args_len + 1; }
if (newLen > args_max) { return null;
args_max = newLen * 2; }
args = Resize(args, args_len, args_max); public Arg_nde_tkn Args_get_by_key(byte[] src, byte[] key) {
} for (int i = 0; i < args_len; i++) {
args[args_len] = arg; Arg_nde_tkn nde = args[i];
arg.Tkn_grp_(this, args_len); if (!nde.KeyTkn_exists()) continue;
args_len = newLen; if (Bry_.Match(src, nde.Key_tkn().Dat_bgn(), nde.Key_tkn().Dat_end(), key)) return nde; // NOTE: dat_ary is guaranteed to exist
} Arg_nde_tkn[] args = Arg_nde_tkn.Ary_empty; int args_max; }
private Arg_nde_tkn[] Resize(Arg_nde_tkn[] src, int cur_len, int new_len) { return null;
Arg_nde_tkn[] rv = new Arg_nde_tkn[new_len]; }
Array_.Copy_to(src, 0, rv, 0, cur_len); public void Args_add(Xop_ctx ctx, Arg_nde_tkn arg) {
return rv; int newLen = args_len + 1;
} if (newLen > args_max) {
private byte[] Get_first_subst_itm(Xol_kwd_mgr kwd_mgr) { args_max = newLen * 2;
Xol_kwd_grp grp = kwd_mgr.Get_at(Xol_kwd_grp_.Id_subst); if (grp == null) return Bry_.Empty; args = Resize(args, args_len, args_max);
Xol_kwd_itm[] itms = grp.Itms(); }
return itms.length == 0 ? Bry_.Empty : itms[0].Val(); args[args_len] = arg;
} arg.Tkn_grp_(this, args_len);
private static final Hash_adp_bry ignore_hash = Hash_adp_bry.ci_a7().Add_str_obj("Citation needed{{subst", "").Add_str_obj("Clarify{{subst", ""); // ignore SafeSubst templates args_len = newLen;
public static boolean Cache_enabled = false; } Arg_nde_tkn[] args = Arg_nde_tkn.Ary_empty; int args_max;
} private Arg_nde_tkn[] Resize(Arg_nde_tkn[] src, int cur_len, int new_len) {
/* Arg_nde_tkn[] rv = new Arg_nde_tkn[new_len];
NOTE_1: if (finder.Colon_pos() != -1) colon_pos = finder.Func().Name().length; Array_.Copy_to(src, 0, rv, 0, cur_len);
Two issues here: return rv;
1) "if (finder.Colon_pos() != -1)" }
Colon_pos can either be -1 or >0 private byte[] Get_first_subst_itm(Xol_kwd_mgr kwd_mgr) {
EX: -1: safesubst:NAMESPACE Xol_kwd_grp grp = kwd_mgr.Get_at(Xol_kwd_grp_.Id_subst); if (grp == null) return Bry_.Empty;
EX: >0: safesubst:#expr:0 Xol_kwd_itm[] itms = grp.Itms();
if -1, don't do anything; this will leave colon_pos as -1 return itms.length == 0 ? Bry_.Empty : itms[0].Val();
2) "finder.Func().Name().length" }
Colon_pos is >0, but refers to String before it was chopped private static final Hash_adp_bry ignore_hash = Hash_adp_bry.ci_a7().Add_str_obj("Citation needed{{subst", "").Add_str_obj("Clarify{{subst", ""); // ignore SafeSubst templates
EX: "safesubst:#expr:0" has Colon_pos of 15 but String is now "#expr:0" }
so, get colon_pos by taking the finder.Func().Name().length /*
NOTE: whitespace does not matter b/c "safesubst: #expr:0" would never be a func; NOTE_1: if (finder.Colon_pos() != -1) colon_pos = finder.Func().Name().length;
Two issues here:
1) "if (finder.Colon_pos() != -1)"
Colon_pos can either be -1 or >0
EX: -1: safesubst:NAMESPACE
EX: >0: safesubst:#expr:0
if -1, don't do anything; this will leave colon_pos as -1
2) "finder.Func().Name().length"
Colon_pos is >0, but refers to String before it was chopped
EX: "safesubst:#expr:0" has Colon_pos of 15 but String is now "#expr:0"
so, get colon_pos by taking the finder.Func().Name().length
NOTE: whitespace does not matter b/c "safesubst: #expr:0" would never be a func;
*/ */

@ -1,6 +1,6 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
@ -13,67 +13,66 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.parsers.tmpls; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*; package gplx.xowa.parsers.tmpls;
import gplx.xowa.langs.kwds.*;
import gplx.xowa.xtns.pfuncs.*; import gplx.Bool_;
import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.caches.*; import gplx.Bry_;
public class Xot_invk_tkn_ { import gplx.Bry_bfr;
public static void Eval_func(Xop_ctx ctx, byte[] src, Xot_invk caller, Xot_invk invk, Bry_bfr bfr, Xot_defn defn, byte[] argx_ary) { import gplx.Bry_bfr_;
Pf_func_base defn_func = (Pf_func_base)defn; import gplx.Byte_ascii;
int defn_func_id = defn_func.Id(); import gplx.xowa.Xoa_ttl;
defn_func = (Pf_func_base)defn_func.New(defn_func_id, defn_func.Name()); // NOTE: always make copy b/c argx_ary may be dynamic import gplx.xowa.Xowe_wiki;
if (argx_ary != Bry_.Empty) defn_func.Argx_dat_(argx_ary); import gplx.xowa.langs.kwds.Xol_kwd_grp_;
defn_func.Eval_argx(ctx, src, caller, invk); import gplx.xowa.parsers.Xop_ctx;
if (defn_func_id == Xol_kwd_grp_.Id_invoke) // NOTE: if #invoke, set frame_ttl to argx, not name; EX:{{#invoke:A}} import gplx.xowa.wikis.caches.Xow_page_cache_itm;
invk.Frame_ttl_(Bry_.Add(Xow_ns_.Bry__module_w_colon, Xoa_ttl.Replace_unders(defn_func.Argx_dat()))); // NOTE: always prepend "Module:" to frame_ttl; DATE:2014-06-13; NOTE: always use spaces; DATE:2014-08-14; always use canonical English "Module"; DATE:2015-11-09 import gplx.xowa.wikis.nss.Xow_ns;
Bry_bfr bfr_func = Bry_bfr_.New(); import gplx.xowa.wikis.nss.Xow_ns_;
defn_func.Func_evaluate(bfr_func, ctx, caller, invk, src); import gplx.xowa.wikis.nss.Xow_ns_mgr;
if (caller.Rslt_is_redirect()) // do not prepend if page is redirect; EX:"#REDIRECT" x> "\n#REDIRECT" DATE:2014-07-11 import gplx.xowa.xtns.pfuncs.Pf_func_base;
caller.Rslt_is_redirect_(false); // reset flag; needed for TEST; kludgy, but Rslt_is_redirect is intended for single use
else public class Xot_invk_tkn_ {
ctx.Page().Tmpl_prepend_mgr().End(ctx, bfr, bfr_func.Bfr(), bfr_func.Len(), Bool_.N); public static void Eval_func(Xop_ctx ctx, byte[] src, Xot_invk caller, Xot_invk invk, Bry_bfr bfr, Xot_defn defn, byte[] argx_ary) {
bfr.Add_bfr_and_clear(bfr_func); Pf_func_base defn_func = (Pf_func_base)defn;
} int defn_func_id = defn_func.Id();
public static void Bld_key(Xot_invk invk, byte[] name_ary, Bry_bfr bfr) { defn_func = (Pf_func_base)defn_func.New(defn_func_id, defn_func.Name()); // NOTE: always make copy b/c argx_ary may be dynamic
bfr.Clear(); if (argx_ary != Bry_.Empty) defn_func.Argx_dat_(argx_ary);
bfr.Add(name_ary); defn_func.Eval_argx(ctx, src, caller, invk);
int args_len = invk.Args_len(); if (defn_func_id == Xol_kwd_grp_.Id_invoke) // NOTE: if #invoke, set frame_ttl to argx, not name; EX:{{#invoke:A}}
for (int i = 0; i < args_len; i++) { invk.Frame_ttl_(Bry_.Add(Xow_ns_.Bry__module_w_colon, Xoa_ttl.Replace_unders(defn_func.Argx_dat()))); // NOTE: always prepend "Module:" to frame_ttl; DATE:2014-06-13; NOTE: always use spaces; DATE:2014-08-14; always use canonical English "Module"; DATE:2015-11-09
Arg_nde_tkn nde = invk.Args_get_by_idx(i); Bry_bfr bfr_func = Bry_bfr_.New();
bfr.Add_byte(Byte_ascii.Pipe); defn_func.Func_evaluate(bfr_func, ctx, caller, invk, src);
if (nde.KeyTkn_exists()) { if (caller.Rslt_is_redirect()) // do not prepend if page is redirect; EX:"#REDIRECT" x> "\n#REDIRECT" DATE:2014-07-11
bfr.Add(nde.Key_tkn().Dat_ary()); caller.Rslt_is_redirect_(false); // reset flag; needed for TEST; kludgy, but Rslt_is_redirect is intended for single use
bfr.Add_byte(Byte_ascii.Eq); else
} ctx.Page().Tmpl_prepend_mgr().End(ctx, bfr, bfr_func.Bfr(), bfr_func.Len(), Bool_.N);
bfr.Add(nde.Val_tkn().Dat_ary()); bfr.Add_bfr_and_clear(bfr_func);
} }
} public static Xot_defn_tmpl Load_defn(Xowe_wiki wiki, Xop_ctx ctx, Xot_invk_tkn invk_tkn, Xoa_ttl ttl, byte[] name_ary) { // recursive loading of templates
public static Xot_defn_tmpl Load_defn(Xowe_wiki wiki, Xop_ctx ctx, Xot_invk_tkn invk_tkn, Xoa_ttl ttl, byte[] name_ary) { // recursive loading of templates Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl); byte[] tmpl_page_bry = tmpl_page_itm == null ? null : tmpl_page_itm.Wtxt__direct();
byte[] tmpl_page_bry = tmpl_page_itm == null ? null : tmpl_page_itm.Wtxt__direct(); Xot_defn_tmpl rv = null;
Xot_defn_tmpl rv = null; if (tmpl_page_bry != null) {
if (tmpl_page_bry != null) { byte old_parse_tid = ctx.Parse_tid(); // NOTE: reusing ctxs is a bad idea; will change Parse_tid and cause strange errors; however, keeping for PERF reasons
byte old_parse_tid = ctx.Parse_tid(); // NOTE: reusing ctxs is a bad idea; will change Parse_tid and cause strange errors; however, keeping for PERF reasons Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template();
Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template(); rv = wiki.Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), ctx.Tkn_mkr(), ns_tmpl, name_ary, tmpl_page_bry); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17
rv = wiki.Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), ctx.Tkn_mkr(), ns_tmpl, name_ary, tmpl_page_bry); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17 Xoa_ttl tmpl_page_ttl = tmpl_page_itm.Ttl();
Xoa_ttl tmpl_page_ttl = tmpl_page_itm.Ttl(); byte[] frame_ttl = Bry_.Add(tmpl_page_ttl.Ns().Name_db(), Byte_ascii.Colon_bry, tmpl_page_ttl.Page_txt()); // NOTE: (1) must have ns (Full); (2) must be txt (space, not underscore); EX:Template:Location map+; DATE:2014-08-22; (3) must be local language; Russian "Шаблон" not English "Template"; PAGE:ru.w:Королевство_Нидерландов DATE:2016-11-23
byte[] frame_ttl = Bry_.Add(tmpl_page_ttl.Ns().Name_db(), Byte_ascii.Colon_bry, tmpl_page_ttl.Page_txt()); // NOTE: (1) must have ns (Full); (2) must be txt (space, not underscore); EX:Template:Location map+; DATE:2014-08-22; (3) must be local language; Russian "Шаблон" not English "Template"; PAGE:ru.w:Королевство_Нидерландов DATE:2016-11-23 rv.Frame_ttl_(frame_ttl); // set defn's frame_ttl; needed for redirect_trg; PAGE:en.w:Statutory_city; DATE:2014-08-22
rv.Frame_ttl_(frame_ttl); // set defn's frame_ttl; needed for redirect_trg; PAGE:en.w:Statutory_city; DATE:2014-08-22 ctx.Parse_tid_(old_parse_tid);
ctx.Parse_tid_(old_parse_tid); wiki.Cache_mgr().Defn_cache().Add(rv, ns_tmpl.Case_match());
wiki.Cache_mgr().Defn_cache().Add(rv, ns_tmpl.Case_match()); }
} return rv;
return rv; }
} public static void Print_not_found__by_transclude(Bry_bfr bfr, Xow_ns_mgr ns_mgr, boolean name_has_template, byte[] name_ary) {// print missing as [[Missing]]; PAGE:en.d:a DATE:2016-06-24
public static void Print_not_found__by_transclude(Bry_bfr bfr, Xow_ns_mgr ns_mgr, boolean name_has_template, byte[] name_ary) {// print missing as [[Missing]]; PAGE:en.d:a DATE:2016-06-24 bfr.Add(Xop_tkn_.Lnki_bgn);
bfr.Add(Xop_tkn_.Lnki_bgn); if (name_has_template)
if (name_has_template) bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon);
bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon); bfr.Add(name_ary);
bfr.Add(name_ary); bfr.Add(Xop_tkn_.Lnki_end);
bfr.Add(Xop_tkn_.Lnki_end); }
} public static void Print_not_found__w_template(Bry_bfr bfr, Xow_ns_mgr ns_mgr, byte[] name_ary) { // print missing as [[:Template:Missing]]; REF:MW: Parser.php|braceSubstitution|$text = "[[:$titleText]]"; EX:en.d:Kazakhstan; DATE:2014-03-25
public static void Print_not_found__w_template(Bry_bfr bfr, Xow_ns_mgr ns_mgr, byte[] name_ary) { // print missing as [[:Template:Missing]]; REF:MW: Parser.php|braceSubstitution|$text = "[[:$titleText]]"; EX:en.d:Kazakhstan; DATE:2014-03-25 bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon);
bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon); bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon);
bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon); bfr.Add(name_ary).Add(Xop_tkn_.Lnki_end);
bfr.Add(name_ary).Add(Xop_tkn_.Lnki_end); }
} }
}

@ -1,6 +1,6 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
@ -13,118 +13,142 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.wikis; import gplx.*; import gplx.xowa.*; package gplx.xowa.wikis;
import gplx.xowa.langs.*; import gplx.xowa.xtns.wbases.*;
import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.domains.crts.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.metas.*; import gplx.xowa.langs.names.*; import gplx.Bry_;
import gplx.xowa.addons.wikis.directorys.dbs.*; import gplx.GfoMsg;
public class Xoae_wiki_mgr implements Xoa_wiki_mgr, Gfo_invk { import gplx.Gfo_invk;
private final Xoae_app app; import gplx.Gfo_invk_;
private final List_adp list = List_adp_.New(); private final Hash_adp_bry hash = Hash_adp_bry.ci_a7(); // ASCII:url_domain; EX:en.wikipedia.org import gplx.GfsCtx;
private Xowdir_db_mgr db_mgr; import gplx.Hash_adp_bry;
public Xoae_wiki_mgr(Xoae_app app) { import gplx.Int_;
this.app = app; import gplx.Io_url;
this.wiki_regy = new Xoa_wiki_regy(app); import gplx.List_adp;
this.wdata_mgr = new Wdata_wiki_mgr(app); import gplx.List_adp_;
} import gplx.String_;
public Xoa_wiki_regy Wiki_regy() {return wiki_regy;} private final Xoa_wiki_regy wiki_regy; import gplx.xowa.Xoae_app;
public Xow_script_mgr Scripts() {return scripts;} private final Xow_script_mgr scripts = new Xow_script_mgr(); import gplx.xowa.Xow_wiki;
public Wdata_wiki_mgr Wdata_mgr() {return wdata_mgr;} private final Wdata_wiki_mgr wdata_mgr; import gplx.xowa.Xowe_wiki;
public Xowe_wiki Wiki_commons() { import gplx.xowa.addons.wikis.directorys.dbs.Xowdir_db_mgr;
synchronized (this) { // LOCK:app-level; DATE:2016-07-06 import gplx.xowa.addons.wikis.directorys.dbs.Xowdir_wiki_itm;
Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(Xow_domain_itm_.Bry__commons); import gplx.xowa.langs.Xol_lang_itm;
if (rv != null) rv.Init_assert(); import gplx.xowa.langs.Xol_lang_itm_;
return rv; import gplx.xowa.langs.Xol_lang_stub_;
} import gplx.xowa.wikis.domains.Xow_domain_itm;
} import gplx.xowa.wikis.domains.Xow_domain_itm_;
public void Init_by_app() { import gplx.xowa.wikis.domains.Xow_domain_tid_;
this.db_mgr = new Xowdir_db_mgr(app.User().User_db_mgr().Conn()); import gplx.xowa.wikis.metas.Xow_script_mgr;
wdata_mgr.Init_by_app(); import gplx.xowa.wikis.nss.Xow_ns_mgr;
} import gplx.xowa.wikis.nss.Xow_ns_mgr_;
public int Count() {return list.Count();} import gplx.xowa.xtns.wbases.Wdata_wiki_mgr;
public boolean Has(byte[] key) {return hash.Has(key);}
public Xow_wiki Get_at(int idx) {return (Xow_wiki)list.Get_at(idx);} public class Xoae_wiki_mgr implements Xoa_wiki_mgr, Gfo_invk {
public Xowe_wiki Get_at_or_null(int i) {return Int_.Between(i, 0, this.Count() - 1) ? (Xowe_wiki)list.Get_at(i) : null;} private final Xoae_app app;
public Xow_wiki Get_by_or_null(byte[] key) {return Bry_.Len_eq_0(key) ? null : (Xowe_wiki)hash.Get_by(key);} private final List_adp list = List_adp_.New(); private final Hash_adp_bry hash = Hash_adp_bry.ci_a7(); // ASCII:url_domain; EX:en.wikipedia.org
public Xow_wiki Get_by_or_make_init_y(byte[] key) { private Xowdir_db_mgr db_mgr;
synchronized (this) { // LOCK:app-level; DATE:2016-07-06 public Xoae_wiki_mgr(Xoae_app app) {
Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key); this.app = app;
rv.Init_assert(); this.wiki_regy = new Xoa_wiki_regy(app);
return rv; this.wdata_mgr = new Wdata_wiki_mgr(app);
} }
} public Xoa_wiki_regy Wiki_regy() {return wiki_regy;} private final Xoa_wiki_regy wiki_regy;
public Xow_wiki Get_by_or_make_init_n(byte[] key) {return Get_by_or_make(key);} public Xow_script_mgr Scripts() {return scripts;} private final Xow_script_mgr scripts = new Xow_script_mgr();
public Xowe_wiki Get_by_or_make(byte[] key) { public Wdata_wiki_mgr Wdata_mgr() {return wdata_mgr;} private final Wdata_wiki_mgr wdata_mgr;
Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key); public Xowe_wiki Wiki_commons() {
return rv; synchronized (this) { // LOCK:app-level; DATE:2016-07-06
} Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(Xow_domain_itm_.Bry__commons);
public void Add(Xow_wiki wiki) { if (rv != null) rv.Init_assert();
if (hash.Get_by_bry(wiki.Domain_bry()) != null) return; // if already there, don't add again; basically, Add_if_dupe_use_1st return rv;
hash.Add(wiki.Domain_bry(), wiki); }
list.Add(wiki); }
} public void Init_by_app() {
public Xowe_wiki Make_and_add(byte[] domain_bry) { this.db_mgr = new Xowdir_db_mgr(app.User().User_db_mgr().Conn());
// get wiki_root_url from either user_wiki or /xowa/wiki/ wdata_mgr.Init_by_app();
Xowdir_wiki_itm user_wiki_itm = db_mgr == null }
? null // TEST: public int Count() {return list.Count();}
: db_mgr.Tbl__wiki().Select_by_key_or_null(String_.new_u8(domain_bry)); public boolean Has(byte[] key) {return hash.Has(key);}
public Xow_wiki Get_at(int idx) {return (Xow_wiki)list.Get_at(idx);}
Xowe_wiki rv = null; public Xowe_wiki Get_at_or_null(int i) {return Int_.Between(i, 0, this.Count() - 1) ? (Xowe_wiki)list.Get_at(i) : null;}
if (user_wiki_itm == null) { public Xow_wiki Get_by_or_null(byte[] key) {return Bry_.Len_eq_0(key) ? null : (Xowe_wiki)hash.Get_by(key);}
rv = (Xowe_wiki)Make(domain_bry, app.Fsys_mgr().Wiki_dir().GenSubDir(String_.new_a7(domain_bry))); public Xow_wiki Get_by_or_make_init_y(byte[] key) {
Add(rv); synchronized (this) { // LOCK:app-level; DATE:2016-07-06
} Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key);
else { rv.Init_assert();
rv = gplx.xowa.addons.wikis.directorys.specials.items.bldrs.Xow_wiki_factory.Load_personal(app, domain_bry, user_wiki_itm.Url().OwnerDir()); return rv;
} }
return rv; }
} public Xow_wiki Get_by_or_make_init_n(byte[] key) {return Get_by_or_make(key);}
public Xow_wiki Make(byte[] domain_bry, Io_url wiki_root_dir) { public Xowe_wiki Get_by_or_make(byte[] key) {
// init domain Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key);
Xow_domain_itm domain_itm = Xow_domain_itm_.parse(domain_bry); return rv;
}
// get lang from domain; if not wmf, default to en public void Add(Xow_wiki wiki) {
byte[] lang_key = domain_itm.Lang_actl_key(); if (hash.Get_by_bry(wiki.Domain_bry()) != null) return; // if already there, don't add again; basically, Add_if_dupe_use_1st
if (lang_key == Xol_lang_stub_.Key__unknown) lang_key = Xol_lang_itm_.Key_en; // unknown langs default to english; note that this makes nonwmf english by default hash.Add(wiki.Domain_bry(), wiki);
Xol_lang_itm lang = app.Lang_mgr().Get_by_or_new(lang_key); list.Add(wiki);
if (domain_itm.Domain_type_id() == Xow_domain_tid_.Tid__other) { }
lang = Xol_lang_itm.New(app.Lang_mgr(), Xol_lang_itm_.Key_en).Kwd_mgr__strx_(true); // create a new english lang, but enable strx functions; DATE:2015-08-23 public Xowe_wiki Make_and_add(byte[] domain_bry) {
Xol_lang_itm_.Lang_init(lang); // get wiki_root_url from either user_wiki or /xowa/wiki/
} Xowdir_wiki_itm user_wiki_itm = db_mgr == null
? null // TEST:
// load ns from site_meta : db_mgr.Tbl__wiki().Select_by_key_or_null(String_.new_u8(domain_bry));
Xow_ns_mgr ns_mgr = app.Dbmeta_mgr().Ns__get_or_load(domain_bry);
if (ns_mgr.Ids_len() == 0) ns_mgr = Xow_ns_mgr_.default_(lang.Case_mgr()); // non-wmf wikis will use default ns_mgr Xowe_wiki rv = null;
if (user_wiki_itm == null) {
return new Xowe_wiki(app, lang, ns_mgr, domain_itm, wiki_root_dir); rv = (Xowe_wiki)Make(domain_bry, app.Fsys_mgr().Wiki_dir().GenSubDir(String_.new_a7(domain_bry)));
} Add(rv);
public Xow_wiki Import_by_url(Io_url url) {return Xoa_wiki_mgr_.Import_by_url(app, this, url);} }
public void Del(byte[] key) {hash.Del(key);} else {
public void Clear() {hash.Clear(); list.Clear();} rv = gplx.xowa.addons.wikis.directorys.specials.items.bldrs.Xow_wiki_factory.Load_personal(app, domain_bry, user_wiki_itm.Url().OwnerDir());
public void Free_mem(boolean clear_ctx) { }
int list_len = list.Count(); return rv;
for (int i = 0; i < list_len; i++) { }
Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i); public Xow_wiki Make(byte[] domain_bry, Io_url wiki_root_dir) {
// wiki.Defn_cache().ReduceCache(); // init domain
if (clear_ctx) wiki.Parser_mgr().Ctx().Clear_all(); // NOTE: clear_ctx will reset toc and refs Xow_domain_itm domain_itm = Xow_domain_itm_.parse(domain_bry);
wiki.Cache_mgr().Page_cache().Free_mem(true);
wiki.Cache_mgr().Tmpl_result_cache().Clear(); // get lang from domain; if not wmf, default to en
} byte[] lang_key = domain_itm.Lang_actl_key();
} if (lang_key == Xol_lang_stub_.Key__unknown) lang_key = Xol_lang_itm_.Key_en; // unknown langs default to english; note that this makes nonwmf english by default
public void Rls() { Xol_lang_itm lang = app.Lang_mgr().Get_by_or_new(lang_key);
int len = list.Count(); if (domain_itm.Domain_type_id() == Xow_domain_tid_.Tid__other) {
for (int i = 0; i < len; i++) { lang = Xol_lang_itm.New(app.Lang_mgr(), Xol_lang_itm_.Key_en).Kwd_mgr__strx_(true); // create a new english lang, but enable strx functions; DATE:2015-08-23
Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i); Xol_lang_itm_.Lang_init(lang);
wiki.Rls(); }
}
} // load ns from site_meta
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { Xow_ns_mgr ns_mgr = app.Dbmeta_mgr().Ns__get_or_load(domain_bry);
if (ctx.Match(k, Invk_get)) return Get_by_or_make(m.ReadBry("v")); if (ns_mgr.Ids_len() == 0) ns_mgr = Xow_ns_mgr_.default_(lang.Case_mgr()); // non-wmf wikis will use default ns_mgr
else if (ctx.Match(k, Invk_scripts)) return scripts;
else if (ctx.Match(k, Invk_wdata)) return wdata_mgr; return new Xowe_wiki(app, lang, ns_mgr, domain_itm, wiki_root_dir);
else if (ctx.Match(k, Invk_len)) return this.Count(); }
else if (ctx.Match(k, Invk_get_at)) return this.Get_at_or_null(m.ReadInt("v")); public Xow_wiki Import_by_url(Io_url url) {return Xoa_wiki_mgr_.Import_by_url(app, this, url);}
else if (ctx.Match(k, Xoa_wiki_mgr_.Invk__import_by_url)) return this.Import_by_url(m.ReadIoUrl("v")); public void Del(byte[] key) {hash.Del(key);}
else return Gfo_invk_.Rv_unhandled; public void Clear() {hash.Clear(); list.Clear();}
} private static final String Invk_get = "get", Invk_scripts = "scripts", Invk_wdata = "wdata"; public void Free_mem(boolean clear_ctx) {
private static final String Invk_len = "len", Invk_get_at = "get_at"; int list_len = list.Count();
} for (int i = 0; i < list_len; i++) {
Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i);
// wiki.Defn_cache().ReduceCache();
if (clear_ctx) wiki.Parser_mgr().Ctx().Clear_all(); // NOTE: clear_ctx will reset toc and refs
wiki.Cache_mgr().Page_cache().Free_mem(true);
}
}
public void Rls() {
int len = list.Count();
for (int i = 0; i < len; i++) {
Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i);
wiki.Rls();
}
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_get)) return Get_by_or_make(m.ReadBry("v"));
else if (ctx.Match(k, Invk_scripts)) return scripts;
else if (ctx.Match(k, Invk_wdata)) return wdata_mgr;
else if (ctx.Match(k, Invk_len)) return this.Count();
else if (ctx.Match(k, Invk_get_at)) return this.Get_at_or_null(m.ReadInt("v"));
else if (ctx.Match(k, Xoa_wiki_mgr_.Invk__import_by_url)) return this.Import_by_url(m.ReadIoUrl("v"));
else return Gfo_invk_.Rv_unhandled;
} private static final String Invk_get = "get", Invk_scripts = "scripts", Invk_wdata = "wdata";
private static final String Invk_len = "len", Invk_get_at = "get_at";
}

@ -1,6 +1,6 @@
/* /*
XOWA: the XOWA Offline Wiki Application XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0. or alternatively under the terms of the Apache License Version 2.0.
@ -13,72 +13,83 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; package gplx.xowa.wikis.caches;
import gplx.core.caches.*;
import gplx.xowa.wikis.xwikis.sitelinks.*; import gplx.Hash_adp;
public class Xow_cache_mgr { import gplx.Hash_adp_;
private final Xowe_wiki wiki; import gplx.Hash_adp_bry;
public Xow_cache_mgr(Xowe_wiki wiki) { import gplx.Io_mgr;
this.wiki = wiki; import gplx.Keyval;
this.page_cache = new Xow_page_cache(wiki); import gplx.Keyval_;
this.defn_cache = new Xow_defn_cache(wiki.Lang()); import gplx.List_adp;
this.ifexist_cache = new Xow_ifexist_cache(wiki, page_cache); import gplx.List_adp_;
} import gplx.String_;
public Hash_adp Tmpl_result_cache() {return tmpl_result_cache;} private final Hash_adp tmpl_result_cache = Hash_adp_bry.cs(); import gplx.core.caches.Gfo_cache_mgr;
public Xow_defn_cache Defn_cache() {return defn_cache;} private final Xow_defn_cache defn_cache; import gplx.xowa.Xowe_wiki;
public Hash_adp_bry Lst_cache() {return lst_cache;} private final Hash_adp_bry lst_cache = Hash_adp_bry.cs(); import gplx.xowa.wikis.xwikis.sitelinks.Xoa_sitelink_itm;
public Hash_adp Misc_cache() {return misc_cache;} private final Hash_adp misc_cache = Hash_adp_.New(); import gplx.xowa.wikis.xwikis.sitelinks.Xoa_sitelink_itm_mgr;
public Xow_page_cache Page_cache() {return page_cache;} private Xow_page_cache page_cache;
public Gfo_cache_mgr Commons_cache() {return commons_cache;} private Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb); public class Xow_cache_mgr {
public Xow_ifexist_cache Ifexist_cache() {return ifexist_cache;} private Xow_ifexist_cache ifexist_cache; private final Xowe_wiki wiki;
public Xow_cache_mgr(Xowe_wiki wiki) {
public Xow_cache_mgr Page_cache_(Xow_page_cache v) {this.page_cache = v; return this;} this.wiki = wiki;
public Xow_cache_mgr Commons_cache_(Gfo_cache_mgr v) {this.commons_cache = v; return this;} this.page_cache = new Xow_page_cache(wiki);
public Xow_cache_mgr Ifexist_cache_(Xow_ifexist_cache v) {this.ifexist_cache = v; return this;} this.defn_cache = new Xow_defn_cache(wiki.Lang());
public void Load_wkr_(Xow_page_cache_wkr v) { this.ifexist_cache = new Xow_ifexist_cache(wiki, page_cache);
page_cache.Load_wkr_(v); }
ifexist_cache.Load_wkr_(v); public Xow_defn_cache Defn_cache() {return defn_cache;} private final Xow_defn_cache defn_cache;
} public Hash_adp_bry Lst_cache() {return lst_cache;} private final Hash_adp_bry lst_cache = Hash_adp_bry.cs();
public Keyval[] Scrib_lang_names() { public Hash_adp Misc_cache() {return misc_cache;} private final Hash_adp misc_cache = Hash_adp_.New();
if (scrib_lang_names == null) { public Xow_page_cache Page_cache() {return page_cache;} private Xow_page_cache page_cache;
List_adp list = List_adp_.New(); public Gfo_cache_mgr Commons_cache() {return commons_cache;} private Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
Xoa_sitelink_itm_mgr itm_mgr = wiki.App().Xwiki_mgr__sitelink_mgr().Itm_mgr(); public Xow_ifexist_cache Ifexist_cache() {return ifexist_cache;} private Xow_ifexist_cache ifexist_cache;
int len = itm_mgr.Len();
for (int i = 0; i < len; ++i) { public Xow_cache_mgr Page_cache_(Xow_page_cache v) {this.page_cache = v; return this;}
Xoa_sitelink_itm itm = itm_mgr.Get_at(i); public Xow_cache_mgr Commons_cache_(Gfo_cache_mgr v) {this.commons_cache = v; return this;}
Keyval kv = Keyval_.new_(String_.new_u8(itm.Key()), String_.new_u8(itm.Name())); public Xow_cache_mgr Ifexist_cache_(Xow_ifexist_cache v) {this.ifexist_cache = v; return this;}
list.Add(kv); public void Load_wkr_(Xow_page_cache_wkr v) {
} page_cache.Load_wkr_(v);
scrib_lang_names = (Keyval[])list.To_ary(Keyval.class); ifexist_cache.Load_wkr_(v);
} }
return scrib_lang_names; public Keyval[] Scrib_lang_names() {
} if (scrib_lang_names == null) {
public void Free_mem__page() {this.Free_mem(Free_mem__page_tid);} List_adp list = List_adp_.New();
public void Free_mem__wbase() {this.Free_mem(Free_mem__wbase_tid);} Xoa_sitelink_itm_mgr itm_mgr = wiki.App().Xwiki_mgr__sitelink_mgr().Itm_mgr();
public void Free_mem__all() {this.Free_mem(Free_mem__all_tid);} int len = itm_mgr.Len();
private void Free_mem(int level) { for (int i = 0; i < len; ++i) {
switch (level) { Xoa_sitelink_itm itm = itm_mgr.Get_at(i);
case Free_mem__page_tid: Keyval kv = Keyval_.new_(String_.new_u8(itm.Key()), String_.new_u8(itm.Name()));
page_cache.Free_mem(false); list.Add(kv);
break; }
case Free_mem__wbase_tid: scrib_lang_names = (Keyval[])list.To_ary(Keyval.class);
page_cache.Free_mem(false); }
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); return scrib_lang_names;
break; }
case Free_mem__all_tid: public void Free_mem__page() {this.Free_mem(Free_mem__page_tid);}
page_cache.Free_mem(true); public void Free_mem__wbase() {this.Free_mem(Free_mem__wbase_tid);}
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); public void Free_mem__all() {this.Free_mem(Free_mem__all_tid);}
commons_cache.Clear(); private void Free_mem(int level) {
ifexist_cache.Clear(); switch (level) {
break; case Free_mem__page_tid:
} page_cache.Free_mem(false);
wiki.Ctg__catpage_mgr().Free_mem_all(); break;
tmpl_result_cache.Clear(); case Free_mem__wbase_tid:
defn_cache.Free_mem_all(); page_cache.Free_mem(false);
lst_cache.Clear(); wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
misc_cache.Clear(); break;
// scrib_lang_names = null; case Free_mem__all_tid:
} page_cache.Free_mem(true);
private static final int Free_mem__page_tid = 0, Free_mem__wbase_tid = 1, Free_mem__all_tid = 2; wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
private static Keyval[] scrib_lang_names; commons_cache.Clear();
} ifexist_cache.Clear();
break;
}
wiki.Ctg__catpage_mgr().Free_mem_all();
defn_cache.Free_mem_all();
lst_cache.Clear();
misc_cache.Clear();
// scrib_lang_names = null;
}
private static final int Free_mem__page_tid = 0, Free_mem__wbase_tid = 1, Free_mem__all_tid = 2;
private static Keyval[] scrib_lang_names;
}

@ -1,135 +1,132 @@
/* package gplx.xowa.xtns.scribunto.libs;
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3, import gplx.Err_;
or alternatively under the terms of the Apache License Version 2.0. import gplx.Keyval_;
import gplx.Object_;
import gplx.String_;
import gplx.Tfds;
import gplx.xowa.xtns.scribunto.Scrib_core;
import gplx.xowa.xtns.scribunto.Scrib_invoke_func;
import gplx.xowa.xtns.scribunto.Scrib_invoke_func_fxt;
import gplx.xowa.xtns.scribunto.Scrib_kv_utl_;
import gplx.xowa.xtns.scribunto.Scrib_lib;
import org.junit.Before;
import org.junit.Test;
You may use XOWA according to either of these licenses as is most appropriate public class Scrib_lib_mw__invoke_tst {
for your project on a case-by-case basis. @Before public void init() {
fxt.Clear_for_invoke();
The terms of each license can be found in the source code repository: lib = fxt.Core().Lib_mw().Init();
} private Scrib_invoke_func_fxt fxt = new Scrib_invoke_func_fxt(); private Scrib_lib lib;
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt @Test public void GetAllExpandedArguments_ws_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}")
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key={{{1}}}}}");
*/ fxt.Init_page("{{test| a }}");
package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a"); // " a " -> "a"
import org.junit.*; }
public class Scrib_lib_mw__invoke_tst { @Test public void GetAllExpandedArguments_ws_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}")
@Before public void init() { fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}");
fxt.Clear_for_invoke(); fxt.Init_page("{{test| a }}");
lib = fxt.Core().Lib_mw().Init(); fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a "); // " a " -> " a "
} private Scrib_invoke_func_fxt fxt = new Scrib_invoke_func_fxt(); private Scrib_lib lib; }
@Test public void GetAllExpandedArguments_ws_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}") @Test public void GetAllExpandedArguments__ignore_empty_key() {// PURPOSE: ignore arguents that have an empty key (|=8|); EX:w:Fool's_mate; DATE:2014-03-05
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key={{{1}}}}}"); fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0}}");
fxt.Init_page("{{test| a }}"); fxt.Init_page("{{test|a1||a2|=a3|a4}}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a"); // " a " -> "a" fxt.Init_server_print_key_y_();
} fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("parent"), "\n 1:a1;2:;3:a2;4:a4"); // NOTE: || is not ignored but |=a3| is
@Test public void GetAllExpandedArguments_ws_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}") fxt.Init_server_print_key_n_();
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}"); }
fxt.Init_page("{{test| a }}"); @Test public void GetExpandedArgument_ws_key_exists() { // PURPOSE: trim val if key exists; literal value
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a "); // " a " -> " a " fxt.Init_page("{{#invoke:Mod_0|Prc_0| key1 = val1 }}");
} fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "val1"); // "key1" -> "key1"
@Test public void GetAllExpandedArguments__ignore_empty_key() {// PURPOSE: ignore arguents that have an empty key (|=8|); EX:w:Fool's_mate; DATE:2014-03-05 }
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0}}"); @Test public void GetExpandedArgument_ws_key_missing() { // PURPOSE: do not trim val if key missing; literal value
fxt.Init_page("{{test|a1||a2|=a3|a4}}"); fxt.Init_page("{{#invoke:Mod_0|Prc_0| a }}");
fxt.Init_server_print_key_y_(); fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a "
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("parent"), "\n 1:a1;2:;3:a2;4:a4"); // NOTE: || is not ignored but |=a3| is }
fxt.Init_server_print_key_n_(); @Test public void GetExpandedArgument_ws_key_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}")
} fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key1={{{1}}}}}");
@Test public void GetExpandedArgument_ws_key_exists() { // PURPOSE: trim val if key exists; literal value fxt.Init_page("{{test| a }}");
fxt.Init_page("{{#invoke:Mod_0|Prc_0| key1 = val1 }}"); fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "a"); // " a " -> "a"
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "val1"); // "key1" -> "key1" }
} @Test public void GetExpandedArgument_ws_key_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}")
@Test public void GetExpandedArgument_ws_key_missing() { // PURPOSE: do not trim val if key missing; literal value fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}");
fxt.Init_page("{{#invoke:Mod_0|Prc_0| a }}"); fxt.Init_page("{{test| a }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a " fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a "
} }
@Test public void GetExpandedArgument_ws_key_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}") @Test public void Preprocess() {
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key1={{{1}}}}}"); this.Init_preprocess();
fxt.Init_page("{{test| a }}"); this.Exec_preprocess(Scrib_core.Frame_key_module , "1", "c");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "a"); // " a " -> "a" this.Exec_preprocess(Scrib_core.Frame_key_module , "2", "d");
} this.Exec_preprocess(Scrib_core.Frame_key_template , "1", "a");
@Test public void GetExpandedArgument_ws_key_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}") this.Exec_preprocess(Scrib_core.Frame_key_template , "2", "b");
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}"); }
fxt.Init_page("{{test| a }}"); @Test public void Preprocess_duplicate_key() {
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a " fxt.Init_page("{{#invoke:Mod_0|Prc_0|key1=a|key2=b|key1=c}}"); // add key1 twice
} fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_preprocess, Object_.Ary("current", "{{#ifeq:1|1|{{{key1}}}|{{{key2}}}}}"), "c");
@Test public void Preprocess() { }
this.Init_preprocess(); @Test public void CallParserFunction() {
this.Exec_preprocess(Scrib_core.Frame_key_module , "1", "c"); fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
this.Exec_preprocess(Scrib_core.Frame_key_module , "2", "d"); fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#expr", "1") , "1"); // named: args is scalar
this.Exec_preprocess(Scrib_core.Frame_key_template , "1", "a"); fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // named: args is table
this.Exec_preprocess(Scrib_core.Frame_key_template , "2", "b"); fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", "", "y", "n") , "n"); // list: args is ary
} fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // list: args is table
@Test public void Preprocess_duplicate_key() { fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if:", "y", "n") , "n"); // colon_in_name
fxt.Init_page("{{#invoke:Mod_0|Prc_0|key1=a|key2=b|key1=c}}"); // add key1 twice }
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_preprocess, Object_.Ary("current", "{{#ifeq:1|1|{{{key1}}}|{{{key2}}}}}"), "c"); @Test public void CallParserFunction_tag() {
} fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
@Test public void CallParserFunction() { fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Scrib_kv_utl_.flat_many_("3", "id=1", "2", "text", "1", "pre")), "<pre 3=\"id=1\">2=text</pre>");// named: sort args; NOTE: keys should probably be stripped; // ISSUE#:462; DATE:2019-05-12
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); }
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#expr", "1") , "1"); // named: args is scalar @Test public void CallParserFunction__no_args() { // PURPOSE.fix: 0 args should not fail
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // named: args is table fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", "", "y", "n") , "n"); // list: args is ary fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Keyval_.Ary_empty), "");// failed with "Script error: index is out of bounds"
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // list: args is table }
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if:", "y", "n") , "n"); // colon_in_name @Test public void CallParserFunction_displayTitle() { // PURPOSE: DISPLAYTITLE not being set when called through CallParserFunction; DATE:2013-08-05
} fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
@Test public void CallParserFunction_tag() { fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "DISPLAYTITLE", "''a''"), "");
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); Tfds.Eq("<i>a</i>", String_.new_a7(fxt.Parser_fxt().Ctx().Page().Html_data().Display_ttl()));
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Scrib_kv_utl_.flat_many_("3", "id=1", "2", "text", "1", "pre")), "<pre 3=\"id=1\">2=text</pre>");// named: sort args; NOTE: keys should probably be stripped; // ISSUE#:462; DATE:2019-05-12 }
} @Test public void CallParserFunction__null() { // PURPOSE.fix: null arg should not fail; PAGE:en.w:Abziri DATE:2017-11-29
@Test public void CallParserFunction__no_args() { // PURPOSE.fix: 0 args should not fail fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#coordinates", 3, Keyval_.Ary(Keyval_.int_(1, "a"), Keyval_.int_(3, "b"), null)), "");// failed with NullPointerException
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Keyval_.Ary_empty), "");// failed with "Script error: index is out of bounds" }
} @Test public void ExpandTemplate_tmpl() {
@Test public void CallParserFunction_displayTitle() { // PURPOSE: DISPLAYTITLE not being set when called through CallParserFunction; DATE:2013-08-05 fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); fxt.Parser_fxt().Data_create("Template:A", "b{{{key1}}}c");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "DISPLAYTITLE", "''a''"), ""); fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary
Tfds.Eq("<i>a</i>", String_.new_a7(fxt.Parser_fxt().Ctx().Page().Html_data().Display_ttl())); }
} @Test public void ExpandTemplate() {
@Test public void CallParserFunction__null() { // PURPOSE.fix: null arg should not fail; PAGE:en.w:Abziri DATE:2017-11-29 fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#coordinates", 3, Keyval_.Ary(Keyval_.int_(1, "a"), Keyval_.int_(3, "b"), null)), "");// failed with NullPointerException fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Format", Scrib_kv_utl_.base1_many_("a")), "a");
} }
@Test public void ExpandTemplate_tmpl() { @Test public void ExpandTemplate_ns_specified() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Data_create("Template:A", "b{{{key1}}}c"); fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Template:Format", Scrib_kv_utl_.base1_many_("a")), "a"); // NOTE: "Template:" specified
} }
@Test public void ExpandTemplate() { @Test public void ExpandTemplate_tmpl_bool() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}"); fxt.Parser_fxt().Data_create("Template:Scribunto_bool", "bool_true={{{bool_true}}};bool_false={{{bool_false}}};");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Format", Scrib_kv_utl_.base1_many_("a")), "a"); fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Scribunto_bool", Scrib_kv_utl_.flat_many_("bool_true", true, "bool_false", false)), "bool_true=1;bool_false={{{bool_false}}};");
} }
@Test public void ExpandTemplate_ns_specified() { @Test public void ExpandTemplate_page() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}"); fxt.Parser_fxt().Data_create("A", "b{{{key1}}}c");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Template:Format", Scrib_kv_utl_.base1_many_("a")), "a"); // NOTE: "Template:" specified fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", ":A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary
} }
@Test public void ExpandTemplate_tmpl_bool() { @Test public void Err_mod_blank() {fxt.Test_parse_err("{{#invoke:}}" , Scrib_invoke_func.Err_mod_missing);}
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); @Test public void Err_mod_missing() {fxt.Test_parse_err("{{#invoke:Missing}}" , Scrib_invoke_func.Err_mod_missing);}
fxt.Parser_fxt().Data_create("Template:Scribunto_bool", "bool_true={{{bool_true}}};bool_false={{{bool_false}}};"); @Test public void Err_mod_custom() {
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Scribunto_bool", Scrib_kv_utl_.flat_many_("bool_true", true, "bool_false", false)), "bool_true=1;bool_false={{{bool_false}}};"); fxt.Test_error(Err_.new_("err_type", "fail", "key0", "val0"), "<strong class=\"error\"><span class=\"scribunto-error\" id=\"mw-scribunto-error-0\">Script error: fail</span></strong>");
} }
@Test public void ExpandTemplate_page() { private void Init_preprocess() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}"); fxt.Init_tmpl("{{#invoke:Mod_0|Func_0|1|c|d}}"); // current
fxt.Parser_fxt().Data_create("A", "b{{{key1}}}c"); fxt.Init_page("{{test|1|a|b|c}}"); // parent
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", ":A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary fxt.Init_cbk(fxt.Core().Lib_mw(), Scrib_lib_mw.Invk_preprocess);
} }
@Test public void Err_mod_blank() {fxt.Test_parse_err("{{#invoke:}}" , Scrib_invoke_func.Err_mod_missing);} private void Exec_preprocess(String frame, String arg_idx, String expd) {
@Test public void Err_mod_missing() {fxt.Test_parse_err("{{#invoke:Missing}}" , Scrib_invoke_func.Err_mod_missing);} fxt.Init_lua_module();
@Test public void Err_mod_custom() { fxt.Init_lua_rcvd_preprocess(frame, "{{#ifeq:" + arg_idx + "|{{{1}}}|{{{2}}}|{{{3}}}}}");
fxt.Test_error(Err_.new_("err_type", "fail", "key0", "val0"), "<strong class=\"error\"><span class=\"scribunto-error\" id=\"mw-scribunto-error-0\">Script error: fail</span></strong>"); fxt.Test_invoke(expd);
} }
private void Init_preprocess() { }
fxt.Init_tmpl("{{#invoke:Mod_0|Func_0|1|c|d}}"); // current
fxt.Init_page("{{test|1|a|b|c}}"); // parent
fxt.Init_cbk(fxt.Core().Lib_mw(), Scrib_lib_mw.Invk_preprocess);
}
private void Exec_preprocess(String frame, String arg_idx, String expd) {
fxt.Parser_fxt().Wiki().Cache_mgr().Tmpl_result_cache().Clear();
fxt.Init_lua_module();
fxt.Init_lua_rcvd_preprocess(frame, "{{#ifeq:" + arg_idx + "|{{{1}}}|{{{2}}}|{{{3}}}}}");
fxt.Test_invoke(expd);
}
}

Loading…
Cancel
Save