Parser: Remove `Tmpl_result_cache`, `Bld_key`, and corresponding code in `Xot_invk_tkn` [#793]

staging
gnosygnu 4 years ago
parent 316c6c6a58
commit d3896bf547

File diff suppressed because it is too large Load Diff

@ -1,6 +1,6 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
@ -13,208 +13,235 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.bldrs.mass_parses.parses.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.files.origs.*;
import gplx.xowa.htmls.core.htmls.*; import gplx.xowa.htmls.core.bldrs.*; import gplx.xowa.htmls.hxtns.pages.*; import gplx.xowa.htmls.core.hzips.*;
import gplx.xowa.wikis.pages.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.pools.*;
import gplx.xowa.addons.wikis.fulltexts.indexers.bldrs.*;
public class Xomp_parse_wkr implements Gfo_invk {
// mgr vars
private final Xomp_parse_mgr mgr;
private final Xomp_mgr_db mgr_db;
private final Xomp_prog_mgr prog_mgr;
private final Xomp_page_pool page_pool;
private final Xof_orig_wkr file_orig_wkr;
private final Xomp_ns_ord_mgr ns_ord_mgr;
// cfg vars
private final Xomp_parse_mgr_cfg cfg;
private int cleanup_interval, commit_interval;
private boolean log_file_lnkis;
// wkr vars
private final Xowe_wiki wiki;
private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
private final int uid;
private Xomp_wkr_db wkr_db;
private Xomp_stat_tbl stat_tbl;
private Hxtn_page_mgr hxtn_mgr;
// indexer vars
private final Xofulltext_indexer_wkr indexer;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private int done_count; private long done_time;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xomp_parse_mgr_cfg cfg
, Xomp_mgr_db mgr_db, Xomp_page_pool page_pool
, Xomp_prog_mgr prog_mgr, Xof_orig_wkr file_orig_wkr, Xomp_ns_ord_mgr ns_ord_mgr
, Xowe_wiki wiki, Xofulltext_indexer_wkr indexer, int uid) {
// mgr vars
this.mgr = mgr; this.mgr_db = mgr_db;
this.page_pool = page_pool; this.prog_mgr = prog_mgr; this.file_orig_wkr = file_orig_wkr;
this.ns_ord_mgr = ns_ord_mgr;
this.indexer = indexer;
// cfg vars
this.cfg = cfg;
this.cleanup_interval = cfg.Cleanup_interval();
this.commit_interval = cfg.Commit_interval();
this.log_file_lnkis = cfg.Log_file_lnkis();
// wkr-specific vars
this.wiki = wiki; this.uid = uid;
this.wkr_db = Xomp_wkr_db.New(Xomp_mgr_db.New__url(wiki), uid);
this.stat_tbl = new Xomp_stat_tbl(wkr_db.Conn());
this.hxtn_mgr = wiki.Hxtn_mgr();
this.hxtn_mgr.Init_by_xomp_wkr(wkr_db.Conn(), cfg.Zip_tid());
}
public void Exec() {
Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
// disable file download
wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
wiki.File__orig_mgr().Wkrs__set(file_orig_wkr);
wiki.File_mgr().Fsdb_mode().Tid__v2__mp__y_();
// enable disable categories according to flag
wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Ctgs_enabled_(cfg.Hdump_catboxs());
// enable lnki_temp
Xomp_lnki_temp_wkr logger = null;
if (log_file_lnkis) {
logger = new Xomp_lnki_temp_wkr(wiki, wkr_db.Conn());
parser_mgr.Ctx().Lnki().File_logger_(logger);
logger.Bgn();
}
// init log_mgr / property_wkr / stats
Xop_log_wkr_factory wkr_factory = new Xop_log_wkr_factory(wkr_db.Conn());
if (cfg.Log_math()) wiki.Parser_mgr().Math__core().Log_wkr_(wkr_factory);
// enable hdump
hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled()).Zip_tid_(cfg.Zip_tid());
hdump_bldr.Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
wkr_db.Conn().Txn_bgn("xomp");
stat_tbl.Stmt_new();
hxtn_mgr.Insert_bgn(false);
Xoh_wtr_ctx hctx = Xoh_wtr_ctx.Hdump_by_hzip_tid(cfg.Hzip_enabled() ? Xoh_hzip_dict_.Hdb__hzip : Xoh_hzip_dict_.Hdb__htxt); // ISSUE#:553; DATE:2019-09-25
// set status to running
mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__running);
// main loop
int prv_ns = -1;
while (true) {
// get page from page pool
Xomp_page_itm ppg = Get_next();
if (ppg == Xomp_page_itm.Null) {
mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__sleeping);
break; // no more pages
}
if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05
try {
long done_bgn = gplx.core.envs.System_.Ticks();
// get ns / ttl
int cur_ns = ppg.Ns_id();
Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry());
// if ns changed and prv_ns is main
if (cur_ns != prv_ns) {
prv_ns = cur_ns;
}
// init page
Xoae_page wpg = Xoae_page.New(wiki, ttl);
wpg.Bldr__ns_ord_(ns_ord_mgr.Get_ord_by_ns_id(cur_ns)); // NOTE: must set ns_id for tier_id in lnki_temp; DATE:2016-09-19
wpg.Db().Text().Text_bry_(ppg.Text());
wpg.Db().Page().Init_by_mp(ppg.Id(), ppg.Page_score());
wpg.Stat_itm().Init(uid);
// parse page
Xop_ctx pctx = parser_mgr.Ctx();
pctx.Clear_all();
parser_mgr.Parse(wpg, true);
// gen_html
hdump_bldr.Insert(pctx, wpg, hctx);
// index
long fulltext_time = 0;
if (indexer != null) {
fulltext_time = gplx.core.envs.System_.Ticks();
indexer.Index(wpg);
fulltext_time = gplx.core.envs.System_.Ticks__elapsed_in_frac(fulltext_time);
}
// mark done for sake of progress
prog_mgr.Mark_done(ppg.Id());
// update stats
long time_cur = gplx.core.envs.System_.Ticks();
long page_time = time_cur - done_bgn;
done_time += page_time;
++done_count;
stat_tbl.Insert(wpg, hdump_bldr.Tmp_hpg(), uid, page_time, fulltext_time);
// cleanup
// ctx.App().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released
if (wiki.Cache_mgr().Tmpl_result_cache().Count() > 50000)
wiki.Cache_mgr().Tmpl_result_cache().Clear();
if (done_count % cleanup_interval == 0) {
wiki.Cache_mgr().Free_mem__page();
wiki.Parser_mgr().Scrib().Core_term();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
}
if (done_count % commit_interval == 0) {
wkr_db.Conn().Txn_sav();
}
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e));
}
}
// cleanup
try {
if (logger != null) logger.End();
wkr_db.Conn().Txn_end();
wkr_db.Conn().Rls_conn();
stat_tbl.Stmt_rls();
hxtn_mgr.Insert_end(false);
mgr.Wkrs_done_add_1(); // NOTE: must release latch last else thread errors
}
catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail_end; err=~{0}", Err_.Message_gplx_log(e));
}
}
public void Bld_stats(Bry_bfr bfr) {
int done_time_in_sec = (int)(done_time / 1000); if (done_time_in_sec == 0) done_time_in_sec = 1;
bfr.Add_int_pad_bgn(Byte_ascii.Space, 4, uid );
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, (int)(done_count / done_time_in_sec));
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_count);
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_time_in_sec);
bfr.Add_byte_nl();
}
private Xomp_page_itm Get_next() {
if (list_idx == list_len) {
mgr_db.Tbl__wkr().Update_exec(uid, done_count, done_time);
list.Clear();
page_pool.Get_next(mgr_db, cfg.Wkr_machine_name(), list);
list_len = list.Len();
if (list_len == 0) return Xomp_page_itm.Null;
list_idx = 0;
}
return (Xomp_page_itm)list.Get_at(list_idx++);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__exec)) this.Exec();
else return Gfo_invk_.Rv_unhandled;
return this;
}
public static final String Invk__exec = "exec";
public static final String Cfg__ns_ids = "xomp.ns_ids";
}
package gplx.xowa.addons.bldrs.mass_parses.parses.wkrs;
import gplx.Bry_bfr;
import gplx.Byte_ascii;
import gplx.Err_;
import gplx.GfoMsg;
import gplx.Gfo_invk;
import gplx.Gfo_invk_;
import gplx.Gfo_usr_dlg_;
import gplx.GfsCtx;
import gplx.List_adp;
import gplx.List_adp_;
import gplx.xowa.Xoa_ttl;
import gplx.xowa.Xoae_page;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_mgr_db;
import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_stat_tbl;
import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_wkr_db;
import gplx.xowa.addons.bldrs.mass_parses.dbs.Xomp_wkr_tbl;
import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.Xomp_parse_mgr;
import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.Xomp_parse_mgr_cfg;
import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.Xomp_prog_mgr;
import gplx.xowa.addons.bldrs.mass_parses.parses.pools.Xomp_page_itm;
import gplx.xowa.addons.bldrs.mass_parses.parses.pools.Xomp_page_pool;
import gplx.xowa.addons.bldrs.mass_parses.parses.utls.Xomp_lnki_temp_wkr;
import gplx.xowa.addons.bldrs.mass_parses.parses.utls.Xomp_ns_ord_mgr;
import gplx.xowa.addons.wikis.fulltexts.indexers.bldrs.Xofulltext_indexer_wkr;
import gplx.xowa.files.origs.Xof_orig_wkr;
import gplx.xowa.htmls.core.bldrs.Xob_hdump_bldr;
import gplx.xowa.htmls.core.htmls.Xoh_wtr_ctx;
import gplx.xowa.htmls.core.hzips.Xoh_hzip_dict_;
import gplx.xowa.htmls.hxtns.pages.Hxtn_page_mgr;
import gplx.xowa.parsers.Xop_ctx;
import gplx.xowa.parsers.Xow_parser_mgr;
import gplx.xowa.parsers.logs.Xop_log_wkr_factory;
import gplx.xowa.wikis.pages.Xopg_view_mode_;
public class Xomp_parse_wkr implements Gfo_invk {
// mgr vars
private final Xomp_parse_mgr mgr;
private final Xomp_mgr_db mgr_db;
private final Xomp_prog_mgr prog_mgr;
private final Xomp_page_pool page_pool;
private final Xof_orig_wkr file_orig_wkr;
private final Xomp_ns_ord_mgr ns_ord_mgr;
// cfg vars
private final Xomp_parse_mgr_cfg cfg;
private int cleanup_interval, commit_interval;
private boolean log_file_lnkis;
// wkr vars
private final Xowe_wiki wiki;
private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
private final int uid;
private Xomp_wkr_db wkr_db;
private Xomp_stat_tbl stat_tbl;
private Hxtn_page_mgr hxtn_mgr;
// indexer vars
private final Xofulltext_indexer_wkr indexer;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private int done_count; private long done_time;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xomp_parse_mgr_cfg cfg
, Xomp_mgr_db mgr_db, Xomp_page_pool page_pool
, Xomp_prog_mgr prog_mgr, Xof_orig_wkr file_orig_wkr, Xomp_ns_ord_mgr ns_ord_mgr
, Xowe_wiki wiki, Xofulltext_indexer_wkr indexer, int uid) {
// mgr vars
this.mgr = mgr; this.mgr_db = mgr_db;
this.page_pool = page_pool; this.prog_mgr = prog_mgr; this.file_orig_wkr = file_orig_wkr;
this.ns_ord_mgr = ns_ord_mgr;
this.indexer = indexer;
// cfg vars
this.cfg = cfg;
this.cleanup_interval = cfg.Cleanup_interval();
this.commit_interval = cfg.Commit_interval();
this.log_file_lnkis = cfg.Log_file_lnkis();
// wkr-specific vars
this.wiki = wiki; this.uid = uid;
this.wkr_db = Xomp_wkr_db.New(Xomp_mgr_db.New__url(wiki), uid);
this.stat_tbl = new Xomp_stat_tbl(wkr_db.Conn());
this.hxtn_mgr = wiki.Hxtn_mgr();
this.hxtn_mgr.Init_by_xomp_wkr(wkr_db.Conn(), cfg.Zip_tid());
}
public void Exec() {
Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
// disable file download
wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
wiki.File__orig_mgr().Wkrs__set(file_orig_wkr);
wiki.File_mgr().Fsdb_mode().Tid__v2__mp__y_();
// enable disable categories according to flag
wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Ctgs_enabled_(cfg.Hdump_catboxs());
// enable lnki_temp
Xomp_lnki_temp_wkr logger = null;
if (log_file_lnkis) {
logger = new Xomp_lnki_temp_wkr(wiki, wkr_db.Conn());
parser_mgr.Ctx().Lnki().File_logger_(logger);
logger.Bgn();
}
// init log_mgr / property_wkr / stats
Xop_log_wkr_factory wkr_factory = new Xop_log_wkr_factory(wkr_db.Conn());
if (cfg.Log_math()) wiki.Parser_mgr().Math__core().Log_wkr_(wkr_factory);
// enable hdump
hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled()).Zip_tid_(cfg.Zip_tid());
hdump_bldr.Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
wkr_db.Conn().Txn_bgn("xomp");
stat_tbl.Stmt_new();
hxtn_mgr.Insert_bgn(false);
Xoh_wtr_ctx hctx = Xoh_wtr_ctx.Hdump_by_hzip_tid(cfg.Hzip_enabled() ? Xoh_hzip_dict_.Hdb__hzip : Xoh_hzip_dict_.Hdb__htxt); // ISSUE#:553; DATE:2019-09-25
// set status to running
mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__running);
// main loop
int prv_ns = -1;
while (true) {
// get page from page pool
Xomp_page_itm ppg = Get_next();
if (ppg == Xomp_page_itm.Null) {
mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__sleeping);
break; // no more pages
}
if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05
try {
long done_bgn = gplx.core.envs.System_.Ticks();
// get ns / ttl
int cur_ns = ppg.Ns_id();
Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry());
// if ns changed and prv_ns is main
if (cur_ns != prv_ns) {
prv_ns = cur_ns;
}
// init page
Xoae_page wpg = Xoae_page.New(wiki, ttl);
wpg.Bldr__ns_ord_(ns_ord_mgr.Get_ord_by_ns_id(cur_ns)); // NOTE: must set ns_id for tier_id in lnki_temp; DATE:2016-09-19
wpg.Db().Text().Text_bry_(ppg.Text());
wpg.Db().Page().Init_by_mp(ppg.Id(), ppg.Page_score());
wpg.Stat_itm().Init(uid);
// parse page
Xop_ctx pctx = parser_mgr.Ctx();
pctx.Clear_all();
parser_mgr.Parse(wpg, true);
// gen_html
hdump_bldr.Insert(pctx, wpg, hctx);
// index
long fulltext_time = 0;
if (indexer != null) {
fulltext_time = gplx.core.envs.System_.Ticks();
indexer.Index(wpg);
fulltext_time = gplx.core.envs.System_.Ticks__elapsed_in_frac(fulltext_time);
}
// mark done for sake of progress
prog_mgr.Mark_done(ppg.Id());
// update stats
long time_cur = gplx.core.envs.System_.Ticks();
long page_time = time_cur - done_bgn;
done_time += page_time;
++done_count;
stat_tbl.Insert(wpg, hdump_bldr.Tmp_hpg(), uid, page_time, fulltext_time);
// cleanup
// ctx.App().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released
if (done_count % cleanup_interval == 0) {
wiki.Cache_mgr().Free_mem__page();
wiki.Parser_mgr().Scrib().Core_term();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
}
if (done_count % commit_interval == 0) {
wkr_db.Conn().Txn_sav();
}
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e));
}
}
// cleanup
try {
if (logger != null) logger.End();
wkr_db.Conn().Txn_end();
wkr_db.Conn().Rls_conn();
stat_tbl.Stmt_rls();
hxtn_mgr.Insert_end(false);
mgr.Wkrs_done_add_1(); // NOTE: must release latch last else thread errors
}
catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail_end; err=~{0}", Err_.Message_gplx_log(e));
}
}
public void Bld_stats(Bry_bfr bfr) {
int done_time_in_sec = (int)(done_time / 1000); if (done_time_in_sec == 0) done_time_in_sec = 1;
bfr.Add_int_pad_bgn(Byte_ascii.Space, 4, uid );
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, (int)(done_count / done_time_in_sec));
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_count);
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_time_in_sec);
bfr.Add_byte_nl();
}
private Xomp_page_itm Get_next() {
if (list_idx == list_len) {
mgr_db.Tbl__wkr().Update_exec(uid, done_count, done_time);
list.Clear();
page_pool.Get_next(mgr_db, cfg.Wkr_machine_name(), list);
list_len = list.Len();
if (list_len == 0) return Xomp_page_itm.Null;
list_idx = 0;
}
return (Xomp_page_itm)list.Get_at(list_idx++);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__exec)) this.Exec();
else return Gfo_invk_.Rv_unhandled;
return this;
}
public static final String Invk__exec = "exec";
public static final String Cfg__ns_ids = "xomp.ns_ids";
}

@ -1,86 +1,85 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.parsers.mediawikis;
import gplx.Bry_;
import gplx.Bry_bfr;
import gplx.Bry_bfr_;
import gplx.String_;
import gplx.xowa.Xoa_ttl;
import gplx.xowa.Xoae_page;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.addons.wikis.ctgs.htmls.pageboxs.Xoctg_pagebox_itm;
import gplx.xowa.htmls.core.htmls.Xoh_wtr_ctx;
import gplx.xowa.parsers.Xop_ctx;
import gplx.xowa.parsers.Xow_parser_mgr;
import gplx.xowa.wikis.Xow_page_tid;
import gplx.xowa.wikis.pages.Xopg_view_mode_;
public class Xop_mediawiki_wkr {
private final Xowe_wiki wiki;
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
public Xop_mediawiki_wkr(Xowe_wiki wiki, Xop_mediawiki_loader loader) {
this.wiki = wiki;
if (loader != null)
wiki.Cache_mgr().Load_wkr_(new Xow_page_cache_wkr__embeddable(wiki, loader));
}
public void Free_memory() {
wiki.Cache_mgr().Tmpl_result_cache().Clear();
wiki.Cache_mgr().Free_mem__page();
wiki.Parser_mgr().Scrib().Core_term();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
}
public String Parse(String page, String wikitext) {
Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));
byte[] wtxt = Bry_.new_u8(wikitext);
Xoae_page wpg = Xoae_page.New(wiki, ttl);
wpg.Db().Text().Text_bry_(wtxt);
Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
// parse page
Xop_ctx pctx = parser_mgr.Ctx();
pctx.Clear_all();
parser_mgr.Parse(wpg, true);
// write to html
boolean is_wikitext = Xow_page_tid.Identify(wpg.Wiki().Domain_tid(), ttl.Ns().Id(), ttl.Page_db()) == Xow_page_tid.Tid_wikitext;
byte[] orig_bry = Bry_.Empty;
if (is_wikitext) {
wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Write_hdump(tmp_bfr, pctx, Xoh_wtr_ctx.Embeddable, wpg);
// write categories
int ctgs_len = wpg.Wtxt().Ctgs__len();
if ( ctgs_len > 0 // skip if no categories found while parsing wikitext
) {
Xoctg_pagebox_itm[] pagebox_itms = new Xoctg_pagebox_itm[ctgs_len];
for (int i = 0; i < ctgs_len; i++) {
pagebox_itms[i] = new Xoctg_pagebox_itm(wpg.Wtxt().Ctgs__get_at(i));
}
wiki.Ctg__pagebox_wtr().Write_pagebox(tmp_bfr, wpg, pagebox_itms);
}
orig_bry = tmp_bfr.To_bry_and_clear();
wpg.Db().Html().Html_bry_(orig_bry);
}
else { // not wikitext; EX: pages in MediaWiki: ns; DATE:2016-09-12
wpg.Db().Html().Html_bry_(wpg.Db().Text().Text_bry());
}
return String_.new_u8(orig_bry);
}
}
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.parsers.mediawikis;
import gplx.Bry_;
import gplx.Bry_bfr;
import gplx.Bry_bfr_;
import gplx.String_;
import gplx.xowa.Xoa_ttl;
import gplx.xowa.Xoae_page;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.addons.wikis.ctgs.htmls.pageboxs.Xoctg_pagebox_itm;
import gplx.xowa.htmls.core.htmls.Xoh_wtr_ctx;
import gplx.xowa.parsers.Xop_ctx;
import gplx.xowa.parsers.Xow_parser_mgr;
import gplx.xowa.wikis.Xow_page_tid;
import gplx.xowa.wikis.pages.Xopg_view_mode_;
public class Xop_mediawiki_wkr {
private final Xowe_wiki wiki;
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
public Xop_mediawiki_wkr(Xowe_wiki wiki, Xop_mediawiki_loader loader) {
this.wiki = wiki;
if (loader != null)
wiki.Cache_mgr().Load_wkr_(new Xow_page_cache_wkr__embeddable(wiki, loader));
}
public void Free_memory() {
wiki.Cache_mgr().Free_mem__page();
wiki.Parser_mgr().Scrib().Core_term();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
}
public String Parse(String page, String wikitext) {
Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));
byte[] wtxt = Bry_.new_u8(wikitext);
Xoae_page wpg = Xoae_page.New(wiki, ttl);
wpg.Db().Text().Text_bry_(wtxt);
Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
// parse page
Xop_ctx pctx = parser_mgr.Ctx();
pctx.Clear_all();
parser_mgr.Parse(wpg, true);
// write to html
boolean is_wikitext = Xow_page_tid.Identify(wpg.Wiki().Domain_tid(), ttl.Ns().Id(), ttl.Page_db()) == Xow_page_tid.Tid_wikitext;
byte[] orig_bry = Bry_.Empty;
if (is_wikitext) {
wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_view_mode_.Tid__read).Write_hdump(tmp_bfr, pctx, Xoh_wtr_ctx.Embeddable, wpg);
// write categories
int ctgs_len = wpg.Wtxt().Ctgs__len();
if ( ctgs_len > 0 // skip if no categories found while parsing wikitext
) {
Xoctg_pagebox_itm[] pagebox_itms = new Xoctg_pagebox_itm[ctgs_len];
for (int i = 0; i < ctgs_len; i++) {
pagebox_itms[i] = new Xoctg_pagebox_itm(wpg.Wtxt().Ctgs__get_at(i));
}
wiki.Ctg__pagebox_wtr().Write_pagebox(tmp_bfr, wpg, pagebox_itms);
}
orig_bry = tmp_bfr.To_bry_and_clear();
wpg.Db().Html().Html_bry_(orig_bry);
}
else { // not wikitext; EX: pages in MediaWiki: ns; DATE:2016-09-12
wpg.Db().Html().Html_bry_(wpg.Db().Text().Text_bry());
}
return String_.new_u8(orig_bry);
}
}

@ -1,6 +1,6 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
@ -13,314 +13,347 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.envs.*;
import gplx.dbs.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.files.origs.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.addons.bldrs.files.utls.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*;
public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xob_cmd, Gfo_invk {
private Xob_dump_src_id page_src;
private Xow_db_mgr db_fsys_mgr; protected Xop_parser parser; protected Xop_ctx ctx; protected Xop_root_tkn root;
private int[] ns_ary; private Xow_db_file[] db_ary;
private int ns_bgn = -1, db_bgn = -1, pg_bgn = -1;
private int ns_end = -1, db_end = -1, pg_end = Int_.Max_value;
private int commit_interval = 1000, progress_interval = 250, cleanup_interval = 2500, select_size = 10 * Io_mgr.Len_mb;
private int exec_count, exec_count_max = Int_.Max_value;
private boolean reset_db = false, exit_after_commit = false, exit_now = false;
private boolean load_tmpls;
private Xob_dump_bmk_mgr bmk_mgr = new Xob_dump_bmk_mgr();
private Xobu_poll_mgr poll_mgr; private int poll_interval = 5000;
private Xob_rate_mgr rate_mgr = new Xob_rate_mgr();
public abstract String Cmd_key();
@Override protected void Cmd_ctor_end(Xob_bldr bldr, Xowe_wiki wiki) {
poll_mgr = new Xobu_poll_mgr(bldr.App()); // init in ctor so gfs can invoke methods
}
public void Cmd_bgn(Xob_bldr bldr) {
parser = wiki.Parser_mgr().Main();
ctx = wiki.Parser_mgr().Ctx();
root = ctx.Tkn_mkr().Root(Bry_.Empty);
wiki.Init_assert(); // NOTE: must init wiki for db_mgr_as_sql
// assert by calling Db_mgr_as_sql
wiki.Db_mgr_as_sql().Core_data_mgr();
// load db_mgr
Xow_db_mgr.Init_by_load(wiki, gplx.xowa.wikis.data.Xow_db_file__core_.Find_core_fil_or_null(wiki)); // NOTE: must reinit providers as previous steps may have rls'd (and left member variable conn which is closed)
wiki.File__orig_mgr().Wkrs__del(Xof_orig_wkr_.Tid_wmf_api);
db_fsys_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
db_ary = Xob_dump_mgr_base_.Init_text_files_ary(db_fsys_mgr);
poll_interval = poll_mgr.Poll_interval();
page_src = new Xob_dump_src_id().Init(wiki, this.Init_redirect(), select_size);
ns_ary = Init_ns_ary();
Db_conn conn = Init_db_file();
Io_url wiki_dir = wiki.Fsys_mgr().Root_dir();
bmk_mgr.Cfg_url_(wiki_dir.GenSubFil("xowa.file.make.cfg.gfs"));
rate_mgr.Log_file_(wiki_dir.GenSubFil("xowa.file.make.log.csv"));
if (reset_db) {
bmk_mgr.Reset();
Init_reset(conn);
}
bmk_mgr.Load(wiki.Appe(), this);
Cmd_bgn_end();
}
protected abstract void Cmd_bgn_end();
public abstract byte Init_redirect();
public abstract int[] Init_ns_ary();
protected abstract void Init_reset(Db_conn p);
protected abstract Db_conn Init_db_file();
private long time_bgn;
public void Cmd_run() {Exec_ns_ary();}
private void Exec_ns_ary() {
if (pg_bgn == Int_.Max_value) return;
if (load_tmpls) Xob_dump_mgr_base_.Load_all_tmpls(usr_dlg, wiki, page_src);
time_bgn = System_.Ticks();
Xob_dump_bmk dump_bmk = new Xob_dump_bmk();
rate_mgr.Init();
int ns_ary_len = ns_ary.length;
for (int i = 0; i < ns_ary_len; i++) {
int ns_id = ns_ary[i];
if (ns_bgn != -1) { // ns_bgn set
if (ns_id == ns_bgn) // ns_id is ns_bgn; null out ns_bgn and continue
ns_bgn = -1;
else // ns_id is not ns_bgn; keep looking
continue;
}
dump_bmk.Ns_id_(ns_id);
Exec_db_ary(i, dump_bmk, ns_id);
if (ns_id == ns_end) exit_now = true; // ns_end set; exit
if (exit_now) break; // exit_now b/c of pg_bgn, db_bgn or something else
}
Exec_commit(dump_bmk.Ns_id(), dump_bmk.Db_id(), dump_bmk.Pg_id(), Bry_.Empty);
}
private void Exec_db_ary(int ns_ord, Xob_dump_bmk dump_bmk, int ns_id) {
int db_ary_len = db_ary.length;
for (int i = 0; i < db_ary_len; i++) {
int db_id = db_ary[i].Id();
if (db_bgn != -1) { // db_bgn set
if (db_id == db_bgn) // db_id is db_bgn; null out db_bgn and continue
db_bgn = -1;
else // db_id is not db_bgn; keep looking
continue;
}
dump_bmk.Db_id_(db_id);
Exec_db_itm(dump_bmk, ns_ord, ns_id, db_id);
if (db_id == db_end) exit_now = true; // db_end set; exit;
if (exit_now) return; // exit_now b/c of pg_bgn, db_bgn or something else
}
}
private void Exec_db_itm(Xob_dump_bmk dump_bmk, int ns_ord, int ns_id, int db_id) {
List_adp pages = List_adp_.New();
Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id);
int pg_id = pg_bgn;
while (true) {
page_src.Get_pages(wiki.Appe(), pages, db_id, ns_id, pg_id);
int pages_len = pages.Count();
if (pages_len == 0) { // no more pages in db;
if (pg_id > pg_bgn) // reset pg_bgn to 0 only if pg_bgn seen;
pg_bgn = 0;
return;
}
usr_dlg.Prog_many("", "", "fetched pages: ~{0}", pages_len);
for (int i = 0; i < pages_len; i++) {
Xowd_page_itm page = (Xowd_page_itm)pages.Get_at(i);
dump_bmk.Pg_id_(pg_id);
Exec_pg_itm(ns_ord, ns, db_id, page);
if ( pg_id >= pg_end
|| exec_count >= exec_count_max) {
exit_now = true;
}
if (exit_now) return;
pg_id = page.Id();
}
}
}
private void Exec_pg_itm(int ns_ord, Xow_ns ns, int db_id, Xowd_page_itm page) {
try {
if ((exec_count % progress_interval) == 0)
usr_dlg.Prog_many("", "", "parsing: ns=~{0} db=~{1} pg=~{2} count=~{3} time=~{4} rate=~{5} ttl=~{6}"
, ns.Id(), db_id, page.Id(), exec_count
, System_.Ticks__elapsed_in_sec(time_bgn), rate_mgr.Rate_as_str(), String_.new_u8(page.Ttl_page_db()));
ctx.Clear_all();
byte[] page_src = page.Text();
if (page_src != null) // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05
Exec_pg_itm_hook(ns_ord, ns, page, page_src);
ctx.Wiki().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released
if (ctx.Wiki().Cache_mgr().Tmpl_result_cache().Count() > 50000)
ctx.Wiki().Cache_mgr().Tmpl_result_cache().Clear();
++exec_count;
rate_mgr.Increment();
if ((exec_count % poll_interval) == 0)
poll_mgr.Poll();
if ((exec_count % commit_interval) == 0)
Exec_commit(ns.Id(), db_id, page.Id(), page.Ttl_page_db());
if ((exec_count % cleanup_interval) == 0)
Free();
}
catch (Exception exc) {
bldr.Usr_dlg().Warn_many("", "", "parse failed: wiki=~{0} ttl=~{1} err=~{2}", wiki.Domain_str(), page.Ttl_full_db(), Err_.Message_gplx_log(exc));
ctx.Wiki().Utl__bfr_mkr().Clear();
this.Free();
}
}
public abstract void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm page, byte[] page_text);
private void Exec_commit(int ns_id, int db_id, int pg_id, byte[] ttl) {
usr_dlg.Prog_many("", "", "committing: ns=~{0} db=~{1} pg=~{2} count=~{3} ttl=~{4}", ns_id, db_id, pg_id, exec_count, String_.new_u8(ttl));
Exec_commit_hook();
bmk_mgr.Save(ns_id, db_id, pg_id);
if (exit_after_commit) exit_now = true;
}
public abstract void Exec_commit_hook();
public abstract void Exec_end_hook();
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_term() {}
public void Cmd_end() {
if (!exit_now)
pg_bgn = Int_.Max_value;
Exec_commit(-1, -1, -1, Bry_.Empty);
Exec_end_hook();
Free();
usr_dlg.Note_many("", "", "done: ~{0} ~{1}", exec_count, Decimal_adp_.divide_safe_(exec_count, System_.Ticks__elapsed_in_sec(time_bgn)).To_str("#,###.000"));
}
private void Free() {
Xowe_wiki_.Rls_mem(wiki, true);
}
protected void Reset_db_y_() {this.reset_db = true;}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_commit_interval_)) commit_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_progress_interval_)) progress_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_rate_interval_)) rate_mgr.Reset_interval_(m.ReadInt("v"));
else if (ctx.Match(k, Invk_cleanup_interval_)) cleanup_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_select_size_)) select_size = m.ReadInt("v") * Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_ns_bgn_)) {ns_bgn = m.ReadInt("v"); Notify_restoring("ns", ns_bgn);}
else if (ctx.Match(k, Invk_db_bgn_)) {db_bgn = m.ReadInt("v"); Notify_restoring("db", db_bgn);}
else if (ctx.Match(k, Invk_pg_bgn_)) {pg_bgn = m.ReadInt("v"); Notify_restoring("pg", pg_bgn);}
else if (ctx.Match(k, Invk_ns_end_)) ns_end = m.ReadInt("v");
else if (ctx.Match(k, Invk_db_end_)) db_end = m.ReadInt("v");
else if (ctx.Match(k, Invk_pg_end_)) pg_end = m.ReadInt("v");
else if (ctx.Match(k, Invk_load_tmpls_)) load_tmpls = m.ReadYn("v");
else if (ctx.Match(k, Invk_poll_mgr)) return poll_mgr;
else if (ctx.Match(k, Invk_reset_db_)) reset_db = m.ReadYn("v");
else if (ctx.Match(k, Invk_exec_count_max_)) exec_count_max = m.ReadInt("v");
else if (ctx.Match(k, Invk_exit_now_)) exit_now = m.ReadYn("v");
else if (ctx.Match(k, Invk_exit_after_commit_)) exit_after_commit = m.ReadYn("v");
else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v"));
else return Gfo_invk_.Rv_unhandled;
return this;
}
private void Notify_restoring(String itm, int val) {
usr_dlg.Note_many("", "", "restoring: itm=~{0} val=~{1}", itm, val);
}
public static final String
Invk_progress_interval_ = "progress_interval_", Invk_commit_interval_ = "commit_interval_", Invk_cleanup_interval_ = "cleanup_interval_", Invk_rate_interval_ = "rate_interval_"
, Invk_select_size_ = "select_size_"
, Invk_ns_bgn_ = "ns_bgn_", Invk_db_bgn_ = "db_bgn_", Invk_pg_bgn_ = "pg_bgn_"
, Invk_ns_end_ = "ns_end_", Invk_db_end_ = "db_end_", Invk_pg_end_ = "pg_end_"
, Invk_load_tmpls_ = "load_tmpls_"
, Invk_poll_mgr = "poll_mgr", Invk_reset_db_ = "reset_db_"
, Invk_exec_count_max_ = "exec_count_max_", Invk_exit_now_ = "exit_now_", Invk_exit_after_commit_ = "exit_after_commit_"
, Invk__manual_now_ = "manual_now_"
;
}
class Xob_dump_mgr_base_ {
public static void Load_all_tmpls(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, Xob_dump_src_id page_src) {
List_adp pages = List_adp_.New();
Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template();
Xow_defn_cache defn_cache = wiki.Cache_mgr().Defn_cache();
int cur_page_id = -1;
int load_count = 0;
usr_dlg.Note_many("", "", "tmpl_load init");
while (true) {
page_src.Get_pages(wiki.Appe(), pages, 0, Xow_ns_.Tid__template, cur_page_id); // 0 is always template db
int page_count = pages.Count();
if (page_count == 0) break; // no more pages in db;
Xowd_page_itm page = null;
for (int i = 0; i < page_count; i++) {
page = (Xowd_page_itm)pages.Get_at(i);
Xot_defn_tmpl defn = new Xot_defn_tmpl();
defn.Init_by_new(ns_tmpl, ns_tmpl.Gen_ttl(page.Ttl_page_db()), page.Text(), null, false); // NOTE: passing null, false; will be overriden later when Parse is called
defn_cache.Add(defn, ns_tmpl.Case_match());
++load_count;
if ((load_count % 10000) == 0) usr_dlg.Prog_many("", "", "tmpl_loading: ~{0}", load_count);
}
cur_page_id = page.Id();
}
usr_dlg.Note_many("", "", "tmpl_load done: ~{0}", load_count);
}
public static Xow_db_file[] Init_text_files_ary(Xow_db_mgr core_data_mgr) {
List_adp text_files_list = List_adp_.New();
int len = core_data_mgr.Dbs__len();
if (len == 1) return new Xow_db_file[] {core_data_mgr.Dbs__get_at(0)}; // single file: return core; note that there are no Tid = Text
for (int i = 0; i < len; i++) {
Xow_db_file file = core_data_mgr.Dbs__get_at(i);
switch (file.Tid()) {
case Xow_db_file_.Tid__text:
case Xow_db_file_.Tid__text_solo:
text_files_list.Add(file);
break;
}
}
return (Xow_db_file[])text_files_list.To_ary_and_clear(Xow_db_file.class);
}
}
class Xob_dump_bmk_mgr {
private Bry_bfr save_bfr = Bry_bfr_.Reset(1024);
public Io_url Cfg_url() {return cfg_url;} public Xob_dump_bmk_mgr Cfg_url_(Io_url v) {cfg_url = v; return this;} private Io_url cfg_url;
public void Reset() {Io_mgr.Instance.DeleteFil(cfg_url);}
public void Load(Xoae_app app, Xob_dump_mgr_base dump_mgr) {
app.Gfs_mgr().Run_url_for(dump_mgr, cfg_url);
}
public void Save(int ns_id, int db_id, int pg_id) {
Save_itm(save_bfr, Xob_dump_mgr_base.Invk_ns_bgn_, ns_id);
Save_itm(save_bfr, Xob_dump_mgr_base.Invk_db_bgn_, db_id);
Save_itm(save_bfr, Xob_dump_mgr_base.Invk_pg_bgn_, pg_id);
Io_mgr.Instance.SaveFilBfr(cfg_url, save_bfr);
}
private void Save_itm(Bry_bfr save_bfr, String key, int val) {
String fmt = "{0}('{1}');\n";
String str = String_.Format(fmt, key, val);
save_bfr.Add_str_u8(str);
}
}
class Xob_rate_mgr {
private long time_bgn;
private int item_len;
private Bry_bfr save_bfr = Bry_bfr_.Reset(255);
public int Reset_interval() {return reset_interval;} public Xob_rate_mgr Reset_interval_(int v) {reset_interval = v; return this;} private int reset_interval = 10000;
public Io_url Log_file_url() {return log_file;} public Xob_rate_mgr Log_file_(Io_url v) {log_file = v; return this;} private Io_url log_file;
public void Init() {time_bgn = System_.Ticks();}
public void Increment() {
++item_len;
if (item_len % reset_interval == 0) {
long time_end = System_.Ticks();
Save(item_len, time_bgn, time_end);
time_bgn = time_end;
item_len = 0;
}
}
private void Save(int count, long bgn, long end) {
int dif = (int)(end - bgn) / 1000;
Decimal_adp rate = Decimal_adp_.divide_safe_(count, dif);
save_bfr
.Add_str_a7(rate.To_str("#,##0.000")).Add_byte_pipe()
.Add_int_variable(count).Add_byte_pipe()
.Add_int_variable(dif).Add_byte_nl()
;
Io_mgr.Instance.AppendFilByt(log_file, save_bfr.To_bry_and_clear());
}
public String Rate_as_str() {return Int_.To_str(Rate());}
public int Rate() {
int elapsed = System_.Ticks__elapsed_in_sec(time_bgn);
return Math_.Div_safe_as_int(item_len, elapsed);
}
}
class Xob_dump_bmk {
public int Ns_id() {return ns_id;} public Xob_dump_bmk Ns_id_(int v) {ns_id = v; return this;} private int ns_id;
public int Db_id() {return db_id;} public Xob_dump_bmk Db_id_(int v) {db_id = v; return this;} private int db_id;
public int Pg_id() {return pg_id;} public Xob_dump_bmk Pg_id_(int v) {pg_id = v; return this;} private int pg_id;
}
package gplx.xowa.bldrs.cmds;
import gplx.Bry_;
import gplx.Bry_bfr;
import gplx.Bry_bfr_;
import gplx.Datetime_now;
import gplx.Decimal_adp;
import gplx.Decimal_adp_;
import gplx.Err_;
import gplx.GfoMsg;
import gplx.Gfo_invk;
import gplx.Gfo_invk_;
import gplx.Gfo_usr_dlg;
import gplx.GfsCtx;
import gplx.Int_;
import gplx.Io_mgr;
import gplx.Io_url;
import gplx.List_adp;
import gplx.List_adp_;
import gplx.Math_;
import gplx.String_;
import gplx.core.envs.System_;
import gplx.dbs.Db_conn;
import gplx.xowa.Xoae_app;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.Xowe_wiki_;
import gplx.xowa.addons.bldrs.files.utls.Xobu_poll_mgr;
import gplx.xowa.bldrs.Xob_bldr;
import gplx.xowa.bldrs.wkrs.Xob_cmd;
import gplx.xowa.bldrs.wkrs.Xob_itm_basic_base;
import gplx.xowa.files.origs.Xof_orig_wkr_;
import gplx.xowa.parsers.Xop_ctx;
import gplx.xowa.parsers.Xop_parser;
import gplx.xowa.parsers.Xop_root_tkn;
import gplx.xowa.parsers.tmpls.Xot_defn_tmpl;
import gplx.xowa.wikis.caches.Xow_defn_cache;
import gplx.xowa.wikis.data.Xow_db_file;
import gplx.xowa.wikis.data.Xow_db_file_;
import gplx.xowa.wikis.data.Xow_db_mgr;
import gplx.xowa.wikis.data.tbls.Xowd_page_itm;
import gplx.xowa.wikis.nss.Xow_ns;
import gplx.xowa.wikis.nss.Xow_ns_;
public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xob_cmd, Gfo_invk {
private Xob_dump_src_id page_src;
private Xow_db_mgr db_fsys_mgr; protected Xop_parser parser; protected Xop_ctx ctx; protected Xop_root_tkn root;
private int[] ns_ary; private Xow_db_file[] db_ary;
private int ns_bgn = -1, db_bgn = -1, pg_bgn = -1;
private int ns_end = -1, db_end = -1, pg_end = Int_.Max_value;
private int commit_interval = 1000, progress_interval = 250, cleanup_interval = 2500, select_size = 10 * Io_mgr.Len_mb;
private int exec_count, exec_count_max = Int_.Max_value;
private boolean reset_db = false, exit_after_commit = false, exit_now = false;
private boolean load_tmpls;
private Xob_dump_bmk_mgr bmk_mgr = new Xob_dump_bmk_mgr();
private Xobu_poll_mgr poll_mgr; private int poll_interval = 5000;
private Xob_rate_mgr rate_mgr = new Xob_rate_mgr();
public abstract String Cmd_key();
@Override protected void Cmd_ctor_end(Xob_bldr bldr, Xowe_wiki wiki) {
poll_mgr = new Xobu_poll_mgr(bldr.App()); // init in ctor so gfs can invoke methods
}
public void Cmd_bgn(Xob_bldr bldr) {
parser = wiki.Parser_mgr().Main();
ctx = wiki.Parser_mgr().Ctx();
root = ctx.Tkn_mkr().Root(Bry_.Empty);
wiki.Init_assert(); // NOTE: must init wiki for db_mgr_as_sql
// assert by calling Db_mgr_as_sql
wiki.Db_mgr_as_sql().Core_data_mgr();
// load db_mgr
Xow_db_mgr.Init_by_load(wiki, gplx.xowa.wikis.data.Xow_db_file__core_.Find_core_fil_or_null(wiki)); // NOTE: must reinit providers as previous steps may have rls'd (and left member variable conn which is closed)
wiki.File__orig_mgr().Wkrs__del(Xof_orig_wkr_.Tid_wmf_api);
db_fsys_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
db_ary = Xob_dump_mgr_base_.Init_text_files_ary(db_fsys_mgr);
poll_interval = poll_mgr.Poll_interval();
page_src = new Xob_dump_src_id().Init(wiki, this.Init_redirect(), select_size);
ns_ary = Init_ns_ary();
Db_conn conn = Init_db_file();
Io_url wiki_dir = wiki.Fsys_mgr().Root_dir();
bmk_mgr.Cfg_url_(wiki_dir.GenSubFil("xowa.file.make.cfg.gfs"));
rate_mgr.Log_file_(wiki_dir.GenSubFil("xowa.file.make.log.csv"));
if (reset_db) {
bmk_mgr.Reset();
Init_reset(conn);
}
bmk_mgr.Load(wiki.Appe(), this);
Cmd_bgn_end();
}
protected abstract void Cmd_bgn_end();
public abstract byte Init_redirect();
public abstract int[] Init_ns_ary();
protected abstract void Init_reset(Db_conn p);
protected abstract Db_conn Init_db_file();
private long time_bgn;
public void Cmd_run() {Exec_ns_ary();}
private void Exec_ns_ary() {
if (pg_bgn == Int_.Max_value) return;
if (load_tmpls) Xob_dump_mgr_base_.Load_all_tmpls(usr_dlg, wiki, page_src);
time_bgn = System_.Ticks();
Xob_dump_bmk dump_bmk = new Xob_dump_bmk();
rate_mgr.Init();
int ns_ary_len = ns_ary.length;
for (int i = 0; i < ns_ary_len; i++) {
int ns_id = ns_ary[i];
if (ns_bgn != -1) { // ns_bgn set
if (ns_id == ns_bgn) // ns_id is ns_bgn; null out ns_bgn and continue
ns_bgn = -1;
else // ns_id is not ns_bgn; keep looking
continue;
}
dump_bmk.Ns_id_(ns_id);
Exec_db_ary(i, dump_bmk, ns_id);
if (ns_id == ns_end) exit_now = true; // ns_end set; exit
if (exit_now) break; // exit_now b/c of pg_bgn, db_bgn or something else
}
Exec_commit(dump_bmk.Ns_id(), dump_bmk.Db_id(), dump_bmk.Pg_id(), Bry_.Empty);
}
private void Exec_db_ary(int ns_ord, Xob_dump_bmk dump_bmk, int ns_id) {
int db_ary_len = db_ary.length;
for (int i = 0; i < db_ary_len; i++) {
int db_id = db_ary[i].Id();
if (db_bgn != -1) { // db_bgn set
if (db_id == db_bgn) // db_id is db_bgn; null out db_bgn and continue
db_bgn = -1;
else // db_id is not db_bgn; keep looking
continue;
}
dump_bmk.Db_id_(db_id);
Exec_db_itm(dump_bmk, ns_ord, ns_id, db_id);
if (db_id == db_end) exit_now = true; // db_end set; exit;
if (exit_now) return; // exit_now b/c of pg_bgn, db_bgn or something else
}
}
private void Exec_db_itm(Xob_dump_bmk dump_bmk, int ns_ord, int ns_id, int db_id) {
List_adp pages = List_adp_.New();
Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id);
int pg_id = pg_bgn;
while (true) {
page_src.Get_pages(wiki.Appe(), pages, db_id, ns_id, pg_id);
int pages_len = pages.Count();
if (pages_len == 0) { // no more pages in db;
if (pg_id > pg_bgn) // reset pg_bgn to 0 only if pg_bgn seen;
pg_bgn = 0;
return;
}
usr_dlg.Prog_many("", "", "fetched pages: ~{0}", pages_len);
for (int i = 0; i < pages_len; i++) {
Xowd_page_itm page = (Xowd_page_itm)pages.Get_at(i);
dump_bmk.Pg_id_(pg_id);
Exec_pg_itm(ns_ord, ns, db_id, page);
if ( pg_id >= pg_end
|| exec_count >= exec_count_max) {
exit_now = true;
}
if (exit_now) return;
pg_id = page.Id();
}
}
}
private void Exec_pg_itm(int ns_ord, Xow_ns ns, int db_id, Xowd_page_itm page) {
try {
if ((exec_count % progress_interval) == 0)
usr_dlg.Prog_many("", "", "parsing: ns=~{0} db=~{1} pg=~{2} count=~{3} time=~{4} rate=~{5} ttl=~{6}"
, ns.Id(), db_id, page.Id(), exec_count
, System_.Ticks__elapsed_in_sec(time_bgn), rate_mgr.Rate_as_str(), String_.new_u8(page.Ttl_page_db()));
ctx.Clear_all();
byte[] page_src = page.Text();
if (page_src != null) // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05
Exec_pg_itm_hook(ns_ord, ns, page, page_src);
ctx.Wiki().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released
++exec_count;
rate_mgr.Increment();
if ((exec_count % poll_interval) == 0)
poll_mgr.Poll();
if ((exec_count % commit_interval) == 0)
Exec_commit(ns.Id(), db_id, page.Id(), page.Ttl_page_db());
if ((exec_count % cleanup_interval) == 0)
Free();
}
catch (Exception exc) {
bldr.Usr_dlg().Warn_many("", "", "parse failed: wiki=~{0} ttl=~{1} err=~{2}", wiki.Domain_str(), page.Ttl_full_db(), Err_.Message_gplx_log(exc));
ctx.Wiki().Utl__bfr_mkr().Clear();
this.Free();
}
}
public abstract void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm page, byte[] page_text);
private void Exec_commit(int ns_id, int db_id, int pg_id, byte[] ttl) {
usr_dlg.Prog_many("", "", "committing: ns=~{0} db=~{1} pg=~{2} count=~{3} ttl=~{4}", ns_id, db_id, pg_id, exec_count, String_.new_u8(ttl));
Exec_commit_hook();
bmk_mgr.Save(ns_id, db_id, pg_id);
if (exit_after_commit) exit_now = true;
}
public abstract void Exec_commit_hook();
public abstract void Exec_end_hook();
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_term() {}
public void Cmd_end() {
if (!exit_now)
pg_bgn = Int_.Max_value;
Exec_commit(-1, -1, -1, Bry_.Empty);
Exec_end_hook();
Free();
usr_dlg.Note_many("", "", "done: ~{0} ~{1}", exec_count, Decimal_adp_.divide_safe_(exec_count, System_.Ticks__elapsed_in_sec(time_bgn)).To_str("#,###.000"));
}
private void Free() {
Xowe_wiki_.Rls_mem(wiki, true);
}
protected void Reset_db_y_() {this.reset_db = true;}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_commit_interval_)) commit_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_progress_interval_)) progress_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_rate_interval_)) rate_mgr.Reset_interval_(m.ReadInt("v"));
else if (ctx.Match(k, Invk_cleanup_interval_)) cleanup_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_select_size_)) select_size = m.ReadInt("v") * Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_ns_bgn_)) {ns_bgn = m.ReadInt("v"); Notify_restoring("ns", ns_bgn);}
else if (ctx.Match(k, Invk_db_bgn_)) {db_bgn = m.ReadInt("v"); Notify_restoring("db", db_bgn);}
else if (ctx.Match(k, Invk_pg_bgn_)) {pg_bgn = m.ReadInt("v"); Notify_restoring("pg", pg_bgn);}
else if (ctx.Match(k, Invk_ns_end_)) ns_end = m.ReadInt("v");
else if (ctx.Match(k, Invk_db_end_)) db_end = m.ReadInt("v");
else if (ctx.Match(k, Invk_pg_end_)) pg_end = m.ReadInt("v");
else if (ctx.Match(k, Invk_load_tmpls_)) load_tmpls = m.ReadYn("v");
else if (ctx.Match(k, Invk_poll_mgr)) return poll_mgr;
else if (ctx.Match(k, Invk_reset_db_)) reset_db = m.ReadYn("v");
else if (ctx.Match(k, Invk_exec_count_max_)) exec_count_max = m.ReadInt("v");
else if (ctx.Match(k, Invk_exit_now_)) exit_now = m.ReadYn("v");
else if (ctx.Match(k, Invk_exit_after_commit_)) exit_after_commit = m.ReadYn("v");
else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v"));
else return Gfo_invk_.Rv_unhandled;
return this;
}
private void Notify_restoring(String itm, int val) {
usr_dlg.Note_many("", "", "restoring: itm=~{0} val=~{1}", itm, val);
}
public static final String
Invk_progress_interval_ = "progress_interval_", Invk_commit_interval_ = "commit_interval_", Invk_cleanup_interval_ = "cleanup_interval_", Invk_rate_interval_ = "rate_interval_"
, Invk_select_size_ = "select_size_"
, Invk_ns_bgn_ = "ns_bgn_", Invk_db_bgn_ = "db_bgn_", Invk_pg_bgn_ = "pg_bgn_"
, Invk_ns_end_ = "ns_end_", Invk_db_end_ = "db_end_", Invk_pg_end_ = "pg_end_"
, Invk_load_tmpls_ = "load_tmpls_"
, Invk_poll_mgr = "poll_mgr", Invk_reset_db_ = "reset_db_"
, Invk_exec_count_max_ = "exec_count_max_", Invk_exit_now_ = "exit_now_", Invk_exit_after_commit_ = "exit_after_commit_"
, Invk__manual_now_ = "manual_now_"
;
}
class Xob_dump_mgr_base_ {
public static void Load_all_tmpls(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, Xob_dump_src_id page_src) {
List_adp pages = List_adp_.New();
Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template();
Xow_defn_cache defn_cache = wiki.Cache_mgr().Defn_cache();
int cur_page_id = -1;
int load_count = 0;
usr_dlg.Note_many("", "", "tmpl_load init");
while (true) {
page_src.Get_pages(wiki.Appe(), pages, 0, Xow_ns_.Tid__template, cur_page_id); // 0 is always template db
int page_count = pages.Count();
if (page_count == 0) break; // no more pages in db;
Xowd_page_itm page = null;
for (int i = 0; i < page_count; i++) {
page = (Xowd_page_itm)pages.Get_at(i);
Xot_defn_tmpl defn = new Xot_defn_tmpl();
defn.Init_by_new(ns_tmpl, ns_tmpl.Gen_ttl(page.Ttl_page_db()), page.Text(), null, false); // NOTE: passing null, false; will be overriden later when Parse is called
defn_cache.Add(defn, ns_tmpl.Case_match());
++load_count;
if ((load_count % 10000) == 0) usr_dlg.Prog_many("", "", "tmpl_loading: ~{0}", load_count);
}
cur_page_id = page.Id();
}
usr_dlg.Note_many("", "", "tmpl_load done: ~{0}", load_count);
}
public static Xow_db_file[] Init_text_files_ary(Xow_db_mgr core_data_mgr) {
List_adp text_files_list = List_adp_.New();
int len = core_data_mgr.Dbs__len();
if (len == 1) return new Xow_db_file[] {core_data_mgr.Dbs__get_at(0)}; // single file: return core; note that there are no Tid = Text
for (int i = 0; i < len; i++) {
Xow_db_file file = core_data_mgr.Dbs__get_at(i);
switch (file.Tid()) {
case Xow_db_file_.Tid__text:
case Xow_db_file_.Tid__text_solo:
text_files_list.Add(file);
break;
}
}
return (Xow_db_file[])text_files_list.To_ary_and_clear(Xow_db_file.class);
}
}
class Xob_dump_bmk_mgr {
private Bry_bfr save_bfr = Bry_bfr_.Reset(1024);
public Io_url Cfg_url() {return cfg_url;} public Xob_dump_bmk_mgr Cfg_url_(Io_url v) {cfg_url = v; return this;} private Io_url cfg_url;
public void Reset() {Io_mgr.Instance.DeleteFil(cfg_url);}
public void Load(Xoae_app app, Xob_dump_mgr_base dump_mgr) {
app.Gfs_mgr().Run_url_for(dump_mgr, cfg_url);
}
public void Save(int ns_id, int db_id, int pg_id) {
Save_itm(save_bfr, Xob_dump_mgr_base.Invk_ns_bgn_, ns_id);
Save_itm(save_bfr, Xob_dump_mgr_base.Invk_db_bgn_, db_id);
Save_itm(save_bfr, Xob_dump_mgr_base.Invk_pg_bgn_, pg_id);
Io_mgr.Instance.SaveFilBfr(cfg_url, save_bfr);
}
private void Save_itm(Bry_bfr save_bfr, String key, int val) {
String fmt = "{0}('{1}');\n";
String str = String_.Format(fmt, key, val);
save_bfr.Add_str_u8(str);
}
}
class Xob_rate_mgr {
private long time_bgn;
private int item_len;
private Bry_bfr save_bfr = Bry_bfr_.Reset(255);
public int Reset_interval() {return reset_interval;} public Xob_rate_mgr Reset_interval_(int v) {reset_interval = v; return this;} private int reset_interval = 10000;
public Io_url Log_file_url() {return log_file;} public Xob_rate_mgr Log_file_(Io_url v) {log_file = v; return this;} private Io_url log_file;
public void Init() {time_bgn = System_.Ticks();}
public void Increment() {
++item_len;
if (item_len % reset_interval == 0) {
long time_end = System_.Ticks();
Save(item_len, time_bgn, time_end);
time_bgn = time_end;
item_len = 0;
}
}
private void Save(int count, long bgn, long end) {
int dif = (int)(end - bgn) / 1000;
Decimal_adp rate = Decimal_adp_.divide_safe_(count, dif);
save_bfr
.Add_str_a7(rate.To_str("#,##0.000")).Add_byte_pipe()
.Add_int_variable(count).Add_byte_pipe()
.Add_int_variable(dif).Add_byte_nl()
;
Io_mgr.Instance.AppendFilByt(log_file, save_bfr.To_bry_and_clear());
}
public String Rate_as_str() {return Int_.To_str(Rate());}
public int Rate() {
int elapsed = System_.Ticks__elapsed_in_sec(time_bgn);
return Math_.Div_safe_as_int(item_len, elapsed);
}
}
class Xob_dump_bmk {
public int Ns_id() {return ns_id;} public Xob_dump_bmk Ns_id_(int v) {ns_id = v; return this;} private int ns_id;
public int Db_id() {return db_id;} public Xob_dump_bmk Db_id_(int v) {db_id = v; return this;} private int db_id;
public int Pg_id() {return pg_id;} public Xob_dump_bmk Pg_id_(int v) {pg_id = v; return this;} private int pg_id;
}

@ -1,6 +1,6 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
@ -13,437 +13,448 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.parsers.tmpls; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.envs.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.kwds.*; import gplx.xowa.langs.funcs.*;
import gplx.xowa.xtns.pfuncs.*; import gplx.xowa.xtns.pfuncs.ttls.*;
import gplx.xowa.wikis.pages.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.wikis.data.tbls.*;
public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk {
public Xot_invk_tkn(int bgn, int end) {this.Tkn_ini_pos(false, bgn, end);}
@Override public byte Tkn_tid() {return typeId;} private byte typeId = Xop_tkn_itm_.Tid_tmpl_invk;
public void Tkn_tid_to_txt() {typeId = Xop_tkn_itm_.Tid_txt;}
public Arg_nde_tkn Name_tkn() {return name_tkn;} public Xot_invk_tkn Name_tkn_(Arg_nde_tkn v) {name_tkn = v; return this;} Arg_nde_tkn name_tkn = Arg_nde_tkn.Null;
public byte Defn_tid() {return defn_tid;} private byte defn_tid = Xot_defn_.Tid_null;
public int Tmpl_subst_bgn() {return tmpl_subst_bgn;} private int tmpl_subst_bgn;
public int Tmpl_subst_end() {return tmpl_subst_end;} private int tmpl_subst_end;
public Xot_invk_tkn Tmpl_subst_props_(byte tid, int bgn, int end) {defn_tid = tid; tmpl_subst_bgn = bgn; tmpl_subst_end = end; return this;}
public Xot_defn Tmpl_defn() {return tmpl_defn;} public Xot_invk_tkn Tmpl_defn_(Xot_defn v) {tmpl_defn = v; return this;} private Xot_defn tmpl_defn = Xot_defn_.Null;
public boolean Frame_is_root() {return false;}
public byte Frame_tid() {return scrib_tid;} public void Frame_tid_(byte v) {scrib_tid = v;} private byte scrib_tid;
public byte[] Frame_ttl() {return frame_ttl;} public void Frame_ttl_(byte[] v) {frame_ttl = v;} private byte[] frame_ttl;
public int Frame_lifetime() {return frame_lifetime;} public void Frame_lifetime_(int v) {frame_lifetime = v;} private int frame_lifetime;
public boolean Rslt_is_redirect() {return rslt_is_redirect;} public void Rslt_is_redirect_(boolean v) {rslt_is_redirect = v;} private boolean rslt_is_redirect;
@Override public void Tmpl_fmt(Xop_ctx ctx, byte[] src, Xot_fmtr fmtr) {fmtr.Reg_tmpl(ctx, src, name_tkn, args_len, args);}
@Override public void Tmpl_compile(Xop_ctx ctx, byte[] src, Xot_compile_data prep_data) {
name_tkn.Tmpl_compile(ctx, src, prep_data);
int args_len = this.Args_len();
for (int i = 0; i < args_len; i++) {
Arg_nde_tkn nde = args[i];
Xop_tkn_itm key = nde.Key_tkn(); int key_subs_len = key.Subs_len();
for (int j = 0; j < key_subs_len; j++)
key.Subs_get(j).Tmpl_compile(ctx, src, prep_data);
Xop_tkn_itm val = nde.Val_tkn(); int val_subs_len = val.Subs_len();
for (int j = 0; j < val_subs_len; j++)
val.Subs_get(j).Tmpl_compile(ctx, src, prep_data);
}
}
@Override public boolean Tmpl_evaluate(Xop_ctx ctx, byte[] src, Xot_invk caller, Bry_bfr bfr) { // this="{{t|{{{0}}}}}" caller="{{t|1}}"
// init common
boolean rv = false;
Xowe_wiki wiki = ctx.Wiki();
Xol_lang_itm lang = wiki.Lang();
// init defn / name
Xot_defn defn = tmpl_defn;
byte[] name_ary = defn.Name();
byte[] name_ary_orig = Bry_.Empty;
int name_bgn = 0, name_ary_len = 0;
Arg_itm_tkn name_key_tkn = name_tkn.Key_tkn();
// init more
byte[] argx_ary = Bry_.Empty;
boolean subst_found = false;
boolean name_had_subst = false;
boolean template_prefix_found = false;
// tmpl_name does not exist in db; may be dynamic, subst, transclusion, etc..
if (defn == Xot_defn_.Null) {
// dynamic tmpl; EX:{{{{{1}}}|a}}
if (name_key_tkn.Itm_static() == Bool_.N_byte) {
Bry_bfr name_tkn_bfr = Bry_bfr_.New_w_size(name_tkn.Src_end() - name_tkn.Src_bgn());
if (defn_tid == Xot_defn_.Tid_subst)
name_tkn_bfr.Add(Get_first_subst_itm(lang.Kwd_mgr()));
name_tkn.Tmpl_evaluate(ctx, src, caller, name_tkn_bfr);
name_ary = name_tkn_bfr.To_bry_and_clear();
}
// tmpl is static; note that dat_ary is still valid but rest of name may not be; EX: {{subst:name{{{1}}}}}
else
name_ary = Bry_.Mid(src, name_key_tkn.Dat_bgn(), name_key_tkn.Dat_end());
name_had_subst = name_key_tkn.Dat_ary_had_subst();
name_ary_orig = name_ary; // cache name_ary_orig
name_ary_len = name_ary.length;
name_bgn = Bry_find_.Find_fwd_while_not_ws(name_ary, 0, name_ary_len);
if ( name_ary_len == 0 // name is blank; can occur with failed inner tmpl; EX: {{ {{does not exist}} }}
|| name_bgn == name_ary_len // name is ws; EX: {{test| }} -> {{{{{1}}}}}is whitespace String; PAGE:en.d:wear_one's_heart_on_one's_sleeve; EX:{{t+|fr|avoir le cœur sur la main| }}
) {
Gfo_usr_dlg_.Instance.Log_many("", "", "parser.tmpl:dynamic is blank; page=~{0}", ctx.Page().Url_bry_safe()); // downgraded from warning to note; PAGE:de.d:país DATE:2016-09-07
return false;
}
if (name_ary[name_bgn] == Byte_ascii.Colon) { // check 1st letter for transclusion
return Transclude(ctx, wiki, bfr, Bool_.N, name_ary, caller, src); // transclusion; EX: {{:Name of page}}
}
// ignore "{{Template:"; EX: {{Template:a}} is the same thing as {{a}}
int tmpl_ns_len = wiki.Ns_mgr().Tmpls_get_w_colon(name_ary, name_bgn, name_ary_len);
if (tmpl_ns_len != Bry_find_.Not_found) {
name_ary = Bry_.Mid(name_ary, name_bgn + tmpl_ns_len, name_ary_len);
name_ary_len = name_ary.length;
name_bgn = 0;
template_prefix_found = true;
}
byte[] ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); int ns_template_prefix_len = ns_template_prefix.length;
if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) {
name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len);
name_ary_len = name_ary.length;
name_bgn = 0;
}
Xow_ns_mgr_name_itm ns_eval = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia
if (ns_eval != null && !template_prefix_found) // do not transclude ns if Template prefix seen earlier; EX: {{Template:Wikipedia:A}} should not transclude "Wikipedia:A"; DATE:2013-04-03
return SubEval(ctx, wiki, bfr, name_ary, caller, src);
Xol_func_itm finder = new Xol_func_itm(); // TS.MEM: DATE:2016-07-12
lang.Func_regy().Find_defn(finder, name_ary, name_bgn, name_ary_len);
defn = finder.Func();
int finder_tid = finder.Tid();
int finder_colon_pos = finder.Colon_pos();
int finder_subst_end = finder.Subst_end();
int colon_pos = -1;
switch (finder_tid) {
case Xot_defn_.Tid_subst: // subst is added verbatim; EX: {{subst:!}} -> {{subst:!}}; logic below is to handle printing of arg which could be standardized if src[] was available for tmpl
bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary);
for (int i = 0; i < args_len; i++) {
Arg_nde_tkn nde = args[i];
bfr.Add_byte(Byte_ascii.Pipe); // add |
bfr.Add_mid(src, nde.Src_bgn(), nde.Src_end()); // add entire arg; "k=v"; note that src must be added, not evaluated, else <nowiki> may be dropped and cause stack overflow; PAGE:ru.w:Близкиерузья_(Сезон_2) DATE:2014-10-21
}
Xot_fmtr_prm.Instance.Print(bfr);
bfr.Add(Xop_curly_end_lxr.Hook);
return true; // NOTE: nothing else to do; return
case Xot_defn_.Tid_safesubst:
name_ary = Bry_.Mid(name_ary, finder_subst_end, name_ary_len); // chop off "safesubst:"
name_ary_len = name_ary.length;
if (defn != Xot_defn_.Null) { // func found
if (finder_colon_pos != -1) colon_pos = defn.Name().length; // set colon_pos; SEE NOTE_1
}
subst_found = true;
break;
case Xot_defn_.Tid_func:
if (defn.Defn_require_colon_arg()) {
colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon);
if (colon_pos == Bry_find_.Not_found)
defn = Xot_defn_.Null;
}
else {
colon_pos = finder_colon_pos;
}
break;
case Xot_defn_.Tid_raw:
case Xot_defn_.Tid_msg:
int raw_colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon);
if (raw_colon_pos == Bry_find_.Not_found) {} // colon missing; EX: {{raw}}; noop and assume template name; DATE:2014-02-11
else { // colon present;
name_ary = Bry_.Mid(name_ary, finder_subst_end + 1, name_ary_len); // chop off "raw"; +1 is for ":"; note that +1 is in bounds b/c raw_colon was found
name_ary_len = name_ary.length;
Xow_ns_mgr_name_itm ns_eval2 = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia
if (ns_eval2 != null) {
Xow_ns ns_eval_itm = ns_eval2.Ns();
int pre_len = ns_eval_itm.Name_enc().length;
if (pre_len < name_ary_len && name_ary[pre_len] == Byte_ascii.Colon)
return SubEval(ctx, wiki, bfr, name_ary, caller, src);
}
}
switch (finder_tid) {
case Xot_defn_.Tid_msg:
defn = Xot_defn_.Null; // null out defn to force template load below; DATE:2014-07-10
break;
}
break;
}
if (subst_found) {// subst found; remove Template: if it exists; EX: {{safesubst:Template:A}} -> {{A}} not {{Template:A}}; EX:en.d:Kazakhstan; DATE:2014-03-25
ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); ns_template_prefix_len = ns_template_prefix.length;
if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) {
name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len);
name_ary_len = name_ary.length;
name_bgn = 0;
template_prefix_found = true;
}
}
if (colon_pos != -1) { // func; separate name_ary into name_ary and arg_x
argx_ary = Bry_.Trim(name_ary, colon_pos + 1, name_ary_len); // trim bgn ws; needed for "{{formatnum:\n{{#expr:2}}\n}}"
name_ary = Bry_.Mid(name_ary, 0, colon_pos);
}
if (defn == Xot_defn_.Null) {
if (ctx.Tid_is_popup()) { // popup && cur_tmpl > tmpl_max
if (Popup_skip(ctx, name_ary, bfr)) return false;
}
defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary);
if (defn == null) {
if (name_ary_len != 0 ) { // name_ary_len != 0 for direct template inclusions; PAGE:en.w:Human evolution and {{:Human evolution/Species chart}}; && ctx.Tmpl_whitelist().Has(name_ary)
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary));
if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args );
if (subst_found || template_prefix_found) { // if "subst:" or "Template:" found, use orig name; DATE:2014-03-31
bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary_orig).Add(Xop_curly_end_lxr.Hook);
return false;
}
else {// output entire tmpl_src WITH args; used to output name only which broke pages; PAGE:en.w:Flag_of_Greenland; DATE:2016-06-21
bfr.Add(Xop_curly_bgn_lxr.Hook);
bfr.Add(name_ary);
for (int i = 0; i < args_len; ++i) {
Arg_nde_tkn nde = this.Args_get_by_idx(i);
bfr.Add_byte(Byte_ascii.Pipe);
// must evaluate args; "size={{{size|}}}" must become "size="; PAGE:en.w:Europe; en.w:Template:Country_data_Guernsey DATE:2016-10-13
nde.Tmpl_compile(ctx, src, Xot_compile_data.Noop);
nde.Tmpl_evaluate(ctx, src, caller, bfr);
}
bfr.Add(Xop_curly_end_lxr.Hook);
return false;
}
}
else { // some templates produce null ttls; EX: "Citation needed{{subst"
defn = wiki.Cache_mgr().Defn_cache().Get_by_key(ttl.Page_db());
if (defn == null && ctx.Tmpl_load_enabled())
defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary);
}
}
}
if (defn == null) defn = Xot_defn_.Null;
}
}
if ( defn.Defn_tid() == Xot_defn_.Tid_null // name is not a known defn
&& lang.Vnt_mgr().Enabled()) { // lang has vnts
Xowd_page_itm page = lang.Vnt_mgr().Convert_mgr().Convert_ttl(wiki, wiki.Ns_mgr().Ns_template(), name_ary);
if (page != Xowd_page_itm.Null) {
name_ary = page.Ttl_page_db();
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary));
if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args );
bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary).Add(Xop_curly_end_lxr.Hook);
return false;
}
defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary);
if (defn == null && ctx.Tmpl_load_enabled())
defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary);
if (defn == null) defn = Xot_defn_.Null;
}
}
Xot_defn_trace trace = ctx.Defn_trace(); int trg_bgn = bfr.Len();
switch (defn.Defn_tid()) {
case Xot_defn_.Tid_null: // defn is unknown
if (ignore_hash.Get_by_bry(name_ary) == null) {
if (Pfunc_rel2abs.Rel2abs_ttl(name_ary, name_bgn, name_ary_len)) {// rel_path; EX: {{/../Peer page}}; DATE:2013-03-27
Bry_bfr tmp_bfr = ctx.Wiki().Utl__bfr_mkr().Get_b512();
name_ary = Pfunc_rel2abs.Rel2abs(tmp_bfr, wiki.Parser_mgr().Rel2abs_ary(), Bry_.Mid(name_ary, name_bgn, name_ary_len), ctx.Page().Ttl().Raw());
tmp_bfr.Mkr_rls();
return SubEval(ctx, wiki, bfr, name_ary, caller, src);
}
if (subst_found)
return Transclude(ctx, wiki, bfr, template_prefix_found, name_ary, caller, src);
Xot_invk_tkn_.Print_not_found__w_template(bfr, wiki.Ns_mgr(), name_ary);
return false;
}
break;
case Xot_defn_.Tid_func:
try {
Xot_invk_tkn_.Eval_func(ctx, src, caller, this, bfr, defn, argx_ary);
rv = true;
} catch (Exception e) {
if (Env_.Mode_testing())
throw Err_.new_exc(e, "xo", "failed to evaluate function", "page", ctx.Page().Ttl().Full_txt(), "defn", defn.Name(), "src", String_.new_u8(src, this.Src_bgn(), this.Src_end()));
else {
wiki.Appe().Usr_dlg().Warn_many("", "", "failed to evaluate function: page=~{0} defn=~{1} src=~{2} err=~{3}", ctx.Page().Ttl().Full_txt(), defn.Name(), Bry_.Replace_nl_w_tab(src, this.Src_bgn(), this.Src_end()), Err_.Message_gplx_log(e));
rv = false;
}
}
break;
default:
Xot_defn_tmpl defn_tmpl = (Xot_defn_tmpl)defn;
if (defn_tmpl.Root() == null) defn_tmpl.Parse_tmpl(ctx); // NOTE: defn_tmpl.Root can be null after clearing out cache; must be non-null else will fail in trace; DATE:2013-02-01
Xot_invk invk_tmpl = Xot_defn_tmpl_.CopyNew(ctx, defn_tmpl, this, caller, src, Xow_ns_.Tid__template, name_ary);
invk_tmpl.Frame_ttl_(defn_tmpl.Frame_ttl()); // set frame_ttl; needed for redirects; PAGE:en.w:Statutory_city; DATE:2014-08-22
trace.Trace_bgn(ctx, src, name_ary, caller, invk_tmpl, defn);
Bry_bfr rslt_bfr = wiki.Utl__bfr_mkr().Get_k004();
try {
Xot_invk_tkn_.Bld_key(invk_tmpl, name_ary, rslt_bfr);
byte[] rslt_key = rslt_bfr.To_bry_and_clear();
Object o = wiki.Cache_mgr().Tmpl_result_cache().Get_by(rslt_key);
Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(bfr);
if (o != null) {
byte[] rslt = (byte[])o;
prepend_mgr.End(ctx, bfr, rslt, rslt.length, Bool_.Y);
bfr.Add(rslt);
}
else {
rv = defn_tmpl.Tmpl_evaluate(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), invk_tmpl, rslt_bfr); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17
prepend_mgr.End(ctx, bfr, rslt_bfr.Bfr(), rslt_bfr.Len(), Bool_.Y);
if (name_had_subst) { // current invk had "subst:"; parse incoming invk again to remove effects of subst; PAGE:pt.w:Argentina DATE:2014-09-24
byte[] tmp_src = rslt_bfr.To_bry_and_clear();
rslt_bfr.Add(wiki.Parser_mgr().Main().Expand_tmpl(tmp_src)); // this could be cleaner / more optimized
}
if (Cache_enabled) {
byte[] rslt_val = rslt_bfr.To_bry_and_clear();
bfr.Add(rslt_val);
Hash_adp cache = wiki.Cache_mgr().Tmpl_result_cache();
cache.Del(rslt_key);
cache.Add(rslt_key, rslt_val);
}
else
bfr.Add_bfr_and_clear(rslt_bfr);
}
trace.Trace_end(trg_bgn, bfr);
} finally {rslt_bfr.Mkr_rls();}
break;
}
return rv;
}
private boolean Popup_skip(Xop_ctx ctx, byte[] ttl, Bry_bfr bfr) {
boolean skip = false;
skip = this.Src_end() - this.Src_bgn() > ctx.Tmpl_tkn_max();
if (!skip) {
gplx.xowa.htmls.modules.popups.keeplists.Xop_keeplist_wiki tmpl_keeplist = ctx.Tmpl_keeplist();
if (tmpl_keeplist != null && tmpl_keeplist.Enabled()) {
byte[] ttl_lower = Xoa_ttl.Replace_spaces(ctx.Wiki().Lang().Case_mgr().Case_build_lower(ttl));
skip = !tmpl_keeplist.Match(ttl_lower);
// if (skip) Tfds.Write_bry(ttl_lower);
}
}
if (skip) {
bfr.Add(gplx.xowa.parsers.miscs.Xop_comm_lxr.Xowa_skip_comment_bry); // add comment tkn; need something to separate ''{{lang|la|Ragusa}}'' else will become ''''; PAGE:en.w:Republic_of_Ragusa; DATE:2014-06-28
return true;
}
else
return false;
}
private boolean Transclude(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, boolean template_prefix_found, byte[] name_ary, Xot_invk caller, byte[] src) {
Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}}
byte[] transclude_src = null;
if (page_ttl.Ns().Id_is_tmpl()) { // ttl is template; check tmpl_regy first before going to data_mgr
Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db());
if (tmpl != null) transclude_src = tmpl.Data_raw();
}
if (transclude_src == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
if ( cache_itm != null
// && Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Page_ttl().Full_db()) // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
) {
transclude_src = cache_itm.Wtxt__direct();
page_ttl = cache_itm.Ttl();
}
}
if (transclude_src != null) {
// NOTE: must use new page, not current, else transcluded page can cause inconsistent TOC state; PAGE:de.w:Game_of_Thrones DATE:2016-11-21
Xot_defn_tmpl transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, Xoae_page.New(wiki, page_ttl)), ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), transclude_src);
return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src, bfr);
}
else {
Xot_invk_tkn_.Print_not_found__by_transclude(bfr, wiki.Ns_mgr(), template_prefix_found, name_ary);
return false;
}
}
private boolean Eval_sub(Xop_ctx ctx, Xoa_ttl transclude_ttl, Xot_defn_tmpl transclude_tmpl, Xot_invk caller, byte[] src, Bry_bfr doc) {
boolean rv = false;
Xot_invk tmp_tmpl = Xot_defn_tmpl_.CopyNew(ctx, transclude_tmpl, this, caller, src, transclude_ttl.Ns().Id(), transclude_tmpl.Name());
Bry_bfr tmp_bfr = Bry_bfr_.New();
Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(doc);
rv = transclude_tmpl.Tmpl_evaluate(ctx, tmp_tmpl, tmp_bfr);
prepend_mgr.End(ctx, doc, tmp_bfr.Bfr(), tmp_bfr.Len(), Bool_.Y);
doc.Add_bfr_and_clear(tmp_bfr);
return rv;
}
private boolean SubEval(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, byte[] name_ary, Xot_invk caller, byte[] src_for_tkn) {
Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}}
Xot_defn_tmpl transclude_tmpl = null;
switch (page_ttl.Ns().Id()) {
case Xow_ns_.Tid__template: // ttl is template not in cache, or some other ns; do load
Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db());
if (tmpl != null) {
if (tmpl.Root() == null) tmpl.Parse_tmpl(ctx);
transclude_tmpl = tmpl;
}
break;
case Xow_ns_.Tid__special:
bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon).Add(name_ary).Add(Xop_tkn_.Lnki_end);
return true;
}
if (transclude_tmpl == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
if ( cache_itm != null) {
if (!Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Ttl().Full_db())) { // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), cache_itm.Wtxt__direct());
page_ttl = cache_itm.Ttl();
}
}
}
if (transclude_tmpl == null) {
bfr.Add(Xop_tkn_.Lnki_bgn).Add(name_ary).Add(Xop_tkn_.Lnki_end); // indicate template was not found; DATE:2014-02-12
return false;
}
return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src_for_tkn, bfr);
}
public int Args_len() {return args_len;} private int args_len = 0;
public Arg_nde_tkn Args_get_by_idx(int idx) {return args[idx];}
public Arg_nde_tkn Args_eval_by_idx(byte[] src, int idx) {
int cur = 0;
for (int i = 0; i < args_len; i++) {
Arg_nde_tkn nde = args[i];
if (nde.KeyTkn_exists()) continue;
if (cur++ == idx) return nde;
}
return null;
}
public Arg_nde_tkn Args_get_by_key(byte[] src, byte[] key) {
for (int i = 0; i < args_len; i++) {
Arg_nde_tkn nde = args[i];
if (!nde.KeyTkn_exists()) continue;
if (Bry_.Match(src, nde.Key_tkn().Dat_bgn(), nde.Key_tkn().Dat_end(), key)) return nde; // NOTE: dat_ary is guaranteed to exist
}
return null;
}
public void Args_add(Xop_ctx ctx, Arg_nde_tkn arg) {
int newLen = args_len + 1;
if (newLen > args_max) {
args_max = newLen * 2;
args = Resize(args, args_len, args_max);
}
args[args_len] = arg;
arg.Tkn_grp_(this, args_len);
args_len = newLen;
} Arg_nde_tkn[] args = Arg_nde_tkn.Ary_empty; int args_max;
private Arg_nde_tkn[] Resize(Arg_nde_tkn[] src, int cur_len, int new_len) {
Arg_nde_tkn[] rv = new Arg_nde_tkn[new_len];
Array_.Copy_to(src, 0, rv, 0, cur_len);
return rv;
}
private byte[] Get_first_subst_itm(Xol_kwd_mgr kwd_mgr) {
Xol_kwd_grp grp = kwd_mgr.Get_at(Xol_kwd_grp_.Id_subst); if (grp == null) return Bry_.Empty;
Xol_kwd_itm[] itms = grp.Itms();
return itms.length == 0 ? Bry_.Empty : itms[0].Val();
}
private static final Hash_adp_bry ignore_hash = Hash_adp_bry.ci_a7().Add_str_obj("Citation needed{{subst", "").Add_str_obj("Clarify{{subst", ""); // ignore SafeSubst templates
public static boolean Cache_enabled = false;
}
/*
NOTE_1: if (finder.Colon_pos() != -1) colon_pos = finder.Func().Name().length;
Two issues here:
1) "if (finder.Colon_pos() != -1)"
Colon_pos can either be -1 or >0
EX: -1: safesubst:NAMESPACE
EX: >0: safesubst:#expr:0
if -1, don't do anything; this will leave colon_pos as -1
2) "finder.Func().Name().length"
Colon_pos is >0, but refers to String before it was chopped
EX: "safesubst:#expr:0" has Colon_pos of 15 but String is now "#expr:0"
so, get colon_pos by taking the finder.Func().Name().length
NOTE: whitespace does not matter b/c "safesubst: #expr:0" would never be a func;
package gplx.xowa.parsers.tmpls;
import gplx.Array_;
import gplx.Bool_;
import gplx.Bry_;
import gplx.Bry_bfr;
import gplx.Bry_bfr_;
import gplx.Bry_find_;
import gplx.Byte_ascii;
import gplx.Err_;
import gplx.Gfo_usr_dlg_;
import gplx.Hash_adp_bry;
import gplx.String_;
import gplx.core.envs.Env_;
import gplx.xowa.Xoa_ttl;
import gplx.xowa.Xoae_page;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.langs.Xol_lang_itm;
import gplx.xowa.langs.funcs.Xol_func_itm;
import gplx.xowa.langs.kwds.Xol_kwd_grp;
import gplx.xowa.langs.kwds.Xol_kwd_grp_;
import gplx.xowa.langs.kwds.Xol_kwd_itm;
import gplx.xowa.langs.kwds.Xol_kwd_mgr;
import gplx.xowa.parsers.Xop_ctx;
import gplx.xowa.parsers.Xop_tkn_itm;
import gplx.xowa.parsers.Xop_tkn_itm_;
import gplx.xowa.parsers.Xop_tkn_itm_base;
import gplx.xowa.wikis.caches.Xow_page_cache_itm;
import gplx.xowa.wikis.data.tbls.Xowd_page_itm;
import gplx.xowa.wikis.nss.Xow_ns;
import gplx.xowa.wikis.nss.Xow_ns_;
import gplx.xowa.wikis.nss.Xow_ns_mgr_name_itm;
import gplx.xowa.wikis.pages.Xopg_tmpl_prepend_mgr;
import gplx.xowa.xtns.pfuncs.ttls.Pfunc_rel2abs;
public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk {
public Xot_invk_tkn(int bgn, int end) {this.Tkn_ini_pos(false, bgn, end);}
@Override public byte Tkn_tid() {return typeId;} private byte typeId = Xop_tkn_itm_.Tid_tmpl_invk;
public void Tkn_tid_to_txt() {typeId = Xop_tkn_itm_.Tid_txt;}
public Arg_nde_tkn Name_tkn() {return name_tkn;} public Xot_invk_tkn Name_tkn_(Arg_nde_tkn v) {name_tkn = v; return this;} Arg_nde_tkn name_tkn = Arg_nde_tkn.Null;
public byte Defn_tid() {return defn_tid;} private byte defn_tid = Xot_defn_.Tid_null;
public int Tmpl_subst_bgn() {return tmpl_subst_bgn;} private int tmpl_subst_bgn;
public int Tmpl_subst_end() {return tmpl_subst_end;} private int tmpl_subst_end;
public Xot_invk_tkn Tmpl_subst_props_(byte tid, int bgn, int end) {defn_tid = tid; tmpl_subst_bgn = bgn; tmpl_subst_end = end; return this;}
public Xot_defn Tmpl_defn() {return tmpl_defn;} public Xot_invk_tkn Tmpl_defn_(Xot_defn v) {tmpl_defn = v; return this;} private Xot_defn tmpl_defn = Xot_defn_.Null;
public boolean Frame_is_root() {return false;}
public byte Frame_tid() {return scrib_tid;} public void Frame_tid_(byte v) {scrib_tid = v;} private byte scrib_tid;
public byte[] Frame_ttl() {return frame_ttl;} public void Frame_ttl_(byte[] v) {frame_ttl = v;} private byte[] frame_ttl;
public int Frame_lifetime() {return frame_lifetime;} public void Frame_lifetime_(int v) {frame_lifetime = v;} private int frame_lifetime;
public boolean Rslt_is_redirect() {return rslt_is_redirect;} public void Rslt_is_redirect_(boolean v) {rslt_is_redirect = v;} private boolean rslt_is_redirect;
@Override public void Tmpl_fmt(Xop_ctx ctx, byte[] src, Xot_fmtr fmtr) {fmtr.Reg_tmpl(ctx, src, name_tkn, args_len, args);}
@Override public void Tmpl_compile(Xop_ctx ctx, byte[] src, Xot_compile_data prep_data) {
name_tkn.Tmpl_compile(ctx, src, prep_data);
int args_len = this.Args_len();
for (int i = 0; i < args_len; i++) {
Arg_nde_tkn nde = args[i];
Xop_tkn_itm key = nde.Key_tkn(); int key_subs_len = key.Subs_len();
for (int j = 0; j < key_subs_len; j++)
key.Subs_get(j).Tmpl_compile(ctx, src, prep_data);
Xop_tkn_itm val = nde.Val_tkn(); int val_subs_len = val.Subs_len();
for (int j = 0; j < val_subs_len; j++)
val.Subs_get(j).Tmpl_compile(ctx, src, prep_data);
}
}
@Override public boolean Tmpl_evaluate(Xop_ctx ctx, byte[] src, Xot_invk caller, Bry_bfr bfr) { // this="{{t|{{{0}}}}}" caller="{{t|1}}"
// init common
boolean rv = false;
Xowe_wiki wiki = ctx.Wiki();
Xol_lang_itm lang = wiki.Lang();
// init defn / name
Xot_defn defn = tmpl_defn;
byte[] name_ary = defn.Name();
byte[] name_ary_orig = Bry_.Empty;
int name_bgn = 0, name_ary_len = 0;
Arg_itm_tkn name_key_tkn = name_tkn.Key_tkn();
// init more
byte[] argx_ary = Bry_.Empty;
boolean subst_found = false;
boolean name_had_subst = false;
boolean template_prefix_found = false;
// tmpl_name does not exist in db; may be dynamic, subst, transclusion, etc..
if (defn == Xot_defn_.Null) {
// dynamic tmpl; EX:{{{{{1}}}|a}}
if (name_key_tkn.Itm_static() == Bool_.N_byte) {
Bry_bfr name_tkn_bfr = Bry_bfr_.New_w_size(name_tkn.Src_end() - name_tkn.Src_bgn());
if (defn_tid == Xot_defn_.Tid_subst)
name_tkn_bfr.Add(Get_first_subst_itm(lang.Kwd_mgr()));
name_tkn.Tmpl_evaluate(ctx, src, caller, name_tkn_bfr);
name_ary = name_tkn_bfr.To_bry_and_clear();
}
// tmpl is static; note that dat_ary is still valid but rest of name may not be; EX: {{subst:name{{{1}}}}}
else
name_ary = Bry_.Mid(src, name_key_tkn.Dat_bgn(), name_key_tkn.Dat_end());
name_had_subst = name_key_tkn.Dat_ary_had_subst();
name_ary_orig = name_ary; // cache name_ary_orig
name_ary_len = name_ary.length;
name_bgn = Bry_find_.Find_fwd_while_not_ws(name_ary, 0, name_ary_len);
if ( name_ary_len == 0 // name is blank; can occur with failed inner tmpl; EX: {{ {{does not exist}} }}
|| name_bgn == name_ary_len // name is ws; EX: {{test| }} -> {{{{{1}}}}}is whitespace String; PAGE:en.d:wear_one's_heart_on_one's_sleeve; EX:{{t+|fr|avoir le cœur sur la main| }}
) {
Gfo_usr_dlg_.Instance.Log_many("", "", "parser.tmpl:dynamic is blank; page=~{0}", ctx.Page().Url_bry_safe()); // downgraded from warning to note; PAGE:de.d:país DATE:2016-09-07
return false;
}
if (name_ary[name_bgn] == Byte_ascii.Colon) { // check 1st letter for transclusion
return Transclude(ctx, wiki, bfr, Bool_.N, name_ary, caller, src); // transclusion; EX: {{:Name of page}}
}
// ignore "{{Template:"; EX: {{Template:a}} is the same thing as {{a}}
int tmpl_ns_len = wiki.Ns_mgr().Tmpls_get_w_colon(name_ary, name_bgn, name_ary_len);
if (tmpl_ns_len != Bry_find_.Not_found) {
name_ary = Bry_.Mid(name_ary, name_bgn + tmpl_ns_len, name_ary_len);
name_ary_len = name_ary.length;
name_bgn = 0;
template_prefix_found = true;
}
byte[] ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); int ns_template_prefix_len = ns_template_prefix.length;
if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) {
name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len);
name_ary_len = name_ary.length;
name_bgn = 0;
}
Xow_ns_mgr_name_itm ns_eval = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia
if (ns_eval != null && !template_prefix_found) // do not transclude ns if Template prefix seen earlier; EX: {{Template:Wikipedia:A}} should not transclude "Wikipedia:A"; DATE:2013-04-03
return SubEval(ctx, wiki, bfr, name_ary, caller, src);
Xol_func_itm finder = new Xol_func_itm(); // TS.MEM: DATE:2016-07-12
lang.Func_regy().Find_defn(finder, name_ary, name_bgn, name_ary_len);
defn = finder.Func();
int finder_tid = finder.Tid();
int finder_colon_pos = finder.Colon_pos();
int finder_subst_end = finder.Subst_end();
int colon_pos = -1;
switch (finder_tid) {
case Xot_defn_.Tid_subst: // subst is added verbatim; EX: {{subst:!}} -> {{subst:!}}; logic below is to handle printing of arg which could be standardized if src[] was available for tmpl
bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary);
for (int i = 0; i < args_len; i++) {
Arg_nde_tkn nde = args[i];
bfr.Add_byte(Byte_ascii.Pipe); // add |
bfr.Add_mid(src, nde.Src_bgn(), nde.Src_end()); // add entire arg; "k=v"; note that src must be added, not evaluated, else <nowiki> may be dropped and cause stack overflow; PAGE:ru.w:Близкиерузья_(Сезон_2) DATE:2014-10-21
}
Xot_fmtr_prm.Instance.Print(bfr);
bfr.Add(Xop_curly_end_lxr.Hook);
return true; // NOTE: nothing else to do; return
case Xot_defn_.Tid_safesubst:
name_ary = Bry_.Mid(name_ary, finder_subst_end, name_ary_len); // chop off "safesubst:"
name_ary_len = name_ary.length;
if (defn != Xot_defn_.Null) { // func found
if (finder_colon_pos != -1) colon_pos = defn.Name().length; // set colon_pos; SEE NOTE_1
}
subst_found = true;
break;
case Xot_defn_.Tid_func:
if (defn.Defn_require_colon_arg()) {
colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon);
if (colon_pos == Bry_find_.Not_found)
defn = Xot_defn_.Null;
}
else {
colon_pos = finder_colon_pos;
}
break;
case Xot_defn_.Tid_raw:
case Xot_defn_.Tid_msg:
int raw_colon_pos = Bry_find_.Find_fwd(name_ary, Byte_ascii.Colon);
if (raw_colon_pos == Bry_find_.Not_found) {} // colon missing; EX: {{raw}}; noop and assume template name; DATE:2014-02-11
else { // colon present;
name_ary = Bry_.Mid(name_ary, finder_subst_end + 1, name_ary_len); // chop off "raw"; +1 is for ":"; note that +1 is in bounds b/c raw_colon was found
name_ary_len = name_ary.length;
Xow_ns_mgr_name_itm ns_eval2 = wiki.Ns_mgr().Names_get_w_colon_or_null(name_ary, 0, name_ary_len); // match {{:Portal or {{:Wikipedia
if (ns_eval2 != null) {
Xow_ns ns_eval_itm = ns_eval2.Ns();
int pre_len = ns_eval_itm.Name_enc().length;
if (pre_len < name_ary_len && name_ary[pre_len] == Byte_ascii.Colon)
return SubEval(ctx, wiki, bfr, name_ary, caller, src);
}
}
switch (finder_tid) {
case Xot_defn_.Tid_msg:
defn = Xot_defn_.Null; // null out defn to force template load below; DATE:2014-07-10
break;
}
break;
}
if (subst_found) {// subst found; remove Template: if it exists; EX: {{safesubst:Template:A}} -> {{A}} not {{Template:A}}; EX:en.d:Kazakhstan; DATE:2014-03-25
ns_template_prefix = wiki.Ns_mgr().Ns_template().Name_db_w_colon(); ns_template_prefix_len = ns_template_prefix.length;
if (name_ary_len > ns_template_prefix_len && Bry_.Match(name_ary, name_bgn, name_bgn + ns_template_prefix_len, ns_template_prefix)) {
name_ary = Bry_.Mid(name_ary, name_bgn + ns_template_prefix_len, name_ary_len);
name_ary_len = name_ary.length;
name_bgn = 0;
template_prefix_found = true;
}
}
if (colon_pos != -1) { // func; separate name_ary into name_ary and arg_x
argx_ary = Bry_.Trim(name_ary, colon_pos + 1, name_ary_len); // trim bgn ws; needed for "{{formatnum:\n{{#expr:2}}\n}}"
name_ary = Bry_.Mid(name_ary, 0, colon_pos);
}
if (defn == Xot_defn_.Null) {
if (ctx.Tid_is_popup()) { // popup && cur_tmpl > tmpl_max
if (Popup_skip(ctx, name_ary, bfr)) return false;
}
defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary);
if (defn == null) {
if (name_ary_len != 0 ) { // name_ary_len != 0 for direct template inclusions; PAGE:en.w:Human evolution and {{:Human evolution/Species chart}}; && ctx.Tmpl_whitelist().Has(name_ary)
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary));
if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args );
if (subst_found || template_prefix_found) { // if "subst:" or "Template:" found, use orig name; DATE:2014-03-31
bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary_orig).Add(Xop_curly_end_lxr.Hook);
return false;
}
else {// output entire tmpl_src WITH args; used to output name only which broke pages; PAGE:en.w:Flag_of_Greenland; DATE:2016-06-21
bfr.Add(Xop_curly_bgn_lxr.Hook);
bfr.Add(name_ary);
for (int i = 0; i < args_len; ++i) {
Arg_nde_tkn nde = this.Args_get_by_idx(i);
bfr.Add_byte(Byte_ascii.Pipe);
// must evaluate args; "size={{{size|}}}" must become "size="; PAGE:en.w:Europe; en.w:Template:Country_data_Guernsey DATE:2016-10-13
nde.Tmpl_compile(ctx, src, Xot_compile_data.Noop);
nde.Tmpl_evaluate(ctx, src, caller, bfr);
}
bfr.Add(Xop_curly_end_lxr.Hook);
return false;
}
}
else { // some templates produce null ttls; EX: "Citation needed{{subst"
defn = wiki.Cache_mgr().Defn_cache().Get_by_key(ttl.Page_db());
if (defn == null && ctx.Tmpl_load_enabled())
defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary);
}
}
}
if (defn == null) defn = Xot_defn_.Null;
}
}
if ( defn.Defn_tid() == Xot_defn_.Tid_null // name is not a known defn
&& lang.Vnt_mgr().Enabled()) { // lang has vnts
Xowd_page_itm page = lang.Vnt_mgr().Convert_mgr().Convert_ttl(wiki, wiki.Ns_mgr().Ns_template(), name_ary);
if (page != Xowd_page_itm.Null) {
name_ary = page.Ttl_page_db();
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.Add(wiki.Ns_mgr().Ns_template().Name_db_w_colon(), name_ary));
if (ttl == null) { // ttl is not valid; just output orig; REF.MW:Parser.php|braceSubstitution|if ( !$found ) $text = $frame->virtualBracketedImplode( '{{', '|', '}}', $titleWithSpaces, $args );
bfr.Add(Xop_curly_bgn_lxr.Hook).Add(name_ary).Add(Xop_curly_end_lxr.Hook);
return false;
}
defn = wiki.Cache_mgr().Defn_cache().Get_by_key(name_ary);
if (defn == null && ctx.Tmpl_load_enabled())
defn = Xot_invk_tkn_.Load_defn(wiki, ctx, this, ttl, name_ary);
if (defn == null) defn = Xot_defn_.Null;
}
}
Xot_defn_trace trace = ctx.Defn_trace(); int trg_bgn = bfr.Len();
switch (defn.Defn_tid()) {
case Xot_defn_.Tid_null: // defn is unknown
if (ignore_hash.Get_by_bry(name_ary) == null) {
if (Pfunc_rel2abs.Rel2abs_ttl(name_ary, name_bgn, name_ary_len)) {// rel_path; EX: {{/../Peer page}}; DATE:2013-03-27
Bry_bfr tmp_bfr = ctx.Wiki().Utl__bfr_mkr().Get_b512();
name_ary = Pfunc_rel2abs.Rel2abs(tmp_bfr, wiki.Parser_mgr().Rel2abs_ary(), Bry_.Mid(name_ary, name_bgn, name_ary_len), ctx.Page().Ttl().Raw());
tmp_bfr.Mkr_rls();
return SubEval(ctx, wiki, bfr, name_ary, caller, src);
}
if (subst_found)
return Transclude(ctx, wiki, bfr, template_prefix_found, name_ary, caller, src);
Xot_invk_tkn_.Print_not_found__w_template(bfr, wiki.Ns_mgr(), name_ary);
return false;
}
break;
case Xot_defn_.Tid_func:
try {
Xot_invk_tkn_.Eval_func(ctx, src, caller, this, bfr, defn, argx_ary);
rv = true;
} catch (Exception e) {
if (Env_.Mode_testing())
throw Err_.new_exc(e, "xo", "failed to evaluate function", "page", ctx.Page().Ttl().Full_txt(), "defn", defn.Name(), "src", String_.new_u8(src, this.Src_bgn(), this.Src_end()));
else {
wiki.Appe().Usr_dlg().Warn_many("", "", "failed to evaluate function: page=~{0} defn=~{1} src=~{2} err=~{3}", ctx.Page().Ttl().Full_txt(), defn.Name(), Bry_.Replace_nl_w_tab(src, this.Src_bgn(), this.Src_end()), Err_.Message_gplx_log(e));
rv = false;
}
}
break;
default:
Xot_defn_tmpl defn_tmpl = (Xot_defn_tmpl)defn;
if (defn_tmpl.Root() == null) defn_tmpl.Parse_tmpl(ctx); // NOTE: defn_tmpl.Root can be null after clearing out cache; must be non-null else will fail in trace; DATE:2013-02-01
Xot_invk invk_tmpl = Xot_defn_tmpl_.CopyNew(ctx, defn_tmpl, this, caller, src, Xow_ns_.Tid__template, name_ary);
invk_tmpl.Frame_ttl_(defn_tmpl.Frame_ttl()); // set frame_ttl; needed for redirects; PAGE:en.w:Statutory_city; DATE:2014-08-22
trace.Trace_bgn(ctx, src, name_ary, caller, invk_tmpl, defn);
Bry_bfr rslt_bfr = wiki.Utl__bfr_mkr().Get_k004();
try {
Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(bfr);
rv = defn_tmpl.Tmpl_evaluate(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), invk_tmpl, rslt_bfr); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17
prepend_mgr.End(ctx, bfr, rslt_bfr.Bfr(), rslt_bfr.Len(), Bool_.Y);
if (name_had_subst) { // current invk had "subst:"; parse incoming invk again to remove effects of subst; PAGE:pt.w:Argentina DATE:2014-09-24
byte[] tmp_src = rslt_bfr.To_bry_and_clear();
rslt_bfr.Add(wiki.Parser_mgr().Main().Expand_tmpl(tmp_src)); // this could be cleaner / more optimized
}
bfr.Add_bfr_and_clear(rslt_bfr);
trace.Trace_end(trg_bgn, bfr);
} finally {rslt_bfr.Mkr_rls();}
break;
}
return rv;
}
private boolean Popup_skip(Xop_ctx ctx, byte[] ttl, Bry_bfr bfr) {
boolean skip = false;
skip = this.Src_end() - this.Src_bgn() > ctx.Tmpl_tkn_max();
if (!skip) {
gplx.xowa.htmls.modules.popups.keeplists.Xop_keeplist_wiki tmpl_keeplist = ctx.Tmpl_keeplist();
if (tmpl_keeplist != null && tmpl_keeplist.Enabled()) {
byte[] ttl_lower = Xoa_ttl.Replace_spaces(ctx.Wiki().Lang().Case_mgr().Case_build_lower(ttl));
skip = !tmpl_keeplist.Match(ttl_lower);
// if (skip) Tfds.Write_bry(ttl_lower);
}
}
if (skip) {
bfr.Add(gplx.xowa.parsers.miscs.Xop_comm_lxr.Xowa_skip_comment_bry); // add comment tkn; need something to separate ''{{lang|la|Ragusa}}'' else will become ''''; PAGE:en.w:Republic_of_Ragusa; DATE:2014-06-28
return true;
}
else
return false;
}
private boolean Transclude(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, boolean template_prefix_found, byte[] name_ary, Xot_invk caller, byte[] src) {
Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}}
byte[] transclude_src = null;
if (page_ttl.Ns().Id_is_tmpl()) { // ttl is template; check tmpl_regy first before going to data_mgr
Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db());
if (tmpl != null) transclude_src = tmpl.Data_raw();
}
if (transclude_src == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
if ( cache_itm != null
// && Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Page_ttl().Full_db()) // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
) {
transclude_src = cache_itm.Wtxt__direct();
page_ttl = cache_itm.Ttl();
}
}
if (transclude_src != null) {
// NOTE: must use new page, not current, else transcluded page can cause inconsistent TOC state; PAGE:de.w:Game_of_Thrones DATE:2016-11-21
Xot_defn_tmpl transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, Xoae_page.New(wiki, page_ttl)), ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), transclude_src);
return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src, bfr);
}
else {
Xot_invk_tkn_.Print_not_found__by_transclude(bfr, wiki.Ns_mgr(), template_prefix_found, name_ary);
return false;
}
}
private boolean Eval_sub(Xop_ctx ctx, Xoa_ttl transclude_ttl, Xot_defn_tmpl transclude_tmpl, Xot_invk caller, byte[] src, Bry_bfr doc) {
boolean rv = false;
Xot_invk tmp_tmpl = Xot_defn_tmpl_.CopyNew(ctx, transclude_tmpl, this, caller, src, transclude_ttl.Ns().Id(), transclude_tmpl.Name());
Bry_bfr tmp_bfr = Bry_bfr_.New();
Xopg_tmpl_prepend_mgr prepend_mgr = ctx.Page().Tmpl_prepend_mgr().Bgn(doc);
rv = transclude_tmpl.Tmpl_evaluate(ctx, tmp_tmpl, tmp_bfr);
prepend_mgr.End(ctx, doc, tmp_bfr.Bfr(), tmp_bfr.Len(), Bool_.Y);
doc.Add_bfr_and_clear(tmp_bfr);
return rv;
}
private boolean SubEval(Xop_ctx ctx, Xowe_wiki wiki, Bry_bfr bfr, byte[] name_ary, Xot_invk caller, byte[] src_for_tkn) {
Xoa_ttl page_ttl = Xoa_ttl.Parse(wiki, name_ary); if (page_ttl == null) return false; // ttl not valid; EX: {{:[[abc]]}}
Xot_defn_tmpl transclude_tmpl = null;
switch (page_ttl.Ns().Id()) {
case Xow_ns_.Tid__template: // ttl is template not in cache, or some other ns; do load
Xot_defn_tmpl tmpl = (Xot_defn_tmpl)wiki.Cache_mgr().Defn_cache().Get_by_key(page_ttl.Page_db());
if (tmpl != null) {
if (tmpl.Root() == null) tmpl.Parse_tmpl(ctx);
transclude_tmpl = tmpl;
}
break;
case Xow_ns_.Tid__special:
bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon).Add(name_ary).Add(Xop_tkn_.Lnki_end);
return true;
}
if (transclude_tmpl == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
if ( cache_itm != null) {
if (!Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Ttl().Full_db())) { // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), cache_itm.Wtxt__direct());
page_ttl = cache_itm.Ttl();
}
}
}
if (transclude_tmpl == null) {
bfr.Add(Xop_tkn_.Lnki_bgn).Add(name_ary).Add(Xop_tkn_.Lnki_end); // indicate template was not found; DATE:2014-02-12
return false;
}
return Eval_sub(ctx, page_ttl, transclude_tmpl, caller, src_for_tkn, bfr);
}
public int Args_len() {return args_len;} private int args_len = 0;
public Arg_nde_tkn Args_get_by_idx(int idx) {return args[idx];}
public Arg_nde_tkn Args_eval_by_idx(byte[] src, int idx) {
int cur = 0;
for (int i = 0; i < args_len; i++) {
Arg_nde_tkn nde = args[i];
if (nde.KeyTkn_exists()) continue;
if (cur++ == idx) return nde;
}
return null;
}
public Arg_nde_tkn Args_get_by_key(byte[] src, byte[] key) {
for (int i = 0; i < args_len; i++) {
Arg_nde_tkn nde = args[i];
if (!nde.KeyTkn_exists()) continue;
if (Bry_.Match(src, nde.Key_tkn().Dat_bgn(), nde.Key_tkn().Dat_end(), key)) return nde; // NOTE: dat_ary is guaranteed to exist
}
return null;
}
public void Args_add(Xop_ctx ctx, Arg_nde_tkn arg) {
int newLen = args_len + 1;
if (newLen > args_max) {
args_max = newLen * 2;
args = Resize(args, args_len, args_max);
}
args[args_len] = arg;
arg.Tkn_grp_(this, args_len);
args_len = newLen;
} Arg_nde_tkn[] args = Arg_nde_tkn.Ary_empty; int args_max;
private Arg_nde_tkn[] Resize(Arg_nde_tkn[] src, int cur_len, int new_len) {
Arg_nde_tkn[] rv = new Arg_nde_tkn[new_len];
Array_.Copy_to(src, 0, rv, 0, cur_len);
return rv;
}
private byte[] Get_first_subst_itm(Xol_kwd_mgr kwd_mgr) {
Xol_kwd_grp grp = kwd_mgr.Get_at(Xol_kwd_grp_.Id_subst); if (grp == null) return Bry_.Empty;
Xol_kwd_itm[] itms = grp.Itms();
return itms.length == 0 ? Bry_.Empty : itms[0].Val();
}
private static final Hash_adp_bry ignore_hash = Hash_adp_bry.ci_a7().Add_str_obj("Citation needed{{subst", "").Add_str_obj("Clarify{{subst", ""); // ignore SafeSubst templates
}
/*
NOTE_1: if (finder.Colon_pos() != -1) colon_pos = finder.Func().Name().length;
Two issues here:
1) "if (finder.Colon_pos() != -1)"
Colon_pos can either be -1 or >0
EX: -1: safesubst:NAMESPACE
EX: >0: safesubst:#expr:0
if -1, don't do anything; this will leave colon_pos as -1
2) "finder.Func().Name().length"
Colon_pos is >0, but refers to String before it was chopped
EX: "safesubst:#expr:0" has Colon_pos of 15 but String is now "#expr:0"
so, get colon_pos by taking the finder.Func().Name().length
NOTE: whitespace does not matter b/c "safesubst: #expr:0" would never be a func;
*/

@ -1,6 +1,6 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
@ -13,67 +13,66 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.parsers.tmpls; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.xowa.langs.kwds.*;
import gplx.xowa.xtns.pfuncs.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.caches.*;
public class Xot_invk_tkn_ {
public static void Eval_func(Xop_ctx ctx, byte[] src, Xot_invk caller, Xot_invk invk, Bry_bfr bfr, Xot_defn defn, byte[] argx_ary) {
Pf_func_base defn_func = (Pf_func_base)defn;
int defn_func_id = defn_func.Id();
defn_func = (Pf_func_base)defn_func.New(defn_func_id, defn_func.Name()); // NOTE: always make copy b/c argx_ary may be dynamic
if (argx_ary != Bry_.Empty) defn_func.Argx_dat_(argx_ary);
defn_func.Eval_argx(ctx, src, caller, invk);
if (defn_func_id == Xol_kwd_grp_.Id_invoke) // NOTE: if #invoke, set frame_ttl to argx, not name; EX:{{#invoke:A}}
invk.Frame_ttl_(Bry_.Add(Xow_ns_.Bry__module_w_colon, Xoa_ttl.Replace_unders(defn_func.Argx_dat()))); // NOTE: always prepend "Module:" to frame_ttl; DATE:2014-06-13; NOTE: always use spaces; DATE:2014-08-14; always use canonical English "Module"; DATE:2015-11-09
Bry_bfr bfr_func = Bry_bfr_.New();
defn_func.Func_evaluate(bfr_func, ctx, caller, invk, src);
if (caller.Rslt_is_redirect()) // do not prepend if page is redirect; EX:"#REDIRECT" x> "\n#REDIRECT" DATE:2014-07-11
caller.Rslt_is_redirect_(false); // reset flag; needed for TEST; kludgy, but Rslt_is_redirect is intended for single use
else
ctx.Page().Tmpl_prepend_mgr().End(ctx, bfr, bfr_func.Bfr(), bfr_func.Len(), Bool_.N);
bfr.Add_bfr_and_clear(bfr_func);
}
public static void Bld_key(Xot_invk invk, byte[] name_ary, Bry_bfr bfr) {
bfr.Clear();
bfr.Add(name_ary);
int args_len = invk.Args_len();
for (int i = 0; i < args_len; i++) {
Arg_nde_tkn nde = invk.Args_get_by_idx(i);
bfr.Add_byte(Byte_ascii.Pipe);
if (nde.KeyTkn_exists()) {
bfr.Add(nde.Key_tkn().Dat_ary());
bfr.Add_byte(Byte_ascii.Eq);
}
bfr.Add(nde.Val_tkn().Dat_ary());
}
}
public static Xot_defn_tmpl Load_defn(Xowe_wiki wiki, Xop_ctx ctx, Xot_invk_tkn invk_tkn, Xoa_ttl ttl, byte[] name_ary) { // recursive loading of templates
Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
byte[] tmpl_page_bry = tmpl_page_itm == null ? null : tmpl_page_itm.Wtxt__direct();
Xot_defn_tmpl rv = null;
if (tmpl_page_bry != null) {
byte old_parse_tid = ctx.Parse_tid(); // NOTE: reusing ctxs is a bad idea; will change Parse_tid and cause strange errors; however, keeping for PERF reasons
Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template();
rv = wiki.Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), ctx.Tkn_mkr(), ns_tmpl, name_ary, tmpl_page_bry); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17
Xoa_ttl tmpl_page_ttl = tmpl_page_itm.Ttl();
byte[] frame_ttl = Bry_.Add(tmpl_page_ttl.Ns().Name_db(), Byte_ascii.Colon_bry, tmpl_page_ttl.Page_txt()); // NOTE: (1) must have ns (Full); (2) must be txt (space, not underscore); EX:Template:Location map+; DATE:2014-08-22; (3) must be local language; Russian "Шаблон" not English "Template"; PAGE:ru.w:Королевство_Нидерландов DATE:2016-11-23
rv.Frame_ttl_(frame_ttl); // set defn's frame_ttl; needed for redirect_trg; PAGE:en.w:Statutory_city; DATE:2014-08-22
ctx.Parse_tid_(old_parse_tid);
wiki.Cache_mgr().Defn_cache().Add(rv, ns_tmpl.Case_match());
}
return rv;
}
public static void Print_not_found__by_transclude(Bry_bfr bfr, Xow_ns_mgr ns_mgr, boolean name_has_template, byte[] name_ary) {// print missing as [[Missing]]; PAGE:en.d:a DATE:2016-06-24
bfr.Add(Xop_tkn_.Lnki_bgn);
if (name_has_template)
bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon);
bfr.Add(name_ary);
bfr.Add(Xop_tkn_.Lnki_end);
}
public static void Print_not_found__w_template(Bry_bfr bfr, Xow_ns_mgr ns_mgr, byte[] name_ary) { // print missing as [[:Template:Missing]]; REF:MW: Parser.php|braceSubstitution|$text = "[[:$titleText]]"; EX:en.d:Kazakhstan; DATE:2014-03-25
bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon);
bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon);
bfr.Add(name_ary).Add(Xop_tkn_.Lnki_end);
}
}
package gplx.xowa.parsers.tmpls;
import gplx.Bool_;
import gplx.Bry_;
import gplx.Bry_bfr;
import gplx.Bry_bfr_;
import gplx.Byte_ascii;
import gplx.xowa.Xoa_ttl;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.langs.kwds.Xol_kwd_grp_;
import gplx.xowa.parsers.Xop_ctx;
import gplx.xowa.wikis.caches.Xow_page_cache_itm;
import gplx.xowa.wikis.nss.Xow_ns;
import gplx.xowa.wikis.nss.Xow_ns_;
import gplx.xowa.wikis.nss.Xow_ns_mgr;
import gplx.xowa.xtns.pfuncs.Pf_func_base;
public class Xot_invk_tkn_ {
public static void Eval_func(Xop_ctx ctx, byte[] src, Xot_invk caller, Xot_invk invk, Bry_bfr bfr, Xot_defn defn, byte[] argx_ary) {
Pf_func_base defn_func = (Pf_func_base)defn;
int defn_func_id = defn_func.Id();
defn_func = (Pf_func_base)defn_func.New(defn_func_id, defn_func.Name()); // NOTE: always make copy b/c argx_ary may be dynamic
if (argx_ary != Bry_.Empty) defn_func.Argx_dat_(argx_ary);
defn_func.Eval_argx(ctx, src, caller, invk);
if (defn_func_id == Xol_kwd_grp_.Id_invoke) // NOTE: if #invoke, set frame_ttl to argx, not name; EX:{{#invoke:A}}
invk.Frame_ttl_(Bry_.Add(Xow_ns_.Bry__module_w_colon, Xoa_ttl.Replace_unders(defn_func.Argx_dat()))); // NOTE: always prepend "Module:" to frame_ttl; DATE:2014-06-13; NOTE: always use spaces; DATE:2014-08-14; always use canonical English "Module"; DATE:2015-11-09
Bry_bfr bfr_func = Bry_bfr_.New();
defn_func.Func_evaluate(bfr_func, ctx, caller, invk, src);
if (caller.Rslt_is_redirect()) // do not prepend if page is redirect; EX:"#REDIRECT" x> "\n#REDIRECT" DATE:2014-07-11
caller.Rslt_is_redirect_(false); // reset flag; needed for TEST; kludgy, but Rslt_is_redirect is intended for single use
else
ctx.Page().Tmpl_prepend_mgr().End(ctx, bfr, bfr_func.Bfr(), bfr_func.Len(), Bool_.N);
bfr.Add_bfr_and_clear(bfr_func);
}
public static Xot_defn_tmpl Load_defn(Xowe_wiki wiki, Xop_ctx ctx, Xot_invk_tkn invk_tkn, Xoa_ttl ttl, byte[] name_ary) { // recursive loading of templates
Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
byte[] tmpl_page_bry = tmpl_page_itm == null ? null : tmpl_page_itm.Wtxt__direct();
Xot_defn_tmpl rv = null;
if (tmpl_page_bry != null) {
byte old_parse_tid = ctx.Parse_tid(); // NOTE: reusing ctxs is a bad idea; will change Parse_tid and cause strange errors; however, keeping for PERF reasons
Xow_ns ns_tmpl = wiki.Ns_mgr().Ns_template();
rv = wiki.Parser_mgr().Main().Parse_text_to_defn_obj(Xop_ctx.New__sub(wiki, ctx, ctx.Page()), ctx.Tkn_mkr(), ns_tmpl, name_ary, tmpl_page_bry); // create new ctx so __NOTOC__ only applies to template, not page; PAGE:de.w:13._Jahrhundert DATE:2017-06-17
Xoa_ttl tmpl_page_ttl = tmpl_page_itm.Ttl();
byte[] frame_ttl = Bry_.Add(tmpl_page_ttl.Ns().Name_db(), Byte_ascii.Colon_bry, tmpl_page_ttl.Page_txt()); // NOTE: (1) must have ns (Full); (2) must be txt (space, not underscore); EX:Template:Location map+; DATE:2014-08-22; (3) must be local language; Russian "Шаблон" not English "Template"; PAGE:ru.w:Королевство_Нидерландов DATE:2016-11-23
rv.Frame_ttl_(frame_ttl); // set defn's frame_ttl; needed for redirect_trg; PAGE:en.w:Statutory_city; DATE:2014-08-22
ctx.Parse_tid_(old_parse_tid);
wiki.Cache_mgr().Defn_cache().Add(rv, ns_tmpl.Case_match());
}
return rv;
}
public static void Print_not_found__by_transclude(Bry_bfr bfr, Xow_ns_mgr ns_mgr, boolean name_has_template, byte[] name_ary) {// print missing as [[Missing]]; PAGE:en.d:a DATE:2016-06-24
bfr.Add(Xop_tkn_.Lnki_bgn);
if (name_has_template)
bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon);
bfr.Add(name_ary);
bfr.Add(Xop_tkn_.Lnki_end);
}
public static void Print_not_found__w_template(Bry_bfr bfr, Xow_ns_mgr ns_mgr, byte[] name_ary) { // print missing as [[:Template:Missing]]; REF:MW: Parser.php|braceSubstitution|$text = "[[:$titleText]]"; EX:en.d:Kazakhstan; DATE:2014-03-25
bfr.Add(Xop_tkn_.Lnki_bgn).Add_byte(Byte_ascii.Colon);
bfr.Add(ns_mgr.Ns_template().Name_db()).Add_byte(Byte_ascii.Colon);
bfr.Add(name_ary).Add(Xop_tkn_.Lnki_end);
}
}

@ -1,6 +1,6 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
@ -13,118 +13,142 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.wikis; import gplx.*; import gplx.xowa.*;
import gplx.xowa.langs.*; import gplx.xowa.xtns.wbases.*;
import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.domains.crts.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.metas.*; import gplx.xowa.langs.names.*;
import gplx.xowa.addons.wikis.directorys.dbs.*;
public class Xoae_wiki_mgr implements Xoa_wiki_mgr, Gfo_invk {
private final Xoae_app app;
private final List_adp list = List_adp_.New(); private final Hash_adp_bry hash = Hash_adp_bry.ci_a7(); // ASCII:url_domain; EX:en.wikipedia.org
private Xowdir_db_mgr db_mgr;
public Xoae_wiki_mgr(Xoae_app app) {
this.app = app;
this.wiki_regy = new Xoa_wiki_regy(app);
this.wdata_mgr = new Wdata_wiki_mgr(app);
}
public Xoa_wiki_regy Wiki_regy() {return wiki_regy;} private final Xoa_wiki_regy wiki_regy;
public Xow_script_mgr Scripts() {return scripts;} private final Xow_script_mgr scripts = new Xow_script_mgr();
public Wdata_wiki_mgr Wdata_mgr() {return wdata_mgr;} private final Wdata_wiki_mgr wdata_mgr;
public Xowe_wiki Wiki_commons() {
synchronized (this) { // LOCK:app-level; DATE:2016-07-06
Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(Xow_domain_itm_.Bry__commons);
if (rv != null) rv.Init_assert();
return rv;
}
}
public void Init_by_app() {
this.db_mgr = new Xowdir_db_mgr(app.User().User_db_mgr().Conn());
wdata_mgr.Init_by_app();
}
public int Count() {return list.Count();}
public boolean Has(byte[] key) {return hash.Has(key);}
public Xow_wiki Get_at(int idx) {return (Xow_wiki)list.Get_at(idx);}
public Xowe_wiki Get_at_or_null(int i) {return Int_.Between(i, 0, this.Count() - 1) ? (Xowe_wiki)list.Get_at(i) : null;}
public Xow_wiki Get_by_or_null(byte[] key) {return Bry_.Len_eq_0(key) ? null : (Xowe_wiki)hash.Get_by(key);}
public Xow_wiki Get_by_or_make_init_y(byte[] key) {
synchronized (this) { // LOCK:app-level; DATE:2016-07-06
Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key);
rv.Init_assert();
return rv;
}
}
public Xow_wiki Get_by_or_make_init_n(byte[] key) {return Get_by_or_make(key);}
public Xowe_wiki Get_by_or_make(byte[] key) {
Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key);
return rv;
}
public void Add(Xow_wiki wiki) {
if (hash.Get_by_bry(wiki.Domain_bry()) != null) return; // if already there, don't add again; basically, Add_if_dupe_use_1st
hash.Add(wiki.Domain_bry(), wiki);
list.Add(wiki);
}
public Xowe_wiki Make_and_add(byte[] domain_bry) {
// get wiki_root_url from either user_wiki or /xowa/wiki/
Xowdir_wiki_itm user_wiki_itm = db_mgr == null
? null // TEST:
: db_mgr.Tbl__wiki().Select_by_key_or_null(String_.new_u8(domain_bry));
Xowe_wiki rv = null;
if (user_wiki_itm == null) {
rv = (Xowe_wiki)Make(domain_bry, app.Fsys_mgr().Wiki_dir().GenSubDir(String_.new_a7(domain_bry)));
Add(rv);
}
else {
rv = gplx.xowa.addons.wikis.directorys.specials.items.bldrs.Xow_wiki_factory.Load_personal(app, domain_bry, user_wiki_itm.Url().OwnerDir());
}
return rv;
}
public Xow_wiki Make(byte[] domain_bry, Io_url wiki_root_dir) {
// init domain
Xow_domain_itm domain_itm = Xow_domain_itm_.parse(domain_bry);
// get lang from domain; if not wmf, default to en
byte[] lang_key = domain_itm.Lang_actl_key();
if (lang_key == Xol_lang_stub_.Key__unknown) lang_key = Xol_lang_itm_.Key_en; // unknown langs default to english; note that this makes nonwmf english by default
Xol_lang_itm lang = app.Lang_mgr().Get_by_or_new(lang_key);
if (domain_itm.Domain_type_id() == Xow_domain_tid_.Tid__other) {
lang = Xol_lang_itm.New(app.Lang_mgr(), Xol_lang_itm_.Key_en).Kwd_mgr__strx_(true); // create a new english lang, but enable strx functions; DATE:2015-08-23
Xol_lang_itm_.Lang_init(lang);
}
// load ns from site_meta
Xow_ns_mgr ns_mgr = app.Dbmeta_mgr().Ns__get_or_load(domain_bry);
if (ns_mgr.Ids_len() == 0) ns_mgr = Xow_ns_mgr_.default_(lang.Case_mgr()); // non-wmf wikis will use default ns_mgr
return new Xowe_wiki(app, lang, ns_mgr, domain_itm, wiki_root_dir);
}
public Xow_wiki Import_by_url(Io_url url) {return Xoa_wiki_mgr_.Import_by_url(app, this, url);}
public void Del(byte[] key) {hash.Del(key);}
public void Clear() {hash.Clear(); list.Clear();}
public void Free_mem(boolean clear_ctx) {
int list_len = list.Count();
for (int i = 0; i < list_len; i++) {
Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i);
// wiki.Defn_cache().ReduceCache();
if (clear_ctx) wiki.Parser_mgr().Ctx().Clear_all(); // NOTE: clear_ctx will reset toc and refs
wiki.Cache_mgr().Page_cache().Free_mem(true);
wiki.Cache_mgr().Tmpl_result_cache().Clear();
}
}
public void Rls() {
int len = list.Count();
for (int i = 0; i < len; i++) {
Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i);
wiki.Rls();
}
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_get)) return Get_by_or_make(m.ReadBry("v"));
else if (ctx.Match(k, Invk_scripts)) return scripts;
else if (ctx.Match(k, Invk_wdata)) return wdata_mgr;
else if (ctx.Match(k, Invk_len)) return this.Count();
else if (ctx.Match(k, Invk_get_at)) return this.Get_at_or_null(m.ReadInt("v"));
else if (ctx.Match(k, Xoa_wiki_mgr_.Invk__import_by_url)) return this.Import_by_url(m.ReadIoUrl("v"));
else return Gfo_invk_.Rv_unhandled;
} private static final String Invk_get = "get", Invk_scripts = "scripts", Invk_wdata = "wdata";
private static final String Invk_len = "len", Invk_get_at = "get_at";
}
package gplx.xowa.wikis;
import gplx.Bry_;
import gplx.GfoMsg;
import gplx.Gfo_invk;
import gplx.Gfo_invk_;
import gplx.GfsCtx;
import gplx.Hash_adp_bry;
import gplx.Int_;
import gplx.Io_url;
import gplx.List_adp;
import gplx.List_adp_;
import gplx.String_;
import gplx.xowa.Xoae_app;
import gplx.xowa.Xow_wiki;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.addons.wikis.directorys.dbs.Xowdir_db_mgr;
import gplx.xowa.addons.wikis.directorys.dbs.Xowdir_wiki_itm;
import gplx.xowa.langs.Xol_lang_itm;
import gplx.xowa.langs.Xol_lang_itm_;
import gplx.xowa.langs.Xol_lang_stub_;
import gplx.xowa.wikis.domains.Xow_domain_itm;
import gplx.xowa.wikis.domains.Xow_domain_itm_;
import gplx.xowa.wikis.domains.Xow_domain_tid_;
import gplx.xowa.wikis.metas.Xow_script_mgr;
import gplx.xowa.wikis.nss.Xow_ns_mgr;
import gplx.xowa.wikis.nss.Xow_ns_mgr_;
import gplx.xowa.xtns.wbases.Wdata_wiki_mgr;
public class Xoae_wiki_mgr implements Xoa_wiki_mgr, Gfo_invk {
private final Xoae_app app;
private final List_adp list = List_adp_.New(); private final Hash_adp_bry hash = Hash_adp_bry.ci_a7(); // ASCII:url_domain; EX:en.wikipedia.org
private Xowdir_db_mgr db_mgr;
public Xoae_wiki_mgr(Xoae_app app) {
this.app = app;
this.wiki_regy = new Xoa_wiki_regy(app);
this.wdata_mgr = new Wdata_wiki_mgr(app);
}
public Xoa_wiki_regy Wiki_regy() {return wiki_regy;} private final Xoa_wiki_regy wiki_regy;
public Xow_script_mgr Scripts() {return scripts;} private final Xow_script_mgr scripts = new Xow_script_mgr();
public Wdata_wiki_mgr Wdata_mgr() {return wdata_mgr;} private final Wdata_wiki_mgr wdata_mgr;
public Xowe_wiki Wiki_commons() {
synchronized (this) { // LOCK:app-level; DATE:2016-07-06
Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(Xow_domain_itm_.Bry__commons);
if (rv != null) rv.Init_assert();
return rv;
}
}
public void Init_by_app() {
this.db_mgr = new Xowdir_db_mgr(app.User().User_db_mgr().Conn());
wdata_mgr.Init_by_app();
}
public int Count() {return list.Count();}
public boolean Has(byte[] key) {return hash.Has(key);}
public Xow_wiki Get_at(int idx) {return (Xow_wiki)list.Get_at(idx);}
public Xowe_wiki Get_at_or_null(int i) {return Int_.Between(i, 0, this.Count() - 1) ? (Xowe_wiki)list.Get_at(i) : null;}
public Xow_wiki Get_by_or_null(byte[] key) {return Bry_.Len_eq_0(key) ? null : (Xowe_wiki)hash.Get_by(key);}
public Xow_wiki Get_by_or_make_init_y(byte[] key) {
synchronized (this) { // LOCK:app-level; DATE:2016-07-06
Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key);
rv.Init_assert();
return rv;
}
}
public Xow_wiki Get_by_or_make_init_n(byte[] key) {return Get_by_or_make(key);}
public Xowe_wiki Get_by_or_make(byte[] key) {
Xowe_wiki rv = (Xowe_wiki)this.Get_by_or_null(key); if (rv == null) rv = Make_and_add(key);
return rv;
}
public void Add(Xow_wiki wiki) {
if (hash.Get_by_bry(wiki.Domain_bry()) != null) return; // if already there, don't add again; basically, Add_if_dupe_use_1st
hash.Add(wiki.Domain_bry(), wiki);
list.Add(wiki);
}
public Xowe_wiki Make_and_add(byte[] domain_bry) {
// get wiki_root_url from either user_wiki or /xowa/wiki/
Xowdir_wiki_itm user_wiki_itm = db_mgr == null
? null // TEST:
: db_mgr.Tbl__wiki().Select_by_key_or_null(String_.new_u8(domain_bry));
Xowe_wiki rv = null;
if (user_wiki_itm == null) {
rv = (Xowe_wiki)Make(domain_bry, app.Fsys_mgr().Wiki_dir().GenSubDir(String_.new_a7(domain_bry)));
Add(rv);
}
else {
rv = gplx.xowa.addons.wikis.directorys.specials.items.bldrs.Xow_wiki_factory.Load_personal(app, domain_bry, user_wiki_itm.Url().OwnerDir());
}
return rv;
}
public Xow_wiki Make(byte[] domain_bry, Io_url wiki_root_dir) {
// init domain
Xow_domain_itm domain_itm = Xow_domain_itm_.parse(domain_bry);
// get lang from domain; if not wmf, default to en
byte[] lang_key = domain_itm.Lang_actl_key();
if (lang_key == Xol_lang_stub_.Key__unknown) lang_key = Xol_lang_itm_.Key_en; // unknown langs default to english; note that this makes nonwmf english by default
Xol_lang_itm lang = app.Lang_mgr().Get_by_or_new(lang_key);
if (domain_itm.Domain_type_id() == Xow_domain_tid_.Tid__other) {
lang = Xol_lang_itm.New(app.Lang_mgr(), Xol_lang_itm_.Key_en).Kwd_mgr__strx_(true); // create a new english lang, but enable strx functions; DATE:2015-08-23
Xol_lang_itm_.Lang_init(lang);
}
// load ns from site_meta
Xow_ns_mgr ns_mgr = app.Dbmeta_mgr().Ns__get_or_load(domain_bry);
if (ns_mgr.Ids_len() == 0) ns_mgr = Xow_ns_mgr_.default_(lang.Case_mgr()); // non-wmf wikis will use default ns_mgr
return new Xowe_wiki(app, lang, ns_mgr, domain_itm, wiki_root_dir);
}
public Xow_wiki Import_by_url(Io_url url) {return Xoa_wiki_mgr_.Import_by_url(app, this, url);}
public void Del(byte[] key) {hash.Del(key);}
public void Clear() {hash.Clear(); list.Clear();}
public void Free_mem(boolean clear_ctx) {
int list_len = list.Count();
for (int i = 0; i < list_len; i++) {
Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i);
// wiki.Defn_cache().ReduceCache();
if (clear_ctx) wiki.Parser_mgr().Ctx().Clear_all(); // NOTE: clear_ctx will reset toc and refs
wiki.Cache_mgr().Page_cache().Free_mem(true);
}
}
public void Rls() {
int len = list.Count();
for (int i = 0; i < len; i++) {
Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i);
wiki.Rls();
}
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_get)) return Get_by_or_make(m.ReadBry("v"));
else if (ctx.Match(k, Invk_scripts)) return scripts;
else if (ctx.Match(k, Invk_wdata)) return wdata_mgr;
else if (ctx.Match(k, Invk_len)) return this.Count();
else if (ctx.Match(k, Invk_get_at)) return this.Get_at_or_null(m.ReadInt("v"));
else if (ctx.Match(k, Xoa_wiki_mgr_.Invk__import_by_url)) return this.Import_by_url(m.ReadIoUrl("v"));
else return Gfo_invk_.Rv_unhandled;
} private static final String Invk_get = "get", Invk_scripts = "scripts", Invk_wdata = "wdata";
private static final String Invk_len = "len", Invk_get_at = "get_at";
}

@ -1,6 +1,6 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
@ -13,72 +13,83 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
import gplx.core.caches.*;
import gplx.xowa.wikis.xwikis.sitelinks.*;
public class Xow_cache_mgr {
private final Xowe_wiki wiki;
public Xow_cache_mgr(Xowe_wiki wiki) {
this.wiki = wiki;
this.page_cache = new Xow_page_cache(wiki);
this.defn_cache = new Xow_defn_cache(wiki.Lang());
this.ifexist_cache = new Xow_ifexist_cache(wiki, page_cache);
}
public Hash_adp Tmpl_result_cache() {return tmpl_result_cache;} private final Hash_adp tmpl_result_cache = Hash_adp_bry.cs();
public Xow_defn_cache Defn_cache() {return defn_cache;} private final Xow_defn_cache defn_cache;
public Hash_adp_bry Lst_cache() {return lst_cache;} private final Hash_adp_bry lst_cache = Hash_adp_bry.cs();
public Hash_adp Misc_cache() {return misc_cache;} private final Hash_adp misc_cache = Hash_adp_.New();
public Xow_page_cache Page_cache() {return page_cache;} private Xow_page_cache page_cache;
public Gfo_cache_mgr Commons_cache() {return commons_cache;} private Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
public Xow_ifexist_cache Ifexist_cache() {return ifexist_cache;} private Xow_ifexist_cache ifexist_cache;
public Xow_cache_mgr Page_cache_(Xow_page_cache v) {this.page_cache = v; return this;}
public Xow_cache_mgr Commons_cache_(Gfo_cache_mgr v) {this.commons_cache = v; return this;}
public Xow_cache_mgr Ifexist_cache_(Xow_ifexist_cache v) {this.ifexist_cache = v; return this;}
public void Load_wkr_(Xow_page_cache_wkr v) {
page_cache.Load_wkr_(v);
ifexist_cache.Load_wkr_(v);
}
public Keyval[] Scrib_lang_names() {
if (scrib_lang_names == null) {
List_adp list = List_adp_.New();
Xoa_sitelink_itm_mgr itm_mgr = wiki.App().Xwiki_mgr__sitelink_mgr().Itm_mgr();
int len = itm_mgr.Len();
for (int i = 0; i < len; ++i) {
Xoa_sitelink_itm itm = itm_mgr.Get_at(i);
Keyval kv = Keyval_.new_(String_.new_u8(itm.Key()), String_.new_u8(itm.Name()));
list.Add(kv);
}
scrib_lang_names = (Keyval[])list.To_ary(Keyval.class);
}
return scrib_lang_names;
}
public void Free_mem__page() {this.Free_mem(Free_mem__page_tid);}
public void Free_mem__wbase() {this.Free_mem(Free_mem__wbase_tid);}
public void Free_mem__all() {this.Free_mem(Free_mem__all_tid);}
private void Free_mem(int level) {
switch (level) {
case Free_mem__page_tid:
page_cache.Free_mem(false);
break;
case Free_mem__wbase_tid:
page_cache.Free_mem(false);
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
break;
case Free_mem__all_tid:
page_cache.Free_mem(true);
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
commons_cache.Clear();
ifexist_cache.Clear();
break;
}
wiki.Ctg__catpage_mgr().Free_mem_all();
tmpl_result_cache.Clear();
defn_cache.Free_mem_all();
lst_cache.Clear();
misc_cache.Clear();
// scrib_lang_names = null;
}
private static final int Free_mem__page_tid = 0, Free_mem__wbase_tid = 1, Free_mem__all_tid = 2;
private static Keyval[] scrib_lang_names;
}
package gplx.xowa.wikis.caches;
import gplx.Hash_adp;
import gplx.Hash_adp_;
import gplx.Hash_adp_bry;
import gplx.Io_mgr;
import gplx.Keyval;
import gplx.Keyval_;
import gplx.List_adp;
import gplx.List_adp_;
import gplx.String_;
import gplx.core.caches.Gfo_cache_mgr;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.wikis.xwikis.sitelinks.Xoa_sitelink_itm;
import gplx.xowa.wikis.xwikis.sitelinks.Xoa_sitelink_itm_mgr;
public class Xow_cache_mgr {
private final Xowe_wiki wiki;
public Xow_cache_mgr(Xowe_wiki wiki) {
this.wiki = wiki;
this.page_cache = new Xow_page_cache(wiki);
this.defn_cache = new Xow_defn_cache(wiki.Lang());
this.ifexist_cache = new Xow_ifexist_cache(wiki, page_cache);
}
public Xow_defn_cache Defn_cache() {return defn_cache;} private final Xow_defn_cache defn_cache;
public Hash_adp_bry Lst_cache() {return lst_cache;} private final Hash_adp_bry lst_cache = Hash_adp_bry.cs();
public Hash_adp Misc_cache() {return misc_cache;} private final Hash_adp misc_cache = Hash_adp_.New();
public Xow_page_cache Page_cache() {return page_cache;} private Xow_page_cache page_cache;
public Gfo_cache_mgr Commons_cache() {return commons_cache;} private Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
public Xow_ifexist_cache Ifexist_cache() {return ifexist_cache;} private Xow_ifexist_cache ifexist_cache;
public Xow_cache_mgr Page_cache_(Xow_page_cache v) {this.page_cache = v; return this;}
public Xow_cache_mgr Commons_cache_(Gfo_cache_mgr v) {this.commons_cache = v; return this;}
public Xow_cache_mgr Ifexist_cache_(Xow_ifexist_cache v) {this.ifexist_cache = v; return this;}
public void Load_wkr_(Xow_page_cache_wkr v) {
page_cache.Load_wkr_(v);
ifexist_cache.Load_wkr_(v);
}
public Keyval[] Scrib_lang_names() {
if (scrib_lang_names == null) {
List_adp list = List_adp_.New();
Xoa_sitelink_itm_mgr itm_mgr = wiki.App().Xwiki_mgr__sitelink_mgr().Itm_mgr();
int len = itm_mgr.Len();
for (int i = 0; i < len; ++i) {
Xoa_sitelink_itm itm = itm_mgr.Get_at(i);
Keyval kv = Keyval_.new_(String_.new_u8(itm.Key()), String_.new_u8(itm.Name()));
list.Add(kv);
}
scrib_lang_names = (Keyval[])list.To_ary(Keyval.class);
}
return scrib_lang_names;
}
public void Free_mem__page() {this.Free_mem(Free_mem__page_tid);}
public void Free_mem__wbase() {this.Free_mem(Free_mem__wbase_tid);}
public void Free_mem__all() {this.Free_mem(Free_mem__all_tid);}
private void Free_mem(int level) {
switch (level) {
case Free_mem__page_tid:
page_cache.Free_mem(false);
break;
case Free_mem__wbase_tid:
page_cache.Free_mem(false);
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
break;
case Free_mem__all_tid:
page_cache.Free_mem(true);
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
commons_cache.Clear();
ifexist_cache.Clear();
break;
}
wiki.Ctg__catpage_mgr().Free_mem_all();
defn_cache.Free_mem_all();
lst_cache.Clear();
misc_cache.Clear();
// scrib_lang_names = null;
}
private static final int Free_mem__page_tid = 0, Free_mem__wbase_tid = 1, Free_mem__all_tid = 2;
private static Keyval[] scrib_lang_names;
}

@ -1,135 +1,132 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
package gplx.xowa.xtns.scribunto.libs;
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
import gplx.Err_;
import gplx.Keyval_;
import gplx.Object_;
import gplx.String_;
import gplx.Tfds;
import gplx.xowa.xtns.scribunto.Scrib_core;
import gplx.xowa.xtns.scribunto.Scrib_invoke_func;
import gplx.xowa.xtns.scribunto.Scrib_invoke_func_fxt;
import gplx.xowa.xtns.scribunto.Scrib_kv_utl_;
import gplx.xowa.xtns.scribunto.Scrib_lib;
import org.junit.Before;
import org.junit.Test;
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*;
import org.junit.*;
public class Scrib_lib_mw__invoke_tst {
@Before public void init() {
fxt.Clear_for_invoke();
lib = fxt.Core().Lib_mw().Init();
} private Scrib_invoke_func_fxt fxt = new Scrib_invoke_func_fxt(); private Scrib_lib lib;
@Test public void GetAllExpandedArguments_ws_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}")
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key={{{1}}}}}");
fxt.Init_page("{{test| a }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a"); // " a " -> "a"
}
@Test public void GetAllExpandedArguments_ws_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}")
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}");
fxt.Init_page("{{test| a }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a "); // " a " -> " a "
}
@Test public void GetAllExpandedArguments__ignore_empty_key() {// PURPOSE: ignore arguents that have an empty key (|=8|); EX:w:Fool's_mate; DATE:2014-03-05
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0}}");
fxt.Init_page("{{test|a1||a2|=a3|a4}}");
fxt.Init_server_print_key_y_();
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("parent"), "\n 1:a1;2:;3:a2;4:a4"); // NOTE: || is not ignored but |=a3| is
fxt.Init_server_print_key_n_();
}
@Test public void GetExpandedArgument_ws_key_exists() { // PURPOSE: trim val if key exists; literal value
fxt.Init_page("{{#invoke:Mod_0|Prc_0| key1 = val1 }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "val1"); // "key1" -> "key1"
}
@Test public void GetExpandedArgument_ws_key_missing() { // PURPOSE: do not trim val if key missing; literal value
fxt.Init_page("{{#invoke:Mod_0|Prc_0| a }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a "
}
@Test public void GetExpandedArgument_ws_key_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}")
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key1={{{1}}}}}");
fxt.Init_page("{{test| a }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "a"); // " a " -> "a"
}
@Test public void GetExpandedArgument_ws_key_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}")
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}");
fxt.Init_page("{{test| a }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a "
}
@Test public void Preprocess() {
this.Init_preprocess();
this.Exec_preprocess(Scrib_core.Frame_key_module , "1", "c");
this.Exec_preprocess(Scrib_core.Frame_key_module , "2", "d");
this.Exec_preprocess(Scrib_core.Frame_key_template , "1", "a");
this.Exec_preprocess(Scrib_core.Frame_key_template , "2", "b");
}
@Test public void Preprocess_duplicate_key() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0|key1=a|key2=b|key1=c}}"); // add key1 twice
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_preprocess, Object_.Ary("current", "{{#ifeq:1|1|{{{key1}}}|{{{key2}}}}}"), "c");
}
@Test public void CallParserFunction() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#expr", "1") , "1"); // named: args is scalar
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // named: args is table
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", "", "y", "n") , "n"); // list: args is ary
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // list: args is table
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if:", "y", "n") , "n"); // colon_in_name
}
@Test public void CallParserFunction_tag() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Scrib_kv_utl_.flat_many_("3", "id=1", "2", "text", "1", "pre")), "<pre 3=\"id=1\">2=text</pre>");// named: sort args; NOTE: keys should probably be stripped; // ISSUE#:462; DATE:2019-05-12
}
@Test public void CallParserFunction__no_args() { // PURPOSE.fix: 0 args should not fail
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Keyval_.Ary_empty), "");// failed with "Script error: index is out of bounds"
}
@Test public void CallParserFunction_displayTitle() { // PURPOSE: DISPLAYTITLE not being set when called through CallParserFunction; DATE:2013-08-05
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "DISPLAYTITLE", "''a''"), "");
Tfds.Eq("<i>a</i>", String_.new_a7(fxt.Parser_fxt().Ctx().Page().Html_data().Display_ttl()));
}
@Test public void CallParserFunction__null() { // PURPOSE.fix: null arg should not fail; PAGE:en.w:Abziri DATE:2017-11-29
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#coordinates", 3, Keyval_.Ary(Keyval_.int_(1, "a"), Keyval_.int_(3, "b"), null)), "");// failed with NullPointerException
}
@Test public void ExpandTemplate_tmpl() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Data_create("Template:A", "b{{{key1}}}c");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary
}
@Test public void ExpandTemplate() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Format", Scrib_kv_utl_.base1_many_("a")), "a");
}
@Test public void ExpandTemplate_ns_specified() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Template:Format", Scrib_kv_utl_.base1_many_("a")), "a"); // NOTE: "Template:" specified
}
@Test public void ExpandTemplate_tmpl_bool() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Data_create("Template:Scribunto_bool", "bool_true={{{bool_true}}};bool_false={{{bool_false}}};");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Scribunto_bool", Scrib_kv_utl_.flat_many_("bool_true", true, "bool_false", false)), "bool_true=1;bool_false={{{bool_false}}};");
}
@Test public void ExpandTemplate_page() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Data_create("A", "b{{{key1}}}c");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", ":A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary
}
@Test public void Err_mod_blank() {fxt.Test_parse_err("{{#invoke:}}" , Scrib_invoke_func.Err_mod_missing);}
@Test public void Err_mod_missing() {fxt.Test_parse_err("{{#invoke:Missing}}" , Scrib_invoke_func.Err_mod_missing);}
@Test public void Err_mod_custom() {
fxt.Test_error(Err_.new_("err_type", "fail", "key0", "val0"), "<strong class=\"error\"><span class=\"scribunto-error\" id=\"mw-scribunto-error-0\">Script error: fail</span></strong>");
}
private void Init_preprocess() {
fxt.Init_tmpl("{{#invoke:Mod_0|Func_0|1|c|d}}"); // current
fxt.Init_page("{{test|1|a|b|c}}"); // parent
fxt.Init_cbk(fxt.Core().Lib_mw(), Scrib_lib_mw.Invk_preprocess);
}
private void Exec_preprocess(String frame, String arg_idx, String expd) {
fxt.Parser_fxt().Wiki().Cache_mgr().Tmpl_result_cache().Clear();
fxt.Init_lua_module();
fxt.Init_lua_rcvd_preprocess(frame, "{{#ifeq:" + arg_idx + "|{{{1}}}|{{{2}}}|{{{3}}}}}");
fxt.Test_invoke(expd);
}
}
public class Scrib_lib_mw__invoke_tst {
@Before public void init() {
fxt.Clear_for_invoke();
lib = fxt.Core().Lib_mw().Init();
} private Scrib_invoke_func_fxt fxt = new Scrib_invoke_func_fxt(); private Scrib_lib lib;
@Test public void GetAllExpandedArguments_ws_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}")
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key={{{1}}}}}");
fxt.Init_page("{{test| a }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a"); // " a " -> "a"
}
@Test public void GetAllExpandedArguments_ws_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}")
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}");
fxt.Init_page("{{test| a }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("current"), "\n a "); // " a " -> " a "
}
@Test public void GetAllExpandedArguments__ignore_empty_key() {// PURPOSE: ignore arguents that have an empty key (|=8|); EX:w:Fool's_mate; DATE:2014-03-05
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0}}");
fxt.Init_page("{{test|a1||a2|=a3|a4}}");
fxt.Init_server_print_key_y_();
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getAllExpandedArguments, Object_.Ary("parent"), "\n 1:a1;2:;3:a2;4:a4"); // NOTE: || is not ignored but |=a3| is
fxt.Init_server_print_key_n_();
}
@Test public void GetExpandedArgument_ws_key_exists() { // PURPOSE: trim val if key exists; literal value
fxt.Init_page("{{#invoke:Mod_0|Prc_0| key1 = val1 }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "val1"); // "key1" -> "key1"
}
@Test public void GetExpandedArgument_ws_key_missing() { // PURPOSE: do not trim val if key missing; literal value
fxt.Init_page("{{#invoke:Mod_0|Prc_0| a }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a "
}
@Test public void GetExpandedArgument_ws_key_prm_key_exists() { // PURPOSE: trim val if key exists; parameterized value ("key={{{1}}}")
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|key1={{{1}}}}}");
fxt.Init_page("{{test| a }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "key1") , "a"); // " a " -> "a"
}
@Test public void GetExpandedArgument_ws_key_prm_key_missing() { // PURPOSE: do not trim val if key missing; parameterized value ("{{{1}}}")
fxt.Init_tmpl("{{#invoke:Mod_0|Prc_0|{{{1}}}}}");
fxt.Init_page("{{test| a }}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_getExpandedArgument, Object_.Ary("current", "1") , " a "); // " a " -> " a "
}
@Test public void Preprocess() {
this.Init_preprocess();
this.Exec_preprocess(Scrib_core.Frame_key_module , "1", "c");
this.Exec_preprocess(Scrib_core.Frame_key_module , "2", "d");
this.Exec_preprocess(Scrib_core.Frame_key_template , "1", "a");
this.Exec_preprocess(Scrib_core.Frame_key_template , "2", "b");
}
@Test public void Preprocess_duplicate_key() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0|key1=a|key2=b|key1=c}}"); // add key1 twice
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_preprocess, Object_.Ary("current", "{{#ifeq:1|1|{{{key1}}}|{{{key2}}}}}"), "c");
}
@Test public void CallParserFunction() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#expr", "1") , "1"); // named: args is scalar
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // named: args is table
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", "", "y", "n") , "n"); // list: args is ary
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if", Scrib_kv_utl_.base1_many_("", "y", "n")) , "n"); // list: args is table
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_callParserFunction, Object_.Ary("current", "#if:", "y", "n") , "n"); // colon_in_name
}
@Test public void CallParserFunction_tag() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Scrib_kv_utl_.flat_many_("3", "id=1", "2", "text", "1", "pre")), "<pre 3=\"id=1\">2=text</pre>");// named: sort args; NOTE: keys should probably be stripped; // ISSUE#:462; DATE:2019-05-12
}
@Test public void CallParserFunction__no_args() { // PURPOSE.fix: 0 args should not fail
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#tag", 3, Keyval_.Ary_empty), "");// failed with "Script error: index is out of bounds"
}
@Test public void CallParserFunction_displayTitle() { // PURPOSE: DISPLAYTITLE not being set when called through CallParserFunction; DATE:2013-08-05
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.base1_many_ary_("current", "DISPLAYTITLE", "''a''"), "");
Tfds.Eq("<i>a</i>", String_.new_a7(fxt.Parser_fxt().Ctx().Page().Html_data().Display_ttl()));
}
@Test public void CallParserFunction__null() { // PURPOSE.fix: null arg should not fail; PAGE:en.w:Abziri DATE:2017-11-29
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Test_lib_proc_kv(lib, Scrib_lib_mw.Invk_callParserFunction, Scrib_kv_utl_.flat_many_(1, "current", 2, "#coordinates", 3, Keyval_.Ary(Keyval_.int_(1, "a"), Keyval_.int_(3, "b"), null)), "");// failed with NullPointerException
}
@Test public void ExpandTemplate_tmpl() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Data_create("Template:A", "b{{{key1}}}c");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary
}
@Test public void ExpandTemplate() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Format", Scrib_kv_utl_.base1_many_("a")), "a");
}
@Test public void ExpandTemplate_ns_specified() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Init_page_create("Template:Format", "{{{1}}}");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Template:Format", Scrib_kv_utl_.base1_many_("a")), "a"); // NOTE: "Template:" specified
}
@Test public void ExpandTemplate_tmpl_bool() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Data_create("Template:Scribunto_bool", "bool_true={{{bool_true}}};bool_false={{{bool_false}}};");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", "Scribunto_bool", Scrib_kv_utl_.flat_many_("bool_true", true, "bool_false", false)), "bool_true=1;bool_false={{{bool_false}}};");
}
@Test public void ExpandTemplate_page() {
fxt.Init_page("{{#invoke:Mod_0|Prc_0}}");
fxt.Parser_fxt().Data_create("A", "b{{{key1}}}c");
fxt.Test_lib_proc(lib, Scrib_lib_mw.Invk_expandTemplate, Object_.Ary("current", ":A", Scrib_kv_utl_.flat_many_("key1", "val1")) , "bval1c"); // list: args is ary
}
@Test public void Err_mod_blank() {fxt.Test_parse_err("{{#invoke:}}" , Scrib_invoke_func.Err_mod_missing);}
@Test public void Err_mod_missing() {fxt.Test_parse_err("{{#invoke:Missing}}" , Scrib_invoke_func.Err_mod_missing);}
@Test public void Err_mod_custom() {
fxt.Test_error(Err_.new_("err_type", "fail", "key0", "val0"), "<strong class=\"error\"><span class=\"scribunto-error\" id=\"mw-scribunto-error-0\">Script error: fail</span></strong>");
}
private void Init_preprocess() {
fxt.Init_tmpl("{{#invoke:Mod_0|Func_0|1|c|d}}"); // current
fxt.Init_page("{{test|1|a|b|c}}"); // parent
fxt.Init_cbk(fxt.Core().Lib_mw(), Scrib_lib_mw.Invk_preprocess);
}
private void Exec_preprocess(String frame, String arg_idx, String expd) {
fxt.Init_lua_module();
fxt.Init_lua_rcvd_preprocess(frame, "{{#ifeq:" + arg_idx + "|{{{1}}}|{{{2}}}|{{{3}}}}}");
fxt.Test_invoke(expd);
}
}

Loading…
Cancel
Save