Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Mass_parse: Change page_cache to LRU cache [#483]

Browse Source
This commit is contained in:
gnosygnu
2019-06-21 23:02:49 -04:00
parent 0cfb0b19ad
commit a01e7409eb
21 changed files with 367 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java
View File

@@ -31,13 +31,14 @@ public class Xomp_parse_mgr {
// init page_pool
Xomp_page_pool_loader page_pool_loader = new Xomp_page_pool_loader(wiki, mgr_db.Conn(), cfg.Num_pages_in_pool(), cfg.Show_msg__fetched_pool());
Xomp_page_pool page_pool = new Xomp_page_pool(page_pool_loader, cfg.Num_pages_per_wkr());
Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval(), cfg.Perf_interval(), mgr_db.Url().GenNewNameAndExt("xomp.perf.csv"));
// cache: preload tmpls and imglinks
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates());
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates(), cfg.Page_cache_max());
wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of user cache
Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
prog_mgr.Init(page_cache, page_pool_loader.Get_pending_count(), cfg.Progress_interval(), cfg.Perf_interval(), mgr_db.Url().GenNewNameAndExt("xomp.perf.csv"));
Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
Xow_ifexist_cache ifexist_cache = new Xow_ifexist_cache(wiki, page_cache).Cache_sizes_(Int_.Max_value, Int_.Max_value);
if (cfg.Load_ifexists_ns() != null) Load_ifexists_ns(wiki, ifexist_cache, cfg.Load_ifexists_ns());
@@ -71,6 +72,7 @@ public class Xomp_parse_mgr {
Xowe_wiki wkr_wiki = Xow_wiki_utl_.Clone_wiki(wiki, wiki.Fsys_mgr().Root_dir());
wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);
// make wkr
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, cfg, mgr_db, page_pool, prog_mgr, file_orig_wkr, ns_ord_mgr, wkr_wiki, indexer, i + wkr_uid_bgn);
wkrs[i] = wkr;

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
View File

@@ -42,6 +42,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
public long Wbase_cache_mru_size() {return wbase_cache_mru_size;} private long wbase_cache_mru_size = 100;
public long Wbase_cache_mru_weight() {return wbase_cache_mru_weight;} private long wbase_cache_mru_weight = 10;
public long Wbase_cache_mru_compress_size() {return wbase_cache_mru_compress_size;} private long wbase_cache_mru_compress_size = 70;
public long Page_cache_max() {return page_cache_max;} private long page_cache_max = 8 * (long)Io_mgr.Len_gb;
public void Init(Xowe_wiki wiki) {
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count();
if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000;
@@ -76,6 +77,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, "wbase_cache_mru_size_")) wbase_cache_mru_size = m.ReadLong("v");
else if (ctx.Match(k, "wbase_cache_mru_weight_")) wbase_cache_mru_weight = m.ReadLong("v");
else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v");
else if (ctx.Match(k, "page_cache_max_")) page_cache_max = m.ReadLong("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}

7
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_prog_mgr.java
View File

@@ -14,14 +14,17 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.xowa.wikis.caches.*;
public class Xomp_prog_mgr {
private final Object thread_lock = new Object();
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
private Xow_page_cache page_cache;
private int prog_interval, perf_interval;
private int pages_done, pages_total;
private long prog_bgn, prog_prv, prog_done, perf_prv;
private Io_url perf_url;
public void Init(int pages_total, int prog_interval, int perf_interval, Io_url perf_url) {
public void Init(Xow_page_cache page_cache, int pages_total, int prog_interval, int perf_interval, Io_url perf_url) {
this.page_cache = page_cache;
this.pages_total = pages_total;
this.prog_interval = prog_interval;
this.perf_interval = perf_interval;
@@ -39,7 +42,7 @@ public class Xomp_prog_mgr {
double rate_cur = pages_done / (prog_done / Time_span_.Ratio_f_to_s);
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)((prog_cur - prog_bgn) / 1000), true, 0);
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)(pages_left / rate_cur), true, 0);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5} cache_stats=~{6}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left, page_cache.To_str());
prog_prv = prog_cur;
}
if (pages_done % perf_interval == 0) {

10
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_tmpl_cache_bldr.java
View File

@@ -17,8 +17,9 @@ package gplx.xowa.addons.bldrs.mass_parses.parses.utls; import gplx.*; import gp
import gplx.dbs.*;
import gplx.xowa.wikis.caches.*;
public class Xomp_tmpl_cache_bldr {
public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all) {
public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all, long page_cache_max) {
Xow_page_cache rv = new Xow_page_cache(wiki);
rv.Max_(page_cache_max);
if (fill_all) Fill_all(rv, wiki);
return rv;
}
@@ -53,12 +54,7 @@ public class Xomp_tmpl_cache_bldr {
text_db_loader.Add(rdr.Read_int("page_text_db_id"), itm);
// ignore duplicate page_titles in cache; EX:ru.n:Модуль:Weather/data DATE:2017-03-16
if (cache.Get_or_null(page_ttl.Full_db()) == null) {
cache.Add(page_ttl.Full_db(), itm);
}
else {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse: ignoring duplicate page title in page cache; title=~{0} id=~{1}", page_ttl.Full_db(), page_id);
}
cache.Add_itm(page_ttl.Full_db_as_str(), itm);
page_regy.Add(page_id, itm);

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java
View File

@@ -124,8 +124,6 @@ public class Xomp_parse_wkr implements Gfo_invk {
Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry());
// if ns changed and prv_ns is main
if (cur_ns != prv_ns) {
if (prv_ns == gplx.xowa.wikis.nss.Xow_ns_.Tid__main)
wiki.Cache_mgr().Free_mem__all(); // NOTE: clears page and wbase cache only; needed else OutOfMemory error for en.w in 25th hour; DATE:2017-01-11
prv_ns = cur_ns;
}

7
400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java
View File

@@ -22,9 +22,6 @@ public class Xop_mediawiki_wkr {
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
public Xop_mediawiki_wkr(Xowe_wiki wiki, Xop_mediawiki_loader loader) {
this.wiki = wiki;
this.Loader_(loader);
}
public void Loader_(Xop_mediawiki_loader loader) {
if (loader != null)
wiki.Cache_mgr().Load_wkr_(new Xow_page_cache_wkr__embeddable(wiki, loader));
}
@@ -34,10 +31,6 @@ public class Xop_mediawiki_wkr {
wiki.Parser_mgr().Scrib().Core_term();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
}
public void Clear_cache(String page) {
Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));
wiki.Cache_mgr().Page_cache().Del(ttl.Full_db());
}
public String Parse(String page, String wikitext) {
Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));

2
400_xowa/src/gplx/xowa/addons/servers/https/utils/Xoa_util_svc.java
View File

@@ -43,7 +43,7 @@ public class Xoa_util_svc {
// get page
Xoa_ttl ttl = wiki.Ttl_parse(page_bry);
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm_2(ttl);
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
byte[] page_text = page_itm == null ? null : page_itm.Wtxt__direct();
if (page_text == null) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "Xoa_utl_svc:page not found: wiki=~{0} page=~{1}", wiki_bry, page_bry);

4
400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn.java
View File

@@ -333,7 +333,7 @@ public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk {
if (tmpl != null) transclude_src = tmpl.Data_raw();
}
if (transclude_src == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(page_ttl);
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
if ( cache_itm != null
// && Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Page_ttl().Full_db()) // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
) {
@@ -377,7 +377,7 @@ public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk {
return true;
}
if (transclude_tmpl == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(page_ttl);
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
if ( cache_itm != null) {
if (!Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Ttl().Full_db())) { // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), cache_itm.Wtxt__direct());

2
400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn_.java
View File

@@ -49,7 +49,7 @@ public class Xot_invk_tkn_ {
}
}
public static Xot_defn_tmpl Load_defn(Xowe_wiki wiki, Xop_ctx ctx, Xot_invk_tkn invk_tkn, Xoa_ttl ttl, byte[] name_ary) { // recursive loading of templates
Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(ttl);
Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
byte[] tmpl_page_bry = tmpl_page_itm == null ? null : tmpl_page_itm.Wtxt__direct();
Xot_defn_tmpl rv = null;
if (tmpl_page_bry != null) {

2
400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java
View File

@@ -53,7 +53,7 @@ public class Xow_ifexist_cache {
if (ns_loaded_hash.Has(ttl.Ns().Id())) return Bool_.N_byte;
// check page_cache since full page + text could be loaded there
Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_or_null(key);
Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_itm_or_null(ttl.Full_db_as_str());
if (itm == Xow_page_cache_itm.Missing)
return Bool_.N_byte;
else if (itm != null)

169
400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java
View File

@@ -14,113 +14,98 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
import gplx.core.lists.caches.*;
public class Xow_page_cache {
private final Object thread_lock = new Object();
private final Object thread_lock = new Object(); // NOTE: thread-safety needed for xomp since one page-cache is shared across all wkrs
private final Xowe_wiki wiki;
private final Ordered_hash cache = Ordered_hash_.New_bry(); // NOTE: wiki titles are not case-sensitive when ns is "1st-letter" (EX: w:earth an w:Earth); in these cases, two entries will be stored
private final List_adp deleted = List_adp_.New();
public Xow_page_cache(Xowe_wiki wiki) {this.wiki = wiki;}
private final Lru_cache cache = new Lru_cache();
private long cache_tries = 0;
private long cache_misses = 0;
public Xow_page_cache(Xowe_wiki wiki) {
this.wiki = wiki;
cache.Max_(16 * Io_mgr.Len_mb);
}
public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;} private Xow_page_cache_wkr load_wkr;
public Xow_page_cache_itm Get_or_null(byte[] ttl_full_db) {return (Xow_page_cache_itm)cache.Get_by(ttl_full_db);}
public byte[] Get_or_load_as_src(Xoa_ttl ttl) {
Xow_page_cache_itm rv = Get_or_load_as_itm(ttl);
return rv == null ? null : rv.Wtxt__direct();
public void Max_(long v) {cache.Max_(v);}
public void Add_itm(String ttl_full_db, Xow_page_cache_itm itm) {
synchronized (thread_lock) {
cache.Set(ttl_full_db, itm, itm.Cache_len());
}
}
public void Add(byte[] ttl_full_db, Xow_page_cache_itm itm) {
cache.Add(ttl_full_db, itm);
public Xow_page_cache_itm Get_itm_or_null(String ttl_full_db) {
synchronized (thread_lock) {
return (Xow_page_cache_itm)cache.Get_or_null(ttl_full_db);
}
}
private void Add_safe(byte[] ttl_full_db, Xow_page_cache_itm itm) {
synchronized (thread_lock) { // LOCK:high-usage;DATE:2016-07-14
if (!cache.Has(ttl_full_db)) { // check again that itm is not in cache; note that this is necessary as cache.Get is not in "synchronized" block (for performance reasons); DATE:2016-12-12
cache.Add(ttl_full_db, itm);
public Xow_page_cache_itm Get_itm_else_load_or_null(Xoa_ttl ttl) {
synchronized (thread_lock) {
cache_tries++;
Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_or_null(ttl.Full_db_as_str());
if (rv == Xow_page_cache_itm.Missing)
return null;
else if (rv == null) {
cache_misses++;
return Load_page(ttl);
}
else
return rv;
}
}
public void Del(byte[] ttl_full_db) {
cache.Del(ttl_full_db);
}
public Xow_page_cache_itm Get_or_load_as_itm(Xoa_ttl ttl) {
byte[] ttl_full_db = ttl.Full_db();
Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by(ttl_full_db);
if (rv == Xow_page_cache_itm.Missing) {
return null;
public byte[] Get_src_else_load_or_null(Xoa_ttl ttl) {
synchronized (thread_lock) {
Xow_page_cache_itm rv = Get_itm_else_load_or_null(ttl);
return rv == null ? null : rv.Wtxt__direct();
}
else if (rv == null) {
return Load_page(ttl, ttl_full_db);
}
return rv;
}
private Xow_page_cache_itm Load_page(Xoa_ttl ttl, byte[] ttl_full_db) {
Xow_page_cache_itm rv = null;
Xoa_ttl page_ttl = ttl;
boolean page_exists = false;
byte[] page_text = null;
byte[] page_redirect_from = null;
int page_id = -1;
// gplx.core.consoles.Console_adp__sys.Instance.Write_str("page_cache:" + String_.new_u8(ttl_full_db));
if (load_wkr != null) {
page_text = load_wkr.Get_page_or_null(ttl_full_db);
page_exists = page_text != null;
}
if (page_text == null) {
Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(ttl); // NOTE: do not call Db_mgr.Load_page; need to handle redirects
page_ttl = page.Ttl();
page_text = page.Db().Text().Text_bry();
page_exists = page.Db().Page().Exists();
page_redirect_from = page.Redirect_trail().Itms__get_wtxt_at_0th_or_null();
page_id = page.Db().Page().Id();
}
if (page_exists) {
rv = new Xow_page_cache_itm(false, page_id, page_ttl, page_text, page_redirect_from);
Add_safe(ttl_full_db, rv);
}
else {
Add_safe(ttl_full_db, Xow_page_cache_itm.Missing);
rv = null;
}
return rv;
}
public Xow_page_cache_itm Get_or_load_as_itm_2(Xoa_ttl ttl) { // NOTE: same as Get_or_load_as_itm, but handles redirects to missing pages; DATE:2016-05-02
byte[] ttl_full_db = ttl.Full_db();
Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by(ttl_full_db);
if (rv == Xow_page_cache_itm.Missing) return null;
else if (rv == null) {
Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(ttl); // NOTE: do not call Db_mgr.Load_page; need to handle redirects
if ( page.Db().Page().Exists() // page exists
|| page.Redirect_trail().Itms__len() > 0 ) { // page redirects to missing page; note that page.Missing == true and page.Redirected_src() != null; PAGE: en.w:Shah_Rukh_Khan; DATE:2016-05-02
rv = new Xow_page_cache_itm(false, page.Db().Page().Id(), page.Ttl(), page.Db().Text().Text_bry(), page.Redirect_trail().Itms__get_wtxt_at_0th_or_null());
Add_safe(ttl_full_db, rv);
}
else {
Add_safe(ttl_full_db, Xow_page_cache_itm.Missing);
rv = null;
}
}
return rv;
}
public void Free_mem(boolean clear_permanent_itms) {
synchronized (thread_lock) { // LOCK:app-level; DATE:2016-07-06
if (clear_permanent_itms) {
cache.Clear();
}
else {
// gather non-permanent items
int len = cache.Count();
for (int i = 0; i < len; i++) {
Xow_page_cache_itm itm = (Xow_page_cache_itm)cache.Get_at(i);
if (!itm.Cache_permanently())
deleted.Add(itm);
}
// remove non-permanent items
len = deleted.Len();
for (int i = 0; i < len; i++) {
Xow_page_cache_itm itm = (Xow_page_cache_itm)deleted.Get_at(i);
if (itm.Ttl() != null) // missing is null
cache.Del(itm.Ttl().Full_db());
}
deleted.Clear();
}
}
}
private Xow_page_cache_itm Load_page(Xoa_ttl ttl) {
// vars
Xow_page_cache_itm rv = null;
Xoa_ttl page_ttl = ttl;
boolean page_exists = false;
byte[] page_text = null;
byte[] page_redirect_from = null;
int page_id = -1;
// load_page if load_wkr exists; only for Xop_mediawiki_mgr
// gplx.core.consoles.Console_adp__sys.Instance.Write_str("page_cache:" + String_.new_u8(ttl_full_db));
if (load_wkr != null) {
page_text = load_wkr.Get_page_or_null(ttl.Full_db());
page_exists = page_text != null;
}
// load_page in other cases
if (page_text == null) {
Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(ttl); // NOTE: do not call Db_mgr.Load_page; need to handle redirects
if ( page.Db().Page().Exists() // page exists
|| page.Redirect_trail().Itms__len() > 0 ) { // page redirects to missing page; note that page.Missing == true and page.Redirected_src() != null; PAGE:en.w:Shah_Rukh_Khan; DATE:2016-05-02
page_exists = true; // NOTE: page.Db().Page().Exists() will be false if page redirects to missing page; PAGE:en.w:Shah_Rukh_Khan; DATE:2019-06-16
page_ttl = page.Ttl();
page_text = page.Db().Text().Text_bry();
page_redirect_from = page.Redirect_trail().Itms__get_wtxt_at_0th_or_null();
page_id = page.Db().Page().Id();
}
}
// create item
if (page_exists) {
rv = new Xow_page_cache_itm(false, page_id, page_ttl, page_text, page_redirect_from);
Add_itm(ttl.Full_db_as_str(), rv);
}
else {
rv = null;
Add_itm(ttl.Full_db_as_str(), Xow_page_cache_itm.Missing);
}
return rv;
}
public String To_str() {
return String_.Format("cache_pct:{0} cache_misses:{1} cache_evicts:{2} cache_tries:{3} cache_size:{4}", Decimal_adp_.divide_(cache_misses * 100, cache_tries).To_str("0.00"), cache_misses, cache.Evicts(), cache_tries, cache.Cur());
}
}

12
400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache_itm.java
View File

@@ -16,9 +16,11 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
import gplx.xowa.wikis.data.tbls.*;
public class Xow_page_cache_itm implements Xowd_text_bry_owner {
private long cache_len;
public Xow_page_cache_itm(boolean cache_permanently, int page_id, Xoa_ttl ttl, byte[] wtxt__direct, byte[] wtxt__redirect) {
this.cache_permanently = cache_permanently;
this.page_id = page_id; this.ttl = ttl; this.wtxt__direct = wtxt__direct; this.wtxt__redirect = wtxt__redirect;
this.page_id = page_id; this.ttl = ttl; this.wtxt__redirect = wtxt__redirect;
Set_text_bry_by_db(wtxt__direct);
}
public Xoa_ttl Ttl() {return ttl;} private Xoa_ttl ttl;
public byte[] Wtxt__direct() {return wtxt__direct;} private byte[] wtxt__direct;
@@ -27,16 +29,20 @@ public class Xow_page_cache_itm implements Xowd_text_bry_owner {
return wtxt__redirect == null ? wtxt__direct : wtxt__redirect;
}
public boolean Cache_permanently() {return cache_permanently;} private final boolean cache_permanently;
public long Cache_len() {return cache_len;}
// used by xomp
public int Page_id() {return page_id;} private int page_id;
public int Redirect_id() {return redirect_id;} private int redirect_id;
public void Set_text_bry_by_db(byte[] v) {this.wtxt__direct = v;}
public void Set_text_bry_by_db(byte[] v) {
this.wtxt__direct = v;
this.cache_len = wtxt__direct == null ? 0 : wtxt__direct.length;
}
public void Redirect_id_(int v) {this.redirect_id = v;}
public void Set_redirect(Xoa_ttl ttl, byte[] trg_wtxt) {
this.ttl = ttl;
Set_text_bry_by_db(trg_wtxt);
this.wtxt__redirect = wtxt__direct;
this.wtxt__direct = trg_wtxt;
}
public static final Xow_page_cache_itm Null = null;

6
400_xowa/src/gplx/xowa/xtns/lst/Lst_pfunc_itm.java
View File

@@ -35,7 +35,8 @@ public class Lst_pfunc_itm {
// get sub_src;
Xop_ctx sub_ctx = Xop_ctx.New__top(wiki).Ref_ignore_(true);
sub_ctx.Page().Ttl_(ctx.Page().Ttl()); // NOTE: must set ttl on page, else test fails;
byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(ttl); if (sub_src == null) return null; // {{#lst:missing}} -> ""
byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl);
if (sub_src == null) return null; // {{#lst:missing}} -> ""
// parse page; note adding to stack to prevent circular recursions
if (!wiki.Parser_mgr().Tmpl_stack_add(ttl.Full_db())) return null;
@@ -75,7 +76,8 @@ public class Lst_pfunc_itm {
// get sub_ctx: note new ctx is needed b/c sub_page objects must not get added to owner_page; for example, references / hdrs / lnki.files
Xop_ctx sub_ctx = Xop_ctx.New__top(wiki).Ref_ignore_(true);
sub_ctx.Page().Ttl_(ctx.Page().Ttl()); // NOTE: must set ttl on page, else test fails;
byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(ttl); if (sub_src == null) return null; // {{#lst:missing}} -> ""
byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl);
if (sub_src == null) return null; // {{#lst:missing}} -> ""
// parse sub_src; note adding to page's stack to prevent circular recursions
if (!wiki.Parser_mgr().Tmpl_stack_add(ttl.Full_db())) return null;

2
400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_index_parser.java
View File

@@ -19,7 +19,7 @@ import gplx.xowa.wikis.nss.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.parsers.xndes.*; import gplx.xowa.parsers.lnkis.*; import gplx.xowa.parsers.tmpls.*;
class Pp_index_parser {
public static Pp_index_page Parse(Xowe_wiki wiki, Xop_ctx ctx, Xoa_ttl index_ttl, int ns_page_id) {
byte[] src = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(index_ttl);
byte[] src = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(index_ttl);
if (src == null) return Pp_index_page.Null;
Xop_parser sub_parser = Xop_parser.new_(wiki, wiki.Parser_mgr().Main().Tmpl_lxr_mgr(), wiki.Parser_mgr().Main().Wtxt_lxr_mgr());
Xop_ctx sub_ctx = Xop_ctx.New__sub__reuse_page(ctx);

2
400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func.java
View File

@@ -51,7 +51,7 @@ public class Scrib_invoke_func extends Pf_func_base {
if (mod == null) {
Xow_ns module_ns = wiki.Ns_mgr().Ids_get_or_null(Xow_ns_.Tid__module);
Xoa_ttl mod_ttl = Xoa_ttl.Parse(wiki, Bry_.Add(module_ns.Name_db_w_colon(), mod_name));
mod_raw = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(mod_ttl);
mod_raw = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(mod_ttl);
if (mod_raw == null) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:missing_mod}}"
}
else

35
400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_mw.java
View File

@@ -97,7 +97,7 @@ public class Scrib_lib_mw implements Scrib_lib {
return rslt.Init_obj(core.Interpreter().LoadString("@" + mod_name + ".lua", mod_code));
Xoa_ttl ttl = Xoa_ttl.Parse(cur_wiki, Bry_.new_u8(mod_name));// NOTE: should have Module: prefix
if (ttl == null) return rslt.Init_ary_empty();
byte[] page_db = cur_wiki.Cache_mgr().Page_cache().Get_or_load_as_src(ttl);
byte[] page_db = cur_wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl);
if (page_db == null) return rslt.Init_ary_empty();
Scrib_lua_mod mod = new Scrib_lua_mod(core, mod_name);
return rslt.Init_obj(mod.LoadString(String_.new_u8(page_db)));
@@ -316,25 +316,28 @@ public class Scrib_lib_mw implements Scrib_lib {
if (!ttl.ForceLiteralLink() && ttl.Ns().Id_is_main()) // title is not literal and is not prefixed with Template; parse again as template; EX: ":A" and "Template:A" are fine; "A" is parsed again as "Template:A"
ttl = Xoa_ttl.Parse(cur_wiki, Bry_.Add(cur_wiki.Ns_mgr().Ns_template().Name_db_w_colon(), ttl_bry)); // parse again, but add "Template:"
Keyval[] args_ary = args.Pull_kv_ary_safe(2);
// BLOCK.bgn:Xot_invk_tkn.Transclude; cannot reuse b/c Transclude needs invk_tkn, and invk_tkn is manufactured late; DATE:2014-01-02
byte[] sub_src = null;
if (ttl.Ns().Id_is_tmpl()) { // ttl is template; check tmpl_regy first before going to data_mgr
// ttl is template; check tmpl_regy first before going to data_mgr
if (ttl.Ns().Id_is_tmpl()) {
Xot_defn_tmpl tmpl = (Xot_defn_tmpl)core.Wiki().Cache_mgr().Defn_cache().Get_by_key(ttl.Page_db());
if (tmpl != null) sub_src = tmpl.Data_raw();
if (tmpl != null)
sub_src = tmpl.Data_raw();
}
if (sub_src == null) // ttl is not in template cache, or is a ttl in non-Template ns; load title
sub_src = core.Wiki().Cache_mgr().Page_cache().Get_or_load_as_src(ttl);
if (sub_src != null) {
Xot_invk_mock sub_frame = Xot_invk_mock.new_(core.Frame_current().Defn_tid(), 0, ttl.Full_txt_w_ttl_case(), args_ary); // NOTE: (1) must have ns (Full); (2) must be txt (space, not underscore); EX:Template:Location map+; DATE:2014-09-21
Xot_defn_tmpl transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), ttl.Ns(), ttl.Page_db(), sub_src);
Bry_bfr sub_bfr = cur_wiki.Utl__bfr_mkr().Get_k004();
transclude_tmpl.Tmpl_evaluate(ctx, sub_frame, sub_bfr);
return rslt.Init_obj(sub_bfr.To_str_and_rls());
}
else {
// ttl is not in template cache; load title
if (sub_src == null)
sub_src = core.Wiki().Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl);
// ttl does not exist; fail
if (sub_src == null)
return rslt.Init_fail("expandTemplate: template \"" + ttl_str + "\" does not exist"); // NOTE: must return error if template is missing; PAGE:en.w:Flag_of_Greenland DATE:2016-05-02
}
// BLOCK.end:Xot_invk_tkn.Transclude
Xot_invk_mock sub_frame = Xot_invk_mock.new_(core.Frame_current().Defn_tid(), 0, ttl.Full_txt_w_ttl_case(), args_ary); // NOTE: (1) must have ns (Full); (2) must be txt (space, not underscore); EX:Template:Location map+; DATE:2014-09-21
Xot_defn_tmpl transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), ttl.Ns(), ttl.Page_db(), sub_src);
Bry_bfr sub_bfr = cur_wiki.Utl__bfr_mkr().Get_k004();
transclude_tmpl.Tmpl_evaluate(ctx, sub_frame, sub_bfr);
return rslt.Init_obj(sub_bfr.To_str_and_rls());
}
public boolean IncrementExpensiveFunctionCount(Scrib_proc_args args, Scrib_proc_rslt rslt) {return rslt.Init_obj(Keyval_.Ary_empty);} // NOTE: for now, always return null (XOWA does not care about expensive parser functions)

2
400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title.java
View File

@@ -195,7 +195,7 @@ public class Scrib_lib_title implements Scrib_lib {
}
public static byte[] GetContentInternal(Scrib_core core, Xowe_wiki wiki, byte[] ttl_bry) {
Xoa_ttl ttl = wiki.Ttl_parse(ttl_bry); if (ttl == null) return null;
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm_2(ttl);
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
byte[] rv = null;
if (page_itm != null) {
byte[] redirected_src = page_itm.Wtxt__redirect();

2
400_xowa/src/gplx/xowa/xtns/template_styles/Template_styles_nde.java
View File

@@ -49,7 +49,7 @@ public class Template_styles_nde implements Xox_xnde, Mwh_atr_itm_owner2 {
}
// get page
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm_2(css_ttl);
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(css_ttl);
if (page_itm != null) {
css_src = page_itm.Wtxt__direct();
css_page_id = page_itm.Page_id();

2
400_xowa/src/gplx/xowa/xtns/wikias/Tabview_xnde.java
View File

@@ -120,7 +120,7 @@ class Tabview_tab_itm {
if (Bry_.Has_at_bgn(page_ttl_bry, Byte_ascii.Angle_bgn) || Bry_.Has_at_end(page_ttl_bry, Byte_ascii.Angle_end)) return null;
Xoa_ttl page_ttl = wiki.Ttl_parse(page_ttl_bry);
if (page_ttl == null) return null;
gplx.xowa.wikis.caches.Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(page_ttl);
gplx.xowa.wikis.caches.Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
if (page_itm == null) return null;
page_body = page_itm.Wtxt__redirect_or_direct();
page_body = Xop_parser_.Parse_text_to_html(wiki, ctx, ctx.Page(), page_body, false);