Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2024-10-27 20:34:16 +00:00
Code Issues Projects Releases Wiki Activity

Mass_parse: Change page_cache to LRU cache [#483]

Browse Source
This commit is contained in:
gnosygnu 2019-06-21 23:02:49 -04:00
parent 0cfb0b19ad
commit a01e7409eb
21 changed files with 367 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
400_xowa/src/gplx/core/lists/caches/Lru_cache.java Normal file
View File

@ -0,0 +1,117 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
public class Lru_cache {
private final Hash_adp map = Hash_adp_.New();
private long max;
private long cur = 0;
private Lru_node head;
private Lru_node tail;
private long evicts;
public void Max_(long max) {
this.max = max;
}
public long Evicts() {return evicts;}
public long Cur() {return cur;}
public Object Get_or_null(Object key) {
Lru_node nde = (Lru_node)map.Get_by(key);
if (nde == null) {
return null;
}
Del_node_from_linked_list(nde);
Add_to_tail(nde);
return nde.Val();
}
public void Set(Object key, Object val, long size) {
Lru_node nde = (Lru_node)map.Get_by(key);
if (nde != null) {
nde.Val_(val);
Del_node_from_linked_list(nde);
Add_to_tail(nde);
}
else {
while (cur + size > max) {
Del_node_from_this(head);
evicts++;
}
nde = new Lru_node(key, val, size);
Add_to_tail(nde);
map.Add(key, nde);
cur += size;
}
}
public void Del(Object key) {
Lru_node nde = (Lru_node)map.Get_by(key);
if (nde != null) {
Del_node_from_this(nde);
}
}
public void Clear() {
map.Clear();
head = null;
tail = null;
cur = 0;
}
private void Del_node_from_this(Lru_node nde) {
map.Del(nde.Key());
cur -= nde.Size();
Del_node_from_linked_list(nde);
}
private void Del_node_from_linked_list(Lru_node nde) {
if (nde.Prv() == null)
head = nde.Nxt();
else
nde.Prv().Nxt_(nde.Nxt());
if (nde.Nxt() == null)
tail = nde.Prv();
else
nde.Nxt().Prv_(nde.Prv());
}
private void Add_to_tail(Lru_node nde) {
if (tail != null)
tail.Nxt_(nde);
nde.Prv_(tail);
nde.Nxt_(null);
tail = nde;
if (head == null)
head = tail;
}
}
class Lru_node {
private final Object key;
private Object val;
private final long size;
private Lru_node prv;
private Lru_node nxt;
public Lru_node(Object key, Object val, long size) {
this.key = key;
this.val = val;
this.size = size;
}
public Object Key() {return key;}
public Object Val() {return val;} public void Val_(Object v) {this.val = v;}
public long Size() {return size;}
public Lru_node Prv() {return prv;} public void Prv_(Lru_node v) {this.prv = v;}
public Lru_node Nxt() {return nxt;} public void Nxt_(Lru_node v) {this.nxt = v;}
}

116
400_xowa/src/gplx/core/lists/caches/Lru_cache_tst.java Normal file
View File

@ -0,0 +1,116 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
import org.junit.*; import gplx.core.tests.*;
import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.nss.*;
public class Lru_cache_tst {
private final Lru_cache_fxt fxt = new Lru_cache_fxt();
@Test public void Get_one() {
fxt.Init__max(10);
fxt.Exec__set("a", 5);
fxt.Test__get_y("a");
}
@Test public void Pop_one() {
fxt.Init__max(10);
fxt.Exec__set("a", 10);
fxt.Exec__set("b", 10);
fxt.Test__get_n("a");
fxt.Test__get_y("b");
}
@Test public void Add_many() {
fxt.Init__max(10);
fxt.Exec__set("a", 4);
fxt.Exec__set("b", 3);
fxt.Exec__set("c", 2);
fxt.Exec__set("d", 1);
fxt.Test__get_y("a", "b", "c", "d");
}
@Test public void Pop_many() {
fxt.Init__max(10);
fxt.Exec__set("a", 4);
fxt.Exec__set("b", 3);
fxt.Exec__set("c", 2);
fxt.Exec__set("d", 1);
fxt.Exec__set("e", 6);
fxt.Test__get_y("c", "d", "e");
fxt.Test__get_n("a", "b");
}
@Test public void Set_repeatedly() {
fxt.Init__max(10);
fxt.Exec__set("a", "a1", 10);
fxt.Exec__set("a", "a2", 10);
fxt.Exec__set("a", "a3", 10);
fxt.Test__get_val("a", "a3");
}
@Test public void Set_bumps_priority() {
fxt.Init__max(10);
fxt.Exec__set("a", 2);
fxt.Exec__set("b", 3);
fxt.Exec__set("c", 2);
fxt.Exec__set("a", 2);
fxt.Exec__set("d", 7);
fxt.Test__get_y("a", "d");
fxt.Test__get_n("b", "c");
}
@Test public void Del() {
fxt.Init__max(10);
fxt.Exec__set("a", 2);
fxt.Exec__set("b", 2);
fxt.Exec__del("b");
fxt.Test__get_y("a");
fxt.Test__get_n("b");
}
@Test public void Clear() {
fxt.Init__max(10);
fxt.Exec__set("a", 2);
fxt.Exec__set("b", 2);
fxt.Exec__clear();
fxt.Test__get_n("a", "b");
}
}
class Lru_cache_fxt {
private final Lru_cache cache = new Lru_cache();
public void Init__max(long max) {
cache.Max_(max);
}
public void Exec__set(String key, long size) {
cache.Set(key, key, size);
}
public void Exec__set(String key, String val, long size) {
cache.Set(key, val, size);
}
public void Exec__del(String key) {
cache.Del(key);
}
public void Exec__clear() {
cache.Clear();
}
public void Test__get_y(String... keys) {
for (String key : keys)
Test__get(key, key);
}
public void Test__get_n(String... keys) {
for (String key : keys)
Test__get(key, null);
}
public void Test__get_val(String key, String val) {
Test__get(key, val);
}
private void Test__get(String key, String expd) {
Object actl = cache.Get_or_null(key);
Gftest.Eq__obj_or_null(expd, actl);
}
}

8
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java
View File

@ -31,13 +31,14 @@ public class Xomp_parse_mgr {
// init page_pool
Xomp_page_pool_loader page_pool_loader = new Xomp_page_pool_loader(wiki, mgr_db.Conn(), cfg.Num_pages_in_pool(), cfg.Show_msg__fetched_pool());
Xomp_page_pool page_pool = new Xomp_page_pool(page_pool_loader, cfg.Num_pages_per_wkr());
Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval(), cfg.Perf_interval(), mgr_db.Url().GenNewNameAndExt("xomp.perf.csv"));
// cache: preload tmpls and imglinks
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates());
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates(), cfg.Page_cache_max());
wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of user cache
Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
prog_mgr.Init(page_cache, page_pool_loader.Get_pending_count(), cfg.Progress_interval(), cfg.Perf_interval(), mgr_db.Url().GenNewNameAndExt("xomp.perf.csv"));
Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
Xow_ifexist_cache ifexist_cache = new Xow_ifexist_cache(wiki, page_cache).Cache_sizes_(Int_.Max_value, Int_.Max_value);
if (cfg.Load_ifexists_ns() != null) Load_ifexists_ns(wiki, ifexist_cache, cfg.Load_ifexists_ns());
@ -71,6 +72,7 @@ public class Xomp_parse_mgr {
Xowe_wiki wkr_wiki = Xow_wiki_utl_.Clone_wiki(wiki, wiki.Fsys_mgr().Root_dir());
wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);
// make wkr
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, cfg, mgr_db, page_pool, prog_mgr, file_orig_wkr, ns_ord_mgr, wkr_wiki, indexer, i + wkr_uid_bgn);
wkrs[i] = wkr;

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
View File

@ -42,6 +42,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
public long Wbase_cache_mru_size() {return wbase_cache_mru_size;} private long wbase_cache_mru_size = 100;
public long Wbase_cache_mru_weight() {return wbase_cache_mru_weight;} private long wbase_cache_mru_weight = 10;
public long Wbase_cache_mru_compress_size() {return wbase_cache_mru_compress_size;} private long wbase_cache_mru_compress_size = 70;
public long Page_cache_max() {return page_cache_max;} private long page_cache_max = 8 * (long)Io_mgr.Len_gb;
public void Init(Xowe_wiki wiki) {
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count();
if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000;
@ -76,6 +77,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, "wbase_cache_mru_size_")) wbase_cache_mru_size = m.ReadLong("v");
else if (ctx.Match(k, "wbase_cache_mru_weight_")) wbase_cache_mru_weight = m.ReadLong("v");
else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v");
else if (ctx.Match(k, "page_cache_max_")) page_cache_max = m.ReadLong("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}

7
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_prog_mgr.java
View File

@ -14,14 +14,17 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.xowa.wikis.caches.*;
public class Xomp_prog_mgr {
private final Object thread_lock = new Object();
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
private Xow_page_cache page_cache;
private int prog_interval, perf_interval;
private int pages_done, pages_total;
private long prog_bgn, prog_prv, prog_done, perf_prv;
private Io_url perf_url;
public void Init(int pages_total, int prog_interval, int perf_interval, Io_url perf_url) {
public void Init(Xow_page_cache page_cache, int pages_total, int prog_interval, int perf_interval, Io_url perf_url) {
this.page_cache = page_cache;
this.pages_total = pages_total;
this.prog_interval = prog_interval;
this.perf_interval = perf_interval;
@ -39,7 +42,7 @@ public class Xomp_prog_mgr {
double rate_cur = pages_done / (prog_done / Time_span_.Ratio_f_to_s);
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)((prog_cur - prog_bgn) / 1000), true, 0);
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)(pages_left / rate_cur), true, 0);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5} cache_stats=~{6}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left, page_cache.To_str());
prog_prv = prog_cur;
}
if (pages_done % perf_interval == 0) {

10
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_tmpl_cache_bldr.java
View File

@ -17,8 +17,9 @@ package gplx.xowa.addons.bldrs.mass_parses.parses.utls; import gplx.*; import gp
import gplx.dbs.*;
import gplx.xowa.wikis.caches.*;
public class Xomp_tmpl_cache_bldr {
public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all) {
public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all, long page_cache_max) {
Xow_page_cache rv = new Xow_page_cache(wiki);
rv.Max_(page_cache_max);
if (fill_all) Fill_all(rv, wiki);
return rv;
}
@ -53,12 +54,7 @@ public class Xomp_tmpl_cache_bldr {
text_db_loader.Add(rdr.Read_int("page_text_db_id"), itm);
// ignore duplicate page_titles in cache; EX:ru.n:Модуль:Weather/data DATE:2017-03-16
if (cache.Get_or_null(page_ttl.Full_db()) == null) {
cache.Add(page_ttl.Full_db(), itm);
}
else {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse: ignoring duplicate page title in page cache; title=~{0} id=~{1}", page_ttl.Full_db(), page_id);
}
cache.Add_itm(page_ttl.Full_db_as_str(), itm);
page_regy.Add(page_id, itm);

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java
View File

@ -124,8 +124,6 @@ public class Xomp_parse_wkr implements Gfo_invk {
Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry());
// if ns changed and prv_ns is main
if (cur_ns != prv_ns) {
if (prv_ns == gplx.xowa.wikis.nss.Xow_ns_.Tid__main)
wiki.Cache_mgr().Free_mem__all(); // NOTE: clears page and wbase cache only; needed else OutOfMemory error for en.w in 25th hour; DATE:2017-01-11
prv_ns = cur_ns;
}

7
400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java
View File

@ -22,9 +22,6 @@ public class Xop_mediawiki_wkr {
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
public Xop_mediawiki_wkr(Xowe_wiki wiki, Xop_mediawiki_loader loader) {
this.wiki = wiki;
this.Loader_(loader);
}
public void Loader_(Xop_mediawiki_loader loader) {
if (loader != null)
wiki.Cache_mgr().Load_wkr_(new Xow_page_cache_wkr__embeddable(wiki, loader));
}
@ -34,10 +31,6 @@ public class Xop_mediawiki_wkr {
wiki.Parser_mgr().Scrib().Core_term();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
}
public void Clear_cache(String page) {
Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));
wiki.Cache_mgr().Page_cache().Del(ttl.Full_db());
}
public String Parse(String page, String wikitext) {
Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));

2
400_xowa/src/gplx/xowa/addons/servers/https/utils/Xoa_util_svc.java
View File

@ -43,7 +43,7 @@ public class Xoa_util_svc {
// get page
Xoa_ttl ttl = wiki.Ttl_parse(page_bry);
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm_2(ttl);
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
byte[] page_text = page_itm == null ? null : page_itm.Wtxt__direct();
if (page_text == null) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "Xoa_utl_svc:page not found: wiki=~{0} page=~{1}", wiki_bry, page_bry);

4
400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn.java
View File

@ -333,7 +333,7 @@ public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk {
if (tmpl != null) transclude_src = tmpl.Data_raw();
}
if (transclude_src == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(page_ttl);
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
if ( cache_itm != null
// && Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Page_ttl().Full_db()) // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
) {
@ -377,7 +377,7 @@ public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk {
return true;
}
if (transclude_tmpl == null && ctx.Tmpl_load_enabled()) { // ttl is template not in cache, or some other ns; do load
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(page_ttl);
Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
if ( cache_itm != null) {
if (!Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Ttl().Full_db())) { // make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), cache_itm.Wtxt__direct());

2
400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn_.java
View File

@ -49,7 +49,7 @@ public class Xot_invk_tkn_ {
}
}
public static Xot_defn_tmpl Load_defn(Xowe_wiki wiki, Xop_ctx ctx, Xot_invk_tkn invk_tkn, Xoa_ttl ttl, byte[] name_ary) { // recursive loading of templates
Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(ttl);
Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
byte[] tmpl_page_bry = tmpl_page_itm == null ? null : tmpl_page_itm.Wtxt__direct();
Xot_defn_tmpl rv = null;
if (tmpl_page_bry != null) {

2
400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java
View File

@ -53,7 +53,7 @@ public class Xow_ifexist_cache {
if (ns_loaded_hash.Has(ttl.Ns().Id())) return Bool_.N_byte;
// check page_cache since full page + text could be loaded there
Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_or_null(key);
Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_itm_or_null(ttl.Full_db_as_str());
if (itm == Xow_page_cache_itm.Missing)
return Bool_.N_byte;
else if (itm != null)

171
400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java
View File

@ -14,113 +14,98 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
import gplx.core.lists.caches.*;
public class Xow_page_cache {
private final Object thread_lock = new Object();
private final Object thread_lock = new Object(); // NOTE: thread-safety needed for xomp since one page-cache is shared across all wkrs
private final Xowe_wiki wiki;
private final Ordered_hash cache = Ordered_hash_.New_bry(); // NOTE: wiki titles are not case-sensitive when ns is "1st-letter" (EX: w:earth an w:Earth); in these cases, two entries will be stored
private final List_adp deleted = List_adp_.New();
public Xow_page_cache(Xowe_wiki wiki) {this.wiki = wiki;}
private final Lru_cache cache = new Lru_cache();
private long cache_tries = 0;
private long cache_misses = 0;
public Xow_page_cache(Xowe_wiki wiki) {
this.wiki = wiki;
cache.Max_(16 * Io_mgr.Len_mb);
}
public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;} private Xow_page_cache_wkr load_wkr;
public Xow_page_cache_itm Get_or_null(byte[] ttl_full_db) {return (Xow_page_cache_itm)cache.Get_by(ttl_full_db);}
public byte[] Get_or_load_as_src(Xoa_ttl ttl) {
Xow_page_cache_itm rv = Get_or_load_as_itm(ttl);
public void Max_(long v) {cache.Max_(v);}
public void Add_itm(String ttl_full_db, Xow_page_cache_itm itm) {
synchronized (thread_lock) {
cache.Set(ttl_full_db, itm, itm.Cache_len());
}
}
public Xow_page_cache_itm Get_itm_or_null(String ttl_full_db) {
synchronized (thread_lock) {
return (Xow_page_cache_itm)cache.Get_or_null(ttl_full_db);
}
}
public Xow_page_cache_itm Get_itm_else_load_or_null(Xoa_ttl ttl) {
synchronized (thread_lock) {
cache_tries++;
Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_or_null(ttl.Full_db_as_str());
if (rv == Xow_page_cache_itm.Missing)
return null;
else if (rv == null) {
cache_misses++;
return Load_page(ttl);
}
else
return rv;
}
}
public byte[] Get_src_else_load_or_null(Xoa_ttl ttl) {
synchronized (thread_lock) {
Xow_page_cache_itm rv = Get_itm_else_load_or_null(ttl);
return rv == null ? null : rv.Wtxt__direct();
}
public void Add(byte[] ttl_full_db, Xow_page_cache_itm itm) {
cache.Add(ttl_full_db, itm);
}
private void Add_safe(byte[] ttl_full_db, Xow_page_cache_itm itm) {
synchronized (thread_lock) { // LOCK:high-usage;DATE:2016-07-14
if (!cache.Has(ttl_full_db)) { // check again that itm is not in cache; note that this is necessary as cache.Get is not in "synchronized" block (for performance reasons); DATE:2016-12-12
cache.Add(ttl_full_db, itm);
}
}
}
public void Del(byte[] ttl_full_db) {
cache.Del(ttl_full_db);
}
public Xow_page_cache_itm Get_or_load_as_itm(Xoa_ttl ttl) {
byte[] ttl_full_db = ttl.Full_db();
Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by(ttl_full_db);
if (rv == Xow_page_cache_itm.Missing) {
return null;
}
else if (rv == null) {
return Load_page(ttl, ttl_full_db);
}
return rv;
}
private Xow_page_cache_itm Load_page(Xoa_ttl ttl, byte[] ttl_full_db) {
Xow_page_cache_itm rv = null;
Xoa_ttl page_ttl = ttl;
boolean page_exists = false;
byte[] page_text = null;
byte[] page_redirect_from = null;
int page_id = -1;
// gplx.core.consoles.Console_adp__sys.Instance.Write_str("page_cache:" + String_.new_u8(ttl_full_db));
if (load_wkr != null) {
page_text = load_wkr.Get_page_or_null(ttl_full_db);
page_exists = page_text != null;
}
if (page_text == null) {
Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(ttl); // NOTE: do not call Db_mgr.Load_page; need to handle redirects
page_ttl = page.Ttl();
page_text = page.Db().Text().Text_bry();
page_exists = page.Db().Page().Exists();
page_redirect_from = page.Redirect_trail().Itms__get_wtxt_at_0th_or_null();
page_id = page.Db().Page().Id();
}
if (page_exists) {
rv = new Xow_page_cache_itm(false, page_id, page_ttl, page_text, page_redirect_from);
Add_safe(ttl_full_db, rv);
}
else {
Add_safe(ttl_full_db, Xow_page_cache_itm.Missing);
rv = null;
}
return rv;
}
public Xow_page_cache_itm Get_or_load_as_itm_2(Xoa_ttl ttl) { // NOTE: same as Get_or_load_as_itm, but handles redirects to missing pages; DATE:2016-05-02
byte[] ttl_full_db = ttl.Full_db();
Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by(ttl_full_db);
if (rv == Xow_page_cache_itm.Missing) return null;
else if (rv == null) {
Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(ttl); // NOTE: do not call Db_mgr.Load_page; need to handle redirects
if ( page.Db().Page().Exists() // page exists
|| page.Redirect_trail().Itms__len() > 0 ) { // page redirects to missing page; note that page.Missing == true and page.Redirected_src() != null; PAGE: en.w:Shah_Rukh_Khan; DATE:2016-05-02
rv = new Xow_page_cache_itm(false, page.Db().Page().Id(), page.Ttl(), page.Db().Text().Text_bry(), page.Redirect_trail().Itms__get_wtxt_at_0th_or_null());
Add_safe(ttl_full_db, rv);
}
else {
Add_safe(ttl_full_db, Xow_page_cache_itm.Missing);
rv = null;
}
}
return rv;
}
public void Free_mem(boolean clear_permanent_itms) {
synchronized (thread_lock) { // LOCK:app-level; DATE:2016-07-06
if (clear_permanent_itms) {
cache.Clear();
}
else {
// gather non-permanent items
int len = cache.Count();
for (int i = 0; i < len; i++) {
Xow_page_cache_itm itm = (Xow_page_cache_itm)cache.Get_at(i);
if (!itm.Cache_permanently())
deleted.Add(itm);
}
}
private Xow_page_cache_itm Load_page(Xoa_ttl ttl) {
// vars
Xow_page_cache_itm rv = null;
Xoa_ttl page_ttl = ttl;
boolean page_exists = false;
byte[] page_text = null;
byte[] page_redirect_from = null;
int page_id = -1;
// load_page if load_wkr exists; only for Xop_mediawiki_mgr
// gplx.core.consoles.Console_adp__sys.Instance.Write_str("page_cache:" + String_.new_u8(ttl_full_db));
if (load_wkr != null) {
page_text = load_wkr.Get_page_or_null(ttl.Full_db());
page_exists = page_text != null;
}
// remove non-permanent items
len = deleted.Len();
for (int i = 0; i < len; i++) {
Xow_page_cache_itm itm = (Xow_page_cache_itm)deleted.Get_at(i);
if (itm.Ttl() != null) // missing is null
cache.Del(itm.Ttl().Full_db());
}
deleted.Clear();
// load_page in other cases
if (page_text == null) {
Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(ttl); // NOTE: do not call Db_mgr.Load_page; need to handle redirects
if ( page.Db().Page().Exists() // page exists
|| page.Redirect_trail().Itms__len() > 0 ) { // page redirects to missing page; note that page.Missing == true and page.Redirected_src() != null; PAGE:en.w:Shah_Rukh_Khan; DATE:2016-05-02
page_exists = true; // NOTE: page.Db().Page().Exists() will be false if page redirects to missing page; PAGE:en.w:Shah_Rukh_Khan; DATE:2019-06-16
page_ttl = page.Ttl();
page_text = page.Db().Text().Text_bry();
page_redirect_from = page.Redirect_trail().Itms__get_wtxt_at_0th_or_null();
page_id = page.Db().Page().Id();
}
}
// create item
if (page_exists) {
rv = new Xow_page_cache_itm(false, page_id, page_ttl, page_text, page_redirect_from);
Add_itm(ttl.Full_db_as_str(), rv);
}
else {
rv = null;
Add_itm(ttl.Full_db_as_str(), Xow_page_cache_itm.Missing);
}
return rv;
}
public String To_str() {
return String_.Format("cache_pct:{0} cache_misses:{1} cache_evicts:{2} cache_tries:{3} cache_size:{4}", Decimal_adp_.divide_(cache_misses * 100, cache_tries).To_str("0.00"), cache_misses, cache.Evicts(), cache_tries, cache.Cur());
}
}

12
400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache_itm.java
View File

@ -16,9 +16,11 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
import gplx.xowa.wikis.data.tbls.*;
public class Xow_page_cache_itm implements Xowd_text_bry_owner {
private long cache_len;
public Xow_page_cache_itm(boolean cache_permanently, int page_id, Xoa_ttl ttl, byte[] wtxt__direct, byte[] wtxt__redirect) {
this.cache_permanently = cache_permanently;
this.page_id = page_id; this.ttl = ttl; this.wtxt__direct = wtxt__direct; this.wtxt__redirect = wtxt__redirect;
this.page_id = page_id; this.ttl = ttl; this.wtxt__redirect = wtxt__redirect;
Set_text_bry_by_db(wtxt__direct);
}
public Xoa_ttl Ttl() {return ttl;} private Xoa_ttl ttl;
public byte[] Wtxt__direct() {return wtxt__direct;} private byte[] wtxt__direct;
@ -27,16 +29,20 @@ public class Xow_page_cache_itm implements Xowd_text_bry_owner {
return wtxt__redirect == null ? wtxt__direct : wtxt__redirect;
}
public boolean Cache_permanently() {return cache_permanently;} private final boolean cache_permanently;
public long Cache_len() {return cache_len;}
// used by xomp
public int Page_id() {return page_id;} private int page_id;
public int Redirect_id() {return redirect_id;} private int redirect_id;
public void Set_text_bry_by_db(byte[] v) {this.wtxt__direct = v;}
public void Set_text_bry_by_db(byte[] v) {
this.wtxt__direct = v;
this.cache_len = wtxt__direct == null ? 0 : wtxt__direct.length;
}
public void Redirect_id_(int v) {this.redirect_id = v;}
public void Set_redirect(Xoa_ttl ttl, byte[] trg_wtxt) {
this.ttl = ttl;
Set_text_bry_by_db(trg_wtxt);
this.wtxt__redirect = wtxt__direct;
this.wtxt__direct = trg_wtxt;
}
public static final Xow_page_cache_itm Null = null;

6
400_xowa/src/gplx/xowa/xtns/lst/Lst_pfunc_itm.java
View File

@ -35,7 +35,8 @@ public class Lst_pfunc_itm {
// get sub_src;
Xop_ctx sub_ctx = Xop_ctx.New__top(wiki).Ref_ignore_(true);
sub_ctx.Page().Ttl_(ctx.Page().Ttl()); // NOTE: must set ttl on page, else test fails;
byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(ttl); if (sub_src == null) return null; // {{#lst:missing}} -> ""
byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl);
if (sub_src == null) return null; // {{#lst:missing}} -> ""
// parse page; note adding to stack to prevent circular recursions
if (!wiki.Parser_mgr().Tmpl_stack_add(ttl.Full_db())) return null;
@ -75,7 +76,8 @@ public class Lst_pfunc_itm {
// get sub_ctx: note new ctx is needed b/c sub_page objects must not get added to owner_page; for example, references / hdrs / lnki.files
Xop_ctx sub_ctx = Xop_ctx.New__top(wiki).Ref_ignore_(true);
sub_ctx.Page().Ttl_(ctx.Page().Ttl()); // NOTE: must set ttl on page, else test fails;
byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(ttl); if (sub_src == null) return null; // {{#lst:missing}} -> ""
byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl);
if (sub_src == null) return null; // {{#lst:missing}} -> ""
// parse sub_src; note adding to page's stack to prevent circular recursions
if (!wiki.Parser_mgr().Tmpl_stack_add(ttl.Full_db())) return null;

2
400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_index_parser.java
View File

@ -19,7 +19,7 @@ import gplx.xowa.wikis.nss.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.parsers.xndes.*; import gplx.xowa.parsers.lnkis.*; import gplx.xowa.parsers.tmpls.*;
class Pp_index_parser {
public static Pp_index_page Parse(Xowe_wiki wiki, Xop_ctx ctx, Xoa_ttl index_ttl, int ns_page_id) {
byte[] src = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(index_ttl);
byte[] src = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(index_ttl);
if (src == null) return Pp_index_page.Null;
Xop_parser sub_parser = Xop_parser.new_(wiki, wiki.Parser_mgr().Main().Tmpl_lxr_mgr(), wiki.Parser_mgr().Main().Wtxt_lxr_mgr());
Xop_ctx sub_ctx = Xop_ctx.New__sub__reuse_page(ctx);

2
400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func.java
View File

@ -51,7 +51,7 @@ public class Scrib_invoke_func extends Pf_func_base {
if (mod == null) {
Xow_ns module_ns = wiki.Ns_mgr().Ids_get_or_null(Xow_ns_.Tid__module);
Xoa_ttl mod_ttl = Xoa_ttl.Parse(wiki, Bry_.Add(module_ns.Name_db_w_colon(), mod_name));
mod_raw = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(mod_ttl);
mod_raw = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(mod_ttl);
if (mod_raw == null) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:missing_mod}}"
}
else

27
400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_mw.java
View File

@ -97,7 +97,7 @@ public class Scrib_lib_mw implements Scrib_lib {
return rslt.Init_obj(core.Interpreter().LoadString("@" + mod_name + ".lua", mod_code));
Xoa_ttl ttl = Xoa_ttl.Parse(cur_wiki, Bry_.new_u8(mod_name));// NOTE: should have Module: prefix
if (ttl == null) return rslt.Init_ary_empty();
byte[] page_db = cur_wiki.Cache_mgr().Page_cache().Get_or_load_as_src(ttl);
byte[] page_db = cur_wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl);
if (page_db == null) return rslt.Init_ary_empty();
Scrib_lua_mod mod = new Scrib_lua_mod(core, mod_name);
return rslt.Init_obj(mod.LoadString(String_.new_u8(page_db)));
@ -316,26 +316,29 @@ public class Scrib_lib_mw implements Scrib_lib {
if (!ttl.ForceLiteralLink() && ttl.Ns().Id_is_main()) // title is not literal and is not prefixed with Template; parse again as template; EX: ":A" and "Template:A" are fine; "A" is parsed again as "Template:A"
ttl = Xoa_ttl.Parse(cur_wiki, Bry_.Add(cur_wiki.Ns_mgr().Ns_template().Name_db_w_colon(), ttl_bry)); // parse again, but add "Template:"
Keyval[] args_ary = args.Pull_kv_ary_safe(2);
// BLOCK.bgn:Xot_invk_tkn.Transclude; cannot reuse b/c Transclude needs invk_tkn, and invk_tkn is manufactured late; DATE:2014-01-02
byte[] sub_src = null;
if (ttl.Ns().Id_is_tmpl()) { // ttl is template; check tmpl_regy first before going to data_mgr
// ttl is template; check tmpl_regy first before going to data_mgr
if (ttl.Ns().Id_is_tmpl()) {
Xot_defn_tmpl tmpl = (Xot_defn_tmpl)core.Wiki().Cache_mgr().Defn_cache().Get_by_key(ttl.Page_db());
if (tmpl != null) sub_src = tmpl.Data_raw();
if (tmpl != null)
sub_src = tmpl.Data_raw();
}
if (sub_src == null) // ttl is not in template cache, or is a ttl in non-Template ns; load title
sub_src = core.Wiki().Cache_mgr().Page_cache().Get_or_load_as_src(ttl);
if (sub_src != null) {
// ttl is not in template cache; load title
if (sub_src == null)
sub_src = core.Wiki().Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl);
// ttl does not exist; fail
if (sub_src == null)
return rslt.Init_fail("expandTemplate: template \"" + ttl_str + "\" does not exist"); // NOTE: must return error if template is missing; PAGE:en.w:Flag_of_Greenland DATE:2016-05-02
Xot_invk_mock sub_frame = Xot_invk_mock.new_(core.Frame_current().Defn_tid(), 0, ttl.Full_txt_w_ttl_case(), args_ary); // NOTE: (1) must have ns (Full); (2) must be txt (space, not underscore); EX:Template:Location map+; DATE:2014-09-21
Xot_defn_tmpl transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), ttl.Ns(), ttl.Page_db(), sub_src);
Bry_bfr sub_bfr = cur_wiki.Utl__bfr_mkr().Get_k004();
transclude_tmpl.Tmpl_evaluate(ctx, sub_frame, sub_bfr);
return rslt.Init_obj(sub_bfr.To_str_and_rls());
}
else {
return rslt.Init_fail("expandTemplate: template \"" + ttl_str + "\" does not exist"); // NOTE: must return error if template is missing; PAGE:en.w:Flag_of_Greenland DATE:2016-05-02
}
// BLOCK.end:Xot_invk_tkn.Transclude
}
public boolean IncrementExpensiveFunctionCount(Scrib_proc_args args, Scrib_proc_rslt rslt) {return rslt.Init_obj(Keyval_.Ary_empty);} // NOTE: for now, always return null (XOWA does not care about expensive parser functions)
public boolean IsSubsting(Scrib_proc_args args, Scrib_proc_rslt rslt) {

2
400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title.java
View File

@ -195,7 +195,7 @@ public class Scrib_lib_title implements Scrib_lib {
}
public static byte[] GetContentInternal(Scrib_core core, Xowe_wiki wiki, byte[] ttl_bry) {
Xoa_ttl ttl = wiki.Ttl_parse(ttl_bry); if (ttl == null) return null;
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm_2(ttl);
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
byte[] rv = null;
if (page_itm != null) {
byte[] redirected_src = page_itm.Wtxt__redirect();

2
400_xowa/src/gplx/xowa/xtns/template_styles/Template_styles_nde.java
View File

@ -49,7 +49,7 @@ public class Template_styles_nde implements Xox_xnde, Mwh_atr_itm_owner2 {
}
// get page
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm_2(css_ttl);
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(css_ttl);
if (page_itm != null) {
css_src = page_itm.Wtxt__direct();
css_page_id = page_itm.Page_id();

2
400_xowa/src/gplx/xowa/xtns/wikias/Tabview_xnde.java
View File

@ -120,7 +120,7 @@ class Tabview_tab_itm {
if (Bry_.Has_at_bgn(page_ttl_bry, Byte_ascii.Angle_bgn) || Bry_.Has_at_end(page_ttl_bry, Byte_ascii.Angle_end)) return null;
Xoa_ttl page_ttl = wiki.Ttl_parse(page_ttl_bry);
if (page_ttl == null) return null;
gplx.xowa.wikis.caches.Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(page_ttl);
gplx.xowa.wikis.caches.Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
if (page_itm == null) return null;
page_body = page_itm.Wtxt__redirect_or_direct();
page_body = Xop_parser_.Parse_text_to_html(wiki, ctx, ctx.Page(), page_body, false);