Mass_parse: Change page_cache to LRU cache [#483]

400_xowa/src/gplx/core/lists/caches/Lru_cache.java

@@ -0,0 +1,117 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012-2017 gnosygnu@gmail.com
+
+XOWA is licensed under the terms of the General Public License (GPL) Version 3,
+or alternatively under the terms of the Apache License Version 2.0.
+
+You may use XOWA according to either of these licenses as is most appropriate
+for your project on a case-by-case basis.
+
+The terms of each license can be found in the source code repository:
+
+GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
+Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
+*/
+package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
+public class Lru_cache {
+	private final    Hash_adp map = Hash_adp_.New();
+	private long max;
+	private long cur = 0;
+	private Lru_node head;
+	private Lru_node tail;
+	private long evicts;
+	public void Max_(long max) {
+		this.max = max;
+	}
+	public long Evicts() {return evicts;}
+	public long Cur() {return cur;}
+	public Object Get_or_null(Object key) {
+		Lru_node nde = (Lru_node)map.Get_by(key);
+		if (nde == null) {
+			return null;
+		}
+
+		Del_node_from_linked_list(nde);
+		Add_to_tail(nde);
+
+		return nde.Val();
+	}
+	public void Set(Object key, Object val, long size) {
+		Lru_node nde = (Lru_node)map.Get_by(key);
+		if (nde != null) {
+			nde.Val_(val);
+
+			Del_node_from_linked_list(nde);
+			Add_to_tail(nde);
+		}
+		else {
+			while (cur + size > max) {
+				Del_node_from_this(head);
+				evicts++;
+			}
+
+			nde = new Lru_node(key, val, size);
+			Add_to_tail(nde);
+			map.Add(key, nde);
+			cur += size;
+		}
+	}
+	public void Del(Object key) {
+		Lru_node nde = (Lru_node)map.Get_by(key);
+		if (nde != null) {
+			Del_node_from_this(nde);
+		}
+	}
+	public void Clear() {
+		map.Clear();
+		head = null;
+		tail = null;
+		cur = 0;
+	}
+	private void Del_node_from_this(Lru_node nde) {
+		map.Del(nde.Key());
+		cur -= nde.Size();
+		Del_node_from_linked_list(nde);
+	}
+	private void Del_node_from_linked_list(Lru_node nde) {
+		if (nde.Prv() == null)
+			head = nde.Nxt();
+		else
+			nde.Prv().Nxt_(nde.Nxt());
+
+		if (nde.Nxt() == null)
+			tail = nde.Prv();
+		else
+			nde.Nxt().Prv_(nde.Prv());
+	}
+	private void Add_to_tail(Lru_node nde) {
+		if (tail != null)
+			tail.Nxt_(nde);
+
+		nde.Prv_(tail);
+		nde.Nxt_(null);
+		tail = nde;
+
+		if (head == null)
+			head = tail;
+	}
+}
+class Lru_node {
+	private final    Object key;
+	private Object val;
+	private final    long size;
+	private Lru_node prv;
+	private Lru_node nxt;
+
+	public Lru_node(Object key, Object val, long size) {
+		this.key = key;
+		this.val = val;
+		this.size = size;
+	}
+	public Object Key() {return key;}
+	public Object Val() {return val;} public void Val_(Object v) {this.val = v;}
+	public long Size() {return size;}
+	public Lru_node Prv() {return prv;} public void Prv_(Lru_node v) {this.prv = v;}
+	public Lru_node Nxt() {return nxt;} public void Nxt_(Lru_node v) {this.nxt = v;}
+}

400_xowa/src/gplx/core/lists/caches/Lru_cache_tst.java

@@ -0,0 +1,116 @@
+/*
+XOWA: the XOWA Offline Wiki Application
+Copyright (C) 2012-2017 gnosygnu@gmail.com
+
+XOWA is licensed under the terms of the General Public License (GPL) Version 3,
+or alternatively under the terms of the Apache License Version 2.0.
+
+You may use XOWA according to either of these licenses as is most appropriate
+for your project on a case-by-case basis.
+
+The terms of each license can be found in the source code repository:
+
+GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
+Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
+*/
+package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
+import org.junit.*; import gplx.core.tests.*;
+import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.nss.*;
+public class Lru_cache_tst {
+	private final    Lru_cache_fxt fxt = new Lru_cache_fxt();
+	@Test  public void Get_one() {
+		fxt.Init__max(10);
+		fxt.Exec__set("a", 5);
+		fxt.Test__get_y("a");
+	}
+	@Test  public void Pop_one() {
+		fxt.Init__max(10);
+		fxt.Exec__set("a", 10);
+		fxt.Exec__set("b", 10);
+		fxt.Test__get_n("a");
+		fxt.Test__get_y("b");
+	}
+	@Test  public void Add_many() {
+		fxt.Init__max(10);
+		fxt.Exec__set("a", 4);
+		fxt.Exec__set("b", 3);
+		fxt.Exec__set("c", 2);
+		fxt.Exec__set("d", 1);
+		fxt.Test__get_y("a", "b", "c", "d");
+	}
+	@Test  public void Pop_many() {
+		fxt.Init__max(10);
+		fxt.Exec__set("a", 4);
+		fxt.Exec__set("b", 3);
+		fxt.Exec__set("c", 2);
+		fxt.Exec__set("d", 1);
+		fxt.Exec__set("e", 6);
+		fxt.Test__get_y("c", "d", "e");
+		fxt.Test__get_n("a", "b");
+	}
+	@Test  public void Set_repeatedly() {
+		fxt.Init__max(10);
+		fxt.Exec__set("a", "a1", 10);
+		fxt.Exec__set("a", "a2", 10);
+		fxt.Exec__set("a", "a3", 10);
+		fxt.Test__get_val("a", "a3");
+	}
+	@Test  public void Set_bumps_priority() {
+		fxt.Init__max(10);
+		fxt.Exec__set("a", 2);
+		fxt.Exec__set("b", 3);
+		fxt.Exec__set("c", 2);
+		fxt.Exec__set("a", 2);
+		fxt.Exec__set("d", 7);
+		fxt.Test__get_y("a", "d");
+		fxt.Test__get_n("b", "c");
+	}
+	@Test  public void Del() {
+		fxt.Init__max(10);
+		fxt.Exec__set("a", 2);
+		fxt.Exec__set("b", 2);
+		fxt.Exec__del("b");
+		fxt.Test__get_y("a");
+		fxt.Test__get_n("b");
+	}
+	@Test  public void Clear() {
+		fxt.Init__max(10);
+		fxt.Exec__set("a", 2);
+		fxt.Exec__set("b", 2);
+		fxt.Exec__clear();
+		fxt.Test__get_n("a", "b");
+	}
+}
+class Lru_cache_fxt {
+	private final    Lru_cache cache = new Lru_cache();
+	public void Init__max(long max) {
+		cache.Max_(max);
+	}
+	public void Exec__set(String key, long size) {
+		cache.Set(key, key, size);
+	}
+	public void Exec__set(String key, String val, long size) {
+		cache.Set(key, val, size);
+	}
+	public void Exec__del(String key) {
+		cache.Del(key);
+	}
+	public void Exec__clear() {
+		cache.Clear();
+	}
+	public void Test__get_y(String... keys) {
+		for (String key : keys)
+			Test__get(key, key);
+	}
+	public void Test__get_n(String... keys) {
+		for (String key : keys)
+			Test__get(key, null);
+	}
+	public void Test__get_val(String key, String val) {
+		Test__get(key, val);
+	}
+	private void Test__get(String key, String expd) {
+		Object actl = cache.Get_or_null(key);
+		Gftest.Eq__obj_or_null(expd, actl);
+	}
+}

400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java

@@ -31,13 +31,14 @@ public class Xomp_parse_mgr {
 		// init page_pool
 		Xomp_page_pool_loader page_pool_loader = new Xomp_page_pool_loader(wiki, mgr_db.Conn(), cfg.Num_pages_in_pool(), cfg.Show_msg__fetched_pool());
 		Xomp_page_pool page_pool = new Xomp_page_pool(page_pool_loader, cfg.Num_pages_per_wkr());
-		Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
-		prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval(), cfg.Perf_interval(), mgr_db.Url().GenNewNameAndExt("xomp.perf.csv"));
 
 		// cache: preload tmpls and imglinks
-		Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates());
+		Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates(), cfg.Page_cache_max());
 		wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_();	// disable db lookups of user cache
 
+		Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
+		prog_mgr.Init(page_cache, page_pool_loader.Get_pending_count(), cfg.Progress_interval(), cfg.Perf_interval(), mgr_db.Url().GenNewNameAndExt("xomp.perf.csv"));
+
 		Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
 		Xow_ifexist_cache ifexist_cache = new Xow_ifexist_cache(wiki, page_cache).Cache_sizes_(Int_.Max_value, Int_.Max_value);
 		if (cfg.Load_ifexists_ns() != null) Load_ifexists_ns(wiki, ifexist_cache, cfg.Load_ifexists_ns());
@@ -71,6 +72,7 @@ public class Xomp_parse_mgr {
 			Xowe_wiki wkr_wiki = Xow_wiki_utl_.Clone_wiki(wiki, wiki.Fsys_mgr().Root_dir());
 			wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);				
 
+
 			// make wkr
 			Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, cfg, mgr_db, page_pool, prog_mgr, file_orig_wkr, ns_ord_mgr, wkr_wiki, indexer, i + wkr_uid_bgn);
 			wkrs[i] = wkr;

400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java

@@ -42,6 +42,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
 	public long		Wbase_cache_mru_size()			{return wbase_cache_mru_size;}			private long wbase_cache_mru_size = 100;
 	public long		Wbase_cache_mru_weight()		{return wbase_cache_mru_weight;}		private long wbase_cache_mru_weight = 10;
 	public long		Wbase_cache_mru_compress_size()	{return wbase_cache_mru_compress_size;}	private long wbase_cache_mru_compress_size = 70;
+	public long     Page_cache_max()                {return page_cache_max;}                private long page_cache_max = 8 * (long)Io_mgr.Len_gb;
 	public void Init(Xowe_wiki wiki) {
 		if (num_wkrs == -1)				num_wkrs = gplx.core.envs.Runtime_.Cpu_count();
 		if (num_pages_in_pool == -1)	num_pages_in_pool = num_wkrs * 1000;
@@ -76,6 +77,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
 		else if	(ctx.Match(k, "wbase_cache_mru_size_"))				wbase_cache_mru_size = m.ReadLong("v");
 		else if	(ctx.Match(k, "wbase_cache_mru_weight_"))			wbase_cache_mru_weight = m.ReadLong("v");
 		else if	(ctx.Match(k, "wbase_cache_mru_compress_size_"))	wbase_cache_mru_compress_size = m.ReadLong("v");
+		else if	(ctx.Match(k, "page_cache_max_"))                   page_cache_max = m.ReadLong("v");
 		else	return Gfo_invk_.Rv_unhandled;
 		return this;
 	}

400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_prog_mgr.java

@@ -14,14 +14,17 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
 Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
 */
 package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
+import gplx.xowa.wikis.caches.*;
 public class Xomp_prog_mgr {
 	private final    Object thread_lock = new Object();
 	private final    Bry_bfr tmp_bfr = Bry_bfr_.New();
+	private Xow_page_cache page_cache;
 	private int prog_interval, perf_interval;
 	private int pages_done, pages_total;
 	private long prog_bgn, prog_prv, prog_done, perf_prv;
 	private Io_url perf_url;
-	public void Init(int pages_total, int prog_interval, int perf_interval, Io_url perf_url) {
+	public void Init(Xow_page_cache page_cache, int pages_total, int prog_interval, int perf_interval, Io_url perf_url) {
+		this.page_cache = page_cache;
 		this.pages_total = pages_total;
 		this.prog_interval = prog_interval;
 		this.perf_interval = perf_interval;
@@ -39,7 +42,7 @@ public class Xomp_prog_mgr {
 				double rate_cur = pages_done / (prog_done / Time_span_.Ratio_f_to_s);
 				String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)((prog_cur - prog_bgn) / 1000), true, 0);
 				String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)(pages_left / rate_cur), true, 0);
-				Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left);
+				Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5} cache_stats=~{6}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left, page_cache.To_str());
 				prog_prv = prog_cur;
 			}
 			if (pages_done % perf_interval == 0) {

400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_tmpl_cache_bldr.java

@@ -17,8 +17,9 @@ package gplx.xowa.addons.bldrs.mass_parses.parses.utls; import gplx.*; import gp
 import gplx.dbs.*;
 import gplx.xowa.wikis.caches.*;
 public class Xomp_tmpl_cache_bldr {
-	public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all) {
+	public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all, long page_cache_max) {
 		Xow_page_cache rv = new Xow_page_cache(wiki);
+		rv.Max_(page_cache_max);
 		if (fill_all) Fill_all(rv, wiki);
 		return rv;
 	}
@@ -53,12 +54,7 @@ public class Xomp_tmpl_cache_bldr {
 				text_db_loader.Add(rdr.Read_int("page_text_db_id"), itm);
 				
 				// ignore duplicate page_titles in cache; EX:ru.n:Модуль:Weather/data DATE:2017-03-16
-				if (cache.Get_or_null(page_ttl.Full_db()) == null) {
+				cache.Add_itm(page_ttl.Full_db_as_str(), itm);
-					cache.Add(page_ttl.Full_db(), itm);
-				}
-				else {
-					Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse: ignoring duplicate page title in page cache; title=~{0} id=~{1}", page_ttl.Full_db(), page_id);
-				}
 
 				page_regy.Add(page_id, itm);
 

400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java

@@ -124,8 +124,6 @@ public class Xomp_parse_wkr implements Gfo_invk {
 				Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry());
 				// if ns changed and prv_ns is main
 				if (cur_ns != prv_ns) {
-					if (prv_ns == gplx.xowa.wikis.nss.Xow_ns_.Tid__main)
-						wiki.Cache_mgr().Free_mem__all();	// NOTE: clears page and wbase cache only; needed else OutOfMemory error for en.w in 25th hour; DATE:2017-01-11
 					prv_ns = cur_ns;
 				}
 

400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java

@@ -22,9 +22,6 @@ public class Xop_mediawiki_wkr {
 	private final    Bry_bfr tmp_bfr = Bry_bfr_.New();
 	public Xop_mediawiki_wkr(Xowe_wiki wiki, Xop_mediawiki_loader loader) {
 		this.wiki = wiki;
-		this.Loader_(loader);
-	}
-	public void Loader_(Xop_mediawiki_loader loader) {
 		if (loader != null)
 			wiki.Cache_mgr().Load_wkr_(new Xow_page_cache_wkr__embeddable(wiki, loader));
 	}
@@ -34,10 +31,6 @@ public class Xop_mediawiki_wkr {
 		wiki.Parser_mgr().Scrib().Core_term();
 		wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
 	}
-	public void Clear_cache(String page) {
-		Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));
-		wiki.Cache_mgr().Page_cache().Del(ttl.Full_db());
-	}
 	public String Parse(String page, String wikitext) {
 		Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));
 

400_xowa/src/gplx/xowa/addons/servers/https/utils/Xoa_util_svc.java

@@ -43,7 +43,7 @@ public class Xoa_util_svc {
 
 		// get page
 		Xoa_ttl ttl = wiki.Ttl_parse(page_bry);
-		Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm_2(ttl);
+		Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
 		byte[] page_text = page_itm == null ? null : page_itm.Wtxt__direct();
 		if (page_text == null) {
 			Gfo_usr_dlg_.Instance.Warn_many("", "", "Xoa_utl_svc:page not found: wiki=~{0} page=~{1}", wiki_bry, page_bry);

400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn.java

@@ -333,7 +333,7 @@ public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk {
 			if (tmpl != null) transclude_src = tmpl.Data_raw();
 		}
 		if (transclude_src == null && ctx.Tmpl_load_enabled()) {	// ttl is template not in cache, or some other ns; do load
-			Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(page_ttl);
+			Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
 			if (	cache_itm != null
 //					&&  Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Page_ttl().Full_db())	// make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
 				) {
@@ -377,7 +377,7 @@ public class Xot_invk_tkn extends Xop_tkn_itm_base implements Xot_invk {
 				return true;
 		}
 		if (transclude_tmpl == null && ctx.Tmpl_load_enabled()) {	// ttl is template not in cache, or some other ns; do load
-			Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(page_ttl);
+			Xow_page_cache_itm cache_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
 			if (	cache_itm != null) {
 				if (!Bry_.Eq(cache_itm.Ttl().Full_db(), ctx.Page().Ttl().Full_db())) {	// make sure that transcluded item is not same as page_ttl; DATE:2014-01-10
 					transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), page_ttl.Ns(), page_ttl.Page_db(), cache_itm.Wtxt__direct());

400_xowa/src/gplx/xowa/parsers/tmpls/Xot_invk_tkn_.java

@@ -49,7 +49,7 @@ public class Xot_invk_tkn_ {
 		}
 	}
 	public static Xot_defn_tmpl Load_defn(Xowe_wiki wiki, Xop_ctx ctx, Xot_invk_tkn invk_tkn, Xoa_ttl ttl, byte[] name_ary) {	// recursive loading of templates
-		Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(ttl);
+		Xow_page_cache_itm tmpl_page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
 		byte[] tmpl_page_bry = tmpl_page_itm == null ? null : tmpl_page_itm.Wtxt__direct();
 		Xot_defn_tmpl rv = null;
 		if (tmpl_page_bry != null) {

400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java

@@ -53,7 +53,7 @@ public class Xow_ifexist_cache {
 		if (ns_loaded_hash.Has(ttl.Ns().Id())) return Bool_.N_byte;
 
 		// check page_cache since full page + text could be loaded there
-		Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_or_null(key);
+		Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_itm_or_null(ttl.Full_db_as_str());
 		if		(itm == Xow_page_cache_itm.Missing)
 			return Bool_.N_byte;
 		else if (itm != null)

400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java

@@ -14,113 +14,98 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
 Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
 */
 package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
+import gplx.core.lists.caches.*;
 public class Xow_page_cache {
-	private final    Object thread_lock = new Object();
+	private final    Object thread_lock = new Object(); // NOTE: thread-safety needed for xomp since one page-cache is shared across all wkrs
 	private final    Xowe_wiki wiki;
-	private final    Ordered_hash cache = Ordered_hash_.New_bry();	// NOTE: wiki titles are not case-sensitive when ns is "1st-letter" (EX: w:earth an w:Earth); in these cases, two entries will be stored
+	private final    Lru_cache cache = new Lru_cache();
-	private final    List_adp deleted = List_adp_.New();		
+	private long cache_tries = 0;
-	public Xow_page_cache(Xowe_wiki wiki) {this.wiki = wiki;}
+	private long cache_misses = 0;
+	public Xow_page_cache(Xowe_wiki wiki) {
+		this.wiki = wiki;
+		cache.Max_(16 * Io_mgr.Len_mb);
+	}
 	public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;} private Xow_page_cache_wkr load_wkr;
-	public Xow_page_cache_itm Get_or_null(byte[] ttl_full_db) {return (Xow_page_cache_itm)cache.Get_by(ttl_full_db);}
+	public void Max_(long v) {cache.Max_(v);}
-	public byte[] Get_or_load_as_src(Xoa_ttl ttl) {
+
-		Xow_page_cache_itm rv = Get_or_load_as_itm(ttl);
+	public void Add_itm(String ttl_full_db, Xow_page_cache_itm itm) {
-		return rv == null ? null : rv.Wtxt__direct();
+		synchronized (thread_lock) {
+			cache.Set(ttl_full_db, itm, itm.Cache_len());
+		}
 	}
-	public void Add(byte[] ttl_full_db, Xow_page_cache_itm itm) {
+	public Xow_page_cache_itm Get_itm_or_null(String ttl_full_db) {
-		cache.Add(ttl_full_db, itm);
+		synchronized (thread_lock) {
+			return (Xow_page_cache_itm)cache.Get_or_null(ttl_full_db);
+		}
 	}
-	private void Add_safe(byte[] ttl_full_db, Xow_page_cache_itm itm) {
+	public Xow_page_cache_itm Get_itm_else_load_or_null(Xoa_ttl ttl) {
-		synchronized (thread_lock) {	// LOCK:high-usage;DATE:2016-07-14
+		synchronized (thread_lock) {
-			if (!cache.Has(ttl_full_db)) {	// check again that itm is not in cache; note that this is necessary as cache.Get is not in "synchronized" block (for performance reasons); DATE:2016-12-12
+			cache_tries++;
-				cache.Add(ttl_full_db, itm);
+			Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_or_null(ttl.Full_db_as_str());
+			if (rv == Xow_page_cache_itm.Missing)
+				return null;
+			else if (rv == null) {
+				cache_misses++;
+				return Load_page(ttl);
 			}
+			else
+				return rv;
 		}
 	}
-	public void Del(byte[] ttl_full_db) {
+	public byte[] Get_src_else_load_or_null(Xoa_ttl ttl) {
-		cache.Del(ttl_full_db);
+		synchronized (thread_lock) {
-	}
+			Xow_page_cache_itm rv = Get_itm_else_load_or_null(ttl);
-	public Xow_page_cache_itm Get_or_load_as_itm(Xoa_ttl ttl) {
+			return rv == null ? null : rv.Wtxt__direct();
-		byte[] ttl_full_db = ttl.Full_db();
-		Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by(ttl_full_db);
-		if		(rv == Xow_page_cache_itm.Missing) {
-			return null;
 		}
-		else if (rv == null) {
-			return Load_page(ttl, ttl_full_db);
-		}
-		return rv;
-	}
-	private Xow_page_cache_itm Load_page(Xoa_ttl ttl, byte[] ttl_full_db) {
-		Xow_page_cache_itm rv = null;
-		Xoa_ttl page_ttl = ttl;
-		boolean page_exists = false;
-		byte[] page_text = null;
-		byte[] page_redirect_from = null;
-		int page_id = -1;
-		// gplx.core.consoles.Console_adp__sys.Instance.Write_str("page_cache:" + String_.new_u8(ttl_full_db));
-		if (load_wkr != null) {
-			page_text = load_wkr.Get_page_or_null(ttl_full_db);
-			page_exists = page_text != null;
-		}
-		if (page_text == null) {
-			Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(ttl);	// NOTE: do not call Db_mgr.Load_page; need to handle redirects
-			page_ttl = page.Ttl();
-			page_text = page.Db().Text().Text_bry();
-			page_exists = page.Db().Page().Exists();
-			page_redirect_from = page.Redirect_trail().Itms__get_wtxt_at_0th_or_null();
-			page_id = page.Db().Page().Id();
-		}
-		if (page_exists) {
-			rv = new Xow_page_cache_itm(false, page_id, page_ttl, page_text, page_redirect_from);
-			Add_safe(ttl_full_db, rv);
-		}
-		else {
-			Add_safe(ttl_full_db, Xow_page_cache_itm.Missing);
-			rv = null;
-		}
-		return rv;
-	}
-	public Xow_page_cache_itm Get_or_load_as_itm_2(Xoa_ttl ttl) {	// NOTE: same as Get_or_load_as_itm, but handles redirects to missing pages; DATE:2016-05-02
-		byte[] ttl_full_db = ttl.Full_db();
-		Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by(ttl_full_db);
-		if		(rv == Xow_page_cache_itm.Missing) return null;
-		else if (rv == null) {
-			Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(ttl);	// NOTE: do not call Db_mgr.Load_page; need to handle redirects
-			if (	page.Db().Page().Exists()				// page exists
-				||	page.Redirect_trail().Itms__len() > 0 ) {		// page redirects to missing page; note that page.Missing == true and page.Redirected_src() != null; PAGE: en.w:Shah_Rukh_Khan; DATE:2016-05-02
-				rv = new Xow_page_cache_itm(false, page.Db().Page().Id(), page.Ttl(), page.Db().Text().Text_bry(), page.Redirect_trail().Itms__get_wtxt_at_0th_or_null());
-				Add_safe(ttl_full_db, rv);
-			}
-			else {
-				Add_safe(ttl_full_db, Xow_page_cache_itm.Missing);
-				rv = null;
-			}
-		}
-		return rv;
 	}
 	public void Free_mem(boolean clear_permanent_itms) {
 		synchronized (thread_lock) {	// LOCK:app-level; DATE:2016-07-06
 			if (clear_permanent_itms) {
 				cache.Clear();
 			}
-			else {
-				// gather non-permanent items
-				int len = cache.Count();
-				for (int i = 0; i < len; i++) {
-					Xow_page_cache_itm itm = (Xow_page_cache_itm)cache.Get_at(i);
-					if (!itm.Cache_permanently())
-						deleted.Add(itm);
-				}
-
-				// remove non-permanent items
-				len = deleted.Len();
-				for (int i = 0; i < len; i++) {
-					Xow_page_cache_itm itm = (Xow_page_cache_itm)deleted.Get_at(i);
-					if (itm.Ttl() != null)	// missing is null
-						cache.Del(itm.Ttl().Full_db());
-				}
-				deleted.Clear();
-			}
 		}
 	}
+	private Xow_page_cache_itm Load_page(Xoa_ttl ttl) {
+		// vars
+		Xow_page_cache_itm rv = null;
+		Xoa_ttl page_ttl = ttl;
+		boolean page_exists = false;
+		byte[] page_text = null;
+		byte[] page_redirect_from = null;
+		int page_id = -1;
+
+		// load_page if load_wkr exists; only for Xop_mediawiki_mgr
+		// gplx.core.consoles.Console_adp__sys.Instance.Write_str("page_cache:" + String_.new_u8(ttl_full_db));
+		if (load_wkr != null) {
+			page_text = load_wkr.Get_page_or_null(ttl.Full_db());
+			page_exists = page_text != null;
+		}
+
+		// load_page in other cases
+		if (page_text == null) {
+			Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(ttl);	// NOTE: do not call Db_mgr.Load_page; need to handle redirects
+			if (    page.Db().Page().Exists()                   // page exists
+				||  page.Redirect_trail().Itms__len() > 0 ) {   // page redirects to missing page; note that page.Missing == true and page.Redirected_src() != null; PAGE:en.w:Shah_Rukh_Khan; DATE:2016-05-02
+				page_exists = true; // NOTE: page.Db().Page().Exists() will be false if page redirects to missing page; PAGE:en.w:Shah_Rukh_Khan; DATE:2019-06-16
+				page_ttl = page.Ttl();
+				page_text = page.Db().Text().Text_bry();
+				page_redirect_from = page.Redirect_trail().Itms__get_wtxt_at_0th_or_null();
+				page_id = page.Db().Page().Id();
+			}
+		}
+
+		// create item
+		if (page_exists) {
+			rv = new Xow_page_cache_itm(false, page_id, page_ttl, page_text, page_redirect_from);
+			Add_itm(ttl.Full_db_as_str(), rv);
+		}
+		else {
+			rv = null;
+			Add_itm(ttl.Full_db_as_str(), Xow_page_cache_itm.Missing);
+		}
+		return rv;
+	}
+	public String To_str() {
+		return String_.Format("cache_pct:{0} cache_misses:{1} cache_evicts:{2} cache_tries:{3} cache_size:{4}", Decimal_adp_.divide_(cache_misses * 100, cache_tries).To_str("0.00"), cache_misses, cache.Evicts(), cache_tries, cache.Cur());
+	}
 }

400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache_itm.java

@@ -16,9 +16,11 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
 package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
 import gplx.xowa.wikis.data.tbls.*;
 public class Xow_page_cache_itm implements Xowd_text_bry_owner {
+	private long cache_len;
 	public Xow_page_cache_itm(boolean cache_permanently, int page_id, Xoa_ttl ttl, byte[] wtxt__direct, byte[] wtxt__redirect) {
 		this.cache_permanently = cache_permanently;
-		this.page_id = page_id; this.ttl = ttl; this.wtxt__direct = wtxt__direct; this.wtxt__redirect = wtxt__redirect;
+		this.page_id = page_id; this.ttl = ttl; this.wtxt__redirect = wtxt__redirect;
+		Set_text_bry_by_db(wtxt__direct);
 	}
 	public Xoa_ttl Ttl() {return ttl;} private Xoa_ttl ttl;
 	public byte[] Wtxt__direct()	{return wtxt__direct;} private byte[] wtxt__direct;
@@ -27,16 +29,20 @@ public class Xow_page_cache_itm implements Xowd_text_bry_owner {
 		return wtxt__redirect == null ? wtxt__direct : wtxt__redirect;
 	}
 	public boolean   Cache_permanently() {return cache_permanently;} private final    boolean cache_permanently;
+	public long Cache_len() {return cache_len;}
 
 	// used by xomp
 	public int Page_id() {return page_id;} private int page_id;
 	public int Redirect_id() {return redirect_id;} private int redirect_id;
-	public void Set_text_bry_by_db(byte[] v) {this.wtxt__direct = v;}
+	public void Set_text_bry_by_db(byte[] v) {
+		this.wtxt__direct = v;
+		this.cache_len = wtxt__direct == null ? 0 : wtxt__direct.length;
+	}
 	public void Redirect_id_(int v) {this.redirect_id = v;}
 	public void Set_redirect(Xoa_ttl ttl, byte[] trg_wtxt) {
 		this.ttl = ttl;
+		Set_text_bry_by_db(trg_wtxt);
 		this.wtxt__redirect = wtxt__direct;
-		this.wtxt__direct = trg_wtxt;
 	}
 
 	public static final    Xow_page_cache_itm Null = null;

400_xowa/src/gplx/xowa/xtns/lst/Lst_pfunc_itm.java

@@ -35,7 +35,8 @@ public class Lst_pfunc_itm {
 			// get sub_src;
 			Xop_ctx sub_ctx = Xop_ctx.New__top(wiki).Ref_ignore_(true);
 			sub_ctx.Page().Ttl_(ctx.Page().Ttl());	// NOTE: must set ttl on page, else test fails;
-			byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(ttl); if (sub_src == null) return null; // {{#lst:missing}} -> ""
+			byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl); 
+			if (sub_src == null) return null; // {{#lst:missing}} -> ""
 			
 			// parse page; note adding to stack to prevent circular recursions
 			if (!wiki.Parser_mgr().Tmpl_stack_add(ttl.Full_db())) return null;
@@ -75,7 +76,8 @@ public class Lst_pfunc_itm {
 			// get sub_ctx: note new ctx is needed b/c sub_page objects must not get added to owner_page; for example, references / hdrs / lnki.files
 			Xop_ctx sub_ctx = Xop_ctx.New__top(wiki).Ref_ignore_(true);
 			sub_ctx.Page().Ttl_(ctx.Page().Ttl());	// NOTE: must set ttl on page, else test fails;
-			byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(ttl); if (sub_src == null) return null; // {{#lst:missing}} -> ""
+			byte[] sub_src = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl);
+			if (sub_src == null) return null; // {{#lst:missing}} -> ""
 
 			// parse sub_src; note adding to page's stack to prevent circular recursions
 			if (!wiki.Parser_mgr().Tmpl_stack_add(ttl.Full_db())) return null;

400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_index_parser.java

@@ -19,7 +19,7 @@ import gplx.xowa.wikis.nss.*;
 import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.parsers.xndes.*; import gplx.xowa.parsers.lnkis.*; import gplx.xowa.parsers.tmpls.*;
 class Pp_index_parser {
 	public static Pp_index_page Parse(Xowe_wiki wiki, Xop_ctx ctx, Xoa_ttl index_ttl, int ns_page_id) {
-		byte[] src = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(index_ttl);
+		byte[] src = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(index_ttl);
 		if (src == null) return Pp_index_page.Null;
 		Xop_parser sub_parser = Xop_parser.new_(wiki, wiki.Parser_mgr().Main().Tmpl_lxr_mgr(), wiki.Parser_mgr().Main().Wtxt_lxr_mgr());
 		Xop_ctx sub_ctx = Xop_ctx.New__sub__reuse_page(ctx);

400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func.java

@@ -51,7 +51,7 @@ public class Scrib_invoke_func extends Pf_func_base {
 		if (mod == null) {
 			Xow_ns module_ns = wiki.Ns_mgr().Ids_get_or_null(Xow_ns_.Tid__module);
 			Xoa_ttl mod_ttl = Xoa_ttl.Parse(wiki, Bry_.Add(module_ns.Name_db_w_colon(), mod_name));
-			mod_raw = wiki.Cache_mgr().Page_cache().Get_or_load_as_src(mod_ttl);
+			mod_raw = wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(mod_ttl);
 			if (mod_raw == null) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:missing_mod}}"
 		}
 		else

400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_mw.java

@@ -97,7 +97,7 @@ public class Scrib_lib_mw implements Scrib_lib {
 			return rslt.Init_obj(core.Interpreter().LoadString("@" + mod_name + ".lua", mod_code));
 		Xoa_ttl ttl = Xoa_ttl.Parse(cur_wiki, Bry_.new_u8(mod_name));// NOTE: should have Module: prefix
 		if (ttl == null) return rslt.Init_ary_empty();
-		byte[] page_db = cur_wiki.Cache_mgr().Page_cache().Get_or_load_as_src(ttl);
+		byte[] page_db = cur_wiki.Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl);
 		if (page_db == null) return rslt.Init_ary_empty();
 		Scrib_lua_mod mod = new Scrib_lua_mod(core, mod_name);
 		return rslt.Init_obj(mod.LoadString(String_.new_u8(page_db)));
@@ -316,25 +316,28 @@ public class Scrib_lib_mw implements Scrib_lib {
 		if (!ttl.ForceLiteralLink() && ttl.Ns().Id_is_main())	// title is not literal and is not prefixed with Template; parse again as template; EX: ":A" and "Template:A" are fine; "A" is parsed again as "Template:A"
 			ttl = Xoa_ttl.Parse(cur_wiki, Bry_.Add(cur_wiki.Ns_mgr().Ns_template().Name_db_w_colon(), ttl_bry));	// parse again, but add "Template:"
 		Keyval[] args_ary = args.Pull_kv_ary_safe(2);
-		// BLOCK.bgn:Xot_invk_tkn.Transclude; cannot reuse b/c Transclude needs invk_tkn, and invk_tkn is manufactured late; DATE:2014-01-02
+
 		byte[] sub_src = null;
-		if (ttl.Ns().Id_is_tmpl()) {				// ttl is template; check tmpl_regy first before going to data_mgr
+		// ttl is template; check tmpl_regy first before going to data_mgr
+		if (ttl.Ns().Id_is_tmpl()) {
 			Xot_defn_tmpl tmpl = (Xot_defn_tmpl)core.Wiki().Cache_mgr().Defn_cache().Get_by_key(ttl.Page_db());
-			if (tmpl != null) sub_src = tmpl.Data_raw();
+			if (tmpl != null)
+				sub_src = tmpl.Data_raw();
 		}
-		if (sub_src == null)						// ttl is not in template cache, or is a ttl in non-Template ns; load title
+
-			sub_src = core.Wiki().Cache_mgr().Page_cache().Get_or_load_as_src(ttl);
+		// ttl is not in template cache; load title
-		if (sub_src !=  null) {
+		if (sub_src == null)
-			Xot_invk_mock sub_frame = Xot_invk_mock.new_(core.Frame_current().Defn_tid(), 0, ttl.Full_txt_w_ttl_case(), args_ary);	// NOTE: (1) must have ns (Full); (2) must be txt (space, not underscore); EX:Template:Location map+; DATE:2014-09-21
+			sub_src = core.Wiki().Cache_mgr().Page_cache().Get_src_else_load_or_null(ttl);
-			Xot_defn_tmpl transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), ttl.Ns(), ttl.Page_db(), sub_src);
+
-			Bry_bfr sub_bfr = cur_wiki.Utl__bfr_mkr().Get_k004();
+		// ttl does not exist; fail
-			transclude_tmpl.Tmpl_evaluate(ctx, sub_frame, sub_bfr);
+		if (sub_src == null)
-			return rslt.Init_obj(sub_bfr.To_str_and_rls());
-		}
-		else {
 			return rslt.Init_fail("expandTemplate: template \"" + ttl_str + "\" does not exist");	// NOTE: must return error if template is missing; PAGE:en.w:Flag_of_Greenland DATE:2016-05-02
-		}
+
-		// BLOCK.end:Xot_invk_tkn.Transclude
+		Xot_invk_mock sub_frame = Xot_invk_mock.new_(core.Frame_current().Defn_tid(), 0, ttl.Full_txt_w_ttl_case(), args_ary);	// NOTE: (1) must have ns (Full); (2) must be txt (space, not underscore); EX:Template:Location map+; DATE:2014-09-21
+		Xot_defn_tmpl transclude_tmpl = ctx.Wiki().Parser_mgr().Main().Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), ttl.Ns(), ttl.Page_db(), sub_src);
+		Bry_bfr sub_bfr = cur_wiki.Utl__bfr_mkr().Get_k004();
+		transclude_tmpl.Tmpl_evaluate(ctx, sub_frame, sub_bfr);
+		return rslt.Init_obj(sub_bfr.To_str_and_rls());
 	}
 
 	public boolean IncrementExpensiveFunctionCount(Scrib_proc_args args, Scrib_proc_rslt rslt) {return rslt.Init_obj(Keyval_.Ary_empty);}	// NOTE: for now, always return null (XOWA does not care about expensive parser functions)

400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title.java

@@ -195,7 +195,7 @@ public class Scrib_lib_title implements Scrib_lib {
 	}
 	public static byte[] GetContentInternal(Scrib_core core, Xowe_wiki wiki, byte[] ttl_bry) {
 		Xoa_ttl ttl = wiki.Ttl_parse(ttl_bry); if (ttl == null) return null;
-		Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm_2(ttl);
+		Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
 		byte[] rv = null;
 		if (page_itm != null) {
 			byte[] redirected_src = page_itm.Wtxt__redirect();

400_xowa/src/gplx/xowa/xtns/template_styles/Template_styles_nde.java

@@ -49,7 +49,7 @@ public class Template_styles_nde implements Xox_xnde, Mwh_atr_itm_owner2 {
 		}
 
 		// get page
-		Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm_2(css_ttl);
+		Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(css_ttl);
 		if (page_itm != null) {
 			css_src = page_itm.Wtxt__direct();
 			css_page_id = page_itm.Page_id();

400_xowa/src/gplx/xowa/xtns/wikias/Tabview_xnde.java

@@ -120,7 +120,7 @@ class Tabview_tab_itm {
 					if (Bry_.Has_at_bgn(page_ttl_bry, Byte_ascii.Angle_bgn) || Bry_.Has_at_end(page_ttl_bry, Byte_ascii.Angle_end)) return null;
 					Xoa_ttl page_ttl = wiki.Ttl_parse(page_ttl_bry);
 					if (page_ttl == null) return null;
-					gplx.xowa.wikis.caches.Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm(page_ttl);
+					gplx.xowa.wikis.caches.Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(page_ttl);
 					if (page_itm == null) return null;
 					page_body = page_itm.Wtxt__redirect_or_direct();
 					page_body = Xop_parser_.Parse_text_to_html(wiki, ctx, ctx.Page(), page_body, false);