Mass_parse: Fix memory leak / slow performance due to over-zealous page caching

400_xowa/src/gplx/xowa/Xoa_app_.java

@@ -32,7 +32,7 @@ public class Xoa_app_ {
 		}
 	}
 	public static final String		Name			= "xowa";
-	public static final String		Version			= "3.12.1.3";
+	public static final String		Version			= "3.12.1.4";
 	public static String	Build_date		= "2012-12-30 00:00:00";
 	public static String	Op_sys_str;
 	public static String	User_agent		= "";

400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_tmpl_cache_bldr.java

@@ -50,7 +50,7 @@ public class Xomp_tmpl_cache_bldr {
 				// add to text_db_loader
 				int page_id = rdr.Read_int("page_id"); 
 				int page_redirect_id = rdr.Read_int("page_redirect_id");
-				Xow_page_cache_itm itm = new Xow_page_cache_itm(page_ttl, null, null);	// NOTE: "null, null;" b/c GetContent in Scrib_title checks specifically for null, not empty String; DATE:2016-10-19
+				Xow_page_cache_itm itm = new Xow_page_cache_itm(true, page_ttl, null, null);	// NOTE: "null, null;" b/c GetContent in Scrib_title checks specifically for null, not empty String; DATE:2016-10-19
 				itm.Set_page_ids(page_id, page_redirect_id);
 				text_db_loader.Add(rdr.Read_int("page_text_db_id"), itm);
 				cache.Add(page_ttl.Full_db(), itm);

400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java

@@ -106,8 +106,9 @@ public class Xomp_parse_wkr implements Gfo_invk {
 				Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry());
 				// if ns changed and prv_ns is main
 				if (cur_ns != prv_ns) {
-					if (prv_ns == gplx.xowa.wikis.nss.Xow_ns_.Tid__main)
+					// COMMENTED: cache now supports 2 levels of permanence; no need to free entire page cache to clear pages added by ifexists and the like; DATE:2016-12-12
-						wiki.Cache_mgr().Free_mem_all(Bool_.Y);	// NOTE: this clears all caches, include imglinks
+					// if (prv_ns == gplx.xowa.wikis.nss.Xow_ns_.Tid__main)
+					//	wiki.Cache_mgr().Free_mem(Bool_.Y);	// NOTE: clears all caches, include imglinks; only Main will have benefit of 
 					prv_ns = cur_ns;
 				}
 				Xoae_page wpg = Xoae_page.New(wiki, ttl);
@@ -137,7 +138,7 @@ public class Xomp_parse_wkr implements Gfo_invk {
 				if (wiki.Cache_mgr().Tmpl_result_cache().Count() > 50000) 
 					wiki.Cache_mgr().Tmpl_result_cache().Clear();
 				if (done_count % cleanup_interval == 0) {
-					wiki.Cache_mgr().Free_mem_all(Bool_.N);
+					wiki.Cache_mgr().Free_mem(Bool_.N);
 					wiki.Parser_mgr().Scrib().Core_term();
 					wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
 				}

400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java

@@ -31,7 +31,7 @@ public class Xop_mediawiki_wkr {
 	}
 	public void Free_memory() {
 		wiki.Cache_mgr().Tmpl_result_cache().Clear();
-		wiki.Cache_mgr().Free_mem_all(Bool_.N);
+		wiki.Cache_mgr().Free_mem(Bool_.N);
 		wiki.Parser_mgr().Scrib().Core_term();
 		wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
 	}

400_xowa/src/gplx/xowa/wikis/Xoae_wiki_mgr.java

@@ -86,7 +86,7 @@ public class Xoae_wiki_mgr implements Xoa_wiki_mgr, Gfo_invk {
 			Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i);
 //				wiki.Defn_cache().ReduceCache();
 			if (clear_ctx) wiki.Parser_mgr().Ctx().Clear_all();	// NOTE: clear_ctx will reset toc and refs
-			wiki.Cache_mgr().Page_cache().Free_mem_all();
+			wiki.Cache_mgr().Page_cache().Free_mem(true);
 			wiki.Cache_mgr().Tmpl_result_cache().Clear();
 		}
 	}

400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java

@@ -50,14 +50,17 @@ public class Xow_cache_mgr {
 		}
 		return scrib_lang_names;
 	}
-	public void Free_mem_all() {this.Free_mem_all(Bool_.Y);}
+	public void Free_mem_all() {this.Free_mem(Bool_.Y);}
-	public void Free_mem_all(boolean free_page_cache) {
+	public void Free_mem(boolean free_page_cache) {
 		if (free_page_cache) {
 			commons_cache.Clear();
 			ifexist_cache.Clear();
 			wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();	// moved from ctx.Clear(); DATE:2016-07-21
+			page_cache.Free_mem(true);
+		}
+		else {
+			page_cache.Free_mem(false);
 		}
-		page_cache.Free_mem_all();
 		wiki.Ctg__catpage_mgr().Free_mem_all();
 		tmpl_result_cache.Clear();
 		defn_cache.Free_mem_all();

400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java

@@ -18,7 +18,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
 public class Xow_page_cache {
 	private final    Xowe_wiki wiki;
-	private final    Hash_adp_bry cache = Hash_adp_bry.cs();	// NOTE: wiki titles are not case-sensitive when ns is "1st-letter" (EX: w:earth an w:Earth); in these cases, two entries will be stored
+	private final    Ordered_hash cache = Ordered_hash_.New_bry();	// NOTE: wiki titles are not case-sensitive when ns is "1st-letter" (EX: w:earth an w:Earth); in these cases, two entries will be stored
+	private final    List_adp deleted = List_adp_.New();
 	private Xow_page_cache_wkr load_wkr;
 	public Xow_page_cache(Xowe_wiki wiki) {this.wiki = wiki;}
 	public Xow_page_cache_wkr Load_wkr() {return load_wkr;}
@@ -32,7 +33,7 @@ public class Xow_page_cache {
 	}
 	public Xow_page_cache_itm Get_or_load_as_itm(Xoa_ttl ttl) {
 		byte[] ttl_full_db = ttl.Full_db();
-		Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by_bry(ttl_full_db);
+		Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by(ttl_full_db);
 		if		(rv == Xow_page_cache_itm.Missing) {
 			return null;
 		}
@@ -59,14 +60,14 @@ public class Xow_page_cache {
 			page_redirect_from = page.Redirect_trail().Itms__get_wtxt_at_0th_or_null();
 		}
 		if (page_exists) {
-			rv = new Xow_page_cache_itm(page_ttl, page_text, page_redirect_from);
+			rv = new Xow_page_cache_itm(false, page_ttl, page_text, page_redirect_from);
 			synchronized (this) {	// LOCK:high-usage;DATE:2016-07-14
-				cache.Add_bry_obj(ttl_full_db, rv);
+				cache.Add(ttl_full_db, rv);
 			}
 		}
 		else {
 			synchronized (this) {	// LOCK:high-usage;DATE:2016-07-14
-				cache.Add_bry_obj(ttl_full_db, Xow_page_cache_itm.Missing);
+				cache.Add(ttl_full_db, Xow_page_cache_itm.Missing);
 				rv = null;
 			}
 		}
@@ -74,29 +75,44 @@ public class Xow_page_cache {
 	}
 	public Xow_page_cache_itm Get_or_load_as_itm_2(Xoa_ttl ttl) {	// NOTE: same as Get_or_load_as_itm, but handles redirects to missing pages; DATE:2016-05-02
 		byte[] ttl_full_db = ttl.Full_db();
-		Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by_bry(ttl_full_db);
+		Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by(ttl_full_db);
 		if		(rv == Xow_page_cache_itm.Missing) return null;
 		else if (rv == null) {
 			Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(ttl);	// NOTE: do not call Db_mgr.Load_page; need to handle redirects
 			if (	page.Db().Page().Exists()				// page exists
 				||	page.Redirect_trail().Itms__len() > 0 ) {		// page redirects to missing page; note that page.Missing == true and page.Redirected_src() != null; PAGE: en.w:Shah_Rukh_Khan; DATE:2016-05-02
-				rv = new Xow_page_cache_itm(page.Ttl(), page.Db().Text().Text_bry(), page.Redirect_trail().Itms__get_wtxt_at_0th_or_null());
+				rv = new Xow_page_cache_itm(false, page.Ttl(), page.Db().Text().Text_bry(), page.Redirect_trail().Itms__get_wtxt_at_0th_or_null());
 				synchronized (this) {	// LOCK:high-usage;DATE:2016-07-14
-					cache.Add_bry_obj(ttl_full_db, rv);
+					cache.Add(ttl_full_db, rv);
 				}
 			}
 			else {
 				synchronized (this) {	// LOCK:high-usage;DATE:2016-07-14
-					cache.Add_bry_obj(ttl_full_db, Xow_page_cache_itm.Missing);
+					cache.Add(ttl_full_db, Xow_page_cache_itm.Missing);
 					rv = null;
 				}
 			}
 		}
 		return rv;
 	}
-	public void Free_mem_all() {
+	public void Free_mem(boolean clear_permanent_itms) {
 		synchronized (this) {	// LOCK:app-level; DATE:2016-07-06
+			if (clear_permanent_itms)
 				cache.Clear();
+			else {
+				int len = cache.Count();
+				for (int i = 0; i < len; i++) {
+					Xow_page_cache_itm itm = (Xow_page_cache_itm)cache.Get_at(i);
+					if (!itm.Cache_permanently())
+						deleted.Add(itm);
+				}
+				len = deleted.Len();
+				for (int i = 0; i < len; i++) {
+					Xow_page_cache_itm itm = (Xow_page_cache_itm)cache.Get_at(i);
+					cache.Del(itm.Ttl().Full_db());
+				}
+				deleted.Clear();
+			}
 		}
 	}
 }

400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache_itm.java

@@ -18,7 +18,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
 import gplx.xowa.wikis.data.tbls.*;
 public class Xow_page_cache_itm implements Xowd_text_bry_owner {
-	public Xow_page_cache_itm(Xoa_ttl ttl, byte[] wtxt__direct, byte[] wtxt__redirect) {
+	public Xow_page_cache_itm(boolean cache_permanently, Xoa_ttl ttl, byte[] wtxt__direct, byte[] wtxt__redirect) {
+		this.cache_permanently = cache_permanently;
 		this.ttl = ttl; this.wtxt__direct = wtxt__direct; this.wtxt__redirect = wtxt__redirect;
 	}
 	public Xoa_ttl Ttl() {return ttl;} private Xoa_ttl ttl;
@@ -27,6 +28,7 @@ public class Xow_page_cache_itm implements Xowd_text_bry_owner {
 	public byte[] Wtxt__redirect_or_direct() {
 		return wtxt__redirect == null ? wtxt__direct : wtxt__redirect;
 	}
+	public boolean   Cache_permanently() {return cache_permanently;} private final    boolean cache_permanently;
 
 	// used by xomp
 	public int Page_id() {return page_id;} private int page_id;
@@ -40,5 +42,5 @@ public class Xow_page_cache_itm implements Xowd_text_bry_owner {
 	}
 
 	public static final    Xow_page_cache_itm Null = null;
-	public static final    Xow_page_cache_itm Missing = new Xow_page_cache_itm(null, null, null);
+	public static final    Xow_page_cache_itm Missing = new Xow_page_cache_itm(false, null, null, null);
 }

400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_hdr_tst.java

@@ -22,7 +22,7 @@ public class Pp_pages_nde_hdr_tst {
 	@Before public void Init() {
 		Io_mgr.Instance.InitEngine_mem();
 		fxt.Wiki().Xtn_mgr().Xtn_proofread().Enabled_y_();
-		fxt.Wiki().Cache_mgr().Page_cache().Free_mem_all();
+		fxt.Wiki().Cache_mgr().Page_cache().Free_mem(true);
 		fxt.Wiki().Db_mgr().Load_mgr().Clear(); // must clear; otherwise fails b/c files get deleted, but wiki.data_mgr caches the Xowd_regy_mgr (the .reg file) in memory;
 		fxt.Wiki().Ns_mgr().Add_new(Xowc_xtn_pages.Ns_page_id_default, "Page").Add_new(Xowc_xtn_pages.Ns_index_id_default, "Index").Init();
 		fxt.Init_page_create("MediaWiki:Proofreadpage_header_template", String_.Concat

400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title_tst.java

@@ -87,7 +87,7 @@ public class Scrib_lib_title_tst {
 	@Test   public void GetContent() {
 		fxt.Test__proc__objs__flat(lib, Scrib_lib_title.Invk_getContent, Object_.Ary("A")										, Scrib_invoke_func_fxt.Null_rslt);
 
-		fxt.Parser_fxt().Ctx().Wiki().Cache_mgr().Page_cache().Free_mem_all();
+		fxt.Parser_fxt().Ctx().Wiki().Cache_mgr().Page_cache().Free_mem(true);
 		fxt.Parser_fxt().Init_page_create("A", "test");
 		fxt.Test__proc__objs__flat(lib, Scrib_lib_title.Invk_getContent, Object_.Ary("A")										, "test");
 	}