Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Mass_parse: Fix memory leak / slow performance due to over-zealous page caching

Browse Source
This commit is contained in:
gnosygnu
2016-12-12 09:25:06 -05:00
parent 8f68854513
commit 5e159233ab
10 changed files with 48 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
400_xowa/src/gplx/xowa/wikis/Xoae_wiki_mgr.java
View File

@@ -86,7 +86,7 @@ public class Xoae_wiki_mgr implements Xoa_wiki_mgr, Gfo_invk {
Xowe_wiki wiki = (Xowe_wiki)list.Get_at(i);
// wiki.Defn_cache().ReduceCache();
if (clear_ctx) wiki.Parser_mgr().Ctx().Clear_all(); // NOTE: clear_ctx will reset toc and refs
wiki.Cache_mgr().Page_cache().Free_mem_all();
wiki.Cache_mgr().Page_cache().Free_mem(true);
wiki.Cache_mgr().Tmpl_result_cache().Clear();
}
}

9
400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java
View File

@@ -50,14 +50,17 @@ public class Xow_cache_mgr {
}
return scrib_lang_names;
}
public void Free_mem_all() {this.Free_mem_all(Bool_.Y);}
public void Free_mem_all(boolean free_page_cache) {
public void Free_mem_all() {this.Free_mem(Bool_.Y);}
public void Free_mem(boolean free_page_cache) {
if (free_page_cache) {
commons_cache.Clear();
ifexist_cache.Clear();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); // moved from ctx.Clear(); DATE:2016-07-21
page_cache.Free_mem(true);
}
else {
page_cache.Free_mem(false);
}
page_cache.Free_mem_all();
wiki.Ctg__catpage_mgr().Free_mem_all();
tmpl_result_cache.Clear();
defn_cache.Free_mem_all();

38
400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java
View File

@@ -18,7 +18,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
public class Xow_page_cache {
private final Xowe_wiki wiki;
private final Hash_adp_bry cache = Hash_adp_bry.cs(); // NOTE: wiki titles are not case-sensitive when ns is "1st-letter" (EX: w:earth an w:Earth); in these cases, two entries will be stored
private final Ordered_hash cache = Ordered_hash_.New_bry(); // NOTE: wiki titles are not case-sensitive when ns is "1st-letter" (EX: w:earth an w:Earth); in these cases, two entries will be stored
private final List_adp deleted = List_adp_.New();
private Xow_page_cache_wkr load_wkr;
public Xow_page_cache(Xowe_wiki wiki) {this.wiki = wiki;}
public Xow_page_cache_wkr Load_wkr() {return load_wkr;}
@@ -32,7 +33,7 @@ public class Xow_page_cache {
}
public Xow_page_cache_itm Get_or_load_as_itm(Xoa_ttl ttl) {
byte[] ttl_full_db = ttl.Full_db();
Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by_bry(ttl_full_db);
Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by(ttl_full_db);
if (rv == Xow_page_cache_itm.Missing) {
return null;
}
@@ -59,14 +60,14 @@ public class Xow_page_cache {
page_redirect_from = page.Redirect_trail().Itms__get_wtxt_at_0th_or_null();
}
if (page_exists) {
rv = new Xow_page_cache_itm(page_ttl, page_text, page_redirect_from);
rv = new Xow_page_cache_itm(false, page_ttl, page_text, page_redirect_from);
synchronized (this) { // LOCK:high-usage;DATE:2016-07-14
cache.Add_bry_obj(ttl_full_db, rv);
cache.Add(ttl_full_db, rv);
}
}
else {
synchronized (this) { // LOCK:high-usage;DATE:2016-07-14
cache.Add_bry_obj(ttl_full_db, Xow_page_cache_itm.Missing);
cache.Add(ttl_full_db, Xow_page_cache_itm.Missing);
rv = null;
}
}
@@ -74,29 +75,44 @@ public class Xow_page_cache {
}
public Xow_page_cache_itm Get_or_load_as_itm_2(Xoa_ttl ttl) { // NOTE: same as Get_or_load_as_itm, but handles redirects to missing pages; DATE:2016-05-02
byte[] ttl_full_db = ttl.Full_db();
Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by_bry(ttl_full_db);
Xow_page_cache_itm rv = (Xow_page_cache_itm)cache.Get_by(ttl_full_db);
if (rv == Xow_page_cache_itm.Missing) return null;
else if (rv == null) {
Xoae_page page = wiki.Data_mgr().Load_page_by_ttl(ttl); // NOTE: do not call Db_mgr.Load_page; need to handle redirects
if ( page.Db().Page().Exists() // page exists
|| page.Redirect_trail().Itms__len() > 0 ) { // page redirects to missing page; note that page.Missing == true and page.Redirected_src() != null; PAGE: en.w:Shah_Rukh_Khan; DATE:2016-05-02
rv = new Xow_page_cache_itm(page.Ttl(), page.Db().Text().Text_bry(), page.Redirect_trail().Itms__get_wtxt_at_0th_or_null());
rv = new Xow_page_cache_itm(false, page.Ttl(), page.Db().Text().Text_bry(), page.Redirect_trail().Itms__get_wtxt_at_0th_or_null());
synchronized (this) { // LOCK:high-usage;DATE:2016-07-14
cache.Add_bry_obj(ttl_full_db, rv);
cache.Add(ttl_full_db, rv);
}
}
else {
synchronized (this) { // LOCK:high-usage;DATE:2016-07-14
cache.Add_bry_obj(ttl_full_db, Xow_page_cache_itm.Missing);
cache.Add(ttl_full_db, Xow_page_cache_itm.Missing);
rv = null;
}
}
}
return rv;
}
public void Free_mem_all() {
public void Free_mem(boolean clear_permanent_itms) {
synchronized (this) { // LOCK:app-level; DATE:2016-07-06
cache.Clear();
if (clear_permanent_itms)
cache.Clear();
else {
int len = cache.Count();
for (int i = 0; i < len; i++) {
Xow_page_cache_itm itm = (Xow_page_cache_itm)cache.Get_at(i);
if (!itm.Cache_permanently())
deleted.Add(itm);
}
len = deleted.Len();
for (int i = 0; i < len; i++) {
Xow_page_cache_itm itm = (Xow_page_cache_itm)cache.Get_at(i);
cache.Del(itm.Ttl().Full_db());
}
deleted.Clear();
}
}
}
}

6
400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache_itm.java
View File

@@ -18,7 +18,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
import gplx.xowa.wikis.data.tbls.*;
public class Xow_page_cache_itm implements Xowd_text_bry_owner {
public Xow_page_cache_itm(Xoa_ttl ttl, byte[] wtxt__direct, byte[] wtxt__redirect) {
public Xow_page_cache_itm(boolean cache_permanently, Xoa_ttl ttl, byte[] wtxt__direct, byte[] wtxt__redirect) {
this.cache_permanently = cache_permanently;
this.ttl = ttl; this.wtxt__direct = wtxt__direct; this.wtxt__redirect = wtxt__redirect;
}
public Xoa_ttl Ttl() {return ttl;} private Xoa_ttl ttl;
@@ -27,6 +28,7 @@ public class Xow_page_cache_itm implements Xowd_text_bry_owner {
public byte[] Wtxt__redirect_or_direct() {
return wtxt__redirect == null ? wtxt__direct : wtxt__redirect;
}
public boolean Cache_permanently() {return cache_permanently;} private final boolean cache_permanently;
// used by xomp
public int Page_id() {return page_id;} private int page_id;
@@ -40,5 +42,5 @@ public class Xow_page_cache_itm implements Xowd_text_bry_owner {
}
public static final Xow_page_cache_itm Null = null;
public static final Xow_page_cache_itm Missing = new Xow_page_cache_itm(null, null, null);
public static final Xow_page_cache_itm Missing = new Xow_page_cache_itm(false, null, null, null);
}