From ad646efb5ca4382621a2df32b1f2a2b4e0f8d040 Mon Sep 17 00:00:00 2001 From: gnosygnu Date: Sat, 7 Jan 2017 10:57:41 -0500 Subject: [PATCH] Mass_parse: Clear ns_loaded cache; Add different cache clearing methods --- 400_xowa/src/gplx/xowa/Xoa_app_.java | 2 +- 400_xowa/src/gplx/xowa/Xowe_wiki_.java | 2 +- .../parses/wkrs/Xomp_parse_wkr.java | 5 ++-- .../parsers/mediawikis/Xop_mediawiki_wkr.java | 2 +- .../xowa/guis/views/Xog_tab_itm_edit_mgr.java | 2 +- .../gplx/xowa/wikis/caches/Xow_cache_mgr.java | 29 ++++++++++++------- .../xowa/wikis/caches/Xow_ifexist_cache.java | 5 +++- .../gplx/xowa/xtns/lst/Lst_pfunc_lst_tst.java | 2 +- .../proofreadPage/Pp_pages_nde_basic_tst.java | 4 +-- .../proofreadPage/Pp_pages_nde_index_tst.java | 10 +++---- .../Pp_pages_nde_recursion_tst.java | 2 +- 11 files changed, 38 insertions(+), 27 deletions(-) diff --git a/400_xowa/src/gplx/xowa/Xoa_app_.java b/400_xowa/src/gplx/xowa/Xoa_app_.java index e7675c340..682f53f54 100644 --- a/400_xowa/src/gplx/xowa/Xoa_app_.java +++ b/400_xowa/src/gplx/xowa/Xoa_app_.java @@ -33,7 +33,7 @@ public class Xoa_app_ { } public static final String Name = "xowa"; public static final int Version_id = 513; - public static final String Version = "4.1.1.1701"; // RELEASE:2017-01-03 20:30 + public static final String Version = "4.0.1.1701"; // RELEASE:2017-01-03 20:30 public static String Build_date = "2012-12-30 00:00:00"; public static String Build_date_fmt = "yyyy-MM-dd HH:mm:ss"; public static String Op_sys_str; diff --git a/400_xowa/src/gplx/xowa/Xowe_wiki_.java b/400_xowa/src/gplx/xowa/Xowe_wiki_.java index db131997d..8e4bc40ed 100644 --- a/400_xowa/src/gplx/xowa/Xowe_wiki_.java +++ b/400_xowa/src/gplx/xowa/Xowe_wiki_.java @@ -34,7 +34,7 @@ public class Xowe_wiki_ { } public static void Rls_mem(Xowe_wiki wiki, boolean clear_ctx) { wiki.Appe().Free_mem(clear_ctx); - wiki.Cache_mgr().Free_mem_all(); + wiki.Cache_mgr().Free_mem__all(); wiki.Parser_mgr().Scrib().Core_term(); } } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java index fc3771185..721699523 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java @@ -110,9 +110,8 @@ public class Xomp_parse_wkr implements Gfo_invk { Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry()); // if ns changed and prv_ns is main if (cur_ns != prv_ns) { - // TOMBSTONE: do not comment free_mem; will cause OutOfMem error on en.w in 25th hour; will need to reinvestigate cache generations later; DATE:2016-12-27 if (prv_ns == gplx.xowa.wikis.nss.Xow_ns_.Tid__main) - wiki.Cache_mgr().Free_mem(Bool_.Y); // NOTE: clears all caches, include imglinks; only Main will have benefit of + wiki.Cache_mgr().Free_mem__wbase(); // NOTE: clears page and wbase cache only; needed else OutOfMemory error for en.w in 25th hour; DATE:2017-01-07 prv_ns = cur_ns; } Xoae_page wpg = Xoae_page.New(wiki, ttl); @@ -142,7 +141,7 @@ public class Xomp_parse_wkr implements Gfo_invk { if (wiki.Cache_mgr().Tmpl_result_cache().Count() > 50000) wiki.Cache_mgr().Tmpl_result_cache().Clear(); if (done_count % cleanup_interval == 0) { - wiki.Cache_mgr().Free_mem(Bool_.N); + wiki.Cache_mgr().Free_mem__page(); wiki.Parser_mgr().Scrib().Core_term(); wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); } diff --git a/400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java b/400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java index b3528a3d1..f5ce6f581 100644 --- a/400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java +++ b/400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java @@ -31,7 +31,7 @@ public class Xop_mediawiki_wkr { } public void Free_memory() { wiki.Cache_mgr().Tmpl_result_cache().Clear(); - wiki.Cache_mgr().Free_mem(Bool_.N); + wiki.Cache_mgr().Free_mem__page(); wiki.Parser_mgr().Scrib().Core_term(); wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); } diff --git a/400_xowa/src/gplx/xowa/guis/views/Xog_tab_itm_edit_mgr.java b/400_xowa/src/gplx/xowa/guis/views/Xog_tab_itm_edit_mgr.java index 2a6a83c09..e3fceecfa 100644 --- a/400_xowa/src/gplx/xowa/guis/views/Xog_tab_itm_edit_mgr.java +++ b/400_xowa/src/gplx/xowa/guis/views/Xog_tab_itm_edit_mgr.java @@ -120,7 +120,7 @@ public class Xog_tab_itm_edit_mgr { } private static void Invalidate(Xowe_wiki wiki) {// invalidate everything on updates; especially needed for page transclusion; {{/my_subpage}} DATE:2014-04-10 wiki.Parser_mgr().Scrib().Core_term(); - wiki.Cache_mgr().Free_mem_all(); + wiki.Cache_mgr().Free_mem__all(); } private static byte[] Get_new_text(Xog_tab_itm tab, byte[] orig) { byte[] rv = tab.Html_itm().Get_elem_value_for_edit_box_as_bry(); diff --git a/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java b/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java index 6d72605b9..21209b19a 100644 --- a/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java +++ b/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java @@ -55,16 +55,24 @@ public class Xow_cache_mgr { } return scrib_lang_names; } - public void Free_mem_all() {this.Free_mem(Bool_.Y);} - public void Free_mem(boolean free_page_cache) { - if (free_page_cache) { - commons_cache.Clear(); - ifexist_cache.Clear(); - wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); // moved from ctx.Clear(); DATE:2016-07-21 - page_cache.Free_mem(true); - } - else { - page_cache.Free_mem(false); + public void Free_mem__page() {this.Free_mem(Free_mem__page_tid);} + public void Free_mem__wbase() {this.Free_mem(Free_mem__wbase_tid);} + public void Free_mem__all() {this.Free_mem(Free_mem__all_tid);} + private void Free_mem(int level) { + switch (level) { + case Free_mem__page_tid: + page_cache.Free_mem(false); + break; + case Free_mem__wbase_tid: + page_cache.Free_mem(false); + wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); + break; + case Free_mem__all_tid: + page_cache.Free_mem(true); + wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); + commons_cache.Clear(); + ifexist_cache.Clear(); + break; } wiki.Ctg__catpage_mgr().Free_mem_all(); tmpl_result_cache.Clear(); @@ -73,5 +81,6 @@ public class Xow_cache_mgr { lst_cache.Clear(); scrib_lang_names = null; } + private static final int Free_mem__page_tid = 0, Free_mem__wbase_tid = 1, Free_mem__all_tid = 2; private static Keyval[] scrib_lang_names; } diff --git a/400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java b/400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java index 624b640ff..116a0e077 100644 --- a/400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java +++ b/400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java @@ -33,7 +33,10 @@ public class Xow_ifexist_cache { return this; } public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;} private Xow_page_cache_wkr load_wkr; - public void Clear() {cache_mgr.Clear();} + public void Clear() { + cache_mgr.Clear(); + ns_loaded_hash.Clear(); + } public void Add(Xoa_ttl ttl) { byte[] key = ttl.Full_db(); cache_mgr.Add(key, itm__exists, key.length); diff --git a/400_xowa/src/gplx/xowa/xtns/lst/Lst_pfunc_lst_tst.java b/400_xowa/src/gplx/xowa/xtns/lst/Lst_pfunc_lst_tst.java index cc0c3b6f2..15f459d8e 100644 --- a/400_xowa/src/gplx/xowa/xtns/lst/Lst_pfunc_lst_tst.java +++ b/400_xowa/src/gplx/xowa/xtns/lst/Lst_pfunc_lst_tst.java @@ -125,7 +125,7 @@ class Lst_pfunc_lst_fxt { public Lst_pfunc_lst_fxt Clear() { if (fxt == null) fxt = new Xop_fxt(); fxt.Reset(); - fxt.Wiki().Cache_mgr().Free_mem_all(); + fxt.Wiki().Cache_mgr().Free_mem__all(); Io_mgr.Instance.InitEngine_mem(); return this; } diff --git a/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_basic_tst.java b/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_basic_tst.java index f614881f4..34dba646e 100644 --- a/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_basic_tst.java +++ b/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_basic_tst.java @@ -18,7 +18,7 @@ along with this program. If not, see . package gplx.xowa.xtns.proofreadPage; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import org.junit.*; import gplx.xowa.apps.cfgs.*; public class Pp_pages_nde_basic_tst { - private final Xop_fxt fxt = new Xop_fxt(); + private final Xop_fxt fxt = new Xop_fxt(); @Before public void Init() { Io_mgr.Instance.InitEngine_mem(); fxt.Wiki().Xtn_mgr().Xtn_proofread().Enabled_y_(); @@ -26,7 +26,7 @@ public class Pp_pages_nde_basic_tst { fxt.Wiki().Ns_mgr().Add_new(Xowc_xtn_pages.Ns_page_id_default, "Page").Add_new(Xowc_xtn_pages.Ns_index_id_default, "Index").Init(); } @After public void term() { - fxt.Wiki().Cache_mgr().Free_mem_all(); + fxt.Wiki().Cache_mgr().Free_mem__all(); } @Test public void Basic() { fxt.Init_page_create("Page:A/1", "abc"); diff --git a/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_index_tst.java b/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_index_tst.java index c9b6c9db9..050c289e4 100644 --- a/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_index_tst.java +++ b/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_index_tst.java @@ -18,7 +18,7 @@ along with this program. If not, see . package gplx.xowa.xtns.proofreadPage; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import org.junit.*; import gplx.xowa.apps.cfgs.*; public class Pp_pages_nde_index_tst { - private final Xop_fxt fxt = new Xop_fxt(); + private final Xop_fxt fxt = new Xop_fxt(); @Before public void Init() { Io_mgr.Instance.InitEngine_mem(); fxt.Wiki().Xtn_mgr().Xtn_proofread().Enabled_y_(); @@ -26,7 +26,7 @@ public class Pp_pages_nde_index_tst { fxt.Wiki().Ns_mgr().Add_new(Xowc_xtn_pages.Ns_page_id_default, "Page").Add_new(Xowc_xtn_pages.Ns_index_id_default, "Index").Init(); } @After public void term() { - fxt.Wiki().Cache_mgr().Free_mem_all(); + fxt.Wiki().Cache_mgr().Free_mem__all(); } @Test public void Repeated() { // PURPOSE: repeated pages should still show (and not be excluded by recursive logic); DATE:2014-01-01 fxt.Init_page_create("Page:A/1", "abc"); // NOTE: recursive call to self @@ -203,7 +203,7 @@ public class Pp_pages_nde_index_tst { fxt.Init_page_create("Index:A", ""); // [[Index:]] has no [[Page:]] links; interpret to=1 as [[Page:A/1]] - fxt.Wiki().Cache_mgr().Free_mem_all(); + fxt.Wiki().Cache_mgr().Free_mem__all(); fxt.Init_page_update("Index:A" , String_.Concat_lines_nl ( "no links" )); @@ -213,7 +213,7 @@ public class Pp_pages_nde_index_tst { )); // [[Index:]] has [[Page:]] links; interpret to=1 as 1st [[Page:]] in [[Index:]]'s [[Page:]] links - fxt.Wiki().Cache_mgr().Free_mem_all(); + fxt.Wiki().Cache_mgr().Free_mem__all(); fxt.Init_page_update("Index:A" , String_.Concat_lines_nl ( "[[Page:A/0]]" )); @@ -223,7 +223,7 @@ public class Pp_pages_nde_index_tst { )); // [[Index:]] has [[Page:]] links but also ; interpret to=1 as [[Page:A/1]] - fxt.Wiki().Cache_mgr().Free_mem_all(); + fxt.Wiki().Cache_mgr().Free_mem__all(); fxt.Init_page_update("Index:A" , String_.Concat_lines_nl ( "[[Page:A/0]]" , "" diff --git a/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_recursion_tst.java b/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_recursion_tst.java index 17067f9f5..ce9466c16 100644 --- a/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_recursion_tst.java +++ b/400_xowa/src/gplx/xowa/xtns/proofreadPage/Pp_pages_nde_recursion_tst.java @@ -21,7 +21,7 @@ public class Pp_pages_nde_recursion_tst { private final Xop_fxt fxt = new Xop_fxt(); @Before public void Init() {fxt.Init_xtn_pages();} @After public void term() { - fxt.Wiki().Cache_mgr().Free_mem_all(); + fxt.Wiki().Cache_mgr().Free_mem__all(); } @Test public void Page() { // PURPOSE: handle recursive calls on page; EX: fr.s:Page:NRF_19.djvu/19; DATE:2014-01-01 fxt.Init_page_create("Page:A/1", "abc"); // NOTE: recursive call to self