From d22c5e5f3ddb5e5104357c0ef6ed140d9b121a43 Mon Sep 17 00:00:00 2001 From: gnosygnu Date: Thu, 12 Jan 2017 13:26:35 -0500 Subject: [PATCH] Mass_parse: Add perf logger --- 100_core/src/gplx/core/envs/Runtime_.java | 1 + .../parses/mgrs/Xomp_parse_mgr.java | 2 +- .../parses/mgrs/Xomp_parse_mgr_cfg.java | 2 + .../parses/mgrs/Xomp_prog_mgr.java | 42 +++++++++++++------ .../gplx/xowa/wikis/caches/Xow_cache_mgr.java | 4 +- .../xowa/wikis/caches/Xow_page_cache.java | 3 ++ 6 files changed, 38 insertions(+), 16 deletions(-) diff --git a/100_core/src/gplx/core/envs/Runtime_.java b/100_core/src/gplx/core/envs/Runtime_.java index 98d4667df..8be8076b1 100644 --- a/100_core/src/gplx/core/envs/Runtime_.java +++ b/100_core/src/gplx/core/envs/Runtime_.java @@ -22,6 +22,7 @@ public class Runtime_ { public static long Memory_max() {return Runtime.getRuntime().maxMemory();} public static long Memory_total() {return Runtime.getRuntime().totalMemory();} public static long Memory_free() {return Runtime.getRuntime().freeMemory();} + public static long Memory_used() {return Memory_total() - Memory_free();} // REF:http://stackoverflow.com/questions/3571203/what-are-runtime-getruntime-totalmemory-and-freememory public static void Exec(String v) { try { diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java index 226de9f1d..39370e2b5 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java @@ -33,7 +33,7 @@ public class Xomp_parse_mgr { Xomp_page_pool_loader page_pool_loader = new Xomp_page_pool_loader(wiki, mgr_db.Conn(), cfg.Num_pages_in_pool(), cfg.Show_msg__fetched_pool()); Xomp_page_pool page_pool = new Xomp_page_pool(page_pool_loader, cfg.Num_pages_per_wkr()); Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr(); - prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval()); + prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval(), cfg.Perf_interval(), mgr_db.Url().GenNewNameAndExt("xomp.perf.csv")); // cache: preload tmpls and imglinks Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates()); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java index b10ae4e65..4e5ba6575 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java @@ -22,6 +22,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk { public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = -1; public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000; public int Progress_interval() {return progress_interval;} private int progress_interval = 1000; + public int Perf_interval() {return perf_interval;} private int perf_interval = 10000; public int Commit_interval() {return commit_interval;} private int commit_interval = 10000; public int Cleanup_interval() {return cleanup_interval;} private int cleanup_interval = 50; // setting at 1000 uses lots of memory public boolean Hdump_enabled() {return hdump_enabled;} private boolean hdump_enabled = true; @@ -49,6 +50,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk { else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v"); else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v"); else if (ctx.Match(k, Invk__progress_interval_)) progress_interval = m.ReadInt("v"); + else if (ctx.Match(k, "perf_interval_")) perf_interval = m.ReadInt("v"); else if (ctx.Match(k, Invk__commit_interval_)) commit_interval = m.ReadInt("v"); else if (ctx.Match(k, Invk__cleanup_interval_)) cleanup_interval = m.ReadInt("v"); else if (ctx.Match(k, Invk__hdump_enabled_)) hdump_enabled = m.ReadYn("v"); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_prog_mgr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_prog_mgr.java index 6d7534106..62656e3fb 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_prog_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_prog_mgr.java @@ -18,27 +18,43 @@ along with this program. If not, see . package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; public class Xomp_prog_mgr { private final Object thread_lock = new Object(); - private int progress_interval; + private final Bry_bfr tmp_bfr = Bry_bfr_.New(); + private int prog_interval, perf_interval; private int pages_done, pages_total; - private long time_bgn, time_prv, time_done; - private final Bry_bfr prog_bfr = Bry_bfr_.New(); - public void Init(int pages_total, int progress_interval) { - this.progress_interval = progress_interval; - this.time_bgn = this.time_prv = gplx.core.envs.System_.Ticks(); + private long prog_bgn, prog_prv, prog_done, perf_prv; + private Io_url perf_url; + public void Init(int pages_total, int prog_interval, int perf_interval, Io_url perf_url) { this.pages_total = pages_total; + this.prog_interval = prog_interval; + this.perf_interval = perf_interval; + this.perf_url = perf_url; + this.prog_bgn = this.prog_prv = this.perf_prv = gplx.core.envs.System_.Ticks(); } public void Mark_done(int id) { synchronized (thread_lock) { pages_done += 1; - if (pages_done % progress_interval == 0) { - long time_cur = gplx.core.envs.System_.Ticks(); + if (pages_done % prog_interval == 0) { + long prog_cur = gplx.core.envs.System_.Ticks(); int pages_left = pages_total - pages_done; - time_done += (time_cur - time_prv); - double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s); - String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)((time_cur - time_bgn) / 1000), true, 0); - String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)(pages_left / rate_cur), true, 0); + prog_done += (prog_cur - prog_prv); + double rate_cur = pages_done / (prog_done / Time_span_.Ratio_f_to_s); + String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)((prog_cur - prog_bgn) / 1000), true, 0); + String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)(pages_left / rate_cur), true, 0); Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left); - time_prv = time_cur; + prog_prv = prog_cur; + } + if (pages_done % perf_interval == 0) { + // get vars + long perf_cur = gplx.core.envs.System_.Ticks(); + long perf_diff = (perf_cur - perf_prv); + this.perf_prv = perf_cur; + int memory_used = (int)(gplx.core.envs.Runtime_.Memory_used() / Io_mgr.Len_mb); + + // save to file + tmp_bfr.Add_int_fixed(pages_done, 12).Add_byte_pipe(); + tmp_bfr.Add_int_fixed(memory_used, 6).Add_byte_pipe(); + tmp_bfr.Add_long_variable(perf_diff).Add_byte_nl(); + Io_mgr.Instance.AppendFilBfr(perf_url, tmp_bfr); } } } diff --git a/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java b/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java index 21209b19a..2ae986f4c 100644 --- a/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java +++ b/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java @@ -77,9 +77,9 @@ public class Xow_cache_mgr { wiki.Ctg__catpage_mgr().Free_mem_all(); tmpl_result_cache.Clear(); defn_cache.Free_mem_all(); - misc_cache.Clear(); lst_cache.Clear(); - scrib_lang_names = null; + misc_cache.Clear(); + // scrib_lang_names = null; } private static final int Free_mem__page_tid = 0, Free_mem__wbase_tid = 1, Free_mem__all_tid = 2; private static Keyval[] scrib_lang_names; diff --git a/400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java b/400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java index 58b12273f..23a89bed0 100644 --- a/400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java +++ b/400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java @@ -101,12 +101,15 @@ public class Xow_page_cache { cache.Clear(); } else { + // gather non-permanent items int len = cache.Count(); for (int i = 0; i < len; i++) { Xow_page_cache_itm itm = (Xow_page_cache_itm)cache.Get_at(i); if (!itm.Cache_permanently()) deleted.Add(itm); } + + // remove non-permanent items len = deleted.Len(); for (int i = 0; i < len; i++) { Xow_page_cache_itm itm = (Xow_page_cache_itm)deleted.Get_at(i);