Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2025-05-30 14:04:56 +00:00
Code Issues Projects Releases Wiki Activity

Mass_parse: Add perf logger

Browse Source
This commit is contained in:
gnosygnu 2017-01-12 13:26:35 -05:00
parent 305c2f9762
commit d22c5e5f3d
6 changed files with 38 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
100_core/src/gplx/core/envs/Runtime_.java
View File

@ -22,6 +22,7 @@ public class Runtime_ {
public static long Memory_max() {return Runtime.getRuntime().maxMemory();}
public static long Memory_total() {return Runtime.getRuntime().totalMemory();}
public static long Memory_free() {return Runtime.getRuntime().freeMemory();}
public static long Memory_used() {return Memory_total() - Memory_free();} // REF:http://stackoverflow.com/questions/3571203/what-are-runtime-getruntime-totalmemory-and-freememory
public static void Exec(String v) {
try {

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java
View File

@ -33,7 +33,7 @@ public class Xomp_parse_mgr {
Xomp_page_pool_loader page_pool_loader = new Xomp_page_pool_loader(wiki, mgr_db.Conn(), cfg.Num_pages_in_pool(), cfg.Show_msg__fetched_pool());
Xomp_page_pool page_pool = new Xomp_page_pool(page_pool_loader, cfg.Num_pages_per_wkr());
Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval());
prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval(), cfg.Perf_interval(), mgr_db.Url().GenNewNameAndExt("xomp.perf.csv"));
// cache: preload tmpls and imglinks
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates());

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
View File

@ -22,6 +22,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = -1;
public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000;
public int Progress_interval() {return progress_interval;} private int progress_interval = 1000;
public int Perf_interval() {return perf_interval;} private int perf_interval = 10000;
public int Commit_interval() {return commit_interval;} private int commit_interval = 10000;
public int Cleanup_interval() {return cleanup_interval;} private int cleanup_interval = 50; // setting at 1000 uses lots of memory
public boolean Hdump_enabled() {return hdump_enabled;} private boolean hdump_enabled = true;
@ -49,6 +50,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
else if (ctx.Match(k, Invk__progress_interval_)) progress_interval = m.ReadInt("v");
else if (ctx.Match(k, "perf_interval_")) perf_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__commit_interval_)) commit_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__cleanup_interval_)) cleanup_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__hdump_enabled_)) hdump_enabled = m.ReadYn("v");

42
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_prog_mgr.java
View File

@ -18,27 +18,43 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
public class Xomp_prog_mgr {
private final Object thread_lock = new Object();
private int progress_interval;
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
private int prog_interval, perf_interval;
private int pages_done, pages_total;
private long time_bgn, time_prv, time_done;
private final Bry_bfr prog_bfr = Bry_bfr_.New();
public void Init(int pages_total, int progress_interval) {
this.progress_interval = progress_interval;
this.time_bgn = this.time_prv = gplx.core.envs.System_.Ticks();
private long prog_bgn, prog_prv, prog_done, perf_prv;
private Io_url perf_url;
public void Init(int pages_total, int prog_interval, int perf_interval, Io_url perf_url) {
this.pages_total = pages_total;
this.prog_interval = prog_interval;
this.perf_interval = perf_interval;
this.perf_url = perf_url;
this.prog_bgn = this.prog_prv = this.perf_prv = gplx.core.envs.System_.Ticks();
}
public void Mark_done(int id) {
synchronized (thread_lock) {
pages_done += 1;
if (pages_done % progress_interval == 0) {
long time_cur = gplx.core.envs.System_.Ticks();
if (pages_done % prog_interval == 0) {
long prog_cur = gplx.core.envs.System_.Ticks();
int pages_left = pages_total - pages_done;
time_done += (time_cur - time_prv);
double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)((time_cur - time_bgn) / 1000), true, 0);
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)(pages_left / rate_cur), true, 0);
prog_done += (prog_cur - prog_prv);
double rate_cur = pages_done / (prog_done / Time_span_.Ratio_f_to_s);
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)((prog_cur - prog_bgn) / 1000), true, 0);
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)(pages_left / rate_cur), true, 0);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left);
time_prv = time_cur;
prog_prv = prog_cur;
}
if (pages_done % perf_interval == 0) {
// get vars
long perf_cur = gplx.core.envs.System_.Ticks();
long perf_diff = (perf_cur - perf_prv);
this.perf_prv = perf_cur;
int memory_used = (int)(gplx.core.envs.Runtime_.Memory_used() / Io_mgr.Len_mb);
// save to file
tmp_bfr.Add_int_fixed(pages_done, 12).Add_byte_pipe();
tmp_bfr.Add_int_fixed(memory_used, 6).Add_byte_pipe();
tmp_bfr.Add_long_variable(perf_diff).Add_byte_nl();
Io_mgr.Instance.AppendFilBfr(perf_url, tmp_bfr);
}
}
}

4
400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java
View File

@ -77,9 +77,9 @@ public class Xow_cache_mgr {
wiki.Ctg__catpage_mgr().Free_mem_all();
tmpl_result_cache.Clear();
defn_cache.Free_mem_all();
misc_cache.Clear();
lst_cache.Clear();
scrib_lang_names = null;
misc_cache.Clear();
// scrib_lang_names = null;
}
private static final int Free_mem__page_tid = 0, Free_mem__wbase_tid = 1, Free_mem__all_tid = 2;
private static Keyval[] scrib_lang_names;

3
400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java
View File

@ -101,12 +101,15 @@ public class Xow_page_cache {
cache.Clear();
}
else {
// gather non-permanent items
int len = cache.Count();
for (int i = 0; i < len; i++) {
Xow_page_cache_itm itm = (Xow_page_cache_itm)cache.Get_at(i);
if (!itm.Cache_permanently())
deleted.Add(itm);
}
// remove non-permanent items
len = deleted.Len();
for (int i = 0; i < len; i++) {
Xow_page_cache_itm itm = (Xow_page_cache_itm)deleted.Get_at(i);