Mass_parse: Add perf logger

100_core/src/gplx/core/envs/Runtime_.java

@@ -22,6 +22,7 @@ public class Runtime_ {
 	public static long Memory_max() {return Runtime.getRuntime().maxMemory();}		
 	public static long Memory_total() {return Runtime.getRuntime().totalMemory();}	
 	public static long Memory_free() {return Runtime.getRuntime().freeMemory();}	
+	public static long Memory_used() {return Memory_total() - Memory_free();}	// REF:http://stackoverflow.com/questions/3571203/what-are-runtime-getruntime-totalmemory-and-freememory
 
 	public static void Exec(String v) {
 				try {

400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java

@@ -33,7 +33,7 @@ public class Xomp_parse_mgr {
 		Xomp_page_pool_loader page_pool_loader = new Xomp_page_pool_loader(wiki, mgr_db.Conn(), cfg.Num_pages_in_pool(), cfg.Show_msg__fetched_pool());
 		Xomp_page_pool page_pool = new Xomp_page_pool(page_pool_loader, cfg.Num_pages_per_wkr());
 		Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
-		prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval());
+		prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval(), cfg.Perf_interval(), mgr_db.Url().GenNewNameAndExt("xomp.perf.csv"));
 
 		// cache: preload tmpls and imglinks
 		Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates());

400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java

@@ -22,6 +22,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
 	public int		Num_pages_in_pool()				{return num_pages_in_pool;}				private int num_pages_in_pool = -1;
 	public int		Num_pages_per_wkr()				{return num_pages_per_wkr;}				private int num_pages_per_wkr = 1000;
 	public int		Progress_interval()				{return progress_interval;}				private int progress_interval = 1000;
+	public int		Perf_interval()					{return perf_interval;}					private int perf_interval = 10000;
 	public int		Commit_interval()				{return commit_interval;}				private int commit_interval = 10000;
 	public int		Cleanup_interval()				{return cleanup_interval;}				private int cleanup_interval = 50;	// setting at 1000 uses lots of memory
 	public boolean		Hdump_enabled()					{return hdump_enabled;}					private boolean hdump_enabled = true;
@@ -49,6 +50,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
 		else if	(ctx.Match(k, Invk__num_pages_in_pool_))			num_pages_in_pool = m.ReadInt("v");
 		else if	(ctx.Match(k, Invk__num_pages_per_wkr_))			num_pages_per_wkr = m.ReadInt("v");
 		else if	(ctx.Match(k, Invk__progress_interval_))			progress_interval = m.ReadInt("v");
+		else if	(ctx.Match(k, "perf_interval_"))					perf_interval = m.ReadInt("v");
 		else if	(ctx.Match(k, Invk__commit_interval_))				commit_interval = m.ReadInt("v");
 		else if	(ctx.Match(k, Invk__cleanup_interval_))				cleanup_interval = m.ReadInt("v");
 		else if	(ctx.Match(k, Invk__hdump_enabled_))				hdump_enabled = m.ReadYn("v");

400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_prog_mgr.java

@@ -18,27 +18,43 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
 public class Xomp_prog_mgr {
 	private final    Object thread_lock = new Object();
-	private int progress_interval;
+	private final    Bry_bfr tmp_bfr = Bry_bfr_.New();
+	private int prog_interval, perf_interval;
 	private int pages_done, pages_total;
-	private long time_bgn, time_prv, time_done;		
+	private long prog_bgn, prog_prv, prog_done, perf_prv;
-	private final    Bry_bfr prog_bfr = Bry_bfr_.New();
+	private Io_url perf_url;
-	public void Init(int pages_total, int progress_interval) {
+	public void Init(int pages_total, int prog_interval, int perf_interval, Io_url perf_url) {
-		this.progress_interval = progress_interval;
-		this.time_bgn = this.time_prv = gplx.core.envs.System_.Ticks();
 		this.pages_total = pages_total;
+		this.prog_interval = prog_interval;
+		this.perf_interval = perf_interval;
+		this.perf_url = perf_url;
+		this.prog_bgn = this.prog_prv = this.perf_prv = gplx.core.envs.System_.Ticks();
 	}
 	public void Mark_done(int id) {
 		synchronized (thread_lock) {
 			pages_done += 1;
-			if (pages_done % progress_interval == 0) {
+			if (pages_done % prog_interval == 0) {
-				long time_cur = gplx.core.envs.System_.Ticks();
+				long prog_cur = gplx.core.envs.System_.Ticks();
 				int pages_left = pages_total - pages_done;
-				time_done += (time_cur - time_prv);
+				prog_done += (prog_cur - prog_prv);
-				double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);
+				double rate_cur = pages_done / (prog_done / Time_span_.Ratio_f_to_s);
-				String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)((time_cur - time_bgn) / 1000), true, 0);
+				String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)((prog_cur - prog_bgn) / 1000), true, 0);
-				String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)(pages_left / rate_cur), true, 0);
+				String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)(pages_left / rate_cur), true, 0);
 				Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left);
-				time_prv = time_cur;
+				prog_prv = prog_cur;
+			}
+			if (pages_done % perf_interval == 0) {
+				// get vars
+				long perf_cur = gplx.core.envs.System_.Ticks();
+				long perf_diff = (perf_cur - perf_prv);
+				this.perf_prv = perf_cur;
+				int memory_used = (int)(gplx.core.envs.Runtime_.Memory_used() / Io_mgr.Len_mb);
+
+				// save to file
+				tmp_bfr.Add_int_fixed(pages_done, 12).Add_byte_pipe();
+				tmp_bfr.Add_int_fixed(memory_used, 6).Add_byte_pipe();
+				tmp_bfr.Add_long_variable(perf_diff).Add_byte_nl();
+				Io_mgr.Instance.AppendFilBfr(perf_url, tmp_bfr);
 			}
 		}
 	}

400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java

@@ -77,9 +77,9 @@ public class Xow_cache_mgr {
 		wiki.Ctg__catpage_mgr().Free_mem_all();
 		tmpl_result_cache.Clear();
 		defn_cache.Free_mem_all();
-		misc_cache.Clear();
 		lst_cache.Clear();
-		scrib_lang_names = null;
+		misc_cache.Clear();
+		// scrib_lang_names = null;
 	}
 	private static final int Free_mem__page_tid = 0, Free_mem__wbase_tid = 1, Free_mem__all_tid = 2;
 	private static Keyval[] scrib_lang_names;

400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java

@@ -101,12 +101,15 @@ public class Xow_page_cache {
 				cache.Clear();
 			}
 			else {
+				// gather non-permanent items
 				int len = cache.Count();
 				for (int i = 0; i < len; i++) {
 					Xow_page_cache_itm itm = (Xow_page_cache_itm)cache.Get_at(i);
 					if (!itm.Cache_permanently())
 						deleted.Add(itm);
 				}
+
+				// remove non-permanent items
 				len = deleted.Len();
 				for (int i = 0; i < len; i++) {
 					Xow_page_cache_itm itm = (Xow_page_cache_itm)deleted.Get_at(i);