Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Mass_parse: Change page_cache to LRU cache [#483]

Browse Source
This commit is contained in:
gnosygnu
2019-06-21 23:02:49 -04:00
parent 0cfb0b19ad
commit a01e7409eb
21 changed files with 367 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java
View File

@@ -31,13 +31,14 @@ public class Xomp_parse_mgr {
// init page_pool
Xomp_page_pool_loader page_pool_loader = new Xomp_page_pool_loader(wiki, mgr_db.Conn(), cfg.Num_pages_in_pool(), cfg.Show_msg__fetched_pool());
Xomp_page_pool page_pool = new Xomp_page_pool(page_pool_loader, cfg.Num_pages_per_wkr());
Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval(), cfg.Perf_interval(), mgr_db.Url().GenNewNameAndExt("xomp.perf.csv"));
// cache: preload tmpls and imglinks
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates());
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates(), cfg.Page_cache_max());
wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of user cache
Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
prog_mgr.Init(page_cache, page_pool_loader.Get_pending_count(), cfg.Progress_interval(), cfg.Perf_interval(), mgr_db.Url().GenNewNameAndExt("xomp.perf.csv"));
Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
Xow_ifexist_cache ifexist_cache = new Xow_ifexist_cache(wiki, page_cache).Cache_sizes_(Int_.Max_value, Int_.Max_value);
if (cfg.Load_ifexists_ns() != null) Load_ifexists_ns(wiki, ifexist_cache, cfg.Load_ifexists_ns());
@@ -71,6 +72,7 @@ public class Xomp_parse_mgr {
Xowe_wiki wkr_wiki = Xow_wiki_utl_.Clone_wiki(wiki, wiki.Fsys_mgr().Root_dir());
wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);
// make wkr
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, cfg, mgr_db, page_pool, prog_mgr, file_orig_wkr, ns_ord_mgr, wkr_wiki, indexer, i + wkr_uid_bgn);
wkrs[i] = wkr;

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
View File

@@ -42,6 +42,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
public long Wbase_cache_mru_size() {return wbase_cache_mru_size;} private long wbase_cache_mru_size = 100;
public long Wbase_cache_mru_weight() {return wbase_cache_mru_weight;} private long wbase_cache_mru_weight = 10;
public long Wbase_cache_mru_compress_size() {return wbase_cache_mru_compress_size;} private long wbase_cache_mru_compress_size = 70;
public long Page_cache_max() {return page_cache_max;} private long page_cache_max = 8 * (long)Io_mgr.Len_gb;
public void Init(Xowe_wiki wiki) {
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count();
if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000;
@@ -76,6 +77,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, "wbase_cache_mru_size_")) wbase_cache_mru_size = m.ReadLong("v");
else if (ctx.Match(k, "wbase_cache_mru_weight_")) wbase_cache_mru_weight = m.ReadLong("v");
else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v");
else if (ctx.Match(k, "page_cache_max_")) page_cache_max = m.ReadLong("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}

7
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_prog_mgr.java
View File

@@ -14,14 +14,17 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.xowa.wikis.caches.*;
public class Xomp_prog_mgr {
private final Object thread_lock = new Object();
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
private Xow_page_cache page_cache;
private int prog_interval, perf_interval;
private int pages_done, pages_total;
private long prog_bgn, prog_prv, prog_done, perf_prv;
private Io_url perf_url;
public void Init(int pages_total, int prog_interval, int perf_interval, Io_url perf_url) {
public void Init(Xow_page_cache page_cache, int pages_total, int prog_interval, int perf_interval, Io_url perf_url) {
this.page_cache = page_cache;
this.pages_total = pages_total;
this.prog_interval = prog_interval;
this.perf_interval = perf_interval;
@@ -39,7 +42,7 @@ public class Xomp_prog_mgr {
double rate_cur = pages_done / (prog_done / Time_span_.Ratio_f_to_s);
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)((prog_cur - prog_bgn) / 1000), true, 0);
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(tmp_bfr, (int)(pages_left / rate_cur), true, 0);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5} cache_stats=~{6}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left, page_cache.To_str());
prog_prv = prog_cur;
}
if (pages_done % perf_interval == 0) {

10
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_tmpl_cache_bldr.java
View File

@@ -17,8 +17,9 @@ package gplx.xowa.addons.bldrs.mass_parses.parses.utls; import gplx.*; import gp
import gplx.dbs.*;
import gplx.xowa.wikis.caches.*;
public class Xomp_tmpl_cache_bldr {
public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all) {
public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all, long page_cache_max) {
Xow_page_cache rv = new Xow_page_cache(wiki);
rv.Max_(page_cache_max);
if (fill_all) Fill_all(rv, wiki);
return rv;
}
@@ -53,12 +54,7 @@ public class Xomp_tmpl_cache_bldr {
text_db_loader.Add(rdr.Read_int("page_text_db_id"), itm);
// ignore duplicate page_titles in cache; EX:ru.n:Модуль:Weather/data DATE:2017-03-16
if (cache.Get_or_null(page_ttl.Full_db()) == null) {
cache.Add(page_ttl.Full_db(), itm);
}
else {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse: ignoring duplicate page title in page cache; title=~{0} id=~{1}", page_ttl.Full_db(), page_id);
}
cache.Add_itm(page_ttl.Full_db_as_str(), itm);
page_regy.Add(page_id, itm);

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java
View File

@@ -124,8 +124,6 @@ public class Xomp_parse_wkr implements Gfo_invk {
Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry());
// if ns changed and prv_ns is main
if (cur_ns != prv_ns) {
if (prv_ns == gplx.xowa.wikis.nss.Xow_ns_.Tid__main)
wiki.Cache_mgr().Free_mem__all(); // NOTE: clears page and wbase cache only; needed else OutOfMemory error for en.w in 25th hour; DATE:2017-01-11
prv_ns = cur_ns;
}

7
400_xowa/src/gplx/xowa/addons/parsers/mediawikis/Xop_mediawiki_wkr.java
View File

@@ -22,9 +22,6 @@ public class Xop_mediawiki_wkr {
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
public Xop_mediawiki_wkr(Xowe_wiki wiki, Xop_mediawiki_loader loader) {
this.wiki = wiki;
this.Loader_(loader);
}
public void Loader_(Xop_mediawiki_loader loader) {
if (loader != null)
wiki.Cache_mgr().Load_wkr_(new Xow_page_cache_wkr__embeddable(wiki, loader));
}
@@ -34,10 +31,6 @@ public class Xop_mediawiki_wkr {
wiki.Parser_mgr().Scrib().Core_term();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
}
public void Clear_cache(String page) {
Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));
wiki.Cache_mgr().Page_cache().Del(ttl.Full_db());
}
public String Parse(String page, String wikitext) {
Xoa_ttl ttl = wiki.Ttl_parse(Bry_.new_u8(page));

2
400_xowa/src/gplx/xowa/addons/servers/https/utils/Xoa_util_svc.java
View File

@@ -43,7 +43,7 @@ public class Xoa_util_svc {
// get page
Xoa_ttl ttl = wiki.Ttl_parse(page_bry);
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_or_load_as_itm_2(ttl);
Xow_page_cache_itm page_itm = wiki.Cache_mgr().Page_cache().Get_itm_else_load_or_null(ttl);
byte[] page_text = page_itm == null ? null : page_itm.Wtxt__direct();
if (page_text == null) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "Xoa_utl_svc:page not found: wiki=~{0} page=~{1}", wiki_bry, page_bry);