Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Mass_parse: Refactor ifexist_mgr into separate class; Load more titles into ifexist_mgr

Browse Source
This commit is contained in:
gnosygnu
2016-12-16 13:23:48 -05:00
parent bfc5c5b68e
commit 5b42a90cd5
8 changed files with 207 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java
View File

@@ -24,18 +24,23 @@ public class Xow_cache_mgr {
this.wiki = wiki;
this.page_cache = new Xow_page_cache(wiki);
this.defn_cache = new Xow_defn_cache(wiki.Lang());
this.ifexist_cache = new Xow_ifexist_cache(wiki, page_cache);
}
public Hash_adp Tmpl_result_cache() {return tmpl_result_cache;} private final Hash_adp tmpl_result_cache = Hash_adp_bry.cs();
public Xow_defn_cache Defn_cache() {return defn_cache;} private final Xow_defn_cache defn_cache;
public Hash_adp_bry Lst_cache() {return lst_cache;} private final Hash_adp_bry lst_cache = Hash_adp_bry.cs();
public Hash_adp Misc_cache() {return misc_cache;} private final Hash_adp misc_cache = Hash_adp_.New();
public Xow_page_cache Page_cache() {return page_cache;} private Xow_page_cache page_cache;
public Gfo_cache_mgr Commons_cache() {return commons_cache;} private Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
public Gfo_cache_mgr Ifexist_cache() {return ifexist_cache;} private Gfo_cache_mgr ifexist_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
public Hash_adp Tmpl_result_cache() {return tmpl_result_cache;} private final Hash_adp tmpl_result_cache = Hash_adp_bry.cs();
public Xow_defn_cache Defn_cache() {return defn_cache;} private final Xow_defn_cache defn_cache;
public Hash_adp_bry Lst_cache() {return lst_cache;} private final Hash_adp_bry lst_cache = Hash_adp_bry.cs();
public Hash_adp Misc_cache() {return misc_cache;} private final Hash_adp misc_cache = Hash_adp_.New();
public Xow_page_cache Page_cache() {return page_cache;} private Xow_page_cache page_cache;
public Gfo_cache_mgr Commons_cache() {return commons_cache;} private Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
public Xow_ifexist_cache Ifexist_cache() {return ifexist_cache;} private Xow_ifexist_cache ifexist_cache;
public Xow_cache_mgr Page_cache_(Xow_page_cache v) {this.page_cache = v; return this;}
public Xow_cache_mgr Commons_cache_(Gfo_cache_mgr v) {this.commons_cache = v; return this;}
public Xow_cache_mgr Ifexist_cache_(Gfo_cache_mgr v) {this.ifexist_cache = v; return this;}
public Xow_cache_mgr Ifexist_cache_(Xow_ifexist_cache v) {this.ifexist_cache = v; return this;}
public void Load_wkr_(Xow_page_cache_wkr v) {
page_cache.Load_wkr_(v);
ifexist_cache.Load_wkr_(v);
}
public Keyval[] Scrib_lang_names() {
if (scrib_lang_names == null) {
List_adp list = List_adp_.New();

90
400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java Normal file
View File

@@ -0,0 +1,90 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
import gplx.core.caches.*;
import gplx.xowa.wikis.data.tbls.*;
public class Xow_ifexist_cache {
private final Xowe_wiki wiki;
private final Xow_page_cache page_cache;
private final Gfo_cache_mgr cache_mgr = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
private final Hash_adp ns_loaded_hash = Hash_adp_.New();
private final Xow_ifexist_itm itm__exists = new Xow_ifexist_itm(Bool_.Y), itm__missing = new Xow_ifexist_itm(Bool_.N);
public Xow_ifexist_cache(Xowe_wiki wiki, Xow_page_cache page_cache) {
this.wiki = wiki;
this.page_cache = page_cache;
}
public Xow_ifexist_cache Cache_sizes_(int max, int reduce) {
cache_mgr.Max_size_(max).Reduce_by_(reduce);
return this;
}
public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;} private Xow_page_cache_wkr load_wkr;
public void Clear() {cache_mgr.Clear();}
public void Add(Xoa_ttl ttl) {
byte[] key = ttl.Full_db();
cache_mgr.Add(key, itm__exists, key.length);
}
public void Add_ns_loaded(int... ns_ids) {
for (int ns_id : ns_ids)
ns_loaded_hash.Add(ns_id, ns_id);
}
public byte Get_by_mem(Xoa_ttl ttl) {
byte[] ttl_full_db = ttl.Full_db();
// check cache_mgr
Xow_ifexist_itm found = (Xow_ifexist_itm)cache_mgr.Get_by_key(ttl_full_db);
if (found != null) return found.Exists() ? Bool_.Y_byte : Bool_.N_byte;
// check ns_loaded cache (xomp only); if exists, return false, since all pages in ns are loaded, and still not found
if (ns_loaded_hash.Has(ttl.Ns().Id())) return Bool_.N_byte;
// check page_cache since full page + text could be loaded there
Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_or_null(ttl_full_db);
if (itm == Xow_page_cache_itm.Missing)
return Bool_.N_byte;
else if (itm != null)
return Bool_.Y_byte;
return Bool_.__byte;
}
public boolean Load(Xoa_ttl ttl) {
byte[] key = ttl.Full_db();
Xow_ifexist_itm itm = null;
// gplx.core.consoles.Console_adp__sys.Instance.Write_str("ifexist_cache:" + String_.new_u8(key));
if (load_wkr != null) {
// load_wkr; should call ifexist method, but for now, load entire page
byte[] page_text = load_wkr.Get_page_or_null(key);
itm = page_text == null ? itm__missing : itm__exists;
}
else {
// page_tbl
Xowd_page_itm page_itm = new Xowd_page_itm();
wiki.Db_mgr().Load_mgr().Load_by_ttl(page_itm, ttl.Ns(), ttl.Page_db());
itm = page_itm.Exists() ? itm__exists : itm__missing;
}
// add
cache_mgr.Add(key, itm, key.length);
return itm == itm__exists;
}
}
class Xow_ifexist_itm implements Rls_able {
public Xow_ifexist_itm(boolean exists) {this.exists = exists;}
public boolean Exists() {return exists;} private final boolean exists;
public void Rls() {}
}

49
400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java
View File

@@ -20,55 +20,14 @@ public class Xow_page_cache {
private final Object thread_lock = new Object();
private final Xowe_wiki wiki;
private final Ordered_hash cache = Ordered_hash_.New_bry(); // NOTE: wiki titles are not case-sensitive when ns is "1st-letter" (EX: w:earth an w:Earth); in these cases, two entries will be stored
private final Hash_adp ifexists_cache = Hash_adp_bry.cs();
private final List_adp deleted = List_adp_.New();
private Xow_page_cache_wkr load_wkr;
private final List_adp deleted = List_adp_.New();
public Xow_page_cache(Xowe_wiki wiki) {this.wiki = wiki;}
public Xow_page_cache_wkr Load_wkr() {return load_wkr;}
public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;}
public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;} private Xow_page_cache_wkr load_wkr;
public Xow_page_cache_itm Get_or_null(byte[] ttl_full_db) {return (Xow_page_cache_itm)cache.Get_by(ttl_full_db);}
public byte[] Get_or_load_as_src(Xoa_ttl ttl) {
Xow_page_cache_itm rv = Get_or_load_as_itm(ttl);
return rv == null ? null : rv.Wtxt__direct();
}
public boolean Get_ifexist_by_mem(byte[] ifexists_key) {
// check ifexists_cache
Xow_page_cache_itm ifexists_itm = (Xow_page_cache_itm)ifexists_cache.Get_by(ifexists_key);
if (ifexists_itm == Xow_page_cache_itm.Missing)
return false;
else if (ifexists_itm != null)
return true;
// check page_cache
ifexists_itm = (Xow_page_cache_itm)cache.Get_by(ifexists_key);
if (ifexists_itm == Xow_page_cache_itm.Missing)
return false;
else if (ifexists_itm != null)
return true;
return false;
}
public boolean Load_ifexist(Xoa_ttl ttl) {
byte[] ifexists_key = ttl.Full_db();
Xow_page_cache_itm ifexists_itm = null;
// do load
if (load_wkr != null) {
// load_wkr
byte[] page_text = load_wkr.Get_page_or_null(ifexists_key);
ifexists_itm = page_text == null ? Xow_page_cache_itm.Missing : new Xow_page_cache_itm(false, ttl, Bry_.Empty, Bry_.Empty);
}
else {
// page_tbl
gplx.xowa.wikis.data.tbls.Xowd_page_itm page_itm = new gplx.xowa.wikis.data.tbls.Xowd_page_itm();
wiki.Db_mgr().Load_mgr().Load_by_ttl(page_itm, ttl.Ns(), ttl.Page_db());
// wiki.Data__core_mgr().Db__core().Tbl__page().Select_by_ttl(page_itm, ttl);
ifexists_itm = page_itm.Exists() ? new Xow_page_cache_itm(false, ttl, Bry_.Empty, Bry_.Empty) : Xow_page_cache_itm.Missing;
}
// add to ifexists_cache only (not page_cache)
ifexists_cache.Add(ifexists_key, ifexists_itm);
return ifexists_itm != Xow_page_cache_itm.Missing;
}
public void Add(byte[] ttl_full_db, Xow_page_cache_itm itm) {
cache.Add(ttl_full_db, itm);
}
@@ -96,6 +55,7 @@ public class Xow_page_cache {
boolean page_exists = false;
byte[] page_text = null;
byte[] page_redirect_from = null;
// gplx.core.consoles.Console_adp__sys.Instance.Write_str("page_cache:" + String_.new_u8(ttl_full_db));
if (load_wkr != null) {
page_text = load_wkr.Get_page_or_null(ttl_full_db);
page_exists = page_text != null;
@@ -139,7 +99,6 @@ public class Xow_page_cache {
synchronized (thread_lock) { // LOCK:app-level; DATE:2016-07-06
if (clear_permanent_itms) {
cache.Clear();
ifexists_cache.Clear();
}
else {
int len = cache.Count();