mirror of
https://github.com/gnosygnu/xowa.git
synced 2025-05-31 22:44:34 +00:00
Mass_parse: Refactor ifexist_mgr into separate class; Load more titles into ifexist_mgr
This commit is contained in:
parent
bfc5c5b68e
commit
5b42a90cd5
@ -35,21 +35,23 @@ public class Xomp_parse_mgr {
|
||||
Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
|
||||
prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval());
|
||||
|
||||
// cache: disable general settings
|
||||
wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of cache
|
||||
Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
|
||||
Gfo_cache_mgr ifexist_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
|
||||
|
||||
// cache: preload tmpls and imglinks
|
||||
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates());
|
||||
wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of user cache
|
||||
|
||||
Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
|
||||
Xow_ifexist_cache ifexist_cache = new Xow_ifexist_cache(wiki, page_cache).Cache_sizes_(Int_.Max_value, Int_.Max_value);
|
||||
if (cfg.Load_ifexists_ns() != null) Load_ifexists_ns(wiki, ifexist_cache, cfg.Load_ifexists_ns());
|
||||
|
||||
Xof_orig_wkr__img_links file_orig_wkr = new Xof_orig_wkr__img_links(wiki);
|
||||
if (cfg.Load_all_imglnks()) Xof_orig_wkr__img_links_.Load_all(file_orig_wkr);
|
||||
if (cfg.Load_all_imglinks()) Xof_orig_wkr__img_links_.Load_all(file_orig_wkr);
|
||||
|
||||
// load_wkr: init and start
|
||||
// Xomp_load_wkr load_wkr = new Xomp_load_wkr(wiki, db_mgr.Mgr_db().Conn(), cfg.Num_pages_in_pool(), cfg.Num_wkrs());
|
||||
// Thread_adp_.Start_by_key("xomp.load", Cancelable_.Never, load_wkr, Xomp_load_wkr.Invk__exec);
|
||||
|
||||
// assert wkr_tbl
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "initing wkrs");
|
||||
int wkr_len = cfg.Num_wkrs();
|
||||
int wkr_uid_bgn = mgr_db.Tbl__wkr().Init_wkrs(cfg.Wkr_machine_name(), wkr_len);
|
||||
latch = new Gfo_countdown_latch(wkr_len);
|
||||
@ -86,4 +88,46 @@ public class Xomp_parse_mgr {
|
||||
}
|
||||
Gfo_usr_dlg_.Instance.Note_many("", "", bfr.To_str_and_clear());
|
||||
}
|
||||
private static void Load_ifexists_ns(Xow_wiki wiki, Xow_ifexist_cache cache, String ns_list) {
|
||||
// expand "*" to all
|
||||
if (String_.Eq(ns_list, "*")) {
|
||||
Bry_bfr bfr = Bry_bfr_.New();
|
||||
gplx.xowa.wikis.nss.Xow_ns_mgr ns_mgr = wiki.Ns_mgr();
|
||||
int len = ns_mgr.Ids_len();
|
||||
for (int i = 0; i < len; i++) {
|
||||
gplx.xowa.wikis.nss.Xow_ns ns = ns_mgr.Ids_get_at(i);
|
||||
if (ns.Id() >= 0) { // skip Media / Special
|
||||
if (bfr.Len() != 0) bfr.Add_byte_comma();
|
||||
bfr.Add_int_variable(ns.Id());
|
||||
}
|
||||
}
|
||||
ns_list = bfr.To_str_and_clear();
|
||||
}
|
||||
// load all titles
|
||||
gplx.xowa.wikis.data.tbls.Xowd_page_tbl page_tbl = wiki.Data__core_mgr().Db__core().Tbl__page();
|
||||
String sql = gplx.dbs.Db_sql_.Make_by_fmt(String_.Ary
|
||||
( "SELECT {0}, {1}"
|
||||
, "FROM {2}"
|
||||
, "WHERE {0} IN ({3})"
|
||||
), page_tbl.Fld_page_ns(), page_tbl.Fld_page_title()
|
||||
, page_tbl.Tbl_name()
|
||||
, ns_list
|
||||
);
|
||||
gplx.dbs.Db_rdr rdr = page_tbl.Conn().Stmt_sql(sql).Exec_select__rls_auto();
|
||||
try {
|
||||
int counter = 0;
|
||||
while (rdr.Move_next()) {
|
||||
int ns = rdr.Read_int(page_tbl.Fld_page_ns());
|
||||
byte[] page_db = rdr.Read_bry_by_str(page_tbl.Fld_page_title());
|
||||
Xoa_ttl ttl = wiki.Ttl_parse(ns, page_db);
|
||||
cache.Add(ttl);
|
||||
if (counter % 100000 == 0) Gfo_usr_dlg_.Instance.Prog_many("", "", "loading ifexists: " + counter);
|
||||
counter++;
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
|
||||
// mark ns
|
||||
int[] ns_ids = Int_.Ary_parse(ns_list, ",");
|
||||
cache.Add_ns_loaded(ns_ids);
|
||||
}
|
||||
}
|
||||
|
@ -18,23 +18,24 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
|
||||
import gplx.core.ios.streams.*;
|
||||
public class Xomp_parse_mgr_cfg implements Gfo_invk {
|
||||
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1;
|
||||
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = -1;
|
||||
public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000;
|
||||
public int Progress_interval() {return progress_interval;} private int progress_interval = 1000;
|
||||
public int Commit_interval() {return commit_interval;} private int commit_interval = 10000;
|
||||
public int Cleanup_interval() {return cleanup_interval;} private int cleanup_interval = 50; // setting at 1000 uses lots of memory
|
||||
public boolean Hdump_enabled() {return hdump_enabled;} private boolean hdump_enabled = true;
|
||||
public boolean Hdump_catboxs() {return hdump_catboxs;} private boolean hdump_catboxs = false;
|
||||
public boolean Hzip_enabled() {return hzip_enabled;} private boolean hzip_enabled = true;
|
||||
public boolean Hdiff_enabled() {return hdiff_enabled;} private boolean hdiff_enabled = true;
|
||||
public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true;
|
||||
public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true;
|
||||
public boolean Load_all_imglnks() {return load_all_imglnks;} private boolean load_all_imglnks = true;
|
||||
public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_tid_.Tid__gzip;
|
||||
public Io_url Mgr_url() {return mgr_url;} private Io_url mgr_url;
|
||||
public String Wkr_machine_name() {return wkr_machine_name;} private String wkr_machine_name;
|
||||
public boolean Show_msg__fetched_pool() {return show_msg__fetched_pool;} private boolean show_msg__fetched_pool;
|
||||
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1;
|
||||
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = -1;
|
||||
public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000;
|
||||
public int Progress_interval() {return progress_interval;} private int progress_interval = 1000;
|
||||
public int Commit_interval() {return commit_interval;} private int commit_interval = 10000;
|
||||
public int Cleanup_interval() {return cleanup_interval;} private int cleanup_interval = 50; // setting at 1000 uses lots of memory
|
||||
public boolean Hdump_enabled() {return hdump_enabled;} private boolean hdump_enabled = true;
|
||||
public boolean Hdump_catboxs() {return hdump_catboxs;} private boolean hdump_catboxs = false;
|
||||
public boolean Hzip_enabled() {return hzip_enabled;} private boolean hzip_enabled = true;
|
||||
public boolean Hdiff_enabled() {return hdiff_enabled;} private boolean hdiff_enabled = true;
|
||||
public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true;
|
||||
public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true;
|
||||
public boolean Load_all_imglinks() {return load_all_imglinks;} private boolean load_all_imglinks = true;
|
||||
public String Load_ifexists_ns() {return load_ifexists_ns;} private String load_ifexists_ns = null;
|
||||
public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_tid_.Tid__gzip;
|
||||
public Io_url Mgr_url() {return mgr_url;} private Io_url mgr_url;
|
||||
public String Wkr_machine_name() {return wkr_machine_name;} private String wkr_machine_name;
|
||||
public boolean Show_msg__fetched_pool() {return show_msg__fetched_pool;} private boolean show_msg__fetched_pool;
|
||||
public void Init(Xowe_wiki wiki) {
|
||||
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count();
|
||||
if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000;
|
||||
@ -54,7 +55,8 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
|
||||
else if (ctx.Match(k, Invk__hdiff_enabled_)) hdiff_enabled = m.ReadYn("v");
|
||||
else if (ctx.Match(k, Invk__zip_tid_)) zip_tid = m.ReadByte("v");
|
||||
else if (ctx.Match(k, Invk__load_all_templates_)) load_all_templates = m.ReadYn("v");
|
||||
else if (ctx.Match(k, Invk__load_all_imglnks_)) load_all_imglnks = m.ReadYn("v");
|
||||
else if (ctx.Match(k, Invk__load_all_imglinks_)) load_all_imglinks = m.ReadYn("v");
|
||||
else if (ctx.Match(k, Invk__load_ifexists_ns_)) load_ifexists_ns = m.ReadStr("v");
|
||||
else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v"));
|
||||
else if (ctx.Match(k, Invk__mgr_url_)) mgr_url = m.ReadIoUrl("v");
|
||||
else if (ctx.Match(k, Invk__wkr_machine_name_)) wkr_machine_name = m.ReadStr("v");
|
||||
@ -67,7 +69,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
|
||||
Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
|
||||
, Invk__progress_interval_ = "progress_interval_", Invk__commit_interval_ = "commit_interval_", Invk__cleanup_interval_ = "cleanup_interval_"
|
||||
, Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_", Invk__zip_tid_ = "zip_tid_"
|
||||
, Invk__load_all_templates_ = "load_all_templates_", Invk__load_all_imglnks_ = "load_all_imglnks_", Invk__manual_now_ = "manual_now_"
|
||||
, Invk__load_all_templates_ = "load_all_templates_", Invk__load_all_imglinks_ = "load_all_imglinks_", Invk__load_ifexists_ns_ = "load_ifexists_ns_", Invk__manual_now_ = "manual_now_"
|
||||
, Invk__hdump_catboxes_ = "hdump_catboxes_"
|
||||
, Invk__mgr_url_ = "mgr_url_", Invk__wkr_machine_name_ = "wkr_machine_name_"
|
||||
, Invk__show_msg__fetched_pool_ = "show_msg__fetched_pool_"
|
||||
|
@ -32,7 +32,7 @@ public class Xomp_tmpl_cache_bldr {
|
||||
, ", pp.page_text_db_id"
|
||||
, ", pp.page_redirect_id"
|
||||
, "FROM page pp"
|
||||
, "WHERE pp.page_namespace IN (10, 828)"
|
||||
, "WHERE pp.page_namespace IN (8, 10, 828)"
|
||||
);
|
||||
|
||||
Xomp_text_db_loader text_db_loader = new Xomp_text_db_loader(wiki);
|
||||
|
@ -27,7 +27,7 @@ public class Xop_mediawiki_wkr {
|
||||
}
|
||||
public void Loader_(Xop_mediawiki_loader loader) {
|
||||
if (loader != null)
|
||||
wiki.Cache_mgr().Page_cache().Load_wkr_(new Xow_page_cache_wkr__embeddable(loader));
|
||||
wiki.Cache_mgr().Load_wkr_(new Xow_page_cache_wkr__embeddable(loader));
|
||||
}
|
||||
public void Free_memory() {
|
||||
wiki.Cache_mgr().Tmpl_result_cache().Clear();
|
||||
|
@ -24,18 +24,23 @@ public class Xow_cache_mgr {
|
||||
this.wiki = wiki;
|
||||
this.page_cache = new Xow_page_cache(wiki);
|
||||
this.defn_cache = new Xow_defn_cache(wiki.Lang());
|
||||
this.ifexist_cache = new Xow_ifexist_cache(wiki, page_cache);
|
||||
}
|
||||
public Hash_adp Tmpl_result_cache() {return tmpl_result_cache;} private final Hash_adp tmpl_result_cache = Hash_adp_bry.cs();
|
||||
public Xow_defn_cache Defn_cache() {return defn_cache;} private final Xow_defn_cache defn_cache;
|
||||
public Hash_adp_bry Lst_cache() {return lst_cache;} private final Hash_adp_bry lst_cache = Hash_adp_bry.cs();
|
||||
public Hash_adp Misc_cache() {return misc_cache;} private final Hash_adp misc_cache = Hash_adp_.New();
|
||||
public Xow_page_cache Page_cache() {return page_cache;} private Xow_page_cache page_cache;
|
||||
public Gfo_cache_mgr Commons_cache() {return commons_cache;} private Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
|
||||
public Gfo_cache_mgr Ifexist_cache() {return ifexist_cache;} private Gfo_cache_mgr ifexist_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
|
||||
public Hash_adp Tmpl_result_cache() {return tmpl_result_cache;} private final Hash_adp tmpl_result_cache = Hash_adp_bry.cs();
|
||||
public Xow_defn_cache Defn_cache() {return defn_cache;} private final Xow_defn_cache defn_cache;
|
||||
public Hash_adp_bry Lst_cache() {return lst_cache;} private final Hash_adp_bry lst_cache = Hash_adp_bry.cs();
|
||||
public Hash_adp Misc_cache() {return misc_cache;} private final Hash_adp misc_cache = Hash_adp_.New();
|
||||
public Xow_page_cache Page_cache() {return page_cache;} private Xow_page_cache page_cache;
|
||||
public Gfo_cache_mgr Commons_cache() {return commons_cache;} private Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
|
||||
public Xow_ifexist_cache Ifexist_cache() {return ifexist_cache;} private Xow_ifexist_cache ifexist_cache;
|
||||
|
||||
public Xow_cache_mgr Page_cache_(Xow_page_cache v) {this.page_cache = v; return this;}
|
||||
public Xow_cache_mgr Commons_cache_(Gfo_cache_mgr v) {this.commons_cache = v; return this;}
|
||||
public Xow_cache_mgr Ifexist_cache_(Gfo_cache_mgr v) {this.ifexist_cache = v; return this;}
|
||||
public Xow_cache_mgr Ifexist_cache_(Xow_ifexist_cache v) {this.ifexist_cache = v; return this;}
|
||||
public void Load_wkr_(Xow_page_cache_wkr v) {
|
||||
page_cache.Load_wkr_(v);
|
||||
ifexist_cache.Load_wkr_(v);
|
||||
}
|
||||
public Keyval[] Scrib_lang_names() {
|
||||
if (scrib_lang_names == null) {
|
||||
List_adp list = List_adp_.New();
|
||||
|
90
400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java
Normal file
90
400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java
Normal file
@ -0,0 +1,90 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
|
||||
import gplx.core.caches.*;
|
||||
import gplx.xowa.wikis.data.tbls.*;
|
||||
public class Xow_ifexist_cache {
|
||||
private final Xowe_wiki wiki;
|
||||
private final Xow_page_cache page_cache;
|
||||
private final Gfo_cache_mgr cache_mgr = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
|
||||
private final Hash_adp ns_loaded_hash = Hash_adp_.New();
|
||||
private final Xow_ifexist_itm itm__exists = new Xow_ifexist_itm(Bool_.Y), itm__missing = new Xow_ifexist_itm(Bool_.N);
|
||||
public Xow_ifexist_cache(Xowe_wiki wiki, Xow_page_cache page_cache) {
|
||||
this.wiki = wiki;
|
||||
this.page_cache = page_cache;
|
||||
}
|
||||
public Xow_ifexist_cache Cache_sizes_(int max, int reduce) {
|
||||
cache_mgr.Max_size_(max).Reduce_by_(reduce);
|
||||
return this;
|
||||
}
|
||||
public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;} private Xow_page_cache_wkr load_wkr;
|
||||
public void Clear() {cache_mgr.Clear();}
|
||||
public void Add(Xoa_ttl ttl) {
|
||||
byte[] key = ttl.Full_db();
|
||||
cache_mgr.Add(key, itm__exists, key.length);
|
||||
}
|
||||
public void Add_ns_loaded(int... ns_ids) {
|
||||
for (int ns_id : ns_ids)
|
||||
ns_loaded_hash.Add(ns_id, ns_id);
|
||||
}
|
||||
public byte Get_by_mem(Xoa_ttl ttl) {
|
||||
byte[] ttl_full_db = ttl.Full_db();
|
||||
|
||||
// check cache_mgr
|
||||
Xow_ifexist_itm found = (Xow_ifexist_itm)cache_mgr.Get_by_key(ttl_full_db);
|
||||
if (found != null) return found.Exists() ? Bool_.Y_byte : Bool_.N_byte;
|
||||
|
||||
// check ns_loaded cache (xomp only); if exists, return false, since all pages in ns are loaded, and still not found
|
||||
if (ns_loaded_hash.Has(ttl.Ns().Id())) return Bool_.N_byte;
|
||||
|
||||
// check page_cache since full page + text could be loaded there
|
||||
Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_or_null(ttl_full_db);
|
||||
if (itm == Xow_page_cache_itm.Missing)
|
||||
return Bool_.N_byte;
|
||||
else if (itm != null)
|
||||
return Bool_.Y_byte;
|
||||
|
||||
return Bool_.__byte;
|
||||
}
|
||||
public boolean Load(Xoa_ttl ttl) {
|
||||
byte[] key = ttl.Full_db();
|
||||
Xow_ifexist_itm itm = null;
|
||||
// gplx.core.consoles.Console_adp__sys.Instance.Write_str("ifexist_cache:" + String_.new_u8(key));
|
||||
|
||||
if (load_wkr != null) {
|
||||
// load_wkr; should call ifexist method, but for now, load entire page
|
||||
byte[] page_text = load_wkr.Get_page_or_null(key);
|
||||
itm = page_text == null ? itm__missing : itm__exists;
|
||||
}
|
||||
else {
|
||||
// page_tbl
|
||||
Xowd_page_itm page_itm = new Xowd_page_itm();
|
||||
wiki.Db_mgr().Load_mgr().Load_by_ttl(page_itm, ttl.Ns(), ttl.Page_db());
|
||||
itm = page_itm.Exists() ? itm__exists : itm__missing;
|
||||
}
|
||||
|
||||
// add
|
||||
cache_mgr.Add(key, itm, key.length);
|
||||
return itm == itm__exists;
|
||||
}
|
||||
}
|
||||
class Xow_ifexist_itm implements Rls_able {
|
||||
public Xow_ifexist_itm(boolean exists) {this.exists = exists;}
|
||||
public boolean Exists() {return exists;} private final boolean exists;
|
||||
public void Rls() {}
|
||||
}
|
@ -20,55 +20,14 @@ public class Xow_page_cache {
|
||||
private final Object thread_lock = new Object();
|
||||
private final Xowe_wiki wiki;
|
||||
private final Ordered_hash cache = Ordered_hash_.New_bry(); // NOTE: wiki titles are not case-sensitive when ns is "1st-letter" (EX: w:earth an w:Earth); in these cases, two entries will be stored
|
||||
private final Hash_adp ifexists_cache = Hash_adp_bry.cs();
|
||||
private final List_adp deleted = List_adp_.New();
|
||||
private Xow_page_cache_wkr load_wkr;
|
||||
private final List_adp deleted = List_adp_.New();
|
||||
public Xow_page_cache(Xowe_wiki wiki) {this.wiki = wiki;}
|
||||
public Xow_page_cache_wkr Load_wkr() {return load_wkr;}
|
||||
public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;}
|
||||
public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;} private Xow_page_cache_wkr load_wkr;
|
||||
public Xow_page_cache_itm Get_or_null(byte[] ttl_full_db) {return (Xow_page_cache_itm)cache.Get_by(ttl_full_db);}
|
||||
public byte[] Get_or_load_as_src(Xoa_ttl ttl) {
|
||||
Xow_page_cache_itm rv = Get_or_load_as_itm(ttl);
|
||||
return rv == null ? null : rv.Wtxt__direct();
|
||||
}
|
||||
public boolean Get_ifexist_by_mem(byte[] ifexists_key) {
|
||||
// check ifexists_cache
|
||||
Xow_page_cache_itm ifexists_itm = (Xow_page_cache_itm)ifexists_cache.Get_by(ifexists_key);
|
||||
if (ifexists_itm == Xow_page_cache_itm.Missing)
|
||||
return false;
|
||||
else if (ifexists_itm != null)
|
||||
return true;
|
||||
|
||||
// check page_cache
|
||||
ifexists_itm = (Xow_page_cache_itm)cache.Get_by(ifexists_key);
|
||||
if (ifexists_itm == Xow_page_cache_itm.Missing)
|
||||
return false;
|
||||
else if (ifexists_itm != null)
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
public boolean Load_ifexist(Xoa_ttl ttl) {
|
||||
byte[] ifexists_key = ttl.Full_db();
|
||||
Xow_page_cache_itm ifexists_itm = null;
|
||||
|
||||
// do load
|
||||
if (load_wkr != null) {
|
||||
// load_wkr
|
||||
byte[] page_text = load_wkr.Get_page_or_null(ifexists_key);
|
||||
ifexists_itm = page_text == null ? Xow_page_cache_itm.Missing : new Xow_page_cache_itm(false, ttl, Bry_.Empty, Bry_.Empty);
|
||||
}
|
||||
else {
|
||||
// page_tbl
|
||||
gplx.xowa.wikis.data.tbls.Xowd_page_itm page_itm = new gplx.xowa.wikis.data.tbls.Xowd_page_itm();
|
||||
wiki.Db_mgr().Load_mgr().Load_by_ttl(page_itm, ttl.Ns(), ttl.Page_db());
|
||||
// wiki.Data__core_mgr().Db__core().Tbl__page().Select_by_ttl(page_itm, ttl);
|
||||
ifexists_itm = page_itm.Exists() ? new Xow_page_cache_itm(false, ttl, Bry_.Empty, Bry_.Empty) : Xow_page_cache_itm.Missing;
|
||||
}
|
||||
|
||||
// add to ifexists_cache only (not page_cache)
|
||||
ifexists_cache.Add(ifexists_key, ifexists_itm);
|
||||
return ifexists_itm != Xow_page_cache_itm.Missing;
|
||||
}
|
||||
public void Add(byte[] ttl_full_db, Xow_page_cache_itm itm) {
|
||||
cache.Add(ttl_full_db, itm);
|
||||
}
|
||||
@ -96,6 +55,7 @@ public class Xow_page_cache {
|
||||
boolean page_exists = false;
|
||||
byte[] page_text = null;
|
||||
byte[] page_redirect_from = null;
|
||||
// gplx.core.consoles.Console_adp__sys.Instance.Write_str("page_cache:" + String_.new_u8(ttl_full_db));
|
||||
if (load_wkr != null) {
|
||||
page_text = load_wkr.Get_page_or_null(ttl_full_db);
|
||||
page_exists = page_text != null;
|
||||
@ -139,7 +99,6 @@ public class Xow_page_cache {
|
||||
synchronized (thread_lock) { // LOCK:app-level; DATE:2016-07-06
|
||||
if (clear_permanent_itms) {
|
||||
cache.Clear();
|
||||
ifexists_cache.Clear();
|
||||
}
|
||||
else {
|
||||
int len = cache.Count();
|
||||
|
@ -20,26 +20,29 @@ import gplx.core.envs.*; import gplx.core.caches.*;
|
||||
import gplx.xowa.apps.wms.apis.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.wikis.nss.*;
|
||||
public class Pfunc_ifexist_mgr {
|
||||
private final Xowd_page_itm db_page = Xowd_page_itm.new_tmp();
|
||||
public boolean Exists(Xowe_wiki wiki, byte[] raw_bry) {
|
||||
if (Bry_.Len_eq_0(raw_bry)) return false; // return early; NOTE: {{autolink}} can pass in "" (see test)
|
||||
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, raw_bry); if (ttl == null) return false;
|
||||
byte[] ttl_bry = ttl.Page_db(); // NOTE: must use Page_db; EX: {{#ifexist:File:Peter & Paul fortress in SPB 03.jpg|y|n}}
|
||||
|
||||
// try to get from cache
|
||||
if (wiki.Cache_mgr().Page_cache().Get_ifexist_by_mem(ttl_bry)) return true;
|
||||
db_page.Clear();
|
||||
Xow_ns ttl_ns = ttl.Ns();
|
||||
byte exists = wiki.Cache_mgr().Ifexist_cache().Get_by_mem(ttl);
|
||||
if (exists == Bool_.Y_byte) return true;
|
||||
else if (exists == Bool_.N_byte) return false;
|
||||
|
||||
byte[] page_db = ttl.Page_db(); // NOTE: must use Page_db; EX: {{#ifexist:File:Peter & Paul fortress in SPB 03.jpg|y|n}}
|
||||
Xow_ns ns = ttl.Ns();
|
||||
boolean rv = false;
|
||||
switch (ttl_ns.Id()) {
|
||||
switch (ns.Id()) {
|
||||
case Xow_ns_.Tid__special: rv = true; break; // NOTE: some pages call for [[Special]]; always return true for now; DATE:2014-07-17
|
||||
case Xow_ns_.Tid__media: rv = Find_ttl_for_media_ns(wiki, ttl_ns, ttl_bry); break;
|
||||
default: rv = Find_ttl_in_db(wiki, ttl, ttl_ns, ttl_bry); break;
|
||||
case Xow_ns_.Tid__media: rv = Find_by_ns__media(wiki, ns, page_db); break;
|
||||
default: rv = Find_by_ns(wiki, ttl, ns, page_db); break;
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
private boolean Find_ttl_in_db(Xowe_wiki wiki, Xoa_ttl ttl, Xow_ns ns, byte[] ttl_bry) {
|
||||
boolean rv = wiki.Cache_mgr().Page_cache().Load_ifexist(ttl);
|
||||
private boolean Find_by_ns(Xowe_wiki wiki, Xoa_ttl ttl, Xow_ns ns, byte[] ttl_bry) {
|
||||
boolean rv = wiki.Cache_mgr().Ifexist_cache().Load(ttl);
|
||||
|
||||
// handle variants
|
||||
if ( !rv
|
||||
&& wiki.Lang().Vnt_mgr().Enabled()) {
|
||||
Xowd_page_itm page = wiki.Lang().Vnt_mgr().Convert_mgr().Convert_ttl(wiki, ns, ttl_bry);
|
||||
@ -48,10 +51,18 @@ public class Pfunc_ifexist_mgr {
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
private boolean Find_ttl_for_media_ns(Xowe_wiki wiki, Xow_ns ns, byte[] ttl_bry) {
|
||||
private boolean Find_by_ns__media(Xowe_wiki wiki, Xow_ns ns, byte[] page_db) {
|
||||
// rarely true, but check local wiki's [[File:]] table anyway
|
||||
Xow_ns file_ns = wiki.Ns_mgr().Ns_file();
|
||||
Xoa_ttl file_ttl = wiki.Ttl_parse(file_ns.Id(), ttl_bry);
|
||||
boolean rv = Find_ttl_in_db(wiki, file_ttl, file_ns, ttl_bry); if (rv) return true; // rarely true, but check local wiki's [[File:]] table anyway
|
||||
Xoa_ttl file_ttl = wiki.Ttl_parse(file_ns.Id(), page_db);
|
||||
|
||||
byte exists = wiki.Cache_mgr().Ifexist_cache().Get_by_mem(file_ttl);
|
||||
if (exists == Bool_.Y_byte) return true;
|
||||
else if (exists == Bool_.N_byte) return false;
|
||||
|
||||
if (Find_by_ns(wiki, file_ttl, file_ns, page_db)) return true;
|
||||
|
||||
// check commons; either (a) commons_wiki directly; or (b) tdb's file_mgr; note that (b) is obsolete
|
||||
Xowe_wiki commons_wiki = wiki.Appe().Wiki_mgr().Wiki_commons();
|
||||
boolean env_is_testing = Env_.Mode_testing();
|
||||
if ( commons_wiki != null // null check
|
||||
@ -60,18 +71,12 @@ public class Pfunc_ifexist_mgr {
|
||||
)
|
||||
) {
|
||||
file_ns = commons_wiki.Ns_mgr().Ns_file();
|
||||
return Find_ttl_in_db(commons_wiki, file_ttl, file_ns, ttl_bry); // accurate test using page table in commons wiki (provided commons is up to date)
|
||||
return Find_by_ns(commons_wiki, file_ttl, file_ns, page_db); // accurate test using page table in commons wiki (provided commons is up to date)
|
||||
}
|
||||
else {
|
||||
if (!env_is_testing)
|
||||
wiki.File_mgr().Init_file_mgr_by_load(wiki); // NOTE: must init Fsdb_mgr (else conn == null), and with bin_wkrs (else no images will ever load); DATE:2014-09-21
|
||||
return wiki.File_mgr().Exists(ttl_bry); // less-accurate test using either (1) orig_wiki table in local wiki (v2) or (2) meta_db_mgr (v1)
|
||||
return wiki.File_mgr().Exists(page_db); // less-accurate test using either (1) orig_wiki table in local wiki (v2) or (2) meta_db_mgr (v1)
|
||||
}
|
||||
}
|
||||
}
|
||||
class Pfunc_ifexist_itm implements Rls_able {
|
||||
public Pfunc_ifexist_itm(byte[] ttl) {this.ttl = ttl;}
|
||||
public byte[] Ttl() {return ttl;} private byte[] ttl;
|
||||
public boolean Exists() {return exists;} public void Exists_(boolean v) {exists = v;} private boolean exists;
|
||||
public void Rls() {}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user