Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2024-10-27 20:34:16 +00:00
Code Issues Projects Releases Wiki Activity

ParserFunctions: Cache ifexist result of common wiki in current wiki

Browse Source
This commit is contained in:
gnosygnu 2018-07-03 08:42:02 -04:00
parent eb9cca66ed
commit f85cf8ad77
4 changed files with 44 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java
View File

@ -124,7 +124,7 @@ public class Xomp_parse_mgr {
int ns = rdr.Read_int(page_tbl.Fld_page_ns());
byte[] page_db = rdr.Read_bry_by_str(page_tbl.Fld_page_title());
Xoa_ttl ttl = wiki.Ttl_parse(ns, page_db);
cache.Add(ttl);
cache.Add(ttl, true);
if (counter % 100000 == 0) Gfo_usr_dlg_.Instance.Prog_many("", "", "loading ifexists: " + counter);
counter++;
}
@ -132,6 +132,6 @@ public class Xomp_parse_mgr {
// mark ns
int[] ns_ids = Int_ary_.Parse(ns_list, ",");
cache.Add_ns_loaded(ns_ids);
cache.Mark_ns_loaded(ns_ids);
}
}

32
400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java
View File

@ -21,7 +21,6 @@ public class Xow_ifexist_cache {
private final Xow_page_cache page_cache;
private final Gfo_cache_mgr cache_mgr = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
private final Hash_adp ns_loaded_hash = Hash_adp_.New();
private final Xow_ifexist_itm itm__exists = new Xow_ifexist_itm(Bool_.Y), itm__missing = new Xow_ifexist_itm(Bool_.N);
public Xow_ifexist_cache(Xowe_wiki wiki, Xow_page_cache page_cache) {
this.wiki = wiki;
this.page_cache = page_cache;
@ -35,26 +34,26 @@ public class Xow_ifexist_cache {
cache_mgr.Clear();
ns_loaded_hash.Clear();
}
public void Add(Xoa_ttl ttl) {
public void Add(Xoa_ttl ttl, boolean exists) {
byte[] key = ttl.Full_db();
cache_mgr.Add(key, itm__exists, key.length);
cache_mgr.Add_replace(key, Xow_ifexist_itm.Get(exists), key.length);
}
public void Add_ns_loaded(int... ns_ids) {
public void Mark_ns_loaded(int... ns_ids) {
for (int ns_id : ns_ids)
ns_loaded_hash.Add(ns_id, ns_id);
}
public byte Get_by_mem(Xoa_ttl ttl) {
byte[] ttl_full_db = ttl.Full_db();
public byte Get_by_cache(Xoa_ttl ttl) {
byte[] key = ttl.Full_db();
// check cache_mgr
Xow_ifexist_itm found = (Xow_ifexist_itm)cache_mgr.Get_by_key(ttl_full_db);
Xow_ifexist_itm found = (Xow_ifexist_itm)cache_mgr.Get_by_key(key);
if (found != null) return found.Exists() ? Bool_.Y_byte : Bool_.N_byte;
// check ns_loaded cache (xomp only); if exists, return false, since all pages in ns are loaded, and still not found
// check ns_loaded cache (xomp only); if ns_exists, return false, since all pages in ns are loaded, and still not found
if (ns_loaded_hash.Has(ttl.Ns().Id())) return Bool_.N_byte;
// check page_cache since full page + text could be loaded there
Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_or_null(ttl_full_db);
Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_or_null(key);
if (itm == Xow_page_cache_itm.Missing)
return Bool_.N_byte;
else if (itm != null)
@ -62,7 +61,7 @@ public class Xow_ifexist_cache {
return Bool_.__byte;
}
public boolean Load(Xoa_ttl ttl) {
public boolean Get_by_load(Xoa_ttl ttl) {
byte[] key = ttl.Full_db();
Xow_ifexist_itm itm = null;
// gplx.core.consoles.Console_adp__sys.Instance.Write_str("ifexist_cache:" + String_.new_u8(key));
@ -70,22 +69,25 @@ public class Xow_ifexist_cache {
if (load_wkr != null) {
// load_wkr; should call ifexist method, but for now, load entire page
byte[] page_text = load_wkr.Get_page_or_null(key);
itm = page_text == null ? itm__missing : itm__exists;
itm = Xow_ifexist_itm.Get(page_text != null);
}
else {
// page_tbl
Xowd_page_itm page_itm = new Xowd_page_itm();
wiki.Db_mgr().Load_mgr().Load_by_ttl(page_itm, ttl.Ns(), ttl.Page_db());
itm = page_itm.Exists() ? itm__exists : itm__missing;
itm = Xow_ifexist_itm.Get(page_itm.Exists());
}
// add
cache_mgr.Add(key, itm, key.length);
return itm == itm__exists;
return itm.Exists();
}
}
class Xow_ifexist_itm implements Rls_able {
public Xow_ifexist_itm(boolean exists) {this.exists = exists;}
Xow_ifexist_itm(boolean exists) {this.exists = exists;}
public boolean Exists() {return exists;} private final boolean exists;
public void Rls() {}
public void Rls() {}
private static final Xow_ifexist_itm Itm__exists = new Xow_ifexist_itm(Bool_.Y), Itm__missing = new Xow_ifexist_itm(Bool_.N);
public static Xow_ifexist_itm Get(boolean exists) {return exists ? Itm__exists : Itm__missing;}
}

42
400_xowa/src/gplx/xowa/xtns/pfuncs/ifs/Pfunc_ifexist_mgr.java
View File

@ -19,63 +19,69 @@ import gplx.xowa.apps.wms.apis.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.wikis.nss.*;
public class Pfunc_ifexist_mgr {
public boolean Exists(Xowe_wiki wiki, byte[] raw_bry) {
// validate ttl; return false if invalid
if (Bry_.Len_eq_0(raw_bry)) return false; // return early; NOTE: {{autolink}} can pass in "" (see test)
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, raw_bry); if (ttl == null) return false;
// try to get from cache
byte exists = wiki.Cache_mgr().Ifexist_cache().Get_by_mem(ttl);
byte exists = wiki.Cache_mgr().Ifexist_cache().Get_by_cache(ttl);
if (exists == Bool_.Y_byte) return true;
else if (exists == Bool_.N_byte) return false;
byte[] page_db = ttl.Page_db(); // NOTE: must use Page_db; EX: {{#ifexist:File:Peter & Paul fortress in SPB 03.jpg|y|n}}
Xow_ns ns = ttl.Ns();
boolean rv = false;
switch (ns.Id()) {
case Xow_ns_.Tid__special: rv = true; break; // NOTE: some pages call for [[Special]]; always return true for now; DATE:2014-07-17
case Xow_ns_.Tid__media: rv = Find_by_ns__media(wiki, ns, page_db); break;
default: rv = Find_by_ns(wiki, ttl, ns, page_db); break;
// not in cache; do find
switch (ttl.Ns().Id()) {
case Xow_ns_.Tid__special: return true; // NOTE: some pages call for [[Special]]; always return true for now; DATE:2014-07-17
case Xow_ns_.Tid__media: return Find_by_ns__media(wiki, ttl);
default: return Find_by_ns__other(wiki, ttl);
}
return rv;
}
private boolean Find_by_ns(Xowe_wiki wiki, Xoa_ttl ttl, Xow_ns ns, byte[] ttl_bry) {
boolean rv = wiki.Cache_mgr().Ifexist_cache().Load(ttl);
private boolean Find_by_ns__other(Xowe_wiki wiki, Xoa_ttl ttl) {
boolean rv = wiki.Cache_mgr().Ifexist_cache().Get_by_load(ttl);
// handle variants
if ( !rv
&& wiki.Lang().Vnt_mgr().Enabled()) {
Xowd_page_itm page = wiki.Lang().Vnt_mgr().Convert_mgr().Convert_ttl(wiki, ns, ttl_bry);
Xowd_page_itm page = wiki.Lang().Vnt_mgr().Convert_mgr().Convert_ttl(wiki, ttl.Ns(), ttl.Page_db());
if (page != Xowd_page_itm.Null)
rv = page.Exists();
}
return rv;
}
private boolean Find_by_ns__media(Xowe_wiki wiki, Xow_ns ns, byte[] page_db) {
private boolean Find_by_ns__media(Xowe_wiki wiki, Xoa_ttl ttl) {
// rarely true, but check local wiki's [[File:]] table anyway
Xow_ns file_ns = wiki.Ns_mgr().Ns_file();
byte[] page_db = ttl.Page_db(); // NOTE: must use Page_db; also handles Media:A.svg -> File:A.svg; EX: {{#ifexist:File:Peter & Paul fortress in SPB 03.jpg|y|n}}
Xoa_ttl file_ttl = wiki.Ttl_parse(file_ns.Id(), page_db);
if (file_ttl == null) return false; // NOTE: must check for NPE: PAGE:es.w:Elecciones_presidenciales_de_Venezuela_de_1998 DATE:2017-09-04
byte exists = wiki.Cache_mgr().Ifexist_cache().Get_by_mem(file_ttl);
// check cache
byte exists = wiki.Cache_mgr().Ifexist_cache().Get_by_cache(file_ttl);
if (exists == Bool_.Y_byte) return true;
else if (exists == Bool_.N_byte) return false;
if (Find_by_ns(wiki, file_ttl, file_ns, page_db)) return true;
// call Find_by_ns_other which will call load, but with "File:" instead of "Media:"
if (Find_by_ns__other(wiki, file_ttl)) return true;
// check commons; either (a) commons_wiki directly; or (b) tdb's file_mgr; note that (b) is obsolete
Xowe_wiki commons_wiki = wiki.Appe().Wiki_mgr().Wiki_commons();
boolean env_is_testing = Env_.Mode_testing();
boolean rv = false;
if ( commons_wiki != null // null check
&& ( commons_wiki.Init_assert().Db_mgr().Tid() == gplx.xowa.wikis.dbs.Xodb_mgr_sql.Tid_sql // make sure tid=sql; tid=txt automatically created for online images; DATE:2014-09-21
|| env_is_testing
)
) {
file_ns = commons_wiki.Ns_mgr().Ns_file();
return Find_by_ns(commons_wiki, file_ttl, file_ns, page_db); // accurate test using page table in commons wiki (provided commons is up to date)
rv = Find_by_ns__other(commons_wiki, file_ttl); // accurate test using page table in commons wiki (provided commons is up to date)
}
else {
if (!env_is_testing)
wiki.File_mgr().Init_file_mgr_by_load(wiki); // NOTE: must init Fsdb_mgr (else conn == null), and with bin_wkrs (else no images will ever load); DATE:2014-09-21
return wiki.File_mgr().Exists(page_db); // less-accurate test using either (1) orig_wiki table in local wiki (v2) or (2) meta_db_mgr (v1)
wiki.File_mgr().Init_file_mgr_by_load(wiki); // NOTE: must init Fsdb_mgr (else conn == null), and with bin_wkrs (else no images will ever load); DATE:2014-09-21
rv = wiki.File_mgr().Exists(page_db); // less-accurate test using either (1) orig_wiki table in local wiki (v2) or (2) meta_db_mgr (v1)
}
// add commons result to this wiki's cache; needed b/c cache will have "false" value (b/c Page does not exist in wiki), but should be "true" (since it exists in commons) PAGE:en.w:Harstad DATE:2018-07-03
wiki.Cache_mgr().Ifexist_cache().Add(file_ttl, rv);
return rv;
}
}

1
400_xowa/src/gplx/xowa/xtns/pfuncs/ifs/Pfunc_ifexist_tst.java
View File

@ -38,6 +38,7 @@ public class Pfunc_ifexist_tst {
Xowe_wiki commons_wiki = fxt.App().Wiki_mgr().Get_by_or_make(gplx.xowa.wikis.domains.Xow_domain_itm_.Bry__commons);
fxt.Init_page_create(commons_wiki, "File:A.png", "");
fxt.Test_parse_tmpl_str_test("{{#ifexist:Media:A.png|y|n}}", "{{test}}", "y");
fxt.Test_parse_tmpl_str_test("{{#ifexist:Media:A.png|y|n}}", "{{test}}", "y"); // BUG:2nd call actually ends up as n; PAGE:en.w:Harstad DATE:2018-07-03
}
@Test public void Media_y_file_v1() {// DATE:2014-07-04
Xof_meta_itm meta_itm = fxt.Wiki().File_mgr().Dbmeta_mgr().Get_itm_or_new(Bry_.new_a7("A.png"));