Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2024-10-27 20:34:16 +00:00
Code Issues Projects Releases Wiki Activity

ParserFunctions: Cache ifexist result of common wiki in current wiki

Browse Source
This commit is contained in:
gnosygnu 2018-07-03 08:42:02 -04:00
parent eb9cca66ed
commit f85cf8ad77
4 changed files with 44 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java
View File

@ -124,7 +124,7 @@ public class Xomp_parse_mgr {
int ns = rdr.Read_int(page_tbl.Fld_page_ns()); int ns = rdr.Read_int(page_tbl.Fld_page_ns());
byte[] page_db = rdr.Read_bry_by_str(page_tbl.Fld_page_title()); byte[] page_db = rdr.Read_bry_by_str(page_tbl.Fld_page_title());
Xoa_ttl ttl = wiki.Ttl_parse(ns, page_db); Xoa_ttl ttl = wiki.Ttl_parse(ns, page_db);
cache.Add(ttl); cache.Add(ttl, true);
if (counter % 100000 == 0) Gfo_usr_dlg_.Instance.Prog_many("", "", "loading ifexists: " + counter); if (counter % 100000 == 0) Gfo_usr_dlg_.Instance.Prog_many("", "", "loading ifexists: " + counter);
counter++; counter++;
} }
@ -132,6 +132,6 @@ public class Xomp_parse_mgr {
// mark ns // mark ns
int[] ns_ids = Int_ary_.Parse(ns_list, ","); int[] ns_ids = Int_ary_.Parse(ns_list, ",");
cache.Add_ns_loaded(ns_ids); cache.Mark_ns_loaded(ns_ids);
} }
} }

30
400_xowa/src/gplx/xowa/wikis/caches/Xow_ifexist_cache.java
View File

@ -21,7 +21,6 @@ public class Xow_ifexist_cache {
private final Xow_page_cache page_cache; private final Xow_page_cache page_cache;
private final Gfo_cache_mgr cache_mgr = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb); private final Gfo_cache_mgr cache_mgr = new Gfo_cache_mgr().Max_size_(64 * Io_mgr.Len_mb).Reduce_by_(32 * Io_mgr.Len_mb);
private final Hash_adp ns_loaded_hash = Hash_adp_.New(); private final Hash_adp ns_loaded_hash = Hash_adp_.New();
private final Xow_ifexist_itm itm__exists = new Xow_ifexist_itm(Bool_.Y), itm__missing = new Xow_ifexist_itm(Bool_.N);
public Xow_ifexist_cache(Xowe_wiki wiki, Xow_page_cache page_cache) { public Xow_ifexist_cache(Xowe_wiki wiki, Xow_page_cache page_cache) {
this.wiki = wiki; this.wiki = wiki;
this.page_cache = page_cache; this.page_cache = page_cache;
@ -35,26 +34,26 @@ public class Xow_ifexist_cache {
cache_mgr.Clear(); cache_mgr.Clear();
ns_loaded_hash.Clear(); ns_loaded_hash.Clear();
} }
public void Add(Xoa_ttl ttl) { public void Add(Xoa_ttl ttl, boolean exists) {
byte[] key = ttl.Full_db(); byte[] key = ttl.Full_db();
cache_mgr.Add(key, itm__exists, key.length); cache_mgr.Add_replace(key, Xow_ifexist_itm.Get(exists), key.length);
} }
public void Add_ns_loaded(int... ns_ids) { public void Mark_ns_loaded(int... ns_ids) {
for (int ns_id : ns_ids) for (int ns_id : ns_ids)
ns_loaded_hash.Add(ns_id, ns_id); ns_loaded_hash.Add(ns_id, ns_id);
} }
public byte Get_by_mem(Xoa_ttl ttl) { public byte Get_by_cache(Xoa_ttl ttl) {
byte[] ttl_full_db = ttl.Full_db(); byte[] key = ttl.Full_db();
// check cache_mgr // check cache_mgr
Xow_ifexist_itm found = (Xow_ifexist_itm)cache_mgr.Get_by_key(ttl_full_db); Xow_ifexist_itm found = (Xow_ifexist_itm)cache_mgr.Get_by_key(key);
if (found != null) return found.Exists() ? Bool_.Y_byte : Bool_.N_byte; if (found != null) return found.Exists() ? Bool_.Y_byte : Bool_.N_byte;
// check ns_loaded cache (xomp only); if exists, return false, since all pages in ns are loaded, and still not found // check ns_loaded cache (xomp only); if ns_exists, return false, since all pages in ns are loaded, and still not found
if (ns_loaded_hash.Has(ttl.Ns().Id())) return Bool_.N_byte; if (ns_loaded_hash.Has(ttl.Ns().Id())) return Bool_.N_byte;
// check page_cache since full page + text could be loaded there // check page_cache since full page + text could be loaded there
Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_or_null(ttl_full_db); Xow_page_cache_itm itm = (Xow_page_cache_itm)page_cache.Get_or_null(key);
if (itm == Xow_page_cache_itm.Missing) if (itm == Xow_page_cache_itm.Missing)
return Bool_.N_byte; return Bool_.N_byte;
else if (itm != null) else if (itm != null)
@ -62,7 +61,7 @@ public class Xow_ifexist_cache {
return Bool_.__byte; return Bool_.__byte;
} }
public boolean Load(Xoa_ttl ttl) { public boolean Get_by_load(Xoa_ttl ttl) {
byte[] key = ttl.Full_db(); byte[] key = ttl.Full_db();
Xow_ifexist_itm itm = null; Xow_ifexist_itm itm = null;
// gplx.core.consoles.Console_adp__sys.Instance.Write_str("ifexist_cache:" + String_.new_u8(key)); // gplx.core.consoles.Console_adp__sys.Instance.Write_str("ifexist_cache:" + String_.new_u8(key));
@ -70,22 +69,25 @@ public class Xow_ifexist_cache {
if (load_wkr != null) { if (load_wkr != null) {
// load_wkr; should call ifexist method, but for now, load entire page // load_wkr; should call ifexist method, but for now, load entire page
byte[] page_text = load_wkr.Get_page_or_null(key); byte[] page_text = load_wkr.Get_page_or_null(key);
itm = page_text == null ? itm__missing : itm__exists; itm = Xow_ifexist_itm.Get(page_text != null);
} }
else { else {
// page_tbl // page_tbl
Xowd_page_itm page_itm = new Xowd_page_itm(); Xowd_page_itm page_itm = new Xowd_page_itm();
wiki.Db_mgr().Load_mgr().Load_by_ttl(page_itm, ttl.Ns(), ttl.Page_db()); wiki.Db_mgr().Load_mgr().Load_by_ttl(page_itm, ttl.Ns(), ttl.Page_db());
itm = page_itm.Exists() ? itm__exists : itm__missing; itm = Xow_ifexist_itm.Get(page_itm.Exists());
} }
// add // add
cache_mgr.Add(key, itm, key.length); cache_mgr.Add(key, itm, key.length);
return itm == itm__exists; return itm.Exists();
} }
} }
class Xow_ifexist_itm implements Rls_able { class Xow_ifexist_itm implements Rls_able {
public Xow_ifexist_itm(boolean exists) {this.exists = exists;} Xow_ifexist_itm(boolean exists) {this.exists = exists;}
public boolean Exists() {return exists;} private final boolean exists; public boolean Exists() {return exists;} private final boolean exists;
public void Rls() {} public void Rls() {}
private static final Xow_ifexist_itm Itm__exists = new Xow_ifexist_itm(Bool_.Y), Itm__missing = new Xow_ifexist_itm(Bool_.N);
public static Xow_ifexist_itm Get(boolean exists) {return exists ? Itm__exists : Itm__missing;}
} }

42
400_xowa/src/gplx/xowa/xtns/pfuncs/ifs/Pfunc_ifexist_mgr.java
View File

@ -19,63 +19,69 @@ import gplx.xowa.apps.wms.apis.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.nss.*;
public class Pfunc_ifexist_mgr { public class Pfunc_ifexist_mgr {
public boolean Exists(Xowe_wiki wiki, byte[] raw_bry) { public boolean Exists(Xowe_wiki wiki, byte[] raw_bry) {
// validate ttl; return false if invalid
if (Bry_.Len_eq_0(raw_bry)) return false; // return early; NOTE: {{autolink}} can pass in "" (see test) if (Bry_.Len_eq_0(raw_bry)) return false; // return early; NOTE: {{autolink}} can pass in "" (see test)
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, raw_bry); if (ttl == null) return false; Xoa_ttl ttl = Xoa_ttl.Parse(wiki, raw_bry); if (ttl == null) return false;
// try to get from cache // try to get from cache
byte exists = wiki.Cache_mgr().Ifexist_cache().Get_by_mem(ttl); byte exists = wiki.Cache_mgr().Ifexist_cache().Get_by_cache(ttl);
if (exists == Bool_.Y_byte) return true; if (exists == Bool_.Y_byte) return true;
else if (exists == Bool_.N_byte) return false; else if (exists == Bool_.N_byte) return false;
byte[] page_db = ttl.Page_db(); // NOTE: must use Page_db; EX: {{#ifexist:File:Peter & Paul fortress in SPB 03.jpg|y|n}} // not in cache; do find
Xow_ns ns = ttl.Ns(); switch (ttl.Ns().Id()) {
boolean rv = false; case Xow_ns_.Tid__special: return true; // NOTE: some pages call for [[Special]]; always return true for now; DATE:2014-07-17
switch (ns.Id()) { case Xow_ns_.Tid__media: return Find_by_ns__media(wiki, ttl);
case Xow_ns_.Tid__special: rv = true; break; // NOTE: some pages call for [[Special]]; always return true for now; DATE:2014-07-17 default: return Find_by_ns__other(wiki, ttl);
case Xow_ns_.Tid__media: rv = Find_by_ns__media(wiki, ns, page_db); break;
default: rv = Find_by_ns(wiki, ttl, ns, page_db); break;
} }
return rv;
} }
private boolean Find_by_ns(Xowe_wiki wiki, Xoa_ttl ttl, Xow_ns ns, byte[] ttl_bry) { private boolean Find_by_ns__other(Xowe_wiki wiki, Xoa_ttl ttl) {
boolean rv = wiki.Cache_mgr().Ifexist_cache().Load(ttl); boolean rv = wiki.Cache_mgr().Ifexist_cache().Get_by_load(ttl);
// handle variants // handle variants
if ( !rv if ( !rv
&& wiki.Lang().Vnt_mgr().Enabled()) { && wiki.Lang().Vnt_mgr().Enabled()) {
Xowd_page_itm page = wiki.Lang().Vnt_mgr().Convert_mgr().Convert_ttl(wiki, ns, ttl_bry); Xowd_page_itm page = wiki.Lang().Vnt_mgr().Convert_mgr().Convert_ttl(wiki, ttl.Ns(), ttl.Page_db());
if (page != Xowd_page_itm.Null) if (page != Xowd_page_itm.Null)
rv = page.Exists(); rv = page.Exists();
} }
return rv; return rv;
} }
private boolean Find_by_ns__media(Xowe_wiki wiki, Xow_ns ns, byte[] page_db) { private boolean Find_by_ns__media(Xowe_wiki wiki, Xoa_ttl ttl) {
// rarely true, but check local wiki's [[File:]] table anyway // rarely true, but check local wiki's [[File:]] table anyway
Xow_ns file_ns = wiki.Ns_mgr().Ns_file(); Xow_ns file_ns = wiki.Ns_mgr().Ns_file();
byte[] page_db = ttl.Page_db(); // NOTE: must use Page_db; also handles Media:A.svg -> File:A.svg; EX: {{#ifexist:File:Peter & Paul fortress in SPB 03.jpg|y|n}}
Xoa_ttl file_ttl = wiki.Ttl_parse(file_ns.Id(), page_db); Xoa_ttl file_ttl = wiki.Ttl_parse(file_ns.Id(), page_db);
if (file_ttl == null) return false; // NOTE: must check for NPE: PAGE:es.w:Elecciones_presidenciales_de_Venezuela_de_1998 DATE:2017-09-04 if (file_ttl == null) return false; // NOTE: must check for NPE: PAGE:es.w:Elecciones_presidenciales_de_Venezuela_de_1998 DATE:2017-09-04
byte exists = wiki.Cache_mgr().Ifexist_cache().Get_by_mem(file_ttl); // check cache
byte exists = wiki.Cache_mgr().Ifexist_cache().Get_by_cache(file_ttl);
if (exists == Bool_.Y_byte) return true; if (exists == Bool_.Y_byte) return true;
else if (exists == Bool_.N_byte) return false; else if (exists == Bool_.N_byte) return false;
if (Find_by_ns(wiki, file_ttl, file_ns, page_db)) return true; // call Find_by_ns_other which will call load, but with "File:" instead of "Media:"
if (Find_by_ns__other(wiki, file_ttl)) return true;
// check commons; either (a) commons_wiki directly; or (b) tdb's file_mgr; note that (b) is obsolete // check commons; either (a) commons_wiki directly; or (b) tdb's file_mgr; note that (b) is obsolete
Xowe_wiki commons_wiki = wiki.Appe().Wiki_mgr().Wiki_commons(); Xowe_wiki commons_wiki = wiki.Appe().Wiki_mgr().Wiki_commons();
boolean env_is_testing = Env_.Mode_testing(); boolean env_is_testing = Env_.Mode_testing();
boolean rv = false;
if ( commons_wiki != null // null check if ( commons_wiki != null // null check
&& ( commons_wiki.Init_assert().Db_mgr().Tid() == gplx.xowa.wikis.dbs.Xodb_mgr_sql.Tid_sql // make sure tid=sql; tid=txt automatically created for online images; DATE:2014-09-21 && ( commons_wiki.Init_assert().Db_mgr().Tid() == gplx.xowa.wikis.dbs.Xodb_mgr_sql.Tid_sql // make sure tid=sql; tid=txt automatically created for online images; DATE:2014-09-21
|| env_is_testing || env_is_testing
) )
) { ) {
file_ns = commons_wiki.Ns_mgr().Ns_file(); file_ns = commons_wiki.Ns_mgr().Ns_file();
return Find_by_ns(commons_wiki, file_ttl, file_ns, page_db); // accurate test using page table in commons wiki (provided commons is up to date) rv = Find_by_ns__other(commons_wiki, file_ttl); // accurate test using page table in commons wiki (provided commons is up to date)
} }
else { else {
if (!env_is_testing) if (!env_is_testing)
wiki.File_mgr().Init_file_mgr_by_load(wiki); // NOTE: must init Fsdb_mgr (else conn == null), and with bin_wkrs (else no images will ever load); DATE:2014-09-21 wiki.File_mgr().Init_file_mgr_by_load(wiki); // NOTE: must init Fsdb_mgr (else conn == null), and with bin_wkrs (else no images will ever load); DATE:2014-09-21
return wiki.File_mgr().Exists(page_db); // less-accurate test using either (1) orig_wiki table in local wiki (v2) or (2) meta_db_mgr (v1) rv = wiki.File_mgr().Exists(page_db); // less-accurate test using either (1) orig_wiki table in local wiki (v2) or (2) meta_db_mgr (v1)
} }
// add commons result to this wiki's cache; needed b/c cache will have "false" value (b/c Page does not exist in wiki), but should be "true" (since it exists in commons) PAGE:en.w:Harstad DATE:2018-07-03
wiki.Cache_mgr().Ifexist_cache().Add(file_ttl, rv);
return rv;
} }
} }

1
400_xowa/src/gplx/xowa/xtns/pfuncs/ifs/Pfunc_ifexist_tst.java
View File

@ -38,6 +38,7 @@ public class Pfunc_ifexist_tst {
Xowe_wiki commons_wiki = fxt.App().Wiki_mgr().Get_by_or_make(gplx.xowa.wikis.domains.Xow_domain_itm_.Bry__commons); Xowe_wiki commons_wiki = fxt.App().Wiki_mgr().Get_by_or_make(gplx.xowa.wikis.domains.Xow_domain_itm_.Bry__commons);
fxt.Init_page_create(commons_wiki, "File:A.png", ""); fxt.Init_page_create(commons_wiki, "File:A.png", "");
fxt.Test_parse_tmpl_str_test("{{#ifexist:Media:A.png|y|n}}", "{{test}}", "y"); fxt.Test_parse_tmpl_str_test("{{#ifexist:Media:A.png|y|n}}", "{{test}}", "y");
fxt.Test_parse_tmpl_str_test("{{#ifexist:Media:A.png|y|n}}", "{{test}}", "y"); // BUG:2nd call actually ends up as n; PAGE:en.w:Harstad DATE:2018-07-03
} }
@Test public void Media_y_file_v1() {// DATE:2014-07-04 @Test public void Media_y_file_v1() {// DATE:2014-07-04
Xof_meta_itm meta_itm = fxt.Wiki().File_mgr().Dbmeta_mgr().Get_itm_or_new(Bry_.new_a7("A.png")); Xof_meta_itm meta_itm = fxt.Wiki().File_mgr().Dbmeta_mgr().Get_itm_or_new(Bry_.new_a7("A.png"));