Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Xtn.Dpl: Change dynamicPageList to always get categories from cache [#556]

Browse Source
This commit is contained in:
gnosygnu
2019-08-25 21:39:21 -04:00
parent 42842f0bcc
commit 2598dee844
7 changed files with 192 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr.java
View File

@@ -49,27 +49,30 @@ public class Xoctg_catpage_mgr implements Gfo_invk {
wiki.App().Cfg().Bind_many_wiki(this, wiki, Cfg__missing_class);
}
public void Free_mem_all() {cache.Clear();}
public Xoctg_catpage_ctg Get_or_load_or_null(byte[] page_ttl, Xoctg_catpage_url catpage_url, Xoa_ttl cat_ttl, int limit) {
// load categories from cat dbs; exit if not found
public Xoctg_catpage_ctg Get_by_cache_or_null(byte[] page_ttl, Xoctg_catpage_url catpage_url, Xoa_ttl cat_ttl, int limit) {
// DynamicPageList categories only (b/c of many members); for regular catpages, always retrieve on demand
Xoctg_catpage_ctg ctg = (Xoctg_catpage_ctg)cache.Get_by(cat_ttl.Full_db());
if (ctg == null) {
if (gplx.core.envs.Env_.Mode_testing()) return null; // needed for dpl test
synchronized (thread_lock) { // LOCK:used by multiple wrks; DATE:2016-09-12
ctg = loader.Load_ctg_or_null(wiki, page_ttl, this, catpage_url, cat_ttl, limit);
}
ctg = Get_by_db_or_null(page_ttl, catpage_url, cat_ttl, limit);
if (ctg == null) return null; // not in cache or db; exit
if (limit == Int_.Max_value) // only add to cache if Max_val (DynamicPageList); for regular catpages, always retrieve on demand
cache.Add(cat_ttl.Full_db(), ctg);
cache.Add(cat_ttl.Full_db(), ctg);
}
return ctg;
}
public Xoctg_catpage_ctg Get_by_db_or_null(byte[] page_ttl, Xoctg_catpage_url catpage_url, Xoa_ttl cat_ttl, int limit) {
// load categories from cat dbs; exit if not found
synchronized (thread_lock) { // LOCK:used by multiple wrks; DATE:2016-09-12
return loader.Load_ctg_or_null(wiki, page_ttl, this, catpage_url, cat_ttl, limit);
}
}
public void Write_catpage(Bry_bfr bfr, Xoa_page page) {
try {
// get catpage_url
Xoctg_catpage_url catpage_url = Xoctg_catpage_url_parser.Parse(page.Url());
// load categories from cat dbs; exit if not found
Xoctg_catpage_ctg ctg = Get_or_load_or_null(page.Ttl().Page_db(), catpage_url, page.Ttl(), grp_max);
Xoctg_catpage_ctg ctg = Get_by_db_or_null(page.Ttl().Page_db(), catpage_url, page.Ttl(), grp_max);
if (ctg == null) return;
// write html

16
400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/dbs/Xoctg_catlink_loader.java
View File

@@ -45,8 +45,7 @@ class Xoctg_catlink_loader {
// sort and reduce list to 200 total
catlink_list.Sort_by(new Xoctg_catlink_sorter(url_is_from));
Xoctg_page_loader catlink_loader = new Xoctg_page_loader(wiki);
Ordered_hash catlink_hash = catlink_loader.Hash();
Ordered_hash catlink_hash = Ordered_hash_.New();
int catlink_list_len = catlink_list.Len();
int max = catlink_list_len < limit ? catlink_list_len : limit;
for (int i = 0; i < max; i++) {
@@ -55,7 +54,6 @@ class Xoctg_catlink_loader {
}
// load ns / ttl for each catlink
page_tbl.Select_in__id(catlink_loader);
Xoctg_catpage_grp grp = rv.Grp_by_tid(grp_tid);
grp.Itms_((Xoctg_catpage_itm[])catlink_hash.To_ary_and_clear(Xoctg_catpage_itm.class));
@@ -93,7 +91,10 @@ class Xoctg_catlink_loader {
, ", cl_type_id"
, ", {0} AS cl_sortkey"
, ", {1} AS cl_sortkey_prefix"
, ", p.page_namespace"
, ", p.page_title"
, "FROM <link_db_{3}>cat_link cl{2}"
, " LEFT JOIN <page_db>page p ON p.page_id = cl{2}.cl_from"
), sortkey_col, sortkey_prefix_fld, sortkey_join, link_db_id);
bfr.Add_str_u8_fmt(String_.Concat_lines_nl
( "WHERE cl_to_id = {0}"
@@ -108,12 +109,15 @@ class Xoctg_catlink_loader {
}
private void Load_catlinks(List_adp catlink_list, String sql) {
Db_rdr rdr = Db_rdr_.Empty;
int count = 0;
try {
attach_mgr.Attach();
rdr = attach_mgr.Conn_main().Stmt_sql(sql).Exec_select__rls_auto();
while (rdr.Move_next()) {
Xoctg_catpage_itm itm = Xoctg_catpage_itm.New_by_rdr(rdr, version);
Xoctg_catpage_itm itm = Xoctg_catpage_itm.New_by_rdr(wiki, rdr, version);
catlink_list.Add(itm);
if (count >= 1000 && (count % 1000) == 0) Gfo_usr_dlg_.Instance.Prog_many("", "", "loading cat_links: count=~{0}", count);
count++;
}
}
finally {
@@ -197,6 +201,10 @@ class Xoctg_catlink_loader {
version = 3;
db_1st = cat_core_conn;
}
// add page_db
db_list.Add(new Db_attach_itm("page_db", page_tbl.Conn()));
Db_attach_mgr attach_mgr = new Db_attach_mgr(db_1st, (Db_attach_itm[])db_list.To_ary_and_clear(Db_attach_itm.class));
return new Xoctg_catlink_loader(wiki, catpage_mgr, page_tbl, version, link_dbs_len, attach_mgr);
}

40
400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/dbs/Xoctg_page_loader.java
View File

@@ -1,40 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.ctgs.htmls.catpages.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*;
import gplx.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*;
public class Xoctg_page_loader implements Select_in_cbk {
private final Xow_wiki wiki;
private final Ordered_hash hash = Ordered_hash_.New();
public Xoctg_page_loader(Xow_wiki wiki) {this.wiki = wiki;}
public Ordered_hash Hash() {return hash;}
public int Hash_max() {return hash.Len();}
public void Write_sql(Bry_bfr bfr, int idx) {
Xoctg_catpage_itm itm = (Xoctg_catpage_itm)hash.Get_at(idx);
bfr.Add_int_variable(itm.Page_id());
}
public void Read_data(Db_rdr rdr) {
// read values from page_tbl
int page_id = rdr.Read_int("page_id");
int page_ns = rdr.Read_int("page_namespace");
byte[] page_ttl = rdr.Read_bry_by_str("page_title");
// get itm and set data
Xoctg_catpage_itm itm = (Xoctg_catpage_itm)hash.Get_by(page_id);
if (itm == null) return; // NOTE: itms can exist in cat_links_tbl, but not in page_tbl; EX:User:Any_page
itm.Page_ttl_(wiki.Ttl_parse(page_ns, page_ttl));
}
}

12
400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_itm.java
View File

@@ -76,7 +76,7 @@ public class Xoctg_catpage_itm {
}
public static final Xoctg_catpage_itm[] Ary_empty = new Xoctg_catpage_itm[0];
public static Xoctg_catpage_itm New_by_rdr(Db_rdr rdr, byte version) {
public static Xoctg_catpage_itm New_by_rdr(Xow_wiki wiki, Db_rdr rdr, byte version) {
byte[] sortkey_binary = Bry_.Empty;
byte[] sortkey_prefix = Bry_.Empty;
if (version == Version__4) {
@@ -87,7 +87,15 @@ public class Xoctg_catpage_itm {
sortkey_binary = Bry_.Empty;
sortkey_prefix = rdr.Read_bry_by_str("cl_sortkey");
}
return new Xoctg_catpage_itm(version, rdr.Read_byte("cl_type_id"), rdr.Read_int("cl_from"), sortkey_prefix, sortkey_binary);
Xoctg_catpage_itm rv = new Xoctg_catpage_itm(version, rdr.Read_byte("cl_type_id"), rdr.Read_int("cl_from"), sortkey_prefix, sortkey_binary);
if (version == Version__4) {
String ttl_str = rdr.Read_str("page_title");
if (ttl_str != null) {// NOTE: ttl_str will be NULL if LEFT JOIN fails on page_db.page
rv.Page_ttl_(wiki.Ttl_parse(rdr.Read_int("page_namespace"), Bry_.new_u8(ttl_str)));
}
}
return rv;
}
public static Xoctg_catpage_itm New_by_ttl(byte grp_tid, int page_id, Xoa_ttl ttl) { // TEST
Xoctg_catpage_itm rv = new Xoctg_catpage_itm(Version__4, grp_tid, page_id, ttl.Page_txt(), Bry_.Empty);