Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Xtns.TemplateStyles: Cache TemplateStyles for HTML databases [#482]

Browse Source
This commit is contained in:
gnosygnu
2019-06-02 12:50:56 -04:00
parent fbe158537e
commit 3d9491e2de
29 changed files with 725 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java
View File

@@ -26,6 +26,8 @@ public class Xomp_make_cmd extends Xob_cmd__base {
new Xomp_make_lnki().Exec(wiki, cfg, 10000);
if (cfg.Mode().Has("stat"))
new Xomp_make_stat().Exec(wiki, cfg);
if (cfg.Mode().Has("hxtn"))
new Xomp_make_hxtn().Exec(wiki, cfg);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__cfg)) return cfg;

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd_cfg.java
View File

@@ -15,7 +15,7 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
public class Xomp_make_cmd_cfg implements Gfo_invk {
public Hash_adp Mode() {return mode;} private Hash_adp mode = Hash_adp_.New().Add_and_more("html", "html").Add_and_more("lnki", "lnki").Add_and_more("stat", "stat");
public Hash_adp Mode() {return mode;} private Hash_adp mode = Hash_adp_.New().Add_and_more("html", "html").Add_and_more("lnki", "lnki").Add_and_more("stat", "stat").Add_and_more("hxtn", "hxtn");
public boolean Delete_html_dbs() {return delete_html_dbs;} private boolean delete_html_dbs = true;
public Ordered_hash Merger_wkrs() {return merger_wkrs;} private final Ordered_hash merger_wkrs = Ordered_hash_.New();
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {

72
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_hxtn.java Normal file
View File

@@ -0,0 +1,72 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.brys.*;
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.htmls.hxtns.pages.*; import gplx.xowa.htmls.hxtns.blobs.*;
class Xomp_make_hxtn {
public void Exec(Xowe_wiki wiki, Xomp_make_cmd_cfg cfg) {
// create mgr
Hxtn_page_mgr page_mgr = new Hxtn_page_mgr();
page_mgr.Init_by_wiki(wiki, Bool_.Y);
Hxtn_page_tbl page_tbl = page_mgr.Page_tbl();
Hxtn_blob_tbl blob_tbl = page_mgr.Blob_tbl();
page_mgr.Insert_bgn(true);
// loop wkrs
Xomp_mgr_db mgr_db = Xomp_mgr_db.New__load(wiki);
int wkrs_len = mgr_db.Tbl__wkr().Select_count();
for (int i = 0; i < wkrs_len; ++i) {
// init count / wkr_db
int count = 0;
Xomp_wkr_db wkr_db = Xomp_wkr_db.New(mgr_db.Dir(), i);
// insert page_tbl
page_tbl.Conn().Txn_bgn("hxtn_page");
Db_rdr rdr = wkr_db.Conn().Stmt_sql("SELECT * FROM hxtn_page;").Exec_select__rls_auto(); // ANSI.Y
try {
while (rdr.Move_next()) {
page_tbl.Insert_by_rdr(rdr);
if (++count % 10000 == 0) {
Gfo_usr_dlg_.Instance.Prog_many("", "", "hxtn.page.insert: db=~{0} count=~{1}", Int_.To_str_pad_bgn_space(i, 3), Int_.To_str_pad_bgn_space(count, 8));
page_tbl.Conn().Txn_sav();
}
}
} finally {rdr.Rls();}
page_tbl.Conn().Txn_end();
// insert blob tbl; note that dupes can exist across wkr_dbs (wkr_db 1 and wkr_db 2 both have Template:Abcd)
count = 0;
blob_tbl.Conn().Env_db_attach("wkr_db", wkr_db.Url());
blob_tbl.Conn().Txn_bgn("hxtn_blob");
rdr = blob_tbl.Conn().Stmt_sql("SELECT src.* FROM wkr_db.hxtn_blob src LEFT JOIN hxtn_blob trg ON src.wiki_id = trg.wiki_id AND src.blob_id = trg.blob_id AND src.blob_tid = trg.blob_tid WHERE trg.blob_id IS NULL;").Exec_select__rls_auto(); // ANSI.Y
try {
while (rdr.Move_next()) {
blob_tbl.Insert_by_rdr(rdr);
if (++count % 10000 == 0) {
Gfo_usr_dlg_.Instance.Prog_many("", "", "hxtn.blob.insert: db=~{0} count=~{1}", Int_.To_str_pad_bgn_space(i, 3), Int_.To_str_pad_bgn_space(count, 8));
blob_tbl.Conn().Txn_sav();
}
}
} finally {rdr.Rls();}
blob_tbl.Conn().Txn_end();
blob_tbl.Conn().Env_db_detach("wkr_db");
}
// cleanup
page_mgr.Insert_end(true);
}
}

1
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_stat.java
View File

@@ -23,6 +23,7 @@ class Xomp_make_stat {
Db_conn mgr_conn = mgr_db.Conn();
Xomp_stat_tbl mgr_tbl = new Xomp_stat_tbl(mgr_conn);
mgr_conn.Txn_bgn("xomp_stats");
mgr_conn.Stmt_delete("xomp_stats", Dbmeta_fld_itm.Str_ary_empty).Exec_delete();
mgr_tbl.Stmt_new();
// loop wkrs

25
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java
View File

@@ -16,7 +16,7 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
package gplx.xowa.addons.bldrs.mass_parses.parses.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.files.origs.*;
import gplx.xowa.htmls.core.bldrs.*;
import gplx.xowa.htmls.core.bldrs.*; import gplx.xowa.htmls.hxtns.pages.*;
import gplx.xowa.wikis.pages.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.pools.*;
@@ -41,6 +41,7 @@ public class Xomp_parse_wkr implements Gfo_invk {
private final int uid;
private Xomp_wkr_db wkr_db;
private Xomp_stat_tbl stat_tbl;
private Hxtn_page_mgr hxtn_mgr;
// indexer vars
private final Xofulltext_indexer_wkr indexer;
@@ -67,6 +68,8 @@ public class Xomp_parse_wkr implements Gfo_invk {
this.wiki = wiki; this.uid = uid;
this.wkr_db = Xomp_wkr_db.New(Xomp_mgr_db.New__url(wiki), uid);
this.stat_tbl = new Xomp_stat_tbl(wkr_db.Conn());
this.hxtn_mgr = wiki.Hxtn_mgr();
this.hxtn_mgr.Init_by_xomp_wkr(wkr_db.Conn(), cfg.Zip_tid());
}
public void Exec() {
Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
@@ -97,6 +100,7 @@ public class Xomp_parse_wkr implements Gfo_invk {
hdump_bldr.Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
wkr_db.Conn().Txn_bgn("xomp");
stat_tbl.Stmt_new();
hxtn_mgr.Insert_bgn(false);
// set status to running
mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__running);
@@ -167,19 +171,26 @@ public class Xomp_parse_wkr implements Gfo_invk {
wiki.Parser_mgr().Scrib().Core_term();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
}
if (done_count % commit_interval == 0)
if (done_count % commit_interval == 0) {
wkr_db.Conn().Txn_sav();
}
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e));
}
}
// cleanup
if (logger != null) logger.End();
wkr_db.Conn().Txn_end();
wkr_db.Conn().Rls_conn();
stat_tbl.Stmt_rls();
mgr.Wkrs_done_add_1(); // NOTE: must release latch last else thread errors
try {
if (logger != null) logger.End();
wkr_db.Conn().Txn_end();
wkr_db.Conn().Rls_conn();
stat_tbl.Stmt_rls();
hxtn_mgr.Insert_end(false);
mgr.Wkrs_done_add_1(); // NOTE: must release latch last else thread errors
}
catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail_end; err=~{0}", Err_.Message_gplx_log(e));
}
}
public void Bld_stats(Bry_bfr bfr) {
int done_time_in_sec = (int)(done_time / 1000); if (done_time_in_sec == 0) done_time_in_sec = 1;