Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

'v3.7.2.1'

Browse Source
This commit is contained in:
gnosygnu
2016-07-10 23:35:32 -04:00
parent f5f48bb9b1
commit b333db45f8
366 changed files with 4468 additions and 3460 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
400_xowa/src/gplx/xowa/addons/bldrs/exports/splits/archives/Reindex_html_dbs_cmd.java
View File

@@ -87,7 +87,7 @@ class Reindex_html_dbs_cmd {
Db_stmt stmt_update = core_conn.Stmt_update(tbl_page, String_.Ary(fld_page_id), fld_page_html_db_id);
Xoh_page_tbl_itm trg_html_tbl = null;
try {
Xoh_page_row src_html_row = new Xoh_page_row();
Xowd_html_row src_html_row = new Xowd_html_row();
long trg_db_size = 0;
int ns_cur = -1, part_id = 0;
int trg_db = -1;

8
400_xowa/src/gplx/xowa/addons/bldrs/exports/splits/htmls/Split_wkr__html.java
View File

@@ -21,15 +21,15 @@ import gplx.xowa.addons.bldrs.exports.splits.metas.*; import gplx.xowa.addons.bl
import gplx.xowa.htmls.core.dbs.*;
public class Split_wkr__html implements Split_wkr {
private Xoh_src_tbl_mgr src_tbl_mgr;
private Xoh_page_tbl tbl; private Db_stmt stmt;
private final Xoh_page_row trg_itm = new Xoh_page_row();
private Xowd_html_tbl tbl; private Db_stmt stmt;
private final Xowd_html_row trg_itm = new Xowd_html_row();
private final Split_rslt_wkr__html rslt_wkr = new Split_rslt_wkr__html();
public void Split__init(Split_ctx ctx, Xow_wiki wiki, Db_conn wkr_conn) {
this.src_tbl_mgr = new Xoh_src_tbl_mgr(wiki);
ctx.Rslt_mgr().Reg_wkr(rslt_wkr);
}
public void Split__trg__nth__new(Split_ctx ctx, Db_conn trg_conn) {
this.tbl = new Xoh_page_tbl(trg_conn);
this.tbl = new Xowd_html_tbl(trg_conn);
Dbmeta_fld_list trg_flds = Make_flds_for_split(tbl.Flds());
trg_conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl.Tbl_name(), trg_flds));
this.stmt = trg_conn.Stmt_insert(tbl.Tbl_name(), trg_flds);
@@ -52,7 +52,7 @@ public class Split_wkr__html implements Split_wkr {
byte[] sidebar_div = trg_itm.Sidebar_div();
// calc db_idx based on db_size
int db_row_size = Xoh_page_row.Db_row_size_fixed + display_ttl.length + content_sub.length + sidebar_div.length + body_len;
int db_row_size = Xowd_html_row.Db_row_size_fixed + display_ttl.length + content_sub.length + sidebar_div.length + body_len;
int trg_db_id = ctx.Html_size_calc().Size_cur_add_(db_row_size);
// do insert

4
400_xowa/src/gplx/xowa/addons/bldrs/exports/splits/htmls/Xoh_page_tbl_itm.java
View File

@@ -22,10 +22,10 @@ public class Xoh_page_tbl_itm {
public Xoh_page_tbl_itm(boolean trg, int db_id, Db_conn conn) {
this.trg = trg;
this.db_id = db_id;
this.html_tbl = new Xoh_page_tbl(conn);
this.html_tbl = new Xowd_html_tbl(conn);
}
public int Db_id() {return db_id;} private final int db_id;
public Xoh_page_tbl Html_tbl() {return html_tbl;} private final Xoh_page_tbl html_tbl;
public Xowd_html_tbl Html_tbl() {return html_tbl;} private final Xowd_html_tbl html_tbl;
public void Rls() {
html_tbl.Conn().Rls_conn();
if (trg) html_tbl.Conn().Env_vacuum();

2
400_xowa/src/gplx/xowa/addons/bldrs/exports/utls/Split_tbl.java
View File

@@ -86,7 +86,7 @@ class Split_tbl__html implements Split_tbl {
public boolean Layout_is_lot() {return Bool_.Y;}
public String[] Fld_pkeys() {return String_.Ary("page_id");}
public String Fld_blob() {return "body";}
public Dbmeta_fld_list Flds() {if (flds == null) flds = new gplx.xowa.htmls.core.dbs.Xoh_page_tbl(Db_conn_.Noop).Flds(); return flds;} private Dbmeta_fld_list flds;
public Dbmeta_fld_list Flds() {if (flds == null) flds = new gplx.xowa.htmls.core.dbs.Xowd_html_tbl(Db_conn_.Noop).Flds(); return flds;} private Dbmeta_fld_list flds;
public Db_conn Wiki_conn__get_or_new(Xow_wiki wiki, int db_id) {
if (db_id == -1) // HACK: return core_conn just so that bin_tbl below can be created
return wiki.Data__core_mgr().Db__core().Conn();

5
400_xowa/src/gplx/xowa/addons/bldrs/files/Xoax_builds_files_addon.java
View File

@@ -18,6 +18,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.files.cmds.*;
import gplx.xowa.addons.bldrs.mass_parses.inits.*;
import gplx.xowa.addons.bldrs.mass_parses.makes.*;
public class Xoax_builds_files_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr {
public Xob_cmd[] Bldr_cmds() {
return new Xob_cmd[]
@@ -38,6 +40,9 @@ public class Xoax_builds_files_addon implements Xoax_addon_itm, Xoax_addon_itm__
, Xobldr__text_db__drop_page.Prototype
, Xobldr__redirect__create.Prototype
, Xobldr__image__create.Prototype
, Xomp_init_cmd.Prototype
, Xomp_parse_cmd.Prototype
};
}

34
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xob_hdump_tbl_retriever__ns_to_db.java Normal file
View File

@@ -0,0 +1,34 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.htmls.core.bldrs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.wikis.data.*;
class Xob_hdump_tbl_retriever__ns_to_db implements Xob_hdump_tbl_retriever {
private final Xob_ns_to_db_mgr ns_to_db_mgr;
public Xob_hdump_tbl_retriever__ns_to_db(Xowe_wiki wiki) {
Xow_db_mgr core_data_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
this.ns_to_db_mgr = new Xob_ns_to_db_mgr(new Xob_ns_to_db_wkr__html(core_data_mgr.Db__core()), core_data_mgr, wiki.Appe().Api_root().Bldr().Wiki().Import().Html_db_max());
Xob_ns_file_itm.Init_ns_bldr_data(Xow_db_file_.Tid__html_data, wiki.Ns_mgr(), gplx.xowa.apps.apis.xowa.bldrs.imports.Xoapi_import.Ns_file_map__each);
}
public Xowd_html_tbl Get_html_tbl(Xow_ns ns, int prv_row_len) {
Xow_db_file html_db = ns_to_db_mgr.Get_by_ns(ns.Bldr_data(), prv_row_len); // get html_db
return html_db.Tbl__html();
}
public void Commit() {ns_to_db_mgr.Commit();}
public void Rls_all() {ns_to_db_mgr.Rls_all();}
}

33
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__lnki_temp__create.java
View File

@@ -46,7 +46,7 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
Xob_db_file make_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
Db_conn make_conn = make_db.Conn();
this.tbl = new Xob_lnki_temp_tbl(make_conn); tbl.Create_tbl();
this.gen_hdump = hdump_bldr.Init(wiki, make_conn);
this.gen_hdump = hdump_bldr.Init(wiki, make_conn, new Xob_hdump_tbl_retriever__ns_to_db(wiki));
Xol_vnt_mgr vnt_mgr = wiki.Lang().Vnt_mgr();
if (vnt_mgr.Enabled()) {
this.converter_lang = vnt_mgr.Convert_lang();
@@ -58,8 +58,10 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
ns_file_is_case_match_all = Ns_file_is_case_match_all(wiki); // NOTE: must call after wiki.init
wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_page_.Tid_read).Ctgs_enabled_(false); // disable categories else progress messages written (also for PERF)
if (wiki.File__bin_mgr() != null)
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // remove wmf wkr, else will try to download images during parsing
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // remove wmf wkr, else will try to download images during parsing
commons_wiki = app.Wiki_mgr().Get_by_or_make(Xow_domain_itm_.Bry__commons);
// init log_mgr / property_wkr
Xop_log_mgr log_mgr = ctx.App().Log_mgr();
log_mgr.Log_dir_(wiki.Fsys_mgr().Root_dir()); // put log in wiki dir, instead of user.temp
invoke_wkr = this.Invoke_wkr(); // set member reference
@@ -68,20 +70,25 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
property_wkr = log_mgr.Make_wkr_property();
wiki.Appe().Wiki_mgr().Wdata_mgr().Enabled_(wdata_enabled);
if (!xtn_ref_enabled) gplx.xowa.xtns.cites.References_nde.Enabled = false;
// init log wkrs
gplx.xowa.xtns.gallery.Gallery_xnde.Log_wkr = log_mgr.Make_wkr().Save_src_str_(Bool_.Y);
gplx.xowa.xtns.imaps.Imap_xnde.Log_wkr = log_mgr.Make_wkr();
gplx.xowa.parsers.xndes.Xop_xnde_wkr.Timeline_log_wkr = log_mgr.Make_wkr();
gplx.xowa.xtns.scores.Score_xnde.Log_wkr = log_mgr.Make_wkr();
gplx.xowa.xtns.hieros.Hiero_xnde.Log_wkr = log_mgr.Make_wkr();
gplx.xowa.xtns.math.Math_nde.Log_wkr = log_mgr.Make_wkr().Save_src_str_(Bool_.Y); // enabled; DATE:2015-10-10
// init fsdb
Xof_fsdb_mgr__sql trg_fsdb_mgr = new Xof_fsdb_mgr__sql();
wiki.File__fsdb_mode().Tid_v2_bld_y_();
Fsdb_db_mgr__v2 fsdb_core = Fsdb_db_mgr__v2_bldr.Get_or_make(wiki, Bool_.Y);
trg_fsdb_mgr.Init_by_wiki(wiki);
Fsm_mnt_mgr trg_mnt_mgr = trg_fsdb_mgr.Mnt_mgr();
wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
wiki.File__orig_mgr().Wkrs_del(gplx.xowa.files.origs.Xof_orig_wkr_.Tid_wmf_api);
trg_mnt_mgr = new Fsm_mnt_mgr(); trg_mnt_mgr.Ctor_by_load(fsdb_core);
trg_mnt_mgr.Mnts__get_insert_idx_(Fsm_mnt_mgr.Mnt_idx_main);
Fsm_mnt_mgr.Patch(trg_mnt_mgr.Mnts__get_main().Cfg_mgr().Tbl()); // NOTE: see fsdb_make; DATE:2014-04-26
@@ -89,21 +96,23 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
log_mgr.Txn_bgn();
}
@Override public void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm db_page, byte[] page_src) {
Xoa_ttl ttl = Xoa_ttl.parse(wiki, ns.Gen_ttl(db_page.Ttl_page_db()));
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, ns.Gen_ttl(db_page.Ttl_page_db()));
byte[] ttl_bry = ttl.Page_db();
byte page_tid = Xow_page_tid.Identify(wiki.Domain_tid(), ns.Id(), ttl_bry);
if (page_tid != Xow_page_tid.Tid_wikitext) return; // ignore js, css, lua, json
Xoae_page page = ctx.Page();
page.Clear_all();
page.Bldr__ns_ord_(ns_ord);
page.Ttl_(ttl).Revision_data().Id_(db_page.Id());
page.Redlink_list().Clear();
page.Url_(Xoa_url.new_(wiki.Domain_bry(), ttl.Full_db()));
page.Ttl_(ttl);
page.Db().Page().Id_(db_page.Id());
page.Html_data().Redlink_list().Clear();
page.Url_(Xoa_url.New(wiki, ttl));
if (ns.Id_is_tmpl())
parser.Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), wiki.Ns_mgr().Ns_template(), ttl_bry, page_src);
else {
parser.Parse_page_all_clear(root, ctx, ctx.Tkn_mkr(), page_src);
if (gen_html && !page.Redirected())
if ( gen_html
&& page.Redirect().Itms__len() == 0) // don't generate html for redirected pages
wiki.Html_mgr().Page_wtr_mgr().Gen(ctx.Page().Root_(root), Xopg_page_.Tid_read);
if (gen_hdump)
hdump_bldr.Insert(page.Root_(root));
@@ -119,9 +128,7 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
}
@Override public void Exec_end_hook() {
// if (converter_lang != null) converter_lang.Log__rls();
if (gen_hdump) {
hdump_bldr.Bld_term();
}
if (gen_hdump) hdump_bldr.Term();
String err_filter_mgr = invoke_wkr.Err_filter_mgr().Print();
if (String_.Len_gt_0(err_filter_mgr)) usr_dlg.Warn_many("", "", err_filter_mgr);
wiki.Appe().Log_mgr().Txn_end();
@@ -139,7 +146,7 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
usr_dlg.Warn_many("", "", "page and thumbtime both set; this may be an issue with fsdb: page=~{0} ttl=~{1}", ctx.Page().Ttl().Page_db_as_str(), String_.new_u8(ttl));
if (lnki.Ns_id() == Xow_ns_.Tid__media)
caller_tid = Xop_file_logger_.Tid__media;
tbl.Insert_cmd_by_batch(ctx.Page().Bldr__ns_ord(), ctx.Page().Revision_data().Id(), ttl, ttl_commons, Byte_.By_int(ext.Id()), lnki.Lnki_type(), caller_tid, lnki.W(), lnki.H(), lnki.Upright(), lnki_time, lnki_page);
tbl.Insert_cmd_by_batch(ctx.Page().Bldr__ns_ord(), ctx.Page().Db().Page().Id(), ttl, ttl_commons, Byte_.By_int(ext.Id()), lnki.Lnki_type(), caller_tid, lnki.W(), lnki.H(), lnki.Upright(), lnki_time, lnki_page);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_wdata_enabled_)) wdata_enabled = m.ReadYn("v");
@@ -172,7 +179,7 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
}
public static byte[] Xto_commons(boolean ns_file_is_case_match_all, Xowe_wiki commons_wiki, byte[] ttl_bry) {
if (!ns_file_is_case_match_all) return null; // return "" if wiki matches common
Xoa_ttl ttl = Xoa_ttl.parse(commons_wiki, Xow_ns_.Tid__file, ttl_bry);
Xoa_ttl ttl = Xoa_ttl.Parse(commons_wiki, Xow_ns_.Tid__file, ttl_bry);
byte[] rv = ttl.Page_db();
return Bry_.Eq(rv, ttl_bry) ? null : rv;
}

2
400_xowa/src/gplx/xowa/addons/bldrs/infos/Xobc_info_html.java
View File

@@ -39,7 +39,7 @@ class Xobc_info_html extends Xow_special_wtr__base {
long total_size = 0;
Bry_bfr tmp_size_bfr = Bry_bfr_.New();
for (int i = 0; i < len; ++i) {
int step_id = (int)list.Get_at(i);
int step_id = Int_.cast(list.Get_at(i));
Xobc_import_step_itm step_itm = data_db.Tbl__import_step().Select_one(step_id);
if (i == 0) {
wiki_domain = Xow_abrv_xo_.To_itm(step_itm.Wiki_abrv()); // ASSUME: 1st step's wiki is same for all steps

37
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/inits/Xomp_init_cmd.java Normal file
View File

@@ -0,0 +1,37 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.inits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xomp_init_cmd extends Xob_cmd__base {
private final Xomp_init_mgr mgr;
public Xomp_init_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);
mgr = new Xomp_init_mgr(wiki);
}
@Override public void Cmd_run() {
wiki.Init_assert();
mgr.Exec();
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__cfg)) return this;
else return super.Invk(ctx, ikey, k, m);
} private static final String Invk__cfg = "cfg";
@Override public String Cmd_key() {return BLDR_CMD_KEY;} private static final String BLDR_CMD_KEY = "wiki.mass_parse.init";
public static final Xob_cmd Prototype = new Xomp_init_cmd(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xomp_init_cmd(bldr, wiki);}
}

52
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/inits/Xomp_init_mgr.java Normal file
View File

@@ -0,0 +1,52 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.inits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.*;
class Xomp_init_mgr {
private final Xow_wiki wiki;
public Xomp_init_mgr(Xow_wiki wiki) {this.wiki = wiki;}
public void Exec() {
Xob_db_file make_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
Db_conn conn = make_db.Conn();
// make table
conn.Meta_tbl_remake(Dbmeta_tbl_itm.New("mp_page", new Dbmeta_fld_itm[]
{ Dbmeta_fld_itm.new_int("page_id").Primary_y_()
, Dbmeta_fld_itm.new_bool("page_done")
}
, Dbmeta_idx_itm.new_normal_by_tbl("mp_page", "page_id__page_done", "page_id", "page_done")
));
// fill table
Db_attach_mgr attach_mgr = new Db_attach_mgr(conn, new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
int[] ns_ary = new int[] {0, 4, 14};
int len = ns_ary.length;
String sql = String_.Concat_lines_nl_skip_last
( "INSERT INTO mp_page (page_id, page_done)"
, "SELECT page_id, 0"
, "FROM <page_db>page"
, "WHERE page_namespace = {0}"
, "AND page_is_redirect = 0"
);
for (int i = 0; i < len; ++i) {
int ns_id = ns_ary[i];
attach_mgr.Exec_sql_w_msg("adding rows for mp_page: ns=" + ns_id, sql, ns_id);// ANSI.Y
}
}
}

34
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xob_hdump_tbl_retriever__xomp.java Normal file
View File

@@ -0,0 +1,34 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.htmls.core.bldrs.*; import gplx.xowa.htmls.core.dbs.*;
class Xob_hdump_tbl_retriever__xomp implements Xob_hdump_tbl_retriever {
private final Db_conn conn;
private final Xowd_html_tbl tbl;
public Xob_hdump_tbl_retriever__xomp(Db_conn conn) {
this.conn = conn;
this.tbl = new Xowd_html_tbl(conn);
conn.Meta_tbl_assert(tbl);
}
public Xowd_html_tbl Get_html_tbl(Xow_ns ns, int prv_row_len) {
return tbl;
}
public void Commit() {conn.Txn_sav();}
public void Rls_all() {conn.Txn_sav(); conn.Rls_conn();}
}

37
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_page_itm.java Normal file
View File

@@ -0,0 +1,37 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
class Xomp_page_itm {
public Xomp_page_itm(int id) {this.id = id;}
public int Id() {return id;} private final int id;
public int Ns_id() {return ns_id;} private int ns_id;
public byte[] Ttl_bry() {return ttl_bry;} private byte[] ttl_bry;
public int Text_db_id() {return text_db_id;} private int text_db_id;
public byte[] Text() {return text;} private byte[] text;
public void Init_by_page(int ns_id, byte[] ttl_bry, int text_db_id) {
this.ns_id = ns_id;
this.ttl_bry = ttl_bry;
this.text_db_id = text_db_id;
}
public void Init_by_text(byte[] text) {
this.text = text;
}
public static final Xomp_page_itm Null = new Xomp_page_itm(-1);
}

84
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_page_pool.java Normal file
View File

@@ -0,0 +1,84 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
class Xomp_page_pool {
private final Object thread_lock = new Object();
private Xomp_page_pool_loader loader;
private List_adp pool = List_adp_.New(); private int pool_idx = 0, pool_len = 0;
private Db_conn make_conn;
private final Bry_bfr prog_bfr = Bry_bfr_.New();
private int pages_done, pages_total;
private long time_bgn, time_prv, time_done;
public void Init(Xow_wiki wiki, int num_pages_per_load) {
this.make_conn = gplx.xowa.bldrs.Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
this.loader = new Xomp_page_pool_loader(wiki, make_conn, num_pages_per_load);
this.pages_done = 0;
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
this.pages_total = loader.Get_pending_count();
}
public boolean Empty() {return empty;} private boolean empty = false;
public void Get_next(List_adp wkr_list, int num_pages_per_wkr) {
synchronized (thread_lock) {
// pool already marked exhausted by another wkr; return;
if (empty) return;
int wkr_end = pool_idx + num_pages_per_wkr;
// need pages to fulfill request
if (wkr_end > pool_len) {
this.pool = loader.Load(pool, pool_idx, pool_len);
this.pool_idx = 0;
this.pool_len = pool.Len();
if (pool_len == 0) { // no more pages; return;
empty = true;
return;
}
wkr_end = num_pages_per_wkr; // recalc wkr_end
}
// reset wkr_end; needed for very last set
if (wkr_end >= pool_len)
wkr_end = pool_len;
// add pages to wkr_list
for (int i = pool_idx; i < wkr_end; ++i) {
Xomp_page_itm page = (Xomp_page_itm)pool.Get_at(i);
wkr_list.Add(page);
}
pool_idx = wkr_end;
}
}
public void Mark_done(int id) {
synchronized (thread_lock) {
pages_done += 1;
if (pages_done % 1000 == 0) {
long time_cur = gplx.core.envs.Env_.TickCount();
int pages_left = pages_total - pages_done;
time_done += (time_cur - time_prv);
double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)((time_cur - time_bgn) / 1000), true, 0);
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)(pages_left / rate_cur), true, 0);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{0} left=~{1} rate=~{2} time_past=~{3} time_left=~{4}", pages_done, pages_left, (int)rate_cur, time_past, time_left);
time_prv = time_cur;
}
}
}
public void Rls() {
make_conn.Rls_conn();
}
}

87
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_page_pool_loader.java Normal file
View File

@@ -0,0 +1,87 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
class Xomp_page_pool_loader {
private final Xow_wiki wiki;
private int prv_page_id = -1;
private final Db_conn make_conn;
private final int num_pages_per_load;
private final Db_attach_mgr attach_mgr;
public Xomp_page_pool_loader(Xow_wiki wiki, Db_conn make_conn, int num_pages_per_load) {
this.wiki = wiki;
this.make_conn = make_conn;
this.attach_mgr = new Db_attach_mgr(make_conn);
this.num_pages_per_load = num_pages_per_load;
}
public int Get_pending_count() {
Db_rdr rdr = make_conn.Stmt_sql("SELECT Count(*) AS Count_of FROM mp_page mp WHERE mp.page_done = 0").Exec_select__rls_auto();
try {
return rdr.Move_next() ? rdr.Read_int("Count_of") : 0;
} finally {rdr.Rls();}
}
public List_adp Load(List_adp list, int list_idx, int list_len) {
List_adp rv = List_adp_.New();
// add remaining pages from old pool to new_pool;
for (int i = list_idx; i < list_len; ++i) {
rv.Add((Xomp_page_itm)list.Get_at(i));
}
// load pages into new pool
this.Load_from_db(rv);
return rv;
}
private void Load_from_db(List_adp list) {
// prepare for page_tbl
String sql = String_.Format(String_.Concat_lines_nl_skip_last // ANSI.Y
( "SELECT mp.page_id"
, ", pp.page_namespace"
, ", pp.page_title"
, ", pp.page_text_db_id"
, "FROM mp_page mp"
, " JOIN <page_db>page pp ON mp.page_id = pp.page_id"
, "WHERE mp.page_id > {0}"
, "AND mp.page_done = 0"
, "LIMIT {1}"
), prv_page_id, num_pages_per_load);
this.attach_mgr.Conn_others_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
sql = attach_mgr.Resolve_sql(sql);
// run page_tbl
Xomp_text_db_loader text_db_loader = new Xomp_text_db_loader(wiki);
attach_mgr.Attach();
Db_rdr rdr = make_conn.Stmt_sql(sql).Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
prv_page_id = rdr.Read_int("page_id");
int text_db_id = rdr.Read_int("page_text_db_id");
Xomp_page_itm ppg = new Xomp_page_itm(prv_page_id);
ppg.Init_by_page
( rdr.Read_int("page_namespace")
, rdr.Read_bry_by_str("page_title")
, text_db_id
);
list.Add(ppg);
text_db_loader.Add(text_db_id, ppg);
}
} finally {rdr.Rls();}
attach_mgr.Detach();
text_db_loader.Load();
}
}

35
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_parse_cmd.java Normal file
View File

@@ -0,0 +1,35 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xomp_parse_cmd extends Xob_cmd__base {
private final Xomp_parse_mgr mgr = new Xomp_parse_mgr();
public Xomp_parse_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
wiki.Init_assert();
mgr.Run(wiki);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__cfg)) return mgr.Cfg();
else return super.Invk(ctx, ikey, k, m);
} private static final String Invk__cfg = "cfg";
@Override public String Cmd_key() {return BLDR_CMD_KEY;} private static final String BLDR_CMD_KEY = "wiki.mass_parse.exec";
public static final Xob_cmd Prototype = new Xomp_parse_cmd(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xomp_parse_cmd(bldr, wiki);}
}

68
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_parse_mgr.java Normal file
View File

@@ -0,0 +1,68 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.threads.*;
import gplx.xowa.langs.*;
class Xomp_parse_mgr {
private final Xomp_page_pool page_pool = new Xomp_page_pool();
public Xomp_parse_mgr_cfg Cfg() {return cfg;} private final Xomp_parse_mgr_cfg cfg = new Xomp_parse_mgr_cfg();
private int wkrs_done;
public void Wkrs_done_add_1() {synchronized (page_pool) {++wkrs_done;}}
public void Run(Xowe_wiki wiki) {
// init pool
cfg.Init(wiki);
page_pool.Init(wiki, cfg.Num_pages_in_pool());
// init threads
int wkr_len = cfg.Num_wkrs();
Xomp_parse_wkr[] wkrs = new Xomp_parse_wkr[wkr_len];
for (int i = 0; i < wkr_len; ++i) {
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, Clone_wiki(wiki), page_pool, i, cfg.Num_pages_per_wkr());
wkrs[i] = wkr;
}
// start threads; done separately b/c thread issues when done right after init
for (int i = 0; i < wkr_len; ++i) {
Xomp_parse_wkr wkr = wkrs[i];
Thread_adp_.Start_by_key("xomp." + Int_.To_str_fmt(i, "000"), Cancelable_.Never, wkr, Xomp_parse_wkr.Invk__exec);
}
// wait until wkrs are wkrs_done
while (true) {
synchronized (page_pool) {
if (wkrs_done == wkr_len) break;
}
Thread_adp_.Sleep(1000);
}
page_pool.Rls();
// print stats
Bry_bfr bfr = Bry_bfr_.New();
for (int i = 0; i < wkr_len; ++i) {
wkrs[i].Bld_stats(bfr);
}
Gfo_usr_dlg_.Instance.Note_many("", "", bfr.To_str_and_clear());
}
private static Xowe_wiki Clone_wiki(Xowe_wiki wiki) {
Xol_lang_itm lang = new Xol_lang_itm(wiki.App().Lang_mgr(), wiki.Lang().Key_bry());
Xol_lang_itm_.Lang_init(lang);
Xowe_wiki rv = new Xowe_wiki(wiki.Appe(), lang, wiki.Ns_mgr(), wiki.Domain_itm(), wiki.Fsys_mgr().Root_dir());
rv.Init_by_wiki();
return rv;
}
}

40
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_parse_mgr_cfg.java Normal file
View File

@@ -0,0 +1,40 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
class Xomp_parse_mgr_cfg implements Gfo_invk {
public Io_url Root_dir() {return root_dir;} private Io_url root_dir;
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1;
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = 1000;
public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000;
public void Init(Xowe_wiki wiki) {
if (root_dir == null) root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Env_.System_cpu_count();
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__num_wkrs_)) num_wkrs = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
else if (ctx.Match(k, Invk__root_dir_)) root_dir = m.ReadIoUrl("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}
private static final String Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
, Invk__root_dir_ = "root_dir_"
;
}

117
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_parse_wkr.java Normal file
View File

@@ -0,0 +1,117 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.xowa.htmls.core.bldrs.*;
import gplx.xowa.parsers.*;
class Xomp_parse_wkr implements Gfo_invk {
private final Xomp_parse_mgr mgr;
private final Xowe_wiki wiki;
private final Xomp_page_pool page_pool;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private final int num_pages_per_wkr;
private final int idx;
private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
// private boolean gen_html = true;// gen_hdump = true;
private int done_count; private long done_time;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xomp_page_pool page_pool, int idx, int num_pages_per_wkr) {
this.mgr = mgr; this.wiki = wiki;
this.page_pool = page_pool; this.num_pages_per_wkr = num_pages_per_wkr;
this.idx = idx;
}
public void Exec() {
// init
Db_conn wkr_conn = Db_conn_bldr.Instance.Get_or_autocreate(true, mgr.Cfg().Root_dir().GenSubFil_nest("xomp_" + Int_.To_str_fmt(idx, "000"), "xomp_wkr.sqlite3"));
Xow_parser_mgr parser = new Xow_parser_mgr(wiki);
wiki.Html_mgr().Page_wtr_mgr().Wkr(gplx.xowa.wikis.pages.Xopg_page_.Tid_read).Ctgs_enabled_(false); // disable categories else progress messages written (also for PERF)
if (wiki.File__bin_mgr() != null)
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // remove wmf wkr, else will try to download images during parsing
hdump_bldr.Init(wiki, wkr_conn, new Xob_hdump_tbl_retriever__xomp(wkr_conn));
while (true) {
Xomp_page_itm ppg = Get_next(); if (ppg == Xomp_page_itm.Null) break; // no more pages
if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05
try {
// init page
long done_bgn = gplx.core.envs.Env_.TickCount();
Xoa_ttl ttl = wiki.Ttl_parse(ppg.Ns_id(), ppg.Ttl_bry());
Xoae_page wpg = Xoae_page.New(wiki, ttl);
wpg.Db().Text().Text_bry_(ppg.Text());
wpg.Db().Page().Id_(ppg.Id());
// parse page
parser.Ctx().Clear_all();
parser.Parse(wpg, true);
// gen_html
// if ( gen_html
// && wpg.Redirect().Itms__len() == 0) // don't generate html for redirected pages
// wiki.Html_mgr().Page_wtr_mgr().Gen(wpg, gplx.xowa.wikis.pages.Xopg_page_.Tid_read);
// if (gen_hdump)
// hdump_bldr.Insert(wpg);
// mark done for sake of progress
page_pool.Mark_done(ppg.Id());
// update stats
long time_cur = gplx.core.envs.Env_.TickCount();
done_time += time_cur - done_bgn;
done_bgn = time_cur;
++done_count;
// cleanup
// ctx.App().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released
if (wiki.Cache_mgr().Tmpl_result_cache().Count() > 50000)
wiki.Cache_mgr().Tmpl_result_cache().Clear();
if (done_count % 50 == 0) {
wiki.Cache_mgr().Free_mem_all();
wiki.Parser_mgr().Scrib().Core_term();
}
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e));
}
}
mgr.Wkrs_done_add_1();
wkr_conn.Rls_conn();
}
public void Bld_stats(Bry_bfr bfr) {
int done_time_in_sec = (int)(done_time / 1000); if (done_time_in_sec == 0) done_time_in_sec = 1;
bfr.Add_int_pad_bgn(Byte_ascii.Space, 4, idx );
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, (int)(done_count / done_time_in_sec));
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_count);
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_time_in_sec);
bfr.Add_byte_nl();
}
private Xomp_page_itm Get_next() {
if (list_idx == list_len) {
list.Clear();
page_pool.Get_next(list, num_pages_per_wkr);
list_len = list.Len();
if (list_len == 0) return Xomp_page_itm.Null;
list_idx = 0;
}
return (Xomp_page_itm)list.Get_at(list_idx++);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__exec)) this.Exec();
else return Gfo_invk_.Rv_unhandled;
return this;
}
public static final String Invk__exec = "exec";
}

94
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_text_db_loader.java Normal file
View File

@@ -0,0 +1,94 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.core.ios.*;
class Xomp_text_db_loader {
private final Xow_wiki wiki;
private final Ordered_hash text_db_hash = Ordered_hash_.New();
private final Io_stream_zip_mgr zip_mgr = Xoa_app_.Utl__zip_mgr();
public Xomp_text_db_loader(Xow_wiki wiki) {
this.wiki = wiki;
}
public void Add(int text_db_id, Xomp_page_itm ppg) {
Xomp_text_db_itm itm = (Xomp_text_db_itm)text_db_hash.Get_by(text_db_id);
if (itm == null) {
itm = new Xomp_text_db_itm(text_db_id);
text_db_hash.Add(text_db_id, itm);
}
itm.Page_list().Add(ppg);
}
public void Load() {
int text_db_hash_len = text_db_hash.Len();
for (int i = 0; i < text_db_hash_len; ++i) {
Xomp_text_db_itm itm = (Xomp_text_db_itm)text_db_hash.Get_at(i);
Load_list(itm.Text_db_id(), itm.Page_list());
}
}
private void Load_list(int text_db_id, List_adp list) {
int list_len = list.Len();
int batch_idx = 0;
Bry_bfr bry = Bry_bfr_.New();
Ordered_hash page_hash = Ordered_hash_.New();
byte zip_tid = wiki.Data__core_mgr().Props().Zip_tid_text();
for (int i = 0; i < list_len; ++i) {
if (batch_idx == 0) {
page_hash.Clear();
bry.Add_str_a7("SELECT page_id, text_data FROM text WHERE page_id IN (");
}
// build WHERE IN for page_ids; EX: "1, 2, 3, 4"
Xomp_page_itm ppg = (Xomp_page_itm)list.Get_at(i);
int page_id = ppg.Id();
if (batch_idx != 0) bry.Add_byte_comma();
bry.Add_int_variable(page_id);
page_hash.Add(page_id, ppg);
++batch_idx;
// load if 255 in list, or last
if ( batch_idx % 255 == 0
|| i == list_len - 1) {
bry.Add_byte(Byte_ascii.Paren_end);
Load_from_text_db(page_hash, zip_tid, text_db_id, bry.To_str_and_clear());
batch_idx = 0;
}
}
}
private void Load_from_text_db(Ordered_hash page_hash, byte zip_tid, int text_db_id, String sql) {
Db_conn text_conn = wiki.Data__core_mgr().Dbs__get_by_id_or_fail(text_db_id).Conn();
Db_rdr rdr = text_conn.Stmt_sql(sql).Exec_select__rls_auto(); // ANSI.Y
try {
while (rdr.Move_next()) {
int page_id = rdr.Read_int("page_id");
byte[] text_data = rdr.Read_bry("text_data");
text_data = zip_mgr.Unzip(zip_tid, text_data);
Xomp_page_itm ppg = (Xomp_page_itm)page_hash.Get_by(page_id);
ppg.Init_by_text(text_data);
}
}
finally {
rdr.Rls();
// text_conn.Rls_conn(); // TOMBSTONE: causes strange errors in tables; DATE:2016-07-06
}
}
}
class Xomp_text_db_itm {
public Xomp_text_db_itm(int text_db_id) {this.text_db_id = text_db_id;}
public int Text_db_id() {return text_db_id;} private final int text_db_id;
public List_adp Page_list() {return page_list;} private final List_adp page_list = List_adp_.New();
}