mirror of
https://github.com/gnosygnu/xowa.git
synced 2026-03-02 03:49:30 +00:00
'v3.8.1.1'
This commit is contained in:
@@ -35,7 +35,7 @@ public class Xobc_data_db_upgrader {
|
||||
//int next_check_interval = cfg_mgr.Select_int_or("", Cfg__next_check_interval, 24 * 7);
|
||||
//DateAdp last_check = DateAdp_.parse_fmt_or(last_check_str, Date_fmt, null);
|
||||
//if (last_check != null) { // check if last_check_str exists
|
||||
// Time_span span = DateAdp_.Now().Diff(last_check);
|
||||
// Time_span span = Datetime_now.Get().Diff(last_check);
|
||||
// if (span.Total_hours().To_double() < next_check_interval) { // check if enough time passed
|
||||
// Gfo_log_.Instance.Info("xobc_db update not needed", "last_check", last_check_str, "next_check_interval", next_check_interval);
|
||||
// return;
|
||||
@@ -47,7 +47,7 @@ public class Xobc_data_db_upgrader {
|
||||
// Gfo_log_.Instance.Info("xobc_db update needed b/c of missing or invalid last_check_str", "last_check", last_check_str);
|
||||
|
||||
// update needed; first, update cfg_key, then get host
|
||||
cfg_mgr.Upsert_str("", Cfg__last_check_date, DateAdp_.Now().XtoStr_fmt(Date_fmt));
|
||||
cfg_mgr.Upsert_str("", Cfg__last_check_date, Datetime_now.Get().XtoStr_fmt(Date_fmt));
|
||||
Xobc_data_db bc_db = task_mgr.Data_db();
|
||||
Xobc_host_regy_itm host_itm = bc_db.Tbl__host_regy().Select(Xobc_host_regy_tbl.Host_id__archive_org);
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ public class Xobc_version_regy_tbl implements Db_tbl {
|
||||
public void Create_tbl() {
|
||||
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
|
||||
conn.Stmt_insert(tbl_name, flds)
|
||||
.Val_int(fld_version_id, 1).Val_str(fld_version_date, DateAdp_.Now().XtoStr_fmt_yyyy_MM_dd_HH_mm()).Val_str(fld_version_note, "initial")
|
||||
.Val_int(fld_version_id, 1).Val_str(fld_version_date, Datetime_now.Get().XtoStr_fmt_yyyy_MM_dd_HH_mm()).Val_str(fld_version_note, "initial")
|
||||
.Exec_insert();
|
||||
}
|
||||
public Xobc_version_regy_itm Select_latest() {
|
||||
|
||||
@@ -81,7 +81,7 @@ class Merge_wkr__core {
|
||||
}
|
||||
private void Merge_cfg(Db_conn src_conn, Db_conn trg_conn, String src_tbl_name) {
|
||||
if (trg_conn.Meta_tbl_exists("xowa_cfg")) {
|
||||
attach_mgr.Conn_main_(trg_conn).Conn_others_(new Db_attach_itm("src_db", src_conn));
|
||||
attach_mgr.Conn_main_(trg_conn).Conn_links_(new Db_attach_itm("src_db", src_conn));
|
||||
attach_mgr.Exec_sql(String_.Concat_lines_nl
|
||||
( "INSERT INTO xowa_cfg"
|
||||
, "SELECT s.cfg_grp, s.cfg_key, s.cfg_val"
|
||||
|
||||
@@ -55,7 +55,7 @@ class Split_init__file {
|
||||
// update bin_db_id
|
||||
wkr_conn.Meta_idx_create("fsdb_img_regy", "img", "img_type", "fil_id", "thm_id");
|
||||
Gfo_log_.Instance.Prog("updating bin_db_id");
|
||||
attach_mgr.Conn_others_(new Db_attach_itm("fsdb_db", atr_conn));
|
||||
attach_mgr.Conn_links_(new Db_attach_itm("fsdb_db", atr_conn));
|
||||
|
||||
// update bin_db_id.fil
|
||||
attach_mgr.Exec_sql(String_.Concat_lines_nl // ANSI.Y
|
||||
|
||||
@@ -35,7 +35,7 @@ class Split_meta_wkr__fil extends Split_meta_wkr_base {
|
||||
}
|
||||
@Override public void On_nth_rls(Split_ctx ctx, Db_conn trg_conn) {this.stmt = Db_stmt_.Rls(stmt);}
|
||||
@Override protected String Load_sql(Db_attach_mgr attach_mgr, int ns_id, int score_bgn, int score_end) {
|
||||
attach_mgr.Conn_others_(new Db_attach_itm("atr_db", atr_conn));
|
||||
attach_mgr.Conn_links_(new Db_attach_itm("atr_db", atr_conn));
|
||||
return String_.Concat_lines_nl
|
||||
( "SELECT f.fil_id, f.fil_owner_id, f.fil_xtn_id, f.fil_ext_id, f.fil_bin_db_id, f.fil_name, f.fil_size, f.fil_modified, f.fil_hash, fir.page_id"
|
||||
, "FROM <atr_db>fsdb_fil f"
|
||||
|
||||
@@ -35,7 +35,7 @@ class Split_meta_wkr__org extends Split_meta_wkr_base {
|
||||
}
|
||||
@Override public void On_nth_rls(Split_ctx ctx, Db_conn trg_conn) {this.stmt = Db_stmt_.Rls(stmt);}
|
||||
@Override protected String Load_sql(Db_attach_mgr attach_mgr, int ns_id, int score_bgn, int score_end) {
|
||||
attach_mgr.Conn_others_(new Db_attach_itm("atr_db", atr_conn));
|
||||
attach_mgr.Conn_links_(new Db_attach_itm("atr_db", atr_conn));
|
||||
return String_.Concat_lines_nl
|
||||
( "SELECT o.orig_ttl, o.orig_repo, o.orig_status, o.orig_ext, o.orig_w, o.orig_h, o.orig_redirect, fir.page_id"
|
||||
, "FROM <atr_db>orig_reg o"
|
||||
|
||||
@@ -35,7 +35,7 @@ class Split_meta_wkr__thm extends Split_meta_wkr_base {
|
||||
}
|
||||
@Override public void On_nth_rls(Split_ctx ctx, Db_conn trg_conn) {this.stmt = Db_stmt_.Rls(stmt);}
|
||||
@Override protected String Load_sql(Db_attach_mgr attach_mgr, int ns_id, int score_bgn, int score_end) {
|
||||
attach_mgr.Conn_others_(new Db_attach_itm("atr_db", atr_conn));
|
||||
attach_mgr.Conn_links_(new Db_attach_itm("atr_db", atr_conn));
|
||||
return String_.Concat_lines_nl
|
||||
( "SELECT t.thm_id, t.thm_owner_id, t.thm_w, t.thm_h, t.thm_time, t.thm_page, t.thm_bin_db_id, t.thm_size, t.thm_modified, t.thm_hash, fir.page_id"
|
||||
, "FROM <atr_db>fsdb_thm t"
|
||||
|
||||
@@ -58,7 +58,7 @@ public class Split_wkr__rndm implements Split_wkr {
|
||||
Rndm_seq_tbl seq_tbl = new Rndm_seq_tbl(trg_conn); seq_tbl.Create_tbl();
|
||||
Split_tbl_.Bld_insert_by_select(tmp_bfr, seq_tbl.Tbl_name(), seq_tbl.Flds());
|
||||
tmp_bfr.Add_str_u8_fmt("WHERE {0} = {1} AND {2} = {3}", seq_tbl.Fld__qry_idx(), bldr.Qry_idx(), seq_tbl.Fld__rng_idx(), bldr.Rng_idx());
|
||||
attach_mgr.Conn_main_(trg_conn).Conn_others_(new Db_attach_itm("src_db", bldr.Conn()));
|
||||
attach_mgr.Conn_main_(trg_conn).Conn_links_(new Db_attach_itm("src_db", bldr.Conn()));
|
||||
attach_mgr.Exec_sql(tmp_bfr.To_str_and_clear());
|
||||
// bldr.Conn().Txn_bgn("rndm");
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ class Split_meta_wkr__link extends Split_meta_wkr_base {
|
||||
}
|
||||
@Override protected String Load_sql(Db_attach_mgr attach_mgr, int ns_id, int score_bgn, int score_end) {
|
||||
int trg_db_id = srch_db_mgr.Tbl__link__get_idx(ns_id);
|
||||
attach_mgr.Conn_others_(new Db_attach_itm("link_db", srch_db_mgr.Tbl__link__get_at(trg_db_id).conn));
|
||||
attach_mgr.Conn_links_(new Db_attach_itm("link_db", srch_db_mgr.Tbl__link__get_at(trg_db_id).conn));
|
||||
return String_.Concat_lines_nl
|
||||
( "SELECT sl.word_id, sl.page_id, sl.link_score, sw.page_id, " + trg_db_id + " AS trg_db_id"
|
||||
, "FROM <link_db>search_link sl"
|
||||
|
||||
@@ -42,7 +42,7 @@ class Split_srch_init {
|
||||
int len = srch_db_mgr.Tbl__link__len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Db_conn link_conn = srch_db_mgr.Tbl__link__get_at(i).conn;
|
||||
attach_mgr.Conn_others_(new Db_attach_itm("word_db", word_conn), new Db_attach_itm("link_db", link_conn));
|
||||
attach_mgr.Conn_links_(new Db_attach_itm("word_db", word_conn), new Db_attach_itm("link_db", link_conn));
|
||||
attach_mgr.Exec_sql(String_.Concat_lines_nl // ANSI.Y
|
||||
( "INSERT INTO split_search_word (word_id, word_text, link_count, link_count_score, link_score_min, link_score_max, page_uid, page_ns, page_id, page_score)"
|
||||
, "SELECT sw.word_id, sw.word_text, sw.link_count, sw.link_count_score, sw.link_score_min, sw.link_score_max, Min(pr.page_uid), -1, -1, -1"
|
||||
|
||||
@@ -21,7 +21,7 @@ import gplx.xowa.files.*;
|
||||
import gplx.xowa.wikis.nss.*;
|
||||
import gplx.xowa.wikis.*; import gplx.xowa.wikis.domains.*;
|
||||
import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.parsers.lnkis.*; import gplx.xowa.parsers.xndes.*;
|
||||
import gplx.xowa.htmls.core.bldrs.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.wdatas.*;
|
||||
import gplx.xowa.htmls.core.bldrs.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.wbases.*;
|
||||
import gplx.fsdb.meta.*; import gplx.xowa.files.fsdb.*; import gplx.fsdb.*;
|
||||
import gplx.xowa.langs.vnts.*; import gplx.xowa.parsers.vnts.*;
|
||||
import gplx.xowa.parsers.lnkis.files.*;
|
||||
@@ -112,7 +112,7 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
|
||||
else {
|
||||
parser.Parse_page_all_clear(root, ctx, ctx.Tkn_mkr(), page_src);
|
||||
if ( gen_html
|
||||
&& page.Redirect().Itms__len() == 0) // don't generate html for redirected pages
|
||||
&& page.Redirect_trail().Itms__len() == 0) // don't generate html for redirected pages
|
||||
wiki.Html_mgr().Page_wtr_mgr().Gen(ctx.Page().Root_(root), Xopg_page_.Tid_read);
|
||||
if (gen_hdump)
|
||||
hdump_bldr.Insert(ctx, page.Root_(root));
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.hdumps.diffs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.hdumps.*;
|
||||
import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Dumpdiff_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr {
|
||||
public Xob_cmd[] Bldr_cmds() {
|
||||
return new Xob_cmd[]
|
||||
{ Dumpdiff_cmd.Prototype
|
||||
};
|
||||
}
|
||||
|
||||
public String Addon__key() {return "xowa.builds.hdumps.diff";}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.hdumps.diffs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.hdumps.*;
|
||||
class Dumpdiff_cfg implements Gfo_invk {
|
||||
public Io_url Prv_dir() {return prv_dir;} private Io_url prv_dir;
|
||||
public Io_url Cur_dir() {return cur_dir;} private Io_url cur_dir;
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk__cur_dir_)) this.cur_dir = m.ReadIoUrl("v");
|
||||
else if (ctx.Match(k, Invk__prv_dir_)) this.prv_dir = m.ReadIoUrl("v");
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
}
|
||||
public static final String Invk__cur_dir_ = "cur_dir_", Invk__prv_dir_ = "prv_dir_";
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.hdumps.diffs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.hdumps.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.htmls.core.htmls.*;
|
||||
public class Dumpdiff_cmd extends Xob_cmd__base {
|
||||
private final Dumpdiff_cfg cfg = new Dumpdiff_cfg();
|
||||
private final Dumpdiff_mgr mgr = new Dumpdiff_mgr();
|
||||
public Dumpdiff_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
|
||||
@Override public void Cmd_run() {
|
||||
wiki.Init_assert();
|
||||
mgr.Exec(bldr.App(), wiki, cfg);
|
||||
}
|
||||
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk__cfg)) return cfg;
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
}
|
||||
private static final String Invk__cfg = "cfg";
|
||||
|
||||
public static final String BLDR_CMD_KEY = "hdump.diff";
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
|
||||
public static final Xob_cmd Prototype = new Dumpdiff_cmd(null, null);
|
||||
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Dumpdiff_cmd(bldr, wiki);}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.hdumps.diffs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.hdumps.*;
|
||||
import gplx.dbs.*;
|
||||
class Dumpdiff_log_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld__page_id, fld__cur_snip, fld__prv_snip;
|
||||
private Db_stmt stmt__insert;
|
||||
public Dumpdiff_log_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
flds.Add_int_pkey("uid");
|
||||
this.fld__page_id = flds.Add_int("page_id");
|
||||
this.fld__cur_snip = flds.Add_str("cur_snip", 1024);
|
||||
this.fld__prv_snip = flds.Add_str("prv_snip", 1024);
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name = "diff_log";
|
||||
public Db_conn Conn() {return conn;} private final Db_conn conn;
|
||||
public void Create_tbl() {
|
||||
conn.Meta_tbl_remake(Dbmeta_tbl_itm.New(tbl_name, flds, Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, fld__page_id, fld__page_id)));
|
||||
}
|
||||
public void Insert_bgn() {
|
||||
stmt__insert = conn.Stmt_insert(tbl_name, fld__page_id, fld__cur_snip, fld__prv_snip);
|
||||
conn.Txn_bgn("diff_log");
|
||||
}
|
||||
public void Insert_by_batch(int page_id, byte[] prv_snip, byte[] cur_snip) {
|
||||
stmt__insert.Clear().Val_int(fld__page_id, page_id).Val_bry_as_str(fld__cur_snip, cur_snip).Val_bry_as_str(fld__prv_snip, prv_snip).Exec_insert();
|
||||
}
|
||||
public void Insert_end() {
|
||||
conn.Txn_end();
|
||||
stmt__insert.Rls();
|
||||
}
|
||||
public void Rls() {}
|
||||
|
||||
public static Dumpdiff_log_tbl New(Xowe_wiki wiki) {
|
||||
Db_conn conn = Db_conn_bldr.Instance.Get_or_autocreate(true, wiki.Fsys_mgr().Root_dir().GenSubFil("xowa.diff.sqlite3"));
|
||||
Dumpdiff_log_tbl rv = new Dumpdiff_log_tbl(conn);
|
||||
conn.Meta_tbl_remake(rv);
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.hdumps.diffs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.hdumps.*;
|
||||
import gplx.core.brys.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.htmls.*;
|
||||
import gplx.xowa.wikis.data.tbls.*;
|
||||
class Dumpdiff_mgr {
|
||||
public void Exec(Xoae_app app, Xowe_wiki wiki, Dumpdiff_cfg cfg) {
|
||||
// init log_tbl, wikis
|
||||
Dumpdiff_log_tbl log_tbl = Dumpdiff_log_tbl.New(wiki);
|
||||
Xowe_wiki cur_wiki = Get_wiki_by_dir(wiki, Bool_.Y, cfg.Cur_dir());
|
||||
Xowe_wiki prv_wiki = Get_wiki_by_dir(wiki, Bool_.N, cfg.Prv_dir());
|
||||
|
||||
// init html_loader, page_loader
|
||||
Hdump_html_loader cur_html_loader = new Hdump_html_loader(cur_wiki);
|
||||
Hdump_html_loader prv_html_loader = new Hdump_html_loader(prv_wiki);
|
||||
Dumpdiff_page_loader page_loader = new Dumpdiff_page_loader(cur_wiki, prv_wiki, 10000);
|
||||
List_adp list = List_adp_.New();
|
||||
|
||||
// prepare for loop
|
||||
int page_count = 0, diff_count = 0;
|
||||
log_tbl.Insert_bgn();
|
||||
|
||||
// loop page_table until no more
|
||||
while (true) {
|
||||
page_loader.Load(list);
|
||||
int list_len = list.Count();
|
||||
if (list_len == 0) break;
|
||||
|
||||
// loop pages, compare, and log
|
||||
for (int i = 0; i < list_len; ++i) {
|
||||
Dumpdiff_page_itm page = (Dumpdiff_page_itm)list.Get_at(i);
|
||||
int page_id = page.Page_id();
|
||||
byte[] cur_html = cur_html_loader.Load(page_id, page.Cur_db_id());
|
||||
byte[] prv_html = prv_html_loader.Load(page_id, page.Prv_db_id());
|
||||
byte[][] diff = Bry_diff_.Diff_1st_line(cur_html, prv_html);
|
||||
++page_count;
|
||||
if (diff != null) {
|
||||
log_tbl.Insert_by_batch(page_id, diff[0], diff[1]);
|
||||
++diff_count;
|
||||
Gfo_usr_dlg_.Instance.Warn_many("", "", "hdump diff: pages=~{0} diffs=~{1} page_id=~{2} cur=~{3} prv=~{4}", page_count, diff_count, page_id, diff[0], diff[1]);
|
||||
}
|
||||
}
|
||||
list.Clear();
|
||||
}
|
||||
|
||||
// cleanup
|
||||
log_tbl.Insert_end();
|
||||
}
|
||||
private static Xowe_wiki Get_wiki_by_dir(Xowe_wiki dflt_wiki, boolean wiki_is_cur, Io_url wiki_dir) {
|
||||
Xowe_wiki rv = null;
|
||||
if (wiki_dir == null) {
|
||||
if (wiki_is_cur)
|
||||
rv = dflt_wiki;
|
||||
else
|
||||
throw Err_.new_("", "prv_dir not specified");
|
||||
}
|
||||
if (rv == null)
|
||||
rv = gplx.xowa.addons.bldrs.mass_parses.parses.Xow_wiki_utl_.Clone_wiki(dflt_wiki, wiki_dir);
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.hdumps.diffs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.hdumps.*;
|
||||
class Dumpdiff_page_itm {
|
||||
public Dumpdiff_page_itm(int page_id, int ns_id, byte[] ttl_bry, int cur_db_id, int prv_db_id) {
|
||||
this.page_id = page_id;
|
||||
this.ns_id = ns_id;
|
||||
this.ttl_bry = ttl_bry;
|
||||
this.cur_db_id = cur_db_id;
|
||||
this.prv_db_id = prv_db_id;
|
||||
}
|
||||
public int Page_id() {return page_id;} private final int page_id;
|
||||
public int Ns_id() {return ns_id;} private final int ns_id;
|
||||
public byte[] Ttl_bry() {return ttl_bry;} private final byte[] ttl_bry;
|
||||
public int Cur_db_id() {return cur_db_id;} private final int cur_db_id;
|
||||
public int Prv_db_id() {return prv_db_id;} private final int prv_db_id;
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.hdumps.diffs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.hdumps.*;
|
||||
import gplx.dbs.*;
|
||||
class Dumpdiff_page_loader {
|
||||
private final Db_attach_mgr attach_mgr = new Db_attach_mgr();
|
||||
private int select_count, cutoff_page_id;
|
||||
public Dumpdiff_page_loader(Xowe_wiki cur_wiki, Xowe_wiki prv_wiki, int select_count) {
|
||||
this.select_count = select_count;
|
||||
attach_mgr.Conn_main_(cur_wiki.Data__core_mgr().Db__core().Conn());
|
||||
attach_mgr.Conn_links_(new Db_attach_itm("prv_db", prv_wiki.Data__core_mgr().Db__core().Conn()));
|
||||
}
|
||||
public void Load(List_adp rv) {
|
||||
String sql = String_.Format(String_.Concat_lines_nl_skip_last // ANSI.Y
|
||||
( "SELECT cur.page_id"
|
||||
, ", cur.page_namespace AS ns_id"
|
||||
, ", cur.page_title AS ttl_bry"
|
||||
, ", cur.page_html_db_id AS cur_html_db_id"
|
||||
, ", prv.page_html_db_id AS prv_html_db_id"
|
||||
, "FROM page cur"
|
||||
, " JOIN <prv_db>page prv ON cur.page_id = prv.page_id"
|
||||
, "WHERE cur.page_id > {0}"
|
||||
, "AND cur.page_html_db_id != -1"
|
||||
, "LIMIT {1}"
|
||||
), cutoff_page_id, select_count);
|
||||
sql = attach_mgr.Resolve_sql(sql);
|
||||
|
||||
attach_mgr.Attach();
|
||||
Db_rdr rdr = attach_mgr.Conn_main().Stmt_sql(sql).Exec_select__rls_auto();
|
||||
while (rdr.Move_next()) {
|
||||
cutoff_page_id = rdr.Read_int("page_id");
|
||||
rv.Add(new Dumpdiff_page_itm(cutoff_page_id
|
||||
, rdr.Read_int("ns_id"), rdr.Read_bry_by_str("ttl_bry")
|
||||
, rdr.Read_int("cur_html_db_id"), rdr.Read_int("prv_html_db_id")
|
||||
));
|
||||
}
|
||||
attach_mgr.Detach();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.hdumps.diffs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.hdumps.*;
|
||||
import gplx.core.ios.*; import gplx.core.ios.streams.*;
|
||||
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.htmls.*;
|
||||
class Hdump_html_loader {
|
||||
private final Xowe_wiki wiki;
|
||||
private Io_stream_zip_mgr stream_zip_mgr = new Io_stream_zip_mgr();
|
||||
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
|
||||
private final Xoh_page tmp_hpg = new Xoh_page();
|
||||
private final Xoa_url tmp_url; private final Xoa_ttl tmp_ttl;
|
||||
|
||||
public Hdump_html_loader(Xowe_wiki wiki) {
|
||||
this.wiki = wiki;
|
||||
this.tmp_url = wiki.Utl__url_parser().Parse(Bry_.new_a7("temp"));
|
||||
this.tmp_ttl = wiki.Ttl_parse(Bry_.new_a7("temp"));
|
||||
}
|
||||
|
||||
public byte[] Load(int page_id, int html_db_id) {
|
||||
// load html_bry from db
|
||||
Xow_db_file html_db = wiki.Data__core_mgr().Dbs__get_by_id_or_fail(html_db_id);
|
||||
tmp_hpg.Ctor_by_hview(wiki, tmp_url, tmp_ttl, page_id);
|
||||
html_db.Tbl__html().Select_by_page(tmp_hpg);
|
||||
|
||||
// unzip it
|
||||
byte[] html_hzip = stream_zip_mgr.Unzip(Io_stream_.Tid_gzip, tmp_hpg.Db().Html().Html_bry());
|
||||
return wiki.Html__hdump_mgr().Load_mgr().Decode_as_bry(tmp_bfr, tmp_hpg, html_hzip, true);
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,7 @@ package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xow
|
||||
import gplx.dbs.*;
|
||||
class Xob_lnki_temp_row implements CompareAble {
|
||||
public int Xomp_uid() {return xomp_uid;} private int xomp_uid;
|
||||
public int Make_uid() {return make_uid;} private int make_uid;
|
||||
public int Lnki_id() {return lnki_id;} private int lnki_id;
|
||||
public int Lnki_tier_id() {return lnki_tier_id;} private int lnki_tier_id;
|
||||
public int Lnki_page_id() {return lnki_page_id;} private int lnki_page_id;
|
||||
public byte[] Lnki_ttl() {return lnki_ttl;} private byte[] lnki_ttl;
|
||||
@@ -34,24 +34,24 @@ class Xob_lnki_temp_row implements CompareAble {
|
||||
public int Lnki_page() {return lnki_page;} private int lnki_page;
|
||||
public void Load(Db_rdr rdr, int xomp_uid) {
|
||||
this.xomp_uid = xomp_uid;
|
||||
this.make_uid = rdr.Read_int("lnki_make_uid");
|
||||
this.lnki_id = rdr.Read_int("lnki_id");
|
||||
this.lnki_tier_id = rdr.Read_int("lnki_tier_id");
|
||||
this.lnki_page_id = rdr.Read_int("lnki_page_id");
|
||||
this.lnki_ttl = rdr.Read_bry_by_str("lnki_ttl");
|
||||
this.lnki_commons_ttl = rdr.Read_bry_by_str("lnki_commons_ttl");
|
||||
this.lnki_commons_ttl = Bry_.new_u8_safe(rdr.Read_str("lnki_commons_ttl"));
|
||||
this.lnki_ext = rdr.Read_byte("lnki_ext");
|
||||
this.lnki_type = rdr.Read_byte("lnki_type");
|
||||
this.lnki_src_tid = rdr.Read_byte("lnki_src_tid");
|
||||
this.lnki_w = rdr.Read_int("lnki_w");
|
||||
this.lnki_h = rdr.Read_int("lnki_h");
|
||||
this.lnki_upright = rdr.Read_double("lnki_upright");
|
||||
this.lnki_time = rdr.Read_int("lnki_time");
|
||||
this.lnki_time = rdr.Read_double("lnki_time");
|
||||
this.lnki_page = rdr.Read_int("lnki_page");
|
||||
}
|
||||
public int compareTo(Object obj) {
|
||||
Xob_lnki_temp_row comp = (Xob_lnki_temp_row)obj;
|
||||
int rv = Int_.Compare(xomp_uid, comp.xomp_uid);
|
||||
if (rv != CompareAble_.Same) return rv;
|
||||
return Int_.Compare(make_uid, comp.make_uid);
|
||||
return Int_.Compare(lnki_id, comp.lnki_id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,8 +21,8 @@ public class Xomp_make_cmd extends Xob_cmd__base {
|
||||
public Xomp_make_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
|
||||
@Override public void Cmd_run() {
|
||||
wiki.Init_assert();
|
||||
Xomp_make_wkr wkr = new Xomp_make_wkr(wiki);
|
||||
wkr.Exec();
|
||||
new Xomp_make_wkr(wiki).Exec();
|
||||
new Xomp_make_lnki().Exec(wiki, 1000);
|
||||
}
|
||||
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;} private static final String BLDR_CMD_KEY = "wiki.mass_parse.make";
|
||||
|
||||
@@ -44,7 +44,8 @@ class Xomp_make_lnki {
|
||||
Xomp_wkr_db wkr_db = xomp_db.Wkr_db(Bool_.N, i);
|
||||
count += Load_rows(rows, xomp_db, wkr_db, uid_bgn, uid_bgn + uid_count);
|
||||
}
|
||||
if (count == 0) break;
|
||||
if (count < 0) break;
|
||||
uid_bgn += count;
|
||||
Save_rows(rows, lnki_temp_tbl);
|
||||
}
|
||||
|
||||
@@ -56,7 +57,7 @@ class Xomp_make_lnki {
|
||||
private int Load_rows(List_adp rows, Xomp_db_core xomp_db, Xomp_wkr_db wkr_db, int uid_bgn, int uid_end) {
|
||||
// build sql
|
||||
Db_attach_mgr attach_mgr = new Db_attach_mgr(xomp_db.Conn());
|
||||
attach_mgr.Conn_others_(new Db_attach_itm("wkr_db", wkr_db.Conn()));
|
||||
attach_mgr.Conn_links_(new Db_attach_itm("wkr_db", wkr_db.Conn()));
|
||||
String sql = String_.Format(String_.Concat_lines_nl_skip_last
|
||||
( "SELECT mgr.xomp_uid"
|
||||
, ", wkr.*"
|
||||
@@ -77,6 +78,7 @@ class Xomp_make_lnki {
|
||||
rv = rdr.Read_int("xomp_uid");
|
||||
Xob_lnki_temp_row row = new Xob_lnki_temp_row();
|
||||
row.Load(rdr, rv);
|
||||
rows.Add(row);
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
attach_mgr.Detach();
|
||||
|
||||
@@ -64,7 +64,7 @@ class Xomp_load_wkr implements Gfo_invk {
|
||||
, "AND mp.page_status = 0"
|
||||
, "LIMIT {1}"
|
||||
), prv_page_id, queue.Capacity());
|
||||
this.attach_mgr.Conn_others_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
|
||||
this.attach_mgr.Conn_links_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
|
||||
sql = attach_mgr.Resolve_sql(sql);
|
||||
|
||||
// page_tbl.load_sql
|
||||
|
||||
@@ -60,7 +60,7 @@ class Xomp_page_pool_loader {
|
||||
, "AND mp.page_status = 0"
|
||||
, "LIMIT {1}"
|
||||
), prv_uid, num_pages_per_load);
|
||||
this.attach_mgr.Conn_others_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
|
||||
this.attach_mgr.Conn_links_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
|
||||
sql = attach_mgr.Resolve_sql(sql);
|
||||
|
||||
// run page_tbl
|
||||
|
||||
@@ -34,7 +34,7 @@ class Xomp_parse_mgr {
|
||||
Xomp_page_pool page_pool = new Xomp_page_pool(pool_loader, cfg.Num_pages_per_wkr());
|
||||
prog_mgr.Init(pool_loader.Get_pending_count(), cfg.Progress_interval());
|
||||
wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of cache
|
||||
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, true);
|
||||
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates());
|
||||
Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
|
||||
Gfo_cache_mgr ifexist_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
|
||||
|
||||
@@ -47,8 +47,8 @@ class Xomp_parse_mgr {
|
||||
latch = new Gfo_countdown_latch(wkr_len);
|
||||
Xomp_parse_wkr[] wkrs = new Xomp_parse_wkr[wkr_len];
|
||||
for (int i = 0; i < wkr_len; ++i) {
|
||||
Xowe_wiki wkr_wiki = Clone_wiki(wiki);
|
||||
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, wkr_wiki, page_pool, i, cfg.Cleanup_interval(), cfg.Progress_interval(), cfg.Log_file_lnkis());
|
||||
Xowe_wiki wkr_wiki = Xow_wiki_utl_.Clone_wiki(wiki, wiki.Fsys_mgr().Root_dir());
|
||||
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, wkr_wiki, page_pool, i, cfg, cfg.Cleanup_interval(), cfg.Progress_interval(), cfg.Log_file_lnkis());
|
||||
wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);
|
||||
// remove wmf wkr, else will try to download images during parsing
|
||||
if (wkr_wiki.File__bin_mgr() != null)
|
||||
@@ -74,11 +74,4 @@ class Xomp_parse_mgr {
|
||||
}
|
||||
Gfo_usr_dlg_.Instance.Note_many("", "", bfr.To_str_and_clear());
|
||||
}
|
||||
private static Xowe_wiki Clone_wiki(Xowe_wiki wiki) {
|
||||
Xol_lang_itm lang = new Xol_lang_itm(wiki.App().Lang_mgr(), wiki.Lang().Key_bry());
|
||||
Xol_lang_itm_.Lang_init(lang);
|
||||
Xowe_wiki rv = new Xowe_wiki(wiki.Appe(), lang, gplx.xowa.wikis.nss.Xow_ns_mgr_.default_(lang.Case_mgr()), wiki.Domain_itm(), wiki.Fsys_mgr().Root_dir());
|
||||
rv.Init_by_wiki();
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.core.ios.streams.*;
|
||||
class Xomp_parse_mgr_cfg implements Gfo_invk {
|
||||
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1; // use env.available_processors
|
||||
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = 1000;
|
||||
@@ -27,6 +28,8 @@ class Xomp_parse_mgr_cfg implements Gfo_invk {
|
||||
public boolean Hzip_enabled() {return hzip_enabled;} private boolean hzip_enabled = true;
|
||||
public boolean Hdiff_enabled() {return hdiff_enabled;} private boolean hdiff_enabled = true;
|
||||
public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true;
|
||||
public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true;
|
||||
public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_.Tid_gzip;
|
||||
public void Init(Xowe_wiki wiki) {
|
||||
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Env_.System_cpu_count();
|
||||
}
|
||||
@@ -41,12 +44,16 @@ class Xomp_parse_mgr_cfg implements Gfo_invk {
|
||||
else if (ctx.Match(k, Invk__hdump_enabled_)) hdump_enabled = m.ReadBool("v");
|
||||
else if (ctx.Match(k, Invk__hzip_enabled_)) hzip_enabled = m.ReadBool("v");
|
||||
else if (ctx.Match(k, Invk__hdiff_enabled_)) hdiff_enabled = m.ReadBool("v");
|
||||
else if (ctx.Match(k, Invk__zip_tid_)) zip_tid = m.ReadByte("v");
|
||||
else if (ctx.Match(k, Invk__load_all_templates_)) load_all_templates = m.ReadYn("v");
|
||||
else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v"));
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
}
|
||||
private static final String
|
||||
Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
|
||||
, Invk__progress_interval_ = "progress_interval_", Invk__commit_interval_ = "commit_interval_", Invk__cleanup_interval_ = "cleanup_interval_"
|
||||
, Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_"
|
||||
, Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_", Invk__zip_tid_ = "zip_tid_"
|
||||
, Invk__load_all_templates_ = "load_all_templates_", Invk__manual_now_ = "manual_now_"
|
||||
;
|
||||
}
|
||||
|
||||
@@ -24,14 +24,16 @@ class Xomp_parse_wkr implements Gfo_invk {
|
||||
private final Xomp_page_pool page_pool;
|
||||
private final int idx;
|
||||
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
|
||||
private final Xomp_parse_mgr_cfg cfg;
|
||||
private int done_count; private long done_time;
|
||||
private Xomp_wkr_db wkr_db; private int cleanup_interval, commit_interval;
|
||||
private boolean log_file_lnkis;
|
||||
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xomp_page_pool page_pool, int idx, int cleanup_interval, int commit_interval, boolean log_file_lnkis) {
|
||||
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xomp_page_pool page_pool, int idx, Xomp_parse_mgr_cfg cfg, int cleanup_interval, int commit_interval, boolean log_file_lnkis) {
|
||||
this.mgr = mgr; this.wiki = wiki;
|
||||
this.page_pool = page_pool;
|
||||
this.idx = idx;
|
||||
this.wkr_db = mgr.Db_core().Wkr_db(Bool_.Y, idx); // NOTE: must go in ctor, or else thread issues
|
||||
this.cfg = cfg;
|
||||
this.cleanup_interval = cleanup_interval;
|
||||
this.commit_interval = commit_interval;
|
||||
this.log_file_lnkis = log_file_lnkis;
|
||||
@@ -40,7 +42,7 @@ class Xomp_parse_wkr implements Gfo_invk {
|
||||
public Xob_hdump_bldr Hdump_bldr() {return hdump_bldr;} private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
|
||||
public void Exec() {
|
||||
// init
|
||||
Xow_parser_mgr parser_mgr = new Xow_parser_mgr(wiki);
|
||||
Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
|
||||
|
||||
// disable file download
|
||||
wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make
|
||||
@@ -58,8 +60,8 @@ class Xomp_parse_wkr implements Gfo_invk {
|
||||
logger.Bgn();
|
||||
}
|
||||
|
||||
// enable hdump
|
||||
hdump_bldr.Enabled_(true).Hzip_enabled_(true).Hzip_diff_(true).Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
|
||||
// enable hdump
|
||||
hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hzip_enabled()).Zip_tid_(cfg.Zip_tid()).Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
|
||||
wkr_db.Conn().Txn_bgn("xomp");
|
||||
|
||||
while (true) {
|
||||
|
||||
@@ -22,9 +22,10 @@ import gplx.xowa.wikis.data.tbls.*;
|
||||
class Xomp_text_db_loader {
|
||||
private final Xow_wiki wiki;
|
||||
private final Ordered_hash text_db_hash = Ordered_hash_.New();
|
||||
private final Io_stream_zip_mgr zip_mgr = Xoa_app_.Utl__zip_mgr();
|
||||
private final Io_stream_zip_mgr zip_mgr;
|
||||
public Xomp_text_db_loader(Xow_wiki wiki) {
|
||||
this.wiki = wiki;
|
||||
this.zip_mgr = wiki.Utl__zip_mgr();
|
||||
}
|
||||
public void Add(int text_db_id, Xowd_text_bry_owner ppg) {
|
||||
Xomp_text_db_itm itm = (Xomp_text_db_itm)text_db_hash.Get_by(text_db_id);
|
||||
|
||||
@@ -71,11 +71,26 @@ class Xomp_tmpl_cache_bldr {
|
||||
for (int i = 0; i < redirect_len; ++i) {
|
||||
Xow_page_cache_itm src_itm = (Xow_page_cache_itm)redirect_list.Get_at(i);
|
||||
Xow_page_cache_itm trg_itm = (Xow_page_cache_itm)page_regy.Get_by(src_itm.Redirect_id());
|
||||
if (trg_itm == null) {
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "missing redirect for tmpl: ~{0}", src_itm.Ttl().Full_db());
|
||||
continue;
|
||||
byte[] trg_itm_wtxt = null;
|
||||
Xoa_ttl trg_ttl = null;
|
||||
if (trg_itm == null) { // template can redirect to non-template pages
|
||||
Xoa_ttl src_ttl = src_itm.Ttl();
|
||||
Xoae_page wpg = Xoae_page.New(wiki, src_ttl);
|
||||
wiki.Data_mgr().Load_from_db(wpg, src_ttl.Ns(), src_ttl, false);
|
||||
if (wpg.Db().Page().Exists()) {
|
||||
trg_itm_wtxt = wpg.Db().Text().Text_bry();
|
||||
trg_ttl = wpg.Ttl();
|
||||
}
|
||||
else {
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "missing redirect for tmpl: ~{0}", src_itm.Ttl().Full_db());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
src_itm.Set_redirect_bry(trg_itm.Wtxt__direct());
|
||||
else {
|
||||
trg_itm_wtxt = trg_itm.Wtxt__direct();
|
||||
trg_ttl = trg_itm.Ttl();
|
||||
}
|
||||
src_itm.Set_redirect(trg_ttl, trg_itm_wtxt); // NOTE: itm must have title of redirect, not original item; EX:Template:Ifempty -> Template:If_empty; DATE:2016-07-26
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.xowa.langs.*; import gplx.xowa.langs.cases.*;
|
||||
public class Xow_wiki_utl_ {
|
||||
public static Xowe_wiki Clone_wiki(Xowe_wiki wiki, Io_url wiki_dir) {
|
||||
Xol_lang_itm lang = new Xol_lang_itm(wiki.App().Lang_mgr(), wiki.Lang().Key_bry());
|
||||
Xol_lang_itm_.Lang_init(lang);
|
||||
Xowe_wiki rv = new Xowe_wiki(wiki.Appe(), lang, gplx.xowa.wikis.nss.Xow_ns_mgr_.default_(lang.Case_mgr()), wiki.Domain_itm(), wiki_dir);
|
||||
rv.Init_by_wiki();
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
@@ -40,7 +40,7 @@ class Xow_import_dir_cbk__xodir implements Xow_import_dir_cbk {
|
||||
wiki.App().User().User_db_mgr().Cfg().Set_app_str("xowa.xodir.custom_dir", path);
|
||||
|
||||
// redirect to import_dir
|
||||
page.Redirect().Itms__add__special(wiki, Xobc_xodir_special.Prototype.Special__meta());
|
||||
page.Redirect_trail().Itms__add__special(wiki, Xobc_xodir_special.Prototype.Special__meta());
|
||||
}
|
||||
public static Xow_import_dir_cbk__xodir Instance = new Xow_import_dir_cbk__xodir(); Xow_import_dir_cbk__xodir() {}
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ public class Xobc_xodir_special implements Xow_special_page {
|
||||
Xobc_xodir_cfg.Set_app_str__selected(wiki.App(), path);
|
||||
// On_path_selected.Invk(null, -1, "", null);
|
||||
// page.Redirect().Itms__add__special(wiki, Xobc_task_special.Prototype.Special__meta());
|
||||
page.Redirect().Itms__add__special(wiki, Prototype.Special__meta());
|
||||
page.Redirect_trail().Itms__add__special(wiki, Prototype.Special__meta());
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user