Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

'v3.7.4.1'

Browse Source
This commit is contained in:
gnosygnu
2016-07-25 21:59:51 -04:00
parent 7a851a41a5
commit 8e91ac0bc4
175 changed files with 2079 additions and 933 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
400_xowa/src/gplx/xowa/addons/bldrs/centrals/Xobc_task_addon.java
View File

@@ -17,8 +17,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.centrals; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.specials.*; import gplx.xowa.htmls.bridges.*;
import gplx.xowa.addons.bldrs.infos.*;
public class Xobc_task_addon implements Xoax_addon_itm, Xoax_addon_itm__special, Xoax_addon_itm__json {
import gplx.xowa.addons.bldrs.infos.*; import gplx.xowa.addons.bldrs.xodirs.*;
public class Xobc_task_addon implements Xoax_addon_itm, Xoax_addon_itm__special, Xoax_addon_itm__json, Xoax_addon_itm__init {
public Xobc_xodir_mgr Xodir_mgr() {return xodir_mgr;} private Xobc_xodir_mgr xodir_mgr;
public void Xodir_mgr_(Xobc_xodir_mgr v) {this.xodir_mgr = v;}
public void Init_addon_by_app(Xoa_app app) {
this.xodir_mgr = new Xobc_xodir_mgr__pc(app);
}
public void Init_addon_by_wiki(Xow_wiki wiki) {}
public Xow_special_page[] Special_pages() {
return new Xow_special_page[]
{ Xobc_task_special.Prototype
@@ -31,5 +37,5 @@ public class Xobc_task_addon implements Xoax_addon_itm, Xoax_addon_itm__special,
};
}
public String Addon__key() {return "xowa.imports.downloads";}
public String Addon__key() {return ADDON__KEY;} public static final String ADDON__KEY = "xowa.imports.downloads";
}

1
400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__unzip.java
View File

@@ -29,6 +29,7 @@ public class Xobc_cmd__unzip extends Xobc_cmd__base {
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.core.ios.zips.zip_unzip";
@Override public String Cmd_name() {return "unzip";}
@Override public boolean Cmd_suspendable() {return true;}
@Override public String Cmd_fallback() {return Xobc_cmd__verify_fil.CMD_TYPE;} // if unzip fails, backtrack to verify; if verify fails, it'll backtrack to download; DATE:2016-07-25
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
if (wkr.Exec(this, src_fil, trg_dir, trg_fils) == Gfo_prog_ui_.Status__fail)

37
400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cfg.java Normal file
View File

@@ -0,0 +1,37 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Pack_file_bldr_cfg implements Gfo_invk {
public Io_url Deploy_dir() {return deploy_dir;} private Io_url deploy_dir;
public boolean Pack_html() {return pack_html;} private boolean pack_html = true;
public boolean Pack_file() {return pack_file;} private boolean pack_file = true;
public DateAdp Pack_file_cutoff() {return pack_file_cutoff;} private DateAdp pack_file_cutoff = null;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__deploy_dir_)) deploy_dir = m.ReadIoUrl("v");
else if (ctx.Match(k, Invk__pack_html_)) pack_html = m.ReadYn("v");
else if (ctx.Match(k, Invk__pack_file_)) pack_file = m.ReadYn("v");
else if (ctx.Match(k, Invk__pack_file_cutoff_)) pack_file_cutoff = m.ReadDate("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}
private static final String Invk__deploy_dir_ = "deploy_dir_"
, Invk__pack_html_ = "pack_html_", Invk__pack_file_ = "pack_file_", Invk__pack_file_cutoff_ = "pack_file_cutoff_"
;
}

9
400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cmd.java
View File

@@ -18,18 +18,17 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Pack_file_bldr_cmd extends Xob_cmd__base {
private Io_url deploy_dir;
private final Pack_file_bldr_cfg cfg = new Pack_file_bldr_cfg();
public Pack_file_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
new Pack_file_mgr().Exec(wiki, deploy_dir);
new Pack_file_mgr().Exec(wiki, cfg);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__deploy_dir_)) deploy_dir = m.ReadIoUrl("v");
if (ctx.Match(k, Invk__cfg)) return cfg;
else return super.Invk (ctx, ikey, k, m);
return this;
}
private static final String Invk__deploy_dir_ = "deploy_dir_";
private static final String Invk__cfg = "cfg";
public static final String BLDR_CMD_KEY = "bldr.export.pack.file";
@Override public String Cmd_key() {return BLDR_CMD_KEY;}

56
400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java
View File

@@ -20,14 +20,14 @@ import gplx.core.progs.*; import gplx.core.ios.zips.*; import gplx.core.ios.stre
import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.fsdb.*; import gplx.fsdb.meta.*;
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
class Pack_file_mgr {
public void Exec(Xowe_wiki wiki, Io_url deploy_dir) {
public void Exec(Xowe_wiki wiki, Pack_file_bldr_cfg cfg) {
// init
wiki.Init_assert();
Io_url wiki_dir = wiki.Fsys_mgr().Root_dir();
Io_url pack_dir = wiki_dir.GenSubDir_nest("tmp", "pack");
Io_mgr.Instance.DeleteDirDeep(pack_dir); Io_mgr.Instance.CreateDirIfAbsent(pack_dir);
String wiki_date = wiki.Props().Modified_latest().XtoStr_fmt("yyyy.MM");
Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date);
Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date, cfg.Pack_html(), cfg.Pack_file(), cfg.Pack_file_cutoff());
// get import_tbl
byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo();
@@ -50,11 +50,14 @@ class Pack_file_mgr {
}
// build tasks
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "html", Xobc_import_type.Tid__wiki__core, Xobc_import_type.Tid__wiki__srch, Xobc_import_type.Tid__wiki__html);
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "file", Xobc_import_type.Tid__file__core, Xobc_import_type.Tid__file__data);
if (cfg.Pack_html())
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "html", Xobc_import_type.Tid__wiki__core, Xobc_import_type.Tid__wiki__srch, Xobc_import_type.Tid__wiki__html);
if (cfg.Pack_file())
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "file", Xobc_import_type.Tid__file__core, Xobc_import_type.Tid__file__data);
bc_conn.Txn_end();
// deploy
Io_url deploy_dir = cfg.Deploy_dir();
if (deploy_dir != null) {
Host_eval_itm host_eval = new Host_eval_itm();
int len = hash.Len();
@@ -160,30 +163,41 @@ class Pack_file_mgr {
}
}
class Pack_hash_bldr {
public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date) {
public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff) {
Pack_hash rv = new Pack_hash();
Pack_zip_name_bldr zip_name_bldr = new Pack_zip_name_bldr(pack_dir, wiki.Domain_str(), String_.new_a7(wiki.Domain_itm().Abrv_wm()), wiki_date);
Xow_db_mgr db_mgr = wiki.Data__core_mgr();
// bld html pack
Xow_db_mgr db_mgr = wiki.Data__core_mgr();
int len = db_mgr.Dbs__len();
for (int i = 0; i < len; ++i) {
Xow_db_file file = db_mgr.Dbs__get_at(i);
int pack_tid = Get_pack_tid(file.Tid());
if (pack_tid == Xobc_import_type.Tid__ignore) continue;
rv.Add(zip_name_bldr, pack_tid, file.Url());
if (pack_html) {
int len = db_mgr.Dbs__len();
for (int i = 0; i < len; ++i) {
Xow_db_file file = db_mgr.Dbs__get_at(i);
int pack_tid = Get_pack_tid(file.Tid());
if (pack_tid == Xobc_import_type.Tid__ignore) continue;
rv.Add(zip_name_bldr, pack_tid, file.Url());
}
rv.Consolidate(Xobc_import_type.Tid__wiki__srch);
}
rv.Consolidate(Xobc_import_type.Tid__wiki__srch);
// bld file pack
Fsm_mnt_itm mnt_itm = wiki.File__mnt_mgr().Mnts__get_at(Fsm_mnt_mgr.Mnt_idx_main);
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__core, wiki_dir.GenSubFil(mnt_itm.Atr_mgr().Db__core().Url_rel()));
if (db_mgr.Props().Layout_file().Tid_is_lot()) {
Fsm_bin_mgr bin_mgr = mnt_itm.Bin_mgr();
int bin_len = bin_mgr.Dbs__len();
for (int i = 0; i < bin_len; ++i) {
Fsm_bin_fil bin_fil = bin_mgr.Dbs__get_at(i);
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__data, bin_fil.Url());
if (pack_file) {
Fsm_mnt_itm mnt_itm = wiki.File__mnt_mgr().Mnts__get_at(Fsm_mnt_mgr.Mnt_idx_main);
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__core, wiki_dir.GenSubFil(mnt_itm.Atr_mgr().Db__core().Url_rel()));
if (db_mgr.Props().Layout_file().Tid_is_lot()) {
Fsm_bin_mgr bin_mgr = mnt_itm.Bin_mgr();
int bin_len = bin_mgr.Dbs__len();
for (int i = 0; i < bin_len; ++i) {
Fsm_bin_fil bin_fil = bin_mgr.Dbs__get_at(i);
Io_url bin_fil_url = bin_fil.Url();
// ignore if bin_fil is earlier than cutoff
if (pack_file_cutoff != null) {
DateAdp bin_fil_date = Io_mgr.Instance.QueryFil(bin_fil_url).ModifiedTime();
if (bin_fil_date.Timestamp_unix() < pack_file_cutoff.Timestamp_unix()) continue;
}
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__data, bin_fil_url);
}
}
}
return rv;

10
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__lnki_temp__create.java
View File

@@ -26,7 +26,7 @@ import gplx.fsdb.meta.*; import gplx.xowa.files.fsdb.*; import gplx.fsdb.*;
import gplx.xowa.langs.vnts.*; import gplx.xowa.parsers.vnts.*;
import gplx.xowa.parsers.lnkis.files.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.files.dbs.*;
import gplx.xowa.addons.bldrs.files.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx.xowa.parsers.lnkis.files.Xop_file_logger {
private Xob_lnki_temp_tbl tbl; private boolean wdata_enabled = true, xtn_ref_enabled = true, gen_html, gen_hdump;
private Xop_log_invoke_wkr invoke_wkr; private Xop_log_property_wkr property_wkr;
@@ -140,7 +140,7 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
Xof_ext ext = Xof_ext_.new_by_ttl_(ttl);
double lnki_time = lnki.Time();
int lnki_page = lnki.Page();
byte[] ttl_commons = Xto_commons(ns_file_is_case_match_all, commons_wiki, ttl);
byte[] ttl_commons = Xomp_file_logger.To_commons_ttl(ns_file_is_case_match_all, commons_wiki, ttl);
if ( Xof_lnki_page.Null_n(lnki_page) // page set
&& Xof_lnki_time.Null_n(lnki_time)) // thumbtime set
usr_dlg.Warn_many("", "", "page and thumbtime both set; this may be an issue with fsdb: page=~{0} ttl=~{1}", ctx.Page().Ttl().Page_db_as_str(), String_.new_u8(ttl));
@@ -177,11 +177,5 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
if (property_wkr == null) property_wkr = bldr.App().Wiki_mgr().Wdata_mgr().Property_wkr_or_new();
return property_wkr;
}
public static byte[] Xto_commons(boolean ns_file_is_case_match_all, Xowe_wiki commons_wiki, byte[] ttl_bry) {
if (!ns_file_is_case_match_all) return null; // return "" if wiki matches common
Xoa_ttl ttl = Xoa_ttl.Parse(commons_wiki, Xow_ns_.Tid__file, ttl_bry);
byte[] rv = ttl.Page_db();
return Bry_.Eq(rv, ttl_bry) ? null : rv;
}
public static boolean Ns_file_is_case_match_all(Xow_wiki wiki) {return wiki.Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Tid__all;}
}

2
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__lnki_temp__create__tst.java
View File

@@ -38,6 +38,6 @@ class Xobldr__lnki_temp__create__fxt {
return this;
}
public void Test__to_commons(String ttl, String expd) {
Tfds.Eq(expd, String_.new_u8(Xobldr__lnki_temp__create.Xto_commons(wiki_ns_file_is_case_match_all, commons_wiki, Bry_.new_u8(ttl))));
Tfds.Eq(expd, String_.new_u8(gplx.xowa.addons.bldrs.mass_parses.parses.Xomp_file_logger.To_commons_ttl(wiki_ns_file_is_case_match_all, commons_wiki, Bry_.new_u8(ttl))));
}
}

10
400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_lnki_temp_tbl.java
View File

@@ -17,9 +17,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*;
import gplx.dbs.*; import gplx.xowa.files.*;
public class Xob_lnki_temp_tbl {
public class Xob_lnki_temp_tbl implements Db_tbl {
private static final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private static final String Tbl_name = "lnki_temp";
private static final String tbl_name = "lnki_temp";
public static final String
Fld_lnki_id = flds.Add_int_pkey_autonum("lnki_id"); // NOTE: insertion order index; public b/c not used and want to bypass warning
private static final String
@@ -39,8 +39,9 @@ public class Xob_lnki_temp_tbl {
private Db_stmt stmt_insert;
public Xob_lnki_temp_tbl(Db_conn conn) {this.conn = conn;}
public Db_conn Conn() {return conn;} private final Db_conn conn;
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(Tbl_name, flds));}
public void Insert_bgn() {conn.Txn_bgn("bldr__lnki_temp"); stmt_insert = conn.Stmt_insert(Tbl_name, flds);}
public String Tbl_name() {return tbl_name;}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Insert_bgn() {conn.Txn_bgn("bldr__lnki_temp"); stmt_insert = conn.Stmt_insert(tbl_name, flds);}
public void Insert_commit() {conn.Txn_sav();}
public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
public void Insert_cmd_by_batch(int tier_id, int page_id, byte[] ttl, byte[] ttl_commons, byte ext_id, byte img_type, byte lnki_src_tid, int w, int h, double upright, double time, int page) {
@@ -59,4 +60,5 @@ public class Xob_lnki_temp_tbl {
.Val_int (Fld_lnki_page , page)
.Exec_insert();
}
public void Rls() {}
}

1
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_db_core.java
View File

@@ -25,6 +25,7 @@ public class Xomp_db_core {
Io_url mgr_url = root_dir.GenSubFil("xomp.sqlite3");
this.mgr_db = new Xomp_mgr_db(mgr_url);
}
public Db_conn Conn() {return mgr_db.Conn();}
public Xomp_mgr_db Mgr_db() {return mgr_db;} private Xomp_mgr_db mgr_db;
public Xomp_wkr_db Wkr_db(boolean delete, int idx) {
Io_url wkr_url = root_dir.GenSubFil_nest("xomp_" + Int_.To_str_fmt(idx, "000"), "xomp_wkr.sqlite3");

9
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_page_tbl.java
View File

@@ -23,7 +23,8 @@ public class Xomp_page_tbl implements Db_tbl {
public Xomp_page_tbl(Db_conn conn) {
this.conn = conn;
this.tbl_name = "xomp_page";
flds.Add_int_pkey("page_id");
flds.Add_int_pkey("xomp_uid");
flds.Add_int("page_id");
flds.Add_int("page_ns");
flds.Add_byte("page_status"); // 0=wait; 1=done; 2=fail
flds.Add_int_dflt("html_len", -1);
@@ -33,7 +34,11 @@ public class Xomp_page_tbl implements Db_tbl {
public String Tbl_name() {return tbl_name;} private final String tbl_name;
public Dbmeta_fld_list Flds() {return flds;} private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
public void Create_tbl() {
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
conn.Meta_tbl_create
( Dbmeta_tbl_itm.New(tbl_name, flds
, Dbmeta_idx_itm.new_normal_by_tbl("xomp_page", "xomp_uid__page_status" , "xomp_uid", "page_status")// for parse
, Dbmeta_idx_itm.new_normal_by_tbl("xomp_page", "page_ns__page_id" , "page_ns", "page_id") // for make
));
}
public void Rls() {}
}

57
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xob_lnki_temp_row.java Normal file
View File

@@ -0,0 +1,57 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
class Xob_lnki_temp_row implements CompareAble {
public int Xomp_uid() {return xomp_uid;} private int xomp_uid;
public int Make_uid() {return make_uid;} private int make_uid;
public int Lnki_tier_id() {return lnki_tier_id;} private int lnki_tier_id;
public int Lnki_page_id() {return lnki_page_id;} private int lnki_page_id;
public byte[] Lnki_ttl() {return lnki_ttl;} private byte[] lnki_ttl;
public byte[] Lnki_commons_ttl() {return lnki_commons_ttl;} private byte[] lnki_commons_ttl;
public byte Lnki_ext() {return lnki_ext;} private byte lnki_ext;
public byte Lnki_type() {return lnki_type;} private byte lnki_type;
public byte Lnki_src_tid() {return lnki_src_tid;} private byte lnki_src_tid;
public int Lnki_w() {return lnki_w;} private int lnki_w;
public int Lnki_h() {return lnki_h;} private int lnki_h;
public double Lnki_upright() {return lnki_upright;} private double lnki_upright;
public double Lnki_time() {return lnki_time;} private double lnki_time;
public int Lnki_page() {return lnki_page;} private int lnki_page;
public void Load(Db_rdr rdr, int xomp_uid) {
this.xomp_uid = xomp_uid;
this.make_uid = rdr.Read_int("lnki_make_uid");
this.lnki_tier_id = rdr.Read_int("lnki_tier_id");
this.lnki_page_id = rdr.Read_int("lnki_page_id");
this.lnki_ttl = rdr.Read_bry_by_str("lnki_ttl");
this.lnki_commons_ttl = rdr.Read_bry_by_str("lnki_commons_ttl");
this.lnki_ext = rdr.Read_byte("lnki_ext");
this.lnki_type = rdr.Read_byte("lnki_type");
this.lnki_src_tid = rdr.Read_byte("lnki_src_tid");
this.lnki_w = rdr.Read_int("lnki_w");
this.lnki_h = rdr.Read_int("lnki_h");
this.lnki_upright = rdr.Read_double("lnki_upright");
this.lnki_time = rdr.Read_int("lnki_time");
this.lnki_page = rdr.Read_int("lnki_page");
}
public int compareTo(Object obj) {
Xob_lnki_temp_row comp = (Xob_lnki_temp_row)obj;
int rv = Int_.Compare(xomp_uid, comp.xomp_uid);
if (rv != CompareAble_.Same) return rv;
return Int_.Compare(make_uid, comp.make_uid);
}
}

29
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_wtr.java
View File

@@ -33,24 +33,33 @@ class Xomp_html_db_wtr {
public int Cur_db_id() {return html_db.Id();}
public Xowd_html_tbl Tbls__get_or_new(int ns_id, long html_len) {
long len_new = len_cur + html_len;
if (html_tbl == null || len_new > len_max) {
Commit();
this.html_db = wiki.Data__core_mgr().Dbs__get_by_tid_or_null(Xow_db_file_.Tid__html_data);
if (html_db == null) {
html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data);
html_db.Conn().Txn_bgn("xomp.html_db_wtr");
this.html_tbl = new Xowd_html_tbl(html_db.Conn());
html_tbl.Create_tbl();
boolean not_inited = html_tbl == null, out_of_space = len_new > len_max;
if (not_inited || out_of_space) {
if (out_of_space)
Commit();
if ( db_mgr.Props().Layout_html().Tid_is_all_or_few() // is not "lot"
&& not_inited // not_inited; set html_db
) {
this.html_db = wiki.Data__core_mgr().Dbs__get_by_tid_or_null(Xow_db_file_.Tid__html_data);
if (html_db == null)
this.html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data);
}
else
this.html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data);
this.html_tbl = new Xowd_html_tbl(html_db.Conn());
html_tbl.Create_tbl();
html_db.Conn().Txn_bgn("xomp.html_db_wtr");
len_cur = html_len; // NOTE: this effectively resets len_new to 0 + html_len;
}
len_cur = len_new;
else // initied and still has space; just update len
len_cur = len_new;
return html_tbl;
}
public void Rls() {
this.Commit();
}
private void Commit() {
if (html_tbl == null) return;
html_tbl.Conn().Txn_end();
html_tbl.Conn().Rls_conn();

95
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java Normal file
View File

@@ -0,0 +1,95 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.brys.*;
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*; import gplx.xowa.addons.bldrs.files.dbs.*;
import gplx.xowa.bldrs.*;
class Xomp_make_lnki {
public void Exec(Xowe_wiki wiki, int uid_count) {
// init
Xomp_db_core xomp_db = Xomp_db_core.New__load(wiki);
Xob_db_file make_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
Xob_lnki_temp_tbl lnki_temp_tbl = new Xob_lnki_temp_tbl(make_db.Conn());
lnki_temp_tbl.Insert_bgn();
// add index
int wkr_count = xomp_db.Wkr_count();
for (int i = 0; i < wkr_count; ++i) {
Xomp_wkr_db wkr_db = xomp_db.Wkr_db(Bool_.N, i);
wkr_db.Conn().Meta_idx_assert("lnki_temp", "lnki_page_id", "lnki_page_id");
}
// loop
List_adp rows = List_adp_.New();
int uid_bgn = -1;
while (true) {
int count = 0;
for (int i = 0; i < wkr_count; ++i) {
Xomp_wkr_db wkr_db = xomp_db.Wkr_db(Bool_.N, i);
count += Load_rows(rows, xomp_db, wkr_db, uid_bgn, uid_bgn + uid_count);
}
if (count == 0) break;
Save_rows(rows, lnki_temp_tbl);
}
// term
lnki_temp_tbl.Insert_end();
xomp_db.Conn().Rls_conn();
make_db.Conn().Rls_conn();
}
private int Load_rows(List_adp rows, Xomp_db_core xomp_db, Xomp_wkr_db wkr_db, int uid_bgn, int uid_end) {
// build sql
Db_attach_mgr attach_mgr = new Db_attach_mgr(xomp_db.Conn());
attach_mgr.Conn_others_(new Db_attach_itm("wkr_db", wkr_db.Conn()));
String sql = String_.Format(String_.Concat_lines_nl_skip_last
( "SELECT mgr.xomp_uid"
, ", wkr.*"
, "FROM <wkr_db>lnki_temp wkr"
, " JOIN xomp_page mgr ON wkr.lnki_page_id = mgr.page_id"
, "WHERE mgr.xomp_uid > {0} AND mgr.xomp_uid <= {1}"
)
, uid_bgn
, uid_end
);
sql = attach_mgr.Resolve_sql(sql);
attach_mgr.Attach();
Db_rdr rdr = xomp_db.Conn().Stmt_sql(sql).Exec_select__rls_auto(); // ANSI.Y
int rv = -1;
try {
while (rdr.Move_next()) {
rv = rdr.Read_int("xomp_uid");
Xob_lnki_temp_row row = new Xob_lnki_temp_row();
row.Load(rdr, rv);
}
} finally {rdr.Rls();}
attach_mgr.Detach();
return rv;
}
private void Save_rows(List_adp rows, Xob_lnki_temp_tbl lnki_temp_tbl) {
int len = rows.Len();
for (int i = 0; i < len; ++i) {
Xob_lnki_temp_row row = (Xob_lnki_temp_row)rows.Get_at(i);
lnki_temp_tbl.Insert_cmd_by_batch(row.Lnki_tier_id(), row.Lnki_page_id(), row.Lnki_ttl(), row.Lnki_commons_ttl()
, row.Lnki_ext(), row.Lnki_src_tid(), row.Lnki_src_tid(), row.Lnki_w(), row.Lnki_h(), row.Lnki_upright()
, row.Lnki_time(), row.Lnki_page());
}
}
}

58
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_file_logger.java Normal file
View File

@@ -0,0 +1,58 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.lnkis.*; import gplx.xowa.parsers.lnkis.files.*;
import gplx.xowa.files.*; import gplx.xowa.addons.bldrs.files.cmds.*; import gplx.xowa.addons.bldrs.files.dbs.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.domains.*;
public class Xomp_file_logger implements Xop_file_logger {
private final Xob_lnki_temp_tbl tbl;
private final Xowe_wiki commons_wiki;
private boolean ns_file_is_case_match_all = true;
public Xomp_file_logger(Xowe_wiki wiki, Db_conn wkr_conn) {
this.tbl = new Xob_lnki_temp_tbl(wkr_conn); wkr_conn.Meta_tbl_assert(tbl);
this.commons_wiki = wiki.Appe().Wiki_mgr().Get_by_or_make(Xow_domain_itm_.Bry__commons);
this.ns_file_is_case_match_all = wiki.Init_assert().Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Tid__all; // NOTE: wiki must be init'd;
}
public void Bgn() {
tbl.Insert_bgn();
}
public void Log_file(Xop_ctx ctx, Xop_lnki_tkn lnki, byte caller_tid) {
if (lnki.Ttl().ForceLiteralLink()) return; // ignore literal links which create a link to file, but do not show the image; EX: [[:File:A.png|thumb|120px]] creates a link to File:A.png, regardless of other display-oriented args
// get caller_tid / tttl
if (lnki.Ns_id() == Xow_ns_.Tid__media) caller_tid = Xop_file_logger_.Tid__media;
// get lnki_data
byte[] ttl = lnki.Ttl().Page_db();
Xof_ext ext = Xof_ext_.new_by_ttl_(ttl);
byte[] ttl_commons = Xomp_file_logger.To_commons_ttl(ns_file_is_case_match_all, commons_wiki, ttl);
// do insert
tbl.Insert_cmd_by_batch(ctx.Page().Bldr__ns_ord(), ctx.Page().Db().Page().Id(), ttl, ttl_commons, Byte_.By_int(ext.Id()), lnki.Lnki_type(), caller_tid, lnki.W(), lnki.H(), lnki.Upright(), lnki.Time(), lnki.Page());
}
public void End() {
tbl.Insert_end();
}
public static byte[] To_commons_ttl(boolean ns_file_is_case_match_all, Xowe_wiki commons_wiki, byte[] ttl_bry) { // handle case-sensitive wikis (en.d) vs case-insensitive commons
if (!ns_file_is_case_match_all) return null; // return "" if wiki matches common
Xoa_ttl ttl = Xoa_ttl.Parse(commons_wiki, Xow_ns_.Tid__file, ttl_bry);
byte[] rv = ttl.Page_db();
return Bry_.Eq(rv, ttl_bry) ? null : rv;
}
}

13
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_page_pool_loader.java
View File

@@ -21,7 +21,7 @@ class Xomp_page_pool_loader {
private final Xow_wiki wiki;
private final int num_pages_per_load;
private final Db_attach_mgr attach_mgr;
private int prv_page_id = -1;
private int prv_uid = -1;
public Xomp_page_pool_loader(Xow_wiki wiki, Db_conn make_conn, int num_pages_per_load) {
this.wiki = wiki;
this.make_conn = make_conn;
@@ -49,16 +49,17 @@ class Xomp_page_pool_loader {
private void Load_from_db(List_adp list) {
// prepare for page_tbl
String sql = String_.Format(String_.Concat_lines_nl_skip_last // ANSI.Y
( "SELECT mp.page_id"
( "SELECT mp.xomp_uid"
, ", pp.page_id"
, ", pp.page_namespace"
, ", pp.page_title"
, ", pp.page_text_db_id"
, "FROM xomp_page mp"
, " JOIN <page_db>page pp ON mp.page_id = pp.page_id"
, "WHERE mp.page_id > {0}"
, "WHERE mp.xomp_uid > {0}"
, "AND mp.page_status = 0"
, "LIMIT {1}"
), prv_page_id, num_pages_per_load);
), prv_uid, num_pages_per_load);
this.attach_mgr.Conn_others_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
sql = attach_mgr.Resolve_sql(sql);
@@ -68,9 +69,9 @@ class Xomp_page_pool_loader {
Db_rdr rdr = make_conn.Stmt_sql(sql).Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
prv_page_id = rdr.Read_int("page_id");
prv_uid = rdr.Read_int("xomp_uid");
int text_db_id = rdr.Read_int("page_text_db_id");
Xomp_page_itm ppg = new Xomp_page_itm(prv_page_id);
Xomp_page_itm ppg = new Xomp_page_itm(rdr.Read_int("page_id"));
ppg.Init_by_page
( rdr.Read_int("page_namespace")
, rdr.Read_bry_by_str("page_title")

17
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_mgr.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.threads.*; import gplx.core.threads.utils.*;
import gplx.core.threads.*; import gplx.core.threads.utils.*; import gplx.core.caches.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.cases.*;
import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.wikis.caches.*;
@@ -32,9 +32,11 @@ class Xomp_parse_mgr {
this.db_core = Xomp_db_core.New__load(wiki);
Xomp_page_pool_loader pool_loader = new Xomp_page_pool_loader(wiki, db_core.Mgr_db().Conn(), cfg.Num_pages_in_pool());
Xomp_page_pool page_pool = new Xomp_page_pool(pool_loader, cfg.Num_pages_per_wkr());
prog_mgr.Init(pool_loader.Get_pending_count());
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, true);
prog_mgr.Init(pool_loader.Get_pending_count(), cfg.Progress_interval());
wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of cache
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, true);
Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
Gfo_cache_mgr ifexist_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
// load_wkr: init and start
// Xomp_load_wkr load_wkr = new Xomp_load_wkr(wiki, db_core.Mgr_db().Conn(), cfg.Num_pages_in_pool(), cfg.Num_wkrs());
@@ -45,8 +47,13 @@ class Xomp_parse_mgr {
latch = new Gfo_countdown_latch(wkr_len);
Xomp_parse_wkr[] wkrs = new Xomp_parse_wkr[wkr_len];
for (int i = 0; i < wkr_len; ++i) {
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, Clone_wiki(wiki), page_pool, i);
wkr.Wiki().Cache_mgr().Page_cache_(page_cache);
Xowe_wiki wkr_wiki = Clone_wiki(wiki);
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, wkr_wiki, page_pool, i, cfg.Cleanup_interval(), cfg.Progress_interval(), cfg.Log_file_lnkis());
wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);
// remove wmf wkr, else will try to download images during parsing
if (wkr_wiki.File__bin_mgr() != null)
wkr_wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf);
wkr.Hdump_bldr().Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled());
wkrs[i] = wkr;
}

24
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_mgr_cfg.java
View File

@@ -17,12 +17,17 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
class Xomp_parse_mgr_cfg implements Gfo_invk {
// public Io_url Root_dir() {return root_dir;} private Io_url root_dir;
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1;
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1; // use env.available_processors
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = 1000;
public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000;
public int Progress_interval() {return progress_interval;} private int progress_interval = 1000;
public int Commit_interval() {return commit_interval;} private int commit_interval = 10000;
public int Cleanup_interval() {return cleanup_interval;} private int cleanup_interval = 50; // setting at 1000 uses lots of memory
public boolean Hdump_enabled() {return hdump_enabled;} private boolean hdump_enabled = true;
public boolean Hzip_enabled() {return hzip_enabled;} private boolean hzip_enabled = true;
public boolean Hdiff_enabled() {return hdiff_enabled;} private boolean hdiff_enabled = true;
public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true;
public void Init(Xowe_wiki wiki) {
// if (root_dir == null) root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Env_.System_cpu_count();
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
@@ -30,11 +35,18 @@ class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
// else if (ctx.Match(k, Invk__root_dir_)) root_dir = m.ReadIoUrl("v");
else if (ctx.Match(k, Invk__progress_interval_)) progress_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__commit_interval_)) commit_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__cleanup_interval_)) cleanup_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__hdump_enabled_)) hdump_enabled = m.ReadBool("v");
else if (ctx.Match(k, Invk__hzip_enabled_)) hzip_enabled = m.ReadBool("v");
else if (ctx.Match(k, Invk__hdiff_enabled_)) hdiff_enabled = m.ReadBool("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}
private static final String Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
// , Invk__root_dir_ = "root_dir_"
private static final String
Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
, Invk__progress_interval_ = "progress_interval_", Invk__commit_interval_ = "commit_interval_", Invk__cleanup_interval_ = "cleanup_interval_"
, Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_"
;
}

43
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_wkr.java
View File

@@ -23,23 +23,42 @@ class Xomp_parse_wkr implements Gfo_invk {
private final Xomp_parse_mgr mgr;
private final Xomp_page_pool page_pool;
private final int idx;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private int done_count; private long done_time;
private Xomp_wkr_db wkr_db;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xomp_page_pool page_pool, int idx) {
private Xomp_wkr_db wkr_db; private int cleanup_interval, commit_interval;
private boolean log_file_lnkis;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xomp_page_pool page_pool, int idx, int cleanup_interval, int commit_interval, boolean log_file_lnkis) {
this.mgr = mgr; this.wiki = wiki;
this.page_pool = page_pool;
this.idx = idx;
this.wkr_db = mgr.Db_core().Wkr_db(Bool_.Y, idx); // NOTE: must go in ctor, or else thread issues
this.cleanup_interval = cleanup_interval;
this.commit_interval = commit_interval;
this.log_file_lnkis = log_file_lnkis;
}
public Xowe_wiki Wiki() {return wiki;} private final Xowe_wiki wiki;
public Xob_hdump_bldr Hdump_bldr() {return hdump_bldr;} private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
public void Exec() {
// init
Xow_parser_mgr parser_mgr = new Xow_parser_mgr(wiki);
wiki.Html_mgr().Page_wtr_mgr().Wkr(gplx.xowa.wikis.pages.Xopg_page_.Tid_read).Ctgs_enabled_(false); // disable categories else progress messages written (also for PERF)
if (wiki.File__bin_mgr() != null)
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // remove wmf wkr, else will try to download images during parsing
// disable file download
wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
wiki.File__orig_mgr().Wkrs_del(gplx.xowa.files.origs.Xof_orig_wkr_.Tid_wmf_api);
// disable categories else progress messages written (also for PERF)
wiki.Html_mgr().Page_wtr_mgr().Wkr(gplx.xowa.wikis.pages.Xopg_page_.Tid_read).Ctgs_enabled_(false);
// enable lnki_temp
Xomp_file_logger logger = null;
if (log_file_lnkis) {
logger = new Xomp_file_logger(wiki, wkr_db.Conn());
parser_mgr.Ctx().Lnki().File_logger_(logger);
logger.Bgn();
}
// enable hdump
hdump_bldr.Enabled_(true).Hzip_enabled_(true).Hzip_diff_(true).Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
wkr_db.Conn().Txn_bgn("xomp");
@@ -76,14 +95,22 @@ class Xomp_parse_wkr implements Gfo_invk {
// ctx.App().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released
if (wiki.Cache_mgr().Tmpl_result_cache().Count() > 50000)
wiki.Cache_mgr().Tmpl_result_cache().Clear();
if (done_count % 50 == 0) {
if (done_count % cleanup_interval == 0) {
wiki.Cache_mgr().Free_mem_all(Bool_.N);
wiki.Parser_mgr().Scrib().Core_term();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
}
if (done_count % commit_interval == 0)
wkr_db.Conn().Txn_sav();
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e));
}
}
if (logger != null)
logger.End();
// cleanup
wkr_db.Conn().Txn_end(); // NOTE: must end txn before running update wkr_id
mgr.Db_core().Update_wkr_id(idx, wkr_db.Conn());
mgr.Wkrs_done_add_1();

6
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_prog_mgr.java
View File

@@ -18,17 +18,19 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
class Xomp_prog_mgr {
private final Object thread_lock = new Object();
private int progress_interval;
private int pages_done, pages_total;
private long time_bgn, time_prv, time_done;
private final Bry_bfr prog_bfr = Bry_bfr_.New();
public void Init(int pages_total) {
public void Init(int pages_total, int progress_interval) {
this.progress_interval = progress_interval;
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
this.pages_total = pages_total;
}
public void Mark_done(int id) {
synchronized (thread_lock) {
pages_done += 1;
if (pages_done % 1000 == 0) {
if (pages_done % progress_interval == 0) {
long time_cur = gplx.core.envs.Env_.TickCount();
int pages_left = pages_total - pages_done;
time_done += (time_cur - time_prv);

46
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_addon.java Normal file
View File

@@ -0,0 +1,46 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.specials.*;
import gplx.xowa.addons.wikis.imports.*;
public class Xobc_xodir_addon implements Xoax_addon_itm, Xoax_addon_itm__special, Xoax_addon_itm__init {
public Xow_special_page[] Special_pages() {
return new Xow_special_page[]
{ Xobc_xodir_special.Prototype
};
}
public void Init_addon_by_app(Xoa_app app) {
}
public void Init_addon_by_wiki(Xow_wiki wiki) {
Xow_import_addon addon = Xow_import_addon.Addon__get(wiki);
addon.Dir_selected_cbks__add(Xow_import_dir_cbk__xodir.Instance);
}
public String Addon__key() {return ADDON__KEY;} private static final String ADDON__KEY = "xowa.bldrs.xodirs";
}
class Xow_import_dir_cbk__xodir implements Xow_import_dir_cbk {
public String Key() {return "xodir";}
public void Cbk__dir_selected(Xow_wiki wiki, Xoa_page page, String path) {
// save to prefs
wiki.App().User().User_db_mgr().Cfg().Set_app_str("xowa.xodir.custom_dir", path);
// redirect to import_dir
page.Redirect().Itms__add__special(wiki, Xobc_xodir_special.Prototype.Special__meta());
}
public static Xow_import_dir_cbk__xodir Instance = new Xow_import_dir_cbk__xodir(); Xow_import_dir_cbk__xodir() {}
}

30
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_cfg.java Normal file
View File

@@ -0,0 +1,30 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
public class Xobc_xodir_cfg {
public static final String
Key__selected_dir = "xowa.xodir.selected_dir"
, Key__custom_dir = "xowa.xodir.custom_dir"
;
public static void Set_app_str__selected(Xoa_app app, byte[] val_bry) {
// if wnt, replace "\"; note that url-encoding while navigating dirs will always convert "\" to "/"
if (gplx.core.envs.Op_sys.Cur().Tid_is_wnt()) val_bry = Bry_.Replace(val_bry, Byte_ascii.Slash, Byte_ascii.Backslash);
app.User().User_db_mgr().Cfg().Set_app_bry(Xobc_xodir_cfg.Key__selected_dir, val_bry);
}
}

51
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_dir.java Normal file
View File

@@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.langs.mustaches.*;
class Xobc_xodir_doc implements Mustache_doc_itm {
private final byte[] import_root, app_root_dir;
private final Xobc_xodir_dir[] dirs;
public Xobc_xodir_doc(Xobc_xodir_dir[] dirs, byte[] import_root, byte[] app_root_dir) {
this.dirs = dirs; this.import_root = import_root; this.app_root_dir = app_root_dir;
}
public boolean Mustache__write(String key, Mustache_bfr bfr) {
if (String_.Eq(key, "import_root")) bfr.Add_bry(import_root);
else if (String_.Eq(key, "app_root_dir")) bfr.Add_bry(app_root_dir);
return false;
}
public Mustache_doc_itm[] Mustache__subs(String key) {
if (String_.Eq(key, "dirs")) return dirs;
return Mustache_doc_itm_.Ary__empty;
}
}
public class Xobc_xodir_dir implements Mustache_doc_itm {
private final boolean is_selected, is_custom;
private final byte[] path;
public Xobc_xodir_dir(boolean is_selected, boolean is_custom, byte[] path) {
this.is_selected = is_selected; this.is_custom = is_custom; this.path = path;
}
public boolean Mustache__write(String key, Mustache_bfr bfr) {
if (String_.Eq(key, "path")) bfr.Add_bry(path);
return false;
}
public Mustache_doc_itm[] Mustache__subs(String key) {
if (String_.Eq(key, "is_selected")) return Mustache_doc_itm_.Ary__bool(is_selected);
else if (String_.Eq(key, "is_custom")) return Mustache_doc_itm_.Ary__bool(is_custom);
return Mustache_doc_itm_.Ary__empty;
}
}

40
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_html.java Normal file
View File

@@ -0,0 +1,40 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.specials.*; import gplx.langs.mustaches.*; import gplx.xowa.wikis.pages.*; import gplx.xowa.wikis.pages.tags.*;
import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
import gplx.xowa.wikis.domains.*; import gplx.core.ios.*;
class Xobc_xodir_html extends Xow_special_wtr__base {
@Override protected Io_url Get_addon_dir(Xoa_app app) {return app.Fsys_mgr().Http_root().GenSubDir_nest("bin", "any", "xowa", "addon", "bldr", "xodir");}
@Override protected Io_url Get_mustache_fil(Io_url addon_dir) {return addon_dir.GenSubFil_nest("bin", "xobc_xodir.mustache.html");}
@Override protected Mustache_doc_itm Bld_mustache_root(Xoa_app app) {
Xobc_task_addon addon = (Xobc_task_addon)app.Addon_mgr().Itms__get_or_null(Xobc_task_addon.ADDON__KEY);
return new Xobc_xodir_doc
( addon.Xodir_mgr().Get_dirs(app)
, gplx.xowa.addons.wikis.imports.Xow_import_special.Get_root_url()
, app.Fsys_mgr().Root_dir().RawBry()
);
}
@Override protected void Bld_tags(Xoa_app app, Io_url addon_dir, Xopage_html_data page_data) {
Xopg_tag_mgr head_tags = page_data.Head_tags();
Xopg_alertify_.Add_tags (head_tags, app.Fsys_mgr().Http_root());
Xopg_tag_wtr_.Add__xocss (head_tags, app.Fsys_mgr().Http_root());
Xopg_tag_wtr_.Add__xohelp (head_tags, app.Fsys_mgr().Http_root());
head_tags.Add(Xopg_tag_itm.New_css_file(addon_dir.GenSubFil_nest("bin", "xobc_xodir.css")));
}
}

21
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_mgr.java Normal file
View File

@@ -0,0 +1,21 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
public interface Xobc_xodir_mgr {
Xobc_xodir_dir[] Get_dirs(Xoa_app app);
}

32
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_mgr__pc.java Normal file
View File

@@ -0,0 +1,32 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
public class Xobc_xodir_mgr__pc implements Xobc_xodir_mgr {
public Xobc_xodir_mgr__pc(Xoa_app app) {
}
public Xobc_xodir_dir[] Get_dirs(Xoa_app app) {
int len = 2;
String dflt = app.Fsys_mgr().Root_dir().Raw();
String selected = app.User().User_db_mgr().Cfg().Get_app_str_or(Xobc_xodir_cfg.Key__selected_dir, dflt);
String custom = app.User().User_db_mgr().Cfg().Get_app_str_or(Xobc_xodir_cfg.Key__custom_dir, "(choose your own folder)");
Xobc_xodir_dir[] rv = new Xobc_xodir_dir[len];
rv[0] = new Xobc_xodir_dir(String_.Eq(selected, dflt), Bool_.N, Bry_.new_u8(dflt));
rv[1] = new Xobc_xodir_dir(String_.Eq(selected, custom), Bool_.Y, Bry_.new_u8(custom));
return rv;
}
}

45
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_special.java Normal file
View File

@@ -0,0 +1,45 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.specials.*; import gplx.core.net.*; import gplx.core.net.qargs.*; import gplx.xowa.wikis.pages.*;
import gplx.xowa.addons.bldrs.centrals.*;
import gplx.xowa.addons.wikis.imports.*;
public class Xobc_xodir_special implements Xow_special_page {
public void Special__gen(Xow_wiki wiki, Xoa_page page, Xoa_url url, Xoa_ttl ttl) {
Gfo_qarg_mgr url_args = new Gfo_qarg_mgr().Init(url.Qargs_ary());
byte[] path = url_args.Read_bry_or(Bry__path, null);
if (path != null) { // path selected; set cfg and redirect to Download Central
Xobc_xodir_cfg.Set_app_str__selected(wiki.App(), path);
// On_path_selected.Invk(null, -1, "", null);
// page.Redirect().Itms__add__special(wiki, Xobc_task_special.Prototype.Special__meta());
page.Redirect().Itms__add__special(wiki, Prototype.Special__meta());
return;
}
new Xobc_xodir_html().Bld_page_by_mustache(wiki.App(), page, this);
}
private static final byte[] Bry__path = Bry_.new_a7("path");
Xobc_xodir_special(Xow_special_meta special__meta) {this.special__meta = special__meta;}
public Xow_special_meta Special__meta() {return special__meta;} private final Xow_special_meta special__meta;
public Xow_special_page Special__clone() {return this;}
public static final Xow_special_page Prototype = new Xobc_xodir_special(Xow_special_meta.New_xo("XowaRootDir", "XOWA Folder Selection"));
public static Gfo_invk On_path_selected = Gfo_invk_.Noop;
}

19
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xow_import_dir_cbk__xodir.java Normal file
View File

@@ -0,0 +1,19 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.addons.wikis.imports.*;