Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

'v3.7.4.1'

Browse Source
This commit is contained in:
gnosygnu
2016-07-25 21:59:51 -04:00
parent 7a851a41a5
commit 8e91ac0bc4
175 changed files with 2079 additions and 933 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
400_xowa/src/gplx/xowa/addons/Xoax_addon_itm__init.java Normal file
View File

@@ -0,0 +1,23 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons; import gplx.*; import gplx.xowa.*;
import gplx.xowa.bldrs.wkrs.*;
public interface Xoax_addon_itm__init {
void Init_addon_by_app(Xoa_app app);
void Init_addon_by_wiki(Xow_wiki wiki);
}

20
400_xowa/src/gplx/xowa/addons/Xoax_addon_mgr.java
View File

@@ -17,8 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons; import gplx.*; import gplx.xowa.*;
public class Xoax_addon_mgr {
private final Ordered_hash hash = Ordered_hash_.New();
// THREAD: must synchronized else two search tabs will fail on startup
private final Ordered_hash hash = Ordered_hash_.New(); // LOCK: must synchronized else two search tabs will fail on startup
public Xoax_addon_itm Itms__get_or_null(String key) {synchronized (hash) {return (Xoax_addon_itm)hash.Get_by(key);}}
public void Itms__add_many(Xoax_addon_itm... ary) {
for (Xoax_addon_itm itm : ary)
@@ -49,6 +48,7 @@ public class Xoax_addon_mgr {
// specials
, new gplx.xowa.addons.wikis.registrys .Wiki_registry_addon()
, new gplx.xowa.addons.wikis.imports .Xow_import_addon()
, new gplx.xowa.addons.bldrs.xodirs .Xobc_xodir_addon()
, new gplx.xowa.addons.bldrs.centrals .Xobc_task_addon()
, new gplx.xowa.addons.apps.helps.logs .Xolog_addon()
@@ -63,6 +63,14 @@ public class Xoax_addon_mgr {
int len = hash.Len();
for (int i = 0; i < len; ++i) {
Xoax_addon_itm addon = (Xoax_addon_itm)hash.Get_at(i);
// init
if (Type_adp_.Implements_intf_obj(addon, Xoax_addon_itm__init.class)) {
Xoax_addon_itm__init addon_init = (Xoax_addon_itm__init)addon;
addon_init.Init_addon_by_app(app);
init_list.Add(addon_init);
}
// add bldr cmds
if (Type_adp_.Implements_intf_obj(addon, Xoax_addon_itm__bldr.class)) {
Xoax_addon_itm__bldr addon_bldr = (Xoax_addon_itm__bldr)addon;
@@ -86,4 +94,12 @@ public class Xoax_addon_mgr {
}
}
}
public void Load_by_wiki(Xow_wiki wiki) {
int len = init_list.Len();
for (int i = 0; i < len; ++i) {
Xoax_addon_itm__init itm = (Xoax_addon_itm__init)init_list.Get_at(i);
itm.Init_addon_by_wiki(wiki);
}
}
private final List_adp init_list = List_adp_.New();
}

16
400_xowa/src/gplx/xowa/addons/apps/helps/logs/Xolog_special.java
View File

@@ -42,11 +42,11 @@ public class Xolog_special implements Xow_special_page {
}
if (redirect) {
String redirect_ttl_str = special__meta.Ttl_str() + "?cmd=view";
Xoa_url_args_bldr args_bldr = new Xoa_url_args_bldr();
args_bldr.Add("cmd", "view");
if (redirect_to_same_file && file != null)
redirect_ttl_str += "&file=" + file;
Xoa_ttl redirect_ttl = wiki.Ttl_parse(Bry_.new_u8(redirect_ttl_str));
page.Redirect().Itms__add__special(Xoa_url.New(wiki, redirect_ttl), redirect_ttl);
args_bldr.Add("file", file);
page.Redirect().Itms__add__special(wiki, Prototype.Special__meta(), args_bldr.To_ary());
return;
}
else
@@ -66,6 +66,14 @@ public class Xolog_special implements Xow_special_page {
public Xow_special_page Special__clone() {return this;}
public static final Xow_special_page Prototype = new Xolog_special(Xow_special_meta.New_xo("XowaLog", "Logs"));
}
class Xoa_url_args_bldr {
private final List_adp list = List_adp_.New();
public Xoa_url_args_bldr Add(String key, Object val) {
list.Add(Keyval_.new_(key, val));
return this;
}
public Keyval[] To_ary() {return (Keyval[])list.To_ary_and_clear(Keyval.class);}
}
class Xolog_file_utl {// yyyyMMdd_HHmmss.log
private static final String Gui__date_fmt = "yyyy-MM-dd HH:mm:ss";
public static String To_name(Io_url url) {return To_name(url.NameOnly());}

12
400_xowa/src/gplx/xowa/addons/bldrs/centrals/Xobc_task_addon.java
View File

@@ -17,8 +17,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.centrals; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.specials.*; import gplx.xowa.htmls.bridges.*;
import gplx.xowa.addons.bldrs.infos.*;
public class Xobc_task_addon implements Xoax_addon_itm, Xoax_addon_itm__special, Xoax_addon_itm__json {
import gplx.xowa.addons.bldrs.infos.*; import gplx.xowa.addons.bldrs.xodirs.*;
public class Xobc_task_addon implements Xoax_addon_itm, Xoax_addon_itm__special, Xoax_addon_itm__json, Xoax_addon_itm__init {
public Xobc_xodir_mgr Xodir_mgr() {return xodir_mgr;} private Xobc_xodir_mgr xodir_mgr;
public void Xodir_mgr_(Xobc_xodir_mgr v) {this.xodir_mgr = v;}
public void Init_addon_by_app(Xoa_app app) {
this.xodir_mgr = new Xobc_xodir_mgr__pc(app);
}
public void Init_addon_by_wiki(Xow_wiki wiki) {}
public Xow_special_page[] Special_pages() {
return new Xow_special_page[]
{ Xobc_task_special.Prototype
@@ -31,5 +37,5 @@ public class Xobc_task_addon implements Xoax_addon_itm, Xoax_addon_itm__special,
};
}
public String Addon__key() {return "xowa.imports.downloads";}
public String Addon__key() {return ADDON__KEY;} public static final String ADDON__KEY = "xowa.imports.downloads";
}

1
400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__unzip.java
View File

@@ -29,6 +29,7 @@ public class Xobc_cmd__unzip extends Xobc_cmd__base {
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.core.ios.zips.zip_unzip";
@Override public String Cmd_name() {return "unzip";}
@Override public boolean Cmd_suspendable() {return true;}
@Override public String Cmd_fallback() {return Xobc_cmd__verify_fil.CMD_TYPE;} // if unzip fails, backtrack to verify; if verify fails, it'll backtrack to download; DATE:2016-07-25
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
if (wkr.Exec(this, src_fil, trg_dir, trg_fils) == Gfo_prog_ui_.Status__fail)

37
400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cfg.java Normal file
View File

@@ -0,0 +1,37 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Pack_file_bldr_cfg implements Gfo_invk {
public Io_url Deploy_dir() {return deploy_dir;} private Io_url deploy_dir;
public boolean Pack_html() {return pack_html;} private boolean pack_html = true;
public boolean Pack_file() {return pack_file;} private boolean pack_file = true;
public DateAdp Pack_file_cutoff() {return pack_file_cutoff;} private DateAdp pack_file_cutoff = null;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__deploy_dir_)) deploy_dir = m.ReadIoUrl("v");
else if (ctx.Match(k, Invk__pack_html_)) pack_html = m.ReadYn("v");
else if (ctx.Match(k, Invk__pack_file_)) pack_file = m.ReadYn("v");
else if (ctx.Match(k, Invk__pack_file_cutoff_)) pack_file_cutoff = m.ReadDate("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}
private static final String Invk__deploy_dir_ = "deploy_dir_"
, Invk__pack_html_ = "pack_html_", Invk__pack_file_ = "pack_file_", Invk__pack_file_cutoff_ = "pack_file_cutoff_"
;
}

9
400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cmd.java
View File

@@ -18,18 +18,17 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Pack_file_bldr_cmd extends Xob_cmd__base {
private Io_url deploy_dir;
private final Pack_file_bldr_cfg cfg = new Pack_file_bldr_cfg();
public Pack_file_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
new Pack_file_mgr().Exec(wiki, deploy_dir);
new Pack_file_mgr().Exec(wiki, cfg);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__deploy_dir_)) deploy_dir = m.ReadIoUrl("v");
if (ctx.Match(k, Invk__cfg)) return cfg;
else return super.Invk (ctx, ikey, k, m);
return this;
}
private static final String Invk__deploy_dir_ = "deploy_dir_";
private static final String Invk__cfg = "cfg";
public static final String BLDR_CMD_KEY = "bldr.export.pack.file";
@Override public String Cmd_key() {return BLDR_CMD_KEY;}

56
400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java
View File

@@ -20,14 +20,14 @@ import gplx.core.progs.*; import gplx.core.ios.zips.*; import gplx.core.ios.stre
import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.fsdb.*; import gplx.fsdb.meta.*;
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
class Pack_file_mgr {
public void Exec(Xowe_wiki wiki, Io_url deploy_dir) {
public void Exec(Xowe_wiki wiki, Pack_file_bldr_cfg cfg) {
// init
wiki.Init_assert();
Io_url wiki_dir = wiki.Fsys_mgr().Root_dir();
Io_url pack_dir = wiki_dir.GenSubDir_nest("tmp", "pack");
Io_mgr.Instance.DeleteDirDeep(pack_dir); Io_mgr.Instance.CreateDirIfAbsent(pack_dir);
String wiki_date = wiki.Props().Modified_latest().XtoStr_fmt("yyyy.MM");
Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date);
Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date, cfg.Pack_html(), cfg.Pack_file(), cfg.Pack_file_cutoff());
// get import_tbl
byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo();
@@ -50,11 +50,14 @@ class Pack_file_mgr {
}
// build tasks
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "html", Xobc_import_type.Tid__wiki__core, Xobc_import_type.Tid__wiki__srch, Xobc_import_type.Tid__wiki__html);
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "file", Xobc_import_type.Tid__file__core, Xobc_import_type.Tid__file__data);
if (cfg.Pack_html())
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "html", Xobc_import_type.Tid__wiki__core, Xobc_import_type.Tid__wiki__srch, Xobc_import_type.Tid__wiki__html);
if (cfg.Pack_file())
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "file", Xobc_import_type.Tid__file__core, Xobc_import_type.Tid__file__data);
bc_conn.Txn_end();
// deploy
Io_url deploy_dir = cfg.Deploy_dir();
if (deploy_dir != null) {
Host_eval_itm host_eval = new Host_eval_itm();
int len = hash.Len();
@@ -160,30 +163,41 @@ class Pack_file_mgr {
}
}
class Pack_hash_bldr {
public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date) {
public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff) {
Pack_hash rv = new Pack_hash();
Pack_zip_name_bldr zip_name_bldr = new Pack_zip_name_bldr(pack_dir, wiki.Domain_str(), String_.new_a7(wiki.Domain_itm().Abrv_wm()), wiki_date);
Xow_db_mgr db_mgr = wiki.Data__core_mgr();
// bld html pack
Xow_db_mgr db_mgr = wiki.Data__core_mgr();
int len = db_mgr.Dbs__len();
for (int i = 0; i < len; ++i) {
Xow_db_file file = db_mgr.Dbs__get_at(i);
int pack_tid = Get_pack_tid(file.Tid());
if (pack_tid == Xobc_import_type.Tid__ignore) continue;
rv.Add(zip_name_bldr, pack_tid, file.Url());
if (pack_html) {
int len = db_mgr.Dbs__len();
for (int i = 0; i < len; ++i) {
Xow_db_file file = db_mgr.Dbs__get_at(i);
int pack_tid = Get_pack_tid(file.Tid());
if (pack_tid == Xobc_import_type.Tid__ignore) continue;
rv.Add(zip_name_bldr, pack_tid, file.Url());
}
rv.Consolidate(Xobc_import_type.Tid__wiki__srch);
}
rv.Consolidate(Xobc_import_type.Tid__wiki__srch);
// bld file pack
Fsm_mnt_itm mnt_itm = wiki.File__mnt_mgr().Mnts__get_at(Fsm_mnt_mgr.Mnt_idx_main);
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__core, wiki_dir.GenSubFil(mnt_itm.Atr_mgr().Db__core().Url_rel()));
if (db_mgr.Props().Layout_file().Tid_is_lot()) {
Fsm_bin_mgr bin_mgr = mnt_itm.Bin_mgr();
int bin_len = bin_mgr.Dbs__len();
for (int i = 0; i < bin_len; ++i) {
Fsm_bin_fil bin_fil = bin_mgr.Dbs__get_at(i);
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__data, bin_fil.Url());
if (pack_file) {
Fsm_mnt_itm mnt_itm = wiki.File__mnt_mgr().Mnts__get_at(Fsm_mnt_mgr.Mnt_idx_main);
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__core, wiki_dir.GenSubFil(mnt_itm.Atr_mgr().Db__core().Url_rel()));
if (db_mgr.Props().Layout_file().Tid_is_lot()) {
Fsm_bin_mgr bin_mgr = mnt_itm.Bin_mgr();
int bin_len = bin_mgr.Dbs__len();
for (int i = 0; i < bin_len; ++i) {
Fsm_bin_fil bin_fil = bin_mgr.Dbs__get_at(i);
Io_url bin_fil_url = bin_fil.Url();
// ignore if bin_fil is earlier than cutoff
if (pack_file_cutoff != null) {
DateAdp bin_fil_date = Io_mgr.Instance.QueryFil(bin_fil_url).ModifiedTime();
if (bin_fil_date.Timestamp_unix() < pack_file_cutoff.Timestamp_unix()) continue;
}
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__data, bin_fil_url);
}
}
}
return rv;

10
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__lnki_temp__create.java
View File

@@ -26,7 +26,7 @@ import gplx.fsdb.meta.*; import gplx.xowa.files.fsdb.*; import gplx.fsdb.*;
import gplx.xowa.langs.vnts.*; import gplx.xowa.parsers.vnts.*;
import gplx.xowa.parsers.lnkis.files.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.files.dbs.*;
import gplx.xowa.addons.bldrs.files.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx.xowa.parsers.lnkis.files.Xop_file_logger {
private Xob_lnki_temp_tbl tbl; private boolean wdata_enabled = true, xtn_ref_enabled = true, gen_html, gen_hdump;
private Xop_log_invoke_wkr invoke_wkr; private Xop_log_property_wkr property_wkr;
@@ -140,7 +140,7 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
Xof_ext ext = Xof_ext_.new_by_ttl_(ttl);
double lnki_time = lnki.Time();
int lnki_page = lnki.Page();
byte[] ttl_commons = Xto_commons(ns_file_is_case_match_all, commons_wiki, ttl);
byte[] ttl_commons = Xomp_file_logger.To_commons_ttl(ns_file_is_case_match_all, commons_wiki, ttl);
if ( Xof_lnki_page.Null_n(lnki_page) // page set
&& Xof_lnki_time.Null_n(lnki_time)) // thumbtime set
usr_dlg.Warn_many("", "", "page and thumbtime both set; this may be an issue with fsdb: page=~{0} ttl=~{1}", ctx.Page().Ttl().Page_db_as_str(), String_.new_u8(ttl));
@@ -177,11 +177,5 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
if (property_wkr == null) property_wkr = bldr.App().Wiki_mgr().Wdata_mgr().Property_wkr_or_new();
return property_wkr;
}
public static byte[] Xto_commons(boolean ns_file_is_case_match_all, Xowe_wiki commons_wiki, byte[] ttl_bry) {
if (!ns_file_is_case_match_all) return null; // return "" if wiki matches common
Xoa_ttl ttl = Xoa_ttl.Parse(commons_wiki, Xow_ns_.Tid__file, ttl_bry);
byte[] rv = ttl.Page_db();
return Bry_.Eq(rv, ttl_bry) ? null : rv;
}
public static boolean Ns_file_is_case_match_all(Xow_wiki wiki) {return wiki.Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Tid__all;}
}

2
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__lnki_temp__create__tst.java
View File

@@ -38,6 +38,6 @@ class Xobldr__lnki_temp__create__fxt {
return this;
}
public void Test__to_commons(String ttl, String expd) {
Tfds.Eq(expd, String_.new_u8(Xobldr__lnki_temp__create.Xto_commons(wiki_ns_file_is_case_match_all, commons_wiki, Bry_.new_u8(ttl))));
Tfds.Eq(expd, String_.new_u8(gplx.xowa.addons.bldrs.mass_parses.parses.Xomp_file_logger.To_commons_ttl(wiki_ns_file_is_case_match_all, commons_wiki, Bry_.new_u8(ttl))));
}
}

10
400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_lnki_temp_tbl.java
View File

@@ -17,9 +17,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*;
import gplx.dbs.*; import gplx.xowa.files.*;
public class Xob_lnki_temp_tbl {
public class Xob_lnki_temp_tbl implements Db_tbl {
private static final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private static final String Tbl_name = "lnki_temp";
private static final String tbl_name = "lnki_temp";
public static final String
Fld_lnki_id = flds.Add_int_pkey_autonum("lnki_id"); // NOTE: insertion order index; public b/c not used and want to bypass warning
private static final String
@@ -39,8 +39,9 @@ public class Xob_lnki_temp_tbl {
private Db_stmt stmt_insert;
public Xob_lnki_temp_tbl(Db_conn conn) {this.conn = conn;}
public Db_conn Conn() {return conn;} private final Db_conn conn;
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(Tbl_name, flds));}
public void Insert_bgn() {conn.Txn_bgn("bldr__lnki_temp"); stmt_insert = conn.Stmt_insert(Tbl_name, flds);}
public String Tbl_name() {return tbl_name;}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Insert_bgn() {conn.Txn_bgn("bldr__lnki_temp"); stmt_insert = conn.Stmt_insert(tbl_name, flds);}
public void Insert_commit() {conn.Txn_sav();}
public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
public void Insert_cmd_by_batch(int tier_id, int page_id, byte[] ttl, byte[] ttl_commons, byte ext_id, byte img_type, byte lnki_src_tid, int w, int h, double upright, double time, int page) {
@@ -59,4 +60,5 @@ public class Xob_lnki_temp_tbl {
.Val_int (Fld_lnki_page , page)
.Exec_insert();
}
public void Rls() {}
}

1
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_db_core.java
View File

@@ -25,6 +25,7 @@ public class Xomp_db_core {
Io_url mgr_url = root_dir.GenSubFil("xomp.sqlite3");
this.mgr_db = new Xomp_mgr_db(mgr_url);
}
public Db_conn Conn() {return mgr_db.Conn();}
public Xomp_mgr_db Mgr_db() {return mgr_db;} private Xomp_mgr_db mgr_db;
public Xomp_wkr_db Wkr_db(boolean delete, int idx) {
Io_url wkr_url = root_dir.GenSubFil_nest("xomp_" + Int_.To_str_fmt(idx, "000"), "xomp_wkr.sqlite3");

9
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_page_tbl.java
View File

@@ -23,7 +23,8 @@ public class Xomp_page_tbl implements Db_tbl {
public Xomp_page_tbl(Db_conn conn) {
this.conn = conn;
this.tbl_name = "xomp_page";
flds.Add_int_pkey("page_id");
flds.Add_int_pkey("xomp_uid");
flds.Add_int("page_id");
flds.Add_int("page_ns");
flds.Add_byte("page_status"); // 0=wait; 1=done; 2=fail
flds.Add_int_dflt("html_len", -1);
@@ -33,7 +34,11 @@ public class Xomp_page_tbl implements Db_tbl {
public String Tbl_name() {return tbl_name;} private final String tbl_name;
public Dbmeta_fld_list Flds() {return flds;} private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
public void Create_tbl() {
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
conn.Meta_tbl_create
( Dbmeta_tbl_itm.New(tbl_name, flds
, Dbmeta_idx_itm.new_normal_by_tbl("xomp_page", "xomp_uid__page_status" , "xomp_uid", "page_status")// for parse
, Dbmeta_idx_itm.new_normal_by_tbl("xomp_page", "page_ns__page_id" , "page_ns", "page_id") // for make
));
}
public void Rls() {}
}

57
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xob_lnki_temp_row.java Normal file
View File

@@ -0,0 +1,57 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
class Xob_lnki_temp_row implements CompareAble {
public int Xomp_uid() {return xomp_uid;} private int xomp_uid;
public int Make_uid() {return make_uid;} private int make_uid;
public int Lnki_tier_id() {return lnki_tier_id;} private int lnki_tier_id;
public int Lnki_page_id() {return lnki_page_id;} private int lnki_page_id;
public byte[] Lnki_ttl() {return lnki_ttl;} private byte[] lnki_ttl;
public byte[] Lnki_commons_ttl() {return lnki_commons_ttl;} private byte[] lnki_commons_ttl;
public byte Lnki_ext() {return lnki_ext;} private byte lnki_ext;
public byte Lnki_type() {return lnki_type;} private byte lnki_type;
public byte Lnki_src_tid() {return lnki_src_tid;} private byte lnki_src_tid;
public int Lnki_w() {return lnki_w;} private int lnki_w;
public int Lnki_h() {return lnki_h;} private int lnki_h;
public double Lnki_upright() {return lnki_upright;} private double lnki_upright;
public double Lnki_time() {return lnki_time;} private double lnki_time;
public int Lnki_page() {return lnki_page;} private int lnki_page;
public void Load(Db_rdr rdr, int xomp_uid) {
this.xomp_uid = xomp_uid;
this.make_uid = rdr.Read_int("lnki_make_uid");
this.lnki_tier_id = rdr.Read_int("lnki_tier_id");
this.lnki_page_id = rdr.Read_int("lnki_page_id");
this.lnki_ttl = rdr.Read_bry_by_str("lnki_ttl");
this.lnki_commons_ttl = rdr.Read_bry_by_str("lnki_commons_ttl");
this.lnki_ext = rdr.Read_byte("lnki_ext");
this.lnki_type = rdr.Read_byte("lnki_type");
this.lnki_src_tid = rdr.Read_byte("lnki_src_tid");
this.lnki_w = rdr.Read_int("lnki_w");
this.lnki_h = rdr.Read_int("lnki_h");
this.lnki_upright = rdr.Read_double("lnki_upright");
this.lnki_time = rdr.Read_int("lnki_time");
this.lnki_page = rdr.Read_int("lnki_page");
}
public int compareTo(Object obj) {
Xob_lnki_temp_row comp = (Xob_lnki_temp_row)obj;
int rv = Int_.Compare(xomp_uid, comp.xomp_uid);
if (rv != CompareAble_.Same) return rv;
return Int_.Compare(make_uid, comp.make_uid);
}
}

29
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_wtr.java
View File

@@ -33,24 +33,33 @@ class Xomp_html_db_wtr {
public int Cur_db_id() {return html_db.Id();}
public Xowd_html_tbl Tbls__get_or_new(int ns_id, long html_len) {
long len_new = len_cur + html_len;
if (html_tbl == null || len_new > len_max) {
Commit();
this.html_db = wiki.Data__core_mgr().Dbs__get_by_tid_or_null(Xow_db_file_.Tid__html_data);
if (html_db == null) {
html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data);
html_db.Conn().Txn_bgn("xomp.html_db_wtr");
this.html_tbl = new Xowd_html_tbl(html_db.Conn());
html_tbl.Create_tbl();
boolean not_inited = html_tbl == null, out_of_space = len_new > len_max;
if (not_inited || out_of_space) {
if (out_of_space)
Commit();
if ( db_mgr.Props().Layout_html().Tid_is_all_or_few() // is not "lot"
&& not_inited // not_inited; set html_db
) {
this.html_db = wiki.Data__core_mgr().Dbs__get_by_tid_or_null(Xow_db_file_.Tid__html_data);
if (html_db == null)
this.html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data);
}
else
this.html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data);
this.html_tbl = new Xowd_html_tbl(html_db.Conn());
html_tbl.Create_tbl();
html_db.Conn().Txn_bgn("xomp.html_db_wtr");
len_cur = html_len; // NOTE: this effectively resets len_new to 0 + html_len;
}
len_cur = len_new;
else // initied and still has space; just update len
len_cur = len_new;
return html_tbl;
}
public void Rls() {
this.Commit();
}
private void Commit() {
if (html_tbl == null) return;
html_tbl.Conn().Txn_end();
html_tbl.Conn().Rls_conn();

95
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java Normal file
View File

@@ -0,0 +1,95 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.brys.*;
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*; import gplx.xowa.addons.bldrs.files.dbs.*;
import gplx.xowa.bldrs.*;
class Xomp_make_lnki {
public void Exec(Xowe_wiki wiki, int uid_count) {
// init
Xomp_db_core xomp_db = Xomp_db_core.New__load(wiki);
Xob_db_file make_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
Xob_lnki_temp_tbl lnki_temp_tbl = new Xob_lnki_temp_tbl(make_db.Conn());
lnki_temp_tbl.Insert_bgn();
// add index
int wkr_count = xomp_db.Wkr_count();
for (int i = 0; i < wkr_count; ++i) {
Xomp_wkr_db wkr_db = xomp_db.Wkr_db(Bool_.N, i);
wkr_db.Conn().Meta_idx_assert("lnki_temp", "lnki_page_id", "lnki_page_id");
}
// loop
List_adp rows = List_adp_.New();
int uid_bgn = -1;
while (true) {
int count = 0;
for (int i = 0; i < wkr_count; ++i) {
Xomp_wkr_db wkr_db = xomp_db.Wkr_db(Bool_.N, i);
count += Load_rows(rows, xomp_db, wkr_db, uid_bgn, uid_bgn + uid_count);
}
if (count == 0) break;
Save_rows(rows, lnki_temp_tbl);
}
// term
lnki_temp_tbl.Insert_end();
xomp_db.Conn().Rls_conn();
make_db.Conn().Rls_conn();
}
private int Load_rows(List_adp rows, Xomp_db_core xomp_db, Xomp_wkr_db wkr_db, int uid_bgn, int uid_end) {
// build sql
Db_attach_mgr attach_mgr = new Db_attach_mgr(xomp_db.Conn());
attach_mgr.Conn_others_(new Db_attach_itm("wkr_db", wkr_db.Conn()));
String sql = String_.Format(String_.Concat_lines_nl_skip_last
( "SELECT mgr.xomp_uid"
, ", wkr.*"
, "FROM <wkr_db>lnki_temp wkr"
, " JOIN xomp_page mgr ON wkr.lnki_page_id = mgr.page_id"
, "WHERE mgr.xomp_uid > {0} AND mgr.xomp_uid <= {1}"
)
, uid_bgn
, uid_end
);
sql = attach_mgr.Resolve_sql(sql);
attach_mgr.Attach();
Db_rdr rdr = xomp_db.Conn().Stmt_sql(sql).Exec_select__rls_auto(); // ANSI.Y
int rv = -1;
try {
while (rdr.Move_next()) {
rv = rdr.Read_int("xomp_uid");
Xob_lnki_temp_row row = new Xob_lnki_temp_row();
row.Load(rdr, rv);
}
} finally {rdr.Rls();}
attach_mgr.Detach();
return rv;
}
private void Save_rows(List_adp rows, Xob_lnki_temp_tbl lnki_temp_tbl) {
int len = rows.Len();
for (int i = 0; i < len; ++i) {
Xob_lnki_temp_row row = (Xob_lnki_temp_row)rows.Get_at(i);
lnki_temp_tbl.Insert_cmd_by_batch(row.Lnki_tier_id(), row.Lnki_page_id(), row.Lnki_ttl(), row.Lnki_commons_ttl()
, row.Lnki_ext(), row.Lnki_src_tid(), row.Lnki_src_tid(), row.Lnki_w(), row.Lnki_h(), row.Lnki_upright()
, row.Lnki_time(), row.Lnki_page());
}
}
}

58
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_file_logger.java Normal file
View File

@@ -0,0 +1,58 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.lnkis.*; import gplx.xowa.parsers.lnkis.files.*;
import gplx.xowa.files.*; import gplx.xowa.addons.bldrs.files.cmds.*; import gplx.xowa.addons.bldrs.files.dbs.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.domains.*;
public class Xomp_file_logger implements Xop_file_logger {
private final Xob_lnki_temp_tbl tbl;
private final Xowe_wiki commons_wiki;
private boolean ns_file_is_case_match_all = true;
public Xomp_file_logger(Xowe_wiki wiki, Db_conn wkr_conn) {
this.tbl = new Xob_lnki_temp_tbl(wkr_conn); wkr_conn.Meta_tbl_assert(tbl);
this.commons_wiki = wiki.Appe().Wiki_mgr().Get_by_or_make(Xow_domain_itm_.Bry__commons);
this.ns_file_is_case_match_all = wiki.Init_assert().Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Tid__all; // NOTE: wiki must be init'd;
}
public void Bgn() {
tbl.Insert_bgn();
}
public void Log_file(Xop_ctx ctx, Xop_lnki_tkn lnki, byte caller_tid) {
if (lnki.Ttl().ForceLiteralLink()) return; // ignore literal links which create a link to file, but do not show the image; EX: [[:File:A.png|thumb|120px]] creates a link to File:A.png, regardless of other display-oriented args
// get caller_tid / tttl
if (lnki.Ns_id() == Xow_ns_.Tid__media) caller_tid = Xop_file_logger_.Tid__media;
// get lnki_data
byte[] ttl = lnki.Ttl().Page_db();
Xof_ext ext = Xof_ext_.new_by_ttl_(ttl);
byte[] ttl_commons = Xomp_file_logger.To_commons_ttl(ns_file_is_case_match_all, commons_wiki, ttl);
// do insert
tbl.Insert_cmd_by_batch(ctx.Page().Bldr__ns_ord(), ctx.Page().Db().Page().Id(), ttl, ttl_commons, Byte_.By_int(ext.Id()), lnki.Lnki_type(), caller_tid, lnki.W(), lnki.H(), lnki.Upright(), lnki.Time(), lnki.Page());
}
public void End() {
tbl.Insert_end();
}
public static byte[] To_commons_ttl(boolean ns_file_is_case_match_all, Xowe_wiki commons_wiki, byte[] ttl_bry) { // handle case-sensitive wikis (en.d) vs case-insensitive commons
if (!ns_file_is_case_match_all) return null; // return "" if wiki matches common
Xoa_ttl ttl = Xoa_ttl.Parse(commons_wiki, Xow_ns_.Tid__file, ttl_bry);
byte[] rv = ttl.Page_db();
return Bry_.Eq(rv, ttl_bry) ? null : rv;
}
}

13
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_page_pool_loader.java
View File

@@ -21,7 +21,7 @@ class Xomp_page_pool_loader {
private final Xow_wiki wiki;
private final int num_pages_per_load;
private final Db_attach_mgr attach_mgr;
private int prv_page_id = -1;
private int prv_uid = -1;
public Xomp_page_pool_loader(Xow_wiki wiki, Db_conn make_conn, int num_pages_per_load) {
this.wiki = wiki;
this.make_conn = make_conn;
@@ -49,16 +49,17 @@ class Xomp_page_pool_loader {
private void Load_from_db(List_adp list) {
// prepare for page_tbl
String sql = String_.Format(String_.Concat_lines_nl_skip_last // ANSI.Y
( "SELECT mp.page_id"
( "SELECT mp.xomp_uid"
, ", pp.page_id"
, ", pp.page_namespace"
, ", pp.page_title"
, ", pp.page_text_db_id"
, "FROM xomp_page mp"
, " JOIN <page_db>page pp ON mp.page_id = pp.page_id"
, "WHERE mp.page_id > {0}"
, "WHERE mp.xomp_uid > {0}"
, "AND mp.page_status = 0"
, "LIMIT {1}"
), prv_page_id, num_pages_per_load);
), prv_uid, num_pages_per_load);
this.attach_mgr.Conn_others_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
sql = attach_mgr.Resolve_sql(sql);
@@ -68,9 +69,9 @@ class Xomp_page_pool_loader {
Db_rdr rdr = make_conn.Stmt_sql(sql).Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
prv_page_id = rdr.Read_int("page_id");
prv_uid = rdr.Read_int("xomp_uid");
int text_db_id = rdr.Read_int("page_text_db_id");
Xomp_page_itm ppg = new Xomp_page_itm(prv_page_id);
Xomp_page_itm ppg = new Xomp_page_itm(rdr.Read_int("page_id"));
ppg.Init_by_page
( rdr.Read_int("page_namespace")
, rdr.Read_bry_by_str("page_title")

17
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_mgr.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.threads.*; import gplx.core.threads.utils.*;
import gplx.core.threads.*; import gplx.core.threads.utils.*; import gplx.core.caches.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.cases.*;
import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.wikis.caches.*;
@@ -32,9 +32,11 @@ class Xomp_parse_mgr {
this.db_core = Xomp_db_core.New__load(wiki);
Xomp_page_pool_loader pool_loader = new Xomp_page_pool_loader(wiki, db_core.Mgr_db().Conn(), cfg.Num_pages_in_pool());
Xomp_page_pool page_pool = new Xomp_page_pool(pool_loader, cfg.Num_pages_per_wkr());
prog_mgr.Init(pool_loader.Get_pending_count());
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, true);
prog_mgr.Init(pool_loader.Get_pending_count(), cfg.Progress_interval());
wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of cache
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, true);
Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
Gfo_cache_mgr ifexist_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
// load_wkr: init and start
// Xomp_load_wkr load_wkr = new Xomp_load_wkr(wiki, db_core.Mgr_db().Conn(), cfg.Num_pages_in_pool(), cfg.Num_wkrs());
@@ -45,8 +47,13 @@ class Xomp_parse_mgr {
latch = new Gfo_countdown_latch(wkr_len);
Xomp_parse_wkr[] wkrs = new Xomp_parse_wkr[wkr_len];
for (int i = 0; i < wkr_len; ++i) {
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, Clone_wiki(wiki), page_pool, i);
wkr.Wiki().Cache_mgr().Page_cache_(page_cache);
Xowe_wiki wkr_wiki = Clone_wiki(wiki);
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, wkr_wiki, page_pool, i, cfg.Cleanup_interval(), cfg.Progress_interval(), cfg.Log_file_lnkis());
wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);
// remove wmf wkr, else will try to download images during parsing
if (wkr_wiki.File__bin_mgr() != null)
wkr_wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf);
wkr.Hdump_bldr().Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled());
wkrs[i] = wkr;
}

24
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_mgr_cfg.java
View File

@@ -17,12 +17,17 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
class Xomp_parse_mgr_cfg implements Gfo_invk {
// public Io_url Root_dir() {return root_dir;} private Io_url root_dir;
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1;
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1; // use env.available_processors
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = 1000;
public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000;
public int Progress_interval() {return progress_interval;} private int progress_interval = 1000;
public int Commit_interval() {return commit_interval;} private int commit_interval = 10000;
public int Cleanup_interval() {return cleanup_interval;} private int cleanup_interval = 50; // setting at 1000 uses lots of memory
public boolean Hdump_enabled() {return hdump_enabled;} private boolean hdump_enabled = true;
public boolean Hzip_enabled() {return hzip_enabled;} private boolean hzip_enabled = true;
public boolean Hdiff_enabled() {return hdiff_enabled;} private boolean hdiff_enabled = true;
public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true;
public void Init(Xowe_wiki wiki) {
// if (root_dir == null) root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Env_.System_cpu_count();
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
@@ -30,11 +35,18 @@ class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
// else if (ctx.Match(k, Invk__root_dir_)) root_dir = m.ReadIoUrl("v");
else if (ctx.Match(k, Invk__progress_interval_)) progress_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__commit_interval_)) commit_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__cleanup_interval_)) cleanup_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__hdump_enabled_)) hdump_enabled = m.ReadBool("v");
else if (ctx.Match(k, Invk__hzip_enabled_)) hzip_enabled = m.ReadBool("v");
else if (ctx.Match(k, Invk__hdiff_enabled_)) hdiff_enabled = m.ReadBool("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}
private static final String Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
// , Invk__root_dir_ = "root_dir_"
private static final String
Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
, Invk__progress_interval_ = "progress_interval_", Invk__commit_interval_ = "commit_interval_", Invk__cleanup_interval_ = "cleanup_interval_"
, Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_"
;
}

43
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_wkr.java
View File

@@ -23,23 +23,42 @@ class Xomp_parse_wkr implements Gfo_invk {
private final Xomp_parse_mgr mgr;
private final Xomp_page_pool page_pool;
private final int idx;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private int done_count; private long done_time;
private Xomp_wkr_db wkr_db;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xomp_page_pool page_pool, int idx) {
private Xomp_wkr_db wkr_db; private int cleanup_interval, commit_interval;
private boolean log_file_lnkis;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xomp_page_pool page_pool, int idx, int cleanup_interval, int commit_interval, boolean log_file_lnkis) {
this.mgr = mgr; this.wiki = wiki;
this.page_pool = page_pool;
this.idx = idx;
this.wkr_db = mgr.Db_core().Wkr_db(Bool_.Y, idx); // NOTE: must go in ctor, or else thread issues
this.cleanup_interval = cleanup_interval;
this.commit_interval = commit_interval;
this.log_file_lnkis = log_file_lnkis;
}
public Xowe_wiki Wiki() {return wiki;} private final Xowe_wiki wiki;
public Xob_hdump_bldr Hdump_bldr() {return hdump_bldr;} private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
public void Exec() {
// init
Xow_parser_mgr parser_mgr = new Xow_parser_mgr(wiki);
wiki.Html_mgr().Page_wtr_mgr().Wkr(gplx.xowa.wikis.pages.Xopg_page_.Tid_read).Ctgs_enabled_(false); // disable categories else progress messages written (also for PERF)
if (wiki.File__bin_mgr() != null)
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // remove wmf wkr, else will try to download images during parsing
// disable file download
wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
wiki.File__orig_mgr().Wkrs_del(gplx.xowa.files.origs.Xof_orig_wkr_.Tid_wmf_api);
// disable categories else progress messages written (also for PERF)
wiki.Html_mgr().Page_wtr_mgr().Wkr(gplx.xowa.wikis.pages.Xopg_page_.Tid_read).Ctgs_enabled_(false);
// enable lnki_temp
Xomp_file_logger logger = null;
if (log_file_lnkis) {
logger = new Xomp_file_logger(wiki, wkr_db.Conn());
parser_mgr.Ctx().Lnki().File_logger_(logger);
logger.Bgn();
}
// enable hdump
hdump_bldr.Enabled_(true).Hzip_enabled_(true).Hzip_diff_(true).Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
wkr_db.Conn().Txn_bgn("xomp");
@@ -76,14 +95,22 @@ class Xomp_parse_wkr implements Gfo_invk {
// ctx.App().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released
if (wiki.Cache_mgr().Tmpl_result_cache().Count() > 50000)
wiki.Cache_mgr().Tmpl_result_cache().Clear();
if (done_count % 50 == 0) {
if (done_count % cleanup_interval == 0) {
wiki.Cache_mgr().Free_mem_all(Bool_.N);
wiki.Parser_mgr().Scrib().Core_term();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear();
}
if (done_count % commit_interval == 0)
wkr_db.Conn().Txn_sav();
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e));
}
}
if (logger != null)
logger.End();
// cleanup
wkr_db.Conn().Txn_end(); // NOTE: must end txn before running update wkr_id
mgr.Db_core().Update_wkr_id(idx, wkr_db.Conn());
mgr.Wkrs_done_add_1();

6
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_prog_mgr.java
View File

@@ -18,17 +18,19 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
class Xomp_prog_mgr {
private final Object thread_lock = new Object();
private int progress_interval;
private int pages_done, pages_total;
private long time_bgn, time_prv, time_done;
private final Bry_bfr prog_bfr = Bry_bfr_.New();
public void Init(int pages_total) {
public void Init(int pages_total, int progress_interval) {
this.progress_interval = progress_interval;
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
this.pages_total = pages_total;
}
public void Mark_done(int id) {
synchronized (thread_lock) {
pages_done += 1;
if (pages_done % 1000 == 0) {
if (pages_done % progress_interval == 0) {
long time_cur = gplx.core.envs.Env_.TickCount();
int pages_left = pages_total - pages_done;
time_done += (time_cur - time_prv);

46
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_addon.java Normal file
View File

@@ -0,0 +1,46 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.specials.*;
import gplx.xowa.addons.wikis.imports.*;
public class Xobc_xodir_addon implements Xoax_addon_itm, Xoax_addon_itm__special, Xoax_addon_itm__init {
public Xow_special_page[] Special_pages() {
return new Xow_special_page[]
{ Xobc_xodir_special.Prototype
};
}
public void Init_addon_by_app(Xoa_app app) {
}
public void Init_addon_by_wiki(Xow_wiki wiki) {
Xow_import_addon addon = Xow_import_addon.Addon__get(wiki);
addon.Dir_selected_cbks__add(Xow_import_dir_cbk__xodir.Instance);
}
public String Addon__key() {return ADDON__KEY;} private static final String ADDON__KEY = "xowa.bldrs.xodirs";
}
class Xow_import_dir_cbk__xodir implements Xow_import_dir_cbk {
public String Key() {return "xodir";}
public void Cbk__dir_selected(Xow_wiki wiki, Xoa_page page, String path) {
// save to prefs
wiki.App().User().User_db_mgr().Cfg().Set_app_str("xowa.xodir.custom_dir", path);
// redirect to import_dir
page.Redirect().Itms__add__special(wiki, Xobc_xodir_special.Prototype.Special__meta());
}
public static Xow_import_dir_cbk__xodir Instance = new Xow_import_dir_cbk__xodir(); Xow_import_dir_cbk__xodir() {}
}

30
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_cfg.java Normal file
View File

@@ -0,0 +1,30 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
public class Xobc_xodir_cfg {
public static final String
Key__selected_dir = "xowa.xodir.selected_dir"
, Key__custom_dir = "xowa.xodir.custom_dir"
;
public static void Set_app_str__selected(Xoa_app app, byte[] val_bry) {
// if wnt, replace "\"; note that url-encoding while navigating dirs will always convert "\" to "/"
if (gplx.core.envs.Op_sys.Cur().Tid_is_wnt()) val_bry = Bry_.Replace(val_bry, Byte_ascii.Slash, Byte_ascii.Backslash);
app.User().User_db_mgr().Cfg().Set_app_bry(Xobc_xodir_cfg.Key__selected_dir, val_bry);
}
}

51
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_dir.java Normal file
View File

@@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.langs.mustaches.*;
class Xobc_xodir_doc implements Mustache_doc_itm {
private final byte[] import_root, app_root_dir;
private final Xobc_xodir_dir[] dirs;
public Xobc_xodir_doc(Xobc_xodir_dir[] dirs, byte[] import_root, byte[] app_root_dir) {
this.dirs = dirs; this.import_root = import_root; this.app_root_dir = app_root_dir;
}
public boolean Mustache__write(String key, Mustache_bfr bfr) {
if (String_.Eq(key, "import_root")) bfr.Add_bry(import_root);
else if (String_.Eq(key, "app_root_dir")) bfr.Add_bry(app_root_dir);
return false;
}
public Mustache_doc_itm[] Mustache__subs(String key) {
if (String_.Eq(key, "dirs")) return dirs;
return Mustache_doc_itm_.Ary__empty;
}
}
public class Xobc_xodir_dir implements Mustache_doc_itm {
private final boolean is_selected, is_custom;
private final byte[] path;
public Xobc_xodir_dir(boolean is_selected, boolean is_custom, byte[] path) {
this.is_selected = is_selected; this.is_custom = is_custom; this.path = path;
}
public boolean Mustache__write(String key, Mustache_bfr bfr) {
if (String_.Eq(key, "path")) bfr.Add_bry(path);
return false;
}
public Mustache_doc_itm[] Mustache__subs(String key) {
if (String_.Eq(key, "is_selected")) return Mustache_doc_itm_.Ary__bool(is_selected);
else if (String_.Eq(key, "is_custom")) return Mustache_doc_itm_.Ary__bool(is_custom);
return Mustache_doc_itm_.Ary__empty;
}
}

40
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_html.java Normal file
View File

@@ -0,0 +1,40 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.specials.*; import gplx.langs.mustaches.*; import gplx.xowa.wikis.pages.*; import gplx.xowa.wikis.pages.tags.*;
import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
import gplx.xowa.wikis.domains.*; import gplx.core.ios.*;
class Xobc_xodir_html extends Xow_special_wtr__base {
@Override protected Io_url Get_addon_dir(Xoa_app app) {return app.Fsys_mgr().Http_root().GenSubDir_nest("bin", "any", "xowa", "addon", "bldr", "xodir");}
@Override protected Io_url Get_mustache_fil(Io_url addon_dir) {return addon_dir.GenSubFil_nest("bin", "xobc_xodir.mustache.html");}
@Override protected Mustache_doc_itm Bld_mustache_root(Xoa_app app) {
Xobc_task_addon addon = (Xobc_task_addon)app.Addon_mgr().Itms__get_or_null(Xobc_task_addon.ADDON__KEY);
return new Xobc_xodir_doc
( addon.Xodir_mgr().Get_dirs(app)
, gplx.xowa.addons.wikis.imports.Xow_import_special.Get_root_url()
, app.Fsys_mgr().Root_dir().RawBry()
);
}
@Override protected void Bld_tags(Xoa_app app, Io_url addon_dir, Xopage_html_data page_data) {
Xopg_tag_mgr head_tags = page_data.Head_tags();
Xopg_alertify_.Add_tags (head_tags, app.Fsys_mgr().Http_root());
Xopg_tag_wtr_.Add__xocss (head_tags, app.Fsys_mgr().Http_root());
Xopg_tag_wtr_.Add__xohelp (head_tags, app.Fsys_mgr().Http_root());
head_tags.Add(Xopg_tag_itm.New_css_file(addon_dir.GenSubFil_nest("bin", "xobc_xodir.css")));
}
}

21
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_mgr.java Normal file
View File

@@ -0,0 +1,21 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
public interface Xobc_xodir_mgr {
Xobc_xodir_dir[] Get_dirs(Xoa_app app);
}

32
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_mgr__pc.java Normal file
View File

@@ -0,0 +1,32 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
public class Xobc_xodir_mgr__pc implements Xobc_xodir_mgr {
public Xobc_xodir_mgr__pc(Xoa_app app) {
}
public Xobc_xodir_dir[] Get_dirs(Xoa_app app) {
int len = 2;
String dflt = app.Fsys_mgr().Root_dir().Raw();
String selected = app.User().User_db_mgr().Cfg().Get_app_str_or(Xobc_xodir_cfg.Key__selected_dir, dflt);
String custom = app.User().User_db_mgr().Cfg().Get_app_str_or(Xobc_xodir_cfg.Key__custom_dir, "(choose your own folder)");
Xobc_xodir_dir[] rv = new Xobc_xodir_dir[len];
rv[0] = new Xobc_xodir_dir(String_.Eq(selected, dflt), Bool_.N, Bry_.new_u8(dflt));
rv[1] = new Xobc_xodir_dir(String_.Eq(selected, custom), Bool_.Y, Bry_.new_u8(custom));
return rv;
}
}

45
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xobc_xodir_special.java Normal file
View File

@@ -0,0 +1,45 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.specials.*; import gplx.core.net.*; import gplx.core.net.qargs.*; import gplx.xowa.wikis.pages.*;
import gplx.xowa.addons.bldrs.centrals.*;
import gplx.xowa.addons.wikis.imports.*;
public class Xobc_xodir_special implements Xow_special_page {
public void Special__gen(Xow_wiki wiki, Xoa_page page, Xoa_url url, Xoa_ttl ttl) {
Gfo_qarg_mgr url_args = new Gfo_qarg_mgr().Init(url.Qargs_ary());
byte[] path = url_args.Read_bry_or(Bry__path, null);
if (path != null) { // path selected; set cfg and redirect to Download Central
Xobc_xodir_cfg.Set_app_str__selected(wiki.App(), path);
// On_path_selected.Invk(null, -1, "", null);
// page.Redirect().Itms__add__special(wiki, Xobc_task_special.Prototype.Special__meta());
page.Redirect().Itms__add__special(wiki, Prototype.Special__meta());
return;
}
new Xobc_xodir_html().Bld_page_by_mustache(wiki.App(), page, this);
}
private static final byte[] Bry__path = Bry_.new_a7("path");
Xobc_xodir_special(Xow_special_meta special__meta) {this.special__meta = special__meta;}
public Xow_special_meta Special__meta() {return special__meta;} private final Xow_special_meta special__meta;
public Xow_special_page Special__clone() {return this;}
public static final Xow_special_page Prototype = new Xobc_xodir_special(Xow_special_meta.New_xo("XowaRootDir", "XOWA Folder Selection"));
public static Gfo_invk On_path_selected = Gfo_invk_.Noop;
}

19
400_xowa/src/gplx/xowa/addons/bldrs/xodirs/Xow_import_dir_cbk__xodir.java Normal file
View File

@@ -0,0 +1,19 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.xodirs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.addons.wikis.imports.*;

29
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__txt.java
View File

@@ -33,7 +33,7 @@ class Xoh_toc_wkr__txt {
public void Init(byte[] page_name) {this.page_name = page_name;}
public void Calc_anch_text(Xoh_toc_itm rv, byte[] src) { // text within hdr; EX: <h2>Abc</h2> -> Abc
int end = src.length;
src = Remove_comment(text_bfr, src, 0, end);
src = Gfh_utl.Del_comments(text_bfr, src, 0, end);
end = src.length;
tag_rdr.Init(page_name, src, 0, end);
try {
@@ -115,6 +115,7 @@ class Xoh_toc_wkr__txt {
case Gfh_tag_.Id__img:
case Gfh_tag_.Id__br:
case Gfh_tag_.Id__hr:
case Gfh_tag_.Id__wbr:
lhs_is_dangling = true;
break;
}
@@ -151,31 +152,5 @@ class Xoh_toc_wkr__txt {
}
}
public static byte[] Remove_comment(Bry_bfr tmp, byte[] src, int bgn, int end) {
boolean dirty = false, append_to_eos = true;
int pos = bgn;
while (true) {
int comm_bgn = Bry_find_.Find_fwd(src, Gfh_tag_.Comm_bgn, pos, end);
if (comm_bgn != -1) { // comment found
int tmp_pos = comm_bgn + Gfh_tag_.Comm_bgn_len;
int comm_end = Bry_find_.Find_fwd(src, Gfh_tag_.Comm_end, tmp_pos, end);
if (comm_end == -1) { // dangling
tmp.Add_mid(src, pos, comm_bgn);
append_to_eos = false;
}
else {
dirty = true;
tmp.Add_mid(src, pos, comm_bgn);
pos = comm_end + Gfh_tag_.Comm_end_len;
continue;
}
}
break;
}
if (dirty && append_to_eos) {
tmp.Add_mid(src, pos, end);
}
return dirty ? tmp.To_bry_and_clear() : src;
}
private static final byte[] id_trim_ary = Bry_.mask_(256, Byte_ascii.Underline_bry);
}

4
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__txt__basic__tst.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
import org.junit.*; import gplx.core.tests.*;
import org.junit.*; import gplx.core.tests.*; import gplx.langs.htmls.*;
public class Xoh_toc_wkr__txt__basic__tst {
@Before public void init() {fxt.Clear();} private final Xoh_toc_wkr__txt__fxt fxt = new Xoh_toc_wkr__txt__fxt();
@Test public void Basic() {
@@ -62,6 +62,6 @@ class Xoh_toc_wkr__txt__fxt {
}
public void Test__remove_comment(String html, String expd) {
byte[] html_bry = Bry_.new_u8(html);
Gftest.Eq__str(expd, Xoh_toc_wkr__txt.Remove_comment(tmp, html_bry, 0, html_bry.length));
Gftest.Eq__str(expd, Gfh_utl.Del_comments(tmp, html_bry, 0, html_bry.length));
}
}

1
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__txt__xnde__tst.java
View File

@@ -34,6 +34,7 @@ public class Xoh_toc_wkr__txt__xnde__tst {
@Test public void A__nest() {fxt.Test__both("<a href=\"/wiki/A\">b<i>c</i>d</a>" , "bcd", "b<i>c</i>d");}
@Test public void Br() {fxt.Test__both("a<br/>b" , "ab");}
@Test public void Br__dangling() {fxt.Test__both("a<br>b" , "ab");}
@Test public void Wbr__dangling() {fxt.Test__both("a<wbr>b" , "ab");}
@Test public void H2() {fxt.Test__both("a<h2>b</h2>c" , "abc");} // NOTE: not a valid test; MW actually generates "ab" b/c of tidy; see corresponding edit test; DATE:2016-06-28
@Test public void Li() {fxt.Test__text("a<ul><li>b</li></ul>c" , "abc");}
@Test public void Table() {fxt.Test__text("a<table><tr><td>b</td></tr></table>c" , "abc");}

16
400_xowa/src/gplx/xowa/addons/wikis/imports/Xow_import_addon.java
View File

@@ -18,11 +18,25 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.wikis.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*;
import gplx.xowa.specials.*;
public class Xow_import_addon implements Xoax_addon_itm, Xoax_addon_itm__special {
private final Ordered_hash cbks = Ordered_hash_.New();
public Xow_special_page[] Special_pages() {
return new Xow_special_page[]
{ Xow_import_special.Prototype
};
}
public void Dir_selected_cbks__add(Xow_import_dir_cbk cbk) {
if (!cbks.Has(cbk.Key()))
cbks.Add(cbk.Key(), cbk);
}
public Xow_import_dir_cbk Dir_selected_cbks__get_by(String key) {return (Xow_import_dir_cbk)cbks.Get_by(key);}
public String Addon__key() {return "xowa.apps.file_browsers";}
public String Addon__key() {return ADDON__KEY;} private static final String ADDON__KEY = "xowa.apps.file_browsers";
public static Xow_import_addon Addon__get(Xow_wiki wiki) {
Xow_import_addon rv = (Xow_import_addon)wiki.Addon_mgr().Itms__get_or_null(ADDON__KEY);
if (rv == null) {
rv = new Xow_import_addon();
wiki.Addon_mgr().Itms__add(rv);
}
return rv;
}
}

22
400_xowa/src/gplx/xowa/addons/wikis/imports/Xow_import_dir_cbk.java Normal file
View File

@@ -0,0 +1,22 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*;
public interface Xow_import_dir_cbk {
String Key();
void Cbk__dir_selected(Xow_wiki wiki, Xoa_page page, String path);
}

21
400_xowa/src/gplx/xowa/addons/wikis/imports/Xow_import_doc.java
View File

@@ -21,12 +21,14 @@ import gplx.langs.mustaches.*;
class Xow_import_doc implements Mustache_doc_itm {
private final boolean is_dir, is_core_xowa;
private final byte[] owner_dir_enc, path, name, date, size, color;
private final byte[] dir_cmd;
private final Xow_import_doc[] subs;
public Xow_import_doc(boolean is_dir, boolean is_core_xowa, int color, byte[] owner_dir, byte[] path, byte[] name, byte[] date, byte[] size, Xow_import_doc[] subs) {
public Xow_import_doc(boolean is_dir, boolean is_core_xowa, int color, byte[] owner_dir, byte[] path, byte[] name, byte[] date, byte[] size, byte[] dir_cmd, Xow_import_doc[] subs) {
this.is_dir = is_dir; this.is_core_xowa = is_core_xowa;
this.color = color % 2 == 0 ? Byte_ascii.Num_0_bry : Byte_ascii.Num_1_bry;
this.owner_dir_enc = gplx.langs.htmls.encoders.Gfo_url_encoder_.Href.Encode(owner_dir);
this.path = path; this.name = name; this.date = date; this.size = size;
this.dir_cmd = dir_cmd;
this.subs = subs;
}
public boolean Mustache__write(String key, Mustache_bfr bfr) {
@@ -37,24 +39,27 @@ class Xow_import_doc implements Mustache_doc_itm {
else if (String_.Eq(key, "date")) bfr.Add_bry(date);
else if (String_.Eq(key, "size")) bfr.Add_bry(size);
else if (String_.Eq(key, "color")) bfr.Add_bry(color);
else if (String_.Eq(key, "dir_cmd")) bfr.Add_bry(dir_cmd);
else if (String_.Eq(key, "dir_cmd_arg")) {bfr.Add_str_u8("&dir_cmd="); bfr.Add_bry(dir_cmd);}
else return false;
return true;
}
public Mustache_doc_itm[] Mustache__subs(String key) {
if (String_.Eq(key, "is_dir")) return Mustache_doc_itm_.Ary__bool(is_dir);
else if (String_.Eq(key, "dir_cmd_exists")) return Mustache_doc_itm_.Ary__bool(Bry_.Len_gt_0(dir_cmd));
else if (String_.Eq(key, "is_core_xowa")) return Mustache_doc_itm_.Ary__bool(is_core_xowa);
else if (String_.Eq(key, "subs")) return subs;
return Mustache_doc_itm_.Ary__empty;
}
public static final Xow_import_doc[] Ary_empty = new Xow_import_doc[0];
public static Xow_import_doc New(IoItmDir owner_dir) {
public static Xow_import_doc New(IoItmDir owner_dir, byte[] dir_cmd) {
List_adp sub_list = List_adp_.New();
New_subs(owner_dir.Url(), sub_list, owner_dir.SubDirs());
New_subs(owner_dir.Url(), sub_list, owner_dir.SubFils());
New_subs(owner_dir.Url(), sub_list, owner_dir.SubDirs(), dir_cmd);
New_subs(owner_dir.Url(), sub_list, owner_dir.SubFils(), Bry_.Empty);
Xow_import_doc[] subs = (Xow_import_doc[])sub_list.To_ary_and_clear(Xow_import_doc.class);
return new Xow_import_doc(Bool_.Y, Bool_.N, 0, owner_dir.Url().OwnerDir().RawBry(), owner_dir.Url().RawBry(), Bry_.new_u8(owner_dir.Name()), Bry_.Empty, Bry_.Empty, subs);
return new Xow_import_doc(Bool_.Y, Bool_.N, 0, owner_dir.Url().OwnerDir().RawBry(), owner_dir.Url().RawBry(), Bry_.new_u8(owner_dir.Name()), Bry_.Empty, Bry_.Empty, dir_cmd, subs);
}
private static void New_subs(Io_url owner_dir, List_adp list, IoItmList subs) {
private static void New_subs(Io_url owner_dir, List_adp list, IoItmList subs, byte[] dir_cmd) {
subs.Sort();
int len = subs.Len();
int list_total = list.Len();
@@ -64,13 +69,13 @@ class Xow_import_doc implements Mustache_doc_itm {
Xow_import_doc trg = null;
if (src.Type_dir()) {
byte[] trg_url = src.Url().RawBry();
trg = new Xow_import_doc(Bool_.Y, Bool_.N, list_total + i, owner_dir_bry, trg_url, Bry_.new_u8(src.Url().NameAndExt_noDirSpr()), Bry_.Empty, Bry_.Empty, Ary_empty);
trg = new Xow_import_doc(Bool_.Y, Bool_.N, list_total + i, owner_dir_bry, trg_url, Bry_.new_u8(src.Url().NameAndExt_noDirSpr()), Bry_.Empty, Bry_.Empty, dir_cmd, Ary_empty);
}
else {
IoItmFil src_as_fil = (IoItmFil)src;
String size_str = Io_size_.To_str(src_as_fil.Size(), "#,###");
boolean is_xowa_core = gplx.xowa.wikis.data.Xow_db_file__core_.Is_core_fil_name(owner_dir.NameOnly(), src.Url().NameAndExt());
trg = new Xow_import_doc(Bool_.N, is_xowa_core, list_total + i, owner_dir_bry, src.Url().RawBry(), Bry_.new_u8(src.Name()), Bry_.new_u8(src_as_fil.ModifiedTime().XtoStr_fmt("yyyy-MM-dd")), Bry_.new_u8(size_str), Ary_empty);
trg = new Xow_import_doc(Bool_.N, is_xowa_core, list_total + i, owner_dir_bry, src.Url().RawBry(), Bry_.new_u8(src.Name()), Bry_.new_u8(src_as_fil.ModifiedTime().XtoStr_fmt("yyyy-MM-dd")), Bry_.new_u8(size_str), dir_cmd, Ary_empty);
}
list.Add(trg);
}

9
400_xowa/src/gplx/xowa/addons/wikis/imports/Xow_import_html.java
View File

@@ -19,13 +19,16 @@ package gplx.xowa.addons.wikis.imports; import gplx.*; import gplx.xowa.*; impor
import gplx.xowa.specials.*; import gplx.langs.mustaches.*; import gplx.xowa.wikis.pages.*; import gplx.xowa.wikis.pages.tags.*;
import gplx.core.ios.*;
class Xow_import_html extends Xow_special_wtr__base {
private final Io_url owner_url;
public Xow_import_html(Io_url owner_url) {this.owner_url = owner_url;}
private final Io_url owner_url; private final byte[] mode;
public Xow_import_html(Io_url owner_url, byte[] mode) {
this.owner_url = owner_url;
this.mode = mode;
}
@Override protected Io_url Get_addon_dir(Xoa_app app) {return app.Fsys_mgr().Http_root().GenSubDir_nest("bin", "any", "xowa", "addon", "wiki", "import");}
@Override protected Io_url Get_mustache_fil(Io_url addon_dir) {return addon_dir.GenSubFil_nest("bin", "xow_import.mustache.html");}
@Override protected Mustache_doc_itm Bld_mustache_root(Xoa_app app) {
IoItmDir owner_dir = Io_mgr.Instance.QueryDir_args(owner_url).DirInclude_(true).ExecAsDir();
return Xow_import_doc.New(owner_dir);
return Xow_import_doc.New(owner_dir, mode);
}
@Override protected void Bld_tags(Xoa_app app, Io_url addon_dir, Xopage_html_data page_data) {
Xopg_tag_mgr head_tags = page_data.Head_tags();

24
400_xowa/src/gplx/xowa/addons/wikis/imports/Xow_import_special.java
View File

@@ -29,7 +29,29 @@ public class Xow_import_special implements Xow_special_page {
return;
}
new Xow_import_html(Io_url_.new_dir_(owner_str)).Bld_page_by_mustache(wiki.App(), page, this);
// check if dir_cmd is available
byte[] dir_cmd = url_args.Read_bry_or_null("dir_cmd");
// check selected
int selected = url_args.Read_int_or("selected", -1);
if ( selected == 1
&& dir_cmd != null) {
Xow_import_addon addon = Xow_import_addon.Addon__get(wiki);
Xow_import_dir_cbk import_cbk = addon.Dir_selected_cbks__get_by(String_.new_u8(dir_cmd));
import_cbk.Cbk__dir_selected(wiki, page, owner_str);
}
new Xow_import_html(Io_url_.new_dir_(owner_str), dir_cmd).Bld_page_by_mustache(wiki.App(), page, this);
}
public static byte[] Get_root_url() {
byte tid = gplx.core.envs.Op_sys.Cur().Tid();
byte[] rv = Bry_.new_a7("/");
switch (tid) {
case gplx.core.envs.Op_sys.Tid_wnt : rv = Bry_.new_a7("C:\\"); break;
}
rv = gplx.langs.htmls.encoders.Gfo_url_encoder_.Href.Encode(rv);
return rv;
}
Xow_import_special(Xow_special_meta special__meta) {this.special__meta = special__meta;}

3
400_xowa/src/gplx/xowa/addons/wikis/registrys/infos/Xow_info_special.java
View File

@@ -31,8 +31,7 @@ public class Xow_info_special implements Xow_special_page {
delete_wiki.Data__core_mgr().Rls(); // release connection if open
Delete_wiki_files(delete_wiki.Fsys_mgr().Root_dir());
}
Xoa_ttl redirect_ttl = wiki.Ttl_parse(gplx.xowa.addons.wikis.registrys.lists.Xow_list_special.Prototype.Special__meta().Ttl_bry());
page.Redirect().Itms__add__special(Xoa_url.New(wiki, redirect_ttl), redirect_ttl);
page.Redirect().Itms__add__special(wiki, gplx.xowa.addons.wikis.registrys.lists.Xow_list_special.Prototype.Special__meta());
return;
}

11
400_xowa/src/gplx/xowa/addons/wikis/registrys/lists/Xow_list_html.java
View File

@@ -33,7 +33,7 @@ class Xow_list_html extends Xow_special_wtr__base {
if (String_.Eq(site_itm.Domain(), gplx.xowa.wikis.domains.Xow_domain_itm_.Str__home)) continue;
list.Add(new Xow_list_doc_wiki(Bry_.new_u8(site_itm.Domain()), site_itm.Date()));
}
return new Xow_list_doc(Get_root_url(), (Xow_list_doc_wiki[])list.To_ary_and_clear(Xow_list_doc_wiki.class));
return new Xow_list_doc(gplx.xowa.addons.wikis.imports.Xow_import_special.Get_root_url(), (Xow_list_doc_wiki[])list.To_ary_and_clear(Xow_list_doc_wiki.class));
}
@Override protected void Bld_tags(Xoa_app app, Io_url addon_dir, Xopage_html_data page_data) {
Xopg_tag_mgr head_tags = page_data.Head_tags();
@@ -41,13 +41,4 @@ class Xow_list_html extends Xow_special_wtr__base {
Xopg_tag_wtr_.Add__xohelp (head_tags, app.Fsys_mgr().Http_root());
head_tags.Add(Xopg_tag_itm.New_css_file(addon_dir.GenSubFil_nest("bin", "xow_list.css")));
}
private static byte[] Get_root_url() {
byte tid = gplx.core.envs.Op_sys.Cur().Tid();
byte[] rv = Bry_.new_a7("/");
switch (tid) {
case gplx.core.envs.Op_sys.Tid_wnt : rv = Bry_.new_a7("C:\\"); break;
}
rv = gplx.langs.htmls.encoders.Gfo_url_encoder_.Href.Encode(rv);
return rv;
}
}

4
400_xowa/src/gplx/xowa/addons/wikis/searchs/bldrs/Srch_temp_tbl_wkr.java
View File

@@ -82,7 +82,7 @@ class Srch_temp_tbl_wkr implements Srch_text_parser_wkr {
Db_attach_mgr attach_mgr = new Db_attach_mgr();
// dump everything into a temp table in order to index it
page_conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_name("page", "page_ns__page_id", "page_namespace", "page_id"));
page_conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl("page", "page_ns__page_id", "page_namespace", "page_id"));
Srch_db_mgr.Optimize_unsafe_(word_conn, Bool_.Y);
word_conn.Meta_tbl_remake(Dbmeta_tbl_itm.New("search_link_temp", Dbmeta_fld_itm.new_int("word_id"), Dbmeta_fld_itm.new_int("page_id"), Dbmeta_fld_itm.new_int("page_namespace")));
attach_mgr.Conn_main_(word_conn).Conn_others_(new Db_attach_itm("page_db", page_conn));
@@ -98,7 +98,7 @@ class Srch_temp_tbl_wkr implements Srch_text_parser_wkr {
));
word_conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_name("search_link_temp", "main", "page_namespace", "word_id", "page_id"));
Srch_db_mgr.Optimize_unsafe_(word_conn, Bool_.N);
page_conn.Meta_idx_delete("page__page_ns__page_id");
page_conn.Meta_idx_delete("page", "page_ns__page_id");
int len = search_db_mgr.Tbl__link__len();
for (int i = 0; i < len; ++i) {

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/specials/Srch_special_page.java
View File

@@ -86,7 +86,7 @@ public class Srch_special_page implements Xow_special_page, Gfo_invk, Gfo_evt_it
page.Root().Data_htm_(search_page.Root().Data_htm());
Xoa_url redirect_url = Xoa_url.New(wiki, search_ttl);
page.Ttl_(search_ttl).Url_(redirect_url);
page.Redirect().Itms__add__special(redirect_url, search_ttl);
page.Redirect().Itms__add__article(redirect_url, search_ttl, null);
}
}
private void Multi_wikis_changed() {