Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.9.2.1

Browse Source
This commit is contained in:
gnosygnu
2015-09-13 21:54:44 -04:00
parent 2145f6382c
commit 5fe27b5b3b
649 changed files with 4726 additions and 3432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
400_xowa/src/gplx/xowa/bldrs/Xob_sql_dump_base.java Normal file
View File

@@ -0,0 +1,60 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.flds.*; import gplx.ios.*; import gplx.xowa.tdbs.*;
import gplx.xowa.bldrs.sqls.*;
public abstract class Xob_sql_dump_base extends Xob_itm_dump_base implements Xob_cmd, GfoInvkAble {
private final Sql_file_parser parser = new Sql_file_parser(); private boolean fail = false;
public abstract String Cmd_key();
public Io_url Src_fil() {return src_fil;} private Io_url src_fil;
public Io_url_gen Make_url_gen() {return make_url_gen;} private Io_url_gen make_url_gen;
public abstract String Sql_file_name();
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
this.Init_dump(this.Cmd_key());
make_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("make"));
if (src_fil == null) {
src_fil = Xotdb_fsys_mgr.Find_file_or_null(wiki.Fsys_mgr().Root_dir(), "*" + Sql_file_name() + "*", ".gz", ".sql");
if (src_fil == null) {
String msg = String_.Format(".sql file not found in dir.\nPlease download the file for your wiki from dumps.wikimedia.org.\nfile={0} dir={1}", Sql_file_name(), wiki.Fsys_mgr().Root_dir());
app.Usr_dlg().Warn_many("", "", msg);
app.Gui_mgr().Kit().Ask_ok("", "", msg);
fail = true;
return;
}
}
parser.Src_fil_(src_fil).Trg_fil_gen_(dump_url_gen);
Cmd_bgn_hook(bldr, parser);
} protected Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
public abstract void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser);
public void Cmd_run() {
if (fail) return;
parser.Parse(bldr.Usr_dlg());
}
@gplx.Virtual public void Cmd_end() {
if (fail) return;
Xobdc_merger.Basic(bldr.Usr_dlg(), dump_url_gen, temp_dir.GenSubDir("sort"), sort_mem_len, Io_line_rdr_key_gen_all._, new Io_sort_fil_basic(bldr.Usr_dlg(), make_url_gen, make_fil_len));
}
public void Cmd_term() {}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_src_fil_)) src_fil = m.ReadIoUrl("v");
else return super.Invk(ctx, ikey, k, m);
return this;
}
public static final String Invk_src_fil_ = "src_fil_";
}

1
400_xowa/src/gplx/xowa/bldrs/cfgs/Xob_wiki_cfg_bldr.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.strings.*;
import gplx.xowa.wikis.xwikis.cfgs.*;
public class Xob_wiki_cfg_bldr implements GfoInvkAble {
public Xob_wiki_cfg_bldr(Xob_bldr bldr) {this.app = bldr.App();} private Xoae_app app;
public void Exec() {

5
400_xowa/src/gplx/xowa/bldrs/cmds/Xob_dump_mgr_base.java
View File

@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.dbs.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.bldrs.cmds.files.*; import gplx.xowa.files.origs.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*;
public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xob_cmd, GfoInvkAble {
private Xob_dump_src_id page_src;
private Xowd_db_mgr db_fsys_mgr; protected Xop_parser parser; protected Xop_ctx ctx; protected Xop_root_tkn root;
@@ -176,9 +177,7 @@ public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xo
usr_dlg.Note_many("", "", "done: ~{0} ~{1}", exec_count, Decimal_adp_.divide_safe_(exec_count, Env_.TickCount_elapsed_in_sec(time_bgn)).To_str("#,###.000"));
}
private void Free() {
ctx.App().Free_mem(true);
gplx.xowa.xtns.scribunto.Scrib_core.Core_invalidate();
wiki.Cache_mgr().Free_mem_all();
Xow_wiki_.Rls_mem(wiki, true);
}
protected void Reset_db_y_() {this.reset_db = true;}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {

1
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_base.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.brys.*; import gplx.ios.*; import gplx.xowa.ctgs.*;
import gplx.xowa.bldrs.sqls.*;
public abstract class Xob_categorylinks_base extends Xob_sql_dump_base implements Sql_file_parser_cmd {
private DateAdp_parser date_parser = DateAdp_parser.new_(); private Sql_file_parser sql_parser; Uca_trie trie; private Bry_bfr uca_bfr = Bry_bfr.reset_(255);
public abstract Io_sort_cmd Make_sort_cmd(Sql_file_parser sql_parser);

3
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_sql.java
View File

@@ -16,7 +16,8 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.ios.*; import gplx.xowa.bldrs.*;
import gplx.ios.*;
import gplx.xowa.bldrs.sqls.*;
public class Xob_categorylinks_sql extends Xob_categorylinks_base {
private Db_idx_mode idx_mode = Db_idx_mode.Itm_end;
public Xob_categorylinks_sql(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}

2
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_sql_make.java
View File

@@ -17,7 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.flds.*; import gplx.ios.*; import gplx.dbs.*; import gplx.xowa.dbs.*; import gplx.xowa.ctgs.*;
import gplx.xowa.bldrs.*;
import gplx.xowa.bldrs.sqls.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_categorylinks_sql_make implements Io_make_cmd {
private Gfo_usr_dlg usr_dlg; private final Xowe_wiki wiki; private final Db_idx_mode idx_mode; private Xowd_db_mgr core_db_mgr;

1
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_txt.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.ios.*;
import gplx.xowa.bldrs.sqls.*;
public class Xob_categorylinks_txt extends Xob_categorylinks_base {
public Xob_categorylinks_txt(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_text_cat_link;}

1
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_parser_base.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.ios.*;
import gplx.xowa.bldrs.sqls.*;
public abstract class Xoctg_hiddencat_parser_base extends Xob_sql_dump_base implements Sql_file_parser_cmd {
public Xoctg_hiddencat_parser_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb; return this;}
@Override public String Sql_file_name() {return "page_props";}

1
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_parser_sql.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.xowa.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.sqls.*;
import gplx.xowa.wikis.data.*;
public class Xoctg_hiddencat_parser_sql extends Xoctg_hiddencat_parser_base {
private Xowd_cat_core_tbl tbl;

1
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_parser_txt.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.ios.*;
import gplx.xowa.bldrs.sqls.*;
public class Xoctg_hiddencat_parser_txt extends Xoctg_hiddencat_parser_base {
public Xoctg_hiddencat_parser_txt(Xob_bldr bldr, Xowe_wiki wiki) {this.Ctor(bldr, wiki);}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_cat_hidden_sql;}

5
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_temp_wkr.java
View File

@@ -19,7 +19,8 @@ package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gp
import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.pages.*;
import gplx.xowa.files.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.domains.*;
import gplx.xowa.parsers.lnkis.redlinks.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.html.hdumps.bldrs.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.wdatas.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.parsers.lnkis.*; import gplx.xowa.parsers.lnkis.redlinks.*; import gplx.xowa.parsers.xndes.*;
import gplx.xowa.html.hdumps.bldrs.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.wdatas.*;
import gplx.fsdb.meta.*; import gplx.xowa.files.fsdb.*; import gplx.fsdb.*;
public class Xob_lnki_temp_wkr extends Xob_dump_mgr_base implements Xopg_redlink_logger {
private Xob_lnki_temp_tbl tbl; private boolean wdata_enabled = true, xtn_ref_enabled = true, gen_html, gen_hdump;
@@ -60,7 +61,7 @@ public class Xob_lnki_temp_wkr extends Xob_dump_mgr_base implements Xopg_redlink
if (!xtn_ref_enabled) gplx.xowa.xtns.cite.References_nde.Enabled = false;
gplx.xowa.xtns.gallery.Gallery_xnde.Log_wkr = log_mgr.Make_wkr().Save_src_str_(Bool_.Y);
gplx.xowa.xtns.imaps.Imap_xnde.Log_wkr = log_mgr.Make_wkr();
gplx.xowa.Xop_xnde_wkr.Timeline_log_wkr = log_mgr.Make_wkr();
gplx.xowa.parsers.xndes.Xop_xnde_wkr.Timeline_log_wkr = log_mgr.Make_wkr();
gplx.xowa.xtns.scores.Score_xnde.Log_wkr = log_mgr.Make_wkr();
gplx.xowa.xtns.hieros.Hiero_xnde.Log_wkr = log_mgr.Make_wkr();
gplx.xowa.xtns.math.Math_nde.Log_wkr = log_mgr.Make_wkr();

2
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_regy_update_cmd.java
View File

@@ -32,7 +32,7 @@ public class Xob_xfer_regy_update_cmd extends Xob_itm_basic_base implements Xob_
this.Update_status(make_db_provider);
}
private void Copy_atrs_to_make_db(Db_conn make_db_provider) {
wiki.File_mgr().Init_file_mgr_by_load(wiki);
wiki.File_mgr().Init_file_mgr_by_load(wiki); // NOTE: this gets current fsdb to update from xfer_regy; this never uses -prv/ or /prv/ databases; DATE:2015-09-10
Fsm_mnt_itm fsdb_abc_mgr = wiki.File_mgr().Fsdb_mgr().Mnt_mgr().Mnts__get_main(); // 0 = fsdb.main
Db_conn conn = fsdb_abc_mgr.Atr_mgr().Db__core().Conn(); // 0 = fsdb.atr.00
Io_url fsdb_atr_url = ((gplx.dbs.engines.sqlite.Sqlite_conn_info)conn.Conn_info()).Url();

1
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_cmd_orig.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.xowa.dbs.*; import gplx.xowa.files.*; import gplx.xowa.files.exts.*;
import gplx.xowa.parsers.lnkis.*;
public class Xob_xfer_temp_cmd_orig extends Xob_itm_basic_base implements Xob_cmd {
private byte[] ext_rules_key = Bry_.Empty;
public Xob_xfer_temp_cmd_orig(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}

1
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_itm.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.xowa.files.*;
import gplx.xowa.parsers.lnkis.*;
class Xob_xfer_temp_itm {
public int Lnki_id() {return lnki_id;} private int lnki_id;
public int Lnki_tier_id() {return lnki_tier_id;} private int lnki_tier_id;

1
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_itm_tst.java
View File

@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*;
import gplx.stores.*; import gplx.xowa.files.*; import gplx.xowa.files.repos.*;
import gplx.xowa.parsers.lnkis.*;
public class Xob_xfer_temp_itm_tst {
private Xob_xfer_temp_itm_fxt fxt = new Xob_xfer_temp_itm_fxt();
@Before public void init() {fxt.Reset();}

3
400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_page_cmd.java
View File

@@ -18,7 +18,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.dbs.*; import gplx.ios.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.wikis.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.dbs.*;
import gplx.xowa.wikis.*; import gplx.xowa.bldrs.filters.dansguardians.*; import gplx.xowa.apis.xowa.bldrs.imports.*;
import gplx.xowa.wikis.*; import gplx.xowa.bldrs.filters.dansguardians.*; import gplx.xowa.apis.xowa.bldrs.imports.*;
import gplx.xowa.parsers.utils.*;
public class Xob_page_cmd extends Xob_itm_basic_base implements Xobd_wkr, GfoInvkAble {
private Xowd_db_mgr db_mgr; private Db_idx_mode idx_mode = Db_idx_mode.Itm_end; private Xowd_page_tbl page_core_tbl; private Io_stream_zip_mgr text_zip_mgr; private byte text_zip_tid;
private Xop_redirect_mgr redirect_mgr; private Xob_redirect_tbl redirect_tbl; private boolean redirect_id_enabled;

1
400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_pagelinks_parser_cmd.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.sqls.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_pagelinks_parser_cmd extends Xob_sql_dump_base implements Sql_file_parser_cmd {
private Db_conn core_conn;

2
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_calc_stats_cmd.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.xowa.tdbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.tdbs.*; import gplx.xowa.tdbs.hives.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_calc_stats_cmd extends Xob_itm_basic_base implements Xob_cmd {
public Xob_calc_stats_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_tdb_calc_stats;}

1
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_page_txt.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.ios.*; import gplx.xowa.tdbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.parsers.utils.*;
public class Xob_page_txt extends Xob_itm_dump_base implements Xobd_wkr, GfoInvkAble {
public Xob_page_txt(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Wkr_key() {return Xob_cmd_keys.Key_tdb_make_page;}

3
400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_deploy_zip_cmd.java
View File

@@ -16,7 +16,8 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.xowa.bldrs.*; import gplx.xowa.tdbs.*;
import gplx.ios.*; import gplx.xowa.bldrs.sqls.*;
import gplx.xowa.tdbs.*;
public class Xob_deploy_zip_cmd extends Xob_itm_basic_base implements Xob_cmd {
public Xob_deploy_zip_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_deploy_zip;}

1
400_xowa/src/gplx/xowa/bldrs/cmds/wikis/Xob_image_cmd.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.wikis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.ios.*; import gplx.xowa.files.*;
import gplx.xowa.bldrs.sqls.*;
public class Xob_image_cmd extends Xob_itm_dump_base implements Xob_cmd, GfoInvkAble, Sql_file_parser_cmd {
private Db_conn conn = null; private Db_stmt stmt = null;
private Xob_image_tbl tbl_image = new Xob_image_tbl();

1
400_xowa/src/gplx/xowa/bldrs/cmds/wikis/Xob_redirect_cmd.java
View File

@@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.wikis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.xowa.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.parsers.utils.*;
public class Xob_redirect_cmd extends Xob_dump_mgr_base {
private Db_conn conn; private Xob_redirect_tbl redirect_tbl;
private Xodb_mgr_sql db_mgr; private Xop_redirect_mgr redirect_mgr; private Url_encoder encoder;

113
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_base.java Normal file
View File

@@ -0,0 +1,113 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.threads.*; import gplx.xowa.bldrs.*;
import gplx.xowa.wikis.domains.*;
import gplx.xowa.wmfs.dumps.*;
abstract class Xoi_cmd_base implements Gfo_thread_cmd {
public void Ctor(Xoi_setup_mgr install_mgr, String wiki_key) {
this.install_mgr = install_mgr; this.wiki_key = wiki_key;
this.Owner_(install_mgr);
} private Xoi_setup_mgr install_mgr; String wiki_key;
@gplx.Virtual public void Cmd_ctor() {}
public abstract String Async_key();
public int Async_sleep_interval() {return Gfo_thread_cmd_.Async_sleep_interval_1_second;}
public boolean Async_prog_enabled() {return false;}
public void Async_prog_run(int async_sleep_sum) {}
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
public boolean Async_term() {
// install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "import.end", "import.end ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
return true;
}
public GfoInvkAble Owner() {return owner;} public Xoi_cmd_base Owner_(GfoInvkAble v) {owner = v; return this;} GfoInvkAble owner;
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
public void Async_run() {
running = true;
// install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "import.bgn", "import.bgn ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
Thread_adp_.invk_(this.Async_key(), this, Invk_process_async).Start();
}
public boolean Async_running() {return running;} private boolean running;
public void Process_async() {
Xoae_app app = install_mgr.App();
Xob_bldr bldr = app.Bldr();
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_a7(wiki_key));
wiki.Init_assert();
bldr.Cmd_mgr().Clear();
Process_async_init(app, wiki, bldr);
bldr.Pause_at_end_(false);
try {bldr.Run();}
catch (Exception e) {
running = false;
install_mgr.Cmd_mgr().Working_(Bool_.N);
throw Err_.new_exc(e, "xo", "error during import");
}
app.Usr_dlg().Prog_none("", "clear", "");
app.Usr_dlg().Note_none("", "clear", "");
Process_async_done(app, wiki, bldr);
running = false;
}
public abstract void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr);
public abstract void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr);
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_process_async)) Process_async();
else if (ctx.Match(k, Invk_owner)) return owner;
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_process_async = "run_async", Invk_owner = "owner";
}
class Xoi_cmd_category2_page_props extends Xoi_cmd_wiki_download { public Xoi_cmd_category2_page_props(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date) {this.Ctor_download_(install_mgr, wiki_key, dump_date, Xowm_dump_type_.Str__page_props);}
@Override public String Download_file_ext() {return ".sql.gz2";}
public static final String KEY_category2 = "wiki.category2.download.page_props";
}
class Xoi_cmd_category2_categorylinks extends Xoi_cmd_wiki_download { public Xoi_cmd_category2_categorylinks(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date) {this.Ctor_download_(install_mgr, wiki_key, dump_date, Xowm_dump_type_.Str__categorylinks);}
@Override public String Download_file_ext() {return ".sql.gz2";}
public static final String KEY_category2 = "wiki.category2.download.categorylinks";
}
class Xoi_cmd_category2_build extends Xoi_cmd_base {
public Xoi_cmd_category2_build(Xoi_setup_mgr install_mgr, String wiki_key) {this.Ctor(install_mgr, wiki_key); this.app = install_mgr.App(); this.wiki_key = wiki_key;} private Xoae_app app; private String wiki_key;
@Override public void Cmd_ctor() {
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_u8(wiki_key));
wiki.Import_cfg().Category_version_(gplx.xowa.ctgs.Xoa_ctg_mgr.Version_2);
}
@Override public String Async_key() {return KEY;} public static final String KEY = "wiki.category2.build";
@Override public void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
if (app.Setup_mgr().Dump_mgr().Wiki_storage_type_is_sql()) {
wiki.Db_mgr_as_sql().Category_version_update(false);
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_text_cat_core, Xob_cmd_keys.Key_text_cat_link, Xob_cmd_keys.Key_text_cat_hidden);
}
else
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_tdb_cat_hidden_sql, Xob_cmd_keys.Key_tdb_cat_hidden_ttl, Xob_cmd_keys.Key_text_cat_link, Xob_cmd_keys.Key_tdb_ctg_link_idx);
}
@Override public void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
app.Usr_dlg().Prog_many("", "", "category2 setup done");
}
}
class Xoi_cmd_search2_build extends Xoi_cmd_base {
public Xoi_cmd_search2_build(Xoi_setup_mgr install_mgr, String wiki_key) {this.Ctor(install_mgr, wiki_key);}
@Override public String Async_key() {return KEY;} public static final String KEY = "wiki.search2.build";
@Override public void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
if (app.Setup_mgr().Dump_mgr().Wiki_storage_type_is_sql()) {
wiki.Db_mgr_as_sql().Category_version_update(false);
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_text_search_cmd);
}
}
@Override public void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
app.Usr_dlg().Prog_many("", "", "search2 setup done");
wiki.Db_mgr().Search_version_refresh();
}
}

60
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_dumpfile.java Normal file
View File

@@ -0,0 +1,60 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.threads.*;
class Xoi_cmd_dumpfile {
public byte[] Domain() {return domain;} private byte[] domain;
public Io_url Bz2_url() {return bz2_url;} Io_url bz2_url;
public Io_url Xml_url() {return xml_url;} Io_url xml_url;
public boolean Bz2_unzip() {return bz2_unzip;} private boolean bz2_unzip;
public void Clear() {domain = null; bz2_url = xml_url = null; bz2_unzip = false;}
public Xoi_cmd_dumpfile Parse_msg(GfoMsg m) {
Io_url dump_url = m.ReadIoUrl("url");
domain = m.ReadBry("domain");
if (Bry_.Len_eq_0(domain)) domain = Bry_.new_u8(dump_url.OwnerDir().NameOnly());
bz2_unzip = String_.Eq(m.ReadStr("args"), "unzip");
String dump_ext = dump_url.Ext();
if (String_.Eq(dump_ext, ".bz2")) {
bz2_url = dump_url;
if (bz2_unzip) {
xml_url = bz2_url.GenNewExt(""); // remove .bz2 extension (new file path should be .xml)
if (!String_.Eq(xml_url.Ext(), ".xml"))
xml_url = xml_url.GenNewExt(".xml");
}
}
else if (String_.Eq(dump_ext, ".xml")) { // user selected xml file;
bz2_url = null;
xml_url = dump_url;
bz2_unzip = false; // ignore unzip arge
}
return this;
}
public Gfo_thread_cmd Exec(Xoi_cmd_mgr cmd_mgr) {
Xowe_wiki wiki = cmd_mgr.App().Wiki_mgr().Get_by_key_or_make(domain);
if (bz2_unzip) { // unzip requested; add unzip cmd
GfoMsg unzip_msg = GfoMsg_.new_parse_(Gfo_thread_cmd_unzip.KEY).Add("v", Gfo_thread_cmd_unzip.KEY).Add("src", bz2_url.Raw()).Add("trg", xml_url.Raw());
Gfo_thread_cmd_unzip unzip_cmd = (Gfo_thread_cmd_unzip)cmd_mgr.Cmd_add(unzip_msg);
unzip_cmd.Term_cmd_for_src_(Gfo_thread_cmd_unzip.Term_cmd_for_src_noop); // don't do anything with bz2 after unzip
}
if (xml_url == null)
wiki.Import_cfg().Src_fil_bz2_(bz2_url);
else
wiki.Import_cfg().Src_fil_xml_(xml_url);
return cmd_mgr.Dump_add_many_custom(String_.new_u8(domain), "", "", true);
}
}

76
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_dumpfile_tst.java Normal file
View File

@@ -0,0 +1,76 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*;
public class Xoi_cmd_dumpfile_tst {
@Before public void init() {fxt.Clear();} private Xoi_cmd_dumpfile_fxt fxt = new Xoi_cmd_dumpfile_fxt();
@Test public void Bz2__unzip() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml.bz2", "", "unzip")
.Test_domain("en.wikipedia.org")
.Test_vals("mem/en.wikipedia.org/fil.xml.bz2", "mem/en.wikipedia.org/fil.xml", true)
;
}
@Test public void Bz2__unzip__assert_xml_ext() { // xml ext relies on removing ".bz2" from ".xml.bz2"; if just ".bz2" add an ".xml"
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.bz2", "", "unzip")
.Test_vals("mem/en.wikipedia.org/fil.bz2", "mem/en.wikipedia.org/fil.xml", true)
;
}
@Test public void Bz2__direct() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.bz2", "", "")
.Test_vals("mem/en.wikipedia.org/fil.bz2", null, false)
;
}
@Test public void Xml__unzip_n() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml", "", "")
.Test_vals(null, "mem/en.wikipedia.org/fil.xml", false)
;
}
@Test public void Xml__unzip_y() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml", "", "")
.Test_vals(null, "mem/en.wikipedia.org/fil.xml", false)
;
}
}
class Xoi_cmd_dumpfile_fxt {
public void Clear() {
dumpfile.Clear();
} private Xoi_cmd_dumpfile dumpfile = new Xoi_cmd_dumpfile();
public Xoi_cmd_dumpfile_fxt Exec_parse_msg(String url, String domain, String args) {
GfoMsg m = GfoMsg_.new_parse_("").Add("url", url).Add("domain", domain).Add("args", args);
dumpfile.Parse_msg(m);
return this;
}
public Xoi_cmd_dumpfile_fxt Test_vals(String expd_bz2, String expd_xml, boolean expd_unzip) {
Eq_url(expd_bz2, dumpfile.Bz2_url());
Eq_url(expd_xml, dumpfile.Xml_url());
Tfds.Eq(expd_unzip, dumpfile.Bz2_unzip());
return this;
}
public Xoi_cmd_dumpfile_fxt Test_domain(String expd_domain) {
Tfds.Eq(expd_domain, String_.new_u8(dumpfile.Domain()));
return this;
}
private void Eq_url(String expd, Io_url actl) {
if (expd == null && actl == null) return;
else if (expd != null && actl != null) {
Tfds.Eq(expd, actl.Raw());
}
else if (expd == null) throw Err_.new_wo_type("actl should be null", "expd", expd);
else if (actl == null) throw Err_.new_wo_type("actl should not be null", "expd", expd);
}
}

67
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_imageMagick_download.java Normal file
View File

@@ -0,0 +1,67 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.gfui.*;
import gplx.core.threads.*;
class Xoi_cmd_imageMagick_download extends Gfo_thread_cmd_download implements Gfo_thread_cmd {// private static final byte[] Bry_windows_zip = Bry_.new_a7("-windows.zip");
// static final String Src_imageMagick = "ftp://ftp.sunet.se/pub/multimedia/graphics/ImageMagick/binaries/";
public Xoi_cmd_imageMagick_download(Gfo_usr_dlg usr_dlg, Gfui_kit kit, Io_url trg) {this.Ctor(usr_dlg, kit); this.trg = trg;} Io_url trg;
@Override public byte Async_init() { // <a href="ImageMagick-6.8.1-9-Q16-x86-windows.zip">
// byte[] raw = xrg.Exec_as_bry(Src_imageMagick);
// int find_pos = Bry_finder.Find_fwd(raw, Bry_windows_zip); if (find_pos == Bry_.NotFound) return Fail();
// int bgn_pos = Bry_finder.Find_bwd(raw, Byte_ascii.Quote, find_pos); if (bgn_pos == Bry_.NotFound) return Fail();
// ++bgn_pos;
// int end_pos = Bry_finder.Find_fwd(raw, Byte_ascii.Quote, bgn_pos); if (end_pos == Bry_.NotFound) return Fail();
// String src = Src_imageMagick + String_.new_a7(Bry_.Mid(raw, bgn_pos, end_pos));
String src = "http://ftp.sunet.se/pub/multimedia/graphics/ImageMagick/binaries/ImageMagick-6.8.8-1-Q16-x86-windows.zip";
this.Init("downloading", src, trg);
return super.Async_init();
}
byte Fail() {
kit.Ask_ok(GRP_KEY, "windows_not_found", "Could not find Windows binary. Please download ImageMagick directly from the site.");
return Gfo_thread_cmd_.Init_cancel_step;
}
public static final String KEY_imageMagick = "download.imageMagick";
static final String GRP_KEY = "xowa.install.cmds.download.imageMagick";
}
class Xoi_cmd_msg_ok extends Gfo_thread_cmd_base implements Gfo_thread_cmd {
public Xoi_cmd_msg_ok(Gfo_usr_dlg usr_dlg, Gfui_kit kit, String msg) {this.msg = msg; this.Ctor(usr_dlg, kit);} private String msg;
@Override public boolean Async_term() {
kit.Ask_ok("msg_ok", "msg", msg);
return true;
}
public static final String KEY = "msg.ok";
}
// class Gfo_thread_exec_sync : Gfo_thread_cmd_base, Gfo_thread_cmd {
// public Gfo_thread_exec_sync(Gfo_usr_dlg usr_dlg, Gfui_kit kit, Io_url exec_url, String exec_args) {this.Ctor(usr_dlg, kit); this.exec_url = exec_url; this.exec_args = exec_args;} Io_url exec_url; String exec_args;
// public override byte Async_init() {
// if (!kit.Ask_yes_no(GRP_KEY, "confirm", "You are about to run '~{0}'. Please confirm that XOWA requested you to run '~{0}' and that you trust it to run on your machine.", exec_url.NameAndExt())) {
// usr_dlg.Warn_many(GRP_KEY, "confirm.fail", "program was untrusted: ~{0} ~{1}", exec_url.Raw(), exec_args);
// return Gfo_thread_cmd_.Init_cancel_all;
// }
// usr_dlg.Prog_many("exec_sync", "bgn", "running process. please wait; ~{0} ~{1}", exec_url.Raw(), exec_args);
// return Gfo_thread_cmd_.Init_ok;
// }
// public override void Async_run() {
// ProcessAdp process = new ProcessAdp().Exe_url_(exec_url).Args_str_(exec_args).Prog_dlg_(usr_dlg);
// process.Run_mode_(ProcessAdp.Run_mode_sync_block);
// process.Run();
// }
// public static final String KEY = "exec.sync";
// static final String GRP_KEY = "exec.sync";
// }

149
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_mgr.java Normal file
View File

@@ -0,0 +1,149 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.threads.*;
public class Xoi_cmd_mgr implements GfoInvkAble {
List_adp cmds = List_adp_.new_();
public Xoi_cmd_mgr(Xoi_setup_mgr install_mgr) {this.app = install_mgr.App(); this.install_mgr = install_mgr;} private Xoae_app app; Xoi_setup_mgr install_mgr;
public Xoae_app App() {return app;}
public void Canceled_y_() {canceled = true;} private boolean canceled = false;
public boolean Working() {return working;} private boolean working;
public void Working_(boolean v) {
working = v;
app.Bldr__running_(v);
}
private void Process_async(Gfo_thread_cmd cmd) {
byte init_rslt = cmd.Async_init();
if (init_rslt == Gfo_thread_cmd_.Init_ok) {
cmd.Async_run();
int async_sleep_interval = cmd.Async_sleep_interval();
boolean async_prog_enabled = cmd.Async_prog_enabled();
int async_sleep_sum = 0;
while (cmd.Async_running()) {
if (canceled) {this.Working_(Bool_.N); return;}
if (async_prog_enabled) cmd.Async_prog_run(async_sleep_sum);
Thread_adp_.Sleep(async_sleep_interval);
async_sleep_sum += async_sleep_interval; // NOTE: this is not exact
}
}
boolean term_pass = cmd.Async_term();
if (cmd.Async_next_cmd() != null && init_rslt != Gfo_thread_cmd_.Init_cancel_all && term_pass)
Run_async(cmd.Async_next_cmd());
else
this.Working_(Bool_.N);
}
private void Run_async(Gfo_thread_cmd cmd) {Thread_adp_.invk_msg_(cmd.Async_key(), this, GfoMsg_.new_cast_(Invk_process_async).Add("v", cmd)).Start();}
private void Cmds_run() {
if (working) {
app.Gui_mgr().Kit().Ask_ok("", "", "An import is in progress. Please wait for it to complete. If you want to do multiple imports at once, see Help:Import/Script.");
return;
}
int cmds_len = cmds.Count();
if (cmds_len == 0) return;
for (int i = 0; i < cmds_len - 1; i++) {
Gfo_thread_cmd cur_cmd = (Gfo_thread_cmd)cmds.Get_at(i);
Gfo_thread_cmd nxt_cmd = (Gfo_thread_cmd)cmds.Get_at(i + 1);
cur_cmd.Cmd_ctor();
cur_cmd.Async_next_cmd_(nxt_cmd);
}
Gfo_thread_cmd cmd = (Gfo_thread_cmd)cmds.Get_at(0);
cmds.Clear();
this.Working_(Bool_.Y);
app.Bldr__running_(true);
this.Run_async(cmd);
}
Object Dump_add_many(GfoMsg m) {
int args_len = m.Args_count();
if (args_len < 4) throw Err_.new_wo_type("Please provide the following: wiki name, wiki date, dump_type, and one command; EX: ('simple.wikipedia.org', 'latest', 'pages-articles', 'wiki.download')");
String wiki_key = m.Args_getAt(0).Val_to_str_or_empty();
String wiki_date = m.Args_getAt(1).Val_to_str_or_empty();
String dump_type = m.Args_getAt(2).Val_to_str_or_empty();
Gfo_thread_cmd cmd = null;
for (int i = 3; i < args_len; i++) {
KeyVal kv = m.Args_getAt(i);
String kv_val = kv.Val_to_str_or_empty();
if (String_.Eq(kv_val, Wiki_cmd_custom))
return Dump_add_many_custom(wiki_key, wiki_date, dump_type, false);
else {
cmd = Dump_cmd_new(wiki_key, wiki_date, dump_type, kv.Val_to_str_or_empty());
cmds.Add(cmd);
}
}
return cmd; // return last cmd
}
public Gfo_thread_cmd Dump_add_many_custom(String wiki_key, String wiki_date, String dump_type, boolean dumpfile_cmd) {
String[] custom_cmds = app.Setup_mgr().Dump_mgr().Custom_cmds();
int custom_cmds_len = custom_cmds.length;
Gfo_thread_cmd cmd = null;
for (int j = 0; j < custom_cmds_len; j++) {
cmd = Dump_cmd_new(wiki_key, wiki_date, dump_type, custom_cmds[j]);
if (dumpfile_cmd) {
if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_download.Key_wiki_download)) continue; // skip download if wiki.dump_file
else if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_unzip.KEY_dump)) {
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_u8(wiki_key));
if (wiki.Import_cfg().Src_fil_xml() != null) continue; // skip unzip if xml exists
}
else if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_import.KEY)) {
((Xoi_cmd_wiki_import)cmd).Import_move_bz2_to_done_(false);
}
}
cmds.Add(cmd);
}
return cmd;
}
Gfo_thread_cmd Dump_cmd_new(String wiki_key, String wiki_date, String dump_type, String cmd_key) {
if (String_.Eq(cmd_key, Xoi_cmd_wiki_download.Key_wiki_download)) return new Xoi_cmd_wiki_download().Ctor_download_(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_unzip.KEY_dump)) return new Xoi_cmd_wiki_unzip(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_import.KEY)) return new Xoi_cmd_wiki_import(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_zip.KEY)) return new Xoi_cmd_wiki_zip(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_category2_build.KEY)) return new Xoi_cmd_category2_build(install_mgr, wiki_key).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_category2_page_props.KEY_category2)) return new Xoi_cmd_category2_page_props(install_mgr, wiki_key, wiki_date).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_category2_categorylinks.KEY_category2)) return new Xoi_cmd_category2_categorylinks(install_mgr, wiki_key, wiki_date).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_search2_build.KEY)) return new Xoi_cmd_search2_build(install_mgr, wiki_key).Owner_(this);
else throw Err_.new_unhandled(cmd_key);
}
public static final String[] Wiki_cmds_valid = new String[] {Xoi_cmd_wiki_download.Key_wiki_download, Xoi_cmd_wiki_unzip.KEY_dump, Xoi_cmd_wiki_import.KEY, Xoi_cmd_wiki_zip.KEY, Xoi_cmd_category2_build.KEY, Xoi_cmd_category2_page_props.KEY_category2, Xoi_cmd_category2_categorylinks.KEY_category2};
public static final String Wiki_cmd_custom = "wiki.custom", Wiki_cmd_dump_file = "wiki.dump_file";
public Gfo_thread_cmd Cmd_add(GfoMsg m) {Gfo_thread_cmd rv = Cmd_new(m); cmds.Add(rv); return rv;}
Gfo_thread_cmd Cmd_new(GfoMsg m) {
String cmd_key = m.ReadStr("v");
if (String_.Eq(cmd_key, Gfo_thread_cmd_download.KEY)) return new Gfo_thread_cmd_download().Init("downloading", m.ReadStr("src"), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg"))).Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());
else if (String_.Eq(cmd_key, Gfo_thread_cmd_unzip.KEY)) return new Gfo_thread_cmd_unzip().Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Init(app.Usr_dlg(), app.Gui_mgr().Kit(), app.Prog_mgr().App_decompress_bz2(), app.Prog_mgr().App_decompress_zip(), app.Prog_mgr().App_decompress_gz(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("src")), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg")));
else if (String_.Eq(cmd_key, Gfo_thread_cmd_replace.KEY)) return new Gfo_thread_cmd_replace().Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Init(app.Usr_dlg(), app.Gui_mgr().Kit(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("fil")));
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_image_cfg.KEY_dump)) return new Xoi_cmd_wiki_image_cfg(app, Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("fil"))).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_goto_page.KEY)) return new Xoi_cmd_wiki_goto_page(app, m.ReadStr("v")).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_msg_ok.KEY)) return new Xoi_cmd_msg_ok(app.Usr_dlg(), app.Gui_mgr().Kit(), m.ReadStr("v")).Owner_(this);
// else if (String_.Eq(cmd_key, Gfo_thread_exec_sync.KEY)) return new Gfo_thread_exec_sync(app.Usr_dlg(), app.Gui_mgr().Kit(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("url")), m.ReadStr("args")).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_imageMagick_download.KEY_imageMagick)) return new Xoi_cmd_imageMagick_download(app.Usr_dlg(), app.Gui_mgr().Kit(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg"))).Owner_(this);
else if (String_.Eq(cmd_key, Wiki_cmd_dump_file)) return Wiki_cmd_dump_file_make(m);
else throw Err_.new_unhandled(cmd_key);
}
Gfo_thread_cmd Wiki_cmd_dump_file_make(GfoMsg m) {
Xoi_cmd_dumpfile dumpfile = new Xoi_cmd_dumpfile().Parse_msg(m);
return dumpfile.Exec(this);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_process_async)) Process_async((Gfo_thread_cmd)m.CastObj("v"));
else if (ctx.Match(k, Invk_dump_add_many)) return Dump_add_many(m);
else if (ctx.Match(k, Invk_cmd_add)) return Cmd_add(m);
else if (ctx.Match(k, Invk_run)) Cmds_run();
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_process_async = "process_async", Invk_dump_add_many = "dump_add_many", Invk_run = "run", Invk_cmd_add = "cmd_add";
static final String GRP_KEY = "xowa.install_mgr.cmd_mgr";
}

54
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_download.java Normal file
View File

@@ -0,0 +1,54 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.gfui.*;
import gplx.core.threads.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.utils.*;
import gplx.xowa.wmfs.dumps.*;
class Xoi_cmd_wiki_download extends Gfo_thread_cmd_download implements Gfo_thread_cmd { private Xoi_setup_mgr install_mgr; private String wiki_key, dump_date, dump_type;
public Xoi_cmd_wiki_download Ctor_download_(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date, String dump_type) {
this.install_mgr = install_mgr;
this.wiki_key = wiki_key;
this.dump_date = dump_date;
this.dump_type = dump_type;
this.Owner_(install_mgr);
return this;
}
@gplx.Virtual public String Download_file_ext() {return ".xml.bz2";} // wiki.download is primarily used for dump files; default to .xml.bz2; NOTE: changed from ".xml"; DATE:2013-11-07
@Override public String Async_key() {return Key_wiki_download;} public static final String Key_wiki_download = "wiki.download";
@Override public byte Async_init() {
Xoae_app app = install_mgr.App();
Xowm_dump_file dump_file = new Xowm_dump_file(wiki_key, dump_date, dump_type);
boolean connected = Xowm_dump_file_.Connect_first(dump_file, install_mgr.Dump_mgr().Server_urls());
if (connected)
app.Usr_dlg().Note_many("", "", "url: ~{0}", dump_file.File_url());
else {
if (!Dump_servers_offline_msg_shown) {
app.Gui_mgr().Kit().Ask_ok("", "", "all dump servers are offline: ~{0}", String_.AryXtoStr(install_mgr.Dump_mgr().Server_urls()));
Dump_servers_offline_msg_shown = true;
}
}
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(dump_file.Domain_itm().Domain_bry());
Io_url root_dir = wiki.Fsys_mgr().Root_dir();
Io_url[] trg_fil_ary = Io_mgr.I.QueryDir_args(root_dir).FilPath_("*." + dump_type + Download_file_ext() + "*").ExecAsUrlAry();
Io_url trg = trg_fil_ary.length == 0 ? root_dir.GenSubFil(dump_file.File_name()) : trg_fil_ary[0];
this.Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());
this.Init("download", dump_file.File_url(), trg);
return super.Async_init();
}
private static boolean Dump_servers_offline_msg_shown = false;
}

30
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_goto_page.java Normal file
View File

@@ -0,0 +1,30 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.threads.*;
class Xoi_cmd_wiki_goto_page extends Gfo_thread_cmd_base implements Gfo_thread_cmd {
public Xoi_cmd_wiki_goto_page(Xoae_app app, String page) {this.app = app; this.page = page; this.Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());} private Xoae_app app; String page;
@Override public void Async_run() {kit.New_cmd_sync(this).Invk(GfsCtx.new_(), 0, Invk_goto_page, GfoMsg_.Null);}
private void Goto_page(String page) {app.Gui_mgr().Browser_win().Page__navigate_by_url_bar(page);}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_goto_page)) Goto_page(page);
else return super.Invk(ctx, ikey, k, m);
return this;
} private static final String Invk_goto_page = "goto_page";
public static final String KEY = "wiki.goto_page";
}

28
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_image_cfg.java Normal file
View File

@@ -0,0 +1,28 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.threads.*;
class Xoi_cmd_wiki_image_cfg extends Gfo_thread_cmd_replace implements Gfo_thread_cmd { public Xoi_cmd_wiki_image_cfg(Xoae_app app, Io_url url) {this.app = app; super.Init(app.Usr_dlg(), app.Gui_mgr().Kit(), url);} private Xoae_app app;
@Override public void Async_run() {
super.Async_run();
app.Cfg_mgr().Set_by_app("app.files.download.enabled", "y");
app.Cfg_mgr().Db_save_txt();
}
static final String GRP_KEY = "xowa.thread.dump.image_cfg";
public static final String KEY_dump = "wiki.image_cfg";
}

117
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_import.java Normal file
View File

@@ -0,0 +1,117 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.threads.*; import gplx.xowa.bldrs.*; import gplx.xowa.gui.views.*; import gplx.xowa.bldrs.cmds.utils.*;
import gplx.xowa.html.hrefs.*;
class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
public Xoi_cmd_wiki_import(Xoi_setup_mgr install_mgr, String wiki_key, String wiki_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.wiki_date = wiki_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, wiki_date, dump_type;
public static final String KEY = "wiki.import";
public void Cmd_ctor() {}
public String Async_key() {return KEY;}
public int Async_sleep_interval() {return Gfo_thread_cmd_.Async_sleep_interval_1_second;}
public boolean Async_prog_enabled() {return false;}
public void Async_prog_run(int async_sleep_sum) {}
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
public boolean Async_term() {
install_mgr.App().Usr_dlg().Log_many(GRP_KEY, "import.end", "import.end ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
return true;
}
public GfoInvkAble Owner() {return owner;} public Xoi_cmd_wiki_import Owner_(GfoInvkAble v) {owner = v; return this;} GfoInvkAble owner;
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
public void Async_run() {
running = true;
install_mgr.App().Usr_dlg().Log_many(GRP_KEY, "import.bgn", "import.bgn ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
Thread_adp_.invk_(this.Async_key(), this, Invk_process_async).Start();
}
public boolean Async_running() {
return running;
}
boolean running;
public boolean Import_move_bz2_to_done() {return import_move_bz2_to_done;} public Xoi_cmd_wiki_import Import_move_bz2_to_done_(boolean v) {import_move_bz2_to_done = v; return this;} private boolean import_move_bz2_to_done = true;
private void Process_txt(Xob_bldr bldr) {
((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_id);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_search_title);
if (wiki.Import_cfg().Category_version() == gplx.xowa.ctgs.Xoa_ctg_mgr.Version_1)
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_category);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_calc_stats);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_core_term);
}
private void Process_sql(Xob_bldr bldr) {
((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_css);
if (wiki.Import_cfg().Category_version() == gplx.xowa.ctgs.Xoa_ctg_mgr.Version_1) {
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_core_v1);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_core);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_link);
}
if (wiki.Appe().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.specials.search.Xows_page__search.Version_2)
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_search_wkr);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_term);
}
private void Process_async() {
Xoae_app app = install_mgr.App();
app.Usr_dlg().Prog_one("", "", "preparing import: ~{0}", wiki_key);
Xob_bldr bldr = app.Bldr();
wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_a7(wiki_key));
wiki.Init_assert();
bldr.Cmd_mgr().Clear();
bldr.Pause_at_end_(false);
Io_url src_url = wiki.Import_cfg().Src_rdr().Url();
if (install_mgr.Dump_mgr().Wiki_storage_type_is_sql())
Process_sql(bldr);
else
Process_txt(bldr);
bldr.Run();
app.Usr_dlg().Prog_none(GRP_KEY, "clear", ""); app.Usr_dlg().Note_none(GRP_KEY, "clear", "");
app.Usere().Available_from_fsys();
wiki.Init_needed_(true);
wiki.Html_mgr().Page_wtr_mgr().Init_(true);
wiki.Init_assert();
if (String_.Eq(src_url.Ext(), ".xml")) {
if (app.Setup_mgr().Dump_mgr().Delete_xml_file())
Io_mgr.I.DeleteFil(src_url);
}
else if (String_.Eq(src_url.Ext(), ".bz2")) {
Io_url trg_fil = app.Fsys_mgr().Wiki_dir().GenSubFil_nest("#dump", "done", src_url.NameAndExt());
if (import_move_bz2_to_done)
Io_mgr.I.MoveFil_args(src_url, trg_fil, true).Exec();
}
running = false;
wiki.Import_cfg().Src_fil_xml_(null).Src_fil_bz2_(null); // reset file else error when going from Import/Script to Import/List
app.Gui_mgr().Kit().New_cmd_sync(this).Invk(GfsCtx.new_(), 0, Invk_open_wiki, GfoMsg_.Null);
} private Xowe_wiki wiki;
private void Open_wiki(String wiki_key) {
Xog_win_itm main_win = install_mgr.App().Gui_mgr().Browser_win();
if (main_win.Active_page() == null) return; // will be null when invoked through cmd-line
byte[] url = Bry_.Add(wiki.Domain_bry(), Xoh_href_.Bry__wiki, wiki.Props().Main_page());
main_win.Page__navigate_by_url_bar(String_.new_u8(url));
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_process_async)) Process_async();
else if (ctx.Match(k, Invk_owner)) return owner;
else if (ctx.Match(k, Invk_open_wiki)) Open_wiki(wiki_key);
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_process_async = "run_async", Invk_owner = "owner", Invk_open_wiki = "open_wiki";
static final String GRP_KEY = "xowa.thread.op.build";
}

129
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_tst.java Normal file
View File

@@ -0,0 +1,129 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*;
import gplx.core.consoles.*;
import gplx.brys.*; import gplx.core.threads.*; import gplx.xowa.setup.maints.*; import gplx.xowa.xtns.wdatas.imports.*;
import gplx.xowa.wikis.domains.*;
import gplx.xowa.wmfs.*; import gplx.xowa.wmfs.dumps.*;
public class Xoi_cmd_wiki_tst {
@Test public void Run() { // MAINT
// Bld_import_list(Xow_wmf_api_mgr.Wikis);
// Bld_cfg_files(Xow_wmf_api_mgr.Wikis); // NOTE: remember to carry over the wikisource / page / index commands from the existing xowa_build_cfg.gfs; also, only run the xowa_build_cfg.gfs once; DATE:2013-10-15; last run: DATE:2014-09-09
}
public void Bld_import_list(String... ary) {
int ary_len = ary.length;
Bry_bfr bfr = Bry_bfr.reset_(255);
Wmf_latest_parser parser = new Wmf_latest_parser();
Bry_fmtr_arg_time time_fmtr = new Bry_fmtr_arg_time();
for (int i = 0; i < ary_len; i++)
Bld_import_list_itm2(bfr, parser, time_fmtr, ary, i);
Io_mgr.I.SaveFilStr("C:\\temp.txt", bfr.Xto_str());
}
private void Bld_import_list_itm2(Bry_bfr bfr, Wmf_latest_parser parser, Bry_fmtr_arg_time time_fmtr, String[] ary, int i) {
String domain_str = ary[i];
byte[] domain_bry = Bry_.new_a7(domain_str);
Xow_domain_itm domain_itm = Xow_domain_itm_.parse(domain_bry);
byte[] wmf_key_bry = Bry_.Replace(Xow_abrv_wm_.To_abrv(domain_itm), Byte_ascii.Dash, Byte_ascii.Underline);
String wmf_key = String_.new_u8(wmf_key_bry);
String url = "https://dumps.wikimedia.org/" + wmf_key + "/latest";
byte[] latest_html = null;
for (int j = 0; j < 5; ++j) {
latest_html = Io_mgr.I.DownloadFil_args("", Io_url_.Empty).Exec_as_bry(url);
if (latest_html != null) break;
Tfds.Write("fail|" + url);
if (j == 4) return;
}
Tfds.Write("pass|" + url);
parser.Parse(latest_html);
Xowm_dump_file dump_file = new Xowm_dump_file(domain_str, "latest", Xowm_dump_type_.Str__pages_articles);
dump_file.Server_url_(Xowm_dump_file_.Server_wmf_https);
byte[] pages_articles_key = Bry_.new_a7(wmf_key + "-latest-pages-articles.xml.bz2");
Wmf_latest_itm latest_itm = parser.Get_by(pages_articles_key);
bfr.Add(domain_bry).Add_byte_pipe();
bfr.Add_str(dump_file.File_url()).Add_byte_pipe();
bfr.Add(Xow_domain_type_.Get_type_as_bry(domain_itm.Domain_type_id())).Add_byte_pipe();
long src_size = latest_itm.Size();
bfr.Add_long_variable(src_size).Add_byte_pipe();
bfr.Add_str(gplx.ios.Io_size_.To_str(src_size)).Add_byte_pipe();
time_fmtr.Seconds_(Math_.Div_safe_as_long(src_size, 1000000)).XferAry(bfr, 0);
bfr.Add_byte_pipe();
bfr.Add_str(latest_itm.Date().XtoStr_fmt_yyyy_MM_dd_HH_mm());
bfr.Add_byte_pipe();
bfr.Add_str(dump_file.Dump_date());
bfr.Add_byte_nl();
}
/*
private void Bld_import_list_itm(Bry_bfr bfr, Xowm_dump_file dump_file, Bry_fmtr_arg_time time_fmtr, String[] ary, int i) {
String itm = ary[i];
dump_file.Ctor(itm, "latest", Xowm_dump_type_.Str__pages_articles);
int count = 0;
while (count++ < 1) {
dump_file.Server_url_(Xowm_dump_file_.Server_wmf);
if (dump_file.Connect()) break;
Tfds.WriteText(String_.Format("retrying: {0} {1}\n", count, dump_file.File_modified()));
Thread_adp_.Sleep(15000); // wait for connection to reset
}
if (count == 10) {
Tfds.WriteText(String_.Format("failed: {0}\n", dump_file.File_url()));
return;
}
else
Tfds.WriteText(String_.Format("passed: {0}\n", itm));
bfr.Add_str(itm).Add_byte_pipe();
bfr.Add_str(dump_file.File_url()).Add_byte_pipe();
bfr.Add(Xow_domain_type_.Get_type_as_bry(dump_file.Wiki_type().Wiki_tid())).Add_byte_pipe();
// Xol_lang_itm lang_itm = Xol_lang_itm_.Get_by_key(wiki_type.Lang_key());
// if (lang_itm == null) lang_itm = Xol_lang_itm_.Get_by_key(Xol_lang_.Key_en); // commons, species, meta, etc will have no lang
// bfr.Add(lang_itm.Local_name()).Add_byte_pipe();
// bfr.Add(lang_itm.Canonical_name()).Add_byte_pipe();
long src_size = dump_file.File_len();
bfr.Add_long_variable(src_size).Add_byte_pipe();
bfr.Add_str(gplx.ios.Io_size_.To_str(src_size)).Add_byte_pipe();
time_fmtr.Seconds_(Math_.Div_safe_as_long(src_size, 1000000)).XferAry(bfr, 0);
bfr.Add_byte_pipe();
bfr.Add_str(dump_file.File_modified().XtoStr_fmt_yyyy_MM_dd_HH_mm());
bfr.Add_byte_pipe();
// bfr.Add_str(String_.Concat_with_obj(",", (Object[])dump_file.Dump_available_dates()));
// bfr.Add_byte_pipe();
bfr.Add_str(dump_file.Dump_date());
bfr.Add_byte_nl();
Thread_adp_.Sleep(1000);
}
*/
public void Bld_cfg_files(String... ary) {
Bry_bfr bfr = Bry_bfr.reset_(255);
gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_api api = new gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_api();
gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_wiki wiki = new gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_wiki();
int ary_len = ary.length;
for (int i = 0; i < ary_len; i++) {
String wiki_domain = ary[i];
try {
byte[] xml = api.Exec_api(api.Api_src(wiki_domain));
wiki.Wiki_domain_(Bry_.new_a7(wiki_domain));
api.Parse(wiki, String_.new_u8(xml));
api.Build_cfg(bfr, wiki);
}
catch (Exception e) {
Console_adp__sys.I.Write_str_w_nl(Err_.Message_gplx_full(e));
}
}
bfr.Add_str_a7("app.bldr.wiki_cfg_bldr.run;").Add_byte_nl();
Io_mgr.I.SaveFilStr("C:\\xowa_build_cfg.gfs", bfr.Xto_str());
}
}

50
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_unzip.java Normal file
View File

@@ -0,0 +1,50 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.gfui.*;
import gplx.core.threads.*;
class Xoi_cmd_wiki_unzip extends Gfo_thread_cmd_unzip implements Gfo_thread_cmd { public static final String KEY_dump = "wiki.unzip";
public Xoi_cmd_wiki_unzip(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.dump_date = dump_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, dump_date, dump_type;
@Override public String Async_key() {return KEY_dump;}
@Override public byte Async_init() {
Xoae_app app = install_mgr.App(); Gfui_kit kit = app.Gui_mgr().Kit();
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_u8(wiki_key));
Io_url wiki_dir = wiki.Import_cfg().Src_dir();
Io_url[] urls = Io_mgr.I.QueryDir_args(wiki_dir).Recur_(false).FilPath_("*.xml.bz2").ExecAsUrlAry();
if (urls.length == 0) {
kit.Ask_ok(GRP_KEY, "dump.unzip_latest.file_missing", "Could not find a dump file for ~{0} in ~{1}", wiki_key, wiki_dir.Raw());
return Gfo_thread_cmd_.Init_cancel_step;
}
Io_url src = urls[urls.length - 1];
Io_url trg = app.Fsys_mgr().Wiki_dir().GenSubFil_nest(wiki_key, src.NameOnly()); // NOTE: NameOnly() will strip trailing .bz2; EX: a.xml.bz2 -> a.xml
super.Init(app.Usr_dlg(), app.Gui_mgr().Kit(), app.Prog_mgr().App_decompress_bz2(), app.Prog_mgr().App_decompress_zip(), app.Prog_mgr().App_decompress_gz(), src, trg);
this.Term_cmd_for_src_(Term_cmd_for_src_move);
this.Term_cmd_for_src_url_(app.Fsys_mgr().Wiki_dir().GenSubFil_nest("#dump", "done", src.NameAndExt()));
if (Io_mgr.I.ExistsFil(trg)) {
int rslt = kit.Ask_yes_no_cancel(GRP_KEY, "target_exists", "Target file already exists: '~{0}'.\nDo you want to delete it?", trg.Raw());
switch (rslt) {
case Gfui_dlg_msg_.Btn_yes: Io_mgr.I.DeleteFil(trg); break;
case Gfui_dlg_msg_.Btn_no: return Gfo_thread_cmd_.Init_cancel_step;
case Gfui_dlg_msg_.Btn_cancel: return Gfo_thread_cmd_.Init_cancel_all;
default: throw Err_.new_unhandled(rslt);
}
}
return Gfo_thread_cmd_.Init_ok;
}
static final String GRP_KEY = "xowa.thread.dump.unzip";
}

69
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_zip.java Normal file
View File

@@ -0,0 +1,69 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.threads.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.utils.*;
class Xoi_cmd_wiki_zip implements Gfo_thread_cmd {
public Xoi_cmd_wiki_zip(Xoi_setup_mgr install_mgr, String wiki_key, String wiki_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.wiki_date = wiki_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, wiki_date, dump_type;
public static final String KEY = "wiki.zip";
public void Cmd_ctor() {}
public String Async_key() {return KEY;}
public int Async_sleep_interval() {return Gfo_thread_cmd_.Async_sleep_interval_1_second;}
public boolean Async_prog_enabled() {return false;}
public void Async_prog_run(int async_sleep_sum) {}
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
public boolean Async_term() {
wiki.Tdb_fsys_mgr().Scan_dirs();
install_mgr.App().Usr_dlg().Log_many(GRP_KEY, "zip.end", "zip.end ~{0}", wiki_key);
install_mgr.App().Usr_dlg().Prog_many(GRP_KEY, "zip.done", "zip done");
return true;
}
public GfoInvkAble Owner() {return owner;} public Xoi_cmd_wiki_zip Owner_(GfoInvkAble v) {owner = v; return this;} GfoInvkAble owner;
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
public void Async_run() {
running = true;
install_mgr.App().Usr_dlg().Log_many(GRP_KEY, "zip.bgn", "zip.bgn ~{0}", wiki_key);
Thread_adp_.invk_(this.Async_key(), this, Invk_process_async).Start();
}
public boolean Async_running() {
return running;
}
boolean running, delete_dirs_page = true, notify_done = true;
private void Process_async() {
Xoae_app app = install_mgr.App();
Xob_bldr bldr = app.Bldr();
wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_a7(wiki_key));
wiki.Init_assert();
bldr.Cmd_mgr().Clear();
bldr.Pause_at_end_(false);
((Xob_deploy_zip_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_deploy_zip)).Delete_dirs_page_(delete_dirs_page);
bldr.Run();
app.Usr_dlg().Prog_none(GRP_KEY, "clear", "");
app.Usr_dlg().Note_none(GRP_KEY, "clear", "");
running = false;
} private Xowe_wiki wiki;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_process_async)) Process_async();
else if (ctx.Match(k, Invk_owner)) return owner;
else if (ctx.Match(k, Invk_delete_dirs_page_)) delete_dirs_page = m.ReadYn("v");
else if (ctx.Match(k, Invk_notify_done_)) notify_done = m.ReadYn("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_process_async = "run_async", Invk_owner = "owner", Invk_delete_dirs_page_ = "delete_dirs_page_", Invk_notify_done_ = "notify_done_";
private static final String GRP_KEY = "xowa.thread.op.wiki.zip";
}

98
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_dump_mgr.java Normal file
View File

@@ -0,0 +1,98 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.ios.*;
import gplx.xowa.wmfs.dumps.*;
public class Xoi_dump_mgr implements GfoInvkAble {
public String[] Server_urls() {return server_urls;} private String[] server_urls = String_.Ary(Xowm_dump_file_.Server_wmf_https, Xowm_dump_file_.Server_your_org, Xowm_dump_file_.Server_c3sl, Xowm_dump_file_.Server_masaryk);
public String[] Custom_cmds() {return custom_cmds;} private String[] custom_cmds = String_.Ary(Xoi_cmd_wiki_download.Key_wiki_download, Xoi_cmd_wiki_import.KEY);
public byte Data_storage_format() {return data_storage_format;} public Xoi_dump_mgr Data_storage_format_(byte v) {data_storage_format = v; return this;} private byte data_storage_format = gplx.ios.Io_stream_.Tid_gzip;
public long Db_text_max() {return db_text_max;} private long db_text_max = (long)3000 * Io_mgr.Len_mb;
public long Db_categorylinks_max() {return db_categorylinks_max;} private long db_categorylinks_max = (long)3600 * Io_mgr.Len_mb;
public long Db_wikidata_max() {return db_wikidata_max;} private long db_wikidata_max = (long)3600 * Io_mgr.Len_mb;
public byte Wiki_storage_type() {return wiki_storage_type;} private byte wiki_storage_type = Wiki_storage_type_sqlite;
public boolean Wiki_storage_type_is_sql() {return wiki_storage_type == Wiki_storage_type_sqlite;}
public String Db_ns_map() {return db_ns_map;} private String db_ns_map = "Template~Module";
public boolean Css_wiki_update() {return css_wiki_update;} private boolean css_wiki_update = true;
public boolean Css_commons_download() {return css_commons_download;} private boolean css_commons_download = true; // changed from false to true; DATE:2014-10-19
public boolean Delete_xml_file() {return delete_xml_file;} private boolean delete_xml_file = true;
public byte Search_version() {return search_version;} private byte search_version = gplx.xowa.specials.search.Xows_page__search.Version_2;
public boolean Import_bz2_by_stdout() {return import_bz2_by_stdout;} private boolean import_bz2_by_stdout = true;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_server_urls)) return String_.Concat_with_str(",\n", server_urls);
else if (ctx.Match(k, Invk_server_urls_)) server_urls = m.ReadStrAryIgnore("v", ",", "\n");
else if (ctx.Match(k, Invk_custom_cmds)) return String_.Concat_with_str(",", custom_cmds);
else if (ctx.Match(k, Invk_custom_cmds_)) custom_cmds = String_.Ary_filter(m.ReadStrAry("v", ","), Xoi_cmd_mgr.Wiki_cmds_valid);
else if (ctx.Match(k, Invk_data_storage_format)) return Io_stream_.Obsolete_to_str(data_storage_format);
else if (ctx.Match(k, Invk_data_storage_format_)) data_storage_format = Io_stream_.Obsolete_to_tid(m.ReadStr("v"));
else if (ctx.Match(k, Invk_data_storage_format_list)) return Options_data_storage_format_list;
else if (ctx.Match(k, Invk_db_text_max)) return db_text_max / Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_text_max_)) db_text_max = m.ReadLong("v") * Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_categorylinks_max)) return db_categorylinks_max / Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_categorylinks_max_)) db_categorylinks_max = m.ReadLong("v") * Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_wikidata_max)) return db_wikidata_max / Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_wikidata_max_)) db_wikidata_max = m.ReadLong("v") * Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_ns_map)) return db_ns_map;
else if (ctx.Match(k, Invk_db_ns_map_)) db_ns_map = m.ReadStr("v");
else if (ctx.Match(k, Invk_wiki_storage_type)) return Wiki_storage_type_str(wiki_storage_type);
else if (ctx.Match(k, Invk_wiki_storage_type_)) wiki_storage_type = Wiki_storage_type_parse(m.ReadStr("v"));
else if (ctx.Match(k, Invk_wiki_storage_type_list)) return Options_storage_type_list;
else if (ctx.Match(k, Invk_css_wiki_update)) return Yn.Xto_str(css_wiki_update);
else if (ctx.Match(k, Invk_css_wiki_update_)) css_wiki_update = m.ReadYn("v");
else if (ctx.Match(k, Invk_css_commons_download)) return Yn.Xto_str(css_commons_download);
else if (ctx.Match(k, Invk_css_commons_download_)) css_commons_download = m.ReadYn("v");
else if (ctx.Match(k, Invk_delete_xml_file)) return Yn.Xto_str(delete_xml_file);
else if (ctx.Match(k, Invk_delete_xml_file_)) delete_xml_file = m.ReadYn("v");
else if (ctx.Match(k, Invk_search_version)) return Options_search_version_str(search_version);
else if (ctx.Match(k, Invk_search_version_)) search_version = Options_search_version_parse(m.ReadStr("v"));
else if (ctx.Match(k, Invk_search_version_list)) return Options_search_version_list;
else if (ctx.Match(k, Invk_import_bz2_by_stdout)) return Yn.Xto_str(import_bz2_by_stdout);
else if (ctx.Match(k, Invk_import_bz2_by_stdout_)) import_bz2_by_stdout = m.ReadYn("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
}
private static final String Invk_server_urls = "server_urls", Invk_server_urls_ = "server_urls_", Invk_custom_cmds = "custom_cmds", Invk_custom_cmds_ = "custom_cmds_"
, Invk_data_storage_format = "data_storage_format", Invk_data_storage_format_ = "data_storage_format_", Invk_data_storage_format_list = "data_storage_format_list"
, Invk_db_text_max = "db_text_max", Invk_db_text_max_ = "db_text_max_", Invk_db_categorylinks_max = "db_categorylinks_max", Invk_db_categorylinks_max_ = "db_categorylinks_max_", Invk_db_wikidata_max = "db_wikidata_max", Invk_db_wikidata_max_ = "db_wikidata_max_"
, Invk_db_ns_map = "db_ns_map", Invk_db_ns_map_ = "db_ns_map_"
, Invk_wiki_storage_type = "wiki_storage_type", Invk_wiki_storage_type_ = "wiki_storage_type_", Invk_wiki_storage_type_list = "wiki_storage_type_list"
, Invk_css_wiki_update = "css_wiki_update", Invk_css_wiki_update_ = "css_wiki_update_"
, Invk_css_commons_download = "css_commons_download", Invk_css_commons_download_ = "css_commons_download_"
, Invk_delete_xml_file = "delete_xml_file", Invk_delete_xml_file_ = "delete_xml_file_"
, Invk_search_version = "search_version", Invk_search_version_ = "search_version_", Invk_search_version_list = "search_version_list"
, Invk_import_bz2_by_stdout = "import_bz2_by_stdout", Invk_import_bz2_by_stdout_ = "import_bz2_by_stdout_"
;
private static KeyVal[] Options_data_storage_format_list = KeyVal_.Ary(KeyVal_.new_(".xdat", "text"), KeyVal_.new_(".gz", "gzip"), KeyVal_.new_(".bz2", "bzip2")); // removed .zip; DATE:2014-05-13; updated aliases; DATE:2014-06-20
static final byte Wiki_storage_type_xdat = 1, Wiki_storage_type_sqlite = 2;
private static final KeyVal[] Options_storage_type_list = KeyVal_.Ary(KeyVal_.new_("sqlite"), KeyVal_.new_("xdat")); // DEPRECATED: KeyVal_.new_("xdat"); DATE:2015-03-30
public static String Wiki_storage_type_str(byte v) {
switch (v) {
case Xoi_dump_mgr.Wiki_storage_type_xdat : return "xdat";
case Xoi_dump_mgr.Wiki_storage_type_sqlite : return "sqlite";
default : throw Err_.new_unhandled(v);
}
}
public static byte Wiki_storage_type_parse(String v) {
if (String_.Eq(v, "xdat")) return Xoi_dump_mgr.Wiki_storage_type_xdat;
else if (String_.Eq(v, "sqlite")) return Xoi_dump_mgr.Wiki_storage_type_sqlite;
else throw Err_.new_unhandled(v);
}
private static final KeyVal[] Options_search_version_list = KeyVal_.Ary(KeyVal_.new_("1"), KeyVal_.new_("2"));
public static String Options_search_version_str(byte v) {return Byte_.To_str(v);}
public static byte Options_search_version_parse(String v) {return Byte_.parse(v);}
}

47
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_mirror_parser.java Normal file
View File

@@ -0,0 +1,47 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public class Xoi_mirror_parser {
public String[] Parse(String raw_str) {
if (String_.Len_eq_0(raw_str)) return String_.Ary_empty;
byte[] raw = Bry_.new_u8(raw_str);
List_adp rv = List_adp_.new_();
int pos = 0;
while (true) {
int bgn = Bry_finder.Find_fwd(raw, CONST_href_bgn, pos); if (bgn == Bry_.NotFound) break;
bgn += CONST_href_bgn.length;
int end = Bry_finder.Find_fwd(raw, CONST_href_end, bgn); if (end == Bry_.NotFound) return String_.Ary_empty;
byte[] date = Bry_.Mid(raw, bgn, end);
pos = end + CONST_href_end.length;
if (Bry_.Match(date, CONST_date_parent_dir)) continue;
int date_pos_last = date.length - 1;
if (date_pos_last == -1) return String_.Ary_empty;
if (date[date_pos_last] == Byte_ascii.Slash) date = Bry_.Mid(date, 0, date_pos_last); // trim trailing /; EX: "20130101/" -> "20130101"
rv.Add(String_.new_u8(date));
}
return rv.To_str_ary();
} static final byte[] CONST_href_bgn = Bry_.new_a7("<a href=\""), CONST_href_end = Bry_.new_a7("\""), CONST_date_parent_dir = Bry_.new_a7("../");
public static String Find_last_lte(String[] ary, String comp) { // assuming sorted ary, find last entry that is lte comp
int len = ary.length;
for (int i = len - 1; i > -1; i--) {
String itm = ary[i];
if (CompareAble_.Is_lessOrSame(itm, comp)) return itm;
}
return "";
}
}

60
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_mirror_parser_tst.java Normal file
View File

@@ -0,0 +1,60 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*;
public class Xoi_mirror_parser_tst {
@Test public void Basic() {
Tst_parse(String_.Concat_lines_nl
( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
, "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.1//EN\" \"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd\">"
, "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\">"
, "<head>"
, "<title>Index of /simplewiki/</title>"
, "<link rel=\"stylesheet\" type=\"text/css\" href=\"/pub/misc/lighttpd-white-dir.css\" />"
, "</head>"
, "<body>"
, "<h2>Index of /simplewiki/</h2>"
, "<div class=\"list\">"
, "<table summary=\"Directory Listing\" cellpadding=\"0\" cellspacing=\"0\">"
, "<thead><tr><th class=\"n\">Name</th><th class=\"m\">Last Modified</th><th class=\"s\">Size</th><th class=\"t\">Type</th></tr></thead>"
, "<tbody>"
, "<tr><td class=\"n\"><a href=\"../\">Parent Directory</a>/</td><td class=\"m\">&nbsp;</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"20120516/\">20120516</a>/</td><td class=\"m\">2012-May-17 01:04:39</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"20121220/\">20121220</a>/</td><td class=\"m\">2012-Dec-20 20:15:55</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"20130214/\">20130214</a>/</td><td class=\"m\">2013-Feb-14 06:28:41</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"latest/\">latest</a>/</td><td class=\"m\">2013-Feb-14 06:28:41</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "</tbody>"
, "</table>"
, "</div>"
, "<div class=\"foot\">lighttpd</div>"
, "</body>"
, "</html>"
), String_.Ary("20120516", "20121220", "20130214", "latest"));
}
@Test public void Find_last_lte() {
Tst_find_last_lte(String_.Ary("20120516", "20121220", "20130214", "latest"), "20130101", "20121220");
Tst_find_last_lte(String_.Ary("20120516", "20121220", "20130214", "latest"), "20120101", "");
}
private void Tst_parse(String raw, String[] expd) {
Xoi_mirror_parser parser = new Xoi_mirror_parser();
Tfds.Eq_ary_str(expd, parser.Parse(raw));
}
private void Tst_find_last_lte(String[] ary, String comp, String expd) {
Tfds.Eq(expd, Xoi_mirror_parser.Find_last_lte(ary, comp));
}
}

44
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_setup_mgr.java Normal file
View File

@@ -0,0 +1,44 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.gfui.*; import gplx.xowa.setup.addons.*;
import gplx.xowa.setup.maints.*;
public class Xoi_setup_mgr implements GfoInvkAble {
public Xoi_setup_mgr(Xoae_app app) {
this.app = app;
cmd_mgr = new Xoi_cmd_mgr(this);
maint_mgr = new Xoa_maint_mgr(app);
}
public void Init_by_app(Xoae_app app) {
addon_mgr.Init_by_app(app);
}
public Xoae_app App() {return app;} private Xoae_app app;
public Xoi_cmd_mgr Cmd_mgr() {return cmd_mgr;} private Xoi_cmd_mgr cmd_mgr;
public Xoi_dump_mgr Dump_mgr() {return dump_mgr;} private Xoi_dump_mgr dump_mgr = new Xoi_dump_mgr();
public Xoi_addon_mgr Addon_mgr() {return addon_mgr;} private Xoi_addon_mgr addon_mgr = new Xoi_addon_mgr();
public Xoa_maint_mgr Maint_mgr() {return maint_mgr;} private Xoa_maint_mgr maint_mgr;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_cmds)) return cmd_mgr;
else if (ctx.Match(k, Invk_dumps)) return dump_mgr;
else if (ctx.Match(k, Invk_addons)) return addon_mgr;
else if (ctx.Match(k, Invk_maint)) return maint_mgr;
else return GfoInvkAble_.Rv_unhandled;
}
static final String Invk_cmds = "cmds", Invk_dumps = "dumps", Invk_addons = "addons", Invk_maint = "maint";
static final String GRP_KEY = "xowa.setup";
}

86
400_xowa/src/gplx/xowa/bldrs/installs/Xow_cfg_wiki_core.java Normal file
View File

@@ -0,0 +1,86 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public class Xow_cfg_wiki_core {
public Xow_cfg_wiki_core(Xowe_wiki wiki) {this.wiki = wiki;} private Xowe_wiki wiki;
public void Save() {
}
public byte[] Build_gfs() {
Gfs_bldr wtr = new Gfs_bldr();
Xow_wiki_props props = wiki.Props();
wtr.Add_proc_init_many(Xowe_wiki.Invk_props).Add_nl();
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_bldr_version_).Add_parens_str(props.Bldr_version()).Add_nl();
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_main_page_).Add_parens_str(props.Main_page()).Add_nl();
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_siteinfo_misc_).Add_parens_str(props.Siteinfo_misc()).Add_nl();
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_siteinfo_mainpage_).Add_parens_str(props.Siteinfo_mainpage()).Add_nl();
wtr.Add_term_nl();
wtr.Add_proc_init_many(Xowe_wiki.Invk_ns_mgr).Add_nl();
wtr.Add_proc_cont_one(Xow_ns_mgr.Invk_clear).Add_nl();
wtr.Add_proc_cont_one(Xow_ns_mgr.Invk_load).Add_paren_bgn().Add_nl();
wtr.Add_quote_xtn_bgn();
Xol_csv_parser csv_parser = Xol_csv_parser._;
int nms_len = wiki.Ns_mgr().Count();
for (int i = 0; i < nms_len; i++) {
Xow_ns ns = wiki.Ns_mgr().Ords_get_at(i);
wtr.Bfr().Add_int_variable(ns.Id()).Add_byte_pipe().Add_int_fixed(ns.Case_match(), 1).Add_byte_pipe();
csv_parser.Save(wtr.Bfr(), ns.Name_txt());
wtr.Add_nl();
}
wtr.Add_quote_xtn_end();
wtr.Add_paren_end().Add_term_nl();
return wtr.Xto_bry();
}
public void Load(String raw) {
wiki.Appe().Gfs_mgr().Run_str_for(wiki, raw);
}
public static void Load_ns_(Xow_ns_mgr ns_mgr, byte[] src) {// 10|1|Template
int len = src.length; int pos = 0, fld_bgn = 0, fld_idx = 0, row_bgn = 0;
int cur_id = Int_.Min_value; byte cur_case_match = Byte_.Max_value_127; byte[] cur_name = Bry_.Empty;
Xol_csv_parser csv_parser = Xol_csv_parser._;
while (true) {
boolean last = pos == len;
byte b = last ? Byte_ascii.Nl : src[pos];
switch (b) {
case Byte_ascii.Pipe:
switch (fld_idx) {
case 0: cur_id = Bry_.To_int_or(src, fld_bgn, pos, Int_.Min_value); if (cur_id == Int_.Min_value) throw Err_.new_wo_type("failed to load id", "id", String_.new_u8(src, fld_bgn, pos)); break;
case 1: cur_case_match = Bry_.To_int_as_byte(src, fld_bgn, pos, Byte_.Max_value_127); if (cur_id == Byte_.Max_value_127) throw Err_.new_wo_type("failed to load match", "id", String_.new_u8(src, fld_bgn, pos)); break;
default: throw Err_.new_unhandled(fld_idx);
}
fld_bgn = pos + 1;
++fld_idx;
break;
case Byte_ascii.Nl:
if (pos > row_bgn) { // guard against blank lines
cur_name = csv_parser.Load(src, fld_bgn, pos);
ns_mgr.Add_new(cur_id, cur_name, cur_case_match, false);
}
cur_id = Int_.Min_value; cur_case_match = Byte_.Max_value_127;
fld_bgn = pos + 1;
fld_idx = 0;
row_bgn = fld_bgn;
break;
}
if (last) break;
++pos;
}
ns_mgr.Init_w_defaults();
}
static final String Url_wiki_core_gfs = "wiki_core.gfs";
static final String GRP_KEY = "xowa.wiki.cfg.wiki_core";
}

111
400_xowa/src/gplx/xowa/bldrs/installs/Xow_cfg_wiki_core_tst.java Normal file
View File

@@ -0,0 +1,111 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*;
public class Xow_cfg_wiki_core_tst {
Xow_cfg_wiki_core_fxt fxt = new Xow_cfg_wiki_core_fxt();
@Before public void init() {fxt.Clear();}
@Test public void Save() {
fxt.Save_tst(Xoa_app_.Version, "Main_Page", "Wikipedia|MediaWiki 1.21wmf5|first-letter|", 0, "User test", Const_wiki_core_cfg);
}
@Test public void Load_and_save() {
fxt.Load_and_save_tst(Const_wiki_core_cfg);
}
@Test public void Load() {
fxt.Load_tst(String_.Concat_lines_nl
( "ns_mgr.clear.load("
, "<:['"
, "0|0|" // DEFECT: test that 0 sets case_match to case-sensitive; empty name was causing it to "default" to 1; DATE:2013-01-30
, "" // test to make sure blank line doesn't fail
, "4|1|Wikipedia" // test to make sure that values are updated after blank line
, ""
, "']:>"
, ");"
)
, fxt.ns_(Xow_ns_.Id_main , true, "")
, fxt.ns_(Xow_ns_.Id_project , false, "Wikipedia")
);
}
public static final String Const_wiki_core_cfg = String_.Concat_lines_nl
( "props"
, ".bldr_version_('" + Xoa_app_.Version + "')"
, ".main_page_('Main_Page')"
, ".siteinfo_misc_('Wikipedia|MediaWiki 1.21wmf5|first-letter|')"
, ".siteinfo_mainpage_('')"
, ";"
, "ns_mgr"
, ".clear"
, ".load("
, "<:['"
, "-2|1|Media"
, "-1|1|Special"
, "0|1|"
, "1|1|Talk"
, "2|0|User test" // NOTE: intentionally changing this to "0|User test" to differ from existing
, "3|1|User talk"
, "4|1|Wikipedia"
, "5|1|Wikipedia talk"
, "6|1|File"
, "7|1|File talk"
, "8|1|MediaWiki"
, "9|1|MediaWiki talk"
, "10|1|Template"
, "11|1|Template talk"
, "12|1|Help"
, "13|1|Help talk"
, "14|1|Category"
, "15|1|Category talk"
, "100|1|Portal"
, "101|1|Portal talk"
, "108|1|Book"
, "109|1|Book talk"
, "828|1|Module"
, "829|1|Module talk"
, "']:>"
, ");"
);
}
class Xow_cfg_wiki_core_fxt {
Xoae_app app; Xowe_wiki wiki;
public Xowe_wiki Wiki() {return wiki;}
public void Clear() {
app = Xoa_app_fxt.app_();
wiki = Xoa_app_fxt.wiki_tst_(app);
}
public void Save_tst(String bldr_version, String main_page, String siteinfo_misc, int ns_user_case_match, String ns_user_name, String expd) {
wiki.Props().Bldr_version_(Bry_.new_a7(bldr_version)).Main_page_(Bry_.new_a7(main_page)).Siteinfo_misc_(Bry_.new_a7(siteinfo_misc));
Xow_ns ns_user = wiki.Ns_mgr().Ids_get_or_null(Xow_ns_.Id_user);
ns_user.Case_match_((byte)ns_user_case_match); ns_user.Name_bry_(Bry_.new_a7(ns_user_name));
Tfds.Eq_str_lines(expd, String_.new_a7(wiki.Cfg_wiki_core().Build_gfs()));
}
public void Load_and_save_tst(String raw) {
wiki.Cfg_wiki_core().Load(raw);
Tfds.Eq_str_lines(raw, String_.new_a7(wiki.Cfg_wiki_core().Build_gfs()));
}
public Xow_ns ns_(int id, boolean case_match, String name) {return new Xow_ns(id, case_match ? Xow_ns_case_.Id_all : Xow_ns_case_.Id_1st, Bry_.new_u8(name), false);}
public void Load_tst(String raw, Xow_ns... expd_ary) {
wiki.Cfg_wiki_core().Load(raw);
int expd_len = expd_ary.length;
for (int i = 0; i < expd_len; i++) {
Xow_ns expd = expd_ary[i];
Xow_ns actl = wiki.Ns_mgr().Ids_get_or_null(expd.Id());
Tfds.Eq(expd.Case_match(), actl.Case_match(), Int_.Xto_str(expd.Id()));
Tfds.Eq(expd.Name_str(), actl.Name_str(), Int_.Xto_str(expd.Id()));
}
}
}

3
400_xowa/src/gplx/xowa/bldrs/langs/Xobc_utl_make_lang.java
View File

@@ -16,7 +16,8 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.langs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.xowa.apps.fsys.*; import gplx.xowa.langs.*;
import gplx.xowa.apps.fsys.*; import gplx.xowa.apps.langs.*;
import gplx.xowa.langs.*;
public class Xobc_utl_make_lang implements GfoInvkAble {
private final Xoa_lang_mgr lang_mgr; private final Xoa_fsys_mgr fsys_mgr; Xol_mw_lang_parser lang_parser;
public Xobc_utl_make_lang(Xoa_lang_mgr lang_mgr, Xoa_fsys_mgr fsys_mgr, Gfo_msg_log msg_log) {

2
400_xowa/src/gplx/xowa/bldrs/langs/Xobc_utl_make_lang_kwds.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.langs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.xowa.langs.*;
import gplx.xowa.langs.*; import gplx.xowa.apps.langs.*;
public class Xobc_utl_make_lang_kwds implements GfoInvkAble, Xol_lang_transform {
private final Xoa_lang_mgr lang_mgr;
public Xobc_utl_make_lang_kwds(Xoa_lang_mgr lang_mgr) {this.lang_mgr = lang_mgr;}

2
400_xowa/src/gplx/xowa/bldrs/langs/Xol_mw_lang_parser_tst.java
View File

@@ -17,7 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.langs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*;
import gplx.intl.*;
import gplx.intl.*; import gplx.xowa.parsers.lnkis.*;
public class Xol_mw_lang_parser_tst {
@Before public void init() {fxt.Clear();} private Xol_mw_lang_parser_fxt fxt = new Xol_mw_lang_parser_fxt();
@Test public void Core_keywords() {

162
400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser.java Normal file
View File

@@ -0,0 +1,162 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.flds.*; import gplx.ios.*;
public class Sql_file_parser {
public Io_url Src_fil() {return src_fil;} public Sql_file_parser Src_fil_(Io_url v) {src_fil = v; return this;} Io_url src_fil;
public int Src_len() {return src_len;} public Sql_file_parser Src_len_(int v) {src_len = v; return this;} private int src_len = 8 * Io_mgr.Len_mb;
public Io_url_gen Trg_fil_gen() {return trg_fil_gen;} public Sql_file_parser Trg_fil_gen_(Io_url_gen v) {trg_fil_gen = v; return this;} Io_url_gen trg_fil_gen;
public int Trg_len() {return trg_len;} public Sql_file_parser Trg_len_(int v) {trg_len = v; return this;} private int trg_len = 4 * Io_mgr.Len_mb;
private Sql_fld_itm[] flds_all; private int flds_all_len;
Gfo_fld_rdr sql_parser = Gfo_fld_rdr.sql_(); Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
static final byte Mode_sql_bgn = 0, Mode_row_bgn = 1, Mode_row_end = 2, Mode_fld = 3, Mode_quote = 4, Mode_escape = 5;
public Sql_file_parser Fld_cmd_(Sql_file_parser_cmd v) {this.fld_cmd = v; return this;} Sql_file_parser_cmd fld_cmd;
public Sql_file_parser Flds_req_(byte[]... v) {flds_req = v; return this;} private byte[][] flds_req;
public Sql_file_parser Flds_req_idx_(int flds_all_len, int... idxs) {
new_flds_all(flds_all_len);
int len = idxs.length;
for (int i = 0; i < len; i++) {
int idx = idxs[i];
Sql_fld_itm itm = new Sql_fld_itm(idx, Bry_.Empty);
flds_all[idx] = itm;
}
return this;
}
private void Identify_flds(byte[] raw) {
Sql_fld_mgr fld_mgr = new Sql_fld_mgr().Parse(raw);
new_flds_all(fld_mgr.Count());
int len = flds_req.length;
for (int i = 0; i < len; i++) {
byte[] fld = flds_req[i];
Sql_fld_itm itm = fld_mgr.Get_by_key(fld); if (itm == null) throw Err_.new_wo_type("could not find field", "fld", fld);
flds_all[itm.Idx()] = itm;
}
}
private void new_flds_all(int len) {
this.flds_all_len = len; // NOTE: must set flds_all_len, else Commit_fld will not be correct;
this.flds_all = new Sql_fld_itm[len];
}
public void Parse(Gfo_usr_dlg usr_dlg) {
Io_buffer_rdr rdr = Io_buffer_rdr.Null;
try {
rdr = Io_buffer_rdr.new_(gplx.ios.Io_stream_rdr_.new_by_url_(src_fil), src_len);
Bry_bfr fil_bfr = Bry_bfr.new_(), val_bfr = Bry_bfr.new_();
byte[] bfr = rdr.Bfr(); int bfr_len = rdr.Bfr_len(), fld_idx = 0, cur_pos = 0;
if (flds_req != null) Identify_flds(bfr);
byte mode = Mode_sql_bgn; byte[] decode_regy = sql_parser.Escape_decode();
boolean reading_file = true; byte mode_prv = Mode_sql_bgn;
Sql_file_parser_data data = new Sql_file_parser_data();
while (reading_file) {
if (cur_pos + 256 > bfr_len && rdr.Fil_pos() != rdr.Fil_len()) { // buffer 256 characters; can be 0, but erring on side of simplicity
rdr.Bfr_load_from(cur_pos);
cur_pos = 0;
bfr = rdr.Bfr();
bfr_len = rdr.Bfr_len();
}
if (cur_pos == bfr_len) break;
byte b = bfr[cur_pos];
switch (mode) {
case Mode_sql_bgn:
cur_pos = Bry_finder.Find_fwd(bfr, Bry_insert_into, cur_pos);
if (cur_pos == Bry_.NotFound || cur_pos > bfr_len) {reading_file = false; continue;}
cur_pos = Bry_finder.Find_fwd(bfr, Bry_values, cur_pos);
if (cur_pos == Bry_.NotFound || cur_pos > bfr_len) throw Err_.new_wo_type("VALUES not found"); // something went wrong;
mode = Mode_fld;
cur_pos += Bry_values.length;
break;
case Mode_row_bgn:
switch (b) {
case Byte_ascii.Paren_bgn: mode = Mode_fld; break;
default: throw Err_.new_unhandled(mode);
}
++cur_pos;
break;
case Mode_row_end:
switch (b) {
case Byte_ascii.Nl: break; // ignore \n
case Byte_ascii.Comma: mode = Mode_row_bgn; break;
case Byte_ascii.Semic: mode = Mode_sql_bgn; break;
default: throw Err_.new_unhandled(mode);
}
++cur_pos;
break;
case Mode_fld:
switch (b) {
case Byte_ascii.Apos: mode = Mode_quote; break; // NOTE: never escape apos by doubling; will fail for empty fields; EX: ", '', ''"; DATE:2013-07-06
case Byte_ascii.Backslash: mode_prv = mode; mode = Mode_escape; break;
default: val_bfr.Add_byte(b); break;
case Byte_ascii.Space: case Byte_ascii.Nl: break;
case Byte_ascii.Comma:
Commit_fld(fld_idx++, val_bfr, fil_bfr, data);
break;
case Byte_ascii.Paren_end:
Commit_fld(fld_idx++, val_bfr, fil_bfr, data);
if (!data.Cancel_row())
Commit_row(usr_dlg, fil_bfr);
fld_idx = 0;
mode = Mode_row_end;
break;
}
++cur_pos;
break;
case Mode_quote:
switch (b) {
case Byte_ascii.Apos: mode = Mode_fld; break;
case Byte_ascii.Backslash: mode_prv = mode; mode = Mode_escape; break;
default: val_bfr.Add_byte(b); break;
}
++cur_pos;
break;
case Mode_escape:
byte escape_val = decode_regy[b];
if (escape_val == Byte_ascii.Null) {val_bfr.Add_byte(Byte_ascii.Backslash).Add_byte(b);}
else val_bfr.Add_byte(escape_val);
mode = mode_prv;
++cur_pos;
break;
default: throw Err_.new_unhandled(mode);
}
}
Io_mgr.I.AppendFilByt(trg_fil_gen.Nxt_url(), fil_bfr.Xto_bry_and_clear());
}
finally {rdr.Rls();}
}
private void Commit_row(Gfo_usr_dlg usr_dlg, Bry_bfr fil_bfr) {
fil_bfr.Add_byte(Byte_ascii.Nl);
if (fil_bfr.Len() > trg_len) {
Io_url trg_fil = trg_fil_gen.Nxt_url();
usr_dlg.Prog_one(GRP_KEY, "make", "making ~{0}", trg_fil.NameAndExt());
Io_mgr.I.AppendFilByt(trg_fil, fil_bfr.Xto_bry_and_clear());
}
}
private void Commit_fld(int fld_idx, Bry_bfr val_bfr, Bry_bfr fil_bfr, Sql_file_parser_data data) {
Sql_fld_itm fld = fld_idx < flds_all_len ? flds_all[fld_idx] : null; // handle new flds added by MW, but not supported by XO; EX:hiddencat and pp_sortkey; DATE:2014-04-28
if (fld != null) {
data.Cancel_row_n_();
if (fld_cmd == null) { // no custom cmd; assume append;
fld_wtr.Bfr_(fil_bfr);
fld_wtr.Write_bry_escape_fld(val_bfr.Bfr(), 0, val_bfr.Len());
}
else
fld_cmd.Exec(val_bfr.Bfr(), fld.Key(), fld_idx, 0, val_bfr.Len(), fil_bfr, data);
}
val_bfr.Clear();
}
private static final byte[] Bry_insert_into = Bry_.new_a7("INSERT INTO "), Bry_values = Bry_.new_a7(" VALUES (");
private static final String GRP_KEY = "xowa.bldr.sql";
}

43
400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_cmd.java Normal file
View File

@@ -0,0 +1,43 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.strings.*;
public interface Sql_file_parser_cmd {
void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data);
}
class Sql_file_parser_cmd_max_len implements Sql_file_parser_cmd {
public void Log_len_(int v) {log_len = v;} private int log_len = 141;
public void Log_print(Io_url url) {
String_bldr sb = String_bldr_.new_();
for (int i = 0; i < log.Count(); i++) {
String itm = (String)log.Get_at(i);
sb.Add(String_.Len(itm) + "|" + itm + "\n");
}
Io_mgr.I.SaveFilStr(url, sb.To_str());
}
public int Max_len() {return max_len;} private int max_len;
public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) {
int fld_len = fld_end - fld_bgn;
if (fld_len > max_len) max_len = fld_len;
if (fld_len > log_len) {
log.Add(String_.new_u8(src, fld_bgn, fld_end));
}
file_bfr.Add_mid(src, fld_bgn, fld_end).Add_byte(Byte_ascii.Pipe);
}
List_adp log = List_adp_.new_();
}

23
400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_data.java Normal file
View File

@@ -0,0 +1,23 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public class Sql_file_parser_data {
public boolean Cancel_row() {return cancel_row;}
public Sql_file_parser_data Cancel_row_n_() {cancel_row = false; return this;}
public Sql_file_parser_data Cancel_row_y_() {cancel_row = true; return this;} private boolean cancel_row;
}

73
400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_tst.java Normal file
View File

@@ -0,0 +1,73 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*; import gplx.ios.*;
public class Sql_file_parser_tst {
Sql_file_parser_fxt fxt = new Sql_file_parser_fxt();
@Before public void init() {fxt.Clear();}
@Test public void One() {
fxt.Init_flds_req_idx_(2, 1).Test_parse("INSERT INTO 'tbl_1' VALUES (1,2);", "2|\n");
}
@Test public void Many() {
fxt.Init_flds_req_idx_(2, 1).Test_parse("INSERT INTO 'tbl_1' VALUES (1,2),(3,4),(5,6);", "2|\n4|\n6|\n");
}
@Test public void Quote_basic() {
fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a','b');", "a|b|\n");
}
@Test public void Escape_pipe() {
fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a|b','c');", "a~pb|c|\n");
}
@Test public void Escape_nl() {
fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a\\nb','c');", "a~nb|c|\n");
}
@Test public void Escape_tab() {
fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a\\tb','c');", "a~tb|c|\n");
}
@Test public void Escape_backslash() {
fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a\\\\b','c');", "a\\b|c|\n");
}
@Test public void Escape_quote() {
fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a\"b','c');", "a\"b|c|\n");
}
@Test public void Fld_paren_end() {
fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'Психостимуляторы_(лекарственные_средства)','c');", "Психостимуляторы_(лекарственные_средства)|c|\n");
}
@Test public void Insert_multiple() {
fxt.Init_flds_req_idx_(2, 1).Test_parse("INSERT INTO 'tbl_1' VALUES (1,2);INSERT INTO 'tbl_1' VALUES (3,4)", "2|\n4|\n");
}
@Test public void Cmds() {
Sql_file_parser_cmd_max_len cmd = new Sql_file_parser_cmd_max_len();
fxt.Init_flds_req_idx_(2, 1).Init_cmd_(cmd).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a'),(3,'abc');", "a|\nabc|\n");
Tfds.Eq(3, cmd.Max_len());
}
}
class Sql_file_parser_fxt {
Sql_file_parser parser = new Sql_file_parser().Src_len_(Io_mgr.Len_kb).Trg_len_(Io_mgr.Len_kb);
public Sql_file_parser_fxt Clear() {Io_mgr.I.InitEngine_mem(); return this;}
public Sql_file_parser_fxt Init_flds_req_idx_(int flds_all_len, int... idxs) {parser.Flds_req_idx_(flds_all_len, idxs); return this;}
public Sql_file_parser_fxt Init_cmd_(Sql_file_parser_cmd cmd) {parser.Fld_cmd_(cmd); return this;}
public void Test_parse(String raw_str, String expd) {
Io_url src_fil = Io_url_.new_fil_("mem/test.sql");
Io_mgr.I.SaveFilBry(src_fil, Bry_.new_u8(raw_str));
Io_url trg_fil = Io_url_.new_fil_("mem/test.csv");
parser.Src_fil_(src_fil).Trg_fil_gen_(Io_url_gen_.fil_(trg_fil));
parser.Parse(Gfo_usr_dlg_.Test());
byte[] actl = Io_mgr.I.LoadFilBry(trg_fil);
Tfds.Eq(expd, String_.new_u8(actl));
}
}

60
400_xowa/src/gplx/xowa/bldrs/sqls/Sql_fld_mgr.java Normal file
View File

@@ -0,0 +1,60 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.ios.*;
class Sql_fld_mgr {
public int Count() {return hash.Count();}
public Sql_fld_itm Get_by_key(String fld) {return Get_by_key(Bry_.new_u8(fld));}
public Sql_fld_itm Get_by_key(byte[] fld) {
return (Sql_fld_itm)hash.Get_by(fld);
} private Ordered_hash hash = Ordered_hash_.new_bry_();
public Sql_fld_mgr Parse(byte[] raw) {
hash.Clear();
int bgn = Bry_finder.Find_fwd(raw, Tkn_create_table); if (bgn == Bry_.NotFound) throw Err_.new_wo_type("could not find 'CREATE TABLE'");
bgn = Bry_finder.Find_fwd(raw, Byte_ascii.Nl, bgn); if (bgn == Bry_.NotFound) throw Err_.new_wo_type("could not find new line after 'CREATE TABLE'");
bgn += Int_.Const_position_after_char;
int end = Bry_finder.Find_fwd(raw, Tkn_unique_index); if (end == Bry_.NotFound) throw Err_.new_wo_type("could not find 'UNIQUE KEY'");
end = Bry_finder.Find_bwd(raw, Byte_ascii.Nl, end); if (bgn == Bry_.NotFound) throw Err_.new_wo_type("could not find new line before 'UNIQUE KEY'");
Parse_lines(Bry_.Mid(raw, bgn, end));
return this;
}
private void Parse_lines(byte[] raw) {
byte[][] lines = Bry_.Split(raw, Byte_ascii.Nl);
int lines_len = lines.length;
int fld_idx = 0;
for (int i = 0; i < lines_len; i++) {
byte[] line = lines[i];
int bgn = Bry_finder.Find_fwd(line, Byte_ascii.Tick); if (bgn == Bry_.NotFound) continue; // skip blank lines
bgn += Int_.Const_position_after_char;
int end = Bry_finder.Find_fwd(line, Byte_ascii.Tick, bgn); if (end == Bry_.NotFound) continue; // skip blank lines
byte[] key = Bry_.Mid(line, bgn, end);
Sql_fld_itm fld = new Sql_fld_itm(fld_idx++, key);
hash.Add(fld.Key(), fld);
}
}
private static final byte[]
Tkn_create_table = Bry_.new_a7("CREATE TABLE")
, Tkn_unique_index = Bry_.new_a7("UNIQUE KEY")
;
public static final int Not_found = -1;
}
class Sql_fld_itm {
public Sql_fld_itm(int idx, byte[] key) {this.idx = idx; this.key = key;}
public int Idx() {return idx;} private int idx;
public byte[] Key() {return key;} private byte[] key;
}

47
400_xowa/src/gplx/xowa/bldrs/sqls/Sql_fld_mgr_tst.java Normal file
View File

@@ -0,0 +1,47 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*;
public class Sql_fld_mgr_tst {
Sql_fld_mgr_fxt fxt = new Sql_fld_mgr_fxt();
@Test public void Basic() {
fxt.Exec_parse(String_.Concat_lines_nl
( "ignore"
, "CREATE TABLE tbl_0 ("
, " `fld_2` int,"
, " `fld_1` int,"
, " `fld_0` int,"
, " UNIQUE KEY idx_0 (fld_2)"
, ");"
));
fxt.Test_count(3);
fxt.Exec_get("fld_0", 2);
fxt.Exec_get("fld_1", 1);
fxt.Exec_get("fld_2", 0);
fxt.Exec_get("fld_3", -1);
}
}
class Sql_fld_mgr_fxt {
Sql_fld_mgr fld_mgr = new Sql_fld_mgr();
public void Exec_parse(String v) {fld_mgr.Parse(Bry_.new_a7(v));}
public void Exec_get(String key, int expd) {
Sql_fld_itm actl_itm = fld_mgr.Get_by_key(key);
Tfds.Eq(expd, actl_itm == null ? Sql_fld_mgr.Not_found : actl_itm.Idx());
}
public void Test_count(int expd) {Tfds.Eq(expd, fld_mgr.Count());}
}