Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Embeddable: Create core dbs in proper subdirectory

Browse Source
This commit is contained in:
gnosygnu
2017-10-23 20:50:50 -04:00
parent 1336d44f34
commit 66877212bf
4537 changed files with 311750 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_base.java
View File

@@ -13,3 +13,89 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.threads.*; import gplx.xowa.bldrs.*;
import gplx.xowa.wikis.domains.*;
import gplx.xowa.bldrs.wms.dumps.*;
abstract class Xoi_cmd_base implements Gfo_thread_cmd {
public void Ctor(Xoi_setup_mgr install_mgr, String wiki_key) {
this.install_mgr = install_mgr; this.wiki_key = wiki_key;
this.Owner_(install_mgr);
} private Xoi_setup_mgr install_mgr; String wiki_key;
@gplx.Virtual public void Cmd_ctor() {}
public abstract String Async_key();
public int Async_sleep_interval() {return Gfo_thread_cmd_.Async_sleep_interval_1_second;}
public boolean Async_prog_enabled() {return false;}
public void Async_prog_run(int async_sleep_sum) {}
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
public boolean Async_term() {return true;}
public Gfo_invk Owner() {return owner;} public Xoi_cmd_base Owner_(Gfo_invk v) {owner = v; return this;} Gfo_invk owner;
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
public void Async_run() {
running = true;
Thread_adp_.Start_by_key(this.Async_key(), this, Invk_process_async);
}
public boolean Async_running() {return running;} private boolean running;
public void Process_async() {
Xoae_app app = install_mgr.App();
Xob_bldr bldr = app.Bldr();
Xowe_wiki wiki = app.Wiki_mgr().Get_by_or_make(Bry_.new_a7(wiki_key));
wiki.Init_assert();
bldr.Cmd_mgr().Clear();
Process_async_init(app, wiki, bldr);
bldr.Pause_at_end_(false);
try {bldr.Run();}
catch (Exception e) {
running = false;
install_mgr.Cmd_mgr().Working_(Bool_.N);
throw Err_.new_exc(e, "xo", "error during import");
}
app.Usr_dlg().Prog_none("", "clear", "");
app.Usr_dlg().Note_none("", "clear", "");
Process_async_done(app, wiki, bldr);
running = false;
}
public abstract void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr);
public abstract void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr);
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_process_async)) Process_async();
else if (ctx.Match(k, Invk_owner)) return owner;
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk_process_async = "run_async", Invk_owner = "owner";
}
class Xoi_cmd_category2_page_props extends Xoi_cmd_wiki_download { public Xoi_cmd_category2_page_props(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date) {this.Ctor_download_(install_mgr, wiki_key, dump_date, Xowm_dump_type_.Str__page_props);}
@Override public String Download_file_ext() {return ".sql.gz";}
public static final String KEY_category2 = "wiki.category2.download.page_props";
}
class Xoi_cmd_category2_categorylinks extends Xoi_cmd_wiki_download { public Xoi_cmd_category2_categorylinks(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date) {this.Ctor_download_(install_mgr, wiki_key, dump_date, Xowm_dump_type_.Str__categorylinks);}
@Override public String Download_file_ext() {return ".sql.gz";}
public static final String KEY_category2 = "wiki.category2.download.categorylinks";
}
class Xoi_cmd_category2_build extends Xoi_cmd_base {
public Xoi_cmd_category2_build(Xoi_setup_mgr install_mgr, String wiki_key) {this.Ctor(install_mgr, wiki_key); this.app = install_mgr.App(); this.wiki_key = wiki_key;} private Xoae_app app; private String wiki_key;
@Override public void Cmd_ctor() {
Xowe_wiki wiki = app.Wiki_mgr().Get_by_or_make(Bry_.new_u8(wiki_key));
wiki.Import_cfg().Category_version_(gplx.xowa.addons.wikis.ctgs.Xoa_ctg_mgr.Version_2);
}
@Override public String Async_key() {return KEY;} public static final String KEY = "wiki.category2.build";
@Override public void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
wiki.Db_mgr_as_sql().Category_version_update(false);
bldr.Cmd_mgr().Add_many(wiki, gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd.BLDR_CMD_KEY, gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd.BLDR_CMD_KEY);
}
@Override public void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
app.Usr_dlg().Prog_many("", "", "category2 setup done");
}
}
class Xoi_cmd_search2_build extends Xoi_cmd_base {
public Xoi_cmd_search2_build(Xoi_setup_mgr install_mgr, String wiki_key) {this.Ctor(install_mgr, wiki_key);}
@Override public String Async_key() {return KEY;} public static final String KEY = "wiki.search2.build";
@Override public void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
wiki.Db_mgr_as_sql().Category_version_update(false);
gplx.xowa.addons.wikis.searchs.bldrs.Srch_bldr_mgr_.Setup(wiki);
}
@Override public void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
app.Usr_dlg().Prog_many("", "", "search2 setup done");
// wiki.Db_mgr().Search_version_refresh();
}
}

43
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_dumpfile.java
View File

@@ -13,3 +13,46 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.threads.*;
class Xoi_cmd_dumpfile {
public byte[] Domain() {return domain;} private byte[] domain;
public Io_url Bz2_url() {return bz2_url;} Io_url bz2_url;
public Io_url Xml_url() {return xml_url;} Io_url xml_url;
public boolean Bz2_unzip() {return bz2_unzip;} private boolean bz2_unzip;
public void Clear() {domain = null; bz2_url = xml_url = null; bz2_unzip = false;}
public Xoi_cmd_dumpfile Parse_msg(GfoMsg m) {
Io_url dump_url = m.ReadIoUrl("url");
domain = m.ReadBry("domain");
if (Bry_.Len_eq_0(domain)) domain = Bry_.new_u8(dump_url.OwnerDir().NameOnly());
bz2_unzip = String_.Eq(m.ReadStr("args"), "unzip");
String dump_ext = dump_url.Ext();
if (String_.Eq(dump_ext, ".bz2")) {
bz2_url = dump_url;
if (bz2_unzip) {
xml_url = bz2_url.GenNewExt(""); // remove .bz2 extension (new file path should be .xml)
if (!String_.Eq(xml_url.Ext(), ".xml"))
xml_url = xml_url.GenNewExt(".xml");
}
}
else if (String_.Eq(dump_ext, ".xml")) { // user selected xml file;
bz2_url = null;
xml_url = dump_url;
bz2_unzip = false; // ignore unzip arge
}
return this;
}
public Gfo_thread_cmd Exec(Xoi_cmd_mgr cmd_mgr) {
Xowe_wiki wiki = cmd_mgr.App().Wiki_mgr().Get_by_or_make(domain);
if (bz2_unzip) { // unzip requested; add unzip cmd
GfoMsg unzip_msg = GfoMsg_.new_parse_(Gfo_thread_cmd_unzip.KEY).Add("v", Gfo_thread_cmd_unzip.KEY).Add("src", bz2_url.Raw()).Add("trg", xml_url.Raw());
Gfo_thread_cmd_unzip unzip_cmd = (Gfo_thread_cmd_unzip)cmd_mgr.Cmd_add(unzip_msg);
unzip_cmd.Term_cmd_for_src_(Gfo_thread_cmd_unzip.Term_cmd_for_src_noop); // don't do anything with bz2 after unzip
}
if (xml_url == null)
wiki.Import_cfg().Src_fil_bz2_(bz2_url);
else
wiki.Import_cfg().Src_fil_xml_(xml_url);
return cmd_mgr.Dump_add_many_custom(String_.new_u8(domain), "", "", true);
}
}

59
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_dumpfile_tst.java
View File

@@ -13,3 +13,62 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*;
public class Xoi_cmd_dumpfile_tst {
@Before public void init() {fxt.Clear();} private Xoi_cmd_dumpfile_fxt fxt = new Xoi_cmd_dumpfile_fxt();
@Test public void Bz2__unzip() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml.bz2", "", "unzip")
.Test_domain("en.wikipedia.org")
.Test_vals("mem/en.wikipedia.org/fil.xml.bz2", "mem/en.wikipedia.org/fil.xml", true)
;
}
@Test public void Bz2__unzip__assert_xml_ext() { // xml ext relies on removing ".bz2" from ".xml.bz2"; if just ".bz2" add an ".xml"
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.bz2", "", "unzip")
.Test_vals("mem/en.wikipedia.org/fil.bz2", "mem/en.wikipedia.org/fil.xml", true)
;
}
@Test public void Bz2__direct() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.bz2", "", "")
.Test_vals("mem/en.wikipedia.org/fil.bz2", null, false)
;
}
@Test public void Xml__unzip_n() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml", "", "")
.Test_vals(null, "mem/en.wikipedia.org/fil.xml", false)
;
}
@Test public void Xml__unzip_y() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml", "", "")
.Test_vals(null, "mem/en.wikipedia.org/fil.xml", false)
;
}
}
class Xoi_cmd_dumpfile_fxt {
public void Clear() {
dumpfile.Clear();
} private Xoi_cmd_dumpfile dumpfile = new Xoi_cmd_dumpfile();
public Xoi_cmd_dumpfile_fxt Exec_parse_msg(String url, String domain, String args) {
GfoMsg m = GfoMsg_.new_parse_("").Add("url", url).Add("domain", domain).Add("args", args);
dumpfile.Parse_msg(m);
return this;
}
public Xoi_cmd_dumpfile_fxt Test_vals(String expd_bz2, String expd_xml, boolean expd_unzip) {
Eq_url(expd_bz2, dumpfile.Bz2_url());
Eq_url(expd_xml, dumpfile.Xml_url());
Tfds.Eq(expd_unzip, dumpfile.Bz2_unzip());
return this;
}
public Xoi_cmd_dumpfile_fxt Test_domain(String expd_domain) {
Tfds.Eq(expd_domain, String_.new_u8(dumpfile.Domain()));
return this;
}
private void Eq_url(String expd, Io_url actl) {
if (expd == null && actl == null) return;
else if (expd != null && actl != null) {
Tfds.Eq(expd, actl.Raw());
}
else if (expd == null) throw Err_.new_wo_type("actl should be null", "expd", expd);
else if (actl == null) throw Err_.new_wo_type("actl should not be null", "expd", expd);
}
}

32
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_imageMagick_download.java
View File

@@ -13,3 +13,35 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.gfui.*; import gplx.gfui.kits.core.*;
import gplx.core.threads.*;
class Xoi_cmd_imageMagick_download extends Gfo_thread_cmd_download implements Gfo_thread_cmd {// private static final byte[] Bry_windows_zip = Bry_.new_a7("-windows.zip");
// static final String Src_imageMagick = "ftp://ftp.sunet.se/pub/multimedia/graphics/ImageMagick/binaries/";
public Xoi_cmd_imageMagick_download(Gfo_usr_dlg usr_dlg, Gfui_kit kit, Io_url trg) {this.Ctor(usr_dlg, kit); this.trg = trg;} Io_url trg;
@Override public byte Async_init() { // <a href="ImageMagick-6.8.1-9-Q16-x86-windows.zip">
// byte[] raw = xrg.Exec_as_bry(Src_imageMagick);
// int find_pos = Bry_find_.Find_fwd(raw, Bry_windows_zip); if (find_pos == Bry_find_.Not_found) return Fail();
// int bgn_pos = Bry_find_.Find_bwd(raw, Byte_ascii.Quote, find_pos); if (bgn_pos == Bry_find_.Not_found) return Fail();
// ++bgn_pos;
// int end_pos = Bry_find_.Find_fwd(raw, Byte_ascii.Quote, bgn_pos); if (end_pos == Bry_find_.Not_found) return Fail();
// String src = Src_imageMagick + String_.new_a7(Bry_.Mid(raw, bgn_pos, end_pos));
String src = "http://ftp.sunet.se/pub/multimedia/graphics/ImageMagick/binaries/ImageMagick-6.8.8-1-Q16-x86-windows.zip";
this.Init("downloading", src, trg);
return super.Async_init();
}
byte Fail() {
kit.Ask_ok(GRP_KEY, "windows_not_found", "Could not find Windows binary. Please download ImageMagick directly from the site.");
return Gfo_thread_cmd_.Init_cancel_step;
}
public static final String KEY_imageMagick = "download.imageMagick";
static final String GRP_KEY = "xowa.install.cmds.download.imageMagick";
}
class Xoi_cmd_msg_ok extends Gfo_thread_cmd_base implements Gfo_thread_cmd {
public Xoi_cmd_msg_ok(Gfo_usr_dlg usr_dlg, Gfui_kit kit, String msg) {this.msg = msg; this.Ctor(usr_dlg, kit);} private String msg;
@Override public boolean Async_term() {
kit.Ask_ok("msg_ok", "msg", msg);
return true;
}
public static final String KEY = "msg.ok";
}

130
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_mgr.java
View File

@@ -13,3 +13,133 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.brys.fmtrs.*; import gplx.core.threads.*;
public class Xoi_cmd_mgr implements Gfo_invk {
List_adp cmds = List_adp_.New();
public Xoi_cmd_mgr(Xoi_setup_mgr install_mgr) {this.app = install_mgr.App(); this.install_mgr = install_mgr;} private Xoae_app app; Xoi_setup_mgr install_mgr;
public Xoae_app App() {return app;}
public void Canceled_y_() {canceled = true;} private boolean canceled = false;
public boolean Working() {return working;} private boolean working;
public void Working_(boolean v) {
working = v;
app.Bldr__running_(v);
}
private void Process_async(Gfo_thread_cmd cmd) {
byte init_rslt = cmd.Async_init();
if (init_rslt == Gfo_thread_cmd_.Init_ok) {
cmd.Async_run();
int async_sleep_interval = cmd.Async_sleep_interval();
boolean async_prog_enabled = cmd.Async_prog_enabled();
int async_sleep_sum = 0;
while (cmd.Async_running()) {
if (canceled) {this.Working_(Bool_.N); return;}
if (async_prog_enabled) cmd.Async_prog_run(async_sleep_sum);
Thread_adp_.Sleep(async_sleep_interval);
async_sleep_sum += async_sleep_interval; // NOTE: this is not exact
}
}
boolean term_pass = cmd.Async_term();
if (cmd.Async_next_cmd() != null && init_rslt != Gfo_thread_cmd_.Init_cancel_all && term_pass)
Run_async(cmd.Async_next_cmd());
else
this.Working_(Bool_.N);
}
private void Run_async(Gfo_thread_cmd cmd) {Thread_adp_.Start_by_val(cmd.Async_key(), this, Invk_process_async, cmd);}
private void Cmds_run() {
if (working) {
app.Gui_mgr().Kit().Ask_ok("", "", "An import is in progress. Please wait for it to complete. If you want to do multiple imports at once, see Dashboard/Import/Offline."); // HOME
return;
}
int cmds_len = cmds.Count();
if (cmds_len == 0) return;
for (int i = 0; i < cmds_len - 1; i++) {
Gfo_thread_cmd cur_cmd = (Gfo_thread_cmd)cmds.Get_at(i);
Gfo_thread_cmd nxt_cmd = (Gfo_thread_cmd)cmds.Get_at(i + 1);
cur_cmd.Cmd_ctor();
cur_cmd.Async_next_cmd_(nxt_cmd);
}
Gfo_thread_cmd cmd = (Gfo_thread_cmd)cmds.Get_at(0);
cmds.Clear();
this.Working_(Bool_.Y);
app.Bldr__running_(true);
this.Run_async(cmd);
}
Object Dump_add_many(GfoMsg m) {
int args_len = m.Args_count();
if (args_len < 4) throw Err_.new_wo_type("Please provide the following: wiki name, wiki date, dump_type, and one command; EX: ('simple.wikipedia.org', 'latest', 'pages-articles', 'wiki.download')");
String wiki_key = m.Args_getAt(0).Val_to_str_or_empty();
String wiki_date = m.Args_getAt(1).Val_to_str_or_empty();
String dump_type = m.Args_getAt(2).Val_to_str_or_empty();
Gfo_thread_cmd cmd = null;
for (int i = 3; i < args_len; i++) {
Keyval kv = m.Args_getAt(i);
String kv_val = kv.Val_to_str_or_empty();
if (String_.Eq(kv_val, Wiki_cmd_custom))
return Dump_add_many_custom(wiki_key, wiki_date, dump_type, false);
else {
cmd = Dump_cmd_new(wiki_key, wiki_date, dump_type, kv.Val_to_str_or_empty());
cmds.Add(cmd);
}
}
return cmd; // return last cmd
}
public Gfo_thread_cmd Dump_add_many_custom(String wiki_key, String wiki_date, String dump_type, boolean dumpfile_cmd) {
String[] custom_cmds = (app.Cfg().Get_bool_app_or("xowa.bldr.import.unzip_bz2_file", false)) // CFG: Cfg__
? String_.Ary(Xoi_cmd_wiki_download.Key_wiki_download, Xoi_cmd_wiki_unzip.KEY_dump, Xoi_cmd_wiki_import.KEY)
: String_.Ary(Xoi_cmd_wiki_download.Key_wiki_download, Xoi_cmd_wiki_import.KEY);
int custom_cmds_len = custom_cmds.length;
Gfo_thread_cmd cmd = null;
for (int j = 0; j < custom_cmds_len; j++) {
cmd = Dump_cmd_new(wiki_key, wiki_date, dump_type, custom_cmds[j]);
if (dumpfile_cmd) {
if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_download.Key_wiki_download)) continue; // skip download if wiki.dump_file
else if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_unzip.KEY_dump)) {
Xowe_wiki wiki = app.Wiki_mgr().Get_by_or_make(Bry_.new_u8(wiki_key));
if (wiki.Import_cfg().Src_fil_xml() != null) continue; // skip unzip if xml exists
}
else if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_import.KEY)) {
((Xoi_cmd_wiki_import)cmd).Import_move_bz2_to_done_(false);
}
}
cmds.Add(cmd);
}
return cmd;
}
Gfo_thread_cmd Dump_cmd_new(String wiki_key, String wiki_date, String dump_type, String cmd_key) {
if (String_.Eq(cmd_key, Xoi_cmd_wiki_download.Key_wiki_download)) return new Xoi_cmd_wiki_download().Ctor_download_(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_unzip.KEY_dump)) return new Xoi_cmd_wiki_unzip(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_import.KEY)) return new Xoi_cmd_wiki_import(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_category2_build.KEY)) return new Xoi_cmd_category2_build(install_mgr, wiki_key).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_category2_page_props.KEY_category2)) return new Xoi_cmd_category2_page_props(install_mgr, wiki_key, wiki_date).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_category2_categorylinks.KEY_category2)) return new Xoi_cmd_category2_categorylinks(install_mgr, wiki_key, wiki_date).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_search2_build.KEY)) return new Xoi_cmd_search2_build(install_mgr, wiki_key).Owner_(this);
else throw Err_.new_unhandled(cmd_key);
}
public static final String Wiki_cmd_custom = "wiki.custom", Wiki_cmd_dump_file = "wiki.dump_file";
public Gfo_thread_cmd Cmd_add(GfoMsg m) {Gfo_thread_cmd rv = Cmd_clone(m); cmds.Add(rv); return rv;}
Gfo_thread_cmd Cmd_clone(GfoMsg m) {
String cmd_key = m.ReadStr("v");
if (String_.Eq(cmd_key, Gfo_thread_cmd_download.KEY)) return new Gfo_thread_cmd_download().Init("downloading", m.ReadStr("src"), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg"))).Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());
else if (String_.Eq(cmd_key, Gfo_thread_cmd_unzip.KEY)) return new Gfo_thread_cmd_unzip().Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Init(app.Usr_dlg(), app.Gui_mgr().Kit(), app.Prog_mgr().App_decompress_bz2(), app.Prog_mgr().App_decompress_zip(), app.Prog_mgr().App_decompress_gz(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("src")), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg")));
else if (String_.Eq(cmd_key, Gfo_thread_cmd_replace.KEY)) return new Gfo_thread_cmd_replace().Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Init(app.Usr_dlg(), app.Gui_mgr().Kit(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("fil")));
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_goto_page.KEY)) return new Xoi_cmd_wiki_goto_page(app, m.ReadStr("v")).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_msg_ok.KEY)) return new Xoi_cmd_msg_ok(app.Usr_dlg(), app.Gui_mgr().Kit(), m.ReadStr("v")).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_imageMagick_download.KEY_imageMagick)) return new Xoi_cmd_imageMagick_download(app.Usr_dlg(), app.Gui_mgr().Kit(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg"))).Owner_(this);
else if (String_.Eq(cmd_key, Wiki_cmd_dump_file)) return Wiki_cmd_dump_file_make(m);
else throw Err_.new_unhandled(cmd_key);
}
Gfo_thread_cmd Wiki_cmd_dump_file_make(GfoMsg m) { // note: might be used directly in home-wiki pages to download files
Xoi_cmd_dumpfile dumpfile = new Xoi_cmd_dumpfile().Parse_msg(m);
return dumpfile.Exec(this);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_process_async)) Process_async((Gfo_thread_cmd)m.CastObj("v"));
else if (ctx.Match(k, Invk_dump_add_many)) return Dump_add_many(m);
else if (ctx.Match(k, Invk_cmd_add)) return Cmd_add(m);
else if (ctx.Match(k, Invk_run)) Cmds_run();
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk_process_async = "process_async", Invk_dump_add_many = "dump_add_many", Invk_run = "run", Invk_cmd_add = "cmd_add";
static final String GRP_KEY = "xowa.install_mgr.cmd_mgr";
}

38
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_download.java
View File

@@ -13,3 +13,41 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.gfui.*;
import gplx.core.threads.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.utils.*;
import gplx.xowa.bldrs.wms.dumps.*;
class Xoi_cmd_wiki_download extends Gfo_thread_cmd_download implements Gfo_thread_cmd { private Xoi_setup_mgr install_mgr; private String wiki_key, dump_date, dump_type;
public Xoi_cmd_wiki_download Ctor_download_(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date, String dump_type) {
this.install_mgr = install_mgr;
this.wiki_key = wiki_key;
this.dump_date = dump_date;
this.dump_type = dump_type;
this.Owner_(install_mgr);
return this;
}
@gplx.Virtual public String Download_file_ext() {return ".xml.bz2";} // wiki.download is primarily used for dump files; default to .xml.bz2; NOTE: changed from ".xml"; DATE:2013-11-07
@Override public String Async_key() {return Key_wiki_download;} public static final String Key_wiki_download = "wiki.download";
@Override public byte Async_init() {
Xoae_app app = install_mgr.App();
Xowm_dump_file dump_file = new Xowm_dump_file(wiki_key, dump_date, dump_type);
String[] server_urls = gplx.xowa.bldrs.installs.Xoi_dump_mgr.Server_urls(app);
boolean connected = Xowm_dump_file_.Connect_first(dump_file, server_urls);
if (connected)
app.Usr_dlg().Note_many("", "", "url: ~{0}", dump_file.File_url());
else {
if (!Dump_servers_offline_msg_shown) {
app.Gui_mgr().Kit().Ask_ok("", "", "all dump servers are offline: ~{0}", String_.AryXtoStr(server_urls));
Dump_servers_offline_msg_shown = true;
}
}
Xowe_wiki wiki = app.Wiki_mgr().Get_by_or_make(dump_file.Domain_itm().Domain_bry());
Io_url root_dir = wiki.Fsys_mgr().Root_dir();
Io_url[] trg_fil_ary = Io_mgr.Instance.QueryDir_args(root_dir).FilPath_("*." + dump_type + Download_file_ext() + "*").ExecAsUrlAry();
Io_url trg = trg_fil_ary.length == 0 ? root_dir.GenSubFil(dump_file.File_name()) : trg_fil_ary[0];
this.Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());
this.Init("download", dump_file.File_url(), trg);
return super.Async_init();
}
private static boolean Dump_servers_offline_msg_shown = false;
}

13
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_goto_page.java
View File

@@ -13,3 +13,16 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.threads.*;
class Xoi_cmd_wiki_goto_page extends Gfo_thread_cmd_base implements Gfo_thread_cmd {
public Xoi_cmd_wiki_goto_page(Xoae_app app, String page) {this.app = app; this.page = page; this.Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());} private Xoae_app app; String page;
@Override public void Async_run() {kit.New_cmd_sync(this).Invk(GfsCtx.new_(), 0, Invk_goto_page, GfoMsg_.Null);}
private void Goto_page(String page) {app.Gui_mgr().Browser_win().Page__navigate_by_url_bar(page);}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_goto_page)) Goto_page(page);
else return super.Invk(ctx, ikey, k, m);
return this;
} private static final String Invk_goto_page = "goto_page";
public static final String KEY = "wiki.goto_page";
}

96
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_import.java
View File

@@ -13,3 +13,99 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.threads.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.utils.*;
import gplx.xowa.guis.views.*;
import gplx.xowa.wikis.domains.*;
import gplx.xowa.htmls.hrefs.*;
import gplx.xowa.addons.wikis.ctgs.bldrs.*;
class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
private boolean running;
private Xowe_wiki wiki;
public Xoi_cmd_wiki_import(Xoi_setup_mgr install_mgr, String wiki_key, String wiki_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.wiki_date = wiki_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, wiki_date, dump_type;
public static final String KEY = "wiki.import";
public void Cmd_ctor() {}
public String Async_key() {return KEY;}
public int Async_sleep_interval() {return Gfo_thread_cmd_.Async_sleep_interval_1_second;}
public boolean Async_prog_enabled() {return false;}
public void Async_prog_run(int async_sleep_sum) {}
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
public boolean Async_term() {
install_mgr.App().Usr_dlg().Log_many(GRP_KEY, "import.end", "import.end ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
return true;
}
public Gfo_invk Owner() {return owner;} public Xoi_cmd_wiki_import Owner_(Gfo_invk v) {owner = v; return this;} Gfo_invk owner;
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
public void Async_run() {
running = true;
install_mgr.App().Usr_dlg().Log_many(GRP_KEY, "import.bgn", "import.bgn ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
Thread_adp_.Start_by_key(this.Async_key(), this, Invk_process_async);
}
public boolean Async_running() {
return running;
}
public boolean Import_move_bz2_to_done() {return import_move_bz2_to_done;} public Xoi_cmd_wiki_import Import_move_bz2_to_done_(boolean v) {import_move_bz2_to_done = v; return this;} private boolean import_move_bz2_to_done = true;
private void Process_async() {
Xoae_app app = install_mgr.App();
app.Usr_dlg().Prog_one("", "", "preparing import: ~{0}", wiki_key);
Xob_bldr bldr = app.Bldr();
wiki = app.Wiki_mgr().Get_by_or_make(Bry_.new_a7(wiki_key));
wiki.Init_assert();
bldr.Cmd_mgr().Clear();
bldr.Pause_at_end_(false);
Io_url src_url = wiki.Import_cfg().Src_rdr().Url();
Process_sql(bldr, src_url);
bldr.Run();
app.Usr_dlg().Prog_none(GRP_KEY, "clear", ""); app.Usr_dlg().Note_none(GRP_KEY, "clear", "");
app.Usere().Available_from_fsys();
wiki.Init_needed_(true);
wiki.Html_mgr().Page_wtr_mgr().Init_(true);
wiki.Init_assert();
if (String_.Eq(src_url.Ext(), ".xml")) {
if ( app.Cfg().Get_bool_app_or("xowa.bldr.import.delete_xml_file", true) // CFG: Cfg__
&& Io_mgr.Instance.ExistsFil(src_url.GenNewExt(".bz2")) // only delete the file if there is a corresponding bz2 file; BUG.GH:#124; DATE:2017-02-02
)
Io_mgr.Instance.DeleteFil(src_url);
}
else if (String_.Eq(src_url.Ext(), ".bz2")) {
Io_url trg_fil = app.Fsys_mgr().Wiki_dir().GenSubFil_nest("#dump", "done", src_url.NameAndExt());
if (import_move_bz2_to_done)
Io_mgr.Instance.MoveFil_args(src_url, trg_fil, true).Exec();
}
running = false;
wiki.Import_cfg().Src_fil_xml_(null).Src_fil_bz2_(null); // reset file else error when going from Import/Script to Import/List
app.Gui_mgr().Kit().New_cmd_sync(this).Invk(GfsCtx.new_(), 0, Invk_open_wiki, GfoMsg_.Null);
}
private void Process_sql(Xob_bldr bldr, Io_url dump_url) {
// setup wiki
((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_css);
// if (wiki.Appe().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.addons.wikis.searchs.specials.Srch_special_page.Version_2)
gplx.xowa.addons.wikis.searchs.bldrs.Srch_bldr_mgr_.Setup(wiki);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_term);
// setup category
if (wiki.Domain_itm().Domain_type_id() != Xow_domain_tid_.Tid__other) { // do not add category if not wmf; note that wikia wikis will not have category dumps; DATE:2016-10-22
Xob_download_cmd.Add_if_not_found_many(bldr, wiki, Xob_catlink_cmd.Dump_file_name, Xob_pageprop_cmd.Dump_file_name);
bldr.Cmd_mgr().Add(new gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd(bldr, wiki).Src_dir_manual_(dump_url.OwnerDir()));
bldr.Cmd_mgr().Add(new gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd(bldr, wiki).Src_dir_manual_(dump_url.OwnerDir()));
}
}
private void Open_wiki(String wiki_key) {
Xog_win_itm main_win = install_mgr.App().Gui_mgr().Browser_win();
if (main_win.Active_page() == null) return; // will be null when invoked through cmd-line
byte[] url = Bry_.Add(wiki.Domain_bry(), Xoh_href_.Bry__wiki, wiki.Props().Main_page());
main_win.Page__navigate_by_url_bar(String_.new_u8(url));
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_process_async)) Process_async();
else if (ctx.Match(k, Invk_owner)) return owner;
else if (ctx.Match(k, Invk_open_wiki)) Open_wiki(wiki_key);
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk_process_async = "run_async", Invk_owner = "owner", Invk_open_wiki = "open_wiki";
static final String GRP_KEY = "xowa.thread.op.build";
}

113
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_tst.java
View File

@@ -13,3 +13,116 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*;
import gplx.core.consoles.*;
import gplx.core.brys.args.*; import gplx.core.threads.*; import gplx.xowa.bldrs.setups.maints.*; import gplx.xowa.xtns.wbases.imports.*;
import gplx.xowa.wikis.domains.*;
import gplx.xowa.bldrs.wms.*; import gplx.xowa.bldrs.wms.dumps.*;
public class Xoi_cmd_wiki_tst {
@Test public void Run() { // MAINT:2017-03-28
// Bld_import_list(Xow_domain_regy.All);
// Bld_cfg_files(Xow_domain_regy.All); // NOTE: remember to carry over the wikisource / page / index commands from the existing xowa_build_cfg.gfs; also, only run the xowa_build_cfg.gfs once; DATE:2013-10-15; last run: DATE:2014-09-09
}
public void Bld_import_list(String... ary) {
int ary_len = ary.length;
Bry_bfr bfr = Bry_bfr_.Reset(255);
Wmf_latest_parser parser = new Wmf_latest_parser();
Bfr_arg__time time_fmtr = new Bfr_arg__time();
for (int i = 0; i < ary_len; i++)
Bld_import_list_itm2(bfr, parser, time_fmtr, ary, i);
Io_mgr.Instance.SaveFilStr("C:\\xowa\\user\\temp.txt", bfr.To_str());
}
private void Bld_import_list_itm2(Bry_bfr bfr, Wmf_latest_parser parser, Bfr_arg__time time_fmtr, String[] ary, int i) {
String domain_str = ary[i];
byte[] domain_bry = Bry_.new_a7(domain_str);
Xow_domain_itm domain_itm = Xow_domain_itm_.parse(domain_bry);
byte[] wmf_key_bry = Bry_.Replace(Xow_abrv_wm_.To_abrv(domain_itm), Byte_ascii.Dash, Byte_ascii.Underline);
String wmf_key = String_.new_u8(wmf_key_bry);
String url = "https://dumps.wikimedia.org/" + wmf_key + "/latest";
byte[] latest_html = null;
for (int j = 0; j < 5; ++j) {
latest_html = Io_mgr.Instance.DownloadFil_args("", Io_url_.Empty).Exec_as_bry(url);
if (latest_html != null) break;
Tfds.Dbg("fail|" + domain_str + "|" + url);
if (j == 4) return;
}
parser.Parse(latest_html);
Xowm_dump_file dump_file = new Xowm_dump_file(domain_str, "latest", Xowm_dump_type_.Str__pages_articles);
dump_file.Server_url_(Xowm_dump_file_.Server_wmf_https);
byte[] pages_articles_key = Bry_.new_a7(wmf_key + "-latest-pages-articles.xml.bz2");
Wmf_latest_itm latest_itm = parser.Get_by(pages_articles_key);
if (latest_itm == null) {Tfds.Dbg("missing|" + domain_str + "|" + url); return;} // NOTE: commonswiki missing entry for commonswiki-latest-pages-articles.xml.bz2 DATE:2016-05-01
Tfds.Dbg("pass|" + domain_str + "|" + url);
bfr.Add(domain_bry).Add_byte_pipe();
bfr.Add_str_u8(dump_file.File_url()).Add_byte_pipe();
bfr.Add(Xow_domain_tid_.Get_type_as_bry(domain_itm.Domain_type_id())).Add_byte_pipe();
long src_size = latest_itm.Size();
bfr.Add_long_variable(src_size).Add_byte_pipe();
bfr.Add_str_a7(gplx.core.ios.Io_size_.To_str(src_size)).Add_byte_pipe();
time_fmtr.Seconds_(Math_.Div_safe_as_long(src_size, 1000000)).Bfr_arg__add(bfr);
bfr.Add_byte_pipe();
bfr.Add_str_a7(latest_itm.Date().XtoStr_fmt_yyyy_MM_dd_HH_mm());
bfr.Add_byte_pipe();
bfr.Add_str_a7(dump_file.Dump_date());
bfr.Add_byte_nl();
}
/*
private void Bld_import_list_itm(Bry_bfr bfr, Xowm_dump_file dump_file, Bry_fmtr_arg_time time_fmtr, String[] ary, int i) {
String itm = ary[i];
dump_file.Ctor(itm, "latest", Xowm_dump_type_.Str__pages_articles);
int count = 0;
while (count++ < 1) {
dump_file.Server_url_(Xowm_dump_file_.Server_wmf);
if (dump_file.Connect()) break;
Tfds.WriteText(String_.Format("retrying: {0} {1}\n", count, dump_file.File_modified()));
Thread_adp_.Sleep(15000); // wait for connection to reset
}
if (count == 10) {
Tfds.WriteText(String_.Format("failed: {0}\n", dump_file.File_url()));
return;
}
else
Tfds.WriteText(String_.Format("passed: {0}\n", itm));
bfr.Add_str(itm).Add_byte_pipe();
bfr.Add_str(dump_file.File_url()).Add_byte_pipe();
bfr.Add(Xow_domain_tid_.Get_type_as_bry(dump_file.Wiki_type().Wiki_tid())).Add_byte_pipe();
// Xol_lang_stub lang_itm = Xol_lang_stub_.Get_by_key(wiki_type.Lang_key());
// if (lang_itm == null) lang_itm = Xol_lang_stub_.Get_by_key(Xol_lang_itm_.Key_en); // commons, species, meta, etc will have no lang
// bfr.Add(lang_itm.Local_name()).Add_byte_pipe();
// bfr.Add(lang_itm.Canonical_name()).Add_byte_pipe();
long src_size = dump_file.File_len();
bfr.Add_long_variable(src_size).Add_byte_pipe();
bfr.Add_str(gplx.core.ios.Io_size_.To_str(src_size)).Add_byte_pipe();
time_fmtr.Seconds_(Math_.Div_safe_as_long(src_size, 1000000)).XferAry(bfr, 0);
bfr.Add_byte_pipe();
bfr.Add_str(dump_file.File_modified().XtoStr_fmt_yyyy_MM_dd_HH_mm());
bfr.Add_byte_pipe();
// bfr.Add_str(String_.Concat_with_obj(",", (Object[])dump_file.Dump_available_dates()));
// bfr.Add_byte_pipe();
bfr.Add_str(dump_file.Dump_date());
bfr.Add_byte_nl();
Thread_adp_.Sleep(1000);
}
*/
public void Bld_cfg_files(String... ary) {
Bry_bfr bfr = Bry_bfr_.Reset(255);
gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_api api = new gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_api();
gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_wiki wiki = new gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_wiki();
int ary_len = ary.length;
for (int i = 0; i < ary_len; i++) {
String wiki_domain = ary[i];
try {
byte[] xml = api.Exec_api(api.Api_src(wiki_domain));
wiki.Wiki_domain_(Bry_.new_a7(wiki_domain));
api.Parse(wiki, String_.new_u8(xml));
api.Build_cfg(bfr, wiki);
}
catch (Exception e) {
Console_adp__sys.Instance.Write_str_w_nl(Err_.Message_gplx_full(e));
}
}
bfr.Add_str_a7("app.bldr.wiki_cfg_bldr.run;").Add_byte_nl();
Io_mgr.Instance.SaveFilStr("C:\\user\\xowa_build_cfg.gfs", bfr.To_str());
}
}

33
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_unzip.java
View File

@@ -13,3 +13,36 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.gfui.*; import gplx.gfui.kits.core.*;
import gplx.core.threads.*;
class Xoi_cmd_wiki_unzip extends Gfo_thread_cmd_unzip implements Gfo_thread_cmd { public static final String KEY_dump = "wiki.unzip";
public Xoi_cmd_wiki_unzip(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.dump_date = dump_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, dump_date, dump_type;
@Override public String Async_key() {return KEY_dump;}
@Override public byte Async_init() {
Xoae_app app = install_mgr.App(); Gfui_kit kit = app.Gui_mgr().Kit();
Xowe_wiki wiki = app.Wiki_mgr().Get_by_or_make(Bry_.new_u8(wiki_key));
Io_url wiki_dir = wiki.Import_cfg().Src_dir();
Io_url[] urls = Io_mgr.Instance.QueryDir_args(wiki_dir).Recur_(false).FilPath_("*.xml.bz2").ExecAsUrlAry();
if (urls.length == 0) {
kit.Ask_ok(GRP_KEY, "dump.unzip_latest.file_missing", "Could not find a dump file for ~{0} in ~{1}", wiki_key, wiki_dir.Raw());
return Gfo_thread_cmd_.Init_cancel_step;
}
Io_url src = urls[urls.length - 1];
Io_url trg = app.Fsys_mgr().Wiki_dir().GenSubFil_nest(wiki_key, src.NameOnly()); // NOTE: NameOnly() will strip trailing .bz2; EX: a.xml.bz2 -> a.xml
super.Init(app.Usr_dlg(), app.Gui_mgr().Kit(), app.Prog_mgr().App_decompress_bz2(), app.Prog_mgr().App_decompress_zip(), app.Prog_mgr().App_decompress_gz(), src, trg);
this.Term_cmd_for_src_(Term_cmd_for_src_move);
this.Term_cmd_for_src_url_(app.Fsys_mgr().Wiki_dir().GenSubFil_nest("#dump", "done", src.NameAndExt()));
if (Io_mgr.Instance.ExistsFil(trg)) {
int rslt = kit.Ask_yes_no_cancel(GRP_KEY, "target_exists", "Target file already exists: '~{0}'.\nDo you want to delete it?", trg.Raw());
switch (rslt) {
case Gfui_dlg_msg_.Btn_yes: Io_mgr.Instance.DeleteFil(trg); break;
case Gfui_dlg_msg_.Btn_no: return Gfo_thread_cmd_.Init_cancel_step;
case Gfui_dlg_msg_.Btn_cancel: return Gfo_thread_cmd_.Init_cancel_all;
default: throw Err_.new_unhandled(rslt);
}
}
return Gfo_thread_cmd_.Init_ok;
}
static final String GRP_KEY = "xowa.thread.dump.unzip";
}

9
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_dump_mgr.java
View File

@@ -13,3 +13,12 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.xowa.bldrs.wms.dumps.*;
public class Xoi_dump_mgr {
public static boolean Import_bz2_by_stdout(Xoa_app app) {return app.Cfg().Get_bool_app_or("xowa.bldr.import.apps.bz2_stdout.enabled", true);} // CFG: Cfg__
public static String[] Server_urls(Xoa_app app) {
String[] or = String_.Ary(Xowm_dump_file_.Server_your_org, Xowm_dump_file_.Server_wmf_https, Xowm_dump_file_.Server_c3sl, Xowm_dump_file_.Server_masaryk); // promote your.org to primary url; DATE:2016-08-07
return app.Cfg().Get_strary_app_or("xowa.bldr.import.dump_servers", ",", or); // CFG: Cfg__
}
}

30
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_mirror_parser.java
View File

@@ -13,3 +13,33 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public class Xoi_mirror_parser {
public String[] Parse(String raw_str) {
if (String_.Len_eq_0(raw_str)) return String_.Ary_empty;
byte[] raw = Bry_.new_u8(raw_str);
List_adp rv = List_adp_.New();
int pos = 0;
while (true) {
int bgn = Bry_find_.Find_fwd(raw, CONST_href_bgn, pos); if (bgn == Bry_find_.Not_found) break;
bgn += CONST_href_bgn.length;
int end = Bry_find_.Find_fwd(raw, CONST_href_end, bgn); if (end == Bry_find_.Not_found) return String_.Ary_empty;
byte[] date = Bry_.Mid(raw, bgn, end);
pos = end + CONST_href_end.length;
if (Bry_.Match(date, CONST_date_parent_dir)) continue;
int date_pos_last = date.length - 1;
if (date_pos_last == -1) return String_.Ary_empty;
if (date[date_pos_last] == Byte_ascii.Slash) date = Bry_.Mid(date, 0, date_pos_last); // trim trailing /; EX: "20130101/" -> "20130101"
rv.Add(String_.new_u8(date));
}
return rv.To_str_ary();
} private static final byte[] CONST_href_bgn = Bry_.new_a7("<a href=\""), CONST_href_end = Bry_.new_a7("\""), CONST_date_parent_dir = Bry_.new_a7("../");
public static String Find_last_lte(String[] ary, String comp) { // assuming sorted ary, find last entry that is lte comp
int len = ary.length;
for (int i = len - 1; i > -1; i--) {
String itm = ary[i];
if (CompareAble_.Is(CompareAble_.Less_or_same, itm, comp)) return itm;
}
return "";
}
}

43
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_mirror_parser_tst.java
View File

@@ -13,3 +13,46 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*;
public class Xoi_mirror_parser_tst {
@Test public void Basic() {
Tst_parse(String_.Concat_lines_nl
( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
, "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.1//EN\" \"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd\">"
, "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\">"
, "<head>"
, "<title>Index of /simplewiki/</title>"
, "<link rel=\"stylesheet\" type=\"text/css\" href=\"/pub/misc/lighttpd-white-dir.css\" />"
, "</head>"
, "<body>"
, "<h2>Index of /simplewiki/</h2>"
, "<div class=\"list\">"
, "<table summary=\"Directory Listing\" cellpadding=\"0\" cellspacing=\"0\">"
, "<thead><tr><th class=\"n\">Name</th><th class=\"m\">Last Modified</th><th class=\"s\">Size</th><th class=\"t\">Type</th></tr></thead>"
, "<tbody>"
, "<tr><td class=\"n\"><a href=\"../\">Parent Directory</a>/</td><td class=\"m\">&nbsp;</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"20120516/\">20120516</a>/</td><td class=\"m\">2012-May-17 01:04:39</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"20121220/\">20121220</a>/</td><td class=\"m\">2012-Dec-20 20:15:55</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"20130214/\">20130214</a>/</td><td class=\"m\">2013-Feb-14 06:28:41</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"latest/\">latest</a>/</td><td class=\"m\">2013-Feb-14 06:28:41</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "</tbody>"
, "</table>"
, "</div>"
, "<div class=\"foot\">lighttpd</div>"
, "</body>"
, "</html>"
), String_.Ary("20120516", "20121220", "20130214", "latest"));
}
@Test public void Find_last_lte() {
Tst_find_last_lte(String_.Ary("20120516", "20121220", "20130214", "latest"), "20130101", "20121220");
Tst_find_last_lte(String_.Ary("20120516", "20121220", "20130214", "latest"), "20120101", "");
}
private void Tst_parse(String raw, String[] expd) {
Xoi_mirror_parser parser = new Xoi_mirror_parser();
Tfds.Eq_ary_str(expd, parser.Parse(raw));
}
private void Tst_find_last_lte(String[] ary, String comp, String expd) {
Tfds.Eq(expd, Xoi_mirror_parser.Find_last_lte(ary, comp));
}
}

25
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_setup_mgr.java
View File

@@ -13,3 +13,28 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.gfui.*; import gplx.xowa.bldrs.setups.addons.*;
import gplx.xowa.bldrs.setups.maints.*;
public class Xoi_setup_mgr implements Gfo_invk {
public Xoi_setup_mgr(Xoae_app app) {
this.app = app;
cmd_mgr = new Xoi_cmd_mgr(this);
maint_mgr = new Xoa_maint_mgr(app);
}
public void Init_by_app(Xoae_app app) {
addon_mgr.Init_by_app(app);
}
public Xoae_app App() {return app;} private Xoae_app app;
public Xoi_cmd_mgr Cmd_mgr() {return cmd_mgr;} private Xoi_cmd_mgr cmd_mgr;
public Xoi_addon_mgr Addon_mgr() {return addon_mgr;} private Xoi_addon_mgr addon_mgr = new Xoi_addon_mgr();
public Xoa_maint_mgr Maint_mgr() {return maint_mgr;} private Xoa_maint_mgr maint_mgr;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_cmds)) return cmd_mgr;
else if (ctx.Match(k, Invk_addons)) return addon_mgr;
else if (ctx.Match(k, Invk_maint)) return maint_mgr;
else return Gfo_invk_.Rv_unhandled;
}
static final String Invk_cmds = "cmds", Invk_addons = "addons", Invk_maint = "maint";
static final String GRP_KEY = "xowa.setup";
}