Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.7.2.1

Browse Source
This commit is contained in:
gnosygnu
2015-07-12 21:10:02 -04:00
commit 794b5a232f
3099 changed files with 238212 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
400_xowa/src_240_install/gplx/xowa/Xob_dump_file.java Normal file
View File

@@ -0,0 +1,89 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*;
import gplx.xowa.wikis.*;
public class Xob_dump_file {
public Xow_domain Wiki_type() {return wiki_domain_itm;} private Xow_domain wiki_domain_itm;
public String Dump_date() {return dump_date;} public void Dump_date_(String v) {dump_date = v;} String dump_date;
public String Dump_file_type() {return dump_file_type;} private String dump_file_type;
public String Server_url() {return server_url;} private String server_url;
public String File_url() {return file_url;} private String file_url;
public String File_name() {return file_name;} private String file_name;
public long File_len() {return file_len;} long file_len;
public DateAdp File_modified() {return file_modified;} DateAdp file_modified;
public byte[] Wiki_alias() {return wiki_alias;} private byte[] wiki_alias;
public Xob_dump_file Ctor(String wiki_domain, String dump_date, String dump_file_type) {
this.dump_date = dump_date; this.dump_file_type = dump_file_type;
this.wiki_domain_itm = Xow_domain_.parse(Bry_.new_a7(wiki_domain));
this.wiki_alias = Xow_wiki_alias.Build_alias(wiki_domain_itm);
byte[] dump_file_bry = Bry_.new_u8(dump_file_type);
byte dump_file_tid = Xow_wiki_alias.Parse__tid(dump_file_bry);
byte[] ext = Xob_dump_file_.Ext_xml_bz2;
switch (dump_file_tid) {
case Xow_wiki_alias.Tid_page_props: case Xow_wiki_alias.Tid_categorylinks: case Xow_wiki_alias.Tid_image:
ext = Xob_dump_file_.Ext_sql_gz;
break;
}
this.file_name = String_.new_u8(Xob_dump_file_.Bld_dump_file_name(wiki_alias, Bry_.new_u8(dump_date), dump_file_bry, ext));
return this;
}
public void Server_url_(String server_url) {
this.server_url = server_url;
String dump_dir_url = String_.new_u8(Xob_dump_file_.Bld_dump_dir_url(Bry_.new_u8(server_url), wiki_alias, Bry_.new_u8(dump_date)));
this.file_url = dump_dir_url + file_name;
}
public boolean Connect() {
IoEngine_xrg_downloadFil args = Io_mgr.I.DownloadFil_args("", Io_url_.Empty);
boolean rv = Connect_exec(args, file_url);
if ( !rv
&& String_.In(server_url, Xob_dump_file_.Server_wmf_http, Xob_dump_file_.Server_wmf_https)
&& String_.Eq(dump_date, Xob_dump_file_.Date_latest)
) { // WMF changed dumping approach to partial dumps; this sometimes causes /latest/ to be missing page_articles; try to get earlier dump; DATE:2015-07-09
byte[] abrv_wm = Xow_wiki_alias.Build_alias(wiki_domain_itm);
String new_dump_root = Xob_dump_file_.Server_wmf_https + String_.new_u8(abrv_wm) + "/"; // EX: http://dumps.wikimedia.org/enwiki/
byte[] wiki_dump_dirs_src = args.Exec_as_bry(new_dump_root);
if (wiki_dump_dirs_src == null) {Xoa_app_.Usr_dlg().Warn_many("", "", "could not connect to dump server; url=~{0}", new_dump_root); return false;}
String[] dates = gplx.xowa.wmfs.dump_pages.Xowmf_wiki_dump_dirs_parser.Parse(wiki_domain_itm.Domain_bry(), wiki_dump_dirs_src);
int dates_len = dates.length;
if ( dates_len > 3 // need at least 3; EX: 20150601,20150701,latest; 20150701 and latest are same; 20150601 is last good
&& String_.Eq(dates[dates_len - 1], Xob_dump_file_.Date_latest)) { // last itm is latest
String new_dump_date = dates[dates_len - 3];
String new_dump_file = String_.Replace(file_name, Xob_dump_file_.Date_latest, new_dump_date); // replace "-latest-" with "-20150602-";
String new_file_url = new_dump_root + new_dump_date + "/" + new_dump_file;
rv = Connect_exec(args, new_file_url);
if (rv) {
dump_date = new_dump_date;
file_name = new_dump_file;
file_url = new_file_url;
}
}
}
return rv;
}
private boolean Connect_exec(IoEngine_xrg_downloadFil args, String cur_file_url) {
boolean rv = args.Src_last_modified_query_(true).Exec_meta(cur_file_url);
if (rv) {
file_len = args.Src_content_length();
file_modified = args.Src_last_modified();
if (file_modified.Timestamp_unix() <= 0) return false; // dump is available, but invalid (stamped with old date)
}
return rv;
}
public static Xob_dump_file new_(String wiki_domain, String dump_date, String dump_type) {return new Xob_dump_file().Ctor(wiki_domain, dump_date, dump_type);}
}

72
400_xowa/src_240_install/gplx/xowa/Xob_dump_file_.java Normal file
View File

@@ -0,0 +1,72 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
public class Xob_dump_file_ {
public static boolean Connect_first(Xob_dump_file rv, String[] server_urls) {
int len = server_urls.length;
for (int i = 0; i < len; ++i) {
String server_url = server_urls[i];
rv.Server_url_(server_url);
Override_dump_date(rv, server_url);
if (rv.Connect()) return true;
}
return false;
}
private static void Override_dump_date(Xob_dump_file rv, String dump_server) {
String dump_date = rv.Dump_date();
if ( String_.Eq(dump_date, Xob_dump_file_.Date_latest)
&& ( String_.Eq(dump_server, Xob_dump_file_.Server_c3sl)
|| String_.Eq(dump_server, Xob_dump_file_.Server_masaryk)
)
){
Xoi_mirror_parser mirror_parser = new Xoi_mirror_parser();
String dump_wiki_url = dump_server + String_.new_a7(rv.Wiki_alias()) + "/";
byte[] dump_url_wiki_html = gplx.ios.IoEngine_xrg_downloadFil.new_("", Io_url_.Empty).Exec_as_bry(dump_wiki_url); if (Bry_.Len_eq_0(dump_url_wiki_html)) return;
String[] dump_available_dates = mirror_parser.Parse(String_.new_u8(dump_url_wiki_html));
String dump_dates_latest = Xoi_mirror_parser.Find_last_lte(dump_available_dates, dump_date);
if (String_.Eq(dump_dates_latest, "")) return; // nothing found
rv.Dump_date_(dump_dates_latest);
}
}
public static byte[] Bld_dump_dir_url(byte[] server_url, byte[] alias, byte[] date) {
return Bry_.Add
( server_url // "http://dumps.wikimedia.org/"
, Bry_.Replace(alias, Byte_ascii.Dash, Byte_ascii.Underline), Bry_slash // "simplewiki/"
, date, Bry_slash // "latest/"
);
}
public static byte[] Bld_dump_file_name(byte[] alias, byte[] date, byte[] dump_file_type, byte[] ext) {
return Bry_.Add
( Bry_.Replace(alias, Byte_ascii.Dash, Byte_ascii.Underline), Bry_dash // "simplewiki-"
, date, Bry_dash // "latest-"
, dump_file_type // "pages-articles"
, ext // ".xml.bz2"
);
}
private static final byte[] Bry_dash = new byte[] {Byte_ascii.Dash}, Bry_slash = new byte[] {Byte_ascii.Slash};
public static final byte[] Ext_xml_bz2 = Bry_.new_a7(".xml.bz2");
public static final byte[] Ext_sql_gz = Bry_.new_a7(".sql.gz");
public static final String
Server_wmf_http = "http://dumps.wikimedia.org/"
, Server_wmf_https = "https://dumps.wikimedia.org/"
, Server_your_org = "http://dumps.wikimedia.your.org/"
, Server_c3sl = "http://wikipedia.c3sl.ufpr.br/"
, Server_masaryk = "http://ftp.fi.muni.cz/pub/wikimedia/"
, Date_latest = "latest"
;
}

111
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_base.java Normal file
View File

@@ -0,0 +1,111 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.threads.*; import gplx.xowa.wikis.*; import gplx.xowa.bldrs.*;
abstract class Xoi_cmd_base implements Gfo_thread_cmd {
public void Ctor(Xoi_setup_mgr install_mgr, String wiki_key) {
this.install_mgr = install_mgr; this.wiki_key = wiki_key;
this.Owner_(install_mgr);
} private Xoi_setup_mgr install_mgr; String wiki_key;
@gplx.Virtual public void Cmd_ctor() {}
public abstract String Async_key();
public int Async_sleep_interval() {return Gfo_thread_cmd_.Async_sleep_interval_1_second;}
public boolean Async_prog_enabled() {return false;}
public void Async_prog_run(int async_sleep_sum) {}
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
public boolean Async_term() {
// install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "import.end", "import.end ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
return true;
}
public GfoInvkAble Owner() {return owner;} public Xoi_cmd_base Owner_(GfoInvkAble v) {owner = v; return this;} GfoInvkAble owner;
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
public void Async_run() {
running = true;
// install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "import.bgn", "import.bgn ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
Thread_adp_.invk_(this.Async_key(), this, Invk_process_async).Start();
}
public boolean Async_running() {return running;} private boolean running;
public void Process_async() {
Xoae_app app = install_mgr.App();
Xob_bldr bldr = app.Bldr();
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_a7(wiki_key));
wiki.Init_assert();
bldr.Cmd_mgr().Clear();
Process_async_init(app, wiki, bldr);
bldr.Pause_at_end_(false);
try {bldr.Run();}
catch (Exception e) {
running = false;
install_mgr.Cmd_mgr().Working_n_();
throw Exc_.new_exc(e, "xo", "error during import");
}
app.Usr_dlg().Prog_none("", "clear", "");
app.Usr_dlg().Note_none("", "clear", "");
Process_async_done(app, wiki, bldr);
running = false;
}
public abstract void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr);
public abstract void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr);
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_process_async)) Process_async();
else if (ctx.Match(k, Invk_owner)) return owner;
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_process_async = "run_async", Invk_owner = "owner";
}
class Xoi_cmd_category2_page_props extends Xoi_cmd_wiki_download { public Xoi_cmd_category2_page_props(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date) {this.Ctor_download_(install_mgr, wiki_key, dump_date, Xow_wiki_alias.Key_page_props);}
@Override public String Download_file_ext() {return ".sql.gz2";}
public static final String KEY_category2 = "wiki.category2.download.page_props";
}
class Xoi_cmd_category2_categorylinks extends Xoi_cmd_wiki_download { public Xoi_cmd_category2_categorylinks(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date) {this.Ctor_download_(install_mgr, wiki_key, dump_date, Xow_wiki_alias.Key_categorylinks);}
@Override public String Download_file_ext() {return ".sql.gz2";}
public static final String KEY_category2 = "wiki.category2.download.categorylinks";
}
class Xoi_cmd_category2_build extends Xoi_cmd_base {
public Xoi_cmd_category2_build(Xoi_setup_mgr install_mgr, String wiki_key) {this.Ctor(install_mgr, wiki_key); this.app = install_mgr.App(); this.wiki_key = wiki_key;} private Xoae_app app; private String wiki_key;
@Override public void Cmd_ctor() {
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_u8(wiki_key));
wiki.Import_cfg().Category_version_(gplx.xowa.ctgs.Xoa_ctg_mgr.Version_2);
}
@Override public String Async_key() {return KEY;} public static final String KEY = "wiki.category2.build";
@Override public void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
if (app.Setup_mgr().Dump_mgr().Wiki_storage_type_is_sql()) {
wiki.Db_mgr_as_sql().Category_version_update(false);
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_text_cat_core, Xob_cmd_keys.Key_text_cat_link, Xob_cmd_keys.Key_text_cat_hidden);
}
else
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_tdb_cat_hidden_sql, Xob_cmd_keys.Key_tdb_cat_hidden_ttl, Xob_cmd_keys.Key_text_cat_link, Xob_cmd_keys.Key_tdb_ctg_link_idx);
}
@Override public void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
app.Usr_dlg().Prog_many("", "", "category2 setup done");
}
}
class Xoi_cmd_search2_build extends Xoi_cmd_base {
public Xoi_cmd_search2_build(Xoi_setup_mgr install_mgr, String wiki_key) {this.Ctor(install_mgr, wiki_key);}
@Override public String Async_key() {return KEY;} public static final String KEY = "wiki.search2.build";
@Override public void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
if (app.Setup_mgr().Dump_mgr().Wiki_storage_type_is_sql()) {
wiki.Db_mgr_as_sql().Category_version_update(false);
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_text_search_cmd);
}
}
@Override public void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
app.Usr_dlg().Prog_many("", "", "search2 setup done");
wiki.Db_mgr().Search_version_refresh();
}
}

60
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_dumpfile.java Normal file
View File

@@ -0,0 +1,60 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.threads.*;
class Xoi_cmd_dumpfile {
public byte[] Domain() {return domain;} private byte[] domain;
public Io_url Bz2_url() {return bz2_url;} Io_url bz2_url;
public Io_url Xml_url() {return xml_url;} Io_url xml_url;
public boolean Bz2_unzip() {return bz2_unzip;} private boolean bz2_unzip;
public void Clear() {domain = null; bz2_url = xml_url = null; bz2_unzip = false;}
public Xoi_cmd_dumpfile Parse_msg(GfoMsg m) {
Io_url dump_url = m.ReadIoUrl("url");
domain = m.ReadBry("domain");
if (Bry_.Len_eq_0(domain)) domain = Bry_.new_u8(dump_url.OwnerDir().NameOnly());
bz2_unzip = String_.Eq(m.ReadStr("args"), "unzip");
String dump_ext = dump_url.Ext();
if (String_.Eq(dump_ext, ".bz2")) {
bz2_url = dump_url;
if (bz2_unzip) {
xml_url = bz2_url.GenNewExt(""); // remove .bz2 extension (new file path should be .xml)
if (!String_.Eq(xml_url.Ext(), ".xml"))
xml_url = xml_url.GenNewExt(".xml");
}
}
else if (String_.Eq(dump_ext, ".xml")) { // user selected xml file;
bz2_url = null;
xml_url = dump_url;
bz2_unzip = false; // ignore unzip arge
}
return this;
}
public Gfo_thread_cmd Exec(Xoi_cmd_mgr cmd_mgr) {
Xowe_wiki wiki = cmd_mgr.App().Wiki_mgr().Get_by_key_or_make(domain);
if (bz2_unzip) { // unzip requested; add unzip cmd
GfoMsg unzip_msg = GfoMsg_.new_parse_(Gfo_thread_cmd_unzip.KEY).Add("v", Gfo_thread_cmd_unzip.KEY).Add("src", bz2_url.Raw()).Add("trg", xml_url.Raw());
Gfo_thread_cmd_unzip unzip_cmd = (Gfo_thread_cmd_unzip)cmd_mgr.Cmd_add(unzip_msg);
unzip_cmd.Term_cmd_for_src_(Gfo_thread_cmd_unzip.Term_cmd_for_src_noop); // don't do anything with bz2 after unzip
}
if (xml_url == null)
wiki.Import_cfg().Src_fil_bz2_(bz2_url);
else
wiki.Import_cfg().Src_fil_xml_(xml_url);
return cmd_mgr.Dump_add_many_custom(String_.new_u8(domain), "", "", true);
}
}

76
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_dumpfile_tst.java Normal file
View File

@@ -0,0 +1,76 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import org.junit.*;
public class Xoi_cmd_dumpfile_tst {
@Before public void init() {fxt.Clear();} private Xoi_cmd_dumpfile_fxt fxt = new Xoi_cmd_dumpfile_fxt();
@Test public void Bz2__unzip() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml.bz2", "", "unzip")
.Test_domain("en.wikipedia.org")
.Test_vals("mem/en.wikipedia.org/fil.xml.bz2", "mem/en.wikipedia.org/fil.xml", true)
;
}
@Test public void Bz2__unzip__assert_xml_ext() { // xml ext relies on removing ".bz2" from ".xml.bz2"; if just ".bz2" add an ".xml"
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.bz2", "", "unzip")
.Test_vals("mem/en.wikipedia.org/fil.bz2", "mem/en.wikipedia.org/fil.xml", true)
;
}
@Test public void Bz2__direct() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.bz2", "", "")
.Test_vals("mem/en.wikipedia.org/fil.bz2", null, false)
;
}
@Test public void Xml__unzip_n() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml", "", "")
.Test_vals(null, "mem/en.wikipedia.org/fil.xml", false)
;
}
@Test public void Xml__unzip_y() {
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml", "", "")
.Test_vals(null, "mem/en.wikipedia.org/fil.xml", false)
;
}
}
class Xoi_cmd_dumpfile_fxt {
public void Clear() {
dumpfile.Clear();
} private Xoi_cmd_dumpfile dumpfile = new Xoi_cmd_dumpfile();
public Xoi_cmd_dumpfile_fxt Exec_parse_msg(String url, String domain, String args) {
GfoMsg m = GfoMsg_.new_parse_("").Add("url", url).Add("domain", domain).Add("args", args);
dumpfile.Parse_msg(m);
return this;
}
public Xoi_cmd_dumpfile_fxt Test_vals(String expd_bz2, String expd_xml, boolean expd_unzip) {
Eq_url(expd_bz2, dumpfile.Bz2_url());
Eq_url(expd_xml, dumpfile.Xml_url());
Tfds.Eq(expd_unzip, dumpfile.Bz2_unzip());
return this;
}
public Xoi_cmd_dumpfile_fxt Test_domain(String expd_domain) {
Tfds.Eq(expd_domain, String_.new_u8(dumpfile.Domain()));
return this;
}
private void Eq_url(String expd, Io_url actl) {
if (expd == null && actl == null) return;
else if (expd != null && actl != null) {
Tfds.Eq(expd, actl.Raw());
}
else if (expd == null) throw Exc_.new_("actl should be null", "expd", expd);
else if (actl == null) throw Exc_.new_("actl should not be null", "expd", expd);
}
}

144
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_mgr.java Normal file
View File

@@ -0,0 +1,144 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.threads.*;
public class Xoi_cmd_mgr implements GfoInvkAble {
List_adp cmds = List_adp_.new_();
public Xoi_cmd_mgr(Xoi_setup_mgr install_mgr) {this.app = install_mgr.App(); this.install_mgr = install_mgr;} private Xoae_app app; Xoi_setup_mgr install_mgr;
public Xoae_app App() {return app;}
public void Canceled_y_() {canceled = true;} private boolean canceled = false;
public boolean Working() {return working;} public void Working_n_() {working = false;} private boolean working;
private void Process_async(Gfo_thread_cmd cmd) {
byte init_rslt = cmd.Async_init();
if (init_rslt == Gfo_thread_cmd_.Init_ok) {
cmd.Async_run();
int async_sleep_interval = cmd.Async_sleep_interval();
boolean async_prog_enabled = cmd.Async_prog_enabled();
int async_sleep_sum = 0;
while (cmd.Async_running()) {
if (canceled) {working = false; return;}
if (async_prog_enabled) cmd.Async_prog_run(async_sleep_sum);
Thread_adp_.Sleep(async_sleep_interval);
async_sleep_sum += async_sleep_interval; // NOTE: this is not exact
}
}
boolean term_pass = cmd.Async_term();
if (cmd.Async_next_cmd() != null && init_rslt != Gfo_thread_cmd_.Init_cancel_all && term_pass)
Run_async(cmd.Async_next_cmd());
else
working = false;
}
private void Run_async(Gfo_thread_cmd cmd) {Thread_adp_.invk_msg_(cmd.Async_key(), this, GfoMsg_.new_cast_(Invk_process_async).Add("v", cmd)).Start();}
private void Cmds_run() {
if (working) {
app.Gui_mgr().Kit().Ask_ok("", "", "An import is in progress. Please wait for it to complete. If you want to do multiple imports at once, see Help:Import/Script.");
return;
}
int cmds_len = cmds.Count();
if (cmds_len == 0) return;
for (int i = 0; i < cmds_len - 1; i++) {
Gfo_thread_cmd cur_cmd = (Gfo_thread_cmd)cmds.Get_at(i);
Gfo_thread_cmd nxt_cmd = (Gfo_thread_cmd)cmds.Get_at(i + 1);
cur_cmd.Cmd_ctor();
cur_cmd.Async_next_cmd_(nxt_cmd);
}
Gfo_thread_cmd cmd = (Gfo_thread_cmd)cmds.Get_at(0);
cmds.Clear();
working = true;
this.Run_async(cmd);
}
Object Dump_add_many(GfoMsg m) {
int args_len = m.Args_count();
if (args_len < 4) throw Exc_.new_("Please provide the following: wiki name, wiki date, dump_type, and one command; EX: ('simple.wikipedia.org', 'latest', 'pages-articles', 'wiki.download')");
String wiki_key = m.Args_getAt(0).Val_to_str_or_empty();
String wiki_date = m.Args_getAt(1).Val_to_str_or_empty();
String dump_type = m.Args_getAt(2).Val_to_str_or_empty();
Gfo_thread_cmd cmd = null;
for (int i = 3; i < args_len; i++) {
KeyVal kv = m.Args_getAt(i);
String kv_val = kv.Val_to_str_or_empty();
if (String_.Eq(kv_val, Wiki_cmd_custom))
return Dump_add_many_custom(wiki_key, wiki_date, dump_type, false);
else {
cmd = Dump_cmd_new(wiki_key, wiki_date, dump_type, kv.Val_to_str_or_empty());
cmds.Add(cmd);
}
}
return cmd; // return last cmd
}
public Gfo_thread_cmd Dump_add_many_custom(String wiki_key, String wiki_date, String dump_type, boolean dumpfile_cmd) {
String[] custom_cmds = app.Setup_mgr().Dump_mgr().Custom_cmds();
int custom_cmds_len = custom_cmds.length;
Gfo_thread_cmd cmd = null;
for (int j = 0; j < custom_cmds_len; j++) {
cmd = Dump_cmd_new(wiki_key, wiki_date, dump_type, custom_cmds[j]);
if (dumpfile_cmd) {
if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_download.Key_wiki_download)) continue; // skip download if wiki.dump_file
else if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_unzip.KEY_dump)) {
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_u8(wiki_key));
if (wiki.Import_cfg().Src_fil_xml() != null) continue; // skip unzip if xml exists
}
else if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_import.KEY)) {
((Xoi_cmd_wiki_import)cmd).Import_move_bz2_to_done_(false);
}
}
cmds.Add(cmd);
}
return cmd;
}
Gfo_thread_cmd Dump_cmd_new(String wiki_key, String wiki_date, String dump_type, String cmd_key) {
if (String_.Eq(cmd_key, Xoi_cmd_wiki_download.Key_wiki_download)) return new Xoi_cmd_wiki_download().Ctor_download_(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_unzip.KEY_dump)) return new Xoi_cmd_wiki_unzip(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_import.KEY)) return new Xoi_cmd_wiki_import(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_zip.KEY)) return new Xoi_cmd_wiki_zip(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_category2_build.KEY)) return new Xoi_cmd_category2_build(install_mgr, wiki_key).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_category2_page_props.KEY_category2)) return new Xoi_cmd_category2_page_props(install_mgr, wiki_key, wiki_date).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_category2_categorylinks.KEY_category2)) return new Xoi_cmd_category2_categorylinks(install_mgr, wiki_key, wiki_date).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_search2_build.KEY)) return new Xoi_cmd_search2_build(install_mgr, wiki_key).Owner_(this);
else throw Exc_.new_unhandled(cmd_key);
}
public static final String[] Wiki_cmds_valid = new String[] {Xoi_cmd_wiki_download.Key_wiki_download, Xoi_cmd_wiki_unzip.KEY_dump, Xoi_cmd_wiki_import.KEY, Xoi_cmd_wiki_zip.KEY, Xoi_cmd_category2_build.KEY, Xoi_cmd_category2_page_props.KEY_category2, Xoi_cmd_category2_categorylinks.KEY_category2};
public static final String Wiki_cmd_custom = "wiki.custom", Wiki_cmd_dump_file = "wiki.dump_file";
public Gfo_thread_cmd Cmd_add(GfoMsg m) {Gfo_thread_cmd rv = Cmd_new(m); cmds.Add(rv); return rv;}
Gfo_thread_cmd Cmd_new(GfoMsg m) {
String cmd_key = m.ReadStr("v");
if (String_.Eq(cmd_key, Gfo_thread_cmd_download.KEY)) return new Gfo_thread_cmd_download().Init("downloading", m.ReadStr("src"), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg"))).Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());
else if (String_.Eq(cmd_key, Gfo_thread_cmd_unzip.KEY)) return new Gfo_thread_cmd_unzip().Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Init(app.Usr_dlg(), app.Gui_mgr().Kit(), app.Prog_mgr().App_decompress_bz2(), app.Prog_mgr().App_decompress_zip(), app.Prog_mgr().App_decompress_gz(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("src")), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg")));
else if (String_.Eq(cmd_key, Gfo_thread_cmd_replace.KEY)) return new Gfo_thread_cmd_replace().Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Init(app.Usr_dlg(), app.Gui_mgr().Kit(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("fil")));
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_image_cfg.KEY_dump)) return new Xoi_cmd_wiki_image_cfg(app, Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("fil"))).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_goto_page.KEY)) return new Xoi_cmd_wiki_goto_page(app, m.ReadStr("v")).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_msg_ok.KEY)) return new Xoi_cmd_msg_ok(app.Usr_dlg(), app.Gui_mgr().Kit(), m.ReadStr("v")).Owner_(this);
// else if (String_.Eq(cmd_key, Gfo_thread_exec_sync.KEY)) return new Gfo_thread_exec_sync(app.Usr_dlg(), app.Gui_mgr().Kit(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("url")), m.ReadStr("args")).Owner_(this);
else if (String_.Eq(cmd_key, Xoi_cmd_imageMagick_download.KEY_imageMagick)) return new Xoi_cmd_imageMagick_download(app.Usr_dlg(), app.Gui_mgr().Kit(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg"))).Owner_(this);
else if (String_.Eq(cmd_key, Wiki_cmd_dump_file)) return Wiki_cmd_dump_file_make(m);
else throw Exc_.new_unhandled(cmd_key);
}
Gfo_thread_cmd Wiki_cmd_dump_file_make(GfoMsg m) {
Xoi_cmd_dumpfile dumpfile = new Xoi_cmd_dumpfile().Parse_msg(m);
return dumpfile.Exec(this);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_process_async)) Process_async((Gfo_thread_cmd)m.CastObj("v"));
else if (ctx.Match(k, Invk_dump_add_many)) return Dump_add_many(m);
else if (ctx.Match(k, Invk_cmd_add)) return Cmd_add(m);
else if (ctx.Match(k, Invk_run)) Cmds_run();
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_process_async = "process_async", Invk_dump_add_many = "dump_add_many", Invk_run = "run", Invk_cmd_add = "cmd_add";
static final String GRP_KEY = "xowa.install_mgr.cmd_mgr";
}

199
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_wiki.java Normal file
View File

@@ -0,0 +1,199 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.gfui.*;
import gplx.core.threads.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.utils.*;
class Xoi_cmd_wiki_download extends Gfo_thread_cmd_download implements Gfo_thread_cmd { private Xoi_setup_mgr install_mgr; private String wiki_key, dump_date, dump_type;
public Xoi_cmd_wiki_download Ctor_download_(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date, String dump_type) {
this.install_mgr = install_mgr;
this.wiki_key = wiki_key;
this.dump_date = dump_date;
this.dump_type = dump_type;
this.Owner_(install_mgr);
return this;
}
@gplx.Virtual public String Download_file_ext() {return ".xml.bz2";} // wiki.download is primarily used for dump files; default to .xml.bz2; NOTE: changed from ".xml"; DATE:2013-11-07
@Override public String Async_key() {return Key_wiki_download;} public static final String Key_wiki_download = "wiki.download";
@Override public byte Async_init() {
Xoae_app app = install_mgr.App();
Xob_dump_file dump_file = Xob_dump_file.new_(wiki_key, dump_date, dump_type);
boolean connected = Xob_dump_file_.Connect_first(dump_file, install_mgr.Dump_mgr().Server_urls());
if (connected)
app.Usr_dlg().Note_many("", "", "url: ~{0}", dump_file.File_url());
else {
if (!Dump_servers_offline_msg_shown) {
app.Gui_mgr().Kit().Ask_ok("", "", "all dump servers are offline: ~{0}", String_.AryXtoStr(install_mgr.Dump_mgr().Server_urls()));
Dump_servers_offline_msg_shown = true;
}
}
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(dump_file.Wiki_type().Domain_bry());
Io_url root_dir = wiki.Fsys_mgr().Root_dir();
Io_url[] trg_fil_ary = Io_mgr.I.QueryDir_args(root_dir).FilPath_("*." + dump_type + Download_file_ext() + "*").ExecAsUrlAry();
Io_url trg = trg_fil_ary.length == 0 ? root_dir.GenSubFil(dump_file.File_name()) : trg_fil_ary[0];
this.Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());
this.Init("download", dump_file.File_url(), trg);
return super.Async_init();
}
private static boolean Dump_servers_offline_msg_shown = false;
}
class Xoi_cmd_wiki_unzip extends Gfo_thread_cmd_unzip implements Gfo_thread_cmd { public static final String KEY_dump = "wiki.unzip";
public Xoi_cmd_wiki_unzip(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.dump_date = dump_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, dump_date, dump_type;
@Override public String Async_key() {return KEY_dump;}
@Override public byte Async_init() {
Xoae_app app = install_mgr.App(); Gfui_kit kit = app.Gui_mgr().Kit();
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_u8(wiki_key));
Io_url wiki_dir = wiki.Import_cfg().Src_dir();
Io_url[] urls = Io_mgr.I.QueryDir_args(wiki_dir).Recur_(false).FilPath_("*.xml.bz2").ExecAsUrlAry();
if (urls.length == 0) {
kit.Ask_ok(GRP_KEY, "dump.unzip_latest.file_missing", "Could not find a dump file for ~{0} in ~{1}", wiki_key, wiki_dir.Raw());
return Gfo_thread_cmd_.Init_cancel_step;
}
Io_url src = urls[urls.length - 1];
Io_url trg = app.Fsys_mgr().Wiki_dir().GenSubFil_nest(wiki_key, src.NameOnly()); // NOTE: NameOnly() will strip trailing .bz2; EX: a.xml.bz2 -> a.xml
super.Init(app.Usr_dlg(), app.Gui_mgr().Kit(), app.Prog_mgr().App_decompress_bz2(), app.Prog_mgr().App_decompress_zip(), app.Prog_mgr().App_decompress_gz(), src, trg);
this.Term_cmd_for_src_(Term_cmd_for_src_move);
this.Term_cmd_for_src_url_(app.Fsys_mgr().Wiki_dir().GenSubFil_nest("#dump", "done", src.NameAndExt()));
if (Io_mgr.I.ExistsFil(trg)) {
int rslt = kit.Ask_yes_no_cancel(GRP_KEY, "target_exists", "Target file already exists: '~{0}'.\nDo you want to delete it?", trg.Raw());
switch (rslt) {
case Gfui_dlg_msg_.Btn_yes: Io_mgr.I.DeleteFil(trg); break;
case Gfui_dlg_msg_.Btn_no: return Gfo_thread_cmd_.Init_cancel_step;
case Gfui_dlg_msg_.Btn_cancel: return Gfo_thread_cmd_.Init_cancel_all;
default: throw Exc_.new_unhandled(rslt);
}
}
return Gfo_thread_cmd_.Init_ok;
}
static final String GRP_KEY = "xowa.thread.dump.unzip";
}
class Xoi_cmd_wiki_image_cfg extends Gfo_thread_cmd_replace implements Gfo_thread_cmd { public Xoi_cmd_wiki_image_cfg(Xoae_app app, Io_url url) {this.app = app; super.Init(app.Usr_dlg(), app.Gui_mgr().Kit(), url);} private Xoae_app app;
@Override public void Async_run() {
super.Async_run();
app.Cfg_mgr().Set_by_app("app.files.download.enabled", "y");
app.Cfg_mgr().Db_save_txt();
}
static final String GRP_KEY = "xowa.thread.dump.image_cfg";
public static final String KEY_dump = "wiki.image_cfg";
}
class Xoi_cmd_wiki_goto_page extends Gfo_thread_cmd_base implements Gfo_thread_cmd {
public Xoi_cmd_wiki_goto_page(Xoae_app app, String page) {this.app = app; this.page = page; this.Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());} private Xoae_app app; String page;
@Override public void Async_run() {kit.New_cmd_sync(this).Invk(GfsCtx.new_(), 0, Invk_goto_page, GfoMsg_.Null);}
private void Goto_page(String page) {app.Gui_mgr().Browser_win().Page__navigate_by_url_bar(page);}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_goto_page)) Goto_page(page);
else return super.Invk(ctx, ikey, k, m);
return this;
} private static final String Invk_goto_page = "goto_page";
public static final String KEY = "wiki.goto_page";
}
class Xoi_cmd_imageMagick_download extends Gfo_thread_cmd_download implements Gfo_thread_cmd {// private static final byte[] Bry_windows_zip = Bry_.new_a7("-windows.zip");
// static final String Src_imageMagick = "ftp://ftp.sunet.se/pub/multimedia/graphics/ImageMagick/binaries/";
public Xoi_cmd_imageMagick_download(Gfo_usr_dlg usr_dlg, Gfui_kit kit, Io_url trg) {this.Ctor(usr_dlg, kit); this.trg = trg;} Io_url trg;
@Override public byte Async_init() { // <a href="ImageMagick-6.8.1-9-Q16-x86-windows.zip">
// byte[] raw = xrg.Exec_as_bry(Src_imageMagick);
// int find_pos = Bry_finder.Find_fwd(raw, Bry_windows_zip); if (find_pos == Bry_.NotFound) return Fail();
// int bgn_pos = Bry_finder.Find_bwd(raw, Byte_ascii.Quote, find_pos); if (bgn_pos == Bry_.NotFound) return Fail();
// ++bgn_pos;
// int end_pos = Bry_finder.Find_fwd(raw, Byte_ascii.Quote, bgn_pos); if (end_pos == Bry_.NotFound) return Fail();
// String src = Src_imageMagick + String_.new_a7(Bry_.Mid(raw, bgn_pos, end_pos));
String src = "http://ftp.sunet.se/pub/multimedia/graphics/ImageMagick/binaries/ImageMagick-6.8.8-1-Q16-x86-windows.zip";
this.Init("downloading", src, trg);
return super.Async_init();
}
byte Fail() {
kit.Ask_ok(GRP_KEY, "windows_not_found", "Could not find Windows binary. Please download ImageMagick directly from the site.");
return Gfo_thread_cmd_.Init_cancel_step;
}
public static final String KEY_imageMagick = "download.imageMagick";
static final String GRP_KEY = "xowa.install.cmds.download.imageMagick";
}
class Xoi_cmd_msg_ok extends Gfo_thread_cmd_base implements Gfo_thread_cmd {
public Xoi_cmd_msg_ok(Gfo_usr_dlg usr_dlg, Gfui_kit kit, String msg) {this.msg = msg; this.Ctor(usr_dlg, kit);} private String msg;
@Override public boolean Async_term() {
kit.Ask_ok("msg_ok", "msg", msg);
return true;
}
public static final String KEY = "msg.ok";
}
// class Gfo_thread_exec_sync : Gfo_thread_cmd_base, Gfo_thread_cmd {
// public Gfo_thread_exec_sync(Gfo_usr_dlg usr_dlg, Gfui_kit kit, Io_url exec_url, String exec_args) {this.Ctor(usr_dlg, kit); this.exec_url = exec_url; this.exec_args = exec_args;} Io_url exec_url; String exec_args;
// public override byte Async_init() {
// if (!kit.Ask_yes_no(GRP_KEY, "confirm", "You are about to run '~{0}'. Please confirm that XOWA requested you to run '~{0}' and that you trust it to run on your machine.", exec_url.NameAndExt())) {
// usr_dlg.Warn_many(GRP_KEY, "confirm.fail", "program was untrusted: ~{0} ~{1}", exec_url.Raw(), exec_args);
// return Gfo_thread_cmd_.Init_cancel_all;
// }
// usr_dlg.Prog_many("exec_sync", "bgn", "running process. please wait; ~{0} ~{1}", exec_url.Raw(), exec_args);
// return Gfo_thread_cmd_.Init_ok;
// }
// public override void Async_run() {
// ProcessAdp process = new ProcessAdp().Exe_url_(exec_url).Args_str_(exec_args).Prog_dlg_(usr_dlg);
// process.Run_mode_(ProcessAdp.Run_mode_sync_block);
// process.Run();
// }
// public static final String KEY = "exec.sync";
// static final String GRP_KEY = "exec.sync";
// }
class Xoi_cmd_wiki_zip implements Gfo_thread_cmd {
public Xoi_cmd_wiki_zip(Xoi_setup_mgr install_mgr, String wiki_key, String wiki_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.wiki_date = wiki_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, wiki_date, dump_type;
public static final String KEY = "wiki.zip";
public void Cmd_ctor() {}
public String Async_key() {return KEY;}
public int Async_sleep_interval() {return Gfo_thread_cmd_.Async_sleep_interval_1_second;}
public boolean Async_prog_enabled() {return false;}
public void Async_prog_run(int async_sleep_sum) {}
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
public boolean Async_term() {
wiki.Tdb_fsys_mgr().Scan_dirs();
install_mgr.App().Usr_dlg().Log_many(GRP_KEY, "zip.end", "zip.end ~{0}", wiki_key);
install_mgr.App().Usr_dlg().Prog_many(GRP_KEY, "zip.done", "zip done");
return true;
}
public GfoInvkAble Owner() {return owner;} public Xoi_cmd_wiki_zip Owner_(GfoInvkAble v) {owner = v; return this;} GfoInvkAble owner;
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
public void Async_run() {
running = true;
install_mgr.App().Usr_dlg().Log_many(GRP_KEY, "zip.bgn", "zip.bgn ~{0}", wiki_key);
Thread_adp_.invk_(this.Async_key(), this, Invk_process_async).Start();
}
public boolean Async_running() {
return running;
}
boolean running, delete_dirs_page = true, notify_done = true;
private void Process_async() {
Xoae_app app = install_mgr.App();
Xob_bldr bldr = app.Bldr();
wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_a7(wiki_key));
wiki.Init_assert();
bldr.Cmd_mgr().Clear();
bldr.Pause_at_end_(false);
((Xob_deploy_zip_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_deploy_zip)).Delete_dirs_page_(delete_dirs_page);
bldr.Run();
app.Usr_dlg().Prog_none(GRP_KEY, "clear", "");
app.Usr_dlg().Note_none(GRP_KEY, "clear", "");
running = false;
} private Xowe_wiki wiki;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_process_async)) Process_async();
else if (ctx.Match(k, Invk_owner)) return owner;
else if (ctx.Match(k, Invk_delete_dirs_page_)) delete_dirs_page = m.ReadYn("v");
else if (ctx.Match(k, Invk_notify_done_)) notify_done = m.ReadYn("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_process_async = "run_async", Invk_owner = "owner", Invk_delete_dirs_page_ = "delete_dirs_page_", Invk_notify_done_ = "notify_done_";
private static final String GRP_KEY = "xowa.thread.op.wiki.zip";
}

117
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_wiki_import.java Normal file
View File

@@ -0,0 +1,117 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.threads.*; import gplx.xowa.bldrs.*; import gplx.xowa.gui.views.*; import gplx.xowa.bldrs.cmds.utils.*;
import gplx.xowa.html.hrefs.*;
class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
public Xoi_cmd_wiki_import(Xoi_setup_mgr install_mgr, String wiki_key, String wiki_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.wiki_date = wiki_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, wiki_date, dump_type;
public static final String KEY = "wiki.import";
public void Cmd_ctor() {}
public String Async_key() {return KEY;}
public int Async_sleep_interval() {return Gfo_thread_cmd_.Async_sleep_interval_1_second;}
public boolean Async_prog_enabled() {return false;}
public void Async_prog_run(int async_sleep_sum) {}
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
public boolean Async_term() {
install_mgr.App().Usr_dlg().Log_many(GRP_KEY, "import.end", "import.end ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
return true;
}
public GfoInvkAble Owner() {return owner;} public Xoi_cmd_wiki_import Owner_(GfoInvkAble v) {owner = v; return this;} GfoInvkAble owner;
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
public void Async_run() {
running = true;
install_mgr.App().Usr_dlg().Log_many(GRP_KEY, "import.bgn", "import.bgn ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
Thread_adp_.invk_(this.Async_key(), this, Invk_process_async).Start();
}
public boolean Async_running() {
return running;
}
boolean running;
public boolean Import_move_bz2_to_done() {return import_move_bz2_to_done;} public Xoi_cmd_wiki_import Import_move_bz2_to_done_(boolean v) {import_move_bz2_to_done = v; return this;} private boolean import_move_bz2_to_done = true;
private void Process_txt(Xob_bldr bldr) {
((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_id);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_search_title);
if (wiki.Import_cfg().Category_version() == gplx.xowa.ctgs.Xoa_ctg_mgr.Version_1)
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_category);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_calc_stats);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_core_term);
}
private void Process_sql(Xob_bldr bldr) {
((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_css);
if (wiki.Import_cfg().Category_version() == gplx.xowa.ctgs.Xoa_ctg_mgr.Version_1) {
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_core_v1);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_core);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_link);
}
if (wiki.Appe().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.specials.search.Xows_page__search.Version_2)
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_search_wkr);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_term);
}
private void Process_async() {
Xoae_app app = install_mgr.App();
app.Usr_dlg().Prog_one("", "", "preparing import: ~{0}", wiki_key);
Xob_bldr bldr = app.Bldr();
wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_a7(wiki_key));
wiki.Init_assert();
bldr.Cmd_mgr().Clear();
bldr.Pause_at_end_(false);
Io_url src_url = wiki.Import_cfg().Src_rdr().Url();
if (install_mgr.Dump_mgr().Wiki_storage_type_is_sql())
Process_sql(bldr);
else
Process_txt(bldr);
bldr.Run();
app.Usr_dlg().Prog_none(GRP_KEY, "clear", ""); app.Usr_dlg().Note_none(GRP_KEY, "clear", "");
app.Usere().Available_from_fsys();
wiki.Init_needed_(true);
wiki.Html_mgr().Page_wtr_mgr().Init_(true);
wiki.Init_assert();
if (String_.Eq(src_url.Ext(), ".xml")) {
if (app.Setup_mgr().Dump_mgr().Delete_xml_file())
Io_mgr.I.DeleteFil(src_url);
}
else if (String_.Eq(src_url.Ext(), ".bz2")) {
Io_url trg_fil = app.Fsys_mgr().Wiki_dir().GenSubFil_nest("#dump", "done", src_url.NameAndExt());
if (import_move_bz2_to_done)
Io_mgr.I.MoveFil_args(src_url, trg_fil, true).Exec();
}
running = false;
wiki.Import_cfg().Src_fil_xml_(null).Src_fil_bz2_(null); // reset file else error when going from Import/Script to Import/List
app.Gui_mgr().Kit().New_cmd_sync(this).Invk(GfsCtx.new_(), 0, Invk_open_wiki, GfoMsg_.Null);
} private Xowe_wiki wiki;
private void Open_wiki(String wiki_key) {
Xog_win_itm main_win = install_mgr.App().Gui_mgr().Browser_win();
if (main_win.Active_page() == null) return; // will be null when invoked through cmd-line
byte[] url = Bry_.Add(wiki.Domain_bry(), Xoh_href_parser.Href_wiki_bry, wiki.Props().Main_page());
main_win.Page__navigate_by_url_bar(String_.new_u8(url));
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_process_async)) Process_async();
else if (ctx.Match(k, Invk_owner)) return owner;
else if (ctx.Match(k, Invk_open_wiki)) Open_wiki(wiki_key);
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_process_async = "run_async", Invk_owner = "owner", Invk_open_wiki = "open_wiki";
static final String GRP_KEY = "xowa.thread.op.build";
}

127
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_wiki_tst.java Normal file
View File

@@ -0,0 +1,127 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import org.junit.*;
import gplx.brys.*; import gplx.core.threads.*; import gplx.xowa.wikis.*; import gplx.xowa.setup.maints.*; import gplx.xowa.xtns.wdatas.imports.*;
public class Xoi_cmd_wiki_tst {
@Test public void Run() { // MAINT
// Bld_import_list(Xow_wmf_api_mgr.Wikis);
// Bld_cfg_files(Xow_wmf_api_mgr.Wikis); // NOTE: remember to carry over the wikisource / page / index commands from the existing xowa_build_cfg.gfs; also, only run the xowa_build_cfg.gfs once; DATE:2013-10-15; last run: DATE:2014-09-09
}
public void Bld_import_list(String... ary) {
int ary_len = ary.length;
Bry_bfr bfr = Bry_bfr.reset_(255);
Wmf_latest_parser parser = new Wmf_latest_parser();
Xob_dump_file dump_file = new Xob_dump_file();
Bry_fmtr_arg_time time_fmtr = new Bry_fmtr_arg_time();
for (int i = 0; i < ary_len; i++)
Bld_import_list_itm2(bfr, parser, dump_file, time_fmtr, ary, i);
Io_mgr.I.SaveFilStr("C:\\temp.txt", bfr.Xto_str());
}
private void Bld_import_list_itm2(Bry_bfr bfr, Wmf_latest_parser parser, Xob_dump_file dump_file, Bry_fmtr_arg_time time_fmtr, String[] ary, int i) {
String domain_str = ary[i];
byte[] domain_bry = Bry_.new_a7(domain_str);
Xow_domain domain_itm = Xow_domain_.parse(domain_bry);
byte[] wmf_key_bry = Bry_.Replace(Xow_wiki_alias.Build_alias(domain_itm), Byte_ascii.Dash, Byte_ascii.Underline);
String wmf_key = String_.new_u8(wmf_key_bry);
String url = "https://dumps.wikimedia.org/" + wmf_key + "/latest";
byte[] latest_html = null;
for (int j = 0; j < 5; ++j) {
latest_html = Io_mgr.I.DownloadFil_args("", Io_url_.Empty).Exec_as_bry(url);
if (latest_html != null) break;
Tfds.Write("fail|" + url);
if (j == 4) return;
}
Tfds.Write("pass|" + url);
parser.Parse(latest_html);
dump_file.Ctor(domain_str, "latest", Xow_wiki_alias.Key_pages_articles);
dump_file.Server_url_(Xob_dump_file_.Server_wmf_https);
byte[] pages_articles_key = Bry_.new_a7(wmf_key + "-latest-pages-articles.xml.bz2");
Wmf_latest_itm latest_itm = parser.Get_by(pages_articles_key);
bfr.Add(domain_bry).Add_byte_pipe();
bfr.Add_str(dump_file.File_url()).Add_byte_pipe();
bfr.Add(Xow_domain_type_.Get_type_as_bry(domain_itm.Domain_tid())).Add_byte_pipe();
long src_size = latest_itm.Size();
bfr.Add_long_variable(src_size).Add_byte_pipe();
bfr.Add_str(gplx.ios.Io_size_.To_str(src_size)).Add_byte_pipe();
time_fmtr.Seconds_(Math_.Div_safe_as_long(src_size, 1000000)).XferAry(bfr, 0);
bfr.Add_byte_pipe();
bfr.Add_str(latest_itm.Date().XtoStr_fmt_yyyy_MM_dd_HH_mm());
bfr.Add_byte_pipe();
bfr.Add_str(dump_file.Dump_date());
bfr.Add_byte_nl();
}
/*
private void Bld_import_list_itm(Bry_bfr bfr, Xob_dump_file dump_file, Bry_fmtr_arg_time time_fmtr, String[] ary, int i) {
String itm = ary[i];
dump_file.Ctor(itm, "latest", Xow_wiki_alias.Key_pages_articles);
int count = 0;
while (count++ < 1) {
dump_file.Server_url_(Xob_dump_file_.Server_wmf);
if (dump_file.Connect()) break;
Tfds.WriteText(String_.Format("retrying: {0} {1}\n", count, dump_file.File_modified()));
Thread_adp_.Sleep(15000); // wait for connection to reset
}
if (count == 10) {
Tfds.WriteText(String_.Format("failed: {0}\n", dump_file.File_url()));
return;
}
else
Tfds.WriteText(String_.Format("passed: {0}\n", itm));
bfr.Add_str(itm).Add_byte_pipe();
bfr.Add_str(dump_file.File_url()).Add_byte_pipe();
bfr.Add(Xow_domain_type_.Get_type_as_bry(dump_file.Wiki_type().Wiki_tid())).Add_byte_pipe();
// Xol_lang_itm lang_itm = Xol_lang_itm_.Get_by_key(wiki_type.Lang_key());
// if (lang_itm == null) lang_itm = Xol_lang_itm_.Get_by_key(Xol_lang_.Key_en); // commons, species, meta, etc will have no lang
// bfr.Add(lang_itm.Local_name()).Add_byte_pipe();
// bfr.Add(lang_itm.Canonical_name()).Add_byte_pipe();
long src_size = dump_file.File_len();
bfr.Add_long_variable(src_size).Add_byte_pipe();
bfr.Add_str(gplx.ios.Io_size_.To_str(src_size)).Add_byte_pipe();
time_fmtr.Seconds_(Math_.Div_safe_as_long(src_size, 1000000)).XferAry(bfr, 0);
bfr.Add_byte_pipe();
bfr.Add_str(dump_file.File_modified().XtoStr_fmt_yyyy_MM_dd_HH_mm());
bfr.Add_byte_pipe();
// bfr.Add_str(String_.ConcatWith_any(",", (Object[])dump_file.Dump_available_dates()));
// bfr.Add_byte_pipe();
bfr.Add_str(dump_file.Dump_date());
bfr.Add_byte_nl();
Thread_adp_.Sleep(1000);
}
*/
public void Bld_cfg_files(String... ary) {
Bry_bfr bfr = Bry_bfr.reset_(255);
gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_api api = new gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_api();
gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_wiki wiki = new gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_wiki();
int ary_len = ary.length;
for (int i = 0; i < ary_len; i++) {
String wiki_domain = ary[i];
try {
byte[] xml = api.Exec_api(api.Api_src(wiki_domain));
wiki.Wiki_domain_(Bry_.new_a7(wiki_domain));
api.Parse(wiki, String_.new_u8(xml));
api.Build_cfg(bfr, wiki);
}
catch (Exception e) {
ConsoleAdp._.WriteLine(Err_.Message_gplx_brief(e));
}
}
bfr.Add_str_a7("app.bldr.wiki_cfg_bldr.run;").Add_byte_nl();
Io_mgr.I.SaveFilStr("C:\\xowa_build_cfg.gfs", bfr.Xto_str());
}
}

97
400_xowa/src_240_install/gplx/xowa/Xoi_dump_mgr.java Normal file
View File

@@ -0,0 +1,97 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*;
public class Xoi_dump_mgr implements GfoInvkAble {
public String[] Server_urls() {return server_urls;} private String[] server_urls = String_.Ary(Xob_dump_file_.Server_wmf_https, Xob_dump_file_.Server_your_org, Xob_dump_file_.Server_c3sl, Xob_dump_file_.Server_masaryk);
public String[] Custom_cmds() {return custom_cmds;} private String[] custom_cmds = String_.Ary(Xoi_cmd_wiki_download.Key_wiki_download, Xoi_cmd_wiki_import.KEY);
public byte Data_storage_format() {return data_storage_format;} public Xoi_dump_mgr Data_storage_format_(byte v) {data_storage_format = v; return this;} private byte data_storage_format = gplx.ios.Io_stream_.Tid_gzip;
public long Db_text_max() {return db_text_max;} private long db_text_max = (long)3000 * Io_mgr.Len_mb;
public long Db_categorylinks_max() {return db_categorylinks_max;} private long db_categorylinks_max = (long)3600 * Io_mgr.Len_mb;
public long Db_wikidata_max() {return db_wikidata_max;} private long db_wikidata_max = (long)3600 * Io_mgr.Len_mb;
public byte Wiki_storage_type() {return wiki_storage_type;} private byte wiki_storage_type = Wiki_storage_type_sqlite;
public boolean Wiki_storage_type_is_sql() {return wiki_storage_type == Wiki_storage_type_sqlite;}
public String Db_ns_map() {return db_ns_map;} private String db_ns_map = "Template~Module";
public boolean Css_wiki_update() {return css_wiki_update;} private boolean css_wiki_update = true;
public boolean Css_commons_download() {return css_commons_download;} private boolean css_commons_download = true; // changed from false to true; DATE:2014-10-19
public boolean Delete_xml_file() {return delete_xml_file;} private boolean delete_xml_file = true;
public byte Search_version() {return search_version;} private byte search_version = gplx.xowa.specials.search.Xows_page__search.Version_2;
public boolean Import_bz2_by_stdout() {return import_bz2_by_stdout;} private boolean import_bz2_by_stdout = true;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_server_urls)) return String_.Concat_with_str(",\n", server_urls);
else if (ctx.Match(k, Invk_server_urls_)) server_urls = m.ReadStrAryIgnore("v", ",", "\n");
else if (ctx.Match(k, Invk_custom_cmds)) return String_.Concat_with_str(",", custom_cmds);
else if (ctx.Match(k, Invk_custom_cmds_)) custom_cmds = String_.Ary_filter(m.ReadStrAry("v", ","), Xoi_cmd_mgr.Wiki_cmds_valid);
else if (ctx.Match(k, Invk_data_storage_format)) return Io_stream_.Obsolete_to_str(data_storage_format);
else if (ctx.Match(k, Invk_data_storage_format_)) data_storage_format = Io_stream_.Obsolete_to_tid(m.ReadStr("v"));
else if (ctx.Match(k, Invk_data_storage_format_list)) return Options_data_storage_format_list;
else if (ctx.Match(k, Invk_db_text_max)) return db_text_max / Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_text_max_)) db_text_max = m.ReadLong("v") * Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_categorylinks_max)) return db_categorylinks_max / Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_categorylinks_max_)) db_categorylinks_max = m.ReadLong("v") * Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_wikidata_max)) return db_wikidata_max / Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_wikidata_max_)) db_wikidata_max = m.ReadLong("v") * Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_ns_map)) return db_ns_map;
else if (ctx.Match(k, Invk_db_ns_map_)) db_ns_map = m.ReadStr("v");
else if (ctx.Match(k, Invk_wiki_storage_type)) return Wiki_storage_type_str(wiki_storage_type);
else if (ctx.Match(k, Invk_wiki_storage_type_)) wiki_storage_type = Wiki_storage_type_parse(m.ReadStr("v"));
else if (ctx.Match(k, Invk_wiki_storage_type_list)) return Options_storage_type_list;
else if (ctx.Match(k, Invk_css_wiki_update)) return Yn.Xto_str(css_wiki_update);
else if (ctx.Match(k, Invk_css_wiki_update_)) css_wiki_update = m.ReadYn("v");
else if (ctx.Match(k, Invk_css_commons_download)) return Yn.Xto_str(css_commons_download);
else if (ctx.Match(k, Invk_css_commons_download_)) css_commons_download = m.ReadYn("v");
else if (ctx.Match(k, Invk_delete_xml_file)) return Yn.Xto_str(delete_xml_file);
else if (ctx.Match(k, Invk_delete_xml_file_)) delete_xml_file = m.ReadYn("v");
else if (ctx.Match(k, Invk_search_version)) return Options_search_version_str(search_version);
else if (ctx.Match(k, Invk_search_version_)) search_version = Options_search_version_parse(m.ReadStr("v"));
else if (ctx.Match(k, Invk_search_version_list)) return Options_search_version_list;
else if (ctx.Match(k, Invk_import_bz2_by_stdout)) return Yn.Xto_str(import_bz2_by_stdout);
else if (ctx.Match(k, Invk_import_bz2_by_stdout_)) import_bz2_by_stdout = m.ReadYn("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
}
private static final String Invk_server_urls = "server_urls", Invk_server_urls_ = "server_urls_", Invk_custom_cmds = "custom_cmds", Invk_custom_cmds_ = "custom_cmds_"
, Invk_data_storage_format = "data_storage_format", Invk_data_storage_format_ = "data_storage_format_", Invk_data_storage_format_list = "data_storage_format_list"
, Invk_db_text_max = "db_text_max", Invk_db_text_max_ = "db_text_max_", Invk_db_categorylinks_max = "db_categorylinks_max", Invk_db_categorylinks_max_ = "db_categorylinks_max_", Invk_db_wikidata_max = "db_wikidata_max", Invk_db_wikidata_max_ = "db_wikidata_max_"
, Invk_db_ns_map = "db_ns_map", Invk_db_ns_map_ = "db_ns_map_"
, Invk_wiki_storage_type = "wiki_storage_type", Invk_wiki_storage_type_ = "wiki_storage_type_", Invk_wiki_storage_type_list = "wiki_storage_type_list"
, Invk_css_wiki_update = "css_wiki_update", Invk_css_wiki_update_ = "css_wiki_update_"
, Invk_css_commons_download = "css_commons_download", Invk_css_commons_download_ = "css_commons_download_"
, Invk_delete_xml_file = "delete_xml_file", Invk_delete_xml_file_ = "delete_xml_file_"
, Invk_search_version = "search_version", Invk_search_version_ = "search_version_", Invk_search_version_list = "search_version_list"
, Invk_import_bz2_by_stdout = "import_bz2_by_stdout", Invk_import_bz2_by_stdout_ = "import_bz2_by_stdout_"
;
private static KeyVal[] Options_data_storage_format_list = KeyVal_.Ary(KeyVal_.new_(".xdat", "text"), KeyVal_.new_(".gz", "gzip"), KeyVal_.new_(".bz2", "bzip2")); // removed .zip; DATE:2014-05-13; updated aliases; DATE:2014-06-20
static final byte Wiki_storage_type_xdat = 1, Wiki_storage_type_sqlite = 2;
private static final KeyVal[] Options_storage_type_list = KeyVal_.Ary(KeyVal_.new_("sqlite"), KeyVal_.new_("xdat")); // DEPRECATED: KeyVal_.new_("xdat"); DATE:2015-03-30
public static String Wiki_storage_type_str(byte v) {
switch (v) {
case Xoi_dump_mgr.Wiki_storage_type_xdat : return "xdat";
case Xoi_dump_mgr.Wiki_storage_type_sqlite : return "sqlite";
default : throw Exc_.new_unhandled(v);
}
}
public static byte Wiki_storage_type_parse(String v) {
if (String_.Eq(v, "xdat")) return Xoi_dump_mgr.Wiki_storage_type_xdat;
else if (String_.Eq(v, "sqlite")) return Xoi_dump_mgr.Wiki_storage_type_sqlite;
else throw Exc_.new_unhandled(v);
}
private static final KeyVal[] Options_search_version_list = KeyVal_.Ary(KeyVal_.new_("1"), KeyVal_.new_("2"));
public static String Options_search_version_str(byte v) {return Byte_.Xto_str(v);}
public static byte Options_search_version_parse(String v) {return Byte_.parse_(v);}
}

47
400_xowa/src_240_install/gplx/xowa/Xoi_mirror_parser.java Normal file
View File

@@ -0,0 +1,47 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
public class Xoi_mirror_parser {
public String[] Parse(String raw_str) {
if (String_.Len_eq_0(raw_str)) return String_.Ary_empty;
byte[] raw = Bry_.new_u8(raw_str);
List_adp rv = List_adp_.new_();
int pos = 0;
while (true) {
int bgn = Bry_finder.Find_fwd(raw, CONST_href_bgn, pos); if (bgn == Bry_.NotFound) break;
bgn += CONST_href_bgn.length;
int end = Bry_finder.Find_fwd(raw, CONST_href_end, bgn); if (end == Bry_.NotFound) return String_.Ary_empty;
byte[] date = Bry_.Mid(raw, bgn, end);
pos = end + CONST_href_end.length;
if (Bry_.Match(date, CONST_date_parent_dir)) continue;
int date_pos_last = date.length - 1;
if (date_pos_last == -1) return String_.Ary_empty;
if (date[date_pos_last] == Byte_ascii.Slash) date = Bry_.Mid(date, 0, date_pos_last); // trim trailing /; EX: "20130101/" -> "20130101"
rv.Add(String_.new_u8(date));
}
return rv.To_str_ary();
} static final byte[] CONST_href_bgn = Bry_.new_a7("<a href=\""), CONST_href_end = Bry_.new_a7("\""), CONST_date_parent_dir = Bry_.new_a7("../");
public static String Find_last_lte(String[] ary, String comp) { // assuming sorted ary, find last entry that is lte comp
int len = ary.length;
for (int i = len - 1; i > -1; i--) {
String itm = ary[i];
if (CompareAble_.Is_lessOrSame(itm, comp)) return itm;
}
return "";
}
}

60
400_xowa/src_240_install/gplx/xowa/Xoi_mirror_parser_tst.java Normal file
View File

@@ -0,0 +1,60 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import org.junit.*;
public class Xoi_mirror_parser_tst {
@Test public void Basic() {
Tst_parse(String_.Concat_lines_nl
( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
, "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.1//EN\" \"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd\">"
, "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\">"
, "<head>"
, "<title>Index of /simplewiki/</title>"
, "<link rel=\"stylesheet\" type=\"text/css\" href=\"/pub/misc/lighttpd-white-dir.css\" />"
, "</head>"
, "<body>"
, "<h2>Index of /simplewiki/</h2>"
, "<div class=\"list\">"
, "<table summary=\"Directory Listing\" cellpadding=\"0\" cellspacing=\"0\">"
, "<thead><tr><th class=\"n\">Name</th><th class=\"m\">Last Modified</th><th class=\"s\">Size</th><th class=\"t\">Type</th></tr></thead>"
, "<tbody>"
, "<tr><td class=\"n\"><a href=\"../\">Parent Directory</a>/</td><td class=\"m\">&nbsp;</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"20120516/\">20120516</a>/</td><td class=\"m\">2012-May-17 01:04:39</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"20121220/\">20121220</a>/</td><td class=\"m\">2012-Dec-20 20:15:55</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"20130214/\">20130214</a>/</td><td class=\"m\">2013-Feb-14 06:28:41</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "<tr><td class=\"n\"><a href=\"latest/\">latest</a>/</td><td class=\"m\">2013-Feb-14 06:28:41</td><td class=\"s\">- &nbsp;</td><td class=\"t\">Directory</td></tr>"
, "</tbody>"
, "</table>"
, "</div>"
, "<div class=\"foot\">lighttpd</div>"
, "</body>"
, "</html>"
), String_.Ary("20120516", "20121220", "20130214", "latest"));
}
@Test public void Find_last_lte() {
Tst_find_last_lte(String_.Ary("20120516", "20121220", "20130214", "latest"), "20130101", "20121220");
Tst_find_last_lte(String_.Ary("20120516", "20121220", "20130214", "latest"), "20120101", "");
}
private void Tst_parse(String raw, String[] expd) {
Xoi_mirror_parser parser = new Xoi_mirror_parser();
Tfds.Eq_ary_str(expd, parser.Parse(raw));
}
private void Tst_find_last_lte(String[] ary, String comp, String expd) {
Tfds.Eq(expd, Xoi_mirror_parser.Find_last_lte(ary, comp));
}
}

44
400_xowa/src_240_install/gplx/xowa/Xoi_setup_mgr.java Normal file
View File

@@ -0,0 +1,44 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.gfui.*; import gplx.xowa.setup.addons.*;
import gplx.xowa.setup.maints.*;
public class Xoi_setup_mgr implements GfoInvkAble {
public Xoi_setup_mgr(Xoae_app app) {
this.app = app;
cmd_mgr = new Xoi_cmd_mgr(this);
maint_mgr = new Xoa_maint_mgr(app);
}
public void Init_by_app(Xoae_app app) {
addon_mgr.Init_by_app(app);
}
public Xoae_app App() {return app;} private Xoae_app app;
public Xoi_cmd_mgr Cmd_mgr() {return cmd_mgr;} private Xoi_cmd_mgr cmd_mgr;
public Xoi_dump_mgr Dump_mgr() {return dump_mgr;} private Xoi_dump_mgr dump_mgr = new Xoi_dump_mgr();
public Xoi_addon_mgr Addon_mgr() {return addon_mgr;} private Xoi_addon_mgr addon_mgr = new Xoi_addon_mgr();
public Xoa_maint_mgr Maint_mgr() {return maint_mgr;} private Xoa_maint_mgr maint_mgr;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_cmds)) return cmd_mgr;
else if (ctx.Match(k, Invk_dumps)) return dump_mgr;
else if (ctx.Match(k, Invk_addons)) return addon_mgr;
else if (ctx.Match(k, Invk_maint)) return maint_mgr;
else return GfoInvkAble_.Rv_unhandled;
}
static final String Invk_cmds = "cmds", Invk_dumps = "dumps", Invk_addons = "addons", Invk_maint = "maint";
static final String GRP_KEY = "xowa.setup";
}

86
400_xowa/src_240_install/gplx/xowa/Xow_cfg_wiki_core.java Normal file
View File

@@ -0,0 +1,86 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
public class Xow_cfg_wiki_core {
public Xow_cfg_wiki_core(Xowe_wiki wiki) {this.wiki = wiki;} private Xowe_wiki wiki;
public void Save() {
}
public byte[] Build_gfs() {
Gfs_bldr wtr = new Gfs_bldr();
Xow_wiki_props props = wiki.Props();
wtr.Add_proc_init_many(Xowe_wiki.Invk_props).Add_nl();
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_bldr_version_).Add_parens_str(props.Bldr_version()).Add_nl();
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_main_page_).Add_parens_str(props.Main_page()).Add_nl();
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_siteinfo_misc_).Add_parens_str(props.Siteinfo_misc()).Add_nl();
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_siteinfo_mainpage_).Add_parens_str(props.Siteinfo_mainpage()).Add_nl();
wtr.Add_term_nl();
wtr.Add_proc_init_many(Xowe_wiki.Invk_ns_mgr).Add_nl();
wtr.Add_proc_cont_one(Xow_ns_mgr.Invk_clear).Add_nl();
wtr.Add_proc_cont_one(Xow_ns_mgr.Invk_load).Add_paren_bgn().Add_nl();
wtr.Add_quote_xtn_bgn();
Xol_csv_parser csv_parser = Xol_csv_parser._;
int nms_len = wiki.Ns_mgr().Count();
for (int i = 0; i < nms_len; i++) {
Xow_ns ns = wiki.Ns_mgr().Ords_get_at(i);
wtr.Bfr().Add_int_variable(ns.Id()).Add_byte_pipe().Add_int_fixed(ns.Case_match(), 1).Add_byte_pipe();
csv_parser.Save(wtr.Bfr(), ns.Name_txt());
wtr.Add_nl();
}
wtr.Add_quote_xtn_end();
wtr.Add_paren_end().Add_term_nl();
return wtr.Xto_bry();
}
public void Load(String raw) {
wiki.Appe().Gfs_mgr().Run_str_for(wiki, raw);
}
public static void Load_ns_(Xow_ns_mgr ns_mgr, byte[] src) {// 10|1|Template
int len = src.length; int pos = 0, fld_bgn = 0, fld_idx = 0, row_bgn = 0;
int cur_id = Int_.MinValue; byte cur_case_match = Byte_.Max_value_127; byte[] cur_name = Bry_.Empty;
Xol_csv_parser csv_parser = Xol_csv_parser._;
while (true) {
boolean last = pos == len;
byte b = last ? Byte_ascii.Nl : src[pos];
switch (b) {
case Byte_ascii.Pipe:
switch (fld_idx) {
case 0: cur_id = Bry_.Xto_int_or(src, fld_bgn, pos, Int_.MinValue); if (cur_id == Int_.MinValue) throw Exc_.new_("failed to load id", "id", String_.new_u8(src, fld_bgn, pos)); break;
case 1: cur_case_match = Bry_.Xto_byte_by_int(src, fld_bgn, pos, Byte_.Max_value_127); if (cur_id == Byte_.Max_value_127) throw Exc_.new_("failed to load match", "id", String_.new_u8(src, fld_bgn, pos)); break;
default: throw Exc_.new_unhandled(fld_idx);
}
fld_bgn = pos + 1;
++fld_idx;
break;
case Byte_ascii.Nl:
if (pos > row_bgn) { // guard against blank lines
cur_name = csv_parser.Load(src, fld_bgn, pos);
ns_mgr.Add_new(cur_id, cur_name, cur_case_match, false);
}
cur_id = Int_.MinValue; cur_case_match = Byte_.Max_value_127;
fld_bgn = pos + 1;
fld_idx = 0;
row_bgn = fld_bgn;
break;
}
if (last) break;
++pos;
}
ns_mgr.Init_w_defaults();
}
static final String Url_wiki_core_gfs = "wiki_core.gfs";
static final String GRP_KEY = "xowa.wiki.cfg.wiki_core";
}

111
400_xowa/src_240_install/gplx/xowa/Xow_cfg_wiki_core_tst.java Normal file
View File

@@ -0,0 +1,111 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import org.junit.*;
public class Xow_cfg_wiki_core_tst {
Xow_cfg_wiki_core_fxt fxt = new Xow_cfg_wiki_core_fxt();
@Before public void init() {fxt.Clear();}
@Test public void Save() {
fxt.Save_tst(Xoa_app_.Version, "Main_Page", "Wikipedia|MediaWiki 1.21wmf5|first-letter|", 0, "User test", Const_wiki_core_cfg);
}
@Test public void Load_and_save() {
fxt.Load_and_save_tst(Const_wiki_core_cfg);
}
@Test public void Load() {
fxt.Load_tst(String_.Concat_lines_nl
( "ns_mgr.clear.load("
, "<:['"
, "0|0|" // DEFECT: test that 0 sets case_match to case-sensitive; empty name was causing it to "default" to 1; DATE:2013-01-30
, "" // test to make sure blank line doesn't fail
, "4|1|Wikipedia" // test to make sure that values are updated after blank line
, ""
, "']:>"
, ");"
)
, fxt.ns_(Xow_ns_.Id_main , true, "")
, fxt.ns_(Xow_ns_.Id_project , false, "Wikipedia")
);
}
public static final String Const_wiki_core_cfg = String_.Concat_lines_nl
( "props"
, ".bldr_version_('" + Xoa_app_.Version + "')"
, ".main_page_('Main_Page')"
, ".siteinfo_misc_('Wikipedia|MediaWiki 1.21wmf5|first-letter|')"
, ".siteinfo_mainpage_('')"
, ";"
, "ns_mgr"
, ".clear"
, ".load("
, "<:['"
, "-2|1|Media"
, "-1|1|Special"
, "0|1|"
, "1|1|Talk"
, "2|0|User test" // NOTE: intentionally changing this to "0|User test" to differ from existing
, "3|1|User talk"
, "4|1|Wikipedia"
, "5|1|Wikipedia talk"
, "6|1|File"
, "7|1|File talk"
, "8|1|MediaWiki"
, "9|1|MediaWiki talk"
, "10|1|Template"
, "11|1|Template talk"
, "12|1|Help"
, "13|1|Help talk"
, "14|1|Category"
, "15|1|Category talk"
, "100|1|Portal"
, "101|1|Portal talk"
, "108|1|Book"
, "109|1|Book talk"
, "828|1|Module"
, "829|1|Module talk"
, "']:>"
, ");"
);
}
class Xow_cfg_wiki_core_fxt {
Xoae_app app; Xowe_wiki wiki;
public Xowe_wiki Wiki() {return wiki;}
public void Clear() {
app = Xoa_app_fxt.app_();
wiki = Xoa_app_fxt.wiki_tst_(app);
}
public void Save_tst(String bldr_version, String main_page, String siteinfo_misc, int ns_user_case_match, String ns_user_name, String expd) {
wiki.Props().Bldr_version_(Bry_.new_a7(bldr_version)).Main_page_(Bry_.new_a7(main_page)).Siteinfo_misc_(Bry_.new_a7(siteinfo_misc));
Xow_ns ns_user = wiki.Ns_mgr().Ids_get_or_null(Xow_ns_.Id_user);
ns_user.Case_match_((byte)ns_user_case_match); ns_user.Name_bry_(Bry_.new_a7(ns_user_name));
Tfds.Eq_str_lines(expd, String_.new_a7(wiki.Cfg_wiki_core().Build_gfs()));
}
public void Load_and_save_tst(String raw) {
wiki.Cfg_wiki_core().Load(raw);
Tfds.Eq_str_lines(raw, String_.new_a7(wiki.Cfg_wiki_core().Build_gfs()));
}
public Xow_ns ns_(int id, boolean case_match, String name) {return new Xow_ns(id, case_match ? Xow_ns_case_.Id_all : Xow_ns_case_.Id_1st, Bry_.new_u8(name), false);}
public void Load_tst(String raw, Xow_ns... expd_ary) {
wiki.Cfg_wiki_core().Load(raw);
int expd_len = expd_ary.length;
for (int i = 0; i < expd_len; i++) {
Xow_ns expd = expd_ary[i];
Xow_ns actl = wiki.Ns_mgr().Ids_get_or_null(expd.Id());
Tfds.Eq(expd.Case_match(), actl.Case_match(), Int_.Xto_str(expd.Id()));
Tfds.Eq(expd.Name_str(), actl.Name_str(), Int_.Xto_str(expd.Id()));
}
}
}