1
0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
This commit is contained in:
gnosygnu
2015-04-05 21:00:31 -04:00
parent 80b9928b5c
commit 18dcd3f89e
615 changed files with 9826 additions and 8619 deletions

View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.threads; import gplx.*;
import gplx.gfui.*; import gplx.xowa.bldrs.files.*;
import gplx.gfui.*; import gplx.xowa.bldrs.cmds.utils.*;
public class Gfo_thread_cmd_unzip implements Gfo_thread_cmd {
public Gfo_thread_cmd_unzip Init(Gfo_usr_dlg usr_dlg, Gfui_kit kit, ProcessAdp bzip2_process, ProcessAdp zip_process, ProcessAdp gz_process, Io_url src, Io_url trg) {
this.src = src; this.trg = trg; this.kit = kit; this.usr_dlg = usr_dlg;

View File

@@ -16,7 +16,8 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*; import gplx.xowa.wikis.*; import gplx.xowa.html.*;
import gplx.ios.*; import gplx.xowa.html.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*;
public class Xoa_css_extractor {
public IoEngine_xrg_downloadFil Download_xrg() {return download_xrg;} private IoEngine_xrg_downloadFil download_xrg = Io_mgr._.DownloadFil_args("", Io_url_.Null);
public Xoa_css_extractor Wiki_domain_(byte[] v) {wiki_domain = v; return this;} private byte[] wiki_domain;
@@ -39,7 +40,7 @@ public class Xoa_css_extractor {
failover_dir = app.Fsys_mgr().Bin_any_dir().GenSubDir_nest("html", "xowa", "import");
url_encoder = Xoa_app_.Utl__encoder_mgr().Url();
}
public void Install_assert(Xowe_wiki wiki, Io_url wiki_html_dir) {
public void Install_assert(boolean download_from_css_tbl, Xowe_wiki wiki, Io_url wiki_html_dir) {
try {
Io_url css_common_url = wiki_html_dir.GenSubFil(Css_common_name);
Io_url css_wiki_url = wiki_html_dir.GenSubFil(Css_wiki_name);
@@ -48,14 +49,17 @@ public class Xoa_css_extractor {
if (wiki.Domain_tid() == Xow_domain_.Tid_int_home || Env_.Mode_testing()) return; // NOTE: do not download if home_wiki; also needed for TEST
if (Io_mgr._.ExistsFil(css_wiki_url)) return; // css file exists; nothing to generate
wiki.Appe().Usr_dlg().Log_many("", "", "generating css for '~{0}'", wiki.Domain_str());
this.Install(wiki, wiki_html_dir);
if (download_from_css_tbl) {
if (Install_by_db(wiki, wiki_html_dir)) return;
}
this.Install_by_wmf(wiki, wiki_html_dir);
}
catch (Exception e) { // if error, failover; paranoia catch for outliers like bad network connectivity fail, or MediaWiki: message not existing; DATE:2013-11-21
wiki.Appe().Usr_dlg().Warn_many("", "", "failed while trying to generate css; failing over; wiki='~{0}' err=~{1}", wiki.Domain_str(), Err_.Message_gplx(e));
Css_common_failover(); // only failover xowa_common.css; xowa_wiki.css comes from MediaWiki:Common.css / Vector.css
}
}
public void Install(Xowe_wiki wiki, Io_url wiki_html_dir) {
private void Install_by_wmf(Xowe_wiki wiki, Io_url wiki_html_dir) {
opt_download_css_common = wiki.Appe().Setup_mgr().Dump_mgr().Css_commons_download();
if (!wiki.Appe().User().Cfg_mgr().Security_mgr().Web_access_enabled()) opt_download_css_common = false; // if !web_access_enabled, don't download
this.wiki_domain = wiki.Domain_bry();
@@ -70,6 +74,17 @@ public class Xoa_css_extractor {
Css_wiki_setup();
Logo_setup();
}
private boolean Install_by_db(Xowe_wiki wiki, Io_url wiki_html_dir) {
Xowd_db_mgr core_db_mgr = wiki.Data_mgr__core_mgr();
if ( core_db_mgr == null
|| core_db_mgr.Props() == null
|| core_db_mgr.Props().Schema_is_1()
|| !core_db_mgr.Tbl__cfg().Select_yn_or(Xow_cfg_consts.Grp__wiki_schema, Xow_cfg_consts.Key__schema__tbl_css_core, Bool_.N)
) return false;
Xowd_db_file core_db = core_db_mgr.Db__core();
gplx.xowa.html.css.Xowd_css_core_mgr.Get(core_db.Tbl__css_core(), core_db.Tbl__css_file(), wiki_html_dir);
return true;
}
public void Css_common_setup() {
if (opt_download_css_common)
Css_common_download();

View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.threads.*; import gplx.xowa.wikis.*;
import gplx.threads.*; import gplx.xowa.wikis.*; import gplx.xowa.bldrs.*;
abstract class Xoi_cmd_base implements Gfo_thread_cmd {
public void Ctor(Xoi_setup_mgr install_mgr, String wiki_key) {
this.install_mgr = install_mgr; this.wiki_key = wiki_key;
@@ -85,11 +85,11 @@ class Xoi_cmd_category2_build extends Xoi_cmd_base {
@Override public String Async_key() {return KEY;} public static final String KEY = "wiki.category2.build";
@Override public void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
if (app.Setup_mgr().Dump_mgr().Wiki_storage_type_is_sql()) {
wiki.Db_mgr_as_sql().Category_version_update(false);
bldr.Cmd_mgr().Add_many(wiki, "import.sql.category_registry", "import.sql.categorylinks", "import.sql.hiddencat");
wiki.Db_mgr_as_sql().Category_version_update(false);
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_text_cat_core, Xob_cmd_keys.Key_text_cat_link, Xob_cmd_keys.Key_text_cat_hidden);
}
else
bldr.Cmd_mgr().Add_many(wiki, "ctg.hiddencat_sql", "ctg.hiddencat_ttl", "ctg.link_sql", "ctg.link_idx");
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_tdb_cat_hidden_sql, Xob_cmd_keys.Key_tdb_cat_hidden_ttl, Xob_cmd_keys.Key_text_cat_link, Xob_cmd_keys.Key_tdb_ctg_link_idx);
}
@Override public void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
app.Usr_dlg().Prog_many("", "", "category2 setup done");
@@ -101,7 +101,7 @@ class Xoi_cmd_search2_build extends Xoi_cmd_base {
@Override public void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
if (app.Setup_mgr().Dump_mgr().Wiki_storage_type_is_sql()) {
wiki.Db_mgr_as_sql().Category_version_update(false);
bldr.Cmd_mgr().Add_many(wiki, "import.sql.search_title.cmd");
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_text_search_cmd);
}
}
@Override public void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {

View File

@@ -17,7 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.gfui.*;
import gplx.threads.*;
import gplx.threads.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.utils.*;
class Xoi_cmd_wiki_download extends Gfo_thread_cmd_download implements Gfo_thread_cmd { private Xoi_setup_mgr install_mgr; private String wiki_key, dump_date, dump_type;
public Xoi_cmd_wiki_download Ctor_download_(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date, String dump_type) {
this.install_mgr = install_mgr;
@@ -181,7 +181,7 @@ class Xoi_cmd_wiki_zip implements Gfo_thread_cmd {
wiki.Init_assert();
bldr.Cmd_mgr().Clear();
bldr.Pause_at_end_(false);
((Xobc_deploy_zip)bldr.Cmd_mgr().Add_cmd(wiki, "deploy.zip")).Delete_dirs_page_(delete_dirs_page);
((Xob_deploy_zip_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_deploy_zip)).Delete_dirs_page_(delete_dirs_page);
bldr.Run();
app.Usr_dlg().Prog_none(GRP_KEY, "clear", "");
app.Usr_dlg().Note_none(GRP_KEY, "clear", "");

View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.threads.*; import gplx.xowa.gui.views.*;
import gplx.threads.*; import gplx.xowa.bldrs.*; import gplx.xowa.gui.views.*; import gplx.xowa.bldrs.cmds.utils.*;
class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
public Xoi_cmd_wiki_import(Xoi_setup_mgr install_mgr, String wiki_key, String wiki_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.wiki_date = wiki_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, wiki_date, dump_type;
public static final String KEY = "wiki.import";
@@ -43,28 +43,29 @@ class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
boolean running;
public boolean Import_move_bz2_to_done() {return import_move_bz2_to_done;} public Xoi_cmd_wiki_import Import_move_bz2_to_done_(boolean v) {import_move_bz2_to_done = v; return this;} private boolean import_move_bz2_to_done = true;
private void Process_txt(Xob_bldr bldr) {
((gplx.xowa.bldrs.imports.Xobc_core_cleanup)bldr.Cmd_mgr().Add_cmd(wiki, "core.cleanup")).Delete_wiki_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, "core.init");
bldr.Cmd_mgr().Add_cmd(wiki, "core.make_page");
bldr.Cmd_mgr().Add_cmd(wiki, "core.make_id");
bldr.Cmd_mgr().Add_cmd(wiki, "core.make_search_title");
((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_id);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_search_title);
if (wiki.Import_cfg().Category_version() == gplx.xowa.ctgs.Xoa_ctg_mgr.Version_1)
bldr.Cmd_mgr().Add_cmd(wiki, "core.make_category");
bldr.Cmd_mgr().Add_cmd(wiki, "core.calc_stats");
bldr.Cmd_mgr().Add_cmd(wiki, "core.term");
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_category);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_calc_stats);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_core_term);
}
private void Process_sql(Xob_bldr bldr) {
((gplx.xowa.bldrs.imports.Xobc_core_cleanup)bldr.Cmd_mgr().Add_cmd(wiki, "core.cleanup")).Delete_wiki_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.init");
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.page");
((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_css);
if (wiki.Import_cfg().Category_version() == gplx.xowa.ctgs.Xoa_ctg_mgr.Version_1) {
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.category_v1");
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.category_registry");
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.categorylinks");
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_core_v1);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_core);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_link);
}
if (wiki.Appe().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.specials.search.Xosrh_core.Version_2)
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.search_title.wkr");
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.term");
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_search_wkr);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_term);
}
private void Process_async() {
Xoae_app app = install_mgr.App();
@@ -80,8 +81,7 @@ class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
else
Process_txt(bldr);
bldr.Run();
app.Usr_dlg().Prog_none(GRP_KEY, "clear", "");
app.Usr_dlg().Note_none(GRP_KEY, "clear", "");
app.Usr_dlg().Prog_none(GRP_KEY, "clear", ""); app.Usr_dlg().Note_none(GRP_KEY, "clear", "");
app.User().Available_from_fsys();
wiki.Init_needed_(true);
wiki.Html_mgr().Page_wtr_mgr().Init_(true);

View File

@@ -16,14 +16,15 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*;
public class Xoi_dump_mgr implements GfoInvkAble {
public String[] Server_urls() {return server_urls;} private String[] server_urls = String_.Ary(Xob_dump_file_.Server_wmf, Xob_dump_file_.Server_your_org, Xob_dump_file_.Server_c3sl, Xob_dump_file_.Server_masaryk);
public String[] Custom_cmds() {return custom_cmds;} private String[] custom_cmds = String_.Ary(Xoi_cmd_wiki_download.Key_wiki_download, Xoi_cmd_wiki_import.KEY);
public byte Data_storage_format() {return data_storage_format;} public Xoi_dump_mgr Data_storage_format_(byte v) {data_storage_format = v; return this;} private byte data_storage_format = gplx.ios.Io_stream_.Tid_gzip;
public long Db_text_max() {return db_text_max;} long db_text_max = (long)3000 * Io_mgr.Len_mb;
public long Db_categorylinks_max() {return db_categorylinks_max;} long db_categorylinks_max = (long)3600 * Io_mgr.Len_mb;
public long Db_wikidata_max() {return db_wikidata_max;} long db_wikidata_max = (long)3600 * Io_mgr.Len_mb;
public byte Wiki_storage_type() {return wiki_storage_type;} private byte wiki_storage_type = Wiki_storage_type_sqlite;
public long Db_text_max() {return db_text_max;} private long db_text_max = (long)3000 * Io_mgr.Len_mb;
public long Db_categorylinks_max() {return db_categorylinks_max;} private long db_categorylinks_max = (long)3600 * Io_mgr.Len_mb;
public long Db_wikidata_max() {return db_wikidata_max;} private long db_wikidata_max = (long)3600 * Io_mgr.Len_mb;
public byte Wiki_storage_type() {return wiki_storage_type;} private byte wiki_storage_type = Wiki_storage_type_sqlite;
public boolean Wiki_storage_type_is_sql() {return wiki_storage_type == Wiki_storage_type_sqlite;}
public String Db_ns_map() {return db_ns_map;} private String db_ns_map = "Template~Module";
public boolean Css_wiki_update() {return css_wiki_update;} private boolean css_wiki_update = true;
@@ -36,8 +37,8 @@ public class Xoi_dump_mgr implements GfoInvkAble {
else if (ctx.Match(k, Invk_server_urls_)) server_urls = m.ReadStrAryIgnore("v", ",", "\n");
else if (ctx.Match(k, Invk_custom_cmds)) return String_.Concat_with_str(",", custom_cmds);
else if (ctx.Match(k, Invk_custom_cmds_)) custom_cmds = String_.Ary_filter(m.ReadStrAry("v", ","), Xoi_cmd_mgr.Wiki_cmds_valid);
else if (ctx.Match(k, Invk_data_storage_format)) return Wtr_tid_to_str(data_storage_format);
else if (ctx.Match(k, Invk_data_storage_format_)) data_storage_format = Wtr_tid_parse(m.ReadStr("v"));
else if (ctx.Match(k, Invk_data_storage_format)) return Io_stream_.Obsolete_to_str(data_storage_format);
else if (ctx.Match(k, Invk_data_storage_format_)) data_storage_format = Io_stream_.Obsolete_to_tid(m.ReadStr("v"));
else if (ctx.Match(k, Invk_data_storage_format_list)) return Options_data_storage_format_list;
else if (ctx.Match(k, Invk_db_text_max)) return db_text_max / Io_mgr.Len_mb;
else if (ctx.Match(k, Invk_db_text_max_)) db_text_max = m.ReadLong("v") * Io_mgr.Len_mb;
@@ -64,7 +65,7 @@ public class Xoi_dump_mgr implements GfoInvkAble {
else return GfoInvkAble_.Rv_unhandled;
return this;
}
static final String Invk_server_urls = "server_urls", Invk_server_urls_ = "server_urls_", Invk_custom_cmds = "custom_cmds", Invk_custom_cmds_ = "custom_cmds_"
private static final String Invk_server_urls = "server_urls", Invk_server_urls_ = "server_urls_", Invk_custom_cmds = "custom_cmds", Invk_custom_cmds_ = "custom_cmds_"
, Invk_data_storage_format = "data_storage_format", Invk_data_storage_format_ = "data_storage_format_", Invk_data_storage_format_list = "data_storage_format_list"
, Invk_db_text_max = "db_text_max", Invk_db_text_max_ = "db_text_max_", Invk_db_categorylinks_max = "db_categorylinks_max", Invk_db_categorylinks_max_ = "db_categorylinks_max_", Invk_db_wikidata_max = "db_wikidata_max", Invk_db_wikidata_max_ = "db_wikidata_max_"
, Invk_db_ns_map = "db_ns_map", Invk_db_ns_map_ = "db_ns_map_"
@@ -76,24 +77,8 @@ public class Xoi_dump_mgr implements GfoInvkAble {
, Invk_import_bz2_by_stdout = "import_bz2_by_stdout", Invk_import_bz2_by_stdout_ = "import_bz2_by_stdout_"
;
private static KeyVal[] Options_data_storage_format_list = KeyVal_.Ary(KeyVal_.new_(".xdat", "text"), KeyVal_.new_(".gz", "gzip"), KeyVal_.new_(".bz2", "bzip2")); // removed .zip; DATE:2014-05-13; updated aliases; DATE:2014-06-20
public static String Wtr_tid_to_str(byte v) {
switch (v) {
case gplx.ios.Io_stream_.Tid_file : return ".xdat";
case gplx.ios.Io_stream_.Tid_zip : return ".zip";
case gplx.ios.Io_stream_.Tid_gzip : return ".gz";
case gplx.ios.Io_stream_.Tid_bzip2 : return ".bz2";
default : throw Err_.unhandled(v);
}
}
public static byte Wtr_tid_parse(String v) {
if (String_.Eq(v, ".xdat")) return gplx.ios.Io_stream_.Tid_file;
else if (String_.Eq(v, ".zip")) return gplx.ios.Io_stream_.Tid_zip;
else if (String_.Eq(v, ".gz")) return gplx.ios.Io_stream_.Tid_gzip;
else if (String_.Eq(v, ".bz2")) return gplx.ios.Io_stream_.Tid_bzip2;
else throw Err_.unhandled(v);
}
static final byte Wiki_storage_type_xdat = 1, Wiki_storage_type_sqlite = 2;
private static final KeyVal[] Options_storage_type_list = KeyVal_.Ary(KeyVal_.new_("sqlite"), KeyVal_.new_("xdat"));
private static final KeyVal[] Options_storage_type_list = KeyVal_.Ary(KeyVal_.new_("sqlite"), KeyVal_.new_("xdat")); // DEPRECATED: KeyVal_.new_("xdat"); DATE:2015-03-30
public static String Wiki_storage_type_str(byte v) {
switch (v) {
case Xoi_dump_mgr.Wiki_storage_type_xdat : return "xdat";

View File

@@ -16,6 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.xowa.wikis.data.tbls.*;
public interface Xow_page_fetcher {
Xow_page_fetcher Wiki_(Xowe_wiki v);
byte[] Fetch(int ns_id, byte[] ttl);
@@ -32,12 +33,12 @@ class Xow_page_fetcher_mok implements Xow_page_fetcher {
public Xow_page_fetcher Wiki_(Xowe_wiki v) {return this;}
public void Clear() {pages.Clear();} private HashAdp pages = HashAdp_.new_();
public void Add(int ns_id, byte[] ttl, byte[] text) {
Xodb_page page = new Xodb_page().Ns_id_(ns_id).Ttl_page_db_(ttl).Wtxt_(text);
Xowd_page_itm page = new Xowd_page_itm().Ns_id_(ns_id).Ttl_page_db_(ttl).Text_(text);
pages.Add(Make_key(ns_id, ttl), page);
}
public byte[] Fetch(int ns_id, byte[] ttl) {
Xodb_page rv = (Xodb_page)pages.Fetch(Make_key(ns_id, ttl));
return rv == null ? null : rv.Wtxt();
Xowd_page_itm rv = (Xowd_page_itm)pages.Fetch(Make_key(ns_id, ttl));
return rv == null ? null : rv.Text();
}
String Make_key(int ns_id, byte[] ttl) {return Int_.Xto_str(ns_id) + "|" + String_.new_utf8_(ttl);}
}