Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2024-10-27 20:34:16 +00:00
Code Issues Projects Releases Wiki Activity

Import: Use the directory of offline dump files for category files

Browse Source
This commit is contained in:
gnosygnu 2016-10-22 10:02:17 -04:00
parent 568060e820
commit 749d355545
3 changed files with 25 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_download_cmd.java
View File

@ -38,6 +38,7 @@ public class Xob_download_cmd extends Xob_cmd__base implements Xob_cmd {
dump_trg_zip = wiki.Fsys_mgr().Root_dir().GenSubFil(dump_file.File_name());
if (dump_trg_bin == null && unzip)
dump_trg_bin = dump_trg_zip.GenNewNameAndExt(dump_trg_zip.NameOnly()); // convert a.sql.gz -> a.sql
// download
usr_dlg.Note_many("", "", "downloading file: now=~{0} src=~{1} trg=~{2}", Datetime_now.Get().XtoStr_fmt_yyyyMMdd_HHmmss(), dump_src, dump_trg_zip.OwnerDir());
IoEngine_xrg_downloadFil download_wkr = app.Wmf_mgr().Download_wkr().Download_xrg();

49
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_import.java
View File

@ -21,6 +21,7 @@ import gplx.xowa.wikis.domains.*;
import gplx.xowa.htmls.hrefs.*;
class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
private boolean running;
private Xowe_wiki wiki;
public Xoi_cmd_wiki_import(Xoi_setup_mgr install_mgr, String wiki_key, String wiki_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.wiki_date = wiki_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, wiki_date, dump_type;
public static final String KEY = "wiki.import";
public void Cmd_ctor() {}
@ -44,29 +45,6 @@ class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
return running;
}
public boolean Import_move_bz2_to_done() {return import_move_bz2_to_done;} public Xoi_cmd_wiki_import Import_move_bz2_to_done_(boolean v) {import_move_bz2_to_done = v; return this;} private boolean import_move_bz2_to_done = true;
private void Process_txt(Xob_bldr bldr) {
((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_id);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_calc_stats);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_core_term);
}
private void Process_sql(Xob_bldr bldr) {
((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_css);
if (wiki.Appe().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.addons.wikis.searchs.specials.Srch_special_page.Version_2)
gplx.xowa.addons.wikis.searchs.bldrs.Srch_bldr_mgr_.Setup(wiki);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_term);
if (wiki.Domain_itm().Domain_type_id() != Xow_domain_tid_.Tid__other) { // do not add category if not wmf; particularly, wikia wikis will not have category dumps; DATE:2016-10-22
bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_download_cmd(bldr, wiki).Dump_type_(gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd.Dump_file_name));
bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_download_cmd(bldr, wiki).Dump_type_(gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd.Dump_file_name));
bldr.Cmd_mgr().Add_many(wiki, gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd.BLDR_CMD_KEY, gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd.BLDR_CMD_KEY);
}
}
private void Process_async() {
Xoae_app app = install_mgr.App();
app.Usr_dlg().Prog_one("", "", "preparing import: ~{0}", wiki_key);
@ -76,10 +54,7 @@ class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
bldr.Cmd_mgr().Clear();
bldr.Pause_at_end_(false);
Io_url src_url = wiki.Import_cfg().Src_rdr().Url();
if (install_mgr.Dump_mgr().Wiki_storage_type_is_sql())
Process_sql(bldr);
else
Process_txt(bldr);
Process_sql(bldr, src_url);
bldr.Run();
app.Usr_dlg().Prog_none(GRP_KEY, "clear", ""); app.Usr_dlg().Note_none(GRP_KEY, "clear", "");
app.Usere().Available_from_fsys();
@ -98,7 +73,25 @@ class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
running = false;
wiki.Import_cfg().Src_fil_xml_(null).Src_fil_bz2_(null); // reset file else error when going from Import/Script to Import/List
app.Gui_mgr().Kit().New_cmd_sync(this).Invk(GfsCtx.new_(), 0, Invk_open_wiki, GfoMsg_.Null);
} private Xowe_wiki wiki;
}
private void Process_sql(Xob_bldr bldr, Io_url dump_url) {
// setup wiki
((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_css);
if (wiki.Appe().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.addons.wikis.searchs.specials.Srch_special_page.Version_2)
gplx.xowa.addons.wikis.searchs.bldrs.Srch_bldr_mgr_.Setup(wiki);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_term);
// setup category
if (wiki.Domain_itm().Domain_type_id() != Xow_domain_tid_.Tid__other) { // do not add category if not wmf; note that wikia wikis will not have category dumps; DATE:2016-10-22
bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_download_cmd(bldr, wiki).Dump_type_(gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd.Dump_file_name));
bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_download_cmd(bldr, wiki).Dump_type_(gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd.Dump_file_name));
bldr.Cmd_mgr().Add(new gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd(bldr, wiki).Src_dir_manual_(dump_url.OwnerDir()));
bldr.Cmd_mgr().Add(new gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd(bldr, wiki).Src_dir_manual_(dump_url.OwnerDir()));
}
}
private void Open_wiki(String wiki_key) {
Xog_win_itm main_win = install_mgr.App().Gui_mgr().Browser_win();
if (main_win.Active_page() == null) return; // will be null when invoked through cmd-line

4
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_sql_dump_base.java
View File

@ -26,6 +26,7 @@ public abstract class Xob_sql_dump_base extends Xob_itm_dump_base implements Xob
}
public Io_url Src_fil() {return src_fil;} private Io_url src_fil;
public Io_url_gen Make_url_gen() {return make_url_gen;} private Io_url_gen make_url_gen;
public Xob_sql_dump_base Src_dir_manual_(Io_url v) {src_dir_manual = v; return this;} private Io_url src_dir_manual;
public abstract String Sql_file_name();
protected abstract Xosql_dump_parser New_parser();
public void Cmd_init(Xob_bldr bldr) {}
@ -33,7 +34,8 @@ public abstract class Xob_sql_dump_base extends Xob_itm_dump_base implements Xob
this.Init_dump(this.Cmd_key());
make_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("make"));
if (src_fil == null) {
src_fil = Xob_io_utl_.Find_nth_by_wildcard_or_null(wiki.Fsys_mgr().Root_dir(), Sql_file_name() + ".sql", ".gz", ".sql");
Io_url src_dir = src_dir_manual == null ? wiki.Fsys_mgr().Root_dir() : src_dir_manual;
src_fil = Xob_io_utl_.Find_nth_by_wildcard_or_null(src_dir, Sql_file_name() + ".sql", ".gz", ".sql");
if (src_fil == null) {
String msg = String_.Format(".sql file not found in dir.\nPlease download the file for your wiki from dumps.wikimedia.org.\nfile={0} dir={1}", Sql_file_name(), wiki.Fsys_mgr().Root_dir());
app.Usr_dlg().Warn_many("", "", msg);