From 749d355545e77ddfb0db78f73738468840ad82a0 Mon Sep 17 00:00:00 2001 From: gnosygnu Date: Sat, 22 Oct 2016 10:02:17 -0400 Subject: [PATCH] Import: Use the directory of offline dump files for category files --- .../bldrs/cmds/utils/Xob_download_cmd.java | 1 + .../bldrs/installs/Xoi_cmd_wiki_import.java | 49 ++++++++----------- .../xowa/bldrs/wkrs/Xob_sql_dump_base.java | 4 +- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_download_cmd.java b/400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_download_cmd.java index 557b361e1..ea68b6ff0 100644 --- a/400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_download_cmd.java +++ b/400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_download_cmd.java @@ -38,6 +38,7 @@ public class Xob_download_cmd extends Xob_cmd__base implements Xob_cmd { dump_trg_zip = wiki.Fsys_mgr().Root_dir().GenSubFil(dump_file.File_name()); if (dump_trg_bin == null && unzip) dump_trg_bin = dump_trg_zip.GenNewNameAndExt(dump_trg_zip.NameOnly()); // convert a.sql.gz -> a.sql + // download usr_dlg.Note_many("", "", "downloading file: now=~{0} src=~{1} trg=~{2}", Datetime_now.Get().XtoStr_fmt_yyyyMMdd_HHmmss(), dump_src, dump_trg_zip.OwnerDir()); IoEngine_xrg_downloadFil download_wkr = app.Wmf_mgr().Download_wkr().Download_xrg(); diff --git a/400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_import.java b/400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_import.java index 8a7ad05a0..835d140e7 100644 --- a/400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_import.java +++ b/400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_import.java @@ -21,6 +21,7 @@ import gplx.xowa.wikis.domains.*; import gplx.xowa.htmls.hrefs.*; class Xoi_cmd_wiki_import implements Gfo_thread_cmd { private boolean running; + private Xowe_wiki wiki; public Xoi_cmd_wiki_import(Xoi_setup_mgr install_mgr, String wiki_key, String wiki_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.wiki_date = wiki_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, wiki_date, dump_type; public static final String KEY = "wiki.import"; public void Cmd_ctor() {} @@ -44,29 +45,6 @@ class Xoi_cmd_wiki_import implements Gfo_thread_cmd { return running; } public boolean Import_move_bz2_to_done() {return import_move_bz2_to_done;} public Xoi_cmd_wiki_import Import_move_bz2_to_done_(boolean v) {import_move_bz2_to_done = v; return this;} private boolean import_move_bz2_to_done = true; - private void Process_txt(Xob_bldr bldr) { - ((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true); - bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_init); - bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_page); - bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_id); - bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_calc_stats); - bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_core_term); - } - private void Process_sql(Xob_bldr bldr) { - ((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true); - bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_init); - bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_page); - bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_css); - if (wiki.Appe().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.addons.wikis.searchs.specials.Srch_special_page.Version_2) - gplx.xowa.addons.wikis.searchs.bldrs.Srch_bldr_mgr_.Setup(wiki); - bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_term); - - if (wiki.Domain_itm().Domain_type_id() != Xow_domain_tid_.Tid__other) { // do not add category if not wmf; particularly, wikia wikis will not have category dumps; DATE:2016-10-22 - bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_download_cmd(bldr, wiki).Dump_type_(gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd.Dump_file_name)); - bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_download_cmd(bldr, wiki).Dump_type_(gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd.Dump_file_name)); - bldr.Cmd_mgr().Add_many(wiki, gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd.BLDR_CMD_KEY, gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd.BLDR_CMD_KEY); - } - } private void Process_async() { Xoae_app app = install_mgr.App(); app.Usr_dlg().Prog_one("", "", "preparing import: ~{0}", wiki_key); @@ -76,10 +54,7 @@ class Xoi_cmd_wiki_import implements Gfo_thread_cmd { bldr.Cmd_mgr().Clear(); bldr.Pause_at_end_(false); Io_url src_url = wiki.Import_cfg().Src_rdr().Url(); - if (install_mgr.Dump_mgr().Wiki_storage_type_is_sql()) - Process_sql(bldr); - else - Process_txt(bldr); + Process_sql(bldr, src_url); bldr.Run(); app.Usr_dlg().Prog_none(GRP_KEY, "clear", ""); app.Usr_dlg().Note_none(GRP_KEY, "clear", ""); app.Usere().Available_from_fsys(); @@ -98,7 +73,25 @@ class Xoi_cmd_wiki_import implements Gfo_thread_cmd { running = false; wiki.Import_cfg().Src_fil_xml_(null).Src_fil_bz2_(null); // reset file else error when going from Import/Script to Import/List app.Gui_mgr().Kit().New_cmd_sync(this).Invk(GfsCtx.new_(), 0, Invk_open_wiki, GfoMsg_.Null); - } private Xowe_wiki wiki; + } + private void Process_sql(Xob_bldr bldr, Io_url dump_url) { + // setup wiki + ((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true); + bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_init); + bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_page); + bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_css); + if (wiki.Appe().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.addons.wikis.searchs.specials.Srch_special_page.Version_2) + gplx.xowa.addons.wikis.searchs.bldrs.Srch_bldr_mgr_.Setup(wiki); + bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_term); + + // setup category + if (wiki.Domain_itm().Domain_type_id() != Xow_domain_tid_.Tid__other) { // do not add category if not wmf; note that wikia wikis will not have category dumps; DATE:2016-10-22 + bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_download_cmd(bldr, wiki).Dump_type_(gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd.Dump_file_name)); + bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_download_cmd(bldr, wiki).Dump_type_(gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd.Dump_file_name)); + bldr.Cmd_mgr().Add(new gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd(bldr, wiki).Src_dir_manual_(dump_url.OwnerDir())); + bldr.Cmd_mgr().Add(new gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd(bldr, wiki).Src_dir_manual_(dump_url.OwnerDir())); + } + } private void Open_wiki(String wiki_key) { Xog_win_itm main_win = install_mgr.App().Gui_mgr().Browser_win(); if (main_win.Active_page() == null) return; // will be null when invoked through cmd-line diff --git a/400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_sql_dump_base.java b/400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_sql_dump_base.java index 8d3d5bfd9..b1b5d8492 100644 --- a/400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_sql_dump_base.java +++ b/400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_sql_dump_base.java @@ -26,6 +26,7 @@ public abstract class Xob_sql_dump_base extends Xob_itm_dump_base implements Xob } public Io_url Src_fil() {return src_fil;} private Io_url src_fil; public Io_url_gen Make_url_gen() {return make_url_gen;} private Io_url_gen make_url_gen; + public Xob_sql_dump_base Src_dir_manual_(Io_url v) {src_dir_manual = v; return this;} private Io_url src_dir_manual; public abstract String Sql_file_name(); protected abstract Xosql_dump_parser New_parser(); public void Cmd_init(Xob_bldr bldr) {} @@ -33,7 +34,8 @@ public abstract class Xob_sql_dump_base extends Xob_itm_dump_base implements Xob this.Init_dump(this.Cmd_key()); make_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("make")); if (src_fil == null) { - src_fil = Xob_io_utl_.Find_nth_by_wildcard_or_null(wiki.Fsys_mgr().Root_dir(), Sql_file_name() + ".sql", ".gz", ".sql"); + Io_url src_dir = src_dir_manual == null ? wiki.Fsys_mgr().Root_dir() : src_dir_manual; + src_fil = Xob_io_utl_.Find_nth_by_wildcard_or_null(src_dir, Sql_file_name() + ".sql", ".gz", ".sql"); if (src_fil == null) { String msg = String_.Format(".sql file not found in dir.\nPlease download the file for your wiki from dumps.wikimedia.org.\nfile={0} dir={1}", Sql_file_name(), wiki.Fsys_mgr().Root_dir()); app.Usr_dlg().Warn_many("", "", msg);