Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Embeddable: Create core dbs in proper subdirectory

Browse Source
This commit is contained in:
gnosygnu
2017-10-23 20:50:22 -04:00
parent dc22c15895
commit 1336d44f34
4537 changed files with 0 additions and 311750 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_cmd.java
View File

@@ -13,13 +13,3 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public interface Xob_cmd extends Gfo_invk {
String Cmd_key();
Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki);
void Cmd_init(Xob_bldr bldr);
void Cmd_bgn(Xob_bldr bldr);
void Cmd_run();
void Cmd_end();
void Cmd_term();
}

18
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_cmd__base.java
View File

@@ -13,21 +13,3 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public abstract class Xob_cmd__base implements Xob_cmd, Gfo_invk {
protected final Xoae_app app; protected final Xob_bldr bldr; protected Xowe_wiki wiki; protected final Gfo_usr_dlg usr_dlg;
public Xob_cmd__base(Xob_bldr bldr, Xowe_wiki wiki) {
this.bldr = bldr;
this.wiki = wiki;
this.app = bldr == null ? null : bldr.App();;
this.usr_dlg = bldr == null ? null : bldr.Usr_dlg();
}
public abstract String Cmd_key();
@gplx.Virtual public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public abstract void Cmd_run();
@gplx.Virtual public void Cmd_init(Xob_bldr bldr) {}
@gplx.Virtual public void Cmd_bgn(Xob_bldr bldr) {}
@gplx.Virtual public void Cmd_end() {}
@gplx.Virtual public void Cmd_term() {}
@gplx.Virtual public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return Gfo_invk_.Rv_unhandled;}
}

11
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_cmd_base.java
View File

@@ -13,14 +13,3 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public abstract class Xob_cmd_base implements Xob_cmd {
public abstract String Cmd_key();
@gplx.Virtual public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
@gplx.Virtual public void Cmd_init(Xob_bldr bldr) {}
@gplx.Virtual public void Cmd_bgn(Xob_bldr bldr) {}
@gplx.Virtual public void Cmd_run() {}
@gplx.Virtual public void Cmd_end() {}
@gplx.Virtual public void Cmd_term() {}
@gplx.Virtual public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return Gfo_invk_.Rv_unhandled;}
}

22
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_idx_base.java
View File

@@ -13,25 +13,3 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.flds.*; import gplx.core.ios.*;
public abstract class Xob_idx_base extends Xob_itm_basic_base implements Xob_cmd, Gfo_invk {
public abstract String Cmd_key();
public Gfo_fld_wtr Fld_wtr() {return fld_wtr;} Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
public Gfo_fld_rdr Fld_rdr() {return fld_rdr;} Gfo_fld_rdr fld_rdr = Gfo_fld_rdr.xowa_();
public Io_url Temp_dir() {return temp_dir;} Io_url temp_dir;
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
temp_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir(this.Cmd_key());
Io_mgr.Instance.DeleteDirDeep(temp_dir);
Cmd_bgn_hook();
}
public abstract void Cmd_bgn_hook();
public abstract void Cmd_run();
@gplx.Virtual public void Cmd_end() {}
public void Cmd_term() {}
public Io_line_rdr rdr_(Io_url dir) {
Io_url[] fils = Io_mgr.Instance.QueryDir_fils(dir);
return new Io_line_rdr(bldr.Usr_dlg(), fils).Key_gen_(Io_line_rdr_key_gen_.first_pipe);
}
}

43
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_io_utl_.java
View File

@@ -13,46 +13,3 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public class Xob_io_utl_ {
public static void Delete_sql_files(Io_url wiki_dir, String sql_file_name) {
Delete_by_wildcard(wiki_dir, sql_file_name + ".sql", ".gz", ".sql");
}
public static void Delete_by_wildcard(Io_url dir, String name_pattern, String... ext_ary) {
List_adp list = Find_by_wildcard(Io_mgr.Instance.QueryDir_args(dir).ExecAsUrlAry(), name_pattern, ext_ary);
int len = list.Len();
for (int i = 0; i < len; ++i) {
Io_url url = (Io_url)list.Get_at(i);
Io_mgr.Instance.DeleteFil(url);
}
}
public static Io_url Find_nth_by_wildcard_or_null(Io_url dir, String name_pattern, String... ext_ary) {
return Find_nth_by_wildcard_or_null(Io_mgr.Instance.QueryDir_args(dir).ExecAsUrlAry(), name_pattern, ext_ary);
}
public static Io_url Find_nth_by_wildcard_or_null(Io_url[] fil_ary, String name_pattern, String... ext_ary) {
List_adp list = Find_by_wildcard(fil_ary, name_pattern, ext_ary);
int list_len = list.Len();
return list_len == 0 ? null : (Io_url)list.Get_at(list_len - 1);
}
public static List_adp Find_by_wildcard(Io_url[] fil_ary, String name_pattern, String... ext_ary) {
List_adp rv = List_adp_.New();
// create ext_hash
Ordered_hash ext_hash = Ordered_hash_.New();
for (String ext : ext_ary)
ext_hash.Add(ext, ext);
// iterate fil_ary
for (Io_url fil : fil_ary) {
// file matches pattern
if ( name_pattern == Pattern__wilcard // empty String means match anything
|| String_.Has(fil.NameAndExt(), name_pattern)) { // name has name_pattern; EX: "enwiki-latest-pages-articles-current.xml" and "pagelinks"
if ( ext_hash.Len() == 0 // empty hash means match any ext
|| ext_hash.Has(fil.Ext())) // ext exists in hash
rv.Add(fil);
}
}
return rv;
}
public static final String Pattern__wilcard = String_.Empty;
}

26
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_io_utl__tst.java
View File

@@ -13,29 +13,3 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*; import gplx.core.tests.*;
public class Xob_io_utl__tst {
private final Xob_io_utl__fxt fxt = new Xob_io_utl__fxt();
@Test public void Basic() {
fxt.Test__match(String_.Ary("a.txt", "b.txt", "c.txt"), "b", String_.Ary(".txt"), "b.txt");
}
@Test public void Include__ext() {// PURPOSE: handle calls like "a.sql", ".sql", ".gz"
fxt.Test__match(String_.Ary("a.txt", "b.txt", "c.txt"), "b.txt", String_.Ary(".txt"), "b.txt");
}
@Test public void Dupe__pick_last() {
fxt.Test__match(String_.Ary("b0.txt", "b1.txt", "b2.txt"), "b", String_.Ary(".txt"), "b2.txt");
}
@Test public void Ext() {
fxt.Test__match(String_.Ary("b.txt", "b.png", "b.xml"), "b", String_.Ary(".xml", ".bz2"), "b.xml");
}
@Test public void Ext__dupes() {
fxt.Test__match(String_.Ary("b.txt", "b.png", "b.xml"), "b", String_.Ary(".txt", ".xml"), "b.xml");
}
}
class Xob_io_utl__fxt {
public void Test__match(String[] path_ary, String name_pattern, String[] ext_ary, String expd) {
Io_url actl = Xob_io_utl_.Find_nth_by_wildcard_or_null(Io_url_.Ary(path_ary), name_pattern, ext_ary);
Gftest.Eq__str(expd, actl == null ? "<<NULL>>" : actl.Raw());
}
}

18
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_itm_basic_base.java
View File

@@ -13,21 +13,3 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public abstract class Xob_itm_basic_base implements Gfo_invk {
protected Xoae_app app; protected Xob_bldr bldr; protected Xowe_wiki wiki; protected Gfo_usr_dlg usr_dlg;
public void Cmd_ctor(Xob_bldr bldr, Xowe_wiki wiki) {
this.bldr = bldr;
this.wiki = wiki;
this.app = bldr == null ? null : bldr.App();
this.usr_dlg = bldr == null ? null : bldr.Usr_dlg();
if (bldr != null && wiki != null)
this.Cmd_ctor_end(bldr, wiki);
}
@gplx.Virtual public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
@gplx.Virtual protected void Cmd_ctor_end(Xob_bldr bldr, Xowe_wiki wiki) {}
@gplx.Virtual public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_owner)) return bldr.Cmd_mgr();
else return Gfo_invk_.Rv_unhandled;
} private static final String Invk_owner = "owner";
}

38
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_itm_dump_base.java
View File

@@ -13,41 +13,3 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.ios.*;
import gplx.xowa.apps.*;
public abstract class Xob_itm_dump_base extends Xob_itm_basic_base {
protected int sort_mem_len = Int_.Neg1, dump_fil_len = Int_.Neg1, make_fil_len = Int_.Neg1;
public Io_url Temp_dir() {return temp_dir;}
public boolean Delete_temp() {return delete_temp;} protected boolean delete_temp = true;
protected Io_url temp_dir, make_dir;
protected Bry_bfr dump_bfr;
public int Make_fil_len() {return make_fil_len;} public void Make_fil_len_(int v) {make_fil_len = v;}
public Io_url_gen Dump_url_gen() {return dump_url_gen;} protected Io_url_gen dump_url_gen;
public void Init_dump(String tmp_dir_key) {Init_dump(tmp_dir_key, null);}
public void Init_dump(String tmp_dir_key, Io_url make_dir_val) {
if (sort_mem_len == Int_.Neg1) sort_mem_len = bldr.Sort_mem_len();
if (dump_fil_len == Int_.Neg1) dump_fil_len = bldr.Dump_fil_len();
if (make_fil_len == Int_.Neg1) make_fil_len = bldr.Make_fil_len();
dump_bfr = Bry_bfr_.New_w_size(dump_fil_len);
temp_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir(tmp_dir_key);
if (make_dir_val == null) make_dir = temp_dir.GenSubDir("make");
else make_dir = make_dir_val;
Io_mgr.Instance.DeleteDirDeep_ary(temp_dir, make_dir);
dump_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("dump"));
}
@gplx.Virtual public void Term_dump(Io_sort_cmd make_cmd) {
Io_mgr.Instance.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr); dump_bfr.Rls();
Xobdc_merger.Basic(bldr.Usr_dlg(), dump_url_gen, temp_dir.GenSubDir("sort"), sort_mem_len, Io_line_rdr_key_gen_.first_pipe, make_cmd);
}
protected void Flush_dump() {Io_mgr.Instance.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_sort_mem_len_)) sort_mem_len = gplx.core.ios.Io_size_.Load_int_(m);
else if (ctx.Match(k, Invk_dump_fil_len_)) dump_fil_len = gplx.core.ios.Io_size_.Load_int_(m);
else if (ctx.Match(k, Invk_make_fil_len_)) make_fil_len = gplx.core.ios.Io_size_.Load_int_(m);
else if (ctx.Match(k, Invk_delete_temp_)) delete_temp = m.ReadBool("v");
else return super.Invk(ctx, ikey, k, m);
return this;
}
public static final String Invk_sort_mem_len_ = "sort_mem_len_", Invk_dump_fil_len_ = "dump_fil_len_", Invk_make_fil_len_ = "make_fil_len_", Invk_delete_temp_ = "delete_temp_";
}

8
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_page_wkr.java
View File

@@ -13,11 +13,3 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public interface Xob_page_wkr extends Gfo_invk {
String Page_wkr__key();
void Page_wkr__bgn();
void Page_wkr__run(gplx.xowa.wikis.data.tbls.Xowd_page_itm page);
void Page_wkr__run_cleanup(); // close txns opened during Page_wkr__run
void Page_wkr__end();
}

55
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_sql_dump_base.java
View File

@@ -13,58 +13,3 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.flds.*; import gplx.core.ios.*; import gplx.xowa.wikis.tdbs.*;
import gplx.xowa.bldrs.sql_dumps.*; import gplx.xowa.wikis.tdbs.bldrs.*;
public abstract class Xob_sql_dump_base extends Xob_itm_dump_base implements Xob_cmd, Gfo_invk {
private final Xosql_dump_parser parser; protected boolean fail = false;
public abstract String Cmd_key();
public Xob_sql_dump_base() {
this.parser = New_parser();
}
public Io_url Src_fil() {return src_fil;} private Io_url src_fil;
public Io_url_gen Make_url_gen() {return make_url_gen;} private Io_url_gen make_url_gen;
public Xob_sql_dump_base Src_dir_manual_(Io_url v) {src_dir_manual = v; return this;} private Io_url src_dir_manual;
public abstract String Sql_file_name();
protected abstract Xosql_dump_parser New_parser();
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
this.Init_dump(this.Cmd_key());
make_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("make"));
if (src_fil == null) {
Io_url src_dir = src_dir_manual == null ? wiki.Fsys_mgr().Root_dir() : src_dir_manual;
src_fil = Xob_io_utl_.Find_nth_by_wildcard_or_null(src_dir, Sql_file_name() + ".sql", ".gz", ".sql");
if (src_fil == null) {
String msg = String_.Format(".sql file not found in dir. Some data will not be imported.\nPlease download the file for your wiki from dumps.wikimedia.org.\nfile={0} dir={1}", Sql_file_name(), wiki.Fsys_mgr().Root_dir());
app.Usr_dlg().Warn_many("", "", msg);
app.Gui_mgr().Kit().Ask_ok("", "", msg);
fail = true;
return;
}
}
parser.Src_fil_(src_fil);
Cmd_bgn_hook(bldr, parser);
} protected Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
public abstract void Cmd_bgn_hook(Xob_bldr bldr, Xosql_dump_parser parser);
public void Cmd_run() {
if (fail) return;
parser.Parse(bldr.Usr_dlg());
}
@gplx.Virtual public void Cmd_end() {
if (fail) return;
Xobdc_merger.Basic(bldr.Usr_dlg(), dump_url_gen, temp_dir.GenSubDir("sort"), sort_mem_len, Io_line_rdr_key_gen_all.Instance, new Io_sort_fil_basic(bldr.Usr_dlg(), make_url_gen, make_fil_len));
}
public void Cmd_term() {}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_src_fil_)) src_fil = m.ReadIoUrl("v");
else return super.Invk(ctx, ikey, k, m);
return this;
}
public static final String Invk_src_fil_ = "src_fil_";
public void Cmd_cleanup_sql() {
// get dump files to delete; EX: "*-categorylinks.sql*" matches "simplewiki-latest-categorylinks.sql" and "simplewiki-latest-categorylinks.sql.gz"
Io_url[] dump_files = Io_mgr.Instance.QueryDir_args(wiki.Fsys_mgr().Root_dir()).FilPath_("*-" + this.Sql_file_name() + ".sql*").ExecAsUrlAry();
for (Io_url dump_file : dump_files)
Io_mgr.Instance.DeleteFil(dump_file);
}
}