mirror of
https://github.com/gnosygnu/xowa.git
synced 2026-03-02 03:49:30 +00:00
Embeddable: Create core dbs in proper subdirectory
This commit is contained in:
gnosygnu
@@ -13,16 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.core.strings.*;
|
||||
public class Xoac_wiki_cfg_bldr_cmd {
|
||||
public Xoac_wiki_cfg_bldr_cmd(String key, String text) {this.key = key; this.text = text;}
|
||||
public String Key() {return key;} private String key;
|
||||
public String Text() {return text;} private String text;
|
||||
public String Exec(String_bldr sb, String wiki, String src) {
|
||||
String sect_txt_bgn = sb.Add("// ").Add(key).Add(".bgn\n").To_str_and_clear();
|
||||
String sect_txt_end = sb.Add("// ").Add(key).Add(".end\n").To_str_and_clear();
|
||||
String sect_txt_all = sb.Add(sect_txt_bgn).Add(text + "\n").Add(sect_txt_end).To_str_and_clear(); // NOTE: always add \n; convenience for single line cmds
|
||||
return src + sect_txt_all;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,21 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
public class Xoac_wiki_cfg_bldr_fil implements Gfo_invk {
|
||||
public Xoac_wiki_cfg_bldr_fil(String wiki) {this.wiki = wiki;}
|
||||
public String Wiki() {return wiki;} private String wiki;
|
||||
public int Itms_count() {return list.Count();}
|
||||
public Xoac_wiki_cfg_bldr_cmd Itms_get_at(int i) {return (Xoac_wiki_cfg_bldr_cmd)list.Get_at(i);}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk_new_cmd_)) {Itms_add(m.ReadStr("id"), m.ReadStr("text"));}
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
} private static final String Invk_new_cmd_ = "new_cmd_";
|
||||
public Xoac_wiki_cfg_bldr_cmd Itms_add(String key, String text) {
|
||||
Xoac_wiki_cfg_bldr_cmd rv = new Xoac_wiki_cfg_bldr_cmd(key, text);
|
||||
list.Add(rv);
|
||||
return rv;
|
||||
}
|
||||
List_adp list = List_adp_.New();
|
||||
}
|
||||
|
||||
@@ -13,43 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.core.strings.*;
|
||||
import gplx.xowa.bldrs.*;
|
||||
public class Xob_wiki_cfg_bldr implements Gfo_invk {
|
||||
public Xob_wiki_cfg_bldr(Xob_bldr bldr) {this.app = bldr.App();} private Xoae_app app;
|
||||
public void Exec() {
|
||||
int len = hash.Count();
|
||||
for (int i = 0; i < len; i++) {
|
||||
Xoac_wiki_cfg_bldr_fil fil = (Xoac_wiki_cfg_bldr_fil)hash.Get_at(i);
|
||||
Exec_fil(fil);
|
||||
}
|
||||
}
|
||||
private void Exec_fil(Xoac_wiki_cfg_bldr_fil fil) {
|
||||
String wiki_key = fil.Wiki();
|
||||
Io_url cfg_file = app.Fsys_mgr().Cfg_wiki_core_dir().GenSubFil(wiki_key + ".gfs");
|
||||
String cfg_text = Io_mgr.Instance.LoadFilStr_args(cfg_file).MissingIgnored_().Exec();
|
||||
int len = fil.Itms_count();
|
||||
String_bldr sb = String_bldr_.new_();
|
||||
for (int i = 0; i < len; i++) {
|
||||
Xoac_wiki_cfg_bldr_cmd cmd = fil.Itms_get_at(i);
|
||||
cfg_text = cmd.Exec(sb, wiki_key, cfg_text);
|
||||
}
|
||||
Io_mgr.Instance.SaveFilStr(cfg_file, cfg_text);
|
||||
}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk_get)) return Itms_get_or_new(m.ReadStr("v"));
|
||||
else if (ctx.Match(k, Invk_run)) Exec();
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
} private static final String Invk_get = "get", Invk_run = "run";
|
||||
public void Clear() {hash.Clear();}
|
||||
public Xoac_wiki_cfg_bldr_fil Itms_get_or_new(String wiki) {
|
||||
Xoac_wiki_cfg_bldr_fil rv = (Xoac_wiki_cfg_bldr_fil)hash.Get_by(wiki);
|
||||
if (rv == null) {
|
||||
rv = new Xoac_wiki_cfg_bldr_fil(wiki);
|
||||
hash.Add(wiki, rv);
|
||||
}
|
||||
return rv;
|
||||
} private Ordered_hash hash = Ordered_hash_.New();
|
||||
}
|
||||
|
||||
@@ -13,165 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import org.junit.*; import gplx.core.strings.*;
|
||||
public class Xob_wiki_cfg_bldr_tst {
|
||||
Xob_wiki_cfg_bldr_fxt fxt = new Xob_wiki_cfg_bldr_fxt();
|
||||
@Before public void init() {fxt.Clear();}
|
||||
@Test public void Exec() {
|
||||
fxt .Init_cmd("en.wikipedia.org", "key0", "en.val0")
|
||||
.Init_cmd("en.wikipedia.org", "key1", "en.val1")
|
||||
.Init_cmd("fr.wikipedia.org", "key0", "fr.val0")
|
||||
.Init_cmd("fr.wikipedia.org", "key1", "fr.val1")
|
||||
.Expd_txt("en.wikipedia.org", String_.Concat_lines_nl
|
||||
( "// key0.bgn"
|
||||
, "en.val0"
|
||||
, "// key0.end"
|
||||
, "// key1.bgn"
|
||||
, "en.val1"
|
||||
, "// key1.end"
|
||||
))
|
||||
.Expd_txt("fr.wikipedia.org", String_.Concat_lines_nl
|
||||
( "// key0.bgn"
|
||||
, "fr.val0"
|
||||
, "// key0.end"
|
||||
, "// key1.bgn"
|
||||
, "fr.val1"
|
||||
, "// key1.end"
|
||||
))
|
||||
.Test()
|
||||
;
|
||||
fxt .Clear()
|
||||
.Init_cmd("en.wikipedia.org", "key2", "en.val2")
|
||||
.Expd_txt("en.wikipedia.org", String_.Concat_lines_nl
|
||||
( "// key0.bgn"
|
||||
, "en.val0"
|
||||
, "// key0.end"
|
||||
, "// key1.bgn"
|
||||
, "en.val1"
|
||||
, "// key1.end"
|
||||
, "// key2.bgn"
|
||||
, "en.val2"
|
||||
, "// key2.end"
|
||||
));
|
||||
}
|
||||
// @Test public void Lang_names_run() {
|
||||
// Io_url dir = Io_url_.new_dir_("/var/www/mediawiki/languages/messages/");
|
||||
// Io_url[] fils = Io_mgr.Instance.QueryDir_args(dir).ExecAsUrlAry();
|
||||
// int fils_len = fils.length;
|
||||
// String_bldr sb = String_bldr_.new_();
|
||||
// for (int i = 0; i < fils_len; i++) {
|
||||
// Io_url fil = fils[i];
|
||||
// String lang_code = String_.Lower(String_.Replace(fil.NameOnly(), "Messages", ""));
|
||||
// String txt = Io_mgr.Instance.LoadFilStr(fil);
|
||||
// String[] lines = String_.Split(txt, '\n');
|
||||
// String line = lines[1];
|
||||
// line = String_.Replace(line, "/** ", "");
|
||||
// int pos = String_.FindBwd(line, " (");
|
||||
// if (pos == String_.Find_none) continue; // en; en_rtl have no "language" line
|
||||
// if ( String_.Has_at_bgn(lang_code, "be_")
|
||||
// || String_.Has_at_bgn(lang_code, "crh_")
|
||||
// || String_.Has_at_bgn(lang_code, "kk_")
|
||||
// || String_.Has_at_bgn(lang_code, "ku_")
|
||||
// || String_.Has_at_bgn(lang_code, "sr_")
|
||||
// || String_.In(lang_code, "de_formal", "nb", "nl_informal", "nn", "no")
|
||||
// ) {
|
||||
// int new_pos = String_.FindBwd(line, " (", pos - 1);
|
||||
// if (new_pos != String_.Find_none)
|
||||
// pos = new_pos;
|
||||
// }
|
||||
// line = Replace_by_pos(line, pos, pos + 2, "|");
|
||||
// int line_len = String_.Len(line);
|
||||
// if (line_len > 0)
|
||||
// line = String_.MidByLen(line, 0, line_len - 1);
|
||||
// String[] terms = String_.Split(line, '|');
|
||||
// sb.Add(lang_code).Add("|").Add(String_.Trim(terms[0])).Add("|").Add(String_.Trim(terms[1])).Add("\n");
|
||||
// }
|
||||
// Tfds.Dbg(sb.To_str_and_clear());
|
||||
// }
|
||||
@Test public void Ns_aliases() {
|
||||
Io_mgr.Instance.InitEngine_mem();
|
||||
Io_mgr.Instance.SaveFilStr("mem/en.wikipedia.org/w/api.php?action=query&format=xml&meta=siteinfo&siprop=namespacealiases", String_.Concat_lines_nl
|
||||
( "<api>"
|
||||
, "<query>"
|
||||
, "<namespacealiases>"
|
||||
, "<ns id=\"4\" xml:space=\"preserve\">WP"
|
||||
, "</ns>"
|
||||
, "<ns id=\"5\" xml:space=\"preserve\">WT"
|
||||
, "</ns>"
|
||||
, "<ns id=\"6\" xml:space=\"preserve\">Image"
|
||||
, "</ns>"
|
||||
, "<ns id=\"7\" xml:space=\"preserve\">Image talk"
|
||||
, "</ns>"
|
||||
, "</namespacealiases>"
|
||||
, "</query>"
|
||||
, "</api>"
|
||||
));
|
||||
String[] wikis = new String[] {"en.wikipedia.org"}; String protocol = "mem/";
|
||||
Tfds.Eq_str_lines(Query_ns(protocol, gplx.core.ios.IoEngine_.MemKey, wikis), String_.Concat_lines_nl
|
||||
( "app.bldr.wiki_cfg_bldr.get('en.wikipedia.org').new_cmd_('wiki.ns_mgr.aliases', 'ns_mgr.add_alias_bulk(\""
|
||||
, "4|WP"
|
||||
, "5|WT"
|
||||
, "6|Image"
|
||||
, "7|Image_talk"
|
||||
, "\");');"
|
||||
));
|
||||
}
|
||||
String Query_ns(String protocol, String trg_engine_key, String[] wikis) {
|
||||
String_bldr sb = String_bldr_.new_();
|
||||
int wikis_len = wikis.length;
|
||||
for (int i = 0; i < wikis_len; i++) {
|
||||
String wiki = wikis[i];
|
||||
if (String_.Len_eq_0(wiki)) continue;
|
||||
try {
|
||||
String api = protocol + wiki + "/w/api.php?action=query&format=xml&meta=siteinfo&siprop=namespacealiases";
|
||||
String xml = String_.new_u8(Io_mgr.Instance.DownloadFil_args("", null).Trg_engine_key_(trg_engine_key).Exec_as_bry(api));
|
||||
if (xml == null) continue; // not found
|
||||
gplx.langs.xmls.XmlDoc xdoc = gplx.langs.xmls.XmlDoc_.parse(xml);
|
||||
gplx.langs.xmls.XmlNde xnde = gplx.langs.xmls.Xpath_.SelectFirst(xdoc.Root(), "query/namespacealiases");
|
||||
sb.Add("app.bldr.wiki_cfg_bldr.get('").Add(wiki).Add("').new_cmd_('wiki.ns_mgr.aliases', 'ns_mgr.add_alias_bulk(\"\n");
|
||||
int xndes_len = xnde.SubNdes().Count();
|
||||
for (int j = 0; j < xndes_len; j++) {
|
||||
gplx.langs.xmls.XmlNde ns_nde = xnde.SubNdes().Get_at(j);
|
||||
if (!String_.Eq(ns_nde.Name(), "ns")) continue;
|
||||
int id = Int_.Parse(ns_nde.Atrs().FetchValOr("id", "-1"));
|
||||
String name = String_.Replace(String_.Replace(ns_nde.Text_inner(), " ", "_"), "'", "''");
|
||||
sb.Add(Int_.To_str(id)).Add("|").Add(String_.Trim(name)).Add_char_nl();
|
||||
}
|
||||
sb.Add("\");');\n");
|
||||
}
|
||||
catch(Exception e) {sb.Add("// fail: " + wiki + " " + Err_.Message_gplx_full(e)).Add_char_nl();}
|
||||
}
|
||||
return sb.To_str_and_clear();
|
||||
}
|
||||
}
|
||||
class Xob_wiki_cfg_bldr_fxt {
|
||||
public Xob_wiki_cfg_bldr_fxt Clear() {
|
||||
if (app == null) {
|
||||
app = Xoa_app_fxt.Make__app__edit();
|
||||
wiki_cfg_bldr = app.Bldr().Wiki_cfg_bldr();
|
||||
}
|
||||
wiki_cfg_bldr.Clear();
|
||||
hash.Clear();
|
||||
return this;
|
||||
} private Xoae_app app; Xob_wiki_cfg_bldr wiki_cfg_bldr; Ordered_hash hash = Ordered_hash_.New();
|
||||
public Xob_wiki_cfg_bldr_fxt Init_cmd(String wiki, String key, String text) {
|
||||
wiki_cfg_bldr.Itms_get_or_new(wiki).Itms_add(key, text);
|
||||
return this;
|
||||
}
|
||||
public Xob_wiki_cfg_bldr_fxt Expd_txt(String wiki, String text) {
|
||||
hash.Add(wiki, Keyval_.new_(wiki, text));
|
||||
return this;
|
||||
}
|
||||
public void Test() {
|
||||
wiki_cfg_bldr.Exec();
|
||||
int len = hash.Count();
|
||||
for (int i = 0; i < len; i++) {
|
||||
Keyval kv = (Keyval)hash.Get_at(i);
|
||||
String wiki = kv.Key();
|
||||
String expd = (String)kv.Val();
|
||||
String actl = Io_mgr.Instance.LoadFilStr(app.Fsys_mgr().Cfg_wiki_core_dir().GenSubFil(wiki + ".gfs"));
|
||||
Tfds.Eq_str_lines(expd, actl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,16 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs.wm_server_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.app_cfgs.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xowm_server_cfg_cmd extends Xob_cmd__base {
|
||||
public Xowm_server_cfg_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
|
||||
@Override public void Cmd_run() {
|
||||
new Xowm_server_cfg_mgr().Exec(bldr.App());
|
||||
}
|
||||
|
||||
public static final String BLDR_CMD_KEY = "cfg.wikis.wm_server_cfg";
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
|
||||
public static final Xob_cmd Prototype = new Xowm_server_cfg_cmd(null, null);
|
||||
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xowm_server_cfg_cmd(bldr, wiki);}
|
||||
}
|
||||
|
||||
@@ -13,69 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs.wm_server_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.app_cfgs.*;
|
||||
import gplx.core.ios.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
import gplx.langs.phps.*;
|
||||
import gplx.xowa.wikis.domains.*;
|
||||
public class Xowm_server_cfg_mgr {
|
||||
public void Exec(Xoa_app app) {
|
||||
// get local file
|
||||
Io_url tmp_url = app.Fsys_mgr().Bin_xowa_dir().GenSubFil_nest("cfg", "wiki", "InitialiseSettings.php.txt");
|
||||
Assert_recent_or_download(tmp_url, "https://noc.wikimedia.org/conf/InitialiseSettings.php.txt", 1440);
|
||||
byte[] src = Io_mgr.Instance.LoadFilBry(tmp_url);
|
||||
Parse_cat_collation(src);
|
||||
}
|
||||
private void Parse_cat_collation(byte[] all) {
|
||||
// extract cat_collation
|
||||
int bgn_pos = Bry_find_.Find_fwd(all, Bry_.new_a7("wgCategoryCollation"));
|
||||
if (bgn_pos == Bry_find_.Not_found) throw Err_.new_wo_type("could not find wgCategoryCollation bgn");
|
||||
bgn_pos = Bry_find_.Move_fwd(all, Byte_ascii.Nl_bry, bgn_pos);
|
||||
|
||||
int end_pos = Bry_find_.Find_fwd(all, Bry_.new_a7("\n],"), bgn_pos);
|
||||
if (end_pos == Bry_find_.Not_found) throw Err_.new_wo_type("could not find wgCategoryCollation end");
|
||||
byte[] src = Bry_.Mid(all, bgn_pos, end_pos + 2); // +2="],"
|
||||
src = Bry_.Add(Bry_.new_a7("$test=["), src, Bry_.new_a7("];"));
|
||||
|
||||
// parse php
|
||||
Php_parser php_parser = new Php_parser();
|
||||
Php_evaluator eval = new Php_evaluator(new gplx.core.log_msgs.Gfo_msg_log("test"));
|
||||
php_parser.Parse_tkns(src, eval);
|
||||
Php_line[] lines = (Php_line[])eval.List().To_ary(Php_line.class);
|
||||
Php_line_assign line = (Php_line_assign)lines[0];
|
||||
Php_itm_ary root_ary = (Php_itm_ary)line.Val();
|
||||
|
||||
// loop tkns and build String
|
||||
Bry_bfr bfr = Bry_bfr_.New();
|
||||
bfr.Add_byte_nl();
|
||||
int len = root_ary.Subs_len();
|
||||
byte[] wiki_abrv__default = Bry_.new_a7("default");
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Php_itm_kv kv = (Php_itm_kv)root_ary.Subs_get(i);
|
||||
byte[] wiki_abrv = kv.Key().Val_obj_bry();
|
||||
byte[] collation = kv.Val().Val_obj_bry();
|
||||
|
||||
if (Bry_.Eq(wiki_abrv, wiki_abrv__default)) continue; // skip "'default' => 'uppercase',"
|
||||
try {
|
||||
byte[] wiki_domain = Xow_abrv_wm_.Parse_to_domain_bry(wiki_abrv);
|
||||
|
||||
Xow_domain_itm itm = Xow_domain_itm_.parse(wiki_domain);
|
||||
bfr.Add_str_u8_fmt("app.bldr.wiki_cfg_bldr.get('{0}').new_cmd_('wiki.ctgs.collations', \"catpage_mgr.collation_('{1}');\");\n", itm.Domain_bry(), collation);
|
||||
} catch (Exception e) {throw Err_.new_("failed to parse line", "wiki", wiki_abrv, "collation", collation, "err", Err_.Message_lang(e));}
|
||||
}
|
||||
Tfds.Write(bfr.To_str_and_clear());
|
||||
}
|
||||
private static void Assert_recent_or_download(Io_url trg, String src, int min) {
|
||||
// get file
|
||||
IoItmFil trg_fil = Io_mgr.Instance.QueryFil(trg);
|
||||
|
||||
// exit if file exists and is recent
|
||||
if (trg_fil != null) {
|
||||
DateAdp now = Datetime_now.Get();
|
||||
if (now.Diff(trg_fil.ModifiedTime()).Total_mins().To_int() < min) return;
|
||||
}
|
||||
|
||||
// load b/c file doesn't exist, or is not recent
|
||||
Io_mgr.Instance.DownloadFil(src, trg);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,27 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.xowa.specials.*; import gplx.xowa.htmls.bridges.*;
|
||||
import gplx.xowa.addons.bldrs.infos.*; import gplx.xowa.addons.bldrs.xodirs.*;
|
||||
public class Xobc_task_addon implements Xoax_addon_itm, Xoax_addon_itm__special, Xoax_addon_itm__json, Xoax_addon_itm__init {
|
||||
public Xobc_xodir_mgr Xodir_mgr() {return xodir_mgr;} private Xobc_xodir_mgr xodir_mgr;
|
||||
public void Xodir_mgr_(Xobc_xodir_mgr v) {this.xodir_mgr = v;}
|
||||
public void Init_addon_by_app(Xoa_app app) {
|
||||
this.xodir_mgr = new Xobc_xodir_mgr__pc(app);
|
||||
}
|
||||
public void Init_addon_by_wiki(Xow_wiki wiki) {}
|
||||
public Xow_special_page[] Special_pages() {
|
||||
return new Xow_special_page[]
|
||||
{ Xobc_task_special.Prototype
|
||||
, Xobc_info_special.Prototype
|
||||
};
|
||||
}
|
||||
public Bridge_cmd_itm[] Json_cmds() {
|
||||
return new Bridge_cmd_itm[]
|
||||
{ gplx.xowa.addons.bldrs.centrals.Xobc_task_bridge.Prototype
|
||||
};
|
||||
}
|
||||
|
||||
public String Addon__key() {return ADDON__KEY;} public static final String ADDON__KEY = "xowa.imports.downloads";
|
||||
}
|
||||
|
||||
@@ -13,50 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.langs.jsons.*;
|
||||
import gplx.xowa.htmls.bridges.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.cmds.*;
|
||||
public class Xobc_task_bridge implements Bridge_cmd_itm {
|
||||
private Xoa_app app;
|
||||
public void Init_by_app(Xoa_app app) {this.app = app;}
|
||||
public String Exec(Json_nde data) {
|
||||
Xobc_task_mgr task_mgr = Xobc_task_special.Task_mgr(app);
|
||||
byte proc_id = proc_hash.Get_as_byte_or(data.Get_as_bry_or(Bridge_cmd_mgr.Msg__proc, null), Byte_ascii.Max_7_bit);
|
||||
Json_nde args = data.Get_kv(Bridge_cmd_mgr.Msg__args).Val_as_nde();
|
||||
switch (proc_id) {
|
||||
case Proc__reload: task_mgr.Reload(); break;
|
||||
case Proc__add_work: task_mgr.Todo_mgr().Add_work(args.Get_as_int("task_id")); break;
|
||||
case Proc__del_work: task_mgr.Work_mgr().Del_work(args.Get_as_int("task_id")); break;
|
||||
case Proc__del_done: task_mgr.Done_mgr().Del_done(args.Get_as_int("task_id")); break;
|
||||
case Proc__del_todo: task_mgr.Todo_mgr().Del_todo(args.Get_as_int("task_id")); break;
|
||||
case Proc__run_next: task_mgr.Work_mgr().Run_next(); break;
|
||||
case Proc__stop_cur: task_mgr.Work_mgr().Stop_cur(); break;
|
||||
case Proc__redo_cur: task_mgr.Work_mgr().Redo_cur(); break;
|
||||
case Proc__filter_todo: task_mgr.Filter_todo(args.Get_as_str("lang_key"), args.Get_as_str("type_key")); break;
|
||||
case Proc__download_db: gplx.xowa.addons.bldrs.centrals.dbs.Xobc_data_db_upgrader.Check_for_updates(task_mgr); break;
|
||||
default: throw Err_.new_unhandled_default(proc_id);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
private static final byte
|
||||
Proc__reload = 0, Proc__add_work = 1, Proc__del_work = 2, Proc__del_done = 3
|
||||
, Proc__run_next = 4, Proc__stop_cur = 5, Proc__redo_cur = 6, Proc__download_db = 7, Proc__filter_todo = 8
|
||||
, Proc__del_todo = 9
|
||||
;
|
||||
private static final Hash_adp_bry proc_hash = Hash_adp_bry.cs()
|
||||
.Add_str_byte("reload" , Proc__reload)
|
||||
.Add_str_byte("add_work" , Proc__add_work)
|
||||
.Add_str_byte("del_work" , Proc__del_work)
|
||||
.Add_str_byte("del_done" , Proc__del_done)
|
||||
.Add_str_byte("run_next" , Proc__run_next)
|
||||
.Add_str_byte("stop_cur" , Proc__stop_cur)
|
||||
.Add_str_byte("redo_cur" , Proc__redo_cur)
|
||||
.Add_str_byte("download_db" , Proc__download_db)
|
||||
.Add_str_byte("filter_todo" , Proc__filter_todo)
|
||||
.Add_str_byte("del_todo" , Proc__del_todo)
|
||||
;
|
||||
|
||||
public byte[] Key() {return BRIDGE_KEY;} public static final byte[] BRIDGE_KEY = Bry_.new_a7("builder_central.exec");
|
||||
public static final Xobc_task_bridge Prototype = new Xobc_task_bridge(); Xobc_task_bridge() {}
|
||||
}
|
||||
|
||||
@@ -13,20 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.langs.mustaches.*;
|
||||
class Xobc_task_doc implements Mustache_doc_itm {
|
||||
private final boolean app_is_drd;
|
||||
private final String link_help;
|
||||
public Xobc_task_doc(boolean app_is_drd, String link_help) {
|
||||
this.app_is_drd = app_is_drd; this.link_help = link_help;
|
||||
}
|
||||
public boolean Mustache__write(String key, Mustache_bfr bfr) {
|
||||
if (String_.Eq(key, "link_help")) bfr.Add_str_u8(link_help);
|
||||
return false;
|
||||
}
|
||||
public Mustache_doc_itm[] Mustache__subs(String key) {
|
||||
if (String_.Eq(key, "app_is_drd")) return Mustache_doc_itm_.Ary__bool(app_is_drd);
|
||||
return Mustache_doc_itm_.Ary__empty;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,30 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.xowa.specials.*; import gplx.langs.mustaches.*; import gplx.xowa.wikis.pages.*; import gplx.xowa.wikis.pages.tags.*;
|
||||
class Xobc_task_html extends Xow_special_wtr__base {
|
||||
@Override protected Io_url Get_addon_dir(Xoa_app app) {return app.Fsys_mgr().Http_root().GenSubDir_nest("bin", "any", "xowa", "addon", "bldr", "central");}
|
||||
@Override protected Io_url Get_mustache_fil(Io_url addon_dir) {return addon_dir.GenSubFil_nest("bin", "xobc.main.mustache.html");}
|
||||
@Override protected Mustache_doc_itm Bld_mustache_root(Xoa_app app) {
|
||||
return new Xobc_task_doc(gplx.core.envs.Op_sys.Cur().Tid_is_drd(), "/site/home/wiki/App/Import/Download_Central");
|
||||
}
|
||||
@Override protected void Bld_tags(Xoa_app app, Io_url addon_dir, Xopage_html_data page_data) {
|
||||
Xopg_tag_mgr head_tags = page_data.Head_tags();
|
||||
Xopg_alertify_.Add_tags (head_tags, app.Fsys_mgr().Http_root());
|
||||
Xopg_tag_wtr_.Add__xocss (head_tags, app.Fsys_mgr().Http_root());
|
||||
Xopg_tag_wtr_.Add__xohelp (head_tags, app.Fsys_mgr().Http_root());
|
||||
Xopg_tag_wtr_.Add__mustache (head_tags, app.Fsys_mgr().Http_root());
|
||||
Xopg_tag_wtr_.Add__jquery (head_tags, app.Fsys_mgr().Http_root());
|
||||
Xopg_tag_wtr_.Add__xonotify (head_tags, app.Fsys_mgr().Http_root());
|
||||
Xopg_tag_wtr_.Add__xolog (head_tags, app.Fsys_mgr().Http_root());
|
||||
Xopg_tag_wtr_.Add__xoajax (head_tags, app.Fsys_mgr().Http_root(), app);
|
||||
|
||||
head_tags.Add(Xopg_tag_itm.New_css_file(addon_dir.GenSubFil_nest("bin", "xobc.css")));
|
||||
head_tags.Add(Xopg_tag_itm.New_htm_frag(addon_dir.GenSubFil_nest("bin", "xobc.row.mustache.html"), "xobc.row"));
|
||||
head_tags.Add(Xopg_tag_itm.New_js_file(addon_dir.GenSubFil_nest("js", "xo.elem.js")));
|
||||
head_tags.Add(Xopg_tag_itm.New_js_file(addon_dir.GenSubFil_nest("js", "xo.tmpl.js")));
|
||||
head_tags.Add(Xopg_tag_itm.New_js_file(addon_dir.GenSubFil_nest("js", "xobc.util.js")));
|
||||
head_tags.Add(Xopg_tag_itm.New_js_file(addon_dir.GenSubFil_nest("js", "xobc.js")));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,69 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.core.brys.evals.*; import gplx.core.gfobjs.*; import gplx.core.progs.rates.*; import gplx.core.threads.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.tasks.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.mgrs.*;
|
||||
import gplx.xowa.guis.cbks.*;
|
||||
public class Xobc_task_mgr implements Xog_json_wkr {
|
||||
private final Xog_cbk_trg cbk_trg = Xog_cbk_trg.New(Xobc_task_special.Prototype.Special__meta().Ttl_bry());
|
||||
public Xobc_task_mgr(Xoa_app app, Io_url data_db_url) {
|
||||
this.app = app;
|
||||
this.cbk_mgr = app.Gui__cbk_mgr();
|
||||
this.user_db = new Xobc_user_db(app.User().User_db_mgr().Conn());
|
||||
this.data_db = new Xobc_data_db(data_db_url);
|
||||
this.work_mgr = new Xobc_task_regy__work(this, app);
|
||||
this.todo_mgr = new Xobc_task_regy__todo(this);
|
||||
this.done_mgr = new Xobc_task_regy__done(this);
|
||||
this.step_mgr = new Xobc_step_factory(this, data_db, app.Fsys_mgr().Wiki_dir());
|
||||
this.skip_mgr = new Xobc_skip_mgr(this, app);
|
||||
this.ary = new Xobc_task_regy__base[] {work_mgr, todo_mgr, done_mgr};
|
||||
this.rate_mgr = Xobc_cmd__base.New_rate_mgr();
|
||||
}
|
||||
public Xoa_app App() {return app;} private final Xoa_app app;
|
||||
public Xog_cbk_mgr Cbk_mgr() {return cbk_mgr;} private final Xog_cbk_mgr cbk_mgr;
|
||||
public Xobc_task_regy__work Work_mgr() {return work_mgr;} private final Xobc_task_regy__work work_mgr;
|
||||
public Xobc_task_regy__todo Todo_mgr() {return todo_mgr;} private final Xobc_task_regy__todo todo_mgr;
|
||||
public Xobc_task_regy__done Done_mgr() {return done_mgr;} private final Xobc_task_regy__done done_mgr;
|
||||
public Xobc_task_regy__base Get_at(int i) {return ary[i];} private final Xobc_task_regy__base[] ary;
|
||||
public Xobc_data_db Data_db() {return data_db;} private final Xobc_data_db data_db;
|
||||
public Xobc_user_db User_db() {return user_db;} private final Xobc_user_db user_db;
|
||||
public Gfo_rate_mgr Rate_mgr() {return rate_mgr;} private final Gfo_rate_mgr rate_mgr;
|
||||
public Xobc_step_factory Step_mgr() {return step_mgr;} private final Xobc_step_factory step_mgr;
|
||||
public Xobc_filter_mgr Filter_mgr() {return filter_mgr;} private final Xobc_filter_mgr filter_mgr = new Xobc_filter_mgr();
|
||||
public Xobc_skip_mgr Skip_mgr() {return skip_mgr;} private final Xobc_skip_mgr skip_mgr;
|
||||
public Xobc_lang_mgr Lang_mgr() {return lang_mgr;} private final Xobc_lang_mgr lang_mgr = new Xobc_lang_mgr();
|
||||
public void Send_json(String func, Gfobj_nde data) {cbk_mgr.Send_json(cbk_trg, func, data);}
|
||||
public Xobc_task_mgr Load_or_init() {
|
||||
Gfo_log_.Instance.Info("task_mgr.load_or_init.bgn");
|
||||
data_db.Tbl__task_regy().Select_all(todo_mgr);
|
||||
user_db.Work_task_tbl().Select_all(this, todo_mgr, work_mgr);
|
||||
user_db.Done_task_tbl().Select_all(todo_mgr, done_mgr);
|
||||
return this;
|
||||
}
|
||||
public void Reload() {
|
||||
Gfo_log_.Instance.Info("task_mgr.reload.bgn");
|
||||
Gfobj_nde root = Gfobj_nde.New();
|
||||
Gfobj_nde lists_nde = root.New_nde("lists");
|
||||
work_mgr.Save_to(lists_nde.New_ary("work"));
|
||||
todo_mgr.Save_to(lists_nde.New_ary("todo"), filter_mgr.Filter(todo_mgr));
|
||||
done_mgr.Save_to(lists_nde.New_ary("done"));
|
||||
root.Add_nde("filters", filter_mgr.Make_init_msg());
|
||||
root.Add_ary("langs", lang_mgr.Make_init_msg(data_db.Tbl__lang_regy().Select_all()));
|
||||
cbk_mgr.Send_json(cbk_trg, "xo.bldr.core.reload__recv", root);
|
||||
}
|
||||
public void Filter_todo(String lang_key, String type_key) {
|
||||
Gfo_log_.Instance.Info("task_mgr.filter_by_lang.bgn");
|
||||
Gfobj_nde root = Gfobj_nde.New();
|
||||
Gfobj_nde lists_nde = root.New_nde("lists").Add_str("list_name", "todo");
|
||||
todo_mgr.Save_to(lists_nde.New_ary("todo"), filter_mgr.Filter(todo_mgr, lang_key, type_key));
|
||||
cbk_mgr.Send_json(cbk_trg, "xo.bldr.core.reload_list__recv", root);
|
||||
}
|
||||
public void Transfer(Xobc_task_regy__base src, Xobc_task_regy__base trg, Xobc_task_itm task) {
|
||||
Gfo_log_.Instance.Info("task_mgr.transfer.bgn", "task_uid", task.Task_id(), "src", src.Name(), "trg", trg.Name());
|
||||
src.Del_by(task.Task_id());
|
||||
trg.Add(task);
|
||||
cbk_mgr.Send_json(cbk_trg, "xo.bldr.core.transfer__recv", Gfobj_nde.New().Add_str("src", src.Name()).Add_str("trg", trg.Name()).Add_nde("task", task.Save_to(Gfobj_nde.New())));
|
||||
}
|
||||
public static final int Lists_len = 3;
|
||||
}
|
||||
|
||||
@@ -13,32 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.xowa.specials.*; import gplx.core.net.*; import gplx.core.net.qargs.*; import gplx.xowa.wikis.pages.*;
|
||||
public class Xobc_task_special implements Xow_special_page {
|
||||
public void Special__gen(Xow_wiki wiki, Xoa_page page, Xoa_url url, Xoa_ttl ttl) {
|
||||
// init task_mgr
|
||||
page.Html_data().Cbk_enabled_(true); // apply cbk_enabled early to capture logging statements
|
||||
Xoa_app app = wiki.App();
|
||||
if (task_mgr == null) task_mgr = New_task_mgr(app);
|
||||
task_mgr.Load_or_init();
|
||||
|
||||
new Xobc_task_html().Bld_page_by_mustache(app, page, this);
|
||||
}
|
||||
public static Xobc_task_mgr Task_mgr(Xoa_app app) {
|
||||
if (task_mgr == null) {
|
||||
task_mgr = New_task_mgr(app);
|
||||
}
|
||||
return task_mgr;
|
||||
} private static Xobc_task_mgr task_mgr;
|
||||
private static Xobc_task_mgr New_task_mgr(Xoa_app app) {
|
||||
Io_url data_db_url = app.Fsys_mgr().Bin_addon_dir().GenSubFil_nest("bldr", "central", "bldr_central.data_db.xowa");
|
||||
app.User().User_db_mgr().Init_site_mgr();
|
||||
return new Xobc_task_mgr(app, data_db_url);
|
||||
}
|
||||
|
||||
Xobc_task_special(Xow_special_meta special__meta) {this.special__meta = special__meta;}
|
||||
public Xow_special_meta Special__meta() {return special__meta;} private final Xow_special_meta special__meta;
|
||||
public Xow_special_page Special__clone() {return this;}
|
||||
public static final Xow_special_page Prototype = new Xobc_task_special(Xow_special_meta.New_xo("XowaDownloadCentral", "Download Central"));
|
||||
}
|
||||
|
||||
@@ -13,144 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.gfobjs.*; import gplx.core.progs.*; import gplx.core.progs.rates.*;
|
||||
import gplx.xowa.apps.apis.*;
|
||||
public abstract class Xobc_cmd__base implements Xobc_cmd_itm {
|
||||
private final Xobc_task_mgr task_mgr;
|
||||
private final Gfo_rate_list rate_list; private final long notify_delay = 1000;
|
||||
private final double delta_threshold = .25d; // allow variance of up to 25% before updating rate
|
||||
private long time_prv;
|
||||
private double rate_cur;
|
||||
private boolean log_verbose;
|
||||
public Xobc_cmd__base(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_id) {
|
||||
this.task_mgr = task_mgr; this.task_id = task_id; this.step_id = step_id; this.cmd_id = cmd_id;
|
||||
this.cmd_uid = String_.Concat_with_str(":", Int_.To_str(task_id), Int_.To_str(step_id), Int_.To_str(cmd_id));
|
||||
this.rate_list = task_mgr == null ? null : task_mgr.Rate_mgr().Get_or_new(this.Cmd_type());
|
||||
}
|
||||
public long Prog_data_cur() {return data_cur;} private long data_cur; public void Prog_data_cur_(long v) {this.data_cur = v;}
|
||||
public long Prog_data_end() {return data_end;} private long data_end; public void Prog_data_end_(long v) {this.data_end = v;}
|
||||
public byte Prog_status() {return status;} private byte status = Gfo_prog_ui_.Status__init; public void Prog_status_(byte v) {status = v;}
|
||||
public void Prog_notify_by_msg(String msg) {task_mgr.Work_mgr().On_stat(task_id, msg);}
|
||||
public boolean Canceled() {return status == Gfo_prog_ui_.Status__suspended;}
|
||||
public void Cancel() {status = Gfo_prog_ui_.Status__suspended;}
|
||||
|
||||
public int Task_id() {return task_id;} private final int task_id;
|
||||
public int Step_id() {return step_id;} private final int step_id;
|
||||
public int Cmd_id() {return cmd_id;} private final int cmd_id;
|
||||
public abstract String Cmd_type();
|
||||
public abstract String Cmd_name();
|
||||
@gplx.Virtual public boolean Cmd_suspendable() {return false;}
|
||||
public String Cmd_uid() {return cmd_uid;} private final String cmd_uid;
|
||||
@gplx.Virtual public String Cmd_fallback() {return this.Cmd_type();}
|
||||
@gplx.Virtual public void Cmd_clear() {// called when restarting failed task
|
||||
this.status = Gfo_prog_ui_.Status__init;
|
||||
this.cmd_exec_err = null; // reset error
|
||||
this.data_cur = 0; // reset progress else bad progress updates; DATE:2016-06-29
|
||||
this.Cmd_cleanup(); // do any cleanup, such as deleting bad downloads
|
||||
}
|
||||
|
||||
public void Cmd_exec(Xobc_cmd_ctx ctx) {
|
||||
// rate_list.Clear(); this.rate_cur = 0; // TOMBSTONE: do not reset rate else pause and resume will show different numbers
|
||||
Xoapi_root api_root = task_mgr.App().Api_root();
|
||||
if (api_root != null)
|
||||
this.log_verbose = api_root.Addon().Bldr().Central().Log_verbose();
|
||||
try {
|
||||
Gfo_log_.Instance.Info("xobc_cmd task bgn", "task_id", task_id, "step_id", step_id, "cmd_id", cmd_id);
|
||||
this.time_prv = gplx.core.envs.System_.Ticks();
|
||||
this.status = Gfo_prog_ui_.Status__working;
|
||||
this.Cmd_exec_hook(ctx);
|
||||
Gfo_log_.Instance.Info("xobc_cmd task end", "task_id", task_id, "step_id", step_id, "cmd_id", cmd_id);
|
||||
switch (status) {
|
||||
case Gfo_prog_ui_.Status__suspended: task_mgr.Work_mgr().On_suspended(this); break;
|
||||
case Gfo_prog_ui_.Status__fail: task_mgr.Work_mgr().On_fail(this, this.Cmd_fail_resumes(), cmd_exec_err); break;
|
||||
case Gfo_prog_ui_.Status__working:
|
||||
this.Prog_notify_and_chk_if_suspended(data_end, data_end); // fire one more time for 100%; note that 100% may not fire due to timer logic below
|
||||
task_mgr.Work_mgr().On_done(this, true);
|
||||
break;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
this.status = Gfo_prog_ui_.Status__fail;
|
||||
Gfo_log_.Instance.Warn("xobc_cmd task fail", "task_id", task_id, "step_id", step_id, "cmd_id", cmd_id, "err", Err_.Message_gplx_log(e));
|
||||
task_mgr.Work_mgr().On_fail(this, this.Cmd_fail_resumes(), Err_.Message_lang(e));
|
||||
}
|
||||
finally {
|
||||
Gfo_log_.Instance.Flush();
|
||||
}
|
||||
}
|
||||
protected abstract void Cmd_exec_hook(Xobc_cmd_ctx ctx);
|
||||
@gplx.Virtual protected boolean Cmd_fail_resumes() {return false;}
|
||||
protected void Cmd_exec_err_(String v) {
|
||||
Gfo_log_.Instance.Warn("xobc_cmd task err", "task_id", task_id, "step_id", step_id, "cmd_id", cmd_id, "err", v);
|
||||
this.status = Gfo_prog_ui_.Status__fail;
|
||||
this.cmd_exec_err = v;
|
||||
} private String cmd_exec_err;
|
||||
@gplx.Virtual public void Cmd_cleanup() {}
|
||||
|
||||
public Gfobj_nde Save_to(Gfobj_nde nde) {
|
||||
nde.Add_int ("task_id" , task_id);
|
||||
nde.Add_int ("step_id" , step_id);
|
||||
nde.Add_int ("cmd_id" , cmd_id);
|
||||
nde.Add_str ("cmd_type" , this.Cmd_type());
|
||||
nde.Add_str ("cmd_name" , this.Cmd_name());
|
||||
nde.Add_bool("cmd_suspendable" , this.Cmd_suspendable());
|
||||
nde.Add_byte("prog_status" , this.Prog_status());
|
||||
nde.Add_long("prog_data_cur" , this.Prog_data_cur());
|
||||
nde.Add_long("prog_data_end" , this.Prog_data_end());
|
||||
nde.Add_long("prog_time_end" , 0);
|
||||
return nde;
|
||||
}
|
||||
public void Load_checkpoint() {
|
||||
long data_cur = this.Load_checkpoint_hook();
|
||||
this.Prog_data_cur_(data_cur);
|
||||
if (data_cur > 0)
|
||||
this.Prog_status_(Gfo_prog_ui_.Status__suspended); // set status to suspended, else js won't warn about accidental removal
|
||||
}
|
||||
@gplx.Virtual protected long Load_checkpoint_hook() {return 0;}
|
||||
|
||||
public boolean Prog_notify_and_chk_if_suspended(long new_data_cur, long new_data_end) {
|
||||
if (status == Gfo_prog_ui_.Status__suspended) return true; // task paused by ui; exit now;
|
||||
long time_cur = gplx.core.envs.System_.Ticks();
|
||||
if (time_cur < time_prv + notify_delay) return false; // message came too soon. ignore it
|
||||
|
||||
// update rate
|
||||
double rate_now = (rate_list.Add(new_data_cur - data_cur, (time_cur - time_prv))) * 1000;
|
||||
double delta = Math_.Abs_double((rate_now - rate_cur) / rate_cur);
|
||||
if ( rate_cur == 0 // rate not set
|
||||
|| delta > delta_threshold) { // rate_now is at least 25% different than rate_prv
|
||||
if (delta > delta_threshold * 2) // rate_now is > 50% different
|
||||
rate_cur = rate_now; // update it now
|
||||
else {
|
||||
double rate_new = ((rate_now - rate_cur) * .05) + rate_cur; // calc new rate as 5% of difference
|
||||
// Tfds.Dbg(delta, rate_now, rate_cur, rate_new);
|
||||
rate_cur = rate_new;
|
||||
}
|
||||
}
|
||||
|
||||
// update prog vals
|
||||
this.time_prv = time_cur;
|
||||
this.data_cur = new_data_cur;
|
||||
this.data_end = new_data_end;
|
||||
|
||||
task_mgr.Send_json("xo.bldr.work.prog__update__recv", Gfobj_nde.New()
|
||||
.Add_int ("task_id", task_id).Add_long("prog_data_cur", data_cur).Add_long("prog_data_end", data_end).Add_int("prog_rate", (int)rate_cur));
|
||||
if (log_verbose)
|
||||
Gfo_usr_dlg_.Instance.Note_many("", "", "xobc:notify: task_id=~{0} cmd_id=~{1} prog_data_cur=~{2} prog_data_end=~{3} rate_cur=~{4}", task_id, cmd_id, data_cur, data_end, rate_cur);
|
||||
return false;
|
||||
}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk__exec)) Cmd_exec((Xobc_cmd_ctx)m.ReadObj("v"));
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
}
|
||||
public static final String Invk__exec = "exec";
|
||||
public static final int Seqn__0 = 0;
|
||||
public static Gfo_rate_mgr New_rate_mgr() {
|
||||
Gfo_rate_mgr rv = new Gfo_rate_mgr(128);
|
||||
rv.Add_new(Xobc_cmd__verify_dir.CMD_TYPE);
|
||||
rv.Add_new(Xobc_cmd__verify_fil.CMD_TYPE);
|
||||
rv.Add_new(Xobc_cmd__download.CMD_TYPE);
|
||||
rv.Add_new(Xobc_cmd__unzip.CMD_TYPE);
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,70 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.progs.*; import gplx.core.net.downloads.*;
|
||||
public class Xobc_cmd__download extends Xobc_cmd__base {
|
||||
private final Xobc_task_mgr task_mgr;
|
||||
private final String src_url; private final Io_url trg_url;
|
||||
private final long expd_size;
|
||||
private final Http_download_wkr wkr;
|
||||
public Xobc_cmd__download(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_id, String src_url, Io_url trg_url, long file_size_zip) {super(task_mgr, task_id, step_id, cmd_id);
|
||||
this.task_mgr = task_mgr;
|
||||
this.src_url = src_url; this.trg_url = trg_url; this.expd_size = file_size_zip;
|
||||
this.wkr = Http_download_wkr_.Proto.Make_new();
|
||||
this.Prog_data_end_(expd_size);
|
||||
}
|
||||
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.core.download";
|
||||
@Override public String Cmd_name() {return "download";}
|
||||
@Override public boolean Cmd_suspendable() {return true;}
|
||||
|
||||
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
|
||||
int error_wait = 10000, error_tries_max = 6, error_tries_cur = 0; // retry every 10 seconds for a total of 6 tries (1 min)
|
||||
|
||||
// loop while "server ... " error
|
||||
while (true) {
|
||||
long trg_size_bgn = Tmp_url_size();
|
||||
byte status = wkr.Exec(this, src_url, trg_url, expd_size);
|
||||
if (status == Gfo_prog_ui_.Status__fail) {
|
||||
|
||||
// check if server error; note: must not loop if bad size; DATE:2016-09-24
|
||||
String fail_msg = wkr.Fail_msg();
|
||||
if (String_.Has_at_bgn(fail_msg, Http_download_wkr__jre.Err__server_download_failed)) {
|
||||
|
||||
// check if anything more downloaded; if so, then reset to 0; DATE:2016-09-03
|
||||
long trg_size_cur = Tmp_url_size();
|
||||
if (trg_size_cur > trg_size_bgn) {
|
||||
error_tries_cur = 0;
|
||||
trg_size_bgn = trg_size_cur;
|
||||
}
|
||||
|
||||
// retry
|
||||
if (error_tries_cur++ < error_tries_max) {
|
||||
task_mgr.Work_mgr().On_stat(this.Task_id(), String_.Format("connection interrupted: retrying in {0} seconds; attempt {1} of {2}", error_wait / 1000, error_tries_cur, error_tries_max));
|
||||
Gfo_usr_dlg_.Instance.Log_many("", "", "xobc_cmd task download interrupted; ~{0} ~{1} ~{2} ~{3}", this.Task_id(), this.Step_id(), trg_url, Io_mgr.Instance.QueryFil(trg_url).Size());
|
||||
gplx.core.threads.Thread_adp_.Sleep(error_wait);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// otherewise exit loop
|
||||
this.Cmd_exec_err_(fail_msg);
|
||||
break;
|
||||
}
|
||||
else
|
||||
break;
|
||||
}
|
||||
Gfo_log_.Instance.Info("xobc_cmd task download", "task_id", this.Task_id(), "step_id", this.Step_id(), "trg_url", trg_url, "trg_len", Io_mgr.Instance.QueryFil(trg_url).Size());
|
||||
}
|
||||
private long Tmp_url_size() {
|
||||
return wkr.Tmp_url() == null ? 0 : Io_mgr.Instance.QueryFil(wkr.Tmp_url()).Size(); // NOTE: wkr.Tmp_url is null in some extreme exceptions; DATE:2016-09-24
|
||||
}
|
||||
@Override public void Cmd_cleanup() {
|
||||
wkr.Exec_cleanup();
|
||||
}
|
||||
@Override protected boolean Cmd_fail_resumes() {return true;}
|
||||
|
||||
@Override protected long Load_checkpoint_hook() {
|
||||
return wkr.Checkpoint__load_by_trg_fil(trg_url);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,32 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.addons.bldrs.updates.files.*;
|
||||
public class Xobc_cmd__fsdb_delete extends Xobc_cmd__base {
|
||||
private final Io_url deletion_db_url;
|
||||
public Xobc_cmd__fsdb_delete(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url deletion_db_url) {super(task_mgr, task_id, step_id, cmd_idx);
|
||||
this.deletion_db_url = deletion_db_url;
|
||||
}
|
||||
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.fsdb.delete";
|
||||
@Override public String Cmd_name() {return "deleting old files";}
|
||||
@Override public boolean Cmd_suspendable() {return true;}
|
||||
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
|
||||
if (!Io_mgr.Instance.ExistsFil(deletion_db_url)) throw Err_.New("deletion db does not exist; file={0}", deletion_db_url.Raw());
|
||||
boolean pass = false;
|
||||
try {
|
||||
new Xodel_exec_mgr().Exec_delete(this, ctx.App().Bldr(), deletion_db_url);
|
||||
pass = true;
|
||||
}
|
||||
catch (Exception e) {
|
||||
this.Cmd_exec_err_(Err_.Message_gplx_log(e));
|
||||
}
|
||||
Gfo_log_.Instance.Info("xobc_cmd task delete", "task_id", this.Task_id(), "step_id", this.Step_id(), "delete_url", deletion_db_url.Raw(), "pass", pass);
|
||||
}
|
||||
@Override public void Cmd_cleanup() {
|
||||
if (Io_mgr.Instance.ExistsFil(deletion_db_url))
|
||||
new Xodel_exec_mgr().Exec_cleanup(deletion_db_url);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,27 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
public class Xobc_cmd__move_fils extends Xobc_cmd__base {
|
||||
private final Io_url src_dir, trg_dir;
|
||||
private final List_adp trg_fils = List_adp_.New();
|
||||
public Xobc_cmd__move_fils(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url src_dir, Io_url trg_dir) {super(task_mgr, task_id, step_id, cmd_idx);
|
||||
this.src_dir = src_dir; this.trg_dir = trg_dir;
|
||||
}
|
||||
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.core.move_fil";
|
||||
@Override public String Cmd_name() {return "move";}
|
||||
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
|
||||
Io_url[] src_fils = Io_mgr.Instance.QueryDir_fils(src_dir);
|
||||
int len = src_fils.length;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Io_url src_fil = src_fils[i];
|
||||
if (String_.Eq(src_fil.Ext(), ".md5")) continue;
|
||||
Io_url trg_fil = trg_dir.GenSubFil(src_fil.NameAndExt());
|
||||
Io_mgr.Instance.MoveFil_args(src_fil, trg_fil, true).Exec();
|
||||
trg_fils.Add(trg_fil);
|
||||
}
|
||||
}
|
||||
@Override public void Cmd_cleanup() {
|
||||
Io_mgr.Instance.Delete_dir_empty(trg_dir.GenSubDir("tmp")); // deletes dir only if empty
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,32 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.ios.zips.*; import gplx.core.progs.*;
|
||||
public class Xobc_cmd__unzip extends Xobc_cmd__base {
|
||||
private final Io_url src_fil, trg_dir;
|
||||
private final Io_zip_decompress_cmd wkr;
|
||||
private final List_adp trg_fils = List_adp_.New();
|
||||
public Xobc_cmd__unzip(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_id, Io_url src_fil, Io_url trg_dir, long prog_data_end) {super(task_mgr, task_id, step_id, cmd_id);
|
||||
this.src_fil = src_fil; this.trg_dir = trg_dir;
|
||||
this.Prog_data_end_(prog_data_end);
|
||||
this.wkr = Io_zip_decompress_cmd_.Proto.Make_new();
|
||||
}
|
||||
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.core.ios.zips.zip_unzip";
|
||||
@Override public String Cmd_name() {return "unzip";}
|
||||
@Override public boolean Cmd_suspendable() {return true;}
|
||||
@Override public String Cmd_fallback() {return Xobc_cmd__verify_fil.CMD_TYPE;} // if unzip fails, backtrack to verify; if verify fails, it'll backtrack to download; DATE:2016-07-25
|
||||
|
||||
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
|
||||
if (wkr.Exec(this, src_fil, trg_dir, trg_fils) == Gfo_prog_ui_.Status__fail)
|
||||
this.Cmd_exec_err_(wkr.Fail_msg());
|
||||
}
|
||||
@Override public void Cmd_cleanup() {
|
||||
wkr.Exec_cleanup();
|
||||
Io_mgr.Instance.DeleteFil(src_fil);
|
||||
Io_mgr.Instance.DeleteDirDeep(trg_dir);
|
||||
}
|
||||
@Override protected long Load_checkpoint_hook() {
|
||||
return wkr.Checkpoint__load_by_src_fil(src_fil);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,46 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.progs.*;
|
||||
import gplx.core.security.*; import gplx.core.security.files.*;
|
||||
public class Xobc_cmd__verify_dir extends Xobc_cmd__base {
|
||||
private final Io_url delete_fil, checksum_fil;
|
||||
public Xobc_cmd__verify_dir(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url checksum_fil, Io_url delete_fil) {super(task_mgr, task_id, step_id, cmd_idx);
|
||||
this.checksum_fil = checksum_fil;
|
||||
this.delete_fil = delete_fil;
|
||||
}
|
||||
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.core.hash_dir";
|
||||
@Override public String Cmd_name() {return "verify";}
|
||||
@Override public String Cmd_fallback() {return Xobc_cmd__unzip.CMD_TYPE;}
|
||||
|
||||
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
|
||||
// parse file
|
||||
Cksum_list list = Cksum_list.Parse_by_fil(checksum_fil);
|
||||
this.Prog_data_end_(list.Itms_size);
|
||||
|
||||
// verify itms
|
||||
Hash_algo algo = Hash_algo_.New_by_tid(list.Type);
|
||||
Cksum_itm[] itms = list.Itms;
|
||||
int len = itms.length;
|
||||
long prog_data_cur = 0;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Cksum_itm itm = itms[i];
|
||||
gplx.core.ios.streams.IoStream stream = Io_mgr.Instance.OpenStreamRead(itm.File_url);
|
||||
byte[] actl_hash = Bry_.Empty;
|
||||
this.Prog_data_cur_(prog_data_cur);
|
||||
try {actl_hash = algo.Hash_stream_as_bry(this, stream);}
|
||||
finally {stream.Rls();}
|
||||
prog_data_cur += itm.File_size;
|
||||
if (this.Prog_notify_and_chk_if_suspended(prog_data_cur, Prog_data_end())) return;
|
||||
if (this.Prog_status() != Gfo_prog_ui_.Status__suspended && !Bry_.Eq(itm.Hash, actl_hash)) {
|
||||
this.Cmd_exec_err_(String_.Format("bad hash; file={0} bad={1} good={2}", itm.File_url.Raw(), actl_hash, itm.Hash));
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
@Override public void Cmd_cleanup() {
|
||||
Io_mgr.Instance.DeleteFil(checksum_fil);
|
||||
Io_mgr.Instance.DeleteFil(delete_fil);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,28 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.progs.*; import gplx.core.security.*;
|
||||
public class Xobc_cmd__verify_fil extends Xobc_cmd__base {
|
||||
private final Io_url src_url; private final byte[] expd_hash;
|
||||
public Xobc_cmd__verify_fil(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_id, Io_url src_url, String expd_hash_str, long prog_data_end) {super(task_mgr, task_id, step_id, cmd_id);
|
||||
this.src_url = src_url;
|
||||
this.expd_hash = Bry_.new_a7(expd_hash_str);
|
||||
this.Prog_data_end_(prog_data_end);
|
||||
}
|
||||
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.core.hash_fil";
|
||||
@Override public String Cmd_name() {return "verify";}
|
||||
@Override public String Cmd_fallback() {return Xobc_cmd__download.CMD_TYPE;}
|
||||
|
||||
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
|
||||
Hash_algo algo = Hash_algo_.New__md5();
|
||||
gplx.core.ios.streams.IoStream stream = Io_mgr.Instance.OpenStreamRead(src_url);
|
||||
byte[] actl_hash = Bry_.Empty;
|
||||
try {actl_hash = algo.Hash_stream_as_bry(this, stream);}
|
||||
finally {stream.Rls();}
|
||||
if (this.Prog_status() != Gfo_prog_ui_.Status__suspended && !Bry_.Eq(expd_hash, actl_hash))
|
||||
this.Cmd_exec_err_(Xobc_cmd__verify_fil.Err_make(actl_hash, expd_hash));
|
||||
}
|
||||
|
||||
public static String Err_make(byte[] actl, byte[] expd) {return String_.Format("bad hash: bad={0} good={1}", String_.new_u8(actl), String_.new_u8(expd));}
|
||||
}
|
||||
|
||||
@@ -13,43 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.addons.bldrs.exports.splits.mgrs.*; import gplx.xowa.addons.bldrs.exports.merges.*;
|
||||
public class Xobc_cmd__wiki_merge extends Xobc_cmd__base {
|
||||
private final String wiki_domain;
|
||||
private final Io_url src_dir;
|
||||
private final Merge2_mgr merge_mgr;
|
||||
private final Merge_prog_wkr prog_wkr;
|
||||
private final int idx_cur;
|
||||
public Xobc_cmd__wiki_merge(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Merge2_mgr merge_mgr, String wiki_domain, Io_url src_dir
|
||||
, long prog_size_end, int prog_count_end, int idx_cur) {super(task_mgr, task_id, step_id, cmd_idx);
|
||||
this.merge_mgr = merge_mgr;
|
||||
this.wiki_domain = wiki_domain;
|
||||
this.src_dir = src_dir;
|
||||
this.prog_wkr = new Merge_prog_wkr(this, src_dir.GenSubFil("merge.checkpoint"), prog_count_end, prog_size_end);
|
||||
merge_mgr.Prog_wkr_(prog_wkr);
|
||||
this.Prog_data_end_(prog_count_end);
|
||||
this.idx_cur = idx_cur;
|
||||
}
|
||||
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.wiki.merge";
|
||||
@Override public String Cmd_name() {return "merge";}
|
||||
@Override public boolean Cmd_suspendable() {return true;}
|
||||
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
|
||||
Xow_wiki wiki = ctx.App().Wiki_mgri().Make(Bry_.new_u8(wiki_domain), ctx.App().Fsys_mgr().Wiki_dir().GenSubDir(wiki_domain));
|
||||
Io_url[] fils = Io_mgr.Instance.QueryDir_fils(src_dir);
|
||||
for (Io_url fil : fils) {
|
||||
if (prog_wkr.Checkpoint__skip_fil(fil)) continue;
|
||||
switch (Split_file_tid_.To_tid(fil)) {
|
||||
case Split_file_tid_.Tid__core: merge_mgr.ctx.Idx_cur_add_(); merge_mgr.Merge_core(wiki, fil); break;
|
||||
case Split_file_tid_.Tid__data: merge_mgr.ctx.Idx_cur_add_(); merge_mgr.Merge_data(wiki, fil, idx_cur); break;
|
||||
}
|
||||
}
|
||||
prog_wkr.Checkpoint__delete();
|
||||
}
|
||||
@Override public void Cmd_cleanup() {
|
||||
Io_mgr.Instance.DeleteDirDeep(src_dir.OwnerDir()); // src_dir is "unzip" dir; owner dir is actual archive dir
|
||||
}
|
||||
@Override protected long Load_checkpoint_hook() {return prog_wkr.Checkpoint__load();}
|
||||
}
|
||||
|
||||
@@ -13,39 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.security.*;
|
||||
import gplx.xowa.wikis.*;
|
||||
public class Xobc_cmd__wiki_reg extends Xobc_cmd__base {
|
||||
private final Io_url wiki_dir;
|
||||
private final String wiki_domain;
|
||||
public Xobc_cmd__wiki_reg(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url wiki_dir, String wiki_domain) {super(task_mgr, task_id, step_id, cmd_idx);
|
||||
this.wiki_dir = wiki_dir;
|
||||
this.wiki_domain = wiki_domain;
|
||||
}
|
||||
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.wiki.reg";
|
||||
@Override public String Cmd_name() {return "import";}
|
||||
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
|
||||
ctx.App().User().User_db_mgr().Init_site_mgr(); // must init for wiki.register cmd
|
||||
|
||||
// get wiki_core_url
|
||||
Io_url[] wiki_fils = Io_mgr.Instance.QueryDir_fils(wiki_dir);
|
||||
Io_url wiki_core_url = null;
|
||||
int len = wiki_fils.length;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Io_url url = wiki_fils[i];
|
||||
if (gplx.xowa.wikis.data.Xow_db_file__core_.Is_core_fil_name(wiki_domain, url.NameAndExt())) {
|
||||
wiki_core_url = url;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (wiki_core_url == null) throw Err_.new_("wiki_import", "import_url not found", "domain", wiki_domain);
|
||||
|
||||
// import; open
|
||||
Xoa_wiki_mgr wiki_mgr = ctx.App().Wiki_mgri();
|
||||
Gfo_invk_.Invk_by_val(wiki_mgr, gplx.xowa.wikis.Xoa_wiki_mgr_.Invk__import_by_url, wiki_core_url);
|
||||
// COMMENTED: do not auto-open wiki; wait for true-pack mode
|
||||
// Xow_wiki wiki = wiki_mgr.Get_by_or_null(Bry_.new_u8(wiki_domain));
|
||||
// Gfo_invk_.Invk_by_msg(ctx.App().Gui__tab_mgr() , gplx.xowa.guis.tabs.Xog_tab_mgr_.Invk__new_tab, GfoMsg_.new_cast_("").Add("focus", true).Add("site", wiki_domain).Add("page", String_.new_u8(wiki.Props().Main_page())));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,8 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
public class Xobc_cmd_ctx {
|
||||
public Xobc_cmd_ctx(Xoa_app app) {this.app = app;}
|
||||
public Xoa_app App() {return app;} private final Xoa_app app;
|
||||
}
|
||||
|
||||
@@ -13,20 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.progs.*; import gplx.core.gfobjs.*;
|
||||
public interface Xobc_cmd_itm extends Gfo_prog_ui, Gfo_invk {
|
||||
int Task_id();
|
||||
int Step_id();
|
||||
int Cmd_id();
|
||||
String Cmd_type(); // "xowa.core.http_download"
|
||||
String Cmd_name(); // "download"
|
||||
boolean Cmd_suspendable(); // "true"
|
||||
String Cmd_uid(); // for thread_pool_mgr: "0:0:0"
|
||||
void Cmd_cleanup();
|
||||
String Cmd_fallback();
|
||||
void Cmd_clear();
|
||||
|
||||
Gfobj_nde Save_to(Gfobj_nde nde);
|
||||
void Load_checkpoint();
|
||||
}
|
||||
|
||||
@@ -13,49 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*;
|
||||
public class Xobc_data_db {
|
||||
public Xobc_data_db(Io_url db_url) {
|
||||
this.url = db_url;
|
||||
this.conn = Db_conn_bldr.Instance.Get_or_autocreate(true, db_url);
|
||||
this.tbl__task_regy = new Xobc_task_regy_tbl(conn);
|
||||
this.tbl__step_regy = new Xobc_step_regy_tbl(conn);
|
||||
this.tbl__step_map = new Xobc_step_map_tbl(conn);
|
||||
this.tbl__import_step = new Xobc_import_step_tbl(conn);
|
||||
this.tbl__host_regy = new Xobc_host_regy_tbl(conn);
|
||||
this.tbl__version_regy = new Xobc_version_regy_tbl(conn);
|
||||
this.tbl__lang_regy = new Xobc_lang_regy_tbl(conn);
|
||||
conn.Meta_tbl_assert(tbl__task_regy, tbl__step_regy, tbl__step_map, tbl__import_step, tbl__host_regy, tbl__version_regy, tbl__lang_regy);
|
||||
}
|
||||
public Db_conn Conn() {return conn;} private final Db_conn conn;
|
||||
public Io_url Url() {return url;} private final Io_url url;
|
||||
public Xobc_task_regy_tbl Tbl__task_regy() {return tbl__task_regy;} private final Xobc_task_regy_tbl tbl__task_regy;
|
||||
public Xobc_step_regy_tbl Tbl__step_regy() {return tbl__step_regy;} private final Xobc_step_regy_tbl tbl__step_regy;
|
||||
public Xobc_step_map_tbl Tbl__step_map() {return tbl__step_map;} private final Xobc_step_map_tbl tbl__step_map;
|
||||
public Xobc_import_step_tbl Tbl__import_step() {return tbl__import_step;} private final Xobc_import_step_tbl tbl__import_step;
|
||||
public Xobc_host_regy_tbl Tbl__host_regy() {return tbl__host_regy;} private final Xobc_host_regy_tbl tbl__host_regy;
|
||||
public Xobc_version_regy_tbl Tbl__version_regy() {return tbl__version_regy;} private final Xobc_version_regy_tbl tbl__version_regy;
|
||||
public Xobc_lang_regy_tbl Tbl__lang_regy() {return tbl__lang_regy;} private final Xobc_lang_regy_tbl tbl__lang_regy;
|
||||
|
||||
public void Delete_by_import(byte[] wiki_abrv, String wiki_date) {
|
||||
// get all step ids from import_regy
|
||||
Xobc_task_step_hash task_step_hash = new Xobc_task_step_hash();
|
||||
tbl__import_step.Select_tasks_steps(task_step_hash, tbl__step_map, wiki_abrv, wiki_date);
|
||||
for (int i = 0; i < task_step_hash.Tasks__len(); ++i) {
|
||||
int task_id = task_step_hash.Tasks__get_at(i);
|
||||
tbl__task_regy.Delete(task_id);
|
||||
tbl__step_map.Delete_by_task_id(task_id);
|
||||
}
|
||||
for (int i = 0; i < task_step_hash.Steps__len(); ++i) {
|
||||
int step_id = task_step_hash.Steps__get_at(i);
|
||||
tbl__step_regy.Delete(step_id);
|
||||
tbl__import_step.Delete(step_id);
|
||||
}
|
||||
}
|
||||
|
||||
public static Xobc_data_db New(gplx.xowa.apps.fsys.Xoa_fsys_mgr fsys_mgr) {
|
||||
return new Xobc_data_db(fsys_mgr.Bin_addon_dir().GenSubFil_nest("bldr", "central", "bldr_central.data_db.xowa"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,75 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.core.progs.*; import gplx.core.net.downloads.*; import gplx.core.gfobjs.*;
|
||||
import gplx.xowa.users.data.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*;
|
||||
import gplx.xowa.wikis.pages.tags.*;
|
||||
public class Xobc_data_db_upgrader {
|
||||
private static final String
|
||||
Cfg__last_check_date = "xowa.bldr_central.data_db.last_check_date"
|
||||
// , Cfg__next_check_interval = "xowa.bldr_central.data_db.next_check_interval"
|
||||
, Date_fmt = "yyyy-MM-dd HH:mm"
|
||||
;
|
||||
public static void Check_for_updates(Xobc_task_mgr task_mgr) {
|
||||
Io_url data_db_url = task_mgr.Data_db().Url();
|
||||
Xoud_cfg_mgr cfg_mgr = task_mgr.App().User().User_db_mgr().Cfg_mgr();
|
||||
|
||||
// check if update needed
|
||||
//String last_check_str = cfg_mgr.Select_str_or("", Cfg__last_check_date, "");
|
||||
//int next_check_interval = cfg_mgr.Select_int_or("", Cfg__next_check_interval, 24 * 7);
|
||||
//DateAdp last_check = DateAdp_.parse_fmt_or(last_check_str, Date_fmt, null);
|
||||
//if (last_check != null) { // check if last_check_str exists
|
||||
// Time_span span = Datetime_now.Get().Diff(last_check);
|
||||
// if (span.Total_hours().To_double() < next_check_interval) { // check if enough time passed
|
||||
// Gfo_log_.Instance.Info("xobc_db update not needed", "last_check", last_check_str, "next_check_interval", next_check_interval);
|
||||
// return;
|
||||
// }
|
||||
// else
|
||||
// Gfo_log_.Instance.Info("xobc_db update needed b/c of last_check", "last_check", last_check_str);
|
||||
//}
|
||||
//else
|
||||
// Gfo_log_.Instance.Info("xobc_db update needed b/c of missing or invalid last_check_str", "last_check", last_check_str);
|
||||
|
||||
// update needed; first, update cfg_key, then get host
|
||||
cfg_mgr.Upsert_str("", Cfg__last_check_date, Datetime_now.Get().XtoStr_fmt(Date_fmt));
|
||||
Xobc_data_db bc_db = task_mgr.Data_db();
|
||||
Xobc_host_regy_itm host_itm = bc_db.Tbl__host_regy().Select(Xobc_host_regy_tbl.Host_id__archive_org);
|
||||
|
||||
// download manifest
|
||||
Http_download_wkr download_wkr = Http_download_wkr_.Proto.Make_new();
|
||||
Io_url manifest_url = data_db_url.GenNewExt(".txt");
|
||||
download_wkr.Exec(Gfo_prog_ui_.Always
|
||||
, String_.Format("http://{0}/{1}/bldr_central.data_db.txt", host_itm.Host_domain(), host_itm.Host_update_dir())
|
||||
, manifest_url, -1);
|
||||
|
||||
// parse manifest and get version_id
|
||||
byte[] manifest_txt = Io_mgr.Instance.LoadFilBry(manifest_url);
|
||||
byte[][] manifest_data = Bry_split_.Split(manifest_txt, Byte_ascii.Pipe);
|
||||
int expd_version_id = Bry_.To_int(manifest_data[0]);
|
||||
Xobc_version_regy_itm actl_version = bc_db.Tbl__version_regy().Select_latest();
|
||||
|
||||
// cleanup
|
||||
Io_mgr.Instance.DeleteFil(manifest_url);
|
||||
bc_db.Conn().Rls_conn();
|
||||
if (expd_version_id == actl_version.Id()) {
|
||||
Gfo_log_.Instance.Info("xobc_db update not needed", "version", expd_version_id);
|
||||
Xopg_alertify_.Exec_log(task_mgr, "Wikis are up-to-date", 30);
|
||||
return; // version matches; exit
|
||||
}
|
||||
|
||||
// version doesn't match; download new
|
||||
// next_check_interval = Bry_.To_int(manifest_data[2]);
|
||||
// cfg_mgr.Upsert_int("", Cfg__next_check_interval, next_check_interval);
|
||||
// Gfo_log_.Instance.Info("xobc_db update needed", "version", expd_version_id, "next_check_interval", next_check_interval);
|
||||
byte[] new_db_url = manifest_data[1];
|
||||
String note = String_.new_u8(manifest_data[3]);
|
||||
download_wkr.Exec(Gfo_prog_ui_.Always
|
||||
, String_.new_u8(new_db_url)
|
||||
, data_db_url, -1);
|
||||
Xopg_alertify_.Exec_log(task_mgr, "Wikis have been updated:<br/>" + note, 30);
|
||||
task_mgr.Load_or_init();
|
||||
task_mgr.Reload();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,17 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
public class Xobc_task_step_hash {
|
||||
private final Ordered_hash tasks_hash = Ordered_hash_.New();
|
||||
private final Ordered_hash steps_hash = Ordered_hash_.New();
|
||||
public int Tasks__len() {return tasks_hash.Len();}
|
||||
public int Tasks__get_at(int i) {return Int_.Cast(tasks_hash.Get_at(i));}
|
||||
public int Steps__len() {return steps_hash.Len();}
|
||||
public int Steps__get_at(int i) {return Int_.Cast(steps_hash.Get_at(i));}
|
||||
public void Clear() {tasks_hash.Clear(); steps_hash.Clear();}
|
||||
public void Add(int task_id, int step_id) {
|
||||
tasks_hash.Add_if_dupe_use_nth(task_id, task_id);
|
||||
steps_hash.Add_if_dupe_use_nth(step_id, step_id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,17 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.users.*;
|
||||
public class Xobc_user_db {
|
||||
public Xobc_user_db(Db_conn conn) {
|
||||
this.work_task_tbl = new Xobc_work_task_tbl(conn);
|
||||
this.done_task_tbl = new Xobc_done_task_tbl(conn);
|
||||
this.done_step_tbl = new Xobc_done_step_tbl(conn);
|
||||
conn.Meta_tbl_assert(work_task_tbl, done_task_tbl, done_step_tbl);
|
||||
}
|
||||
public Xobc_work_task_tbl Work_task_tbl() {return work_task_tbl;} private final Xobc_work_task_tbl work_task_tbl;
|
||||
public Xobc_done_task_tbl Done_task_tbl() {return done_task_tbl;} private final Xobc_done_task_tbl done_task_tbl;
|
||||
public Xobc_done_step_tbl Done_step_tbl() {return done_step_tbl;} private final Xobc_done_step_tbl done_step_tbl;
|
||||
}
|
||||
|
||||
@@ -13,16 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
public class Xobc_host_regy_itm {
|
||||
public Xobc_host_regy_itm(int host_id, String host_domain, String host_data_dir, String host_update_dir) {
|
||||
this.host_id = host_id;
|
||||
this.host_domain = host_domain;
|
||||
this.host_data_dir = host_data_dir;
|
||||
this.host_update_dir = host_update_dir;
|
||||
}
|
||||
public int Host_id() {return host_id;} private final int host_id;
|
||||
public String Host_domain() {return host_domain;} private final String host_domain;
|
||||
public String Host_data_dir() {return host_data_dir;} private final String host_data_dir;
|
||||
public String Host_update_dir() {return host_update_dir;} private final String host_update_dir;
|
||||
}
|
||||
|
||||
@@ -13,37 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
import gplx.dbs.*;
|
||||
public class Xobc_host_regy_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld_host_id, fld_host_domain, fld_host_data_dir, fld_host_update_dir;
|
||||
private final Db_conn conn;
|
||||
public Xobc_host_regy_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "host_regy";
|
||||
this.fld_host_id = flds.Add_int_pkey("host_id");
|
||||
this.fld_host_domain = flds.Add_str("host_domain", 255); // EX: archive.org
|
||||
this.fld_host_data_dir = flds.Add_str("host_data_dir", 255); // EX: download/Xowa_~{host_regy|wiki_abrv}_latest
|
||||
this.fld_host_update_dir = flds.Add_str("host_update_dir", 255); // EX: download/Xowa_app_support
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public void Create_tbl() {
|
||||
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
|
||||
conn.Stmt_insert(tbl_name, flds)
|
||||
.Val_int(fld_host_id, Host_id__archive_org).Val_str(fld_host_domain, "archive.org")
|
||||
.Val_str(fld_host_data_dir, "download/Xowa_~{host_regy|wiki_abrv}_latest").Val_str(fld_host_update_dir, "download/Xowa_app_support")
|
||||
.Exec_insert();
|
||||
}
|
||||
public Xobc_host_regy_itm Select(int host_id) {
|
||||
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_host_id).Crt_int(fld_host_id, host_id).Exec_select__rls_auto();
|
||||
try {return rdr.Move_next() ? Load_itm(rdr) : null;}
|
||||
finally {rdr.Rls();}
|
||||
}
|
||||
private Xobc_host_regy_itm Load_itm(Db_rdr rdr) {
|
||||
return new Xobc_host_regy_itm(rdr.Read_int(fld_host_id), rdr.Read_str(fld_host_domain), rdr.Read_str(fld_host_data_dir), rdr.Read_str(fld_host_update_dir));
|
||||
}
|
||||
public void Rls() {}
|
||||
public static final int Host_id__archive_org = 1;
|
||||
}
|
||||
|
||||
@@ -13,14 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
public class Xobc_lang_regy_itm {
|
||||
public Xobc_lang_regy_itm(int id, String key, String name) {
|
||||
this.id = id;
|
||||
this.key = key;
|
||||
this.name = name;
|
||||
}
|
||||
public int Id() {return id;} private final int id;
|
||||
public String Key() {return key;} private final String key;
|
||||
public String Name() {return name;} private final String name;
|
||||
}
|
||||
|
||||
@@ -13,31 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
import gplx.dbs.*;
|
||||
public class Xobc_lang_regy_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld__lang_id, fld__lang_key, fld__lang_name;
|
||||
private final Db_conn conn;
|
||||
public Xobc_lang_regy_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "lang_regy";
|
||||
this.fld__lang_id = flds.Add_int_pkey("lang_id");
|
||||
this.fld__lang_key = flds.Add_str("lang_key", 255);
|
||||
this.fld__lang_name = flds.Add_str("lang_name", 255);
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
|
||||
public Xobc_lang_regy_itm[] Select_all() {
|
||||
List_adp list = List_adp_.New();
|
||||
Db_rdr rdr = conn.Stmt_select(tbl_name, flds).Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next())
|
||||
list.Add(new Xobc_lang_regy_itm(rdr.Read_int(fld__lang_id), rdr.Read_str(fld__lang_key), rdr.Read_str(fld__lang_name)));
|
||||
}
|
||||
finally {rdr.Rls();}
|
||||
return (Xobc_lang_regy_itm[])list.To_ary_and_clear(Xobc_lang_regy_itm.class);
|
||||
}
|
||||
public void Rls() {}
|
||||
}
|
||||
|
||||
@@ -13,16 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
public class Xobc_step_map_itm {
|
||||
public Xobc_step_map_itm(int sm_id, int task_id, int step_id, int step_seqn) {
|
||||
this.Sm_id = sm_id;
|
||||
this.Task_id = task_id;
|
||||
this.Step_id = step_id;
|
||||
this.Step_seqn = step_seqn;
|
||||
}
|
||||
public final int Sm_id;
|
||||
public final int Task_id;
|
||||
public final int Step_id;
|
||||
public final int Step_seqn;
|
||||
}
|
||||
|
||||
@@ -13,78 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
import gplx.dbs.*;
|
||||
public class Xobc_step_map_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld_sm_id, fld_step_seqn, fld_task_id, fld_step_id;
|
||||
private final Db_conn conn; private Db_stmt insert_stmt;
|
||||
public Xobc_step_map_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "step_map";
|
||||
this.fld_sm_id = flds.Add_int_pkey("sm_id");
|
||||
this.fld_task_id = flds.Add_int("task_id");
|
||||
this.fld_step_id = flds.Add_int("step_id");
|
||||
this.fld_step_seqn = flds.Add_int("step_seqn");
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
|
||||
public int Select_step_id(int task_id, int step_seqn) {
|
||||
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_task_id, fld_step_seqn).Crt_int(fld_task_id, task_id).Crt_int(fld_step_seqn, step_seqn).Exec_select__rls_auto();
|
||||
try {
|
||||
if (rdr.Move_next())
|
||||
return rdr.Read_int(fld_step_id);
|
||||
else
|
||||
throw Err_.new_("", "xobc:could not find step_id", "task_id", task_id, "step_seqn", step_seqn);
|
||||
}
|
||||
finally {rdr.Rls();}
|
||||
}
|
||||
public Xobc_step_map_itm Select_one(int task_id, int step_id) {
|
||||
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_task_id, fld_step_id).Crt_int(fld_task_id, task_id).Crt_int(fld_step_id, step_id).Exec_select__rls_auto();
|
||||
try {
|
||||
if (rdr.Move_next())
|
||||
return new Xobc_step_map_itm
|
||||
( rdr.Read_int(fld_sm_id)
|
||||
, rdr.Read_int(fld_task_id)
|
||||
, rdr.Read_int(fld_step_id)
|
||||
, rdr.Read_int(fld_step_seqn)
|
||||
);
|
||||
else
|
||||
throw Err_.new_("", "bldr.central:could not find step_id", "task_id", task_id, "step_id", step_id);
|
||||
}
|
||||
finally {rdr.Rls();}
|
||||
}
|
||||
public int Select_seqn_but_skip_done(int task_id, Hash_adp step_ids) {
|
||||
Db_rdr rdr = conn.Stmt_select_order(tbl_name, flds, String_.Ary(fld_task_id), fld_step_seqn).Crt_int(fld_task_id, task_id).Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
int step_id = rdr.Read_int("step_id");
|
||||
if (step_ids.Has(step_id))
|
||||
return rdr.Read_int("step_seqn");
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
throw Err_.new_("", "xobc:could not find next sort", "task_id", task_id);
|
||||
}
|
||||
public List_adp Select_all(int task_id) {
|
||||
List_adp rv = List_adp_.New();
|
||||
Db_rdr rdr = conn.Stmt_select_order(tbl_name, flds, String_.Ary(fld_task_id), fld_step_seqn).Crt_int(fld_task_id, task_id).Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
rv.Add(rdr.Read_int("step_id"));
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
return rv;
|
||||
}
|
||||
public void Insert(int sm_id, int task_id, int step_id, int step_seqn) {
|
||||
if (insert_stmt == null) insert_stmt = conn.Stmt_insert(tbl_name, flds);
|
||||
insert_stmt.Clear().Val_int(fld_sm_id, sm_id).Val_int(fld_task_id, task_id).Val_int(fld_step_id, step_id).Val_int(fld_step_seqn, step_seqn)
|
||||
.Exec_insert();
|
||||
}
|
||||
public void Delete_by_task_id(int task_id) {
|
||||
conn.Stmt_delete(tbl_name, fld_task_id).Crt_int(fld_task_id, task_id).Exec_delete();
|
||||
}
|
||||
public void Rls() {
|
||||
insert_stmt = Db_stmt_.Rls(insert_stmt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,41 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.tasks.*;
|
||||
public class Xobc_step_regy_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld_step_id, fld_step_type;
|
||||
private final Db_conn conn; private Db_stmt insert_stmt;
|
||||
public Xobc_step_regy_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "step_regy";
|
||||
this.fld_step_id = flds.Add_int_pkey("step_id");
|
||||
this.fld_step_type = flds.Add_int("step_type");
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
|
||||
public int Select_type(int step_id) {
|
||||
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_step_id).Crt_int(fld_step_id, step_id).Exec_select__rls_auto();
|
||||
try {
|
||||
if (rdr.Move_next())
|
||||
return rdr.Read_int(fld_step_type);
|
||||
else
|
||||
throw Err_.new_("", "bldr.central:could not find step_type", "step_id", step_id);
|
||||
}
|
||||
finally {rdr.Rls();}
|
||||
}
|
||||
public void Insert(int step_id, int step_type) {
|
||||
if (insert_stmt == null) insert_stmt = conn.Stmt_insert(tbl_name, flds);
|
||||
insert_stmt.Clear().Val_int(fld_step_id, step_id).Val_int(fld_step_type, step_type)
|
||||
.Exec_insert();
|
||||
}
|
||||
public void Delete(int step_id) {
|
||||
conn.Stmt_delete(tbl_name, fld_step_id).Crt_int(fld_step_id, step_id).Exec_delete();
|
||||
}
|
||||
public void Rls() {
|
||||
insert_stmt = Db_stmt_.Rls(insert_stmt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,21 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
public class Xobc_task_regy_itm {
|
||||
public Xobc_task_regy_itm(int id, int seqn, byte[] key, byte[] name, int step_count) {
|
||||
this.id = id;
|
||||
this.seqn = seqn;
|
||||
this.key = key;
|
||||
this.name = name;
|
||||
this.step_count = step_count;
|
||||
}
|
||||
public int Id() {return id;} private final int id;
|
||||
public int Seqn() {return seqn;} private final int seqn;
|
||||
public byte[] Key() {return key;} private final byte[] key;
|
||||
public byte[] Name() {return name;} private final byte[] name;
|
||||
public int Step_count() {return step_count;} private final int step_count;
|
||||
|
||||
public static final String Type__text = "text", Type__html = "html", Type__file = "file";
|
||||
public static final int Seqn__obsolete = 999999;
|
||||
}
|
||||
|
||||
@@ -13,86 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.tasks.*;
|
||||
public class Xobc_task_regy_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld_task_id, fld_task_seqn, fld_step_count, fld_task_key, fld_task_name;
|
||||
private final Db_conn conn; private Db_stmt insert_stmt;
|
||||
public Xobc_task_regy_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "task_regy";
|
||||
this.fld_task_id = flds.Add_int_pkey("task_id");
|
||||
this.fld_task_seqn = flds.Add_int("task_seqn");
|
||||
this.fld_step_count = flds.Add_int("step_count");
|
||||
this.fld_task_key = flds.Add_str("task_key", 255);
|
||||
this.fld_task_name = flds.Add_str("task_name", 255);
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public void Create_tbl() {
|
||||
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
|
||||
conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "task_seqn", fld_task_seqn));
|
||||
}
|
||||
public void Select_all(Xobc_task_regy__base todo_regy) {
|
||||
todo_regy.Clear();
|
||||
Db_rdr rdr = conn.Stmt_select_order(tbl_name, flds, String_.Ary_empty, fld_task_seqn).Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
int task_seqn = rdr.Read_int(fld_task_seqn);
|
||||
if (task_seqn == Xobc_task_regy_itm.Seqn__obsolete) continue; // WORKAROUND: do not show old tasks; should add a status column, but don't want to change schema yet; DATE:2016-10-20
|
||||
|
||||
int task_id = rdr.Read_int(fld_task_id);
|
||||
int step_count = rdr.Read_int(fld_step_count);
|
||||
String task_key = rdr.Read_str(fld_task_key);
|
||||
String task_name = rdr.Read_str(fld_task_name);
|
||||
todo_regy.Add(new Xobc_task_itm(task_id, task_seqn, step_count, task_key, task_name));
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
}
|
||||
public int Select_by_key(String key) {
|
||||
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_task_key).Exec_select__rls_auto();
|
||||
try {return rdr.Move_next() ? rdr.Read_int(fld_task_id) : -1;}
|
||||
finally {rdr.Rls();}
|
||||
}
|
||||
public String Select_key_by_id_or_null(int id) {
|
||||
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_task_id).Crt_int(fld_task_id, id).Exec_select__rls_auto();
|
||||
try {return rdr.Move_next() ? rdr.Read_str(fld_task_key) : null;}
|
||||
finally {rdr.Rls();}
|
||||
}
|
||||
public void Insert(int task_id, int task_seqn, int step_count, String task_key, String task_name) {
|
||||
if (insert_stmt == null) insert_stmt = conn.Stmt_insert(tbl_name, flds);
|
||||
insert_stmt.Clear().Val_int(fld_task_id, task_id).Val_int(fld_task_seqn, task_seqn).Val_int(fld_step_count, step_count)
|
||||
.Val_str(fld_task_key, task_key).Val_str(fld_task_name, task_name)
|
||||
.Exec_insert();
|
||||
}
|
||||
public Xobc_task_regy_itm[] Select_by_wiki(byte[] wiki_domain) {
|
||||
String sql = Db_sql_.Make_by_fmt(String_.Ary
|
||||
( "SELECT *"
|
||||
, "FROM task_regy"
|
||||
, "WHERE task_key LIKE '{0}%'" // DEPENDENCY:Xobc_task_key
|
||||
, "AND task_seqn != 999999"
|
||||
), wiki_domain);
|
||||
List_adp list = List_adp_.New();
|
||||
Db_rdr rdr = conn.Stmt_sql(sql).Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
list.Add(new Xobc_task_regy_itm
|
||||
( rdr.Read_int("task_id")
|
||||
, rdr.Read_int("task_seqn")
|
||||
, rdr.Read_bry_by_str("task_key")
|
||||
, rdr.Read_bry_by_str("task_name")
|
||||
, rdr.Read_int("step_count")
|
||||
));
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
return (Xobc_task_regy_itm[])list.To_ary_and_clear(Xobc_task_regy_itm.class);
|
||||
}
|
||||
public void Delete(int task_id) {
|
||||
conn.Stmt_delete(tbl_name, fld_task_id).Crt_int(fld_task_id, task_id).Exec_delete();
|
||||
}
|
||||
public void Rls() {
|
||||
insert_stmt = Db_stmt_.Rls(insert_stmt);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,14 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
public class Xobc_version_regy_itm {
|
||||
public Xobc_version_regy_itm(int id, String date, String note) {
|
||||
this.id = id;
|
||||
this.date = date;
|
||||
this.note = note;
|
||||
}
|
||||
public int Id() {return id;} private final int id;
|
||||
public String Date() {return date;} private final String date;
|
||||
public String Note() {return note;} private final String note;
|
||||
}
|
||||
|
||||
@@ -13,36 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
import gplx.dbs.*;
|
||||
public class Xobc_version_regy_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld_version_id, fld_version_date, fld_version_note;
|
||||
private final Db_conn conn;
|
||||
public Xobc_version_regy_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "version_regy";
|
||||
this.fld_version_id = flds.Add_int_pkey("version_id");
|
||||
this.fld_version_date = flds.Add_str("version_date", 16);
|
||||
this.fld_version_note = flds.Add_str("version_note", 255);
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public void Create_tbl() {
|
||||
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
|
||||
conn.Stmt_insert(tbl_name, flds)
|
||||
.Val_int(fld_version_id, 1).Val_str(fld_version_date, Datetime_now.Get().XtoStr_fmt_yyyy_MM_dd_HH_mm()).Val_str(fld_version_note, "initial")
|
||||
.Exec_insert();
|
||||
}
|
||||
public Xobc_version_regy_itm Select_latest() {
|
||||
Db_rdr rdr = conn.Stmt_select(tbl_name, flds).Exec_select__rls_auto();
|
||||
try {
|
||||
if (rdr.Move_next())
|
||||
return new Xobc_version_regy_itm(rdr.Read_int(fld_version_id), rdr.Read_str(fld_version_date), rdr.Read_str(fld_version_note));
|
||||
else
|
||||
throw Err_.new_("", "version_regy does not have version");
|
||||
}
|
||||
finally {rdr.Rls();}
|
||||
}
|
||||
public void Rls() {}
|
||||
}
|
||||
|
||||
@@ -13,9 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*;
|
||||
public class Xobc_import_date {
|
||||
public static String To_str__yyyy_mm(String raw, String dlm) {
|
||||
return String_.Mid(raw, 0, 4) + dlm + String_.Mid(raw, 4, 6);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,38 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*;
|
||||
import gplx.dbs.*;
|
||||
public class Xobc_import_step_itm {
|
||||
public Xobc_import_step_itm(int step_id, int host_id, byte[] wiki_abrv, String wiki_date
|
||||
, String import_name, int import_type, byte import_zip_type, long import_size_zip, long import_size_raw, String import_md5
|
||||
, long import_prog_data_max, int import_prog_row_max
|
||||
) {
|
||||
this.Step_id = step_id;
|
||||
this.Host_id = host_id;
|
||||
this.wiki_abrv = wiki_abrv;
|
||||
this.Wiki_date = wiki_date;
|
||||
this.Import_name = import_name;
|
||||
this.Import_type = import_type;
|
||||
this.Import_zip_type = import_zip_type;
|
||||
this.Import_size_zip = import_size_zip;
|
||||
this.Import_size_raw = import_size_raw;
|
||||
this.Import_md5 = import_md5;
|
||||
this.Import_prog_data_max = import_prog_data_max;
|
||||
this.Import_prog_row_max = import_prog_row_max;
|
||||
}
|
||||
public final int Step_id;
|
||||
public final int Host_id;
|
||||
public byte[] Wiki_abrv() {return wiki_abrv;} private final byte[] wiki_abrv;
|
||||
public final String Wiki_date;
|
||||
public final String Import_name;
|
||||
public final int Import_type;
|
||||
public final byte Import_zip_type;
|
||||
public final long Import_size_zip;
|
||||
public final long Import_size_raw;
|
||||
public final String Import_md5;
|
||||
public final long Import_prog_data_max;
|
||||
public final int Import_prog_row_max;
|
||||
|
||||
public static final Xobc_import_step_itm Null = null;
|
||||
}
|
||||
|
||||
@@ -13,106 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*;
|
||||
import gplx.dbs.*;
|
||||
public class Xobc_import_step_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld_step_id, fld_host_id, fld_wiki_abrv, fld_wiki_date, fld_import_name, fld_import_type, fld_import_zip, fld_import_md5, fld_import_size_zip, fld_import_size_raw, fld_prog_size_end, fld_prog_count_end;
|
||||
public final Db_conn conn; private Db_stmt insert_stmt;
|
||||
public Xobc_import_step_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "import_step";
|
||||
this.fld_step_id = flds.Add_int_pkey("step_id");
|
||||
this.fld_host_id = flds.Add_int("host_id");
|
||||
this.fld_wiki_abrv = flds.Add_str("wiki_abrv", 255);
|
||||
this.fld_wiki_date = flds.Add_str("wiki_date", 8);
|
||||
this.fld_import_name = flds.Add_str("import_name", 255);
|
||||
this.fld_import_type = flds.Add_int("import_type");
|
||||
this.fld_import_zip = flds.Add_byte("import_zip");
|
||||
this.fld_import_size_zip = flds.Add_long("import_size_zip");
|
||||
this.fld_import_size_raw = flds.Add_long("import_size_raw");
|
||||
this.fld_import_md5 = flds.Add_str("import_md5", 48);
|
||||
this.fld_prog_size_end = flds.Add_long("prog_size_end");
|
||||
this.fld_prog_count_end = flds.Add_long("prog_count_end");
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
|
||||
public Xobc_import_step_itm Select_one(int step_id) {
|
||||
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_step_id).Crt_int(fld_step_id, step_id).Exec_select__rls_auto();
|
||||
try {
|
||||
return (rdr.Move_next())
|
||||
? New_itm(rdr)
|
||||
: Xobc_import_step_itm.Null;
|
||||
}
|
||||
finally {rdr.Rls();}
|
||||
}
|
||||
public void Delete(int step_id) {
|
||||
conn.Stmt_delete(tbl_name, fld_step_id).Crt_int(fld_step_id, step_id).Exec_delete();
|
||||
}
|
||||
public void Insert(int step_id, int host_id, byte[] wiki_abrv, String wiki_date, String import_name, int import_type, byte zip_type, byte[] md5, long size_zip, long size_raw
|
||||
, long prog_size_end, int prog_count_end) {
|
||||
if (insert_stmt == null) insert_stmt = conn.Stmt_insert(tbl_name, flds);
|
||||
insert_stmt.Clear().Val_int(fld_step_id, step_id).Val_int(fld_host_id, host_id)
|
||||
.Val_bry_as_str(fld_wiki_abrv, wiki_abrv).Val_str(fld_wiki_date, wiki_date)
|
||||
.Val_str(fld_import_name, import_name)
|
||||
.Val_int(fld_import_type, import_type).Val_byte(fld_import_zip, zip_type)
|
||||
.Val_long(fld_import_size_zip, size_zip).Val_long(fld_import_size_raw, size_raw).Val_bry_as_str(fld_import_md5, md5)
|
||||
.Val_long(fld_prog_size_end, prog_size_end).Val_int(fld_prog_count_end, prog_count_end)
|
||||
.Exec_insert();
|
||||
}
|
||||
public void Select_tasks_steps(Xobc_task_step_hash task_step_hash, Xobc_step_map_tbl step_map_tbl, byte[] wiki_abrv, String wiki_date) {
|
||||
task_step_hash.Clear();
|
||||
Db_rdr rdr = conn.Stmt_sql(String_.Concat_lines_nl_skip_last
|
||||
( "SELECT DISTINCT sm.task_id, sm.step_id"
|
||||
, "FROM " + tbl_name + " imps"
|
||||
, " JOIN " + step_map_tbl.Tbl_name() + " sm ON sm.step_id = imps.step_id"
|
||||
, "WHERE imps.wiki_abrv = ?"
|
||||
, "AND imps.wiki_date = ?"
|
||||
))
|
||||
.Crt_bry_as_str(fld_wiki_abrv, wiki_abrv)
|
||||
.Crt_str(fld_wiki_date, wiki_date)
|
||||
.Exec_select__rls_auto()
|
||||
;
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
task_step_hash.Add(rdr.Read_int("task_id"), rdr.Read_int("step_id"));
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
}
|
||||
public Xobc_import_step_itm[] Select_by_task_id(int task_id) {
|
||||
List_adp list = List_adp_.New();
|
||||
Db_rdr rdr = conn.Stmt_sql(Db_sql_.Make_by_fmt(String_.Ary
|
||||
( "SELECT s.*"
|
||||
, "FROM import_step s"
|
||||
, " JOIN step_map sm ON s.step_id = sm.step_id"
|
||||
, "WHERE sm.task_id = {0}"
|
||||
), task_id))
|
||||
.Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
list.Add(New_itm(rdr));
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
return (Xobc_import_step_itm[])list.To_ary_and_clear(Xobc_import_step_itm.class);
|
||||
}
|
||||
public void Rls() {
|
||||
insert_stmt = Db_stmt_.Rls(insert_stmt);
|
||||
}
|
||||
private Xobc_import_step_itm New_itm(Db_rdr rdr) {
|
||||
return new Xobc_import_step_itm
|
||||
( rdr.Read_int(fld_step_id)
|
||||
, rdr.Read_int(fld_host_id)
|
||||
, rdr.Read_bry_by_str(fld_wiki_abrv)
|
||||
, rdr.Read_str(fld_wiki_date)
|
||||
, rdr.Read_str(fld_import_name)
|
||||
, rdr.Read_int(fld_import_type)
|
||||
, rdr.Read_byte(fld_import_zip)
|
||||
, rdr.Read_long(fld_import_size_zip)
|
||||
, rdr.Read_long(fld_import_size_raw)
|
||||
, rdr.Read_str(fld_import_md5)
|
||||
, rdr.Read_long(fld_prog_size_end)
|
||||
, rdr.Read_int(fld_prog_count_end)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,21 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*;
|
||||
public class Xobc_import_type {
|
||||
public static final int // SERIALIZED: bc_db; import_step
|
||||
Tid__ignore = 0
|
||||
, Tid__pack = 1
|
||||
, Tid__wiki__core = 2
|
||||
, Tid__wiki__srch = 3
|
||||
, Tid__wiki__html = 4
|
||||
, Tid__wiki__text = 5
|
||||
, Tid__file__core = 6
|
||||
, Tid__file__data = 7
|
||||
, Tid__fsdb__delete = 8
|
||||
, Tid__wiki__ctg = 9
|
||||
, Tid__misc = 10
|
||||
, Tid__wiki__wbase = 11
|
||||
, Tid__wiki__lucene = 12
|
||||
;
|
||||
}
|
||||
|
||||
@@ -13,13 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*;
|
||||
public class Xobc_zip_type {
|
||||
public static String To_str(byte type) {
|
||||
switch (type) {
|
||||
case Type__zip: return "zip";
|
||||
default: throw Err_.new_unhandled_default(type);
|
||||
}
|
||||
}
|
||||
public static final byte Type__zip = 1; // SERIALIZED
|
||||
}
|
||||
|
||||
@@ -13,41 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.users; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.tasks.*;
|
||||
public class Xobc_done_step_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld_task_id, fld_step_id;
|
||||
private final Db_conn conn;
|
||||
public Xobc_done_step_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "xobc_done_step";
|
||||
this.fld_task_id = flds.Add_int("task_id");
|
||||
this.fld_step_id = flds.Add_int("step_id");
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public void Create_tbl() {
|
||||
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
|
||||
conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "main", fld_task_id, fld_step_id));
|
||||
}
|
||||
public void Insert(int task_id, int step_id) {
|
||||
conn.Stmt_insert(tbl_name, fld_task_id, fld_step_id)
|
||||
.Val_int(fld_task_id, task_id).Val_int(fld_step_id, step_id)
|
||||
.Exec_insert();
|
||||
}
|
||||
public Hash_adp Select_all(int task_id) {
|
||||
Hash_adp rv = null;
|
||||
Db_rdr rdr = conn.Stmt_select(tbl_name, String_.Ary(fld_step_id), fld_task_id).Crt_int(fld_task_id, task_id).Exec_select__rls_auto();
|
||||
try {
|
||||
if (rdr.Move_next()) {
|
||||
if (rv == null) rv = Hash_adp_.New();
|
||||
rv.Add_as_key_and_val(rdr.Read_int("step_id"));
|
||||
}
|
||||
}
|
||||
finally {rdr.Rls();}
|
||||
return null;
|
||||
}
|
||||
public void Rls() {}
|
||||
}
|
||||
|
||||
@@ -13,50 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.users; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.tasks.*;
|
||||
public class Xobc_done_task_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld_task_id, fld_task_seqn;
|
||||
private final Db_conn conn;
|
||||
public Xobc_done_task_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "xobc_done_task";
|
||||
this.fld_task_id = flds.Add_int_pkey("task_id");
|
||||
this.fld_task_seqn = flds.Add_int("task_seqn");
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
|
||||
public void Insert(int task_id, int task_seqn) {
|
||||
conn.Stmt_insert(tbl_name, fld_task_id, fld_task_seqn)
|
||||
.Val_int(fld_task_id, task_id).Val_int(fld_task_seqn, task_seqn)
|
||||
.Exec_insert();
|
||||
}
|
||||
public void Delete(int task_id) {
|
||||
conn.Stmt_delete(tbl_name, fld_task_id)
|
||||
.Crt_int(fld_task_id, task_id)
|
||||
.Exec_delete();
|
||||
}
|
||||
public void Select_all(Xobc_task_regy__base todo_regy, Xobc_task_regy__base done_regy) {
|
||||
done_regy.Clear();
|
||||
Db_rdr rdr = conn.Stmt_select_all(tbl_name, flds).Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
int task_id = rdr.Read_int(fld_task_id);
|
||||
int task_seqn = rdr.Read_int(fld_task_seqn);
|
||||
if (task_seqn == gplx.xowa.addons.bldrs.centrals.dbs.datas.Xobc_task_regy_itm.Seqn__obsolete) continue; // WORKAROUND: do not show old tasks; should add a status column, but don't want to change schema yet; DATE:2016-11-03
|
||||
Xobc_task_itm itm = (Xobc_task_itm)todo_regy.Get_by(task_id);
|
||||
if (itm == null) {
|
||||
Gfo_log_.Instance.Warn("task exists in done, but not in todo", "task_id", task_id);
|
||||
continue;
|
||||
}
|
||||
done_regy.Add(itm);
|
||||
todo_regy.Del_by(task_id);
|
||||
itm.Task_seqn_(task_seqn);
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
}
|
||||
public void Rls() {}
|
||||
}
|
||||
|
||||
@@ -13,59 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.users; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.tasks.*;
|
||||
public class Xobc_work_task_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld_task_id, fld_task_seqn, fld_step_id, fld_cmd_id;
|
||||
private final Db_conn conn;
|
||||
public Xobc_work_task_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "xobc_work_task";
|
||||
this.fld_task_id = flds.Add_int_pkey("task_id");
|
||||
this.fld_task_seqn = flds.Add_int("task_seqn");
|
||||
this.fld_step_id = flds.Add_int("step_id");
|
||||
this.fld_cmd_id = flds.Add_int("cmd_id");
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
|
||||
public void Select_all(Xobc_task_mgr task_mgr, Xobc_task_regy__base todo_regy, Xobc_task_regy__base work_regy) {
|
||||
work_regy.Clear();
|
||||
Db_rdr rdr = conn.Stmt_select_all(tbl_name, flds).Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
int task_id = rdr.Read_int(fld_task_id);
|
||||
int task_seqn = rdr.Read_int(fld_task_seqn);
|
||||
int step_id = rdr.Read_int(fld_step_id);
|
||||
int cmd_id = rdr.Read_int(fld_cmd_id);
|
||||
Xobc_task_itm task = (Xobc_task_itm)todo_regy.Get_by(task_id);
|
||||
if (task == null) {
|
||||
Gfo_log_.Instance.Warn("task exists in work, but not in todo", "task_id", task_id);
|
||||
continue;
|
||||
}
|
||||
work_regy.Add(task);
|
||||
todo_regy.Del_by(task_id);
|
||||
task.Task_seqn_(task_seqn);
|
||||
task.Task_status_(cmd_id == Xobc_cmd__base.Seqn__0 ? gplx.core.progs.Gfo_prog_ui_.Status__init : gplx.core.progs.Gfo_prog_ui_.Status__working);
|
||||
task_mgr.Step_mgr().Load(task, step_id, cmd_id);
|
||||
}
|
||||
work_regy.Sort();
|
||||
} finally {rdr.Rls();}
|
||||
}
|
||||
public void Insert(int task_id, int task_seqn, int step_id, int cmd_id) {
|
||||
conn.Stmt_insert(tbl_name, fld_task_id, fld_task_seqn, fld_step_id, fld_cmd_id)
|
||||
.Val_int(fld_task_id, task_id).Val_int(fld_task_seqn, task_seqn).Val_int(fld_step_id, step_id).Val_int(fld_cmd_id, cmd_id)
|
||||
.Exec_insert();
|
||||
}
|
||||
public void Update(int task_id, int task_seqn, int step_id, int cmd_id) {
|
||||
conn.Stmt_update_exclude(tbl_name, flds, fld_task_id)
|
||||
.Val_int(fld_task_seqn, task_seqn).Val_int(fld_step_id, step_id).Val_int(fld_cmd_id, cmd_id).Crt_int(fld_task_id, task_id)
|
||||
.Exec_update();
|
||||
}
|
||||
public void Delete(int task_id) {
|
||||
conn.Stmt_delete(tbl_name, fld_task_id).Val_int(fld_task_id, task_id).Exec_insert();
|
||||
}
|
||||
public void Rls() {}
|
||||
}
|
||||
|
||||
@@ -13,32 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.hosts; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*;
|
||||
import gplx.core.brys.evals.*;
|
||||
import gplx.xowa.wikis.domains.*;
|
||||
public class Host_eval_itm {
|
||||
private final Bry_eval_mgr mgr = Bry_eval_mgr.Dflt();
|
||||
private final Host_eval_wkr wkr = new Host_eval_wkr();
|
||||
private final Hash_adp host_hash = Hash_adp_.New();
|
||||
public Host_eval_itm() {
|
||||
mgr.Add_many(wkr);
|
||||
}
|
||||
public byte[] Eval_dir_name(Xow_domain_itm domain_itm) {
|
||||
wkr.Domain_itm_(domain_itm);
|
||||
return mgr.Eval(Bry_.new_u8("Xowa_~{host_regy|wiki_abrv}_latest"));
|
||||
}
|
||||
public String Eval_src_fil(Xobc_data_db data_db, int host_id, Xow_domain_itm domain, String file_name) {
|
||||
return Eval_src_dir(data_db, host_id, domain) + file_name;
|
||||
}
|
||||
public String Eval_src_dir(Xobc_data_db data_db, int host_id, Xow_domain_itm domain) {
|
||||
Xobc_host_regy_itm host_itm = (Xobc_host_regy_itm)host_hash.Get_by(host_id);
|
||||
if (host_itm == null) {
|
||||
host_itm = data_db.Tbl__host_regy().Select(host_id);
|
||||
host_hash.Add(host_id, host_itm);
|
||||
}
|
||||
wkr.Domain_itm_(domain);
|
||||
String host_dir = String_.new_u8(mgr.Eval(Bry_.new_u8(host_itm.Host_data_dir())));
|
||||
return String_.Format("http://{0}/{1}/", host_itm.Host_domain(), host_dir);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,37 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.hosts; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.brys.evals.*;
|
||||
import gplx.xowa.wikis.domains.*;
|
||||
public class Host_eval_wkr implements Bry_eval_wkr {
|
||||
private Xow_domain_itm domain_itm;
|
||||
public String Key() {return "host_regy";}
|
||||
public Host_eval_wkr Domain_itm_(Xow_domain_itm domain_itm) {this.domain_itm = domain_itm; return this;}
|
||||
public void Resolve(Bry_bfr rv, byte[] src, int args_bgn, int args_end) {
|
||||
// EX: "~{host_regy|wiki_abrv}" -> "enwiki"
|
||||
int type = hash.Get_as_byte_or(src, args_bgn, args_end, Byte_.Max_value_127);
|
||||
switch (type) {
|
||||
case Type__wiki_abrv:
|
||||
// handle wikidata, commonswiki separately; DATE:2016-10-20
|
||||
if (String_.Eq(domain_itm.Domain_str(), "www.wikidata.org"))
|
||||
rv.Add_str_a7("wikidatawiki");
|
||||
else if (String_.Eq(domain_itm.Domain_str(), "commons.wikimedia.org"))
|
||||
rv.Add_str_a7("commonswiki");
|
||||
// do not use Abrv_mw(); all other wikis will be "generalized" to their language url; EX:"en.wiktionary.org" -> "enwiki" x> "enwiktionary"
|
||||
else {
|
||||
byte[] lang_key = domain_itm.Lang_orig_key();
|
||||
if (lang_key == Bry_.Empty) lang_key = Bry_.new_a7("en"); // handle species
|
||||
rv.Add(lang_key);
|
||||
rv.Add_str_a7("wiki");
|
||||
}
|
||||
break;
|
||||
default: throw Err_.new_unhandled_default(type);
|
||||
}
|
||||
}
|
||||
|
||||
public static final byte Type__wiki_abrv = 0;
|
||||
private static final Hash_adp_bry hash = Hash_adp_bry.cs()
|
||||
.Add_str_byte("wiki_abrv", Type__wiki_abrv)
|
||||
;
|
||||
}
|
||||
|
||||
@@ -13,18 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.hosts; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import org.junit.*; import gplx.core.tests.*; import gplx.xowa.wikis.domains.*;
|
||||
public class Host_eval_wkr__tst {
|
||||
private final Host_eval_wkr__fxt fxt = new Host_eval_wkr__fxt();
|
||||
@Test public void En_w() {fxt.Test__resolve_quick("en.wikipedia.org" , "Xowa_enwiki_latest");}
|
||||
@Test public void Fr_d() {fxt.Test__resolve_quick("fr.wiktionary.org" , "Xowa_frwiki_latest");}
|
||||
@Test public void Species() {fxt.Test__resolve_quick("species.wikimedia.org" , "Xowa_enwiki_latest");}
|
||||
}
|
||||
class Host_eval_wkr__fxt {
|
||||
public void Test__resolve_quick(String domain_str, String expd) {
|
||||
Host_eval_itm eval_itm = new Host_eval_itm();
|
||||
Xow_domain_itm domain_itm = Xow_domain_itm_.parse(Bry_.new_u8(domain_str));
|
||||
Gftest.Eq__bry(Bry_.new_u8(expd), eval_itm.Eval_dir_name(domain_itm));
|
||||
}
|
||||
}
|
||||
@@ -13,38 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.gfobjs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.tasks.*;
|
||||
import gplx.xowa.wikis.domains.*;
|
||||
import gplx.xowa.langs.*;
|
||||
public class Xobc_filter_mgr {
|
||||
private String lang_key_str = Xow_domain_itm_.Lang_key__all, type_key_str = Xow_domain_itm_.Type_key__all;
|
||||
public Xobc_task_itm[] Filter(Xobc_task_regy__base task_list) {return Filter(task_list, lang_key_str, type_key_str);}
|
||||
public Xobc_task_itm[] Filter(Xobc_task_regy__base task_list, String lang_key_str, String type_key_str) {
|
||||
this.lang_key_str = lang_key_str;
|
||||
this.type_key_str = type_key_str;
|
||||
|
||||
List_adp tmp = List_adp_.New();
|
||||
|
||||
// loop tasks and find matches
|
||||
int len = task_list.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xobc_task_itm task = (Xobc_task_itm)task_list.Get_at(i);
|
||||
Xobc_task_key task_key_itm = Xobc_task_key.To_itm(task.Task_key());
|
||||
Xow_domain_itm task_domain = task_key_itm.Wiki_domain_itm();
|
||||
if ( Xow_domain_itm_.Match_lang(task_domain, lang_key_str)
|
||||
&& Xow_domain_itm_.Match_type(task_domain, type_key_str)
|
||||
)
|
||||
tmp.Add(task);
|
||||
}
|
||||
|
||||
return (Xobc_task_itm[])tmp.To_ary_and_clear(Xobc_task_itm.class);
|
||||
}
|
||||
public Gfobj_nde Make_init_msg() {
|
||||
Gfobj_nde root = Gfobj_nde.New();
|
||||
root.New_nde("langs").Add_str("active", lang_key_str);
|
||||
root.New_nde("types").Add_str("active", type_key_str);
|
||||
return root;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,20 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.gfobjs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.datas.*;
|
||||
public class Xobc_lang_mgr {
|
||||
public Gfobj_ary Make_init_msg(Xobc_lang_regy_itm[] itms) {
|
||||
List_adp list = List_adp_.New();
|
||||
int len = itms.length;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xobc_lang_regy_itm itm = itms[i];
|
||||
Gfobj_nde itm_nde = Gfobj_nde.New();
|
||||
list.Add(itm_nde);
|
||||
itm_nde.Add_str("key", itm.Key());
|
||||
itm_nde.Add_str("name", itm.Name());
|
||||
}
|
||||
return new Gfobj_ary((Gfobj_nde[])list.To_ary_and_clear(Gfobj_nde.class));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,25 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*;
|
||||
public class Xobc_skip_mgr implements Gfo_invk {
|
||||
private boolean category_enabled;
|
||||
public Xobc_skip_mgr(Xobc_task_mgr task_mgr, Xoa_app app) {
|
||||
app.Cfg().Bind_many_app(this, Cfg__namespaces_category);
|
||||
}
|
||||
public boolean Should_skip(Xobc_import_step_itm step_itm) {
|
||||
if (category_enabled && step_itm.Import_type == Xobc_import_type.Tid__wiki__ctg)
|
||||
return true;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Cfg__namespaces_category)) category_enabled = m.ReadBool("v");
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
}
|
||||
private static final String
|
||||
Cfg__namespaces_category = "xowa.bldr.download_central.namespaces.category"
|
||||
;
|
||||
}
|
||||
|
||||
@@ -13,77 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.steps; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.brys.evals.*; import gplx.core.primitives.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.tasks.*; import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.utils.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
|
||||
import gplx.xowa.addons.bldrs.exports.merges.*;
|
||||
import gplx.xowa.addons.bldrs.updates.files.*;
|
||||
import gplx.xowa.addons.bldrs.exports.packs.files.*;
|
||||
import gplx.xowa.bldrs.*;
|
||||
import gplx.xowa.wikis.domains.*;
|
||||
public class Xobc_step_factory {
|
||||
private final Xobc_task_mgr task_mgr;
|
||||
private final Xobc_data_db data_db;
|
||||
private final Bry_eval_mgr eval_mgr = Bry_eval_mgr.Dflt(); private final Host_eval_itm host_eval = new Host_eval_itm();
|
||||
private final Host_eval_wkr eval_wkr__host_regy = new Host_eval_wkr();
|
||||
public Xobc_step_factory(Xobc_task_mgr task_mgr, Xobc_data_db data_db, Io_url wiki_dir) {
|
||||
this.task_mgr = task_mgr;
|
||||
this.data_db = data_db;
|
||||
eval_mgr.Add_many(new Bry_eval_wkr__builder_central(wiki_dir));
|
||||
}
|
||||
public void Load(Xobc_task_itm task, int step_id, int cmd_idx) {
|
||||
int step_type = data_db.Tbl__step_regy().Select_type(step_id);
|
||||
switch (step_type) {
|
||||
case Xobc_step_itm.Type__wiki_import: Load_wiki_import(task, step_id, cmd_idx); break;
|
||||
default: throw Err_.new_unhandled_default(step_type);
|
||||
}
|
||||
}
|
||||
private void Load_wiki_import(Xobc_task_itm task, int step_id, int cmd_idx) {
|
||||
int step_seqn = data_db.Tbl__step_map().Select_one(task.Task_id(), step_id).Step_seqn;
|
||||
Xobc_import_step_itm import_itm = data_db.Tbl__import_step().Select_one(step_id); String_obj_ref step_name = String_obj_ref.empty_();
|
||||
Xobc_cmd_itm[] cmds = Make_wiki_import_cmds(import_itm, task.Task_id(), step_id, step_name, step_seqn);
|
||||
Xobc_step_itm step = new Xobc_step_itm(step_id, step_seqn, cmds).Cmd_idx_(cmd_idx);
|
||||
step.Step_name_(String_.Format("{0} ·({1}/{2})", step_name.Val(), step_seqn + List_adp_.Base1, task.Step_count()));
|
||||
task.Step_(step);
|
||||
step.Cmd().Load_checkpoint();
|
||||
if (step.Cmd().Prog_status() == gplx.core.progs.Gfo_prog_ui_.Status__suspended)
|
||||
task.Task_status_(step.Cmd().Prog_status());
|
||||
}
|
||||
private Xobc_cmd_itm[] Make_wiki_import_cmds(Xobc_import_step_itm import_itm, int task_id, int step_id, String_obj_ref step_name, int step_seqn) {
|
||||
List_adp list = List_adp_.New();
|
||||
Xow_domain_itm domain_itm = Xow_abrv_xo_.To_itm(import_itm.Wiki_abrv());
|
||||
String wiki_domain = domain_itm.Domain_str();
|
||||
String file_name = import_itm.Import_name;
|
||||
step_name.Val_(file_name);
|
||||
eval_wkr__host_regy.Domain_itm_(domain_itm);
|
||||
String src_http_url = host_eval.Eval_src_fil(data_db, import_itm.Host_id, domain_itm, file_name);
|
||||
Io_url zip_file_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__download_fil, wiki_domain, file_name));
|
||||
Io_url unzip_dir_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__unzip_dir, wiki_domain, file_name));
|
||||
Io_url wiki_dir_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__wiki_dir, wiki_domain, file_name));
|
||||
|
||||
// if lucene, move to /data/search/
|
||||
if (import_itm.Import_type == Xobc_import_type.Tid__wiki__lucene) {
|
||||
wiki_dir_url = gplx.xowa.addons.wikis.fulltexts.Xosearch_fulltext_addon.Get_index_dir(wiki_dir_url);
|
||||
}
|
||||
|
||||
Io_url checksum_url = unzip_dir_url.GenSubFil(file_name + ".md5");
|
||||
int cmd_idx = 0;
|
||||
list.Add(new Xobc_cmd__download (task_mgr, task_id, step_id, cmd_idx++, src_http_url, zip_file_url, import_itm.Import_size_zip));
|
||||
list.Add(new Xobc_cmd__verify_fil (task_mgr, task_id, step_id, cmd_idx++, zip_file_url, import_itm.Import_md5, import_itm.Import_size_zip));
|
||||
list.Add(new Xobc_cmd__unzip (task_mgr, task_id, step_id, cmd_idx++, zip_file_url, unzip_dir_url, import_itm.Import_size_raw));
|
||||
list.Add(new Xobc_cmd__verify_dir (task_mgr, task_id, step_id, cmd_idx++, checksum_url, zip_file_url));
|
||||
// list.Add(new Xobc_cmd__wiki_merge (task_mgr, task_id, step_id, cmd_idx++, merge_mgr, wiki_domain, unzip_dir_url, import_itm.Import_prog_data_max, import_itm.Import_prog_row_max, step_seqn));
|
||||
list.Add(new Xobc_cmd__move_fils (task_mgr, task_id, step_id, cmd_idx++, unzip_dir_url, wiki_dir_url));
|
||||
switch (import_itm.Import_type) {
|
||||
case Xobc_import_type.Tid__wiki__core: list.Add(new Xobc_cmd__wiki_reg (task_mgr, task_id, step_id, cmd_idx++, wiki_dir_url, wiki_domain)); break;
|
||||
case Xobc_import_type.Tid__fsdb__delete: list.Add(new Xobc_cmd__fsdb_delete (task_mgr, task_id, step_id, cmd_idx++, Pack_zip_name_bldr.To_wiki_url(wiki_dir_url, zip_file_url.OwnerDir()))); break;
|
||||
}
|
||||
return (Xobc_cmd_itm[])list.To_ary_and_clear(Xobc_cmd_itm.class);
|
||||
}
|
||||
private Io_url Eval_url(String src) {return Io_url_.new_any_(String_.new_u8(eval_mgr.Eval(Bry_.new_u8(src))));}
|
||||
public static Xow_wiki Get_wiki_by_abrv(Xoa_app app, byte[] wiki_abrv) {
|
||||
Xow_domain_itm domain_itm = Xow_abrv_xo_.To_itm(wiki_abrv);
|
||||
return app.Wiki_mgri().Get_by_or_make_init_y(domain_itm.Domain_bry());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,49 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.steps; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.gfobjs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.cmds.*;
|
||||
public class Xobc_step_itm {
|
||||
private int cmd_idx = 0;
|
||||
private final Xobc_cmd_itm[] cmds;
|
||||
public Xobc_step_itm(int step_id, int step_seqn, Xobc_cmd_itm[] cmds) {
|
||||
this.step_id = step_id; this.step_seqn = step_seqn; this.cmds = cmds;
|
||||
}
|
||||
public int Step_id() {return step_id;} private final int step_id;
|
||||
public int Step_seqn() {return step_seqn;} private final int step_seqn;
|
||||
public String Step_name() {return step_name;} private String step_name;
|
||||
public Xobc_cmd_itm Cmd() {return cmds[cmd_idx];}
|
||||
public Xobc_step_itm Cmd_idx_(int v) {cmd_idx = v; return this;}
|
||||
public void Cmd_idx_next_() {++cmd_idx;}
|
||||
public boolean Cmd_is_last() {return cmd_idx == cmds.length - 1;}
|
||||
public Xobc_cmd_itm Step_fallback_to(String fallback_id) {
|
||||
int fallback_idx = 0;
|
||||
int len = cmds.length;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xobc_cmd_itm cmd = cmds[i];
|
||||
if (String_.Eq(cmd.Cmd_type(), fallback_id)) {
|
||||
fallback_idx = cmd.Cmd_id();
|
||||
break;
|
||||
}
|
||||
}
|
||||
cmd_idx = fallback_idx;
|
||||
return Cmd();
|
||||
}
|
||||
public void Step_name_(String v) {this.step_name = v;}
|
||||
public void Step_cleanup() {
|
||||
int len = cmds.length;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
cmds[i].Cmd_cleanup();
|
||||
}
|
||||
}
|
||||
public Gfobj_nde Save_to(Gfobj_nde nde) {
|
||||
nde.Add_int ("step_id", step_id);
|
||||
nde.Add_str ("step_name", step_name);
|
||||
this.Cmd().Save_to(nde.New_nde("cmd"));
|
||||
return nde; // fluent
|
||||
}
|
||||
|
||||
public static final int Type__wiki_import = 1;
|
||||
public static final int Seqn__0 = 0;
|
||||
}
|
||||
|
||||
@@ -13,39 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.tasks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.gfobjs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.steps.*;
|
||||
public class Xobc_task_itm implements gplx.CompareAble {
|
||||
private final int task_sort;
|
||||
public Xobc_task_itm(int task_id, int task_seqn, int step_count, String task_key, String task_name) {
|
||||
this.task_id = task_id;
|
||||
this.task_seqn = task_seqn;
|
||||
this.step_count = step_count;
|
||||
this.task_key = task_key;
|
||||
this.task_name = task_name;
|
||||
this.task_sort = task_seqn; // set task_sort to task_seqn; note that task_sort needs to be memorialized b/c "work" changes task_seqn to add-order
|
||||
}
|
||||
public int Task_id() {return task_id;} private final int task_id;
|
||||
public String Task_key() {return task_key;} private final String task_key;
|
||||
public String Task_name() {return task_name;} private final String task_name;
|
||||
public int Step_count() {return step_count;} private final int step_count;
|
||||
public int Task_seqn() {return task_seqn;} private int task_seqn;
|
||||
public byte Task_status() {return task_status;} private byte task_status;
|
||||
public Xobc_step_itm Step() {return step;} public void Step_(Xobc_step_itm v) {this.step = v;} private Xobc_step_itm step;
|
||||
public boolean Step_is_last() {return step.Step_seqn() == step_count - 1;}
|
||||
public void Task_status_(byte v) {task_status = v;} // called when task moves from init -> working -> suspended -> done
|
||||
public void Task_seqn_(int v) {this.task_seqn = v;} // called when task is init'd from db, or added to list
|
||||
public int compareTo(Object obj) {Xobc_task_itm comp = (Xobc_task_itm)obj; return Int_.Compare(task_seqn, comp.task_seqn);}
|
||||
|
||||
public Gfobj_nde Save_to(Gfobj_nde nde) {
|
||||
nde.Add_int ("task_id" , task_id);
|
||||
nde.Add_str ("task_name" , task_name);
|
||||
nde.Add_byte("task_status" , task_status);
|
||||
nde.Add_int ("task_sort" , task_sort);
|
||||
if (step != null) {
|
||||
step.Save_to(nde.New_nde("step"));
|
||||
}
|
||||
return nde; // FLUENT
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,32 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.tasks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.xowa.wikis.domains.*;
|
||||
public class Xobc_task_key {
|
||||
public Xobc_task_key(String wiki_domain, String wiki_date, String task_type) {
|
||||
this.wiki_domain = wiki_domain;
|
||||
this.wiki_date = wiki_date;
|
||||
this.task_type = task_type;
|
||||
}
|
||||
public String Wiki_domain() {return wiki_domain;} private final String wiki_domain;
|
||||
public String Wiki_date() {return wiki_date;} private final String wiki_date;
|
||||
public String Wiki_date_ui() {return String_.Replace(wiki_date, ".", "-");}
|
||||
public String Task_type() {return task_type;} private final String task_type;
|
||||
public String Task_type_ui() {
|
||||
if (String_.Eq(task_type, "html")) return "Articles";
|
||||
else if (String_.Eq(task_type, "file")) return "Images";
|
||||
else if (String_.Eq(task_type, "text")) return "Source";
|
||||
else if (String_.Eq(task_type, "patch")) return "Patch";
|
||||
else return task_type;
|
||||
}
|
||||
public Xow_domain_itm Wiki_domain_itm() {return Xow_domain_itm_.parse(Bry_.new_u8(wiki_domain));}
|
||||
|
||||
public static Xobc_task_key To_itm(String task_key) {
|
||||
String[] ary = String_.Split(task_key, "|");
|
||||
return new Xobc_task_key(ary[0], ary[1], ary[2]);
|
||||
}
|
||||
public static String To_str(String wiki_domain, String wiki_date, String task_type) {
|
||||
return String_.Concat(wiki_domain, "|", wiki_date, "|", task_type);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,29 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.tasks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.gfobjs.*;
|
||||
public abstract class Xobc_task_regy__base {
|
||||
private final Ordered_hash hash = Ordered_hash_.New();
|
||||
protected final Xobc_task_mgr task_mgr;
|
||||
public Xobc_task_regy__base(Xobc_task_mgr task_mgr, String name) {this.task_mgr = task_mgr; this.name = name;}
|
||||
public String Name() {return name;} private final String name;
|
||||
public int Len() {return hash.Len();}
|
||||
public void Add(Xobc_task_itm t) {hash.Add(t.Task_id(), t);}
|
||||
public void Clear() {hash.Clear();}
|
||||
public Xobc_task_itm Get_at(int i) {return (Xobc_task_itm)hash.Get_at(i);}
|
||||
public Xobc_task_itm Get_by(int i) {return (Xobc_task_itm)hash.Get_by(i);}
|
||||
public void Del_by(int i) {hash.Del(i);}
|
||||
public void Sort() {hash.Sort();}
|
||||
|
||||
public void Save_to(Gfobj_ary ary) {Save_to(ary, (Xobc_task_itm[])hash.To_ary(Xobc_task_itm.class));}
|
||||
public void Save_to(Gfobj_ary ary, Xobc_task_itm[] itms) {
|
||||
int len = itms.length;
|
||||
Gfobj_nde[] sub_ndes = new Gfobj_nde[len];
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Gfobj_nde sub_nde = sub_ndes[i] = Gfobj_nde.New();
|
||||
itms[i].Save_to(sub_nde);
|
||||
}
|
||||
ary.Ary_(sub_ndes);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,11 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.tasks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
public class Xobc_task_regy__done extends Xobc_task_regy__base {
|
||||
public Xobc_task_regy__done(Xobc_task_mgr task_mgr) {super(task_mgr, "done");}
|
||||
public void Del_done(int task_id) {
|
||||
task_mgr.User_db().Done_task_tbl().Delete(task_id);
|
||||
task_mgr.Transfer(this, task_mgr.Todo_mgr(), this.Get_by(task_id));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,34 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.tasks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.gfobjs.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
public class Xobc_task_regy__todo extends Xobc_task_regy__base {
|
||||
public Xobc_task_regy__todo(Xobc_task_mgr task_mgr) {super(task_mgr, "todo");}
|
||||
public void Add_work(int task_id) {
|
||||
Xobc_user_db user_db = task_mgr.User_db();
|
||||
Xobc_task_itm task = task_mgr.Todo_mgr().Get_by(task_id);
|
||||
task.Task_status_(gplx.core.progs.Gfo_prog_ui_.Status__init);
|
||||
task.Task_seqn_(task_mgr.Work_mgr().Len());
|
||||
|
||||
// get step_id; default is 1st step with seqn=0, but skip any steps that have already been downloaded; handles user accidentally removing item from work;
|
||||
int step_seqn = Xobc_step_itm.Seqn__0;
|
||||
Hash_adp done_steps = user_db.Done_step_tbl().Select_all(task_id);
|
||||
if (done_steps != null)
|
||||
step_seqn = task_mgr.Data_db().Tbl__step_map().Select_seqn_but_skip_done(task_id, done_steps);
|
||||
int step_id = task_mgr.Data_db().Tbl__step_map().Select_step_id(task_id, step_seqn);
|
||||
|
||||
// load task, cur_step, and all cmds
|
||||
task_mgr.Step_mgr().Load(task, step_id, Xobc_cmd__base.Seqn__0);
|
||||
|
||||
// do transfer
|
||||
user_db.Work_task_tbl().Insert(task_id, task.Task_seqn(), step_id, Xobc_cmd__base.Seqn__0);
|
||||
task_mgr.Transfer(this, task_mgr.Work_mgr(), task);
|
||||
}
|
||||
public void Del_todo(int task_id) {
|
||||
Xobc_task_itm task = this.Get_by(task_id);
|
||||
task_mgr.User_db().Done_task_tbl().Insert(task_id, task.Task_seqn());
|
||||
task_mgr.Transfer(this, task_mgr.Done_mgr(), task);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,149 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.tasks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.gfobjs.*; import gplx.core.progs.*; import gplx.core.threads.*;
|
||||
import gplx.xowa.drds.powers.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.cmds.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*;
|
||||
public class Xobc_task_regy__work extends Xobc_task_regy__base {
|
||||
private final Thread_adp_mgr thread_mgr = new Thread_adp_mgr(1000, 5000);
|
||||
private final Xobc_cmd_ctx ctx;
|
||||
public Xobc_task_regy__work(Xobc_task_mgr task_mgr, Xoa_app app) {super(task_mgr, "work");
|
||||
this.ctx = new Xobc_cmd_ctx(app);
|
||||
}
|
||||
public void Del_work(int task_id) {
|
||||
Xobc_task_itm task = this.Get_by(task_id);
|
||||
if (task.Step() != null) task.Step().Step_cleanup();
|
||||
this.Del_task(task, task_mgr.Todo_mgr());
|
||||
}
|
||||
private void Del_task(Xobc_task_itm task, Xobc_task_regy__base trg) {
|
||||
task_mgr.User_db().Work_task_tbl().Delete(task.Task_id());
|
||||
task_mgr.Transfer(this, trg, task);
|
||||
}
|
||||
public void Run_next() {
|
||||
Xod_power_mgr_.Instance.Wake_lock__get("task_mgr");
|
||||
int len = this.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xobc_task_itm task = this.Get_at(i);
|
||||
Xobc_cmd_itm cmd = task.Step().Cmd();
|
||||
if (gplx.core.bits.Bitmask_.Has_byte(Gfo_prog_ui_.Status__runnable, cmd.Prog_status())) { // must be runnable
|
||||
Run_task(task, cmd);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void Run_task(Xobc_task_itm task, Xobc_cmd_itm cmd) {
|
||||
// if task marked for skip, launch skip-cmd on separate thread and exit;
|
||||
if (task_mgr.Skip_mgr().Should_skip(task_mgr.Data_db().Tbl__import_step().Select_one(cmd.Step_id()))) {
|
||||
thread_mgr.Add("skip_" + cmd.Cmd_uid(), Thread_adp_.Start_by_key("skip_xobc: " + cmd.Cmd_name(), new Xobc_task_skip(this, cmd), ""));
|
||||
return;
|
||||
}
|
||||
|
||||
task.Task_status_(gplx.core.progs.Gfo_prog_ui_.Status__working);
|
||||
task_mgr.Send_json("xo.bldr.work.prog__start__recv", task.Save_to(Gfobj_nde.New()));
|
||||
thread_mgr.Add(cmd.Cmd_uid(), Thread_adp_.Start_by_val("xobc: " + cmd.Cmd_name(), cmd, cmd, Xobc_cmd__base.Invk__exec, ctx));
|
||||
}
|
||||
public void Stop_cur() {
|
||||
Xod_power_mgr_.Instance.Wake_lock__rls("task_mgr");
|
||||
int len = this.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xobc_task_itm task = this.Get_at(i);
|
||||
Xobc_cmd_itm cmd = task.Step().Cmd();
|
||||
if (gplx.core.bits.Bitmask_.Has_byte(Gfo_prog_ui_.Status__working, cmd.Prog_status())) { // must be runnable
|
||||
Stop_task(cmd.Cmd_uid());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void Redo_cur() {
|
||||
int len = this.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xobc_task_itm task = this.Get_at(i);
|
||||
Xobc_step_itm step = task.Step();
|
||||
Xobc_cmd_itm cmd = step.Cmd();
|
||||
if (gplx.core.bits.Bitmask_.Has_byte(Gfo_prog_ui_.Status__fail, cmd.Prog_status())) { // must be runnable
|
||||
Redo_task(task, step, cmd);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
private void Stop_task(String cmd_uid) {
|
||||
thread_mgr.Halt(cmd_uid, Thread_halt_cbk_.Noop);
|
||||
}
|
||||
private void Redo_task(Xobc_task_itm task, Xobc_step_itm step, Xobc_cmd_itm cmd) {
|
||||
cmd = step.Step_fallback_to(cmd.Cmd_fallback());
|
||||
cmd.Cmd_clear();
|
||||
task.Task_status_(gplx.core.progs.Gfo_prog_ui_.Status__working);
|
||||
task_mgr.Send_json("xo.bldr.work.prog__start__recv", task.Save_to(Gfobj_nde.New()));
|
||||
thread_mgr.Add(cmd.Cmd_uid(), Thread_adp_.Start_by_val("xobc: " + cmd.Cmd_name(), cmd, cmd, Xobc_cmd__base.Invk__exec, ctx));
|
||||
}
|
||||
public void On_done(Xobc_cmd_itm cmd, boolean notify) {
|
||||
Xobc_task_itm task = this.Get_by(cmd.Task_id());
|
||||
Xobc_step_itm step = task.Step();
|
||||
boolean step_is_done = step.Cmd_is_last();
|
||||
boolean task_is_done = step_is_done && task.Step_is_last();
|
||||
if (notify)
|
||||
task_mgr.Send_json("xo.bldr.work.prog__done__recv", Gfobj_nde.New().Add_int("task_id", task.Task_id()).Add_bool("task_is_done", task_is_done));
|
||||
|
||||
// step.done -> task.done || step.next
|
||||
if (step_is_done) {
|
||||
step.Step_cleanup();
|
||||
task_mgr.User_db().Done_step_tbl().Insert(cmd.Task_id(), cmd.Step_id());
|
||||
// task.done
|
||||
if (task_is_done) {
|
||||
task.Task_status_(Gfo_prog_ui_.Status__done);
|
||||
task_mgr.User_db().Done_task_tbl().Insert(cmd.Task_id(), task.Task_seqn()); // NOTE: this will order by todo's sort, not by actually completed sorted
|
||||
Del_task(task, task_mgr.Done_mgr());
|
||||
}
|
||||
// step.next
|
||||
else {
|
||||
int next_step = task_mgr.Data_db().Tbl__step_map().Select_step_id(task.Task_id(), step.Step_seqn() + 1);
|
||||
task_mgr.Step_mgr().Load(task, next_step, Xobc_cmd__base.Seqn__0);
|
||||
}
|
||||
}
|
||||
// step.work -> cmd.next
|
||||
else {
|
||||
step.Cmd_idx_next_();
|
||||
}
|
||||
|
||||
// release wake_lock; will be acquired when task is run_next; DATE:2016-06-29
|
||||
Xod_power_mgr_.Instance.Wake_lock__rls("task_mgr");
|
||||
|
||||
// task_regy.done
|
||||
if (task_is_done) {
|
||||
if (this.Len() > 0)
|
||||
this.Run_next();
|
||||
}
|
||||
// task_regy.work
|
||||
else {
|
||||
cmd = task.Step().Cmd();
|
||||
task_mgr.User_db().Work_task_tbl().Update(cmd.Task_id(), task.Task_seqn(), cmd.Step_id(), cmd.Cmd_id());
|
||||
this.Run_next();
|
||||
}
|
||||
}
|
||||
public void On_suspended(Xobc_cmd_itm cmd) {
|
||||
Xobc_task_itm task = this.Get_by(cmd.Task_id());
|
||||
task.Task_status_(Gfo_prog_ui_.Status__suspended);
|
||||
task_mgr.Send_json("xo.bldr.work.stop_cur__recv", Gfobj_nde.New().Add_int("task_id", task.Task_id()));
|
||||
}
|
||||
public void On_fail(Xobc_cmd_itm task, boolean resume, String msg) {
|
||||
Xod_power_mgr_.Instance.Wake_lock__rls("task_mgr");
|
||||
task_mgr.Send_json("xo.bldr.work.prog__fail__recv", Gfobj_nde.New().Add_int("task_id", task.Task_id()).Add_str("err", msg).Add_bool("resume", resume));
|
||||
}
|
||||
public void On_stat(int task_id, String msg) {
|
||||
task_mgr.Send_json("xo.bldr.work.prog__stat__recv", Gfobj_nde.New().Add_int("task_id", task_id).Add_str("msg", msg));
|
||||
}
|
||||
}
|
||||
class Xobc_task_skip implements Gfo_invk {
|
||||
private final Xobc_task_regy__work work_regy;
|
||||
private final Xobc_cmd_itm cmd;
|
||||
public Xobc_task_skip(Xobc_task_regy__work work_regy, Xobc_cmd_itm cmd) {
|
||||
this.work_regy = work_regy;
|
||||
this.cmd = cmd;
|
||||
}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
work_regy.On_done(cmd, false);
|
||||
return Gfo_invk_.Rv_handled;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,53 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import gplx.core.brys.evals.*;
|
||||
public class Bry_eval_wkr__builder_central implements Bry_eval_wkr {
|
||||
private final byte[] wiki_dir;
|
||||
public Bry_eval_wkr__builder_central(Io_url wiki_dir) {this.wiki_dir = wiki_dir.RawBry();}
|
||||
public String Key() {return "builder_central";}
|
||||
public void Resolve(Bry_bfr rv, byte[] src, int args_bgn, int args_end) {
|
||||
// EX: "~{builder_central|download_fil|en.wikipedia.org|Xowa_enwiki_2016-05_html_ns.000_db.001.zip}" -> "/xowa/wiki/en.wikipedia.org/tmp/bldr/Xowa_enwiki_2016-05_html_ns.000_db.001.zip/download.zip"
|
||||
byte[][] args = Bry_split_.Split(src, args_bgn, args_end, Byte_ascii.Pipe, Bool_.N);
|
||||
int type = hash.Get_as_byte_or(args[0], Byte_.Max_value_127);
|
||||
if (type == Byte_.Max_value_127) throw Err_.new_wo_type("unknown eval type", "src", src);
|
||||
byte dir_spr = gplx.core.envs.Op_sys.Cur().Fsys_dir_spr_byte();
|
||||
rv.Add(wiki_dir); // "/xowa/wiki/"
|
||||
rv.Add(args[1]).Add_byte(dir_spr); // "en.wikipedia.org/"
|
||||
if (type == Type__wiki_dir) return;
|
||||
rv.Add_str_a7("tmp").Add_byte(dir_spr); // "tmp/"
|
||||
rv.Add_str_a7("bldr").Add_byte(dir_spr); // "bldr/"
|
||||
rv.Add(args[2]).Add_byte(dir_spr); // "Xowa_enwiki_2016-05_html_ns.000_db.001.zip/"
|
||||
switch (type) {
|
||||
case Type__download_fil: rv.Add_str_a7("download.zip"); break;
|
||||
case Type__unzip_dir: rv.Add_str_a7("unzip").Add_byte(dir_spr); break;
|
||||
default: throw Err_.new_unhandled_default(type);
|
||||
}
|
||||
}
|
||||
|
||||
public static final byte Type__download_fil = 0, Type__unzip_dir = 1, Type__wiki_dir = 2;
|
||||
private static final Hash_adp_bry hash = Hash_adp_bry.cs()
|
||||
.Add_str_byte("download_fil", Type__download_fil)
|
||||
.Add_str_byte("unzip_dir", Type__unzip_dir)
|
||||
.Add_str_byte("wiki_dir", Type__wiki_dir)
|
||||
;
|
||||
public static String Make_str(byte type, String domain, String file_name) {
|
||||
String type_name = null;
|
||||
switch (type) {
|
||||
case Type__download_fil: type_name = "download_fil"; break;
|
||||
case Type__unzip_dir: type_name = "unzip_dir"; break;
|
||||
case Type__wiki_dir: type_name = "wiki_dir"; break;
|
||||
default: throw Err_.new_unhandled_default(type);
|
||||
}
|
||||
return "~{" + String_.Format("builder_central|{0}|{1}|{2}", type_name, domain, file_name) + "}";
|
||||
}
|
||||
public static String Make_str(byte type, String domain) {
|
||||
String type_name = null;
|
||||
switch (type) {
|
||||
case Type__wiki_dir: type_name = "wiki_dir"; break;
|
||||
default: throw Err_.new_unhandled_default(type);
|
||||
}
|
||||
return "~{" + String_.Format("builder_central|{0}|{1}", type_name, domain) + "}";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,40 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
public class Time_dhms_ {
|
||||
public static String To_str(Bry_bfr bfr, long val, boolean show_unit, int max_places) {To_bfr(bfr, val, show_unit, max_places); return bfr.To_str_and_clear();}
|
||||
public static void To_bfr(Bry_bfr bfr, long val, boolean show_unit, int max_places) {
|
||||
byte suffix = Byte_ascii.Null;
|
||||
int count = 0;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
// get factor and unit
|
||||
long factor = 0;
|
||||
byte unit = Byte_ascii.Null;
|
||||
switch(i) {
|
||||
case 0: unit = Byte_ascii.Ltr_d; factor = 86400; break;
|
||||
case 1: unit = Byte_ascii.Ltr_h; factor = 3600; break;
|
||||
case 2: unit = Byte_ascii.Ltr_m; factor = 60; break;
|
||||
case 3: unit = Byte_ascii.Ltr_s; factor = 1; break;
|
||||
}
|
||||
// calc cur and update val; EX: 3690 -> cur:1,mod:90
|
||||
int cur = (int)(val / factor);
|
||||
val %= factor;
|
||||
if (count == 0) { // no str yet
|
||||
if ( cur == 0 // cur is 0; EX: 3690 and factor = 86400 -> 0 days; don't write anything
|
||||
&& i != 3) // unless it is the seconds place; need to handle "0 s" specifically
|
||||
continue;
|
||||
suffix = unit; // set suffix
|
||||
bfr.Add_int_variable(cur); // write cur; note that this is not zero-padded
|
||||
}
|
||||
else { // str exists
|
||||
bfr.Add_int_pad_bgn(Byte_ascii.Num_0, 2, cur); // write cur; note that this zero-padded; also, note will write "00" if cur == 0
|
||||
}
|
||||
if (++count == max_places) break; // stop if max-places reached; EX: 86400 should write 1:00, not 1:00:00:00
|
||||
if (i != 3) // do not add ":" if seconds
|
||||
bfr.Add_byte_colon();
|
||||
}
|
||||
if (show_unit) // add units; EX: " s" for seconds
|
||||
bfr.Add_byte_space().Add_byte(suffix);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,23 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
|
||||
import org.junit.*; import gplx.core.tests.*;
|
||||
public class Time_dhms__tst {
|
||||
private final Time_dhms__fxt fxt = new Time_dhms__fxt();
|
||||
@Test public void S__0() {fxt.Test__to_str( 0, 4, "0 s");}
|
||||
@Test public void S__1() {fxt.Test__to_str( 1, 4, "1 s");}
|
||||
@Test public void S__2() {fxt.Test__to_str( 30, 4, "30 s");}
|
||||
@Test public void M__1() {fxt.Test__to_str( 60, 4, "1:00 m");}
|
||||
@Test public void M__2() {fxt.Test__to_str( 600, 4, "10:00 m");}
|
||||
@Test public void H__1() {fxt.Test__to_str( 3600, 4, "1:00:00 h");}
|
||||
@Test public void H__2() {fxt.Test__to_str( 5025, 4, "1:23:45 h");}
|
||||
@Test public void D__1() {fxt.Test__to_str( 86400, 4, "1:00:00:00 d");}
|
||||
@Test public void Max_places() {fxt.Test__to_str( 86400, 2, "1:00 d");}
|
||||
}
|
||||
class Time_dhms__fxt {
|
||||
private final Bry_bfr bfr = Bry_bfr_.New();
|
||||
public Time_dhms__fxt Test__to_str(long v, int max_places, String expd) {
|
||||
Gftest.Eq__str(expd, Time_dhms_.To_str(bfr, v, Bool_.Y, max_places));
|
||||
return this;}
|
||||
}
|
||||
@@ -13,17 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Export_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr {
|
||||
public Xob_cmd[] Bldr_cmds() {
|
||||
return new Xob_cmd[]
|
||||
{ gplx.xowa.addons.bldrs.exports.splits.Split_bldr_cmd.Prototype
|
||||
, gplx.xowa.addons.bldrs.exports.merges.Merge_bldr_cmd.Prototype
|
||||
, gplx.xowa.addons.bldrs.exports.packs.splits.Pack_split_bldr_cmd.Prototype
|
||||
, gplx.xowa.addons.bldrs.exports.packs.files.Pack_file_bldr_cmd.Prototype
|
||||
};
|
||||
}
|
||||
|
||||
public String Addon__key() {return "xowa.builds.repacks";}
|
||||
}
|
||||
|
||||
@@ -13,19 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
public class Xow_db_file_mgr {
|
||||
private final Ordered_hash hash = Ordered_hash_.New();
|
||||
public int Len() {return hash.Len();}
|
||||
public Xow_db_file Get_at(int i) {return (Xow_db_file)hash.Get_at(i);}
|
||||
public Xow_db_file_mgr Load_by_type(Xow_db_mgr db_mgr, int type) {
|
||||
int len = db_mgr.Dbs__len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xow_db_file db_file = db_mgr.Dbs__get_at(i);
|
||||
if (db_file.Tid() != type) continue;
|
||||
hash.Add(db_file.Url().Raw(), db_file);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,98 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*;
|
||||
public class Merge2_copy_utl {
|
||||
private final Bry_bfr bfr = Bry_bfr_.New();
|
||||
private final String tbl_name, fld_pkey, fld_blob;
|
||||
private final boolean tbl_has_blob;
|
||||
private Db_conn conn;
|
||||
public Merge2_copy_utl(String tbl_name, String fld_pkey, String fld_blob, boolean tbl_has_blob) {
|
||||
this.tbl_name = tbl_name;
|
||||
this.fld_pkey = fld_pkey;
|
||||
this.fld_blob = fld_blob;
|
||||
this.tbl_has_blob = tbl_has_blob;
|
||||
}
|
||||
public void Init_conn(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
}
|
||||
public byte[] Select_blob(Db_rdr data_rdr, String fld_blob_pkey) { // NOTE: need to SELECT entire BLOB; cannot UPDATE in 1 MB increments b/c SQLite does not really concat BLOBS; REF.SQLITE: http://sqlite.1065341.n5.nabble.com/Append-data-to-a-BLOB-field-td46003.html
|
||||
int id = data_rdr.Read_int(fld_blob_pkey);
|
||||
int len_max = data_rdr.Read_int("blob_len");
|
||||
int len_cur = 1, len_gap = 1000000;
|
||||
while (len_cur < len_max) {
|
||||
// determine substr args; EX: Substr(blob_col, 1, 1000000); Substr(blob_col, 1000001, 1000000); Substr(blob_col, 2000001, 1234);
|
||||
int len_new = len_cur + len_gap;
|
||||
if (len_new >= len_max) len_gap = len_max - len_cur; // last SELECT; Substr remainder, not full 1 MB
|
||||
|
||||
// read data; note that stmt needs to be new'd for each loop b/c SQL is different
|
||||
Db_stmt stmt = conn.Stmt_sql(String_.Format(String_.Concat_lines_nl_skip_last // ANSI.N; NOTE: will only run on Android SQLite
|
||||
( "SELECT Substr({0}, {1}, {2}) AS blob_data"
|
||||
, "FROM {3}"
|
||||
, "WHERE {4} = {5}"
|
||||
), fld_blob, len_cur, len_gap, tbl_name, fld_pkey, id));
|
||||
Db_rdr rdr = stmt.Exec_select__rls_auto();
|
||||
try {
|
||||
if (rdr.Move_next())
|
||||
bfr.Add(rdr.Read_bry("blob_data"));
|
||||
else
|
||||
throw Err_.new_wo_type("failed to read blob in increments", "id", id, "len_cur", len_cur);
|
||||
}
|
||||
finally {
|
||||
rdr.Rls();
|
||||
stmt.Rls();
|
||||
}
|
||||
|
||||
len_cur = len_new;
|
||||
}
|
||||
return bfr.To_bry_and_clear();
|
||||
}
|
||||
public String Bld_sql(Dbmeta_fld_list flds, int flds_end, boolean src_is_pack, byte mode, int resume__db_id) {
|
||||
bfr.Add_str_a7("SELECT");
|
||||
for (int i = 0; i < flds_end; ++i) {
|
||||
bfr.Add_str_a7(i == 0 ? " " : ", ");
|
||||
bfr.Add_str_u8(flds.Get_at(i).Name());
|
||||
}
|
||||
bfr.Add_str_u8(Bld_select_fld(mode));
|
||||
bfr.Add_str_a7("\nFROM ").Add_str_u8(tbl_name);
|
||||
if (!src_is_pack) {
|
||||
}
|
||||
else {
|
||||
bfr.Add_str_a7("\nWHERE ");
|
||||
if (src_is_pack) {
|
||||
bfr.Add_str_a7("trg_db_id >= ");
|
||||
bfr.Add_int_variable(resume__db_id);
|
||||
if (mode != Merge2_copy_utl.Mode__all)
|
||||
bfr.Add_str_a7("\nAND ");
|
||||
}
|
||||
bfr.Add_str_a7(Bld_where(mode));
|
||||
}
|
||||
if (!src_is_pack) {
|
||||
bfr.Add_str_a7("\nORDER BY ");
|
||||
bfr.Add_str_u8(fld_pkey);
|
||||
}
|
||||
else {
|
||||
bfr.Add_str_a7("\nORDER BY trg_db_id, ");
|
||||
bfr.Add_str_u8(fld_pkey);
|
||||
}
|
||||
return bfr.To_str_and_clear();
|
||||
}
|
||||
public String Bld_select_fld(byte mode) {
|
||||
switch (mode) {
|
||||
case Merge2_copy_utl.Mode__all:
|
||||
case Merge2_copy_utl.Mode__drd__small: return tbl_has_blob ? ", " + fld_blob : "";
|
||||
case Merge2_copy_utl.Mode__drd__large: return ", blob_len";
|
||||
default: throw Err_.new_unhandled_default(mode);
|
||||
}
|
||||
}
|
||||
public String Bld_where(byte mode) {
|
||||
switch (mode) {
|
||||
case Merge2_copy_utl.Mode__all: return "";
|
||||
case Merge2_copy_utl.Mode__drd__small: return "blob_len <= 1000000";
|
||||
case Merge2_copy_utl.Mode__drd__large: return "blob_len > 1000000";
|
||||
default: throw Err_.new_unhandled_default(mode);
|
||||
}
|
||||
}
|
||||
public static final byte Mode__all = 0, Mode__drd__small = 1, Mode__drd__large = 2;
|
||||
}
|
||||
|
||||
@@ -13,131 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.xowa.addons.bldrs.exports.utls.*;
|
||||
class Merge2_copy_wkr__lot {
|
||||
private final Split_tbl tbl;
|
||||
private final String tbl_name, fld_blob_pkey, fld_blob_data;
|
||||
private final Merge2_copy_utl copy_utl;
|
||||
private int resume__db_id;
|
||||
private boolean tbl_has_blob;
|
||||
public Merge2_copy_wkr__lot(Split_tbl tbl) {
|
||||
this.tbl = tbl;
|
||||
this.tbl_name = tbl.Tbl_name();
|
||||
this.fld_blob_pkey = tbl.Fld_pkeys()[0];
|
||||
this.fld_blob_data = tbl.Fld_blob();
|
||||
this.tbl_has_blob = Split_tbl_.Tbl_has_blob(tbl);
|
||||
this.copy_utl = new Merge2_copy_utl(tbl.Tbl_name(), fld_blob_pkey, fld_blob_data, tbl_has_blob);
|
||||
}
|
||||
public void Copy_by_sql(String msg, Merge_ctx ctx, Merge_prog_wkr prog_wkr, Db_conn src_conn, Merge2_trg_itm trg_db, Merge2_trg_mgr trg_mgr, boolean src_is_pack) {
|
||||
Xow_wiki wiki = ctx.Wiki();
|
||||
if (src_is_pack) {
|
||||
int[] trg_db_idxs = Get_trg_dbs(src_conn, tbl_name);
|
||||
int trg_db_idxs_len = trg_db_idxs.length;
|
||||
for (int i = 0; i < trg_db_idxs_len; ++i) {
|
||||
int trg_db_id = trg_db_idxs[i];
|
||||
Merge2_trg_itm trg_db_new = trg_mgr.Cur_or_new(ctx, prog_wkr, wiki, trg_db_id);
|
||||
Copy_by_trg_conn(msg, prog_wkr, src_conn, trg_db_new, trg_db_id, src_is_pack);
|
||||
}
|
||||
}
|
||||
else {
|
||||
Copy_by_trg_conn(msg, prog_wkr, src_conn, trg_db, trg_db.Idx(), src_is_pack);
|
||||
}
|
||||
}
|
||||
private void Copy_by_trg_conn(String msg, Merge_prog_wkr prog_wkr, Db_conn src_conn, Merge2_trg_itm trg_db, int trg_db_id, boolean src_is_pack) {
|
||||
Dbmeta_fld_list src_flds = tbl.Flds().Clone(); // src_flds.Insert(0, Dbmeta_fld_itm.new_int("trg_db_id"));
|
||||
Merge_wkr_utl.Merge_by_sql(prog_wkr, msg, tbl.Tbl_name(), src_flds, src_conn, trg_db, trg_db_id, !src_is_pack);
|
||||
}
|
||||
private int[] Get_trg_dbs(Db_conn conn, String tbl_name) {
|
||||
List_adp list = List_adp_.New();
|
||||
Db_rdr rdr = conn.Stmt_sql(String_.Format("SELECT trg_db_id FROM {0} GROUP BY trg_db_id", tbl_name)).Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
list.Add(rdr.Read_int("trg_db_id"));
|
||||
}
|
||||
}
|
||||
finally {
|
||||
rdr.Rls();
|
||||
}
|
||||
return (int[])list.To_ary_and_clear(int.class);
|
||||
}
|
||||
public void Copy_entire_src(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Db_conn src_conn, Merge2_trg_itm trg_db, Merge2_trg_mgr trg_mgr, boolean src_is_pack) {
|
||||
Xow_wiki wiki = ctx.Wiki();
|
||||
|
||||
if (gplx.core.envs.Op_sys.Cur().Tid_is_drd() && tbl_has_blob) {
|
||||
Copy_rows(ctx, prog_wkr, wiki, tbl, src_conn, trg_db, trg_mgr, src_is_pack, Merge2_copy_utl.Mode__drd__small);
|
||||
Copy_rows(ctx, prog_wkr, wiki, tbl, src_conn, trg_db, trg_mgr, src_is_pack, Merge2_copy_utl.Mode__drd__large);
|
||||
}
|
||||
else
|
||||
Copy_rows(ctx, prog_wkr, wiki, tbl, src_conn, trg_db, trg_mgr, src_is_pack, Merge2_copy_utl.Mode__all);
|
||||
}
|
||||
private void Copy_rows(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Xow_wiki wiki, Split_tbl tbl, Db_conn src_conn, Merge2_trg_itm trg_db, Merge2_trg_mgr trg_mgr, boolean src_is_pack, byte select_mode) {
|
||||
// init src
|
||||
Dbmeta_fld_list src_flds = tbl.Flds().Clone();
|
||||
if (src_is_pack)
|
||||
src_flds.Insert(0, Dbmeta_fld_itm.new_int("trg_db_id"));
|
||||
int flds_nth = src_flds.Len();
|
||||
if (tbl_has_blob) {
|
||||
Split_tbl_.Flds__add_blob_len(src_flds);
|
||||
flds_nth = src_flds.Len() - 1; // ignore blob which is last fld by convention
|
||||
}
|
||||
String src_sql = copy_utl.Bld_sql(src_flds, flds_nth, src_is_pack, select_mode, resume__db_id);
|
||||
Db_rdr src_rdr = src_conn.Stmt_sql(src_sql).Exec_select__rls_auto();
|
||||
copy_utl.Init_conn(src_conn);
|
||||
|
||||
Dbmeta_fld_list trg_flds = tbl.Flds().Clone();
|
||||
if (tbl_has_blob && src_is_pack) {
|
||||
Split_tbl_.Flds__add_blob_len(trg_flds);
|
||||
}
|
||||
|
||||
Db_stmt trg_stmt = null; Db_conn trg_conn = null; int cur_trg = -1;
|
||||
if (trg_db != null) {
|
||||
trg_conn = trg_db.Conn();
|
||||
trg_conn.Txn_bgn("merge_" + tbl_name);
|
||||
trg_stmt = trg_conn.Stmt_insert(tbl_name, trg_flds);
|
||||
cur_trg = trg_db.Idx();
|
||||
}
|
||||
try {
|
||||
while (src_rdr.Move_next()) {
|
||||
// switch trg_db if null, or diff than cur; note that trg_stmt also changes
|
||||
if (src_is_pack) {
|
||||
int trg_db_id = src_rdr.Read_int("trg_db_id");
|
||||
if (cur_trg != trg_db_id) {
|
||||
if (trg_conn != null) trg_conn.Txn_end();
|
||||
trg_conn = trg_mgr.Cur_or_new(ctx, prog_wkr, wiki, trg_db_id).Conn();
|
||||
trg_conn.Txn_bgn("merge_" + tbl_name);
|
||||
trg_stmt = trg_conn.Stmt_insert(tbl_name, trg_flds);
|
||||
cur_trg = trg_db_id;
|
||||
resume__db_id = trg_db_id;
|
||||
prog_wkr.Checkpoint__save();
|
||||
}
|
||||
}
|
||||
|
||||
// read and insert
|
||||
trg_stmt.Clear();
|
||||
gplx.dbs.diffs.Gfdb_rdr_utl_.Stmt_args(trg_stmt, trg_flds, 0, trg_flds.Len() - (tbl_has_blob ? 1 : 0), src_rdr);
|
||||
if (tbl_has_blob) {
|
||||
byte[] val_blob = select_mode == Merge2_copy_utl.Mode__drd__large
|
||||
? copy_utl.Select_blob(src_rdr, fld_blob_pkey)
|
||||
: src_rdr.Read_bry(fld_blob_data);
|
||||
trg_stmt.Val_bry(fld_blob_data , val_blob);
|
||||
}
|
||||
trg_stmt.Exec_insert();
|
||||
|
||||
if (prog_wkr.Prog__insert_and_stop_if_suspended(0)) break;
|
||||
}
|
||||
}
|
||||
finally {
|
||||
src_rdr.Rls();
|
||||
if (trg_conn != null) {
|
||||
trg_conn.Txn_end();
|
||||
}
|
||||
trg_stmt = Db_stmt_.Rls(trg_stmt);
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
2 situations:
|
||||
- pack (trg_db,blob_len,*) -> heap (blob_len,*)
|
||||
- heap (blob_len,*) -> wiki (*)
|
||||
*/
|
||||
|
||||
@@ -13,16 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.xowa.addons.bldrs.exports.utls.*;
|
||||
public class Merge2_heap_db implements Merge2_trg_itm {
|
||||
public Merge2_heap_db(Split_tbl tbl, Dbmeta_fld_list flds, int idx, Io_url url, Db_conn conn) {
|
||||
this.tbl = tbl; this.flds = flds; this.idx = idx;
|
||||
this.url = url; this.conn = conn;
|
||||
}
|
||||
public Split_tbl Tbl() {return tbl;} private final Split_tbl tbl;
|
||||
public Dbmeta_fld_list Flds() {return flds;} private final Dbmeta_fld_list flds;
|
||||
public int Idx() {return idx;} private final int idx;
|
||||
public Io_url Url() {return url;} private final Io_url url;
|
||||
public Db_conn Conn() {return conn;} private final Db_conn conn;
|
||||
}
|
||||
|
||||
@@ -13,99 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.xowa.addons.bldrs.exports.utls.*;
|
||||
public class Merge2_heap_mgr {// data tbls will only have 1 itms; blob tbls will have N itms
|
||||
private final Split_tbl tbl;
|
||||
private final List_adp list = List_adp_.New(), deleted = List_adp_.New();
|
||||
private Dbmeta_fld_list heap_flds;
|
||||
public Merge2_heap_mgr(Split_tbl tbl) {this.tbl = tbl;}
|
||||
public Merge2_heap_db Cur() {return cur;} private Merge2_heap_db cur;
|
||||
public Merge2_heap_db Make_itm(Xow_wiki wiki, int trg_db_id) {
|
||||
// clone tbl_flds; disable primary key
|
||||
this.heap_flds = tbl.Flds().Clone();
|
||||
int len = heap_flds.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Dbmeta_fld_itm itm = heap_flds.Get_at(i);
|
||||
if (itm.Primary()) itm.Primary_n_();
|
||||
}
|
||||
|
||||
// if blob, add "blob_len" in penultimate pos; note that last fld is "blob_fld"
|
||||
// if (Split_tbl_.Tbl_has_blob(tbl))
|
||||
// Split_tbl_.Flds__add_blob_len(heap_flds);
|
||||
|
||||
// init heap_conn
|
||||
String tbl_name = tbl.Tbl_name();
|
||||
Io_url heap_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "merge", tbl_name);
|
||||
Io_url heap_url = heap_dir.GenSubFil(String_.Format("xowa.merge.temp.{0}{1}.sqlite3", tbl_name, trg_db_id == -1 ? "" : "." + Int_.To_str_pad_bgn_zero(trg_db_id, 4)));
|
||||
Db_conn_bldr_data bldr_data = Db_conn_bldr.Instance.Get_or_new(heap_url);
|
||||
Db_conn heap_conn = bldr_data.Conn();
|
||||
if (bldr_data.Created())
|
||||
heap_conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, heap_flds));
|
||||
|
||||
// make itm and return it
|
||||
Merge2_heap_db rv = new Merge2_heap_db(tbl, heap_flds, trg_db_id, heap_url, heap_conn);
|
||||
this.Add(rv);
|
||||
return rv;
|
||||
}
|
||||
public void Copy_to_wiki(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Merge2_wkr__heap_base wkr) {
|
||||
// do copy for all itms; only 1 itm for page, fsdb_fil, etc..; but many for html, fsdb_bin
|
||||
int len = list.Len();
|
||||
for (int i = 0; i < len; ++i)
|
||||
Copy_to_wiki__itm(ctx, prog_wkr, wkr, this.Get_at(i), i == len - 1);
|
||||
|
||||
// delete any old dbs
|
||||
len = deleted.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Merge2_heap_db itm = (Merge2_heap_db)deleted.Get_at(i);
|
||||
list.Del(itm);
|
||||
}
|
||||
deleted.Clear();
|
||||
|
||||
// if last idx, cleanup
|
||||
if (ctx.Heap__last_idx()) this.Cleanup();
|
||||
}
|
||||
public void Cleanup() {
|
||||
int len = list.Len();
|
||||
for (int i = 0; i < len; ++i)
|
||||
Cleanup_file(this.Get_at(i));
|
||||
}
|
||||
private Merge2_heap_db Get_at(int i) {return (Merge2_heap_db)list.Get_at(i);}
|
||||
private void Add(Merge2_heap_db itm) {
|
||||
list.Add(itm);
|
||||
cur = itm;
|
||||
}
|
||||
private void Copy_to_wiki__itm(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Merge2_wkr__heap_base wkr, Merge2_heap_db itm, boolean itm_is_cur) {
|
||||
// init
|
||||
Split_tbl tbl = wkr.Tbl();
|
||||
String tbl_name = tbl.Tbl_name();
|
||||
Db_conn heap_conn = itm.Conn();
|
||||
Dbmeta_fld_list tbl_flds = tbl.Flds();
|
||||
Db_conn wiki_conn = tbl.Wiki_conn__get_or_new(ctx.Wiki(), itm.Idx());
|
||||
|
||||
// index heap table
|
||||
// String[] pkey_flds = tbl.Fld_pkeys();
|
||||
// heap_conn.Meta_idx_create(tbl_name, "sort", pkey_flds);
|
||||
|
||||
// do bulk copy into wiki tbls; note ORDER BY
|
||||
wkr.Copy_to_wiki(ctx, prog_wkr, tbl_name, tbl_flds, heap_conn, new Merge2_trg_itm__wiki(itm.Idx(), wiki_conn), String_.Ary_empty);
|
||||
|
||||
// cleanup
|
||||
if (itm_is_cur) { // if cur, rebuild heap table; vaccum;
|
||||
// heap_conn.Meta_tbl_remake(Dbmeta_tbl_itm.New(tbl_name, heap_flds));
|
||||
// heap_conn.Env_vacuum();
|
||||
heap_conn.Rls_conn();
|
||||
Io_mgr.Instance.DeleteFil(itm.Url()); // SQLite: file delete is faster than DROP TABLE or DELETE FROM
|
||||
heap_conn.Reopen_conn();
|
||||
heap_conn.Meta_tbl_remake(Dbmeta_tbl_itm.New(tbl_name, heap_flds));
|
||||
}
|
||||
else { // else, delete file; EX: fsdb_bin has 2 heap files; delete 1st, but still keep 2nd
|
||||
Cleanup_file(itm);
|
||||
deleted.Add(itm);
|
||||
}
|
||||
}
|
||||
private void Cleanup_file(Merge2_heap_db itm) {
|
||||
itm.Conn().Rls_conn();
|
||||
Io_mgr.Instance.DeleteFil(itm.Url());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,57 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.xowa.addons.bldrs.exports.utls.*;
|
||||
public class Merge2_mgr {
|
||||
public final Merge_ctx ctx;
|
||||
private final Ordered_hash wkr_hash;
|
||||
|
||||
public Merge2_mgr() {
|
||||
this.ctx = new Merge_ctx();
|
||||
this.wkr_hash = Make_wkrs
|
||||
( new Merge2_wkr__heap_one(Split_tbl_.Page)
|
||||
, new Merge2_wkr__heap_lot(Split_tbl_.Html)
|
||||
, new Merge2_wkr__heap_one(Split_tbl_.Srch_word)
|
||||
, new Merge2_wkr__heap_lot(Split_tbl_.Srch_link)
|
||||
, new Merge2_wkr__heap_one(Split_tbl_.Fsdb_fil)
|
||||
, new Merge2_wkr__heap_one(Split_tbl_.Fsdb_thm)
|
||||
, new Merge2_wkr__heap_one(Split_tbl_.Fsdb_org)
|
||||
, new Merge2_wkr__heap_lot(Split_tbl_.Fsdb_bin)
|
||||
);
|
||||
}
|
||||
public void Prog_wkr_(Merge_prog_wkr prog_wkr) {this.prog_wkr = prog_wkr;} private Merge_prog_wkr prog_wkr;
|
||||
public void Merge_core(Xow_wiki wiki, Io_url src_url) {
|
||||
Db_conn src_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, src_url);
|
||||
new Merge_wkr__core().Copy_to_temp(null, wiki, src_conn);
|
||||
Gfo_invk_.Invk_by_val(wiki.App().Wiki_mgri(), gplx.xowa.wikis.Xoa_wiki_mgr_.Invk__import_by_url, src_url);
|
||||
src_conn.Rls_conn(); // NOTE: must close conn else pack_conn will stay open
|
||||
// fails b/c no Main_Page; Gfo_invk_.Invk_by_msg(wiki.App().Gui__tab_mgr(), gplx.xowa.guis.tabs.Xog_tab_mgr_.Invk__new_tab, GfoMsg_.new_cast_("").Add("focus", true).Add("site", wiki.Domain_str()).Add("page", String_.new_u8(wiki.Props().Main_page())));
|
||||
}
|
||||
public void Merge_data(Xow_wiki wiki, Io_url src_url, int idx_cur) {
|
||||
long all_time_bgn = gplx.core.envs.System_.Ticks();
|
||||
wiki.Init_by_wiki();
|
||||
Db_conn src_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, src_url);
|
||||
ctx.Init(wiki, src_conn);
|
||||
|
||||
// merge data
|
||||
int hash_len = wkr_hash.Len();
|
||||
for (int i = 0; i < hash_len; ++i) {
|
||||
if (prog_wkr.Canceled()) break;
|
||||
Merge2_wkr wkr = (Merge2_wkr)wkr_hash.Get_at(i);
|
||||
// if (prog_wkr.Checkpoint__skip_wkr(src_url, wkr.Tbl_name())) continue;
|
||||
long wkr_time_bgn = gplx.core.envs.System_.Ticks();
|
||||
wkr.Merge_data(ctx, prog_wkr);
|
||||
Gfo_log_.Instance.Info("merge.wkr.done", "data", src_url.NameAndExt() + "|" + wkr.Tbl().Tbl_name() + "|" + gplx.core.envs.System_.Ticks__elapsed_in_frac(wkr_time_bgn));
|
||||
}
|
||||
if (ctx.Heap__copy_to_wiki()) ctx.Heap__increment_nxt();
|
||||
Gfo_log_.Instance.Info("merge.wkr.done", "data", src_url.NameAndExt() + "|-1|" + gplx.core.envs.System_.Ticks__elapsed_in_frac(all_time_bgn));
|
||||
src_conn.Rls_conn(); // NOTE: must close conn else pack_conn will stay open
|
||||
}
|
||||
private static Ordered_hash Make_wkrs(Merge2_wkr... wkrs) {
|
||||
Ordered_hash rv = Ordered_hash_.New();
|
||||
for (Merge2_wkr wkr : wkrs)
|
||||
rv.Add(wkr.Tbl().Tbl_name(), wkr);
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,14 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*;
|
||||
public interface Merge2_trg_itm {
|
||||
int Idx();
|
||||
Db_conn Conn();
|
||||
}
|
||||
class Merge2_trg_itm__wiki implements Merge2_trg_itm {
|
||||
public Merge2_trg_itm__wiki(int idx, Db_conn conn) {this.idx = idx; this.conn = conn;}
|
||||
public int Idx() {return idx;} private final int idx;
|
||||
public Db_conn Conn() {return conn;} private final Db_conn conn;
|
||||
}
|
||||
|
||||
@@ -13,33 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.xowa.addons.bldrs.exports.utls.*;
|
||||
interface Merge2_trg_mgr {
|
||||
Merge2_trg_itm Cur();
|
||||
Merge2_trg_itm Cur_or_new(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Xow_wiki wiki, int trg_db_id);
|
||||
}
|
||||
class Merge2_trg_mgr__heap implements Merge2_trg_mgr {
|
||||
private final Merge2_heap_mgr heap_mgr;
|
||||
public Merge2_trg_mgr__heap(Merge2_heap_mgr heap_mgr) {this.heap_mgr = heap_mgr;}
|
||||
public Merge2_trg_itm Cur() {return heap_mgr.Cur();}
|
||||
public Merge2_trg_itm Cur_or_new(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Xow_wiki wiki, int trg_db_id) {
|
||||
Merge2_heap_db cur = heap_mgr.Cur();
|
||||
if (cur == null || cur.Idx() != trg_db_id) {
|
||||
cur = heap_mgr.Make_itm(ctx.Wiki(), trg_db_id);
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
}
|
||||
class Merge2_trg_mgr__wiki implements Merge2_trg_mgr {
|
||||
private final Split_tbl tbl;
|
||||
public Merge2_trg_mgr__wiki(Split_tbl tbl) {this.tbl = tbl;}
|
||||
public Merge2_trg_itm Cur() {return cur;} private Merge2_trg_itm__wiki cur;
|
||||
public Merge2_trg_itm Cur_or_new(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Xow_wiki wiki, int trg_db_id) {
|
||||
if (cur == null || cur.Idx() != trg_db_id) {
|
||||
Db_conn conn = tbl.Wiki_conn__get_or_new(wiki, trg_db_id);
|
||||
cur = new Merge2_trg_itm__wiki(trg_db_id, conn);
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,9 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.xowa.addons.bldrs.exports.utls.*;
|
||||
interface Merge2_wkr {
|
||||
Split_tbl Tbl();
|
||||
void Merge_data(Merge_ctx ctx, Merge_prog_wkr prog_wkr);
|
||||
}
|
||||
|
||||
@@ -13,20 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.xowa.addons.bldrs.exports.utls.*;
|
||||
public abstract class Merge2_wkr__heap_base implements Merge2_wkr {
|
||||
public Merge2_wkr__heap_base(Split_tbl tbl) {
|
||||
this.tbl = tbl;
|
||||
this.heap_mgr = new Merge2_heap_mgr(tbl);
|
||||
}
|
||||
public Split_tbl Tbl() {return tbl;} private final Split_tbl tbl;
|
||||
protected Merge2_heap_mgr Heap_mgr() {return heap_mgr;} private final Merge2_heap_mgr heap_mgr;
|
||||
public void Merge_data(Merge_ctx ctx, Merge_prog_wkr prog_wkr) { // fires once per file
|
||||
this.Copy_to_heap(ctx, prog_wkr, heap_mgr, tbl);
|
||||
this.Copy_to_wiki(ctx, prog_wkr, heap_mgr, tbl);
|
||||
}
|
||||
protected abstract void Copy_to_heap(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Merge2_heap_mgr heap_mgr, Split_tbl tbl);
|
||||
protected abstract void Copy_to_wiki(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Merge2_heap_mgr heap_mgr, Split_tbl tbl);
|
||||
public abstract void Copy_to_wiki(Merge_ctx ctx, Merge_prog_wkr prog_wkr, String tbl_name, Dbmeta_fld_list src_flds, Db_conn src_conn, Merge2_trg_itm trg_db, String[] fld_pkeys);
|
||||
}
|
||||
|
||||
@@ -13,25 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.xowa.addons.bldrs.exports.utls.*;
|
||||
public class Merge2_wkr__heap_lot extends Merge2_wkr__heap_base {
|
||||
private final Merge2_copy_wkr__lot copy_wkr;
|
||||
private final Merge2_trg_mgr trg_mgr__heap;
|
||||
public Merge2_wkr__heap_lot(Split_tbl tbl) {super(tbl);
|
||||
this.copy_wkr = new Merge2_copy_wkr__lot(tbl);
|
||||
this.trg_mgr__heap = new Merge2_trg_mgr__heap(this.Heap_mgr());
|
||||
}
|
||||
@Override protected void Copy_to_heap(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Merge2_heap_mgr heap_mgr, Split_tbl tbl) {
|
||||
copy_wkr.Copy_by_sql("merging " + tbl.Tbl_name(), ctx, prog_wkr, ctx.Pack_conn(), null, trg_mgr__heap, Bool_.Y);
|
||||
// copy_wkr.Copy_entire_src(ctx, prog_wkr, ctx.Pack_conn(), null, trg_mgr__heap, Bool_.Y);
|
||||
}
|
||||
@Override protected void Copy_to_wiki(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Merge2_heap_mgr heap_mgr, Split_tbl tbl) {
|
||||
if (ctx.Heap__copy_to_wiki())
|
||||
heap_mgr.Copy_to_wiki(ctx, prog_wkr, this);
|
||||
}
|
||||
@Override public void Copy_to_wiki(Merge_ctx ctx, Merge_prog_wkr prog_wkr, String tbl_name, Dbmeta_fld_list trg_flds, Db_conn src_conn, Merge2_trg_itm trg_db, String[] fld_pkeys) {
|
||||
copy_wkr.Copy_by_sql("merging " + tbl_name, ctx, prog_wkr, src_conn, trg_db, trg_mgr__heap, Bool_.N);
|
||||
// copy_wkr.Copy_entire_src(ctx, prog_wkr, src_conn, trg_db, trg_mgr__heap, Bool_.N);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,25 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.xowa.addons.bldrs.exports.utls.*;
|
||||
public class Merge2_wkr__heap_one extends Merge2_wkr__heap_base {
|
||||
public Merge2_wkr__heap_one(Split_tbl tbl) {super(tbl);}
|
||||
@Override protected void Copy_to_heap(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Merge2_heap_mgr heap_mgr, Split_tbl tbl) {
|
||||
// init
|
||||
String tbl_name = tbl.Tbl_name();
|
||||
Dbmeta_fld_list flds = tbl.Flds();
|
||||
Merge2_heap_db heap_db = heap_mgr.Cur();
|
||||
if (heap_db == null) heap_db = heap_mgr.Make_itm(ctx.Wiki(), -1);
|
||||
|
||||
// copy
|
||||
Merge_wkr_utl.Merge_by_sql(prog_wkr, "merging " + tbl_name, tbl_name, flds, ctx.Pack_conn(), heap_db, -1, Bool_.N);
|
||||
}
|
||||
@Override protected void Copy_to_wiki(Merge_ctx ctx, Merge_prog_wkr prog_wkr, Merge2_heap_mgr heap_mgr, Split_tbl tbl) {
|
||||
if (ctx.Heap__copy_to_wiki())
|
||||
heap_mgr.Copy_to_wiki(ctx, prog_wkr, this);
|
||||
}
|
||||
@Override public void Copy_to_wiki(Merge_ctx ctx, Merge_prog_wkr prog_wkr, String tbl_name, Dbmeta_fld_list trg_flds, Db_conn src_conn, Merge2_trg_itm trg_db, String[] fld_pkeys) {
|
||||
Merge_wkr_utl.Merge_by_sql(prog_wkr, "merging " + tbl_name, tbl_name, trg_flds, src_conn, trg_db, -1, Bool_.Y);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,38 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
import gplx.xowa.addons.bldrs.exports.splits.mgrs.*; import gplx.xowa.addons.bldrs.exports.splits.rslts.*;
|
||||
public class Merge_bldr_cmd extends Xob_cmd__base {
|
||||
public Merge_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
|
||||
@Override public void Cmd_run() {
|
||||
Io_mgr.Instance.DeleteDirDeep(wiki.Fsys_mgr().Root_dir());
|
||||
wiki.Init_assert();
|
||||
|
||||
Merge2_mgr merge_mgr = new Merge2_mgr();
|
||||
Io_url[] urls = Io_mgr.Instance.QueryDir_fils(Io_url_.new_fil_("C:\\xowa\\wiki\\simple.wikipedia.org\\tmp\\split\\"));
|
||||
int i = 0;
|
||||
for (Io_url url : urls) {
|
||||
// if (++i == 5) break;
|
||||
switch (Split_file_tid_.To_tid(url)) {
|
||||
case Split_file_tid_.Tid__core: merge_mgr.Merge_core(wiki, url); break;
|
||||
case Split_file_tid_.Tid__data:
|
||||
Db_conn conn = Db_conn_bldr.Instance.Get_or_noop(url);
|
||||
Wkr_stats_tbl stats_tbl = new Wkr_stats_tbl(conn);
|
||||
Wkr_stats_itm summary_itm = stats_tbl.Select_all__summary();
|
||||
Merge_prog_wkr prog_wkr = new Merge_prog_wkr(gplx.core.progs.Gfo_prog_ui_.Always, url.GenNewNameAndExt("merge.checkpoint"), summary_itm.Count, summary_itm.Size);
|
||||
merge_mgr.Prog_wkr_(prog_wkr);
|
||||
merge_mgr.Merge_data(wiki, url, i); break;
|
||||
default: continue;
|
||||
}
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
public static final String BLDR_CMD_KEY = "bldr.export.merge";
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
|
||||
public static final Xob_cmd Prototype = new Merge_bldr_cmd(null, null);
|
||||
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Merge_bldr_cmd(bldr, wiki);}
|
||||
}
|
||||
|
||||
@@ -13,20 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*;
|
||||
public class Merge_ctx {
|
||||
public void Init(Xow_wiki wiki, Db_conn pack_conn) {
|
||||
this.wiki = wiki;
|
||||
this.pack_conn = pack_conn;
|
||||
}
|
||||
public Xow_wiki Wiki() {return wiki;} private Xow_wiki wiki;
|
||||
public Db_conn Pack_conn() {return pack_conn;} private Db_conn pack_conn;
|
||||
public int Idx_cur() {return idx_cur;} private int idx_cur; public void Idx_cur_add_() {++idx_cur;}
|
||||
public int Idx_end = 70;
|
||||
public int Idx_gap = 10;
|
||||
public int Idx_nxt = 10;
|
||||
public boolean Heap__copy_to_wiki() {return idx_cur == Idx_nxt || idx_cur == Idx_end;}
|
||||
public void Heap__increment_nxt() {Idx_nxt += Idx_gap;}
|
||||
public boolean Heap__last_idx() {return idx_cur == Idx_end;}
|
||||
}
|
||||
|
||||
@@ -13,47 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
public class Merge_prog_checkpoint {
|
||||
private final Bry_bfr bfr = Bry_bfr_.New();
|
||||
private final Io_url url;
|
||||
private String resume_fil;
|
||||
private int resume_wkr = -1;
|
||||
private int resume_db = -1;
|
||||
public Merge_prog_checkpoint(Io_url url) {this.url = url;}
|
||||
public int Resume_db() {return resume_db;}
|
||||
public boolean Skip_fil(Io_url fil) {
|
||||
if (resume_fil == null) return false; // not resume; do not skip
|
||||
boolean rv = String_.Eq(fil.NameAndExt(), resume_fil);
|
||||
if (rv) resume_fil = null; // null out for next call
|
||||
return !rv;
|
||||
}
|
||||
public boolean Skip_wkr(byte wkr_tid) {
|
||||
if (resume_wkr == -1) return false; // not resume; do not skip;
|
||||
boolean rv = wkr_tid == resume_wkr;
|
||||
if (rv) resume_wkr = -1; // null out for next call
|
||||
return !rv;
|
||||
}
|
||||
public int Load() {
|
||||
byte[] bry = Io_mgr.Instance.LoadFilBryOrNull(url); if (bry == null) return 0;
|
||||
byte[][] parts = Bry_split_.Split(bry, Byte_ascii.Pipe);
|
||||
this.resume_fil = String_.new_u8(parts[0]);
|
||||
this.resume_wkr = Bry_.To_int(parts[1]);
|
||||
this.resume_db = Bry_.To_int(parts[2]);
|
||||
return Bry_.To_int(parts[3]);
|
||||
}
|
||||
public void Save(Io_url fil, byte wkr_tid, int db_id, int prog_count) { // EX: file.xowa|0|4|1234
|
||||
if (fil == null) return;
|
||||
bfr.Add_str_u8(fil.NameAndExt()).Add_byte_pipe();
|
||||
bfr.Add_int_variable(wkr_tid).Add_byte_pipe();
|
||||
bfr.Add_int_variable(db_id).Add_byte_pipe();
|
||||
bfr.Add_int_variable(prog_count);
|
||||
Io_mgr.Instance.SaveFilBry(url, bfr.To_bry_and_clear());
|
||||
}
|
||||
public void Delete() {
|
||||
resume_fil = null;
|
||||
resume_wkr = -1;
|
||||
resume_db = -1;
|
||||
Io_mgr.Instance.DeleteFil(url);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,50 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.core.progs.*; import gplx.dbs.bulks.*;
|
||||
public class Merge_prog_wkr implements Db_bulk_prog {
|
||||
private final Gfo_prog_ui prog_ui;
|
||||
private final Merge_prog_checkpoint checkpoint;
|
||||
private final int prog_count_end;
|
||||
private int prog_count_cur;
|
||||
private long time_nxt, time_gap = 100;
|
||||
// private final int[] mergepoint_ary = new int[gplx.xowa.addons.bldrs.exports.splits.rslts.Split_rslt_tid_.Tid_max];
|
||||
public Merge_prog_wkr(Gfo_prog_ui prog_ui, Io_url checkpoint_fil, int prog_count_end, long prog_size_end) {
|
||||
this.prog_ui = prog_ui;
|
||||
this.checkpoint = new Merge_prog_checkpoint(checkpoint_fil);
|
||||
this.prog_count_end = prog_count_end;
|
||||
}
|
||||
public boolean Canceled() {return prog_ui.Canceled();}
|
||||
public long Checkpoint__load() {
|
||||
long rv = checkpoint.Load();
|
||||
this.prog_count_cur = (int)rv;
|
||||
return rv;
|
||||
}
|
||||
public int Checkpoint__resume_db() {return checkpoint.Resume_db();}
|
||||
public void Checkpoint__delete() {checkpoint.Delete();}
|
||||
public boolean Checkpoint__skip_fil(Io_url fil) {return checkpoint.Skip_fil(fil);}
|
||||
private Io_url cur_fil; private byte cur_wkr_tid;
|
||||
public boolean Checkpoint__skip_wkr(Io_url fil, byte wkr_tid) {
|
||||
boolean rv = checkpoint.Skip_wkr(wkr_tid);
|
||||
if (rv) return true;
|
||||
this.cur_fil = fil;
|
||||
this.cur_wkr_tid = wkr_tid;
|
||||
this.Checkpoint__save();
|
||||
time_nxt = gplx.core.envs.System_.Ticks() + time_gap;
|
||||
return false;
|
||||
}
|
||||
public void Checkpoint__save() {
|
||||
// cur_wkr.Resume__db_id()
|
||||
checkpoint.Save(cur_fil, cur_wkr_tid, -1, prog_count_cur);
|
||||
}
|
||||
public boolean Prog__insert_and_stop_if_suspended(int row_size) {
|
||||
++prog_count_cur;
|
||||
long time_cur = gplx.core.envs.System_.Ticks();
|
||||
if (time_cur < time_nxt) return false;
|
||||
// gplx.core.threads.Thread_adp_.Sleep(10);
|
||||
time_nxt = time_cur + time_gap;
|
||||
boolean rv = prog_ui.Prog_notify_and_chk_if_suspended(prog_count_cur, prog_count_end);
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,82 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.dbs.bulks.*;
|
||||
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.infos.*;
|
||||
import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.dbs.*;
|
||||
import gplx.fsdb.*; import gplx.fsdb.meta.*;
|
||||
class Merge_wkr__core {
|
||||
private final Db_attach_mgr attach_mgr = new Db_attach_mgr();
|
||||
private final Db_tbl_copy copy_mgr = new Db_tbl_copy();
|
||||
public void Copy_to_temp(Merge_prog_wkr prog_wkr, Xow_wiki wiki, Db_conn src_conn) {
|
||||
Create_dbs(wiki, src_conn);
|
||||
Merge_core(wiki, src_conn);
|
||||
Merge_srch(wiki, src_conn);
|
||||
Merge_file(wiki, src_conn);
|
||||
}
|
||||
private void Create_dbs(Xow_wiki wiki, Db_conn src_conn) {
|
||||
Db_cfg_tbl cfg_tbl = Xowd_cfg_tbl_.New(src_conn, "xowa_cfg__core");
|
||||
Xowd_core_db_props core_db_props = Xowd_core_db_props.Cfg_load(src_conn, cfg_tbl);
|
||||
Xob_info_session session = Xob_info_session.Load(cfg_tbl);
|
||||
|
||||
Xow_wiki_.Create_sql_backend(wiki, core_db_props, session);
|
||||
}
|
||||
private void Merge_core(Xow_wiki wiki, Db_conn src_conn) {
|
||||
Db_conn trg_conn = wiki.Data__core_mgr().Db__core().Conn();
|
||||
copy_mgr.Copy_many(src_conn, trg_conn, "site_ns", "css_core", "css_file"); // NOTE: "xowa_db" skipped; NOTE: css_core must go before Init_by_wiki
|
||||
copy_mgr.Copy_one(src_conn, trg_conn, "xowa_cfg__core", "xowa_cfg");
|
||||
wiki.Init_by_wiki();
|
||||
|
||||
// make text_db b/c page entries have Load_page_wkr will try to load from text
|
||||
Xow_db_file text_db = null;
|
||||
if (wiki.Data__core_mgr().Props().Layout_text().Tid_is_lot()) {
|
||||
text_db = wiki.Data__core_mgr().Dbs__get_by_tid_or_null(Xow_db_file_.Tid__text);
|
||||
if (text_db == null)
|
||||
text_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__text);
|
||||
}
|
||||
else
|
||||
text_db = wiki.Data__core_mgr().Db__core();
|
||||
text_db.Tbl__text().Create_tbl();
|
||||
|
||||
// merge at end for site_stats
|
||||
copy_mgr.Copy_many(src_conn, trg_conn, "site_stats");
|
||||
}
|
||||
private void Merge_srch(Xow_wiki wiki, Db_conn src_conn) {
|
||||
Srch_search_addon addon = Srch_search_addon.Get(wiki);
|
||||
|
||||
Srch_db_mgr srch_db_mgr = addon.Db_mgr();
|
||||
srch_db_mgr.Create_all();
|
||||
srch_db_mgr.Tbl__word().Create_idx();
|
||||
int len = srch_db_mgr.Tbl__link__len();
|
||||
for (int i = 0; i < len; ++i)
|
||||
srch_db_mgr.Tbl__link__get_at(i).Create_idx__link_score();
|
||||
|
||||
Db_conn trg_conn = addon.Db_mgr().Tbl__word().conn;
|
||||
// copy_mgr.Copy_many(src_conn, trg_conn, "search_link_reg");
|
||||
Merge_cfg(src_conn, trg_conn, "xowa_cfg__srch");
|
||||
}
|
||||
private void Merge_file(Xow_wiki wiki, Db_conn src_conn) {
|
||||
if (wiki.Type_is_edit())
|
||||
((Xowe_wiki)wiki).File_mgr().Init_file_mgr_by_load(wiki);
|
||||
else
|
||||
wiki.File__mnt_mgr().Ctor_by_load(wiki.File__fsdb_core());
|
||||
Db_conn trg_conn = wiki.File__fsdb_core().File__atr_file__at(Fsm_mnt_mgr.Mnt_idx_main).Conn();
|
||||
copy_mgr.Copy_many(src_conn, trg_conn, "fsdb_dba", "fsdb_dbb", "fsdb_dir", "fsdb_mnt");
|
||||
Merge_cfg(src_conn, trg_conn, "xowa_cfg__file");
|
||||
}
|
||||
private void Merge_cfg(Db_conn src_conn, Db_conn trg_conn, String src_tbl_name) {
|
||||
if (trg_conn.Meta_tbl_exists("xowa_cfg")) {
|
||||
attach_mgr.Conn_main_(trg_conn).Conn_links_(new Db_attach_itm("src_db", src_conn));
|
||||
attach_mgr.Exec_sql(String_.Concat_lines_nl
|
||||
( "INSERT INTO xowa_cfg"
|
||||
, "SELECT s.cfg_grp, s.cfg_key, s.cfg_val"
|
||||
, "FROM <src_db>" + src_tbl_name + " s"
|
||||
, " LEFT JOIN xowa_cfg t ON s.cfg_grp = t.cfg_grp AND s.cfg_key = t.cfg_key"
|
||||
, "WHERE t.cfg_grp IS NULL"
|
||||
));
|
||||
}
|
||||
else
|
||||
copy_mgr.Copy_one (src_conn, trg_conn, src_tbl_name, "xowa_cfg");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,55 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.merges; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.dbs.bulks.*; import gplx.dbs.engines.sqlite.*;
|
||||
import gplx.xowa.addons.bldrs.exports.utls.*;
|
||||
class Merge_wkr_utl {
|
||||
public static void Merge_by_sql(Db_bulk_prog prog_wkr, String msg, String tbl_name, Dbmeta_fld_list flds, Db_conn src_conn, Merge2_trg_itm trg_db, int trg_db_id, boolean disable_synchronous) {
|
||||
Bry_bfr bfr = Bry_bfr_.New();
|
||||
Db_conn trg_conn = trg_db.Conn();
|
||||
Db_attach_mgr attach_mgr = new Db_attach_mgr(trg_conn, new Db_attach_itm("src_db", src_conn));
|
||||
Io_url trg_url = Sqlite_conn_info.To_url(trg_conn);
|
||||
if (disable_synchronous) {
|
||||
Io_mgr.Instance.CopyFil(trg_url, trg_url.GenNewExt(".bak"), true);
|
||||
trg_conn.Exec_qry(Sqlite_pragma.New__synchronous__off());
|
||||
Set_journal(trg_conn, "OFF");
|
||||
}
|
||||
attach_mgr.Exec_sql(Bld_insert_into(bfr, tbl_name, flds, trg_db_id, disable_synchronous));
|
||||
if (disable_synchronous) {
|
||||
trg_conn.Exec_qry(Sqlite_pragma.New__synchronous__normal());
|
||||
Io_mgr.Instance.DeleteFil(trg_url.GenNewExt(".bak"));
|
||||
Set_journal(trg_conn, "DELETE");
|
||||
}
|
||||
prog_wkr.Prog__insert_and_stop_if_suspended(1);
|
||||
}
|
||||
private static void Set_journal(Db_conn conn, String mode) {
|
||||
Db_rdr rdr = conn.Stmt_sql("PRAGMA journal_mode=" + mode).Exec_select__rls_auto();
|
||||
try {
|
||||
rdr.Move_next();
|
||||
rdr.Read_at(0);
|
||||
}
|
||||
finally {
|
||||
rdr.Rls();
|
||||
}
|
||||
}
|
||||
private static String Bld_insert_into(Bry_bfr bfr, String tbl_name, Dbmeta_fld_list flds, int trg_db_id, boolean disable_synchronous) {
|
||||
Split_tbl_.Bld_insert_by_select(bfr, tbl_name, flds);
|
||||
if (trg_db_id != -1 && !disable_synchronous)
|
||||
bfr.Add_str_u8_fmt("WHERE trg_db_id = {0}", trg_db_id);
|
||||
return bfr.To_str_and_clear();
|
||||
}
|
||||
public static void Merge_by_rows(Db_bulk_prog prog_wkr, String msg, String tbl_name, Dbmeta_fld_list flds, Db_conn src_conn, Db_conn trg_conn, String[] order_bys) {
|
||||
Db_stmt src_stmt = order_bys == String_.Ary_empty
|
||||
? src_conn.Stmt_select_all(tbl_name, flds)
|
||||
: src_conn.Stmt_select_order(tbl_name, flds, String_.Ary_empty, order_bys)
|
||||
;
|
||||
Db_rdr src = src_stmt.Exec_select__rls_auto();
|
||||
Db_stmt trg = trg_conn.Stmt_insert(tbl_name, flds);
|
||||
try {Db_bulk_exec_.Insert(prog_wkr, msg, flds.To_fld_ary(), src, trg, trg_conn);}
|
||||
finally {
|
||||
src.Rls();
|
||||
trg.Rls();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,23 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Pack_file_bldr_cmd extends Xob_cmd__base {
|
||||
private final Pack_file_cfg cfg = new Pack_file_cfg();
|
||||
public Pack_file_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
|
||||
@Override public void Cmd_run() {
|
||||
new Pack_file_mgr().Exec(wiki, cfg);
|
||||
}
|
||||
|
||||
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk__cfg)) return cfg;
|
||||
else return super.Invk (ctx, ikey, k, m);
|
||||
}
|
||||
private static final String Invk__cfg = "cfg";
|
||||
|
||||
public static final String BLDR_CMD_KEY = "bldr.export.pack.file";
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
|
||||
public static final Xob_cmd Prototype = new Pack_file_bldr_cmd(null, null);
|
||||
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Pack_file_bldr_cmd(bldr, wiki);}
|
||||
}
|
||||
|
||||
@@ -13,37 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Pack_file_cfg implements Gfo_invk {
|
||||
public Io_url Deploy_dir() {return deploy_dir;} private Io_url deploy_dir;
|
||||
public boolean Pack_text() {return pack_text;} private boolean pack_text = false;
|
||||
public boolean Pack_html() {return pack_html;} private boolean pack_html = true;
|
||||
public boolean Pack_file() {return pack_file;} private boolean pack_file = true;
|
||||
public boolean Pack_lucene() {return pack_lucene;} private boolean pack_lucene;
|
||||
public long Lucene_max() {return lucene_max;} private long lucene_max = Io_mgr.Len_mb * 1500;
|
||||
public boolean Pack_fsdb_delete() {return pack_fsdb_delete;} private boolean pack_fsdb_delete;
|
||||
public boolean Pack_custom() {return pack_custom_files != null;}
|
||||
public String Pack_custom_files() {return pack_custom_files;} private String pack_custom_files;
|
||||
public String Pack_custom_name() {return pack_custom_name;} private String pack_custom_name;
|
||||
public DateAdp Pack_file_cutoff() {return pack_file_cutoff;} private DateAdp pack_file_cutoff = null;
|
||||
public String Wiki_date(DateAdp wiki_last_modified) {
|
||||
return wiki_date == null ? wiki_last_modified.XtoStr_fmt("yyyy.MM") : wiki_date;
|
||||
} private String wiki_date = null;
|
||||
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, "deploy_dir_")) deploy_dir = m.ReadIoUrl("v");
|
||||
else if (ctx.Match(k, "pack_text_")) pack_text = m.ReadYn("v");
|
||||
else if (ctx.Match(k, "pack_html_")) pack_html = m.ReadYn("v");
|
||||
else if (ctx.Match(k, "pack_file_")) pack_file = m.ReadYn("v");
|
||||
else if (ctx.Match(k, "pack_file_cutoff_")) pack_file_cutoff = m.ReadDate("v");
|
||||
else if (ctx.Match(k, "pack_fsdb_delete_")) pack_fsdb_delete = m.ReadYn("v");
|
||||
else if (ctx.Match(k, "pack_custom_name_")) pack_custom_name = m.ReadStr("v");
|
||||
else if (ctx.Match(k, "pack_custom_files_")) pack_custom_files = m.ReadStr("v"); // pack_custom {files='en.wikipedia.org-core.xowa|en.wikipedia.org-html-ns.008.xowa'}}
|
||||
else if (ctx.Match(k, "wiki_date_")) wiki_date = m.ReadStr("v");
|
||||
else if (ctx.Match(k, "pack_lucene_")) pack_lucene = m.ReadYn("v");
|
||||
else if (ctx.Match(k, "lucene_max_")) lucene_max = m.ReadLong("v") * Io_mgr.Len_mb;
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,161 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
import gplx.core.progs.*; import gplx.core.ios.zips.*; import gplx.core.ios.streams.*; import gplx.core.security.*;
|
||||
import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.fsdb.*; import gplx.fsdb.meta.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.hosts.*; import gplx.xowa.addons.bldrs.centrals.tasks.*;
|
||||
public class Pack_file_mgr {
|
||||
public void Exec(Xowe_wiki wiki, Pack_file_cfg cfg) {
|
||||
// init
|
||||
wiki.Init_assert();
|
||||
Io_url wiki_dir = wiki.Fsys_mgr().Root_dir();
|
||||
Io_url pack_dir = wiki_dir.GenSubDir_nest("tmp", "pack");
|
||||
Io_mgr.Instance.DeleteDirDeep(pack_dir); Io_mgr.Instance.CreateDirIfAbsent(pack_dir);
|
||||
String wiki_date = cfg.Wiki_date(wiki.Props().Modified_latest());
|
||||
Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date, cfg);
|
||||
|
||||
// get import_tbl
|
||||
byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo();
|
||||
Xobc_data_db bc_db = Xobc_data_db.New(wiki.App().Fsys_mgr());
|
||||
Db_conn bc_conn = bc_db.Conn();
|
||||
if (!cfg.Pack_custom()) // only delete files if not custom
|
||||
bc_db.Delete_by_import(wiki_abrv, wiki_date);
|
||||
bc_conn.Txn_bgn("xobc_import_insert");
|
||||
|
||||
// build zip packs
|
||||
Hash_algo hash_algo = Hash_algo_.New__md5();
|
||||
Bry_bfr tmp_bfr = Bry_bfr_.New();
|
||||
int hash_len = hash.Len();
|
||||
for (int i = 0; i < hash_len; ++i) {
|
||||
Pack_list list = (Pack_list)hash.Get_at(i);
|
||||
int list_len = list.Len();
|
||||
for (int j = 0; j < list_len; ++j) {
|
||||
Pack_itm itm = (Pack_itm)list.Get_at(j);
|
||||
Make_pack(wiki, wiki_dir, wiki_abrv, wiki_date, bc_db, hash_algo, tmp_bfr, itm, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// build tasks
|
||||
if (cfg.Pack_html())
|
||||
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, Xobc_task_regy_itm.Type__html, Xobc_import_type.Tid__wiki__core, Xobc_import_type.Tid__wiki__srch, Xobc_import_type.Tid__wiki__html, Xobc_import_type.Tid__wiki__ctg, Xobc_import_type.Tid__wiki__lucene);
|
||||
if (cfg.Pack_file())
|
||||
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, Xobc_task_regy_itm.Type__file, Xobc_import_type.Tid__file__core, Xobc_import_type.Tid__file__data); // , Xobc_import_type.Tid__fsdb__delete
|
||||
if (cfg.Pack_text()) // NOTE: will only add wikitext and wikibase packs; wikidata needs to pack html also; DATE:2017-04-09
|
||||
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, Xobc_task_regy_itm.Type__text, Xobc_import_type.Tid__wiki__text, Xobc_import_type.Tid__wiki__wbase);
|
||||
if (cfg.Pack_custom())
|
||||
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, cfg.Pack_custom_name(), Xobc_import_type.Tid__misc);
|
||||
bc_conn.Txn_end();
|
||||
|
||||
// deploy
|
||||
Io_url deploy_dir = cfg.Deploy_dir();
|
||||
if (deploy_dir != null) {
|
||||
Host_eval_itm host_eval = new Host_eval_itm();
|
||||
int len = hash.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Pack_list list = (Pack_list)hash.Get_at(i);
|
||||
int list_len = list.Len();
|
||||
for (int j = 0; j < list_len; ++j) {
|
||||
Pack_itm itm = (Pack_itm)list.Get_at(j);
|
||||
byte[] owner_dir = host_eval.Eval_dir_name(wiki.Domain_itm());
|
||||
Io_url src_url = itm.Zip_url();
|
||||
Io_url trg_url = deploy_dir.GenSubFil_nest(String_.new_u8(owner_dir), src_url.NameAndExt());
|
||||
Io_mgr.Instance.MoveFil_args(src_url, trg_url, true).Exec();
|
||||
}
|
||||
}
|
||||
Io_mgr.Instance.Delete_dir_empty(pack_dir);
|
||||
}
|
||||
}
|
||||
private static void Make_task(Bry_bfr tmp_bfr, Xow_wiki wiki, String wiki_date, Xobc_data_db bc_db, Pack_hash hash, String task_type, int... list_tids) {
|
||||
// get packs
|
||||
List_adp pack_list = List_adp_.New();
|
||||
int list_tids_len = list_tids.length;
|
||||
long raw_len = 0;
|
||||
for (int i = 0; i < list_tids_len; ++i) {
|
||||
Pack_list list = hash.Get_by(list_tids[i]);
|
||||
if (list == null) continue; // no lists for that tid;
|
||||
int list_len = list.Len();
|
||||
for (int j = 0; j < list_len; ++j) {
|
||||
Pack_itm itm = (Pack_itm)list.Get_at(j);
|
||||
raw_len += itm.Raw_size();
|
||||
pack_list.Add(itm);
|
||||
}
|
||||
}
|
||||
int pack_list_len = pack_list.Len();
|
||||
|
||||
// create task
|
||||
String task_key = Xobc_task_key.To_str(wiki.Domain_str(), wiki_date, task_type);
|
||||
String task_name = Build_task_name(tmp_bfr, wiki, wiki_date, task_type, raw_len);
|
||||
Xobc_task_regy_tbl task_regy_tbl = bc_db.Tbl__task_regy();
|
||||
int task_id = bc_db.Conn().Sys_mgr().Autonum_next("task_regy.task_id");
|
||||
task_regy_tbl.Insert(task_id, task_id, pack_list_len, task_key, task_name);
|
||||
|
||||
// map steps
|
||||
for (int i = 0; i < pack_list_len; ++i) {
|
||||
Pack_itm itm = (Pack_itm)pack_list.Get_at(i);
|
||||
int sm_id = bc_db.Conn().Sys_mgr().Autonum_next("step_map.sm_id");
|
||||
bc_db.Tbl__step_map().Insert(sm_id, task_id, itm.Step_id(), i);
|
||||
}
|
||||
}
|
||||
public static String Build_task_name(Bry_bfr tmp_bfr, Xow_wiki wiki, String wiki_date, String task_type, long raw_len) {// Simple Wikipedia - Articles (2016-06) [420.31 MB]
|
||||
byte[] lang_key = wiki.Domain_itm().Lang_orig_key();
|
||||
byte[] lang_name = Bry_.Len_eq_0(lang_key) // species.wikimedia.org and other wikimedia wikis have no lang;
|
||||
? Bry_.Empty
|
||||
: Bry_.Add(gplx.xowa.langs.Xol_lang_stub_.Get_by_key_or_null(lang_key).Canonical_name(), Byte_ascii.Space); // EX: "Deutsch "
|
||||
byte[] wiki_name = wiki.Domain_itm().Domain_type().Display_bry(); // EX: Wikipedia
|
||||
String type_name = Get_task_name_by_task_type(task_type);
|
||||
wiki_date = String_.Replace(wiki_date, ".", "-");
|
||||
String file_size = gplx.core.ios.Io_size_.To_str_new(tmp_bfr, raw_len, 2);
|
||||
return String_.Format("{0}{1} - {2} ({3}) [{4}]", lang_name, wiki_name, type_name, wiki_date, file_size);
|
||||
}
|
||||
private static String Get_task_name_by_task_type(String task_type) {
|
||||
if (String_.Eq(task_type, Xobc_task_regy_itm.Type__html)) return "Articles";
|
||||
else if (String_.Eq(task_type, Xobc_task_regy_itm.Type__file)) return "Images";
|
||||
else if (String_.Eq(task_type, Xobc_task_regy_itm.Type__text)) return "Wikitext";
|
||||
else return task_type;
|
||||
}
|
||||
private static void Make_pack(Xowe_wiki wiki, Io_url wiki_dir, byte[] wiki_abrv, String wiki_date, Xobc_data_db bc_db, Hash_algo hash_algo, Bry_bfr tmp_bfr, Pack_itm itm, int task_id) {
|
||||
// hash raws
|
||||
Io_url zip_url = itm.Zip_url();
|
||||
Gfo_log_.Instance.Prog("hashing raw: " + zip_url.NameAndExt());
|
||||
Io_url md5_url = wiki_dir.GenSubFil(zip_url.NameOnly() + ".md5");
|
||||
long raw_size = 0;
|
||||
Io_url[] raw_urls = itm.Raw_urls();
|
||||
int raw_urls_len = raw_urls.length;
|
||||
for (int i = 0; i < raw_urls_len; ++i) {
|
||||
Io_url raw_url = raw_urls[i];
|
||||
IoStream raw_stream = Io_mgr.Instance.OpenStreamRead(raw_url);
|
||||
byte[] raw_md5 = null;
|
||||
try {raw_md5 = hash_algo.Hash_stream_as_bry(Gfo_prog_ui_.Noop, raw_stream);}
|
||||
finally {raw_stream.Rls();}
|
||||
tmp_bfr.Add(raw_md5).Add_byte_space().Add_byte(Byte_ascii.Star).Add_str_a7(raw_url.NameAndExt()).Add_byte_nl();
|
||||
raw_size += raw_stream.Len();
|
||||
}
|
||||
Io_mgr.Instance.SaveFilBfr(md5_url, tmp_bfr);
|
||||
itm.Raw_size_(raw_size);
|
||||
|
||||
// compress raws
|
||||
Gfo_log_.Instance.Prog("compressing raw");
|
||||
Io_zip_compress_cmd__jre zip_cmd = new Io_zip_compress_cmd__jre();
|
||||
raw_urls = (Io_url[])Array_.Insert(raw_urls, new Io_url[] {md5_url}, 0); // add ".md5" to .zip
|
||||
zip_cmd.Exec_hook(Gfo_prog_ui_.Noop, raw_urls, zip_url, "", 0, 0);
|
||||
|
||||
// hash zip
|
||||
Gfo_log_.Instance.Prog("hashing zip");
|
||||
IoStream zip_stream = Io_mgr.Instance.OpenStreamRead(zip_url);
|
||||
byte[] zip_md5 = null;
|
||||
try {zip_md5 = hash_algo.Hash_stream_as_bry(Gfo_prog_ui_.Noop, zip_stream);}
|
||||
finally {zip_stream.Rls();}
|
||||
long zip_len = Io_mgr.Instance.QueryFil(zip_url).Size();
|
||||
|
||||
// cleanup
|
||||
Io_mgr.Instance.DeleteFil(md5_url);
|
||||
|
||||
// generate import
|
||||
Gfo_log_.Instance.Prog("generating tasks");
|
||||
int step_id = bc_db.Conn().Sys_mgr().Autonum_next("step_regy.step_id");
|
||||
itm.Step_id_(step_id);
|
||||
bc_db.Tbl__step_regy().Insert(step_id, Xobc_step_itm.Type__wiki_import);
|
||||
bc_db.Tbl__import_step().Insert(step_id, gplx.xowa.addons.bldrs.centrals.dbs.datas.Xobc_host_regy_tbl.Host_id__archive_org, wiki_abrv, wiki_date, zip_url.NameAndExt(), itm.Tid(), Xobc_zip_type.Type__zip, zip_md5, zip_len, raw_size, 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,41 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
class Pack_hash {
|
||||
private final Ordered_hash hash = Ordered_hash_.New();
|
||||
public int Len() {return hash.Len();}
|
||||
public Pack_list Get_at(int i) {return (Pack_list)hash.Get_at(i);}
|
||||
public Pack_list Get_by(int tid) {return (Pack_list)hash.Get_by(tid);}
|
||||
public Pack_itm Add(Pack_zip_name_bldr bldr, int list_tid, Io_url file_url) {return Add(list_tid, bldr.Bld(file_url), file_url);}
|
||||
public Pack_itm Add(int list_tid, Io_url pack_url, Io_url... raw_urls) {
|
||||
Pack_list list = (Pack_list)hash.Get_by(list_tid);
|
||||
if (list == null) {
|
||||
list = new Pack_list(list_tid);
|
||||
hash.Add(list_tid, list);
|
||||
}
|
||||
Pack_itm itm = new Pack_itm(list_tid, pack_url, raw_urls);
|
||||
|
||||
// check if file exists; needed for wikitext packs which add same urls as html packs; DATE:2017-04-09
|
||||
if (list.Has(pack_url))
|
||||
return itm;
|
||||
list.Add(itm);
|
||||
return itm;
|
||||
}
|
||||
public void Consolidate(int... tids) { // merge n itms into 1 itm; needed for search-core + search-link -> search
|
||||
int tids_len = tids.length;
|
||||
for (int i = 0; i < tids_len; ++i) {
|
||||
Pack_list list = (Pack_list)hash.Get_by(tids[i]);
|
||||
if (list == null) continue; // tid doesn't exist; EX: search in Tid_few
|
||||
int list_len = list.Len();
|
||||
Pack_itm itm_0 = (Pack_itm)list.Get_at(0);
|
||||
Io_url[] urls = new Io_url[list_len];
|
||||
for (int j = 0; j < list_len; ++j) {
|
||||
urls[j] = ((Pack_itm)list.Get_at(j)).Raw_urls()[0];
|
||||
}
|
||||
list.Clear();
|
||||
itm_0.Raw_urls_(urls);
|
||||
list.Add(itm_0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,149 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
import gplx.core.ios.*;
|
||||
import gplx.fsdb.meta.*;
|
||||
import gplx.xowa.wikis.data.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*;
|
||||
class Pack_hash_bldr {
|
||||
public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, Pack_file_cfg cfg) {
|
||||
// boolean pack_text, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff, boolean pack_fsdb_delete
|
||||
Pack_hash rv = new Pack_hash();
|
||||
Pack_zip_name_bldr zip_name_bldr = new Pack_zip_name_bldr(pack_dir, wiki.Domain_str(), String_.new_a7(wiki.Domain_itm().Abrv_wm()), wiki_date, cfg.Pack_custom_name());
|
||||
Xow_db_mgr db_mgr = wiki.Data__core_mgr();
|
||||
|
||||
// bld custom_files
|
||||
if (cfg.Pack_custom())
|
||||
return Bld_custom_files(rv, wiki, wiki_dir, zip_name_bldr, cfg.Pack_custom_files());
|
||||
|
||||
// bld html pack
|
||||
if (cfg.Pack_html()) {
|
||||
int len = db_mgr.Dbs__len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xow_db_file file = db_mgr.Dbs__get_at(i);
|
||||
int pack_tid = Get_pack_tid(file.Tid());
|
||||
if (Int_.In(pack_tid, Xobc_import_type.Tid__ignore, Xobc_import_type.Tid__wiki__text)) continue;
|
||||
rv.Add(zip_name_bldr, pack_tid, file.Url());
|
||||
}
|
||||
rv.Consolidate(Xobc_import_type.Tid__wiki__srch);
|
||||
}
|
||||
|
||||
// bld text pack
|
||||
if (cfg.Pack_text()) {
|
||||
int len = db_mgr.Dbs__len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xow_db_file file = db_mgr.Dbs__get_at(i);
|
||||
int pack_tid = Get_pack_tid(file.Tid());
|
||||
if (pack_tid == Xobc_import_type.Tid__ignore) continue; // NOTE: will try to add same urls as html, but noops b/c rv.Add checks for unique urls; DATE:2017-04-09
|
||||
rv.Add(zip_name_bldr, pack_tid, file.Url());
|
||||
}
|
||||
}
|
||||
|
||||
// bld lucene pack
|
||||
if (cfg.Pack_lucene()) {
|
||||
Pack_lucene(rv, wiki, zip_name_bldr, cfg);
|
||||
}
|
||||
|
||||
// bld file pack
|
||||
if (cfg.Pack_file()) {
|
||||
Fsm_mnt_itm mnt_itm = wiki.File__mnt_mgr().Mnts__get_at(Fsm_mnt_mgr.Mnt_idx_main);
|
||||
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__core, wiki_dir.GenSubFil(mnt_itm.Atr_mgr().Db__core().Url_rel()));
|
||||
if (db_mgr.Props().Layout_file().Tid_is_lot()) {
|
||||
Fsm_bin_mgr bin_mgr = mnt_itm.Bin_mgr();
|
||||
int bin_len = bin_mgr.Dbs__len();
|
||||
for (int i = 0; i < bin_len; ++i) {
|
||||
Fsm_bin_fil bin_fil = bin_mgr.Dbs__get_at(i);
|
||||
Io_url bin_fil_url = bin_fil.Url();
|
||||
|
||||
// ignore if bin_fil is earlier than cutoff
|
||||
if (cfg.Pack_file_cutoff() != null) {
|
||||
DateAdp bin_fil_date = Io_mgr.Instance.QueryFil(bin_fil_url).ModifiedTime();
|
||||
if (bin_fil_date.Timestamp_unix() < cfg.Pack_file_cutoff().Timestamp_unix()) continue;
|
||||
}
|
||||
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__data, bin_fil_url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bld pack_fsdb_delete
|
||||
if (cfg.Pack_fsdb_delete()) {
|
||||
gplx.xowa.bldrs.Xob_db_file fsdb_deletion_db = gplx.xowa.bldrs.Xob_db_file.New__deletion_db(wiki);
|
||||
if (!Io_mgr.Instance.ExistsFil(fsdb_deletion_db.Url())) throw Err_.new_wo_type("deletion db does not exists: url=" + fsdb_deletion_db.Url().Raw());
|
||||
rv.Add(zip_name_bldr, Xobc_import_type.Tid__fsdb__delete, fsdb_deletion_db.Url());
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
private static void Pack_lucene(Pack_hash rv, Xow_wiki wiki, Pack_zip_name_bldr zip_name_bldr, Pack_file_cfg cfg) {
|
||||
// read files from lucene_dir
|
||||
Io_url lucene_dir = gplx.xowa.addons.wikis.fulltexts.Xosearch_fulltext_addon.Get_index_dir(wiki);
|
||||
IoItmHash fils = Io_mgr.Instance.QueryDir_args(lucene_dir).ExecAsItmHash();
|
||||
|
||||
// init vars
|
||||
int pack_num = 0;
|
||||
long size_cur = 0;
|
||||
long size_max = cfg.Lucene_max();
|
||||
List_adp url_list = List_adp_.New();
|
||||
int fil_idx = 0;
|
||||
int fils_len = fils.Len();
|
||||
|
||||
// loop over each file
|
||||
while (fil_idx < fils_len) {
|
||||
IoItmFil fil = (IoItmFil)fils.Get_at(fil_idx);
|
||||
|
||||
// calc size_new
|
||||
long size_new = size_cur + fil.Size();
|
||||
|
||||
// if last file, set size_new to max and add file
|
||||
boolean add_file = true;
|
||||
if (fil_idx == fils_len - 1) {
|
||||
size_new = size_max;
|
||||
url_list.Add(fil.Url());
|
||||
add_file = false;
|
||||
}
|
||||
|
||||
// size exceeded; make new pack
|
||||
if (size_new >= size_max) {
|
||||
rv.Add(Xobc_import_type.Tid__wiki__lucene, zip_name_bldr.Bld_by_suffix("xtn.fulltext_search", pack_num), (Io_url[])url_list.To_ary_and_clear(Io_url.class));
|
||||
pack_num++;
|
||||
size_cur = 0;
|
||||
}
|
||||
// size too small; just update
|
||||
else {
|
||||
size_cur = size_new;
|
||||
}
|
||||
|
||||
// add file to list; ignore if last file and added above
|
||||
if (add_file)
|
||||
url_list.Add(fil.Url());
|
||||
fil_idx++;
|
||||
}
|
||||
}
|
||||
private static Pack_hash Bld_custom_files(Pack_hash rv, Xow_wiki wiki, Io_url wiki_dir, Pack_zip_name_bldr zip_name_bldr, String custom_files_blob) {
|
||||
String[] custom_files = String_.Split(custom_files_blob, "|");
|
||||
int len = custom_files.length;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Io_url file_url = wiki_dir.GenSubFil(custom_files[i]);
|
||||
rv.Add(zip_name_bldr, Xobc_import_type.Tid__misc, file_url);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
private static int Get_pack_tid(byte db_file_tid) {
|
||||
switch (db_file_tid) {
|
||||
case Xow_db_file_.Tid__core: return Xobc_import_type.Tid__wiki__core;
|
||||
case Xow_db_file_.Tid__search_core:
|
||||
case Xow_db_file_.Tid__search_link: return Xobc_import_type.Tid__wiki__srch;
|
||||
case Xow_db_file_.Tid__html_solo:
|
||||
case Xow_db_file_.Tid__html_data: return Xobc_import_type.Tid__wiki__html;
|
||||
case Xow_db_file_.Tid__cat:
|
||||
case Xow_db_file_.Tid__cat_core:
|
||||
case Xow_db_file_.Tid__cat_link: return Xobc_import_type.Tid__wiki__ctg;
|
||||
case Xow_db_file_.Tid__file_core: return Xobc_import_type.Tid__file__core;
|
||||
case Xow_db_file_.Tid__file_solo:
|
||||
case Xow_db_file_.Tid__file_data: return Xobc_import_type.Tid__file__data;
|
||||
case Xow_db_file_.Tid__text: return Xobc_import_type.Tid__wiki__text;
|
||||
case Xow_db_file_.Tid__wbase: return Xobc_import_type.Tid__wiki__wbase;
|
||||
default: return Xobc_import_type.Tid__ignore;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,16 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
class Pack_itm {
|
||||
public Pack_itm(int tid, Io_url zip_url, Io_url... raw_urls) {
|
||||
this.tid = tid;
|
||||
this.zip_url = zip_url;
|
||||
this.raw_urls = raw_urls;
|
||||
}
|
||||
public int Tid() {return tid;} private int tid;
|
||||
public Io_url Zip_url() {return zip_url;} private Io_url zip_url;
|
||||
public Io_url[] Raw_urls() {return raw_urls;} private Io_url[] raw_urls; public void Raw_urls_(Io_url[] v) {this.raw_urls = v;}
|
||||
public long Raw_size() {return raw_size;} private long raw_size; public void Raw_size_(long v) {this.raw_size = v;}
|
||||
public int Step_id() {return step_id;} private int step_id; public void Step_id_(int v) {this.step_id = v;}
|
||||
}
|
||||
|
||||
@@ -13,16 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
class Pack_list {
|
||||
private final Ordered_hash list = Ordered_hash_.New();
|
||||
public Pack_list(int tid) {this.tid = tid;}
|
||||
public int Tid() {return tid;} private final int tid;
|
||||
public int Len() {return list.Len();}
|
||||
public Pack_itm Get_at(int i) {return (Pack_itm)list.Get_at(i);}
|
||||
public void Add(Pack_itm itm) {list.Add(itm.Zip_url().Raw(), itm);}
|
||||
public void Clear() {list.Clear();}
|
||||
public boolean Has(Io_url url) {
|
||||
return list.Has(url.Raw());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,54 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
public class Pack_zip_name_bldr { // en.wikipedia.org-file-ns.000-db.001.xowa -> Xowa_enwiki_2016-05_file_ns.000_db.001.zip
|
||||
private final Io_url pack_dir;
|
||||
private final byte[] wiki_domain, zip_name_prefix;
|
||||
|
||||
public Pack_zip_name_bldr(Io_url pack_dir, String wiki_domain_str, String wiki_abrv, String wiki_date, String custom_name) {
|
||||
this.pack_dir = pack_dir;
|
||||
this.wiki_domain = Bry_.new_u8(wiki_domain_str);
|
||||
|
||||
String zip_name_suffix = custom_name == null ? String_.Replace(wiki_date, ".", "-") : custom_name; // "Xowa_enwiki_2016-10_core.zip" vs "Xowa_enwiki_custom_core.zip"
|
||||
this.zip_name_prefix = Bry_.new_u8("Xowa_" + wiki_abrv + "_" + zip_name_suffix);
|
||||
}
|
||||
public Io_url Bld(Io_url orig_url) {
|
||||
// get name and add .zip; EX: "en.wikipedia.org-file-core.xowa" -> "en.wikipedia.org-file-core.zip"
|
||||
byte[] orig_bry = Bry_.new_u8(orig_url.NameOnly() + ".zip");
|
||||
|
||||
// swap dashes with unders; EX: "en.wikipedia.org-file-core.xowa" -> "en.wikipedia.org_file_core.zip"
|
||||
orig_bry = Bry_.Replace(orig_bry, Byte_ascii.Dash, Byte_ascii.Underline);
|
||||
|
||||
// swap domain with xobc-style-prefix; EX: "en.wikipedia.org_file_core.zip" -> "Xowa_enwiki_2017-03_file_core.zip"
|
||||
orig_bry = Bry_.Replace(orig_bry, wiki_domain, zip_name_prefix);
|
||||
|
||||
return pack_dir.GenSubFil(String_.new_u8(orig_bry));
|
||||
}
|
||||
public Io_url Bld_by_suffix(String suffix, int pack_num) {
|
||||
// make fil_name EX: "Xowa_enwiki_2017-03" + "_" + "xtn.fulltext_search.001" + .zip
|
||||
String fil_name = String_.new_u8(zip_name_prefix) + "_" + suffix + "." + Int_.To_str_pad_bgn_zero(pack_num + List_adp_.Base1, 3) + ".zip";
|
||||
return pack_dir.GenSubFil(fil_name);
|
||||
}
|
||||
public static Io_url To_wiki_url(Io_url wiki_dir, Io_url zip_dir) {
|
||||
// get wiki_url based on wiki_dir and xobc_zip_fil; EX: "/wiki/en.wikipedia.org/", "/wiki/tmp/Xowa_enwiki_2016-09_file_core_deletion_2016-09/" -> "/wiki/en.wikipedia.org-file-core-deletion-2016.09.zip"
|
||||
String name_str = zip_dir.NameOnly() + ".xowa";
|
||||
byte[] name_bry = Bry_.new_u8(name_str);
|
||||
int pos = Bry_find__Find_fwd_idx(name_bry, Byte_ascii.Underline, 2);
|
||||
name_bry = Bry_.Mid(name_bry, pos, name_bry.length);
|
||||
name_bry = Bry_.Add(Bry_.new_u8(wiki_dir.NameOnly()), name_bry);
|
||||
name_bry = Bry_.Replace(name_bry, Byte_ascii.Underline, Byte_ascii.Dash);
|
||||
return wiki_dir.GenSubFil(String_.new_u8(name_bry));
|
||||
}
|
||||
private static int Bry_find__Find_fwd_idx(byte[] src, byte val, int find_max) {
|
||||
int src_len = src.length;
|
||||
int find_cur = 0;
|
||||
int cur_pos = 0;
|
||||
while (true) {
|
||||
int new_pos = Bry_find_.Find_fwd(src, val, cur_pos, src_len);
|
||||
if (new_pos == -1) throw Err_.New("failed to find value; src={0} val={1}", src, val);
|
||||
if (find_cur++ == find_max) return new_pos;
|
||||
cur_pos = new_pos + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,27 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
import org.junit.*; import gplx.core.tests.*;
|
||||
public class Pack_zip_name_bldr__tst {
|
||||
private Pack_zip_name_bldr__fxt fxt = new Pack_zip_name_bldr__fxt();
|
||||
@Test public void Basic() {
|
||||
fxt.Test__to_wiki_url("mem/wiki/en.wikipedia.org/", "mem/wiki/en.wikipedia.org/tmp/Xowa_enwiki_2016-09_file_deletion_2016.09/", "mem/wiki/en.wikipedia.org/en.wikipedia.org-file-deletion-2016.09.xowa");
|
||||
}
|
||||
@Test public void Bld_by_suffix() {
|
||||
Pack_zip_name_bldr bldr = fxt.Make__bldr("mem/wiki/en.wikipedia.org/tmp/pack/", "en.wikipedia.org", "enwiki", "2017-03", null);
|
||||
fxt.Test__bld_by_suffix(bldr, "xtn.fulltext_search", 1, "mem/wiki/en.wikipedia.org/tmp/pack/Xowa_enwiki_2017-03_xtn.fulltext_search.002.zip");
|
||||
}
|
||||
}
|
||||
class Pack_zip_name_bldr__fxt {
|
||||
public void Test__to_wiki_url(String wiki_dir, String zip_fil, String expd) {
|
||||
Gftest.Eq__str(expd, Pack_zip_name_bldr.To_wiki_url(Io_url_.mem_fil_(wiki_dir), Io_url_.mem_dir_(zip_fil)).Raw(), "wiki_url");
|
||||
}
|
||||
|
||||
public Pack_zip_name_bldr Make__bldr(String wiki_dir, String domain, String wiki_abrv, String wiki_date, String custom_name) {
|
||||
return new Pack_zip_name_bldr(Io_url_.new_dir_(wiki_dir), domain, wiki_abrv, wiki_date, custom_name);
|
||||
}
|
||||
public void Test__bld_by_suffix(Pack_zip_name_bldr bldr, String suffix, int pack_num, String expd) {
|
||||
Gftest.Eq__str(expd, bldr.Bld_by_suffix(suffix, pack_num).Xto_api());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,16 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.splits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
class Pack_itm {
|
||||
public Pack_itm(int idx, int type, Io_url zip_url, Io_url[] raw_urls) {
|
||||
this.idx = idx;
|
||||
this.type = type;
|
||||
this.zip_url = zip_url;
|
||||
this.raw_urls = raw_urls;
|
||||
}
|
||||
public int Idx() {return idx;} private final int idx;
|
||||
public int Type() {return type;} private final int type;
|
||||
public Io_url[] Raw_urls() {return raw_urls;} private final Io_url[] raw_urls;
|
||||
public Io_url Zip_url() {return zip_url;} private final Io_url zip_url;
|
||||
}
|
||||
|
||||
@@ -13,13 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.splits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
import gplx.xowa.wikis.data.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*;
|
||||
import gplx.xowa.addons.bldrs.exports.splits.*; import gplx.xowa.addons.bldrs.exports.splits.mgrs.*;
|
||||
class Pack_list {
|
||||
private final List_adp list = List_adp_.New();
|
||||
public int Len() {return list.Len();}
|
||||
public Pack_itm Get_at(int i) {return (Pack_itm)list.Get_at(i);}
|
||||
public void Add(Pack_itm itm) {list.Add(itm);}
|
||||
}
|
||||
|
||||
@@ -13,132 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.splits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
import gplx.core.progs.*; import gplx.core.ios.zips.*; import gplx.core.ios.streams.*; import gplx.core.security.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.wikis.data.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.steps.*;
|
||||
import gplx.xowa.addons.bldrs.exports.splits.mgrs.*; import gplx.xowa.addons.bldrs.exports.splits.rslts.*;
|
||||
// NOTE: used for experimental pack / split approach (html,file,search in one db)
|
||||
class Pack_mgr {
|
||||
public void Exec(Xowe_wiki wiki, long pack_size_max) {
|
||||
// init
|
||||
Io_url wiki_dir = wiki.Fsys_mgr().Root_dir();
|
||||
wiki.Init_assert();
|
||||
String wiki_date = wiki.Props().Modified_latest().XtoStr_fmt("yyyyMMdd");
|
||||
|
||||
// build pack list
|
||||
Pack_list pack_list = Make_list(wiki, wiki_date, pack_size_max);
|
||||
|
||||
// get import_tbl
|
||||
byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo();
|
||||
Xobc_data_db bc_db = Xobc_data_db.New(wiki.App().Fsys_mgr());
|
||||
Db_conn bc_conn = bc_db.Conn();
|
||||
bc_db.Delete_by_import(wiki_abrv, wiki_date);
|
||||
bc_conn.Txn_bgn("xobc_import_insert");
|
||||
|
||||
// loop packs and (a) zip raws; (b) create entry
|
||||
Hash_algo hash_algo = Hash_algo_.New__md5();
|
||||
Bry_bfr tmp_bfr = Bry_bfr_.New();
|
||||
int len = pack_list.Len();
|
||||
int task_id = bc_conn.Sys_mgr().Autonum_next("task_regy.task_id");
|
||||
bc_db.Tbl__task_regy().Insert(task_id, task_id, len, wiki.Domain_str(), wiki.Domain_str());
|
||||
for (int i = 0; i < len; ++i)
|
||||
Make_pack(wiki, wiki_dir, wiki_abrv, wiki_date, bc_db, hash_algo, tmp_bfr, pack_list.Get_at(i), task_id);
|
||||
bc_conn.Txn_end();
|
||||
}
|
||||
private static Pack_list Make_list(Xow_wiki wiki, String wiki_date, long pack_size_max) {
|
||||
// init; delete dir;
|
||||
String wiki_abrv = String_.new_a7(wiki.Domain_itm().Abrv_wm());
|
||||
Io_url wiki_dir = wiki.Fsys_mgr().Root_dir();
|
||||
Io_url pack_root = wiki_dir.GenSubDir_nest("tmp", "pack");
|
||||
Io_url split_root = wiki_dir.GenSubDir_nest("tmp", "split");
|
||||
Io_mgr.Instance.DeleteDirDeep(pack_root);
|
||||
Io_mgr.Instance.CreateDirIfAbsent(pack_root);
|
||||
Io_url[] fils = Io_mgr.Instance.QueryDir_fils(split_root);
|
||||
List_adp list = List_adp_.New();
|
||||
|
||||
// delete pack dir
|
||||
Pack_list rv = new Pack_list();
|
||||
long pack_size_cur = 0;
|
||||
int len = fils.length;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Io_url fil = fils[i];
|
||||
long fil_size = Io_mgr.Instance.QueryFil(fil).Size();
|
||||
if (Split_file_tid_.To_tid(fil) == Split_file_tid_.Tid__temp) continue; // ignore temp file
|
||||
list.Add(fil);
|
||||
|
||||
// calc pack size
|
||||
long pack_size_new = pack_size_cur + fil_size;
|
||||
if ( pack_size_new > pack_size_max
|
||||
|| i == len - 1) {
|
||||
int pack_idx = rv.Len();
|
||||
Io_url fil_1st = (Io_url)list.Get_at(0);
|
||||
int ns_id = Split_file_tid_.Get_ns_by_url(fil_1st);
|
||||
Io_url zip_url = pack_root.GenSubFil(Split_file_tid_.Make_file_name(wiki_abrv, wiki_date, rv.Len(), ns_id, ".zip"));
|
||||
Pack_itm itm = new Pack_itm(pack_idx, Xobc_import_type.Tid__pack, zip_url, (Io_url[])list.To_ary_and_clear(Io_url.class));
|
||||
rv.Add(itm);
|
||||
pack_size_cur = 0;
|
||||
}
|
||||
else
|
||||
pack_size_cur = pack_size_new;
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
private static void Make_pack(Xowe_wiki wiki, Io_url wiki_dir, byte[] wiki_abrv, String wiki_date, Xobc_data_db bc_db, Hash_algo hash_algo, Bry_bfr tmp_bfr, Pack_itm itm, int task_id) {
|
||||
// hash raws
|
||||
Io_url zip_url = itm.Zip_url();
|
||||
Gfo_log_.Instance.Prog("hashing raw: " + zip_url.NameAndExt());
|
||||
Io_url md5_url = wiki_dir.GenSubFil(zip_url.NameOnly() + ".md5");
|
||||
long raw_size = 0;
|
||||
Io_url[] raw_urls = itm.Raw_urls();
|
||||
int raw_urls_len = raw_urls.length;
|
||||
for (int i = 0; i < raw_urls_len; ++i) {
|
||||
Io_url raw_url = raw_urls[i];
|
||||
IoStream raw_stream = Io_mgr.Instance.OpenStreamRead(raw_url);
|
||||
byte[] raw_md5 = null;
|
||||
try {raw_md5 = hash_algo.Hash_stream_as_bry(Gfo_prog_ui_.Noop, raw_stream);}
|
||||
finally {raw_stream.Rls();}
|
||||
tmp_bfr.Add(raw_md5).Add_byte_space().Add_byte(Byte_ascii.Star).Add_str_a7(raw_url.NameAndExt()).Add_byte_nl();
|
||||
raw_size += raw_stream.Len();
|
||||
}
|
||||
Io_mgr.Instance.SaveFilBfr(md5_url, tmp_bfr);
|
||||
|
||||
// calc prog_size_end
|
||||
long prog_size_end = 0; int prog_count_end =0;
|
||||
for (int i = 0; i < raw_urls_len; ++i) {
|
||||
Io_url raw_url = raw_urls[i];
|
||||
if (Split_file_tid_.To_tid(raw_url) != Split_file_tid_.Tid__data) continue;
|
||||
Db_conn src_conn = Db_conn_bldr.Instance.Get_or_noop(raw_url);
|
||||
Wkr_stats_tbl tbl = new Wkr_stats_tbl(src_conn);
|
||||
Wkr_stats_itm stat = tbl.Select_all__summary();
|
||||
prog_count_end += stat.Count;
|
||||
prog_size_end += stat.Size;
|
||||
}
|
||||
|
||||
// compress raws
|
||||
Gfo_log_.Instance.Prog("compressing raw");
|
||||
Io_zip_compress_cmd__jre zip_cmd = new Io_zip_compress_cmd__jre();
|
||||
raw_urls = (Io_url[])Array_.Insert(raw_urls, new Io_url[] {md5_url}, 0); // add ".md5" to .zip
|
||||
zip_cmd.Exec_hook(Gfo_prog_ui_.Noop, raw_urls, zip_url, "", 0, 0);
|
||||
|
||||
// hash zip
|
||||
Gfo_log_.Instance.Prog("hashing zip");
|
||||
IoStream zip_stream = Io_mgr.Instance.OpenStreamRead(zip_url);
|
||||
byte[] zip_md5 = null;
|
||||
try {zip_md5 = hash_algo.Hash_stream_as_bry(Gfo_prog_ui_.Noop, zip_stream);}
|
||||
finally {zip_stream.Rls();}
|
||||
long zip_len = Io_mgr.Instance.QueryFil(zip_url).Size();
|
||||
|
||||
// generate import
|
||||
Gfo_log_.Instance.Prog("generating tasks");
|
||||
int step_id = bc_db.Conn().Sys_mgr().Autonum_next("step_regy.step_id");
|
||||
int sm_id = bc_db.Conn().Sys_mgr().Autonum_next("step_map.sm_id");
|
||||
bc_db.Tbl__step_regy().Insert(step_id, Xobc_step_itm.Type__wiki_import);
|
||||
bc_db.Tbl__step_map().Insert(sm_id, task_id, step_id, itm.Idx());
|
||||
bc_db.Tbl__import_step().Insert(step_id, gplx.xowa.addons.bldrs.centrals.dbs.datas.Xobc_host_regy_tbl.Host_id__archive_org, wiki_abrv, wiki_date, zip_url.NameAndExt(), itm.Type(), Xobc_zip_type.Type__zip, zip_md5, zip_len, raw_size, prog_size_end, prog_count_end);
|
||||
|
||||
// cleanup
|
||||
Io_mgr.Instance.DeleteFil(md5_url);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,23 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.packs.splits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Pack_split_bldr_cmd extends Xob_cmd__base {
|
||||
private long pack_size_max = 30 * Io_mgr.Len_mb;
|
||||
public Pack_split_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
|
||||
@Override public void Cmd_run() {
|
||||
new Pack_mgr().Exec(wiki, pack_size_max);
|
||||
}
|
||||
|
||||
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk__pack_size_max_)) pack_size_max = m.ReadLong("v");
|
||||
return super.Invk (ctx, ikey, k, m);
|
||||
}
|
||||
private static final String Invk__pack_size_max_ = "pack_size_max_";
|
||||
|
||||
public static final String BLDR_CMD_KEY = "bldr.export.pack.split";
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
|
||||
public static final Xob_cmd Prototype = new Pack_split_bldr_cmd(null, null);
|
||||
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Pack_split_bldr_cmd(bldr, wiki);}
|
||||
}
|
||||
|
||||
@@ -13,24 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.splits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
import gplx.xowa.addons.bldrs.exports.splits.mgrs.*;
|
||||
public class Split_bldr_cmd extends Xob_cmd__base {
|
||||
private final Split_cfg cfg = new Split_cfg();
|
||||
public Split_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
|
||||
@Override public void Cmd_run() {
|
||||
wiki.Init_assert();
|
||||
new Split_mgr().Exec(wiki, cfg);
|
||||
}
|
||||
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk__cfg)) return cfg;
|
||||
return super.Invk (ctx, ikey, k, m);
|
||||
}
|
||||
private static final String Invk__cfg = "cfg";
|
||||
|
||||
public static final String BLDR_CMD_KEY = "bldr.export.split";
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
|
||||
public static final Xob_cmd Prototype = new Split_bldr_cmd(null, null);
|
||||
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Split_bldr_cmd(bldr, wiki);}
|
||||
}
|
||||
|
||||
@@ -13,64 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.splits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.addons.bldrs.exports.splits.mgrs.*; import gplx.xowa.addons.bldrs.exports.splits.metas.*; import gplx.xowa.addons.bldrs.exports.splits.rslts.*;
|
||||
public class Split_ctx {
|
||||
private int trg_idx = -1;
|
||||
private final Split_wkr[] wkrs;
|
||||
private boolean trg_make = true;
|
||||
private long trg_max = 32 * Io_mgr.Len_mb;
|
||||
public Split_ctx(Split_cfg cfg, Xow_wiki wiki, Split_wkr[] wkrs, Split_ns_itm[] ns_itms, Db_conn wkr_conn) {
|
||||
this.cfg = cfg; this.wiki = wiki; this.wkrs = wkrs;
|
||||
this.trg_max = cfg.Trg_max();
|
||||
this.ns_itms = ns_itms; this.wkr_conn = wkr_conn;
|
||||
this.rslt_mgr = new Split_rslt_mgr(wkr_conn);
|
||||
this.html_size_calc = new Split_db_size_calc(cfg.Html().Db_max(), cfg.Html().Db_idx());
|
||||
this.file_size_calc = new Split_db_size_calc(cfg.File().Db_max(), cfg.File().Db_idx());
|
||||
}
|
||||
public Split_cfg Cfg() {return cfg;} private final Split_cfg cfg;
|
||||
public Xow_wiki Wiki() {return wiki;} private final Xow_wiki wiki;
|
||||
public Split_ns_itm[] Ns_itms() {return ns_itms;} private final Split_ns_itm[] ns_itms;
|
||||
public Db_conn Wkr_conn() {return wkr_conn;} private final Db_conn wkr_conn;
|
||||
public Db_conn Trg_conn() {return trg_conn;} private Db_conn trg_conn;
|
||||
public int Trg_ns() {return trg_ns;} private int trg_ns; public void Trg_ns_(int v) {this.trg_ns = v;}
|
||||
public Split_page_mgr Page_mgr() {return page_mgr;} private final Split_page_mgr page_mgr = new Split_page_mgr();
|
||||
public Split_rslt_mgr Rslt_mgr() {return rslt_mgr;} private final Split_rslt_mgr rslt_mgr;
|
||||
public Split_db_size_calc Html_size_calc() {return html_size_calc;} private final Split_db_size_calc html_size_calc;
|
||||
public Split_db_size_calc File_size_calc() {return file_size_calc;} private final Split_db_size_calc file_size_calc;
|
||||
|
||||
public void Trg_db__completed() {trg_make = true;}
|
||||
public void Trg_db__assert(int ns_id) {
|
||||
if (rslt_mgr.Db_size() < trg_max && !trg_make) return;
|
||||
trg_make = false;
|
||||
|
||||
// term trg_conn
|
||||
if (trg_conn != null) Trg_db__term();
|
||||
|
||||
// init trg_con
|
||||
Io_url trg_url = this.Trg_db__make(ns_id);
|
||||
for (Split_wkr wkr : wkrs)
|
||||
wkr.Split__trg__nth__new(this, trg_conn);
|
||||
rslt_mgr.On__nth__new(trg_idx, trg_url, ns_id);
|
||||
trg_conn.Txn_bgn("split");
|
||||
}
|
||||
public Io_url Trg_db__make(int ns_id) {
|
||||
// create new trg_conn
|
||||
String trg_name = Split_file_tid_.Make_file_name(String_.new_u8(wiki.Domain_itm().Abrv_wm()), wiki.Props().Modified_latest().XtoStr_fmt("yyyy.MM"), ++trg_idx, ns_id, ".xowa");
|
||||
Io_url trg_url = wiki.Fsys_mgr().Root_dir().GenSubFil_nest("tmp", "split", trg_name);
|
||||
this.trg_conn = Db_conn_bldr.Instance.Get_or_autocreate(true, trg_url);
|
||||
return trg_url;
|
||||
}
|
||||
public void Trg_db__null() {trg_conn = null;} // null conn, else Trg_db_assert will try to close txn
|
||||
public void Term() {
|
||||
Trg_db__term();
|
||||
rslt_mgr.Term();
|
||||
}
|
||||
private void Trg_db__term() {
|
||||
rslt_mgr.On__nth__rls(trg_conn);
|
||||
trg_conn.Txn_end();
|
||||
for (Split_wkr wkr : wkrs)
|
||||
wkr.Split__trg__nth__rls(this, trg_conn);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,68 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.splits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.addons.bldrs.exports.splits.mgrs.*; import gplx.xowa.addons.bldrs.exports.splits.rslts.*;
|
||||
class Split_mgr {
|
||||
public void Exec(Xow_wiki wiki, Split_cfg cfg) {
|
||||
// init
|
||||
Split_ns_itm[] ns_itms = cfg.Ns_itms();
|
||||
Split_wkr[] wkrs = new Split_wkr[]
|
||||
{ new gplx.xowa.addons.bldrs.exports.splits.srchs.Split_wkr__srch()
|
||||
, new gplx.xowa.addons.bldrs.exports.splits.htmls.Split_wkr__html()
|
||||
, new gplx.xowa.addons.bldrs.exports.splits.pages.Split_wkr__page() // NOTE: page needs to follow html b/c of trg_db_id
|
||||
, new gplx.xowa.addons.bldrs.exports.splits.files.Split_wkr__file()
|
||||
, new gplx.xowa.addons.bldrs.exports.splits.rndms.Split_wkr__rndm()
|
||||
};
|
||||
|
||||
// init ctx
|
||||
Io_url split_root = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "split");
|
||||
Io_mgr.Instance.DeleteDirDeep(split_root);
|
||||
Io_mgr.Instance.CreateDirIfAbsent(split_root);
|
||||
Db_conn wkr_conn = Db_conn_bldr.Instance.Get_or_autocreate(true, split_root.GenSubFil("xowa.split.sqlite3"));
|
||||
Split_ctx ctx = new Split_ctx(cfg, wiki, wkrs, ns_itms, wkr_conn);
|
||||
ctx.Trg_db__make(-1);
|
||||
new Split_mgr_init().Init(ctx, ctx.Wkr_conn(), wiki.Data__core_mgr().Tbl__page().Conn());
|
||||
for (Split_wkr wkr : wkrs) {
|
||||
wkr.Split__init(ctx, wiki, wkr_conn);
|
||||
wkr.Split__trg__1st__new(ctx, ctx.Trg_conn());
|
||||
}
|
||||
ctx.Trg_db__null();
|
||||
|
||||
// split by ns
|
||||
List_adp page_list = List_adp_.New();
|
||||
Split_page_loader loader = new Split_page_loader(wiki, cfg.Loader_rows());
|
||||
for (Split_ns_itm ns_itm : ns_itms) {
|
||||
int ns_id = ns_itm.Ns_id();
|
||||
loader.Init_ns(ns_id);
|
||||
ctx.Trg_ns_(ns_id);
|
||||
while (true) {
|
||||
ctx.Trg_db__assert(ns_id); // new db will be needed when moving between ns; EX: ns.000 goes into one db; ns.004 goes into another
|
||||
boolean reading = loader.Load_pages(ctx, page_list, wkrs, ns_id);
|
||||
Split_pages(ctx, page_list, wkrs, ns_id);
|
||||
if (!reading) {// no more rows; ns is done; stop loop and go to next ns;
|
||||
ctx.Trg_db__completed();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanup
|
||||
loader.Rls();
|
||||
ctx.Term();
|
||||
for (Split_wkr wkr : wkrs)
|
||||
wkr.Split__term(ctx);
|
||||
}
|
||||
private void Split_pages(Split_ctx ctx, List_adp page_list, Split_wkr[] wkrs, int ns_id) {
|
||||
Split_rslt_mgr rslt_mgr = ctx.Rslt_mgr();
|
||||
int len = page_list.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
ctx.Trg_db__assert(ns_id); // new db may be needed; EX: 10,000 will be read, and 1st 100 needs 1 db; next 100 needs another db
|
||||
Xowd_page_itm page = (Xowd_page_itm)page_list.Get_at(i);
|
||||
int page_id = page.Id();
|
||||
for (Split_wkr wkr : wkrs)
|
||||
wkr.Split__exec(ctx, rslt_mgr, page, page_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,14 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.splits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*;
|
||||
import gplx.dbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.addons.bldrs.exports.splits.rslts.*;
|
||||
public interface Split_wkr {
|
||||
void Split__init (Split_ctx ctx, Xow_wiki wiki, Db_conn wkr_conn);
|
||||
void Split__pages_loaded (Split_ctx ctx, int ns_id, int score_bgn, int score_end);
|
||||
void Split__trg__1st__new (Split_ctx ctx, Db_conn trg_conn);
|
||||
void Split__trg__nth__new (Split_ctx ctx, Db_conn trg_conn);
|
||||
void Split__trg__nth__rls (Split_ctx ctx, Db_conn trg_conn);
|
||||
void Split__exec (Split_ctx ctx, Split_rslt_mgr rslt_mgr, Xowd_page_itm page, int page_id);
|
||||
void Split__term (Split_ctx ctx);
|
||||
}
|
||||
@@ -13,164 +13,3 @@ The terms of each license can be found in the source code repository:
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.exports.splits.archives; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.splits.*;
|
||||
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.dbs.metas.*;
|
||||
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.htmls.core.dbs.*;
|
||||
import gplx.xowa.addons.bldrs.exports.splits.htmls.*;
|
||||
class Reindex_html_dbs_cmd {
|
||||
private Db_conn core_conn;
|
||||
private Xowd_page_tbl page_tbl;
|
||||
private Xoh_src_tbl_mgr src_tbl_mgr;
|
||||
private Xoh_trg_tbl_mgr trg_tbl_mgr;
|
||||
private String tbl_page, fld_page_id, fld_page_ns, fld_page_len, fld_page_score, fld_page_html_db_id;
|
||||
private final String Idx_name = "page__repack";
|
||||
public void Exec(Xowe_wiki wiki, long trg_db_size_max) {
|
||||
// init
|
||||
wiki.Init_assert();
|
||||
Xow_db_mgr db_mgr = wiki.Data__core_mgr();
|
||||
this.page_tbl = db_mgr.Tbl__page();
|
||||
this.core_conn = page_tbl.Conn();
|
||||
this.src_tbl_mgr = new Xoh_src_tbl_mgr(wiki);
|
||||
this.trg_tbl_mgr = new Xoh_trg_tbl_mgr(wiki);
|
||||
this.tbl_page = page_tbl.Tbl_name();
|
||||
this.fld_page_id = page_tbl.Fld_page_id();
|
||||
this.fld_page_ns = page_tbl.Fld_page_ns();
|
||||
this.fld_page_len = page_tbl.Fld_page_len();
|
||||
this.fld_page_score = page_tbl.Fld_page_score();
|
||||
this.fld_page_html_db_id = page_tbl.Fld_html_db_id();
|
||||
|
||||
Create_repack_idx_on_page();
|
||||
Create_repack_tbl();
|
||||
Insert_repack_rows();
|
||||
Move_html_data(trg_db_size_max);
|
||||
Cleanup_dbs(wiki);
|
||||
core_conn.Meta_idx_delete(Idx_name);
|
||||
core_conn.Meta_tbl_delete("repack");
|
||||
core_conn.Env_vacuum();
|
||||
}
|
||||
private void Create_repack_idx_on_page() {
|
||||
// add idx: page (page_ns DESC, page_score DESC, page_len DESC)
|
||||
if (!core_conn.Meta_idx_exists(Idx_name)) {
|
||||
core_conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_name(tbl_page, Idx_name
|
||||
, Dbmeta_idx_fld.Dsc(fld_page_ns)
|
||||
, Dbmeta_idx_fld.Dsc(fld_page_score)
|
||||
, Dbmeta_idx_fld.Dsc(fld_page_len)));
|
||||
}
|
||||
}
|
||||
private void Create_repack_tbl() {
|
||||
core_conn.Meta_tbl_remake(Dbmeta_tbl_itm.New("repack"
|
||||
, Dbmeta_fld_itm.new_int("sort_idx").Autonum_y_().Primary_y_()
|
||||
, Dbmeta_fld_itm.new_int("page_id")
|
||||
, Dbmeta_fld_itm.new_int("page_ns")
|
||||
, Dbmeta_fld_itm.new_int("page_score")
|
||||
, Dbmeta_fld_itm.new_int("page_len")
|
||||
, Dbmeta_fld_itm.new_int("src_db")
|
||||
));
|
||||
}
|
||||
private void Insert_repack_rows() {
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "inserting rows into repack table");
|
||||
Db_qry__select_cmd select_qry = (Db_qry__select_cmd)Db_qry_.select_
|
||||
( tbl_page, fld_page_id, fld_page_ns, fld_page_len, fld_page_score, fld_page_html_db_id)
|
||||
.Order_(fld_page_ns, Bool_.Y)
|
||||
.Order_(fld_page_score, Bool_.N)
|
||||
.Order_(fld_page_len, Bool_.N)
|
||||
.Where_(Db_crt_.New_eq_not(fld_page_html_db_id, -1));
|
||||
Db_qry_.insert_("repack").Cols_("page_id", "page_ns", "page_len", "page_score", "src_db")
|
||||
.Select_(select_qry)
|
||||
.Exec_qry(core_conn);
|
||||
}
|
||||
private void Move_html_data(long trg_db_size_max) {
|
||||
// read rows and move
|
||||
Db_rdr rdr = core_conn.Stmt_select_order("repack", String_.Ary("sort_idx", "page_id", "page_ns", "page_len", "src_db"), String_.Ary_empty, "sort_idx").Exec_select__rls_auto();
|
||||
Db_stmt stmt_update = core_conn.Stmt_update(tbl_page, String_.Ary(fld_page_id), fld_page_html_db_id);
|
||||
Xoh_page_tbl_itm trg_html_tbl = null;
|
||||
try {
|
||||
Xowd_html_row src_html_row = new Xowd_html_row();
|
||||
long trg_db_size = 0;
|
||||
int ns_cur = -1, part_id = 0;
|
||||
int trg_db = -1;
|
||||
core_conn.Txn_bgn("update page");
|
||||
while (rdr.Move_next()) {
|
||||
// check if ns changed
|
||||
int page_ns = rdr.Read_int("page_ns");
|
||||
boolean ns_changed = false;
|
||||
if (ns_cur != page_ns ) { // ns changed
|
||||
ns_cur = page_ns;
|
||||
part_id = 0; // reset part_id; note that 1st part will be base_1
|
||||
ns_changed = true;
|
||||
}
|
||||
|
||||
Xoh_page_tbl_itm src_html_tbl = src_tbl_mgr.Get_or_load(rdr.Read_int("src_db"));
|
||||
int page_id = rdr.Read_int("page_id");
|
||||
if (!src_html_tbl.Html_tbl().Select_as_row(src_html_row, page_id)) throw Err_.new_wo_type("could not find html", "page_id", page_id);
|
||||
|
||||
// check if new file needed
|
||||
int page_size = src_html_row.Body().length;
|
||||
trg_db_size += page_size;
|
||||
if ( trg_html_tbl == null // will be null for 1st pass
|
||||
|| ns_changed // ns_changed
|
||||
|| trg_db_size > trg_db_size_max // file filled
|
||||
) {
|
||||
if (trg_html_tbl != null) { // close trg_db if open
|
||||
trg_html_tbl.Html_tbl().Insert_end();
|
||||
trg_html_tbl.Rls();
|
||||
}
|
||||
trg_html_tbl = trg_tbl_mgr.Make_new(ns_cur, ++part_id);
|
||||
trg_html_tbl.Html_tbl().Insert_bgn();
|
||||
trg_db = trg_html_tbl.Db_id();
|
||||
trg_db_size = page_size;
|
||||
}
|
||||
|
||||
// move row
|
||||
trg_html_tbl.Html_tbl().Insert(src_html_row.Page_id(), src_html_row.Head_flag(), src_html_row.Body_flag()
|
||||
, src_html_row.Display_ttl(), src_html_row.Content_sub(), src_html_row.Sidebar_div()
|
||||
, src_html_row.Body());
|
||||
|
||||
// update page_html_db_id
|
||||
stmt_update.Clear().Val_int(fld_page_html_db_id, trg_db).Crt_int(fld_page_id, page_id).Exec_update();
|
||||
}
|
||||
} finally {
|
||||
rdr.Rls();
|
||||
if (trg_html_tbl != null) {
|
||||
trg_html_tbl.Html_tbl().Insert_end();
|
||||
trg_html_tbl.Rls();
|
||||
}
|
||||
core_conn.Txn_end();
|
||||
stmt_update.Rls();
|
||||
src_tbl_mgr.Cleanup();
|
||||
}
|
||||
}
|
||||
private void Cleanup_dbs(Xowe_wiki wiki) {
|
||||
// delete old dbs
|
||||
wiki.Data__core_mgr().Rls();
|
||||
String repack_suffix = Xoh_trg_tbl_mgr.Repack_suffix;
|
||||
Db_stmt delete_stmt = core_conn.Stmt_delete(Xowd_xowa_db_tbl.TBL_NAME, Xowd_xowa_db_tbl.Fld_id);
|
||||
Db_rdr rdr = core_conn.Stmt_select(Xowd_xowa_db_tbl.TBL_NAME, String_.Ary(Xowd_xowa_db_tbl.Fld_id, Xowd_xowa_db_tbl.Fld_type, Xowd_xowa_db_tbl.Fld_url)).Exec_select__rls_auto();
|
||||
while (rdr.Move_next()) {
|
||||
byte file_tid = rdr.Read_byte(Xowd_xowa_db_tbl.Fld_type);
|
||||
if (file_tid != Xow_db_file_.Tid__html_data) continue;
|
||||
String file_url = rdr.Read_str(Xowd_xowa_db_tbl.Fld_url);
|
||||
if (String_.Has(file_url, repack_suffix)) continue;
|
||||
delete_stmt.Clear().Crt_int(Xowd_xowa_db_tbl.Fld_id, rdr.Read_int(Xowd_xowa_db_tbl.Fld_id)).Exec_delete();
|
||||
Io_mgr.Instance.DeleteFil(wiki.Fsys_mgr().Root_dir().GenSubFil(file_url));
|
||||
}
|
||||
rdr.Rls();
|
||||
delete_stmt.Rls();
|
||||
|
||||
// update new dbs
|
||||
Db_stmt update_stmt = core_conn.Stmt_update(Xowd_xowa_db_tbl.TBL_NAME, String_.Ary(Xowd_xowa_db_tbl.Fld_id), Xowd_xowa_db_tbl.Fld_url);
|
||||
rdr = core_conn.Stmt_select(Xowd_xowa_db_tbl.TBL_NAME, String_.Ary(Xowd_xowa_db_tbl.Fld_id, Xowd_xowa_db_tbl.Fld_type, Xowd_xowa_db_tbl.Fld_url)).Exec_select__rls_auto();
|
||||
while (rdr.Move_next()) {
|
||||
byte file_tid = rdr.Read_byte(Xowd_xowa_db_tbl.Fld_type);
|
||||
if (file_tid != Xow_db_file_.Tid__html_data) continue;
|
||||
Io_url old_url = wiki.Fsys_mgr().Root_dir().GenSubFil(rdr.Read_str(Xowd_xowa_db_tbl.Fld_url));
|
||||
String old_raw = old_url.Raw();
|
||||
Io_url new_url = Io_url_.new_fil_(String_.Replace(old_raw, repack_suffix, ""));
|
||||
if (!String_.Has(old_raw, repack_suffix)) throw Err_.new_wo_type("html db should be repack", "db_name", old_raw);
|
||||
update_stmt.Clear().Val_str(Xowd_xowa_db_tbl.Fld_url, new_url.NameAndExt()).Crt_int(Xowd_xowa_db_tbl.Fld_id, rdr.Read_int(Xowd_xowa_db_tbl.Fld_id)).Exec_update();
|
||||
Io_mgr.Instance.MoveFil(old_url, new_url);
|
||||
}
|
||||
rdr.Rls();
|
||||
update_stmt.Rls();
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user