mirror of
https://github.com/gnosygnu/xowa.git
synced 2026-03-02 03:49:30 +00:00
uca category support and other changes
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.core.strings.*;
|
||||
public class Xoac_wiki_cfg_bldr_cmd {
|
||||
public Xoac_wiki_cfg_bldr_cmd(String key, String text) {this.key = key; this.text = text;}
|
||||
public String Key() {return key;} private String key;
|
||||
public String Text() {return text;} private String text;
|
||||
public String Exec(String_bldr sb, String wiki, String src) {
|
||||
String sect_txt_bgn = sb.Add("// ").Add(key).Add(".bgn\n").To_str_and_clear();
|
||||
String sect_txt_end = sb.Add("// ").Add(key).Add(".end\n").To_str_and_clear();
|
||||
String sect_txt_all = sb.Add(sect_txt_bgn).Add(text + "\n").Add(sect_txt_end).To_str_and_clear(); // NOTE: always add \n; convenience for single line cmds
|
||||
return src + sect_txt_all;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
public class Xoac_wiki_cfg_bldr_fil implements Gfo_invk {
|
||||
public Xoac_wiki_cfg_bldr_fil(String wiki) {this.wiki = wiki;}
|
||||
public String Wiki() {return wiki;} private String wiki;
|
||||
public int Itms_count() {return list.Count();}
|
||||
public Xoac_wiki_cfg_bldr_cmd Itms_get_at(int i) {return (Xoac_wiki_cfg_bldr_cmd)list.Get_at(i);}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk_new_cmd_)) {Itms_add(m.ReadStr("id"), m.ReadStr("text"));}
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
} private static final String Invk_new_cmd_ = "new_cmd_";
|
||||
public Xoac_wiki_cfg_bldr_cmd Itms_add(String key, String text) {
|
||||
Xoac_wiki_cfg_bldr_cmd rv = new Xoac_wiki_cfg_bldr_cmd(key, text);
|
||||
list.Add(rv);
|
||||
return rv;
|
||||
}
|
||||
List_adp list = List_adp_.New();
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import gplx.core.strings.*;
|
||||
import gplx.xowa.bldrs.*;
|
||||
public class Xob_wiki_cfg_bldr implements Gfo_invk {
|
||||
public Xob_wiki_cfg_bldr(Xob_bldr bldr) {this.app = bldr.App();} private Xoae_app app;
|
||||
public void Exec() {
|
||||
int len = hash.Count();
|
||||
for (int i = 0; i < len; i++) {
|
||||
Xoac_wiki_cfg_bldr_fil fil = (Xoac_wiki_cfg_bldr_fil)hash.Get_at(i);
|
||||
Exec_fil(fil);
|
||||
}
|
||||
}
|
||||
private void Exec_fil(Xoac_wiki_cfg_bldr_fil fil) {
|
||||
String wiki_key = fil.Wiki();
|
||||
Io_url cfg_file = app.Fsys_mgr().Cfg_wiki_core_dir().GenSubFil(wiki_key + ".gfs");
|
||||
String cfg_text = Io_mgr.Instance.LoadFilStr_args(cfg_file).MissingIgnored_().Exec();
|
||||
int len = fil.Itms_count();
|
||||
String_bldr sb = String_bldr_.new_();
|
||||
for (int i = 0; i < len; i++) {
|
||||
Xoac_wiki_cfg_bldr_cmd cmd = fil.Itms_get_at(i);
|
||||
cfg_text = cmd.Exec(sb, wiki_key, cfg_text);
|
||||
}
|
||||
Io_mgr.Instance.SaveFilStr(cfg_file, cfg_text);
|
||||
}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk_get)) return Itms_get_or_new(m.ReadStr("v"));
|
||||
else if (ctx.Match(k, Invk_run)) Exec();
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
} private static final String Invk_get = "get", Invk_run = "run";
|
||||
public void Clear() {hash.Clear();}
|
||||
public Xoac_wiki_cfg_bldr_fil Itms_get_or_new(String wiki) {
|
||||
Xoac_wiki_cfg_bldr_fil rv = (Xoac_wiki_cfg_bldr_fil)hash.Get_by(wiki);
|
||||
if (rv == null) {
|
||||
rv = new Xoac_wiki_cfg_bldr_fil(wiki);
|
||||
hash.Add(wiki, rv);
|
||||
}
|
||||
return rv;
|
||||
} private Ordered_hash hash = Ordered_hash_.New();
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
|
||||
import org.junit.*; import gplx.core.strings.*;
|
||||
public class Xob_wiki_cfg_bldr_tst {
|
||||
Xob_wiki_cfg_bldr_fxt fxt = new Xob_wiki_cfg_bldr_fxt();
|
||||
@Before public void init() {fxt.Clear();}
|
||||
@Test public void Exec() {
|
||||
fxt .Init_cmd("en.wikipedia.org", "key0", "en.val0")
|
||||
.Init_cmd("en.wikipedia.org", "key1", "en.val1")
|
||||
.Init_cmd("fr.wikipedia.org", "key0", "fr.val0")
|
||||
.Init_cmd("fr.wikipedia.org", "key1", "fr.val1")
|
||||
.Expd_txt("en.wikipedia.org", String_.Concat_lines_nl
|
||||
( "// key0.bgn"
|
||||
, "en.val0"
|
||||
, "// key0.end"
|
||||
, "// key1.bgn"
|
||||
, "en.val1"
|
||||
, "// key1.end"
|
||||
))
|
||||
.Expd_txt("fr.wikipedia.org", String_.Concat_lines_nl
|
||||
( "// key0.bgn"
|
||||
, "fr.val0"
|
||||
, "// key0.end"
|
||||
, "// key1.bgn"
|
||||
, "fr.val1"
|
||||
, "// key1.end"
|
||||
))
|
||||
.Test()
|
||||
;
|
||||
fxt .Clear()
|
||||
.Init_cmd("en.wikipedia.org", "key2", "en.val2")
|
||||
.Expd_txt("en.wikipedia.org", String_.Concat_lines_nl
|
||||
( "// key0.bgn"
|
||||
, "en.val0"
|
||||
, "// key0.end"
|
||||
, "// key1.bgn"
|
||||
, "en.val1"
|
||||
, "// key1.end"
|
||||
, "// key2.bgn"
|
||||
, "en.val2"
|
||||
, "// key2.end"
|
||||
));
|
||||
}
|
||||
// @Test public void Lang_names_run() {
|
||||
// Io_url dir = Io_url_.new_dir_("/var/www/mediawiki/languages/messages/");
|
||||
// Io_url[] fils = Io_mgr.Instance.QueryDir_args(dir).ExecAsUrlAry();
|
||||
// int fils_len = fils.length;
|
||||
// String_bldr sb = String_bldr_.new_();
|
||||
// for (int i = 0; i < fils_len; i++) {
|
||||
// Io_url fil = fils[i];
|
||||
// String lang_code = String_.Lower(String_.Replace(fil.NameOnly(), "Messages", ""));
|
||||
// String txt = Io_mgr.Instance.LoadFilStr(fil);
|
||||
// String[] lines = String_.Split(txt, '\n');
|
||||
// String line = lines[1];
|
||||
// line = String_.Replace(line, "/** ", "");
|
||||
// int pos = String_.FindBwd(line, " (");
|
||||
// if (pos == String_.Find_none) continue; // en; en_rtl have no "language" line
|
||||
// if ( String_.Has_at_bgn(lang_code, "be_")
|
||||
// || String_.Has_at_bgn(lang_code, "crh_")
|
||||
// || String_.Has_at_bgn(lang_code, "kk_")
|
||||
// || String_.Has_at_bgn(lang_code, "ku_")
|
||||
// || String_.Has_at_bgn(lang_code, "sr_")
|
||||
// || String_.In(lang_code, "de_formal", "nb", "nl_informal", "nn", "no")
|
||||
// ) {
|
||||
// int new_pos = String_.FindBwd(line, " (", pos - 1);
|
||||
// if (new_pos != String_.Find_none)
|
||||
// pos = new_pos;
|
||||
// }
|
||||
// line = Replace_by_pos(line, pos, pos + 2, "|");
|
||||
// int line_len = String_.Len(line);
|
||||
// if (line_len > 0)
|
||||
// line = String_.MidByLen(line, 0, line_len - 1);
|
||||
// String[] terms = String_.Split(line, '|');
|
||||
// sb.Add(lang_code).Add("|").Add(String_.Trim(terms[0])).Add("|").Add(String_.Trim(terms[1])).Add("\n");
|
||||
// }
|
||||
// Tfds.Dbg(sb.To_str_and_clear());
|
||||
// }
|
||||
@Test public void Ns_aliases() {
|
||||
Io_mgr.Instance.InitEngine_mem();
|
||||
Io_mgr.Instance.SaveFilStr("mem/en.wikipedia.org/w/api.php?action=query&format=xml&meta=siteinfo&siprop=namespacealiases", String_.Concat_lines_nl
|
||||
( "<api>"
|
||||
, "<query>"
|
||||
, "<namespacealiases>"
|
||||
, "<ns id=\"4\" xml:space=\"preserve\">WP"
|
||||
, "</ns>"
|
||||
, "<ns id=\"5\" xml:space=\"preserve\">WT"
|
||||
, "</ns>"
|
||||
, "<ns id=\"6\" xml:space=\"preserve\">Image"
|
||||
, "</ns>"
|
||||
, "<ns id=\"7\" xml:space=\"preserve\">Image talk"
|
||||
, "</ns>"
|
||||
, "</namespacealiases>"
|
||||
, "</query>"
|
||||
, "</api>"
|
||||
));
|
||||
String[] wikis = new String[] {"en.wikipedia.org"}; String protocol = "mem/";
|
||||
Tfds.Eq_str_lines(Query_ns(protocol, gplx.core.ios.IoEngine_.MemKey, wikis), String_.Concat_lines_nl
|
||||
( "app.bldr.wiki_cfg_bldr.get('en.wikipedia.org').new_cmd_('wiki.ns_mgr.aliases', 'ns_mgr.add_alias_bulk(\""
|
||||
, "4|WP"
|
||||
, "5|WT"
|
||||
, "6|Image"
|
||||
, "7|Image_talk"
|
||||
, "\");');"
|
||||
));
|
||||
}
|
||||
String Query_ns(String protocol, String trg_engine_key, String[] wikis) {
|
||||
String_bldr sb = String_bldr_.new_();
|
||||
int wikis_len = wikis.length;
|
||||
for (int i = 0; i < wikis_len; i++) {
|
||||
String wiki = wikis[i];
|
||||
if (String_.Len_eq_0(wiki)) continue;
|
||||
try {
|
||||
String api = protocol + wiki + "/w/api.php?action=query&format=xml&meta=siteinfo&siprop=namespacealiases";
|
||||
String xml = String_.new_u8(Io_mgr.Instance.DownloadFil_args("", null).Trg_engine_key_(trg_engine_key).Exec_as_bry(api));
|
||||
if (xml == null) continue; // not found
|
||||
gplx.langs.xmls.XmlDoc xdoc = gplx.langs.xmls.XmlDoc_.parse(xml);
|
||||
gplx.langs.xmls.XmlNde xnde = gplx.langs.xmls.Xpath_.SelectFirst(xdoc.Root(), "query/namespacealiases");
|
||||
sb.Add("app.bldr.wiki_cfg_bldr.get('").Add(wiki).Add("').new_cmd_('wiki.ns_mgr.aliases', 'ns_mgr.add_alias_bulk(\"\n");
|
||||
int xndes_len = xnde.SubNdes().Count();
|
||||
for (int j = 0; j < xndes_len; j++) {
|
||||
gplx.langs.xmls.XmlNde ns_nde = xnde.SubNdes().Get_at(j);
|
||||
if (!String_.Eq(ns_nde.Name(), "ns")) continue;
|
||||
int id = Int_.parse(ns_nde.Atrs().FetchValOr("id", "-1"));
|
||||
String name = String_.Replace(String_.Replace(ns_nde.Text_inner(), " ", "_"), "'", "''");
|
||||
sb.Add(Int_.To_str(id)).Add("|").Add(String_.Trim(name)).Add_char_nl();
|
||||
}
|
||||
sb.Add("\");');\n");
|
||||
}
|
||||
catch(Exception e) {sb.Add("// fail: " + wiki + " " + Err_.Message_gplx_full(e)).Add_char_nl();}
|
||||
}
|
||||
return sb.To_str_and_clear();
|
||||
}
|
||||
}
|
||||
class Xob_wiki_cfg_bldr_fxt {
|
||||
public Xob_wiki_cfg_bldr_fxt Clear() {
|
||||
if (app == null) {
|
||||
app = Xoa_app_fxt.Make__app__edit();
|
||||
wiki_cfg_bldr = app.Bldr().Wiki_cfg_bldr();
|
||||
}
|
||||
wiki_cfg_bldr.Clear();
|
||||
hash.Clear();
|
||||
return this;
|
||||
} private Xoae_app app; Xob_wiki_cfg_bldr wiki_cfg_bldr; Ordered_hash hash = Ordered_hash_.New();
|
||||
public Xob_wiki_cfg_bldr_fxt Init_cmd(String wiki, String key, String text) {
|
||||
wiki_cfg_bldr.Itms_get_or_new(wiki).Itms_add(key, text);
|
||||
return this;
|
||||
}
|
||||
public Xob_wiki_cfg_bldr_fxt Expd_txt(String wiki, String text) {
|
||||
hash.Add(wiki, Keyval_.new_(wiki, text));
|
||||
return this;
|
||||
}
|
||||
public void Test() {
|
||||
wiki_cfg_bldr.Exec();
|
||||
int len = hash.Count();
|
||||
for (int i = 0; i < len; i++) {
|
||||
Keyval kv = (Keyval)hash.Get_at(i);
|
||||
String wiki = kv.Key();
|
||||
String expd = (String)kv.Val();
|
||||
String actl = Io_mgr.Instance.LoadFilStr(app.Fsys_mgr().Cfg_wiki_core_dir().GenSubFil(wiki + ".gfs"));
|
||||
Tfds.Eq_str_lines(expd, actl);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs.wm_server_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.app_cfgs.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xowm_server_cfg_cmd extends Xob_cmd__base {
|
||||
public Xowm_server_cfg_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
|
||||
@Override public void Cmd_run() {
|
||||
// wiki.Init_assert();
|
||||
// mgr.Exec(wiki);
|
||||
}
|
||||
|
||||
public static final String BLDR_CMD_KEY = "cfg.wikis.wm_server_cfg";
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
|
||||
public static final Xob_cmd Prototype = new Xowm_server_cfg_cmd(null, null);
|
||||
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xowm_server_cfg_cmd(bldr, wiki);}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.app_cfgs.wm_server_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.app_cfgs.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xowm_server_cfg_mgr {
|
||||
public void Exec() {
|
||||
byte[] src = Load_or_download();
|
||||
Parse_cat_collation(src);
|
||||
}
|
||||
private byte[] Load_or_download() {
|
||||
// Io_mgr.Instance.DownloadFil_args("", Io_url_.NullPtr).Exec_as_bry("https://noc.wikimedia.org/conf/InitialiseSettings.php.txt");
|
||||
return null;
|
||||
}
|
||||
private void Parse_cat_collation(byte[] src) {
|
||||
int bgn_pos = Bry_find_.Find_fwd(src, Bry_.new_a7("wgCategoryCollation"));
|
||||
if (bgn_pos == Bry_find_.Not_found) throw Err_.new_wo_type("could not find wgCategoryCollation bgn");
|
||||
int end_pos = Bry_find_.Find_fwd(src, Bry_.new_a7("],"));
|
||||
if (end_pos == Bry_find_.Not_found) throw Err_.new_wo_type("could not find wgCategoryCollation end");
|
||||
Tfds.Write(src, bgn_pos, end_pos);
|
||||
}
|
||||
}
|
||||
@@ -54,4 +54,8 @@ public class Xobc_data_db {
|
||||
tbl__import_step.Delete(step_id);
|
||||
}
|
||||
}
|
||||
|
||||
public static Xobc_data_db New(gplx.xowa.apps.fsys.Xoa_fsys_mgr fsys_mgr) {
|
||||
return new Xobc_data_db(fsys_mgr.Bin_addon_dir().GenSubFil_nest("bldr", "central", "bldr_central.data_db.xowa"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
|
||||
public class Xobc_task_regy_itm {
|
||||
public Xobc_task_regy_itm(int id, int seqn, byte[] key, byte[] name, int step_count) {
|
||||
this.id = id;
|
||||
this.seqn = seqn;
|
||||
this.key = key;
|
||||
this.name = name;
|
||||
this.step_count = step_count;
|
||||
}
|
||||
public int Id() {return id;} private final int id;
|
||||
public int Seqn() {return seqn;} private final int seqn;
|
||||
public byte[] Key() {return key;} private final byte[] key;
|
||||
public byte[] Name() {return name;} private final byte[] name;
|
||||
public int Step_count() {return step_count;} private final int step_count;
|
||||
}
|
||||
@@ -67,6 +67,28 @@ public class Xobc_task_regy_tbl implements Db_tbl {
|
||||
.Val_str(fld_task_key, task_key).Val_str(fld_task_name, task_name)
|
||||
.Exec_insert();
|
||||
}
|
||||
public Xobc_task_regy_itm[] Select_by_wiki(byte[] wiki_domain) {
|
||||
String sql = Db_sql_.Make_by_fmt(String_.Ary
|
||||
( "SELECT *"
|
||||
, "FROM task_regy"
|
||||
, "WHERE task_key LIKE '{0}%'"
|
||||
, "AND task_seqn != 999999"
|
||||
), wiki_domain);
|
||||
List_adp list = List_adp_.New();
|
||||
Db_rdr rdr = conn.Stmt_sql(sql).Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
list.Add(new Xobc_task_regy_itm
|
||||
( rdr.Read_int("task_id")
|
||||
, rdr.Read_int("task_seqn")
|
||||
, rdr.Read_bry_by_str("task_key")
|
||||
, rdr.Read_bry_by_str("task_name")
|
||||
, rdr.Read_int("step_count")
|
||||
));
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
return (Xobc_task_regy_itm[])list.To_ary_and_clear(Xobc_task_regy_itm.class);
|
||||
}
|
||||
public void Delete(int task_id) {
|
||||
conn.Stmt_delete(tbl_name, fld_task_id).Crt_int(fld_task_id, task_id).Exec_delete();
|
||||
}
|
||||
|
||||
@@ -44,20 +44,7 @@ public class Xobc_import_step_tbl implements Db_tbl {
|
||||
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_step_id).Crt_int(fld_step_id, step_id).Exec_select__rls_auto();
|
||||
try {
|
||||
return (rdr.Move_next())
|
||||
? new Xobc_import_step_itm
|
||||
( rdr.Read_int(fld_step_id)
|
||||
, rdr.Read_int(fld_host_id)
|
||||
, rdr.Read_bry_by_str(fld_wiki_abrv)
|
||||
, rdr.Read_str(fld_wiki_date)
|
||||
, rdr.Read_str(fld_import_name)
|
||||
, rdr.Read_int(fld_import_type)
|
||||
, rdr.Read_byte(fld_import_zip)
|
||||
, rdr.Read_long(fld_import_size_zip)
|
||||
, rdr.Read_long(fld_import_size_raw)
|
||||
, rdr.Read_str(fld_import_md5)
|
||||
, rdr.Read_long(fld_prog_size_end)
|
||||
, rdr.Read_int(fld_prog_count_end)
|
||||
)
|
||||
? New_itm(rdr)
|
||||
: Xobc_import_step_itm.Null;
|
||||
}
|
||||
finally {rdr.Rls();}
|
||||
@@ -95,7 +82,39 @@ public class Xobc_import_step_tbl implements Db_tbl {
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
}
|
||||
public Xobc_import_step_itm[] Select_by_task_id(int task_id) {
|
||||
List_adp list = List_adp_.New();
|
||||
Db_rdr rdr = conn.Stmt_sql(Db_sql_.Make_by_fmt(String_.Ary
|
||||
( "SELECT s.*"
|
||||
, "FROM import_step s"
|
||||
, " JOIN step_map sm ON s.step_id = sm.step_id"
|
||||
, "WHERE sm.task_id = {0}"
|
||||
), task_id))
|
||||
.Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
list.Add(New_itm(rdr));
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
return (Xobc_import_step_itm[])list.To_ary_and_clear(Xobc_import_step_itm.class);
|
||||
}
|
||||
public void Rls() {
|
||||
insert_stmt = Db_stmt_.Rls(insert_stmt);
|
||||
}
|
||||
private Xobc_import_step_itm New_itm(Db_rdr rdr) {
|
||||
return new Xobc_import_step_itm
|
||||
( rdr.Read_int(fld_step_id)
|
||||
, rdr.Read_int(fld_host_id)
|
||||
, rdr.Read_bry_by_str(fld_wiki_abrv)
|
||||
, rdr.Read_str(fld_wiki_date)
|
||||
, rdr.Read_str(fld_import_name)
|
||||
, rdr.Read_int(fld_import_type)
|
||||
, rdr.Read_byte(fld_import_zip)
|
||||
, rdr.Read_long(fld_import_size_zip)
|
||||
, rdr.Read_long(fld_import_size_raw)
|
||||
, rdr.Read_str(fld_import_md5)
|
||||
, rdr.Read_long(fld_prog_size_end)
|
||||
, rdr.Read_int(fld_prog_count_end)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.x
|
||||
import gplx.core.progs.*; import gplx.core.ios.zips.*; import gplx.core.ios.streams.*; import gplx.core.security.*;
|
||||
import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.fsdb.*; import gplx.fsdb.meta.*;
|
||||
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
|
||||
class Pack_file_mgr {
|
||||
public class Pack_file_mgr {
|
||||
public void Exec(Xowe_wiki wiki, Pack_file_bldr_cfg cfg) {
|
||||
// init
|
||||
wiki.Init_assert();
|
||||
@@ -31,7 +31,7 @@ class Pack_file_mgr {
|
||||
|
||||
// get import_tbl
|
||||
byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo();
|
||||
Xobc_data_db bc_db = new Xobc_data_db(wiki.App().Fsys_mgr().Bin_addon_dir().GenSubFil_nest("bldr", "central", "bldr_central.data_db.xowa"));
|
||||
Xobc_data_db bc_db = Xobc_data_db.New(wiki.App().Fsys_mgr());
|
||||
Db_conn bc_conn = bc_db.Conn();
|
||||
bc_db.Delete_by_import(wiki_abrv, wiki_date);
|
||||
bc_conn.Txn_bgn("xobc_import_insert");
|
||||
@@ -93,7 +93,7 @@ class Pack_file_mgr {
|
||||
int pack_list_len = pack_list.Len();
|
||||
|
||||
// create task
|
||||
String task_key = wiki.Domain_str() + "|" + wiki_date + "|" + task_type;
|
||||
String task_key = Task_key__build(wiki.Domain_str(), wiki_date, task_type);
|
||||
String task_name = Build_task_name(tmp_bfr, wiki, wiki_date, task_type, raw_len);
|
||||
Xobc_task_regy_tbl task_regy_tbl = bc_db.Tbl__task_regy();
|
||||
int task_id = bc_db.Conn().Sys_mgr().Autonum_next("task_regy.task_id");
|
||||
@@ -161,6 +161,12 @@ class Pack_file_mgr {
|
||||
bc_db.Tbl__step_regy().Insert(step_id, Xobc_step_itm.Type__wiki_import);
|
||||
bc_db.Tbl__import_step().Insert(step_id, gplx.xowa.addons.bldrs.centrals.dbs.datas.Xobc_host_regy_tbl.Host_id__archive_org, wiki_abrv, wiki_date, zip_url.NameAndExt(), itm.Tid(), Xobc_zip_type.Type__zip, zip_md5, zip_len, raw_size, 0, 0);
|
||||
}
|
||||
private static String Task_key__build(String wiki_domain, String wiki_date, String task_type) {
|
||||
return String_.Concat(wiki_domain, "|", wiki_date, "|", task_type);
|
||||
}
|
||||
public static String[] Task_key__parse(String task_key) {
|
||||
return String_.Split(task_key, "|");
|
||||
}
|
||||
}
|
||||
class Pack_hash_bldr {
|
||||
public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff, boolean pack_fsdb_delete) {
|
||||
|
||||
@@ -33,7 +33,7 @@ class Pack_mgr {
|
||||
|
||||
// get import_tbl
|
||||
byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo();
|
||||
Xobc_data_db bc_db = new Xobc_data_db(wiki.App().Fsys_mgr().Bin_addon_dir().GenSubFil_nest("bldr", "central", "bldr_central.data_db.xowa"));
|
||||
Xobc_data_db bc_db = Xobc_data_db.New(wiki.App().Fsys_mgr());
|
||||
Db_conn bc_conn = bc_db.Conn();
|
||||
bc_db.Delete_by_import(wiki_abrv, wiki_date);
|
||||
bc_conn.Txn_bgn("xobc_import_insert");
|
||||
|
||||
@@ -19,6 +19,8 @@ package gplx.xowa.addons.bldrs.files; import gplx.*; import gplx.xowa.*; import
|
||||
import gplx.xowa.bldrs.wkrs.*;
|
||||
import gplx.xowa.addons.bldrs.files.cmds.*;
|
||||
import gplx.xowa.addons.bldrs.mass_parses.inits.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.makes.*;
|
||||
import gplx.xowa.addons.bldrs.files.cksums.*;
|
||||
import gplx.xowa.addons.bldrs.app_cfgs.wm_server_cfgs.*;
|
||||
public class Xoax_builds_files_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr {
|
||||
public Xob_cmd[] Bldr_cmds() {
|
||||
return new Xob_cmd[]
|
||||
@@ -43,6 +45,9 @@ public class Xoax_builds_files_addon implements Xoax_addon_itm, Xoax_addon_itm__
|
||||
, Xomp_init_cmd.Prototype
|
||||
, Xomp_parse_cmd.Prototype
|
||||
, Xomp_make_cmd.Prototype
|
||||
, Xocksum_calc_cmd.Prototype
|
||||
|
||||
, Xowm_server_cfg_cmd.Prototype
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.files.cksums; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xocksum_calc_cmd extends Xob_cmd__base {
|
||||
public Xocksum_calc_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
|
||||
@Override public void Cmd_run() {
|
||||
wiki.Init_assert();
|
||||
new Xocksum_calc_mgr().Exec(wiki);
|
||||
}
|
||||
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;} private static final String BLDR_CMD_KEY = "fsdb.cksums.calc";
|
||||
public static final Xob_cmd Prototype = new Xocksum_calc_cmd(null, null);
|
||||
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xocksum_calc_cmd(bldr, wiki);}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.files.cksums; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*;
|
||||
import gplx.core.ios.streams.*; import gplx.core.security.*;
|
||||
import gplx.dbs.*; import gplx.xowa.addons.bldrs.files.cksums.dbs.*;
|
||||
import gplx.xowa.files.*; import gplx.fsdb.*; import gplx.fsdb.data.*;
|
||||
public class Xocksum_calc_mgr {
|
||||
public void Exec(Xowe_wiki wiki) {
|
||||
// get conn variables
|
||||
Xocksum_cksum_db db = Xocksum_cksum_db.Get(wiki);
|
||||
Db_conn conn = db.Conn();
|
||||
Xocksum_cksum_tbl tbl = db.Tbl__cksum();
|
||||
conn.Meta_tbl_assert(tbl);
|
||||
|
||||
// insert missing items
|
||||
tbl.Insert_missing();
|
||||
tbl.Create_idx();
|
||||
|
||||
// get updates
|
||||
int count = 0;
|
||||
Hash_algo md5_algo = Hash_algo_.New__md5();
|
||||
List_adp updates = List_adp_.New();
|
||||
String cur_date = Datetime_now.Get().XtoStr_gplx();
|
||||
Db_stmt select_stmt = tbl.Select_samples_stmt(10000);
|
||||
while (true) {
|
||||
// get cksum_rows
|
||||
Xocksum_cksum_row[] rows = tbl.Select_samples(select_stmt);
|
||||
|
||||
// loop cksum_rows and (a) get bin_data; (b) if md5 diff, then add to updates
|
||||
int len = rows.length; if (len == 0) break;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xocksum_cksum_row row = rows[i];
|
||||
byte[] bin_bry = Get_bin(wiki, row);
|
||||
if (bin_bry == null) {
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "null; fil_id=~{0} thm_id=~{1}", row.Fil_id(), row.Thm_id());
|
||||
bin_bry = Bry_.Empty;
|
||||
}
|
||||
row.Bin_size_(bin_bry.length);
|
||||
byte[] md5 = md5_algo.Hash_bry_as_bry(bin_bry);
|
||||
if (!Bry_.Eq(md5, row.Cksum_val())) {
|
||||
row.Cksum_val_(md5);
|
||||
updates.Add(row);
|
||||
}
|
||||
}
|
||||
|
||||
// run updates
|
||||
conn.Txn_bgn("cksum_update");
|
||||
len = updates.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xocksum_cksum_row row = (Xocksum_cksum_row)updates.Get_at(i);
|
||||
tbl.Update(row.Fil_id(), row.Thm_id(), row.Bin_db_id(), row.Bin_size(), row.Cksum_tid(), 0, row.Cksum_val(), cur_date);
|
||||
if (++count % 2000 == 0) Gfo_usr_dlg_.Instance.Prog_many("", "", "updating; rows=~{0}", count);
|
||||
}
|
||||
updates.Clear();
|
||||
conn.Txn_end();
|
||||
}
|
||||
select_stmt.Rls();
|
||||
}
|
||||
private byte[] Get_bin(Xowe_wiki wiki, Xocksum_cksum_row row) {
|
||||
int bin_id = row.Thm_id() == -1 ? row.Fil_id() : row.Thm_id();
|
||||
Fsd_bin_itm bin_itm = wiki.File__mnt_mgr().Mnts__get_main().Bin_mgr().Dbs__get_at(row.Bin_db_id()).Select_as_itm(bin_id);
|
||||
return bin_itm.Bin_data();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.files.cksums.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.addons.bldrs.files.cksums.*;
|
||||
import gplx.dbs.*; import gplx.fsdb.meta.*;
|
||||
public class Xocksum_cksum_db {
|
||||
public Xocksum_cksum_db(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl__cksum = new Xocksum_cksum_tbl(conn);
|
||||
}
|
||||
public Db_conn Conn() {return conn;} private final Db_conn conn;
|
||||
public Xocksum_cksum_tbl Tbl__cksum() {return tbl__cksum;} private final Xocksum_cksum_tbl tbl__cksum;
|
||||
|
||||
public static Xocksum_cksum_db Get(Xowe_wiki wiki) {
|
||||
return new Xocksum_cksum_db(wiki.File__fsdb_core().File__abc_file__at(Fsm_mnt_mgr.Mnt_idx_main).Conn());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.files.cksums.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.addons.bldrs.files.cksums.*;
|
||||
public class Xocksum_cksum_row {
|
||||
public Xocksum_cksum_row(int fil_id, int thm_id, int bin_db_id, long bin_size, byte cksum_tid, int cksum_count, byte[] cksum_val, String cksum_date) {
|
||||
this.fil_id = fil_id;
|
||||
this.thm_id = thm_id;
|
||||
this.bin_db_id = bin_db_id;
|
||||
this.bin_size = bin_size;
|
||||
this.cksum_tid = cksum_tid;
|
||||
this.cksum_count = cksum_count;
|
||||
this.cksum_val = cksum_val;
|
||||
this.cksum_date = cksum_date;
|
||||
}
|
||||
public int Fil_id() {return fil_id;} private final int fil_id;
|
||||
public int Thm_id() {return thm_id;} private final int thm_id;
|
||||
public int Bin_db_id() {return bin_db_id;} private final int bin_db_id;
|
||||
public long Bin_size() {return bin_size;} private long bin_size;
|
||||
public byte Cksum_tid() {return cksum_tid;} private final byte cksum_tid;
|
||||
public int Cksum_count() {return cksum_count;} private final int cksum_count;
|
||||
public byte[] Cksum_val() {return cksum_val;} private byte[] cksum_val;
|
||||
public String Cksum_date() {return cksum_date;} private final String cksum_date;
|
||||
|
||||
public void Bin_size_(long v) {this.bin_size = v;}
|
||||
public void Cksum_val_(byte[] v) {this.cksum_val = v;}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.files.cksums.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.addons.bldrs.files.cksums.*;
|
||||
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.addons.wikis.ctgs.*;
|
||||
public class Xocksum_cksum_tbl implements Db_tbl {
|
||||
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
private final String fld__fil_id, fld__thm_id, fld__bin_db_id, fld__bin_len, fld__cksum_tid, fld__cksum_count, fld__cksum_val, fld__cksum_date;
|
||||
private Db_stmt stmt__update;
|
||||
public Xocksum_cksum_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "fsdb_cksum";
|
||||
this.fld__fil_id = flds.Add_int("fil_id");
|
||||
this.fld__thm_id = flds.Add_int("thm_id");
|
||||
this.fld__bin_db_id = flds.Add_int("bin_db_id");
|
||||
this.fld__bin_len = flds.Add_long("bin_size");
|
||||
this.fld__cksum_tid = flds.Add_byte("cksum_tid");
|
||||
this.fld__cksum_count = flds.Add_int("cksum_count");
|
||||
this.fld__cksum_val = flds.Add_str("cksum_val", 255);
|
||||
this.fld__cksum_date = flds.Add_str("cksum_date", 16);
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public Db_conn Conn() {return conn;} private final Db_conn conn;
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
|
||||
public void Create_idx() {
|
||||
conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "fil_id__thm_id", fld__fil_id, fld__thm_id));
|
||||
conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, fld__cksum_val, fld__cksum_val));
|
||||
}
|
||||
public void Insert_missing() {
|
||||
// insert from fsdb_fil
|
||||
conn.Exec_sql(Db_sql_.Make_by_fmt(String_.Ary
|
||||
( "INSERT INTO fsdb_cksum (fil_id, thm_id, bin_db_id, bin_size, cksum_tid, cksum_count, cksum_val, cksum_date)"
|
||||
, "SELECT f.fil_id, -1, f.fil_bin_db_id, f.fil_size, {0}, 0, '', ''"
|
||||
, "FROM fsdb_fil f"
|
||||
, " LEFT JOIN fsdb_cksum c ON c.fil_id = f.fil_id AND c.thm_id = -1"
|
||||
, "WHERE c.fil_id IS NULL"
|
||||
, "AND f.fil_bin_db_id != -1"
|
||||
), Cksum_tid__md5));
|
||||
|
||||
// insert from fsdb_fil
|
||||
conn.Exec_sql(Db_sql_.Make_by_fmt(String_.Ary
|
||||
( "INSERT INTO fsdb_cksum (fil_id, thm_id, bin_db_id, bin_size, cksum_tid, cksum_count, cksum_val, cksum_date)"
|
||||
, "SELECT t.thm_owner_id, t.thm_id, t.thm_bin_db_id, t.thm_size, {0}, 0, '', ''"
|
||||
, "FROM fsdb_thm t"
|
||||
, " LEFT JOIN fsdb_cksum c ON c.fil_id = t.thm_owner_id AND c.thm_id = t.thm_id"
|
||||
, "WHERE c.fil_id IS NULL"
|
||||
), Cksum_tid__md5));
|
||||
}
|
||||
public Db_stmt Select_samples_stmt(int count) {
|
||||
return conn.Stmt_sql(Db_sql_.Make_by_fmt(String_.Ary
|
||||
( "SELECT *"
|
||||
, "FROM fsdb_cksum"
|
||||
, "WHERE cksum_val = ''"
|
||||
// , "ORDER BY cksum_count, cksum_date"
|
||||
, "LIMIT {0}"
|
||||
), count));
|
||||
}
|
||||
public Xocksum_cksum_row[] Select_samples(Db_stmt stmt) {
|
||||
List_adp rv = List_adp_.New();
|
||||
|
||||
Db_rdr rdr = stmt.Exec_select__rls_manual();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
rv.Add(new Xocksum_cksum_row
|
||||
( rdr.Read_int("fil_id")
|
||||
, rdr.Read_int("thm_id")
|
||||
, rdr.Read_int("bin_db_id")
|
||||
, rdr.Read_long("bin_size")
|
||||
, rdr.Read_byte("cksum_tid")
|
||||
, rdr.Read_int("cksum_count")
|
||||
, rdr.Read_bry_by_str("cksum_val")
|
||||
, rdr.Read_str("cksum_date")
|
||||
));
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
|
||||
return (Xocksum_cksum_row[])rv.To_ary_and_clear(Xocksum_cksum_row.class);
|
||||
}
|
||||
public void Update(int fil_id, int thm_id, int bin_db_id, long bin_size, byte cksum_tid, int cksum_count, byte[] cksum_val, String cksum_date) {
|
||||
if (stmt__update == null) stmt__update = conn.Stmt_update_exclude(tbl_name, flds, fld__fil_id, fld__thm_id);
|
||||
stmt__update.Clear()
|
||||
.Val_int(fld__bin_db_id, bin_db_id).Val_long(fld__bin_len, bin_size)
|
||||
.Val_byte(fld__cksum_tid, cksum_tid).Val_int(fld__cksum_count, cksum_count)
|
||||
.Val_bry_as_str(fld__cksum_val, cksum_val).Val_str(fld__cksum_date, cksum_date)
|
||||
.Crt_int(fld__fil_id, fil_id).Crt_int(fld__thm_id, thm_id)
|
||||
.Exec_update();
|
||||
}
|
||||
public void Rls() {
|
||||
this.stmt__update = Db_stmt_.Rls(stmt__update);
|
||||
}
|
||||
public static final byte Cksum_tid__md5 = 1;
|
||||
}
|
||||
@@ -39,7 +39,7 @@ class Hdump_html_loader {
|
||||
html_db.Tbl__html().Select_by_page(tmp_hpg);
|
||||
|
||||
// unzip it
|
||||
byte[] html_hzip = stream_zip_mgr.Unzip(Io_stream_.Tid_gzip, tmp_hpg.Db().Html().Html_bry());
|
||||
byte[] html_hzip = stream_zip_mgr.Unzip(Io_stream_tid_.Tid__gzip, tmp_hpg.Db().Html().Html_bry());
|
||||
return wiki.Html__hdump_mgr().Load_mgr().Decode_as_bry(tmp_bfr, tmp_hpg, html_hzip, true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
|
||||
public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true;
|
||||
public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true;
|
||||
public boolean Load_all_imglnks() {return load_all_imglnks;} private boolean load_all_imglnks = true;
|
||||
public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_.Tid_gzip;
|
||||
public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_tid_.Tid__gzip;
|
||||
public Io_url Mgr_url() {return mgr_url;} private Io_url mgr_url;
|
||||
public String Wkr_machine_name() {return wkr_machine_name;} private String wkr_machine_name;
|
||||
public boolean Show_msg__fetched_pool() {return show_msg__fetched_pool;} private boolean show_msg__fetched_pool;
|
||||
|
||||
@@ -32,7 +32,7 @@ class Xodel_make_mgr {
|
||||
, "FROM xfer_regy xr"
|
||||
, "WHERE xr.lnki_ttl = fsdb_regy.fsdb_name"
|
||||
, "AND xr.file_is_orig = fsdb_regy.fsdb_is_orig"
|
||||
, "AND xr.orig_repo = fsdb_regy.fsdb_repo"
|
||||
// , "AND xr.orig_repo = fsdb_regy.fsdb_repo" // TOMBSTONE: do no reinstate; some images exist in both repos, and this will delete images from one repo; DATE:2016-09-28
|
||||
, "AND xr.file_w = fsdb_regy.fsdb_w"
|
||||
, "AND xr.lnki_time = fsdb_regy.fsdb_time"
|
||||
, "AND xr.lnki_page = fsdb_regy.fsdb_page"
|
||||
|
||||
Reference in New Issue
Block a user