1
0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00

uca category support and other changes

This commit is contained in:
gnosygnu
2016-10-12 08:57:22 -04:00
parent e3b393650d
commit 3fc2e0741f
187 changed files with 3486 additions and 2984 deletions

View File

@@ -0,0 +1,30 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.core.strings.*;
public class Xoac_wiki_cfg_bldr_cmd {
public Xoac_wiki_cfg_bldr_cmd(String key, String text) {this.key = key; this.text = text;}
public String Key() {return key;} private String key;
public String Text() {return text;} private String text;
public String Exec(String_bldr sb, String wiki, String src) {
String sect_txt_bgn = sb.Add("// ").Add(key).Add(".bgn\n").To_str_and_clear();
String sect_txt_end = sb.Add("// ").Add(key).Add(".end\n").To_str_and_clear();
String sect_txt_all = sb.Add(sect_txt_bgn).Add(text + "\n").Add(sect_txt_end).To_str_and_clear(); // NOTE: always add \n; convenience for single line cmds
return src + sect_txt_all;
}
}

View File

@@ -0,0 +1,35 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
public class Xoac_wiki_cfg_bldr_fil implements Gfo_invk {
public Xoac_wiki_cfg_bldr_fil(String wiki) {this.wiki = wiki;}
public String Wiki() {return wiki;} private String wiki;
public int Itms_count() {return list.Count();}
public Xoac_wiki_cfg_bldr_cmd Itms_get_at(int i) {return (Xoac_wiki_cfg_bldr_cmd)list.Get_at(i);}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_new_cmd_)) {Itms_add(m.ReadStr("id"), m.ReadStr("text"));}
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk_new_cmd_ = "new_cmd_";
public Xoac_wiki_cfg_bldr_cmd Itms_add(String key, String text) {
Xoac_wiki_cfg_bldr_cmd rv = new Xoac_wiki_cfg_bldr_cmd(key, text);
list.Add(rv);
return rv;
}
List_adp list = List_adp_.New();
}

View File

@@ -0,0 +1,57 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.core.strings.*;
import gplx.xowa.bldrs.*;
public class Xob_wiki_cfg_bldr implements Gfo_invk {
public Xob_wiki_cfg_bldr(Xob_bldr bldr) {this.app = bldr.App();} private Xoae_app app;
public void Exec() {
int len = hash.Count();
for (int i = 0; i < len; i++) {
Xoac_wiki_cfg_bldr_fil fil = (Xoac_wiki_cfg_bldr_fil)hash.Get_at(i);
Exec_fil(fil);
}
}
private void Exec_fil(Xoac_wiki_cfg_bldr_fil fil) {
String wiki_key = fil.Wiki();
Io_url cfg_file = app.Fsys_mgr().Cfg_wiki_core_dir().GenSubFil(wiki_key + ".gfs");
String cfg_text = Io_mgr.Instance.LoadFilStr_args(cfg_file).MissingIgnored_().Exec();
int len = fil.Itms_count();
String_bldr sb = String_bldr_.new_();
for (int i = 0; i < len; i++) {
Xoac_wiki_cfg_bldr_cmd cmd = fil.Itms_get_at(i);
cfg_text = cmd.Exec(sb, wiki_key, cfg_text);
}
Io_mgr.Instance.SaveFilStr(cfg_file, cfg_text);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_get)) return Itms_get_or_new(m.ReadStr("v"));
else if (ctx.Match(k, Invk_run)) Exec();
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk_get = "get", Invk_run = "run";
public void Clear() {hash.Clear();}
public Xoac_wiki_cfg_bldr_fil Itms_get_or_new(String wiki) {
Xoac_wiki_cfg_bldr_fil rv = (Xoac_wiki_cfg_bldr_fil)hash.Get_by(wiki);
if (rv == null) {
rv = new Xoac_wiki_cfg_bldr_fil(wiki);
hash.Add(wiki, rv);
}
return rv;
} private Ordered_hash hash = Ordered_hash_.New();
}

View File

@@ -0,0 +1,179 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.app_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import org.junit.*; import gplx.core.strings.*;
public class Xob_wiki_cfg_bldr_tst {
Xob_wiki_cfg_bldr_fxt fxt = new Xob_wiki_cfg_bldr_fxt();
@Before public void init() {fxt.Clear();}
@Test public void Exec() {
fxt .Init_cmd("en.wikipedia.org", "key0", "en.val0")
.Init_cmd("en.wikipedia.org", "key1", "en.val1")
.Init_cmd("fr.wikipedia.org", "key0", "fr.val0")
.Init_cmd("fr.wikipedia.org", "key1", "fr.val1")
.Expd_txt("en.wikipedia.org", String_.Concat_lines_nl
( "// key0.bgn"
, "en.val0"
, "// key0.end"
, "// key1.bgn"
, "en.val1"
, "// key1.end"
))
.Expd_txt("fr.wikipedia.org", String_.Concat_lines_nl
( "// key0.bgn"
, "fr.val0"
, "// key0.end"
, "// key1.bgn"
, "fr.val1"
, "// key1.end"
))
.Test()
;
fxt .Clear()
.Init_cmd("en.wikipedia.org", "key2", "en.val2")
.Expd_txt("en.wikipedia.org", String_.Concat_lines_nl
( "// key0.bgn"
, "en.val0"
, "// key0.end"
, "// key1.bgn"
, "en.val1"
, "// key1.end"
, "// key2.bgn"
, "en.val2"
, "// key2.end"
));
}
// @Test public void Lang_names_run() {
// Io_url dir = Io_url_.new_dir_("/var/www/mediawiki/languages/messages/");
// Io_url[] fils = Io_mgr.Instance.QueryDir_args(dir).ExecAsUrlAry();
// int fils_len = fils.length;
// String_bldr sb = String_bldr_.new_();
// for (int i = 0; i < fils_len; i++) {
// Io_url fil = fils[i];
// String lang_code = String_.Lower(String_.Replace(fil.NameOnly(), "Messages", ""));
// String txt = Io_mgr.Instance.LoadFilStr(fil);
// String[] lines = String_.Split(txt, '\n');
// String line = lines[1];
// line = String_.Replace(line, "/** ", "");
// int pos = String_.FindBwd(line, " (");
// if (pos == String_.Find_none) continue; // en; en_rtl have no "language" line
// if ( String_.Has_at_bgn(lang_code, "be_")
// || String_.Has_at_bgn(lang_code, "crh_")
// || String_.Has_at_bgn(lang_code, "kk_")
// || String_.Has_at_bgn(lang_code, "ku_")
// || String_.Has_at_bgn(lang_code, "sr_")
// || String_.In(lang_code, "de_formal", "nb", "nl_informal", "nn", "no")
// ) {
// int new_pos = String_.FindBwd(line, " (", pos - 1);
// if (new_pos != String_.Find_none)
// pos = new_pos;
// }
// line = Replace_by_pos(line, pos, pos + 2, "|");
// int line_len = String_.Len(line);
// if (line_len > 0)
// line = String_.MidByLen(line, 0, line_len - 1);
// String[] terms = String_.Split(line, '|');
// sb.Add(lang_code).Add("|").Add(String_.Trim(terms[0])).Add("|").Add(String_.Trim(terms[1])).Add("\n");
// }
// Tfds.Dbg(sb.To_str_and_clear());
// }
@Test public void Ns_aliases() {
Io_mgr.Instance.InitEngine_mem();
Io_mgr.Instance.SaveFilStr("mem/en.wikipedia.org/w/api.php?action=query&format=xml&meta=siteinfo&siprop=namespacealiases", String_.Concat_lines_nl
( "<api>"
, "<query>"
, "<namespacealiases>"
, "<ns id=\"4\" xml:space=\"preserve\">WP"
, "</ns>"
, "<ns id=\"5\" xml:space=\"preserve\">WT"
, "</ns>"
, "<ns id=\"6\" xml:space=\"preserve\">Image"
, "</ns>"
, "<ns id=\"7\" xml:space=\"preserve\">Image talk"
, "</ns>"
, "</namespacealiases>"
, "</query>"
, "</api>"
));
String[] wikis = new String[] {"en.wikipedia.org"}; String protocol = "mem/";
Tfds.Eq_str_lines(Query_ns(protocol, gplx.core.ios.IoEngine_.MemKey, wikis), String_.Concat_lines_nl
( "app.bldr.wiki_cfg_bldr.get('en.wikipedia.org').new_cmd_('wiki.ns_mgr.aliases', 'ns_mgr.add_alias_bulk(\""
, "4|WP"
, "5|WT"
, "6|Image"
, "7|Image_talk"
, "\");');"
));
}
String Query_ns(String protocol, String trg_engine_key, String[] wikis) {
String_bldr sb = String_bldr_.new_();
int wikis_len = wikis.length;
for (int i = 0; i < wikis_len; i++) {
String wiki = wikis[i];
if (String_.Len_eq_0(wiki)) continue;
try {
String api = protocol + wiki + "/w/api.php?action=query&format=xml&meta=siteinfo&siprop=namespacealiases";
String xml = String_.new_u8(Io_mgr.Instance.DownloadFil_args("", null).Trg_engine_key_(trg_engine_key).Exec_as_bry(api));
if (xml == null) continue; // not found
gplx.langs.xmls.XmlDoc xdoc = gplx.langs.xmls.XmlDoc_.parse(xml);
gplx.langs.xmls.XmlNde xnde = gplx.langs.xmls.Xpath_.SelectFirst(xdoc.Root(), "query/namespacealiases");
sb.Add("app.bldr.wiki_cfg_bldr.get('").Add(wiki).Add("').new_cmd_('wiki.ns_mgr.aliases', 'ns_mgr.add_alias_bulk(\"\n");
int xndes_len = xnde.SubNdes().Count();
for (int j = 0; j < xndes_len; j++) {
gplx.langs.xmls.XmlNde ns_nde = xnde.SubNdes().Get_at(j);
if (!String_.Eq(ns_nde.Name(), "ns")) continue;
int id = Int_.parse(ns_nde.Atrs().FetchValOr("id", "-1"));
String name = String_.Replace(String_.Replace(ns_nde.Text_inner(), " ", "_"), "'", "''");
sb.Add(Int_.To_str(id)).Add("|").Add(String_.Trim(name)).Add_char_nl();
}
sb.Add("\");');\n");
}
catch(Exception e) {sb.Add("// fail: " + wiki + " " + Err_.Message_gplx_full(e)).Add_char_nl();}
}
return sb.To_str_and_clear();
}
}
class Xob_wiki_cfg_bldr_fxt {
public Xob_wiki_cfg_bldr_fxt Clear() {
if (app == null) {
app = Xoa_app_fxt.Make__app__edit();
wiki_cfg_bldr = app.Bldr().Wiki_cfg_bldr();
}
wiki_cfg_bldr.Clear();
hash.Clear();
return this;
} private Xoae_app app; Xob_wiki_cfg_bldr wiki_cfg_bldr; Ordered_hash hash = Ordered_hash_.New();
public Xob_wiki_cfg_bldr_fxt Init_cmd(String wiki, String key, String text) {
wiki_cfg_bldr.Itms_get_or_new(wiki).Itms_add(key, text);
return this;
}
public Xob_wiki_cfg_bldr_fxt Expd_txt(String wiki, String text) {
hash.Add(wiki, Keyval_.new_(wiki, text));
return this;
}
public void Test() {
wiki_cfg_bldr.Exec();
int len = hash.Count();
for (int i = 0; i < len; i++) {
Keyval kv = (Keyval)hash.Get_at(i);
String wiki = kv.Key();
String expd = (String)kv.Val();
String actl = Io_mgr.Instance.LoadFilStr(app.Fsys_mgr().Cfg_wiki_core_dir().GenSubFil(wiki + ".gfs"));
Tfds.Eq_str_lines(expd, actl);
}
}
}

View File

@@ -0,0 +1,31 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.app_cfgs.wm_server_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.app_cfgs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xowm_server_cfg_cmd extends Xob_cmd__base {
public Xowm_server_cfg_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
// wiki.Init_assert();
// mgr.Exec(wiki);
}
public static final String BLDR_CMD_KEY = "cfg.wikis.wm_server_cfg";
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
public static final Xob_cmd Prototype = new Xowm_server_cfg_cmd(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xowm_server_cfg_cmd(bldr, wiki);}
}

View File

@@ -0,0 +1,36 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.app_cfgs.wm_server_cfgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.app_cfgs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xowm_server_cfg_mgr {
public void Exec() {
byte[] src = Load_or_download();
Parse_cat_collation(src);
}
private byte[] Load_or_download() {
// Io_mgr.Instance.DownloadFil_args("", Io_url_.NullPtr).Exec_as_bry("https://noc.wikimedia.org/conf/InitialiseSettings.php.txt");
return null;
}
private void Parse_cat_collation(byte[] src) {
int bgn_pos = Bry_find_.Find_fwd(src, Bry_.new_a7("wgCategoryCollation"));
if (bgn_pos == Bry_find_.Not_found) throw Err_.new_wo_type("could not find wgCategoryCollation bgn");
int end_pos = Bry_find_.Find_fwd(src, Bry_.new_a7("],"));
if (end_pos == Bry_find_.Not_found) throw Err_.new_wo_type("could not find wgCategoryCollation end");
Tfds.Write(src, bgn_pos, end_pos);
}
}

View File

@@ -54,4 +54,8 @@ public class Xobc_data_db {
tbl__import_step.Delete(step_id);
}
}
public static Xobc_data_db New(gplx.xowa.apps.fsys.Xoa_fsys_mgr fsys_mgr) {
return new Xobc_data_db(fsys_mgr.Bin_addon_dir().GenSubFil_nest("bldr", "central", "bldr_central.data_db.xowa"));
}
}

View File

@@ -0,0 +1,32 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.centrals.dbs.datas; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*;
public class Xobc_task_regy_itm {
public Xobc_task_regy_itm(int id, int seqn, byte[] key, byte[] name, int step_count) {
this.id = id;
this.seqn = seqn;
this.key = key;
this.name = name;
this.step_count = step_count;
}
public int Id() {return id;} private final int id;
public int Seqn() {return seqn;} private final int seqn;
public byte[] Key() {return key;} private final byte[] key;
public byte[] Name() {return name;} private final byte[] name;
public int Step_count() {return step_count;} private final int step_count;
}

View File

@@ -67,6 +67,28 @@ public class Xobc_task_regy_tbl implements Db_tbl {
.Val_str(fld_task_key, task_key).Val_str(fld_task_name, task_name)
.Exec_insert();
}
public Xobc_task_regy_itm[] Select_by_wiki(byte[] wiki_domain) {
String sql = Db_sql_.Make_by_fmt(String_.Ary
( "SELECT *"
, "FROM task_regy"
, "WHERE task_key LIKE '{0}%'"
, "AND task_seqn != 999999"
), wiki_domain);
List_adp list = List_adp_.New();
Db_rdr rdr = conn.Stmt_sql(sql).Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
list.Add(new Xobc_task_regy_itm
( rdr.Read_int("task_id")
, rdr.Read_int("task_seqn")
, rdr.Read_bry_by_str("task_key")
, rdr.Read_bry_by_str("task_name")
, rdr.Read_int("step_count")
));
}
} finally {rdr.Rls();}
return (Xobc_task_regy_itm[])list.To_ary_and_clear(Xobc_task_regy_itm.class);
}
public void Delete(int task_id) {
conn.Stmt_delete(tbl_name, fld_task_id).Crt_int(fld_task_id, task_id).Exec_delete();
}

View File

@@ -44,20 +44,7 @@ public class Xobc_import_step_tbl implements Db_tbl {
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_step_id).Crt_int(fld_step_id, step_id).Exec_select__rls_auto();
try {
return (rdr.Move_next())
? new Xobc_import_step_itm
( rdr.Read_int(fld_step_id)
, rdr.Read_int(fld_host_id)
, rdr.Read_bry_by_str(fld_wiki_abrv)
, rdr.Read_str(fld_wiki_date)
, rdr.Read_str(fld_import_name)
, rdr.Read_int(fld_import_type)
, rdr.Read_byte(fld_import_zip)
, rdr.Read_long(fld_import_size_zip)
, rdr.Read_long(fld_import_size_raw)
, rdr.Read_str(fld_import_md5)
, rdr.Read_long(fld_prog_size_end)
, rdr.Read_int(fld_prog_count_end)
)
? New_itm(rdr)
: Xobc_import_step_itm.Null;
}
finally {rdr.Rls();}
@@ -95,7 +82,39 @@ public class Xobc_import_step_tbl implements Db_tbl {
}
} finally {rdr.Rls();}
}
public Xobc_import_step_itm[] Select_by_task_id(int task_id) {
List_adp list = List_adp_.New();
Db_rdr rdr = conn.Stmt_sql(Db_sql_.Make_by_fmt(String_.Ary
( "SELECT s.*"
, "FROM import_step s"
, " JOIN step_map sm ON s.step_id = sm.step_id"
, "WHERE sm.task_id = {0}"
), task_id))
.Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
list.Add(New_itm(rdr));
}
} finally {rdr.Rls();}
return (Xobc_import_step_itm[])list.To_ary_and_clear(Xobc_import_step_itm.class);
}
public void Rls() {
insert_stmt = Db_stmt_.Rls(insert_stmt);
}
private Xobc_import_step_itm New_itm(Db_rdr rdr) {
return new Xobc_import_step_itm
( rdr.Read_int(fld_step_id)
, rdr.Read_int(fld_host_id)
, rdr.Read_bry_by_str(fld_wiki_abrv)
, rdr.Read_str(fld_wiki_date)
, rdr.Read_str(fld_import_name)
, rdr.Read_int(fld_import_type)
, rdr.Read_byte(fld_import_zip)
, rdr.Read_long(fld_import_size_zip)
, rdr.Read_long(fld_import_size_raw)
, rdr.Read_str(fld_import_md5)
, rdr.Read_long(fld_prog_size_end)
, rdr.Read_int(fld_prog_count_end)
);
}
}

View File

@@ -19,7 +19,7 @@ package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.x
import gplx.core.progs.*; import gplx.core.ios.zips.*; import gplx.core.ios.streams.*; import gplx.core.security.*;
import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.fsdb.*; import gplx.fsdb.meta.*;
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
class Pack_file_mgr {
public class Pack_file_mgr {
public void Exec(Xowe_wiki wiki, Pack_file_bldr_cfg cfg) {
// init
wiki.Init_assert();
@@ -31,7 +31,7 @@ class Pack_file_mgr {
// get import_tbl
byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo();
Xobc_data_db bc_db = new Xobc_data_db(wiki.App().Fsys_mgr().Bin_addon_dir().GenSubFil_nest("bldr", "central", "bldr_central.data_db.xowa"));
Xobc_data_db bc_db = Xobc_data_db.New(wiki.App().Fsys_mgr());
Db_conn bc_conn = bc_db.Conn();
bc_db.Delete_by_import(wiki_abrv, wiki_date);
bc_conn.Txn_bgn("xobc_import_insert");
@@ -93,7 +93,7 @@ class Pack_file_mgr {
int pack_list_len = pack_list.Len();
// create task
String task_key = wiki.Domain_str() + "|" + wiki_date + "|" + task_type;
String task_key = Task_key__build(wiki.Domain_str(), wiki_date, task_type);
String task_name = Build_task_name(tmp_bfr, wiki, wiki_date, task_type, raw_len);
Xobc_task_regy_tbl task_regy_tbl = bc_db.Tbl__task_regy();
int task_id = bc_db.Conn().Sys_mgr().Autonum_next("task_regy.task_id");
@@ -161,6 +161,12 @@ class Pack_file_mgr {
bc_db.Tbl__step_regy().Insert(step_id, Xobc_step_itm.Type__wiki_import);
bc_db.Tbl__import_step().Insert(step_id, gplx.xowa.addons.bldrs.centrals.dbs.datas.Xobc_host_regy_tbl.Host_id__archive_org, wiki_abrv, wiki_date, zip_url.NameAndExt(), itm.Tid(), Xobc_zip_type.Type__zip, zip_md5, zip_len, raw_size, 0, 0);
}
private static String Task_key__build(String wiki_domain, String wiki_date, String task_type) {
return String_.Concat(wiki_domain, "|", wiki_date, "|", task_type);
}
public static String[] Task_key__parse(String task_key) {
return String_.Split(task_key, "|");
}
}
class Pack_hash_bldr {
public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff, boolean pack_fsdb_delete) {

View File

@@ -33,7 +33,7 @@ class Pack_mgr {
// get import_tbl
byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo();
Xobc_data_db bc_db = new Xobc_data_db(wiki.App().Fsys_mgr().Bin_addon_dir().GenSubFil_nest("bldr", "central", "bldr_central.data_db.xowa"));
Xobc_data_db bc_db = Xobc_data_db.New(wiki.App().Fsys_mgr());
Db_conn bc_conn = bc_db.Conn();
bc_db.Delete_by_import(wiki_abrv, wiki_date);
bc_conn.Txn_bgn("xobc_import_insert");

View File

@@ -19,6 +19,8 @@ package gplx.xowa.addons.bldrs.files; import gplx.*; import gplx.xowa.*; import
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.files.cmds.*;
import gplx.xowa.addons.bldrs.mass_parses.inits.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.makes.*;
import gplx.xowa.addons.bldrs.files.cksums.*;
import gplx.xowa.addons.bldrs.app_cfgs.wm_server_cfgs.*;
public class Xoax_builds_files_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr {
public Xob_cmd[] Bldr_cmds() {
return new Xob_cmd[]
@@ -43,6 +45,9 @@ public class Xoax_builds_files_addon implements Xoax_addon_itm, Xoax_addon_itm__
, Xomp_init_cmd.Prototype
, Xomp_parse_cmd.Prototype
, Xomp_make_cmd.Prototype
, Xocksum_calc_cmd.Prototype
, Xowm_server_cfg_cmd.Prototype
};
}

View File

@@ -0,0 +1,30 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cksums; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xocksum_calc_cmd extends Xob_cmd__base {
public Xocksum_calc_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
wiki.Init_assert();
new Xocksum_calc_mgr().Exec(wiki);
}
@Override public String Cmd_key() {return BLDR_CMD_KEY;} private static final String BLDR_CMD_KEY = "fsdb.cksums.calc";
public static final Xob_cmd Prototype = new Xocksum_calc_cmd(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xocksum_calc_cmd(bldr, wiki);}
}

View File

@@ -0,0 +1,79 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cksums; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*;
import gplx.core.ios.streams.*; import gplx.core.security.*;
import gplx.dbs.*; import gplx.xowa.addons.bldrs.files.cksums.dbs.*;
import gplx.xowa.files.*; import gplx.fsdb.*; import gplx.fsdb.data.*;
public class Xocksum_calc_mgr {
public void Exec(Xowe_wiki wiki) {
// get conn variables
Xocksum_cksum_db db = Xocksum_cksum_db.Get(wiki);
Db_conn conn = db.Conn();
Xocksum_cksum_tbl tbl = db.Tbl__cksum();
conn.Meta_tbl_assert(tbl);
// insert missing items
tbl.Insert_missing();
tbl.Create_idx();
// get updates
int count = 0;
Hash_algo md5_algo = Hash_algo_.New__md5();
List_adp updates = List_adp_.New();
String cur_date = Datetime_now.Get().XtoStr_gplx();
Db_stmt select_stmt = tbl.Select_samples_stmt(10000);
while (true) {
// get cksum_rows
Xocksum_cksum_row[] rows = tbl.Select_samples(select_stmt);
// loop cksum_rows and (a) get bin_data; (b) if md5 diff, then add to updates
int len = rows.length; if (len == 0) break;
for (int i = 0; i < len; ++i) {
Xocksum_cksum_row row = rows[i];
byte[] bin_bry = Get_bin(wiki, row);
if (bin_bry == null) {
Gfo_usr_dlg_.Instance.Prog_many("", "", "null; fil_id=~{0} thm_id=~{1}", row.Fil_id(), row.Thm_id());
bin_bry = Bry_.Empty;
}
row.Bin_size_(bin_bry.length);
byte[] md5 = md5_algo.Hash_bry_as_bry(bin_bry);
if (!Bry_.Eq(md5, row.Cksum_val())) {
row.Cksum_val_(md5);
updates.Add(row);
}
}
// run updates
conn.Txn_bgn("cksum_update");
len = updates.Len();
for (int i = 0; i < len; ++i) {
Xocksum_cksum_row row = (Xocksum_cksum_row)updates.Get_at(i);
tbl.Update(row.Fil_id(), row.Thm_id(), row.Bin_db_id(), row.Bin_size(), row.Cksum_tid(), 0, row.Cksum_val(), cur_date);
if (++count % 2000 == 0) Gfo_usr_dlg_.Instance.Prog_many("", "", "updating; rows=~{0}", count);
}
updates.Clear();
conn.Txn_end();
}
select_stmt.Rls();
}
private byte[] Get_bin(Xowe_wiki wiki, Xocksum_cksum_row row) {
int bin_id = row.Thm_id() == -1 ? row.Fil_id() : row.Thm_id();
Fsd_bin_itm bin_itm = wiki.File__mnt_mgr().Mnts__get_main().Bin_mgr().Dbs__get_at(row.Bin_db_id()).Select_as_itm(bin_id);
return bin_itm.Bin_data();
}
}

View File

@@ -0,0 +1,31 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cksums.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.addons.bldrs.files.cksums.*;
import gplx.dbs.*; import gplx.fsdb.meta.*;
public class Xocksum_cksum_db {
public Xocksum_cksum_db(Db_conn conn) {
this.conn = conn;
this.tbl__cksum = new Xocksum_cksum_tbl(conn);
}
public Db_conn Conn() {return conn;} private final Db_conn conn;
public Xocksum_cksum_tbl Tbl__cksum() {return tbl__cksum;} private final Xocksum_cksum_tbl tbl__cksum;
public static Xocksum_cksum_db Get(Xowe_wiki wiki) {
return new Xocksum_cksum_db(wiki.File__fsdb_core().File__abc_file__at(Fsm_mnt_mgr.Mnt_idx_main).Conn());
}
}

View File

@@ -0,0 +1,41 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cksums.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.addons.bldrs.files.cksums.*;
public class Xocksum_cksum_row {
public Xocksum_cksum_row(int fil_id, int thm_id, int bin_db_id, long bin_size, byte cksum_tid, int cksum_count, byte[] cksum_val, String cksum_date) {
this.fil_id = fil_id;
this.thm_id = thm_id;
this.bin_db_id = bin_db_id;
this.bin_size = bin_size;
this.cksum_tid = cksum_tid;
this.cksum_count = cksum_count;
this.cksum_val = cksum_val;
this.cksum_date = cksum_date;
}
public int Fil_id() {return fil_id;} private final int fil_id;
public int Thm_id() {return thm_id;} private final int thm_id;
public int Bin_db_id() {return bin_db_id;} private final int bin_db_id;
public long Bin_size() {return bin_size;} private long bin_size;
public byte Cksum_tid() {return cksum_tid;} private final byte cksum_tid;
public int Cksum_count() {return cksum_count;} private final int cksum_count;
public byte[] Cksum_val() {return cksum_val;} private byte[] cksum_val;
public String Cksum_date() {return cksum_date;} private final String cksum_date;
public void Bin_size_(long v) {this.bin_size = v;}
public void Cksum_val_(byte[] v) {this.cksum_val = v;}
}

View File

@@ -0,0 +1,107 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cksums.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.addons.bldrs.files.cksums.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.addons.wikis.ctgs.*;
public class Xocksum_cksum_tbl implements Db_tbl {
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private final String fld__fil_id, fld__thm_id, fld__bin_db_id, fld__bin_len, fld__cksum_tid, fld__cksum_count, fld__cksum_val, fld__cksum_date;
private Db_stmt stmt__update;
public Xocksum_cksum_tbl(Db_conn conn) {
this.conn = conn;
this.tbl_name = "fsdb_cksum";
this.fld__fil_id = flds.Add_int("fil_id");
this.fld__thm_id = flds.Add_int("thm_id");
this.fld__bin_db_id = flds.Add_int("bin_db_id");
this.fld__bin_len = flds.Add_long("bin_size");
this.fld__cksum_tid = flds.Add_byte("cksum_tid");
this.fld__cksum_count = flds.Add_int("cksum_count");
this.fld__cksum_val = flds.Add_str("cksum_val", 255);
this.fld__cksum_date = flds.Add_str("cksum_date", 16);
conn.Rls_reg(this);
}
public Db_conn Conn() {return conn;} private final Db_conn conn;
public String Tbl_name() {return tbl_name;} private final String tbl_name;
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_idx() {
conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "fil_id__thm_id", fld__fil_id, fld__thm_id));
conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, fld__cksum_val, fld__cksum_val));
}
public void Insert_missing() {
// insert from fsdb_fil
conn.Exec_sql(Db_sql_.Make_by_fmt(String_.Ary
( "INSERT INTO fsdb_cksum (fil_id, thm_id, bin_db_id, bin_size, cksum_tid, cksum_count, cksum_val, cksum_date)"
, "SELECT f.fil_id, -1, f.fil_bin_db_id, f.fil_size, {0}, 0, '', ''"
, "FROM fsdb_fil f"
, " LEFT JOIN fsdb_cksum c ON c.fil_id = f.fil_id AND c.thm_id = -1"
, "WHERE c.fil_id IS NULL"
, "AND f.fil_bin_db_id != -1"
), Cksum_tid__md5));
// insert from fsdb_fil
conn.Exec_sql(Db_sql_.Make_by_fmt(String_.Ary
( "INSERT INTO fsdb_cksum (fil_id, thm_id, bin_db_id, bin_size, cksum_tid, cksum_count, cksum_val, cksum_date)"
, "SELECT t.thm_owner_id, t.thm_id, t.thm_bin_db_id, t.thm_size, {0}, 0, '', ''"
, "FROM fsdb_thm t"
, " LEFT JOIN fsdb_cksum c ON c.fil_id = t.thm_owner_id AND c.thm_id = t.thm_id"
, "WHERE c.fil_id IS NULL"
), Cksum_tid__md5));
}
public Db_stmt Select_samples_stmt(int count) {
return conn.Stmt_sql(Db_sql_.Make_by_fmt(String_.Ary
( "SELECT *"
, "FROM fsdb_cksum"
, "WHERE cksum_val = ''"
// , "ORDER BY cksum_count, cksum_date"
, "LIMIT {0}"
), count));
}
public Xocksum_cksum_row[] Select_samples(Db_stmt stmt) {
List_adp rv = List_adp_.New();
Db_rdr rdr = stmt.Exec_select__rls_manual();
try {
while (rdr.Move_next()) {
rv.Add(new Xocksum_cksum_row
( rdr.Read_int("fil_id")
, rdr.Read_int("thm_id")
, rdr.Read_int("bin_db_id")
, rdr.Read_long("bin_size")
, rdr.Read_byte("cksum_tid")
, rdr.Read_int("cksum_count")
, rdr.Read_bry_by_str("cksum_val")
, rdr.Read_str("cksum_date")
));
}
} finally {rdr.Rls();}
return (Xocksum_cksum_row[])rv.To_ary_and_clear(Xocksum_cksum_row.class);
}
public void Update(int fil_id, int thm_id, int bin_db_id, long bin_size, byte cksum_tid, int cksum_count, byte[] cksum_val, String cksum_date) {
if (stmt__update == null) stmt__update = conn.Stmt_update_exclude(tbl_name, flds, fld__fil_id, fld__thm_id);
stmt__update.Clear()
.Val_int(fld__bin_db_id, bin_db_id).Val_long(fld__bin_len, bin_size)
.Val_byte(fld__cksum_tid, cksum_tid).Val_int(fld__cksum_count, cksum_count)
.Val_bry_as_str(fld__cksum_val, cksum_val).Val_str(fld__cksum_date, cksum_date)
.Crt_int(fld__fil_id, fil_id).Crt_int(fld__thm_id, thm_id)
.Exec_update();
}
public void Rls() {
this.stmt__update = Db_stmt_.Rls(stmt__update);
}
public static final byte Cksum_tid__md5 = 1;
}

View File

@@ -39,7 +39,7 @@ class Hdump_html_loader {
html_db.Tbl__html().Select_by_page(tmp_hpg);
// unzip it
byte[] html_hzip = stream_zip_mgr.Unzip(Io_stream_.Tid_gzip, tmp_hpg.Db().Html().Html_bry());
byte[] html_hzip = stream_zip_mgr.Unzip(Io_stream_tid_.Tid__gzip, tmp_hpg.Db().Html().Html_bry());
return wiki.Html__hdump_mgr().Load_mgr().Decode_as_bry(tmp_bfr, tmp_hpg, html_hzip, true);
}
}

View File

@@ -31,7 +31,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true;
public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true;
public boolean Load_all_imglnks() {return load_all_imglnks;} private boolean load_all_imglnks = true;
public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_.Tid_gzip;
public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_tid_.Tid__gzip;
public Io_url Mgr_url() {return mgr_url;} private Io_url mgr_url;
public String Wkr_machine_name() {return wkr_machine_name;} private String wkr_machine_name;
public boolean Show_msg__fetched_pool() {return show_msg__fetched_pool;} private boolean show_msg__fetched_pool;

View File

@@ -32,7 +32,7 @@ class Xodel_make_mgr {
, "FROM xfer_regy xr"
, "WHERE xr.lnki_ttl = fsdb_regy.fsdb_name"
, "AND xr.file_is_orig = fsdb_regy.fsdb_is_orig"
, "AND xr.orig_repo = fsdb_regy.fsdb_repo"
// , "AND xr.orig_repo = fsdb_regy.fsdb_repo" // TOMBSTONE: do no reinstate; some images exist in both repos, and this will delete images from one repo; DATE:2016-09-28
, "AND xr.file_w = fsdb_regy.fsdb_w"
, "AND xr.lnki_time = fsdb_regy.fsdb_time"
, "AND xr.lnki_page = fsdb_regy.fsdb_page"