mirror of https://github.com/gnosygnu/xowa
gnosygnu
8 years ago
parent
f5f48bb9b1
commit
b333db45f8
@ -0,0 +1,292 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.langs.phps; import gplx.*; import gplx.langs.*;
|
||||
import gplx.core.tests.*; import gplx.core.log_msgs.*;
|
||||
class Php_parser_fxt {
|
||||
Php_tkn_factory tkn_factory = new Php_tkn_factory();
|
||||
Php_parser parser = new Php_parser();
|
||||
Php_tkn_wkr_tkn tkn_wkr = new Php_tkn_wkr_tkn();
|
||||
Php_evaluator line_wkr = new Php_evaluator(new Gfo_msg_log("test"));
|
||||
Tst_mgr tst_mgr = new Tst_mgr();
|
||||
Gfo_msg_log_chkr log_mgr_chkr = new Gfo_msg_log_chkr();
|
||||
public void Clear() {log_mgr_chkr.Clear(); tkn_wkr.Clear(); line_wkr.Clear();}
|
||||
public Php_tkn_chkr_base tkn_declaration() {return Php_tkn_declaration_chkr.Instance;}
|
||||
public Php_tkn_chkr_base tkn_txt(int bgn, int end) {return new Php_tkn_txt_chkr(bgn, end);}
|
||||
public Php_tkn_chkr_base tkn_ws(int bgn, int end) {return new Php_tkn_ws_chkr(bgn, end);}
|
||||
public Php_tkn_chkr_base tkn_generic(int bgn, int end, byte tid) {return new Php_tkn_generic_chkr(bgn, end, tid);}
|
||||
public Php_tkn_comment_chkr tkn_comment_mult(int bgn, int end) {return new Php_tkn_comment_chkr(bgn, end).Comment_tid_(Php_tkn_comment.Tid_mult);}
|
||||
public Php_tkn_comment_chkr tkn_comment_slash(int bgn, int end) {return new Php_tkn_comment_chkr(bgn, end).Comment_tid_(Php_tkn_comment.Tid_slash);}
|
||||
public Php_tkn_comment_chkr tkn_comment_hash(int bgn, int end) {return new Php_tkn_comment_chkr(bgn, end).Comment_tid_(Php_tkn_comment.Tid_hash);}
|
||||
public Php_tkn_quote_chkr tkn_quote_apos(int bgn, int end) {return new Php_tkn_quote_chkr(bgn, end).Quote_tid_(Byte_ascii.Apos);}
|
||||
public Php_tkn_quote_chkr tkn_quote_quote(int bgn, int end) {return new Php_tkn_quote_chkr(bgn, end).Quote_tid_(Byte_ascii.Quote);}
|
||||
public Php_parser_fxt Msg(Gfo_msg_itm itm, int bgn, int end) {
|
||||
log_mgr_chkr.Add_itm(itm, bgn, end);
|
||||
return this;
|
||||
}
|
||||
public Php_tkn_var_chkr tkn_var(int bgn, int end, String v) {return new Php_tkn_var_chkr(bgn, end).Var_name_(v);}
|
||||
public Php_tkn_num_chkr tkn_num(int bgn, int end, int v) {return new Php_tkn_num_chkr(bgn, end).Num_val_int_(v);}
|
||||
public Php_line_assign_chkr line_assign(String key, Php_itm_chkr_base val) {return new Php_line_assign_chkr().Key_(key).Val_(val);}
|
||||
public Php_line_assign_chkr line_assign_subs(String key, String[] subs, Php_itm_chkr_base val) {return new Php_line_assign_chkr().Key_(key).Subs_(subs).Val_(val);}
|
||||
public Php_itm_chkr_base itm_bool_true() {return new Php_itm_generic_chkr(Php_itm_.Tid_bool_true);}
|
||||
public Php_itm_chkr_base itm_bool_false() {return new Php_itm_generic_chkr(Php_itm_.Tid_bool_false);}
|
||||
public Php_itm_chkr_base itm_null() {return new Php_itm_generic_chkr(Php_itm_.Tid_null);}
|
||||
public Php_itm_chkr_base itm_quote(String v) {return new Php_itm_quote_chkr().Val_obj_str_(v);}
|
||||
public Php_itm_chkr_base itm_int(int v) {return new Php_itm_int_chkr().Val_obj_int_(v);}
|
||||
public Php_itm_chkr_base itm_txt(String v) {return new Php_itm_txt_chkr().Val_obj_str_(v);}
|
||||
public Php_itm_ary_chkr itm_ary() {return new Php_itm_ary_chkr();}
|
||||
public Php_itm_kv_chkr itm_kv_quote(String k, String v) {return new Php_itm_kv_chkr().Key_(k).Val_(itm_quote(v));}
|
||||
public Php_itm_kv_chkr itm_kv_txt(String k, String v) {return new Php_itm_kv_chkr().Key_(k).Val_(itm_txt(v));}
|
||||
public Php_itm_kv_chkr itm_kv_int(String k, int v) {return new Php_itm_kv_chkr().Key_(k).Val_(itm_int(v));}
|
||||
public Php_itm_kv_chkr itm_kv_itm(String k, Php_itm_chkr_base v) {return new Php_itm_kv_chkr().Key_(k).Val_(v);}
|
||||
public void tst_tkns(String raw, Php_tkn_chkr_base... expd) {
|
||||
byte[] raw_bry = Bry_.new_u8(raw);
|
||||
parser.Parse_tkns(raw_bry, tkn_wkr);
|
||||
Php_tkn[] actl = (Php_tkn[])tkn_wkr.List().To_ary(Php_tkn.class);
|
||||
tst_mgr.Vars().Clear().Add("raw_bry", raw_bry);
|
||||
tst_mgr.Tst_ary("", expd, actl);
|
||||
log_mgr_chkr.tst(tst_mgr, tkn_wkr.Msg_log());
|
||||
}
|
||||
public void tst_lines(String raw, Php_line_assign_chkr... expd) {
|
||||
byte[] raw_bry = Bry_.new_u8(raw);
|
||||
parser.Parse_tkns(raw_bry, line_wkr);
|
||||
Php_line[] actl = (Php_line[])line_wkr.List().To_ary(Php_line.class);
|
||||
tst_mgr.Vars().Clear().Add("raw_bry", raw_bry);
|
||||
tst_mgr.Tst_ary("", expd, actl);
|
||||
log_mgr_chkr.tst(tst_mgr, line_wkr.Msg_log());
|
||||
}
|
||||
}
|
||||
abstract class Php_tkn_chkr_base implements Tst_chkr {
|
||||
public abstract byte Tkn_tid();
|
||||
public abstract Class<?> TypeOf();
|
||||
public int Src_bgn() {return src_bgn;} private int src_bgn = -1;
|
||||
public int Src_end() {return src_end;} private int src_end = -1;
|
||||
public void Src_rng_(int src_bgn, int src_end) {this.src_bgn = src_bgn; this.src_end = src_end;}
|
||||
public int Chk(Tst_mgr mgr, String path, Object actl_obj) {
|
||||
Php_tkn actl = (Php_tkn)actl_obj;
|
||||
int rv = 0;
|
||||
rv += mgr.Tst_val(false, path, "tkn_tid", this.Tkn_tid(), actl.Tkn_tid());
|
||||
rv += mgr.Tst_val(src_bgn == -1, path, "src_bgn", src_bgn, actl.Src_bgn());
|
||||
rv += mgr.Tst_val(src_end == -1, path, "src_end", src_end, actl.Src_end());
|
||||
rv += Chk_tkn(mgr, path, actl);
|
||||
return rv;
|
||||
}
|
||||
@gplx.Virtual public int Chk_tkn(Tst_mgr mgr, String path, Php_tkn actl_obj) {return 0;}
|
||||
}
|
||||
class Php_tkn_declaration_chkr extends Php_tkn_chkr_base {
|
||||
@Override public Class<?> TypeOf() {return Php_tkn_declaration.class;}
|
||||
@Override public byte Tkn_tid() {return Php_tkn_.Tid_declaration;}
|
||||
public static final Php_tkn_declaration_chkr Instance = new Php_tkn_declaration_chkr();
|
||||
}
|
||||
class Php_tkn_txt_chkr extends Php_tkn_chkr_base {
|
||||
public Php_tkn_txt_chkr(int src_bgn, int src_end) {this.Src_rng_(src_bgn, src_end);}
|
||||
@Override public Class<?> TypeOf() {return Php_tkn_txt.class;}
|
||||
@Override public byte Tkn_tid() {return Php_tkn_.Tid_txt;}
|
||||
}
|
||||
class Php_tkn_ws_chkr extends Php_tkn_chkr_base {
|
||||
public Php_tkn_ws_chkr(int src_bgn, int src_end) {this.Src_rng_(src_bgn, src_end);}
|
||||
@Override public Class<?> TypeOf() {return Php_tkn_ws.class;}
|
||||
@Override public byte Tkn_tid() {return Php_tkn_.Tid_ws;}
|
||||
}
|
||||
class Php_tkn_comment_chkr extends Php_tkn_chkr_base {
|
||||
public Php_tkn_comment_chkr(int src_bgn, int src_end) {this.Src_rng_(src_bgn, src_end);}
|
||||
@Override public Class<?> TypeOf() {return Php_tkn_comment.class;}
|
||||
@Override public byte Tkn_tid() {return Php_tkn_.Tid_comment;}
|
||||
public Php_tkn_comment_chkr Comment_tid_(byte v) {this.comment_tid = v; return this;} private byte comment_tid = Php_tkn_comment.Tid_null;
|
||||
@Override public int Chk_tkn(Tst_mgr mgr, String path, Php_tkn actl_obj) {
|
||||
Php_tkn_comment actl = (Php_tkn_comment)actl_obj;
|
||||
int rv = 0;
|
||||
rv += mgr.Tst_val(comment_tid == Php_tkn_comment.Tid_null, path, "comment_tid", comment_tid, actl.Comment_tid());
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
class Php_tkn_quote_chkr extends Php_tkn_chkr_base {
|
||||
public Php_tkn_quote_chkr(int src_bgn, int src_end) {this.Src_rng_(src_bgn, src_end);}
|
||||
@Override public Class<?> TypeOf() {return Php_tkn_quote.class;}
|
||||
@Override public byte Tkn_tid() {return Php_tkn_.Tid_quote;}
|
||||
public Php_tkn_quote_chkr Quote_tid_(byte v) {this.quote_tid = v; return this;} private byte quote_tid = Byte_ascii.Null;
|
||||
@Override public int Chk_tkn(Tst_mgr mgr, String path, Php_tkn actl_obj) {
|
||||
Php_tkn_quote actl = (Php_tkn_quote)actl_obj;
|
||||
int rv = 0;
|
||||
rv += mgr.Tst_val(quote_tid == Byte_ascii.Null, path, "quote_tid", quote_tid, actl.Quote_tid());
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
class Php_tkn_var_chkr extends Php_tkn_chkr_base {
|
||||
public Php_tkn_var_chkr(int src_bgn, int src_end) {this.Src_rng_(src_bgn, src_end);}
|
||||
@Override public Class<?> TypeOf() {return Php_tkn_var.class;}
|
||||
@Override public byte Tkn_tid() {return Php_tkn_.Tid_var;}
|
||||
public Php_tkn_var_chkr Var_name_(String v) {this.var_name = v; return this;} private String var_name;
|
||||
@Override public int Chk_tkn(Tst_mgr mgr, String path, Php_tkn actl_obj) {
|
||||
Php_tkn_var actl = (Php_tkn_var)actl_obj;
|
||||
int rv = 0;
|
||||
byte[] raw_bry = (byte[])mgr.Vars_get_by_key("raw_bry");
|
||||
rv += mgr.Tst_val(var_name == null, path, "var_name", var_name, String_.new_u8(actl.Var_name(raw_bry)));
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
class Php_tkn_num_chkr extends Php_tkn_chkr_base {
|
||||
public Php_tkn_num_chkr(int src_bgn, int src_end) {this.Src_rng_(src_bgn, src_end);}
|
||||
@Override public Class<?> TypeOf() {return Php_tkn_num.class;}
|
||||
@Override public byte Tkn_tid() {return Php_tkn_.Tid_num;}
|
||||
public Php_tkn_num_chkr Num_val_int_(int v) {this.num_val_int = v; return this;} private int num_val_int = Int_.Min_value;
|
||||
@Override public int Chk_tkn(Tst_mgr mgr, String path, Php_tkn actl_obj) {
|
||||
Php_tkn_num actl = (Php_tkn_num)actl_obj;
|
||||
int rv = 0;
|
||||
byte[] raw_bry = (byte[])mgr.Vars_get_by_key("raw_bry");
|
||||
rv += mgr.Tst_val(num_val_int == Int_.Min_value, path, "num_val_int", num_val_int, actl.Num_val_int(raw_bry));
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
class Php_tkn_generic_chkr extends Php_tkn_chkr_base {
|
||||
public Php_tkn_generic_chkr(int src_bgn, int src_end, byte tkn_tid) {this.Src_rng_(src_bgn, src_end); this.tkn_tid = tkn_tid;}
|
||||
@Override public Class<?> TypeOf() {return Php_tkn.class;}
|
||||
@Override public byte Tkn_tid() {return tkn_tid;} private byte tkn_tid;
|
||||
}
|
||||
class Php_line_assign_chkr implements Tst_chkr {
|
||||
public Class<?> TypeOf() {return Php_line_assign.class;}
|
||||
public Php_line_assign_chkr Key_(String v) {key = v; return this;} private String key;
|
||||
public Php_line_assign_chkr Subs_(String[] v) {
|
||||
int subs_len = v.length;
|
||||
subs = new Php_itm_quote_chkr[subs_len];
|
||||
for (int i = 0; i < subs_len; i++)
|
||||
subs[i] = new Php_itm_quote_chkr().Val_obj_str_(v[i]);
|
||||
return this;
|
||||
} Php_itm_chkr_base[] subs;
|
||||
public Php_line_assign_chkr Val_(Php_itm_chkr_base v) {val = v; return this;} Php_itm_chkr_base val;
|
||||
public int Chk(Tst_mgr mgr, String path, Object actl_obj) {
|
||||
Php_line_assign actl = (Php_line_assign)actl_obj;
|
||||
int rv = 0;
|
||||
rv += mgr.Tst_val(key == null, path, "key", key, String_.new_u8(actl.Key().Val_obj_bry()));
|
||||
if (subs != null) rv += mgr.Tst_sub_ary(subs, actl.Key_subs(), "subs", rv);
|
||||
rv += mgr.Tst_sub_obj(val, actl.Val(), "val", rv);
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
abstract class Php_itm_chkr_base implements Tst_chkr {
|
||||
public abstract byte Itm_tid();
|
||||
public abstract Class<?> TypeOf();
|
||||
public int Chk(Tst_mgr mgr, String path, Object actl_obj) {
|
||||
Php_itm actl = (Php_itm)actl_obj;
|
||||
int rv = 0;
|
||||
rv += mgr.Tst_val(false, path, "tkn_tid", this.Itm_tid(), actl.Itm_tid());
|
||||
rv += Chk_itm(mgr, path, actl);
|
||||
return rv;
|
||||
}
|
||||
@gplx.Virtual public int Chk_itm(Tst_mgr mgr, String path, Php_itm actl_obj) {return 0;}
|
||||
public static final Php_itm_chkr_base[] Ary_empty = new Php_itm_chkr_base[0];
|
||||
}
|
||||
class Php_itm_generic_chkr extends Php_itm_chkr_base {
|
||||
public Php_itm_generic_chkr(byte itm_tid) {this.itm_tid = itm_tid;} private byte itm_tid;
|
||||
@Override public byte Itm_tid() {return itm_tid;}
|
||||
@Override public Class<?> TypeOf() {return Php_itm.class;}
|
||||
}
|
||||
class Php_itm_int_chkr extends Php_itm_chkr_base {
|
||||
@Override public byte Itm_tid() {return Php_itm_.Tid_int;}
|
||||
@Override public Class<?> TypeOf() {return Php_itm.class;}
|
||||
public Php_itm_int_chkr Val_obj_int_(int v) {this.val_obj_int = v; return this;} private int val_obj_int;
|
||||
@Override public int Chk_itm(Tst_mgr mgr, String path, Php_itm actl_obj) {
|
||||
Php_itm_int actl = (Php_itm_int)actl_obj;
|
||||
int rv = 0;
|
||||
rv += mgr.Tst_val(false, path, "val_obj_str", val_obj_int, actl.Val_obj_int());
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
class Php_itm_txt_chkr extends Php_itm_chkr_base {
|
||||
@Override public byte Itm_tid() {return Php_itm_.Tid_var;}
|
||||
@Override public Class<?> TypeOf() {return Php_itm.class;}
|
||||
public Php_itm_txt_chkr Val_obj_str_(String v) {this.val_obj_str = v; return this;} private String val_obj_str;
|
||||
@Override public int Chk_itm(Tst_mgr mgr, String path, Php_itm actl_obj) {
|
||||
Php_itm_var actl = (Php_itm_var)actl_obj;
|
||||
int rv = 0;
|
||||
rv += mgr.Tst_val(false, path, "val_obj_str", val_obj_str, String_.new_u8(actl.Val_obj_bry()));
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
class Php_itm_quote_chkr extends Php_itm_chkr_base {
|
||||
@Override public byte Itm_tid() {return Php_itm_.Tid_quote;}
|
||||
@Override public Class<?> TypeOf() {return Php_itm.class;}
|
||||
public Php_itm_quote_chkr Val_obj_str_(String v) {this.val_obj_str = v; return this;} private String val_obj_str;
|
||||
@Override public int Chk_itm(Tst_mgr mgr, String path, Php_itm actl_obj) {
|
||||
Php_itm_quote actl = (Php_itm_quote)actl_obj;
|
||||
int rv = 0;
|
||||
rv += mgr.Tst_val(false, path, "val_obj_str", val_obj_str, String_.new_u8(actl.Val_obj_bry()));
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
class Php_itm_ary_chkr extends Php_itm_chkr_base {
|
||||
@Override public byte Itm_tid() {return Php_itm_.Tid_ary;}
|
||||
@Override public Class<?> TypeOf() {return Php_itm.class;}
|
||||
public Php_itm_ary_chkr Subs_(Php_itm_chkr_base... v) {this.itms = v; return this;} Php_itm_chkr_base[] itms = Php_itm_chkr_base.Ary_empty;
|
||||
@Override public int Chk_itm(Tst_mgr mgr, String path, Php_itm actl_obj) {
|
||||
Php_itm_ary actl = (Php_itm_ary)actl_obj;
|
||||
int rv = 0;
|
||||
int actl_subs_len = actl.Subs_len();
|
||||
Php_itm[] actl_ary = new Php_itm[actl_subs_len];
|
||||
for (int i = 0; i < actl_subs_len; i++) {
|
||||
actl_ary[i] = (Php_itm)actl.Subs_get(i);
|
||||
}
|
||||
rv += mgr.Tst_sub_ary(itms, actl_ary, "subs", rv);
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
class Php_itm_kv_chkr extends Php_itm_chkr_base {
|
||||
@Override public byte Itm_tid() {return Php_itm_.Tid_kv;}
|
||||
@Override public Class<?> TypeOf() {return Php_itm.class;}
|
||||
public Php_itm_kv_chkr Key_(String v) {key = v; return this;} private String key;
|
||||
public Php_itm_kv_chkr Val_(Php_itm_chkr_base v) {val = v; return this;} Php_itm_chkr_base val;
|
||||
@Override public int Chk_itm(Tst_mgr mgr, String path, Php_itm actl_obj) {
|
||||
Php_itm_kv actl = (Php_itm_kv)actl_obj;
|
||||
int rv = 0;
|
||||
rv += mgr.Tst_val(false, path, "key", key, String_.new_u8(actl.Key().Val_obj_bry()));
|
||||
rv += mgr.Tst_sub_obj(val, actl.Val(), path, rv);
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
class Gfo_msg_log_chkr implements Tst_chkr {
|
||||
List_adp itms = List_adp_.New();
|
||||
public Class<?> TypeOf() {return Gfo_msg_log.class;}
|
||||
public void Clear() {itms.Clear();}
|
||||
public void Add_itm(Gfo_msg_itm itm, int bgn, int end) {
|
||||
Gfo_msg_data_chkr chkr = new Gfo_msg_data_chkr();
|
||||
chkr.Itm_(itm).Excerpt_bgn_(bgn).Excerpt_end_(end);
|
||||
itms.Add(chkr);
|
||||
}
|
||||
public int Chk(Tst_mgr mgr, String path, Object actl_obj) {return 0;}
|
||||
public void tst(Tst_mgr mgr, Object actl_obj) {
|
||||
Gfo_msg_log actl = (Gfo_msg_log)actl_obj;
|
||||
int actl_itms_len = actl.Ary_len();
|
||||
Gfo_msg_data[] actl_itms = new Gfo_msg_data[actl_itms_len];
|
||||
for (int i = 0; i < actl_itms_len; i++)
|
||||
actl_itms[i] = actl.Ary_get(i);
|
||||
mgr.Tst_ary("itms", (Gfo_msg_data_chkr[])itms.To_ary(Gfo_msg_data_chkr.class), actl_itms);
|
||||
}
|
||||
}
|
||||
class Gfo_msg_data_chkr implements Tst_chkr {
|
||||
public Class<?> TypeOf() {return Gfo_msg_data.class;}
|
||||
public Gfo_msg_data_chkr Itm_(Gfo_msg_itm v) {itm = v; return this;} Gfo_msg_itm itm;
|
||||
public Gfo_msg_data_chkr Excerpt_bgn_(int v) {excerpt_bgn = v; return this;} private int excerpt_bgn = -1;
|
||||
public Gfo_msg_data_chkr Excerpt_end_(int v) {excerpt_end = v; return this;} private int excerpt_end = -1;
|
||||
public int Chk(Tst_mgr mgr, String path, Object actl_obj) {
|
||||
Gfo_msg_data actl = (Gfo_msg_data)actl_obj;
|
||||
int rv = 0;
|
||||
rv += mgr.Tst_val(itm == null, path, "itm", itm.Path_str(), actl.Item().Path_str());
|
||||
rv += mgr.Tst_val(excerpt_bgn == -1, path, "excerpt_bgn", excerpt_bgn, actl.Src_bgn());
|
||||
rv += mgr.Tst_val(excerpt_end == -1, path, "excerpt_end", excerpt_end, actl.Src_end());
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.files.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*;
|
||||
import gplx.xowa.wikis.nss.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.htmls.core.bldrs.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.wikis.data.*;
|
||||
class Xob_hdump_tbl_retriever__ns_to_db implements Xob_hdump_tbl_retriever {
|
||||
private final Xob_ns_to_db_mgr ns_to_db_mgr;
|
||||
public Xob_hdump_tbl_retriever__ns_to_db(Xowe_wiki wiki) {
|
||||
Xow_db_mgr core_data_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
|
||||
this.ns_to_db_mgr = new Xob_ns_to_db_mgr(new Xob_ns_to_db_wkr__html(core_data_mgr.Db__core()), core_data_mgr, wiki.Appe().Api_root().Bldr().Wiki().Import().Html_db_max());
|
||||
Xob_ns_file_itm.Init_ns_bldr_data(Xow_db_file_.Tid__html_data, wiki.Ns_mgr(), gplx.xowa.apps.apis.xowa.bldrs.imports.Xoapi_import.Ns_file_map__each);
|
||||
}
|
||||
public Xowd_html_tbl Get_html_tbl(Xow_ns ns, int prv_row_len) {
|
||||
Xow_db_file html_db = ns_to_db_mgr.Get_by_ns(ns.Bldr_data(), prv_row_len); // get html_db
|
||||
return html_db.Tbl__html();
|
||||
}
|
||||
public void Commit() {ns_to_db_mgr.Commit();}
|
||||
public void Rls_all() {ns_to_db_mgr.Rls_all();}
|
||||
}
|
||||
@ -0,0 +1,37 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.inits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xomp_init_cmd extends Xob_cmd__base {
|
||||
private final Xomp_init_mgr mgr;
|
||||
public Xomp_init_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);
|
||||
mgr = new Xomp_init_mgr(wiki);
|
||||
}
|
||||
@Override public void Cmd_run() {
|
||||
wiki.Init_assert();
|
||||
mgr.Exec();
|
||||
}
|
||||
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk__cfg)) return this;
|
||||
else return super.Invk(ctx, ikey, k, m);
|
||||
} private static final String Invk__cfg = "cfg";
|
||||
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;} private static final String BLDR_CMD_KEY = "wiki.mass_parse.init";
|
||||
public static final Xob_cmd Prototype = new Xomp_init_cmd(null, null);
|
||||
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xomp_init_cmd(bldr, wiki);}
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.inits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.bldrs.*;
|
||||
class Xomp_init_mgr {
|
||||
private final Xow_wiki wiki;
|
||||
public Xomp_init_mgr(Xow_wiki wiki) {this.wiki = wiki;}
|
||||
public void Exec() {
|
||||
Xob_db_file make_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
|
||||
Db_conn conn = make_db.Conn();
|
||||
|
||||
// make table
|
||||
conn.Meta_tbl_remake(Dbmeta_tbl_itm.New("mp_page", new Dbmeta_fld_itm[]
|
||||
{ Dbmeta_fld_itm.new_int("page_id").Primary_y_()
|
||||
, Dbmeta_fld_itm.new_bool("page_done")
|
||||
}
|
||||
, Dbmeta_idx_itm.new_normal_by_tbl("mp_page", "page_id__page_done", "page_id", "page_done")
|
||||
));
|
||||
|
||||
// fill table
|
||||
Db_attach_mgr attach_mgr = new Db_attach_mgr(conn, new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
|
||||
int[] ns_ary = new int[] {0, 4, 14};
|
||||
int len = ns_ary.length;
|
||||
String sql = String_.Concat_lines_nl_skip_last
|
||||
( "INSERT INTO mp_page (page_id, page_done)"
|
||||
, "SELECT page_id, 0"
|
||||
, "FROM <page_db>page"
|
||||
, "WHERE page_namespace = {0}"
|
||||
, "AND page_is_redirect = 0"
|
||||
);
|
||||
for (int i = 0; i < len; ++i) {
|
||||
int ns_id = ns_ary[i];
|
||||
attach_mgr.Exec_sql_w_msg("adding rows for mp_page: ns=" + ns_id, sql, ns_id);// ANSI.Y
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.wikis.nss.*; import gplx.xowa.htmls.core.bldrs.*; import gplx.xowa.htmls.core.dbs.*;
|
||||
class Xob_hdump_tbl_retriever__xomp implements Xob_hdump_tbl_retriever {
|
||||
private final Db_conn conn;
|
||||
private final Xowd_html_tbl tbl;
|
||||
public Xob_hdump_tbl_retriever__xomp(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl = new Xowd_html_tbl(conn);
|
||||
conn.Meta_tbl_assert(tbl);
|
||||
}
|
||||
public Xowd_html_tbl Get_html_tbl(Xow_ns ns, int prv_row_len) {
|
||||
return tbl;
|
||||
}
|
||||
public void Commit() {conn.Txn_sav();}
|
||||
public void Rls_all() {conn.Txn_sav(); conn.Rls_conn();}
|
||||
}
|
||||
@ -0,0 +1,37 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
class Xomp_page_itm {
|
||||
public Xomp_page_itm(int id) {this.id = id;}
|
||||
public int Id() {return id;} private final int id;
|
||||
public int Ns_id() {return ns_id;} private int ns_id;
|
||||
public byte[] Ttl_bry() {return ttl_bry;} private byte[] ttl_bry;
|
||||
public int Text_db_id() {return text_db_id;} private int text_db_id;
|
||||
public byte[] Text() {return text;} private byte[] text;
|
||||
|
||||
public void Init_by_page(int ns_id, byte[] ttl_bry, int text_db_id) {
|
||||
this.ns_id = ns_id;
|
||||
this.ttl_bry = ttl_bry;
|
||||
this.text_db_id = text_db_id;
|
||||
}
|
||||
public void Init_by_text(byte[] text) {
|
||||
this.text = text;
|
||||
}
|
||||
|
||||
public static final Xomp_page_itm Null = new Xomp_page_itm(-1);
|
||||
}
|
||||
@ -0,0 +1,84 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
class Xomp_page_pool {
|
||||
private final Object thread_lock = new Object();
|
||||
private Xomp_page_pool_loader loader;
|
||||
private List_adp pool = List_adp_.New(); private int pool_idx = 0, pool_len = 0;
|
||||
private Db_conn make_conn;
|
||||
private final Bry_bfr prog_bfr = Bry_bfr_.New();
|
||||
private int pages_done, pages_total;
|
||||
private long time_bgn, time_prv, time_done;
|
||||
public void Init(Xow_wiki wiki, int num_pages_per_load) {
|
||||
this.make_conn = gplx.xowa.bldrs.Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
|
||||
this.loader = new Xomp_page_pool_loader(wiki, make_conn, num_pages_per_load);
|
||||
this.pages_done = 0;
|
||||
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
|
||||
this.pages_total = loader.Get_pending_count();
|
||||
}
|
||||
public boolean Empty() {return empty;} private boolean empty = false;
|
||||
public void Get_next(List_adp wkr_list, int num_pages_per_wkr) {
|
||||
synchronized (thread_lock) {
|
||||
// pool already marked exhausted by another wkr; return;
|
||||
if (empty) return;
|
||||
int wkr_end = pool_idx + num_pages_per_wkr;
|
||||
|
||||
// need pages to fulfill request
|
||||
if (wkr_end > pool_len) {
|
||||
this.pool = loader.Load(pool, pool_idx, pool_len);
|
||||
this.pool_idx = 0;
|
||||
this.pool_len = pool.Len();
|
||||
if (pool_len == 0) { // no more pages; return;
|
||||
empty = true;
|
||||
return;
|
||||
}
|
||||
wkr_end = num_pages_per_wkr; // recalc wkr_end
|
||||
}
|
||||
|
||||
// reset wkr_end; needed for very last set
|
||||
if (wkr_end >= pool_len)
|
||||
wkr_end = pool_len;
|
||||
|
||||
// add pages to wkr_list
|
||||
for (int i = pool_idx; i < wkr_end; ++i) {
|
||||
Xomp_page_itm page = (Xomp_page_itm)pool.Get_at(i);
|
||||
wkr_list.Add(page);
|
||||
}
|
||||
pool_idx = wkr_end;
|
||||
}
|
||||
}
|
||||
public void Mark_done(int id) {
|
||||
synchronized (thread_lock) {
|
||||
pages_done += 1;
|
||||
if (pages_done % 1000 == 0) {
|
||||
long time_cur = gplx.core.envs.Env_.TickCount();
|
||||
int pages_left = pages_total - pages_done;
|
||||
time_done += (time_cur - time_prv);
|
||||
double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);
|
||||
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)((time_cur - time_bgn) / 1000), true, 0);
|
||||
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)(pages_left / rate_cur), true, 0);
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{0} left=~{1} rate=~{2} time_past=~{3} time_left=~{4}", pages_done, pages_left, (int)rate_cur, time_past, time_left);
|
||||
time_prv = time_cur;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void Rls() {
|
||||
make_conn.Rls_conn();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,87 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
class Xomp_page_pool_loader {
|
||||
private final Xow_wiki wiki;
|
||||
private int prv_page_id = -1;
|
||||
private final Db_conn make_conn;
|
||||
private final int num_pages_per_load;
|
||||
private final Db_attach_mgr attach_mgr;
|
||||
public Xomp_page_pool_loader(Xow_wiki wiki, Db_conn make_conn, int num_pages_per_load) {
|
||||
this.wiki = wiki;
|
||||
this.make_conn = make_conn;
|
||||
this.attach_mgr = new Db_attach_mgr(make_conn);
|
||||
this.num_pages_per_load = num_pages_per_load;
|
||||
}
|
||||
public int Get_pending_count() {
|
||||
Db_rdr rdr = make_conn.Stmt_sql("SELECT Count(*) AS Count_of FROM mp_page mp WHERE mp.page_done = 0").Exec_select__rls_auto();
|
||||
try {
|
||||
return rdr.Move_next() ? rdr.Read_int("Count_of") : 0;
|
||||
} finally {rdr.Rls();}
|
||||
}
|
||||
public List_adp Load(List_adp list, int list_idx, int list_len) {
|
||||
List_adp rv = List_adp_.New();
|
||||
// add remaining pages from old pool to new_pool;
|
||||
for (int i = list_idx; i < list_len; ++i) {
|
||||
rv.Add((Xomp_page_itm)list.Get_at(i));
|
||||
}
|
||||
|
||||
// load pages into new pool
|
||||
this.Load_from_db(rv);
|
||||
return rv;
|
||||
}
|
||||
private void Load_from_db(List_adp list) {
|
||||
// prepare for page_tbl
|
||||
String sql = String_.Format(String_.Concat_lines_nl_skip_last // ANSI.Y
|
||||
( "SELECT mp.page_id"
|
||||
, ", pp.page_namespace"
|
||||
, ", pp.page_title"
|
||||
, ", pp.page_text_db_id"
|
||||
, "FROM mp_page mp"
|
||||
, " JOIN <page_db>page pp ON mp.page_id = pp.page_id"
|
||||
, "WHERE mp.page_id > {0}"
|
||||
, "AND mp.page_done = 0"
|
||||
, "LIMIT {1}"
|
||||
), prv_page_id, num_pages_per_load);
|
||||
this.attach_mgr.Conn_others_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
|
||||
sql = attach_mgr.Resolve_sql(sql);
|
||||
|
||||
// run page_tbl
|
||||
Xomp_text_db_loader text_db_loader = new Xomp_text_db_loader(wiki);
|
||||
attach_mgr.Attach();
|
||||
Db_rdr rdr = make_conn.Stmt_sql(sql).Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
prv_page_id = rdr.Read_int("page_id");
|
||||
int text_db_id = rdr.Read_int("page_text_db_id");
|
||||
Xomp_page_itm ppg = new Xomp_page_itm(prv_page_id);
|
||||
ppg.Init_by_page
|
||||
( rdr.Read_int("page_namespace")
|
||||
, rdr.Read_bry_by_str("page_title")
|
||||
, text_db_id
|
||||
);
|
||||
list.Add(ppg);
|
||||
text_db_loader.Add(text_db_id, ppg);
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
attach_mgr.Detach();
|
||||
|
||||
text_db_loader.Load();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xomp_parse_cmd extends Xob_cmd__base {
|
||||
private final Xomp_parse_mgr mgr = new Xomp_parse_mgr();
|
||||
public Xomp_parse_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
|
||||
@Override public void Cmd_run() {
|
||||
wiki.Init_assert();
|
||||
mgr.Run(wiki);
|
||||
}
|
||||
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk__cfg)) return mgr.Cfg();
|
||||
else return super.Invk(ctx, ikey, k, m);
|
||||
} private static final String Invk__cfg = "cfg";
|
||||
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;} private static final String BLDR_CMD_KEY = "wiki.mass_parse.exec";
|
||||
public static final Xob_cmd Prototype = new Xomp_parse_cmd(null, null);
|
||||
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xomp_parse_cmd(bldr, wiki);}
|
||||
}
|
||||
@ -0,0 +1,68 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.core.threads.*;
|
||||
import gplx.xowa.langs.*;
|
||||
class Xomp_parse_mgr {
|
||||
private final Xomp_page_pool page_pool = new Xomp_page_pool();
|
||||
public Xomp_parse_mgr_cfg Cfg() {return cfg;} private final Xomp_parse_mgr_cfg cfg = new Xomp_parse_mgr_cfg();
|
||||
private int wkrs_done;
|
||||
public void Wkrs_done_add_1() {synchronized (page_pool) {++wkrs_done;}}
|
||||
public void Run(Xowe_wiki wiki) {
|
||||
// init pool
|
||||
cfg.Init(wiki);
|
||||
page_pool.Init(wiki, cfg.Num_pages_in_pool());
|
||||
|
||||
// init threads
|
||||
int wkr_len = cfg.Num_wkrs();
|
||||
Xomp_parse_wkr[] wkrs = new Xomp_parse_wkr[wkr_len];
|
||||
for (int i = 0; i < wkr_len; ++i) {
|
||||
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, Clone_wiki(wiki), page_pool, i, cfg.Num_pages_per_wkr());
|
||||
wkrs[i] = wkr;
|
||||
}
|
||||
|
||||
// start threads; done separately b/c thread issues when done right after init
|
||||
for (int i = 0; i < wkr_len; ++i) {
|
||||
Xomp_parse_wkr wkr = wkrs[i];
|
||||
Thread_adp_.Start_by_key("xomp." + Int_.To_str_fmt(i, "000"), Cancelable_.Never, wkr, Xomp_parse_wkr.Invk__exec);
|
||||
}
|
||||
|
||||
// wait until wkrs are wkrs_done
|
||||
while (true) {
|
||||
synchronized (page_pool) {
|
||||
if (wkrs_done == wkr_len) break;
|
||||
}
|
||||
Thread_adp_.Sleep(1000);
|
||||
}
|
||||
page_pool.Rls();
|
||||
|
||||
// print stats
|
||||
Bry_bfr bfr = Bry_bfr_.New();
|
||||
for (int i = 0; i < wkr_len; ++i) {
|
||||
wkrs[i].Bld_stats(bfr);
|
||||
}
|
||||
Gfo_usr_dlg_.Instance.Note_many("", "", bfr.To_str_and_clear());
|
||||
}
|
||||
private static Xowe_wiki Clone_wiki(Xowe_wiki wiki) {
|
||||
Xol_lang_itm lang = new Xol_lang_itm(wiki.App().Lang_mgr(), wiki.Lang().Key_bry());
|
||||
Xol_lang_itm_.Lang_init(lang);
|
||||
Xowe_wiki rv = new Xowe_wiki(wiki.Appe(), lang, wiki.Ns_mgr(), wiki.Domain_itm(), wiki.Fsys_mgr().Root_dir());
|
||||
rv.Init_by_wiki();
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
class Xomp_parse_mgr_cfg implements Gfo_invk {
|
||||
public Io_url Root_dir() {return root_dir;} private Io_url root_dir;
|
||||
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1;
|
||||
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = 1000;
|
||||
public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000;
|
||||
public void Init(Xowe_wiki wiki) {
|
||||
if (root_dir == null) root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
|
||||
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Env_.System_cpu_count();
|
||||
}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk__num_wkrs_)) num_wkrs = m.ReadInt("v");
|
||||
else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v");
|
||||
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
|
||||
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
|
||||
else if (ctx.Match(k, Invk__root_dir_)) root_dir = m.ReadIoUrl("v");
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
}
|
||||
private static final String Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
|
||||
, Invk__root_dir_ = "root_dir_"
|
||||
;
|
||||
}
|
||||
@ -0,0 +1,117 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.htmls.core.bldrs.*;
|
||||
import gplx.xowa.parsers.*;
|
||||
class Xomp_parse_wkr implements Gfo_invk {
|
||||
private final Xomp_parse_mgr mgr;
|
||||
private final Xowe_wiki wiki;
|
||||
private final Xomp_page_pool page_pool;
|
||||
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
|
||||
private final int num_pages_per_wkr;
|
||||
private final int idx;
|
||||
private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
|
||||
// private boolean gen_html = true;// gen_hdump = true;
|
||||
private int done_count; private long done_time;
|
||||
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xomp_page_pool page_pool, int idx, int num_pages_per_wkr) {
|
||||
this.mgr = mgr; this.wiki = wiki;
|
||||
this.page_pool = page_pool; this.num_pages_per_wkr = num_pages_per_wkr;
|
||||
this.idx = idx;
|
||||
}
|
||||
public void Exec() {
|
||||
// init
|
||||
Db_conn wkr_conn = Db_conn_bldr.Instance.Get_or_autocreate(true, mgr.Cfg().Root_dir().GenSubFil_nest("xomp_" + Int_.To_str_fmt(idx, "000"), "xomp_wkr.sqlite3"));
|
||||
Xow_parser_mgr parser = new Xow_parser_mgr(wiki);
|
||||
wiki.Html_mgr().Page_wtr_mgr().Wkr(gplx.xowa.wikis.pages.Xopg_page_.Tid_read).Ctgs_enabled_(false); // disable categories else progress messages written (also for PERF)
|
||||
if (wiki.File__bin_mgr() != null)
|
||||
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // remove wmf wkr, else will try to download images during parsing
|
||||
hdump_bldr.Init(wiki, wkr_conn, new Xob_hdump_tbl_retriever__xomp(wkr_conn));
|
||||
|
||||
while (true) {
|
||||
Xomp_page_itm ppg = Get_next(); if (ppg == Xomp_page_itm.Null) break; // no more pages
|
||||
if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05
|
||||
|
||||
try {
|
||||
// init page
|
||||
long done_bgn = gplx.core.envs.Env_.TickCount();
|
||||
Xoa_ttl ttl = wiki.Ttl_parse(ppg.Ns_id(), ppg.Ttl_bry());
|
||||
Xoae_page wpg = Xoae_page.New(wiki, ttl);
|
||||
wpg.Db().Text().Text_bry_(ppg.Text());
|
||||
wpg.Db().Page().Id_(ppg.Id());
|
||||
|
||||
// parse page
|
||||
parser.Ctx().Clear_all();
|
||||
parser.Parse(wpg, true);
|
||||
|
||||
// gen_html
|
||||
// if ( gen_html
|
||||
// && wpg.Redirect().Itms__len() == 0) // don't generate html for redirected pages
|
||||
// wiki.Html_mgr().Page_wtr_mgr().Gen(wpg, gplx.xowa.wikis.pages.Xopg_page_.Tid_read);
|
||||
// if (gen_hdump)
|
||||
// hdump_bldr.Insert(wpg);
|
||||
|
||||
// mark done for sake of progress
|
||||
page_pool.Mark_done(ppg.Id());
|
||||
|
||||
// update stats
|
||||
long time_cur = gplx.core.envs.Env_.TickCount();
|
||||
done_time += time_cur - done_bgn;
|
||||
done_bgn = time_cur;
|
||||
++done_count;
|
||||
|
||||
// cleanup
|
||||
// ctx.App().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released
|
||||
if (wiki.Cache_mgr().Tmpl_result_cache().Count() > 50000)
|
||||
wiki.Cache_mgr().Tmpl_result_cache().Clear();
|
||||
if (done_count % 50 == 0) {
|
||||
wiki.Cache_mgr().Free_mem_all();
|
||||
wiki.Parser_mgr().Scrib().Core_term();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e));
|
||||
}
|
||||
}
|
||||
mgr.Wkrs_done_add_1();
|
||||
wkr_conn.Rls_conn();
|
||||
}
|
||||
public void Bld_stats(Bry_bfr bfr) {
|
||||
int done_time_in_sec = (int)(done_time / 1000); if (done_time_in_sec == 0) done_time_in_sec = 1;
|
||||
bfr.Add_int_pad_bgn(Byte_ascii.Space, 4, idx );
|
||||
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, (int)(done_count / done_time_in_sec));
|
||||
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_count);
|
||||
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_time_in_sec);
|
||||
bfr.Add_byte_nl();
|
||||
}
|
||||
private Xomp_page_itm Get_next() {
|
||||
if (list_idx == list_len) {
|
||||
list.Clear();
|
||||
page_pool.Get_next(list, num_pages_per_wkr);
|
||||
list_len = list.Len();
|
||||
if (list_len == 0) return Xomp_page_itm.Null;
|
||||
list_idx = 0;
|
||||
}
|
||||
return (Xomp_page_itm)list.Get_at(list_idx++);
|
||||
}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk__exec)) this.Exec();
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
}
|
||||
public static final String Invk__exec = "exec";
|
||||
}
|
||||
@ -0,0 +1,94 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.core.ios.*;
|
||||
class Xomp_text_db_loader {
|
||||
private final Xow_wiki wiki;
|
||||
private final Ordered_hash text_db_hash = Ordered_hash_.New();
|
||||
private final Io_stream_zip_mgr zip_mgr = Xoa_app_.Utl__zip_mgr();
|
||||
public Xomp_text_db_loader(Xow_wiki wiki) {
|
||||
this.wiki = wiki;
|
||||
}
|
||||
public void Add(int text_db_id, Xomp_page_itm ppg) {
|
||||
Xomp_text_db_itm itm = (Xomp_text_db_itm)text_db_hash.Get_by(text_db_id);
|
||||
if (itm == null) {
|
||||
itm = new Xomp_text_db_itm(text_db_id);
|
||||
text_db_hash.Add(text_db_id, itm);
|
||||
}
|
||||
itm.Page_list().Add(ppg);
|
||||
}
|
||||
public void Load() {
|
||||
int text_db_hash_len = text_db_hash.Len();
|
||||
for (int i = 0; i < text_db_hash_len; ++i) {
|
||||
Xomp_text_db_itm itm = (Xomp_text_db_itm)text_db_hash.Get_at(i);
|
||||
Load_list(itm.Text_db_id(), itm.Page_list());
|
||||
}
|
||||
}
|
||||
private void Load_list(int text_db_id, List_adp list) {
|
||||
int list_len = list.Len();
|
||||
int batch_idx = 0;
|
||||
Bry_bfr bry = Bry_bfr_.New();
|
||||
Ordered_hash page_hash = Ordered_hash_.New();
|
||||
byte zip_tid = wiki.Data__core_mgr().Props().Zip_tid_text();
|
||||
for (int i = 0; i < list_len; ++i) {
|
||||
if (batch_idx == 0) {
|
||||
page_hash.Clear();
|
||||
bry.Add_str_a7("SELECT page_id, text_data FROM text WHERE page_id IN (");
|
||||
}
|
||||
|
||||
// build WHERE IN for page_ids; EX: "1, 2, 3, 4"
|
||||
Xomp_page_itm ppg = (Xomp_page_itm)list.Get_at(i);
|
||||
int page_id = ppg.Id();
|
||||
if (batch_idx != 0) bry.Add_byte_comma();
|
||||
bry.Add_int_variable(page_id);
|
||||
page_hash.Add(page_id, ppg);
|
||||
++batch_idx;
|
||||
|
||||
// load if 255 in list, or last
|
||||
if ( batch_idx % 255 == 0
|
||||
|| i == list_len - 1) {
|
||||
bry.Add_byte(Byte_ascii.Paren_end);
|
||||
Load_from_text_db(page_hash, zip_tid, text_db_id, bry.To_str_and_clear());
|
||||
batch_idx = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
private void Load_from_text_db(Ordered_hash page_hash, byte zip_tid, int text_db_id, String sql) {
|
||||
Db_conn text_conn = wiki.Data__core_mgr().Dbs__get_by_id_or_fail(text_db_id).Conn();
|
||||
Db_rdr rdr = text_conn.Stmt_sql(sql).Exec_select__rls_auto(); // ANSI.Y
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
int page_id = rdr.Read_int("page_id");
|
||||
byte[] text_data = rdr.Read_bry("text_data");
|
||||
text_data = zip_mgr.Unzip(zip_tid, text_data);
|
||||
Xomp_page_itm ppg = (Xomp_page_itm)page_hash.Get_by(page_id);
|
||||
ppg.Init_by_text(text_data);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
rdr.Rls();
|
||||
// text_conn.Rls_conn(); // TOMBSTONE: causes strange errors in tables; DATE:2016-07-06
|
||||
}
|
||||
}
|
||||
}
|
||||
class Xomp_text_db_itm {
|
||||
public Xomp_text_db_itm(int text_db_id) {this.text_db_id = text_db_id;}
|
||||
public int Text_db_id() {return text_db_id;} private final int text_db_id;
|
||||
public List_adp Page_list() {return page_list;} private final List_adp page_list = List_adp_.New();
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue