Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.7.2.1

Browse Source
This commit is contained in:
gnosygnu
2015-07-12 21:10:02 -04:00
commit 794b5a232f
3099 changed files with 238212 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
400_xowa/src/gplx/xowa/parsers/amps/Xop_amp_lxr.java Normal file
View File

@@ -0,0 +1,28 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.amps; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*;
public class Xop_amp_lxr implements Xop_lxr {
public byte Lxr_tid() {return Xop_lxr_.Tid_amp;}
public void Init_by_wiki(Xowe_wiki wiki, Btrie_fast_mgr core_trie) {core_trie.Add(Byte_ascii.Amp, this);}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
return ctx.Amp().Make_tkn(ctx, tkn_mkr, root, src, src_len, bgn_pos, cur_pos);
}
public static final Xop_amp_lxr _ = new Xop_amp_lxr();
}

121
400_xowa/src/gplx/xowa/parsers/amps/Xop_amp_mgr.java Normal file
View File

@@ -0,0 +1,121 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.amps; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*;
public class Xop_amp_mgr {
private final Bry_bfr tmp_bfr = Bry_bfr.reset_(32);
public Btrie_slim_mgr Amp_trie() {return amp_trie;} private final Btrie_slim_mgr amp_trie = Xop_amp_trie._;
public int Rslt_pos() {return rslt_pos;} private int rslt_pos;
public int Rslt_val() {return rslt_val;} private int rslt_val;
public Xop_tkn_itm Parse_as_tkn(Xop_tkn_mkr tkn_mkr, byte[] src, int src_len, int amp_pos, int cur_pos) {
rslt_pos = amp_pos + 1; // default to fail pos; after amp;
Object o = amp_trie.Match_bgn(src, cur_pos, src_len);
cur_pos = amp_trie.Match_pos();
if (o == null) return null;
Xop_amp_trie_itm itm = (Xop_amp_trie_itm)o;
switch (itm.Tid()) {
case Xop_amp_trie_itm.Tid_name_std:
case Xop_amp_trie_itm.Tid_name_xowa:
rslt_pos = cur_pos;
return tkn_mkr.Amp_txt(amp_pos, cur_pos, itm);
case Xop_amp_trie_itm.Tid_num_hex:
case Xop_amp_trie_itm.Tid_num_dec:
boolean ncr_is_hex = itm.Tid() == Xop_amp_trie_itm.Tid_num_hex;
boolean pass = Parse_as_int(ncr_is_hex, src, src_len, amp_pos, cur_pos);
return pass ? tkn_mkr.Amp_num(amp_pos, rslt_pos, rslt_val) : null;
default: throw Exc_.new_unhandled(itm.Tid());
}
}
public boolean Parse_as_int(boolean ncr_is_hex, byte[] src, int src_len, int amp_pos, int int_bgn) {
rslt_pos = amp_pos + 1; // default to fail pos; after amp;
rslt_val = -1; // clear any previous setting
int cur_pos = int_bgn, int_end = -1;
int semic_pos = Bry_finder.Find_fwd(src, Byte_ascii.Semic, cur_pos, src_len);
if (semic_pos == Bry_finder.Not_found) return false;
int_end = semic_pos - 1; // int_end = pos before semicolon
int multiple = ncr_is_hex ? 16 : 10, val = 0, factor = 1, cur = 0;
for (int i = int_end; i >= int_bgn; i--) {
byte b = src[i];
if (ncr_is_hex) {
if (b >= 48 && b <= 57) cur = b - 48;
else if (b >= 65 && b <= 70) cur = b - 55;
else if (b >= 97 && b <= 102) cur = b - 87;
else if((b >= 71 && b <= 90)
|| (b >= 91 && b <= 122)) continue; // NOTE: wiki discards letters G-Z; PAGE:en.w:Miscellaneous_Symbols "{{Unicode|&#xx26D0;}}"; NOTE 2nd x is discarded
else return false;
}
else {
cur = b - Byte_ascii.Num_0;
if (cur < 0 || cur > 10) return false;
}
val += cur * factor;
if (val > gplx.intl.Utf8_.Codepoint_max) return false; // fail if value > largest_unicode_codepoint
factor *= multiple;
}
rslt_val = val;
rslt_pos = semic_pos + 1; // position after semic
return true;
}
public byte[] Decode_as_bry(byte[] src) {
if (src == null) return src;
int src_len = src.length;
boolean dirty = false;
int pos = 0;
while (pos < src_len) {
byte b = src[pos];
if (b == Byte_ascii.Amp) {
int nxt_pos = pos + 1;
if (nxt_pos < src_len) {
byte nxt_b = src[nxt_pos];
Object amp_obj = amp_trie.Match_bgn_w_byte(nxt_b, src, nxt_pos, src_len);
if (amp_obj != null) {
if (!dirty) {
tmp_bfr.Add_mid(src, 0, pos);
dirty = true;
}
Xop_amp_trie_itm amp_itm = (Xop_amp_trie_itm)amp_obj;
switch (amp_itm.Tid()) {
case Xop_amp_trie_itm.Tid_name_std:
case Xop_amp_trie_itm.Tid_name_xowa:
tmp_bfr.Add(amp_itm.Utf8_bry());
pos = amp_trie.Match_pos();
break;
case Xop_amp_trie_itm.Tid_num_hex:
case Xop_amp_trie_itm.Tid_num_dec:
boolean ncr_is_hex = amp_itm.Tid() == Xop_amp_trie_itm.Tid_num_hex;
int int_bgn = amp_trie.Match_pos();
if (Parse_as_int(ncr_is_hex, src, src_len, pos, int_bgn))
tmp_bfr.Add_u8_int(rslt_val);
else
tmp_bfr.Add_mid(src, pos, nxt_pos);
pos = rslt_pos;
break;
default:
throw Exc_.new_unhandled(amp_itm.Tid());
}
continue;
}
}
}
if (dirty)
tmp_bfr.Add_byte(b);
++pos;
}
return dirty ? tmp_bfr.Xto_bry_and_clear() : src;
}
}

44
400_xowa/src/gplx/xowa/parsers/amps/Xop_amp_mgr_decode_tst.java Normal file
View File

@@ -0,0 +1,44 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.amps; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import org.junit.*;
public class Xop_amp_mgr_decode_tst {
@Before public void init() {fxt.Reset();} private Xop_amp_mgr_fxt fxt = new Xop_amp_mgr_fxt();
@Test public void Text() {fxt.Test_decode_as_bry("a" , "a");}
@Test public void Name() {fxt.Test_decode_as_bry("&amp;" , "&");}
@Test public void Name_w_text() {fxt.Test_decode_as_bry("a&amp;b" , "a&b");}
@Test public void Name_fail_semic_missing() {fxt.Test_decode_as_bry("a&ampb" , "a&ampb");}
@Test public void Name_fail_amp_only() {fxt.Test_decode_as_bry("a&" , "a&");}
@Test public void Num_fail() {fxt.Test_decode_as_bry("&#!;" , "&#!;");} // ! is not valid num
@Test public void Hex_fail() {fxt.Test_decode_as_bry("&#x!;" , "&#x!;");} // ! is not valid hex
@Test public void Num_basic() {fxt.Test_decode_as_bry("&#0931;" , "Σ");}
@Test public void Num_zero_padded() {fxt.Test_decode_as_bry("&#00931;" , "Σ");}
@Test public void Hex_upper() {fxt.Test_decode_as_bry("&#x3A3;" , "Σ");}
@Test public void Hex_lower() {fxt.Test_decode_as_bry("&#x3a3;" , "Σ");}
@Test public void Hex_zero_padded() {fxt.Test_decode_as_bry("&#x03a3;" , "Σ");}
@Test public void Hex_upper_x() {fxt.Test_decode_as_bry("&#X3A3;" , "Σ");}
@Test public void Num_fail_large_codepoint() {fxt.Test_decode_as_bry("&#538189831;" , "&#538189831;");}
@Test public void Num_ignore_extra_x() {fxt.Test_decode_as_bry("&#xx26D0;" , Char_.XtoStr(Char_.XbyInt(9936)));} // 2nd x is ignored
}
class Xop_amp_mgr_fxt {
private Xop_amp_mgr amp_mgr = new Xop_amp_mgr();
public void Reset() {}
public void Test_decode_as_bry(String raw, String expd) {
Tfds.Eq(expd, String_.new_u8(amp_mgr.Decode_as_bry(Bry_.new_u8(raw))));
}
}

27
400_xowa/src/gplx/xowa/parsers/amps/Xop_amp_tkn_num.java Normal file
View File

@@ -0,0 +1,27 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.amps; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
public class Xop_amp_tkn_num extends Xop_tkn_itm_base {
public Xop_amp_tkn_num(int bgn, int end, int val, byte[] str_as_bry) {
this.val = val; this.str_as_bry = str_as_bry;
this.Tkn_ini_pos(false, bgn, end);
}
@Override public byte Tkn_tid() {return Xop_tkn_itm_.Tid_html_ncr;}
public int Val() {return val;} private int val;
public byte[] Str_as_bry() {return str_as_bry;} private byte[] str_as_bry;
}

31
400_xowa/src/gplx/xowa/parsers/amps/Xop_amp_tkn_txt.java Normal file
View File

@@ -0,0 +1,31 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.amps; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
public class Xop_amp_tkn_txt extends Xop_tkn_itm_base {
private Xop_amp_trie_itm html_ref_itm;
public Xop_amp_tkn_txt(int bgn, int end, Xop_amp_trie_itm html_ref_itm) {
this.html_ref_itm = html_ref_itm;
this.Tkn_ini_pos(false, bgn, end);
}
@Override public byte Tkn_tid() {return Xop_tkn_itm_.Tid_html_ref;}
public int Char_int() {return html_ref_itm.Char_int();}
public byte[] Xml_name_bry() {return html_ref_itm.Xml_name_bry();}
public boolean Itm_is_custom() {return html_ref_itm.Tid() == Xop_amp_trie_itm.Tid_name_xowa;}
public void Print_ncr(Bry_bfr bfr) {html_ref_itm.Print_ncr(bfr);}
public void Print_literal(Bry_bfr bfr) {html_ref_itm.Print_literal(bfr);}
}

318
400_xowa/src/gplx/xowa/parsers/amps/Xop_amp_trie.java Normal file
View File

@@ -0,0 +1,318 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.amps; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*;
public class Xop_amp_trie {
public static final byte[] // NOTE: top_define
Bry_xowa_lt = Bry_.new_a7("&xowa_lt;")
, Bry_xowa_brack_bgn = Bry_.new_a7("&xowa_brack_bgn;")
, Bry_xowa_brack_end = Bry_.new_a7("&xowa_brack_end;")
, Bry_xowa_pipe = Bry_.new_a7("&xowa_pipe;")
, Bry_xowa_apos = Bry_.new_a7("&xowa_apos;")
, Bry_xowa_colon = Bry_.new_a7("&xowa_colon;")
, Bry_xowa_underline = Bry_.new_a7("&xowa_underline;")
, Bry_xowa_asterisk = Bry_.new_a7("&xowa_asterisk;")
, Bry_xowa_space = Bry_.new_a7("&xowa_space;")
, Bry_xowa_nl = Bry_.new_a7("&xowa_nl;")
, Bry_xowa_dash = Bry_.new_a7("&xowa_dash;")
;
public static final Btrie_slim_mgr _ = new_(); Xop_amp_trie() {}
private static Btrie_slim_mgr new_() {// REF.MW: Sanitizer|$wgHtmlEntities; NOTE:added apos
Btrie_slim_mgr rv = Btrie_slim_mgr.cs_();
Reg_name(rv, Bool_.Y, 60, Bry_xowa_lt);
Reg_name(rv, Bool_.Y, 91, Bry_xowa_brack_bgn);
Reg_name(rv, Bool_.Y, 93, Bry_xowa_brack_end);
Reg_name(rv, Bool_.Y, 124, Bry_xowa_pipe);
Reg_name(rv, Bool_.Y, 39, Bry_xowa_apos);
Reg_name(rv, Bool_.Y, 58, Bry_xowa_colon);
Reg_name(rv, Bool_.Y, 95, Bry_xowa_underline);
Reg_name(rv, Bool_.Y, 42, Bry_xowa_asterisk);
Reg_name(rv, Bool_.Y, 32, Bry_xowa_space);
Reg_name(rv, Bool_.Y, 10, Bry_xowa_nl);
Reg_name(rv, Bool_.Y, 45, Bry_xowa_dash);
Reg_name(rv, Bool_.N, 39, "&apos;");
Reg_name(rv, Bool_.N, 193, "&Aacute;");
Reg_name(rv, Bool_.N, 225, "&aacute;");
Reg_name(rv, Bool_.N, 194, "&Acirc;");
Reg_name(rv, Bool_.N, 226, "&acirc;");
Reg_name(rv, Bool_.N, 180, "&acute;");
Reg_name(rv, Bool_.N, 198, "&AElig;");
Reg_name(rv, Bool_.N, 230, "&aelig;");
Reg_name(rv, Bool_.N, 192, "&Agrave;");
Reg_name(rv, Bool_.N, 224, "&agrave;");
Reg_name(rv, Bool_.N, 8501, "&alefsym;");
Reg_name(rv, Bool_.N, 913, "&Alpha;");
Reg_name(rv, Bool_.N, 945, "&alpha;");
Reg_name(rv, Bool_.N, 38, "&amp;");
Reg_name(rv, Bool_.N, 8743, "&and;");
Reg_name(rv, Bool_.N, 8736, "&ang;");
Reg_name(rv, Bool_.N, 197, "&Aring;");
Reg_name(rv, Bool_.N, 229, "&aring;");
Reg_name(rv, Bool_.N, 8776, "&asymp;");
Reg_name(rv, Bool_.N, 195, "&Atilde;");
Reg_name(rv, Bool_.N, 227, "&atilde;");
Reg_name(rv, Bool_.N, 196, "&Auml;");
Reg_name(rv, Bool_.N, 228, "&auml;");
Reg_name(rv, Bool_.N, 8222, "&bdquo;");
Reg_name(rv, Bool_.N, 914, "&Beta;");
Reg_name(rv, Bool_.N, 946, "&beta;");
Reg_name(rv, Bool_.N, 166, "&brvbar;");
Reg_name(rv, Bool_.N, 8226, "&bull;");
Reg_name(rv, Bool_.N, 8745, "&cap;");
Reg_name(rv, Bool_.N, 199, "&Ccedil;");
Reg_name(rv, Bool_.N, 231, "&ccedil;");
Reg_name(rv, Bool_.N, 184, "&cedil;");
Reg_name(rv, Bool_.N, 162, "&cent;");
Reg_name(rv, Bool_.N, 935, "&Chi;");
Reg_name(rv, Bool_.N, 967, "&chi;");
Reg_name(rv, Bool_.N, 710, "&circ;");
Reg_name(rv, Bool_.N, 9827, "&clubs;");
Reg_name(rv, Bool_.N, 8773, "&cong;");
Reg_name(rv, Bool_.N, 169, "&copy;");
Reg_name(rv, Bool_.N, 8629, "&crarr;");
Reg_name(rv, Bool_.N, 8746, "&cup;");
Reg_name(rv, Bool_.N, 164, "&curren;");
Reg_name(rv, Bool_.N, 8224, "&dagger;");
Reg_name(rv, Bool_.N, 8225, "&Dagger;");
Reg_name(rv, Bool_.N, 8595, "&darr;");
Reg_name(rv, Bool_.N, 8659, "&dArr;");
Reg_name(rv, Bool_.N, 176, "&deg;");
Reg_name(rv, Bool_.N, 916, "&Delta;");
Reg_name(rv, Bool_.N, 948, "&delta;");
Reg_name(rv, Bool_.N, 9830, "&diams;");
Reg_name(rv, Bool_.N, 247, "&divide;");
Reg_name(rv, Bool_.N, 201, "&Eacute;");
Reg_name(rv, Bool_.N, 233, "&eacute;");
Reg_name(rv, Bool_.N, 202, "&Ecirc;");
Reg_name(rv, Bool_.N, 234, "&ecirc;");
Reg_name(rv, Bool_.N, 200, "&Egrave;");
Reg_name(rv, Bool_.N, 232, "&egrave;");
Reg_name(rv, Bool_.N, 8709, "&empty;");
Reg_name(rv, Bool_.N, 8195, "&emsp;");
Reg_name(rv, Bool_.N, 8194, "&ensp;");
Reg_name(rv, Bool_.N, 917, "&Epsilon;");
Reg_name(rv, Bool_.N, 949, "&epsilon;");
Reg_name(rv, Bool_.N, 8801, "&equiv;");
Reg_name(rv, Bool_.N, 919, "&Eta;");
Reg_name(rv, Bool_.N, 951, "&eta;");
Reg_name(rv, Bool_.N, 208, "&ETH;");
Reg_name(rv, Bool_.N, 240, "&eth;");
Reg_name(rv, Bool_.N, 203, "&Euml;");
Reg_name(rv, Bool_.N, 235, "&euml;");
Reg_name(rv, Bool_.N, 8364, "&euro;");
Reg_name(rv, Bool_.N, 8707, "&exist;");
Reg_name(rv, Bool_.N, 402, "&fnof;");
Reg_name(rv, Bool_.N, 8704, "&forall;");
Reg_name(rv, Bool_.N, 189, "&frac12;");
Reg_name(rv, Bool_.N, 188, "&frac14;");
Reg_name(rv, Bool_.N, 190, "&frac34;");
Reg_name(rv, Bool_.N, 8260, "&frasl;");
Reg_name(rv, Bool_.N, 915, "&Gamma;");
Reg_name(rv, Bool_.N, 947, "&gamma;");
Reg_name(rv, Bool_.N, 8805, "&ge;");
Reg_name(rv, Bool_.N, 62, "&gt;");
Reg_name(rv, Bool_.N, 8596, "&harr;");
Reg_name(rv, Bool_.N, 8660, "&hArr;");
Reg_name(rv, Bool_.N, 9829, "&hearts;");
Reg_name(rv, Bool_.N, 8230, "&hellip;");
Reg_name(rv, Bool_.N, 205, "&Iacute;");
Reg_name(rv, Bool_.N, 237, "&iacute;");
Reg_name(rv, Bool_.N, 206, "&Icirc;");
Reg_name(rv, Bool_.N, 238, "&icirc;");
Reg_name(rv, Bool_.N, 161, "&iexcl;");
Reg_name(rv, Bool_.N, 204, "&Igrave;");
Reg_name(rv, Bool_.N, 236, "&igrave;");
Reg_name(rv, Bool_.N, 8465, "&image;");
Reg_name(rv, Bool_.N, 8734, "&infin;");
Reg_name(rv, Bool_.N, 8747, "&int;");
Reg_name(rv, Bool_.N, 921, "&Iota;");
Reg_name(rv, Bool_.N, 953, "&iota;");
Reg_name(rv, Bool_.N, 191, "&iquest;");
Reg_name(rv, Bool_.N, 8712, "&isin;");
Reg_name(rv, Bool_.N, 207, "&Iuml;");
Reg_name(rv, Bool_.N, 239, "&iuml;");
Reg_name(rv, Bool_.N, 922, "&Kappa;");
Reg_name(rv, Bool_.N, 954, "&kappa;");
Reg_name(rv, Bool_.N, 923, "&Lambda;");
Reg_name(rv, Bool_.N, 955, "&lambda;");
Reg_name(rv, Bool_.N, 9001, "&lang;");
Reg_name(rv, Bool_.N, 171, "&laquo;");
Reg_name(rv, Bool_.N, 8592, "&larr;");
Reg_name(rv, Bool_.N, 8656, "&lArr;");
Reg_name(rv, Bool_.N, 8968, "&lceil;");
Reg_name(rv, Bool_.N, 8220, "&ldquo;");
Reg_name(rv, Bool_.N, 8804, "&le;");
Reg_name(rv, Bool_.N, 8970, "&lfloor;");
Reg_name(rv, Bool_.N, 8727, "&lowast;");
Reg_name(rv, Bool_.N, 9674, "&loz;");
Reg_name(rv, Bool_.N, 8206, "&lrm;");
Reg_name(rv, Bool_.N, 8249, "&lsaquo;");
Reg_name(rv, Bool_.N, 8216, "&lsquo;");
Reg_name(rv, Bool_.N, 60, "&lt;");
Reg_name(rv, Bool_.N, 175, "&macr;");
Reg_name(rv, Bool_.N, 8212, "&mdash;");
Reg_name(rv, Bool_.N, 181, "&micro;");
Reg_name(rv, Bool_.N, 183, "&middot;");
Reg_name(rv, Bool_.N, 8722, "&minus;");
Reg_name(rv, Bool_.N, 924, "&Mu;");
Reg_name(rv, Bool_.N, 956, "&mu;");
Reg_name(rv, Bool_.N, 8711, "&nabla;");
Reg_name(rv, Bool_.N, 160, "&nbsp;");
Reg_name(rv, Bool_.N, 8211, "&ndash;");
Reg_name(rv, Bool_.N, 8800, "&ne;");
Reg_name(rv, Bool_.N, 8715, "&ni;");
Reg_name(rv, Bool_.N, 172, "&not;");
Reg_name(rv, Bool_.N, 8713, "&notin;");
Reg_name(rv, Bool_.N, 8836, "&nsub;");
Reg_name(rv, Bool_.N, 209, "&Ntilde;");
Reg_name(rv, Bool_.N, 241, "&ntilde;");
Reg_name(rv, Bool_.N, 925, "&Nu;");
Reg_name(rv, Bool_.N, 957, "&nu;");
Reg_name(rv, Bool_.N, 211, "&Oacute;");
Reg_name(rv, Bool_.N, 243, "&oacute;");
Reg_name(rv, Bool_.N, 212, "&Ocirc;");
Reg_name(rv, Bool_.N, 244, "&ocirc;");
Reg_name(rv, Bool_.N, 338, "&OElig;");
Reg_name(rv, Bool_.N, 339, "&oelig;");
Reg_name(rv, Bool_.N, 210, "&Ograve;");
Reg_name(rv, Bool_.N, 242, "&ograve;");
Reg_name(rv, Bool_.N, 8254, "&oline;");
Reg_name(rv, Bool_.N, 937, "&Omega;");
Reg_name(rv, Bool_.N, 969, "&omega;");
Reg_name(rv, Bool_.N, 927, "&Omicron;");
Reg_name(rv, Bool_.N, 959, "&omicron;");
Reg_name(rv, Bool_.N, 8853, "&oplus;");
Reg_name(rv, Bool_.N, 8744, "&or;");
Reg_name(rv, Bool_.N, 170, "&ordf;");
Reg_name(rv, Bool_.N, 186, "&ordm;");
Reg_name(rv, Bool_.N, 216, "&Oslash;");
Reg_name(rv, Bool_.N, 248, "&oslash;");
Reg_name(rv, Bool_.N, 213, "&Otilde;");
Reg_name(rv, Bool_.N, 245, "&otilde;");
Reg_name(rv, Bool_.N, 8855, "&otimes;");
Reg_name(rv, Bool_.N, 214, "&Ouml;");
Reg_name(rv, Bool_.N, 246, "&ouml;");
Reg_name(rv, Bool_.N, 182, "&para;");
Reg_name(rv, Bool_.N, 8706, "&part;");
Reg_name(rv, Bool_.N, 8240, "&permil;");
Reg_name(rv, Bool_.N, 8869, "&perp;");
Reg_name(rv, Bool_.N, 934, "&Phi;");
Reg_name(rv, Bool_.N, 966, "&phi;");
Reg_name(rv, Bool_.N, 928, "&Pi;");
Reg_name(rv, Bool_.N, 960, "&pi;");
Reg_name(rv, Bool_.N, 982, "&piv;");
Reg_name(rv, Bool_.N, 177, "&plusmn;");
Reg_name(rv, Bool_.N, 163, "&pound;");
Reg_name(rv, Bool_.N, 8242, "&prime;");
Reg_name(rv, Bool_.N, 8243, "&Prime;");
Reg_name(rv, Bool_.N, 8719, "&prod;");
Reg_name(rv, Bool_.N, 8733, "&prop;");
Reg_name(rv, Bool_.N, 936, "&Psi;");
Reg_name(rv, Bool_.N, 968, "&psi;");
Reg_name(rv, Bool_.N, 34, "&quot;");
Reg_name(rv, Bool_.N, 8730, "&radic;");
Reg_name(rv, Bool_.N, 9002, "&rang;");
Reg_name(rv, Bool_.N, 187, "&raquo;");
Reg_name(rv, Bool_.N, 8594, "&rarr;");
Reg_name(rv, Bool_.N, 8658, "&rArr;");
Reg_name(rv, Bool_.N, 8969, "&rceil;");
Reg_name(rv, Bool_.N, 8221, "&rdquo;");
Reg_name(rv, Bool_.N, 8476, "&real;");
Reg_name(rv, Bool_.N, 174, "&reg;");
Reg_name(rv, Bool_.N, 8971, "&rfloor;");
Reg_name(rv, Bool_.N, 929, "&Rho;");
Reg_name(rv, Bool_.N, 961, "&rho;");
Reg_name(rv, Bool_.N, 8207, "&rlm;");
Reg_name(rv, Bool_.N, 8250, "&rsaquo;");
Reg_name(rv, Bool_.N, 8217, "&rsquo;");
Reg_name(rv, Bool_.N, 8218, "&sbquo;");
Reg_name(rv, Bool_.N, 352, "&Scaron;");
Reg_name(rv, Bool_.N, 353, "&scaron;");
Reg_name(rv, Bool_.N, 8901, "&sdot;");
Reg_name(rv, Bool_.N, 167, "&sect;");
Reg_name(rv, Bool_.N, 173, "&shy;");
Reg_name(rv, Bool_.N, 931, "&Sigma;");
Reg_name(rv, Bool_.N, 963, "&sigma;");
Reg_name(rv, Bool_.N, 962, "&sigmaf;");
Reg_name(rv, Bool_.N, 8764, "&sim;");
Reg_name(rv, Bool_.N, 9824, "&spades;");
Reg_name(rv, Bool_.N, 8834, "&sub;");
Reg_name(rv, Bool_.N, 8838, "&sube;");
Reg_name(rv, Bool_.N, 8721, "&sum;");
Reg_name(rv, Bool_.N, 8835, "&sup;");
Reg_name(rv, Bool_.N, 185, "&sup1;");
Reg_name(rv, Bool_.N, 178, "&sup2;");
Reg_name(rv, Bool_.N, 179, "&sup3;");
Reg_name(rv, Bool_.N, 8839, "&supe;");
Reg_name(rv, Bool_.N, 223, "&szlig;");
Reg_name(rv, Bool_.N, 932, "&Tau;");
Reg_name(rv, Bool_.N, 964, "&tau;");
Reg_name(rv, Bool_.N, 8756, "&there4;");
Reg_name(rv, Bool_.N, 920, "&Theta;");
Reg_name(rv, Bool_.N, 952, "&theta;");
Reg_name(rv, Bool_.N, 977, "&thetasym;");
Reg_name(rv, Bool_.N, 8201, "&thinsp;");
Reg_name(rv, Bool_.N, 222, "&THORN;");
Reg_name(rv, Bool_.N, 254, "&thorn;");
Reg_name(rv, Bool_.N, 732, "&tilde;");
Reg_name(rv, Bool_.N, 215, "&times;");
Reg_name(rv, Bool_.N, 8482, "&trade;");
Reg_name(rv, Bool_.N, 218, "&Uacute;");
Reg_name(rv, Bool_.N, 250, "&uacute;");
Reg_name(rv, Bool_.N, 8593, "&uarr;");
Reg_name(rv, Bool_.N, 8657, "&uArr;");
Reg_name(rv, Bool_.N, 219, "&Ucirc;");
Reg_name(rv, Bool_.N, 251, "&ucirc;");
Reg_name(rv, Bool_.N, 217, "&Ugrave;");
Reg_name(rv, Bool_.N, 249, "&ugrave;");
Reg_name(rv, Bool_.N, 168, "&uml;");
Reg_name(rv, Bool_.N, 978, "&upsih;");
Reg_name(rv, Bool_.N, 933, "&Upsilon;");
Reg_name(rv, Bool_.N, 965, "&upsilon;");
Reg_name(rv, Bool_.N, 220, "&Uuml;");
Reg_name(rv, Bool_.N, 252, "&uuml;");
Reg_name(rv, Bool_.N, 8472, "&weierp;");
Reg_name(rv, Bool_.N, 926, "&Xi;");
Reg_name(rv, Bool_.N, 958, "&xi;");
Reg_name(rv, Bool_.N, 221, "&Yacute;");
Reg_name(rv, Bool_.N, 253, "&yacute;");
Reg_name(rv, Bool_.N, 165, "&yen;");
Reg_name(rv, Bool_.N, 376, "&Yuml;");
Reg_name(rv, Bool_.N, 255, "&yuml;");
Reg_name(rv, Bool_.N, 918, "&Zeta;");
Reg_name(rv, Bool_.N, 950, "&zeta;");
Reg_name(rv, Bool_.N, 8205, "&zwj;");
Reg_name(rv, Bool_.N, 8204, "&zwnj;");
Reg_prefix(rv, Xop_amp_trie_itm.Tid_num_hex, "#x");
Reg_prefix(rv, Xop_amp_trie_itm.Tid_num_hex, "#X");
Reg_prefix(rv, Xop_amp_trie_itm.Tid_num_dec, "#");
return rv;
}
private static void Reg_name(Btrie_slim_mgr trie, boolean tid_is_xowa, int char_int, String xml_name_str) {Reg_name(trie, tid_is_xowa, char_int, Bry_.new_a7(xml_name_str));}
private static void Reg_name(Btrie_slim_mgr trie, boolean tid_is_xowa, int char_int, byte[] xml_name_bry) {
byte itm_tid = tid_is_xowa ? Xop_amp_trie_itm.Tid_name_xowa : Xop_amp_trie_itm.Tid_name_std;
Xop_amp_trie_itm itm = new Xop_amp_trie_itm(itm_tid, char_int, xml_name_bry);
byte[] key = Bry_.Mid(xml_name_bry, 1, xml_name_bry.length); // ignore & for purpose of trie; EX: "amp;"; NOTE: must keep trailing ";" else "&amp " will be valid;
trie.Add_obj(key, itm);
}
private static void Reg_prefix(Btrie_slim_mgr trie, byte prefix_type, String prefix) {
byte[] prefix_ary = Bry_.new_a7(prefix);
Xop_amp_trie_itm itm = new Xop_amp_trie_itm(prefix_type, Xop_amp_trie_itm.Char_int_null, prefix_ary);
trie.Add_obj(prefix_ary, itm);
}
}

58
400_xowa/src/gplx/xowa/parsers/amps/Xop_amp_trie_itm.java Normal file
View File

@@ -0,0 +1,58 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.amps; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.html.*; import gplx.xowa.html.lnkis.*;
public class Xop_amp_trie_itm {
public Xop_amp_trie_itm(byte tid, int char_int, byte[] xml_name_bry) {
this.tid = tid;
this.char_int = char_int;
this.utf8_bry = gplx.intl.Utf16_.Encode_int_to_bry(char_int);
this.xml_name_bry = xml_name_bry;
this.key_name_len = xml_name_bry.length - 2; // 2 for & and ;
}
public byte Tid() {return tid;} private final byte tid;
public int Char_int() {return char_int;} private final int char_int; // val; EX: 160
public byte[] Utf8_bry() {return utf8_bry;} private final byte[] utf8_bry; // EX: new byte[] {192, 160}; (C2, A0)
public byte[] Xml_name_bry() {return xml_name_bry;} private final byte[] xml_name_bry; // EX: "&nbsp;"
public int Key_name_len() {return key_name_len;} private final int key_name_len; // EX: "nbsp".Len
public void Print_ncr(Bry_bfr bfr) {
switch (char_int) {
case Byte_ascii.Lt: case Byte_ascii.Gt: case Byte_ascii.Quote: case Byte_ascii.Amp:
bfr.Add(xml_name_bry); // NOTE: never write actual char; EX: "&lt;" should be written as "&lt;", not "<"
break;
default:
bfr.Add(Xoh_lnki_title_fmtr.Escape_bgn); // &#
bfr.Add_int_variable(char_int); // 160
bfr.Add_byte(Byte_ascii.Semic); // ;
break;
}
}
public void Print_literal(Bry_bfr bfr) {
switch (char_int) {
case Byte_ascii.Lt: bfr.Add(Html_entity_.Lt_bry); break; // NOTE: never write actual char; EX: "&lt;" should be written as "&lt;", not "<"; MW does same; DATE:2014-11-07
case Byte_ascii.Gt: bfr.Add(Html_entity_.Gt_bry); break;
case Byte_ascii.Quote: bfr.Add(Html_entity_.Quote_bry); break;
case Byte_ascii.Amp: bfr.Add(Html_entity_.Amp_bry); break;
default:
bfr.Add(utf8_bry); // write literal; EX: "[" not "&#91;"
break;
}
}
public static final byte Tid_name_std = 1, Tid_name_xowa = 2, Tid_num_hex = 3, Tid_num_dec = 4;
public static final int Char_int_null = -1;
}

32
400_xowa/src/gplx/xowa/parsers/amps/Xop_amp_wkr.java Normal file
View File

@@ -0,0 +1,32 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.amps; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
public class Xop_amp_wkr implements Xop_ctx_wkr {
public void Ctor_ctx(Xop_ctx ctx) {}
public void Page_bgn(Xop_ctx ctx, Xop_root_tkn root) {}
public void Page_end(Xop_ctx ctx, Xop_root_tkn root, byte[] src, int src_len) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn, int cur_pos) {
if (cur_pos == src_len) return ctx.Lxr_make_txt_(cur_pos); // NOTE: & is last char in page; strange and rare, but don't raise error
Xop_amp_mgr amp_mgr = ctx.App().Parser_amp_mgr();
Xop_tkn_itm amp_tkn = amp_mgr.Parse_as_tkn(tkn_mkr, src, src_len, bgn, cur_pos);
int rv_pos = amp_mgr.Rslt_pos();
if (amp_tkn == null) return ctx.Lxr_make_txt_(rv_pos);
ctx.Subs_add(root, amp_tkn);
return rv_pos;
}
}

41
400_xowa/src/gplx/xowa/parsers/amps/Xop_amp_wkr_tst.java Normal file
View File

@@ -0,0 +1,41 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.amps; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import org.junit.*;
public class Xop_amp_wkr_tst {
private Xop_fxt fxt = new Xop_fxt();
@Test public void Name() {fxt.Test_parse_page_wiki("&amp;" , fxt.tkn_html_ref_("&amp;"));} // check for html_ref
@Test public void Name_fail() {fxt.Test_parse_page_wiki("&nil;" , fxt.tkn_txt_(0, 5));} // check for text
@Test public void Hex() {fxt.Test_parse_page_wiki("&#x3A3;" , fxt.tkn_html_ncr_(931));} // check for html_ncr; Σ: http://en.wikipedia.org/wiki/Numeric_character_reference
@Test public void Num_fail_incomplete() {fxt.Test_parse_page_wiki("&#" , fxt.tkn_txt_());}
@Test public void Convert_to_named() {fxt.Test_parse_page_wiki_str("&amp;" , "&amp;");} // note that &amp; is printed, not &
@Test public void Convert_to_named_amp() {fxt.Test_parse_page_wiki_str("&" , "&amp;");} // PURPOSE: html_wtr was not handling & only
@Test public void Convert_to_numeric() {fxt.Test_parse_page_wiki_str("&aacute;" , "&#225;");} // testing that &#225; is outputted, not á
@Test public void Defect_bad_code_fails() { // PURPOSE: early rewrite of Xop_amp_mgr caused Xoh_html_wtr_escaper to fail with array out of bounds error; EX:w:Czech_Republic; DATE:2014-05-11
fxt.Test_parse_page_wiki_str
( "[[File:A.png|alt=<p>&#10;</p>]]" // basically checks amp parsing inside xnde inside lnki's alt (which uses different parsing code
, "<a href=\"/wiki/File:A.png\" class=\"image\" xowa_title=\"A.png\"><img id=\"xowa_file_img_0\" alt=\"&#10;\" src=\"file:///mem/wiki/repo/trg/orig/7/0/A.png\" width=\"0\" height=\"0\" /></a>"
);
}
@Test public void Ignore_ncr() { // PURPOSE: check that ncr is unescaped; PAGE:de.w:Cross-Site-Scripting; DATE:2014-07-23
fxt.Test_parse_page_all_str
( "a <code>&#60;iframe&#62;</code>) b"
, "a <code>&#60;iframe&#62;</code>) b" // &#60; should not become <
);
}
}