Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v1.7.2.1

Browse Source
This commit is contained in:
gnosygnu
2014-07-13 23:23:30 -04:00
parent ecbe2918d8
commit bc10cd76b6
316 changed files with 3251 additions and 1652 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
400_xowa/src_409_tkn_misc/gplx/xowa/Xop_comm_lxr.java
View File

@@ -16,10 +16,11 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
class Xop_comm_lxr implements Xop_lxr {
import gplx.core.btries.*;
public class Xop_comm_lxr implements Xop_lxr {
public byte Lxr_tid() {return Xop_lxr_.Tid_comment;}
public void Init_by_wiki(Xow_wiki wiki, ByteTrieMgr_fast core_trie) {core_trie.Add(Bgn_ary, this);}
public void Init_by_lang(Xol_lang lang, ByteTrieMgr_fast core_trie) {}
public void Init_by_wiki(Xow_wiki wiki, Btrie_fast_mgr core_trie) {core_trie.Add(Bgn_ary, this);}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
int lhs_end = cur_pos;
int end_pos = Bry_finder.Find_fwd(src, End_ary, cur_pos, src_len); // search for "-->" // NOTE: do not reuse cur_pos, else cur_pos may become -1 and fatal error in ctx.Msg_log() below; DATE:2014-06-08
@@ -37,7 +38,7 @@ class Xop_comm_lxr implements Xop_lxr {
private static int Trim_ws_if_entire_line_is_commment(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int cur_pos, int lhs_end, int rhs_bgn) {// REF.MW:Preprocessor_DOM.php|preprocessToXml|handle comments; DATE:2014-02-24
if ( ctx.Tid_is_popup()
&& ctx.Parse_tid() == Xop_parser_.Parse_tid_page_wiki // note that only popup parse can generate <!-- --> that makes it to wtxt
&& Bry_.Match(src, lhs_end, rhs_bgn, gplx.xowa.html.modules.popups.Xow_popup_parser.Comment_txt) // <!--XOWA_SKIP-->
&& Bry_.Match(src, lhs_end, rhs_bgn, Xowa_skip_text_bry) // <!--XOWA_SKIP-->
)
return cur_pos; // in popup mode only do not gobble trailing \n; PAGE:en.w:Gwynedd; DATE:2014-07-01
int nl_lhs = -1;
@@ -89,5 +90,7 @@ class Xop_comm_lxr implements Xop_lxr {
public static final byte[] Bgn_ary = new byte[] {60, 33, 45, 45}, /*<!--*/ End_ary = new byte[] {45, 45, 62}; /*-->*/
private static final int End_len = End_ary.length;
public static final Xop_comm_lxr _ = new Xop_comm_lxr(); Xop_comm_lxr() {}
private static final String Xowa_skip_text_str = "XOWA_SKIP";
private static final byte[] Xowa_skip_text_bry = Bry_.new_ascii_(Xowa_skip_text_str);
public static final byte[] Xowa_skip_comment_bry = Bry_.new_ascii_("<!--" + Xowa_skip_text_str + "-->");
}

5
400_xowa/src_409_tkn_misc/gplx/xowa/Xop_cr_tkn.java
View File

@@ -16,16 +16,17 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.btries.*;
public class Xop_cr_tkn extends Xop_tkn_itm_base {
public Xop_cr_tkn(int bgn, int end) {this.Tkn_ini_pos(true, -1, -1);}
@Override public byte Tkn_tid() {return Xop_tkn_itm_.Tid_cr;}
}
class Xop_cr_lxr implements Xop_lxr {
public byte Lxr_tid() {return Xop_lxr_.Tid_cr;}
public void Init_by_wiki(Xow_wiki wiki, ByteTrieMgr_fast core_trie) {
public void Init_by_wiki(Xow_wiki wiki, Btrie_fast_mgr core_trie) {
core_trie.Add(Byte_ascii.CarriageReturn, this);
}
public void Init_by_lang(Xol_lang lang, ByteTrieMgr_fast core_trie) {}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
return cur_pos; //ignore
}

5
400_xowa/src_409_tkn_misc/gplx/xowa/Xop_eq_lxr.java
View File

@@ -16,11 +16,12 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.btries.*;
class Xop_eq_lxr implements Xop_lxr {//20111222
public Xop_eq_lxr(boolean tmpl_mode) {this.tmpl_mode = tmpl_mode;} boolean tmpl_mode;
public byte Lxr_tid() {return Xop_lxr_.Tid_eq;}
public void Init_by_wiki(Xow_wiki wiki, ByteTrieMgr_fast core_trie) {core_trie.Add(Byte_ascii.Eq, this);}
public void Init_by_lang(Xol_lang lang, ByteTrieMgr_fast core_trie) {}
public void Init_by_wiki(Xow_wiki wiki, Btrie_fast_mgr core_trie) {core_trie.Add(Byte_ascii.Eq, this);}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
cur_pos = Bry_finder.Find_fwd_while(src, cur_pos, src_len, Byte_ascii.Eq); // gobble up eq; "==" should produce 1 eq_tkn with len of 2, not 2 eq_tkn with len of 1; DATE:2014-04-17
int eq_len = cur_pos - bgn_pos;

5
400_xowa/src_409_tkn_misc/gplx/xowa/Xop_hr_lxr.java
View File

@@ -16,10 +16,11 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.btries.*;
public class Xop_hr_lxr implements Xop_lxr {
public byte Lxr_tid() {return Xop_lxr_.Tid_hr;}
public void Init_by_wiki(Xow_wiki wiki, ByteTrieMgr_fast parse_trie) {parse_trie.Add(Hook_ary, this);} static final byte[] Hook_ary = new byte[] {Byte_ascii.NewLine, Byte_ascii.Dash, Byte_ascii.Dash, Byte_ascii.Dash, Byte_ascii.Dash};
public void Init_by_lang(Xol_lang lang, ByteTrieMgr_fast core_trie) {}
public void Init_by_wiki(Xow_wiki wiki, Btrie_fast_mgr parse_trie) {parse_trie.Add(Hook_ary, this);} static final byte[] Hook_ary = new byte[] {Byte_ascii.NewLine, Byte_ascii.Dash, Byte_ascii.Dash, Byte_ascii.Dash, Byte_ascii.Dash};
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
int nl_adj = -1; // -1 to ignore nl at bgn for hr_len
boolean bos = bgn_pos == Xop_parser_.Doc_bgn_bos;

5
400_xowa/src_409_tkn_misc/gplx/xowa/Xop_pipe_lxr.java
View File

@@ -16,14 +16,15 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.btries.*;
class Xop_pipe_tkn extends Xop_tkn_itm_base {
public Xop_pipe_tkn(int bgn, int end) {this.Tkn_ini_pos(false, bgn, end);}
@Override public byte Tkn_tid() {return Xop_tkn_itm_.Tid_pipe;}
}
class Xop_pipe_lxr implements Xop_lxr {
public byte Lxr_tid() {return Xop_lxr_.Tid_pipe;}
public void Init_by_wiki(Xow_wiki wiki, ByteTrieMgr_fast core_trie) {core_trie.Add(Byte_ascii.Pipe, this);}
public void Init_by_lang(Xol_lang lang, ByteTrieMgr_fast core_trie) {}
public void Init_by_wiki(Xow_wiki wiki, Btrie_fast_mgr core_trie) {core_trie.Add(Byte_ascii.Pipe, this);}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
int cur_stack_tid = ctx.Cur_tkn_tid(), rv = -1;
switch (cur_stack_tid) {

5
400_xowa/src_409_tkn_misc/gplx/xowa/Xop_space_tkn.java
View File

@@ -16,6 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.btries.*;
public class Xop_space_tkn extends Xop_tkn_itm_base {
public Xop_space_tkn(boolean immutable, int bgn, int end) {this.Tkn_ini_pos(immutable, bgn, end);}
@Override public byte Tkn_tid() {return Xop_tkn_itm_.Tid_space;}
@@ -34,8 +35,8 @@ public class Xop_space_tkn extends Xop_tkn_itm_base {
}
class Xop_space_lxr implements Xop_lxr {
public byte Lxr_tid() {return Xop_lxr_.Tid_space;}
public void Init_by_wiki(Xow_wiki wiki, ByteTrieMgr_fast core_trie) {core_trie.Add(Byte_ascii.Space, this);}
public void Init_by_lang(Xol_lang lang, ByteTrieMgr_fast core_trie) {}
public void Init_by_wiki(Xow_wiki wiki, Btrie_fast_mgr core_trie) {core_trie.Add(Byte_ascii.Space, this);}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
cur_pos = Bry_finder.Find_fwd_while(src, cur_pos, src_len, Byte_ascii.Space);
ctx.Subs_add(root, tkn_mkr.Space(root, bgn_pos, cur_pos));

5
400_xowa/src_409_tkn_misc/gplx/xowa/Xop_tab_tkn.java
View File

@@ -16,6 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.btries.*;
public class Xop_tab_tkn extends Xop_tkn_itm_base {
public Xop_tab_tkn(int bgn, int end) {this.Tkn_ini_pos(false, bgn, end);}
@Override public byte Tkn_tid() {return Xop_tkn_itm_.Tid_tab;}
@@ -23,11 +24,11 @@ public class Xop_tab_tkn extends Xop_tkn_itm_base {
}
class Xop_tab_lxr implements Xop_lxr {
public byte Lxr_tid() {return Xop_lxr_.Tid_tab;}
public void Init_by_wiki(Xow_wiki wiki, ByteTrieMgr_fast core_trie) {
public void Init_by_wiki(Xow_wiki wiki, Btrie_fast_mgr core_trie) {
core_trie.Add(Byte_ascii.Tab, this);
core_trie.Add(Xop_tab_tkn.Bry_tab_ent, this);
}
public void Init_by_lang(Xol_lang lang, ByteTrieMgr_fast core_trie) {}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
cur_pos = Bry_finder.Find_fwd_while(src, cur_pos, src_len, Byte_ascii.Tab);
src[bgn_pos] = Byte_ascii.Tab; // HACK: SEE:NOTE_1:tabs

61
400_xowa/src_409_tkn_misc/gplx/xowa/Xop_under_lxr.java
View File

@@ -16,17 +16,18 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.btries.*;
import gplx.xowa.html.tocs.*;
class Xop_under_lxr implements Xop_lxr {
private ByteTrieMgr_slim under_words_cs, under_words_ci;
private Btrie_mgr words_trie_ci, words_trie_cs;
public byte Lxr_tid() {return Xop_lxr_.Tid_under;}
public void Init_by_wiki(Xow_wiki wiki, ByteTrieMgr_fast core_trie) {}
public void Init_by_lang(Xol_lang lang, ByteTrieMgr_fast core_trie) {
public void Init_by_wiki(Xow_wiki wiki, Btrie_fast_mgr core_trie) {}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {
Xol_kwd_mgr kwd_mgr = lang.Kwd_mgr();
int under_kwds_len = under_kwds.length;
Xop_under_lxr lxr = new Xop_under_lxr();
lxr.under_words_cs = ByteTrieMgr_slim.cs_();
lxr.under_words_ci = ByteTrieMgr_slim.ci_utf_8_();
lxr.words_trie_cs = Btrie_slim_mgr.cs_();
lxr.words_trie_ci = Btrie_utf8_mgr.new_(lang.Case_mgr());
core_trie.Add(Xop_under_hook.Key_std, lxr);
boolean hook_alt_null = true;
for (int i = 0; i < under_kwds_len; i++) {
@@ -35,15 +36,16 @@ class Xop_under_lxr implements Xop_lxr {
Xol_kwd_itm[] kwd_itms = kwd_grp.Itms(); if (kwd_itms == null) continue;
int kwd_itms_len = kwd_itms.length;
boolean kwd_case_match = kwd_grp.Case_match();
Btrie_mgr words_trie = kwd_grp.Case_match() ? lxr.words_trie_cs : lxr.words_trie_ci;
for (int j = 0; j < kwd_itms_len; j++) {
Xol_kwd_itm kwd_itm = kwd_itms[j];
byte[] kwd_bry = kwd_itm.Val();
int kwd_len = kwd_bry.length;
Object hook_obj = Hook_trie.MatchAtCur(kwd_bry, 0, kwd_len);
Object hook_obj = Hook_trie.Match_bgn(kwd_bry, 0, kwd_len);
if (hook_obj != null) {
Xop_under_hook hook = (Xop_under_hook)hook_obj;
ByteTrieMgr_slim under_words = kwd_case_match ? lxr.under_words_cs : lxr.under_words_ci;
under_words.Add(Bry_.Mid(kwd_bry, hook.Key_len(), kwd_bry.length), Int_obj_val.new_(kwd_id));
byte[] word_bry = Bry_.Mid(kwd_bry, hook.Key_len(), kwd_bry.length);
words_trie.Add_obj(word_bry, new Xop_under_word(kwd_id, word_bry));
if (hook_alt_null && hook.Tid() == Xop_under_hook.Tid_alt) {
core_trie.Add(Xop_under_hook.Key_alt, lxr);
hook_alt_null = false;
@@ -69,24 +71,25 @@ class Xop_under_lxr implements Xop_lxr {
, Xol_kwd_grp_.Id_hiddencat, Xol_kwd_grp_.Id_index, Xol_kwd_grp_.Id_noindex, Xol_kwd_grp_.Id_staticredirect
, Xol_kwd_grp_.Id_disambig
};
private static final ByteTrieMgr_fast Hook_trie = ByteTrieMgr_fast.cs_()
.Add(Xop_under_hook.Key_std, Xop_under_hook.Itm_std)
.Add(Xop_under_hook.Key_alt, Xop_under_hook.Itm_alt)
;
private static final Btrie_fast_mgr Hook_trie = Btrie_fast_mgr.cs_()
.Add(Xop_under_hook.Key_std, Xop_under_hook.Itm_std)
.Add(Xop_under_hook.Key_alt, Xop_under_hook.Itm_alt)
;
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
if (cur_pos == src_len) return ctx.Lxr_make_txt_(cur_pos); // eos
if (cur_pos == src_len) return ctx.Lxr_make_txt_(cur_pos); // eos
int rv = cur_pos;
Object o = under_words_cs.MatchAtCur(src, cur_pos, src_len); // check cs
if (o == null) {
o = under_words_ci.MatchAtCur(src, cur_pos, src_len); // check ci
if (o != null)
rv = under_words_ci.Match_pos();
Object word_obj = words_trie_cs.Match_bgn(src, cur_pos, src_len); // check cs
if (word_obj == null) {
word_obj = words_trie_ci.Match_bgn(src, cur_pos, src_len); // check ci
if (word_obj == null)
return ctx.Lxr_make_txt_(cur_pos); // kwd not found; EX: "TOCA__"
else
rv = words_trie_ci.Match_pos();
}
else
rv = under_words_cs.Match_pos();
if (o == null) return ctx.Lxr_make_txt_(cur_pos); // kwd not found; EX: "TOCA__"
int kwd_id = ((Int_obj_val)(o)).Val();
Xop_under_lxr.Make_tkn(ctx, tkn_mkr, root, src, src_len, bgn_pos, rv, kwd_id);
rv = words_trie_cs.Match_pos();
Xop_under_word word_itm = (Xop_under_word)word_obj;
Xop_under_lxr.Make_tkn(ctx, tkn_mkr, root, src, src_len, bgn_pos, rv, word_itm.Kwd_id());
return rv;
}
public static void Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos, int kwd_id) {
@@ -112,8 +115,8 @@ class Xop_word_lxr implements Xop_lxr {
private int kwd_id;
public Xop_word_lxr(int kwd_id) {this.kwd_id = kwd_id;}
public byte Lxr_tid() {return Xop_lxr_.Tid_word;}
public void Init_by_wiki(Xow_wiki wiki, ByteTrieMgr_fast core_trie) {}
public void Init_by_lang(Xol_lang lang, ByteTrieMgr_fast core_trie) {}
public void Init_by_wiki(Xow_wiki wiki, Btrie_fast_mgr core_trie) {}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
Xop_under_lxr.Make_tkn(ctx, tkn_mkr, root, src, src_len, bgn_pos, cur_pos, kwd_id); // for now, all word_lxrs only call the under_lxr; DATE:2014-02-14
return cur_pos;
@@ -131,3 +134,13 @@ class Xop_under_hook {
, Itm_alt = new Xop_under_hook(Tid_alt, Key_alt)
;
}
class Xop_under_word {
public Xop_under_word(int kwd_id, byte[] word_bry) {
this.kwd_id = kwd_id;
this.word_bry = word_bry;
this.word_len = word_bry.length;
}
public int Kwd_id() {return kwd_id;} private int kwd_id;
public byte[] Word_bry() {return word_bry;} private byte[] word_bry;
public int Word_len() {return word_len;} private int word_len;
}

34
400_xowa/src_409_tkn_misc/gplx/xowa/Xop_under_lxr_tst.java
View File

@@ -144,11 +144,43 @@ public class Xop_under_lxr_tst {
wiki.Parser().Init_by_lang(lang);
fxt.Test_parse_page_all_str("a＿＿TOC＿＿b", "ab");
}
@Test public void Hook_utf8() { // PURPOSE: ja wikis use alternate __; DATE:2014-03-04
@Test public void Ascii_ci() { // PURPOSE: case-insensitive ascii; DATE:2014-07-10
Xow_wiki wiki = fxt.Wiki(); Xol_lang lang = wiki.Lang();
fxt.Init_lang_kwds(lang, Xol_kwd_grp_.Id_toc, false, "__TOC__");
wiki.Parser().Init_by_lang(lang);
fxt.Test_parse_page_all_str("a__TOC__b", "ab");
fxt.Test_parse_page_all_str("a__toc__b", "ab");
}
@Test public void Utf8_ci() { // PURPOSE: case-insensitive UTF8; DATE:2014-07-10
Xow_wiki wiki = fxt.Wiki(); Xol_lang lang = wiki.Lang();
lang.Case_mgr_utf8_();
fxt.Init_lang_kwds(lang, Xol_kwd_grp_.Id_toc, false, "__AÉI__");
wiki.Parser().Init_by_lang(lang);
fxt.Test_parse_page_all_str("a__AÉI__b", "ab");
fxt.Test_parse_page_all_str("a__aéi__b", "ab");
}
@Test public void Utf8_ci_asymmetric() { // PURPOSE: case-insensitive UTF8; asymmetric; DATE:2014-07-10
Xow_wiki wiki = fxt.Wiki(); Xol_lang lang = wiki.Lang();
lang.Case_mgr_utf8_();
fxt.Init_lang_kwds(lang, Xol_kwd_grp_.Id_toc, false, "__İÇİNDEKİLER__"); // __TOC__ for tr.w
wiki.Parser().Init_by_lang(lang);
fxt.Test_parse_page_all_str("a__İçindekiler__b", "ab");
}
@Test public void Cs() { // PURPOSE: cs (ascii / utf8 doesn't matter); DATE:2014-07-11
Xow_wiki wiki = fxt.Wiki(); Xol_lang lang = wiki.Lang();
fxt.Init_lang_kwds(lang, Xol_kwd_grp_.Id_toc , Bool_.Y, "__TOC__");
wiki.Parser().Init_by_lang(lang);
fxt.Test_parse_page_all_str("a__TOC__b" , "ab"); // ci.pass
fxt.Test_parse_page_all_str("a__toc__b" , "a__toc__b"); // ci.pass
}
@Test public void Ascii_cs_ci() { // PURPOSE: test simultaneous cs and ci; DATE:2014-07-11
Xow_wiki wiki = fxt.Wiki(); Xol_lang lang = wiki.Lang();
fxt.Init_lang_kwds(lang, Xol_kwd_grp_.Id_toc , Bool_.N, "__TOC__");
fxt.Init_lang_kwds(lang, Xol_kwd_grp_.Id_notoc , Bool_.Y, "__NOTOC__");
wiki.Parser().Init_by_lang(lang);
fxt.Test_parse_page_all_str("a__TOC__b" , "ab"); // ci.pass
fxt.Test_parse_page_all_str("a__toc__b" , "ab"); // ci.pass
fxt.Test_parse_page_all_str("a__NOTOC__b" , "ab"); // cs.pass
fxt.Test_parse_page_all_str("a__notoc__b" , "a__notoc__b"); // cs.fail
}
}