Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Source: Restore broken commit

Browse Source
This commit is contained in:
gnosygnu
2017-02-06 22:14:55 -05:00
parent 938beac9f9
commit 3bfeb94b43
4380 changed files with 328018 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_convert_lang.java Normal file
View File

@@ -0,0 +1,188 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*; import gplx.core.primitives.*; import gplx.dbs.*;
import gplx.xowa.langs.vnts.*; import gplx.xowa.langs.vnts.converts.*;
import gplx.xowa.parsers.htmls.*;
public class Vnt_convert_lang {
private final Xol_convert_mgr convert_mgr; private final Xol_vnt_regy vnt_regy;
private final Vnt_convert_rule converter_rule; private final Vnt_html_doc_wkr html_convert_wkr; private final Mwh_doc_parser doc_parser = new Mwh_doc_parser();
private final Bry_bfr bfr = Bry_bfr_.New_w_size(255), tmp_frame_bfr = Bry_bfr_.New_w_size(255), tmp_convert_bfr = Bry_bfr_.New_w_size(255);
private byte[] src; private int src_len; private int pos;
private Vnt_log_mgr log_mgr; private int tag_bgn, tag_end;
public Vnt_convert_lang(Xol_convert_mgr convert_mgr, Xol_vnt_regy vnt_regy) {
this.convert_mgr = convert_mgr; this.vnt_regy = vnt_regy;
this.html_convert_wkr = new Vnt_html_doc_wkr(convert_mgr, vnt_regy);
this.converter_rule = new Vnt_convert_rule(this, vnt_regy, log_mgr);
}
public byte[] Converted_title() {return converted_title;} private byte[] converted_title;
public void Log__init(Db_conn conn) {
log_mgr = new Vnt_log_mgr();
log_mgr.Init_by_db(conn, vnt_regy);
}
public byte[] Parse_page(Xol_vnt_itm vnt_itm, int page_id, byte[] src) {// REF.MW:/languages/LanguageConverter.php!recursiveConvertTopLevel
if (log_mgr != null) log_mgr.Init_by_page(page_id);
this.converted_title = null;
return Parse_bry(vnt_itm, src);
}
public byte[] Parse_bry(Xol_vnt_itm vnt_itm, byte[] src) {
boolean convert_needed = true; // false for sr lang; SEE:LanguageSr.php !$this->guessVariant(src, vnt);
this.pos = 0;
this.src = src; this.src_len = src.length;
while (pos < src_len) {
int curly_bgn = Bry_find_.Find_fwd(src, Bry__curly_bgn, pos, src_len);
if (curly_bgn == Bry_find_.Not_found) { // No more markup, append final segment
Add_output(vnt_itm, convert_needed, src, pos, src_len);
return bfr.To_bry_and_clear();
}
boolean inside_tag = Is_inside_tag(pos, curly_bgn);
if (inside_tag) {
Add_output(vnt_itm, convert_needed, src, pos, tag_bgn); // Markup found; append segment
Auto_convert(bfr, vnt_itm, src, tag_bgn, tag_end);
pos = tag_end;
}
else {
Add_output(vnt_itm, convert_needed, src, pos, curly_bgn); // Markup found; append segment
pos = curly_bgn; // Advance position
bfr.Add(Parse_recursive(tmp_frame_bfr, vnt_itm, 1)); // Do recursive conversion
}
}
return bfr.To_bry_and_clear();
}
private boolean Is_inside_tag(int prev_pos, int curly_bgn) {
if ( curly_bgn == 0 // -{ starts at BOS; EX: "-{A}-"
|| curly_bgn == prev_pos // -{ starts after last pair; EX: "-{A}--{B}-"
) return false;
int cur = curly_bgn - 1;
tag_bgn = tag_end = -1;
boolean loop = true;
while (loop) { // scan bwd for <
byte b = src[cur];
switch (b) {
case Byte_ascii.Angle_bgn: tag_bgn = cur; loop = false; break;
case Byte_ascii.Angle_end: return false; // ">" found; "-{}-" not inside tag
default: --cur; break;
}
if (cur == prev_pos - 1) break;
}
if (tag_bgn == -1) return false; // no "<" found;
loop = true;
cur = curly_bgn + 1; // TODO_OLD: resume at }-
while (loop) { // scan fwd for >
byte b = src[cur];
switch (b) {
case Byte_ascii.Angle_bgn: return false; // "<" found; "-{}-" not inside tag
case Byte_ascii.Angle_end: tag_end = cur + 1; return true;
default: ++cur; break;
}
if (cur == src_len) break;
}
return false; // no ">" foud
}
private byte[] Parse_recursive(Bry_bfr frame_bfr, Xol_vnt_itm vnt_itm, int depth) {
pos += 2; // skip "-{"
boolean warning_done = false;
boolean frame_bfr_dirty = false;
int bgn_pos = pos;
while (pos < src_len) {
byte b = src[pos];
Object o = trie.Match_bgn_w_byte(b, src,pos, src_len);
if (o == null) { // char;
++pos;
continue;
}
switch (((Byte_obj_val)o).Val()) {
case Tid__curly_bgn:
frame_bfr.Add_mid(src, bgn_pos, pos); // add everything from bgn of frame to cur pos; EX: "a" in "-{a-{b}-c}-"
frame_bfr_dirty = true;
if (depth >= max_depth) {
pos += 2; // skip "-{"
frame_bfr.Add(Bry__curly_bgn);
if (!warning_done) {
frame_bfr.Add_str_a7("<span class=\"error\">max depth");
// wfMessage('language-converter-depth-warning')->numParams($this->mMaxDepth)->inContentLanguage()->text()
frame_bfr.Add_str_a7("</span>");
warning_done = true;
}
continue;
}
frame_bfr.Add(Parse_recursive(Bry_bfr_.New_w_size(16), vnt_itm, depth + 1)); // Recursively parse another rule
bgn_pos = pos;
break;
case Tid__curly_end:
if (frame_bfr_dirty) { // recursive; use frame_bfr
frame_bfr.Add_mid(src, bgn_pos, pos); // add everything from bgn of frame to cur pos; EX: "a" in "-{a-{b}-c}-"
byte[] frame_bry = frame_bfr.To_bry_and_clear();
converter_rule.Parse(vnt_itm, frame_bry, 0, frame_bry.length);
}
else // not recursive
converter_rule.Parse(vnt_itm, src, bgn_pos, pos);
Apply_manual_conv(converter_rule);
pos += 2;
return converter_rule.Display();
default: throw Err_.new_unhandled(-1); // never happens
}
}
Auto_convert(frame_bfr, vnt_itm, src, bgn_pos, src_len); // Unclosed rule
pos = src_len;
return Bry_.Add(Bry__curly_bgn, frame_bfr.To_bry_and_clear());
}
private void Add_output(Xol_vnt_itm vnt_itm, boolean convert_needed, byte[] src, int bgn, int end) {
if (end - bgn == 0) return;
if (convert_needed) {
Auto_convert(bfr, vnt_itm, src, bgn, end);
}
else
bfr.Add_mid(src, bgn, end);
}
public byte[] Auto_convert(Xol_vnt_itm vnt_itm, byte[] src) {
Auto_convert(tmp_convert_bfr, vnt_itm, src, 0, src.length);
return tmp_convert_bfr.To_bry_and_clear();
}
private void Auto_convert(Bry_bfr bfr, Xol_vnt_itm vnt_itm, byte[] src, int bgn, int end) {
html_convert_wkr.Init(bfr, vnt_itm);
doc_parser.Parse(html_convert_wkr, src, bgn, end);
}
private void Apply_manual_conv(Vnt_convert_rule rule) {
this.converted_title = rule.Title();
byte action = rule.Action();
Vnt_rule_undi_mgr cnv_tbl = rule.Cnv_tbl();
int len = cnv_tbl.Len();
for (int i = 0; i < len; ++i) {
Vnt_rule_undi_grp grp = cnv_tbl.Get_at(i);
byte[] grp_key = grp.Vnt();
Xol_vnt_itm vnt_itm = vnt_regy.Get_by(grp_key); if (vnt_itm == null) continue;
int grp_len = grp.Len();
Xol_convert_wkr wkr = convert_mgr.Converter_ary()[vnt_itm.Idx()];
for (int j = 0; j < grp_len; ++j) {
Vnt_rule_undi_itm itm = grp.Get_at(j);
if (action == Byte_ascii.Plus) {
wkr.Add(itm.Src(), itm.Trg());
}
else if (action == Byte_ascii.Dash)
wkr.Del(itm.Src());
}
}
}
private static final byte Tid__curly_bgn = 1, Tid__curly_end = 2;
private static final byte[] Bry__curly_bgn = Bry_.new_a7("-{"), Bry__curly_end = Bry_.new_a7("}-");
private static final Btrie_fast_mgr trie = Btrie_fast_mgr.cs()
.Add_bry_byte(Bry__curly_bgn, Tid__curly_bgn)
.Add_bry_byte(Bry__curly_end, Tid__curly_end);
private static final int max_depth = 32;
}

102
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_convert_lang__html__tst.java Normal file
View File

@@ -0,0 +1,102 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import org.junit.*; import gplx.xowa.langs.vnts.*; import gplx.xowa.langs.vnts.converts.*;
public class Vnt_convert_lang__html__tst { // REF: https://www.mediawiki.org/wiki/Writing_systems/Syntax
private final Vnt_convert_lang_fxt fxt = new Vnt_convert_lang_fxt();
private String rule;
@Before public void init() {
rule = "-{H|zh-cn:cn;zh-hk:hk;zh-tw:tw}-";
fxt.Clear();
}
@Test public void Node() {
fxt.Test_parse(rule + "hk<span>hk</span>hk", "cn<span>cn</span>cn");
}
@Test public void Attribs() {
fxt.Test_parse(rule + "<span class='hk'>hk</span>", "<span class='hk'>cn</span>");
}
@Test public void Attribs__title() {
fxt.Test_parse(rule + "<span title='hk'>hk</span>", "<span title='cn'>cn</span>");
}
@Test public void Attribs__alt() {
fxt.Test_parse(rule + "<span alt='hk'>hk</span>", "<span alt='cn'>cn</span>");
}
@Test public void Attribs__alt_w_embedded_vnt() { // PURPOSE: handle embedded variants inside attribute tags; PAGE:sr.n:Проглашени_победници_„Вики_воли_Земљу" DATE:2015-10-13
fxt.Test_parse(rule + "<img id='hk' alt='hk-{hk}-hk' src='hk'>", "<img id='hk' alt='cnhkcn' src='hk'>");
}
@Test public void Attribs__skip_url() {
fxt.Test_parse(rule + "<span alt='http://hk.org'>hk</span>", "<span alt='http://hk.org'>cn</span>");
}
@Test public void Node__script() {
fxt.Test_parse(rule + "hk<script>hk</script>hk", "cn<script>hk</script>cn");
}
@Test public void Node__code() {
fxt.Test_parse(rule + "hk<code>hk</code>hk", "cn<code>hk</code>cn");
}
@Test public void Node__pre() {
fxt.Test_parse(rule + "hk<pre>hk</pre>hk", "cn<pre>hk</pre>cn");
}
@Test public void Node__pre__nested() {
fxt.Test_parse(rule + "hk<pre><span>hk</span></pre>hk", "cn<pre><span>hk</span></pre>cn");
}
@Test public void Recursive__deep() {
fxt.Test_parse("a-{b-{c-{d}-e}-f}-g", "abcdefg");
}
@Test public void Recursive__many() {
fxt.Test_parse("a-{b-{c}-d-{e}-f}-g", "abcdefg");
}
@Test public void Recursive__unclosed() {
fxt.Test_parse("a-{b-{c", "a-{b-{c");
}
@Test public void Recursive__unclosed_2() {
fxt.Test_parse("a-{b-{c}-", "a-{bc");
}
@Test public void Recursive__failed() { // PURPOSE: handle out of bounds exception; PAGE:zh.w:重庆市 ;DATE:2015-10-01
fxt.Test_parse("-{zh-cn:a-{b}-c; zh-tw:d-{e}-f; }-", "abc");
}
@Test public void Unclosed() {
fxt.Test_parse("a-{bc", "a-{bc");
}
@Test public void Entity__body() {
fxt.Test_parse("-{H|zh-cn:nbsp-cn;zh-hk:nbsp;}-" + "&nbsp;nbsp", "&nbsp;nbsp-cn");
}
@Test public void Entity__atr() {
fxt.Test_parse("-{H|zh-cn:nbsp-cn;zh-hk:nbsp;}-" + "<div title='nbsp&nbsp;nbsp'/>" , "<div title='nbsp-cn&nbsp;nbsp-cn'/>");
}
@Test public void Node__example() {
fxt.Test_parse(rule + String_.Concat_lines_nl_skip_last
( "{|"
, "|-"
, "|A<br />"
, "|B<br/>"
, "<span style=''>-{zh-hans:C;zh-hant:D;}-</span>"
, "|}")
, String_.Concat_lines_nl_skip_last
( "{|"
, "|-"
, "|A<br />"
, "|B<br/>"
, "<span style=''>C</span>"
, "|}"
));
}
@Test public void Attribs__title__w_vnt() {
fxt.Init_cur("zh-tw");
fxt.Test_parse("<span title=\"-{zh-cn:cn;zh-hant:hk;}-cn\" style=\"color:red;\">cn</span>", "<span title=\"hkcn\" style=\"color:red;\">cn</span>"); // cn not converted to hk
}
}

83
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_convert_lang__syntax__tst.java Normal file
View File

@@ -0,0 +1,83 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import org.junit.*; import gplx.xowa.langs.vnts.*; import gplx.xowa.langs.vnts.converts.*;
public class Vnt_convert_lang__syntax__tst { // REF: https://www.mediawiki.org/wiki/Writing_systems/Syntax
private final Vnt_convert_lang_fxt fxt = new Vnt_convert_lang_fxt();
@Test public void Bidi() {
String text = "-{zh-hans:a;zh-hant:b}-";
fxt.Test_parse_many(text, "a", "zh-hans", "zh-cn", "zh-sg", "zh");
fxt.Test_parse_many(text, "b", "zh-hant", "zh-hk", "zh-tw");
}
@Test public void Undi() {
String text = "-{H|cn_k=>zh-cn:cn_v}-cn_k";
fxt.Test_parse_many(text, "cn_k", "zh", "zh-hans", "zh-hant", "zh-hk", "zh-my", "zh-mo", "zh-sg", "zh-tw");
fxt.Test_parse_many(text, "cn_v", "zh-cn");
}
@Test public void Raw() {
fxt.Test_parse_many("-{a}-", "a", "zh-hans", "zh-cn", "zh-sg", "zh", "zh-hant", "zh-hk", "zh-tw");
fxt.Test_parse_many("-{R|a}-", "a", "zh-hans", "zh-cn", "zh-sg", "zh", "zh-hant", "zh-hk", "zh-tw");
}
@Test public void Hide() {
String text = "-{H|zh-cn:cn;zh-hk:hk;zh-tw:tw}-cn hk tw";
fxt.Test_parse_many(text, "cn cn cn", "zh-cn", "zh-sg");
fxt.Test_parse_many(text, "hk hk hk", "zh-hk");
fxt.Test_parse_many(text, "tw tw tw", "zh-tw");
fxt.Test_parse_many(text, "cn hk tw", "zh", "zh-hans", "zh-hant");
}
@Test public void Aout() {
String text = "-{A|zh-cn:cn;zh-hk:hk;zh-tw:tw}- cn hk tw";
fxt.Test_parse_many(text, "cn cn cn cn", "zh-cn", "zh-sg");
fxt.Test_parse_many(text, "hk hk hk hk", "zh-hk");
fxt.Test_parse_many(text, "tw tw tw tw", "zh-tw");
fxt.Test_parse_many(text, "cn cn hk tw", "zh", "zh-hans");
fxt.Test_parse_many(text, "tw cn hk tw", "zh-hant");
fxt.Test_parse_many("h-{}-k", "hk", "zh-cn"); // semi-disabled
}
@Test public void Del() {
String text = "-{H|zh-cn:cn;zh-hk:hk;zh-tw:tw}-cn hk tw-{-|zh-cn:cn;zh-hk:hk;zh-tw:tw}- cn hk tw";
fxt.Test_parse_many(text, "cn cn cn cn hk tw", "zh-cn", "zh-sg");
fxt.Test_parse_many(text, "hk hk hk cn hk tw", "zh-hk");
fxt.Test_parse_many(text, "tw tw tw cn hk tw", "zh-tw");
fxt.Test_parse_many(text, "cn hk tw cn hk tw", "zh", "zh-hans", "zh-hant");
}
@Test public void Title() {
fxt.Test_parse_title("-{}-", null, "", "zh-cn");
String text = "-{T|zh-cn:cn;zh-hk:hk;zh-tw:tw}-cn hk tw";
fxt.Test_parse_title(text, "cn", "cn hk tw", "zh-cn");
fxt.Test_parse_title(text, "cn", "cn hk tw", "zh-sg");
fxt.Test_parse_title(text, "hk", "cn hk tw", "zh-hk");
fxt.Test_parse_title(text, "tw", "cn hk tw", "zh-tw");
fxt.Test_parse_title(text, "cn", "cn hk tw", "zh-hans");
fxt.Test_parse_title(text, "tw", "cn hk tw", "zh-hant");
fxt.Test_parse_title(text, null, "cn hk tw", "zh");
}
@Test public void Descrip() {
String text = "-{D|zh-cn:cn;zh-hk:hk;zh-tw:tw}-";
fxt.Test_parse_many(text, "ZH-CN:cn;ZH-HK:hk;ZH-TW:tw;", "zh", "zh-hans", "zh-hant", "zh-cn", "zh-hk", "zh-my", "zh-mo", "zh-sg", "zh-tw");
}
@Test public void Mixture() {
String text = "-{H|zh-cn:cn;zh-hk:hk;zh-tw:tw}--{zh;zh-hans;zh-hant|cn hk tw}- -{zh;zh-cn;zh-hk;zh-tw|cn hk tw}-";
fxt.Test_parse_many(text, "cn hk tw cn cn cn", "zh-cn", "zh-sg", "zh-hans");
fxt.Test_parse_many(text, "cn hk tw hk hk hk", "zh-hk");
fxt.Test_parse_many(text, "cn hk tw tw tw tw", "zh-tw", "zh-hant");
fxt.Test_parse_many(text, "cn hk tw cn hk tw", "zh");
}
@Test public void Descrip__undi() {fxt.Test_parse("-{D|cn_k=>zh-cn:cn_v;hk_k=>zh-hk:hk_v}-", "cn_k⇒ZH-CN:cn_v;hk_k⇒ZH-HK:hk_v;");}
@Test public void Descrip__mixd() {fxt.Test_parse("-{D|zh-tw:tw_v;cn_k=>zh-cn:cn_v;hk_k=>zh-hk:hk_v;zh-mo:mo_v}-", "ZH-TW:tw_v;ZH-MO:mo_v;cn_k⇒ZH-CN:cn_v;hk_k⇒ZH-HK:hk_v;");}
}

56
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_convert_lang_fxt.java Normal file
View File

@@ -0,0 +1,56 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.xowa.langs.vnts.*; import gplx.xowa.langs.vnts.converts.*;
class Vnt_convert_lang_fxt {
private final Vnt_convert_lang converter_lang;
private final Xol_convert_mgr convert_mgr = new Xol_convert_mgr();
private final Xol_vnt_regy vnt_regy = Xol_vnt_regy_fxt.new_chinese();
private Xol_vnt_itm vnt_itm;
public Vnt_convert_lang_fxt() {
converter_lang = new Vnt_convert_lang(convert_mgr, vnt_regy);
this.Clear();
}
public void Clear() {
convert_mgr.Init(vnt_regy);
Init_cur("zh-cn");
}
public Vnt_convert_lang_fxt Init_cur(String vnt) {
byte[] cur_vnt = Bry_.new_a7(vnt);
this.vnt_itm = vnt_regy.Get_by(cur_vnt);
return this;
}
public void Test_parse(String raw, String expd) {
Tfds.Eq_str(expd, String_.new_u8(converter_lang.Parse_page(vnt_itm, -1, Bry_.new_u8(raw))));
}
public void Test_parse_many(String raw, String expd, String... vnts) {
int len = vnts.length;
for (int i = 0; i < len; ++i) {
String vnt_key = vnts[i];
Init_cur(vnt_key);
Xol_vnt_itm vnt = vnt_regy.Get_by(Bry_.new_a7(vnt_key));
Tfds.Eq_str(expd, String_.new_u8(converter_lang.Parse_page(vnt, -1, Bry_.new_u8(raw))), vnt_key);
}
}
public void Test_parse_title(String raw, String expd_title, String expd_text, String vnt_key) {
Init_cur(vnt_key);
Xol_vnt_itm vnt = vnt_regy.Get_by(Bry_.new_a7(vnt_key));
Tfds.Eq_str(expd_text, String_.new_u8(converter_lang.Parse_page(vnt, -1, Bry_.new_u8(raw))), vnt_key);
Tfds.Eq_str(expd_title, converter_lang.Converted_title());
}
}

194
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_convert_rule.java Normal file
View File

@@ -0,0 +1,194 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*; import gplx.core.primitives.*;
import gplx.xowa.langs.vnts.*;
class Vnt_convert_rule { // REF.MW: /languages/LanguageConverter.php|ConverterRule
private final Vnt_flag_parser flag_parser = new Vnt_flag_parser(); private final Vnt_flag_code_mgr flag_codes = new Vnt_flag_code_mgr(); private final Vnt_flag_lang_mgr flag_langs = new Vnt_flag_lang_mgr();
private final Vnt_rule_parser rule_parser = new Vnt_rule_parser(); private final Vnt_rule_undi_mgr rule_undis = new Vnt_rule_undi_mgr(); private final Vnt_rule_bidi_mgr rule_bidis = new Vnt_rule_bidi_mgr();
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
private final Ordered_hash cnv_marked_hash = Ordered_hash_.New_bry();
private Vnt_convert_lang converter;
private Xol_vnt_regy vnt_regy; private byte[] vnt_key;
private Vnt_log_mgr log_mgr;
private byte[] rule_raw;
private boolean rule_parser_init = true;
public byte[] Display() {return display;} private byte[] display;
public byte[] Title() {return title;} private byte[] title;
public byte Action() {return action;} private byte action;
public Vnt_rule_undi_mgr Cnv_tbl() {return cnv_tbl;} private final Vnt_rule_undi_mgr cnv_tbl = new Vnt_rule_undi_mgr();
public Vnt_convert_rule(Vnt_convert_lang converter, Xol_vnt_regy vnt_regy, Vnt_log_mgr log_mgr) {
this.converter = converter; this.log_mgr = log_mgr;
this.vnt_regy = vnt_regy;
flag_parser.Init(log_mgr);
rule_parser.Init(log_mgr, vnt_regy);
}
public void Parse(Xol_vnt_itm vnt_itm, byte[] src, int src_bgn, int src_end) {
if (rule_parser_init) { // WORKAROUND: initialized vnt_regy on first parse; initializing during constructor doesn't work b/c vnt_regy == 0 due to load order of gfs script; DATE:2016-11-09
rule_parser_init = false;
rule_parser.Init(log_mgr, vnt_regy);
}
this.vnt_key = vnt_itm.Key();
this.display = this.title = null;
this.action = Byte_ascii.Null;
int pipe_pos = Bry_find_.Find_fwd(src, Byte_ascii.Pipe, src_bgn, src_end);
flag_parser.Parse(flag_codes, flag_langs, vnt_regy, src, src_bgn, pipe_pos);
int rule_bgn = pipe_pos == -1 ? src_bgn : pipe_pos + 1;
this.rule_raw = Bry_.Mid(src, rule_bgn, src_end);
int flag_langs_count = flag_langs.Count();
if (flag_langs_count > 0) { // vnts exist in flag; EX: -{zh-hans;zh-hant|text}-
if (flag_langs.Has(vnt_key))
rule_raw = converter.Auto_convert(vnt_itm, rule_raw); // convert rule text to current language; EX:-{|convert}-
else {
byte[][] fallbacks = vnt_itm.Fallback_ary();
int fallbacks_len = fallbacks.length;
for (int i = 0; i < fallbacks_len; ++i) {
byte[] fallback = fallbacks[i];
if (flag_langs.Has(fallback)) {
Xol_vnt_itm fallback_itm = (Xol_vnt_itm)vnt_regy.Get_by(fallback);
rule_raw = converter.Auto_convert(fallback_itm, rule_raw);
break;
}
}
}
flag_codes.Limit(Vnt_flag_code_.Tid_raw);
}
rule_parser.Clear(rule_undis, rule_bidis, rule_raw);
if (!flag_codes.Get(Vnt_flag_code_.Tid_raw) && !flag_codes.Get(Vnt_flag_code_.Tid_name))
rule_parser.Parse(src, rule_bgn, src_end);
if (log_mgr != null) log_mgr.Log_rule(src_bgn, src_end, Bry_.Mid(src, src_bgn, src_end), flag_codes, flag_langs, rule_undis, rule_bidis);
if (rule_undis.Has_none() && rule_bidis.Has_none()) {
if ( flag_codes.Get(Vnt_flag_code_.Tid_add)
|| flag_codes.Get(Vnt_flag_code_.Tid_del)
) { // fill all variants if text in -{A/H/-|text} without rules
for (int i = 0; i < flag_langs_count; ++i) {
Xol_vnt_itm itm = flag_langs.Get_at(i);
rule_bidis.Set(itm.Key(), rule_raw);
}
}
else if ( !flag_codes.Get(Vnt_flag_code_.Tid_name)
&& !flag_codes.Get(Vnt_flag_code_.Tid_title)
) {
flag_codes.Limit(Vnt_flag_code_.Tid_raw);
}
}
int flag_count = Vnt_flag_code_.Tid__max;
for (int flag = 0; flag < flag_count; ++flag) {
if (!flag_codes.Get(flag)) continue;
switch (flag) {
case Vnt_flag_code_.Tid_raw: display = rule_parser.Raw(); break; // if we don't do content convert, still strip the -{}- tags
case Vnt_flag_code_.Tid_name: // process N flag: output current variant name
byte[] vnt_key_trim = Bry_.Trim(rule_parser.Raw());
Xol_vnt_itm vnt_itm_trim = vnt_regy.Get_by(vnt_key_trim);
display = vnt_itm_trim == null ? display = Bry_.Empty : vnt_itm_trim.Name();
break;
case Vnt_flag_code_.Tid_descrip: display = Make_descrip(); break; // process D flag: output rules description
case Vnt_flag_code_.Tid_hide: display = Bry_.Empty; break; // process H,- flag or T only: output nothing
case Vnt_flag_code_.Tid_del: display = Bry_.Empty; action = Byte_ascii.Dash; break;
case Vnt_flag_code_.Tid_add: display = Bry_.Empty; action = Byte_ascii.Plus; break;
case Vnt_flag_code_.Tid_show: display = Make_converted(vnt_itm); break;
case Vnt_flag_code_.Tid_title: display = Bry_.Empty; title = Make_title(vnt_itm); break;
default: break; // ignore unknown flags (but see error case below)
}
}
if (display == null)
display = Bry_.Add(Bry__error_bgn, Bry__error_end); // wfMessage( 'converter-manual-rule-error' )->inContentLanguage()->escaped()
Make_conv_tbl();
}
private void Make_conv_tbl() {
if (rule_undis.Has_none() && rule_bidis.Has_none()) return; // Special case optimisation
cnv_tbl.Clear(); cnv_marked_hash.Clear();
int vnt_regy_len = vnt_regy.Len();
for (int i = 0; i < vnt_regy_len; ++i) {
Xol_vnt_itm vnt = vnt_regy.Get_at(i);
byte[] vnt_key = vnt.Key();
// bidi: fill in missing variants with fallbacks
byte[] bidi_bry = rule_bidis.Get_text_by_key_or_null(vnt_key);
if (bidi_bry == null) {
bidi_bry = rule_bidis.Get_text_by_ary_or_null(vnt.Fallback_ary());
if (bidi_bry != null) rule_bidis.Set(vnt_key, bidi_bry);
}
if (bidi_bry != null) {
int marked_len = cnv_marked_hash.Count();
for (int j = 0; j < marked_len; ++j) {
Xol_vnt_itm marked_itm = (Xol_vnt_itm)cnv_marked_hash.Get_at(j);
byte[] marked_key = marked_itm.Key();
byte[] marked_bry = rule_bidis.Get_text_by_key_or_null(marked_key);
byte[] cur_bidi_bry = rule_bidis.Get_text_by_key_or_null(vnt_key);
if (vnt.Dir() == Xol_vnt_dir_.Tid__bi)
cnv_tbl.Set(vnt_key, marked_bry, cur_bidi_bry);
if (marked_itm.Dir() == Xol_vnt_dir_.Tid__bi)
cnv_tbl.Set(marked_key, cur_bidi_bry, marked_bry);
}
cnv_marked_hash.Add(vnt_key, vnt);
}
// undi: fill to convert tables
byte[] undi_bry = rule_undis.Get_text_by_key_or_null(vnt_key);
if (vnt.Dir() != Xol_vnt_dir_.Tid__none && undi_bry != null) {
Vnt_rule_undi_grp undi_grp = rule_undis.Get_by(vnt_key);
int undi_grp_len = undi_grp.Len();
for (int j = 0; j < undi_grp_len; ++j) {
Vnt_rule_undi_itm undi_itm = undi_grp.Get_at(j);
cnv_tbl.Set(vnt_key, undi_itm.Src(), undi_itm.Trg());
}
}
}
}
private byte[] Make_descrip() {
int len = rule_bidis.Len();
for (int i = 0; i < len; ++i) {
Vnt_rule_bidi_itm bidi_itm = rule_bidis.Get_at(i);
Xol_vnt_itm vnt_itm = vnt_regy.Get_by(bidi_itm.Vnt());
tmp_bfr.Add(vnt_itm.Name()).Add_byte_colon().Add(bidi_itm.Text()).Add_byte_semic();
}
len = rule_undis.Len();
for (int i = 0; i < len; ++i) {
Vnt_rule_undi_grp undi_grp = rule_undis.Get_at(i);
int sub_len = undi_grp.Len();
for (int j = 0; j < sub_len; ++j) {
Vnt_rule_undi_itm undi_itm = (Vnt_rule_undi_itm)undi_grp.Get_at(j);
Xol_vnt_itm undi_vnt = vnt_regy.Get_by(undi_grp.Vnt());
tmp_bfr.Add(undi_itm.Src()).Add(Bry__undi_spr).Add(undi_vnt.Name()).Add_byte_colon().Add(undi_itm.Trg()).Add_byte_semic();
}
}
return tmp_bfr.To_bry_and_clear();
}
private byte[] Make_title(Xol_vnt_itm vnt) {
if (vnt.Idx() == 0) { // for mainLanguageCode; EX: "zh"
byte[] rv = rule_bidis.Get_text_by_key_or_null(vnt.Key());
return rv == null ? rule_undis.Get_text_by_key_or_null(vnt.Key()) : rv;
}
else
return Make_converted(vnt);
}
private byte[] Make_converted(Xol_vnt_itm vnt) {
if (rule_bidis.Len() == 0 && rule_undis.Len() == 0) return rule_raw;
byte[] rv = rule_bidis.Get_text_by_key_or_null(vnt.Key()); // display current variant in bidirectional array
if (rv == null) rv = rule_bidis.Get_text_by_ary_or_null(vnt.Fallback_ary()); // or display current variant in fallbacks
if (rv == null) rv = rule_undis.Get_text_by_key_or_null(vnt.Key()); // or display current variant in unidirectional array
if (rv == null && vnt.Dir() == Xol_vnt_dir_.Tid__none) { // or display first text under disable manual convert
rv = (rule_bidis.Len() > 0) ? rule_bidis.Get_text_at(0) : rule_undis.Get_text_at(0);
}
return rv;
}
private final static byte[]
Bry__error_bgn = Bry_.new_a7("<span class=\"error\">vnt error")
, Bry__error_end = Bry_.new_a7("</span>")
, Bry__undi_spr = Bry_.new_u8("⇒")
;
}

50
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_code_.java Normal file
View File

@@ -0,0 +1,50 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
class Vnt_flag_code_ {
public static final int
Tid_add = 0 // +: EX: -{+|zh-hans:A;zh-hant:B}- -> "A"
, Tid_del = 1 // -: remove; EX: -{-|zh-hans:A;zh-hant:B}- -> ""
, Tid_aout = 2 // A: Add and output; EX: -{A|zh-hans:A;zh-hant:B}- -> "A"
, Tid_hide = 3 // H: Hide macro; EX: -{H|zh-hans:A;zh-hant:B}- -> ""
, Tid_raw = 4 // R: Raw: no convert; EX: -{R|zh-hans:A;zh-hant:B}- -> "zh-hans:A;zh-hant:B"
, Tid_show = 5 // S: Show EX: -{S|zh-hans:A;zh-hant:B}- -> "A"
, Tid_descrip = 6 // D: Describe; EX: -{D|zh-hans:A;zh-hant:B}- -> "简体：A；繁體：B；" (简体=Simplified;繁體=Traditional)
, Tid_name = 7 // N: variant Name EX: -{N|zh-hans:A;zh-hant:B}- -> ""
, Tid_title = 8 // T: page Title; EX: -{T|zh-hans:A;zh-hant:B}- -> ""
, Tid_err = 9 // E: Error EX: -{E|zh-hans:A;zh-hant:B}- -> "A"
, Tid__max = 10
;
private static final String[] Tid__names = new String[]
{ "+", "-", "A", "H", "R"
, "S", "D", "N", "T", "E"
};
public static String To_str(int tid) {return Tid__names[tid];}
public static final Hash_adp_bry Regy = Hash_adp_bry.ci_a7() // NOTE: match either lc or uc; EX: -{D}- or -{d}-;
.Add_byte_int(Byte_ascii.Plus , Tid_add)
.Add_byte_int(Byte_ascii.Dash , Tid_del)
.Add_byte_int(Byte_ascii.Ltr_A , Tid_aout)
.Add_byte_int(Byte_ascii.Ltr_H , Tid_hide)
.Add_byte_int(Byte_ascii.Ltr_R , Tid_raw)
.Add_byte_int(Byte_ascii.Ltr_S , Tid_show)
.Add_byte_int(Byte_ascii.Ltr_D , Tid_descrip)
.Add_byte_int(Byte_ascii.Ltr_N , Tid_name)
.Add_byte_int(Byte_ascii.Ltr_T , Tid_title)
.Add_byte_int(Byte_ascii.Ltr_E , Tid_err)
;
}

56
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_code_mgr.java Normal file
View File

@@ -0,0 +1,56 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
class Vnt_flag_code_mgr {
private final boolean[] ary = new boolean[Ary_len]; private final static int Ary_len = Vnt_flag_code_.Tid__max;
public int Count() {return count;} private int count = 0;
public boolean Get(int tid) {return ary[tid];}
public void Clear() {
count = 0;
for (int i = 0; i < Ary_len; ++i)
ary[i] = false;
}
public void Add(int tid) {
this.Set_y(tid);
++count;
}
public void Set_y(int tid) {ary[tid] = Bool_.Y;}
public void Set_y_many(int... vals) {
int len = vals.length;
for (int i = 0; i < len; ++i)
ary[vals[i]] = Bool_.Y;
}
public void Set_n(int tid) {ary[tid] = Bool_.N;}
public void Limit(int tid) {
for (int i = 0; i < Ary_len; ++i)
ary[i] = i == tid;
}
public boolean Limit_if_exists(int tid) {
boolean exists = ary[tid]; if (!exists) return false;
this.Limit(tid);
return true;
}
public void To_bfr__dbg(Bry_bfr bfr) {
for (int i = 0; i < Ary_len; ++i) {
if (ary[i]) {
if (bfr.Len_gt_0()) bfr.Add_byte_semic();
bfr.Add_str_a7(Vnt_flag_code_.To_str(i));
}
}
}
}

35
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_lang_mgr.java Normal file
View File

@@ -0,0 +1,35 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.xowa.langs.vnts.*;
class Vnt_flag_lang_mgr {
private final Ordered_hash regy = Ordered_hash_.New_bry();
public int Count() {return regy.Count();}
public boolean Has(byte[] vnt) {return regy.Has(vnt);}
public void Clear() {regy.Clear();}
public void Add(Xol_vnt_itm itm) {regy.Add_if_dupe_use_1st(itm.Key(), itm);}
public Xol_vnt_itm Get_at(int i) {return (Xol_vnt_itm)regy.Get_at(i);}
public void To_bfr__dbg(Bry_bfr bfr) {
int len = regy.Count();
for (int i = 0; i < len; ++i) {
Xol_vnt_itm itm = (Xol_vnt_itm)regy.Get_at(i);
if (bfr.Len_gt_0()) bfr.Add_byte_semic();
bfr.Add(itm.Key());
}
}
}

68
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_parser.java Normal file
View File

@@ -0,0 +1,68 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.xowa.langs.vnts.*;
class Vnt_flag_parser implements gplx.core.brys.Bry_split_wkr {
private final Hash_adp_bry codes_regy = Vnt_flag_code_.Regy;
private Vnt_flag_code_mgr codes; private Vnt_flag_lang_mgr langs;
private Xol_vnt_regy vnt_regy;
private Vnt_log_mgr log_mgr;
public void Init(Vnt_log_mgr log_mgr) {
this.log_mgr = log_mgr;
}
public void Parse(Vnt_flag_code_mgr codes, Vnt_flag_lang_mgr langs, Xol_vnt_regy vnt_regy, byte[] src, int src_bgn, int src_end) {
this.codes = codes; this.langs = langs; this.vnt_regy = vnt_regy;
codes.Clear(); langs.Clear();
if (src_end != Bry_find_.Not_found) // "|" found; EX: -{A|}-
Bry_split_.Split(src, src_bgn, src_end, Byte_ascii.Semic, true, this);
int codes_count = codes.Count(), langs_count = langs.Count();
if (codes_count == 0) codes.Set_y(Vnt_flag_code_.Tid_show);
else if (codes.Limit_if_exists(Vnt_flag_code_.Tid_raw)) {}
else if (codes.Limit_if_exists(Vnt_flag_code_.Tid_name)) {}
else if (codes.Limit_if_exists(Vnt_flag_code_.Tid_del)) {}
else if (codes_count == 1 && codes.Get(Vnt_flag_code_.Tid_title)) codes.Set_y(Vnt_flag_code_.Tid_hide);
else if (codes.Get(Vnt_flag_code_.Tid_hide)) {
boolean exists_d = codes.Get(Vnt_flag_code_.Tid_descrip);
boolean exists_t = codes.Get(Vnt_flag_code_.Tid_title);
codes.Clear();
codes.Set_y_many(Vnt_flag_code_.Tid_add, Vnt_flag_code_.Tid_hide);
if (exists_d) codes.Set_y(Vnt_flag_code_.Tid_descrip);
if (exists_t) codes.Set_y(Vnt_flag_code_.Tid_title);
}
else {
if (codes.Get(Vnt_flag_code_.Tid_aout))
codes.Set_y_many(Vnt_flag_code_.Tid_add, Vnt_flag_code_.Tid_show);
if (codes.Get(Vnt_flag_code_.Tid_descrip))
codes.Set_n(Vnt_flag_code_.Tid_show);
if (langs_count > 0)
codes.Clear();
}
}
public int Split(byte[] src, int itm_bgn, int itm_end) {
int flag_tid = codes_regy.Get_as_int_or(src, itm_bgn, itm_end, -1);
if (flag_tid == -1) { // try to find flags like "zh-hans", "zh-hant"; allow syntaxes like "-{zh-hans;zh-hant|XXXX}-"
Xol_vnt_itm vnt_itm = vnt_regy.Get_by(src, itm_bgn, itm_end);
if (vnt_itm == null) return Bry_split_.Rv__ok; // unknown flag; ignore
if (log_mgr != null) log_mgr.Log_lang(vnt_itm, Vnt_log_mgr.Scope__lang);
langs.Add(vnt_itm);
return Bry_split_.Rv__ok;
}
codes.Add(flag_tid);
return Bry_split_.Rv__ok;
}
}

56
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_parser_tst.java Normal file
View File

@@ -0,0 +1,56 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import org.junit.*; import gplx.xowa.langs.vnts.*;
public class Vnt_flag_parser_tst {
private final Vnt_flag_parser_fxt fxt = new Vnt_flag_parser_fxt();
@Test public void Basic() {fxt.Test_parse("D" , "D");}
@Test public void Multiple() {fxt.Test_parse("+;S;E" , "+;S;E");}
@Test public void Ws() {fxt.Test_parse(" + ; S ; E " , "+;S;E");}
@Test public void None() {fxt.Test_parse("" , "S");}
@Test public void Wrong() {fxt.Test_parse("XYZ" , "S");}
@Test public void Raw__limit() {fxt.Test_parse("R;S" , "R");}
@Test public void Name__limit() {fxt.Test_parse("N;S" , "N");}
@Test public void Del_limit() {fxt.Test_parse("-;S" , "-");}
@Test public void Title__also_macro_y() {fxt.Test_parse("T" , "H;T");}
@Test public void Title__also_macro_n() {fxt.Test_parse("T;S" , "S;T");}
@Test public void Hide__remove_all() {fxt.Test_parse("H;S" , "+;H");}
@Test public void Hide__keep_descrip() {fxt.Test_parse("H;S;D" , "+;H;D");}
@Test public void Hide__keep_title() {fxt.Test_parse("H;S;T" , "+;H;T");}
@Test public void Aout__also_show_all() {fxt.Test_parse("A" , "+;A;S");}
@Test public void Descrip__remove_show() {fxt.Test_parse("D;S" , "D");}
@Test public void Aout_w_descrip() {fxt.Test_parse("A;D;S" , "+;A;D");}
@Test public void Lang__one() {fxt.Test_parse("zh-hans" , "S;zh-hans");}
@Test public void Lang__many() {fxt.Test_parse("zh-cn;zh-hk" , "S;zh-cn;zh-hk");}
@Test public void Lang__many__ws() {fxt.Test_parse(" zh-cn ; zh-hk " , "S;zh-cn;zh-hk");}
@Test public void Lang__many__dupe() {fxt.Test_parse("zh-cn;zh-cn" , "S;zh-cn");}
@Test public void Lang__zap__codes() {fxt.Test_parse("+;S;zh-hans;" , "zh-hans");}
}
class Vnt_flag_parser_fxt {
private final Vnt_flag_parser parser = new Vnt_flag_parser();
private final Vnt_flag_code_mgr codes = new Vnt_flag_code_mgr(); private final Vnt_flag_lang_mgr langs = new Vnt_flag_lang_mgr();
private final Xol_vnt_regy vnt_regy = Xol_vnt_regy_fxt.new_chinese();
private final Bry_bfr bfr = Bry_bfr_.New();
public void Test_parse(String raw, String expd) {
byte[] src = Bry_.new_u8(raw);
parser.Parse(codes, langs, vnt_regy, src, 0, src.length);
codes.To_bfr__dbg(bfr);
langs.To_bfr__dbg(bfr);
Tfds.Eq_str(expd, bfr.To_str_and_clear());
}
}

82
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_html_doc_wkr.java Normal file
View File

@@ -0,0 +1,82 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.primitives.*; import gplx.core.btries.*;
import gplx.xowa.parsers.htmls.*; import gplx.xowa.parsers.xndes.*; import gplx.xowa.parsers.amps.*;
import gplx.xowa.langs.vnts.*; import gplx.xowa.langs.vnts.converts.*;
import gplx.xowa.htmls.*;
class Vnt_html_doc_wkr implements Mwh_doc_wkr {
private final Hash_adp_bry atr_hash = Hash_adp_bry.ci_a7();
private final Xol_convert_mgr convert_mgr; private final Xol_vnt_regy vnt_regy;
private Vnt_convert_lang atr_converter;
private Xol_vnt_itm vnt_itm; private int convert_vnt_idx;
private Bry_bfr bfr;
public Vnt_html_doc_wkr(Xol_convert_mgr convert_mgr, Xol_vnt_regy vnt_regy) {
this.convert_mgr = convert_mgr; this.vnt_regy = vnt_regy;
atr_hash.Add_many_str("title", "alt");
}
public Hash_adp_bry Nde_regy() {return nde_regy;} private final Hash_adp_bry nde_regy = Mwh_doc_wkr_.Nde_regy__mw();
public void Init(Bry_bfr bfr, Xol_vnt_itm vnt_itm) {this.bfr = bfr; this.vnt_itm = vnt_itm; this.convert_vnt_idx = vnt_itm.Idx();}
public void On_atr_each (Mwh_atr_parser mgr, byte[] src, int nde_tid, boolean valid, boolean repeated, boolean key_exists, byte[] key_bry, byte[] val_bry_manual, int[] itm_ary, int itm_idx) {
boolean literal = true;
if (atr_hash.Get_by_mid(key_bry, 0, key_bry.length) != null) { // title, alt
int val_bgn = itm_ary[itm_idx + Mwh_atr_mgr.Idx_val_bgn];
int val_end = itm_ary[itm_idx + Mwh_atr_mgr.Idx_val_end];
if (Bry_find_.Find_fwd(src, Bry__url_frag, val_bgn, val_end) == Bry_find_.Not_found) { // do not convert if urls are present
literal = false;
byte[] val_bry = val_bry_manual == null ? Bry_.Mid(src, val_bgn, val_end) : val_bry_manual;
if (atr_converter == null) atr_converter = new Vnt_convert_lang(convert_mgr, vnt_regy);// NOTE: late instantiation, or else StackOverflow error
val_bry = atr_converter.Parse_bry(vnt_itm, val_bry);
bfr.Add_byte_space();
bfr.Add(key_bry);
bfr.Add_byte(Byte_ascii.Eq);
byte quote_byte = Mwh_atr_itm_.Calc_qte_byte(itm_ary, itm_idx);
bfr.Add_byte(quote_byte);
bfr.Add(val_bry);
bfr.Add_byte(quote_byte);
}
}
if (literal) {
int atr_bgn = itm_ary[itm_idx + Mwh_atr_mgr.Idx_atr_bgn];
int atr_end = itm_ary[itm_idx + Mwh_atr_mgr.Idx_atr_end];
bfr.Add_mid(src, atr_bgn, atr_end);
}
}
public void On_txt_end (Mwh_doc_parser mgr, byte[] src, int nde_tid, int itm_bgn, int itm_end) {
switch (nde_tid) {
case Xop_xnde_tag_.Tid__code:
case Xop_xnde_tag_.Tid__script:
case Xop_xnde_tag_.Tid__pre:
bfr.Add_mid(src, itm_bgn, itm_end);
break;
default:
bfr.Add(convert_mgr.Convert_text(convert_vnt_idx, src, itm_bgn, itm_end));
break;
}
}
public void On_nde_head_bgn(Mwh_doc_parser mgr, byte[] src, int nde_tid, int key_bgn, int key_end) {
bfr.Add_byte(Byte_ascii.Angle_bgn).Add_mid(src, key_bgn, key_end); // EX: "<span"
}
public void On_nde_head_end(Mwh_doc_parser mgr, byte[] src, int nde_tid, int itm_bgn, int itm_end, boolean inline) {
bfr.Add(inline ? Xoh_consts.__inline : Xoh_consts.__end); // add "/>" or ">"
}
public void On_nde_tail_end (Mwh_doc_parser mgr, byte[] src, int nde_tid, int itm_bgn, int itm_end) {bfr.Add_mid(src, itm_bgn, itm_end);}
public void On_comment_end (Mwh_doc_parser mgr, byte[] src, int nde_tid, int itm_bgn, int itm_end) {bfr.Add_mid(src, itm_bgn, itm_end);}
public void On_entity_end (Mwh_doc_parser mgr, byte[] src, int nde_tid, int itm_bgn, int itm_end) {bfr.Add_mid(src, itm_bgn, itm_end);}
private static final byte[] Bry__url_frag = Bry_.new_a7("://"); // REF.MW: if ( !strpos( $attr, '://' ) ) {
}

58
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_log_mgr.java Normal file
View File

@@ -0,0 +1,58 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.dbs.*;
import gplx.xowa.langs.vnts.*;
class Vnt_log_mgr {
private int uid;
private int page_id, rule_idx;
private Xol_vnt_regy vnt_regy;
private int[] vnt_ary = new int[10];
private Vnt_log_tbl tbl;
public void Init_by_db(Db_conn conn, Xol_vnt_regy vnt_regy) {
this.vnt_regy = vnt_regy;
this.tbl = new Vnt_log_tbl(conn);
tbl.Create_tbl();
this.uid = 0;
this.page_id = 0;
}
public void Init_by_page(int page_id) {
this.page_id = page_id;
this.rule_idx = -1;
for (int i = 0; i < 10; ++i)
vnt_ary[i] = 0;
}
public void Log_lang(byte[] vnt, int scope) {Log_lang(vnt_regy.Get_by(vnt), scope);}
public void Log_lang(Xol_vnt_itm itm, int scope) {
int idx = itm.Idx();
int val = vnt_ary[idx];
vnt_ary[idx] = val == 0 ? scope : val | scope;
}
public void Log_rule(int src_bgn, int src_end, byte[] src_txt, Vnt_flag_code_mgr flag_codes, Vnt_flag_lang_mgr flag_langs, Vnt_rule_undi_mgr rule_undis, Vnt_rule_bidi_mgr rule_bidis) {
tbl.Insert(uid, page_id, ++rule_idx
, flag_codes.Count(), flag_langs.Count(), rule_undis.Len(), rule_bidis.Len()
, flag_codes.Get(Vnt_flag_code_.Tid_add), flag_codes.Get(Vnt_flag_code_.Tid_del), flag_codes.Get(Vnt_flag_code_.Tid_aout), flag_codes.Get(Vnt_flag_code_.Tid_hide), flag_codes.Get(Vnt_flag_code_.Tid_raw), flag_codes.Get(Vnt_flag_code_.Tid_show), flag_codes.Get(Vnt_flag_code_.Tid_descrip), flag_codes.Get(Vnt_flag_code_.Tid_name), flag_codes.Get(Vnt_flag_code_.Tid_title), flag_codes.Get(Vnt_flag_code_.Tid_err)
, vnt_ary[0], vnt_ary[1], vnt_ary[2], vnt_ary[3], vnt_ary[4], vnt_ary[5], vnt_ary[6], vnt_ary[7], vnt_ary[8], vnt_ary[9]
, src_bgn, src_end, src_txt
);
}
public void Rls() {
tbl.Rls();
}
public static final int Scope__lang = 1, Scope__undi = 2, Scope__bidi = 4;
}

106
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_log_tbl.java Normal file
View File

@@ -0,0 +1,106 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.dbs.*;
public class Vnt_log_tbl implements Rls_able {
private final String tbl_name = "log_vnt"; private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private final String fld_uid, fld_page_id, fld_rule_idx
, fld_flag_count, fld_lang_count, fld_undi_count, fld_bidi_count
, fld_flag_add, fld_flag_del, fld_flag_aout, fld_flag_hide, fld_flag_raw, fld_flag_show, fld_flag_descrip, fld_flag_name, fld_flag_title, fld_flag_err
, fld_vnt_0, fld_vnt_1, fld_vnt_2, fld_vnt_3, fld_vnt_4, fld_vnt_5, fld_vnt_6, fld_vnt_7, fld_vnt_8, fld_vnt_9
, fld_src_bgn, fld_src_end, fld_src_txt;
private Db_stmt stmt_insert;
public Vnt_log_tbl(Db_conn conn) {
this.conn = conn;
this.fld_uid = flds.Add_int("uid");
this.fld_page_id = flds.Add_int("page_id");
this.fld_rule_idx = flds.Add_int("rule_idx");
this.fld_flag_count = flds.Add_int("flag_count");
this.fld_lang_count = flds.Add_int("lang_count");
this.fld_undi_count = flds.Add_int("undi_count");
this.fld_bidi_count = flds.Add_int("bidi_count");
this.fld_flag_add = flds.Add_int("flag_add");
this.fld_flag_del = flds.Add_int("flag_del");
this.fld_flag_aout = flds.Add_int("flag_aout");
this.fld_flag_hide = flds.Add_int("flag_hide");
this.fld_flag_raw = flds.Add_int("flag_raw");
this.fld_flag_show = flds.Add_int("flag_show");
this.fld_flag_descrip = flds.Add_int("flag_descrip");
this.fld_flag_name = flds.Add_int("flag_name");
this.fld_flag_title = flds.Add_int("flag_title");
this.fld_flag_err = flds.Add_int("flag_err");
this.fld_vnt_0 = flds.Add_int("vnt_0");
this.fld_vnt_1 = flds.Add_int("vnt_1");
this.fld_vnt_2 = flds.Add_int("vnt_2");
this.fld_vnt_3 = flds.Add_int("vnt_3");
this.fld_vnt_4 = flds.Add_int("vnt_4");
this.fld_vnt_5 = flds.Add_int("vnt_5");
this.fld_vnt_6 = flds.Add_int("vnt_6");
this.fld_vnt_7 = flds.Add_int("vnt_7");
this.fld_vnt_8 = flds.Add_int("vnt_8");
this.fld_vnt_9 = flds.Add_int("vnt_9");
this.fld_src_bgn = flds.Add_int("src_bgn");
this.fld_src_end = flds.Add_int("src_end");
this.fld_src_txt = flds.Add_text("src_txt");
conn.Rls_reg(this);
}
public Db_conn Conn() {return conn;} private final Db_conn conn;
public void Rls() {
stmt_insert = Db_stmt_.Rls(stmt_insert);
}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Insert(int uid, int page_id, int rule_idx, int flag_count, int lang_count, int undi_count, int bidi_count
, boolean flag_add, boolean flag_del, boolean flag_aout, boolean flag_hide, boolean flag_raw, boolean flag_show, boolean flag_descrip, boolean flag_name, boolean flag_title, boolean flag_err
, int vnt_0, int vnt_1, int vnt_2, int vnt_3, int vnt_4, int vnt_5, int vnt_6, int vnt_7, int vnt_8, int vnt_9
, int src_bgn, int src_end, byte[] src_txt
) {
if (stmt_insert == null) stmt_insert = conn.Stmt_insert(tbl_name, flds);
stmt_insert.Clear()
.Val_int(fld_uid, uid)
.Val_int(fld_page_id, page_id)
.Val_int(fld_rule_idx, rule_idx)
.Val_int(fld_flag_count, flag_count)
.Val_int(fld_lang_count, lang_count)
.Val_int(fld_undi_count, undi_count)
.Val_int(fld_bidi_count, bidi_count)
.Val_int_by_bool(fld_flag_add, flag_add)
.Val_int_by_bool(fld_flag_del, flag_del)
.Val_int_by_bool(fld_flag_aout, flag_aout)
.Val_int_by_bool(fld_flag_hide, flag_hide)
.Val_int_by_bool(fld_flag_raw, flag_raw)
.Val_int_by_bool(fld_flag_show, flag_show)
.Val_int_by_bool(fld_flag_descrip, flag_descrip)
.Val_int_by_bool(fld_flag_name, flag_name)
.Val_int_by_bool(fld_flag_title, flag_title)
.Val_int_by_bool(fld_flag_err, flag_err)
.Val_int(fld_vnt_0, vnt_0)
.Val_int(fld_vnt_1, vnt_1)
.Val_int(fld_vnt_2, vnt_2)
.Val_int(fld_vnt_3, vnt_3)
.Val_int(fld_vnt_4, vnt_4)
.Val_int(fld_vnt_5, vnt_5)
.Val_int(fld_vnt_6, vnt_6)
.Val_int(fld_vnt_7, vnt_7)
.Val_int(fld_vnt_8, vnt_8)
.Val_int(fld_vnt_9, vnt_9)
.Val_int(fld_src_bgn, src_bgn)
.Val_int(fld_src_end, src_end)
.Val_bry_as_str(fld_src_txt, src_txt)
.Exec_insert();
}
}

68
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_bidi_mgr.java Normal file
View File

@@ -0,0 +1,68 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
class Vnt_rule_bidi_mgr {
private final Ordered_hash hash = Ordered_hash_.New_bry();
public int Len() {return hash.Count();}
public boolean Has_none() {return hash.Count() == 0;}
public void Clear() {hash.Clear();}
public Vnt_rule_bidi_itm Get_at(int i) {return (Vnt_rule_bidi_itm)hash.Get_at(i);}
public Vnt_rule_bidi_itm Get_by(byte[] k) {return (Vnt_rule_bidi_itm)hash.Get_by(k);}
public byte[] Get_text_by_ary_or_null(byte[]... ary) {
int len = ary.length;
byte[] rv = null;
for (int i = 0; i < len; ++i) {
byte[] itm = ary[i];
Vnt_rule_bidi_itm bidi_itm = (Vnt_rule_bidi_itm)hash.Get_by(itm); if (bidi_itm == null) continue;
rv = Get_text_by_key_or_null(bidi_itm.Vnt());
if (rv != null) return rv;
}
return rv;
}
public byte[] Get_text_by_key_or_null(byte[] vnt) {
Vnt_rule_bidi_itm rv = (Vnt_rule_bidi_itm)hash.Get_by(vnt);
return rv == null ? null : rv.Text();
}
public byte[] Get_text_at(int i) {
Vnt_rule_bidi_itm itm = (Vnt_rule_bidi_itm)hash.Get_at(i);
return itm == null ? null : itm.Text();
}
public void Set(byte[] vnt, byte[] text) {
Vnt_rule_bidi_itm itm = (Vnt_rule_bidi_itm)hash.Get_by(vnt);
if (itm == null) {
itm = new Vnt_rule_bidi_itm(vnt, text);
hash.Add(vnt, itm);
}
else
itm.Text_(text);
}
public void To_bry__dbg(Bry_bfr bfr) {
int len = hash.Count();
for (int i = 0; i < len; ++i) {
if (i != 0) bfr.Add_byte_nl();
Vnt_rule_bidi_itm itm = (Vnt_rule_bidi_itm)hash.Get_at(i);
bfr.Add(itm.Vnt()).Add_byte_colon().Add(itm.Text());
}
}
}
class Vnt_rule_bidi_itm {
public Vnt_rule_bidi_itm(byte[] vnt, byte[] text) {this.vnt = vnt; this.text = text;}
public byte[] Vnt() {return vnt;} private final byte[] vnt;
public byte[] Text() {return text;} private byte[] text;
public void Text_(byte[] v) {this.text = v;}
}

98
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_parser.java Normal file
View File

@@ -0,0 +1,98 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*;
import gplx.xowa.langs.vnts.*;
class Vnt_rule_parser implements gplx.core.brys.Bry_split_wkr {
private final Btrie_slim_mgr vnt_trie = Btrie_slim_mgr.ci_a7();
private final Btrie_rv trv = new Btrie_rv();
private Vnt_rule_undi_mgr undis; private Vnt_rule_bidi_mgr bidis;
private int src_end; private byte[] rule_raw;
public byte[] Raw() {return rule_raw;}
private Vnt_log_mgr log_mgr;
public void Init(Vnt_log_mgr log_mgr, Xol_vnt_regy vnt_regy) {
this.log_mgr = log_mgr;
this.vnt_trie.Clear();
int len = vnt_regy.Len();
for (int i = 0; i < len; ++i) {
Xol_vnt_itm itm = (Xol_vnt_itm)vnt_regy.Get_at(i);
vnt_trie.Add_obj(itm.Key(), itm);
}
}
public void Clear(Vnt_rule_undi_mgr undis, Vnt_rule_bidi_mgr bidis, byte[] rule_raw) {
this.undis = undis; this.bidis = bidis;
undis.Clear(); bidis.Clear();
this.rule_raw = rule_raw;
}
public void Parse(byte[] src, int src_bgn, int src_end) {
this.src_end = src_end;
Bry_split_.Split(src, src_bgn, src_end, Byte_ascii.Semic, false, this); // trim=false for "&#entity;" check below
}
public int Split(byte[] src, int itm_bgn, int itm_end) { // macro=>zh-hans:text;
int html_entity_pos = Bry_find_.Find_bwd_while_alphanum(src, itm_end);
byte html_entity_byte = src[html_entity_pos];
if (html_entity_byte == Byte_ascii.Hash) html_entity_byte = src[html_entity_pos - 2]; // skip #; EX: &#123;
if (html_entity_byte == Byte_ascii.Amp) return Bry_split_.Rv__extend; // reject "&#entity;"; EX: "&nbsp;zh-hans;"
if (itm_end != src_end) {
int nxt_lang_bgn = Bry_find_.Find_fwd(src, Bry__bidi_dlm, itm_end + 1, src_end); // look for next "=>"
if (nxt_lang_bgn == Bry_find_.Not_found)
nxt_lang_bgn = Bry_find_.Find_fwd_while_ws(src, itm_end + 1, src_end); // skip any ws after end ";"; EX: "a:1; b:2"; NOTE: +1 to skip semic;
else
nxt_lang_bgn += 2;
int nxt_lang_end = Bry_find_.Find_fwd(src, Byte_ascii.Colon, nxt_lang_bgn, src_end); // get colon;
if (nxt_lang_end != Bry_find_.Not_found) {
nxt_lang_end = Bry_find_.Find_bwd__skip_ws(src, nxt_lang_end, src_end); // trim
if (vnt_trie.Match_bgn(src, nxt_lang_bgn, nxt_lang_end) == null) return Bry_split_.Rv__extend; // reject ";not_variant"; EX: ";border" in "zh-hans:<span style='color:blue;border:1px;'>;zh-hant:"
}
}
int undi_bgn = Bry_find_.Find_fwd_while_ws(src, itm_bgn, itm_end); // skip any ws after bgn ";"; EX: " a=>b:c;"
int undi_end = Bry_find_.Find_fwd(src, Bry__bidi_dlm, undi_bgn, itm_end); // look for "=>"
int lang_bgn = undi_bgn; // default lang_bgn to undi_bgn; assumes no bidi found
if (undi_end != Bry_find_.Not_found) { // "=>" found; bidi exists
lang_bgn = Bry_find_.Find_fwd_while_ws(src, undi_end + 2, itm_end); // set lang_bgn after => and gobble up ws
undi_end = Bry_find_.Find_bwd__skip_ws(src, undi_end, undi_bgn); // trim ws from end of bd;
}
Object vnt_obj = vnt_trie.Match_at(trv, src, lang_bgn, itm_end);
if (vnt_obj == null)
return (itm_bgn == 0) ? Bry_split_.Rv__cancel : Bry_split_.Rv__extend; // if 1st item; cancel rest; otherwise, extend
int lang_end = trv.Pos();
int text_bgn = Bry_find_.Find_fwd_while_ws(src, lang_end, itm_end); if (src[text_bgn] != Byte_ascii.Colon) return Bry_split_.Rv__extend;
++text_bgn;
Xol_vnt_itm vnt_itm = (Xol_vnt_itm)vnt_obj;
byte[] vnt_key = vnt_itm.Key();
byte[] text_bry = Bry_.Mid_w_trim(src, text_bgn, itm_end);
if (undi_end == Bry_find_.Not_found) {
if (log_mgr != null) log_mgr.Log_lang(vnt_itm, Vnt_log_mgr.Scope__bidi);
bidis.Set(vnt_key, text_bry);
}
else {
byte[] undi_bry = Bry_.Mid(src, undi_bgn, undi_end);
if (itm_end - text_bgn > 0) {
if (log_mgr != null) log_mgr.Log_lang(vnt_itm, Vnt_log_mgr.Scope__undi);
undis.Set(vnt_key, undi_bry, text_bry);
}
}
return Bry_split_.Rv__ok;
}
public void To_bry__dbg(Bry_bfr bfr) {
undis.To_bry__dbg(bfr);
if (bfr.Len_gt_0()) bfr.Add_byte_nl();
bidis.To_bry__dbg(bfr);
}
private static final byte[] Bry__bidi_dlm = Bry_.new_a7("=>");
}

27
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_parser__bidi_tst.java Normal file
View File

@@ -0,0 +1,27 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import org.junit.*;
public class Vnt_rule_parser__bidi_tst {
private final Vnt_rule_parser_fxt fxt = new Vnt_rule_parser_fxt();
@Test public void Basic() {fxt.Test_parse("x1:v1;" , "x1:v1");}
@Test public void Ws() {fxt.Test_parse(" x1 : v1 ;" , "x1:v1");}
@Test public void Entity() {fxt.Test_parse("x1:a&nbsp;x2:b;x2:b;" , "x1:a&nbsp;x2:b" , "x2:b");}
@Test public void Unknown__nth() {fxt.Test_parse("x1:a;wx2:b;x2:b;" , "x1:a;wx2:b" , "x2:b");}
@Test public void Unknown__1st() {fxt.Test_parse("wx1:a;x1:b;" , "");}
}

24
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_parser__undi_tst.java Normal file
View File

@@ -0,0 +1,24 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import org.junit.*;
public class Vnt_rule_parser__undi_tst {
private final Vnt_rule_parser_fxt fxt = new Vnt_rule_parser_fxt();
@Test public void One() {fxt.Test_parse("k1=>x1:v1;" , "x1:k1=v1");}
@Test public void Many() {fxt.Test_parse("k1=>x1:v1;k2=>x2:v2;" , "x1:k1=v1", "x2:k2=v2");}
}

37
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_parser_fxt.java Normal file
View File

@@ -0,0 +1,37 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.xowa.langs.vnts.*;
class Vnt_rule_parser_fxt {
private final Vnt_rule_parser parser = new Vnt_rule_parser(); private final Vnt_rule_undi_mgr undis = new Vnt_rule_undi_mgr(); private final Vnt_rule_bidi_mgr bidis = new Vnt_rule_bidi_mgr();
private final Bry_bfr bfr = Bry_bfr_.New_w_size(255);
public Vnt_rule_parser_fxt() {
Xol_vnt_regy vnt_regy = new Xol_vnt_regy();
vnt_regy.Add(Bry_.new_a7("x1"), Bry_.new_a7("lang1"));
vnt_regy.Add(Bry_.new_a7("x2"), Bry_.new_a7("lang2"));
vnt_regy.Add(Bry_.new_a7("x3"), Bry_.new_a7("lang3"));
parser.Init(null, vnt_regy);
}
public void Test_parse(String raw, String... expd_ary) {
byte[] src = Bry_.new_u8(raw);
parser.Clear(undis, bidis, src);
parser.Parse(src, 0, src.length);
parser.To_bry__dbg(bfr);
Tfds.Eq_str_lines(String_.Concat_lines_nl_skip_last(expd_ary), bfr.To_str_and_clear());
}
}

80
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_undi_mgr.java Normal file
View File

@@ -0,0 +1,80 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
class Vnt_rule_undi_mgr {
private final Ordered_hash hash = Ordered_hash_.New_bry();
public int Len() {return hash.Count();}
public boolean Has_none() {return hash.Count() == 0;}
public void Clear() {hash.Clear();}
public Vnt_rule_undi_grp Get_at(int i) {return (Vnt_rule_undi_grp)hash.Get_at(i);}
public Vnt_rule_undi_grp Get_by(byte[] key) {return (Vnt_rule_undi_grp)hash.Get_by(key);}
public byte[] Get_text_by_key_or_null(byte[] key) {
Vnt_rule_undi_grp grp = (Vnt_rule_undi_grp)hash.Get_by(key); if (grp == null) return null;
return grp.Len() == 0 ? null : grp.Get_at(0).Trg(); // REF.MW: $disp = $disp[0];
}
public byte[] Get_text_at(int i) {
Vnt_rule_undi_grp grp = (Vnt_rule_undi_grp)hash.Get_at(i); if (grp == null) return null;
return grp.Len() == 0 ? null : grp.Get_at(0).Trg();
}
public Vnt_rule_undi_grp Set(byte[] vnt, byte[] src, byte[] trg) {
Vnt_rule_undi_grp grp = (Vnt_rule_undi_grp)hash.Get_by(vnt);
if (grp == null) {
grp = new Vnt_rule_undi_grp(vnt);
hash.Add(vnt, grp);
}
grp.Set(src, trg);
return grp;
}
public void To_bry__dbg(Bry_bfr bfr) {
int len = hash.Count();
for (int i = 0; i < len; ++i) {
if (i != 0) bfr.Add_byte_nl();
Vnt_rule_undi_grp grp = (Vnt_rule_undi_grp)hash.Get_at(i);
bfr.Add(grp.Vnt()).Add_byte_colon();
grp.To_bry__dbg(bfr);
}
}
}
class Vnt_rule_undi_grp {
private final Ordered_hash hash = Ordered_hash_.New_bry();
public Vnt_rule_undi_grp(byte[] vnt) {this.vnt = vnt;}
public int Len() {return hash.Count();}
public Vnt_rule_undi_itm Get_at(int i) {return (Vnt_rule_undi_itm)hash.Get_at(i);}
public byte[] Vnt() {return vnt;} private final byte[] vnt;
public Vnt_rule_undi_itm Set(byte[] src, byte[] trg) {
Vnt_rule_undi_itm itm = (Vnt_rule_undi_itm)hash.Get_by(src);
if (itm == null) {
itm = new Vnt_rule_undi_itm(src, trg);
hash.Add(src, itm);
}
return itm;
}
public void To_bry__dbg(Bry_bfr bfr) {
int len = hash.Count();
for (int i = 0; i < len; ++i) {
Vnt_rule_undi_itm itm = (Vnt_rule_undi_itm)hash.Get_at(i);
bfr.Add(itm.Src()).Add_byte_eq().Add(itm.Trg());
}
}
}
class Vnt_rule_undi_itm {
public Vnt_rule_undi_itm(byte[] src, byte[] trg) {this.src = src; this.trg = trg;}
public byte[] Src() {return src;} private final byte[] src;
public byte[] Trg() {return trg;} private byte[] trg;
public void Trg_(byte[] v) {this.trg = v;}
}