mirror of
https://github.com/gnosygnu/xowa.git
synced 2026-03-02 03:49:30 +00:00
v2.9.4.1
This commit is contained in:
gnosygnu
144
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_converter_lang.java
Normal file
144
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_converter_lang.java
Normal file
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import gplx.core.btries.*; import gplx.core.primitives.*;
|
||||
import gplx.xowa.langs.vnts.*; import gplx.xowa.langs.vnts.converts.*;
|
||||
import gplx.xowa.parsers.htmls.*;
|
||||
public class Vnt_converter_lang {
|
||||
private final Bry_bfr bfr = Bry_bfr.new_();
|
||||
private int max_depth = 32;
|
||||
private byte[] src; private int src_len;
|
||||
private boolean convert_needed;
|
||||
private int pos;
|
||||
private final Vnt_converter_rule converter_rule = new Vnt_converter_rule();
|
||||
private Xol_convert_mgr convert_mgr; private Xol_vnt_regy vnt_regy; // private Xol_vnt_mgr vnt_mgr; // private Xol_vnt_itm vnt_itm;
|
||||
private final Mwh_doc_parser doc_parser = new Mwh_doc_parser();
|
||||
private final Vnt_html_doc_wkr html_convert_wkr;
|
||||
private final Bry_bfr tmp_convert_bfr = Bry_bfr.new_();
|
||||
public Vnt_converter_lang(Xol_convert_mgr convert_mgr, Xol_vnt_regy vnt_regy) {
|
||||
this.html_convert_wkr = new Vnt_html_doc_wkr(convert_mgr);
|
||||
this.convert_mgr = convert_mgr; this.vnt_regy = vnt_regy;
|
||||
}
|
||||
public byte[] Converted_title() {return converted_title;} private byte[] converted_title;
|
||||
public byte[] Parse(Xol_vnt_itm vnt_itm, byte[] src) {// REF.MW:/languages/LanguageConverter.php!recursiveConvertTopLevel
|
||||
this.converted_title = null;
|
||||
converter_rule.Init(this, vnt_regy, vnt_itm);
|
||||
this.converted_title = null;
|
||||
int markup_count = 0;
|
||||
this.pos = 0;
|
||||
this.convert_needed = true; // false for sr lang; SEE:LanguageSr.php !$this->guessVariant(src, vnt);
|
||||
this.src = src; this.src_len = src.length;
|
||||
while (pos < src_len) {
|
||||
int curly_bgn = Bry_find_.Find_fwd(src, Bry__curly_bgn, pos, src_len);
|
||||
if (curly_bgn == Bry_find_.Not_found) { // No more markup, append final segment
|
||||
if (markup_count == 0) return src; // no markups found; just return original
|
||||
Add_output(vnt_itm, convert_needed, src, pos, src_len);
|
||||
return bfr.Xto_bry_and_clear();
|
||||
}
|
||||
Add_output(vnt_itm, convert_needed, src, pos, curly_bgn); // Markup found; append segment
|
||||
pos = curly_bgn; // Advance position
|
||||
++markup_count;
|
||||
Parse_recursive(vnt_itm, 1); // Do recursive conversion
|
||||
}
|
||||
return bfr.Xto_bry_and_clear();
|
||||
}
|
||||
private void Parse_recursive(Xol_vnt_itm vnt_itm, int depth) {
|
||||
pos += 2; // skip "-{"
|
||||
boolean warning_done = false;
|
||||
int bgn_pos = pos;
|
||||
while (pos < src_len) {
|
||||
byte b = src[pos];
|
||||
Object o = trie.Match_bgn_w_byte(b, src,pos, src_len);
|
||||
if (o == null) { // char;
|
||||
++pos;
|
||||
continue;
|
||||
}
|
||||
switch (((Byte_obj_val)o).Val()) {
|
||||
case Tid__curly_bgn:
|
||||
if (depth >= max_depth) {
|
||||
bfr.Add(Bry__curly_bgn);
|
||||
if (!warning_done) {
|
||||
bfr.Add_str("<span class=\"error\">");
|
||||
// wfMessage('language-converter-depth-warning')->numParams($this->mMaxDepth)->inContentLanguage()->text()
|
||||
bfr.Add_str("</span>");
|
||||
warning_done = true;
|
||||
}
|
||||
pos += 2; // skip "-{"
|
||||
continue;
|
||||
}
|
||||
bgn_pos = pos;
|
||||
Parse_recursive(vnt_itm, depth + 1); // Recursively parse another rule
|
||||
break;
|
||||
case Tid__curly_end:
|
||||
converter_rule.Parse(src, bgn_pos, pos);
|
||||
Apply_manual_conv(converter_rule);
|
||||
bfr.Add(converter_rule.Display());
|
||||
pos += 2;
|
||||
return;
|
||||
default: throw Err_.new_unhandled(-1); // never happens
|
||||
}
|
||||
}
|
||||
if (pos < src_len) { // Unclosed rule
|
||||
bfr.Add(Bry__curly_bgn);
|
||||
Auto_convert(bfr, vnt_itm, src, pos, src_len);
|
||||
}
|
||||
pos = src_len;
|
||||
}
|
||||
private void Add_output(Xol_vnt_itm vnt_itm, boolean convert_needed, byte[] src, int bgn, int end) {
|
||||
if (end - bgn == 0) return;
|
||||
if (convert_needed) {
|
||||
Auto_convert(bfr, vnt_itm, src, bgn, end);
|
||||
}
|
||||
else
|
||||
bfr.Add_mid(src, bgn, end);
|
||||
}
|
||||
public byte[] Auto_convert(Xol_vnt_itm vnt_itm, byte[] src) {
|
||||
Auto_convert(tmp_convert_bfr, vnt_itm, src, 0, src.length);
|
||||
return tmp_convert_bfr.Xto_bry_and_clear();
|
||||
}
|
||||
private void Auto_convert(Bry_bfr bfr, Xol_vnt_itm vnt_itm, byte[] src, int bgn, int end) {
|
||||
html_convert_wkr.Init(bfr, vnt_itm.Idx());
|
||||
doc_parser.Parse(html_convert_wkr, src, bgn, end);
|
||||
}
|
||||
private void Apply_manual_conv(Vnt_converter_rule rule) {
|
||||
this.converted_title = rule.Title();
|
||||
byte action = rule.Action();
|
||||
Vnt_rule_undi_mgr cnv_tbl = rule.Cnv_tbl();
|
||||
int len = cnv_tbl.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Vnt_rule_undi_grp grp = cnv_tbl.Get_at(i);
|
||||
byte[] grp_key = grp.Vnt();
|
||||
Xol_vnt_itm vnt_itm = vnt_regy.Get_by(grp_key); if (vnt_itm == null) continue;
|
||||
int grp_len = grp.Len();
|
||||
Xol_convert_wkr wkr = convert_mgr.Converter_ary()[vnt_itm.Idx()];
|
||||
for (int j = 0; j < grp_len; ++j) {
|
||||
Vnt_rule_undi_itm itm = grp.Get_at(j);
|
||||
if (action == Byte_ascii.Plus) {
|
||||
wkr.Add(itm.Src(), itm.Trg());
|
||||
}
|
||||
else if (action == Byte_ascii.Dash)
|
||||
wkr.Del(itm.Src());
|
||||
}
|
||||
}
|
||||
}
|
||||
private static final byte Tid__curly_bgn = 1, Tid__curly_end = 2;
|
||||
private static final byte[] Bry__curly_bgn = Bry_.new_a7("-{"), Bry__curly_end = Bry_.new_a7("}-");
|
||||
private static final Btrie_fast_mgr trie = Btrie_fast_mgr.cs()
|
||||
.Add_bry_byte(Bry__curly_bgn, Tid__curly_bgn)
|
||||
.Add_bry_byte(Bry__curly_end, Tid__curly_end);
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import org.junit.*; import gplx.xowa.langs.vnts.*; import gplx.xowa.langs.vnts.converts.*;
|
||||
public class Vnt_converter_lang__html__tst { // REF: https://www.mediawiki.org/wiki/Writing_systems/Syntax
|
||||
private final Vnt_converter_lang_fxt fxt = new Vnt_converter_lang_fxt();
|
||||
private String rule;
|
||||
@Before public void init() {
|
||||
rule = "-{H|zh-cn:cn;zh-hk:hk;zh-tw:tw}-";
|
||||
}
|
||||
@Test public void Node() {
|
||||
fxt.Test_parse(rule + "hk<span>hk</span>hk", "cn<span>cn</span>cn");
|
||||
}
|
||||
@Test public void Attribs() {
|
||||
fxt.Test_parse(rule + "<span class='hk'>hk</span>", "<span class='hk'>cn</span>");
|
||||
}
|
||||
@Test public void Attribs__title() {
|
||||
fxt.Test_parse(rule + "<span title='hk'>hk</span>", "<span title='cn'>cn</span>");
|
||||
}
|
||||
@Test public void Attribs__alt() {
|
||||
fxt.Test_parse(rule + "<span alt='hk'>hk</span>", "<span alt='cn'>cn</span>");
|
||||
}
|
||||
@Test public void Attribs__skip_url() {
|
||||
fxt.Test_parse(rule + "<span alt='http://hk.org'>hk</span>", "<span alt='http://hk.org'>cn</span>");
|
||||
}
|
||||
@Test public void Node__style() {
|
||||
fxt.Test_parse(rule + "hk<script>hk</script>hk", "cn<script>hk</script>cn");
|
||||
}
|
||||
@Test public void Node__code() {
|
||||
fxt.Test_parse(rule + "hk<code>hk</code>hk", "cn<code>hk</code>cn");
|
||||
}
|
||||
@Test public void Node__pre() {
|
||||
fxt.Test_parse(rule + "hk<pre>hk</pre>hk", "cn<pre>hk</pre>cn");
|
||||
}
|
||||
@Test public void Node__pre__nested() {
|
||||
fxt.Test_parse(rule + "hk<pre><span>hk</span></pre>hk", "cn<pre><span>hk</span></pre>cn");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import org.junit.*; import gplx.xowa.langs.vnts.*; import gplx.xowa.langs.vnts.converts.*;
|
||||
public class Vnt_converter_lang__syntax__tst { // REF: https://www.mediawiki.org/wiki/Writing_systems/Syntax
|
||||
private final Vnt_converter_lang_fxt fxt = new Vnt_converter_lang_fxt();
|
||||
@Test public void Bidi() {
|
||||
String text = "-{zh-hans:a;zh-hant:b}-";
|
||||
fxt.Test_parse_many(text, "a", "zh-hans", "zh-cn", "zh-sg", "zh");
|
||||
fxt.Test_parse_many(text, "b", "zh-hant", "zh-hk", "zh-tw");
|
||||
}
|
||||
@Test public void Undi() {
|
||||
String text = "-{H|cn_k=>zh-cn:cn_v}-cn_k";
|
||||
fxt.Test_parse_many(text, "cn_k", "zh", "zh-hans", "zh-hant", "zh-hk", "zh-my", "zh-mo", "zh-sg", "zh-tw");
|
||||
fxt.Test_parse_many(text, "cn_v", "zh-cn");
|
||||
}
|
||||
@Test public void Raw() {
|
||||
fxt.Test_parse_many("-{a}-", "a", "zh-hans", "zh-cn", "zh-sg", "zh", "zh-hant", "zh-hk", "zh-tw");
|
||||
fxt.Test_parse_many("-{R|a}-", "a", "zh-hans", "zh-cn", "zh-sg", "zh", "zh-hant", "zh-hk", "zh-tw");
|
||||
}
|
||||
@Test public void Hide() {
|
||||
String text = "-{H|zh-cn:cn;zh-hk:hk;zh-tw:tw}-cn hk tw";
|
||||
fxt.Test_parse_many(text, "cn cn cn", "zh-cn", "zh-sg");
|
||||
fxt.Test_parse_many(text, "hk hk hk", "zh-hk");
|
||||
fxt.Test_parse_many(text, "tw tw tw", "zh-tw");
|
||||
fxt.Test_parse_many(text, "cn hk tw", "zh", "zh-hans", "zh-hant");
|
||||
}
|
||||
@Test public void Aout() {
|
||||
String text = "-{A|zh-cn:cn;zh-hk:hk;zh-tw:tw}- cn hk tw";
|
||||
fxt.Test_parse_many(text, "cn cn cn cn", "zh-cn", "zh-sg");
|
||||
fxt.Test_parse_many(text, "hk hk hk hk", "zh-hk");
|
||||
fxt.Test_parse_many(text, "tw tw tw tw", "zh-tw");
|
||||
fxt.Test_parse_many(text, "cn cn hk tw", "zh", "zh-hans");
|
||||
fxt.Test_parse_many(text, "tw cn hk tw", "zh-hant");
|
||||
fxt.Test_parse_many("h-{}-k", "hk", "zh-cn"); // semi-disabled
|
||||
}
|
||||
@Test public void Del() {
|
||||
String text = "-{H|zh-cn:cn;zh-hk:hk;zh-tw:tw}-cn hk tw-{-|zh-cn:cn;zh-hk:hk;zh-tw:tw}- cn hk tw";
|
||||
fxt.Test_parse_many(text, "cn cn cn cn hk tw", "zh-cn", "zh-sg");
|
||||
fxt.Test_parse_many(text, "hk hk hk cn hk tw", "zh-hk");
|
||||
fxt.Test_parse_many(text, "tw tw tw cn hk tw", "zh-tw");
|
||||
fxt.Test_parse_many(text, "cn hk tw cn hk tw", "zh", "zh-hans", "zh-hant");
|
||||
}
|
||||
@Test public void Title() {
|
||||
fxt.Test_parse_title("-{}-", null, "", "zh-cn");
|
||||
String text = "-{T|zh-cn:cn;zh-hk:hk;zh-tw:tw}-cn hk tw";
|
||||
fxt.Test_parse_title(text, "cn", "cn hk tw", "zh-cn");
|
||||
fxt.Test_parse_title(text, "cn", "cn hk tw", "zh-sg");
|
||||
fxt.Test_parse_title(text, "hk", "cn hk tw", "zh-hk");
|
||||
fxt.Test_parse_title(text, "tw", "cn hk tw", "zh-tw");
|
||||
fxt.Test_parse_title(text, "cn", "cn hk tw", "zh-hans");
|
||||
fxt.Test_parse_title(text, "tw", "cn hk tw", "zh-hant");
|
||||
fxt.Test_parse_title(text, null, "cn hk tw", "zh");
|
||||
}
|
||||
@Test public void Descrip() {
|
||||
String text = "-{D|zh-cn:cn;zh-hk:hk;zh-tw:tw}-";
|
||||
fxt.Test_parse_many(text, "ZH-CN:cn;ZH-HK:hk;ZH-TW:tw;", "zh", "zh-hans", "zh-hant", "zh-cn", "zh-hk", "zh-my", "zh-mo", "zh-sg", "zh-tw");
|
||||
}
|
||||
@Test public void Mixture() {
|
||||
String text = "-{H|zh-cn:cn;zh-hk:hk;zh-tw:tw}--{zh;zh-hans;zh-hant|cn hk tw}- -{zh;zh-cn;zh-hk;zh-tw|cn hk tw}-";
|
||||
fxt.Test_parse_many(text, "cn hk tw cn cn cn", "zh-cn", "zh-sg", "zh-hans");
|
||||
fxt.Test_parse_many(text, "cn hk tw hk hk hk", "zh-hk");
|
||||
fxt.Test_parse_many(text, "cn hk tw tw tw tw", "zh-tw", "zh-hant");
|
||||
fxt.Test_parse_many(text, "cn hk tw cn hk tw", "zh");
|
||||
}
|
||||
@Test public void Descrip__undi() {fxt.Test_parse("-{D|cn_k=>zh-cn:cn_v;hk_k=>zh-hk:hk_v}-", "cn_k⇒ZH-CN:cn_v;hk_k⇒ZH-HK:hk_v;");}
|
||||
@Test public void Descrip__mixd() {fxt.Test_parse("-{D|zh-tw:tw_v;cn_k=>zh-cn:cn_v;hk_k=>zh-hk:hk_v;zh-mo:mo_v}-", "ZH-TW:tw_v;ZH-MO:mo_v;cn_k⇒ZH-CN:cn_v;hk_k⇒ZH-HK:hk_v;");}
|
||||
}
|
||||
class Vnt_converter_lang_fxt {
|
||||
private final Vnt_converter_lang converter_lang;
|
||||
private final Xol_convert_mgr convert_mgr = new Xol_convert_mgr();
|
||||
private final Xol_vnt_regy vnt_regy = Xol_vnt_regy_fxt.new_chinese();
|
||||
private Xol_vnt_itm vnt_itm;
|
||||
public Vnt_converter_lang_fxt() {
|
||||
converter_lang = new Vnt_converter_lang(convert_mgr, vnt_regy);
|
||||
convert_mgr.Init(vnt_regy);
|
||||
Init_cur("zh-cn");
|
||||
}
|
||||
public void Init_cur(String vnt) {
|
||||
byte[] cur_vnt = Bry_.new_a7(vnt);
|
||||
this.vnt_itm = vnt_regy.Get_by(cur_vnt);
|
||||
convert_mgr.Cur_vnt_(cur_vnt);
|
||||
}
|
||||
public void Test_parse(String raw, String expd) {
|
||||
Tfds.Eq_str(expd, String_.new_u8(converter_lang.Parse(vnt_itm, Bry_.new_u8(raw))));
|
||||
}
|
||||
public void Test_parse_many(String raw, String expd, String... vnts) {
|
||||
int len = vnts.length;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
String vnt_key = vnts[i];
|
||||
Init_cur(vnt_key);
|
||||
Xol_vnt_itm vnt = vnt_regy.Get_by(Bry_.new_a7(vnt_key));
|
||||
Tfds.Eq_str(expd, String_.new_u8(converter_lang.Parse(vnt, Bry_.new_u8(raw))), vnt_key);
|
||||
}
|
||||
}
|
||||
public void Test_parse_title(String raw, String expd_title, String expd_text, String vnt_key) {
|
||||
Init_cur(vnt_key);
|
||||
Xol_vnt_itm vnt = vnt_regy.Get_by(Bry_.new_a7(vnt_key));
|
||||
Tfds.Eq_str(expd_text, String_.new_u8(converter_lang.Parse(vnt, Bry_.new_u8(raw))), vnt_key);
|
||||
Tfds.Eq_str(expd_title, converter_lang.Converted_title());
|
||||
}
|
||||
}
|
||||
@@ -17,42 +17,170 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import gplx.core.btries.*; import gplx.core.primitives.*;
|
||||
import gplx.xowa.langs.vnts.*;
|
||||
class Vnt_converter_rule { // REF.MW: /languages/LanguageConverter.php|ConverterRule
|
||||
private final byte[] src;
|
||||
private final int src_bgn, src_end;
|
||||
private int pipe_pos = -1;
|
||||
public Vnt_converter_rule(byte[] src, int src_bgn, int src_end) {
|
||||
this.src = src; this.src_bgn = src_bgn; this.src_end = src_end;
|
||||
private final Vnt_flag_parser flag_parser = new Vnt_flag_parser(); private final Vnt_flag_code_mgr flag_codes = new Vnt_flag_code_mgr(); private final Vnt_flag_lang_mgr flag_langs = new Vnt_flag_lang_mgr();
|
||||
private final Vnt_rule_parser rule_parser = new Vnt_rule_parser(); private final Vnt_rule_undi_mgr rule_undis = new Vnt_rule_undi_mgr(); private final Vnt_rule_bidi_mgr rule_bidis = new Vnt_rule_bidi_mgr();
|
||||
private final Bry_bfr tmp_bfr = Bry_bfr.new_();
|
||||
private final Ordered_hash cnv_marked_hash = Ordered_hash_.new_bry_();
|
||||
private Vnt_converter_lang converter;
|
||||
private Xol_vnt_regy vnt_regy; private Xol_vnt_itm vnt_itm; private byte[] vnt_key;
|
||||
private byte[] rule_raw;
|
||||
public byte[] Display() {return display;} private byte[] display;
|
||||
public byte[] Title() {return title;} private byte[] title;
|
||||
public byte Action() {return action;} private byte action;
|
||||
public Vnt_rule_undi_mgr Cnv_tbl() {return cnv_tbl;} private final Vnt_rule_undi_mgr cnv_tbl = new Vnt_rule_undi_mgr();
|
||||
public void Init(Vnt_converter_lang converter, Xol_vnt_regy vnt_regy, Xol_vnt_itm vnt_itm) {
|
||||
this.converter = converter;
|
||||
this.vnt_regy = vnt_regy; this.vnt_itm = vnt_itm; this.vnt_key = vnt_itm.Key();
|
||||
rule_parser.Init(vnt_regy);
|
||||
}
|
||||
public void Parse() {
|
||||
}
|
||||
public void Parse_flags(Vnt_flag_parser parser) {
|
||||
this.pipe_pos = Bry_find_.Find_fwd(src, Byte_ascii.Pipe, src_bgn, src_end);
|
||||
if (pipe_pos != Bry_find_.Not_found) // "|" found; EX: -{A|}-
|
||||
parser.Parse(src, src_bgn, pipe_pos);
|
||||
int flag_count = parser.Count();
|
||||
if (flag_count == 0) parser.Set_y(Vnt_flag_itm_.Tid_show);
|
||||
else if (parser.Limit_if_exists(Vnt_flag_itm_.Tid_raw)) {}
|
||||
else if (parser.Limit_if_exists(Vnt_flag_itm_.Tid_name)) {}
|
||||
else if (parser.Limit_if_exists(Vnt_flag_itm_.Tid_del)) {}
|
||||
else if (flag_count == 1 && parser.Get(Vnt_flag_itm_.Tid_title)) parser.Set_y(Vnt_flag_itm_.Tid_macro);
|
||||
else if (parser.Get(Vnt_flag_itm_.Tid_macro)) {
|
||||
boolean exists_d = parser.Get(Vnt_flag_itm_.Tid_descrip);
|
||||
boolean exists_t = parser.Get(Vnt_flag_itm_.Tid_title);
|
||||
parser.Clear();
|
||||
parser.Set_y_many(Vnt_flag_itm_.Tid_all, Vnt_flag_itm_.Tid_macro);
|
||||
if (exists_d) parser.Set_y(Vnt_flag_itm_.Tid_descrip);
|
||||
if (exists_t) parser.Set_y(Vnt_flag_itm_.Tid_title);
|
||||
public void Parse(byte[] src, int src_bgn, int src_end) {
|
||||
this.display = this.title = null;
|
||||
this.action = Byte_ascii.Null;
|
||||
int pipe_pos = Bry_find_.Find_fwd(src, Byte_ascii.Pipe, src_bgn, src_end);
|
||||
flag_parser.Parse(flag_codes, flag_langs, vnt_regy, src, src_bgn, pipe_pos);
|
||||
int rule_bgn = pipe_pos == -1 ? src_bgn : pipe_pos + 1;
|
||||
this.rule_raw = Bry_.Mid(src, rule_bgn, src_end);
|
||||
int flag_langs_count = flag_langs.Count();
|
||||
if (flag_langs_count > 0) { // vnts exist in flag; EX: -{zh-hans;zh-hant|text}-
|
||||
if (flag_langs.Has(vnt_key))
|
||||
rule_raw = converter.Auto_convert(vnt_itm, rule_raw); // convert rule text to current language; EX:-{|convert}-
|
||||
else {
|
||||
byte[][] fallbacks = vnt_itm.Fallback_ary();
|
||||
int fallbacks_len = fallbacks.length;
|
||||
for (int i = 0; i < fallbacks_len; ++i) {
|
||||
byte[] fallback = fallbacks[i];
|
||||
if (flag_langs.Has(fallback)) {
|
||||
Xol_vnt_itm fallback_itm = (Xol_vnt_itm)vnt_regy.Get_by(fallback);
|
||||
rule_raw = converter.Auto_convert(fallback_itm, rule_raw);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
flag_codes.Limit(Vnt_flag_code_.Tid_raw);
|
||||
}
|
||||
else {
|
||||
if (parser.Get(Vnt_flag_itm_.Tid_add))
|
||||
parser.Set_y_many(Vnt_flag_itm_.Tid_all, Vnt_flag_itm_.Tid_show);
|
||||
if (parser.Get(Vnt_flag_itm_.Tid_descrip))
|
||||
parser.Set_n(Vnt_flag_itm_.Tid_show);
|
||||
parser.Limit_if_exists_vnts(); // try to find flags like "zh-hans", "zh-hant"; allow syntaxes like "-{zh-hans;zh-hant|XXXX}-"
|
||||
rule_parser.Clear(rule_undis, rule_bidis, rule_raw);
|
||||
if (!flag_codes.Get(Vnt_flag_code_.Tid_raw) && !flag_codes.Get(Vnt_flag_code_.Tid_name)) {
|
||||
rule_parser.Parse(src, rule_bgn, src_end);
|
||||
}
|
||||
if (rule_undis.Has_none() && rule_bidis.Has_none()) {
|
||||
if ( flag_codes.Get(Vnt_flag_code_.Tid_add)
|
||||
|| flag_codes.Get(Vnt_flag_code_.Tid_del)
|
||||
) { // fill all variants if text in -{A/H/-|text} without rules
|
||||
for (int i = 0; i < flag_langs_count; ++i) {
|
||||
Xol_vnt_itm itm = flag_langs.Get_at(i);
|
||||
rule_bidis.Set(itm.Key(), rule_raw);
|
||||
}
|
||||
}
|
||||
else if ( !flag_codes.Get(Vnt_flag_code_.Tid_name)
|
||||
&& !flag_codes.Get(Vnt_flag_code_.Tid_title)
|
||||
) {
|
||||
flag_codes.Limit(Vnt_flag_code_.Tid_raw);
|
||||
}
|
||||
}
|
||||
int flag_count = Vnt_flag_code_.Tid__max;
|
||||
for (int flag = 0; flag < flag_count; ++flag) {
|
||||
if (!flag_codes.Get(flag)) continue;
|
||||
switch (flag) {
|
||||
case Vnt_flag_code_.Tid_raw: display = rule_parser.Raw(); break; // if we don't do content convert, still strip the -{}- tags
|
||||
case Vnt_flag_code_.Tid_name: // process N flag: output current variant name
|
||||
byte[] vnt_key_trim = Bry_.Trim(rule_parser.Raw());
|
||||
Xol_vnt_itm vnt_itm_trim = vnt_regy.Get_by(vnt_key_trim);
|
||||
display = vnt_itm_trim == null ? display = Bry_.Empty : vnt_itm_trim.Name();
|
||||
break;
|
||||
case Vnt_flag_code_.Tid_descrip: display = Make_descrip(); break; // process D flag: output rules description
|
||||
case Vnt_flag_code_.Tid_hide: display = Bry_.Empty; break; // process H,- flag or T only: output nothing
|
||||
case Vnt_flag_code_.Tid_del: display = Bry_.Empty; action = Byte_ascii.Dash; break;
|
||||
case Vnt_flag_code_.Tid_add: display = Bry_.Empty; action = Byte_ascii.Plus; break;
|
||||
case Vnt_flag_code_.Tid_show: display = Make_converted(vnt_itm); break;
|
||||
case Vnt_flag_code_.Tid_title: display = Bry_.Empty; title = Make_title(vnt_itm); break;
|
||||
default: break; // ignore unknown flags (but see error case below)
|
||||
}
|
||||
}
|
||||
if (display == null)
|
||||
display = Bry_.Add(Bry__error_bgn, Bry__error_end); // wfMessage( 'converter-manual-rule-error' )->inContentLanguage()->escaped()
|
||||
Make_conv_tbl();
|
||||
}
|
||||
private void Make_conv_tbl() {
|
||||
if (rule_undis.Has_none() && rule_bidis.Has_none()) return; // Special case optimisation
|
||||
cnv_tbl.Clear(); cnv_marked_hash.Clear();
|
||||
int vnt_regy_len = vnt_regy.Len();
|
||||
for (int i = 0; i < vnt_regy_len; ++i) {
|
||||
Xol_vnt_itm vnt = vnt_regy.Get_at(i);
|
||||
byte[] vnt_key = vnt.Key();
|
||||
// bidi: fill in missing variants with fallbacks
|
||||
byte[] bidi_bry = rule_bidis.Get_text_by_key_or_null(vnt_key);
|
||||
if (bidi_bry == null) {
|
||||
bidi_bry = rule_bidis.Get_text_by_ary_or_null(vnt.Fallback_ary());
|
||||
if (bidi_bry != null) rule_bidis.Set(vnt_key, bidi_bry);
|
||||
}
|
||||
if (bidi_bry != null) {
|
||||
int marked_len = cnv_marked_hash.Count();
|
||||
for (int j = 0; j < marked_len; ++j) {
|
||||
Xol_vnt_itm marked_itm = (Xol_vnt_itm)cnv_marked_hash.Get_at(j);
|
||||
byte[] marked_key = marked_itm.Key();
|
||||
byte[] marked_bry = rule_bidis.Get_text_by_key_or_null(marked_key);
|
||||
byte[] cur_bidi_bry = rule_bidis.Get_text_by_key_or_null(vnt_key);
|
||||
if (vnt.Dir() == Xol_vnt_dir_.Tid__bi)
|
||||
cnv_tbl.Set(vnt_key, marked_bry, cur_bidi_bry);
|
||||
if (marked_itm.Dir() == Xol_vnt_dir_.Tid__bi)
|
||||
cnv_tbl.Set(marked_key, cur_bidi_bry, marked_bry);
|
||||
}
|
||||
cnv_marked_hash.Add(vnt_key, vnt);
|
||||
}
|
||||
// undi: fill to convert tables
|
||||
byte[] undi_bry = rule_undis.Get_text_by_key_or_null(vnt_key);
|
||||
if (vnt.Dir() != Xol_vnt_dir_.Tid__none && undi_bry != null) {
|
||||
Vnt_rule_undi_grp undi_grp = rule_undis.Get_by(vnt_key);
|
||||
int undi_grp_len = undi_grp.Len();
|
||||
for (int j = 0; j < undi_grp_len; ++j) {
|
||||
Vnt_rule_undi_itm undi_itm = undi_grp.Get_at(j);
|
||||
cnv_tbl.Set(vnt_key, undi_itm.Src(), undi_itm.Trg());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
public void Parse_rules(Vnt_rule_parser parser) {
|
||||
parser.Parse(src, src_bgn, src_end);
|
||||
private byte[] Make_descrip() {
|
||||
int len = rule_bidis.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Vnt_rule_bidi_itm bidi_itm = rule_bidis.Get_at(i);
|
||||
Xol_vnt_itm vnt_itm = vnt_regy.Get_by(bidi_itm.Vnt());
|
||||
tmp_bfr.Add(vnt_itm.Name()).Add_byte_colon().Add(bidi_itm.Text()).Add_byte_semic();
|
||||
}
|
||||
len = rule_undis.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Vnt_rule_undi_grp undi_grp = rule_undis.Get_at(i);
|
||||
int sub_len = undi_grp.Len();
|
||||
for (int j = 0; j < sub_len; ++j) {
|
||||
Vnt_rule_undi_itm undi_itm = (Vnt_rule_undi_itm)undi_grp.Get_at(j);
|
||||
Xol_vnt_itm undi_vnt = vnt_regy.Get_by(undi_grp.Vnt());
|
||||
tmp_bfr.Add(undi_itm.Src()).Add(Bry__undi_spr).Add(undi_vnt.Name()).Add_byte_colon().Add(undi_itm.Trg()).Add_byte_semic();
|
||||
}
|
||||
}
|
||||
return tmp_bfr.Xto_bry_and_clear();
|
||||
}
|
||||
private byte[] Make_title(Xol_vnt_itm vnt) {
|
||||
if (vnt.Idx() == 0) { // for mainLanguageCode; EX: "zh"
|
||||
byte[] rv = rule_bidis.Get_text_by_key_or_null(vnt.Key());
|
||||
return rv == null ? rule_undis.Get_text_by_key_or_null(vnt.Key()) : rv;
|
||||
}
|
||||
else
|
||||
return Make_converted(vnt);
|
||||
}
|
||||
private byte[] Make_converted(Xol_vnt_itm vnt) {
|
||||
if (rule_bidis.Len() == 0 && rule_undis.Len() == 0) return rule_raw;
|
||||
byte[] rv = rule_bidis.Get_text_by_key_or_null(vnt.Key()); // display current variant in bidirectional array
|
||||
if (rv == null) rv = rule_bidis.Get_text_by_ary_or_null(vnt.Fallback_ary()); // or display current variant in fallbacks
|
||||
if (rv == null) rv = rule_undis.Get_text_by_key_or_null(vnt.Key()); // or display current variant in unidirectional array
|
||||
if (rv == null && vnt.Dir() == Xol_vnt_dir_.Tid__none) { // or display first text under disable manual convert
|
||||
rv = (rule_bidis.Len() > 0) ? rule_bidis.Get_text_at(0) : rule_undis.Get_text_at(0);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
private final static byte[]
|
||||
Bry__error_bgn = Bry_.new_a7("<span class=\"error\">")
|
||||
, Bry__error_end = Bry_.new_a7("</span>")
|
||||
, Bry__undi_spr = Bry_.new_u8("⇒")
|
||||
;
|
||||
}
|
||||
|
||||
@@ -16,37 +16,35 @@ You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
class Vnt_flag_itm_ {
|
||||
class Vnt_flag_code_ {
|
||||
public static final int
|
||||
Tid_unknown = 0
|
||||
, Tid_show = 1 // S: EX: -{S|zh-hans:A;zh-hant:B}- -> "A"
|
||||
, Tid_all = 2 // +: EX: -{+|zh-hans:A;zh-hant:B}- -> "A"
|
||||
, Tid_err = 3 // E: EX: -{E|zh-hans:A;zh-hant:B}- -> "A"
|
||||
, Tid_add = 4 // A: add and output; EX: -{A|zh-hans:A;zh-hant:B}- -> "A"
|
||||
, Tid_title = 5 // T: page_title; EX: -{T|zh-hans:A;zh-hant:B}- -> ""
|
||||
, Tid_raw = 6 // R: raw: no convert; EX: -{R|zh-hans:A;zh-hant:B}- -> "zh-hans:A;zh-hant:B"
|
||||
, Tid_descrip = 7 // D: describe; EX: -{D|zh-hans:A;zh-hant:B}- -> "简体：A；繁體：B；" (简体=Simplified;繁體=Traditional)
|
||||
, Tid_del = 8 // -: remove; EX: -{-|zh-hans:A;zh-hant:B}- -> ""
|
||||
, Tid_macro = 9 // H: macro; EX: -{H|zh-hans:A;zh-hant:B}- -> ""
|
||||
, Tid_name = 10 // N: EX: -{N|zh-hans:A;zh-hant:B}- -> ""
|
||||
, Tid_lang = 11 // vnt: EX: -{zh-hant|B}- -> "B"
|
||||
, Tid__max = 12
|
||||
Tid_add = 0 // +: EX: -{+|zh-hans:A;zh-hant:B}- -> "A"
|
||||
, Tid_del = 1 // -: remove; EX: -{-|zh-hans:A;zh-hant:B}- -> ""
|
||||
, Tid_aout = 2 // A: Add and output; EX: -{A|zh-hans:A;zh-hant:B}- -> "A"
|
||||
, Tid_hide = 3 // H: Hide macro; EX: -{H|zh-hans:A;zh-hant:B}- -> ""
|
||||
, Tid_raw = 4 // R: Raw: no convert; EX: -{R|zh-hans:A;zh-hant:B}- -> "zh-hans:A;zh-hant:B"
|
||||
, Tid_show = 5 // S: Show EX: -{S|zh-hans:A;zh-hant:B}- -> "A"
|
||||
, Tid_descrip = 6 // D: Describe; EX: -{D|zh-hans:A;zh-hant:B}- -> "简体：A；繁體：B；" (简体=Simplified;繁體=Traditional)
|
||||
, Tid_name = 7 // N: variant Name EX: -{N|zh-hans:A;zh-hant:B}- -> ""
|
||||
, Tid_title = 8 // T: page Title; EX: -{T|zh-hans:A;zh-hant:B}- -> ""
|
||||
, Tid_err = 9 // E: Error EX: -{E|zh-hans:A;zh-hant:B}- -> "A"
|
||||
, Tid__max = 10
|
||||
;
|
||||
private static final String[] Tid__names = new String[]
|
||||
{ "unknown", "show", "all", "err", "add", "title"
|
||||
, "raw", "descrip", "del", "macro", "name", "lang"
|
||||
{ "+", "-", "A", "H", "R"
|
||||
, "S", "D", "N", "T", "E"
|
||||
};
|
||||
public static String To_name(int tid) {return Tid__names[tid];}
|
||||
public static String To_str(int tid) {return Tid__names[tid];}
|
||||
public static final Hash_adp_bry Regy = Hash_adp_bry.ci_a7() // NOTE: match either lc or uc; EX: -{D}- or -{d}-;
|
||||
.Add_byte_int(Byte_ascii.Ltr_S , Tid_show)
|
||||
.Add_byte_int(Byte_ascii.Plus , Tid_all)
|
||||
.Add_byte_int(Byte_ascii.Ltr_E , Tid_err)
|
||||
.Add_byte_int(Byte_ascii.Ltr_A , Tid_add)
|
||||
.Add_byte_int(Byte_ascii.Ltr_T , Tid_title)
|
||||
.Add_byte_int(Byte_ascii.Ltr_R , Tid_raw)
|
||||
.Add_byte_int(Byte_ascii.Ltr_D , Tid_descrip)
|
||||
.Add_byte_int(Byte_ascii.Plus , Tid_add)
|
||||
.Add_byte_int(Byte_ascii.Dash , Tid_del)
|
||||
.Add_byte_int(Byte_ascii.Ltr_H , Tid_macro)
|
||||
.Add_byte_int(Byte_ascii.Ltr_A , Tid_aout)
|
||||
.Add_byte_int(Byte_ascii.Ltr_H , Tid_hide)
|
||||
.Add_byte_int(Byte_ascii.Ltr_R , Tid_raw)
|
||||
.Add_byte_int(Byte_ascii.Ltr_S , Tid_show)
|
||||
.Add_byte_int(Byte_ascii.Ltr_D , Tid_descrip)
|
||||
.Add_byte_int(Byte_ascii.Ltr_N , Tid_name)
|
||||
.Add_byte_int(Byte_ascii.Ltr_T , Tid_title)
|
||||
.Add_byte_int(Byte_ascii.Ltr_E , Tid_err)
|
||||
;
|
||||
}
|
||||
56
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_code_mgr.java
Normal file
56
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_code_mgr.java
Normal file
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
class Vnt_flag_code_mgr {
|
||||
private final boolean[] ary = new boolean[Ary_len]; private final static int Ary_len = Vnt_flag_code_.Tid__max;
|
||||
public int Count() {return count;} private int count = 0;
|
||||
public boolean Get(int tid) {return ary[tid];}
|
||||
public void Clear() {
|
||||
count = 0;
|
||||
for (int i = 0; i < Ary_len; ++i)
|
||||
ary[i] = false;
|
||||
}
|
||||
public void Add(int tid) {
|
||||
this.Set_y(tid);
|
||||
++count;
|
||||
}
|
||||
public void Set_y(int tid) {ary[tid] = Bool_.Y;}
|
||||
public void Set_y_many(int... vals) {
|
||||
int len = vals.length;
|
||||
for (int i = 0; i < len; ++i)
|
||||
ary[vals[i]] = Bool_.Y;
|
||||
}
|
||||
public void Set_n(int tid) {ary[tid] = Bool_.N;}
|
||||
public void Limit(int tid) {
|
||||
for (int i = 0; i < Ary_len; ++i)
|
||||
ary[i] = i == tid;
|
||||
}
|
||||
public boolean Limit_if_exists(int tid) {
|
||||
boolean exists = ary[tid]; if (!exists) return false;
|
||||
this.Limit(tid);
|
||||
return true;
|
||||
}
|
||||
public void To_bfr__dbg(Bry_bfr bfr) {
|
||||
for (int i = 0; i < Ary_len; ++i) {
|
||||
if (ary[i]) {
|
||||
if (bfr.Len_gt_0()) bfr.Add_byte_semic();
|
||||
bfr.Add_str_a7(Vnt_flag_code_.To_str(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
35
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_lang_mgr.java
Normal file
35
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_lang_mgr.java
Normal file
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import gplx.xowa.langs.vnts.*;
|
||||
class Vnt_flag_lang_mgr {
|
||||
private final Ordered_hash regy = Ordered_hash_.new_bry_();
|
||||
public int Count() {return regy.Count();}
|
||||
public boolean Has(byte[] vnt) {return regy.Has(vnt);}
|
||||
public void Clear() {regy.Clear();}
|
||||
public void Add(Xol_vnt_itm itm) {regy.Add(itm.Key(), itm);}
|
||||
public Xol_vnt_itm Get_at(int i) {return (Xol_vnt_itm)regy.Get_at(i);}
|
||||
public void To_bfr__dbg(Bry_bfr bfr) {
|
||||
int len = regy.Count();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xol_vnt_itm itm = (Xol_vnt_itm)regy.Get_at(i);
|
||||
if (bfr.Len_gt_0()) bfr.Add_byte_semic();
|
||||
bfr.Add(itm.Key());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,52 +16,48 @@ You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import gplx.xowa.langs.vnts.*;
|
||||
class Vnt_flag_parser implements gplx.core.brys.Bry_split_wkr {
|
||||
private final Hash_adp_bry flag_regy = Vnt_flag_itm_.Regy;
|
||||
private final Hash_adp_bry vnt_regy = Hash_adp_bry.cs();
|
||||
private final boolean[] flag_ary = new boolean[Vnt_flag_itm_.Tid__max];
|
||||
private int count = 0;
|
||||
public int Count() {return count;}
|
||||
public boolean Get(int tid) {return flag_ary[tid];}
|
||||
public void Set_y(int tid) {flag_ary[tid] = Bool_.Y;}
|
||||
public void Set_y_many(int... ary) {
|
||||
int len = ary.length;
|
||||
for (int i = 0; i < len; ++i)
|
||||
flag_ary[ary[i]] = Bool_.Y;
|
||||
}
|
||||
public void Set_n(int tid) {flag_ary[tid] = Bool_.N;}
|
||||
public void Limit(int tid) {
|
||||
for (int i = 0; i < Vnt_flag_itm_.Tid__max; ++i) {
|
||||
if (i != tid) flag_ary[i] = false;
|
||||
private final Hash_adp_bry codes_regy = Vnt_flag_code_.Regy;
|
||||
private Vnt_flag_code_mgr codes; private Vnt_flag_lang_mgr langs;
|
||||
private Xol_vnt_regy vnt_regy;
|
||||
public void Parse(Vnt_flag_code_mgr codes, Vnt_flag_lang_mgr langs, Xol_vnt_regy vnt_regy, byte[] src, int src_bgn, int src_end) {
|
||||
this.codes = codes; this.langs = langs; this.vnt_regy = vnt_regy;
|
||||
codes.Clear(); langs.Clear();
|
||||
if (src_end != Bry_find_.Not_found) // "|" found; EX: -{A|}-
|
||||
Bry_split_.Split(src, src_bgn, src_end, Byte_ascii.Semic, true, this);
|
||||
int codes_count = codes.Count(), langs_count = langs.Count();
|
||||
if (codes_count == 0) codes.Set_y(Vnt_flag_code_.Tid_show);
|
||||
else if (codes.Limit_if_exists(Vnt_flag_code_.Tid_raw)) {}
|
||||
else if (codes.Limit_if_exists(Vnt_flag_code_.Tid_name)) {}
|
||||
else if (codes.Limit_if_exists(Vnt_flag_code_.Tid_del)) {}
|
||||
else if (codes_count == 1 && codes.Get(Vnt_flag_code_.Tid_title)) codes.Set_y(Vnt_flag_code_.Tid_hide);
|
||||
else if (codes.Get(Vnt_flag_code_.Tid_hide)) {
|
||||
boolean exists_d = codes.Get(Vnt_flag_code_.Tid_descrip);
|
||||
boolean exists_t = codes.Get(Vnt_flag_code_.Tid_title);
|
||||
codes.Clear();
|
||||
codes.Set_y_many(Vnt_flag_code_.Tid_add, Vnt_flag_code_.Tid_hide);
|
||||
if (exists_d) codes.Set_y(Vnt_flag_code_.Tid_descrip);
|
||||
if (exists_t) codes.Set_y(Vnt_flag_code_.Tid_title);
|
||||
}
|
||||
}
|
||||
public boolean Limit_if_exists(int tid) {
|
||||
boolean exists = flag_ary[tid]; if (!exists) return false;
|
||||
for (int i = 0; i < Vnt_flag_itm_.Tid__max; ++i) {
|
||||
if (i != tid) flag_ary[i] = false;
|
||||
else {
|
||||
if (codes.Get(Vnt_flag_code_.Tid_aout))
|
||||
codes.Set_y_many(Vnt_flag_code_.Tid_add, Vnt_flag_code_.Tid_show);
|
||||
if (codes.Get(Vnt_flag_code_.Tid_descrip))
|
||||
codes.Set_n(Vnt_flag_code_.Tid_show);
|
||||
if (langs_count > 0)
|
||||
codes.Clear();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
public boolean Limit_if_exists_vnts() {
|
||||
return false;
|
||||
}
|
||||
public void Clear() {
|
||||
count = 0;
|
||||
for (int i = 0; i < Vnt_flag_itm_.Tid__max; ++i)
|
||||
flag_ary[i] = false;
|
||||
}
|
||||
public void Parse(byte[] src, int src_bgn, int src_end) {
|
||||
this.Clear();
|
||||
Bry_split_.Split(src, Byte_ascii.Semic, true, this);
|
||||
}
|
||||
public int Split(byte[] src, int itm_bgn, int itm_end) {
|
||||
int flag_tid = flag_regy.Get_as_int_or(src, itm_bgn, itm_end, -1);
|
||||
if (flag_tid == -1) {
|
||||
int vnt_tid = vnt_regy.Get_as_int_or(src, itm_bgn, itm_end, -1);
|
||||
if (vnt_tid == -1) return Bry_split_.Rv__ok; // unknown flag; ignore
|
||||
int flag_tid = codes_regy.Get_as_int_or(src, itm_bgn, itm_end, -1);
|
||||
if (flag_tid == -1) { // try to find flags like "zh-hans", "zh-hant"; allow syntaxes like "-{zh-hans;zh-hant|XXXX}-"
|
||||
Xol_vnt_itm vnt_itm = vnt_regy.Get_by(src, itm_bgn, itm_end);
|
||||
if (vnt_itm == null) return Bry_split_.Rv__ok; // unknown flag; ignore
|
||||
langs.Add(vnt_itm);
|
||||
return Bry_split_.Rv__ok;
|
||||
}
|
||||
flag_ary[flag_tid] = true;
|
||||
++count;
|
||||
codes.Add(flag_tid);
|
||||
return Bry_split_.Rv__ok;
|
||||
}
|
||||
}
|
||||
|
||||
55
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_parser_tst.java
Normal file
55
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_parser_tst.java
Normal file
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import org.junit.*; import gplx.xowa.langs.vnts.*;
|
||||
public class Vnt_flag_parser_tst {
|
||||
private final Vnt_flag_parser_fxt fxt = new Vnt_flag_parser_fxt();
|
||||
@Test public void Basic() {fxt.Test_parse("D" , "D");}
|
||||
@Test public void Multiple() {fxt.Test_parse("+;S;E" , "+;S;E");}
|
||||
@Test public void Ws() {fxt.Test_parse(" + ; S ; E " , "+;S;E");}
|
||||
@Test public void None() {fxt.Test_parse("" , "S");}
|
||||
@Test public void Wrong() {fxt.Test_parse("XYZ" , "S");}
|
||||
@Test public void Raw__limit() {fxt.Test_parse("R;S" , "R");}
|
||||
@Test public void Name__limit() {fxt.Test_parse("N;S" , "N");}
|
||||
@Test public void Del_limit() {fxt.Test_parse("-;S" , "-");}
|
||||
@Test public void Title__also_macro_y() {fxt.Test_parse("T" , "H;T");}
|
||||
@Test public void Title__also_macro_n() {fxt.Test_parse("T;S" , "S;T");}
|
||||
@Test public void Hide__remove_all() {fxt.Test_parse("H;S" , "+;H");}
|
||||
@Test public void Hide__keep_descrip() {fxt.Test_parse("H;S;D" , "+;H;D");}
|
||||
@Test public void Hide__keep_title() {fxt.Test_parse("H;S;T" , "+;H;T");}
|
||||
@Test public void Aout__also_show_all() {fxt.Test_parse("A" , "+;A;S");}
|
||||
@Test public void Descrip__remove_show() {fxt.Test_parse("D;S" , "D");}
|
||||
@Test public void Aout_w_descrip() {fxt.Test_parse("A;D;S" , "+;A;D");}
|
||||
@Test public void Lang__one() {fxt.Test_parse("zh-hans" , "S;zh-hans");}
|
||||
@Test public void Lang__many() {fxt.Test_parse("zh-cn;zh-hk" , "S;zh-cn;zh-hk");}
|
||||
@Test public void Lang__many__ws() {fxt.Test_parse(" zh-cn ; zh-hk " , "S;zh-cn;zh-hk");}
|
||||
@Test public void Lang__zap__codes() {fxt.Test_parse("+;S;zh-hans;" , "zh-hans");}
|
||||
}
|
||||
class Vnt_flag_parser_fxt {
|
||||
private final Vnt_flag_parser parser = new Vnt_flag_parser();
|
||||
private final Vnt_flag_code_mgr codes = new Vnt_flag_code_mgr(); private final Vnt_flag_lang_mgr langs = new Vnt_flag_lang_mgr();
|
||||
private final Xol_vnt_regy vnt_regy = Xol_vnt_regy_fxt.new_chinese();
|
||||
private final Bry_bfr bfr = Bry_bfr.new_();
|
||||
public void Test_parse(String raw, String expd) {
|
||||
byte[] src = Bry_.new_u8(raw);
|
||||
parser.Parse(codes, langs, vnt_regy, src, 0, src.length);
|
||||
codes.To_bfr__dbg(bfr);
|
||||
langs.To_bfr__dbg(bfr);
|
||||
Tfds.Eq_str(expd, bfr.Xto_str_and_clear());
|
||||
}
|
||||
}
|
||||
74
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_html_doc_wkr.java
Normal file
74
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_html_doc_wkr.java
Normal file
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import gplx.xowa.parsers.htmls.*; import gplx.xowa.parsers.xndes.*;
|
||||
import gplx.xowa.langs.vnts.converts.*;
|
||||
import gplx.xowa.html.*;
|
||||
class Vnt_html_doc_wkr implements Mwh_doc_wkr {
|
||||
private final Hash_adp_bry atr_hash = Hash_adp_bry.ci_a7();
|
||||
private Bry_bfr bfr;
|
||||
private final Xol_convert_mgr convert_mgr; private int convert_vnt_idx;
|
||||
public Vnt_html_doc_wkr(Xol_convert_mgr convert_mgr) {
|
||||
this.convert_mgr = convert_mgr;
|
||||
atr_hash.Add_many_str("title", "alt");
|
||||
}
|
||||
public Hash_adp_bry Nde_regy() {return nde_regy;} private final Hash_adp_bry nde_regy = Mwh_doc_wkr_.Nde_regy__mw();
|
||||
public void Init(Bry_bfr bfr, int convert_vnt_idx) {this.bfr = bfr; this.convert_vnt_idx = convert_vnt_idx;}
|
||||
public void On_atr_each (Mwh_atr_parser mgr, byte[] src, int nde_tid, boolean valid, boolean repeated, boolean key_exists, byte[] key_bry, byte[] val_bry_manual, int[] itm_ary, int itm_idx) {
|
||||
int val_bgn = itm_ary[itm_idx + Mwh_atr_mgr.Idx_val_bgn];
|
||||
int val_end = itm_ary[itm_idx + Mwh_atr_mgr.Idx_val_end];
|
||||
if ( atr_hash.Get_by_mid(key_bry, 0, key_bry.length) == null // title, alt
|
||||
|| !key_exists
|
||||
|| Bry_find_.Find_fwd(src, Bry__url_frag, val_bgn, val_end) != Bry_find_.Not_found
|
||||
) { // handle name-only attribs like "<span title>"
|
||||
int atr_bgn = itm_ary[itm_idx + Mwh_atr_mgr.Idx_atr_bgn];
|
||||
int atr_end = itm_ary[itm_idx + Mwh_atr_mgr.Idx_atr_end];
|
||||
bfr.Add_mid(src, atr_bgn, atr_end);
|
||||
}
|
||||
else {
|
||||
bfr.Add_byte_space();
|
||||
bfr.Add(key_bry);
|
||||
bfr.Add_byte(Byte_ascii.Eq);
|
||||
byte quote_byte = Mwh_atr_itm.Calc_qte_byte(itm_ary, itm_idx);
|
||||
bfr.Add_byte(quote_byte);
|
||||
bfr.Add(convert_mgr.Convert_text(convert_vnt_idx, src, val_bgn, val_end));
|
||||
bfr.Add_byte(quote_byte);
|
||||
}
|
||||
}
|
||||
public void On_txt_end (Mwh_doc_parser mgr, byte[] src, int nde_tid, int itm_bgn, int itm_end) {
|
||||
switch (nde_tid) {
|
||||
case Xop_xnde_tag_.Tid_code:
|
||||
case Xop_xnde_tag_.Tid_script:
|
||||
case Xop_xnde_tag_.Tid_pre:
|
||||
bfr.Add_mid(src, itm_bgn, itm_end);
|
||||
break;
|
||||
default:
|
||||
bfr.Add(convert_mgr.Convert_text(convert_vnt_idx, src, itm_bgn, itm_end));
|
||||
break;
|
||||
}
|
||||
}
|
||||
public void On_nde_head_bgn(Mwh_doc_parser mgr, byte[] src, int nde_tid, int key_bgn, int key_end) {
|
||||
bfr.Add_byte(Byte_ascii.Angle_bgn).Add_mid(src, key_bgn, key_end); // EX: "<span"
|
||||
}
|
||||
public void On_nde_head_end(Mwh_doc_parser mgr, byte[] src, int nde_tid, int itm_bgn, int itm_end, boolean inline) {
|
||||
bfr.Add(inline ? Xoh_consts.__inline : Xoh_consts.__end); // add "/>" or ">"
|
||||
}
|
||||
public void On_nde_tail_end (Mwh_doc_parser mgr, byte[] src, int nde_tid, int itm_bgn, int itm_end) {bfr.Add_mid(src, itm_bgn, itm_end);}
|
||||
public void On_comment_end (Mwh_doc_parser mgr, byte[] src, int nde_tid, int itm_bgn, int itm_end) {bfr.Add_mid(src, itm_bgn, itm_end);}
|
||||
private static final byte[] Bry__url_frag = Bry_.new_a7("://"); // REF.MW: if ( !strpos( $attr, '://' ) ) {
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import gplx.core.btries.*; import gplx.core.primitives.*;
|
||||
public class Vnt_language_converter {
|
||||
private final Bry_bfr bfr = Bry_bfr.new_();
|
||||
private int max_depth = 32;
|
||||
private byte[] src; private int src_len;
|
||||
private boolean convert_needed;
|
||||
private int pos;
|
||||
public byte[] Parse(byte[] vnt, byte[] src) {// REF.MW:/languages/LanguageConverter.php!recursiveConvertTopLevel
|
||||
synchronized (bfr) {
|
||||
int markup_count = 0;
|
||||
this.pos = 0;
|
||||
this.convert_needed = false; // for sr lang; SEE:LanguageSr.php !$this->guessVariant(src, vnt);
|
||||
this.src = src; this.src_len = src.length;
|
||||
while (pos < src_len) {
|
||||
int curly_bgn = Bry_find_.Find_fwd(src, Bry__curly_bgn, pos, src_len);
|
||||
if (curly_bgn == Bry_find_.Not_found) { // No more markup, append final segment
|
||||
if (markup_count == 0) return src; // no markups found; just return original
|
||||
Add_output(vnt, convert_needed, src, pos, src_len);
|
||||
return bfr.Xto_bry_and_clear();
|
||||
}
|
||||
bfr.Add_mid(src, pos, curly_bgn); // Markup found; append segment
|
||||
Add_output(vnt, convert_needed, src, pos, src_len);
|
||||
pos = curly_bgn; // Advance position
|
||||
++markup_count;
|
||||
Parse_recursive(vnt, pos, 1); // Do recursive conversion
|
||||
}
|
||||
return bfr.Xto_bry_and_clear();
|
||||
}
|
||||
}
|
||||
private void Parse_recursive(byte[] vnt, int pos, int depth) {
|
||||
pos += 2; // skip "-{"
|
||||
boolean warning_done = false;
|
||||
// $inner = '';
|
||||
while (pos < src_len) {
|
||||
byte b = src[pos];
|
||||
Object o = trie.Match_bgn_w_byte(b, src,pos, src_len);
|
||||
if (o == null) { // char;
|
||||
++pos;
|
||||
continue;
|
||||
}
|
||||
int new_pos = trie.Match_pos(); // Markup found; Append initial segment
|
||||
bfr.Add_mid(src, pos, new_pos);
|
||||
pos = new_pos; // Advance position
|
||||
switch (((Byte_obj_val)o).Val()) {
|
||||
case Tid__curly_bgn:
|
||||
if (depth >= max_depth) {
|
||||
bfr.Add(Bry__curly_bgn);
|
||||
if (!warning_done) {
|
||||
bfr.Add_str("<span class=\"error\">");
|
||||
// wfMessage('language-converter-depth-warning')->numParams($this->mMaxDepth)->inContentLanguage()->text()
|
||||
bfr.Add_str("</span>");
|
||||
warning_done = true;
|
||||
}
|
||||
pos += 2; // skip "-{"
|
||||
continue;
|
||||
}
|
||||
Parse_recursive(vnt, pos, depth + 1); // Recursively parse another rule
|
||||
break;
|
||||
case Tid__curly_end:
|
||||
pos += 2;
|
||||
/*
|
||||
// Apply the rule
|
||||
$rule = new ConverterRule($inner, $this);
|
||||
$rule->parse($variant);
|
||||
$this->applyManualConv($rule);
|
||||
return $rule->getDisplay();
|
||||
*/
|
||||
return;
|
||||
default: throw Err_.new_unhandled(-1); // never happens
|
||||
}
|
||||
}
|
||||
if (pos < src_len) { // Unclosed rule
|
||||
byte[] frag = Auto_convert(vnt, src, pos, src_len);
|
||||
bfr.Add(Bry__curly_bgn).Add(frag);
|
||||
}
|
||||
pos = src_len;
|
||||
}
|
||||
private void Add_output(byte[] vnt, boolean convert_needed, byte[] src, int pos, int src_len) {
|
||||
if (convert_needed) {
|
||||
byte[] frag = Auto_convert(vnt, src, pos, src_len);
|
||||
bfr.Add(frag);
|
||||
}
|
||||
else
|
||||
bfr.Add_mid(src, pos, src_len);
|
||||
}
|
||||
private byte[] Auto_convert(byte[] vnt, byte[] src, int bgn, int end) {return src;}
|
||||
private static final byte Tid__curly_bgn = 1, Tid__curly_end = 2;
|
||||
private static final byte[] Bry__curly_bgn = Bry_.new_a7("-{"), Bry__curly_end = Bry_.new_a7("}-");
|
||||
private static final Btrie_fast_mgr trie = Btrie_fast_mgr.cs()
|
||||
.Add_bry_byte(Bry__curly_bgn, Tid__curly_bgn)
|
||||
.Add_bry_byte(Bry__curly_end, Tid__curly_end);
|
||||
}
|
||||
68
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_bidi_mgr.java
Normal file
68
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_bidi_mgr.java
Normal file
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
class Vnt_rule_bidi_mgr {
|
||||
private final Ordered_hash hash = Ordered_hash_.new_bry_();
|
||||
public int Len() {return hash.Count();}
|
||||
public boolean Has_none() {return hash.Count() == 0;}
|
||||
public void Clear() {hash.Clear();}
|
||||
public Vnt_rule_bidi_itm Get_at(int i) {return (Vnt_rule_bidi_itm)hash.Get_at(i);}
|
||||
public Vnt_rule_bidi_itm Get_by(byte[] k) {return (Vnt_rule_bidi_itm)hash.Get_by(k);}
|
||||
public byte[] Get_text_by_ary_or_null(byte[]... ary) {
|
||||
int len = ary.length;
|
||||
byte[] rv = null;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
byte[] itm = ary[i];
|
||||
Vnt_rule_bidi_itm bidi_itm = (Vnt_rule_bidi_itm)hash.Get_by(itm); if (bidi_itm == null) continue;
|
||||
rv = Get_text_by_key_or_null(bidi_itm.Vnt());
|
||||
if (rv != null) return rv;
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public byte[] Get_text_by_key_or_null(byte[] vnt) {
|
||||
Vnt_rule_bidi_itm rv = (Vnt_rule_bidi_itm)hash.Get_by(vnt);
|
||||
return rv == null ? null : rv.Text();
|
||||
}
|
||||
public byte[] Get_text_at(int i) {
|
||||
Vnt_rule_bidi_itm itm = (Vnt_rule_bidi_itm)hash.Get_at(i);
|
||||
return itm == null ? null : itm.Text();
|
||||
}
|
||||
public void Set(byte[] vnt, byte[] text) {
|
||||
Vnt_rule_bidi_itm itm = (Vnt_rule_bidi_itm)hash.Get_by(vnt);
|
||||
if (itm == null) {
|
||||
itm = new Vnt_rule_bidi_itm(vnt, text);
|
||||
hash.Add(vnt, itm);
|
||||
}
|
||||
else
|
||||
itm.Text_(text);
|
||||
}
|
||||
public void To_bry__dbg(Bry_bfr bfr) {
|
||||
int len = hash.Count();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
if (i != 0) bfr.Add_byte_nl();
|
||||
Vnt_rule_bidi_itm itm = (Vnt_rule_bidi_itm)hash.Get_at(i);
|
||||
bfr.Add(itm.Vnt()).Add_byte_colon().Add(itm.Text());
|
||||
}
|
||||
}
|
||||
}
|
||||
class Vnt_rule_bidi_itm {
|
||||
public Vnt_rule_bidi_itm(byte[] vnt, byte[] text) {this.vnt = vnt; this.text = text;}
|
||||
public byte[] Vnt() {return vnt;} private final byte[] vnt;
|
||||
public byte[] Text() {return text;} private byte[] text;
|
||||
public void Text_(byte[] v) {this.text = v;}
|
||||
}
|
||||
@@ -17,45 +17,76 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import gplx.core.btries.*;
|
||||
import gplx.xowa.langs.vnts.*;
|
||||
class Vnt_rule_parser implements gplx.core.brys.Bry_split_wkr {
|
||||
// private final Btrie_slim_mgr vnt_trie = Btrie_slim_mgr.ci_a7();
|
||||
public void Parse(byte[] src, int src_bgn, int src_end) {
|
||||
Bry_split_.Split(src, Byte_ascii.Semic, false, this); // trim=false for "&#entity;" check below
|
||||
private final Btrie_slim_mgr vnt_trie = Btrie_slim_mgr.ci_a7();
|
||||
private Vnt_rule_undi_mgr undis; private Vnt_rule_bidi_mgr bidis;
|
||||
private int src_end, src_len; private byte[] rule_raw;
|
||||
public byte[] Raw() {return rule_raw;}
|
||||
public void Init(Xol_vnt_regy vnt_regy) {
|
||||
this.vnt_trie.Clear();
|
||||
int len = vnt_regy.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Xol_vnt_itm itm = (Xol_vnt_itm)vnt_regy.Get_at(i);
|
||||
vnt_trie.Add_obj(itm.Key(), itm);
|
||||
}
|
||||
}
|
||||
public int Split(byte[] src, int itm_bgn, int itm_end) {
|
||||
int html_entity_pos = Bry_find_.Find_bwd_while_alphanum(src, itm_bgn);
|
||||
if (Bry_.Eq(src, html_entity_pos - 2, html_entity_pos, Bry__html_entity)) return Bry_split_.Rv__extend; // reject "&#entity;"; EX: " zh-hans;"
|
||||
/*
|
||||
itm_bgn = skip fwd for ws;
|
||||
itm_bgn = skip fwd for "=>"
|
||||
Object vnt_obj = vnt_trie.Match_bgn(src, itm_bgn, itm_end); if (vnt_obj == null) return Bry_split_.Rv__extend; // reject ";not_variant"; EX: ";border" in "zh-hans:<span style='color:blue;border:1px;'>;zh-hant:"
|
||||
itm_end = skip bwd for ws
|
||||
// val = trim( val[0] );
|
||||
// trg = trim( val[1] );
|
||||
// $u = explode( '=>', val, 2 );
|
||||
// // if trg is empty, strtr() could return a wrong result
|
||||
// if ( count( $u ) == 1 && trg && in_array( val, $variants ) ) {
|
||||
// bidi_ary[val] = trg;
|
||||
// } elseif ( count( $u ) == 2 ) {
|
||||
// $from = trim( $u[0] );
|
||||
// val = trim( $u[1] );
|
||||
// if ( array_key_exists( val, $unidtable )
|
||||
// && !is_array( $unidtable[val] )
|
||||
// && trg
|
||||
// && in_array( val, $variants ) ) {
|
||||
// $unidtable[val] = array( $from => trg );
|
||||
// } elseif ( trg && in_array( val, $variants ) ) {
|
||||
// $unidtable[val][$from] = trg;
|
||||
// }
|
||||
// }
|
||||
// // syntax error, pass
|
||||
// if ( !isset( $this->mConverter->mVariantNames[val] ) ) {
|
||||
// bidi_ary = array();
|
||||
// $unidtable = array();
|
||||
// break;
|
||||
// }
|
||||
*/
|
||||
public void Clear(Vnt_rule_undi_mgr undis, Vnt_rule_bidi_mgr bidis, byte[] rule_raw) {
|
||||
this.undis = undis; this.bidis = bidis;
|
||||
undis.Clear(); bidis.Clear();
|
||||
this.rule_raw = rule_raw;
|
||||
}
|
||||
public void Parse(byte[] src, int src_bgn, int src_end) {
|
||||
this.src_end = src_end; this.src_len = src.length;
|
||||
Bry_split_.Split(src, src_bgn, src_end, Byte_ascii.Semic, false, this); // trim=false for "&#entity;" check below
|
||||
}
|
||||
public int Split(byte[] src, int itm_bgn, int itm_end) { // macro=>zh-hans:text;
|
||||
int html_entity_pos = Bry_find_.Find_bwd_while_alphanum(src, itm_end);
|
||||
byte html_entity_byte = src[html_entity_pos];
|
||||
if (html_entity_byte == Byte_ascii.Hash) html_entity_byte = src[html_entity_pos - 2]; // skip #; EX: {
|
||||
if (html_entity_byte == Byte_ascii.Amp) return Bry_split_.Rv__extend; // reject "&#entity;"; EX: " zh-hans;"
|
||||
if (itm_end != src_end) {
|
||||
int nxt_lang_bgn = Bry_find_.Find_fwd(src, Bry__bidi_dlm, itm_end + 1, src_len); // look for next "=>"
|
||||
if (nxt_lang_bgn == Bry_find_.Not_found)
|
||||
nxt_lang_bgn = Bry_find_.Find_fwd_while_ws(src, itm_end + 1, src_len); // skip any ws after end ";"; EX: "a:1; b:2"; NOTE: +1 to skip semic;
|
||||
else
|
||||
nxt_lang_bgn += 2;
|
||||
int nxt_lang_end = Bry_find_.Find_fwd(src, Byte_ascii.Colon, nxt_lang_bgn, src_len); // get colon;
|
||||
if (nxt_lang_end != Bry_find_.Not_found) {
|
||||
nxt_lang_end = Bry_find_.Find_bwd__skip_ws(src, nxt_lang_end, src_len); // trim
|
||||
if (vnt_trie.Match_bgn(src, nxt_lang_bgn, nxt_lang_end) == null) return Bry_split_.Rv__extend; // reject ";not_variant"; EX: ";border" in "zh-hans:<span style='color:blue;border:1px;'>;zh-hant:"
|
||||
}
|
||||
}
|
||||
int undi_bgn = Bry_find_.Find_fwd_while_ws(src, itm_bgn, itm_end); // skip any ws after bgn ";"; EX: " a=>b:c;"
|
||||
int undi_end = Bry_find_.Find_fwd(src, Bry__bidi_dlm, undi_bgn, itm_end); // look for "=>"
|
||||
int lang_bgn = undi_bgn; // default lang_bgn to undi_bgn; assumes no bidi found
|
||||
if (undi_end != Bry_find_.Not_found) { // "=>" found; bidi exists
|
||||
lang_bgn = Bry_find_.Find_fwd_while_ws(src, undi_end + 2, itm_end); // set lang_bgn after => and gobble up ws
|
||||
undi_end = Bry_find_.Find_bwd__skip_ws(src, undi_end, undi_bgn); // trim ws from end of bd;
|
||||
}
|
||||
Object vnt_obj = vnt_trie.Match_bgn(src, lang_bgn, itm_end);
|
||||
if (vnt_obj == null) {
|
||||
return (itm_bgn == 0) ? Bry_split_.Rv__cancel : Bry_split_.Rv__extend; // if 1st item; cancel rest; otherwise, extend
|
||||
}
|
||||
int lang_end = vnt_trie.Match_pos();
|
||||
int text_bgn = Bry_find_.Find_fwd_while_ws(src, lang_end, itm_end); if (src[text_bgn] != Byte_ascii.Colon) return Bry_split_.Rv__extend;
|
||||
++text_bgn;
|
||||
Xol_vnt_itm vnt_itm = (Xol_vnt_itm)vnt_obj;
|
||||
byte[] vnt_key = vnt_itm.Key();
|
||||
byte[] text_bry = Bry_.Mid_w_trim(src, text_bgn, itm_end);
|
||||
if (undi_end == Bry_find_.Not_found)
|
||||
bidis.Set(vnt_key, text_bry);
|
||||
else {
|
||||
byte[] undi_bry = Bry_.Mid(src, undi_bgn, undi_end);
|
||||
if (itm_end - text_bgn > 0)
|
||||
undis.Set(vnt_key, undi_bry, text_bry);
|
||||
}
|
||||
return Bry_split_.Rv__ok;
|
||||
}
|
||||
private static final byte[] Bry__html_entity = Bry_.new_a7("&#");
|
||||
public void To_bry__dbg(Bry_bfr bfr) {
|
||||
undis.To_bry__dbg(bfr);
|
||||
if (bfr.Len_gt_0()) bfr.Add_byte_nl();
|
||||
bidis.To_bry__dbg(bfr);
|
||||
}
|
||||
private static final byte[] Bry__bidi_dlm = Bry_.new_a7("=>");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import org.junit.*;
|
||||
public class Vnt_rule_parser__bidi_tst {
|
||||
private final Vnt_rule_parser_fxt fxt = new Vnt_rule_parser_fxt();
|
||||
@Test public void Basic() {fxt.Test_parse("x1:v1;" , "x1:v1");}
|
||||
@Test public void Ws() {fxt.Test_parse(" x1 : v1 ;" , "x1:v1");}
|
||||
@Test public void Entity() {fxt.Test_parse("x1:a x2:b;x2:b;" , "x1:a x2:b" , "x2:b");}
|
||||
@Test public void Unknown__nth() {fxt.Test_parse("x1:a;wx2:b;x2:b;" , "x1:a;wx2:b" , "x2:b");}
|
||||
@Test public void Unknown__1st() {fxt.Test_parse("wx1:a;x1:b;" , "");}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import org.junit.*;
|
||||
public class Vnt_rule_parser__undi_tst {
|
||||
private final Vnt_rule_parser_fxt fxt = new Vnt_rule_parser_fxt();
|
||||
@Test public void One() {fxt.Test_parse("k1=>x1:v1;" , "x1:k1=v1");}
|
||||
@Test public void Many() {fxt.Test_parse("k1=>x1:v1;k2=>x2:v2;" , "x1:k1=v1", "x2:k2=v2");}
|
||||
}
|
||||
37
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_parser_fxt.java
Normal file
37
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_parser_fxt.java
Normal file
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
import gplx.xowa.langs.vnts.*;
|
||||
class Vnt_rule_parser_fxt {
|
||||
private final Vnt_rule_parser parser = new Vnt_rule_parser(); private final Vnt_rule_undi_mgr undis = new Vnt_rule_undi_mgr(); private final Vnt_rule_bidi_mgr bidis = new Vnt_rule_bidi_mgr();
|
||||
private final Bry_bfr bfr = Bry_bfr.new_(255);
|
||||
public Vnt_rule_parser_fxt() {
|
||||
Xol_vnt_regy vnt_regy = new Xol_vnt_regy();
|
||||
vnt_regy.Add(Bry_.new_a7("x1"), Bry_.new_a7("lang1"));
|
||||
vnt_regy.Add(Bry_.new_a7("x2"), Bry_.new_a7("lang2"));
|
||||
vnt_regy.Add(Bry_.new_a7("x3"), Bry_.new_a7("lang3"));
|
||||
parser.Init(vnt_regy);
|
||||
}
|
||||
public void Test_parse(String raw, String... expd_ary) {
|
||||
byte[] src = Bry_.new_u8(raw);
|
||||
parser.Clear(undis, bidis, src);
|
||||
parser.Parse(src, 0, src.length);
|
||||
parser.To_bry__dbg(bfr);
|
||||
Tfds.Eq_str_lines(String_.Concat_lines_nl_skip_last(expd_ary), bfr.Xto_str_and_clear());
|
||||
}
|
||||
}
|
||||
80
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_undi_mgr.java
Normal file
80
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_undi_mgr.java
Normal file
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
|
||||
class Vnt_rule_undi_mgr {
|
||||
private final Ordered_hash hash = Ordered_hash_.new_bry_();
|
||||
public int Len() {return hash.Count();}
|
||||
public boolean Has_none() {return hash.Count() == 0;}
|
||||
public void Clear() {hash.Clear();}
|
||||
public Vnt_rule_undi_grp Get_at(int i) {return (Vnt_rule_undi_grp)hash.Get_at(i);}
|
||||
public Vnt_rule_undi_grp Get_by(byte[] key) {return (Vnt_rule_undi_grp)hash.Get_by(key);}
|
||||
public byte[] Get_text_by_key_or_null(byte[] key) {
|
||||
Vnt_rule_undi_grp grp = (Vnt_rule_undi_grp)hash.Get_by(key); if (grp == null) return null;
|
||||
return grp.Len() == 0 ? null : grp.Get_at(0).Trg(); // REF.MW: $disp = $disp[0];
|
||||
}
|
||||
public byte[] Get_text_at(int i) {
|
||||
Vnt_rule_undi_grp grp = (Vnt_rule_undi_grp)hash.Get_at(i); if (grp == null) return null;
|
||||
return grp.Len() == 0 ? null : grp.Get_at(0).Trg();
|
||||
}
|
||||
public Vnt_rule_undi_grp Set(byte[] vnt, byte[] src, byte[] trg) {
|
||||
Vnt_rule_undi_grp grp = (Vnt_rule_undi_grp)hash.Get_by(vnt);
|
||||
if (grp == null) {
|
||||
grp = new Vnt_rule_undi_grp(vnt);
|
||||
hash.Add(vnt, grp);
|
||||
}
|
||||
grp.Set(src, trg);
|
||||
return grp;
|
||||
}
|
||||
public void To_bry__dbg(Bry_bfr bfr) {
|
||||
int len = hash.Count();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
if (i != 0) bfr.Add_byte_nl();
|
||||
Vnt_rule_undi_grp grp = (Vnt_rule_undi_grp)hash.Get_at(i);
|
||||
bfr.Add(grp.Vnt()).Add_byte_colon();
|
||||
grp.To_bry__dbg(bfr);
|
||||
}
|
||||
}
|
||||
}
|
||||
class Vnt_rule_undi_grp {
|
||||
private final Ordered_hash hash = Ordered_hash_.new_bry_();
|
||||
public Vnt_rule_undi_grp(byte[] vnt) {this.vnt = vnt;}
|
||||
public int Len() {return hash.Count();}
|
||||
public Vnt_rule_undi_itm Get_at(int i) {return (Vnt_rule_undi_itm)hash.Get_at(i);}
|
||||
public byte[] Vnt() {return vnt;} private final byte[] vnt;
|
||||
public Vnt_rule_undi_itm Set(byte[] src, byte[] trg) {
|
||||
Vnt_rule_undi_itm itm = (Vnt_rule_undi_itm)hash.Get_by(src);
|
||||
if (itm == null) {
|
||||
itm = new Vnt_rule_undi_itm(src, trg);
|
||||
hash.Add(src, itm);
|
||||
}
|
||||
return itm;
|
||||
}
|
||||
public void To_bry__dbg(Bry_bfr bfr) {
|
||||
int len = hash.Count();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Vnt_rule_undi_itm itm = (Vnt_rule_undi_itm)hash.Get_at(i);
|
||||
bfr.Add(itm.Src()).Add_byte_eq().Add(itm.Trg());
|
||||
}
|
||||
}
|
||||
}
|
||||
class Vnt_rule_undi_itm {
|
||||
public Vnt_rule_undi_itm(byte[] src, byte[] trg) {this.src = src; this.trg = trg;}
|
||||
public byte[] Src() {return src;} private final byte[] src;
|
||||
public byte[] Trg() {return trg;} private byte[] trg;
|
||||
public void Trg_(byte[] v) {this.trg = v;}
|
||||
}
|
||||
@@ -25,7 +25,7 @@ class Xop_vnt_lang_bldr { // performant way of building langs; EX: -{zh;zh-hans;
|
||||
public void Add(byte[] key) {
|
||||
Xol_vnt_itm vnt = vnt_regy.Get_by(key); if (vnt == null) return; // ignore invalid vnts; EX: -{zh;zhx}-
|
||||
int vnt_mask = vnt.Mask__vnt();
|
||||
this.rslt_mask = (rslt_mask == 0) ? vnt_mask : Enm_.Flip_int(true, rslt_mask, vnt_mask);
|
||||
this.rslt_mask = (rslt_mask == 0) ? vnt_mask : Bitmask_.Flip_int(true, rslt_mask, vnt_mask);
|
||||
}
|
||||
public Xop_vnt_flag Bld() {
|
||||
return (rslt_mask == 0) ? Xop_vnt_flag_.Flag_unknown : Xop_vnt_flag.new_lang(rslt_mask);
|
||||
|
||||
@@ -53,7 +53,7 @@ class Xop_vnt_tkn_mok {
|
||||
}
|
||||
public Xop_vnt_tkn_mok Flags_none_() {flags_list.Clear(); return this;}
|
||||
public Xop_vnt_tkn_mok Flags_unknown_(String... v) {flags_list.Add(Xop_vnt_flag_.Flag_unknown); return this;}
|
||||
public Xop_vnt_tkn_mok Flags_langs_(int... ary) {flags_list.Add(Xop_vnt_flag.new_lang(Enm_.Add_int_ary(ary))); return this;}
|
||||
public Xop_vnt_tkn_mok Flags_langs_(int... ary) {flags_list.Add(Xop_vnt_flag.new_lang(Bitmask_.Add_int_ary(ary))); return this;}
|
||||
public Xop_vnt_tkn_mok Flags_codes_(String... ary) {
|
||||
int len = ary.length;
|
||||
for (int i = 0; i < len; i++) {
|
||||
@@ -122,7 +122,7 @@ class Xop_vnt_lxr_fxt {
|
||||
int itm_mask = itm.Mask();
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
int mask = gplx.core.brys.Bit_.Get_flag(i);
|
||||
if (Enm_.Has_int(mask, itm_mask)) {
|
||||
if (Bitmask_.Has_int(mask, itm_mask)) {
|
||||
Xol_vnt_itm vnt = vnt_regy.Get_at(i);
|
||||
bfr.Add(vnt.Key()).Add_byte(Byte_ascii.Semic);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user