Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.9.3.1

Browse Source
This commit is contained in:
gnosygnu
2015-09-20 23:43:51 -04:00
parent 5fe27b5b3b
commit fa70c05354
1056 changed files with 8375 additions and 7095 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_converter_rule.java Normal file
View File

@@ -0,0 +1,58 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*; import gplx.core.primitives.*;
class Vnt_converter_rule { // REF.MW: /languages/LanguageConverter.php|ConverterRule
private final byte[] src;
private final int src_bgn, src_end;
private int pipe_pos = -1;
public Vnt_converter_rule(byte[] src, int src_bgn, int src_end) {
this.src = src; this.src_bgn = src_bgn; this.src_end = src_end;
}
public void Parse() {
}
public void Parse_flags(Vnt_flag_parser parser) {
this.pipe_pos = Bry_find_.Find_fwd(src, Byte_ascii.Pipe, src_bgn, src_end);
if (pipe_pos != Bry_find_.Not_found) // "|" found; EX: -{A|}-
parser.Parse(src, src_bgn, pipe_pos);
int flag_count = parser.Count();
if (flag_count == 0) parser.Set_y(Vnt_flag_itm_.Tid_show);
else if (parser.Limit_if_exists(Vnt_flag_itm_.Tid_raw)) {}
else if (parser.Limit_if_exists(Vnt_flag_itm_.Tid_name)) {}
else if (parser.Limit_if_exists(Vnt_flag_itm_.Tid_del)) {}
else if (flag_count == 1 && parser.Get(Vnt_flag_itm_.Tid_title)) parser.Set_y(Vnt_flag_itm_.Tid_macro);
else if (parser.Get(Vnt_flag_itm_.Tid_macro)) {
boolean exists_d = parser.Get(Vnt_flag_itm_.Tid_descrip);
boolean exists_t = parser.Get(Vnt_flag_itm_.Tid_title);
parser.Clear();
parser.Set_y_many(Vnt_flag_itm_.Tid_all, Vnt_flag_itm_.Tid_macro);
if (exists_d) parser.Set_y(Vnt_flag_itm_.Tid_descrip);
if (exists_t) parser.Set_y(Vnt_flag_itm_.Tid_title);
}
else {
if (parser.Get(Vnt_flag_itm_.Tid_add))
parser.Set_y_many(Vnt_flag_itm_.Tid_all, Vnt_flag_itm_.Tid_show);
if (parser.Get(Vnt_flag_itm_.Tid_descrip))
parser.Set_n(Vnt_flag_itm_.Tid_show);
parser.Limit_if_exists_vnts(); // try to find flags like "zh-hans", "zh-hant"; allow syntaxes like "-{zh-hans;zh-hant|XXXX}-"
}
}
public void Parse_rules(Vnt_rule_parser parser) {
parser.Parse(src, src_bgn, src_end);
}
}

52
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_itm_.java Normal file
View File

@@ -0,0 +1,52 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
class Vnt_flag_itm_ {
public static final int
Tid_unknown = 0
, Tid_show = 1 // S: EX: -{S|zh-hans:A;zh-hant:B}- -> "A"
, Tid_all = 2 // +: EX: -{+|zh-hans:A;zh-hant:B}- -> "A"
, Tid_err = 3 // E: EX: -{E|zh-hans:A;zh-hant:B}- -> "A"
, Tid_add = 4 // A: add and output; EX: -{A|zh-hans:A;zh-hant:B}- -> "A"
, Tid_title = 5 // T: page_title; EX: -{T|zh-hans:A;zh-hant:B}- -> ""
, Tid_raw = 6 // R: raw: no convert; EX: -{R|zh-hans:A;zh-hant:B}- -> "zh-hans:A;zh-hant:B"
, Tid_descrip = 7 // D: describe; EX: -{D|zh-hans:A;zh-hant:B}- -> "简体：A；繁體：B；" (简体=Simplified;繁體=Traditional)
, Tid_del = 8 // -: remove; EX: -{-|zh-hans:A;zh-hant:B}- -> ""
, Tid_macro = 9 // H: macro; EX: -{H|zh-hans:A;zh-hant:B}- -> ""
, Tid_name = 10 // N: EX: -{N|zh-hans:A;zh-hant:B}- -> ""
, Tid_lang = 11 // vnt: EX: -{zh-hant|B}- -> "B"
, Tid__max = 12
;
private static final String[] Tid__names = new String[]
{ "unknown", "show", "all", "err", "add", "title"
, "raw", "descrip", "del", "macro", "name", "lang"
};
public static String To_name(int tid) {return Tid__names[tid];}
public static final Hash_adp_bry Regy = Hash_adp_bry.ci_a7() // NOTE: match either lc or uc; EX: -{D}- or -{d}-;
.Add_byte_int(Byte_ascii.Ltr_S , Tid_show)
.Add_byte_int(Byte_ascii.Plus , Tid_all)
.Add_byte_int(Byte_ascii.Ltr_E , Tid_err)
.Add_byte_int(Byte_ascii.Ltr_A , Tid_add)
.Add_byte_int(Byte_ascii.Ltr_T , Tid_title)
.Add_byte_int(Byte_ascii.Ltr_R , Tid_raw)
.Add_byte_int(Byte_ascii.Ltr_D , Tid_descrip)
.Add_byte_int(Byte_ascii.Dash , Tid_del)
.Add_byte_int(Byte_ascii.Ltr_H , Tid_macro)
.Add_byte_int(Byte_ascii.Ltr_N , Tid_name)
;
}

67
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_flag_parser.java Normal file
View File

@@ -0,0 +1,67 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
class Vnt_flag_parser implements gplx.core.brys.Bry_split_wkr {
private final Hash_adp_bry flag_regy = Vnt_flag_itm_.Regy;
private final Hash_adp_bry vnt_regy = Hash_adp_bry.cs();
private final boolean[] flag_ary = new boolean[Vnt_flag_itm_.Tid__max];
private int count = 0;
public int Count() {return count;}
public boolean Get(int tid) {return flag_ary[tid];}
public void Set_y(int tid) {flag_ary[tid] = Bool_.Y;}
public void Set_y_many(int... ary) {
int len = ary.length;
for (int i = 0; i < len; ++i)
flag_ary[ary[i]] = Bool_.Y;
}
public void Set_n(int tid) {flag_ary[tid] = Bool_.N;}
public void Limit(int tid) {
for (int i = 0; i < Vnt_flag_itm_.Tid__max; ++i) {
if (i != tid) flag_ary[i] = false;
}
}
public boolean Limit_if_exists(int tid) {
boolean exists = flag_ary[tid]; if (!exists) return false;
for (int i = 0; i < Vnt_flag_itm_.Tid__max; ++i) {
if (i != tid) flag_ary[i] = false;
}
return true;
}
public boolean Limit_if_exists_vnts() {
return false;
}
public void Clear() {
count = 0;
for (int i = 0; i < Vnt_flag_itm_.Tid__max; ++i)
flag_ary[i] = false;
}
public void Parse(byte[] src, int src_bgn, int src_end) {
this.Clear();
Bry_split_.Split(src, Byte_ascii.Semic, true, this);
}
public int Split(byte[] src, int itm_bgn, int itm_end) {
int flag_tid = flag_regy.Get_as_int_or(src, itm_bgn, itm_end, -1);
if (flag_tid == -1) {
int vnt_tid = vnt_regy.Get_as_int_or(src, itm_bgn, itm_end, -1);
if (vnt_tid == -1) return Bry_split_.Rv__ok; // unknown flag; ignore
}
flag_ary[flag_tid] = true;
++count;
return Bry_split_.Rv__ok;
}
}

110
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_language_converter.java Normal file
View File

@@ -0,0 +1,110 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*; import gplx.core.primitives.*;
public class Vnt_language_converter {
private final Bry_bfr bfr = Bry_bfr.new_();
private int max_depth = 32;
private byte[] src; private int src_len;
private boolean convert_needed;
private int pos;
public byte[] Parse(byte[] vnt, byte[] src) {// REF.MW:/languages/LanguageConverter.php!recursiveConvertTopLevel
synchronized (bfr) {
int markup_count = 0;
this.pos = 0;
this.convert_needed = false; // for sr lang; SEE:LanguageSr.php !$this->guessVariant(src, vnt);
this.src = src; this.src_len = src.length;
while (pos < src_len) {
int curly_bgn = Bry_find_.Find_fwd(src, Bry__curly_bgn, pos, src_len);
if (curly_bgn == Bry_find_.Not_found) { // No more markup, append final segment
if (markup_count == 0) return src; // no markups found; just return original
Add_output(vnt, convert_needed, src, pos, src_len);
return bfr.Xto_bry_and_clear();
}
bfr.Add_mid(src, pos, curly_bgn); // Markup found; append segment
Add_output(vnt, convert_needed, src, pos, src_len);
pos = curly_bgn; // Advance position
++markup_count;
Parse_recursive(vnt, pos, 1); // Do recursive conversion
}
return bfr.Xto_bry_and_clear();
}
}
private void Parse_recursive(byte[] vnt, int pos, int depth) {
pos += 2; // skip "-{"
boolean warning_done = false;
// $inner = '';
while (pos < src_len) {
byte b = src[pos];
Object o = trie.Match_bgn_w_byte(b, src,pos, src_len);
if (o == null) { // char;
++pos;
continue;
}
int new_pos = trie.Match_pos(); // Markup found; Append initial segment
bfr.Add_mid(src, pos, new_pos);
pos = new_pos; // Advance position
switch (((Byte_obj_val)o).Val()) {
case Tid__curly_bgn:
if (depth >= max_depth) {
bfr.Add(Bry__curly_bgn);
if (!warning_done) {
bfr.Add_str("<span class=\"error\">");
// wfMessage('language-converter-depth-warning')->numParams($this->mMaxDepth)->inContentLanguage()->text()
bfr.Add_str("</span>");
warning_done = true;
}
pos += 2; // skip "-{"
continue;
}
Parse_recursive(vnt, pos, depth + 1); // Recursively parse another rule
break;
case Tid__curly_end:
pos += 2;
/*
// Apply the rule
$rule = new ConverterRule($inner, $this);
$rule->parse($variant);
$this->applyManualConv($rule);
return $rule->getDisplay();
*/
return;
default: throw Err_.new_unhandled(-1); // never happens
}
}
if (pos < src_len) { // Unclosed rule
byte[] frag = Auto_convert(vnt, src, pos, src_len);
bfr.Add(Bry__curly_bgn).Add(frag);
}
pos = src_len;
}
private void Add_output(byte[] vnt, boolean convert_needed, byte[] src, int pos, int src_len) {
if (convert_needed) {
byte[] frag = Auto_convert(vnt, src, pos, src_len);
bfr.Add(frag);
}
else
bfr.Add_mid(src, pos, src_len);
}
private byte[] Auto_convert(byte[] vnt, byte[] src, int bgn, int end) {return src;}
private static final byte Tid__curly_bgn = 1, Tid__curly_end = 2;
private static final byte[] Bry__curly_bgn = Bry_.new_a7("-{"), Bry__curly_end = Bry_.new_a7("}-");
private static final Btrie_fast_mgr trie = Btrie_fast_mgr.cs()
.Add_bry_byte(Bry__curly_bgn, Tid__curly_bgn)
.Add_bry_byte(Bry__curly_end, Tid__curly_end);
}

61
400_xowa/src/gplx/xowa/parsers/vnts/Vnt_rule_parser.java Normal file
View File

@@ -0,0 +1,61 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*;
class Vnt_rule_parser implements gplx.core.brys.Bry_split_wkr {
// private final Btrie_slim_mgr vnt_trie = Btrie_slim_mgr.ci_a7();
public void Parse(byte[] src, int src_bgn, int src_end) {
Bry_split_.Split(src, Byte_ascii.Semic, false, this); // trim=false for "&#entity;" check below
}
public int Split(byte[] src, int itm_bgn, int itm_end) {
int html_entity_pos = Bry_find_.Find_bwd_while_alphanum(src, itm_bgn);
if (Bry_.Eq(src, html_entity_pos - 2, html_entity_pos, Bry__html_entity)) return Bry_split_.Rv__extend; // reject "&#entity;"; EX: "&nbsp;zh-hans;"
/*
itm_bgn = skip fwd for ws;
itm_bgn = skip fwd for "=>"
Object vnt_obj = vnt_trie.Match_bgn(src, itm_bgn, itm_end); if (vnt_obj == null) return Bry_split_.Rv__extend; // reject ";not_variant"; EX: ";border" in "zh-hans:<span style='color:blue;border:1px;'>;zh-hant:"
itm_end = skip bwd for ws
// val = trim( val[0] );
// trg = trim( val[1] );
// $u = explode( '=>', val, 2 );
// // if trg is empty, strtr() could return a wrong result
// if ( count( $u ) == 1 && trg && in_array( val, $variants ) ) {
// bidi_ary[val] = trg;
// } elseif ( count( $u ) == 2 ) {
// $from = trim( $u[0] );
// val = trim( $u[1] );
// if ( array_key_exists( val, $unidtable )
// && !is_array( $unidtable[val] )
// && trg
// && in_array( val, $variants ) ) {
// $unidtable[val] = array( $from => trg );
// } elseif ( trg && in_array( val, $variants ) ) {
// $unidtable[val][$from] = trg;
// }
// }
// // syntax error, pass
// if ( !isset( $this->mConverter->mVariantNames[val] ) ) {
// bidi_ary = array();
// $unidtable = array();
// break;
// }
*/
return Bry_split_.Rv__ok;
}
private static final byte[] Bry__html_entity = Bry_.new_a7("&#");
}

22
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_eqgt_tkn.java Normal file
View File

@@ -0,0 +1,22 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
public class Xop_vnt_eqgt_tkn extends Xop_tkn_itm_base { // "=>"
public Xop_vnt_eqgt_tkn(int bgn, int end) {this.Tkn_ini_pos(false, bgn, end);}
@Override public byte Tkn_tid() {return Xop_tkn_itm_.Tid_vnt_eqgt;}
}

75
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_flag.java Normal file
View File

@@ -0,0 +1,75 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*;
public class Xop_vnt_flag {
public Xop_vnt_flag(int tid) {this.tid = tid; this.mask = 0;}
public Xop_vnt_flag(int tid, int mask) {this.tid = tid; this.mask = mask;}
public int Tid() {return tid;} private final int tid;
public int Mask() {return mask;} private final int mask;
public static Xop_vnt_flag new_lang(int mask) {return new Xop_vnt_flag(Xop_vnt_flag_.Tid_lang, mask);}
}
class Xop_vnt_flag_ { // REF.MW: /languages/LanguageConverter.php
public static final Xop_vnt_flag[] Ary_empty = new Xop_vnt_flag[0];
public static final int
Tid_unknown = 0
, Tid_show = 1 // EX: -{S|zh-hans:A;zh-hant:B}- -> "A"
, Tid_all = 2 // EX: -{+|zh-hans:A;zh-hant:B}- -> "A"
, Tid_err = 3 // EX: -{E|zh-hans:A;zh-hant:B}- -> "A"
, Tid_add = 4 // add and output; EX: -{A|zh-hans:A;zh-hant:B}- -> "A"
, Tid_title = 5 // page_title; EX: -{T|zh-hans:A;zh-hant:B}- -> ""
, Tid_raw = 6 // raw: no convert; EX: -{R|zh-hans:A;zh-hant:B}- -> "zh-hans:A;zh-hant:B"
, Tid_descrip = 7 // describe; EX: -{D|zh-hans:A;zh-hant:B}- -> "简体：A；繁體：B；" (简体=Simplified;繁體=Traditional)
, Tid_del = 8 // remove; EX: -{-|zh-hans:A;zh-hant:B}- -> ""
, Tid_macro = 9 // macro; EX: -{H|zh-hans:A;zh-hant:B}- -> ""
, Tid_name = 10 // EX: -{N|zh-hans:A;zh-hant:B}- -> ""
, Tid_lang = 11 // EX: -{zh-hant|B}- -> "B"
, Tid_len = 12
;
public static final byte Tid__max = 12;
private static final String[] Tid__names = new String[]
{ "unknown", "show", "all", "err", "add", "title"
, "raw", "descrip", "del", "macro", "name", "lang"
};
public static String To_name(int tid) {return Tid__names[tid];}
public static final Xop_vnt_flag
Flag_unknown = new Xop_vnt_flag(Tid_unknown)
, Flag_show = new Xop_vnt_flag(Tid_show)
, Flag_all = new Xop_vnt_flag(Tid_all)
, Flag_err = new Xop_vnt_flag(Tid_err)
, Flag_add = new Xop_vnt_flag(Tid_add)
, Flag_title = new Xop_vnt_flag(Tid_title)
, Flag_raw = new Xop_vnt_flag(Tid_raw)
, Flag_descrip = new Xop_vnt_flag(Tid_descrip)
, Flag_del = new Xop_vnt_flag(Tid_del)
, Flag_macro = new Xop_vnt_flag(Tid_macro)
, Flag_name = new Xop_vnt_flag(Tid_name)
;
public static final Btrie_fast_mgr Trie = Btrie_fast_mgr.ci_a7() // NOTE: match either lc or uc; EX: -{D}- or -{d}-; // NOTE:ci.ascii:MW_const.en; flag keys; EX: -{S|a}-
.Add(Byte_ascii.Ltr_S , Xop_vnt_flag_.Flag_show)
.Add(Byte_ascii.Plus , Xop_vnt_flag_.Flag_all)
.Add(Byte_ascii.Ltr_E , Xop_vnt_flag_.Flag_err)
.Add(Byte_ascii.Ltr_A , Xop_vnt_flag_.Flag_add)
.Add(Byte_ascii.Ltr_T , Xop_vnt_flag_.Flag_title)
.Add(Byte_ascii.Ltr_R , Xop_vnt_flag_.Flag_raw)
.Add(Byte_ascii.Ltr_D , Xop_vnt_flag_.Flag_descrip)
.Add(Byte_ascii.Dash , Xop_vnt_flag_.Flag_del)
.Add(Byte_ascii.Ltr_H , Xop_vnt_flag_.Flag_macro)
.Add(Byte_ascii.Ltr_N , Xop_vnt_flag_.Flag_name)
;
}

92
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_flag_parser.java Normal file
View File

@@ -0,0 +1,92 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*;
import gplx.xowa.langs.vnts.*;
class Xop_vnt_flag_parser {
private final Xop_vnt_lang_bldr flag_lang_bldr;
private final Xol_vnt_regy vnt_regy;
public Xop_vnt_flag_parser(Xol_vnt_mgr vnt_mgr) {this.flag_lang_bldr = new Xop_vnt_lang_bldr(vnt_mgr); this.vnt_regy = vnt_mgr.Regy();}
public int Rslt_tkn_pos() {return rslt_tkn_pos;} private int rslt_tkn_pos;
public int Rslt_pipe_last() {return rslt_pipe_last;} private int rslt_pipe_last;
public Xop_vnt_flag[] Rslt_flags() {return rslt_flags;} private Xop_vnt_flag[] rslt_flags;
public void Parse(Xowe_wiki wiki, Xoa_url page_url, Xop_vnt_tkn vnt_tkn, int pipe_tkn_count, byte[] src) {
flag_lang_bldr.Clear();
int rv_idx = 0;
int subs_len = vnt_tkn.Subs_len();
this.rslt_flags = new Xop_vnt_flag[pipe_tkn_count];
this.rslt_tkn_pos = 0;
Bry_bfr flag_bfr = wiki.Utl__bfr_mkr().Get_b128();
boolean loop = true;
while (loop) {
Xop_tkn_itm sub = vnt_tkn.Subs_get(rslt_tkn_pos);
switch (sub.Tkn_tid()) {
case Xop_tkn_itm_.Tid_space: case Xop_tkn_itm_.Tid_tab: case Xop_tkn_itm_.Tid_newLine: break; // skip ws
case Xop_tkn_itm_.Tid_txt: flag_bfr.Add_mid(src, sub.Src_bgn(), sub.Src_end()); break; // just add text
case Xop_tkn_itm_.Tid_pipe:
rslt_flags[rv_idx++] = Parse_flag_bry(flag_bfr.Xto_bry_and_clear());
if (rv_idx == pipe_tkn_count) {
loop = false;
rslt_pipe_last = sub.Src_end();
}
break;
default:
wiki.Appe().Usr_dlg().Log_many("", "", "unknown tkn in vnt flag; url=~{0} tid=~{1} txt=~{2}", page_url.To_str(), sub.Tkn_tid(), String_.new_u8(src, sub.Src_bgn(), sub.Src_end()));
flag_bfr.Add_mid(src, sub.Src_bgn(), sub.Src_end());
break;
}
++rslt_tkn_pos;
if (rslt_tkn_pos == subs_len) break;
}
flag_bfr.Mkr_rls();
}
private Xop_vnt_flag Parse_flag_bry(byte[] bry) {
int bry_len = bry.length; if (bry_len == 0) return Xop_vnt_flag_.Flag_unknown; // EX: exit early if 0 len, else trie will fail; EX: "-{|}-"
Object flag_obj = flag_trie.Match_exact(bry, 0, bry_len);
return flag_obj == null
? Parse_flag_vnts(bry, bry_len) // unknown tid sequence; either (a) "lang" cmd ("-{zh-hans;zh-hant|a}-") or (b) invalid cmd ("-{X|a}-")
: (Xop_vnt_flag)flag_obj; // known flag; check that next non_ws is |
}
private Xop_vnt_flag Parse_flag_vnts(byte[] bry, int bry_len) {
int vnt_pos = 0;
boolean loop = true;
Btrie_slim_mgr trie = vnt_regy.Trie();
while (loop) {
boolean last = false, valid = true;
Object vnt_obj = trie.Match_bgn(bry, vnt_pos, bry_len);
if (vnt_obj == null) break; // no more vnts found; stop
vnt_pos = trie.Match_pos(); // update pos to end of vnt
int semic_pos = Bry_find_.Find_fwd_while_not_ws(bry, vnt_pos, bry_len);
if (semic_pos == bry_len) // note that Find_fwd_non_ws will return bry_len if no non-ws found;
last = true;
else { // char found; make sure it is semic
if (bry[semic_pos] != Byte_ascii.Semic) // invalid vnt; ignore; EX: -{zh-hansx|}-
valid = false;
vnt_pos = semic_pos + 1; // update pos to after semic
if (vnt_pos == bry_len) last = true; // EX: "a;"
}
if (valid)
flag_lang_bldr.Add(((Xol_vnt_itm)vnt_obj).Key());
else // invalid entry clears list; EX: -{zh-hans;zh-bad}-
flag_lang_bldr.Clear();
if (last) break;
}
return flag_lang_bldr.Bld();
}
private static final Btrie_fast_mgr flag_trie = Xop_vnt_flag_.Trie;
}

77
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_html_wtr.java Normal file
View File

@@ -0,0 +1,77 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.xowa.html.*; import gplx.xowa.langs.vnts.*;
public class Xop_vnt_html_wtr {
public static void Write(Bry_bfr bfr, Xoh_html_wtr html_wtr, Xop_ctx ctx, Xoh_wtr_ctx hctx, Xoae_page page, byte[] src, Xop_vnt_tkn vnt) {
Xol_vnt_mgr vnt_mgr = ctx.Wiki().Lang().Vnt_mgr(); Xol_vnt_regy vnt_regy = vnt_mgr.Regy();
byte[] cur_lang_vnt = vnt_mgr.Cur_key();
Xop_vnt_rule_tkn[] rules = vnt.Vnt_rules(); if (rules == null) return; // shouldn't happen, but guard anyway
int rules_len = rules.length;
switch (vnt.Vnt_cmd()) {
case Xop_vnt_html_wtr.Cmd_empty: break; // nothing: ""
case Xop_vnt_html_wtr.Cmd_error: // original token; "-{A}-"
bfr.Add_mid(src, vnt.Src_bgn(), vnt.Src_end());
break;
case Xop_vnt_html_wtr.Cmd_literal: { // val only; "A"
Xop_vnt_rule_tkn rule_0 = rules[0]; // Cmd_calc guarantees there will always be 1 item
html_wtr.Write_tkn_ary(bfr, ctx, hctx, src, rule_0.Rule_subs());
break;
}
case Xop_vnt_html_wtr.Cmd_bidi: { // matching rule: "A" if zh-hans; -{zh-hans:A}-
Xop_vnt_rule_tkn rule = Get_rule_by_key(vnt_mgr, vnt_regy, rules, rules_len, cur_lang_vnt);
if (rule != null) html_wtr.Write_tkn_ary(bfr, ctx, hctx, src, rule.Rule_subs());
break;
}
case Xop_vnt_html_wtr.Cmd_lang: { // matching lang: "A" if zh-hans; -{zh-hans|A}-
Xop_vnt_rule_tkn rule_0 = rules[0]; // Cmd_calc guarantees there will always be 1 rule
Xop_vnt_flag flag_0 = vnt.Vnt_flags()[0]; // parse guarantees there will always be 1 flag
if (vnt_regy.Mask__match_any(flag_0.Mask(), vnt_mgr.Cur_itm().Mask__fallbacks()))
html_wtr.Write_tkn_ary(bfr, ctx, hctx, src, rule_0.Rule_subs());
break;
}
case Xop_vnt_html_wtr.Cmd_raw: { // raw; everything between last flag and }-: "-{R|zh-hans:A;zh-hant:B}- -> "zh-hans:A;zh-hant:B"
bfr.Add_mid(src, vnt.Vnt_pipe_idx_last(), vnt.Src_end() - 2);
break;
}
case Xop_vnt_html_wtr.Cmd_descrip: { // descrip; similar to raw, but use localized lang
// bfr.Add_mid(src, vnt.Vnt_pipe_idx_last(), vnt.Src_end() - 2);
break;
}
case Xop_vnt_html_wtr.Cmd_title: break; // title: ignore; already handled during parse; DATE:2014-08-29
}
}
public static Xop_vnt_rule_tkn Get_rule_by_key(Xol_vnt_mgr vnt_mgr, Xol_vnt_regy vnt_regy, Xop_vnt_rule_tkn[] rules, int rules_len, byte[] cur_lang_vnt) {
vnt_regy.Mask__sort(rules); // sort to put more specific in front; EX: -{zh-hans:A;zh-cn:B}- should be "B", not "A"
for (int i = 0; i < rules_len; i++) {
Xop_vnt_rule_tkn rule = rules[i];
if (vnt_regy.Mask__match_any(vnt_regy.Mask__calc(rule.Rule_lang()), vnt_mgr.Cur_itm().Mask__fallbacks())) return rule;
}
return null;
}
public static final byte
Cmd_error = 0 // eror -> output literal; EX: "-{some_unknown_error}-" -> "-{some_unknown_error}-"
, Cmd_empty = 1 // empty -> output nothing; EX: "-{}-" -> ""
, Cmd_literal = 2 // literal EX: "-{A}-" -> "A"
, Cmd_bidi = 3 // bidi EX: "-{zh-hans:A;zh-hant:B}-" -> "A" if zh-hans; "B" if zh-hant
, Cmd_lang = 4 // lang EX: "-{zh-hans|A}-" -> "A" if zh-hans; "" if zh-hant
, Cmd_raw = 5 // raw; text in -{}- EX: "-{R|zh-hans:A;zh-hant:B}- -> "zh-hans:A;zh-hant:B"
, Cmd_descrip = 6 // describe; output rules EX: "-{D|zh-hans:A;zh-hant:B}- -> "简体：A；繁體：B；"
, Cmd_title = 7 // title; change title EX: "-{T|zh-hans:A;zh-hant:B}- -> "A" as display title
;
}

33
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_lang_bldr.java Normal file
View File

@@ -0,0 +1,33 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.xowa.langs.vnts.*;
class Xop_vnt_lang_bldr { // performant way of building langs; EX: -{zh;zh-hans;zh-cn|B}-
private final Xol_vnt_regy vnt_regy;
private int rslt_mask;
public Xop_vnt_lang_bldr(Xol_vnt_mgr vnt_mgr) {this.vnt_regy = vnt_mgr.Regy();}
public void Clear() {rslt_mask = 0;}
public void Add(byte[] key) {
Xol_vnt_itm vnt = vnt_regy.Get_by(key); if (vnt == null) return; // ignore invalid vnts; EX: -{zh;zhx}-
int vnt_mask = vnt.Mask__vnt();
this.rslt_mask = (rslt_mask == 0) ? vnt_mask : Enm_.Flip_int(true, rslt_mask, vnt_mask);
}
public Xop_vnt_flag Bld() {
return (rslt_mask == 0) ? Xop_vnt_flag_.Flag_unknown : Xop_vnt_flag.new_lang(rslt_mask);
}
}

73
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_lxr.java Normal file
View File

@@ -0,0 +1,73 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*; import gplx.xowa.langs.*;
import gplx.xowa.langs.vnts.*;
class Xop_vnt_lxr_bgn implements Xop_lxr {
public int Lxr_tid() {return Xop_lxr_.Tid_vnt_bgn;}
public void Init_by_wiki(Xowe_wiki wiki, Btrie_fast_mgr core_trie) {core_trie.Add(Xop_vnt_lxr_.Hook_bgn, this);}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
ctx.Subs_add_and_stack(root, tkn_mkr.Vnt(bgn_pos, cur_pos));
return cur_pos;
}
public void Term(Btrie_fast_mgr core_trie) {}
public static final Xop_vnt_lxr_bgn _ = new Xop_vnt_lxr_bgn(); Xop_vnt_lxr_bgn() {}
}
class Xop_vnt_lxr_end implements Xop_lxr {
private Xop_vnt_flag_parser flag_parser;
private Xop_vnt_rules_parser rule_parser;
public int Lxr_tid() {return Xop_lxr_.Tid_vnt_end;}
public void Init_by_wiki(Xowe_wiki wiki, Btrie_fast_mgr core_trie) {
core_trie.Add(Xop_vnt_lxr_.Hook_end, this);
Xol_vnt_mgr vnt_mgr = wiki.Lang().Vnt_mgr();
flag_parser = new Xop_vnt_flag_parser(vnt_mgr);
rule_parser = new Xop_vnt_rules_parser(vnt_mgr);
}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public void Term(Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
int stack_pos = ctx.Stack_idx_typ(Xop_tkn_itm_.Tid_vnt);
if (stack_pos == Xop_ctx.Stack_not_found) return ctx.Lxr_make_txt_(cur_pos); // "}-" found but no "-{" in stack;
Xop_vnt_tkn vnt_tkn = (Xop_vnt_tkn)ctx.Stack_pop_til(root, src, stack_pos, false, bgn_pos, cur_pos, Xop_tkn_itm_.Tid_vnt);
Xowe_wiki wiki = ctx.Wiki();
try {
vnt_tkn.Src_end_(cur_pos);
vnt_tkn.Subs_move(root);
Xop_vnt_flag[] vnt_flag_ary = Xop_vnt_flag_.Ary_empty;
int rule_subs_bgn = 0;
int pipe_tkn_count = vnt_tkn.Vnt_pipe_tkn_count();
if (pipe_tkn_count > 0) {
flag_parser.Parse(wiki, ctx.Cur_page().Url(), vnt_tkn, pipe_tkn_count, src);
vnt_flag_ary = flag_parser.Rslt_flags();
rule_subs_bgn = flag_parser.Rslt_tkn_pos();
vnt_tkn.Vnt_pipe_idx_last_(flag_parser.Rslt_pipe_last());
}
vnt_tkn.Vnt_flags_(vnt_flag_ary);
Xop_vnt_rule_tkn[] rules = rule_parser.Parse(ctx, vnt_tkn, src, rule_subs_bgn);
vnt_tkn.Vnt_rules_(rules);
vnt_tkn.Vnt_cmd_calc(wiki, ctx.Cur_page(), ctx, src);
}
catch (Exception e) {
ctx.App().Usr_dlg().Warn_many("", "", "vnt.parse failed: page=~{0} src=~{1} err=~{2}", ctx.Cur_page().Ttl().Raw(), String_.new_u8(src, bgn_pos, cur_pos), Err_.Message_gplx_full(e));
if (vnt_tkn != null)
root.Subs_add(tkn_mkr.Bry_mid(src, vnt_tkn.Src_bgn(), cur_pos));
}
return cur_pos;
}
}

46
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_lxr_.java Normal file
View File

@@ -0,0 +1,46 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*; import gplx.xowa.langs.*;
import gplx.xowa.langs.vnts.*;
public class Xop_vnt_lxr_ {
public static void Init(Xowe_wiki wiki) {
Btrie_fast_mgr wiki_trie = wiki.Parser_mgr().Main().Wtxt_lxr_mgr().Trie();
Object exists = wiki_trie.Match_bgn(Xop_vnt_lxr_.Hook_bgn, 0, Xop_vnt_lxr_.Hook_bgn.length);
if (exists == null) {
Xop_vnt_lxr_eqgt._.Init_by_wiki(wiki, wiki_trie);
Xop_vnt_lxr_bgn._.Init_by_wiki(wiki, wiki_trie);
new Xop_vnt_lxr_end().Init_by_wiki(wiki, wiki_trie);
// Btrie_fast_mgr tmpl_trie = wiki.Parser_mgr().Main().Tmpl_trie(); // do not add to tmpl trie
// Xop_vnt_lxr_bgn._.Init_by_wiki(wiki, tmpl_trie);
}
}
public static final byte[] Hook_bgn = new byte[] {Byte_ascii.Dash, Byte_ascii.Curly_bgn}, Hook_end = new byte[] {Byte_ascii.Curly_end, Byte_ascii.Dash};
}
class Xop_vnt_lxr_eqgt implements Xop_lxr {
public int Lxr_tid() {return Xop_lxr_.Tid_vnt_eqgt;}
public void Init_by_wiki(Xowe_wiki wiki, Btrie_fast_mgr core_trie) {core_trie.Add(Hook, this);}
public void Init_by_lang(Xol_lang lang, Btrie_fast_mgr core_trie) {}
public int Make_tkn(Xop_ctx ctx, Xop_tkn_mkr tkn_mkr, Xop_root_tkn root, byte[] src, int src_len, int bgn_pos, int cur_pos) {
ctx.Subs_add_and_stack(root, tkn_mkr.Vnt_eqgt(bgn_pos, cur_pos));
return cur_pos;
}
public void Term(Btrie_fast_mgr core_trie) {}
public static final byte[] Hook = new byte[] {Byte_ascii.Eq, Byte_ascii.Gt};
public static final Xop_vnt_lxr_eqgt _ = new Xop_vnt_lxr_eqgt(); Xop_vnt_lxr_eqgt() {}
}

79
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_parser__html__tst.java Normal file
View File

@@ -0,0 +1,79 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import org.junit.*;
public class Xop_vnt_parser__html__tst { // NOTE: cur_vnt is zh-cn
private final Xop_vnt_parser_fxt fxt = new Xop_vnt_parser_fxt();
@Before public void init() {fxt.Clear();}
@Test public void Literal() {fxt.Test_parse("-{A}-", "A");}
@Test public void Bidi_fwd() {fxt.Test_parse("-{zh-cn:B;zh-hans:A}-", "B");}
@Test public void Bidi_bwd() {fxt.Test_parse("-{zh-hans:A;zh-cn:B}-", "B");}
@Test public void Empty() {fxt.Test_parse("a-{}-b", "ab");}
@Test public void Unknown_empty() {fxt.Test_parse("a-{|}-c", "ac");}
@Test public void Unknown_text() {fxt.Test_parse("a-{|b}-c", "abc");}
@Test public void Unknown_flag() {fxt.Test_parse("a-{x|b}-c", "abc");}
@Test public void Lang_y() {fxt.Test_parse("-{zh-hant|A}-", "A");}
@Test public void Lang_n() {fxt.Test_parse("-{zh-sg|A}-", "");}
@Test public void Raw() {fxt.Test_parse("-{R|zh-hans:A;}-", "zh-hans:A;");}
// @Test public void Descrip() {fxt.Test_parse("-{D|zh-hans:A;}-", "zh-hans:A");}
@Test public void Tmpl() {
fxt.Parser_fxt().Init_page_create("Template:A", "B");
fxt.Test_parse("-{{{A}}}-", "B");
}
@Test public void Tmpl_arg_4() { // PURPOSE: handle "-{" + "{{{"
fxt.Parser_fxt().Init_page_create("Template:A", "-{{{{1}}}}-");
fxt.Test_parse("{{A|B}}", "B"); // -{ {{{1}}} }- -> -{B}- -> B
}
@Test public void Tmpl_arg_3() { // PURPOSE: handle "-" + "{{{"; PAGE:sr.w:ДНК; EX:<span id="interwiki-{{{1}}}-fa"></span> DATE:2014-07-03
fxt.Parser_fxt().Init_page_create("Template:A", "-{{{1}}}-");
fxt.Test_parse("{{A|B}}", "-B-");
}
@Test public void Parser_function() {
fxt.Test_parse("-{{{#expr:1}}}-", "1");
}
@Test public void Ignore() {
fxt.Test_parse("-{{#expr:1}}-", "-1-");
}
@Test public void Expr() {
fxt.Parser_fxt().Init_page_create("Template:A", "{{#expr: 0-{{{1|2}}}}}");
fxt.Test_parse("{{A}}", "-2");
}
@Test public void Invalid() { // PURPOSE: invalid flags should cause vnt to render text only; DATE:2014-04-10
fxt.Test_parse("-{:a|b}-", "b");
}
@Test public void Macro_ignore() { // PURPOSE: ignore macro (implement later); EX:zh.v:西安; Template:pagebanner; DATE:2014-05-03
fxt.Test_parse("-{H|zh-cn:亚琛; zh-tw:阿亨;}-", "");
}
@Test public void Title() { // PURPOSE: implement title; PAGE:zh.w:Help:進階字詞轉換處理 DATE:2014-08-29
fxt.Test_parse("-{T|zh-hant:A;zh-hans:B}-", "");
Tfds.Eq("A", String_.new_u8(fxt.Parser_fxt().Page().Html_data().Display_ttl_vnt()));
}
// @Test public void Disabled() {
// Xop_fxt fxt = new Xop_fxt();
// fxt.Wiki().Vnt_mgr().Set(null, null);
// fxt.Test_parse_page_all_str("a-{b}-c", "a-{b}-c");
// }
// @Test public void Enabled() {
// Xoae_app app = Xoa_app_fxt.app_();
// Xol_lang lang = new Xol_lang(app, Bry_.new_a7("zh"));
// Xowe_wiki wiki = Xoa_app_fxt.wiki_(app, "zh.wikipedia.org", lang);
// Xop_fxt fxt = new Xop_fxt(app, wiki);
// fxt.Test_parse_page_all_str("a-{b}-c", "ac");
// fxt.Wiki().Vnt_mgr().Set(null, null); // set it back to null for other tests
// }
}

153
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_parser__tkn__basic__tst.java Normal file
View File

@@ -0,0 +1,153 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import org.junit.*; import gplx.xowa.langs.vnts.*; import gplx.xowa.parsers.miscs.*;
public class Xop_vnt_parser__tkn__basic__tst {
private final Xop_vnt_lxr_fxt fxt = new Xop_vnt_lxr_fxt();
@Before public void init() {fxt.Clear();}
@Test public void Add__flag__basic() {fxt.Test_parse("-{A|b}-" , fxt.vnt_().Flags_codes_("A").Rule_("b"));}
@Test public void Add__flag__ws() {fxt.Test_parse("-{ A |b}-" , fxt.vnt_().Flags_codes_("A").Rule_("b"));}
@Test public void Add__rule__semic_n() {fxt.Test_parse("-{A|zh-hans:bcd}-" , fxt.vnt_().Flags_codes_("A").Rule_("zh-hans", "bcd"));}
@Test public void Add__rule__semic_y() {fxt.Test_parse("-{A|zh-hans:bcd;}-" , fxt.vnt_().Flags_codes_("A").Rule_("zh-hans", "bcd"));}
@Test public void Add__rule__semic_empty() {fxt.Test_parse("-{A|zh-hans:bcd;;}-" , fxt.vnt_().Flags_codes_("A").Rule_("zh-hans", "bcd"));}
@Test public void Add__rule__ws() {fxt.Test_parse("-{A|zh-hans : b c ;}-" , fxt.vnt_().Flags_codes_("A").Rule_("zh-hans", "b c"));}
@Test public void Add__rule__many() {fxt.Test_parse("-{A|zh-hans:b;zh-hant:c}-" , fxt.vnt_().Flags_codes_("A").Rule_("zh-hans", "b").Rule_("zh-hant", "c"));}
@Test public void Langs__flag__semic_n() {fxt.Test_parse("-{zh-hans;zh-hant|b}-" , fxt.vnt_().Flags_langs_(Xop_vnt_tkn_mok.Mask__hans, Xop_vnt_tkn_mok.Mask__hant).Rule_("b"));}
@Test public void Langs__flag__semic_y() {fxt.Test_parse("-{zh-hans;zh-hant;|b}-" , fxt.vnt_().Flags_langs_(Xop_vnt_tkn_mok.Mask__hans, Xop_vnt_tkn_mok.Mask__hant).Rule_("b"));}
@Test public void Langs__flag__ws() {fxt.Test_parse("-{ zh-hans ; zh-hant ; |b}-" , fxt.vnt_().Flags_langs_(Xop_vnt_tkn_mok.Mask__hans, Xop_vnt_tkn_mok.Mask__hant).Rule_("b"));}
@Test public void Langs__unknown__1st() {fxt.Test_parse("-{ zh-hans x ; zh-hant ; |b}-" , fxt.vnt_().Flags_unknown_().Rule_("b"));}
@Test public void Langs__unknown__nth() {fxt.Test_parse("-{ zh-hans ; zh-hant x; |b}-" , fxt.vnt_().Flags_unknown_().Rule_("b"));}
@Test public void Langs__unknown__all() {fxt.Test_parse("-{ zh-hans x ; zh-hant x;|b}-" , fxt.vnt_().Flags_unknown_().Rule_("b"));}
@Test public void Multiple() {fxt.Test_parse("-{A|D|E|b}-" , fxt.vnt_().Flags_codes_("A", "D", "E").Rule_("b"));}
@Test public void Unknown__flag_only() {fxt.Test_parse("-{a}-" , fxt.vnt_().Flags_none_().Rule_("a"));}
@Test public void Unknown__flag_w_text() {fxt.Test_parse("-{ A x |b}-" , fxt.vnt_().Flags_unknown_().Rule_("b"));}
@Test public void Bidi__basic() {fxt.Test_parse("-{zh-hans:a;zh-hant:b}-" , fxt.vnt_().Flags_none_().Rule_("zh-hans", "a").Rule_("zh-hant", "b"));}
@Test public void Bidi__invalid__1st() {fxt.Test_parse("-{zh-x:x;zh-hans:a;zh-hant:b}-" , fxt.vnt_().Flags_none_().Rule_("zh-x:x;zh-hans:a;zh-hant:b"));}
@Test public void Bidi__invalid__nth() {fxt.Test_parse("-{zh-hans:a;zh-x:x;zh-hant:b}-" , fxt.vnt_().Flags_none_().Rule_("zh-hans", "a;zh-x:x").Rule_("zh-hant", "b"));}
// @Test public void Bidi__html() {
// fxt.Test_parse("-{zh-cn:<span class='border:1px;text-align:center;'>text1</span>;zh-tw:<span class='border:1px;text-align:center;'>tex21</span>;}-"
// , fxt.vnt_().Flags_none_().Rule_("zh-hans", "a;zh-x:x").Rule_("zh-hant", "b"));
// }
}
class Xop_vnt_tkn_mok {
private final List_adp rules_list = List_adp_.new_(), flags_list = List_adp_.new_();
private Xop_vnt_flag[] flags;
public Xop_vnt_flag[] Flags() {
if (flags == null) flags = (Xop_vnt_flag[])flags_list.To_ary(Xop_vnt_flag.class);
return flags;
}
public Xop_vnt_tkn_mok Flags_none_() {flags_list.Clear(); return this;}
public Xop_vnt_tkn_mok Flags_unknown_(String... v) {flags_list.Add(Xop_vnt_flag_.Flag_unknown); return this;}
public Xop_vnt_tkn_mok Flags_langs_(int... ary) {flags_list.Add(Xop_vnt_flag.new_lang(Enm_.Add_int_ary(ary))); return this;}
public Xop_vnt_tkn_mok Flags_codes_(String... ary) {
int len = ary.length;
for (int i = 0; i < len; i++) {
byte[] bry = Bry_.new_a7(ary[i]);
Xop_vnt_flag flag = (Xop_vnt_flag)Xop_vnt_flag_.Trie.Match_bgn(bry, 0, bry.length);
flags_list.Add(flag);
}
return this;
}
public Xop_vnt_rule_tkn[] Rules() {
if (rules == null) rules = (Xop_vnt_rule_tkn[])rules_list.To_ary(Xop_vnt_rule_tkn.class);
return rules;
} private Xop_vnt_rule_tkn[] rules;
public Xop_vnt_tkn_mok Rule_(String rule) {return Rule_(Xop_vnt_rule_tkn.Null_lang, rule);}
public Xop_vnt_tkn_mok Rule_(byte[] lang, String rule) {return Rule_(Xop_vnt_rule_tkn.Null_macro, lang, new Xop_bry_tkn(-1, -1, Bry_.new_u8(rule)));}
public Xop_vnt_tkn_mok Rule_(String lang, String rule) {return Rule_(Xop_vnt_rule_tkn.Null_macro, Bry_.new_a7(lang), new Xop_bry_tkn(-1, -1, Bry_.new_u8(rule)));}
public Xop_vnt_tkn_mok Rule_(String macro, String lang, String rule) {return Rule_(Bry_.new_a7(macro), Bry_.new_a7(lang), new Xop_bry_tkn(-1, -1, Bry_.new_u8(rule)));}
public Xop_vnt_tkn_mok Rule_(byte[] macro, byte[] lang, Xop_tkn_itm... tkns) {rules_list.Add(new Xop_vnt_rule_tkn(macro, lang, tkns)); return this;}
public static final int Mask__hans = 2, Mask__hant = 4;
}
class Xop_vnt_lxr_fxt {
private Xop_fxt fxt;
private Xol_vnt_regy vnt_regy;
private final Bry_bfr tmp_bfr = Bry_bfr.new_();
public Xop_vnt_lxr_fxt Clear() {
Xoae_app app = Xoa_app_fxt.app_();
Xowe_wiki wiki = Xoa_app_fxt.wiki_(app, "zh.wikipedia.org");
fxt = new Xop_fxt(app, wiki);
Xop_vnt_lxr_fxt.Init_vnt_mgr(wiki.Lang().Vnt_mgr(), "zh-hans", "zh-hant", "zh-cn");
Xop_vnt_lxr_.Init(wiki);
this.vnt_regy = wiki.Lang().Vnt_mgr().Regy();
return this;
}
public Xop_vnt_tkn_mok vnt_() {return new Xop_vnt_tkn_mok();}
public static void Init_vnt_mgr(Xol_vnt_mgr vnt_mgr, String... vnts_str) {
byte[][] vnts_bry = Bry_.Ary(vnts_str);
int vnts_bry_len = vnts_bry.length;
for (int i = 0; i < vnts_bry_len; i++)
vnt_mgr.Regy__get_or_new(vnts_bry[i]);
vnt_mgr.Convert_mgr().Init(vnt_mgr.Regy());
}
public Xop_vnt_lxr_fxt Test_parse(String raw, Xop_vnt_tkn_mok expd) {
byte[] raw_bry = Bry_.new_u8(raw);
Xop_root_tkn root = fxt.Exec_parse_page_all_as_root(raw_bry);
Xop_vnt_tkn actl = (Xop_vnt_tkn)root.Subs_get(0);
Test_vnt_tkn(raw_bry, expd, actl);
return this;
}
private void Test_vnt_tkn(byte[] raw_bry, Xop_vnt_tkn_mok expd, Xop_vnt_tkn actl) {
Tfds.Eq(Vnt_flag_ary_to_str(tmp_bfr, expd.Flags()), Vnt_flag_ary_to_str(tmp_bfr, actl.Vnt_flags()), "flags");
Tfds.Eq_str_lines(Vnt_rule_ary_to_str(tmp_bfr, raw_bry, expd.Rules()), Vnt_rule_ary_to_str(tmp_bfr, raw_bry, actl.Vnt_rules()), "rules");
}
private String Vnt_flag_ary_to_str(Bry_bfr bfr, Xop_vnt_flag[] ary) {
int len = ary.length;
for (int i = 0; i < len; i++) {
Xop_vnt_flag itm = ary[i];
int itm_tid = itm.Tid();
if (itm_tid == Xop_vnt_flag_.Tid_lang)
Vnt_flag_lang_to_bfr(bfr, itm);
else
bfr.Add_str(Xop_vnt_flag_.To_name(itm_tid)).Add_byte(Byte_ascii.Semic);
}
return bfr.Xto_str_and_clear();
}
private void Vnt_flag_lang_to_bfr(Bry_bfr bfr, Xop_vnt_flag itm) {
int itm_mask = itm.Mask();
for (int i = 0; i < 32; ++i) {
int mask = gplx.core.brys.Bit_.Get_flag(i);
if (Enm_.Has_int(mask, itm_mask)) {
Xol_vnt_itm vnt = vnt_regy.Get_at(i);
bfr.Add(vnt.Key()).Add_byte(Byte_ascii.Semic);
}
}
}
private String Vnt_rule_ary_to_str(Bry_bfr bfr, byte[] src, Xop_vnt_rule_tkn[] ary) {
if (ary == null) return "";
int len = ary.length;
for (int i = 0; i < len; i++) {
Xop_vnt_rule_tkn itm = ary[i];
if (itm.Rule_macro() != Xop_vnt_rule_tkn.Null_macro) // macro exists
bfr.Add(itm.Rule_macro()).Add_str("=>");
if (itm.Rule_lang() != Xop_vnt_rule_tkn.Null_lang) // lang exists
bfr.Add(itm.Rule_lang()).Add_byte(Byte_ascii.Colon);
Xop_tkn_itm[] subs = itm.Rule_subs();
int subs_len = subs.length;
for (int j = 0; j < subs_len; j++) {
Xop_tkn_itm sub = subs[j];
if (sub.Tkn_tid() == Xop_tkn_itm_.Tid_bry) // tests uses Xop_tkn_bry
bfr.Add(((Xop_bry_tkn)sub).Val());
else
bfr.Add_mid(src, sub.Src_bgn(), sub.Src_end());
}
bfr.Add_byte(Byte_ascii.Semic).Add_byte_nl();
}
return bfr.Xto_str_and_clear();
}
}

42
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_parser__tkn__macro__tst.java Normal file
View File

@@ -0,0 +1,42 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import org.junit.*; import gplx.xowa.langs.vnts.*; import gplx.xowa.parsers.miscs.*;
public class Xop_vnt_parser__tkn__macro__tst {
private final Xop_vnt_lxr_fxt fxt = new Xop_vnt_lxr_fxt();
@Before public void init() {fxt.Clear();}
@Test public void Basic() {
fxt.Test_parse("-{H|A1=>zh-hans:B1}-", fxt.vnt_().Flags_codes_("H")
.Rule_("A1", "zh-hans", "B1"));
}
@Test public void Many() {
fxt.Test_parse("-{H|A1=>zh-hans:B1;A2=>zh-hant:B2;A3=>zh-cn:B3;}-", fxt.vnt_().Flags_codes_("H")
.Rule_("A1", "zh-hans", "B1")
.Rule_("A2", "zh-hant", "B2")
.Rule_("A3", "zh-cn" , "B3")
);
}
@Test public void Mixed() {
fxt.Test_parse("-{H|A1=>zh-hans:B1;zh-hant:B2;A3=>zh-cn:B3}-"
, fxt.vnt_().Flags_codes_("H")
.Rule_("A1" , "zh-hans" , "B1")
.Rule_( "zh-hant" , "B2")
.Rule_("A3", "zh-cn" , "B3")
);
}
}

46
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_parser_fxt.java Normal file
View File

@@ -0,0 +1,46 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.vnts.*;
public class Xop_vnt_parser_fxt {
public Xop_fxt Parser_fxt() {return fxt;} private Xop_fxt fxt;
public Xop_vnt_parser_fxt Clear() {
Xoae_app app = Xoa_app_fxt.app_();
Xowe_wiki wiki = Xoa_app_fxt.wiki_(app, "zh.wikipedia.org");
fxt = new Xop_fxt(app, wiki);
Xol_lang lang = wiki.Lang();
lang.Fallback_bry_(Bry_.new_a7("zh-cn,zh-hans,zh-hant"));
Xop_vnt_parser_fxt.Vnt_mgr__init(lang.Vnt_mgr(), 3, Vnts_chinese);
Xop_vnt_lxr_.Init(wiki);
return this;
}
public Xop_vnt_parser_fxt Test_parse(String raw, String expd) {
fxt.Test_parse_page_all_str(raw, expd);
return this;
}
public static void Vnt_mgr__init(Xol_vnt_mgr vnt_mgr, int cur_idx, String[] ary) {
int len = ary.length;
for (int i = 0; i < len; ++i) {
Xol_vnt_itm itm = vnt_mgr.Regy__get_or_new(Bry_.new_a7(ary[i]));
vnt_mgr.Lang().Lang_mgr().Get_by_key_or_load(itm.Key()).Fallback_bry_(Bry_.new_a7("zh-hans,zh-hant"));
}
vnt_mgr.Init_end();
vnt_mgr.Cur_vnt_(Bry_.new_a7(ary[cur_idx]));
}
public static final String[] Vnts_chinese = String_.Ary("zh", "zh-hans", "zh-hant", "zh-cn", "zh-hk", "zh-mo", "zh-sg", "zh-tw");
}

28
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_rule_tkn.java Normal file
View File

@@ -0,0 +1,28 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
public class Xop_vnt_rule_tkn extends Xop_tkn_itm_base {
public Xop_vnt_rule_tkn(byte[] rule_macro, byte[] rule_lang, Xop_tkn_itm[] rule_subs) {
this.rule_macro = rule_macro; this.rule_lang = rule_lang; this.rule_subs = rule_subs;
}
@Override public byte Tkn_tid() {return Xop_tkn_itm_.Tid_vnt_rule;}
public byte[] Rule_macro() {return rule_macro;} private final byte[] rule_macro;
public byte[] Rule_lang() {return rule_lang;} private final byte[] rule_lang;
public Xop_tkn_itm[] Rule_subs() {return rule_subs;} private final Xop_tkn_itm[] rule_subs;
public static final byte[] Null_lang = null, Null_macro = null;
}

172
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_rules_parser.java Normal file
View File

@@ -0,0 +1,172 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*;
import gplx.xowa.langs.vnts.*;
import gplx.xowa.parsers.miscs.*;
class Xop_vnt_rules_parser {
private final Btrie_slim_mgr trie;
private final List_adp rules_list = List_adp_.new_(), text_tkns = List_adp_.new_();
private int tmp_area; private byte[] tmp_macro, tmp_lang; private int tmp_key_bgn = -1;
private int text_tkns__last_text_tkn; private boolean check_for_ws_at_bgn = false; private int semic_pos;
private Xop_tkn_mkr tkn_mkr;
public Xop_vnt_rules_parser(Xol_vnt_mgr vnt_mgr) {this.trie = Xop_vnt_rules_parser_.new_trie(vnt_mgr.Regy());}
public void Clear() {
rules_list.Clear(); text_tkns.Clear();
this.tmp_area = Area_key;
this.tmp_macro = tmp_lang = null;
this.tmp_key_bgn = -1;
this.text_tkns__last_text_tkn = 0;
this.check_for_ws_at_bgn = false;
this.semic_pos = -1;
}
public Xop_vnt_rule_tkn[] Parse(Xop_ctx ctx, Xop_vnt_tkn vnt_tkn, byte[] src, int subs_bgn) {// parse for macro;lang;text; EX: -{macro1=>lang1:text1;macro2=>lang2:text2;}
synchronized (rules_list) {
this.Clear();
this.tkn_mkr = ctx.Tkn_mkr();
int subs_len = vnt_tkn.Subs_len(); int subs_idx = subs_bgn;
boolean valid = true;
while (subs_idx < subs_len) {
if (!Parse_tkn(vnt_tkn, src, subs_idx)) {
valid = false;
break;
}
++subs_idx;
}
if (valid)
Make_rule(); // make rules for any pending items; EX: "-{A|text}-"; "text" is unclosed by semic and would need to be processed
if (rules_list.Count() == 0) { // no rules made; assume text is just literal; EX: "-{Abc}-"
Xop_tkn_itm[] rule_subs = Xop_vnt_rules_parser_.Get_subs_as_ary(vnt_tkn, subs_bgn, subs_len);
rules_list.Add(new Xop_vnt_rule_tkn(Xop_vnt_rule_tkn.Null_macro, Xop_vnt_rule_tkn.Null_lang, rule_subs));
}
return (Xop_vnt_rule_tkn[])rules_list.To_ary_and_clear(Xop_vnt_rule_tkn.class);
}
}
private boolean Parse_tkn(Xop_vnt_tkn vnt_tkn, byte[] src, int subs_idx) {
Xop_tkn_itm sub = vnt_tkn.Subs_get(subs_idx);
boolean add_to_text_tkns = true, sub_is_middle_ws = false;
switch (sub.Tkn_tid()) {
case Xop_tkn_itm_.Tid_colon:
switch (tmp_area) {
case Area_lang_found: // colon should only follow lang; EX: -{zh-hant:text}-
if (tmp_lang == null) { // if tmp_lang already exists, ignore colon; treat as part of text; EX: -{zh-hant:text1:text2}-
tmp_lang = Bry_.Mid_w_trim(src, tmp_key_bgn, sub.Src_bgn());
tmp_area = Area_text;
add_to_text_tkns = false; // do not add ":"
check_for_ws_at_bgn = true;
}
break;
default: // colon found after unknown lang; EX: "-{zh-hant:a;zh-x:b}-"
if (rules_list.Count() == 0) return false; // invalid lang at start renders entire rule invalid;
Xop_vnt_rule_tkn rule = (Xop_vnt_rule_tkn)List_adp_.Pop(rules_list); // invalid rule in middle just adds text to previous rule
Xop_tkn_itm[] text_tkns_ary = rule.Rule_subs();
for (Xop_tkn_itm itm : text_tkns_ary)
text_tkns.Add(itm);
tmp_macro = rule.Rule_macro();
tmp_lang = rule.Rule_lang();
text_tkns.Add(tkn_mkr.Bry_raw(semic_pos, sub.Src_end(), Bry_.Mid_w_trim(src, semic_pos, sub.Src_end())));
tmp_area = Area_text;
add_to_text_tkns = false;
break;
}
break;
case Xop_tkn_itm_.Tid_space: case Xop_tkn_itm_.Tid_tab: case Xop_tkn_itm_.Tid_newLine: // skip ws
if (check_for_ws_at_bgn) add_to_text_tkns = false; // prevents ws at bgn from being added to text_tkns; EX: " a"
else sub_is_middle_ws = true;
break;
case Xop_tkn_itm_.Tid_vnt_eqgt:
if (tmp_area == Area_key) {
tmp_macro = Bry_.Mid_w_trim(src, tmp_key_bgn, sub.Src_bgn());
add_to_text_tkns = false;
}
break;
default:
break;
case Xop_tkn_itm_.Tid_txt:
add_to_text_tkns = Parse_bry(sub, src, sub.Src_bgn(), sub.Src_end());
break;
case Xop_tkn_itm_.Tid_bry:
byte[] bry = ((Xop_bry_tkn)sub).Val();
add_to_text_tkns = Parse_bry(sub, bry, 0, bry.length);
break;
}
if (sub_is_middle_ws)
text_tkns__last_text_tkn = text_tkns.Count(); // ws found in middle or end; update text_tkns__last_text_tkn; EX: "a "
else
text_tkns__last_text_tkn = -1; // not a middle_ws; set to -1; EX: "a b"
if (tmp_area == Area_text && add_to_text_tkns) { // tmp_area is text && not ":", "=>", or leading "\s"
text_tkns.Add(sub);
check_for_ws_at_bgn = false; // stop checking for ws at bgn
}
return true;
}
private boolean Parse_bry(Xop_tkn_itm sub, byte[] src, int src_bgn, int src_end) {// NOTE: parse byte-per-byte b/c ";" is not a tkn; note that class is designed around this behavior, particulary tmp_key_bgn
int cur_pos = src_bgn;
while (cur_pos < src_end) {
if (tmp_key_bgn == -1) tmp_key_bgn = cur_pos;
byte b = src[cur_pos];
Object itm_obj = trie.Match_bgn_w_byte(b, src, cur_pos, src_end);
if (itm_obj == null) // not a lang or semic; ignore
++cur_pos;
else {
Xop_vnt_rule_trie_itm itm = (Xop_vnt_rule_trie_itm)itm_obj;
int new_pos = trie.Match_pos();
switch (itm.Tid()) {
case Xop_vnt_rule_trie_itm.Tid_lang: // lang; EX: "zh-hant"; only process inside Area_key;
if (tmp_area == Area_key) {
int nxt_pos = Bry_find_.Find_fwd_while_space_or_tab(src, new_pos, src_end); // skip any ws at end
if (nxt_pos == src_end) { // eos for tkn; valid; EX: "zh-hant", "zh-hant "
tmp_key_bgn = cur_pos;
tmp_area = Area_lang_found;
}
return false;
}
break;
case Xop_vnt_rule_trie_itm.Tid_semic: // delimiter for multiple langs; EX: -{zh-hans:A;zh-cn:B}-
semic_pos = cur_pos;
switch (tmp_area) {
case Area_text: // in Area_text; make rule;
text_tkns.Add(tkn_mkr.Bry_raw(src_bgn, cur_pos, Bry_.Mid_w_trim(src, src_bgn, cur_pos))); // add everything up to ";"; EX: "text1;zh-hant"; add "text1"
Make_rule();
break;
case Area_key: // ignore; empty semic's; EX: "zh-hant:a;;"
break;
}
break;
}
cur_pos = new_pos;
}
}
return tmp_area == Area_text;
}
private void Make_rule() {// called after ";" or before "}-"
int text_tkns_len = text_tkns.Count();
if ( tmp_area != Area_text // stil in key area; EX: -{text;}-
|| text_tkns_len == 0 // no text tkns; EX: -{zh-hans:;}
) return;
if (text_tkns__last_text_tkn != -1) text_tkns.Del_range(text_tkns__last_text_tkn, text_tkns.Count() - 1); // ignore trailing ws tkns
Xop_tkn_itm[] rule_subs = (Xop_tkn_itm[])text_tkns.To_ary_and_clear(Xop_tkn_itm.class);
Xop_vnt_rule_tkn rule = new Xop_vnt_rule_tkn(tmp_macro, tmp_lang, rule_subs);
rules_list.Add(rule);
tmp_macro = null;
tmp_lang = null;
tmp_area = Area_key;
tmp_key_bgn = -1;
}
private static final int Area_key = 1, Area_lang_found = 2, Area_text = 3;
}

60
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_rules_parser_.java Normal file
View File

@@ -0,0 +1,60 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.core.btries.*;
import gplx.xowa.langs.vnts.*;
class Xop_vnt_rules_parser_ {
public static Btrie_slim_mgr new_trie(Xol_vnt_regy regy) {
Btrie_slim_mgr rv = Btrie_slim_mgr.ci_a7(); // NOTE:ci.ascii:MW_const.en; lang variant name; EX:zh-hans
int len = regy.Len();
for (int i = 0; i < len; ++i) {
Xol_vnt_itm itm = regy.Get_at(i);
byte[] key = itm.Key();
rv.Add_obj(key, Xop_vnt_rule_trie_itm.lang_(key));
}
rv.Add_obj(";", Xop_vnt_rule_trie_itm.Dlm_semic);
return rv;
}
public static Xop_tkn_itm[] Get_subs_as_ary(Xop_tkn_itm owner, int bgn, int end) {
int len = end - bgn;
Xop_tkn_itm[] rv = new Xop_tkn_itm[len];
for (int i = bgn; i < end; i++)
rv[i - bgn] = owner.Subs_get(i);
return rv;
}
}
class Xop_vnt_rule_trie_itm {
public Xop_vnt_rule_trie_itm(byte tid, byte[] lang) {this.tid = tid; this.lang = lang;}
public byte Tid() {return tid;} private byte tid;
public byte[] Lang() {return lang;} private byte[] lang;
public static final byte Tid_semic = 1, Tid_lang = 2;
public static Xop_vnt_rule_trie_itm lang_(byte[] lang) {return new Xop_vnt_rule_trie_itm(Tid_lang, lang);}
public static final Xop_vnt_rule_trie_itm
Dlm_semic = new Xop_vnt_rule_trie_itm(Tid_semic, null)
;
}
/*
-{flags|lang:rule}- EX: -{A|zh-hant:a}-
-{lang:rule;lang:rule} EX: -{zh-hans:a;zh-hant:b}-
-{lang;lang|rule}- EX: -{zh-hans;zh-hant|XXXX}-
-{rule}- EX: -{a}-
-{flags|from=>variant:to;}- EX: -{H|HUGEBLOCK=>zh-cn:macro;}-
-{lang:data_0;data_1;}- EX: -{zh-hans:<span style='border:solid;color:blue;'>;zh-hant:b}-
. where data_0 and data_1 is actually one itm since ; is not delimiter b/c data_1 must be variant_code
-{zh-hans:a-{zh-hans:b}-c}-
*/

71
400_xowa/src/gplx/xowa/parsers/vnts/Xop_vnt_tkn.java Normal file
View File

@@ -0,0 +1,71 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.parsers.vnts; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*;
import gplx.xowa.html.*; import gplx.xowa.langs.vnts.*;
public class Xop_vnt_tkn extends Xop_tkn_itm_base {
public Xop_vnt_tkn(int bgn, int end) {
this.Tkn_ini_pos(false, bgn, end);
vnt_pipe_idx_last = bgn + Xop_vnt_lxr_.Hook_bgn.length; // default last pipe to pos after -{
}
@Override public byte Tkn_tid() {return Xop_tkn_itm_.Tid_vnt;}
public int Vnt_pipe_tkn_count() {return vnt_pipe_tkn_count;}
public Xop_vnt_tkn Vnt_pipe_tkn_count_add_() {++vnt_pipe_tkn_count; return this;} private int vnt_pipe_tkn_count;
public int Vnt_pipe_idx_last() {return vnt_pipe_idx_last;} public Xop_vnt_tkn Vnt_pipe_idx_last_(int v) {vnt_pipe_idx_last = v; return this;} private int vnt_pipe_idx_last = -1;
public Xop_vnt_flag[] Vnt_flags() {return vnt_flags;} public Xop_vnt_tkn Vnt_flags_(Xop_vnt_flag[] v) {vnt_flags = v; return this;} private Xop_vnt_flag[] vnt_flags;
public Xop_vnt_rule_tkn[] Vnt_rules() {return vnt_rules;} public Xop_vnt_tkn Vnt_rules_(Xop_vnt_rule_tkn[] v) {vnt_rules = v; return this;} private Xop_vnt_rule_tkn[] vnt_rules;
public byte Vnt_cmd() {return vnt_cmd;} private byte vnt_cmd;
public void Vnt_cmd_calc(Xowe_wiki wiki, Xoae_page page, Xop_ctx ctx, byte[] src) {
int flags_len = vnt_flags.length;
int rules_len = vnt_rules.length;
if (flags_len == 0) { // no flags; either literal ("-{A}-") or bidi ("-{zh-hans:A;zh-hant:B}-");
if (rules_len == 0) vnt_cmd = Xop_vnt_html_wtr.Cmd_empty;
else {
Xop_vnt_rule_tkn rule_0 = vnt_rules[0];
if ( rules_len == 1 // only one rule
&& rule_0.Rule_lang() == Xop_vnt_rule_tkn.Null_lang // no lang; EX: -{A}-
)
vnt_cmd = Xop_vnt_html_wtr.Cmd_literal;
else // bidi: either one rule which has lang ("-{zh-hans:A}-") or more than one rule (which can't be literal)
vnt_cmd = Xop_vnt_html_wtr.Cmd_bidi;
}
}
else if (flags_len == 1){ // 1 flag; common case
Xop_vnt_flag flag_0 = vnt_flags[0];
switch (flag_0.Tid()) {
case Xop_vnt_flag_.Tid_lang : vnt_cmd = Xop_vnt_html_wtr.Cmd_lang; break;
case Xop_vnt_flag_.Tid_raw : vnt_cmd = Xop_vnt_html_wtr.Cmd_raw; break;
case Xop_vnt_flag_.Tid_descrip : vnt_cmd = Xop_vnt_html_wtr.Cmd_descrip; break;
case Xop_vnt_flag_.Tid_unknown : vnt_cmd = Xop_vnt_html_wtr.Cmd_literal; break; // flag is unknown; output text as literal; EX: "-{|a}-"; "-{X|a}-"
case Xop_vnt_flag_.Tid_macro : vnt_cmd = Xop_vnt_html_wtr.Cmd_empty; break; // TODO: implement macro; ignore for now; DATE:2014-05-03
case Xop_vnt_flag_.Tid_title: { // title; same as {{DISPLAYTITLE}} but variant aware; PAGE:zh.w:Help:進階字詞轉換處理 DATE:2014-08-29
vnt_cmd = Xop_vnt_html_wtr.Cmd_title;
Xol_vnt_mgr vnt_mgr = wiki.Lang().Vnt_mgr();
byte[] cur_lang_vnt = vnt_mgr.Cur_key();
Xop_vnt_rule_tkn rule = Xop_vnt_html_wtr.Get_rule_by_key(vnt_mgr, vnt_mgr.Regy(), vnt_rules, vnt_rules.length, cur_lang_vnt);
if (rule != null) {
Bry_bfr tmp_bfr = wiki.Utl__bfr_mkr().Get_b512();
wiki.Html_mgr().Html_wtr().Write_tkn_ary(tmp_bfr, ctx, Xoh_wtr_ctx.Alt, src, rule.Rule_subs());
byte[] display_ttl = tmp_bfr.To_bry_and_rls();
page.Html_data().Display_ttl_vnt_(display_ttl);
}
break;
}
}
}
}
}