1
0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00

v2.10.3.1

This commit is contained in:
gnosygnu
2015-10-18 22:17:57 -04:00
parent 8e18af05b6
commit 4f43f51b18
1935 changed files with 12500 additions and 12889 deletions

View File

@@ -0,0 +1,83 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.langs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.langs.*;
import gplx.core.primitives.*; import gplx.langs.jsons.*; import gplx.langs.phps.*; import gplx.langs.gfs.*;
import gplx.xowa.apps.gfs.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.msgs.*; import gplx.xowa.langs.parsers.*;
interface Json_itm_wkr {
void Read_kv_sub(byte[] key, byte[] val);
}
abstract class Json_itm_wkr__base implements Json_itm_wkr {
private Json_parser json_parser = new Json_parser();
private Php_text_itm_parser php_quote_parser = new Php_text_itm_parser().Quote_is_single_(true); // assume values are equivalent to php single quote; DATE:2014-08-06
public void Exec(byte[] src) {
List_adp tmp_list = List_adp_.new_(); Byte_obj_ref tmp_result = Byte_obj_ref.zero_(); Bry_bfr tmp_bfr = Bry_bfr.reset_(16);
Json_doc jdoc = json_parser.Parse(src);
this.Exec_bgn();
Json_nde root = jdoc.Root_nde();
int subs_len = root.Len();
for (int i = 0; i < subs_len; ++i) {
Json_itm itm = root.Get_at(i);
switch (itm.Tid()) {
case Json_itm_.Tid__kv:
Json_kv kv = (Json_kv)itm;
if (kv.Key().Data_eq(Name_metadata)) continue; // ignore @metadata node
byte[] kv_key = kv.Key().Data_bry();
byte[] kv_val = kv.Val().Data_bry();
kv_val = php_quote_parser.Parse_as_bry(tmp_list, kv_val, tmp_result, tmp_bfr);
Read_kv_sub(kv_key, kv_val);
break;
}
}
this.Exec_end();
}
@gplx.Virtual public void Exec_bgn() {}
@gplx.Virtual public void Exec_end() {}
public abstract void Read_kv_sub(byte[] key, byte[] val);
private static final byte[] Name_metadata = Bry_.new_a7("@metadata");
}
class Json_itm_wkr__gfs extends Json_itm_wkr__base {
private Xoa_gfs_bldr gfs_bldr = new Xoa_gfs_bldr();
private Xol_csv_parser csv_parser = Xol_csv_parser.Instance;
private Bry_bfr bfr;
public byte[] Xto_bry() {return gfs_bldr.Xto_bry();}
@Override public void Exec_bgn() {
bfr = gfs_bldr.Bfr();
gfs_bldr.Add_proc_init_many("this", "messages", "load_text").Add_paren_bgn().Add_nl();
gfs_bldr.Add_quote_xtn_bgn();
}
@Override public void Exec_end() {
gfs_bldr.Add_quote_xtn_end().Add_paren_end().Add_term_nl();
}
@Override public void Read_kv_sub(byte[] key, byte[] val) {
csv_parser.Save(bfr, key); // key
bfr.Add_byte_pipe(); // |
csv_parser.Save(bfr, val); // val
bfr.Add_byte_nl(); // \n
}
}
class Json_itm_wkr__msgs extends Json_itm_wkr__base {
private Xol_msg_mgr msg_mgr; private boolean dirty;
public void Ctor(boolean dirty, Xol_msg_mgr msg_mgr) {this.dirty = dirty; this.msg_mgr = msg_mgr;}
@Override public void Read_kv_sub(byte[] key, byte[] val) {
Xol_msg_itm msg_itm = msg_mgr.Itm_by_key_or_new(key);
Xol_msg_itm_.update_val_(msg_itm, val);
if (dirty) // bldr needs to dirty message to generate lang.gfs; DATE:2014-08-05
msg_itm.Dirty_(true);
}
}

View File

@@ -0,0 +1,32 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.langs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.langs.*;
import gplx.xowa.langs.*;
public class Xob_i18n_parser {
public static void Load_msgs(boolean dirty, Xol_lang_itm lang, Io_url i18n_fil) {
String i18n_str = Io_mgr.Instance.LoadFilStr_args(i18n_fil).MissingIgnored_().Exec(); if (String_.Len_eq_0(i18n_str)) return;
Json_itm_wkr__msgs wkr = new Json_itm_wkr__msgs();
wkr.Ctor(dirty, lang.Msg_mgr());
wkr.Exec(Bry_.new_u8(i18n_str));
}
public static byte[] Xto_gfs(byte[] raw) {
Json_itm_wkr__gfs wkr = new Json_itm_wkr__gfs();
wkr.Exec(raw);
return wkr.Xto_bry();
}
}

View File

@@ -0,0 +1,64 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.langs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.langs.*;
import org.junit.*;
import gplx.core.intls.*;
public class Xob_i18n_parser_tst {
@Before public void init() {fxt.Clear();} private Xob_i18n_parser_fxt fxt = new Xob_i18n_parser_fxt();
@Test public void Basic() {
fxt.Test_xto_gfs(String_.Concat_lines_nl_skip_last
( "{"
, " \"@metadata\": {"
, " \"authors\": []"
, " },"
, "\"key_1\": \"val_1\","
, "\"key_2\": \"val_2\","
, "\"key_3\": \"val $1\","
, "}"
), String_.Concat_lines_nl_skip_last
( "this.messages.load_text("
, "<:['"
, "key_1|val_1"
, "key_2|val_2"
, "key_3|val ~{0}"
, "']:>"
, ");"
));
}
// @Test public void Load_msgs_validate() {
// fxt.Test_load_msgs_dir("C:\\xowa\\bin\\any\\xowa\\xtns\\Insider\\i18n\\");
// }
}
class Xob_i18n_parser_fxt {
public void Clear() {
}
public void Test_xto_gfs(String raw, String expd) {
byte[] actl = Xob_i18n_parser.Xto_gfs(Bry_.new_u8(raw));
Tfds.Eq_str_lines(expd, String_.new_u8(actl));
}
public void Test_load_msgs_dir(String dir_str) {
Xoae_app app = Xoa_app_fxt.app_();
Xowe_wiki wiki = Xoa_app_fxt.wiki_tst_(app);
Io_url dir_url = Io_url_.new_dir_(dir_str);
Io_url[] fil_urls = Io_mgr.Instance.QueryDir_fils(dir_url);
int len = fil_urls.length;
for (int i = 0; i < len; ++i) {
Xob_i18n_parser.Load_msgs(false, wiki.Lang(), fil_urls[i]);
}
}
}

View File

@@ -0,0 +1,49 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.langs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.langs.*;
import gplx.xowa.apps.fsys.*;
import gplx.xowa.langs.*;
public class Xobc_utl_make_lang implements GfoInvkAble {
private final Xoa_lang_mgr lang_mgr; private final Xoa_fsys_mgr fsys_mgr; Xol_mw_lang_parser lang_parser;
public Xobc_utl_make_lang(Xoa_lang_mgr lang_mgr, Xoa_fsys_mgr fsys_mgr, Gfo_msg_log msg_log) {
this.lang_mgr = lang_mgr; this.fsys_mgr = fsys_mgr;
kwd_mgr = new Xobc_utl_make_lang_kwds(lang_mgr);
lang_parser = new Xol_mw_lang_parser(msg_log);
}
public Xobc_utl_make_lang_kwds Kwd_mgr() {return kwd_mgr;} private Xobc_utl_make_lang_kwds kwd_mgr;
public Ordered_hash Manual_text_bgn_hash() {return manual_text_bgn_hash;} private final Ordered_hash manual_text_bgn_hash = Ordered_hash_.New_bry();
public Ordered_hash Manual_text_end_hash() {return manual_text_end_hash;} private final Ordered_hash manual_text_end_hash = Ordered_hash_.New_bry();
public void Bld_all() {
Io_url lang_root = fsys_mgr.Cfg_lang_core_dir().OwnerDir(); // OwnerDir to get "/lang/" in "/cfg/lang/core/"
lang_parser.Parse_mediawiki(lang_mgr, lang_root.GenSubDir("mediawiki"), kwd_mgr);
kwd_mgr.Add_words();
lang_parser.Save_langs(lang_mgr, lang_root.GenSubDir(Xol_mw_lang_parser.Dir_name_core), manual_text_bgn_hash, manual_text_end_hash);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done");
}
public void Parse_manual_text(byte[] key, byte[] text, Ordered_hash manual_text) {
manual_text.Add(key, new byte[][] {key, text});
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_kwds)) return kwd_mgr;
else if (ctx.Match(k, Invk_build_all)) Bld_all();
else if (ctx.Match(k, Invk_manual_text_bgn)) Parse_manual_text(m.ReadBry("langs"), m.ReadBry("text"), manual_text_bgn_hash);
else if (ctx.Match(k, Invk_manual_text_end)) Parse_manual_text(m.ReadBry("langs"), m.ReadBry("text"), manual_text_end_hash);
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_kwds = "keywords", Invk_manual_text_bgn = "manual_text_bgn", Invk_manual_text_end = "manual_text_end", Invk_build_all = "build_all";
}

View File

@@ -0,0 +1,149 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.langs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.langs.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.kwds.*; import gplx.xowa.langs.parsers.*;
public class Xobc_utl_make_lang_kwds implements GfoInvkAble, Xol_lang_transform {
private final Xoa_lang_mgr lang_mgr;
public Xobc_utl_make_lang_kwds(Xoa_lang_mgr lang_mgr) {this.lang_mgr = lang_mgr;}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_keep_trailing_colon)) Parse_keep_trailing_colon(m.ReadBry("langs"), m.ReadBry("text"));
else if (ctx.Match(k, Invk_prepend_hash)) Parse_prepend_hash(m.ReadBry("langs"), m.ReadBry("text"));
else if (ctx.Match(k, Invk_add_words)) Parse_add_words(m.ReadBry("langs"), m.ReadBry("text"));
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_keep_trailing_colon = "keep_trailing_colon", Invk_prepend_hash = "prepend_hash", Invk_add_words = "add_words";
public byte[] Kwd_transform(byte[] lang_key, byte[] kwd_key, byte[] kwd_word) {
byte[] rv = kwd_word;
if (!Hash_itm_applies(trailing_colons, lang_key, kwd_key, kwd_word)) {
int kwd_last = rv.length - 1;
if (kwd_last > 0 && rv[kwd_last] == Byte_ascii.Colon)
rv = Bry_.Mid(rv, 0, rv.length - 1);
}
if (Hash_itm_applies(prepend_hash, lang_key, kwd_key, kwd_word)) {
if (rv.length > 0 && rv[0] != Byte_ascii.Hash)
rv = Bry_.Add(Byte_ascii.Hash, rv);
}
return rv;
}
public void Add_words() {
Ordered_hash hash = add_words_hash;
Ordered_hash tmp = Ordered_hash_.New_bry();
int hash_len = hash.Count();
for (int i = 0; i < hash_len; i++) {
Xobcl_kwd_lang cfg_lang = (Xobcl_kwd_lang)hash.Get_at(i);
Xol_lang_itm lang = lang_mgr.Get_by(cfg_lang.Key_bry()); if (lang == null) continue;
int cfg_grp_len = cfg_lang.Grps().length;
for (int j = 0; j < cfg_grp_len; j++) {
Xobcl_kwd_row cfg_grp = cfg_lang.Grps()[j];
int kwd_id = Xol_kwd_grp_.Id_by_bry(cfg_grp.Key());
if (kwd_id == -1) throw Err_.new_wo_type("could not find kwd for key", "key", String_.new_u8(cfg_grp.Key()));
Xol_kwd_grp kwd_grp = lang.Kwd_mgr().Get_at(kwd_id);
tmp.Clear();
if (kwd_grp == null) {
kwd_grp = lang.Kwd_mgr().Get_or_new(kwd_id);
kwd_grp.Srl_load(Bool_.N, Bry_.Ary_empty); // ASSUME: kwd explicitly added, but does not exist in language; default to !case_match
}
for (Xol_kwd_itm itm : kwd_grp.Itms())
tmp.Add(itm.Val(), itm.Val());
if (cfg_grp.Itms().length == 0) {
if (!tmp.Has(cfg_grp.Key())) tmp.Add(cfg_grp.Key(), cfg_grp.Key());
}
else {
for (byte[] itm : cfg_grp.Itms()) {
if (!tmp.Has(itm)) tmp.Add(itm, itm);
}
}
byte[][] words = (byte[][])tmp.To_ary(byte[].class);
kwd_grp.Srl_load(kwd_grp.Case_match(), words);
}
}
}
boolean Hash_itm_applies(Ordered_hash hash, byte[] lang_key, byte[] kwd_key, byte[] kwd_word) {
Xobcl_kwd_lang cfg_lang = (Xobcl_kwd_lang)hash.Get_by(lang_key); if (cfg_lang == null) return false;
Xobcl_kwd_row cfg_grp = cfg_lang.Grps_get_by_key(kwd_key); if (cfg_grp == null) return false;
return cfg_grp.Itms().length == 0 || cfg_grp.Itms_has(kwd_word);
}
public void Parse_add_words(byte[] langs_bry, byte[] kwds) {Parse(langs_bry, kwds, add_words_hash);} private Ordered_hash add_words_hash = Ordered_hash_.New_bry();
public void Parse_prepend_hash(byte[] langs_bry, byte[] kwds) {Parse(langs_bry, kwds, prepend_hash);} private Ordered_hash prepend_hash = Ordered_hash_.New_bry();
public void Parse_keep_trailing_colon(byte[] langs_bry, byte[] kwds) {Parse(langs_bry, kwds, trailing_colons);} private Ordered_hash trailing_colons = Ordered_hash_.New_bry();
private void Parse(byte[] langs_bry, byte[] kwds, Ordered_hash hash) {
Xobcl_kwd_row[] itms = Parse(kwds);
Xol_lang_stub[] stub_ary = Xol_lang_stub_.Ary(); // NOTE: was "lang_mgr.To_hash(langs_bry);" which was effectively "wiki_types" -> all langs; DATE:2015-10-07
int len = stub_ary.length;
for (int i = 0; i < len; ++i) {
Xol_lang_stub stub_itm = stub_ary[i];
byte[] key = stub_itm.Key();
Xobcl_kwd_lang grp = (Xobcl_kwd_lang)hash.Get_by(key);
if (grp == null) {
grp = new Xobcl_kwd_lang(key, itms);
hash.Add(key, grp);
}
else
grp.Merge(itms);
}
}
@gplx.Internal protected static Xobcl_kwd_row[] Parse(byte[] src) {
int src_len = src.length, pos = 0, fld_bgn = 0;
byte[] cur_key = Bry_.Empty;
Xol_csv_parser csv_parser = Xol_csv_parser.Instance;
List_adp rv = List_adp_.new_(); int fld_idx = 0;
while (true) {
boolean last = pos == src_len; // NOTE: logic occurs b/c of \n}~-> dlm which gobbles up last \n
byte b = last ? Byte_ascii.Nl : src[pos];
switch (b) {
case Byte_ascii.Pipe:
cur_key = csv_parser.Load(src, fld_bgn, pos);
fld_bgn = pos + 1;
++fld_idx;
break;
case Byte_ascii.Nl:
if (pos - fld_bgn > 0 || fld_idx == 1) {
byte[] cur_val = csv_parser.Load(src, fld_bgn, pos);
Xobcl_kwd_row row = new Xobcl_kwd_row(cur_key, Bry_split_.Split(cur_val, Byte_ascii.Tilde));
rv.Add(row);
}
fld_bgn = pos + 1;
fld_idx = 0;
break;
default:
break;
}
if (last) break;
++pos;
}
return (Xobcl_kwd_row[])rv.To_ary(Xobcl_kwd_row.class);
}
}
class Xobcl_kwd_lang {
public Xobcl_kwd_lang(byte[] key_bry, Xobcl_kwd_row[] grps) {
this.key_bry = key_bry; this.grps = grps;
for (Xobcl_kwd_row grp : grps)
grps_hash.Add(grp.Key(), grp);
}
public void Merge(Xobcl_kwd_row[] v) {
grps = (Xobcl_kwd_row[])Array_.Resize_add(grps, v);
for (Xobcl_kwd_row grp : v) {
grps_hash.Add_if_dupe_use_nth(grp.Key(), grp); // NOTE: Add_if_dupe_use_nth instead of Add b/c kwds may be expanded; EX: lst is added to all langs but de requires #lst~#section~Abschnitt~; DATE:2013-06-02
}
}
public Xobcl_kwd_row Grps_get_by_key(byte[] key) {return (Xobcl_kwd_row)grps_hash.Get_by(key);} private Ordered_hash grps_hash = Ordered_hash_.New_bry();
public byte[] Key_bry() {return key_bry;} private byte[] key_bry;
public Xobcl_kwd_row[] Grps() {return grps;} private Xobcl_kwd_row[] grps;
}

View File

@@ -0,0 +1,180 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.langs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.langs.*;
import org.junit.*; import gplx.core.strings.*;
import gplx.xowa.langs.*;
public class Xobc_utl_make_lang_tst {
@Before public void init() {fxt.Clear();} private Xobc_utl_make_lang_fxt fxt = new Xobc_utl_make_lang_fxt();
@Test public void Parse() {
fxt.Parse_rows(String_.Concat_lines_nl
( ""
, "if|#if~#si~"
, ""
, "ifeq|#ifeq~#sieq"
, ""
, "expr|"
)
, fxt.row_("if", "#if", "#si")
, fxt.row_("ifeq", "#ifeq", "#sieq")
, fxt.row_("expr")
);
}
@Test public void Trailing_colon() {
fxt.Kwd_mgr().Parse_keep_trailing_colon(Bry_.new_a7("fr"), Bry_.new_u8(String_.Concat_lines_nl
( "if|if:~si:~"
, "ifeq|"
)));
fxt.Ini_file_mw_core("fr", String_.Concat_lines_nl
( "$magicWords = array("
, " 'expr' => array(0, 'expr:'),"
, " 'if' => array(0, 'if:', 'si:', 'if_unchanged:'),"
, " 'ifeq' => array(0, 'ifeq:', 'sieq:'),"
, ");"
)
);
fxt.Mgr().Bld_all();
fxt.Tst_file_xo("fr", String_.Concat_lines_nl
( "this"
, ".keywords"
, " .load_text("
, "<:['"
, "expr|0|expr~"
, "if|0|if:~si:~if_unchanged~"
, "ifeq|0|ifeq:~sieq:~"
, "']:>"
, ").lang"
, ";"
));
}
@Test public void Prepend_hash() {
fxt.Kwd_mgr().Parse_prepend_hash(Bry_.new_a7("fr"), Bry_.new_u8(String_.Concat_lines_nl
( "if|if:~si:~"
, "ifeq|"
, "tag|tag~"
)));
fxt.Ini_file_mw_core("fr", String_.Concat_lines_nl
( "$magicWords = array("
, " 'tag' => array( '0', 'etiqueta', 'ETIQUETA', 'tag' ),"
, " 'expr' => array(0, 'expr:'),"
, " 'if' => array(0, 'if:', 'si:', 'if_unchanged:'),"
, " 'ifeq' => array(0, 'ifeq:', 'sieq:'),"
, ");"
)
);
fxt.Mgr().Bld_all();
fxt.Tst_file_xo("fr", String_.Concat_lines_nl
( "this"
, ".keywords"
, " .load_text("
, "<:['"
, "tag|0|etiqueta~ETIQUETA~#tag~"
, "expr|0|expr~"
, "if|0|#if~#si~if_unchanged~"
, "ifeq|0|#ifeq~#sieq~"
, "']:>"
, ").lang"
, ";"
));
}
@Test public void Add_words_hash() {
fxt.Kwd_mgr().Parse_add_words(Bry_.new_a7("fr"), Bry_.new_u8(String_.Concat_lines_nl
( "if|if_new:~if~"
, "ifeq|"
)));
fxt.Ini_file_mw_core("fr", String_.Concat_lines_nl
( "$magicWords = array("
, " 'if' => array(0, 'if:', 'si:'),"
, " 'ifeq' => array(0, 'sieq:'),"
, ");"
)
);
fxt.Mgr().Bld_all();
fxt.Tst_file_xo("fr", String_.Concat_lines_nl
( "this"
, ".keywords"
, " .load_text("
, "<:['"
, "if|0|if~si~if_new:~"
, "ifeq|0|sieq~ifeq~"
, "']:>"
, ").lang"
, ";"
));
}
@Test public void Manual_text() {
fxt.Mgr().Parse_manual_text(Bry_.new_a7("fr"), Bry_.new_u8(String_.Concat_lines_nl
( "app;"
))
, fxt.Mgr().Manual_text_end_hash());
fxt.Ini_file_mw_core("fr", String_.Concat_lines_nl
( "$magicWords = array("
, " 'if' => array(0, 'if:', 'si:'),"
, ");"
)
);
fxt.Mgr().Bld_all();
fxt.Tst_file_xo("fr", String_.Concat_lines_nl
( "this"
, ".keywords"
, " .load_text("
, "<:['"
, "if|0|if~si~"
, "']:>"
, ").lang"
, ";"
, "app;"
));
}
}
class Xobc_utl_make_lang_fxt {
public Xobc_utl_make_lang Mgr() {return mgr;} private Xobc_utl_make_lang mgr;
public Xobc_utl_make_lang_kwds Kwd_mgr() {return mgr.Kwd_mgr();}
public Xobc_utl_make_lang_fxt Clear() {
app = Xoa_app_fxt.app_();
mgr = new Xobc_utl_make_lang(app.Lang_mgr(), app.Fsys_mgr(), app.Msg_log());
return this;
} private String_bldr sb = String_bldr_.new_(); private Xoae_app app;
public Xobcl_kwd_row row_(String key, String... itms) {return new Xobcl_kwd_row(Bry_.new_a7(key), Bry_.Ary(itms));}
public void Parse_rows(String raw, Xobcl_kwd_row... expd) {Tfds.Eq_str_lines(Xto_str(expd), Xto_str(Xobc_utl_make_lang_kwds.Parse(Bry_.new_a7(raw))));}
public void Ini_file_mw_core(String lang, String raw) {
Io_url fil = app.Fsys_mgr().Cfg_lang_core_dir().OwnerDir().GenSubFil_nest("mediawiki", "core_php", "Messages" + String_.UpperFirst(lang) + ".php");
Io_mgr.Instance.SaveFilStr(fil, raw);
}
public void Tst_file_xo(String lang, String expd) {
Io_url fil = Xol_lang_itm_.xo_lang_fil_(app.Fsys_mgr(), lang);
Tfds.Eq_str_lines(expd, Io_mgr.Instance.LoadFilStr(fil));
}
private String Xto_str(Xobcl_kwd_row[] expd) {
int len = expd.length;
for (int i = 0; i < len; i++) {
Xobcl_kwd_row row = expd[i];
sb.Add(row.Key());
byte[][] itms = row.Itms();
int itms_len = itms.length;
if (itms_len > 0) {
sb.Add_char_pipe();
for (int j = 0; j < itms_len; j++) {
byte[] itm = itms[j];
sb.Add(itm).Add_char_pipe();
}
}
sb.Add_char_nl();
}
return sb.To_str_and_clear();
}
}

View File

@@ -0,0 +1,28 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.langs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.langs.*;
public class Xobcl_kwd_row {
public Xobcl_kwd_row(byte[] key, byte[][] itms) {
this.key = key; this.itms = itms;
for (byte[] itm : itms)
itms_hash.Add(itm, itm);
}
public byte[] Key() {return key;} private byte[] key;
public byte[][] Itms() {return itms;} private byte[][] itms;
public boolean Itms_has(byte[] key) {return itms_hash.Has(key);} private Ordered_hash itms_hash = Ordered_hash_.New_bry();
}

View File

@@ -0,0 +1,25 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.langs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.langs.*;
public interface Xol_lang_transform {
byte[] Kwd_transform(byte[] lang_key, byte[] kwd_key, byte[] kwd_word);
}
class Xol_lang_transform_null implements Xol_lang_transform {
public byte[] Kwd_transform(byte[] lang_key, byte[] kwd_key, byte[] kwd_word) {return kwd_word;}
public static final Xol_lang_transform_null Instance = new Xol_lang_transform_null(); Xol_lang_transform_null() {}
}

View File

@@ -0,0 +1,370 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.langs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.langs.*;
import gplx.core.primitives.*; import gplx.core.btries.*; import gplx.core.consoles.*; import gplx.core.intls.*; import gplx.langs.phps.*;
import gplx.xowa.apps.gfs.*;
import gplx.xowa.apps.fsys.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.msgs.*; import gplx.xowa.langs.kwds.*; import gplx.xowa.langs.numbers.*; import gplx.xowa.langs.parsers.*; import gplx.xowa.langs.specials.*;
import gplx.xowa.wikis.nss.*;
public class Xol_mw_lang_parser {
private Php_parser parser = new Php_parser(); private Php_evaluator evaluator;
public Xol_mw_lang_parser(Gfo_msg_log msg_log) {evaluator = new Php_evaluator(msg_log);}
public void Bld_all(Xoa_lang_mgr lang_mgr, Xoa_fsys_mgr fsys_mgr) {Bld_all(lang_mgr, fsys_mgr, Xol_lang_transform_null.Instance);}
public void Bld_all(Xoa_lang_mgr lang_mgr, Xoa_fsys_mgr fsys_mgr, Xol_lang_transform lang_transform) {
Io_url lang_root = fsys_mgr.Cfg_lang_core_dir().OwnerDir();
Parse_mediawiki(lang_mgr, lang_root.GenSubDir("mediawiki"), lang_transform);
Save_langs(lang_mgr, lang_root.GenSubDir(Xol_mw_lang_parser.Dir_name_core), Ordered_hash_.New_bry(), Ordered_hash_.New_bry());
}
public void Save_langs(Xoa_lang_mgr lang_mgr, Io_url xowa_root, Ordered_hash manual_text_bgn, Ordered_hash manual_text_end) {
int len = lang_mgr.Len();
Xoa_gfs_bldr bldr = new Xoa_gfs_bldr();
for (int i = 0; i < len; i++) {
Xol_lang_itm lang = lang_mgr.Get_at(i);
String lang_key = lang.Key_str();
Io_url lang_url = xowa_root.GenSubFil(lang_key + ".gfs");
Save_langs_by_manual_text(bldr, manual_text_bgn, lang_key);
Xol_lang_srl.Save_num_mgr(bldr, lang.Num_mgr());
bldr.Add_proc_init_many("this").Add_nl();
if (lang.Fallback_bry() != null) // NOTE: fallback will often be null; EX: en
bldr.Add_proc_cont_one(Xol_lang_itm.Invk_fallback_load).Add_parens_str(lang.Fallback_bry()).Add_nl();
if (!lang.Dir_ltr()) // NOTE: only save dir_ltr if false; EX: en
bldr.Add_proc_cont_one(Xol_lang_itm.Invk_dir_rtl_).Add_parens_str(Yn.To_str(!lang.Dir_ltr())).Add_nl();
Xol_lang_srl.Save_ns_grps(bldr, lang.Ns_names(), Xol_lang_itm.Invk_ns_names);
Xol_lang_srl.Save_ns_grps(bldr, lang.Ns_aliases(), Xol_lang_itm.Invk_ns_aliases);
Xol_lang_srl.Save_specials(bldr, lang.Specials_mgr());
Xol_lang_srl.Save_keywords(bldr, lang.Kwd_mgr());
Xol_lang_srl.Save_messages(bldr, lang.Msg_mgr(), true);
bldr.Add_term_nl();
Save_langs_by_manual_text(bldr, manual_text_end, lang_key);
Io_mgr.Instance.SaveFilBfr(lang_url, bldr.Bfr());
}
}
private void Save_langs_by_manual_text(Xoa_gfs_bldr bldr, Ordered_hash manual_text_hash, String lang_key) {
byte[][] itm = (byte[][])manual_text_hash.Get_by(Bry_.new_u8(lang_key));
if (itm != null) bldr.Bfr().Add(itm[1]);
}
public void Parse_mediawiki(Xoa_lang_mgr lang_mgr, Io_url mediawiki_root, Xol_lang_transform lang_transform) {
Bry_bfr bfr = Bry_bfr.new_();
Parse_file_core_php(lang_mgr, mediawiki_root, bfr, lang_transform);
Parse_file_xtns_php(lang_mgr, mediawiki_root, bfr, lang_transform);
Parse_file_json(lang_mgr, bfr, lang_transform, mediawiki_root.GenSubDir("core_json"));
Parse_file_json(lang_mgr, bfr, lang_transform, mediawiki_root.GenSubDir("xtns_json"));
}
private void Parse_file_core_php(Xoa_lang_mgr lang_mgr, Io_url mediawiki_root, Bry_bfr bfr, Xol_lang_transform lang_transform) {
Io_url dir = mediawiki_root.GenSubDir("core_php");
Io_url[] urls = Io_mgr.Instance.QueryDir_fils(dir);
int len = urls.length;
for (int i = 0; i < len; i++) {
Io_url url = urls[i];
try {
String lang_key = String_.Replace(String_.Lower(String_.Mid(url.NameOnly(), 8)), "_", "-"); // 8=Messages.length; lower b/c format is MessagesEn.php (need "en")
// if (String_.In(lang_key, "qqq", "enrtl", "bbc", "bbc-latn")) continue;
String text = Io_mgr.Instance.LoadFilStr(url);
Xol_lang_itm lang = lang_mgr.Get_by_or_new(Bry_.new_u8(lang_key));
this.Parse_core(text, lang, bfr, lang_transform);
} catch (Exception exc) {Err_.Noop(exc); Console_adp__sys.Instance.Write_str_w_nl("failed to parse " + url.NameOnly() + Err_.Message_gplx_full(exc));}
}
}
private void Parse_file_xtns_php(Xoa_lang_mgr lang_mgr, Io_url mediawiki_root, Bry_bfr bfr, Xol_lang_transform lang_transform) {
Io_url dir = mediawiki_root.GenSubDir("xtns_php");
Io_url[] urls = Io_mgr.Instance.QueryDir_fils(dir);
int len = urls.length;
for (int i = 0; i < len; i++) {
Io_url url = urls[i];
try {
String text = Io_mgr.Instance.LoadFilStr(url);
boolean prepend_hash = String_.Eq("ParserFunctions.i18n.magic", url.NameOnly());
this.Parse_xtn(text, url, lang_mgr, bfr, prepend_hash, lang_transform);
} catch (Exception exc) {Err_.Noop(exc); Console_adp__sys.Instance.Write_str_w_nl("failed to parse " + url.NameOnly() + Err_.Message_gplx_full(exc));}
}
}
private void Parse_file_json(Xoa_lang_mgr lang_mgr, Bry_bfr bfr, Xol_lang_transform lang_transform, Io_url root_dir) {
Io_url[] dirs = Io_mgr.Instance.QueryDir_args(root_dir).DirOnly_().ExecAsUrlAry();
int dirs_len = dirs.length;
for (int i = 0; i < dirs_len; i++) {
Io_url dir = dirs[i];
Io_url[] fils = Io_mgr.Instance.QueryDir_args(dir).ExecAsUrlAry();
int fils_len = fils.length;
for (int j = 0; j < fils_len; ++j) {
Io_url fil = fils[j];
try {
Xol_lang_itm lang = lang_mgr.Get_by_or_new(Bry_.new_u8(fil.NameOnly()));
Xob_i18n_parser.Load_msgs(true, lang, fil);
} catch (Exception exc) {Err_.Noop(exc); Console_adp__sys.Instance.Write_str_w_nl(String_.Format("failed to parse json file; url={0} err={1}\n", fil.Raw(), Err_.Message_gplx_full(exc)));}
}
}
}
public void Parse_core(String text, Xol_lang_itm lang, Bry_bfr bfr, Xol_lang_transform lang_transform) {
evaluator.Clear();
parser.Parse_tkns(text, evaluator);
Php_line[] lines = (Php_line[])evaluator.List().To_ary(Php_line.class);
int lines_len = lines.length;
List_adp bry_list = List_adp_.new_();
for (int i = 0; i < lines_len; i++) {
Php_line_assign line = (Php_line_assign)lines[i];
byte[] key = line.Key().Val_obj_bry();
Object o = Tid_hash.Get_by_bry(key);
if (o != null) {
Byte_obj_val stub = (Byte_obj_val)o;
switch (stub.Val()) {
case Tid_namespaceNames:
Parse_array_kv(bry_list, line);
lang.Ns_names().Ary_set_(Parse_namespace_strings(bry_list, true));
break;
case Tid_namespaceAliases:
Parse_array_kv(bry_list, line);
lang.Ns_aliases().Ary_set_(Parse_namespace_strings(bry_list, false));
break;
case Tid_specialPageAliases:
Parse_specials(line, lang.Key_bry(), lang.Specials_mgr());
break;
case Tid_magicwords:
Parse_magicwords(line, lang.Key_bry(), lang.Kwd_mgr(), false, lang_transform);
break;
case Tid_messages:
Parse_array_kv(bry_list, line);
Parse_messages(bry_list, lang, bfr);
break;
case Tid_fallback:
byte[] fallback_bry = Php_itm_.Parse_bry(line.Val());
if (!Bry_.Eq(fallback_bry, Bool_.True_bry)) // MessagesEn.php has fallback = false; ignore
lang.Fallback_bry_(fallback_bry);
break;
case Tid_rtl:
byte[] rtl_bry = Php_itm_.Parse_bry(line.Val());
boolean dir_rtl = Bry_.Eq(rtl_bry, Bool_.True_bry);
lang.Dir_ltr_(!dir_rtl);
break;
case Tid_separatorTransformTable:
Parse_separatorTransformTable(line, lang.Num_mgr());
break;
case Tid_digitTransformTable:
Parse_digitTransformTable(line, lang.Num_mgr());
break;
case Tid_digitGroupingPattern:
byte[] digitGroupingPattern = Php_itm_.Parse_bry(line.Val());
lang.Num_mgr().Num_grp_fmtr().Digit_grouping_pattern_(digitGroupingPattern);
break;
}
}
}
lang.Evt_lang_changed();
}
// public static String[] Lang_skip = new String[] {"qqq", "enrtl", "akz", "sxu", "test", "mwv", "rup", "hu-formal", "tzm", "bbc", "bbc-latn", "lrc", "ttt", "gom", "gom-latn"};
public static String[] Lang_skip = String_.Ary_empty;
public void Parse_xtn(String text, Io_url url, Xoa_lang_mgr lang_mgr, Bry_bfr bfr, boolean prepend_hash, Xol_lang_transform lang_transform) {
evaluator.Clear();
parser.Parse_tkns(text, evaluator);
List_adp bry_list = List_adp_.new_();
Php_line[] lines = (Php_line[])evaluator.List().To_ary(Php_line.class);
int lines_len = lines.length;
for (int i = 0; i < lines_len; i++) {
Php_line_assign line = (Php_line_assign)lines[i];
byte[] key = line.Key().Val_obj_bry();
Object o = Tid_hash.Get_by_bry(key);
if (o != null) {
Php_key[] subs = line.Key_subs();
if (subs.length == 0) continue; // ignore if no lang_key; EX: ['en']
byte[] lang_key = subs[0].Val_obj_bry();
try {
if (String_.In(String_.new_u8(lang_key), Lang_skip)) continue;
Xol_lang_itm lang = lang_mgr.Get_by_or_new(lang_key);
Byte_obj_val stub = (Byte_obj_val)o;
switch (stub.Val()) {
case Tid_messages:
Parse_array_kv(bry_list, line);
Parse_messages(bry_list, lang, bfr);
break;
case Tid_magicwords:
if (line.Key_subs().length == 0) continue; // ignore lines like $magicWords = array();
if (line.Key_subs().length > 1) throw Err_.new_wo_type("magicWords in xtn must have only 1 accessor", "len", line.Key_subs().length);
Php_key accessor = line.Key_subs()[0];
byte[] accessor_bry = accessor.Val_obj_bry();
if (Bry_.Eq(accessor_bry, lang_key)) // accessor must match lang_key
Parse_magicwords(line, lang.Key_bry(), lang.Kwd_mgr(), prepend_hash, lang_transform);
break;
}
} catch (Exception exc) {Err_.Noop(exc); Console_adp__sys.Instance.Write_str_w_nl("failed to parse " + url.NameOnly() + Err_.Message_gplx_full(exc));}
}
}
}
private void Parse_array_kv(List_adp rv, Php_line_assign line) {
rv.Clear();
Php_itm_ary ary = (Php_itm_ary)line.Val();
int subs_len = ary.Subs_len();
for (int i = 0; i < subs_len; i++) {
Php_itm_kv kv = (Php_itm_kv)ary.Subs_get(i);
rv.Add(Php_itm_.Parse_bry(kv.Key()));
rv.Add(Php_itm_.Parse_bry(kv.Val()));
}
}
public Xow_ns[] Parse_namespace_strings(List_adp list, boolean ns_names) {
byte[][] brys = (byte[][])list.To_ary(byte[].class);
int brys_len = brys.length;
Xow_ns[] rv = new Xow_ns[brys_len / 2];
int key_dif = ns_names ? 0 : 1;
int val_dif = ns_names ? 1 : 0;
for (int i = 0; i < brys_len; i+=2) {
byte[] kv_key = brys[i + key_dif];
byte[] kv_val = brys[i + val_dif];
Bry_.Replace_all_direct(kv_val, Byte_ascii.Underline, Byte_ascii.Space); // NOTE: siteInfo.xml names have " " not "_" (EX: "User talk"). for now, follow that convention
int ns_id = Id_by_mw_name(kv_key);
// if (ns_id == Xow_ns_.Id_null) throw Err_mgr.Instance.fmt_auto_(GRP_KEY, "namespace_names", String_.new_u8(kv_key));
rv[i / 2] = new Xow_ns(ns_id, Xow_ns_case_.Id_1st, kv_val, false); // note that Xow_ns is being used as glorified id-name struct; case_match and alias values do not matter
}
return rv;
}
private void Parse_messages(List_adp rv, Xol_lang_itm lang, Bry_bfr bfr) {
byte[][] brys = (byte[][])rv.To_ary(byte[].class);
int brys_len = brys.length;
Xol_msg_mgr mgr = lang.Msg_mgr();
List_adp quote_itm_list = List_adp_.new_(); Byte_obj_ref quote_parse_result = Byte_obj_ref.zero_();
for (int i = 0; i < brys_len; i+=2) {
byte[] kv_key = brys[i];
Xol_msg_itm itm = mgr.Itm_by_key_or_new(kv_key);
if (itm == null) continue;
byte[] kv_val = brys[i + 1];
kv_val = php_quote_parser.Parse_as_bry(quote_itm_list, kv_val, quote_parse_result, bfr);
Xol_msg_itm_.update_val_(itm, kv_val);
itm.Dirty_(true);
}
} private Php_text_itm_parser php_quote_parser = new Php_text_itm_parser();
private void Parse_magicwords(Php_line_assign line, byte[] lang_key, Xol_kwd_mgr mgr, boolean prepend_hash, Xol_lang_transform lang_transform) {
Php_itm_ary ary = (Php_itm_ary)line.Val();
int subs_len = ary.Subs_len();
for (int i = 0; i < subs_len; i++) {
Php_itm_sub sub = ary.Subs_get(i);
Php_itm_kv kv = (Php_itm_kv)sub;
byte[] kv_key = kv.Key().Val_obj_bry();
Php_itm_ary kv_ary = (Php_itm_ary)kv.Val();
int kv_ary_len = kv_ary.Subs_len();
boolean case_match = false; // if 1 arg, default to false
int kv_ary_bgn = 0; int words_len = kv_ary_len; // if 1 arg, default to entire kv_ary; words_len
int case_match_int = Php_itm_.Parse_int_or(kv_ary.Subs_get(0), Int_.Min_value);
if (case_match_int != Int_.Min_value) {
case_match = Parse_int_as_bool(kv_ary.Subs_get(0)); // arg[0] is case_match
kv_ary_bgn = 1; // arg[1] is 1st word
words_len = kv_ary_len - 1; // words.len = kv_len - 1 (skip case_match
}
byte[][] words = new byte[words_len][];
for (int j = kv_ary_bgn; j < kv_ary_len; j++) {
Php_itm_sub kv_sub = kv_ary.Subs_get(j);
byte[] word = Php_itm_.Parse_bry(kv_sub);
// if (prepend_hash && word[0] != Byte_ascii.Hash) word = Bry_.Add(Byte_ascii.Hash, word);
words[j - kv_ary_bgn] = lang_transform.Kwd_transform(lang_key, kv_key, word);
}
int keyword_id = Xol_kwd_grp_.Id_by_bry(kv_key); if (keyword_id == -1) continue; //throw Err_mgr.Instance.fmt_(Err_scope_keywords, "invalid_key", "key does not have id", String_.new_u8(kv_key));
Xol_kwd_grp grp = mgr.Get_or_new(keyword_id);
grp.Srl_load(case_match, words);
}
}
private void Parse_specials(Php_line_assign line, byte[] lang_key, Xol_specials_mgr specials_mgr) {
specials_mgr.Clear(); // NOTE: always clear, else will try to readd to en.gfs
Php_itm_ary ary = (Php_itm_ary)line.Val();
int subs_len = ary.Subs_len();
for (int i = 0; i < subs_len; i++) {
Php_itm_sub sub = ary.Subs_get(i);
Php_itm_kv kv = (Php_itm_kv)sub;
byte[] kv_key = kv.Key().Val_obj_bry();
Php_itm_ary kv_ary = (Php_itm_ary)kv.Val();
int kv_ary_len = kv_ary.Subs_len();
byte[][] aliases = new byte[kv_ary_len][];
for (int j = 0; j < kv_ary_len; j++) {
Php_itm_sub kv_sub = kv_ary.Subs_get(j);
aliases[j] = Php_itm_.Parse_bry(kv_sub);
}
specials_mgr.Add(kv_key, aliases);
}
}
private boolean Parse_int_as_bool(Php_itm itm) {
int rv = Php_itm_.Parse_int_or(itm, Int_.Min_value);
if (rv == Int_.Min_value) throw Err_.new_wo_type("value must be 0 or 1", "val", String_.new_u8(itm.Val_obj_bry()));
return rv == 1;
}
private void Parse_separatorTransformTable(Php_line_assign line, Xol_num_mgr num_mgr) {
if (line.Val().Itm_tid() == Php_itm_.Tid_null) return;// en is null; $separatorTransformTable = null;
Php_itm_ary ary = (Php_itm_ary)line.Val();
int subs_len = ary.Subs_len();
List_adp tmp_list = List_adp_.new_(); Byte_obj_ref tmp_result = Byte_obj_ref.zero_(); Bry_bfr tmp_bfr = Bry_bfr.reset_(16);
for (int i = 0; i < subs_len; i++) {
Php_itm_kv kv = (Php_itm_kv)ary.Subs_get(i);
byte[] key_bry = Php_itm_.Parse_bry(kv.Key()), val_bry = Php_itm_.Parse_bry(kv.Val());
val_bry = php_quote_parser.Parse_as_bry(tmp_list, val_bry, tmp_result, tmp_bfr);
Xol_csv_parser.Instance.Load(tmp_bfr, val_bry);
val_bry = tmp_bfr.To_bry_and_clear();
if ( Bry_.Eq(key_bry, Bry_separatorTransformTable_dot)
|| Bry_.Eq(key_bry, Bry_separatorTransformTable_comma)
)
num_mgr.Separators_mgr().Set(key_bry, val_bry);
else throw Err_.new_unhandled(String_.new_u8(key_bry)); // NOTE: as of v1.22.2, all Messages only have a key of "." or "," DATE:2014-04-15
}
} private static final byte[] Bry_separatorTransformTable_comma = new byte[] {Byte_ascii.Comma}, Bry_separatorTransformTable_dot = new byte[] {Byte_ascii.Dot};
private void Parse_digitTransformTable(Php_line_assign line, Xol_num_mgr num_mgr) {
if (line.Val().Itm_tid() == Php_itm_.Tid_null) return;// en is null; $digitTransformTable = null;
Php_itm_ary ary = (Php_itm_ary)line.Val();
int subs_len = ary.Subs_len();
List_adp tmp_list = List_adp_.new_(); Byte_obj_ref tmp_result = Byte_obj_ref.zero_(); Bry_bfr tmp_bfr = Bry_bfr.reset_(16);
for (int i = 0; i < subs_len; i++) {
Php_itm_kv kv = (Php_itm_kv)ary.Subs_get(i);
byte[] key_bry = Php_itm_.Parse_bry(kv.Key()), val_bry = Php_itm_.Parse_bry(kv.Val());
val_bry = php_quote_parser.Parse_as_bry(tmp_list, val_bry, tmp_result, tmp_bfr);
num_mgr.Digits_mgr().Set(key_bry, val_bry);
}
}
private static final byte
Tid_namespaceNames = 0, Tid_namespaceAliases = 1, Tid_specialPageAliases = 2
, Tid_messages = 3, Tid_magicwords = 4
, Tid_fallback = 5, Tid_rtl = 6
, Tid_separatorTransformTable = 7, Tid_digitTransformTable = 8, Tid_digitGroupingPattern = 9
;
private static Hash_adp_bry Tid_hash = Hash_adp_bry.cs()
.Add_str_byte("namespaceNames", Tid_namespaceNames).Add_str_byte("namespaceAliases", Tid_namespaceAliases).Add_str_byte("specialPageAliases", Tid_specialPageAliases)
.Add_str_byte("messages", Tid_messages).Add_str_byte("magicWords", Tid_magicwords)
.Add_str_byte("fallback", Tid_fallback).Add_str_byte("rtl", Tid_rtl)
.Add_str_byte("separatorTransformTable", Tid_separatorTransformTable)
.Add_str_byte("digitTransformTable", Tid_digitTransformTable).Add_str_byte("digitGroupingPattern", Tid_digitGroupingPattern)
;
public static int Id_by_mw_name(byte[] src) {
if (mw_names == null) {
mw_names = Btrie_slim_mgr.cs();
mw_names.Add_obj("NS_MEDIA", Int_obj_val.new_(Xow_ns_.Id_media));
mw_names.Add_obj("NS_SPECIAL", Int_obj_val.new_(Xow_ns_.Id_special));
mw_names.Add_obj("NS_MAIN", Int_obj_val.new_(Xow_ns_.Id_main));
mw_names.Add_obj("NS_TALK", Int_obj_val.new_(Xow_ns_.Id_talk));
mw_names.Add_obj("NS_USER", Int_obj_val.new_(Xow_ns_.Id_user));
mw_names.Add_obj("NS_USER_TALK", Int_obj_val.new_(Xow_ns_.Id_user_talk));
mw_names.Add_obj("NS_PROJECT", Int_obj_val.new_(Xow_ns_.Id_project));
mw_names.Add_obj("NS_PROJECT_TALK", Int_obj_val.new_(Xow_ns_.Id_project_talk));
mw_names.Add_obj("NS_FILE", Int_obj_val.new_(Xow_ns_.Id_file));
mw_names.Add_obj("NS_FILE_TALK", Int_obj_val.new_(Xow_ns_.Id_file_talk));
mw_names.Add_obj("NS_MEDIAWIKI", Int_obj_val.new_(Xow_ns_.Id_mediawiki));
mw_names.Add_obj("NS_MEDIAWIKI_TALK", Int_obj_val.new_(Xow_ns_.Id_mediaWiki_talk));
mw_names.Add_obj("NS_TEMPLATE", Int_obj_val.new_(Xow_ns_.Id_template));
mw_names.Add_obj("NS_TEMPLATE_TALK", Int_obj_val.new_(Xow_ns_.Id_template_talk));
mw_names.Add_obj("NS_HELP", Int_obj_val.new_(Xow_ns_.Id_help));
mw_names.Add_obj("NS_HELP_TALK", Int_obj_val.new_(Xow_ns_.Id_help_talk));
mw_names.Add_obj("NS_CATEGORY", Int_obj_val.new_(Xow_ns_.Id_category));
mw_names.Add_obj("NS_CATEGORY_TALK", Int_obj_val.new_(Xow_ns_.Id_category_talk));
}
Object o = mw_names.Match_exact(src, 0, src.length);
return o == null ? Xow_ns_.Id_null : ((Int_obj_val)o).Val();
} private static Btrie_slim_mgr mw_names;
public static final String Dir_name_core = "core";
}

View File

@@ -0,0 +1,317 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.langs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.langs.*;
import org.junit.*;
import gplx.core.intls.*; import gplx.xowa.parsers.lnkis.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.kwds.*; import gplx.xowa.langs.msgs.*; import gplx.xowa.langs.specials.*;
public class Xol_mw_lang_parser_tst {
@Before public void init() {fxt.Clear();} private Xol_mw_lang_parser_fxt fxt = new Xol_mw_lang_parser_fxt();
@Test public void Core_keywords() {
fxt.Parse_core("$magicWords = array('toc' => array(0, 'a1', 'a2', 'a3'), 'notoc' => array(1, 'b1', 'b2', 'b3'));")
.Tst_keyword(Xol_kwd_grp_.Id_toc, false, "a1", "a2", "a3")
.Tst_keyword(Xol_kwd_grp_.Id_notoc, true, "b1", "b2", "b3")
;
}
@Test public void Core_keywords_img_thumb() {
fxt.Parse_core("$magicWords = array('img_thumbnail' => array(1, 'thumb', 'thmb'));")
.Tst_keyword_img("thumb", Xop_lnki_arg_parser.Tid_thumb)
.Tst_keyword_img("thmb" , Xop_lnki_arg_parser.Tid_thumb)
.Tst_keyword_img("thum" , Xop_lnki_arg_parser.Tid_caption)
;
}
@Test public void Core_keywords_img_upright() {
fxt.Parse_core("$magicWords = array('img_upright' => array(1, 'upright', 'upright=$1', 'upright $1'));")
.Tst_keyword_img("upright" , Xop_lnki_arg_parser.Tid_upright)
.Tst_keyword_img("upright " , Xop_lnki_arg_parser.Tid_upright)
;
}
@Test public void Core_keywords_func_currentmonth() {
Tfds.Now_enabled_y_();
fxt.Parse_core("$magicWords = array('currentmonth' => array(0, 'MOISACTUEL'));")
.Tst_parse("{{MOISACTUEL}}", "01")
;
Tfds.Now_enabled_n_();
}
@Test public void Core_keywords_func_ns() {
fxt.Parse_core("$magicWords = array('ns' => array(0, 'ESPACEN'));")
.Tst_parse("{{ESPACEN:6}}", "File")
;
}
@Test public void Keywords_img_dim() {
fxt.Parse_core("$magicWords = array('img_width' => array(1, '$1pxl'));")
.Tst_keyword_img("50pxl", Xop_lnki_arg_parser.Tid_dim)
.Tst_keyword_img("50pxlpxl", Xop_lnki_arg_parser.Tid_dim)
.Tst_keyword_img("50xl", Xop_lnki_arg_parser.Tid_caption)
;
}
@Test public void Core_namespaces_names() {
fxt.Parse_core("$namespaceNames = array(NS_FILE => 'Fichier');")
.Sync_ns()
.Tst_ns_lkp("Fichier", Xow_ns_.Id_file)
.Tst_ns_lkp("File" , Xow_ns_.Id_null)
;
}
@Test public void Core_namespaces_aliases() {
fxt.Parse_core("$namespaceAliases = array('Discussion_Fichier' => NS_FILE_TALK);")
.Sync_ns()
.Tst_ns_lkp("Discussion Fichier", Xow_ns_.Id_file_talk)
.Tst_ns_lkp("Discussion Fichierx", Xow_ns_.Id_null)
;
}
@Test public void Core_specialPageAliases() {
fxt.Parse_core("$specialPageAliases = array('Randompage' => array('PageAuHasard', 'Page_au_hasard'));")
.Test_specialPageAliases("Randompage", "PageAuHasard", "Page_au_hasard")
;
}
@Test public void Xtn_keywords_fr() {fxt.Parse_xtn("$magicWords['fr'] = array('if' => array(0, 'si' ));").Tst_parse("{{si:|y|n}}", "n");}
@Test public void Xtn_keywords_de() {fxt.Parse_xtn("$magicWords['de'] = array('if' => array(0, 'si' ));").Tst_parse("{{si:|y|n}}", "<a href=\"/wiki/Template:Si:\">Template:Si:</a>");} // should be "Si", not "si"; ALSO, should probably be "{{si:|y|n}}" not "[[:Template:si:]]" DATE:2014-07-04
@Test public void Core_messages() {
fxt.Parse_core("$messages = array('sunday' => 'dimanche');")
.Tst_message("sunday", 0, "dimanche", false)
;
}
@Test public void Fallback() {
fxt.Parse_core("$fallback = 'zh-hans';");
Tfds.Eq("zh-hans", String_.new_u8(fxt.Lang().Fallback_bry()));
}
@Test public void Separator_transform_table() {
fxt.Parse_core("$separatorTransformTable = array( ',' => '.', '.' => ',' );");
fxt.Num_fmt_tst("1234,56", "1.234.56"); // NOTE: dot is repeated; confirmed with dewiki and {{formatnum:1234,56}}
}
@Test public void Separator_transform_table_fr() {
fxt.Parse_core("$separatorTransformTable = array( ',' => '\\xc2\\xa0', '.' => ',' );");
fxt.Num_fmt_tst("1234,56", "1 234 56"); // NOTE: nbsp here; also, nbsp is repeated. see dewiki and {{formatnum:1234,56}}
}
@Test public void Digit_transform_table() {
fxt.Save_file("mem/xowa/bin/any/xowa/cfg/lang/mediawiki/core_php/MessagesFr.php"
, "$digitTransformTable = array("
, " '0' => 'nulla',"
, " '1' => 'I',"
, " '2' => 'II',"
, " '3' => 'III',"
, " '4' => 'IV',"
, " '5' => 'V',"
, " '6' => 'VI',"
, " '7' => 'VII',"
, " '8' => 'VIII',"
, " '9' => 'IX',"
, ");"
);
fxt.Run_bld_all();
fxt.Tst_file("mem/xowa/bin/any/xowa/cfg/lang/core/fr.gfs", String_.Concat_lines_nl
( "numbers {"
, " digits {"
, " clear;"
, " set('0', 'nulla');"
, " set('1', 'I');"
, " set('2', 'II');"
, " set('3', 'III');"
, " set('4', 'IV');"
, " set('5', 'V');"
, " set('6', 'VI');"
, " set('7', 'VII');"
, " set('8', 'VIII');"
, " set('9', 'IX');"
, " }"
, "}"
, "this"
, ";"
));
}
@Test public void Digit_grouping_pattern() {
fxt.Save_file("mem/xowa/bin/any/xowa/cfg/lang/mediawiki/core_php/MessagesFr.php"
, "$digitGroupingPattern = '##,##,###'"
, ");"
);
fxt.Run_bld_all();
fxt.Tst_file("mem/xowa/bin/any/xowa/cfg/lang/core/fr.gfs", String_.Concat_lines_nl
( "numbers {"
, " digit_grouping_pattern = '##,##,###';"
, "}"
, "this"
, ";"
));
}
@Test public void Bld() {
fxt.Save_file("mem/xowa/bin/any/xowa/cfg/lang/mediawiki/core_php/MessagesFr.php"
, "$fallback = 'zh-hans';"
, "$rtl = true;"
, "$namespaceNames = array(NS_FILE => 'Filex');"
, "$namespaceAliases = array('File Discussion' => NS_FILE_TALK);"
, "$magicWords = array('currentmonth' => array(0, 'CUR_MONTH'));"
, "$messages = array('sunday' => 'sunday');"
);
fxt.Save_file("mem/xowa/bin/any/xowa/cfg/lang/mediawiki/xtns_php/Test.il8n.php"
, "$magicWords['fr'] = array('currentyear' => array(0, 'CUR_YEAR'));"
, "$messages['fr'] = array('monday' => 'monday');"
);
fxt.Save_file("mem/xowa/bin/any/xowa/cfg/lang/mediawiki/core_json/Messages/fr.json"
, "{"
, " \"@metadata\": {"
, " \"authors\": []"
, " },"
, "\"key_1\": \"val_1\","
, "\"key_2\": \"val $1\","
, "}"
);
fxt.Save_file("mem/xowa/bin/any/xowa/cfg/lang/mediawiki/xtns_json/Test2/fr.json"
, "{"
, " \"@metadata\": {"
, " \"authors\": []"
, " },"
, "\"key_3\": \"val_3\","
, "\"key_4\": \"val $1\","
, "}"
);
fxt.Run_bld_all();
fxt.Tst_file("mem/xowa/bin/any/xowa/cfg/lang/core/fr.gfs", String_.Concat_lines_nl
( "this"
, ".fallback_load('zh-hans')"
, ".dir_rtl_('y')"
, ".ns_names"
, " .load_text("
, "<:['"
, "6|Filex"
, "']:>"
, ").lang"
, ".ns_aliases"
, " .load_text("
, "<:['"
, "7|File Discussion"
, "']:>"
, ").lang"
, ".keywords"
, " .load_text("
, "<:['"
, "currentmonth|0|CUR_MONTH~"
, "currentyear|0|CUR_YEAR~"
, "']:>"
, ").lang"
, ".messages"
, " .load_text("
, "<:['"
, "sunday|sunday"
, "monday|monday"
, "key_1|val_1"
, "key_2|val ~{0}"
, "key_3|val_3"
, "key_4|val ~{0}"
, "']:>"
, ").lang"
, ";"
));
}
@Test public void Dir_ltr() {
fxt.Parse_core("$rtl = 'true';");
Tfds.Eq(false, fxt.Lang().Dir_ltr());
}
@Test public void Core_keywords__only_1() { // PURPOSE: some magic words don't specify case-match; EX: Disambiguator.php; DATE:2013-12-24
fxt.Parse_core("$magicWords = array('toc' => array('a1'));")
.Tst_keyword(Xol_kwd_grp_.Id_toc, false, "a1")
;
}
@Test public void Core_keywords__skip_case_match__1() { // PURPOSE: some magic words don't specify case-match; EX: Disambiguator.php; DATE:2013-12-24
fxt.Parse_core("$magicWords = array('toc' => array('a1'));")
.Tst_keyword(Xol_kwd_grp_.Id_toc, false, "a1")
;
}
@Test public void Core_keywords__skip_case_match__2() { // PURPOSE: some magic words don't specify case-match; EX: Disambiguator.php; DATE:2013-12-24
fxt.Parse_core("$magicWords = array('toc' => array('a1', 'a2'));")
.Tst_keyword(Xol_kwd_grp_.Id_toc, false, "a1", "a2")
;
}
}
class Xol_mw_lang_parser_fxt {
Xoae_app app; Xowe_wiki wiki; private Xop_fxt fxt;
Xol_mw_lang_parser parser = new Xol_mw_lang_parser(Gfo_msg_log.Test()); Bry_bfr tmp_bfr = Bry_bfr.reset_(255);
public void Clear() {
if (app == null) {
app = Xoa_app_fxt.app_();
}
app.Lang_mgr().Clear();// NOTE: always clear the lang
lang = app.Lang_mgr().Get_by_or_new(Bry_.new_a7("fr"));
wiki = Xoa_app_fxt.wiki_(app, "en.wikipedia.org", lang);
fxt = new Xop_fxt(app, wiki);
lang.Kwd_mgr().Clear(); lang.Msg_mgr().Clear(); // NOTE: clear kwds and msgs else they will be printed to file; this line must go last b/c various xtns will fill in kwds dynamically
}
public Xol_lang_itm Lang() {return lang;} private Xol_lang_itm lang;
public void Num_fmt_tst(String raw, String expd) {Tfds.Eq(expd, String_.new_u8(lang.Num_mgr().Format_num(Bry_.new_u8(raw))));}
public void Run_bld_all() {parser.Bld_all(app.Lang_mgr(), app.Fsys_mgr());}
public void Save_file(String path, String... lines) {
Io_mgr.Instance.SaveFilStr(Io_url_.mem_fil_(path), String_.Concat_lines_nl(lines));
}
public void Tst_file(String path, String expd) {
Io_url url = Io_url_.mem_fil_(path);
String actl = Io_mgr.Instance.LoadFilStr(url);
Tfds.Eq_str_lines(expd, actl);
}
public Xol_mw_lang_parser_fxt Parse_core(String raw) {parser.Parse_core(raw, lang, tmp_bfr, Xol_lang_transform_null.Instance); return this;}
public Xol_mw_lang_parser_fxt Parse_xtn (String raw) {parser.Parse_xtn(raw, Io_url_.Empty, app.Lang_mgr(), tmp_bfr, false, Xol_lang_transform_null.Instance); lang.Evt_lang_changed(); return this;}
public Xol_mw_lang_parser_fxt Tst_keyword(int id, boolean case_sensitive, String... words) {
Xol_kwd_grp lst = lang.Kwd_mgr().Get_at(id); if (lst == null) throw Err_.new_wo_type("list should not be null");
Tfds.Eq(case_sensitive, lst.Case_match());
int actl_len = lst.Itms().length;
String[] actl = new String[actl_len];
for (int i = 0; i < actl_len; i++)
actl[i] = String_.new_u8(lst.Itms()[i].Val());
Tfds.Eq_ary_str(words, actl);
return this;
}
public Xol_mw_lang_parser_fxt Tst_keyword_img(String key_str, byte tid) {
byte[] key = Bry_.new_u8(key_str);
Tfds.Eq(tid, lang.Lnki_arg_parser().Identify_tid(key, 0, key.length));
return this;
}
public Xol_mw_lang_parser_fxt Tst_parse(String raw, String expd) {
fxt.Reset();
fxt.Test_parse_page_all_str(raw, expd);
return this;
}
public Xol_mw_lang_parser_fxt Tst_ns_lkp(String key_str, int id) {
byte[] key = Bry_.new_u8(key_str);
Xow_ns ns = (Xow_ns)wiki.Ns_mgr().Names_get_or_null(key, 0, key.length);
int actl = ns == null ? Xow_ns_.Id_null : ns.Id();
Tfds.Eq(id, actl);
return this;
}
public Xol_mw_lang_parser_fxt Test_specialPageAliases(String special, String... expd_aliases) {
Xol_specials_itm actl_aliases = lang.Specials_mgr().Get_by_key(Bry_.new_u8(special));
Tfds.Eq_ary_str(expd_aliases, To_str_ary(actl_aliases));
return this;
}
private String[] To_str_ary(Xol_specials_itm itm) {
int len = itm.Aliases().length;
String[] rv = new String[len];
for (int i = 0; i < len; i++) {
rv[i] = String_.new_u8((byte[])itm.Aliases()[i]);
}
return rv;
}
public Xol_mw_lang_parser_fxt Sync_ns() {
Xow_ns_mgr_.rebuild_(lang, wiki.Ns_mgr());
return this;
}
public Xol_mw_lang_parser_fxt Tst_message(String key, int id, String val, boolean fmt) {
Xol_msg_itm itm = lang.Msg_mgr().Itm_by_key_or_new(Bry_.new_a7(key));
Tfds.Eq(id, itm.Id());
Tfds.Eq(val, String_.new_u8(itm.Val()));
Tfds.Eq(fmt, itm.Has_fmt_arg());
return this;
}
}

View File

@@ -0,0 +1,46 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.langs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.langs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.langs.parsers.*;
public class Xol_ns_grp implements GfoInvkAble {
public Xol_ns_grp(Xol_lang_itm lang) {this.lang = lang;} private Xol_lang_itm lang;
public int Len() {return ary.length;}
public Xow_ns Get_at(int i) {return ary[i];} private Xow_ns[] ary = Ary_empty;
public void Ary_set_(Xow_ns[] v) {this.ary = v;}
public void Ary_add_(Xow_ns[] add_ary) {
int old_ary_len = ary.length;
int add_ary_len = add_ary.length;
Xow_ns[] new_ary = new Xow_ns[old_ary_len + add_ary_len];
for (int i = 0; i < old_ary_len; i++)
new_ary[i] = ary[i];
for (int i = 0; i < add_ary_len; i++)
new_ary[i + old_ary_len] = add_ary[i];
this.ary = new_ary;
}
private static final Xow_ns[] Ary_empty = new Xow_ns[0];
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_lang)) return lang;
else if (ctx.Match(k, Invk_load_text)) Exec_load_text(m.ReadBry("v"));
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_lang = Xol_lang_srl.Invk_lang, Invk_load_text = Xol_lang_srl.Invk_load_text;
private void Exec_load_text(byte[] bry) {
ary = (Xow_ns[])Array_.Resize_add(ary, Xol_lang_srl.Load_ns_grps(bry));
}
}