From 1e28aa6e79b0804096e519fadfd44913905fb38d Mon Sep 17 00:00:00 2001 From: gnosygnu Date: Thu, 1 Dec 2016 19:55:43 -0500 Subject: [PATCH] Xtn.Math: Swap out math tags with UNIQ --- 400_xowa/src/gplx/xowa/Xoa_app_.java | 2 +- 400_xowa/src/gplx/xowa/Xop_fxt.java | 25 ++++++++++++++ .../gplx/xowa/htmls/Xoh_page_wtr_mgr_tst.java | 14 ++++++++ .../src/gplx/xowa/htmls/Xoh_page_wtr_wkr.java | 3 ++ .../htmls/core/htmls/Xoh_html_wtr_tst.java | 7 ---- .../modules/popups/Xow_popup_parser.java | 1 + .../src/gplx/xowa/parsers/Xow_parser_mgr.java | 3 ++ .../gplx/xowa/parsers/uniqs/Xop_uniq_mgr.java | 5 +++ .../xowa/parsers/vnts/Vnt_convert_lang.java | 1 - .../xndes/Xop_xnde_wkr__basic_tst.java | 13 +++----- .../xowa/xtns/math/Xof_math_html_wtr.java | 13 +++++--- .../xowa/xtns/math/Xof_math_mgr_html_tst.java | 33 ++++++++++++------- 12 files changed, 86 insertions(+), 34 deletions(-) diff --git a/400_xowa/src/gplx/xowa/Xoa_app_.java b/400_xowa/src/gplx/xowa/Xoa_app_.java index 10e22bf3e..0882e0134 100644 --- a/400_xowa/src/gplx/xowa/Xoa_app_.java +++ b/400_xowa/src/gplx/xowa/Xoa_app_.java @@ -32,7 +32,7 @@ public class Xoa_app_ { } } public static final String Name = "xowa"; - public static final String Version = "3.11.2.8"; + public static final String Version = "3.11.2.9"; public static String Build_date = "2012-12-30 00:00:00"; public static String Op_sys_str; public static String User_agent = ""; diff --git a/400_xowa/src/gplx/xowa/Xop_fxt.java b/400_xowa/src/gplx/xowa/Xop_fxt.java index 1dc2fab5a..f2a8b4994 100644 --- a/400_xowa/src/gplx/xowa/Xop_fxt.java +++ b/400_xowa/src/gplx/xowa/Xop_fxt.java @@ -417,6 +417,18 @@ public class Xop_fxt { Xol_kwd_grp kwd_grp = kwd_mgr.Get_or_new(kwd_id); kwd_grp.Srl_load(case_match, Bry_.Ary(kwds)); } + public void Init_lang_vnts(String... vnts) { + wiki.Lang().Vnt_mgr().Enabled_(true); + gplx.xowa.langs.vnts.Xol_vnt_regy vnt_regy = wiki.Lang().Vnt_mgr().Regy(); + for (int i = 0; i < vnts.length; i++) { + byte[] vnt = Bry_.new_u8(vnts[i]); + vnt_regy.Add(vnt, vnt); + if (i == 0) { + wiki.Lang().Vnt_mgr().Cur_itm_(vnt); + } + } + wiki.Lang().Vnt_mgr().Init_end(); + } public void Init_xtn_pages() { Io_mgr.Instance.InitEngine_mem(); wiki.Xtn_mgr().Xtn_proofread().Enabled_y_(); @@ -440,4 +452,17 @@ public class Xop_fxt { // Tfds.Dbg(tmp_bfr.To_str()); return tmp_bfr.To_str_and_clear(); } + public void Test__parse_to_html_mgr(String src_str, String expd) { + byte[] src_bry = Bry_.new_u8(src_str); + Xop_root_tkn root = Exec_parse_page_all_as_root(src_bry); + Xoae_page page = this.Page(); + page.Root_(root); + byte[] actl = wiki.Html_mgr().Page_wtr_mgr().Gen(page, gplx.xowa.wikis.pages.Xopg_page_.Tid_read); + Tfds.Eq_str_lines(expd, String_.new_u8(actl)); + } + public static Xop_fxt New_app_html() { + Xop_fxt fxt = new Xop_fxt(); + fxt.Wiki().Html_mgr().Page_wtr_mgr().Page_read_fmtr().Fmt_("~{page_data}"); + return fxt; + } } diff --git a/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_mgr_tst.java b/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_mgr_tst.java index e89916fc8..1ebc4050e 100644 --- a/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_mgr_tst.java +++ b/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_mgr_tst.java @@ -30,4 +30,18 @@ public class Xoh_page_wtr_mgr_tst { page_wtr_mgr.Gen(wiki.Parser_mgr().Ctx().Page(), Xopg_page_.Tid_read); Tfds.Eq(String_.new_a7(portal_mgr.Div_logo_bry()), "/site/en.wikipedia.org/wiki/"); } + @Test public void Skip__math__basic() { + Xop_fxt fxt = Xop_fxt.New_app_html(); + fxt.Init_lang_vnts("zh-hans", "zh-hant"); + + fxt.Test__parse_to_html_mgr(String_.Concat_lines_nl_skip_last + ( "x_{1}-1" + , "x-{1+2}-1" + , "-{zh-hans:A;zh-hant:B;}-" + ), String_.Concat_lines_nl_skip_last + ( "x_{1}-1" // not converted + , "x-{1+2}-1" // not converted + , "A" // converted + )); + } } diff --git a/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_wkr.java b/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_wkr.java index cb3537470..76a8de880 100644 --- a/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_wkr.java +++ b/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_wkr.java @@ -207,6 +207,9 @@ public class Xoh_page_wtr_wkr { // translate if variants are enabled Xol_vnt_mgr vnt_mgr = wiki.Lang().Vnt_mgr(); if (vnt_mgr.Enabled()) bfr.Add(vnt_mgr.Convert_lang().Parse_page(vnt_mgr.Cur_itm(), page.Db().Page().Id(), bfr.To_bry_and_clear())); + + // handle uniqs + wiki.Parser_mgr().Uniq_mgr().Parse(bfr); } private void Write_body_pre(Bry_bfr bfr, Xoae_app app, Xowe_wiki wiki, Xoh_wtr_ctx hctx, byte[] data_raw, Bry_bfr tmp_bfr) { Xoh_html_wtr_escaper.Escape(app.Parser_amp_mgr(), tmp_bfr, data_raw, 0, data_raw.length, false, false); diff --git a/400_xowa/src/gplx/xowa/htmls/core/htmls/Xoh_html_wtr_tst.java b/400_xowa/src/gplx/xowa/htmls/core/htmls/Xoh_html_wtr_tst.java index fc7ee0c7c..89142fe3a 100644 --- a/400_xowa/src/gplx/xowa/htmls/core/htmls/Xoh_html_wtr_tst.java +++ b/400_xowa/src/gplx/xowa/htmls/core/htmls/Xoh_html_wtr_tst.java @@ -320,13 +320,6 @@ public class Xoh_html_wtr_tst { fxt.Test_parse_page_wiki_str("
", "
"); fxt.Test_parse_page_wiki_str("
", "
"); } - @Test public void Math() { - fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(false); - fxt.Test_parse_page_all_str("x + y", "x + y"); // latex has img - fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(true); - fxt.Test_parse_page_all_str("x + y", "x + y"); // mathjax has no img - fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(false); - } @Test public void Timeline() {// PURPOSE: embed timeline contents in pre; DATE:2014-05-22 fxt.Test_parse_page_wiki_str("a", "
a
"); } diff --git a/400_xowa/src/gplx/xowa/htmls/modules/popups/Xow_popup_parser.java b/400_xowa/src/gplx/xowa/htmls/modules/popups/Xow_popup_parser.java index 59ac1d39e..98e0e38eb 100644 --- a/400_xowa/src/gplx/xowa/htmls/modules/popups/Xow_popup_parser.java +++ b/400_xowa/src/gplx/xowa/htmls/modules/popups/Xow_popup_parser.java @@ -128,6 +128,7 @@ public class Xow_popup_parser { parser.Parse_to_src_end(wtxt_root, wtxt_ctx, tkn_mkr, wrdx_bry, wtxt_trie, Xop_parser_.Doc_bgn_bos, wrdx_bry.length); wtxt_ctx.Parser__page_term(wtxt_root, wrdx_bry, wrdx_bry.length); wiki.Html_mgr().Html_wtr().Write_doc(wrdx_bfr, wtxt_ctx, hctx, wrdx_bry, wtxt_root); + wiki.Parser_mgr().Uniq_mgr().Parse(wrdx_bfr); } private void Adjust_wrdx_end(Xow_popup_itm popup_itm, Bry_bfr wrdx_bfr) { popup_itm.Words_found_(data.Words_found()); diff --git a/400_xowa/src/gplx/xowa/parsers/Xow_parser_mgr.java b/400_xowa/src/gplx/xowa/parsers/Xow_parser_mgr.java index 770e70ba3..8560c38ad 100644 --- a/400_xowa/src/gplx/xowa/parsers/Xow_parser_mgr.java +++ b/400_xowa/src/gplx/xowa/parsers/Xow_parser_mgr.java @@ -20,6 +20,7 @@ import gplx.core.primitives.*; import gplx.core.brys.fmtrs.*; import gplx.xowa.wikis.*; import gplx.core.envs.*; import gplx.xowa.files.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.wbases.hwtrs.*; import gplx.xowa.xtns.pfuncs.ifs.*; import gplx.xowa.xtns.pfuncs.times.*; import gplx.xowa.xtns.pfuncs.ttls.*; +import gplx.xowa.parsers.uniqs.*; public class Xow_parser_mgr { private final Xowe_wiki wiki; private final Xop_tkn_mkr tkn_mkr; public Xow_parser_mgr(Xowe_wiki wiki) { @@ -37,6 +38,7 @@ public class Xow_parser_mgr { public Pft_func_formatdate_bldr Date_fmt_bldr() {return date_fmt_bldr;} private final Pft_func_formatdate_bldr date_fmt_bldr = new Pft_func_formatdate_bldr(); public Gfo_number_parser Pp_num_parser() {return pp_num_parser;} private final Gfo_number_parser pp_num_parser = new Gfo_number_parser().Ignore_space_at_end_y_(); public int[] Rel2abs_ary() {return rel2abs_ary;} private final int[] rel2abs_ary = new int[Pfunc_rel2abs.Ttl_max]; + public Xop_uniq_mgr Uniq_mgr() {return uniq_mgr;} private final Xop_uniq_mgr uniq_mgr = new Xop_uniq_mgr(); public boolean Lst__recursing() {return lst_recursing;} private boolean lst_recursing; public void Lst__recursing_(boolean v) {lst_recursing = v;} public Bry_bfr Wbase__time__bfr() {return wbase__time__bfr;} private final Bry_bfr wbase__time__bfr = Bry_bfr_.New(); public Bry_fmtr Wbase__time__fmtr() {return wbase__time__fmtr;} private final Bry_fmtr wbase__time__fmtr = Bry_fmtr.new_(); @@ -76,6 +78,7 @@ public class Xow_parser_mgr { if (!Env_.Mode_testing()) wiki.Init_assert(); tmpl_stack_ary = Bry_.Ary_empty; tmpl_stack_ary_len = tmpl_stack_ary_max = 0; + uniq_mgr.Clear(); scrib.When_page_changed(page); // notify scribunto about page changed ctx.Page_(page); diff --git a/400_xowa/src/gplx/xowa/parsers/uniqs/Xop_uniq_mgr.java b/400_xowa/src/gplx/xowa/parsers/uniqs/Xop_uniq_mgr.java index 1f01e16ec..916651911 100644 --- a/400_xowa/src/gplx/xowa/parsers/uniqs/Xop_uniq_mgr.java +++ b/400_xowa/src/gplx/xowa/parsers/uniqs/Xop_uniq_mgr.java @@ -31,6 +31,11 @@ public class Xop_uniq_mgr { // REF.MW:/parser/StripState.php return key; } public byte[] Get(byte[] key) {return (byte[])general_trie.Match_exact(key, 0, key.length);} + public void Parse(Bry_bfr bfr) { + if (general_trie.Count() == 0) return; + byte[] rv = Parse(key_bfr, general_trie, bfr.To_bry_and_clear()); + bfr.Add(rv); + } public byte[] Parse(byte[] src) {return Parse(key_bfr, general_trie, src);} private byte[] Parse(Bry_bfr bfr, Btrie_slim_mgr trie, byte[] src) { int src_len = src.length; diff --git a/400_xowa/src/gplx/xowa/parsers/vnts/Vnt_convert_lang.java b/400_xowa/src/gplx/xowa/parsers/vnts/Vnt_convert_lang.java index 804f70250..474944a99 100644 --- a/400_xowa/src/gplx/xowa/parsers/vnts/Vnt_convert_lang.java +++ b/400_xowa/src/gplx/xowa/parsers/vnts/Vnt_convert_lang.java @@ -184,6 +184,5 @@ public class Vnt_convert_lang { private static final Btrie_fast_mgr trie = Btrie_fast_mgr.cs() .Add_bry_byte(Bry__curly_bgn, Tid__curly_bgn) .Add_bry_byte(Bry__curly_end, Tid__curly_end); - public static final byte[] Bry__armor_bgn = Bry_.new_a7("-{R|"), Bry__armor_end = Bry_.new_a7("}-"); private static final int max_depth = 32; } diff --git a/400_xowa/src/gplx/xowa/parsers/xndes/Xop_xnde_wkr__basic_tst.java b/400_xowa/src/gplx/xowa/parsers/xndes/Xop_xnde_wkr__basic_tst.java index 130e80a7c..ff78fc195 100644 --- a/400_xowa/src/gplx/xowa/parsers/xndes/Xop_xnde_wkr__basic_tst.java +++ b/400_xowa/src/gplx/xowa/parsers/xndes/Xop_xnde_wkr__basic_tst.java @@ -18,7 +18,7 @@ along with this program. If not, see . package gplx.xowa.parsers.xndes; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*; import org.junit.*; import gplx.xowa.wikis.nss.*; public class Xop_xnde_wkr__basic_tst { - private final Xop_fxt fxt = new Xop_fxt(); + private final Xop_fxt fxt = new Xop_fxt(); @After public void term() {fxt.Init_para_n_();} @Test public void Escape_lt() { // PURPOSE: some templates have unknown tags; PAGE:en.w:PHP fxt.Init_para_y_(); @@ -103,9 +103,9 @@ public class Xop_xnde_wkr__basic_tst { } @Test public void CaseSensitivity_xtn_3() {// PURPOSE: xtn xnde must do case-insensitive match DATE:2013-12-02 fxt.Test_parse_page_all_str - ( "a b c" // should match not - , "a b c" - ); + ( "a b c" // should match not + , "a b c" + ); } @Test public void Whitelist() { fxt.Test_parse_page_all_str("", ""); @@ -122,11 +122,6 @@ public class Xop_xnde_wkr__basic_tst { @Test public void Script_in_syntaxhighlight() { fxt.Test_parse_page_all_str("", "
<script>alert('fail');</script>
"); } - @Test public void Script_in_math() { - fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(false); - fxt.Test_parse_page_all_str("", "<script>alert('fail');"); - fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(true); - } @Test public void Html5_time() {// PURPOSE: HTML5; should output self (i.e.: must be whitelisted) fxt.Test_parse_page_wiki_str("", ""); } diff --git a/400_xowa/src/gplx/xowa/xtns/math/Xof_math_html_wtr.java b/400_xowa/src/gplx/xowa/xtns/math/Xof_math_html_wtr.java index f6e2f3182..be4754d31 100644 --- a/400_xowa/src/gplx/xowa/xtns/math/Xof_math_html_wtr.java +++ b/400_xowa/src/gplx/xowa/xtns/math/Xof_math_html_wtr.java @@ -20,6 +20,7 @@ import gplx.core.brys.fmtrs.*; import gplx.xowa.htmls.*; import gplx.langs.htmls.entitys.*; import gplx.xowa.htmls.core.htmls.*; import gplx.xowa.parsers.*; import gplx.xowa.parsers.xndes.*; import gplx.xowa.parsers.vnts.*; public class Xof_math_html_wtr { + public static final byte[] Bry__id_prefix = Bry_.new_a7("xowa_math_txt_"); private final Bry_fmtr math_fmtr_latex = Bry_fmtr.new_("~{math_text}", "math_idx", "math_text"); private final Bry_fmtr math_fmtr_mathjax = Bry_fmtr.new_("~{math_text}", "math_idx", "math_text"); public void Write(Xoh_html_wtr wtr, Xop_ctx ctx, Xoh_wtr_ctx opts, Bry_bfr bfr, byte[] src, Xop_xnde_tkn xnde) { @@ -45,12 +46,16 @@ public class Xof_math_html_wtr { private void Write_for_mathjax(Bry_bfr bfr, Xoae_page page, boolean enabled, boolean renderer_is_latex, byte[] math_bry, Bry_bfr tmp_bfr, Xof_math_itm math_itm) { int id = page.File_math().Count(); Xof_math_itm new_math_itm = math_itm.Clone().Id_(id); + +// boolean armor_math = page.Lang().Vnt_mgr().Enabled() && !renderer_is_latex; // REF.MW:LangConverter.php|armourMath +// if (armor_math) bfr.Add(Vnt_convert_lang.Bry__armor_bgn); +// bfr.Add_bfr_and_clear(tmp_bfr); +// if (armor_math) bfr.Add(Vnt_convert_lang.Bry__armor_end); + byte[] unique_bry = page.Wikie().Parser_mgr().Uniq_mgr().Add(math_bry); Bry_fmtr math_fmtr = renderer_is_latex ? math_fmtr_latex : math_fmtr_mathjax; - boolean armor_math = page.Lang().Vnt_mgr().Enabled() && !renderer_is_latex; // REF.MW:LangConverter.php|armourMath - if (armor_math) bfr.Add(Vnt_convert_lang.Bry__armor_bgn); - math_fmtr.Bld_bfr_many(tmp_bfr, id, math_bry); + math_fmtr.Bld_bfr_many(tmp_bfr, id, unique_bry); bfr.Add_bfr_and_clear(tmp_bfr); - if (armor_math) bfr.Add(Vnt_convert_lang.Bry__armor_end); + if (enabled && renderer_is_latex) // NOTE: only generate images if math is enabled; otherwise "downloading" prints at bottom of screen, but no action (also a lot of file IO) page.File_math().Add(new_math_itm); } diff --git a/400_xowa/src/gplx/xowa/xtns/math/Xof_math_mgr_html_tst.java b/400_xowa/src/gplx/xowa/xtns/math/Xof_math_mgr_html_tst.java index 4fec61ed4..7e3dd1827 100644 --- a/400_xowa/src/gplx/xowa/xtns/math/Xof_math_mgr_html_tst.java +++ b/400_xowa/src/gplx/xowa/xtns/math/Xof_math_mgr_html_tst.java @@ -18,24 +18,33 @@ along with this program. If not, see . package gplx.xowa.xtns.math; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import org.junit.*; public class Xof_math_mgr_html_tst { - @Before public void init() {} private final Xop_fxt fxt = new Xop_fxt(); - @Test public void Escape_lt_gt_mathjax() { // PURPOSE: escape <>&"; EX:de.w:Vergleich_(Zahlen); DATE:2014-05-10; PAGE:s.w:Matrix_(mathematics) DATE:2014-07-19 - fxt.Test_html_full_str("a<>b", "a<>b"); - } - @Test public void Escape_lt_gt_latex() { + @Before public void init() {} private final Xop_fxt fxt = Xop_fxt.New_app_html(); + @Test public void Basic__latex() { fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(false); - fxt.Test_html_full_str("a<>b", "a<>b"); + fxt.Test__parse_to_html_mgr("x + y", "x + y"); // latex has img + } + @Test public void Basic__mathjax() { fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(true); + fxt.Test__parse_to_html_mgr("x + y", "x + y"); // mathjax has no img + } + @Test public void Escape__mathjax() { // PURPOSE: escape <>&"; EX:de.w:Vergleich_(Zahlen); DATE:2014-05-10; PAGE:s.w:Matrix_(mathematics) DATE:2014-07-19 + fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(true); + fxt.Test__parse_to_html_mgr("a<>b", "a<>b"); + } + @Test public void Escape__latex() { + fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(false); + fxt.Test__parse_to_html_mgr("a<>b", "a<>b"); } @Test public void Amp() { // PURPOSE: assert that amp is not escaped; DATE:2014-07-20 - fxt.Test_html_full_str("a&b", "a&b"); + fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(true); + fxt.Test__parse_to_html_mgr("a&b", "a&b"); } @Test public void Quote() { // PURPOSE: assert that quote is not escaped; DATE:2014-07-20 - fxt.Test_html_full_str("a\"b", "a\"b"); + fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(true); + fxt.Test__parse_to_html_mgr("a\"b", "a\"b"); } - @Test public void Vnt() { // PURPOSE: if vnt, armor math; PAGE:sr.w:Земља; DATE:2015-10-11 - fxt.Wiki().Lang().Vnt_mgr().Enabled_(Bool_.Y); - fxt.Test_html_full_str("abc", "-{R|abc}-"); // NOTE: this also armors the id which is good for the mathjax.js - fxt.Wiki().Lang().Vnt_mgr().Enabled_(Bool_.N); + @Test public void Script() { + fxt.App().File_mgr().Math_mgr().Renderer_is_mathjax_(false); + fxt.Test__parse_to_html_mgr("", "<script>alert('fail');"); } }